


'''simple docstring'''


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
						from .tokenization_nllb import NllbTokenizer
else:
						lowerCamelCase__    	=     None


lowerCamelCase__    	=     logging.get_logger(__name__)


lowerCamelCase__    	=     {'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

lowerCamelCase__    	=     {
    'vocab_file': {
        'facebook/nllb-200-distilled-600M': (
            'https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/nllb-200-distilled-600M': (
            'https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json'
        ),
    },
}

lowerCamelCase__    	=     {
    'facebook/nllb-large-en-ro': 1_024,
    'facebook/nllb-200-distilled-600M': 1_024,
}

# fmt: off
lowerCamelCase__    	=     ['ace_Arab', 'ace_Latn', 'acm_Arab', 'acq_Arab', 'aeb_Arab', 'afr_Latn', 'ajp_Arab', 'aka_Latn', 'amh_Ethi', 'apc_Arab', 'arb_Arab', 'ars_Arab', 'ary_Arab', 'arz_Arab', 'asm_Beng', 'ast_Latn', 'awa_Deva', 'ayr_Latn', 'azb_Arab', 'azj_Latn', 'bak_Cyrl', 'bam_Latn', 'ban_Latn', 'bel_Cyrl', 'bem_Latn', 'ben_Beng', 'bho_Deva', 'bjn_Arab', 'bjn_Latn', 'bod_Tibt', 'bos_Latn', 'bug_Latn', 'bul_Cyrl', 'cat_Latn', 'ceb_Latn', 'ces_Latn', 'cjk_Latn', 'ckb_Arab', 'crh_Latn', 'cym_Latn', 'dan_Latn', 'deu_Latn', 'dik_Latn', 'dyu_Latn', 'dzo_Tibt', 'ell_Grek', 'eng_Latn', 'epo_Latn', 'est_Latn', 'eus_Latn', 'ewe_Latn', 'fao_Latn', 'pes_Arab', 'fij_Latn', 'fin_Latn', 'fon_Latn', 'fra_Latn', 'fur_Latn', 'fuv_Latn', 'gla_Latn', 'gle_Latn', 'glg_Latn', 'grn_Latn', 'guj_Gujr', 'hat_Latn', 'hau_Latn', 'heb_Hebr', 'hin_Deva', 'hne_Deva', 'hrv_Latn', 'hun_Latn', 'hye_Armn', 'ibo_Latn', 'ilo_Latn', 'ind_Latn', 'isl_Latn', 'ita_Latn', 'jav_Latn', 'jpn_Jpan', 'kab_Latn', 'kac_Latn', 'kam_Latn', 'kan_Knda', 'kas_Arab', 'kas_Deva', 'kat_Geor', 'knc_Arab', 'knc_Latn', 'kaz_Cyrl', 'kbp_Latn', 'kea_Latn', 'khm_Khmr', 'kik_Latn', 'kin_Latn', 'kir_Cyrl', 'kmb_Latn', 'kon_Latn', 'kor_Hang', 'kmr_Latn', 'lao_Laoo', 'lvs_Latn', 'lij_Latn', 'lim_Latn', 'lin_Latn', 'lit_Latn', 'lmo_Latn', 'ltg_Latn', 'ltz_Latn', 'lua_Latn', 'lug_Latn', 'luo_Latn', 'lus_Latn', 'mag_Deva', 'mai_Deva', 'mal_Mlym', 'mar_Deva', 'min_Latn', 'mkd_Cyrl', 'plt_Latn', 'mlt_Latn', 'mni_Beng', 'khk_Cyrl', 'mos_Latn', 'mri_Latn', 'zsm_Latn', 'mya_Mymr', 'nld_Latn', 'nno_Latn', 'nob_Latn', 'npi_Deva', 'nso_Latn', 'nus_Latn', 'nya_Latn', 'oci_Latn', 'gaz_Latn', 'ory_Orya', 'pag_Latn', 'pan_Guru', 'pap_Latn', 'pol_Latn', 'por_Latn', 'prs_Arab', 'pbt_Arab', 'quy_Latn', 'ron_Latn', 'run_Latn', 'rus_Cyrl', 'sag_Latn', 'san_Deva', 'sat_Beng', 'scn_Latn', 'shn_Mymr', 'sin_Sinh', 'slk_Latn', 'slv_Latn', 'smo_Latn', 'sna_Latn', 'snd_Arab', 'som_Latn', 'sot_Latn', 'spa_Latn', 'als_Latn', 'srd_Latn', 'srp_Cyrl', 'ssw_Latn', 'sun_Latn', 'swe_Latn', 'swh_Latn', 'szl_Latn', 'tam_Taml', 'tat_Cyrl', 'tel_Telu', 'tgk_Cyrl', 'tgl_Latn', 'tha_Thai', 'tir_Ethi', 'taq_Latn', 'taq_Tfng', 'tpi_Latn', 'tsn_Latn', 'tso_Latn', 'tuk_Latn', 'tum_Latn', 'tur_Latn', 'twi_Latn', 'tzm_Tfng', 'uig_Arab', 'ukr_Cyrl', 'umb_Latn', 'urd_Arab', 'uzn_Latn', 'vec_Latn', 'vie_Latn', 'war_Latn', 'wol_Latn', 'xho_Latn', 'ydd_Hebr', 'yor_Latn', 'yue_Hant', 'zho_Hans', 'zho_Hant', 'zul_Latn']






class 	lowerCAmelCase__			(			UpperCAmelCase__	):
			lowerCAmelCase :	Optional[int]          =     VOCAB_FILES_NAMES
			lowerCAmelCase :	List[str]          =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase :	Dict          =     PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase :	Optional[int]          =     ["input_ids", "attention_mask"]
			lowerCAmelCase :	Dict          =     NllbTokenizer

			lowerCAmelCase :	List[int]          =     []
			lowerCAmelCase :	List[int]          =     []




			def __init__(		self  :     Union[str, Any]	,  lowerCamelCase__  :     List[Any]=None	,  lowerCamelCase__  :     Optional[Any]=None	,  lowerCamelCase__  :     Dict="<s>"	,  lowerCamelCase__  :     Optional[int]="</s>"	,  lowerCamelCase__  :     str="</s>"	,  lowerCamelCase__  :     List[str]="<s>"	,  lowerCamelCase__  :     List[Any]="<unk>"	,  lowerCamelCase__  :     int="<pad>"	,  lowerCamelCase__  :     Union[str, Any]="<mask>"	,  lowerCamelCase__  :     List[Any]=None	,  lowerCamelCase__  :     Dict=None	,  lowerCamelCase__  :     int=None	,  lowerCamelCase__  :     Dict=False	,  **lowerCamelCase__  :     str	,  )   ->int:



					'''simple docstring'''







					_UpperCAmelCase		:       Any								=      AddedToken(lowerCamelCase__	,  lstrip=lowerCamelCase__	,  rstrip=lowerCamelCase__ ) if isinstance(lowerCamelCase__	,  lowerCamelCase__ ) else mask_token
					_UpperCAmelCase		:       str								=      legacy_behaviour
					super().__init__(
					    vocab_file=lowerCamelCase__	,  tokenizer_file=lowerCamelCase__	,  bos_token=lowerCamelCase__	,  eos_token=lowerCamelCase__	,  sep_token=lowerCamelCase__	,  cls_token=lowerCamelCase__	,  unk_token=lowerCamelCase__	,  pad_token=lowerCamelCase__	,  mask_token=lowerCamelCase__	,  src_lang=lowerCamelCase__	,  tgt_lang=lowerCamelCase__	,  additional_special_tokens=lowerCamelCase__	,  legacy_behaviour=lowerCamelCase__	,  **lowerCamelCase__	,  )

					_UpperCAmelCase		:       Any								=      vocab_file
					_UpperCAmelCase		:       Optional[Any]								=      False if not self.vocab_file else True

					_UpperCAmelCase		:       Optional[int]								=      FAIRSEQ_LANGUAGE_CODES.copy()

					if additional_special_tokens is not None:
							# Only add those special tokens if they are not already there.
							_additional_special_tokens.extend(
							    [t for t in additional_special_tokens if t not in _additional_special_tokens] )

					self.add_special_tokens({"additional_special_tokens": _additional_special_tokens} )
					_UpperCAmelCase		:       Tuple								=      {
					    lang_code: self.convert_tokens_to_ids(lowerCamelCase__ ) for lang_code in FAIRSEQ_LANGUAGE_CODES
					}

					_UpperCAmelCase		:       Dict								=      src_lang if src_lang is not None else "eng_Latn"
					_UpperCAmelCase		:       Optional[int]								=      self.convert_tokens_to_ids(self._src_lang )
					_UpperCAmelCase		:       Dict								=      tgt_lang
					self.set_src_lang_special_tokens(self._src_lang )




			@property
			def 							lowerCAmelCase__    (		self  :     Optional[int] )   ->str:



					'''simple docstring'''







					return self._src_lang




			@src_lang.setter
			def 							lowerCAmelCase__    (		self  :     Optional[int]	,  lowerCamelCase__  :     str )   ->None:



					'''simple docstring'''







					_UpperCAmelCase		:       Optional[int]								=      new_src_lang
					self.set_src_lang_special_tokens(self._src_lang )




			def 							lowerCAmelCase__    (		self  :     Union[str, Any]	,  lowerCamelCase__  :     List[int]	,  lowerCamelCase__  :     Optional[List[int]] = None )   ->List[int]:



					'''simple docstring'''







					if token_ids_a is None:
							return self.prefix_tokens + token_ids_a + self.suffix_tokens
					# We don't expect to process pairs, but leave the pair logic for API consistency
					return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




			def 							lowerCAmelCase__    (		self  :     Dict	,  lowerCamelCase__  :     List[int]	,  lowerCamelCase__  :     Optional[List[int]] = None )   ->List[int]:



					'''simple docstring'''







					_UpperCAmelCase		:       Optional[int]								=      [self.sep_token_id]
					_UpperCAmelCase		:       Optional[Any]								=      [self.cls_token_id]

					if token_ids_a is None:
							return len(cls + token_ids_a + sep ) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]




			def 							lowerCAmelCase__    (		self  :     Union[str, Any]	,  lowerCamelCase__  :     List[Any]	,  lowerCamelCase__  :     str	,  lowerCamelCase__  :     Optional[str]	,  lowerCamelCase__  :     Optional[str]	,  **lowerCamelCase__  :     Any )   ->Tuple:



					'''simple docstring'''







					if src_lang is None or tgt_lang is None:
							raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model" )
					_UpperCAmelCase		:       Any								=      src_lang
					_UpperCAmelCase		:       List[str]								=      self(lowerCamelCase__	,  add_special_tokens=lowerCamelCase__	,  return_tensors=lowerCamelCase__	,  **lowerCamelCase__ )
					_UpperCAmelCase		:       List[Any]								=      self.convert_tokens_to_ids(lowerCamelCase__ )
					_UpperCAmelCase		:       str								=      tgt_lang_id
					return inputs




			def 							lowerCAmelCase__    (		self  :     str	,  lowerCamelCase__  :     List[str]	,  lowerCamelCase__  :     str = "eng_Latn"	,  lowerCamelCase__  :     Optional[List[str]] = None	,  lowerCamelCase__  :     str = "fra_Latn"	,  **lowerCamelCase__  :     int	,  )   ->BatchEncoding:



					'''simple docstring'''







					_UpperCAmelCase		:       str								=      src_lang
					_UpperCAmelCase		:       Optional[Any]								=      tgt_lang
					return super().prepare_seqaseq_batch(lowerCamelCase__	,  lowerCamelCase__	,  **lowerCamelCase__ )




			def 							lowerCAmelCase__    (		self  :     str )   ->List[str]:



					'''simple docstring'''







					return self.set_src_lang_special_tokens(self.src_lang )




			def 							lowerCAmelCase__    (		self  :     Optional[int] )   ->Union[str, Any]:



					'''simple docstring'''







					return self.set_tgt_lang_special_tokens(self.tgt_lang )




			def 							lowerCAmelCase__    (		self  :     Union[str, Any]	,  lowerCamelCase__  :     Union[str, Any] )   ->None:



					'''simple docstring'''







					_UpperCAmelCase		:       str								=      self.convert_tokens_to_ids(lowerCamelCase__ )

					if self.legacy_behaviour:
							_UpperCAmelCase		:       Optional[Any]								=      []
							_UpperCAmelCase		:       Tuple								=      [self.eos_token_id, self.cur_lang_code]
					else:
							_UpperCAmelCase		:       Union[str, Any]								=      [self.cur_lang_code]
							_UpperCAmelCase		:       Dict								=      [self.eos_token_id]

					_UpperCAmelCase		:       str								=      self.convert_ids_to_tokens(self.prefix_tokens )
					_UpperCAmelCase		:       List[str]								=      self.convert_ids_to_tokens(self.suffix_tokens )

					_UpperCAmelCase		:       List[Any]								=      processors.TemplateProcessing(
					    single=prefix_tokens_str + ["$A"] + suffix_tokens_str	,  pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str	,  special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str	,  self.prefix_tokens + self.suffix_tokens ) )	,  )




			def 							lowerCAmelCase__    (		self  :     List[Any]	,  lowerCamelCase__  :     str )   ->None:



					'''simple docstring'''







					_UpperCAmelCase		:       Tuple								=      self.convert_tokens_to_ids(lowerCamelCase__ )
					if self.legacy_behaviour:
							_UpperCAmelCase		:       Optional[int]								=      []
							_UpperCAmelCase		:       Optional[int]								=      [self.eos_token_id, self.cur_lang_code]
					else:
							_UpperCAmelCase		:       Union[str, Any]								=      [self.cur_lang_code]
							_UpperCAmelCase		:       Union[str, Any]								=      [self.eos_token_id]

					_UpperCAmelCase		:       List[Any]								=      self.convert_ids_to_tokens(self.prefix_tokens )
					_UpperCAmelCase		:       int								=      self.convert_ids_to_tokens(self.suffix_tokens )

					_UpperCAmelCase		:       Dict								=      processors.TemplateProcessing(
					    single=prefix_tokens_str + ["$A"] + suffix_tokens_str	,  pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str	,  special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str	,  self.prefix_tokens + self.suffix_tokens ) )	,  )






			def 							lowerCAmelCase__    (		self  :     List[Any]	,  lowerCamelCase__  :     str	,  lowerCamelCase__  :     Optional[str] = None )   ->Tuple[str]:



					'''simple docstring'''







					if not self.can_save_slow_tokenizer:
							raise ValueError(
							    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
							    "tokenizer." )

					if not os.path.isdir(lowerCamelCase__ ):
							logger.error(F"""Vocabulary path ({save_directory}) should be a directory.""" )
							return
					_UpperCAmelCase		:       List[str]								=      os.path.join(
					    lowerCamelCase__	,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )

					if os.path.abspath(self.vocab_file ) != os.path.abspath(lowerCamelCase__ ):
							copyfile(self.vocab_file	,  lowerCamelCase__ )

					return (out_vocab_file,)







'''simple docstring'''


import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


lowerCamelCase__    	=     ['\nclass', '\ndef', '\n#', '\n@', '\nprint', '\nif']






class 	lowerCAmelCase__			(			UpperCAmelCase__	):




			def __init__(		self  :     Tuple	,  lowerCamelCase__  :     List[Any]	,  lowerCamelCase__  :     Optional[int]	,  lowerCamelCase__  :     Optional[Any]=None	,  lowerCamelCase__  :     int=1 )   ->Optional[int]:



					'''simple docstring'''







					_UpperCAmelCase		:       List[Any]								=      tokenizer
					_UpperCAmelCase		:       Tuple								=      dataset
					_UpperCAmelCase		:       Union[str, Any]								=      len(lowerCamelCase__ ) if n_tasks is None else n_tasks
					_UpperCAmelCase		:       Any								=      n_copies






			def __iter__(		self  :     Any )   ->Dict:



					'''simple docstring'''







					_UpperCAmelCase		:       Optional[int]								=      []
					for task in range(self.n_tasks ):
							# without strip, the model generate commented codes ...
							prompts.append(self.tokenizer.eos_token + self.dataset[task]["prompt"].strip() )
					_UpperCAmelCase		:       Optional[Any]								=      self.tokenizer(lowerCamelCase__	,  padding=lowerCamelCase__	,  return_tensors="pt" )
					for task in range(self.n_tasks ):
							for _ in range(self.n_copies ):
									yield {
									    "ids": outputs.input_ids[task],
									    "task_id": task,
									    "input_len": outputs.attention_mask[task].sum(),
									}






class 	lowerCAmelCase__			(			UpperCAmelCase__	):




			def __init__(		self  :     Optional[int]	,  lowerCamelCase__  :     Tuple	,  lowerCamelCase__  :     Any	,  lowerCamelCase__  :     Dict )   ->Any:



					'''simple docstring'''







					_UpperCAmelCase		:       List[str]								=      start_length
					_UpperCAmelCase		:       Union[str, Any]								=      eof_strings
					_UpperCAmelCase		:       Union[str, Any]								=      tokenizer






			def __call__(		self  :     Tuple	,  lowerCamelCase__  :     Union[str, Any]	,  lowerCamelCase__  :     List[str]	,  **lowerCamelCase__  :     Optional[int] )   ->Union[str, Any]:



					'''simple docstring'''







					_UpperCAmelCase		:       Tuple								=      self.tokenizer.batch_decode(input_ids[:, self.start_length :] )
					_UpperCAmelCase		:       Optional[int]								=      []
					for decoded_generation in decoded_generations:
							done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings ) )
					return all(lowerCamelCase__ )






def  __lowerCAmelCase (__lowerCAmelCase   ):
		_UpperCAmelCase		:       Tuple								=      re.split("(%s)" % "|".join(__lowerCAmelCase   )      , __lowerCAmelCase   )
		# last string should be ""
		return "".join(string_list[:-2]   )






def  __lowerCAmelCase (__lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase=20      , **__lowerCAmelCase   ):
		_UpperCAmelCase		:       Tuple								=      defaultdict(__lowerCAmelCase   )  # dict of list of generated tokens
		for step, batch in tqdm(enumerate(__lowerCAmelCase   )   ):
				with torch.no_grad():
						_UpperCAmelCase		:       Tuple								=      batch["ids"].shape[-1]
						_UpperCAmelCase		:       Optional[int]								=      accelerator.unwrap_model(__lowerCAmelCase   ).generate(
						    input_ids=batch["ids"][:, : batch["input_len"]]      , num_return_sequences=__lowerCAmelCase      , **__lowerCAmelCase   )
						# each task is generated batch_size times
						_UpperCAmelCase		:       str								=      batch["task_id"].repeat(__lowerCAmelCase   )
						_UpperCAmelCase		:       str								=      accelerator.pad_across_processes(
						    __lowerCAmelCase      , dim=1      , pad_index=tokenizer.pad_token_id   )

						_UpperCAmelCase					,    _UpperCAmelCase		:       int								=      accelerator.gather((generated_tokens, generated_tasks)   )
						_UpperCAmelCase		:       Dict								=      generated_tokens.cpu().numpy()
						_UpperCAmelCase		:       Dict								=      generated_tasks.cpu().numpy()

						for task, generated_tokens in zip(__lowerCAmelCase      , __lowerCAmelCase   ):
								gen_token_dict[task].append(__lowerCAmelCase   )

		_UpperCAmelCase		:       int								=      [[] for _ in range(__lowerCAmelCase   )]
		for task, generated_tokens in gen_token_dict.items():
				for s in generated_tokens:
						_UpperCAmelCase		:       List[Any]								=      tokenizer.decode(__lowerCAmelCase      , skip_special_tokens=__lowerCAmelCase      , clean_up_tokenization_spaces=__lowerCAmelCase   )
						code_gens[task].append(remove_last_block(__lowerCAmelCase   )   )
		return code_gens






def  __lowerCAmelCase ():
		# Setup configuration
		_UpperCAmelCase		:       List[str]								=      HfArgumentParser(__lowerCAmelCase   )
		_UpperCAmelCase		:       Tuple								=      parser.parse_args()

		transformers.logging.set_verbosity_error()
		# enables code execution in code_eval metric
		_UpperCAmelCase		:       Any								=      args.HF_ALLOW_CODE_EVAL
		# make sure tokenizer plays nice with multiprocessing
		_UpperCAmelCase		:       List[str]								=      "false"

		if args.num_workers is None:
				_UpperCAmelCase		:       List[str]								=      multiprocessing.cpu_count()

		# Use dataset load to feed to accelerate
		_UpperCAmelCase		:       List[Any]								=      Accelerator()
		set_seed(args.seed      , device_specific=__lowerCAmelCase   )

		# Load model and tokenizer
		_UpperCAmelCase		:       Tuple								=      AutoTokenizer.from_pretrained(args.model_ckpt   )
		_UpperCAmelCase		:       List[str]								=      tokenizer.eos_token
		_UpperCAmelCase		:       str								=      AutoModelForCausalLM.from_pretrained(args.model_ckpt   )

		# Generation settings
		_UpperCAmelCase		:       Tuple								=      {
		    "do_sample": args.do_sample,
		    "temperature": args.temperature,
		    "max_new_tokens": args.max_new_tokens,
		    "top_p": args.top_p,
		    "top_k": args.top_k,
		    "stopping_criteria": StoppingCriteriaList([EndOfFunctionCriteria(0      , __lowerCAmelCase      , __lowerCAmelCase   )]   ),
		}

		# Load evaluation dataset and metric
		_UpperCAmelCase		:       Union[str, Any]								=      load_dataset("openai_humaneval"   )
		_UpperCAmelCase		:       List[Any]								=      load_metric("code_eval"   )

		_UpperCAmelCase		:       Optional[Any]								=      args.num_tasks if args.num_tasks is not None else len(human_eval["test"]   )
		_UpperCAmelCase		:       Any								=      args.n_samples // args.batch_size

		_UpperCAmelCase		:       Tuple								=      TokenizedDataset(__lowerCAmelCase      , human_eval["test"]      , n_copies=__lowerCAmelCase      , n_tasks=__lowerCAmelCase   )
		# do not confuse args.batch_size, which is actually the num_return_sequences
		_UpperCAmelCase		:       List[str]								=      DataLoader(__lowerCAmelCase      , batch_size=1   )

		# Run a quick test to see if code evaluation is enabled
		try:
				_UpperCAmelCase		:       Optional[int]								=      code_eval_metric.compute(references=[""]      , predictions=[[""]]   )
		except ValueError as exception:
				print(
				    "Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL=\"1\"`"
				    " flag to enable code evaluation."   )
				raise exception

		_UpperCAmelCase					,    _UpperCAmelCase		:       Any								=      accelerator.prepare(__lowerCAmelCase      , __lowerCAmelCase   )

		_UpperCAmelCase		:       Dict								=      complete_code(
		    __lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase      , n_tasks=__lowerCAmelCase      , batch_size=args.batch_size      , **__lowerCAmelCase      , )

		if accelerator.is_main_process:
				_UpperCAmelCase		:       List[Any]								=      []

				for task in tqdm(range(__lowerCAmelCase   )   ):
						_UpperCAmelCase		:       str								=      human_eval["test"][task]["test"]
						_UpperCAmelCase		:       Union[str, Any]								=      F"""check({human_eval['test'][task]['entry_point']})"""
						references.append("\n" + test_func + "\n" + entry_point   )

				# Evaluate completions with "code_eval" metric
				_UpperCAmelCase					,    _UpperCAmelCase		:       str								=      code_eval_metric.compute(
				    references=__lowerCAmelCase      , predictions=__lowerCAmelCase      , num_workers=args.num_workers   )
				print(F"""Results: {pass_at_k}"""   )

				# Save results to json file
				with open(args.output_file      , "w"   ) as fp:
						json.dump(__lowerCAmelCase      , __lowerCAmelCase   )


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
						main()



