

import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


_lowerCAmelCase						=		["""\nclass""", """\ndef""", """\n#""", """\n@""", """\nprint""", """\nif"""]







class 	__UpperCamelCase    (    a__						):

  def __init__(		self    ,_A    ,_A    ,_A=None    ,_A=1      ):
      '''simple docstring'''

      _lowerCAmelCase       :   Dict  =    tokenizer
      _lowerCAmelCase       :   Union[str, Any]  =    dataset
      _lowerCAmelCase       :   List[Any]  =    len(_A      ) if n_tasks is None else n_tasks
      _lowerCAmelCase       :   List[Any]  =    n_copies







  def __iter__(		self      ):
      '''simple docstring'''

      _lowerCAmelCase       :   Optional[Any]  =    []
      for task in range(self.n_tasks      ):
          # without strip, the model generate commented codes ...
          prompts.append(self.tokenizer.eos_token + self.dataset[task]['prompt'].strip()      )
      _lowerCAmelCase       :   List[str]  =    self.tokenizer(_A    ,padding=_A    ,return_tensors='pt'      )
      for task in range(self.n_tasks      ):
          for _ in range(self.n_copies      ):
              yield {
                  "ids": outputs.input_ids[task],
                  "task_id": task,
                  "input_len": outputs.attention_mask[task].sum(),
              }







class 	__UpperCamelCase    (    a__						):

  def __init__(		self    ,_A    ,_A    ,_A      ):
      '''simple docstring'''

      _lowerCAmelCase       :   List[str]  =    start_length
      _lowerCAmelCase       :   Tuple  =    eof_strings
      _lowerCAmelCase       :   str  =    tokenizer







  def __call__(		self    ,_A    ,_A    ,**_A      ):
      '''simple docstring'''

      _lowerCAmelCase       :   Union[str, Any]  =    self.tokenizer.batch_decode(input_ids[:, self.start_length :]      )
      _lowerCAmelCase       :   str  =    []
      for decoded_generation in decoded_generations:
          done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings      )      )
      return all(_A      )





def 		lowerCamelCase__		(					_lowerCamelCase   ):

    '''simple docstring'''



    _lowerCAmelCase       :   Optional[Any]  =    re.split('(%s)' % '|'.join(_lowerCamelCase   ) ,   _lowerCamelCase   )
    # last string should be ""
    return "".join(string_list[:-2]   )

def 		lowerCamelCase__		(					_lowerCamelCase ,   _lowerCamelCase ,   _lowerCamelCase ,   _lowerCamelCase ,   _lowerCamelCase ,   _lowerCamelCase=20 ,   **_lowerCamelCase   ):

    '''simple docstring'''



    _lowerCAmelCase       :   List[Any]  =    defaultdict(_lowerCamelCase   )  # dict of list of generated tokens
    for step, batch in tqdm(enumerate(_lowerCamelCase   )   ):
        with torch.no_grad():
            _lowerCAmelCase       :   List[str]  =    batch['ids'].shape[-1]
            _lowerCAmelCase       :   Optional[Any]  =    accelerator.unwrap_model(_lowerCamelCase   ).generate(
                input_ids=batch['ids'][:, : batch['input_len']] ,   num_return_sequences=_lowerCamelCase ,   **_lowerCamelCase   )
            # each task is generated batch_size times
            _lowerCAmelCase       :   Optional[Any]  =    batch['task_id'].repeat(_lowerCamelCase   )
            _lowerCAmelCase       :   Union[str, Any]  =    accelerator.pad_across_processes(
                _lowerCamelCase ,   dim=1 ,   pad_index=tokenizer.pad_token_id   )

            _lowerCAmelCase       :   int  =    accelerator.gather((generated_tokens, generated_tasks)   )
            _lowerCAmelCase       :   int  =    generated_tokens.cpu().numpy()
            _lowerCAmelCase       :   int  =    generated_tasks.cpu().numpy()

            for task, generated_tokens in zip(_lowerCamelCase ,   _lowerCamelCase   ):
                gen_token_dict[task].append(_lowerCamelCase   )

    _lowerCAmelCase       :   Union[str, Any]  =    [[] for _ in range(_lowerCamelCase   )]
    for task, generated_tokens in gen_token_dict.items():
        for s in generated_tokens:
            _lowerCAmelCase       :   int  =    tokenizer.decode(_lowerCamelCase ,   skip_special_tokens=_lowerCamelCase ,   clean_up_tokenization_spaces=_lowerCamelCase   )
            code_gens[task].append(remove_last_block(_lowerCamelCase   )   )
    return code_gens

def 		lowerCamelCase__		(					):

    '''simple docstring'''



    _lowerCAmelCase       :   Tuple  =    HfArgumentParser(_lowerCamelCase   )
    _lowerCAmelCase       :   int  =    parser.parse_args()

    transformers.logging.set_verbosity_error()
    # enables code execution in code_eval metric
    _lowerCAmelCase       :   Union[str, Any]  =    args.HF_ALLOW_CODE_EVAL
    # make sure tokenizer plays nice with multiprocessing
    _lowerCAmelCase       :   Dict  =    'false'

    if args.num_workers is None:
        _lowerCAmelCase       :   List[str]  =    multiprocessing.cpu_count()

    # Use dataset load to feed to accelerate
    _lowerCAmelCase       :   Optional[Any]  =    Accelerator()
    set_seed(args.seed ,   device_specific=_lowerCamelCase   )

    # Load model and tokenizer
    _lowerCAmelCase       :   List[Any]  =    AutoTokenizer.from_pretrained(args.model_ckpt   )
    _lowerCAmelCase       :   Optional[Any]  =    tokenizer.eos_token
    _lowerCAmelCase       :   str  =    AutoModelForCausalLM.from_pretrained(args.model_ckpt   )

    # Generation settings
    _lowerCAmelCase       :   str  =    {
        'do_sample': args.do_sample,
        'temperature': args.temperature,
        'max_new_tokens': args.max_new_tokens,
        'top_p': args.top_p,
        'top_k': args.top_k,
        'stopping_criteria': StoppingCriteriaList([EndOfFunctionCriteria(0 ,   _lowerCamelCase ,   _lowerCamelCase   )]   ),
    }

    # Load evaluation dataset and metric
    _lowerCAmelCase       :   Any  =    load_dataset('openai_humaneval'   )
    _lowerCAmelCase       :   Optional[Any]  =    load_metric('code_eval'   )

    _lowerCAmelCase       :   Dict  =    args.num_tasks if args.num_tasks is not None else len(human_eval['test']   )
    _lowerCAmelCase       :   Dict  =    args.n_samples // args.batch_size

    _lowerCAmelCase       :   Optional[int]  =    TokenizedDataset(_lowerCamelCase ,   human_eval['test'] ,   n_copies=_lowerCamelCase ,   n_tasks=_lowerCamelCase   )
    # do not confuse args.batch_size, which is actually the num_return_sequences
    _lowerCAmelCase       :   Dict  =    DataLoader(_lowerCamelCase ,   batch_size=1   )

    # Run a quick test to see if code evaluation is enabled
    try:
        _lowerCAmelCase       :   Optional[Any]  =    code_eval_metric.compute(references=[''] ,   predictions=[['']]   )
    except ValueError as exception:
        print(
            'Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL="1"`'
            ' flag to enable code evaluation.'   )
        raise exception

    _lowerCAmelCase       :   Dict  =    accelerator.prepare(_lowerCamelCase ,   _lowerCamelCase   )

    _lowerCAmelCase       :   int  =    complete_code(
        _lowerCamelCase ,   _lowerCamelCase ,   _lowerCamelCase ,   _lowerCamelCase ,   n_tasks=_lowerCamelCase ,   batch_size=args.batch_size ,   **_lowerCamelCase ,   )

    if accelerator.is_main_process:
        _lowerCAmelCase       :   str  =    []

        for task in tqdm(range(_lowerCamelCase   )   ):
            _lowerCAmelCase       :   List[Any]  =    human_eval['test'][task]['test']
            _lowerCAmelCase       :   Optional[Any]  =    f"""check({human_eval['test'][task]['entry_point']})"""
            references.append('\n' + test_func + '\n' + entry_point   )

        # Evaluate completions with "code_eval" metric
        _lowerCAmelCase       :   Optional[Any]  =    code_eval_metric.compute(
            references=_lowerCamelCase ,   predictions=_lowerCamelCase ,   num_workers=args.num_workers   )
        print(f"""Results: {pass_at_k}"""   )

        # Save results to json file
        with open(args.output_file ,   'w'   ) as fp:
            json.dump(_lowerCamelCase ,   _lowerCamelCase   )


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
 main()








"""simple docstring"""






from __future__ import annotations

def 		lowerCamelCase__		(					_lowerCamelCase   ):

    '''simple docstring'''



    create_state_space_tree(_lowerCamelCase ,   [] ,   0 ,   [0 for i in range(len(_lowerCamelCase   )   )]   )

def 		lowerCamelCase__		(					_lowerCamelCase ,   _lowerCamelCase ,   _lowerCamelCase ,   _lowerCamelCase ,   ):

    '''simple docstring'''



    if index == len(_lowerCamelCase   ):
        print(_lowerCamelCase   )
        return

    for i in range(len(_lowerCamelCase   )   ):
        if not index_used[i]:
            current_sequence.append(sequence[i]   )
            _lowerCAmelCase       :   List[str]  =    True
            create_state_space_tree(_lowerCamelCase ,   _lowerCamelCase ,   index + 1 ,   _lowerCamelCase   )
            current_sequence.pop()
            _lowerCAmelCase       :   int  =    False
_lowerCAmelCase						=		[3, 1, 2, 4]
generate_all_permutations(sequence)

_lowerCAmelCase						=		["A", "B", "C"]

generate_all_permutations(sequence_a)





