






import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


snake_case    				=		["""\nclass""", """\ndef""", """\n#""", """\n@""", """\nprint""", """\nif"""]


class  SCREAMING_SNAKE_CASE			(			lowerCAmelCase					):
     '''simple docstring'''


     def __init__(				self       :	Tuple					, UpperCAmelCase_       :	Tuple					, UpperCAmelCase_       :	Dict					, UpperCAmelCase_       :	List[Any]=None					, UpperCAmelCase_       :	List[Any]=1      ):
       SCREAMING_SNAKE_CASE       :      Optional[Any]							=    tokenizer
       SCREAMING_SNAKE_CASE       :      Any							=    dataset
       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    len(UpperCAmelCase_      ) if n_tasks is None else n_tasks
       SCREAMING_SNAKE_CASE       :      Dict							=    n_copies






     def __iter__(				self       :	Union[str, Any]      ):
       SCREAMING_SNAKE_CASE       :      int							=    []
       for task in range(self.n_tasks      ):
         # without strip, the model generate commented codes ...
         prompts.append(self.tokenizer.eos_token + self.dataset[task]["prompt"].strip()      )
       SCREAMING_SNAKE_CASE       :      int							=    self.tokenizer(UpperCAmelCase_					, padding=UpperCAmelCase_					, return_tensors="pt"      )
       for task in range(self.n_tasks      ):
         for _ in range(self.n_copies      ):
           yield {
               "ids": outputs.input_ids[task],
               "task_id": task,
               "input_len": outputs.attention_mask[task].sum(),
           }





class  SCREAMING_SNAKE_CASE			(			lowerCAmelCase					):
     '''simple docstring'''


     def __init__(				self       :	Dict					, UpperCAmelCase_       :	List[str]					, UpperCAmelCase_       :	List[str]					, UpperCAmelCase_       :	Any      ):
       SCREAMING_SNAKE_CASE       :      Any							=    start_length
       SCREAMING_SNAKE_CASE       :      List[str]							=    eof_strings
       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    tokenizer






     def __call__(				self       :	Optional[int]					, UpperCAmelCase_       :	Dict					, UpperCAmelCase_       :	str					, **UpperCAmelCase_       :	Any      ):
       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    self.tokenizer.batch_decode(input_ids[:, self.start_length :]      )
       SCREAMING_SNAKE_CASE       :      List[Any]							=    []
       for decoded_generation in decoded_generations:
         done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings      )      )
       return all(UpperCAmelCase_      )





def   lowerCamelCase__			(     lowercase				):

  """simple docstring"""






  SCREAMING_SNAKE_CASE       :      Any							=    re.split("(%s)" % "|".join(lowercase				)						,     lowercase				)
  # last string should be ""
  return "".join(string_list[:-2]				)


def   lowerCamelCase__			(     lowercase						,     lowercase						,     lowercase						,     lowercase						,     lowercase						,     lowercase=20						,     **lowercase				):

  """simple docstring"""






  SCREAMING_SNAKE_CASE       :      Any							=    defaultdict(lowercase				)  # dict of list of generated tokens
  for step, batch in tqdm(enumerate(lowercase				)				):
    with torch.no_grad():
      SCREAMING_SNAKE_CASE       :      Dict							=    batch["ids"].shape[-1]
      SCREAMING_SNAKE_CASE       :      Dict							=    accelerator.unwrap_model(lowercase				).generate(
          input_ids=batch["ids"][:, : batch["input_len"]]						,     num_return_sequences=lowercase						,     **lowercase				)
      # each task is generated batch_size times
      SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    batch["task_id"].repeat(lowercase				)
      SCREAMING_SNAKE_CASE       :      List[Any]							=    accelerator.pad_across_processes(
          lowercase						,     dim=1						,     pad_index=tokenizer.pad_token_id				)

      SCREAMING_SNAKE_CASE						,  SCREAMING_SNAKE_CASE       :      Dict							=    accelerator.gather((generated_tokens, generated_tasks)				)
      SCREAMING_SNAKE_CASE       :      Tuple							=    generated_tokens.cpu().numpy()
      SCREAMING_SNAKE_CASE       :      str							=    generated_tasks.cpu().numpy()

      for task, generated_tokens in zip(lowercase						,     lowercase				):
        gen_token_dict[task].append(lowercase				)

  SCREAMING_SNAKE_CASE       :      Any							=    [[] for _ in range(lowercase				)]
  for task, generated_tokens in gen_token_dict.items():
    for s in generated_tokens:
      SCREAMING_SNAKE_CASE       :      int							=    tokenizer.decode(lowercase						,     skip_special_tokens=lowercase						,     clean_up_tokenization_spaces=lowercase				)
      code_gens[task].append(remove_last_block(lowercase				)				)
  return code_gens


def   lowerCamelCase__			(     ):

  """simple docstring"""






  SCREAMING_SNAKE_CASE       :      List[str]							=    HfArgumentParser(lowercase				)
  SCREAMING_SNAKE_CASE       :      List[str]							=    parser.parse_args()

  transformers.logging.set_verbosity_error()
  # enables code execution in code_eval metric
  SCREAMING_SNAKE_CASE       :      Tuple							=    args.HF_ALLOW_CODE_EVAL
  # make sure tokenizer plays nice with multiprocessing
  SCREAMING_SNAKE_CASE       :      Optional[Any]							=    "false"

  if args.num_workers is None:
    SCREAMING_SNAKE_CASE       :      str							=    multiprocessing.cpu_count()

  # Use dataset load to feed to accelerate
  SCREAMING_SNAKE_CASE       :      List[Any]							=    Accelerator()
  set_seed(args.seed						,     device_specific=lowercase				)

  # Load model and tokenizer
  SCREAMING_SNAKE_CASE       :      Optional[Any]							=    AutoTokenizer.from_pretrained(args.model_ckpt				)
  SCREAMING_SNAKE_CASE       :      List[str]							=    tokenizer.eos_token
  SCREAMING_SNAKE_CASE       :      str							=    AutoModelForCausalLM.from_pretrained(args.model_ckpt				)

  # Generation settings
  SCREAMING_SNAKE_CASE       :      str							=    {
      "do_sample": args.do_sample,
      "temperature": args.temperature,
      "max_new_tokens": args.max_new_tokens,
      "top_p": args.top_p,
      "top_k": args.top_k,
      "stopping_criteria": StoppingCriteriaList([EndOfFunctionCriteria(0						,     lowercase						,     lowercase				)]				),
  }

  # Load evaluation dataset and metric
  SCREAMING_SNAKE_CASE       :      List[str]							=    load_dataset("openai_humaneval"				)
  SCREAMING_SNAKE_CASE       :      List[Any]							=    load_metric("code_eval"				)

  SCREAMING_SNAKE_CASE       :      Optional[Any]							=    args.num_tasks if args.num_tasks is not None else len(human_eval["test"]				)
  SCREAMING_SNAKE_CASE       :      Tuple							=    args.n_samples // args.batch_size

  SCREAMING_SNAKE_CASE       :      Optional[int]							=    TokenizedDataset(lowercase						,     human_eval["test"]						,     n_copies=lowercase						,     n_tasks=lowercase				)
  # do not confuse args.batch_size, which is actually the num_return_sequences
  SCREAMING_SNAKE_CASE       :      Any							=    DataLoader(lowercase						,     batch_size=1				)

  # Run a quick test to see if code evaluation is enabled
  try:
    SCREAMING_SNAKE_CASE       :      str							=    code_eval_metric.compute(references=[""]						,     predictions=[[""]]				)
  except ValueError as exception:
    print(
        "Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL=\"1\"`"
        " flag to enable code evaluation."				)
    raise exception

  SCREAMING_SNAKE_CASE						,  SCREAMING_SNAKE_CASE       :      int							=    accelerator.prepare(lowercase						,     lowercase				)

  SCREAMING_SNAKE_CASE       :      Dict							=    complete_code(
      lowercase						,     lowercase						,     lowercase						,     lowercase						,     n_tasks=lowercase						,     batch_size=args.batch_size						,     **lowercase						,     )

  if accelerator.is_main_process:
    SCREAMING_SNAKE_CASE       :      Tuple							=    []

    for task in tqdm(range(lowercase				)				):
      SCREAMING_SNAKE_CASE       :      Any							=    human_eval["test"][task]["test"]
      SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    F'''check({human_eval['test'][task]['entry_point']})'''
      references.append("\n" + test_func + "\n" + entry_point				)

    # Evaluate completions with "code_eval" metric
    SCREAMING_SNAKE_CASE						,  SCREAMING_SNAKE_CASE       :      Dict							=    code_eval_metric.compute(
        references=lowercase						,     predictions=lowercase						,     num_workers=args.num_workers				)
    print(F'''Results: {pass_at_k}'''				)

    # Save results to json file
    with open(args.output_file						,     "w"				) as fp:
      json.dump(lowercase						,     lowercase				)


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
   main()














from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


snake_case    				=		{"""configuration_swin""": ["""SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP""", """SwinConfig""", """SwinOnnxConfig"""]}


try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   snake_case    				=		[
       """SWIN_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """SwinForImageClassification""",
       """SwinForMaskedImageModeling""",
       """SwinModel""",
       """SwinPreTrainedModel""",
       """SwinBackbone""",
   ]

try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   snake_case    				=		[
       """TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """TFSwinForImageClassification""",
       """TFSwinForMaskedImageModeling""",
       """TFSwinModel""",
       """TFSwinPreTrainedModel""",
   ]

if TYPE_CHECKING:
   from .configuration_swin import SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP, SwinConfig, SwinOnnxConfig

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_swin import (
          SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
          SwinBackbone,
          SwinForImageClassification,
          SwinForMaskedImageModeling,
          SwinModel,
          SwinPreTrainedModel,
      )

   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_swin import (
          TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFSwinForImageClassification,
          TFSwinForMaskedImageModeling,
          TFSwinModel,
          TFSwinPreTrainedModel,
      )

else:
   import sys

   snake_case    				=		_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)






