

"""simple docstring"""





import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


lowerCamelCase      =		["""\nclass""", """\ndef""", """\n#""", """\n@""", """\nprint""", """\nif"""]
class    lowercase__     (					SCREAMING_SNAKE_CASE   ):







       '''simple docstring'''


       def __init__(							self    :						Optional[Any]	,       _UpperCAmelCase    :						str	,       _UpperCAmelCase    :						Optional[Any]	,       _UpperCAmelCase    :						List[Any]=None	,       _UpperCAmelCase    :						str=1					)				->		List[Any]:


             '''simple docstring'''




             UpperCAmelCase_    				=							tokenizer
             UpperCAmelCase_    				=							dataset
             UpperCAmelCase_    				=							len(_UpperCAmelCase					) if n_tasks is None else n_tasks
             UpperCAmelCase_    				=							n_copies







       def __iter__(							self    :						Tuple					)				->		List[Any]:


             '''simple docstring'''




             UpperCAmelCase_    				=							[]
             for task in range(self.n_tasks					):
                   # without strip, the model generate commented codes ...
                   prompts.append(self.tokenizer.eos_token + self.dataset[task]["prompt"].strip()					)
             UpperCAmelCase_    				=							self.tokenizer(_UpperCAmelCase	,       padding=_UpperCAmelCase	,       return_tensors="pt"					)
             for task in range(self.n_tasks					):
                   for _ in range(self.n_copies					):
                         yield {
                             "ids": outputs.input_ids[task],
                             "task_id": task,
                             "input_len": outputs.attention_mask[task].sum(),
                         }



class    lowercase__     (					SCREAMING_SNAKE_CASE   ):







       '''simple docstring'''


       def __init__(							self    :						List[Any]	,       _UpperCAmelCase    :						Optional[Any]	,       _UpperCAmelCase    :						Tuple	,       _UpperCAmelCase    :						List[str]					)				->		Dict:


             '''simple docstring'''




             UpperCAmelCase_    				=							start_length
             UpperCAmelCase_    				=							eof_strings
             UpperCAmelCase_    				=							tokenizer







       def __call__(							self    :						str	,       _UpperCAmelCase    :						List[Any]	,       _UpperCAmelCase    :						List[Any]	,       **_UpperCAmelCase    :						Optional[Any]					)				->		Union[str, Any]:


             '''simple docstring'''




             UpperCAmelCase_    				=							self.tokenizer.batch_decode(input_ids[:, self.start_length :]					)
             UpperCAmelCase_    				=							[]
             for decoded_generation in decoded_generations:
                   done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings					)					)
             return all(_UpperCAmelCase					)



def      a__			(   lowerCAmelCase__   ):
      UpperCAmelCase_    				=							re.split("(%s)" % "|".join(lowerCAmelCase__   )			,    lowerCAmelCase__   )
      # last string should be ""
      return "".join(string_list[:-2]   )
def      a__			(   lowerCAmelCase__			,    lowerCAmelCase__			,    lowerCAmelCase__			,    lowerCAmelCase__			,    lowerCAmelCase__			,    lowerCAmelCase__=20			,    **lowerCAmelCase__   ):
      UpperCAmelCase_    				=							defaultdict(lowerCAmelCase__   )  # dict of list of generated tokens
      for step, batch in tqdm(enumerate(lowerCAmelCase__   )   ):
            with torch.no_grad():
                  UpperCAmelCase_    				=							batch["ids"].shape[-1]
                  UpperCAmelCase_    				=							accelerator.unwrap_model(lowerCAmelCase__   ).generate(
                      input_ids=batch["ids"][:, : batch["input_len"]]			,    num_return_sequences=lowerCAmelCase__			,    **lowerCAmelCase__   )
                  # each task is generated batch_size times
                  UpperCAmelCase_    				=							batch["task_id"].repeat(lowerCAmelCase__   )
                  UpperCAmelCase_    				=							accelerator.pad_across_processes(
                      lowerCAmelCase__			,    dim=1			,    pad_index=tokenizer.pad_token_id   )

                  UpperCAmelCase_				,					UpperCAmelCase_    				=							accelerator.gather((generated_tokens, generated_tasks)   )
                  UpperCAmelCase_    				=							generated_tokens.cpu().numpy()
                  UpperCAmelCase_    				=							generated_tasks.cpu().numpy()

                  for task, generated_tokens in zip(lowerCAmelCase__			,    lowerCAmelCase__   ):
                        gen_token_dict[task].append(lowerCAmelCase__   )

      UpperCAmelCase_    				=							[[] for _ in range(lowerCAmelCase__   )]
      for task, generated_tokens in gen_token_dict.items():
            for s in generated_tokens:
                  UpperCAmelCase_    				=							tokenizer.decode(lowerCAmelCase__			,    skip_special_tokens=lowerCAmelCase__			,    clean_up_tokenization_spaces=lowerCAmelCase__   )
                  code_gens[task].append(remove_last_block(lowerCAmelCase__   )   )
      return code_gens
def      a__			(   ):
      # Setup configuration
      UpperCAmelCase_    				=							HfArgumentParser(lowerCAmelCase__   )
      UpperCAmelCase_    				=							parser.parse_args()

      transformers.logging.set_verbosity_error()
      # enables code execution in code_eval metric
      UpperCAmelCase_    				=							args.HF_ALLOW_CODE_EVAL
      # make sure tokenizer plays nice with multiprocessing
      UpperCAmelCase_    				=							"false"

      if args.num_workers is None:
            UpperCAmelCase_    				=							multiprocessing.cpu_count()

      # Use dataset load to feed to accelerate
      UpperCAmelCase_    				=							Accelerator()
      set_seed(args.seed			,    device_specific=lowerCAmelCase__   )

      # Load model and tokenizer
      UpperCAmelCase_    				=							AutoTokenizer.from_pretrained(args.model_ckpt   )
      UpperCAmelCase_    				=							tokenizer.eos_token
      UpperCAmelCase_    				=							AutoModelForCausalLM.from_pretrained(args.model_ckpt   )

      # Generation settings
      UpperCAmelCase_    				=							{
          "do_sample": args.do_sample,
          "temperature": args.temperature,
          "max_new_tokens": args.max_new_tokens,
          "top_p": args.top_p,
          "top_k": args.top_k,
          "stopping_criteria": StoppingCriteriaList([EndOfFunctionCriteria(0			,    lowerCAmelCase__			,    lowerCAmelCase__   )]   ),
      }

      # Load evaluation dataset and metric
      UpperCAmelCase_    				=							load_dataset("openai_humaneval"   )
      UpperCAmelCase_    				=							load_metric("code_eval"   )

      UpperCAmelCase_    				=							args.num_tasks if args.num_tasks is not None else len(human_eval["test"]   )
      UpperCAmelCase_    				=							args.n_samples // args.batch_size

      UpperCAmelCase_    				=							TokenizedDataset(lowerCAmelCase__			,    human_eval["test"]			,    n_copies=lowerCAmelCase__			,    n_tasks=lowerCAmelCase__   )
      # do not confuse args.batch_size, which is actually the num_return_sequences
      UpperCAmelCase_    				=							DataLoader(lowerCAmelCase__			,    batch_size=1   )

      # Run a quick test to see if code evaluation is enabled
      try:
            UpperCAmelCase_    				=							code_eval_metric.compute(references=[""]			,    predictions=[[""]]   )
      except ValueError as exception:
            print(
                "Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL=\"1\"`"
                " flag to enable code evaluation."   )
            raise exception

      UpperCAmelCase_				,					UpperCAmelCase_    				=							accelerator.prepare(lowerCAmelCase__			,    lowerCAmelCase__   )

      UpperCAmelCase_    				=							complete_code(
          lowerCAmelCase__			,    lowerCAmelCase__			,    lowerCAmelCase__			,    lowerCAmelCase__			,    n_tasks=lowerCAmelCase__			,    batch_size=args.batch_size			,    **lowerCAmelCase__			,    )

      if accelerator.is_main_process:
            UpperCAmelCase_    				=							[]

            for task in tqdm(range(lowerCAmelCase__   )   ):
                  UpperCAmelCase_    				=							human_eval["test"][task]["test"]
                  UpperCAmelCase_    				=							f"""check({human_eval['test'][task]['entry_point']})"""
                  references.append("\n" + test_func + "\n" + entry_point   )

            # Evaluate completions with "code_eval" metric
            UpperCAmelCase_				,					UpperCAmelCase_    				=							code_eval_metric.compute(
                references=lowerCAmelCase__			,    predictions=lowerCAmelCase__			,    num_workers=args.num_workers   )
            print(f"""Results: {pass_at_k}"""   )

            # Save results to json file
            with open(args.output_file			,    "w"   ) as fp:
                  json.dump(lowerCAmelCase__			,    lowerCAmelCase__   )


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
  main()









"""simple docstring"""





from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
  import PIL


lowerCamelCase      =		logging.get_logger(__name__)
class    lowercase__     (					SCREAMING_SNAKE_CASE   ):







       '''simple docstring'''


       UpperCamelCase		=  ['''pixel_values''']



       def __init__(							self    :						str	,       _UpperCAmelCase    :						bool = True	,       _UpperCAmelCase    :						Dict[str, int] = None	,       _UpperCAmelCase    :						float = None	,       _UpperCAmelCase    :						PILImageResampling = PILImageResampling.BILINEAR	,       _UpperCAmelCase    :						bool = True	,       _UpperCAmelCase    :						Union[int, float] = 1 / 255	,       _UpperCAmelCase    :						bool = True	,       _UpperCAmelCase    :						Optional[Union[float, List[float]]] = None	,       _UpperCAmelCase    :						Optional[Union[float, List[float]]] = None	,       **_UpperCAmelCase    :						Optional[int]	,       )				->		None:


             '''simple docstring'''




             super().__init__(**_UpperCAmelCase					)
             UpperCAmelCase_    				=							size if size is not None else {"shortest_edge": 384}
             UpperCAmelCase_    				=							get_size_dict(_UpperCAmelCase	,       default_to_square=_UpperCAmelCase					)

             UpperCAmelCase_    				=							do_resize
             UpperCAmelCase_    				=							size
             # Default value set here for backwards compatibility where the value in config is None
             UpperCAmelCase_    				=							crop_pct if crop_pct is not None else 224 / 256
             UpperCAmelCase_    				=							resample
             UpperCAmelCase_    				=							do_rescale
             UpperCAmelCase_    				=							rescale_factor
             UpperCAmelCase_    				=							do_normalize
             UpperCAmelCase_    				=							image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
             UpperCAmelCase_    				=							image_std if image_std is not None else IMAGENET_STANDARD_STD



       def   lowercase__       (							self    :						Union[str, Any]	,       _UpperCAmelCase    :						np.ndarray	,       _UpperCAmelCase    :						Dict[str, int]	,       _UpperCAmelCase    :						float	,       _UpperCAmelCase    :						PILImageResampling = PILImageResampling.BICUBIC	,       _UpperCAmelCase    :						Optional[Union[str, ChannelDimension]] = None	,       **_UpperCAmelCase    :						Optional[int]	,       )				->		np.ndarray:


             '''simple docstring'''




             UpperCAmelCase_    				=							get_size_dict(_UpperCAmelCase	,       default_to_square=_UpperCAmelCase					)
             if "shortest_edge" not in size:
                   raise ValueError(F"""Size dictionary must contain 'shortest_edge' key. Got {size.keys()}"""					)
             UpperCAmelCase_    				=							size["shortest_edge"]

             if shortest_edge < 384:
                   # maintain same ratio, resizing shortest edge to shortest_edge/crop_pct
                   UpperCAmelCase_    				=							int(shortest_edge / crop_pct					)
                   UpperCAmelCase_    				=							get_resize_output_image_size(_UpperCAmelCase	,       size=_UpperCAmelCase	,       default_to_square=_UpperCAmelCase					)
                   UpperCAmelCase_    				=							resize(image=_UpperCAmelCase	,       size=_UpperCAmelCase	,       resample=_UpperCAmelCase	,       data_format=_UpperCAmelCase	,       **_UpperCAmelCase					)
                   # then crop to (shortest_edge, shortest_edge)
                   return center_crop(image=_UpperCAmelCase	,       size=(shortest_edge, shortest_edge)	,       data_format=_UpperCAmelCase	,       **_UpperCAmelCase					)
             else:
                   # warping (no cropping) when evaluated at 384 or larger
                   return resize(
                       _UpperCAmelCase	,       size=(shortest_edge, shortest_edge)	,       resample=_UpperCAmelCase	,       data_format=_UpperCAmelCase	,       **_UpperCAmelCase					)



       def   lowercase__       (							self    :						Tuple	,       _UpperCAmelCase    :						np.ndarray	,       _UpperCAmelCase    :						Union[int, float]	,       _UpperCAmelCase    :						Optional[Union[str, ChannelDimension]] = None	,       **_UpperCAmelCase    :						List[Any]	,       )				->		Any:


             '''simple docstring'''




             return rescale(_UpperCAmelCase	,       scale=_UpperCAmelCase	,       data_format=_UpperCAmelCase	,       **_UpperCAmelCase					)



       def   lowercase__       (							self    :						Dict	,       _UpperCAmelCase    :						np.ndarray	,       _UpperCAmelCase    :						Union[float, List[float]]	,       _UpperCAmelCase    :						Union[float, List[float]]	,       _UpperCAmelCase    :						Optional[Union[str, ChannelDimension]] = None	,       **_UpperCAmelCase    :						List[str]	,       )				->		np.ndarray:


             '''simple docstring'''




             return normalize(_UpperCAmelCase	,       mean=_UpperCAmelCase	,       std=_UpperCAmelCase	,       data_format=_UpperCAmelCase	,       **_UpperCAmelCase					)







       def   lowercase__       (							self    :						Union[str, Any]	,       _UpperCAmelCase    :						ImageInput	,       _UpperCAmelCase    :						bool = None	,       _UpperCAmelCase    :						Dict[str, int] = None	,       _UpperCAmelCase    :						float = None	,       _UpperCAmelCase    :						PILImageResampling = None	,       _UpperCAmelCase    :						bool = None	,       _UpperCAmelCase    :						float = None	,       _UpperCAmelCase    :						bool = None	,       _UpperCAmelCase    :						Optional[Union[float, List[float]]] = None	,       _UpperCAmelCase    :						Optional[Union[float, List[float]]] = None	,       _UpperCAmelCase    :						Optional[Union[str, TensorType]] = None	,       _UpperCAmelCase    :						ChannelDimension = ChannelDimension.FIRST	,       **_UpperCAmelCase    :						Optional[int]	,       )				->		PIL.Image.Image:


             '''simple docstring'''




             UpperCAmelCase_    				=							do_resize if do_resize is not None else self.do_resize
             UpperCAmelCase_    				=							crop_pct if crop_pct is not None else self.crop_pct
             UpperCAmelCase_    				=							resample if resample is not None else self.resample
             UpperCAmelCase_    				=							do_rescale if do_rescale is not None else self.do_rescale
             UpperCAmelCase_    				=							rescale_factor if rescale_factor is not None else self.rescale_factor
             UpperCAmelCase_    				=							do_normalize if do_normalize is not None else self.do_normalize
             UpperCAmelCase_    				=							image_mean if image_mean is not None else self.image_mean
             UpperCAmelCase_    				=							image_std if image_std is not None else self.image_std

             UpperCAmelCase_    				=							size if size is not None else self.size
             UpperCAmelCase_    				=							get_size_dict(_UpperCAmelCase	,       default_to_square=_UpperCAmelCase					)

             UpperCAmelCase_    				=							make_list_of_images(_UpperCAmelCase					)

             if not valid_images(_UpperCAmelCase					):
                   raise ValueError(
                       "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
                       "torch.Tensor, tf.Tensor or jax.ndarray."					)

             if do_resize and size is None or resample is None:
                   raise ValueError("Size and resample must be specified if do_resize is True."					)

             if do_resize and size["shortest_edge"] < 384 and crop_pct is None:
                   raise ValueError("crop_pct must be specified if size < 384."					)

             if do_rescale and rescale_factor is None:
                   raise ValueError("Rescale factor must be specified if do_rescale is True."					)

             if do_normalize and (image_mean is None or image_std is None):
                   raise ValueError("Image mean and std must be specified if do_normalize is True."					)

             # All transformations expect numpy arrays.
             UpperCAmelCase_    				=							[to_numpy_array(_UpperCAmelCase					) for image in images]

             if do_resize:
                   UpperCAmelCase_    				=							[self.resize(image=_UpperCAmelCase	,       size=_UpperCAmelCase	,       crop_pct=_UpperCAmelCase	,       resample=_UpperCAmelCase					) for image in images]

             if do_rescale:
                   UpperCAmelCase_    				=							[self.rescale(image=_UpperCAmelCase	,       scale=_UpperCAmelCase					) for image in images]

             if do_normalize:
                   UpperCAmelCase_    				=							[self.normalize(image=_UpperCAmelCase	,       mean=_UpperCAmelCase	,       std=_UpperCAmelCase					) for image in images]

             UpperCAmelCase_    				=							[to_channel_dimension_format(_UpperCAmelCase	,       _UpperCAmelCase					) for image in images]

             UpperCAmelCase_    				=							{"pixel_values": images}
             return BatchFeature(data=_UpperCAmelCase	,       tensor_type=_UpperCAmelCase					)




