



'''simple docstring'''



import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


lowerCAmelCase	:		Dict				=   ['''\nclass''', '''\ndef''', '''\n#''', '''\n@''', '''\nprint''', '''\nif''']






class 					UpperCAmelCase__	(		__SCREAMING_SNAKE_CASE ):



     def __init__(    self			,				UpperCamelCase			,				UpperCamelCase			,				UpperCamelCase=None			,				UpperCamelCase=1     )     ->			List[str]:
      __lowerCAmelCase												=  tokenizer
      __lowerCAmelCase												=  dataset
      __lowerCAmelCase												=  len(__snake_case     ) if n_tasks is None else n_tasks
      __lowerCAmelCase												=  n_copies



     def __iter__(    self     )     ->			Union[str, Any]:
      __lowerCAmelCase												=  []
      for task in range(self.n_tasks     ):
       # without strip, the model generate commented codes ...
       prompts.append(self.tokenizer.eos_token + self.dataset[task]["prompt"].strip()     )
      __lowerCAmelCase												=  self.tokenizer(__snake_case			,				padding=__snake_case			,				return_tensors="pt"     )
      for task in range(self.n_tasks     ):
       for _ in range(self.n_copies     ):
        yield {
            "ids": outputs.input_ids[task],
            "task_id": task,
            "input_len": outputs.attention_mask[task].sum(),
        }






class 					UpperCAmelCase__	(		__SCREAMING_SNAKE_CASE ):



     def __init__(    self			,				UpperCamelCase			,				UpperCamelCase			,				UpperCamelCase     )     ->			Union[str, Any]:
      __lowerCAmelCase												=  start_length
      __lowerCAmelCase												=  eof_strings
      __lowerCAmelCase												=  tokenizer



     def __call__(    self			,				UpperCamelCase			,				UpperCamelCase			,				**UpperCamelCase     )     ->			Optional[int]:
      __lowerCAmelCase												=  self.tokenizer.batch_decode(input_ids[:, self.start_length :]     )
      __lowerCAmelCase												=  []
      for decoded_generation in decoded_generations:
       done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings     )     )
      return all(__snake_case     )



def   __lowerCAmelCase					(							lowerCamelCase   :						Tuple					):






 '''simple docstring'''




 __lowerCAmelCase												=  re.split("(%s)" % "|".join(a_					)				,		a_					)
 # last string should be ""
 return "".join(string_list[:-2]					)



def   __lowerCAmelCase					(							lowerCamelCase   :						List[str]				,		lowerCamelCase   :						str				,		lowerCamelCase   :						Optional[int]				,		lowerCamelCase   :						str				,		lowerCamelCase   :						Optional[int]				,		lowerCamelCase   :						Tuple=20				,		**lowerCamelCase   :						int					):






 '''simple docstring'''




 __lowerCAmelCase												=  defaultdict(a_					)  # dict of list of generated tokens
 for step, batch in tqdm(enumerate(a_					)					):
  with torch.no_grad():
   __lowerCAmelCase												=  batch['''ids'''].shape[-1]
   __lowerCAmelCase												=  accelerator.unwrap_model(a_					).generate(
       input_ids=batch["ids"][:, : batch["input_len"]]				,		num_return_sequences=a_				,		**a_					)
   # each task is generated batch_size times
   __lowerCAmelCase												=  batch['''task_id'''].repeat(a_					)
   __lowerCAmelCase												=  accelerator.pad_across_processes(
       a_				,		dim=1				,		pad_index=tokenizer.pad_token_id					)

   __lowerCAmelCase												=  accelerator.gather((generated_tokens, generated_tasks)					)
   __lowerCAmelCase												=  generated_tokens.cpu().numpy()
   __lowerCAmelCase												=  generated_tasks.cpu().numpy()

   for task, generated_tokens in zip(a_				,		a_					):
    gen_token_dict[task].append(a_					)

 __lowerCAmelCase												=  [[] for _ in range(a_					)]
 for task, generated_tokens in gen_token_dict.items():
  for s in generated_tokens:
   __lowerCAmelCase												=  tokenizer.decode(a_				,		skip_special_tokens=a_				,		clean_up_tokenization_spaces=a_					)
   code_gens[task].append(remove_last_block(a_					)					)
 return code_gens



def   __lowerCAmelCase					(							):






 '''simple docstring'''




 __lowerCAmelCase												=  HfArgumentParser(a_					)
 __lowerCAmelCase												=  parser.parse_args()

 transformers.logging.set_verbosity_error()
 # enables code execution in code_eval metric
 __lowerCAmelCase												=  args.HF_ALLOW_CODE_EVAL
 # make sure tokenizer plays nice with multiprocessing
 __lowerCAmelCase												=  '''false'''

 if args.num_workers is None:
  __lowerCAmelCase												=  multiprocessing.cpu_count()

 # Use dataset load to feed to accelerate
 __lowerCAmelCase												=  Accelerator()
 set_seed(args.seed				,		device_specific=a_					)

 # Load model and tokenizer
 __lowerCAmelCase												=  AutoTokenizer.from_pretrained(args.model_ckpt					)
 __lowerCAmelCase												=  tokenizer.eos_token
 __lowerCAmelCase												=  AutoModelForCausalLM.from_pretrained(args.model_ckpt					)

 # Generation settings
 __lowerCAmelCase												=  {
     '''do_sample''': args.do_sample,
     '''temperature''': args.temperature,
     '''max_new_tokens''': args.max_new_tokens,
     '''top_p''': args.top_p,
     '''top_k''': args.top_k,
     '''stopping_criteria''': StoppingCriteriaList([EndOfFunctionCriteria(0				,		a_				,		a_					)]					),
 }

 # Load evaluation dataset and metric
 __lowerCAmelCase												=  load_dataset("openai_humaneval"					)
 __lowerCAmelCase												=  load_metric("code_eval"					)

 __lowerCAmelCase												=  args.num_tasks if args.num_tasks is not None else len(human_eval["test"]					)
 __lowerCAmelCase												=  args.n_samples // args.batch_size

 __lowerCAmelCase												=  TokenizedDataset(a_				,		human_eval["test"]				,		n_copies=a_				,		n_tasks=a_					)
 # do not confuse args.batch_size, which is actually the num_return_sequences
 __lowerCAmelCase												=  DataLoader(a_				,		batch_size=1					)

 # Run a quick test to see if code evaluation is enabled
 try:
  __lowerCAmelCase												=  code_eval_metric.compute(references=[""]				,		predictions=[[""]]					)
 except ValueError as exception:
  print(
      "Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL=\"1\"`"
      " flag to enable code evaluation."					)
  raise exception

 __lowerCAmelCase												=  accelerator.prepare(a_				,		a_					)

 __lowerCAmelCase												=  complete_code(
     a_				,		a_				,		a_				,		a_				,		n_tasks=a_				,		batch_size=args.batch_size				,		**a_				,		)

 if accelerator.is_main_process:
  __lowerCAmelCase												=  []

  for task in tqdm(range(a_					)					):
   __lowerCAmelCase												=  human_eval['''test'''][task]['''test''']
   __lowerCAmelCase												=  f'''check({human_eval['test'][task]['entry_point']})'''
   references.append("\n" + test_func + "\n" + entry_point					)

  # Evaluate completions with "code_eval" metric
  __lowerCAmelCase												=  code_eval_metric.compute(
      references=a_				,		predictions=a_				,		num_workers=args.num_workers					)
  print(f'''Results: {pass_at_k}'''					)

  # Save results to json file
  with open(args.output_file				,		"w"					) as fp:
   json.dump(a_				,		a_					)


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
 main()




'''simple docstring'''



import requests
from bsa import BeautifulSoup



def   __lowerCAmelCase					(							lowerCamelCase   :						str = "https://www.worldometers.info/coronavirus"					):






	'''simple docstring'''




	__lowerCAmelCase												=  BeautifulSoup(requests.get(lowerCamelCase					).text				,		"html.parser"					)
	__lowerCAmelCase												=  soup.findAll("h1"					)
	__lowerCAmelCase												=  soup.findAll("div"				,		{"class": "maincounter-number"}					)
	keys += soup.findAll("span"				,		{"class": "panel-title"}					)
	values += soup.findAll("div"				,		{"class": "number-table-main"}					)
	return {key.text.strip(): value.text.strip() for key, value in zip(lowerCamelCase				,		lowerCamelCase					)}


if __name__ == "__main__":
	print('''\033[1m''' + '''COVID-19 Status of the World''' + '''\033[0m\n''')
	for key, value in world_covidaa_stats().items():
		print(f'{key}\n{value}\n')