






"""simple docstring"""



import copy
import tempfile
import unittest

from huggingface_hub import HfFolder, delete_repo
from parameterized import parameterized
from requests.exceptions import HTTPError

from transformers import AutoConfig, GenerationConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test






class 							__SCREAMING_SNAKE_CASE						(				unittest.TestCase		):


       @parameterized.expand([(None,), ("""foo.json""",)]							)
       def 						_UpperCamelCase       (			self      :						Dict			,    snake_case      :						Tuple							):
             '''simple docstring'''





             A__						:  Optional[int]		  =  GenerationConfig(
                 do_sample=snake_case			,    temperature=0.7			,    length_penalty=1.0			,    bad_words_ids=[[1, 2, 3], [4, 5]]			,    )
             with tempfile.TemporaryDirectory() as tmp_dir:
                   config.save_pretrained(snake_case			,    config_name=snake_case							)
                   A__						:  List[str]		  =  GenerationConfig.from_pretrained(snake_case			,    config_name=snake_case							)

             # Checks parameters that were specified
             self.assertEqual(loaded_config.do_sample			,    snake_case							)
             self.assertEqual(loaded_config.temperature			,    0.7							)
             self.assertEqual(loaded_config.length_penalty			,    1.0							)
             self.assertEqual(loaded_config.bad_words_ids			,    [[1, 2, 3], [4, 5]]							)

             # Checks parameters that were not specified (defaults)
             self.assertEqual(loaded_config.top_k			,    50							)
             self.assertEqual(loaded_config.max_length			,    20							)
             self.assertEqual(loaded_config.max_time			,    snake_case							)


       def 						_UpperCamelCase       (			self      :						Any							):
             '''simple docstring'''





             A__						:  Optional[int]		  =  AutoConfig.from_pretrained("""gpt2"""							)
             A__						:  Optional[int]		  =  GenerationConfig.from_model_config(snake_case							)
             A__						:  List[Any]		  =  GenerationConfig()

             # The generation config has loaded a few non-default parameters from the model config
             self.assertNotEqual(snake_case			,    snake_case							)

             # One of those parameters is eos_token_id -- check if it matches
             self.assertNotEqual(generation_config_from_model.eos_token_id			,    default_generation_config.eos_token_id							)
             self.assertEqual(generation_config_from_model.eos_token_id			,    model_config.eos_token_id							)


       def 						_UpperCamelCase       (			self      :						Dict							):
             '''simple docstring'''





             A__						:  str		  =  GenerationConfig()
             A__						:  Union[str, Any]		  =  {
                 """max_new_tokens""": 1024,
                 """foo""": """bar""",
             }
             A__						:  List[Any]		  =  copy.deepcopy(snake_case							)
             A__						:  List[str]		  =  generation_config.update(**snake_case							)

             # update_kwargs was not modified (no side effects)
             self.assertEqual(snake_case			,    snake_case							)

             # update_kwargs was used to update the config on valid attributes
             self.assertEqual(generation_config.max_new_tokens			,    1024							)

             # `.update()` returns a dictionary of unused kwargs
             self.assertEqual(snake_case			,    {"""foo""": """bar"""}							)


       def 						_UpperCamelCase       (			self      :						str							):
             '''simple docstring'''





             A__						:  List[str]		  =  GenerationConfig()
             A__						:  Tuple		  =  """bar"""

             with tempfile.TemporaryDirectory("""test-generation-config"""							) as tmp_dir:
                   generation_config.save_pretrained(snake_case							)

                   A__						:  Union[str, Any]		  =  GenerationConfig.from_pretrained(snake_case							)
             # update_kwargs was used to update the config on valid attributes
             self.assertEqual(new_config.foo			,    """bar"""							)

             A__						:  List[Any]		  =  GenerationConfig.from_model_config(snake_case							)
             assert not hasattr(snake_case			,    """foo"""							)  # no new kwargs should be initialized if from config


       def 						_UpperCamelCase       (			self      :						Any							):
             '''simple docstring'''





             A__						:  Dict		  =  GenerationConfig()
             self.assertEqual(default_config.temperature			,    1.0							)
             self.assertEqual(default_config.do_sample			,    snake_case							)
             self.assertEqual(default_config.num_beams			,    1							)

             A__						:  List[str]		  =  GenerationConfig(
                 do_sample=snake_case			,    temperature=0.7			,    length_penalty=1.0			,    bad_words_ids=[[1, 2, 3], [4, 5]]			,    )
             self.assertEqual(config.temperature			,    0.7							)
             self.assertEqual(config.do_sample			,    snake_case							)
             self.assertEqual(config.num_beams			,    1							)

             with tempfile.TemporaryDirectory() as tmp_dir:
                   config.save_pretrained(snake_case							)
                   A__						:  Any		  =  GenerationConfig.from_pretrained(snake_case			,    temperature=1.0							)

             self.assertEqual(loaded_config.temperature			,    1.0							)
             self.assertEqual(loaded_config.do_sample			,    snake_case							)
             self.assertEqual(loaded_config.num_beams			,    1							)  # default value






@is_staging_test
class 							__SCREAMING_SNAKE_CASE						(				unittest.TestCase		):


       @classmethod
       def 						_UpperCamelCase       (			cls      :						Optional[Any]							):
             '''simple docstring'''





             A__						:  List[Any]		  =  TOKEN
             HfFolder.save_token(snake_case							)


       @classmethod
       def 						_UpperCamelCase       (			cls      :						Dict							):
             '''simple docstring'''





             try:
                   delete_repo(token=cls._token			,    repo_id="""test-generation-config"""							)
             except HTTPError:
                   pass

             try:
                   delete_repo(token=cls._token			,    repo_id="""valid_org/test-generation-config-org"""							)
             except HTTPError:
                   pass


       def 						_UpperCamelCase       (			self      :						Optional[Any]							):
             '''simple docstring'''





             A__						:  List[str]		  =  GenerationConfig(
                 do_sample=snake_case			,    temperature=0.7			,    length_penalty=1.0			,    )
             config.push_to_hub("""test-generation-config"""			,    use_auth_token=self._token							)

             A__						:  Union[str, Any]		  =  GenerationConfig.from_pretrained(F'{USER}/test-generation-config'							)
             for k, v in config.to_dict().items():
                   if k != "transformers_version":
                         self.assertEqual(snake_case			,    getattr(snake_case			,    snake_case							)							)

        # Reset repo
             delete_repo(token=self._token			,    repo_id="""test-generation-config"""							)

             # Push to hub via save_pretrained
             with tempfile.TemporaryDirectory() as tmp_dir:
                   config.save_pretrained(
                       snake_case			,    repo_id="""test-generation-config"""			,    push_to_hub=snake_case			,    use_auth_token=self._token							)

             A__						:  Any		  =  GenerationConfig.from_pretrained(F'{USER}/test-generation-config'							)
             for k, v in config.to_dict().items():
                   if k != "transformers_version":
                         self.assertEqual(snake_case			,    getattr(snake_case			,    snake_case							)							)


       def 						_UpperCamelCase       (			self      :						Optional[int]							):
             '''simple docstring'''





             A__						:  List[str]		  =  GenerationConfig(
                 do_sample=snake_case			,    temperature=0.7			,    length_penalty=1.0			,    )
             config.push_to_hub("""valid_org/test-generation-config-org"""			,    use_auth_token=self._token							)

             A__						:  str		  =  GenerationConfig.from_pretrained("""valid_org/test-generation-config-org"""							)
             for k, v in config.to_dict().items():
                   if k != "transformers_version":
                         self.assertEqual(snake_case			,    getattr(snake_case			,    snake_case							)							)

        # Reset repo
             delete_repo(token=self._token			,    repo_id="""valid_org/test-generation-config-org"""							)

             # Push to hub via save_pretrained
             with tempfile.TemporaryDirectory() as tmp_dir:
                   config.save_pretrained(
                       snake_case			,    repo_id="""valid_org/test-generation-config-org"""			,    push_to_hub=snake_case			,    use_auth_token=self._token							)

             A__						:  Union[str, Any]		  =  GenerationConfig.from_pretrained("""valid_org/test-generation-config-org"""							)
             for k, v in config.to_dict().items():
                   if k != "transformers_version":
                         self.assertEqual(snake_case			,    getattr(snake_case			,    snake_case							)							)














"""simple docstring"""



import multiprocessing
import time

from arguments import PretokenizationArguments
from datasets import load_dataset

from transformers import AutoTokenizer, HfArgumentParser




def      _lowerCAmelCase		(							UpperCAmelCase__       :					List[str]      )		->str:
						A__						:  Tuple		  =  {}
						A__						:  Union[str, Any]		  =  tokenizer(example["""content"""],						truncation=UpperCAmelCase__      )["""input_ids"""]
						A__						:  Any		  =  len(example["""content"""]      ) / len(output["""input_ids"""]      )
						return output


A_    =							HfArgumentParser(PretokenizationArguments)
A_    =							parser.parse_args()
if args.num_workers is None:
						A_    =							multiprocessing.cpu_count()
A_    =							AutoTokenizer.from_pretrained(args.tokenizer_dir)

A_    =							time.time()
A_    =							load_dataset(args.dataset_name, split='''train''')
print(F'Dataset loaded in {time.time()-t_start:.2f}s')

A_    =							time.time()
A_    =							ds.map(
    tokenize,
    num_proc=args.num_workers,
    remove_columns=[
        '''repo_name''',
        '''path''',
        '''copies''',
        '''size''',
        '''content''',
        '''license''',
        '''hash''',
        '''line_mean''',
        '''line_max''',
        '''alpha_frac''',
        '''autogenerated''',
    ],
)
print(F'Dataset tokenized in {time.time()-t_start:.2f}s')

A_    =							time.time()
ds.push_to_hub(args.tokenized_data_repo)

print(F'Data pushed to the hub in {time.time()-t_start:.2f}s')






