



import multiprocessing
import time

from arguments import PretokenizationArguments
from datasets import load_dataset

from transformers import AutoTokenizer, HfArgumentParser







def    _snake_case (       __snake_case    )     ->     List[str]:






      '''simple docstring'''







      UpperCAmelCase_  :	int				      =		{}
      UpperCAmelCase_  :	Optional[int]				      =		tokenizer(example["content"]					,  truncation=__snake_case    )["input_ids"]
      UpperCAmelCase_  :	Tuple				      =		len(example["content"]    ) / len(output["input_ids"]    )
      return output


__lowerCamelCase         =			HfArgumentParser(PretokenizationArguments)
__lowerCamelCase         =			parser.parse_args()
if args.num_workers is None:
    __lowerCamelCase         =			multiprocessing.cpu_count()
__lowerCamelCase         =			AutoTokenizer.from_pretrained(args.tokenizer_dir)

__lowerCamelCase         =			time.time()
__lowerCamelCase         =			load_dataset(args.dataset_name, split='''train''')
print(F'''Dataset loaded in {time.time()-t_start:.2f}s''')

__lowerCamelCase         =			time.time()
__lowerCamelCase         =			ds.map(
    tokenize,
    num_proc=args.num_workers,
    remove_columns=[
        '''repo_name''',
        '''path''',
        '''copies''',
        '''size''',
        '''content''',
        '''license''',
        '''hash''',
        '''line_mean''',
        '''line_max''',
        '''alpha_frac''',
        '''autogenerated''',
    ],
)
print(F'''Dataset tokenized in {time.time()-t_start:.2f}s''')

__lowerCamelCase         =			time.time()
ds.push_to_hub(args.tokenized_data_repo)





print(F'''Data pushed to the hub in {time.time()-t_start:.2f}s''')





def    _snake_case (       __snake_case					,  __snake_case    )     ->     int:






      '''simple docstring'''







      return int((input_a, input_a).count(0    ) != 0    )







def    _snake_case (       )     ->     None:






      '''simple docstring'''







      assert nand_gate(0					,  0    ) == 1
      assert nand_gate(0					,  1    ) == 1
      assert nand_gate(1					,  0    ) == 1
      assert nand_gate(1					,  1    ) == 0


if __name__ == "__main__":
   print(nand_gate(0, 0))
   print(nand_gate(0, 1))
   print(nand_gate(1, 0))
   print(nand_gate(1, 1))
