





"""simple docstring"""




import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam


class  __UpperCamelCase       (		datasets.BeamBasedBuilder		):






							def 					__a     (     self   )			->				Any:
								return datasets.DatasetInfo(
								    features=datasets.Features({"content": datasets.Value("string"   )}   )   ,      supervised_keys=lowerCAmelCase__   ,      )






							def 					__a     (     self   ,      lowerCAmelCase__   ,      lowerCAmelCase__   )			->				Any:
								return [datasets.SplitGenerator(name=datasets.Split.TRAIN   ,      gen_kwargs={"examples": get_test_dummy_examples()}   )]






							def 					__a     (     self   ,      lowerCAmelCase__   ,      lowerCAmelCase__   )			->				List[Any]:
								import apache_beam as beam

								return pipeline | "Load Examples" >> beam.Create(lowerCAmelCase__   )


class  __UpperCamelCase       (		datasets.BeamBasedBuilder		):






							def 					__a     (     self   )			->				str:
								return datasets.DatasetInfo(
								    features=datasets.Features({"a": datasets.Sequence({"b": datasets.Value("string"   )}   )}   )   ,      supervised_keys=lowerCAmelCase__   ,      )






							def 					__a     (     self   ,      lowerCAmelCase__   ,      lowerCAmelCase__   )			->				Optional[int]:
								return [
								    datasets.SplitGenerator(name=datasets.Split.TRAIN   ,      gen_kwargs={"examples": get_test_nested_examples()}   )
								]






							def 					__a     (     self   ,      lowerCAmelCase__   ,      lowerCAmelCase__   )			->				str:
								import apache_beam as beam

								return pipeline | "Load Examples" >> beam.Create(lowerCAmelCase__   )






def    _SCREAMING_SNAKE_CASE	(			)				->List[str]:






	'''simple docstring'''






	return [(i, {"content": content}) for i, content in enumerate(["foo", "bar", "foobar"]      )]






def    _SCREAMING_SNAKE_CASE	(			)				->Tuple:






	'''simple docstring'''






	return [(i, {"a": {"b": [content]}}) for i, content in enumerate(["foo", "bar", "foobar"]      )]







class  __UpperCamelCase       (		a__		):






							@require_beam
							def 					__a     (     self   )			->				Optional[int]:
								a   :	List[Any]			=       len(get_test_dummy_examples()   )
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
									a   :	Any			=       DummyBeamDataset(cache_dir=lowerCAmelCase__   ,      beam_runner="DirectRunner"   )
									builder.download_and_prepare()
									self.assertTrue(
									    os.path.exists(
									        os.path.join(lowerCAmelCase__   ,      builder.name   ,      "default"   ,      "0.0.0"   ,      f"""{builder.name}-train.arrow"""   )   )   )
									self.assertDictEqual(builder.info.features   ,      datasets.Features({"content": datasets.Value("string"   )}   )   )
									a   :	Optional[Any]			=       builder.as_dataset()
									self.assertEqual(dset["train"].num_rows   ,      lowerCAmelCase__   )
									self.assertEqual(dset["train"].info.splits["train"].num_examples   ,      lowerCAmelCase__   )
									self.assertDictEqual(dset["train"][0]   ,      get_test_dummy_examples()[0][1]   )
									self.assertDictEqual(
									    dset["train"][expected_num_examples - 1]   ,      get_test_dummy_examples()[expected_num_examples - 1][1]   )
									self.assertTrue(
									    os.path.exists(os.path.join(lowerCAmelCase__   ,      builder.name   ,      "default"   ,      "0.0.0"   ,      "dataset_info.json"   )   )   )
									del dset






							@require_beam
							def 					__a     (     self   )			->				Union[str, Any]:
								import apache_beam as beam

								a   :	int			=       beam.io.parquetio.WriteToParquet

								a   :	Any			=       len(get_test_dummy_examples()   )
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
									a   :	Dict			=       DummyBeamDataset(cache_dir=lowerCAmelCase__   ,      beam_runner="DirectRunner"   )
									with patch("apache_beam.io.parquetio.WriteToParquet"   ) as write_parquet_mock:
										a   :	str			=       partial(lowerCAmelCase__   ,      num_shards=2   )
										builder.download_and_prepare()
									self.assertTrue(
									    os.path.exists(
									        os.path.join(
									            lowerCAmelCase__   ,      builder.name   ,      "default"   ,      "0.0.0"   ,      f"""{builder.name}-train-00000-of-00002.arrow"""   )   )   )
									self.assertTrue(
									    os.path.exists(
									        os.path.join(
									            lowerCAmelCase__   ,      builder.name   ,      "default"   ,      "0.0.0"   ,      f"""{builder.name}-train-00000-of-00002.arrow"""   )   )   )
									self.assertDictEqual(builder.info.features   ,      datasets.Features({"content": datasets.Value("string"   )}   )   )
									a   :	List[str]			=       builder.as_dataset()
									self.assertEqual(dset["train"].num_rows   ,      lowerCAmelCase__   )
									self.assertEqual(dset["train"].info.splits["train"].num_examples   ,      lowerCAmelCase__   )
									# Order is not preserved when sharding, so we just check that all the elements are there
									self.assertListEqual(sorted(dset["train"]["content"]   )   ,      sorted(["foo", "bar", "foobar"]   )   )
									self.assertTrue(
									    os.path.exists(os.path.join(lowerCAmelCase__   ,      builder.name   ,      "default"   ,      "0.0.0"   ,      "dataset_info.json"   )   )   )
									del dset






							@require_beam
							def 					__a     (     self   )			->				Optional[Any]:
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
									a   :	int			=       DummyBeamDataset(cache_dir=lowerCAmelCase__   )
									self.assertRaises(datasets.builder.MissingBeamOptions   ,      builder.download_and_prepare   )






							@require_beam
							def 					__a     (     self   )			->				Optional[Any]:
								a   :	List[Any]			=       len(get_test_nested_examples()   )
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
									a   :	int			=       NestedBeamDataset(cache_dir=lowerCAmelCase__   ,      beam_runner="DirectRunner"   )
									builder.download_and_prepare()
									self.assertTrue(
									    os.path.exists(
									        os.path.join(lowerCAmelCase__   ,      builder.name   ,      "default"   ,      "0.0.0"   ,      f"""{builder.name}-train.arrow"""   )   )   )
									self.assertDictEqual(
									    builder.info.features   ,      datasets.Features({"a": datasets.Sequence({"b": datasets.Value("string"   )}   )}   )   )
									a   :	Dict			=       builder.as_dataset()
									self.assertEqual(dset["train"].num_rows   ,      lowerCAmelCase__   )
									self.assertEqual(dset["train"].info.splits["train"].num_examples   ,      lowerCAmelCase__   )
									self.assertDictEqual(dset["train"][0]   ,      get_test_nested_examples()[0][1]   )
									self.assertDictEqual(
									    dset["train"][expected_num_examples - 1]   ,      get_test_nested_examples()[expected_num_examples - 1][1]   )
									self.assertTrue(
									    os.path.exists(os.path.join(lowerCAmelCase__   ,      builder.name   ,      "default"   ,      "0.0.0"   ,      "dataset_info.json"   )   )   )
									del dset









"""simple docstring"""




import multiprocessing
import time

from arguments import PretokenizationArguments
from datasets import load_dataset

from transformers import AutoTokenizer, HfArgumentParser






def    _SCREAMING_SNAKE_CASE	(			_lowercase		:					List[str]      )				->int:






	'''simple docstring'''






	a   :	int			=       {}
	a   :	Union[str, Any]			=       tokenizer(example["content"]					, truncation=_lowercase      )["input_ids"]
	a   :	Any			=       len(example["content"]      ) / len(output["input_ids"]      )
	return output


a		:				int  			=    HfArgumentParser(PretokenizationArguments)
a		:				Optional[int]  			=    parser.parse_args()
if args.num_workers is None:
		a		:				Tuple  			=    multiprocessing.cpu_count()
a		:				Optional[int]  			=    AutoTokenizer.from_pretrained(args.tokenizer_dir)

a		:				Dict  			=    time.time()
a		:				Tuple  			=    load_dataset(args.dataset_name, split='''train''')
print(F'''Dataset loaded in {time.time()-t_start:.2f}s''')

a		:				Dict  			=    time.time()
a		:				Tuple  			=    ds.map(
    tokenize,
    num_proc=args.num_workers,
    remove_columns=[
        '''repo_name''',
        '''path''',
        '''copies''',
        '''size''',
        '''content''',
        '''license''',
        '''hash''',
        '''line_mean''',
        '''line_max''',
        '''alpha_frac''',
        '''autogenerated''',
    ],
)
print(F'''Dataset tokenized in {time.time()-t_start:.2f}s''')

a		:				Tuple  			=    time.time()
ds.push_to_hub(args.tokenized_data_repo)







print(F'''Data pushed to the hub in {time.time()-t_start:.2f}s''')


