






'''simple docstring'''

import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


UpperCAmelCase__						:List[Any]          =			logging.getLogger(__name__)
def     __lowercase  ()   ->     Optional[int]:


     """simple docstring"""
     __lowerCamelCase						:       Optional[int]				   =				argparse.ArgumentParser(
         description="""Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."""     )
     parser.add_argument(
         """--dataset_name""",						type=_lowercase,						default="""wikitext""",						help="""Name of the training. Explore datasets at: hf.co/datasets.""",						)
     parser.add_argument(
         """--dataset_config""",						type=_lowercase,						default="""wikitext-103-raw-v1""",						help="""Configuration name of the dataset."""     )
     parser.add_argument(
         """--tokenizer_name_or_path""",						type=_lowercase,						default="""sayakpaul/unigram-tokenizer-wikitext""",						help="""Tokenizer identifier. Can be a local filepath or a Hub identifier.""",						)
     parser.add_argument(
         """--shard_size""",						type=_lowercase,						default=1_000,						help="""Number of entries to go in a single shard.""",						)
     parser.add_argument("""--split""",						type=_lowercase,						default="""train""",						choices=["""train""", """test""", """validation"""]     )
     parser.add_argument(
         """--limit""",						default=_lowercase,						type=_lowercase,						help="""Limit the number of shards (used for debugging).""",						)
     parser.add_argument(
         """--max_length""",						type=_lowercase,						default=512,						help="""Maximum sequence length. For training on TPUs, it helps to have a maximum"""
         """ sequence length that is a multiple of 8.""",						)
     parser.add_argument(
         """--output_dir""",						default="""tf-tpu""",						type=_lowercase,						help="""Output directory where the TFRecord shards will be saved. If the"""
         """ path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"""
         """ shards will be directly saved to a Google Cloud Storage bucket.""",						)

     __lowerCamelCase						:       Optional[int]				   =				parser.parse_args()
     return args
def     __lowercase  (_lowercase     )   ->     int:


     """simple docstring"""
     def fn(_lowercase     ):
          return tokenizer(examples["""text"""]     )

     return fn
def     __lowercase  (_lowercase     )   ->     Dict:


     """simple docstring"""
     __lowerCamelCase						:       Tuple				   =				[]
     for i in range(len(tokenized_data["""input_ids"""]     )     ):
          __lowerCamelCase						:       Tuple				   =				{
              """input_ids""": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["""input_ids"""][i]     )     ),
              """attention_mask""": tf.train.Feature(
                  intaa_list=tf.train.IntaaList(value=tokenized_data["""attention_mask"""][i]     )     ),
          }
          __lowerCamelCase						:       Union[str, Any]				   =				tf.train.Features(feature=_lowercase     )
          __lowerCamelCase						:       List[str]				   =				tf.train.Example(features=_lowercase     )
          __lowerCamelCase						:       int				   =				example.SerializeToString()
          records.append(_lowercase     )
     return records
def     __lowercase  (_lowercase     )   ->     Dict:


     """simple docstring"""
     __lowerCamelCase						:       int				   =				datasets.load_dataset(args.dataset_name,						args.dataset_config,						split=args.split     )

     if args.limit is not None:
          __lowerCamelCase						:       Dict				   =				min(len(_lowercase     ),						args.limit     )
          __lowerCamelCase						:       Optional[int]				   =				dataset.select(range(_lowercase     )     )
          print(f"Limiting the dataset to {args.limit} entries."     )

     __lowerCamelCase						:       Tuple				   =				AutoTokenizer.from_pretrained(args.tokenizer_name_or_path     )

     # Handle output directory creation.
     # For serializing into a Google Cloud Storage Bucket, one needs to first
     # create a bucket.
     if "gs" not in args.output_dir:
          if not os.path.exists(args.output_dir     ):
               os.makedirs(args.output_dir     )
          __lowerCamelCase						:       Optional[Any]				   =				os.path.join(args.output_dir,						args.split     )
          if not os.path.exists(_lowercase     ):
               os.makedirs(_lowercase     )
     else:
          __lowerCamelCase						:       str				   =				os.path.join(args.output_dir,						args.split     )

     # Tokenize the whole dataset at once.
     __lowerCamelCase						:       str				   =				tokenize_function(_lowercase     )
     __lowerCamelCase						:       Any				   =				dataset.map(_lowercase,						batched=_lowercase,						num_proc=4,						remove_columns=["""text"""]     )

     # We need to concatenate all our texts together, and then split the result
     # into chunks of a fixed size, which we will call block_size. To do this, we
     # will use the map method again, with the option batched=True. When we use batched=True,
     # the function we pass to map() will be passed multiple inputs at once, allowing us
     # to group them into more or fewer examples than we had in the input.
     # This allows us to create our new fixed-length samples. The advantage of this
     # method is that we don't lose a whole lot of content from the dataset compared to the
     # case where we simply tokenize with a pre-defined max_length.

     def group_texts(_lowercase     ):
          # Concatenate all texts.
          __lowerCamelCase						:       Any				   =				{k: sum(examples[k],						[]     ) for k in examples.keys()}
          __lowerCamelCase						:       str				   =				len(concatenated_examples[list(examples.keys()     )[0]]     )
          # We drop the small remainder, though you could add padding instead if the model supports it
          # In this, as in all things, we advise you to follow your heart 🫀
          __lowerCamelCase						:       Optional[int]				   =				(total_length // args.max_length) * args.max_length
          # Split by chunks of max_len.
          __lowerCamelCase						:       Union[str, Any]				   =				{
              k: [t[i : i + args.max_length] for i in range(0,						_lowercase,						args.max_length     )]
              for k, t in concatenated_examples.items()
          }
          return result

     __lowerCamelCase						:       Dict				   =				dataset_tokenized.map(_lowercase,						batched=_lowercase,						batch_size=1_000,						num_proc=4     )

     __lowerCamelCase						:       Any				   =				0
     __lowerCamelCase						:       List[str]				   =				0
     for shard in range(0,						len(_lowercase     ),						args.shard_size     ):
          __lowerCamelCase						:       Any				   =				grouped_dataset[shard : shard + args.shard_size]
          __lowerCamelCase						:       Union[str, Any]				   =				len(dataset_snapshot["""input_ids"""]     )
          __lowerCamelCase						:       Union[str, Any]				   =				os.path.join(_lowercase,						f"dataset-{shard_count}-{records_containing}.tfrecord"     )
          __lowerCamelCase						:       str				   =				get_serialized_examples(_lowercase     )

          with tf.io.TFRecordWriter(_lowercase     ) as out_file:
               for i in range(len(_lowercase     )     ):
                    __lowerCamelCase						:       Dict				   =				serialized_examples[i]
                    out_file.write(_lowercase     )
               print("""Wrote file {} containing {} records""".format(_lowercase,						_lowercase     )     )

          shard_count += 1
          total_records += records_containing

     with open(f"split-{args.split}-records-count.txt",						"""w"""     ) as f:
          print(f"Total {args.split} records: {total_records}",						file=_lowercase     )


if __name__ == "__main__":
   UpperCAmelCase__						:Optional[int]          =			parse_args()
   main(args)











'''simple docstring'''

from ..utils import DummyObject, requires_backends







class 	SCREAMING_SNAKE_CASE	(      metaclass=lowerCAmelCase_  ):
       snake_case__    :     Union[str, Any]   		=			['torch', 'torchsde']




       def __init__(					self      : int	,    *A__      : Optional[Any]	,    **A__      : Dict					):


            """simple docstring"""







            requires_backends(self	,    ["""torch""", """torchsde"""]					)




       @classmethod
       def 					a_					(					cls      : Optional[Any]	,    *A__      : str	,    **A__      : Optional[Any]					):


            """simple docstring"""







            requires_backends(cls	,    ["""torch""", """torchsde"""]					)




       @classmethod
       def 					a_					(					cls      : Union[str, Any]	,    *A__      : Any	,    **A__      : Union[str, Any]					):


            """simple docstring"""







            requires_backends(cls	,    ["""torch""", """torchsde"""]					)



