

"""simple docstring"""



import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


SCREAMING_SNAKE_CASE_					:	Union[str, Any]							  =						logging.getLogger(__name__)





def  _snake_case     (    ):
     A__										=  argparse.ArgumentParser(
         description="""Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."""			)
     parser.add_argument(
         """--dataset_name"""					,   type=UpperCAmelCase_					,   default="""wikitext"""					,   help="""Name of the training. Explore datasets at: hf.co/datasets."""					,   )
     parser.add_argument(
         """--dataset_config"""					,   type=UpperCAmelCase_					,   default="""wikitext-103-raw-v1"""					,   help="""Configuration name of the dataset."""			)
     parser.add_argument(
         """--tokenizer_name_or_path"""					,   type=UpperCAmelCase_					,   default="""sayakpaul/unigram-tokenizer-wikitext"""					,   help="""Tokenizer identifier. Can be a local filepath or a Hub identifier."""					,   )
     parser.add_argument(
         """--shard_size"""					,   type=UpperCAmelCase_					,   default=1000					,   help="""Number of entries to go in a single shard."""					,   )
     parser.add_argument("""--split"""					,   type=UpperCAmelCase_					,   default="""train"""					,   choices=["""train""", """test""", """validation"""]			)
     parser.add_argument(
         """--limit"""					,   default=UpperCAmelCase_					,   type=UpperCAmelCase_					,   help="""Limit the number of shards (used for debugging)."""					,   )
     parser.add_argument(
         """--max_length"""					,   type=UpperCAmelCase_					,   default=512					,   help="""Maximum sequence length. For training on TPUs, it helps to have a maximum"""
         """ sequence length that is a multiple of 8."""					,   )
     parser.add_argument(
         """--output_dir"""					,   default="""tf-tpu"""					,   type=UpperCAmelCase_					,   help="""Output directory where the TFRecord shards will be saved. If the"""
         """ path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"""
         """ shards will be directly saved to a Google Cloud Storage bucket."""					,   )

     A__										=  parser.parse_args()
     return args





def  _snake_case     (    UpperCAmelCase_  :					Any			):
     def fn(UpperCAmelCase_  :					Union[str, Any]			):
          return tokenizer(examples["""text"""]			)

     return fn





def  _snake_case     (    UpperCAmelCase_  :					Optional[Any]			):
     A__										=  []
     for i in range(len(tokenized_data["""input_ids"""]			)			):
          A__										=  {
              """input_ids""": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["""input_ids"""][i]			)			),
              """attention_mask""": tf.train.Feature(
                  intaa_list=tf.train.IntaaList(value=tokenized_data["""attention_mask"""][i]			)			),
          }
          A__										=  tf.train.Features(feature=UpperCAmelCase_			)
          A__										=  tf.train.Example(features=UpperCAmelCase_			)
          A__										=  example.SerializeToString()
          records.append(UpperCAmelCase_			)
     return records





def  _snake_case     (    UpperCAmelCase_  :					Optional[int]			):
     A__										=  datasets.load_dataset(args.dataset_name					,   args.dataset_config					,   split=args.split			)

     if args.limit is not None:
          A__										=  min(len(UpperCAmelCase_			)					,   args.limit			)
          A__										=  dataset.select(range(UpperCAmelCase_			)			)
          print(F"""Limiting the dataset to {args.limit} entries."""			)

     A__										=  AutoTokenizer.from_pretrained(args.tokenizer_name_or_path			)

     # Handle output directory creation.
     # For serializing into a Google Cloud Storage Bucket, one needs to first
     # create a bucket.
     if "gs" not in args.output_dir:
          if not os.path.exists(args.output_dir			):
               os.makedirs(args.output_dir			)
          A__										=  os.path.join(args.output_dir					,   args.split			)
          if not os.path.exists(UpperCAmelCase_			):
               os.makedirs(UpperCAmelCase_			)
     else:
          A__										=  os.path.join(args.output_dir					,   args.split			)

     # Tokenize the whole dataset at once.
     A__										=  tokenize_function(UpperCAmelCase_			)
     A__										=  dataset.map(UpperCAmelCase_					,   batched=UpperCAmelCase_					,   num_proc=4					,   remove_columns=["""text"""]			)

     # We need to concatenate all our texts together, and then split the result
     # into chunks of a fixed size, which we will call block_size. To do this, we
     # will use the map method again, with the option batched=True. When we use batched=True,
     # the function we pass to map() will be passed multiple inputs at once, allowing us
     # to group them into more or fewer examples than we had in the input.
     # This allows us to create our new fixed-length samples. The advantage of this
     # method is that we don't lose a whole lot of content from the dataset compared to the
     # case where we simply tokenize with a pre-defined max_length.

     def group_texts(UpperCAmelCase_  :					Tuple			):
          # Concatenate all texts.
          A__										=  {k: sum(examples[k]					,   []			) for k in examples.keys()}
          A__										=  len(concatenated_examples[list(examples.keys()			)[0]]			)
          # We drop the small remainder, though you could add padding instead if the model supports it
          # In this, as in all things, we advise you to follow your heart 🫀
          A__										=  (total_length // args.max_length) * args.max_length
          # Split by chunks of max_len.
          A__										=  {
              k: [t[i : i + args.max_length] for i in range(0					,   UpperCAmelCase_					,   args.max_length			)]
              for k, t in concatenated_examples.items()
          }
          return result

     A__										=  dataset_tokenized.map(UpperCAmelCase_					,   batched=UpperCAmelCase_					,   batch_size=1000					,   num_proc=4			)

     A__										=  0
     A__										=  0
     for shard in range(0					,   len(UpperCAmelCase_			)					,   args.shard_size			):
          A__										=  grouped_dataset[shard : shard + args.shard_size]
          A__										=  len(dataset_snapshot["""input_ids"""]			)
          A__										=  os.path.join(UpperCAmelCase_					,   F"""dataset-{shard_count}-{records_containing}.tfrecord"""			)
          A__										=  get_serialized_examples(UpperCAmelCase_			)

          with tf.io.TFRecordWriter(UpperCAmelCase_			) as out_file:
               for i in range(len(UpperCAmelCase_			)			):
                    A__										=  serialized_examples[i]
                    out_file.write(UpperCAmelCase_			)
               print("""Wrote file {} containing {} records""".format(UpperCAmelCase_					,   UpperCAmelCase_			)			)

          shard_count += 1
          total_records += records_containing

     with open(F"""split-{args.split}-records-count.txt"""					,   """w"""			) as f:
          print(F"""Total {args.split} records: {total_records}"""					,   file=UpperCAmelCase_			)


if __name__ == "__main__":
 SCREAMING_SNAKE_CASE_					:	Union[str, Any]							  =						parse_args()
 main(args)









"""simple docstring"""



def  _snake_case     (    UpperCAmelCase_  :					int			):
     A__										=  generate_pascal_triangle(UpperCAmelCase_			)
     for row_idx in range(UpperCAmelCase_			):
          # Print left spaces
          for _ in range(num_rows - row_idx - 1			):
               print(end=""" """			)
          # Print row values
          for col_idx in range(row_idx + 1			):
               if col_idx != row_idx:
                    print(triangle[row_idx][col_idx]					,   end=""" """			)
               else:
                    print(triangle[row_idx][col_idx]					,   end=""""""			)
          print()





def  _snake_case     (    UpperCAmelCase_  :					int			):

     if not isinstance(UpperCAmelCase_					,   UpperCAmelCase_			):
          raise TypeError("""The input value of 'num_rows' should be 'int'"""			)

     if num_rows == 0:
          return []
     elif num_rows < 0:
          raise ValueError(
              """The input value of 'num_rows' should be greater than or equal to 0"""			)

     A__										=  []
     for current_row_idx in range(UpperCAmelCase_			):
          A__										=  populate_current_row(UpperCAmelCase_					,   UpperCAmelCase_			)
          triangle.append(UpperCAmelCase_			)
     return triangle





def  _snake_case     (    UpperCAmelCase_  :					list[list[int]]					,   UpperCAmelCase_  :					int			):
     A__										=  [-1] * (current_row_idx + 1)
     # first and last elements of current row are equal to 1
     A__			,				A__										=  1, 1
     for current_col_idx in range(1					,   UpperCAmelCase_			):
          calculate_current_element(
              UpperCAmelCase_					,   UpperCAmelCase_					,   UpperCAmelCase_					,   UpperCAmelCase_			)
     return current_row





def  _snake_case     (    UpperCAmelCase_  :					list[list[int]]					,   UpperCAmelCase_  :					list[int]					,   UpperCAmelCase_  :					int					,   UpperCAmelCase_  :					int					,   ):
     A__										=  triangle[current_row_idx - 1][current_col_idx - 1]
     A__										=  triangle[current_row_idx - 1][current_col_idx]
     A__										=  above_to_left_elt + above_to_right_elt





def  _snake_case     (    UpperCAmelCase_  :					int			):

     if not isinstance(UpperCAmelCase_					,   UpperCAmelCase_			):
          raise TypeError("""The input value of 'num_rows' should be 'int'"""			)

     if num_rows == 0:
          return []
     elif num_rows < 0:
          raise ValueError(
              """The input value of 'num_rows' should be greater than or equal to 0"""			)

     A__										=  [[1]]

     for row_index in range(1					,   UpperCAmelCase_			):
          A__										=  [0] + result[-1] + [0]
          A__										=  row_index + 1
          # Calculate the number of distinct elements in a row
          A__										=  sum(divmod(UpperCAmelCase_					,   2			)			)
          A__										=  [
              temp_row[i - 1] + temp_row[i] for i in range(1					,   distinct_elements + 1			)
          ]
          A__										=  row_first_half[: (row_index + 1) // 2]
          row_second_half.reverse()
          A__										=  row_first_half + row_second_half
          result.append(UpperCAmelCase_			)

     return result





def  _snake_case     (    ):
     from collections.abc import Callable
     from timeit import timeit

     def benchmark_a_function(UpperCAmelCase_  :					Callable					,   UpperCAmelCase_  :					int			) -> None:
          A__										=  F"""{func.__name__}({value})"""
          A__										=  timeit(F"""__main__.{call}"""					,   setup="""import __main__"""			)
          # print(f"{call:38} = {func(value)} -- {timing:.4f} seconds")
          print(F"""{call:38} -- {timing:.4f} seconds"""			)

     for value in range(15			):  # (1, 7, 14):
          for func in (generate_pascal_triangle, generate_pascal_triangle_optimized):
               benchmark_a_function(UpperCAmelCase_					,   UpperCAmelCase_			)
          print()


if __name__ == "__main__":
 import doctest

 doctest.testmod()
 benchmark()






