






"""simple docstring"""
import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


lowerCamelCase_       			=   logging.getLogger(__name__)

def        snake_case   (     ):
			UpperCAmelCase_  :				List[Any]      =      argparse.ArgumentParser(
			    description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."		)
			parser.add_argument(
			    "--dataset_name"			,type=A__			,default="wikitext"			,help="Name of the training. Explore datasets at: hf.co/datasets."			,)
			parser.add_argument(
			    "--dataset_config"			,type=A__			,default="wikitext-103-raw-v1"			,help="Configuration name of the dataset."		)
			parser.add_argument(
			    "--tokenizer_name_or_path"			,type=A__			,default="sayakpaul/unigram-tokenizer-wikitext"			,help="Tokenizer identifier. Can be a local filepath or a Hub identifier."			,)
			parser.add_argument(
			    "--shard_size"			,type=A__			,default=10_00			,help="Number of entries to go in a single shard."			,)
			parser.add_argument("--split"			,type=A__			,default="train"			,choices=["train", "test", "validation"]		)
			parser.add_argument(
			    "--limit"			,default=A__			,type=A__			,help="Limit the number of shards (used for debugging)."			,)
			parser.add_argument(
			    "--max_length"			,type=A__			,default=5_12			,help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
			    " sequence length that is a multiple of 8."			,)
			parser.add_argument(
			    "--output_dir"			,default="tf-tpu"			,type=A__			,help="Output directory where the TFRecord shards will be saved. If the"
			    " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
			    " shards will be directly saved to a Google Cloud Storage bucket."			,)

			UpperCAmelCase_  :				Dict      =      parser.parse_args()
			return args

def        snake_case   (     A__		):
			def fn(A__		):
						return tokenizer(examples["text"]		)

			return fn

def        snake_case   (     A__		):
			UpperCAmelCase_  :				Optional[Any]      =      []
			for i in range(len(tokenized_data["input_ids"]		)		):
						UpperCAmelCase_  :				int      =      {
						    "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]		)		),
						    "attention_mask": tf.train.Feature(
						        intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]		)		),
						}
						UpperCAmelCase_  :				Tuple      =      tf.train.Features(feature=A__		)
						UpperCAmelCase_  :				str      =      tf.train.Example(features=A__		)
						UpperCAmelCase_  :				Tuple      =      example.SerializeToString()
						records.append(A__		)
			return records

def        snake_case   (     A__		):
			UpperCAmelCase_  :				Tuple      =      datasets.load_dataset(args.dataset_name			,args.dataset_config			,split=args.split		)

			if args.limit is not None:
						UpperCAmelCase_  :				Dict      =      min(len(A__		)			,args.limit		)
						UpperCAmelCase_  :				Optional[Any]      =      dataset.select(range(A__		)		)
						print(F"""Limiting the dataset to {args.limit} entries."""		)

			UpperCAmelCase_  :				str      =      AutoTokenizer.from_pretrained(args.tokenizer_name_or_path		)

			# Handle output directory creation.
			# For serializing into a Google Cloud Storage Bucket, one needs to first
			# create a bucket.
			if "gs" not in args.output_dir:
						if not os.path.exists(args.output_dir		):
									os.makedirs(args.output_dir		)
						UpperCAmelCase_  :				Any      =      os.path.join(args.output_dir			,args.split		)
						if not os.path.exists(A__		):
									os.makedirs(A__		)
			else:
						UpperCAmelCase_  :				List[Any]      =      os.path.join(args.output_dir			,args.split		)

			# Tokenize the whole dataset at once.
			UpperCAmelCase_  :				Optional[int]      =      tokenize_function(A__		)
			UpperCAmelCase_  :				str      =      dataset.map(A__			,batched=A__			,num_proc=4			,remove_columns=["text"]		)

			# We need to concatenate all our texts together, and then split the result
			# into chunks of a fixed size, which we will call block_size. To do this, we
			# will use the map method again, with the option batched=True. When we use batched=True,
			# the function we pass to map() will be passed multiple inputs at once, allowing us
			# to group them into more or fewer examples than we had in the input.
			# This allows us to create our new fixed-length samples. The advantage of this
			# method is that we don't lose a whole lot of content from the dataset compared to the
			# case where we simply tokenize with a pre-defined max_length.

			def group_texts(A__		):
						# Concatenate all texts.
						UpperCAmelCase_  :				List[Any]      =      {k: sum(examples[k]			,[]		) for k in examples.keys()}
						UpperCAmelCase_  :				Optional[int]      =      len(concatenated_examples[list(examples.keys()		)[0]]		)
						# We drop the small remainder, though you could add padding instead if the model supports it
						# In this, as in all things, we advise you to follow your heart 🫀
						UpperCAmelCase_  :				Dict      =      (total_length // args.max_length) * args.max_length
						# Split by chunks of max_len.
						UpperCAmelCase_  :				Any      =      {
						    k: [t[i : i + args.max_length] for i in range(0			,A__			,args.max_length		)]
						    for k, t in concatenated_examples.items()
						}
						return result

			UpperCAmelCase_  :				Tuple      =      dataset_tokenized.map(A__			,batched=A__			,batch_size=10_00			,num_proc=4		)

			UpperCAmelCase_  :				int      =      0
			UpperCAmelCase_  :				Any      =      0
			for shard in range(0			,len(A__		)			,args.shard_size		):
						UpperCAmelCase_  :				str      =      grouped_dataset[shard : shard + args.shard_size]
						UpperCAmelCase_  :				List[Any]      =      len(dataset_snapshot["input_ids"]		)
						UpperCAmelCase_  :				List[str]      =      os.path.join(A__			,F"""dataset-{shard_count}-{records_containing}.tfrecord"""		)
						UpperCAmelCase_  :				str      =      get_serialized_examples(A__		)

						with tf.io.TFRecordWriter(A__		) as out_file:
									for i in range(len(A__		)		):
												UpperCAmelCase_  :				Optional[Any]      =      serialized_examples[i]
												out_file.write(A__		)
									print("Wrote file {} containing {} records".format(A__			,A__		)		)

						shard_count += 1
						total_records += records_containing

			with open(F"""split-{args.split}-records-count.txt"""			,"w"		) as f:
						print(F"""Total {args.split} records: {total_records}"""			,file=A__		)


if __name__ == "__main__":
			lowerCamelCase_       			=   parse_args()
			main(args)











"""simple docstring"""
import doctest
import logging
import os
import unittest
from pathlib import Path
from typing import List, Union

import transformers
from transformers.testing_utils import require_tf, require_torch, slow


lowerCamelCase_       			=   logging.getLogger()







@unittest.skip('''Temporarily disable the doc tests.'''  )
@require_torch
@require_tf
@slow
class     UpperCamelCase_ (unittest.TestCase  ):





	def   _SCREAMING_SNAKE_CASE   (   self      :				Union[str, Any]			,			lowerCAmelCase_      :				Path			,			lowerCAmelCase_      :				Union[str, None] = None			,			lowerCAmelCase_      :				Union[List[str], None] = None			,			lowerCAmelCase_      :				Union[str, List[str], None] = None			,			lowerCAmelCase_      :				bool = True			,			)		->    Tuple:
				UpperCAmelCase_  :				Optional[Any]      =      [file for file in os.listdir(lowerCAmelCase_		) if os.path.isfile(os.path.join(lowerCAmelCase_			,			lowerCAmelCase_		)		)]

				if identifier is not None:
							UpperCAmelCase_  :				Optional[int]      =      [file for file in files if identifier in file]

				if n_identifier is not None:
							if isinstance(lowerCAmelCase_			,			lowerCAmelCase_		):
										for n_ in n_identifier:
													UpperCAmelCase_  :				List[Any]      =      [file for file in files if n_ not in file]
							else:
										UpperCAmelCase_  :				Optional[Any]      =      [file for file in files if n_identifier not in file]

				UpperCAmelCase_  :				List[Any]      =      ignore_files or []
				ignore_files.append("__init__.py"		)
				UpperCAmelCase_  :				Optional[Any]      =      [file for file in files if file not in ignore_files]

				for file in files:
							# Open all files
							print("Testing"			,			lowerCAmelCase_		)

							if only_modules:
										UpperCAmelCase_  :				str      =      file.split("."		)[0]
										try:
													UpperCAmelCase_  :				Tuple      =      getattr(lowerCAmelCase_			,			lowerCAmelCase_		)
													UpperCAmelCase_  :				int      =      doctest.DocTestSuite(lowerCAmelCase_		)
													UpperCAmelCase_  :				str      =      unittest.TextTestRunner().run(lowerCAmelCase_		)
													self.assertIs(len(result.failures		)			,			0		)
										except AttributeError:
													logger.info(f"""{module_identifier} is not a module."""		)
							else:
										UpperCAmelCase_  :				List[Any]      =      doctest.testfile(str(".." / directory / file		)			,			optionflags=doctest.ELLIPSIS		)
										self.assertIs(result.failed			,			0		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Optional[int]		)		->    List[Any]:
				UpperCAmelCase_  :				Optional[int]      =      Path("src/transformers"		)
				UpperCAmelCase_  :				List[Any]      =      "modeling"
				UpperCAmelCase_  :				Tuple      =      [
				    "modeling_ctrl.py",
				    "modeling_tf_ctrl.py",
				]
				self.analyze_directory(lowerCAmelCase_			,			identifier=lowerCAmelCase_			,			ignore_files=lowerCAmelCase_		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Union[str, Any]		)		->    List[Any]:
				UpperCAmelCase_  :				List[str]      =      Path("src/transformers"		)
				UpperCAmelCase_  :				int      =      "tokenization"
				self.analyze_directory(lowerCAmelCase_			,			identifier=lowerCAmelCase_		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Optional[Any]		)		->    Optional[Any]:
				UpperCAmelCase_  :				int      =      Path("src/transformers"		)
				UpperCAmelCase_  :				Any      =      "configuration"
				self.analyze_directory(lowerCAmelCase_			,			identifier=lowerCAmelCase_		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Any		)		->    Any:
				UpperCAmelCase_  :				int      =      Path("src/transformers"		)
				UpperCAmelCase_  :				List[Any]      =      ["configuration", "modeling", "tokenization"]
				self.analyze_directory(lowerCAmelCase_			,			n_identifier=lowerCAmelCase_		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				List[str]		)		->    Dict:
				UpperCAmelCase_  :				List[str]      =      Path("docs/source"		)
				UpperCAmelCase_  :				int      =      ["favicon.ico"]
				self.analyze_directory(lowerCAmelCase_			,			ignore_files=lowerCAmelCase_			,			only_modules=lowerCAmelCase_		)



