






"""simple docstring"""
import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


lowerCamelCase_       			=   logging.getLogger(__name__)

def        snake_case   (     ):
			UpperCAmelCase_  :				List[Any]      =      argparse.ArgumentParser(
			    description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."		)
			parser.add_argument(
			    "--dataset_name"			,type=A__			,default="wikitext"			,help="Name of the training. Explore datasets at: hf.co/datasets."			,)
			parser.add_argument(
			    "--dataset_config"			,type=A__			,default="wikitext-103-raw-v1"			,help="Configuration name of the dataset."		)
			parser.add_argument(
			    "--tokenizer_name_or_path"			,type=A__			,default="sayakpaul/unigram-tokenizer-wikitext"			,help="Tokenizer identifier. Can be a local filepath or a Hub identifier."			,)
			parser.add_argument(
			    "--shard_size"			,type=A__			,default=10_00			,help="Number of entries to go in a single shard."			,)
			parser.add_argument("--split"			,type=A__			,default="train"			,choices=["train", "test", "validation"]		)
			parser.add_argument(
			    "--limit"			,default=A__			,type=A__			,help="Limit the number of shards (used for debugging)."			,)
			parser.add_argument(
			    "--max_length"			,type=A__			,default=5_12			,help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
			    " sequence length that is a multiple of 8."			,)
			parser.add_argument(
			    "--output_dir"			,default="tf-tpu"			,type=A__			,help="Output directory where the TFRecord shards will be saved. If the"
			    " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
			    " shards will be directly saved to a Google Cloud Storage bucket."			,)

			UpperCAmelCase_  :				Dict      =      parser.parse_args()
			return args

def        snake_case   (     A__		):
			def fn(A__		):
						return tokenizer(examples["text"]		)

			return fn

def        snake_case   (     A__		):
			UpperCAmelCase_  :				Optional[Any]      =      []
			for i in range(len(tokenized_data["input_ids"]		)		):
						UpperCAmelCase_  :				int      =      {
						    "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]		)		),
						    "attention_mask": tf.train.Feature(
						        intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]		)		),
						}
						UpperCAmelCase_  :				Tuple      =      tf.train.Features(feature=A__		)
						UpperCAmelCase_  :				str      =      tf.train.Example(features=A__		)
						UpperCAmelCase_  :				Tuple      =      example.SerializeToString()
						records.append(A__		)
			return records

def        snake_case   (     A__		):
			UpperCAmelCase_  :				Tuple      =      datasets.load_dataset(args.dataset_name			,args.dataset_config			,split=args.split		)

			if args.limit is not None:
						UpperCAmelCase_  :				Dict      =      min(len(A__		)			,args.limit		)
						UpperCAmelCase_  :				Optional[Any]      =      dataset.select(range(A__		)		)
						print(F"""Limiting the dataset to {args.limit} entries."""		)

			UpperCAmelCase_  :				str      =      AutoTokenizer.from_pretrained(args.tokenizer_name_or_path		)

			# Handle output directory creation.
			# For serializing into a Google Cloud Storage Bucket, one needs to first
			# create a bucket.
			if "gs" not in args.output_dir:
						if not os.path.exists(args.output_dir		):
									os.makedirs(args.output_dir		)
						UpperCAmelCase_  :				Any      =      os.path.join(args.output_dir			,args.split		)
						if not os.path.exists(A__		):
									os.makedirs(A__		)
			else:
						UpperCAmelCase_  :				List[Any]      =      os.path.join(args.output_dir			,args.split		)

			# Tokenize the whole dataset at once.
			UpperCAmelCase_  :				Optional[int]      =      tokenize_function(A__		)
			UpperCAmelCase_  :				str      =      dataset.map(A__			,batched=A__			,num_proc=4			,remove_columns=["text"]		)

			# We need to concatenate all our texts together, and then split the result
			# into chunks of a fixed size, which we will call block_size. To do this, we
			# will use the map method again, with the option batched=True. When we use batched=True,
			# the function we pass to map() will be passed multiple inputs at once, allowing us
			# to group them into more or fewer examples than we had in the input.
			# This allows us to create our new fixed-length samples. The advantage of this
			# method is that we don't lose a whole lot of content from the dataset compared to the
			# case where we simply tokenize with a pre-defined max_length.

			def group_texts(A__		):
						# Concatenate all texts.
						UpperCAmelCase_  :				List[Any]      =      {k: sum(examples[k]			,[]		) for k in examples.keys()}
						UpperCAmelCase_  :				Optional[int]      =      len(concatenated_examples[list(examples.keys()		)[0]]		)
						# We drop the small remainder, though you could add padding instead if the model supports it
						# In this, as in all things, we advise you to follow your heart 🫀
						UpperCAmelCase_  :				Dict      =      (total_length // args.max_length) * args.max_length
						# Split by chunks of max_len.
						UpperCAmelCase_  :				Any      =      {
						    k: [t[i : i + args.max_length] for i in range(0			,A__			,args.max_length		)]
						    for k, t in concatenated_examples.items()
						}
						return result

			UpperCAmelCase_  :				Tuple      =      dataset_tokenized.map(A__			,batched=A__			,batch_size=10_00			,num_proc=4		)

			UpperCAmelCase_  :				int      =      0
			UpperCAmelCase_  :				Any      =      0
			for shard in range(0			,len(A__		)			,args.shard_size		):
						UpperCAmelCase_  :				str      =      grouped_dataset[shard : shard + args.shard_size]
						UpperCAmelCase_  :				List[Any]      =      len(dataset_snapshot["input_ids"]		)
						UpperCAmelCase_  :				List[str]      =      os.path.join(A__			,F"""dataset-{shard_count}-{records_containing}.tfrecord"""		)
						UpperCAmelCase_  :				str      =      get_serialized_examples(A__		)

						with tf.io.TFRecordWriter(A__		) as out_file:
									for i in range(len(A__		)		):
												UpperCAmelCase_  :				Optional[Any]      =      serialized_examples[i]
												out_file.write(A__		)
									print("Wrote file {} containing {} records".format(A__			,A__		)		)

						shard_count += 1
						total_records += records_containing

			with open(F"""split-{args.split}-records-count.txt"""			,"w"		) as f:
						print(F"""Total {args.split} records: {total_records}"""			,file=A__		)


if __name__ == "__main__":
			lowerCamelCase_       			=   parse_args()
			main(args)











"""simple docstring"""
import argparse
import shlex

import runhouse as rh


if __name__ == "__main__":
			# Refer to https://runhouse-docs.readthedocs-hosted.com/en/latest/api/python/cluster.html#hardware-setup for cloud access
			# setup instructions, if using on-demand hardware

			# If user passes --user <user> --host <host> --key_path <key_path> <example> <args>, fill them in as BYO cluster
			# If user passes --instance <instance> --provider <provider> <example> <args>, fill them in as on-demand cluster
			# Throw an error if user passes both BYO and on-demand cluster args
			# Otherwise, use default values
			lowerCamelCase_       			=   argparse.ArgumentParser()
			parser.add_argument('''--user''', type=str, default='''ubuntu''')
			parser.add_argument('''--host''', type=str, default='''localhost''')
			parser.add_argument('''--key_path''', type=str, default=None)
			parser.add_argument('''--instance''', type=str, default='''V100:1''')
			parser.add_argument('''--provider''', type=str, default='''cheapest''')
			parser.add_argument('''--use_spot''', type=bool, default=False)
			parser.add_argument('''--example''', type=str, default='''pytorch/text-generation/run_generation.py''')
			lowerCamelCase_			,	lowerCamelCase_       			=   parser.parse_known_args()
			if args.host != "localhost":
						if args.instance != "V100:1" or args.provider != "cheapest":
									raise ValueError('''Cannot specify both BYO and on-demand cluster args''')
						lowerCamelCase_       			=   rh.cluster(
						    name='''rh-cluster''', ips=[args.host], ssh_creds={'''ssh_user''': args.user, '''ssh_private_key''': args.key_path}
						)
			else:
						lowerCamelCase_       			=   rh.cluster(
						    name='''rh-cluster''', instance_type=args.instance, provider=args.provider, use_spot=args.use_spot
						)
			lowerCamelCase_       			=   args.example.rsplit('''/''', 1)[0]

			# Set up remote environment
			cluster.install_packages(['''pip:./'''])  # Installs transformers from local source
			# Note transformers is copied into the home directory on the remote machine, so we can install from there
			cluster.run([f'pip install -r transformers/examples/{example_dir}/requirements.txt'])
			cluster.run(['''pip install torch --upgrade --extra-index-url https://download.pytorch.org/whl/cu117'''])

			# Run example. You can bypass the CLI wrapper and paste your own code here.
			cluster.run([f'python transformers/examples/{args.example} {" ".join(shlex.quote(arg) for arg in unknown)}'])

			# Alternatively, we can just import and run a training function (especially if there's no wrapper CLI):
			# from my_script... import train
			# reqs = ['pip:./', 'torch', 'datasets', 'accelerate', 'evaluate', 'tqdm', 'scipy', 'scikit-learn', 'tensorboard']
			# launch_train_gpu = rh.function(fn=train,
			#                                system=gpu,
			#                                reqs=reqs,
			#                                name='train_bert_glue')
			#
			# We can pass in arguments just like we would to a function:
			# launch_train_gpu(num_epochs = 3, lr = 2e-5, seed = 42, batch_size = 16
			#                  stream_logs=True)



