






import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__A      =   logging.getLogger(__name__)




def      __A					(     ):

					'''simple docstring'''







					_A								= argparse.ArgumentParser(
					    description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.''' )
					parser.add_argument(
					    '''--dataset_name'''					,			type=_lowercase					,			default='''wikitext'''					,			help='''Name of the training. Explore datasets at: hf.co/datasets.'''					,			)
					parser.add_argument(
					    '''--dataset_config'''					,			type=_lowercase					,			default='''wikitext-103-raw-v1'''					,			help='''Configuration name of the dataset.''' )
					parser.add_argument(
					    '''--tokenizer_name_or_path'''					,			type=_lowercase					,			default='''sayakpaul/unigram-tokenizer-wikitext'''					,			help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''					,			)
					parser.add_argument(
					    '''--shard_size'''					,			type=_lowercase					,			default=10_00					,			help='''Number of entries to go in a single shard.'''					,			)
					parser.add_argument('''--split'''					,			type=_lowercase					,			default='''train'''					,			choices=['''train''', '''test''', '''validation'''] )
					parser.add_argument(
					    '''--limit'''					,			default=_lowercase					,			type=_lowercase					,			help='''Limit the number of shards (used for debugging).'''					,			)
					parser.add_argument(
					    '''--max_length'''					,			type=_lowercase					,			default=5_12					,			help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
					    ''' sequence length that is a multiple of 8.'''					,			)
					parser.add_argument(
					    '''--output_dir'''					,			default='''tf-tpu'''					,			type=_lowercase					,			help='''Output directory where the TFRecord shards will be saved. If the'''
					    ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
					    ''' shards will be directly saved to a Google Cloud Storage bucket.'''					,			)

					_A								= parser.parse_args()
					return args




def      __A					(     _lowercase ):

					'''simple docstring'''







					def fn(_lowercase ):
										return tokenizer(examples['''text'''] )

					return fn




def      __A					(     _lowercase ):

					'''simple docstring'''







					_A								= []
					for i in range(len(tokenized_data['''input_ids'''] ) ):
										_A								= {
										    '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i] ) ),
										    '''attention_mask''': tf.train.Feature(
										        intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i] ) ),
										}
										_A								= tf.train.Features(feature=_lowercase )
										_A								= tf.train.Example(features=_lowercase )
										_A								= example.SerializeToString()
										records.append(_lowercase )
					return records




def      __A					(     _lowercase ):

					'''simple docstring'''







					_A								= datasets.load_dataset(args.dataset_name					,			args.dataset_config					,			split=args.split )

					if args.limit is not None:
										_A								= min(len(_lowercase )					,			args.limit )
										_A								= dataset.select(range(_lowercase ) )
										print(f"""Limiting the dataset to {args.limit} entries.""" )

					_A								= AutoTokenizer.from_pretrained(args.tokenizer_name_or_path )

					# Handle output directory creation.
					# For serializing into a Google Cloud Storage Bucket, one needs to first
					# create a bucket.
					if "gs" not in args.output_dir:
										if not os.path.exists(args.output_dir ):
															os.makedirs(args.output_dir )
										_A								= os.path.join(args.output_dir					,			args.split )
										if not os.path.exists(_lowercase ):
															os.makedirs(_lowercase )
					else:
										_A								= os.path.join(args.output_dir					,			args.split )

					# Tokenize the whole dataset at once.
					_A								= tokenize_function(_lowercase )
					_A								= dataset.map(_lowercase					,			batched=_lowercase					,			num_proc=4					,			remove_columns=['''text'''] )

					# We need to concatenate all our texts together, and then split the result
					# into chunks of a fixed size, which we will call block_size. To do this, we
					# will use the map method again, with the option batched=True. When we use batched=True,
					# the function we pass to map() will be passed multiple inputs at once, allowing us
					# to group them into more or fewer examples than we had in the input.
					# This allows us to create our new fixed-length samples. The advantage of this
					# method is that we don't lose a whole lot of content from the dataset compared to the
					# case where we simply tokenize with a pre-defined max_length.

					def group_texts(_lowercase ):
										# Concatenate all texts.
										_A								= {k: sum(examples[k]					,			[] ) for k in examples.keys()}
										_A								= len(concatenated_examples[list(examples.keys() )[0]] )
										# We drop the small remainder, though you could add padding instead if the model supports it
										# In this, as in all things, we advise you to follow your heart 🫀
										_A								= (total_length // args.max_length) * args.max_length
										# Split by chunks of max_len.
										_A								= {
										    k: [t[i : i + args.max_length] for i in range(0					,			_lowercase					,			args.max_length )]
										    for k, t in concatenated_examples.items()
										}
										return result

					_A								= dataset_tokenized.map(_lowercase					,			batched=_lowercase					,			batch_size=10_00					,			num_proc=4 )

					_A								= 0
					_A								= 0
					for shard in range(0					,			len(_lowercase )					,			args.shard_size ):
										_A								= grouped_dataset[shard : shard + args.shard_size]
										_A								= len(dataset_snapshot['''input_ids'''] )
										_A								= os.path.join(_lowercase					,			f"""dataset-{shard_count}-{records_containing}.tfrecord""" )
										_A								= get_serialized_examples(_lowercase )

										with tf.io.TFRecordWriter(_lowercase ) as out_file:
															for i in range(len(_lowercase ) ):
																				_A								= serialized_examples[i]
																				out_file.write(_lowercase )
															print('''Wrote file {} containing {} records'''.format(_lowercase					,			_lowercase ) )

										shard_count += 1
										total_records += records_containing

					with open(f"""split-{args.split}-records-count.txt"""					,			'''w''' ) as f:
										print(f"""Total {args.split} records: {total_records}"""					,			file=_lowercase )


if __name__ == "__main__":
				__A      =   parse_args()
				main(args)











import logging
import os

import quant_trainer
import torch
from torch.utils.data import DataLoader

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput


__A      =				logging.getLogger(__name__)

if is_torch_tpu_available(check_device=False):
				import torch_xla.core.xla_model as xm
				import torch_xla.debug.metrics as met





class 						SCREAMING_SNAKE_CASE				( snake_case					):





							"""simple docstring"""

							def __init__(  self:   int		,     *__A:   str		,     __A:   List[Any]=None		,     __A:   Union[str, Any]=None		,     __A:   List[Any]=None		,     **__A:   int   )    ->				List[Any]:
												super().__init__(*__A		,     **__A   )
												_A								= eval_examples
												_A								= post_process_function
												_A								= quant_trainer_args
												_A								= 1_28  # default number of calibration samples

							def       __A (  self:   Union[str, Any]		,     __A:   List[Any]=None   )    ->				Optional[Any]:
												if calib_dataset is None and self.calib_dataset is None:
																	raise ValueError('''Trainer: calibration requires an calib_dataset.'''   )
												_A								= calib_dataset if calib_dataset is not None else self.calib_dataset

												_A								= self._remove_unused_columns(__A		,     description='''Calibration'''   )

												return DataLoader(
												    __A		,     batch_size=self.args.eval_batch_size		,     collate_fn=self.data_collator		,     drop_last=self.args.dataloader_drop_last		,     num_workers=self.args.dataloader_num_workers		,     pin_memory=self.args.dataloader_pin_memory		,     shuffle=__A		,     )

							def       __A (  self:   List[Any]		,     __A:   Any=None   )    ->				Optional[int]:
												_A								= self.train_dataset if calib_dataset is None else calib_dataset
												_A								= self.get_calib_dataloader(__A   )

												_A								= self.model
												quant_trainer.configure_model(__A		,     self.quant_trainer_args		,     calib=__A   )
												model.eval()
												quant_trainer.enable_calibration(__A   )

												logger.info('''***** Running calibration *****'''   )
												logger.info(f"""  Num examples = {self.calib_num}"""   )
												logger.info(f"""  Batch size = {calib_dataloader.batch_size}"""   )

												for step, inputs in enumerate(__A   ):
																	# Prediction step
																	_A							,_A							,_A								= self.prediction_step(__A		,     __A		,     prediction_loss_only=__A   )
																	if (step + 1) * calib_dataloader.batch_size >= self.calib_num:
																						break

												quant_trainer.finish_calibration(__A		,     self.quant_trainer_args   )
												_A								= model

							def       __A (  self:   Any		,     __A:   Dict=None		,     __A:   Tuple=None		,     __A:   List[Any]=None		,     __A:   str = "eval"   )    ->				int:
												_A								= self.eval_dataset if eval_dataset is None else eval_dataset
												_A								= self.get_eval_dataloader(__A   )
												_A								= self.eval_examples if eval_examples is None else eval_examples

												# Temporarily disable metric computation, we will do it in the loop here.
												_A								= self.compute_metrics
												_A								= None
												_A								= self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
												try:
																	_A								= eval_loop(
																	    __A		,     description='''Evaluation'''		,     prediction_loss_only=True if compute_metrics is None else None		,     ignore_keys=__A		,     )
												finally:
																	_A								= compute_metrics

												if self.post_process_function is not None and self.compute_metrics is not None:
																	_A								= self.post_process_function(__A		,     __A		,     output.predictions   )
																	_A								= self.compute_metrics(__A   )

																	# Prefix all keys with metric_key_prefix + '_'
																	for key in list(metrics.keys()   ):
																						if not key.startswith(f"""{metric_key_prefix}_"""   ):
																											_A								= metrics.pop(__A   )

																	self.log(__A   )
												else:
																	_A								= {}

												if self.args.tpu_metrics_debug or self.args.debug:
																	# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																	xm.master_print(met.metrics_report()   )

												_A								= self.callback_handler.on_evaluate(self.args		,     self.state		,     self.control		,     __A   )
												return metrics

							def       __A (  self:   Union[str, Any]		,     __A:   Optional[int]		,     __A:   int		,     __A:   List[Any]=None		,     __A:   str = "test"   )    ->				Union[str, Any]:
												_A								= self.get_test_dataloader(__A   )

												# Temporarily disable metric computation, we will do it in the loop here.
												_A								= self.compute_metrics
												_A								= None
												_A								= self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
												try:
																	_A								= eval_loop(
																	    __A		,     description='''Prediction'''		,     prediction_loss_only=True if compute_metrics is None else None		,     ignore_keys=__A		,     )
												finally:
																	_A								= compute_metrics

												if self.post_process_function is None or self.compute_metrics is None:
																	return output

												_A								= self.post_process_function(__A		,     __A		,     output.predictions		,     '''predict'''   )
												_A								= self.compute_metrics(__A   )

												# Prefix all keys with metric_key_prefix + '_'
												for key in list(metrics.keys()   ):
																	if not key.startswith(f"""{metric_key_prefix}_"""   ):
																						_A								= metrics.pop(__A   )

												return PredictionOutput(predictions=predictions.predictions		,     label_ids=predictions.label_ids		,     metrics=__A   )



							def       __A (  self:   Tuple		,     __A:   Optional[Any]="./"   )    ->				List[str]:
												_A								= self.eval_dataset
												_A								= self.get_eval_dataloader(__A   )

												_A								= next(iter(__A   )   )

												# saving device - to make it consistent
												_A								= torch.device('''cuda''' if torch.cuda.is_available() else '''cpu'''   )

												# convert to tuple
												_A								= tuple(v.to(__A   ) for k, v in batch.items()   )

												logger.info('''Converting model to be onnx compatible'''   )
												from pytorch_quantization.nn import TensorQuantizer

												_A								= True

												_A								= self.model.to(__A   )

												model.eval()
												model.float()

												_A								= model.module if hasattr(__A		,     '''module'''   ) else model
												quant_trainer.configure_model(__A		,     self.quant_trainer_args   )

												_A								= os.path.join(__A		,     '''model.onnx'''   )
												logger.info(f"""exporting model to {output_model_file}"""   )

												_A								= {0: '''batch_size''', 1: '''seq_len'''}

												torch.onnx.export(
												    __A		,     __A		,     __A		,     export_params=__A		,     opset_version=13		,     do_constant_folding=__A		,     input_names=['''input_ids''', '''attention_mask''', '''token_type_ids''']		,     output_names=['''output_start_logits''', '''output_end_logits''']		,     dynamic_axes={
												        '''input_ids''': axes,
												        '''attention_mask''': axes,
												        '''token_type_ids''': axes,
												        '''output_start_logits''': axes,
												        '''output_end_logits''': axes,
												    }		,     verbose=__A		,     )
												logger.info('''onnx export finished'''   )



