



'''simple docstring'''

import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


_A				:    List[str]     						=logging.getLogger(__name__)







def 						__UpperCamelCase				(							) ->			List[str]:

	'''simple docstring'''





	_lowercase					:       Tuple       =		argparse.ArgumentParser(
	    description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'					)
	parser.add_argument(
	    '--dataset_name',				type=_lowercase,				default='wikitext',				help='Name of the training. Explore datasets at: hf.co/datasets.',				)
	parser.add_argument(
	    '--dataset_config',				type=_lowercase,				default='wikitext-103-raw-v1',				help='Configuration name of the dataset.'					)
	parser.add_argument(
	    '--tokenizer_name_or_path',				type=_lowercase,				default='sayakpaul/unigram-tokenizer-wikitext',				help='Tokenizer identifier. Can be a local filepath or a Hub identifier.',				)
	parser.add_argument(
	    '--shard_size',				type=_lowercase,				default=1000,				help='Number of entries to go in a single shard.',				)
	parser.add_argument('--split',				type=_lowercase,				default='train',				choices=['train', 'test', 'validation']					)
	parser.add_argument(
	    '--limit',				default=_lowercase,				type=_lowercase,				help='Limit the number of shards (used for debugging).',				)
	parser.add_argument(
	    '--max_length',				type=_lowercase,				default=512,				help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
	    ' sequence length that is a multiple of 8.',				)
	parser.add_argument(
	    '--output_dir',				default='tf-tpu',				type=_lowercase,				help='Output directory where the TFRecord shards will be saved. If the'
	    ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
	    ' shards will be directly saved to a Google Cloud Storage bucket.',				)

	_lowercase					:       Tuple       =		parser.parse_args()
	return args







def 						__UpperCamelCase				(							_lowercase					) ->			str:

	'''simple docstring'''





	def fn(_lowercase					):
		return tokenizer(examples['text']					)

	return fn







def 						__UpperCamelCase				(							_lowercase					) ->			List[Any]:

	'''simple docstring'''





	_lowercase					:       str       =		[]
	for i in range(len(tokenized_data['input_ids']					)					):
		_lowercase					:       int       =		{
		    'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i]					)					),
		    'attention_mask': tf.train.Feature(
		        intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i]					)					),
		}
		_lowercase					:       List[str]       =		tf.train.Features(feature=_lowercase					)
		_lowercase					:       Dict       =		tf.train.Example(features=_lowercase					)
		_lowercase					:       Tuple       =		example.SerializeToString()
		records.append(_lowercase					)
	return records







def 						__UpperCamelCase				(							_lowercase					) ->			int:

	'''simple docstring'''





	_lowercase					:       Optional[int]       =		datasets.load_dataset(args.dataset_name,				args.dataset_config,				split=args.split					)

	if args.limit is not None:
		_lowercase					:       Dict       =		min(len(_lowercase					),				args.limit					)
		_lowercase					:       Any       =		dataset.select(range(_lowercase					)					)
		print(f'''Limiting the dataset to {args.limit} entries.'''					)

	_lowercase					:       List[Any]       =		AutoTokenizer.from_pretrained(args.tokenizer_name_or_path					)

	# Handle output directory creation.
	# For serializing into a Google Cloud Storage Bucket, one needs to first
	# create a bucket.
	if "gs" not in args.output_dir:
		if not os.path.exists(args.output_dir					):
			os.makedirs(args.output_dir					)
		_lowercase					:       Optional[Any]       =		os.path.join(args.output_dir,				args.split					)
		if not os.path.exists(_lowercase					):
			os.makedirs(_lowercase					)
	else:
		_lowercase					:       List[Any]       =		os.path.join(args.output_dir,				args.split					)

	# Tokenize the whole dataset at once.
	_lowercase					:       List[Any]       =		tokenize_function(_lowercase					)
	_lowercase					:       Any       =		dataset.map(_lowercase,				batched=_lowercase,				num_proc=4,				remove_columns=['text']					)

	# We need to concatenate all our texts together, and then split the result
	# into chunks of a fixed size, which we will call block_size. To do this, we
	# will use the map method again, with the option batched=True. When we use batched=True,
	# the function we pass to map() will be passed multiple inputs at once, allowing us
	# to group them into more or fewer examples than we had in the input.
	# This allows us to create our new fixed-length samples. The advantage of this
	# method is that we don't lose a whole lot of content from the dataset compared to the
	# case where we simply tokenize with a pre-defined max_length.

	def group_texts(_lowercase					):
		# Concatenate all texts.
		_lowercase					:       Tuple       =		{k: sum(examples[k],				[]					) for k in examples.keys()}
		_lowercase					:       int       =		len(concatenated_examples[list(examples.keys()					)[0]]					)
		# We drop the small remainder, though you could add padding instead if the model supports it
		# In this, as in all things, we advise you to follow your heart 🫀
		_lowercase					:       str       =		(total_length // args.max_length) * args.max_length
		# Split by chunks of max_len.
		_lowercase					:       Dict       =		{
		    k: [t[i : i + args.max_length] for i in range(0,				_lowercase,				args.max_length					)]
		    for k, t in concatenated_examples.items()
		}
		return result

	_lowercase					:       Any       =		dataset_tokenized.map(_lowercase,				batched=_lowercase,				batch_size=1000,				num_proc=4					)

	_lowercase					:       Union[str, Any]       =		0
	_lowercase					:       Union[str, Any]       =		0
	for shard in range(0,				len(_lowercase					),				args.shard_size					):
		_lowercase					:       Optional[Any]       =		grouped_dataset[shard : shard + args.shard_size]
		_lowercase					:       Any       =		len(dataset_snapshot['input_ids']					)
		_lowercase					:       Any       =		os.path.join(_lowercase,				f'''dataset-{shard_count}-{records_containing}.tfrecord'''					)
		_lowercase					:       Optional[int]       =		get_serialized_examples(_lowercase					)

		with tf.io.TFRecordWriter(_lowercase					) as out_file:
			for i in range(len(_lowercase					)					):
				_lowercase					:       Dict       =		serialized_examples[i]
				out_file.write(_lowercase					)
			print('Wrote file {} containing {} records'.format(_lowercase,				_lowercase					)					)

		shard_count += 1
		total_records += records_containing

	with open(f'''split-{args.split}-records-count.txt''',				'w'					) as f:
		print(f'''Total {args.split} records: {total_records}''',				file=_lowercase					)


if __name__ == "__main__":
		_A				:    Optional[int]     						=parse_args()
		main(args)





'''simple docstring'''

import unittest

import numpy as np

from transformers import RoFormerConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
		import jax.numpy as jnp

		from transformers.models.roformer.modeling_flax_roformer import (
		    FlaxRoFormerForMaskedLM,
		    FlaxRoFormerForMultipleChoice,
		    FlaxRoFormerForQuestionAnswering,
		    FlaxRoFormerForSequenceClassification,
		    FlaxRoFormerForTokenClassification,
		    FlaxRoFormerModel,
		)



class 		lowerCamelCase__      (						unittest.TestCase     ):




					'''simple docstring'''



					def __init__(  self				:	Dict     ,  UpperCamelCase_				:	Union[str, Any]     ,  UpperCamelCase_				:	List[str]=13     ,  UpperCamelCase_				:	Union[str, Any]=7     ,  UpperCamelCase_				:	Union[str, Any]=True     ,  UpperCamelCase_				:	Any=True     ,  UpperCamelCase_				:	Dict=True     ,  UpperCamelCase_				:	List[str]=True     ,  UpperCamelCase_				:	int=99     ,  UpperCamelCase_				:	Tuple=32     ,  UpperCamelCase_				:	List[str]=5     ,  UpperCamelCase_				:	Dict=4     ,  UpperCamelCase_				:	Tuple=37     ,  UpperCamelCase_				:	int="gelu"     ,  UpperCamelCase_				:	int=0.1     ,  UpperCamelCase_				:	Dict=0.1     ,  UpperCamelCase_				:	Any=512     ,  UpperCamelCase_				:	List[Any]=16     ,  UpperCamelCase_				:	Optional[int]=2     ,  UpperCamelCase_				:	Tuple=0.02     ,  UpperCamelCase_				:	Union[str, Any]=4     ,  )  ->       Tuple:



						'''simple docstring'''





						_lowercase					:       int       =		parent
						_lowercase					:       str       =		batch_size
						_lowercase					:       List[str]       =		seq_length
						_lowercase					:       Dict       =		is_training
						_lowercase					:       Optional[int]       =		use_attention_mask
						_lowercase					:       List[Any]       =		use_token_type_ids
						_lowercase					:       Union[str, Any]       =		use_labels
						_lowercase					:       Dict       =		vocab_size
						_lowercase					:       List[Any]       =		hidden_size
						_lowercase					:       Any       =		num_hidden_layers
						_lowercase					:       int       =		num_attention_heads
						_lowercase					:       Optional[int]       =		intermediate_size
						_lowercase					:       Any       =		hidden_act
						_lowercase					:       List[str]       =		hidden_dropout_prob
						_lowercase					:       Union[str, Any]       =		attention_probs_dropout_prob
						_lowercase					:       Optional[int]       =		max_position_embeddings
						_lowercase					:       int       =		type_vocab_size
						_lowercase					:       Any       =		type_sequence_label_size
						_lowercase					:       Any       =		initializer_range
						_lowercase					:       str       =		num_choices




					def 						__UpperCAmelCase     (  self				:	str					)  ->       int:



						'''simple docstring'''





						_lowercase					:       Optional[int]       =		ids_tensor([self.batch_size, self.seq_length]     ,  self.vocab_size					)

						_lowercase					:       int       =		None
						if self.use_attention_mask:
							_lowercase					:       Optional[int]       =		random_attention_mask([self.batch_size, self.seq_length]					)

						_lowercase					:       Any       =		None
						if self.use_token_type_ids:
							_lowercase					:       Optional[Any]       =		ids_tensor([self.batch_size, self.seq_length]     ,  self.type_vocab_size					)

						_lowercase					:       str       =		RoFormerConfig(
						    vocab_size=self.vocab_size     ,  hidden_size=self.hidden_size     ,  num_hidden_layers=self.num_hidden_layers     ,  num_attention_heads=self.num_attention_heads     ,  intermediate_size=self.intermediate_size     ,  hidden_act=self.hidden_act     ,  hidden_dropout_prob=self.hidden_dropout_prob     ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,  max_position_embeddings=self.max_position_embeddings     ,  type_vocab_size=self.type_vocab_size     ,  is_decoder=UpperCamelCase_     ,  initializer_range=self.initializer_range     ,  )

						return config, input_ids, token_type_ids, attention_mask




					def 						__UpperCAmelCase     (  self				:	List[Any]					)  ->       int:



						'''simple docstring'''





						_lowercase					:       Dict       =		self.prepare_config_and_inputs()
						_lowercase  ,    _lowercase  ,    _lowercase  ,    _lowercase					:       Union[str, Any]       =		config_and_inputs
						_lowercase					:       Optional[int]       =		{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': attention_mask}
						return config, inputs_dict






@require_flax
class 		lowerCamelCase__      (						A	,   unittest.TestCase     ):




					'''simple docstring'''



					A_						    =						True

					A_						    =						(
					    (
					        FlaxRoFormerModel,
					        FlaxRoFormerForMaskedLM,
					        FlaxRoFormerForSequenceClassification,
					        FlaxRoFormerForTokenClassification,
					        FlaxRoFormerForMultipleChoice,
					        FlaxRoFormerForQuestionAnswering,
					    )
					    if is_flax_available()
					    else ()
					)




					def 						__UpperCAmelCase     (  self				:	str					)  ->       int:



						'''simple docstring'''





						_lowercase					:       Tuple       =		FlaxRoFormerModelTester(self					)




					@slow
					def 						__UpperCAmelCase     (  self				:	List[str]					)  ->       Optional[int]:



						'''simple docstring'''





						for model_class_name in self.all_model_classes:
							_lowercase					:       Optional[int]       =		model_class_name.from_pretrained('junnyu/roformer_chinese_small'     ,  from_pt=UpperCamelCase_					)
							_lowercase					:       str       =		model(np.ones((1, 1)					)					)
							self.assertIsNotNone(UpperCamelCase_					)






@require_flax
class 		lowerCamelCase__      (						unittest.TestCase     ):




					'''simple docstring'''



					@slow
					def 						__UpperCAmelCase     (  self				:	List[str]					)  ->       List[Any]:



						'''simple docstring'''





						_lowercase					:       Dict       =		FlaxRoFormerForMaskedLM.from_pretrained('junnyu/roformer_chinese_base'					)
						_lowercase					:       Any       =		jnp.array([[0, 1, 2, 3, 4, 5]]					)
						_lowercase					:       int       =		model(UpperCamelCase_					)[0]

						_lowercase					:       Union[str, Any]       =		5_0000

						_lowercase					:       str       =		(1, 6, vocab_size)
						self.assertEqual(output.shape     ,  UpperCamelCase_					)

						_lowercase					:       int       =		jnp.array(
						    [[[-0.12_05, -1.02_65, 0.29_22], [-1.51_34, 0.19_74, 0.15_19], [-5.01_35, -3.90_03, -0.84_04]]]					)

						self.assertTrue(jnp.allclose(output[:, :3, :3]     ,  UpperCamelCase_     ,  atol=1E-4					)					)
