





'''simple docstring'''

import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


UpperCAmelCase					:		Optional[Any]		=							logging.getLogger(__name__)




def    _a    (							):





      """simple docstring"""



      _snake_case		:	int					     =     argparse.ArgumentParser(
          description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''	)
      parser.add_argument(
          '''--dataset_name'''			,      type=lowerCAmelCase_			,      default='''wikitext'''			,      help='''Name of the training. Explore datasets at: hf.co/datasets.'''			,      )
      parser.add_argument(
          '''--dataset_config'''			,      type=lowerCAmelCase_			,      default='''wikitext-103-raw-v1'''			,      help='''Configuration name of the dataset.'''	)
      parser.add_argument(
          '''--tokenizer_name_or_path'''			,      type=lowerCAmelCase_			,      default='''sayakpaul/unigram-tokenizer-wikitext'''			,      help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''			,      )
      parser.add_argument(
          '''--shard_size'''			,      type=lowerCAmelCase_			,      default=1_000			,      help='''Number of entries to go in a single shard.'''			,      )
      parser.add_argument('''--split'''			,      type=lowerCAmelCase_			,      default='''train'''			,      choices=['''train''', '''test''', '''validation''']	)
      parser.add_argument(
          '''--limit'''			,      default=lowerCAmelCase_			,      type=lowerCAmelCase_			,      help='''Limit the number of shards (used for debugging).'''			,      )
      parser.add_argument(
          '''--max_length'''			,      type=lowerCAmelCase_			,      default=512			,      help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
          ''' sequence length that is a multiple of 8.'''			,      )
      parser.add_argument(
          '''--output_dir'''			,      default='''tf-tpu'''			,      type=lowerCAmelCase_			,      help='''Output directory where the TFRecord shards will be saved. If the'''
          ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
          ''' shards will be directly saved to a Google Cloud Storage bucket.'''			,      )

      _snake_case		:	Union[str, Any]					     =     parser.parse_args()
      return args




def    _a    (							lowerCAmelCase_	):





      """simple docstring"""



      def fn(lowerCAmelCase_	):
            return tokenizer(examples['''text''']	)

      return fn




def    _a    (							lowerCAmelCase_	):





      """simple docstring"""



      _snake_case		:	Optional[Any]					     =     []
      for i in range(len(tokenized_data['''input_ids''']	)	):
            _snake_case		:	Tuple					     =     {
                '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]	)	),
                '''attention_mask''': tf.train.Feature(
                    intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]	)	),
            }
            _snake_case		:	List[Any]					     =     tf.train.Features(feature=lowerCAmelCase_	)
            _snake_case		:	str					     =     tf.train.Example(features=lowerCAmelCase_	)
            _snake_case		:	List[str]					     =     example.SerializeToString()
            records.append(lowerCAmelCase_	)
      return records




def    _a    (							lowerCAmelCase_	):





      """simple docstring"""



      _snake_case		:	List[Any]					     =     datasets.load_dataset(args.dataset_name			,      args.dataset_config			,      split=args.split	)

      if args.limit is not None:
            _snake_case		:	str					     =     min(len(lowerCAmelCase_	)			,      args.limit	)
            _snake_case		:	Optional[Any]					     =     dataset.select(range(lowerCAmelCase_	)	)
            print(f'''Limiting the dataset to {args.limit} entries.'''	)

      _snake_case		:	int					     =     AutoTokenizer.from_pretrained(args.tokenizer_name_or_path	)

      # Handle output directory creation.
      # For serializing into a Google Cloud Storage Bucket, one needs to first
      # create a bucket.
      if "gs" not in args.output_dir:
            if not os.path.exists(args.output_dir	):
                  os.makedirs(args.output_dir	)
            _snake_case		:	Dict					     =     os.path.join(args.output_dir			,      args.split	)
            if not os.path.exists(lowerCAmelCase_	):
                  os.makedirs(lowerCAmelCase_	)
      else:
            _snake_case		:	Optional[int]					     =     os.path.join(args.output_dir			,      args.split	)

      # Tokenize the whole dataset at once.
      _snake_case		:	List[str]					     =     tokenize_function(lowerCAmelCase_	)
      _snake_case		:	List[str]					     =     dataset.map(lowerCAmelCase_			,      batched=lowerCAmelCase_			,      num_proc=4			,      remove_columns=['''text''']	)

      # We need to concatenate all our texts together, and then split the result
      # into chunks of a fixed size, which we will call block_size. To do this, we
      # will use the map method again, with the option batched=True. When we use batched=True,
      # the function we pass to map() will be passed multiple inputs at once, allowing us
      # to group them into more or fewer examples than we had in the input.
      # This allows us to create our new fixed-length samples. The advantage of this
      # method is that we don't lose a whole lot of content from the dataset compared to the
      # case where we simply tokenize with a pre-defined max_length.

      def group_texts(lowerCAmelCase_	):
            # Concatenate all texts.
            _snake_case		:	Optional[int]					     =     {k: sum(examples[k]			,      []	) for k in examples.keys()}
            _snake_case		:	Optional[Any]					     =     len(concatenated_examples[list(examples.keys()	)[0]]	)
            # We drop the small remainder, though you could add padding instead if the model supports it
            # In this, as in all things, we advise you to follow your heart 🫀
            _snake_case		:	int					     =     (total_length // args.max_length) * args.max_length
            # Split by chunks of max_len.
            _snake_case		:	int					     =     {
                k: [t[i : i + args.max_length] for i in range(0			,      lowerCAmelCase_			,      args.max_length	)]
                for k, t in concatenated_examples.items()
            }
            return result

      _snake_case		:	Tuple					     =     dataset_tokenized.map(lowerCAmelCase_			,      batched=lowerCAmelCase_			,      batch_size=1_000			,      num_proc=4	)

      _snake_case		:	Tuple					     =     0
      _snake_case		:	Union[str, Any]					     =     0
      for shard in range(0			,      len(lowerCAmelCase_	)			,      args.shard_size	):
            _snake_case		:	Union[str, Any]					     =     grouped_dataset[shard : shard + args.shard_size]
            _snake_case		:	Union[str, Any]					     =     len(dataset_snapshot['''input_ids''']	)
            _snake_case		:	Any					     =     os.path.join(lowerCAmelCase_			,      f'''dataset-{shard_count}-{records_containing}.tfrecord'''	)
            _snake_case		:	int					     =     get_serialized_examples(lowerCAmelCase_	)

            with tf.io.TFRecordWriter(lowerCAmelCase_	) as out_file:
                  for i in range(len(lowerCAmelCase_	)	):
                        _snake_case		:	int					     =     serialized_examples[i]
                        out_file.write(lowerCAmelCase_	)
                  print('''Wrote file {} containing {} records'''.format(lowerCAmelCase_			,      lowerCAmelCase_	)	)

            shard_count += 1
            total_records += records_containing

      with open(f'''split-{args.split}-records-count.txt'''			,      '''w'''	) as f:
            print(f'''Total {args.split} records: {total_records}'''			,      file=lowerCAmelCase_	)


if __name__ == "__main__":
   UpperCAmelCase					:		int		=							parse_args()
   main(args)









'''simple docstring'''

from __future__ import annotations

import unittest

from transformers import is_tf_available, is_torch_available
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER, is_pt_tf_cross_test, slow


if is_tf_available():
   from transformers import (
       AutoConfig,
       BertConfig,
       GPTaConfig,
       TaConfig,
       TFAutoModel,
       TFAutoModelForCausalLM,
       TFAutoModelForMaskedLM,
       TFAutoModelForPreTraining,
       TFAutoModelForQuestionAnswering,
       TFAutoModelForSeqaSeqLM,
       TFAutoModelForSequenceClassification,
       TFAutoModelWithLMHead,
       TFBertForMaskedLM,
       TFBertForPreTraining,
       TFBertForQuestionAnswering,
       TFBertForSequenceClassification,
       TFBertModel,
       TFGPTaLMHeadModel,
       TFRobertaForMaskedLM,
       TFTaForConditionalGeneration,
   )
   from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
   from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
   from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST

if is_torch_available():
   from transformers import (
       AutoModel,
       AutoModelForCausalLM,
       AutoModelForMaskedLM,
       AutoModelForPreTraining,
       AutoModelForQuestionAnswering,
       AutoModelForSeqaSeqLM,
       AutoModelForSequenceClassification,
       AutoModelWithLMHead,
       BertForMaskedLM,
       BertForPreTraining,
       BertForQuestionAnswering,
       BertForSequenceClassification,
       BertModel,
       GPTaLMHeadModel,
       RobertaForMaskedLM,
       TaForConditionalGeneration,
   )





@is_pt_tf_cross_test
class lowerCamelCase						(unittest.TestCase					):







  @slow
  def 				UpperCAmelCase_				(		self		)		->						List[Any]:






        """simple docstring"""





        for model_name in ["bert-base-uncased"]:
              _snake_case		:	Union[str, Any]					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Any					     =     TFAutoModel.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	str					     =     AutoModel.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						List[str]:






        """simple docstring"""





        for model_name in ["bert-base-uncased"]:
              _snake_case		:	Optional[Any]					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Dict					     =     TFAutoModelForPreTraining.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Tuple					     =     AutoModelForPreTraining.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						Union[str, Any]:






        """simple docstring"""





        for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              _snake_case		:	Optional[int]					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	List[Any]					     =     TFAutoModelForCausalLM.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              _snake_case     ,							_snake_case		:	Tuple					     =     TFAutoModelForCausalLM.from_pretrained(
                  lowercase__     ,     output_loading_info=lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Optional[int]					     =     AutoModelForCausalLM.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              _snake_case     ,							_snake_case		:	Optional[Any]					     =     AutoModelForCausalLM.from_pretrained(
                  lowercase__     ,     output_loading_info=lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						Optional[Any]:






        """simple docstring"""





        for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              _snake_case		:	List[Any]					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Tuple					     =     TFAutoModelWithLMHead.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	int					     =     AutoModelWithLMHead.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						Any:






        """simple docstring"""





        for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              _snake_case		:	List[str]					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Union[str, Any]					     =     TFAutoModelForMaskedLM.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              _snake_case     ,							_snake_case		:	List[str]					     =     TFAutoModelForMaskedLM.from_pretrained(
                  lowercase__     ,     output_loading_info=lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	int					     =     AutoModelForMaskedLM.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              _snake_case     ,							_snake_case		:	Optional[int]					     =     AutoModelForMaskedLM.from_pretrained(
                  lowercase__     ,     output_loading_info=lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						List[str]:






        """simple docstring"""





        for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              _snake_case		:	List[str]					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Tuple					     =     TFAutoModelForSeqaSeqLM.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              _snake_case     ,							_snake_case		:	List[str]					     =     TFAutoModelForSeqaSeqLM.from_pretrained(
                  lowercase__     ,     output_loading_info=lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	List[str]					     =     AutoModelForSeqaSeqLM.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              _snake_case     ,							_snake_case		:	Dict					     =     AutoModelForSeqaSeqLM.from_pretrained(
                  lowercase__     ,     output_loading_info=lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						Dict:






        """simple docstring"""





        for model_name in ["bert-base-uncased"]:
              _snake_case		:	Any					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Any					     =     TFAutoModelForSequenceClassification.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Dict					     =     AutoModelForSequenceClassification.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  @slow
  def 				UpperCAmelCase_				(		self		)		->						Optional[int]:






        """simple docstring"""





        for model_name in ["bert-base-uncased"]:
              _snake_case		:	str					     =     AutoConfig.from_pretrained(lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	str					     =     TFAutoModelForQuestionAnswering.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)

              _snake_case		:	Union[str, Any]					     =     AutoModelForQuestionAnswering.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
              self.assertIsNotNone(lowercase__		)
              self.assertIsInstance(lowercase__     ,     lowercase__		)







  def 				UpperCAmelCase_				(		self		)		->						str:






        """simple docstring"""





        _snake_case		:	Union[str, Any]					     =     TFAutoModelWithLMHead.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
        self.assertIsInstance(lowercase__     ,     lowercase__		)
        self.assertEqual(model.num_parameters()     ,     14_410		)
        self.assertEqual(model.num_parameters(only_trainable=lowercase__		)     ,     14_410		)

        _snake_case		:	Tuple					     =     AutoModelWithLMHead.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
        self.assertIsInstance(lowercase__     ,     lowercase__		)
        self.assertEqual(model.num_parameters()     ,     14_410		)
        self.assertEqual(model.num_parameters(only_trainable=lowercase__		)     ,     14_410		)







  def 				UpperCAmelCase_				(		self		)		->						str:






        """simple docstring"""





        _snake_case		:	List[str]					     =     TFAutoModelWithLMHead.from_pretrained(lowercase__     ,     from_pt=lowercase__		)
        self.assertIsInstance(lowercase__     ,     lowercase__		)
        self.assertEqual(model.num_parameters()     ,     14_410		)
        self.assertEqual(model.num_parameters(only_trainable=lowercase__		)     ,     14_410		)

        _snake_case		:	int					     =     AutoModelWithLMHead.from_pretrained(lowercase__     ,     from_tf=lowercase__		)
        self.assertIsInstance(lowercase__     ,     lowercase__		)
        self.assertEqual(model.num_parameters()     ,     14_410		)
        self.assertEqual(model.num_parameters(only_trainable=lowercase__		)     ,     14_410		)


