



'''simple docstring'''



import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


lowerCAmelCase	:		List[Any]				=   logging.getLogger(__name__)



def   __lowerCAmelCase					(							):






 '''simple docstring'''




 __lowerCAmelCase												=  argparse.ArgumentParser(
     description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."					)
 parser.add_argument(
     "--dataset_name"				,		type=lowerCamelCase				,		default="wikitext"				,		help="Name of the training. Explore datasets at: hf.co/datasets."				,		)
 parser.add_argument(
     "--dataset_config"				,		type=lowerCamelCase				,		default="wikitext-103-raw-v1"				,		help="Configuration name of the dataset."					)
 parser.add_argument(
     "--tokenizer_name_or_path"				,		type=lowerCamelCase				,		default="sayakpaul/unigram-tokenizer-wikitext"				,		help="Tokenizer identifier. Can be a local filepath or a Hub identifier."				,		)
 parser.add_argument(
     "--shard_size"				,		type=lowerCamelCase				,		default=10_00				,		help="Number of entries to go in a single shard."				,		)
 parser.add_argument("--split"				,		type=lowerCamelCase				,		default="train"				,		choices=["train", "test", "validation"]					)
 parser.add_argument(
     "--limit"				,		default=lowerCamelCase				,		type=lowerCamelCase				,		help="Limit the number of shards (used for debugging)."				,		)
 parser.add_argument(
     "--max_length"				,		type=lowerCamelCase				,		default=5_12				,		help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
     " sequence length that is a multiple of 8."				,		)
 parser.add_argument(
     "--output_dir"				,		default="tf-tpu"				,		type=lowerCamelCase				,		help="Output directory where the TFRecord shards will be saved. If the"
     " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
     " shards will be directly saved to a Google Cloud Storage bucket."				,		)

 __lowerCAmelCase												=  parser.parse_args()
 return args



def   __lowerCAmelCase					(							lowerCamelCase   :						int					):






 '''simple docstring'''




 def fn(lowerCamelCase   :						Optional[Any]					):
  return tokenizer(examples["text"]					)

 return fn



def   __lowerCAmelCase					(							lowerCamelCase   :						List[str]					):






 '''simple docstring'''




 __lowerCAmelCase												=  []
 for i in range(len(tokenized_data["input_ids"]					)					):
  __lowerCAmelCase												=  {
      "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]					)					),
      "attention_mask": tf.train.Feature(
          intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]					)					),
  }
  __lowerCAmelCase												=  tf.train.Features(feature=lowerCamelCase					)
  __lowerCAmelCase												=  tf.train.Example(features=lowerCamelCase					)
  __lowerCAmelCase												=  example.SerializeToString()
  records.append(lowerCamelCase					)
 return records



def   __lowerCAmelCase					(							lowerCamelCase   :						Optional[Any]					):






 '''simple docstring'''




 __lowerCAmelCase												=  datasets.load_dataset(args.dataset_name				,		args.dataset_config				,		split=args.split					)

 if args.limit is not None:
  __lowerCAmelCase												=  min(len(lowerCamelCase					)				,		args.limit					)
  __lowerCAmelCase												=  dataset.select(range(lowerCamelCase					)					)
  print(f'''Limiting the dataset to {args.limit} entries.'''					)

 __lowerCAmelCase												=  AutoTokenizer.from_pretrained(args.tokenizer_name_or_path					)

 # Handle output directory creation.
 # For serializing into a Google Cloud Storage Bucket, one needs to first
 # create a bucket.
 if "gs" not in args.output_dir:
  if not os.path.exists(args.output_dir					):
   os.makedirs(args.output_dir					)
  __lowerCAmelCase												=  os.path.join(args.output_dir				,		args.split					)
  if not os.path.exists(lowerCamelCase					):
   os.makedirs(lowerCamelCase					)
 else:
  __lowerCAmelCase												=  os.path.join(args.output_dir				,		args.split					)

 # Tokenize the whole dataset at once.
 __lowerCAmelCase												=  tokenize_function(lowerCamelCase					)
 __lowerCAmelCase												=  dataset.map(lowerCamelCase				,		batched=lowerCamelCase				,		num_proc=4				,		remove_columns=["text"]					)

 # We need to concatenate all our texts together, and then split the result
 # into chunks of a fixed size, which we will call block_size. To do this, we
 # will use the map method again, with the option batched=True. When we use batched=True,
 # the function we pass to map() will be passed multiple inputs at once, allowing us
 # to group them into more or fewer examples than we had in the input.
 # This allows us to create our new fixed-length samples. The advantage of this
 # method is that we don't lose a whole lot of content from the dataset compared to the
 # case where we simply tokenize with a pre-defined max_length.

 def group_texts(lowerCamelCase   :						Optional[Any]					):
  # Concatenate all texts.
  __lowerCAmelCase												=  {k: sum(examples[k]				,		[]					) for k in examples.keys()}
  __lowerCAmelCase												=  len(concatenated_examples[list(examples.keys()					)[0]]					)
  # We drop the small remainder, though you could add padding instead if the model supports it
  # In this, as in all things, we advise you to follow your heart 🫀
  __lowerCAmelCase												=  (total_length // args.max_length) * args.max_length
  # Split by chunks of max_len.
  __lowerCAmelCase												=  {
      k: [t[i : i + args.max_length] for i in range(0				,		lowerCamelCase				,		args.max_length					)]
      for k, t in concatenated_examples.items()
  }
  return result

 __lowerCAmelCase												=  dataset_tokenized.map(lowerCamelCase				,		batched=lowerCamelCase				,		batch_size=10_00				,		num_proc=4					)

 __lowerCAmelCase												=  0
 __lowerCAmelCase												=  0
 for shard in range(0				,		len(lowerCamelCase					)				,		args.shard_size					):
  __lowerCAmelCase												=  grouped_dataset[shard : shard + args.shard_size]
  __lowerCAmelCase												=  len(dataset_snapshot["input_ids"]					)
  __lowerCAmelCase												=  os.path.join(lowerCamelCase				,		f'''dataset-{shard_count}-{records_containing}.tfrecord'''					)
  __lowerCAmelCase												=  get_serialized_examples(lowerCamelCase					)

  with tf.io.TFRecordWriter(lowerCamelCase					) as out_file:
   for i in range(len(lowerCamelCase					)					):
    __lowerCAmelCase												=  serialized_examples[i]
    out_file.write(lowerCamelCase					)
   print("Wrote file {} containing {} records".format(lowerCamelCase				,		lowerCamelCase					)					)

  shard_count += 1
  total_records += records_containing

 with open(f'''split-{args.split}-records-count.txt'''				,		"w"					) as f:
  print(f'''Total {args.split} records: {total_records}'''				,		file=lowerCamelCase					)


if __name__ == "__main__":
 lowerCAmelCase	:		Optional[Any]				=   parse_args()
 main(args)




'''simple docstring'''



import os
import sys
import unittest


lowerCAmelCase	:		str				=   os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, '''utils'''))

import get_test_info  # noqa: E402
from get_test_info import (  # noqa: E402
    get_model_to_test_mapping,
    get_model_to_tester_mapping,
    get_test_to_tester_mapping,
)


lowerCAmelCase	:		Tuple				=   os.path.join('''tests''', '''models''', '''bert''', '''test_modeling_bert.py''')
lowerCAmelCase	:		Tuple				=   os.path.join('''tests''', '''models''', '''blip''', '''test_modeling_blip.py''')






class 					UpperCAmelCase__	(		unittest.TestCase ):



					def        UpperCAmelCase_ (    self     )     ->			str:
						__lowerCAmelCase												=  get_test_to_tester_mapping(UpperCamelCase     )
						__lowerCAmelCase												=  get_test_to_tester_mapping(UpperCamelCase     )

						__lowerCAmelCase												=  {"BertModelTest": "BertModelTester"}

						__lowerCAmelCase												=  {
						    "BlipModelTest": "BlipModelTester",
						    "BlipTextImageModelTest": "BlipTextImageModelsModelTester",
						    "BlipTextModelTest": "BlipTextModelTester",
						    "BlipTextRetrievalModelTest": "BlipTextRetrievalModelTester",
						    "BlipVQAModelTest": "BlipVQAModelTester",
						    "BlipVisionModelTest": "BlipVisionModelTester",
						}

						self.assertEqual(get_test_info.to_json(UpperCamelCase     )			,				UpperCamelCase     )
						self.assertEqual(get_test_info.to_json(UpperCamelCase     )			,				UpperCamelCase     )



					def        UpperCAmelCase_ (    self     )     ->			List[Any]:
						__lowerCAmelCase												=  get_model_to_test_mapping(UpperCamelCase     )
						__lowerCAmelCase												=  get_model_to_test_mapping(UpperCamelCase     )

						__lowerCAmelCase												=  {
						    "BertForMaskedLM": ["BertModelTest"],
						    "BertForMultipleChoice": ["BertModelTest"],
						    "BertForNextSentencePrediction": ["BertModelTest"],
						    "BertForPreTraining": ["BertModelTest"],
						    "BertForQuestionAnswering": ["BertModelTest"],
						    "BertForSequenceClassification": ["BertModelTest"],
						    "BertForTokenClassification": ["BertModelTest"],
						    "BertLMHeadModel": ["BertModelTest"],
						    "BertModel": ["BertModelTest"],
						}

						__lowerCAmelCase												=  {
						    "BlipForConditionalGeneration": ["BlipTextImageModelTest"],
						    "BlipForImageTextRetrieval": ["BlipTextRetrievalModelTest"],
						    "BlipForQuestionAnswering": ["BlipVQAModelTest"],
						    "BlipModel": ["BlipModelTest"],
						    "BlipTextModel": ["BlipTextModelTest"],
						    "BlipVisionModel": ["BlipVisionModelTest"],
						}

						self.assertEqual(get_test_info.to_json(UpperCamelCase     )			,				UpperCamelCase     )
						self.assertEqual(get_test_info.to_json(UpperCamelCase     )			,				UpperCamelCase     )



					def        UpperCAmelCase_ (    self     )     ->			str:
						__lowerCAmelCase												=  get_model_to_tester_mapping(UpperCamelCase     )
						__lowerCAmelCase												=  get_model_to_tester_mapping(UpperCamelCase     )

						__lowerCAmelCase												=  {
						    "BertForMaskedLM": ["BertModelTester"],
						    "BertForMultipleChoice": ["BertModelTester"],
						    "BertForNextSentencePrediction": ["BertModelTester"],
						    "BertForPreTraining": ["BertModelTester"],
						    "BertForQuestionAnswering": ["BertModelTester"],
						    "BertForSequenceClassification": ["BertModelTester"],
						    "BertForTokenClassification": ["BertModelTester"],
						    "BertLMHeadModel": ["BertModelTester"],
						    "BertModel": ["BertModelTester"],
						}

						__lowerCAmelCase												=  {
						    "BlipForConditionalGeneration": ["BlipTextImageModelsModelTester"],
						    "BlipForImageTextRetrieval": ["BlipTextRetrievalModelTester"],
						    "BlipForQuestionAnswering": ["BlipVQAModelTester"],
						    "BlipModel": ["BlipModelTester"],
						    "BlipTextModel": ["BlipTextModelTester"],
						    "BlipVisionModel": ["BlipVisionModelTester"],
						}

						self.assertEqual(get_test_info.to_json(UpperCamelCase     )			,				UpperCamelCase     )
						self.assertEqual(get_test_info.to_json(UpperCamelCase     )			,				UpperCamelCase     )