



import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


a_						:     Optional[Any]	     =		logging.getLogger(__name__)

def     __lowercase(	)     ->   str:


      """simple docstring"""



      lowerCamelCase								=		argparse.ArgumentParser(
          description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset." )
      parser.add_argument(
          "--dataset_name"   ,   type=lowerCAmelCase_   ,   default="wikitext"   ,   help="Name of the training. Explore datasets at: hf.co/datasets."   ,   )
      parser.add_argument(
          "--dataset_config"   ,   type=lowerCAmelCase_   ,   default="wikitext-103-raw-v1"   ,   help="Configuration name of the dataset." )
      parser.add_argument(
          "--tokenizer_name_or_path"   ,   type=lowerCAmelCase_   ,   default="sayakpaul/unigram-tokenizer-wikitext"   ,   help="Tokenizer identifier. Can be a local filepath or a Hub identifier."   ,   )
      parser.add_argument(
          "--shard_size"   ,   type=lowerCAmelCase_   ,   default=1000   ,   help="Number of entries to go in a single shard."   ,   )
      parser.add_argument("--split"   ,   type=lowerCAmelCase_   ,   default="train"   ,   choices=["train", "test", "validation"] )
      parser.add_argument(
          "--limit"   ,   default=lowerCAmelCase_   ,   type=lowerCAmelCase_   ,   help="Limit the number of shards (used for debugging)."   ,   )
      parser.add_argument(
          "--max_length"   ,   type=lowerCAmelCase_   ,   default=512   ,   help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
          " sequence length that is a multiple of 8."   ,   )
      parser.add_argument(
          "--output_dir"   ,   default="tf-tpu"   ,   type=lowerCAmelCase_   ,   help="Output directory where the TFRecord shards will be saved. If the"
          " path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord"
          " shards will be directly saved to a Google Cloud Storage bucket."   ,   )

      lowerCamelCase								=		parser.parse_args()
      return args

def     __lowercase(	UpperCAmelCase__ )     ->   Union[str, Any]:


      """simple docstring"""



      def fn(UpperCAmelCase__ ):
            return tokenizer(examples["text"] )

      return fn

def     __lowercase(	UpperCAmelCase__ )     ->   Any:


      """simple docstring"""



      lowerCamelCase								=		[]
      for i in range(len(tokenized_data["input_ids"] ) ):
            lowerCamelCase								=		{
                '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i] ) ),
                '''attention_mask''': tf.train.Feature(
                    intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i] ) ),
            }
            lowerCamelCase								=		tf.train.Features(feature=lowerCAmelCase_ )
            lowerCamelCase								=		tf.train.Example(features=lowerCAmelCase_ )
            lowerCamelCase								=		example.SerializeToString()
            records.append(lowerCAmelCase_ )
      return records

def     __lowercase(	UpperCAmelCase__ )     ->   Union[str, Any]:


      """simple docstring"""



      lowerCamelCase								=		datasets.load_dataset(args.dataset_name   ,   args.dataset_config   ,   split=args.split )

      if args.limit is not None:
            lowerCamelCase								=		min(len(lowerCAmelCase_ )   ,   args.limit )
            lowerCamelCase								=		dataset.select(range(lowerCAmelCase_ ) )
            print(F"""Limiting the dataset to {args.limit} entries.""" )

      lowerCamelCase								=		AutoTokenizer.from_pretrained(args.tokenizer_name_or_path )

      # Handle output directory creation.
      # For serializing into a Google Cloud Storage Bucket, one needs to first
      # create a bucket.
      if "gs" not in args.output_dir:
            if not os.path.exists(args.output_dir ):
                  os.makedirs(args.output_dir )
            lowerCamelCase								=		os.path.join(args.output_dir   ,   args.split )
            if not os.path.exists(lowerCAmelCase_ ):
                  os.makedirs(lowerCAmelCase_ )
      else:
            lowerCamelCase								=		os.path.join(args.output_dir   ,   args.split )

      # Tokenize the whole dataset at once.
      lowerCamelCase								=		tokenize_function(lowerCAmelCase_ )
      lowerCamelCase								=		dataset.map(lowerCAmelCase_   ,   batched=lowerCAmelCase_   ,   num_proc=4   ,   remove_columns=["text"] )

      # We need to concatenate all our texts together, and then split the result
      # into chunks of a fixed size, which we will call block_size. To do this, we
      # will use the map method again, with the option batched=True. When we use batched=True,
      # the function we pass to map() will be passed multiple inputs at once, allowing us
      # to group them into more or fewer examples than we had in the input.
      # This allows us to create our new fixed-length samples. The advantage of this
      # method is that we don't lose a whole lot of content from the dataset compared to the
      # case where we simply tokenize with a pre-defined max_length.

      def group_texts(UpperCAmelCase__ ):
            # Concatenate all texts.
            lowerCamelCase								=		{k: sum(examples[k]   ,   [] ) for k in examples.keys()}
            lowerCamelCase								=		len(concatenated_examples[list(examples.keys() )[0]] )
            # We drop the small remainder, though you could add padding instead if the model supports it
            # In this, as in all things, we advise you to follow your heart 🫀
            lowerCamelCase								=		(total_length // args.max_length) * args.max_length
            # Split by chunks of max_len.
            lowerCamelCase								=		{
                k: [t[i : i + args.max_length] for i in range(0   ,   lowerCAmelCase_   ,   args.max_length )]
                for k, t in concatenated_examples.items()
            }
            return result

      lowerCamelCase								=		dataset_tokenized.map(lowerCAmelCase_   ,   batched=lowerCAmelCase_   ,   batch_size=1000   ,   num_proc=4 )

      lowerCamelCase								=		0
      lowerCamelCase								=		0
      for shard in range(0   ,   len(lowerCAmelCase_ )   ,   args.shard_size ):
            lowerCamelCase								=		grouped_dataset[shard : shard + args.shard_size]
            lowerCamelCase								=		len(dataset_snapshot["input_ids"] )
            lowerCamelCase								=		os.path.join(lowerCAmelCase_   ,   F"""dataset-{shard_count}-{records_containing}.tfrecord""" )
            lowerCamelCase								=		get_serialized_examples(lowerCAmelCase_ )

            with tf.io.TFRecordWriter(lowerCAmelCase_ ) as out_file:
                  for i in range(len(lowerCAmelCase_ ) ):
                        lowerCamelCase								=		serialized_examples[i]
                        out_file.write(lowerCAmelCase_ )
                  print("Wrote file {} containing {} records".format(lowerCAmelCase_   ,   lowerCAmelCase_ ) )

            shard_count += 1
            total_records += records_containing

      with open(F"""split-{args.split}-records-count.txt"""   ,   "w" ) as f:
            print(F"""Total {args.split} records: {total_records}"""   ,   file=lowerCAmelCase_ )


if __name__ == "__main__":
     a_						:     int	     =		parse_args()
     main(args)




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
     from .tokenization_rembert import RemBertTokenizer
else:
     a_						:     Any	     =		None

a_						:     Optional[Any]	     =		logging.get_logger(__name__)
a_						:     int	     =		{'vocab_file': 'sentencepiece.model', 'tokenizer_file': 'tokenizer.json'}

a_						:     Dict	     =		{
    'vocab_file': {
        'google/rembert': 'https://huggingface.co/google/rembert/resolve/main/sentencepiece.model',
    },
    'tokenizer_file': {
        'google/rembert': 'https://huggingface.co/google/rembert/resolve/main/tokenizer.json',
    },
}

a_						:     List[str]	     =		{
    'google/rembert': 2_5_6,
}

a_						:     str	     =		'▁'





class    lowerCamelCase__  (						UpperCAmelCase_):
     """simple docstring"""



     _A       =			VOCAB_FILES_NAMES
     _A       =			PRETRAINED_VOCAB_FILES_MAP
     _A       =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _A       =			RemBertTokenizer







     def __init__(self				,			__a=None				,			__a=None				,			__a=True				,			__a=True				,			__a=False				,			__a="[CLS]"				,			__a="[SEP]"				,			__a="<unk>"				,			__a="[SEP]"				,			__a="<pad>"				,			__a="[CLS]"				,			__a="[MASK]"				,			**__a				,			):






           '''simple docstring'''



           lowerCamelCase								=		AddedToken(__a				,			lstrip=__a				,			rstrip=__a    ) if isinstance(__a				,			__a    ) else mask_token

           super().__init__(
               __a				,			tokenizer_file=__a				,			do_lower_case=__a				,			remove_space=__a				,			keep_accents=__a				,			bos_token=__a				,			eos_token=__a				,			unk_token=__a				,			sep_token=__a				,			pad_token=__a				,			cls_token=__a				,			mask_token=__a				,			**__a				,			)

           lowerCamelCase								=		do_lower_case
           lowerCamelCase								=		remove_space
           lowerCamelCase								=		keep_accents
           lowerCamelCase								=		vocab_file
           lowerCamelCase								=		False if not self.vocab_file else True







     def 						_a      (self				,			__a				,			__a = None    ):






           '''simple docstring'''



           lowerCamelCase								=		[self.sep_token_id]
           lowerCamelCase								=		[self.cls_token_id]
           if token_ids_a is None:
                 return cls + token_ids_a + sep
           return cls + token_ids_a + sep + token_ids_a + sep







     def 						_a      (self				,			__a				,			__a = None				,			__a = False    ):






           '''simple docstring'''



           if already_has_special_tokens:
                 if token_ids_a is not None:
                       raise ValueError(
                           "You should not supply a second sequence if the provided sequence of "
                           "ids is already formatted with special tokens for the model."    )
                 return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

           if token_ids_a is not None:
                 return [1] + ([0] * len(__a    )) + [1] + ([0] * len(__a    )) + [1]
           return [1] + ([0] * len(__a    )) + [1]







     def 						_a      (self				,			__a				,			__a = None    ):






           '''simple docstring'''



           lowerCamelCase								=		[self.sep_token_id]
           lowerCamelCase								=		[self.cls_token_id]

           if token_ids_a is None:
                 return len(cls + token_ids_a + sep    ) * [0]
           return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]







     def 						_a      (self				,			__a				,			__a = None    ):






           '''simple docstring'''



           if not os.path.isdir(__a    ):
                 logger.error("Vocabulary path ({}) should be a directory".format(__a    )    )
                 return
           lowerCamelCase								=		os.path.join(
               __a				,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )

           if os.path.abspath(self.vocab_file    ) != os.path.abspath(__a    ):
                 copyfile(self.vocab_file				,			__a    )

           return (out_vocab_file,)