


import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__UpperCamelCase    :						List[Any]     =							logging.getLogger(__name__)





def   _UpperCAmelCase       (     ):
      """simple docstring"""




      __lowerCamelCase     :      Optional[int]       						=						argparse.ArgumentParser(
          description="""Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."""      )
      parser.add_argument(
          """--dataset_name"""						,	type=UpperCAmelCase						,	default="""wikitext"""						,	help="""Name of the training. Explore datasets at: hf.co/datasets."""						,	)
      parser.add_argument(
          """--dataset_config"""						,	type=UpperCAmelCase						,	default="""wikitext-103-raw-v1"""						,	help="""Configuration name of the dataset."""      )
      parser.add_argument(
          """--tokenizer_name_or_path"""						,	type=UpperCAmelCase						,	default="""sayakpaul/unigram-tokenizer-wikitext"""						,	help="""Tokenizer identifier. Can be a local filepath or a Hub identifier."""						,	)
      parser.add_argument(
          """--shard_size"""						,	type=UpperCAmelCase						,	default=1_000						,	help="""Number of entries to go in a single shard."""						,	)
      parser.add_argument("""--split"""						,	type=UpperCAmelCase						,	default="""train"""						,	choices=["""train""", """test""", """validation"""]      )
      parser.add_argument(
          """--limit"""						,	default=UpperCAmelCase						,	type=UpperCAmelCase						,	help="""Limit the number of shards (used for debugging)."""						,	)
      parser.add_argument(
          """--max_length"""						,	type=UpperCAmelCase						,	default=512						,	help="""Maximum sequence length. For training on TPUs, it helps to have a maximum"""
          """ sequence length that is a multiple of 8."""						,	)
      parser.add_argument(
          """--output_dir"""						,	default="""tf-tpu"""						,	type=UpperCAmelCase						,	help="""Output directory where the TFRecord shards will be saved. If the"""
          """ path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"""
          """ shards will be directly saved to a Google Cloud Storage bucket."""						,	)

      __lowerCamelCase     :      Tuple       						=						parser.parse_args()
      return args





def   _UpperCAmelCase       (     UpperCAmelCase      :			int      ):
      """simple docstring"""




      def fn(UpperCAmelCase      :			Optional[Any]      ):
            return tokenizer(examples["""text"""]      )

      return fn





def   _UpperCAmelCase       (     UpperCAmelCase      :			List[str]      ):
      """simple docstring"""




      __lowerCamelCase     :      Union[str, Any]       						=						[]
      for i in range(len(tokenized_data["""input_ids"""]      )      ):
            __lowerCamelCase     :      Optional[Any]       						=						{
                """input_ids""": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["""input_ids"""][i]      )      ),
                """attention_mask""": tf.train.Feature(
                    intaa_list=tf.train.IntaaList(value=tokenized_data["""attention_mask"""][i]      )      ),
            }
            __lowerCamelCase     :      int       						=						tf.train.Features(feature=UpperCAmelCase      )
            __lowerCamelCase     :      List[Any]       						=						tf.train.Example(features=UpperCAmelCase      )
            __lowerCamelCase     :      Dict       						=						example.SerializeToString()
            records.append(UpperCAmelCase      )
      return records





def   _UpperCAmelCase       (     UpperCAmelCase      :			Optional[Any]      ):
      """simple docstring"""




      __lowerCamelCase     :      str       						=						datasets.load_dataset(args.dataset_name						,	args.dataset_config						,	split=args.split      )

      if args.limit is not None:
            __lowerCamelCase     :      Dict       						=						min(len(UpperCAmelCase      )						,	args.limit      )
            __lowerCamelCase     :      Optional[Any]       						=						dataset.select(range(UpperCAmelCase      )      )
            print(f"""Limiting the dataset to {args.limit} entries."""      )

      __lowerCamelCase     :      Optional[Any]       						=						AutoTokenizer.from_pretrained(args.tokenizer_name_or_path      )

      # Handle output directory creation.
      # For serializing into a Google Cloud Storage Bucket, one needs to first
      # create a bucket.
      if "gs" not in args.output_dir:
            if not os.path.exists(args.output_dir      ):
                  os.makedirs(args.output_dir      )
            __lowerCamelCase     :      Optional[int]       						=						os.path.join(args.output_dir						,	args.split      )
            if not os.path.exists(UpperCAmelCase      ):
                  os.makedirs(UpperCAmelCase      )
      else:
            __lowerCamelCase     :      List[Any]       						=						os.path.join(args.output_dir						,	args.split      )

      # Tokenize the whole dataset at once.
      __lowerCamelCase     :      str       						=						tokenize_function(UpperCAmelCase      )
      __lowerCamelCase     :      List[Any]       						=						dataset.map(UpperCAmelCase						,	batched=UpperCAmelCase						,	num_proc=4						,	remove_columns=["""text"""]      )

      # We need to concatenate all our texts together, and then split the result
      # into chunks of a fixed size, which we will call block_size. To do this, we
      # will use the map method again, with the option batched=True. When we use batched=True,
      # the function we pass to map() will be passed multiple inputs at once, allowing us
      # to group them into more or fewer examples than we had in the input.
      # This allows us to create our new fixed-length samples. The advantage of this
      # method is that we don't lose a whole lot of content from the dataset compared to the
      # case where we simply tokenize with a pre-defined max_length.

      def group_texts(UpperCAmelCase      :			Optional[Any]      ):
            # Concatenate all texts.
            __lowerCamelCase     :      Optional[int]       						=						{k: sum(examples[k]						,	[]      ) for k in examples.keys()}
            __lowerCamelCase     :      Dict       						=						len(concatenated_examples[list(examples.keys()      )[0]]      )
            # We drop the small remainder, though you could add padding instead if the model supports it
            # In this, as in all things, we advise you to follow your heart 🫀
            __lowerCamelCase     :      Optional[Any]       						=						(total_length // args.max_length) * args.max_length
            # Split by chunks of max_len.
            __lowerCamelCase     :      Union[str, Any]       						=						{
                k: [t[i : i + args.max_length] for i in range(0						,	UpperCAmelCase						,	args.max_length      )]
                for k, t in concatenated_examples.items()
            }
            return result

      __lowerCamelCase     :      Union[str, Any]       						=						dataset_tokenized.map(UpperCAmelCase						,	batched=UpperCAmelCase						,	batch_size=1_000						,	num_proc=4      )

      __lowerCamelCase     :      Optional[Any]       						=						0
      __lowerCamelCase     :      Tuple       						=						0
      for shard in range(0						,	len(UpperCAmelCase      )						,	args.shard_size      ):
            __lowerCamelCase     :      Dict       						=						grouped_dataset[shard : shard + args.shard_size]
            __lowerCamelCase     :      Union[str, Any]       						=						len(dataset_snapshot["""input_ids"""]      )
            __lowerCamelCase     :      Tuple       						=						os.path.join(UpperCAmelCase						,	f"""dataset-{shard_count}-{records_containing}.tfrecord"""      )
            __lowerCamelCase     :      Optional[Any]       						=						get_serialized_examples(UpperCAmelCase      )

            with tf.io.TFRecordWriter(UpperCAmelCase      ) as out_file:
                  for i in range(len(UpperCAmelCase      )      ):
                        __lowerCamelCase     :      List[Any]       						=						serialized_examples[i]
                        out_file.write(UpperCAmelCase      )
                  print("""Wrote file {} containing {} records""".format(UpperCAmelCase						,	UpperCAmelCase      )      )

            shard_count += 1
            total_records += records_containing

      with open(f"""split-{args.split}-records-count.txt"""						,	"""w"""      ) as f:
            print(f"""Total {args.split} records: {total_records}"""						,	file=UpperCAmelCase      )


if __name__ == "__main__":
    __UpperCamelCase    :						Optional[Any]     =							parse_args()
    main(args)







from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCamelCase    :						Tuple     =							logging.get_logger(__name__)

__UpperCamelCase    :						Any     =							{
    'facebook/nllb-moe-54B': 'https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json',
}



class 							_UpperCamelCase (						A						):

   '''simple docstring'''







   a_				:							int    = "nllb-moe"
   a_				:							Optional[int]    = ["past_key_values"]
   a_				:							Optional[Any]    = {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}

   def __init__(  self  : int       ,  _lowerCamelCase  : Union[str, Any]=1_2_8_1_1_2       ,  _lowerCamelCase  : List[str]=1_0_2_4       ,  _lowerCamelCase  : Optional[int]=1_2       ,  _lowerCamelCase  : int=4_0_9_6       ,  _lowerCamelCase  : Union[str, Any]=1_6       ,  _lowerCamelCase  : int=1_2       ,  _lowerCamelCase  : Optional[int]=4_0_9_6       ,  _lowerCamelCase  : Any=1_6       ,  _lowerCamelCase  : Any=0.05       ,  _lowerCamelCase  : List[Any]=0.05       ,  _lowerCamelCase  : Tuple=True       ,  _lowerCamelCase  : Any=True       ,  _lowerCamelCase  : Optional[int]="relu"       ,  _lowerCamelCase  : Dict=1_0_2_4       ,  _lowerCamelCase  : Optional[int]=0.1       ,  _lowerCamelCase  : Optional[int]=0.1       ,  _lowerCamelCase  : Optional[int]=0.0       ,  _lowerCamelCase  : List[Any]=0.02       ,  _lowerCamelCase  : List[str]=2       ,  _lowerCamelCase  : List[str]=True       ,  _lowerCamelCase  : Union[str, Any]=False       ,  _lowerCamelCase  : Optional[Any]="float32"       ,  _lowerCamelCase  : Union[str, Any]=False       ,  _lowerCamelCase  : Tuple=1_2_8       ,  _lowerCamelCase  : Dict=6_4       ,  _lowerCamelCase  : Any=4       ,  _lowerCamelCase  : List[Any]=4       ,  _lowerCamelCase  : str=0.001       ,  _lowerCamelCase  : Dict=0.001       ,  _lowerCamelCase  : Any="all"       ,  _lowerCamelCase  : Optional[Any]=False       ,  _lowerCamelCase  : List[Any]=False       ,  _lowerCamelCase  : Optional[Any]=1.0       ,  _lowerCamelCase  : List[str]=0.2       ,  _lowerCamelCase  : List[Any]=1       ,  _lowerCamelCase  : str=0       ,  _lowerCamelCase  : int=2       ,  _lowerCamelCase  : int=False       ,  **_lowerCamelCase  : List[str]       ,  ):






         '''simple docstring'''



         __lowerCamelCase     :      Optional[Any]       						=						vocab_size
         __lowerCamelCase     :      str       						=						max_position_embeddings
         __lowerCamelCase     :      int       						=						d_model
         __lowerCamelCase     :      Optional[Any]       						=						encoder_ffn_dim
         __lowerCamelCase     :      str       						=						encoder_layers
         __lowerCamelCase     :      Tuple       						=						encoder_attention_heads
         __lowerCamelCase     :      Union[str, Any]       						=						decoder_ffn_dim
         __lowerCamelCase     :      Dict       						=						decoder_layers
         __lowerCamelCase     :      Any       						=						decoder_attention_heads
         __lowerCamelCase     :      Union[str, Any]       						=						dropout
         __lowerCamelCase     :      Dict       						=						attention_dropout
         __lowerCamelCase     :      int       						=						activation_dropout
         __lowerCamelCase     :      Union[str, Any]       						=						activation_function
         __lowerCamelCase     :      Optional[int]       						=						init_std
         __lowerCamelCase     :      Union[str, Any]       						=						encoder_layerdrop
         __lowerCamelCase     :      Dict       						=						decoder_layerdrop
         __lowerCamelCase     :      List[str]       						=						use_cache
         __lowerCamelCase     :      Any       						=						encoder_layers
         __lowerCamelCase     :      List[Any]       						=						scale_embedding  # scale factor will be sqrt(d_model) if True
         __lowerCamelCase     :      Optional[Any]       						=						router_z_loss_coef
         __lowerCamelCase     :      Union[str, Any]       						=						router_aux_loss_coef
         __lowerCamelCase     :      Dict       						=						decoder_sparse_step
         __lowerCamelCase     :      Optional[int]       						=						encoder_sparse_step
         __lowerCamelCase     :      Any       						=						num_experts
         __lowerCamelCase     :      int       						=						expert_capacity
         __lowerCamelCase     :      Dict       						=						router_bias
         if router_dtype not in ["float32", "float16", "bfloat16"]:
               raise ValueError(F"""`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}"""							)
         __lowerCamelCase     :      Tuple       						=						router_dtype

         __lowerCamelCase     :      Dict       						=						router_ignore_padding_tokens
         __lowerCamelCase     :      str       						=						batch_prioritized_routing
         __lowerCamelCase     :      List[str]       						=						second_expert_policy
         __lowerCamelCase     :      Tuple       						=						normalize_router_prob_before_dropping
         __lowerCamelCase     :      Dict       						=						moe_eval_capacity_token_fraction
         __lowerCamelCase     :      Optional[int]       						=						moe_token_dropout
         __lowerCamelCase     :      Dict       						=						output_router_logits
         super().__init__(
             pad_token_id=_lowerCamelCase       ,  bos_token_id=_lowerCamelCase       ,  eos_token_id=_lowerCamelCase       ,  is_encoder_decoder=_lowerCamelCase       ,  decoder_start_token_id=_lowerCamelCase       ,  **_lowerCamelCase       ,  )



