




import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


_snake_case            = logging.getLogger(__name__)

_snake_case            = tf.data.AUTOTUNE







def        lowerCAmelCase_			(  ):
      _A							:  Tuple    =		argparse.ArgumentParser(description="""Train a masked language model on TPU."""  )
      parser.add_argument(
          """--pretrained_model_config""",type=snake_case_,default="""roberta-base""",help="""The model config to use. Note that we don't copy the model's weights, only the config!""",)
      parser.add_argument(
          """--tokenizer""",type=snake_case_,default="""unigram-tokenizer-wikitext""",help="""The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model's vocab size.""",)

      parser.add_argument(
          """--per_replica_batch_size""",type=snake_case_,default=8,help="""Batch size per TPU core.""",)

      parser.add_argument(
          """--no_tpu""",action="""store_true""",help="""If set, run on CPU and don't try to initialize a TPU. Useful for debugging on non-TPU instances.""",)

      parser.add_argument(
          """--tpu_name""",type=snake_case_,help="""Name of TPU resource to initialize. Should be blank on Colab, and 'local' on TPU VMs.""",default="""local""",)

      parser.add_argument(
          """--tpu_zone""",type=snake_case_,help="""Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes.""",)

      parser.add_argument(
          """--gcp_project""",type=snake_case_,help="""Google cloud project name. Only used for non-Colab TPU nodes."""  )

      parser.add_argument(
          """--bfloat16""",action="""store_true""",help="""Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU.""",)

      parser.add_argument(
          """--train_dataset""",type=snake_case_,help="""Path to training dataset to load. If the path begins with `gs://`"""
          """ then the dataset will be loaded from a Google Cloud Storage bucket.""",)

      parser.add_argument(
          """--shuffle_buffer_size""",type=snake_case_,default=2**18,help="""Size of the shuffle buffer (in samples)""",)

      parser.add_argument(
          """--eval_dataset""",type=snake_case_,help="""Path to evaluation dataset to load. If the path begins with `gs://`"""
          """ then the dataset will be loaded from a Google Cloud Storage bucket.""",)

      parser.add_argument(
          """--num_epochs""",type=snake_case_,default=1,help="""Number of epochs to train for.""",)

      parser.add_argument(
          """--learning_rate""",type=snake_case_,default=1e-4,help="""Learning rate to use for training.""",)

      parser.add_argument(
          """--weight_decay_rate""",type=snake_case_,default=1e-3,help="""Weight decay rate to use for training.""",)

      parser.add_argument(
          """--max_length""",type=snake_case_,default=512,help="""Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py""",)

      parser.add_argument(
          """--mlm_probability""",type=snake_case_,default=0.15,help="""Fraction of tokens to mask during training.""",)

      parser.add_argument("""--output_dir""",type=snake_case_,required=snake_case_,help="""Path to save model checkpoints to."""  )
      parser.add_argument("""--hub_model_id""",type=snake_case_,help="""Model ID to upload to on the Hugging Face Hub."""  )

      _A							:  Any    =		parser.parse_args()
      return args







def        lowerCAmelCase_			(  snake_case_  ):
      try:
            if args.tpu_name:
                  _A							:  List[str]    =		tf.distribute.cluster_resolver.TPUClusterResolver(
                      args.tpu_name,zone=args.tpu_zone,project=args.gcp_project  )
            else:
                  _A							:  Optional[int]    =		tf.distribute.cluster_resolver.TPUClusterResolver()
      except ValueError:
            raise RuntimeError(
                """Couldn't connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or """
                """--gcp_project. When running on a TPU VM, use --tpu_name local."""  )

      tf.config.experimental_connect_to_cluster(snake_case_  )
      tf.tpu.experimental.initialize_tpu_system(snake_case_  )

      return tpu







def        lowerCAmelCase_			(  snake_case_  ):
      _A							:  Optional[Any]    =		0
      for file in file_list:
            _A							:  Optional[Any]    =		file.split("""/"""  )[-1]
            _A							:  List[Any]    =		re.search(r"""-\d+-(\d+)\.tfrecord""",snake_case_  ).group(1  )
            _A							:  Dict    =		int(snake_case_  )
            num_samples += sample_count

      return num_samples







def        lowerCAmelCase_			(  snake_case_,snake_case_,snake_case_,snake_case_,snake_case_,snake_case_=None  ):
      _A							:  Tuple    =		count_samples(snake_case_  )
      _A							:  List[str]    =		tf.data.Dataset.from_tensor_slices(snake_case_  )
      if shuffle:
            _A							:  Union[str, Any]    =		dataset.shuffle(len(snake_case_  )  )
      _A							:  Dict    =		tf.data.TFRecordDataset(snake_case_,num_parallel_reads=snake_case_  )
      # TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
      _A							:  Union[str, Any]    =		dataset.apply(tf.data.experimental.assert_cardinality(snake_case_  )  )
      _A							:  int    =		dataset.map(snake_case_,num_parallel_calls=snake_case_  )
      if shuffle:
            assert shuffle_buffer_size is not None
            _A							:  int    =		dataset.shuffle(args.shuffle_buffer_size  )
      _A							:  str    =		dataset.batch(snake_case_,drop_remainder=snake_case_  )
      _A							:  Tuple    =		dataset.map(snake_case_,num_parallel_calls=snake_case_  )
      _A							:  Any    =		dataset.prefetch(snake_case_  )
      return dataset







def        lowerCAmelCase_			(  snake_case_  ):
      if not args.no_tpu:
            _A							:  Dict    =		initialize_tpu(snake_case_  )
            _A							:  str    =		tf.distribute.TPUStrategy(snake_case_  )
      else:
            _A							:  Optional[int]    =		tf.distribute.OneDeviceStrategy(device="""/gpu:0"""  )

      if args.bfloataa:
            tf.keras.mixed_precision.set_global_policy("""mixed_bfloat16"""  )

      _A							:  List[Any]    =		AutoTokenizer.from_pretrained(args.tokenizer  )
      _A							:  int    =		AutoConfig.from_pretrained(args.pretrained_model_config  )
      _A							:  int    =		tokenizer.vocab_size

      _A							:  Tuple    =		tf.io.gfile.glob(os.path.join(args.train_dataset,"""*.tfrecord"""  )  )
      if not training_records:
            raise ValueError(f'''No .tfrecord files found in {args.train_dataset}.'''  )
      _A							:  Dict    =		tf.io.gfile.glob(os.path.join(args.eval_dataset,"""*.tfrecord"""  )  )
      if not eval_records:
            raise ValueError(f'''No .tfrecord files found in {args.eval_dataset}.'''  )

      _A							:  Optional[Any]    =		count_samples(snake_case_  )

      _A							:  List[str]    =		num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
      _A							:  Any    =		steps_per_epoch * args.num_epochs

      with strategy.scope():
            _A							:  List[str]    =		TFAutoModelForMaskedLM.from_config(snake_case_  )
            model(model.dummy_inputs  )  # Pass some dummy inputs through the model to ensure all the weights are built
            _A    ,     _A							:  List[str]    =		create_optimizer(
                num_train_steps=snake_case_,num_warmup_steps=total_train_steps // 20,init_lr=args.learning_rate,weight_decay_rate=args.weight_decay_rate,)

            # Transformers models compute the right loss for their task by default when labels are passed, and will
            # use this for training unless you specify your own loss function in compile().
            model.compile(optimizer=snake_case_,metrics=["""accuracy"""]  )

      def decode_fn(snake_case_  ):
            _A							:  Union[str, Any]    =		{
                """input_ids""": tf.io.FixedLenFeature(dtype=tf.intaa,shape=(args.max_length,)  ),
                """attention_mask""": tf.io.FixedLenFeature(dtype=tf.intaa,shape=(args.max_length,)  ),
            }
            return tf.io.parse_single_example(snake_case_,snake_case_  )

      # Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
      # use their methods in our data pipeline.
      _A							:  List[Any]    =		DataCollatorForLanguageModeling(
          tokenizer=snake_case_,mlm_probability=args.mlm_probability,mlm=snake_case_,return_tensors="""tf"""  )

      def mask_with_collator(snake_case_  ):
            # TF really needs an isin() function
            _A							:  Tuple    =		(
                ~tf.cast(batch["""attention_mask"""],tf.bool  )
                | (batch["""input_ids"""] == tokenizer.cls_token_id)
                | (batch["""input_ids"""] == tokenizer.sep_token_id)
            )
            _A    ,     _A							:  str    =		data_collator.tf_mask_tokens(
                batch["""input_ids"""],vocab_size=len(snake_case_  ),mask_token_id=tokenizer.mask_token_id,special_tokens_mask=snake_case_,)
            return batch

      _A							:  List[str]    =		args.per_replica_batch_size * strategy.num_replicas_in_sync

      _A							:  Optional[int]    =		prepare_dataset(
          snake_case_,decode_fn=snake_case_,mask_fn=snake_case_,batch_size=snake_case_,shuffle=snake_case_,shuffle_buffer_size=args.shuffle_buffer_size,)

      _A							:  int    =		prepare_dataset(
          snake_case_,decode_fn=snake_case_,mask_fn=snake_case_,batch_size=snake_case_,shuffle=snake_case_,)

      _A							:  Optional[int]    =		[]
      if args.hub_model_id:
            callbacks.append(
                PushToHubCallback(output_dir=args.output_dir,hub_model_id=args.hub_model_id,tokenizer=snake_case_  )  )

      model.fit(
          snake_case_,validation_data=snake_case_,epochs=args.num_epochs,callbacks=snake_case_,)

      model.save_pretrained(args.output_dir  )


if __name__ == "__main__":
  _snake_case            = parse_args()
  main(args)







import os
import re
import shutil
import sys
import tempfile
import unittest

import black


_snake_case            = os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, "utils"))

import check_copies  # noqa: E402


# This is the reference code that will be used in the tests.
# If BertLMPredictionHead is changed in modeling_bert.py, this code needs to be manually updated.
_snake_case            = "    def __init__(self, config):\n        super().__init__()\n        self.transform = BertPredictionHeadTransform(config)\n\n        # The output weights are the same as the input embeddings, but there is\n        # an output-only bias for each token.\n        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)\n\n        self.bias = nn.Parameter(torch.zeros(config.vocab_size))\n\n        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`\n        self.decoder.bias = self.bias\n\n    def forward(self, hidden_states):\n        hidden_states = self.transform(hidden_states)\n        hidden_states = self.decoder(hidden_states)\n        return hidden_states\n"







class 		lowercase							(     unittest.TestCase	):

     def       a__			(						self       )		->							Union[str, Any]:
           _A							:  List[str]    =		tempfile.mkdtemp()
           os.makedirs(os.path.join(self.transformer_dir	,  """models/bert/"""       )       )
           _A							:  str    =		self.transformer_dir
           shutil.copy(
               os.path.join(_a	,  """src/transformers/models/bert/modeling_bert.py"""       )	,  os.path.join(self.transformer_dir	,  """models/bert/modeling_bert.py"""       )	,  )

     def       a__			(						self       )		->							Optional[int]:
           _A							:  List[str]    =		"""src/transformers"""
           shutil.rmtree(self.transformer_dir       )

     def       a__			(						self	,  _a	,  _a	,  _a	,  _a=None       )		->							Optional[Any]:
           _A							:  Optional[Any]    =		comment + F'''\nclass {class_name}(nn.Module):\n''' + class_code
           if overwrite_result is not None:
                 _A							:  List[str]    =		comment + F'''\nclass {class_name}(nn.Module):\n''' + overwrite_result
           _A							:  List[str]    =		black.Mode(target_versions={black.TargetVersion.PYaa}	,  line_length=119       )
           _A							:  Optional[int]    =		black.format_str(_a	,  mode=_a       )
           _A							:  Optional[Any]    =		os.path.join(self.transformer_dir	,  """new_code.py"""       )
           with open(_a	,  """w"""	,  newline="""\n"""       ) as f:
                 f.write(_a       )
           if overwrite_result is None:
                 self.assertTrue(len(check_copies.is_copy_consistent(_a       )       ) == 0       )
           else:
                 check_copies.is_copy_consistent(f.name	,  overwrite=_a       )
                 with open(_a	,  """r"""       ) as f:
                       self.assertTrue(f.read()	,  _a       )

     def       a__			(						self       )		->							str:
           _A							:  Union[str, Any]    =		check_copies.find_code_in_transformers("""models.bert.modeling_bert.BertLMPredictionHead"""       )
           self.assertEqual(_a	,  _a       )

     def       a__			(						self       )		->							int:
           # Base copy consistency
           self.check_copy_consistency(
               """# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead"""	,  """BertLMPredictionHead"""	,  REFERENCE_CODE + """\n"""	,  )

           # With no empty line at the end
           self.check_copy_consistency(
               """# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead"""	,  """BertLMPredictionHead"""	,  _a	,  )

           # Copy consistency with rename
           self.check_copy_consistency(
               """# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel"""	,  """TestModelLMPredictionHead"""	,  re.sub("""Bert"""	,  """TestModel"""	,  _a       )	,  )

           # Copy consistency with a really long name
           _A							:  List[str]    =		"""TestModelWithAReallyLongNameBecauseSomePeopleLikeThatForSomeReason"""
           self.check_copy_consistency(
               F'''# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->{long_class_name}'''	,  F'''{long_class_name}LMPredictionHead'''	,  re.sub("""Bert"""	,  _a	,  _a       )	,  )

           # Copy consistency with overwrite
           self.check_copy_consistency(
               """# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel"""	,  """TestModelLMPredictionHead"""	,  _a	,  overwrite_result=re.sub("""Bert"""	,  """TestModel"""	,  _a       )	,  )



     def       a__			(						self       )		->							Tuple:
           _A							:  Union[str, Any]    =		check_copies.LOCALIZED_READMES["""README_zh-hans.md"""]

           _A							:  str    =		(
               """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the"""
               """ Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for"""
               """ Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong"""
               """ Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.\n1."""
               """ **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (from HuggingFace),"""
               """ released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and"""
               """ lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same"""
               """ method has been applied to compress GPT2 into"""
               """ [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into"""
               """ [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),"""
               """ Multilingual BERT into"""
               """ [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German"""
               """ version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)**"""
               """ (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders"""
               """ as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang"""
               """ Luong, Quoc V. Le, Christopher D. Manning."""
           )
           _A							:  str    =		(
               """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"""
               """ Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
               """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
               """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"""
           )
           _A							:  Any    =		(
               """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"""
               """ Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
               """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
               """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n1."""
               """ **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (来自 HuggingFace) 伴随论文"""
               """ [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and"""
               """ lighter](https://arxiv.org/abs/1910.01108) 由 Victor Sanh, Lysandre Debut and Thomas Wolf 发布。 The same"""
               """ method has been applied to compress GPT2 into"""
               """ [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into"""
               """ [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),"""
               """ Multilingual BERT into"""
               """ [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German"""
               """ version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)** (来自"""
               """ Google Research/Stanford University) 伴随论文 [ELECTRA: Pre-training text encoders as discriminators rather"""
               """ than generators](https://arxiv.org/abs/2003.10555) 由 Kevin Clark, Minh-Thang Luong, Quoc V. Le,"""
               """ Christopher D. Manning 发布。\n"""
           )

           _A    ,     _A							:  Tuple    =		check_copies.convert_to_localized_md(
               _a	,  _a	,  localized_readme["""format_model_list"""]       )

           self.assertFalse(_a       )
           self.assertEqual(_a	,  _a       )

           _A    ,     _A							:  List[str]    =		check_copies.convert_to_localized_md(
               _a	,  _a	,  localized_readme["""format_model_list"""]       )

           # Check whether the number of models is equal to README.md after conversion.
           self.assertTrue(_a       )

           _A							:  Tuple    =		(
               """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the"""
               """ Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for"""
               """ Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong"""
               """ Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut."""
           )
           _A							:  Dict    =		(
               """1. **[ALBERT](https://huggingface.co/transformers/main/model_doc/albert.html)** (来自 Google Research and"""
               """ the Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
               """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
               """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"""
           )
           _A							:  Optional[Any]    =		(
               """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"""
               """ Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
               """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
               """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"""
           )

           _A    ,     _A							:  Optional[int]    =		check_copies.convert_to_localized_md(
               _a	,  _a	,  localized_readme["""format_model_list"""]       )

           # Check if the model link is synchronized.
           self.assertEqual(_a	,  _a       )

