






import os
import re
import shutil
import sys
import tempfile
import unittest

import black


__UpperCAmelCase							:   List[str]							=  os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, "utils"))

import check_copies  # noqa: E402


# This is the reference code that will be used in the tests.
# If BertLMPredictionHead is changed in modeling_bert.py, this code needs to be manually updated.
__UpperCAmelCase							:   List[str]							=  "    def __init__(self, config):\n        super().__init__()\n        self.transform = BertPredictionHeadTransform(config)\n\n        # The output weights are the same as the input embeddings, but there is\n        # an output-only bias for each token.\n        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)\n\n        self.bias = nn.Parameter(torch.zeros(config.vocab_size))\n\n        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`\n        self.decoder.bias = self.bias\n\n    def forward(self, hidden_states):\n        hidden_states = self.transform(hidden_states)\n        hidden_states = self.decoder(hidden_states)\n        return hidden_states\n"


class 					_snake_case						(			unittest.TestCase ):






 def    lowerCAmelCase_							(		self       )						->    List[str]:
     snake_case__      :str         =   tempfile.mkdtemp()
     os.makedirs(os.path.join(self.transformer_dir					,"models/bert/"       )       )
     snake_case__      :Optional[int]         =   self.transformer_dir
     shutil.copy(
         os.path.join(A_					,"src/transformers/models/bert/modeling_bert.py"       )					,os.path.join(self.transformer_dir					,"models/bert/modeling_bert.py"       )					,)






 def    lowerCAmelCase_							(		self       )						->    int:
     snake_case__      :List[str]         =   "src/transformers"
     shutil.rmtree(self.transformer_dir       )






 def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase					,UpperCamelCase					,UpperCamelCase=None       )						->    int:
     snake_case__      :Union[str, Any]         =   comment + f'\nclass {class_name}(nn.Module):\n' + class_code
     if overwrite_result is not None:
         snake_case__      :Tuple         =   comment + f'\nclass {class_name}(nn.Module):\n' + overwrite_result
     snake_case__      :Optional[Any]         =   black.Mode(target_versions={black.TargetVersion.PYaa}					,line_length=119       )
     snake_case__      :List[str]         =   black.format_str(A_					,mode=A_       )
     snake_case__      :List[str]         =   os.path.join(self.transformer_dir					,"new_code.py"       )
     with open(A_					,"w"					,newline="\n"       ) as f:
         f.write(A_       )
     if overwrite_result is None:
         self.assertTrue(len(check_copies.is_copy_consistent(A_       )       ) == 0       )
     else:
         check_copies.is_copy_consistent(f.name					,overwrite=A_       )
         with open(A_					,"r"       ) as f:
             self.assertTrue(f.read()					,A_       )






 def    lowerCAmelCase_							(		self       )						->    Optional[int]:
     snake_case__      :int         =   check_copies.find_code_in_transformers("models.bert.modeling_bert.BertLMPredictionHead"       )
     self.assertEqual(A_					,A_       )






 def    lowerCAmelCase_							(		self       )						->    str:
     # Base copy consistency
     self.check_copy_consistency(
         "# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead"					,"BertLMPredictionHead"					,REFERENCE_CODE + "\n"					,)

     # With no empty line at the end
     self.check_copy_consistency(
         "# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead"					,"BertLMPredictionHead"					,A_					,)

     # Copy consistency with rename
     self.check_copy_consistency(
         "# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel"					,"TestModelLMPredictionHead"					,re.sub("Bert"					,"TestModel"					,A_       )					,)

     # Copy consistency with a really long name
     snake_case__      :Tuple         =   "TestModelWithAReallyLongNameBecauseSomePeopleLikeThatForSomeReason"
     self.check_copy_consistency(
         f'# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->{long_class_name}'					,f'{long_class_name}LMPredictionHead'					,re.sub("Bert"					,A_					,A_       )					,)

     # Copy consistency with overwrite
     self.check_copy_consistency(
         "# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel"					,"TestModelLMPredictionHead"					,A_					,overwrite_result=re.sub("Bert"					,"TestModel"					,A_       )					,)






 def    lowerCAmelCase_							(		self       )						->    str:
     snake_case__      :Optional[int]         =   check_copies.LOCALIZED_READMES["README_zh-hans.md"]

     snake_case__      :Any         =   (
         "1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the"
         " Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for"
         " Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong"
         " Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.\n1."
         " **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (from HuggingFace),"
         " released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and"
         " lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same"
         " method has been applied to compress GPT2 into"
         " [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into"
         " [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),"
         " Multilingual BERT into"
         " [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German"
         " version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)**"
         " (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders"
         " as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang"
         " Luong, Quoc V. Le, Christopher D. Manning."
     )
     snake_case__      :Union[str, Any]         =   (
         "1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"
         " Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"
         " Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"
         " Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"
     )
     snake_case__      :Optional[Any]         =   (
         "1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"
         " Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"
         " Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"
         " Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n1."
         " **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (来自 HuggingFace) 伴随论文"
         " [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and"
         " lighter](https://arxiv.org/abs/1910.01108) 由 Victor Sanh, Lysandre Debut and Thomas Wolf 发布。 The same"
         " method has been applied to compress GPT2 into"
         " [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into"
         " [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),"
         " Multilingual BERT into"
         " [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German"
         " version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)** (来自"
         " Google Research/Stanford University) 伴随论文 [ELECTRA: Pre-training text encoders as discriminators rather"
         " than generators](https://arxiv.org/abs/2003.10555) 由 Kevin Clark, Minh-Thang Luong, Quoc V. Le,"
         " Christopher D. Manning 发布。\n"
     )

     snake_case__    ,						snake_case__      :Optional[Any]         =   check_copies.convert_to_localized_md(
         A_					,A_					,localized_readme["format_model_list"]       )

     self.assertFalse(A_       )
     self.assertEqual(A_					,A_       )

     snake_case__    ,						snake_case__      :Tuple         =   check_copies.convert_to_localized_md(
         A_					,A_					,localized_readme["format_model_list"]       )

     # Check whether the number of models is equal to README.md after conversion.
     self.assertTrue(A_       )

     snake_case__      :int         =   (
         "1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the"
         " Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for"
         " Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong"
         " Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut."
     )
     snake_case__      :Union[str, Any]         =   (
         "1. **[ALBERT](https://huggingface.co/transformers/main/model_doc/albert.html)** (来自 Google Research and"
         " the Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"
         " Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"
         " Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"
     )
     snake_case__      :Any         =   (
         "1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"
         " Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"
         " Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"
         " Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"
     )

     snake_case__    ,						snake_case__      :Dict         =   check_copies.convert_to_localized_md(
         A_					,A_					,localized_readme["format_model_list"]       )

     # Check if the model link is synchronized.
     self.assertEqual(A_					,A_       )







from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


__UpperCAmelCase							:   str							=  logging.get_logger(__name__)

__UpperCAmelCase							:   List[Any]							=  {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

__UpperCAmelCase							:   List[Any]							=  {
    "vocab_file": {
        "allegro/herbert-base-cased": "https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json"
    },
    "merges_file": {
        "allegro/herbert-base-cased": "https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt"
    },
}

__UpperCAmelCase							:   str							=  {"allegro/herbert-base-cased": 5_1_4}
__UpperCAmelCase							:   List[str]							=  {}


class 					_snake_case						(			_A ):
	_A          =   VOCAB_FILES_NAMES
	_A          =   PRETRAINED_VOCAB_FILES_MAP
	_A          =   PRETRAINED_INIT_CONFIGURATION
	_A          =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_A          =   HerbertTokenizer






	def __init__(		self					,UpperCamelCase=None					,UpperCamelCase=None					,UpperCamelCase=None					,UpperCamelCase="<s>"					,UpperCamelCase="<unk>"					,UpperCamelCase="<pad>"					,UpperCamelCase="<mask>"					,UpperCamelCase="</s>"					,**UpperCamelCase					,)						->    Dict:
					super().__init__(
					    UpperCamelCase					,UpperCamelCase					,tokenizer_file=UpperCamelCase					,cls_token=UpperCamelCase					,unk_token=UpperCamelCase					,pad_token=UpperCamelCase					,mask_token=UpperCamelCase					,sep_token=UpperCamelCase					,**UpperCamelCase					,)






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None       )						->    List[int]:
					snake_case__      :Optional[int]         =   [self.cls_token_id]
					snake_case__      :Any         =   [self.sep_token_id]
					if token_ids_a is None:
									return cls + token_ids_a + sep

					return cls + token_ids_a + sep + token_ids_a + sep






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None					,UpperCamelCase = False       )						->    List[int]:
					if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=UpperCamelCase					,token_ids_a=UpperCamelCase					,already_has_special_tokens=UpperCamelCase       )

					if token_ids_a is None:
									return [1] + ([0] * len(UpperCamelCase       )) + [1]
					return [1] + ([0] * len(UpperCamelCase       )) + [1] + ([0] * len(UpperCamelCase       )) + [1]






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None       )						->    List[int]:
					snake_case__      :Any         =   [self.sep_token_id]
					snake_case__      :Union[str, Any]         =   [self.cls_token_id]

					if token_ids_a is None:
									return len(cls + token_ids_a + sep       ) * [0]
					return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None       )						->    Tuple[str]:
					snake_case__      :List[str]         =   self._tokenizer.model.save(UpperCamelCase					,name=UpperCamelCase       )
					return tuple(UpperCamelCase       )