



import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 				_snake_case			(					UpperCAmelCase_    ,	unittest.TestCase):
    UpperCamelCase__				:					List[str]     		=BertTokenizer
    UpperCamelCase__				:					Dict     		=BertTokenizerFast
    UpperCamelCase__				:					Any     		=True
    UpperCamelCase__				:					Any     		=True
    UpperCamelCase__				:					int     		=filter_non_english




    def 							A__      (						self					:						Optional[Any]						):
       super().setUp()

       lowercase__					  =				[
           "[UNK]",
           "[CLS]",
           "[SEP]",
           "[PAD]",
           "[MASK]",
           "want",
           "##want",
           "##ed",
           "wa",
           "un",
           "runn",
           "##ing",
           ",",
           "low",
           "lowest",
       ]
       lowercase__					  =				os.path.join(self.tmpdirname,					VOCAB_FILES_NAMES["vocab_file"]						)
       with open(self.vocab_file,					"w",					encoding="utf-8"						) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)




    def 							A__      (						self					:						int,					__lowercase					:						Any						):
       lowercase__					  =				"UNwant\u00E9d,running"
       lowercase__					  =				"unwanted, running"
       return input_text, output_text




    def 							A__      (						self					:						int						):
       lowercase__					  =				self.tokenizer_class(self.vocab_file						)

       lowercase__					  =				tokenizer.tokenize("UNwant\u00E9d,running"						)
       self.assertListEqual(_lowercase,					["un", "##want", "##ed", ",", "runn", "##ing"]						)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase						),					[9, 6, 7, 12, 10, 11]						)




    def 							A__      (						self					:						Union[str, Any]						):
       if not self.test_rust_tokenizer:
          return

       lowercase__					  =				self.get_tokenizer()
       lowercase__					  =				self.get_rust_tokenizer()

       lowercase__					  =				"UNwant\u00E9d,running"

       lowercase__					  =				tokenizer.tokenize(_lowercase						)
       lowercase__					  =				rust_tokenizer.tokenize(_lowercase						)
       self.assertListEqual(_lowercase,					_lowercase						)

       lowercase__					  =				tokenizer.encode(_lowercase,					add_special_tokens=_lowercase						)
       lowercase__					  =				rust_tokenizer.encode(_lowercase,					add_special_tokens=_lowercase						)
       self.assertListEqual(_lowercase,					_lowercase						)

       lowercase__					  =				self.get_rust_tokenizer()
       lowercase__					  =				tokenizer.encode(_lowercase						)
       lowercase__					  =				rust_tokenizer.encode(_lowercase						)
       self.assertListEqual(_lowercase,					_lowercase						)

       # With lower casing
       lowercase__					  =				self.get_tokenizer(do_lower_case=_lowercase						)
       lowercase__					  =				self.get_rust_tokenizer(do_lower_case=_lowercase						)

       lowercase__					  =				"UNwant\u00E9d,running"

       lowercase__					  =				tokenizer.tokenize(_lowercase						)
       lowercase__					  =				rust_tokenizer.tokenize(_lowercase						)
       self.assertListEqual(_lowercase,					_lowercase						)

       lowercase__					  =				tokenizer.encode(_lowercase,					add_special_tokens=_lowercase						)
       lowercase__					  =				rust_tokenizer.encode(_lowercase,					add_special_tokens=_lowercase						)
       self.assertListEqual(_lowercase,					_lowercase						)

       lowercase__					  =				self.get_rust_tokenizer()
       lowercase__					  =				tokenizer.encode(_lowercase						)
       lowercase__					  =				rust_tokenizer.encode(_lowercase						)
       self.assertListEqual(_lowercase,					_lowercase						)




    def 							A__      (						self					:						List[Any]						):
       lowercase__					  =				BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"						),					["ah", "\u535A", "\u63A8", "zz"]						)




    def 							A__      (						self					:						int						):
       lowercase__					  =				BasicTokenizer(do_lower_case=_lowercase						)

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						),					["hello", "!", "how", "are", "you", "?"]						)
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						),					["hello"]						)




    def 							A__      (						self					:						List[Any]						):
       lowercase__					  =				BasicTokenizer(do_lower_case=_lowercase,					strip_accents=_lowercase						)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						),					["hällo", "!", "how", "are", "you", "?"]						)
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						),					["h\u00E9llo"]						)




    def 							A__      (						self					:						Any						):
       lowercase__					  =				BasicTokenizer(do_lower_case=_lowercase,					strip_accents=_lowercase						)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						),					["hallo", "!", "how", "are", "you", "?"]						)
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						),					["hello"]						)




    def 							A__      (						self					:						Tuple						):
       lowercase__					  =				BasicTokenizer(do_lower_case=_lowercase						)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						),					["hallo", "!", "how", "are", "you", "?"]						)
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						),					["hello"]						)




    def 							A__      (						self					:						int						):
       lowercase__					  =				BasicTokenizer(do_lower_case=_lowercase						)

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						),					["HeLLo", "!", "how", "Are", "yoU", "?"]						)




    def 							A__      (						self					:						Tuple						):
       lowercase__					  =				BasicTokenizer(do_lower_case=_lowercase,					strip_accents=_lowercase						)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						),					["HäLLo", "!", "how", "Are", "yoU", "?"]						)




    def 							A__      (						self					:						Any						):
       lowercase__					  =				BasicTokenizer(do_lower_case=_lowercase,					strip_accents=_lowercase						)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						),					["HaLLo", "!", "how", "Are", "yoU", "?"]						)




    def 							A__      (						self					:						List[Any]						):
       lowercase__					  =				BasicTokenizer(do_lower_case=_lowercase,					never_split=["[UNK]"]						)

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"						),					["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]						)




    def 							A__      (						self					:						List[Any]						):
       lowercase__					  =				BasicTokenizer()
       lowercase__					  =				"a\n\'ll !!to?\'d of, can\'t."
       lowercase__					  =				["a", "\'", "ll", "!", "!", "to", "?", "\'", "d", "of", ",", "can", "\'", "t", "."]
       self.assertListEqual(tokenizer.tokenize(_lowercase						),					_lowercase						)




    def 							A__      (						self					:						List[Any]						):
       lowercase__					  =				["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

       lowercase__					  =				{}
       for i, token in enumerate(_lowercase						):
          lowercase__					  =				i
       lowercase__					  =				WordpieceTokenizer(vocab=_lowercase,					unk_token="[UNK]"						)

       self.assertListEqual(tokenizer.tokenize(""						),					[]						)

       self.assertListEqual(tokenizer.tokenize("unwanted running"						),					["un", "##want", "##ed", "runn", "##ing"]						)

       self.assertListEqual(tokenizer.tokenize("unwantedX running"						),					["[UNK]", "runn", "##ing"]						)




    def 							A__      (						self					:						int						):
       self.assertTrue(_is_whitespace(" "						)						)
       self.assertTrue(_is_whitespace("\t"						)						)
       self.assertTrue(_is_whitespace("\r"						)						)
       self.assertTrue(_is_whitespace("\n"						)						)
       self.assertTrue(_is_whitespace("\u00A0"						)						)

       self.assertFalse(_is_whitespace("A"						)						)
       self.assertFalse(_is_whitespace("-"						)						)




    def 							A__      (						self					:						Optional[int]						):
       self.assertTrue(_is_control("\u0005"						)						)

       self.assertFalse(_is_control("A"						)						)
       self.assertFalse(_is_control(" "						)						)
       self.assertFalse(_is_control("\t"						)						)
       self.assertFalse(_is_control("\r"						)						)




    def 							A__      (						self					:						Optional[int]						):
       self.assertTrue(_is_punctuation("-"						)						)
       self.assertTrue(_is_punctuation("$"						)						)
       self.assertTrue(_is_punctuation("`"						)						)
       self.assertTrue(_is_punctuation("."						)						)

       self.assertFalse(_is_punctuation("A"						)						)
       self.assertFalse(_is_punctuation(" "						)						)




    def 							A__      (						self					:						Union[str, Any]						):
       lowercase__					  =				self.get_tokenizer()
       lowercase__					  =				self.get_rust_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(_lowercase						) for t in ["Test", "\xad", "test"]],					[["[UNK]"], [], ["[UNK]"]]						)

       self.assertListEqual(
           [rust_tokenizer.tokenize(_lowercase						) for t in ["Test", "\xad", "test"]],					[["[UNK]"], [], ["[UNK]"]]						)




    @slow
    def 							A__      (						self					:						List[str]						):
       lowercase__					  =				self.tokenizer_class.from_pretrained("bert-base-uncased"						)

       lowercase__					  =				tokenizer.encode("sequence builders",					add_special_tokens=_lowercase						)
       lowercase__					  =				tokenizer.encode("multi-sequence build",					add_special_tokens=_lowercase						)

       lowercase__					  =				tokenizer.build_inputs_with_special_tokens(_lowercase						)
       lowercase__					  =				tokenizer.build_inputs_with_special_tokens(_lowercase,					_lowercase						)

       assert encoded_sentence == [101] + text + [102]
       assert encoded_pair == [101] + text + [102] + text_a + [102]




    def 							A__      (						self					:						Any						):
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
             lowercase__					  =				self.rust_tokenizer_class.from_pretrained(_lowercase,					**_lowercase						)

             lowercase__					  =				F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
             lowercase__					  =				tokenizer_r.encode_plus(
                 _lowercase,					return_attention_mask=_lowercase,					return_token_type_ids=_lowercase,					return_offsets_mapping=_lowercase,					add_special_tokens=_lowercase,					)

             lowercase__					  =				tokenizer_r.do_lower_case if hasattr(_lowercase,					"do_lower_case"						) else False
             lowercase__					  =				(
                 [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "A"),
                     ((1, 2), ","),
                     ((3, 5), "na"),
                     ((5, 6), "##ï"),
                     ((6, 8), "##ve"),
                     ((9, 15), tokenizer_r.mask_token),
                     ((16, 21), "Allen"),
                     ((21, 23), "##NL"),
                     ((23, 24), "##P"),
                     ((25, 33), "sentence"),
                     ((33, 34), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
                 if not do_lower_case
                 else [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "a"),
                     ((1, 2), ","),
                     ((3, 8), "naive"),
                     ((9, 15), tokenizer_r.mask_token),
                     ((16, 21), "allen"),
                     ((21, 23), "##nl"),
                     ((23, 24), "##p"),
                     ((25, 33), "sentence"),
                     ((33, 34), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
             )

             self.assertEqual(
                 [e[1] for e in expected_results],					tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]						)						)
             self.assertEqual([e[0] for e in expected_results],					tokens["offset_mapping"]						)




    def 							A__      (						self					:						List[str]						):
       lowercase__					  =				["的", "人", "有"]
       lowercase__					  =				"".join(_lowercase						)
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
             lowercase__					  =				True
             lowercase__					  =				self.tokenizer_class.from_pretrained(_lowercase,					**_lowercase						)
             lowercase__					  =				self.rust_tokenizer_class.from_pretrained(_lowercase,					**_lowercase						)

             lowercase__					  =				tokenizer_p.encode(_lowercase,					add_special_tokens=_lowercase						)
             lowercase__					  =				tokenizer_r.encode(_lowercase,					add_special_tokens=_lowercase						)

             lowercase__					  =				tokenizer_r.convert_ids_to_tokens(_lowercase						)
             lowercase__					  =				tokenizer_p.convert_ids_to_tokens(_lowercase						)

             # it is expected that each Chinese character is not preceded by "##"
             self.assertListEqual(_lowercase,					_lowercase						)
             self.assertListEqual(_lowercase,					_lowercase						)

             lowercase__					  =				False
             lowercase__					  =				self.rust_tokenizer_class.from_pretrained(_lowercase,					**_lowercase						)
             lowercase__					  =				self.tokenizer_class.from_pretrained(_lowercase,					**_lowercase						)

             lowercase__					  =				tokenizer_r.encode(_lowercase,					add_special_tokens=_lowercase						)
             lowercase__					  =				tokenizer_p.encode(_lowercase,					add_special_tokens=_lowercase						)

             lowercase__					  =				tokenizer_r.convert_ids_to_tokens(_lowercase						)
             lowercase__					  =				tokenizer_p.convert_ids_to_tokens(_lowercase						)

             # it is expected that only the first Chinese character is not preceded by "##".
             lowercase__					  =				[
                 F'''##{token}''' if idx != 0 else token for idx, token in enumerate(_lowercase						)
             ]
             self.assertListEqual(_lowercase,					_lowercase						)
             self.assertListEqual(_lowercase,					_lowercase						)









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


lowercase_								=     {
    """configuration_roberta_prelayernorm""": [
        """ROBERTA_PRELAYERNORM_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """RobertaPreLayerNormConfig""",
        """RobertaPreLayerNormOnnxConfig""",
    ],
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowercase_								=     [
							    """ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """RobertaPreLayerNormForCausalLM""",
							    """RobertaPreLayerNormForMaskedLM""",
							    """RobertaPreLayerNormForMultipleChoice""",
							    """RobertaPreLayerNormForQuestionAnswering""",
							    """RobertaPreLayerNormForSequenceClassification""",
							    """RobertaPreLayerNormForTokenClassification""",
							    """RobertaPreLayerNormModel""",
							    """RobertaPreLayerNormPreTrainedModel""",
							]

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowercase_								=     [
							    """TF_ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """TFRobertaPreLayerNormForCausalLM""",
							    """TFRobertaPreLayerNormForMaskedLM""",
							    """TFRobertaPreLayerNormForMultipleChoice""",
							    """TFRobertaPreLayerNormForQuestionAnswering""",
							    """TFRobertaPreLayerNormForSequenceClassification""",
							    """TFRobertaPreLayerNormForTokenClassification""",
							    """TFRobertaPreLayerNormMainLayer""",
							    """TFRobertaPreLayerNormModel""",
							    """TFRobertaPreLayerNormPreTrainedModel""",
							]

try:
							if not is_flax_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowercase_								=     [
							    """FlaxRobertaPreLayerNormForCausalLM""",
							    """FlaxRobertaPreLayerNormForMaskedLM""",
							    """FlaxRobertaPreLayerNormForMultipleChoice""",
							    """FlaxRobertaPreLayerNormForQuestionAnswering""",
							    """FlaxRobertaPreLayerNormForSequenceClassification""",
							    """FlaxRobertaPreLayerNormForTokenClassification""",
							    """FlaxRobertaPreLayerNormModel""",
							    """FlaxRobertaPreLayerNormPreTrainedModel""",
							]


if TYPE_CHECKING:
							from .configuration_roberta_prelayernorm import (
							    ROBERTA_PRELAYERNORM_PRETRAINED_CONFIG_ARCHIVE_MAP,
							    RobertaPreLayerNormConfig,
							    RobertaPreLayerNormOnnxConfig,
							)

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_roberta_prelayernorm import (
														    ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST,
														    RobertaPreLayerNormForCausalLM,
														    RobertaPreLayerNormForMaskedLM,
														    RobertaPreLayerNormForMultipleChoice,
														    RobertaPreLayerNormForQuestionAnswering,
														    RobertaPreLayerNormForSequenceClassification,
														    RobertaPreLayerNormForTokenClassification,
														    RobertaPreLayerNormModel,
														    RobertaPreLayerNormPreTrainedModel,
														)

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_roberta_prelayernorm import (
														    TF_ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFRobertaPreLayerNormForCausalLM,
														    TFRobertaPreLayerNormForMaskedLM,
														    TFRobertaPreLayerNormForMultipleChoice,
														    TFRobertaPreLayerNormForQuestionAnswering,
														    TFRobertaPreLayerNormForSequenceClassification,
														    TFRobertaPreLayerNormForTokenClassification,
														    TFRobertaPreLayerNormMainLayer,
														    TFRobertaPreLayerNormModel,
														    TFRobertaPreLayerNormPreTrainedModel,
														)

							try:
														if not is_flax_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_flax_roberta_prelayernorm import (
														    FlaxRobertaPreLayerNormForCausalLM,
														    FlaxRobertaPreLayerNormForMaskedLM,
														    FlaxRobertaPreLayerNormForMultipleChoice,
														    FlaxRobertaPreLayerNormForQuestionAnswering,
														    FlaxRobertaPreLayerNormForSequenceClassification,
														    FlaxRobertaPreLayerNormForTokenClassification,
														    FlaxRobertaPreLayerNormModel,
														    FlaxRobertaPreLayerNormPreTrainedModel,
														)

else:
							import sys

							lowercase_								=     _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




