



'''simple docstring'''


import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 	UpperCAmelCase	(		__a					,	unittest.TestCase):


 '''simple docstring'''




 __magic_name__					: Optional[Any]  =      MobileBertTokenizer
 __magic_name__					: List[str]  =      MobileBertTokenizerFast
 __magic_name__					: Optional[int]  =      True
 __magic_name__					: Any  =      True
 __magic_name__					: Any  =      filter_non_english
 __magic_name__					: List[str]  =      "google/mobilebert-uncased"




 def       lowercase_  (   self) ->      Tuple:



      """simple docstring"""







      super().setUp()

      a_       		=[
          "[UNK]",
          "[CLS]",
          "[SEP]",
          "[PAD]",
          "[MASK]",
          "want",
          "##want",
          "##ed",
          "wa",
          "un",
          "runn",
          "##ing",
          ",",
          "low",
          "lowest",
      ]
      a_       		=os.path.join(self.tmpdirname							,  VOCAB_FILES_NAMES["vocab_file"])
      with open(self.vocab_file							,  "w"							,  encoding="utf-8") as vocab_writer:
           vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))

      a_       		=[
          (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
          for tokenizer_def in self.tokenizers_list
      ]




 def       lowercase_  (   self							,  lowerCAmelCase_) ->      Optional[Any]:



      """simple docstring"""







      a_       		="UNwant\u00E9d,running"
      a_       		="unwanted, running"
      return input_text, output_text




 def       lowercase_  (   self) ->      Tuple:



      """simple docstring"""







      a_       		=self.tokenizer_class(self.vocab_file)

      a_       		=tokenizer.tokenize("UNwant\u00E9d,running")
      self.assertListEqual(lowerCAmelCase_							,  ["un", "##want", "##ed", ",", "runn", "##ing"])
      self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_)							,  [9, 6, 7, 1_2, 1_0, 1_1])




 def       lowercase_  (   self) ->      Dict:



      """simple docstring"""







      if not self.test_rust_tokenizer:
           return

      a_       		=self.get_tokenizer()
      a_       		=self.get_rust_tokenizer()

      a_       		="UNwant\u00E9d,running"

      a_       		=tokenizer.tokenize(lowerCAmelCase_)
      a_       		=rust_tokenizer.tokenize(lowerCAmelCase_)
      self.assertListEqual(lowerCAmelCase_							,  lowerCAmelCase_)

      a_       		=tokenizer.encode(lowerCAmelCase_							,  add_special_tokens=lowerCAmelCase_)
      a_       		=rust_tokenizer.encode(lowerCAmelCase_							,  add_special_tokens=lowerCAmelCase_)
      self.assertListEqual(lowerCAmelCase_							,  lowerCAmelCase_)

      a_       		=self.get_rust_tokenizer()
      a_       		=tokenizer.encode(lowerCAmelCase_)
      a_       		=rust_tokenizer.encode(lowerCAmelCase_)
      self.assertListEqual(lowerCAmelCase_							,  lowerCAmelCase_)

      # With lower casing
      a_       		=self.get_tokenizer(do_lower_case=lowerCAmelCase_)
      a_       		=self.get_rust_tokenizer(do_lower_case=lowerCAmelCase_)

      a_       		="UNwant\u00E9d,running"

      a_       		=tokenizer.tokenize(lowerCAmelCase_)
      a_       		=rust_tokenizer.tokenize(lowerCAmelCase_)
      self.assertListEqual(lowerCAmelCase_							,  lowerCAmelCase_)

      a_       		=tokenizer.encode(lowerCAmelCase_							,  add_special_tokens=lowerCAmelCase_)
      a_       		=rust_tokenizer.encode(lowerCAmelCase_							,  add_special_tokens=lowerCAmelCase_)
      self.assertListEqual(lowerCAmelCase_							,  lowerCAmelCase_)

      a_       		=self.get_rust_tokenizer()
      a_       		=tokenizer.encode(lowerCAmelCase_)
      a_       		=rust_tokenizer.encode(lowerCAmelCase_)
      self.assertListEqual(lowerCAmelCase_							,  lowerCAmelCase_)




 def       lowercase_  (   self) ->      Any:



      """simple docstring"""







      a_       		=BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz")							,  ["ah", "\u535A", "\u63A8", "zz"])




 def       lowercase_  (   self) ->      Optional[Any]:



      """simple docstring"""







      a_       		=BasicTokenizer(do_lower_case=lowerCAmelCase_)

      self.assertListEqual(
          tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")							,  ["hello", "!", "how", "are", "you", "?"])
      self.assertListEqual(tokenizer.tokenize("H\u00E9llo")							,  ["hello"])




 def       lowercase_  (   self) ->      Any:



      """simple docstring"""







      a_       		=BasicTokenizer(do_lower_case=lowerCAmelCase_							,  strip_accents=lowerCAmelCase_)

      self.assertListEqual(
          tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")							,  ["hällo", "!", "how", "are", "you", "?"])
      self.assertListEqual(tokenizer.tokenize("H\u00E9llo")							,  ["h\u00E9llo"])




 def       lowercase_  (   self) ->      str:



      """simple docstring"""







      a_       		=BasicTokenizer(do_lower_case=lowerCAmelCase_							,  strip_accents=lowerCAmelCase_)

      self.assertListEqual(
          tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")							,  ["hallo", "!", "how", "are", "you", "?"])
      self.assertListEqual(tokenizer.tokenize("H\u00E9llo")							,  ["hello"])




 def       lowercase_  (   self) ->      List[str]:



      """simple docstring"""







      a_       		=BasicTokenizer(do_lower_case=lowerCAmelCase_)

      self.assertListEqual(
          tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")							,  ["hallo", "!", "how", "are", "you", "?"])
      self.assertListEqual(tokenizer.tokenize("H\u00E9llo")							,  ["hello"])




 def       lowercase_  (   self) ->      Any:



      """simple docstring"""







      a_       		=BasicTokenizer(do_lower_case=lowerCAmelCase_)

      self.assertListEqual(
          tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")							,  ["HeLLo", "!", "how", "Are", "yoU", "?"])




 def       lowercase_  (   self) ->      Tuple:



      """simple docstring"""







      a_       		=BasicTokenizer(do_lower_case=lowerCAmelCase_							,  strip_accents=lowerCAmelCase_)

      self.assertListEqual(
          tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")							,  ["HäLLo", "!", "how", "Are", "yoU", "?"])




 def       lowercase_  (   self) ->      List[str]:



      """simple docstring"""







      a_       		=BasicTokenizer(do_lower_case=lowerCAmelCase_							,  strip_accents=lowerCAmelCase_)

      self.assertListEqual(
          tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")							,  ["HaLLo", "!", "how", "Are", "yoU", "?"])




 def       lowercase_  (   self) ->      Dict:



      """simple docstring"""







      a_       		=BasicTokenizer(do_lower_case=lowerCAmelCase_							,  never_split=["[UNK]"])

      self.assertListEqual(
          tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]")							,  ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"])




 def       lowercase_  (   self) ->      Tuple:



      """simple docstring"""







      a_       		=["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

      a_       		={}
      for i, token in enumerate(lowerCAmelCase_):
           a_       		=i
      a_       		=WordpieceTokenizer(vocab=lowerCAmelCase_							,  unk_token="[UNK]")

      self.assertListEqual(tokenizer.tokenize("")							,  [])

      self.assertListEqual(tokenizer.tokenize("unwanted running")							,  ["un", "##want", "##ed", "runn", "##ing"])

      self.assertListEqual(tokenizer.tokenize("unwantedX running")							,  ["[UNK]", "runn", "##ing"])




 def       lowercase_  (   self) ->      Dict:



      """simple docstring"""







      self.assertTrue(_is_whitespace(" "))
      self.assertTrue(_is_whitespace("\t"))
      self.assertTrue(_is_whitespace("\r"))
      self.assertTrue(_is_whitespace("\n"))
      self.assertTrue(_is_whitespace("\u00A0"))

      self.assertFalse(_is_whitespace("A"))
      self.assertFalse(_is_whitespace("-"))




 def       lowercase_  (   self) ->      List[Any]:



      """simple docstring"""







      self.assertTrue(_is_control("\u0005"))

      self.assertFalse(_is_control("A"))
      self.assertFalse(_is_control(" "))
      self.assertFalse(_is_control("\t"))
      self.assertFalse(_is_control("\r"))




 def       lowercase_  (   self) ->      Dict:



      """simple docstring"""







      self.assertTrue(_is_punctuation("-"))
      self.assertTrue(_is_punctuation("$"))
      self.assertTrue(_is_punctuation("`"))
      self.assertTrue(_is_punctuation("."))

      self.assertFalse(_is_punctuation("A"))
      self.assertFalse(_is_punctuation(" "))




 def       lowercase_  (   self) ->      List[Any]:



      """simple docstring"""







      a_       		=self.get_tokenizer()
      a_       		=self.get_rust_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(lowerCAmelCase_) for t in ["Test", "\xad", "test"]]							,  [["[UNK]"], [], ["[UNK]"]])

      self.assertListEqual(
          [rust_tokenizer.tokenize(lowerCAmelCase_) for t in ["Test", "\xad", "test"]]							,  [["[UNK]"], [], ["[UNK]"]])




 @slow
 def       lowercase_  (   self) ->      Optional[Any]:



      """simple docstring"""







      a_       		=self.tokenizer_class.from_pretrained("google/mobilebert-uncased")

      a_       		=tokenizer.encode("sequence builders"							,  add_special_tokens=lowerCAmelCase_)
      a_       		=tokenizer.encode("multi-sequence build"							,  add_special_tokens=lowerCAmelCase_)

      a_       		=tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_)
      a_       		=tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_							,  lowerCAmelCase_)

      assert encoded_sentence == [1_0_1] + text + [1_0_2]
      assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]




 def       lowercase_  (   self) ->      Dict:



      """simple docstring"""







      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                a_       		=self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_							,  **lowerCAmelCase_)

                a_       		=f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                a_       		=tokenizer_r.encode_plus(
                    lowerCAmelCase_							,  return_attention_mask=lowerCAmelCase_							,  return_token_type_ids=lowerCAmelCase_							,  return_offsets_mapping=lowerCAmelCase_							,  add_special_tokens=lowerCAmelCase_							,  )

                a_       		=tokenizer_r.do_lower_case if hasattr(lowerCAmelCase_							,  "do_lower_case") else False
                a_       		=(
                    [
                        ((0, 0), tokenizer_r.cls_token),
                        ((0, 1), "A"),
                        ((1, 2), ","),
                        ((3, 5), "na"),
                        ((5, 6), "##ï"),
                        ((6, 8), "##ve"),
                        ((9, 1_5), tokenizer_r.mask_token),
                        ((1_6, 2_1), "Allen"),
                        ((2_1, 2_3), "##NL"),
                        ((2_3, 2_4), "##P"),
                        ((2_5, 3_3), "sentence"),
                        ((3_3, 3_4), "."),
                        ((0, 0), tokenizer_r.sep_token),
                    ]
                    if not do_lower_case
                    else [
                        ((0, 0), tokenizer_r.cls_token),
                        ((0, 1), "a"),
                        ((1, 2), ","),
                        ((3, 8), "naive"),
                        ((9, 1_5), tokenizer_r.mask_token),
                        ((1_6, 2_1), "allen"),
                        ((2_1, 2_3), "##nl"),
                        ((2_3, 2_4), "##p"),
                        ((2_5, 3_3), "sentence"),
                        ((3_3, 3_4), "."),
                        ((0, 0), tokenizer_r.sep_token),
                    ]
                )

                self.assertEqual(
                    [e[1] for e in expected_results]							,  tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]))
                self.assertEqual([e[0] for e in expected_results]							,  tokens["offset_mapping"])




 def       lowercase_  (   self) ->      int:



      """simple docstring"""







      a_       		=["的", "人", "有"]
      a_       		="".join(lowerCAmelCase_)
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                a_       		=True
                a_       		=self.tokenizer_class.from_pretrained(lowerCAmelCase_							,  **lowerCAmelCase_)
                a_       		=self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_							,  **lowerCAmelCase_)

                a_       		=tokenizer_p.encode(lowerCAmelCase_							,  add_special_tokens=lowerCAmelCase_)
                a_       		=tokenizer_r.encode(lowerCAmelCase_							,  add_special_tokens=lowerCAmelCase_)

                a_       		=tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_)
                a_       		=tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_)

                # it is expected that each Chinese character is not preceded by "##"
                self.assertListEqual(lowerCAmelCase_							,  lowerCAmelCase_)
                self.assertListEqual(lowerCAmelCase_							,  lowerCAmelCase_)

                a_       		=False
                a_       		=self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_							,  **lowerCAmelCase_)
                a_       		=self.tokenizer_class.from_pretrained(lowerCAmelCase_							,  **lowerCAmelCase_)

                a_       		=tokenizer_r.encode(lowerCAmelCase_							,  add_special_tokens=lowerCAmelCase_)
                a_       		=tokenizer_p.encode(lowerCAmelCase_							,  add_special_tokens=lowerCAmelCase_)

                a_       		=tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_)
                a_       		=tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_)

                # it is expected that only the first Chinese character is not preceded by "##".
                a_       		=[
                    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCAmelCase_)
                ]
                self.assertListEqual(lowerCAmelCase_							,  lowerCAmelCase_)
                self.assertListEqual(lowerCAmelCase_							,  lowerCAmelCase_)







'''simple docstring'''


from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig


lowercase							       =			{
    '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/config.json''',
    '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/config.json''',
    '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/config.json''',
    '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/config.json''',
    '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/config.json''',
    '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/config.json''',
    '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/config.json''',
    '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/config.json''',
}







class 	UpperCAmelCase	(		__a):


 '''simple docstring'''




 __magic_name__					: int  =      "albert"




 def __init__(   self							,  lowerCAmelCase_=3_0_0_0_0							,  lowerCAmelCase_=1_2_8							,  lowerCAmelCase_=4_0_9_6							,  lowerCAmelCase_=1_2							,  lowerCAmelCase_=1							,  lowerCAmelCase_=6_4							,  lowerCAmelCase_=1_6_3_8_4							,  lowerCAmelCase_=1							,  lowerCAmelCase_="gelu_new"							,  lowerCAmelCase_=0							,  lowerCAmelCase_=0							,  lowerCAmelCase_=5_1_2							,  lowerCAmelCase_=2							,  lowerCAmelCase_=0.0_2							,  lowerCAmelCase_=1e-12							,  lowerCAmelCase_=0.1							,  lowerCAmelCase_="absolute"							,  lowerCAmelCase_=0							,  lowerCAmelCase_=2							,  lowerCAmelCase_=3							,  **lowerCAmelCase_							,  ) ->      Any:



      """simple docstring"""







      super().__init__(pad_token_id=lowerCAmelCase_							,  bos_token_id=lowerCAmelCase_							,  eos_token_id=lowerCAmelCase_							,  **lowerCAmelCase_)

      a_       		=vocab_size
      a_       		=embedding_size
      a_       		=hidden_size
      a_       		=num_hidden_layers
      a_       		=num_hidden_groups
      a_       		=num_attention_heads
      a_       		=inner_group_num
      a_       		=hidden_act
      a_       		=intermediate_size
      a_       		=hidden_dropout_prob
      a_       		=attention_probs_dropout_prob
      a_       		=max_position_embeddings
      a_       		=type_vocab_size
      a_       		=initializer_range
      a_       		=layer_norm_eps
      a_       		=classifier_dropout_prob
      a_       		=position_embedding_type







class 	UpperCAmelCase	(		__a):


 '''simple docstring'''




 @property
 def       lowercase_  (   self) ->      Mapping[str, Mapping[int, str]]:



      """simple docstring"""







      if self.task == "multiple-choice":
           a_       		={0: "batch", 1: "choice", 2: "sequence"}
      else:
           a_       		={0: "batch", 1: "sequence"}
      return OrderedDict(
          [
              ("input_ids", dynamic_axis),
              ("attention_mask", dynamic_axis),
              ("token_type_ids", dynamic_axis),
          ])


