'''simple docstring'''






import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class  UpperCAmelCase__						(       __UpperCAmelCase						,	unittest.TestCase ):
    lowerCAmelCase_				    =		BertTokenizer
    lowerCAmelCase_				    =		BertTokenizerFast
    lowerCAmelCase_				    =		True
    lowerCAmelCase_				    =		True
    lowerCAmelCase_				    =		filter_non_english



    def   lowerCamelCase_  (	self      :       List[Any]     ):
          super().setUp()

          _lowerCamelCase				:   Any								=   [
              "[UNK]",
              "[CLS]",
              "[SEP]",
              "[PAD]",
              "[MASK]",
              "want",
              "##want",
              "##ed",
              "wa",
              "un",
              "runn",
              "##ing",
              ",",
              "low",
              "lowest",
          ]
          _lowerCamelCase				:   int								=   os.path.join(self.tmpdirname,VOCAB_FILES_NAMES["vocab_file"]     )
          with open(self.vocab_file,"w",encoding="utf-8"     ) as vocab_writer:
                vocab_writer.write("".join([x + "\n" for x in vocab_tokens]     )     )



    def   lowerCamelCase_  (	self      :       Any,__A      :       Optional[int]     ):
          _lowerCamelCase				:   List[str]								=   "UNwant\u00E9d,running"
          _lowerCamelCase				:   List[str]								=   "unwanted, running"
          return input_text, output_text



    def   lowerCamelCase_  (	self      :       List[Any]     ):
          _lowerCamelCase				:   Union[str, Any]								=   self.tokenizer_class(self.vocab_file     )

          _lowerCamelCase				:   Dict								=   tokenizer.tokenize("UNwant\u00E9d,running"     )
          self.assertListEqual(__A,["un", "##want", "##ed", ",", "runn", "##ing"]     )
          self.assertListEqual(tokenizer.convert_tokens_to_ids(__A     ),[9, 6, 7, 1_2, 1_0, 1_1]     )



    def   lowerCamelCase_  (	self      :       Optional[int]     ):

          if not self.test_rust_tokenizer:
                return

          _lowerCamelCase				:   Tuple								=   self.get_tokenizer()
          _lowerCamelCase				:   str								=   self.get_rust_tokenizer()

          _lowerCamelCase				:   Optional[Any]								=   "UNwant\u00E9d,running"

          _lowerCamelCase				:   str								=   tokenizer.tokenize(__A     )
          _lowerCamelCase				:   str								=   rust_tokenizer.tokenize(__A     )
          self.assertListEqual(__A,__A     )

          _lowerCamelCase				:   Optional[int]								=   tokenizer.encode(__A,add_special_tokens=__A     )
          _lowerCamelCase				:   Optional[int]								=   rust_tokenizer.encode(__A,add_special_tokens=__A     )
          self.assertListEqual(__A,__A     )

          _lowerCamelCase				:   Any								=   self.get_rust_tokenizer()
          _lowerCamelCase				:   Any								=   tokenizer.encode(__A     )
          _lowerCamelCase				:   List[Any]								=   rust_tokenizer.encode(__A     )
          self.assertListEqual(__A,__A     )

          # With lower casing
          _lowerCamelCase				:   Optional[int]								=   self.get_tokenizer(do_lower_case=__A     )
          _lowerCamelCase				:   Tuple								=   self.get_rust_tokenizer(do_lower_case=__A     )

          _lowerCamelCase				:   List[str]								=   "UNwant\u00E9d,running"

          _lowerCamelCase				:   Tuple								=   tokenizer.tokenize(__A     )
          _lowerCamelCase				:   Union[str, Any]								=   rust_tokenizer.tokenize(__A     )
          self.assertListEqual(__A,__A     )

          _lowerCamelCase				:   int								=   tokenizer.encode(__A,add_special_tokens=__A     )
          _lowerCamelCase				:   Optional[Any]								=   rust_tokenizer.encode(__A,add_special_tokens=__A     )
          self.assertListEqual(__A,__A     )

          _lowerCamelCase				:   List[Any]								=   self.get_rust_tokenizer()
          _lowerCamelCase				:   List[str]								=   tokenizer.encode(__A     )
          _lowerCamelCase				:   Any								=   rust_tokenizer.encode(__A     )
          self.assertListEqual(__A,__A     )



    def   lowerCamelCase_  (	self      :       Optional[int]     ):
          _lowerCamelCase				:   Optional[Any]								=   BasicTokenizer()

          self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"     ),["ah", "\u535A", "\u63A8", "zz"]     )



    def   lowerCamelCase_  (	self      :       List[Any]     ):
          _lowerCamelCase				:   Dict								=   BasicTokenizer(do_lower_case=__A     )

          self.assertListEqual(
              tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "     ),["hello", "!", "how", "are", "you", "?"]     )
          self.assertListEqual(tokenizer.tokenize("H\u00E9llo"     ),["hello"]     )



    def   lowerCamelCase_  (	self      :       Optional[int]     ):
          _lowerCamelCase				:   Dict								=   BasicTokenizer(do_lower_case=__A,strip_accents=__A     )

          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "     ),["hällo", "!", "how", "are", "you", "?"]     )
          self.assertListEqual(tokenizer.tokenize("H\u00E9llo"     ),["h\u00E9llo"]     )



    def   lowerCamelCase_  (	self      :       Union[str, Any]     ):
          _lowerCamelCase				:   Optional[int]								=   BasicTokenizer(do_lower_case=__A,strip_accents=__A     )

          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "     ),["hallo", "!", "how", "are", "you", "?"]     )
          self.assertListEqual(tokenizer.tokenize("H\u00E9llo"     ),["hello"]     )



    def   lowerCamelCase_  (	self      :       str     ):
          _lowerCamelCase				:   Union[str, Any]								=   BasicTokenizer(do_lower_case=__A     )

          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "     ),["hallo", "!", "how", "are", "you", "?"]     )
          self.assertListEqual(tokenizer.tokenize("H\u00E9llo"     ),["hello"]     )



    def   lowerCamelCase_  (	self      :       int     ):
          _lowerCamelCase				:   List[str]								=   BasicTokenizer(do_lower_case=__A     )

          self.assertListEqual(
              tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "     ),["HeLLo", "!", "how", "Are", "yoU", "?"]     )



    def   lowerCamelCase_  (	self      :       Optional[int]     ):
          _lowerCamelCase				:   int								=   BasicTokenizer(do_lower_case=__A,strip_accents=__A     )

          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "     ),["HäLLo", "!", "how", "Are", "yoU", "?"]     )



    def   lowerCamelCase_  (	self      :       List[str]     ):
          _lowerCamelCase				:   int								=   BasicTokenizer(do_lower_case=__A,strip_accents=__A     )

          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "     ),["HaLLo", "!", "how", "Are", "yoU", "?"]     )



    def   lowerCamelCase_  (	self      :       Union[str, Any]     ):
          _lowerCamelCase				:   int								=   BasicTokenizer(do_lower_case=__A,never_split=["[UNK]"]     )

          self.assertListEqual(
              tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"     ),["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]     )



    def   lowerCamelCase_  (	self      :       List[str]     ):
          _lowerCamelCase				:   Dict								=   BasicTokenizer()
          _lowerCamelCase				:   List[Any]								=   "a\n\'ll !!to?\'d of, can\'t."
          _lowerCamelCase				:   Any								=   ["a", "\'", "ll", "!", "!", "to", "?", "\'", "d", "of", ",", "can", "\'", "t", "."]
          self.assertListEqual(tokenizer.tokenize(__A     ),__A     )



    def   lowerCamelCase_  (	self      :       List[Any]     ):
          _lowerCamelCase				:   Any								=   ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

          _lowerCamelCase				:   int								=   {}
          for i, token in enumerate(__A     ):
                _lowerCamelCase				:   Optional[int]								=   i
          _lowerCamelCase				:   int								=   WordpieceTokenizer(vocab=__A,unk_token="[UNK]"     )

          self.assertListEqual(tokenizer.tokenize(""     ),[]     )

          self.assertListEqual(tokenizer.tokenize("unwanted running"     ),["un", "##want", "##ed", "runn", "##ing"]     )

          self.assertListEqual(tokenizer.tokenize("unwantedX running"     ),["[UNK]", "runn", "##ing"]     )



    def   lowerCamelCase_  (	self      :       List[Any]     ):
          self.assertTrue(_is_whitespace(" "     )     )
          self.assertTrue(_is_whitespace("\t"     )     )
          self.assertTrue(_is_whitespace("\r"     )     )
          self.assertTrue(_is_whitespace("\n"     )     )
          self.assertTrue(_is_whitespace("\u00A0"     )     )

          self.assertFalse(_is_whitespace("A"     )     )
          self.assertFalse(_is_whitespace("-"     )     )



    def   lowerCamelCase_  (	self      :       Union[str, Any]     ):
          self.assertTrue(_is_control("\u0005"     )     )

          self.assertFalse(_is_control("A"     )     )
          self.assertFalse(_is_control(" "     )     )
          self.assertFalse(_is_control("\t"     )     )
          self.assertFalse(_is_control("\r"     )     )



    def   lowerCamelCase_  (	self      :       Dict     ):
          self.assertTrue(_is_punctuation("-"     )     )
          self.assertTrue(_is_punctuation("$"     )     )
          self.assertTrue(_is_punctuation("`"     )     )
          self.assertTrue(_is_punctuation("."     )     )

          self.assertFalse(_is_punctuation("A"     )     )
          self.assertFalse(_is_punctuation(" "     )     )



    def   lowerCamelCase_  (	self      :       str     ):
          _lowerCamelCase				:   Optional[Any]								=   self.get_tokenizer()
          _lowerCamelCase				:   Tuple								=   self.get_rust_tokenizer()

          # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
          self.assertListEqual([tokenizer.tokenize(__A     ) for t in ["Test", "\xad", "test"]],[["[UNK]"], [], ["[UNK]"]]     )

          self.assertListEqual(
              [rust_tokenizer.tokenize(__A     ) for t in ["Test", "\xad", "test"]],[["[UNK]"], [], ["[UNK]"]]     )



    @slow
    def   lowerCamelCase_  (	self      :       str     ):
          _lowerCamelCase				:   List[Any]								=   self.tokenizer_class.from_pretrained("bert-base-uncased"     )

          _lowerCamelCase				:   Dict								=   tokenizer.encode("sequence builders",add_special_tokens=__A     )
          _lowerCamelCase				:   List[Any]								=   tokenizer.encode("multi-sequence build",add_special_tokens=__A     )

          _lowerCamelCase				:   int								=   tokenizer.build_inputs_with_special_tokens(__A     )
          _lowerCamelCase				:   Optional[int]								=   tokenizer.build_inputs_with_special_tokens(__A,__A     )

          assert encoded_sentence == [1_0_1] + text + [1_0_2]
          assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]



    def   lowerCamelCase_  (	self      :       Union[str, Any]     ):

          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'     ):
                      _lowerCamelCase				:   Dict								=   self.rust_tokenizer_class.from_pretrained(__A,**__A     )

                      _lowerCamelCase				:   List[str]								=   f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
                      _lowerCamelCase				:   Any								=   tokenizer_r.encode_plus(
                          __A,return_attention_mask=__A,return_token_type_ids=__A,return_offsets_mapping=__A,add_special_tokens=__A,)

                      _lowerCamelCase				:   str								=   tokenizer_r.do_lower_case if hasattr(__A,"do_lower_case"     ) else False
                      _lowerCamelCase				:   str								=   (
                          [
                              ((0, 0), tokenizer_r.cls_token),
                              ((0, 1), "A"),
                              ((1, 2), ","),
                              ((3, 5), "na"),
                              ((5, 6), "##ï"),
                              ((6, 8), "##ve"),
                              ((9, 1_5), tokenizer_r.mask_token),
                              ((1_6, 2_1), "Allen"),
                              ((2_1, 2_3), "##NL"),
                              ((2_3, 2_4), "##P"),
                              ((2_5, 3_3), "sentence"),
                              ((3_3, 3_4), "."),
                              ((0, 0), tokenizer_r.sep_token),
                          ]
                          if not do_lower_case
                          else [
                              ((0, 0), tokenizer_r.cls_token),
                              ((0, 1), "a"),
                              ((1, 2), ","),
                              ((3, 8), "naive"),
                              ((9, 1_5), tokenizer_r.mask_token),
                              ((1_6, 2_1), "allen"),
                              ((2_1, 2_3), "##nl"),
                              ((2_3, 2_4), "##p"),
                              ((2_5, 3_3), "sentence"),
                              ((3_3, 3_4), "."),
                              ((0, 0), tokenizer_r.sep_token),
                          ]
                      )

                      self.assertEqual(
                          [e[1] for e in expected_results],tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]     )     )
                      self.assertEqual([e[0] for e in expected_results],tokens["offset_mapping"]     )



    def   lowerCamelCase_  (	self      :       int     ):
          _lowerCamelCase				:   Any								=   ["的", "人", "有"]
          _lowerCamelCase				:   Union[str, Any]								=   "".join(__A     )
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'     ):
                      _lowerCamelCase				:   str								=   True
                      _lowerCamelCase				:   Optional[int]								=   self.tokenizer_class.from_pretrained(__A,**__A     )
                      _lowerCamelCase				:   str								=   self.rust_tokenizer_class.from_pretrained(__A,**__A     )

                      _lowerCamelCase				:   Tuple								=   tokenizer_p.encode(__A,add_special_tokens=__A     )
                      _lowerCamelCase				:   Optional[Any]								=   tokenizer_r.encode(__A,add_special_tokens=__A     )

                      _lowerCamelCase				:   Dict								=   tokenizer_r.convert_ids_to_tokens(__A     )
                      _lowerCamelCase				:   str								=   tokenizer_p.convert_ids_to_tokens(__A     )

                      # it is expected that each Chinese character is not preceded by "##"
                      self.assertListEqual(__A,__A     )
                      self.assertListEqual(__A,__A     )

                      _lowerCamelCase				:   Any								=   False
                      _lowerCamelCase				:   List[str]								=   self.rust_tokenizer_class.from_pretrained(__A,**__A     )
                      _lowerCamelCase				:   Tuple								=   self.tokenizer_class.from_pretrained(__A,**__A     )

                      _lowerCamelCase				:   Optional[int]								=   tokenizer_r.encode(__A,add_special_tokens=__A     )
                      _lowerCamelCase				:   Any								=   tokenizer_p.encode(__A,add_special_tokens=__A     )

                      _lowerCamelCase				:   List[Any]								=   tokenizer_r.convert_ids_to_tokens(__A     )
                      _lowerCamelCase				:   int								=   tokenizer_p.convert_ids_to_tokens(__A     )

                      # it is expected that only the first Chinese character is not preceded by "##".
                      _lowerCamelCase				:   Dict								=   [
                          f'##{token}' if idx != 0 else token for idx, token in enumerate(__A     )
                      ]
                      self.assertListEqual(__A,__A     )
                      self.assertListEqual(__A,__A     )

'''simple docstring'''






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
 from .tokenization_albert import AlbertTokenizer
else:
 UpperCAmelCase_       :   Any   =       None

UpperCAmelCase_       :   Any   =       logging.get_logger(__name__)
UpperCAmelCase_       :   Optional[Any]   =       {'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase_       :   List[Any]   =       {
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json',
    },
}

UpperCAmelCase_       :   str   =       {
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

UpperCAmelCase_       :   str   =       '▁'

class  UpperCAmelCase__						(       A ):
    lowerCAmelCase_				    =		VOCAB_FILES_NAMES
    lowerCAmelCase_				    =		PRETRAINED_VOCAB_FILES_MAP
    lowerCAmelCase_				    =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCAmelCase_				    =		AlbertTokenizer



    def __init__(	self      :       Dict,__A      :       str=None,__A      :       Union[str, Any]=None,__A      :       Union[str, Any]=True,__A      :       int=True,__A      :       List[Any]=False,__A      :       Any="[CLS]",__A      :       List[str]="[SEP]",__A      :       Tuple="<unk>",__A      :       Dict="[SEP]",__A      :       Dict="<pad>",__A      :       List[str]="[CLS]",__A      :       Any="[MASK]",**__A      :       Tuple,):
          # Mask token behave like a normal word, i.e. include the space before it and
          # is included in the raw text, there should be a match in a non-normalized sentence.
          _lowerCamelCase				:   Union[str, Any]								=   (
              AddedToken(__A,lstrip=__A,rstrip=__A,normalized=__A     )
              if isinstance(__A,__A     )
              else mask_token
          )

          super().__init__(
              __A,tokenizer_file=__A,do_lower_case=__A,remove_space=__A,keep_accents=__A,bos_token=__A,eos_token=__A,unk_token=__A,sep_token=__A,pad_token=__A,cls_token=__A,mask_token=__A,**__A,)

          _lowerCamelCase				:   Dict								=   do_lower_case
          _lowerCamelCase				:   List[str]								=   remove_space
          _lowerCamelCase				:   Any								=   keep_accents
          _lowerCamelCase				:   Tuple								=   vocab_file
          _lowerCamelCase				:   Optional[int]								=   False if not self.vocab_file else True



    def   lowerCamelCase_  (	self      :       List[str],__A      :       List[int],__A      :       Optional[List[int]] = None     ):
          _lowerCamelCase				:   List[Any]								=   [self.sep_token_id]
          _lowerCamelCase				:   Any								=   [self.cls_token_id]
          if token_ids_a is None:
                return cls + token_ids_a + sep
          return cls + token_ids_a + sep + token_ids_a + sep



    def   lowerCamelCase_  (	self      :       List[str],__A      :       List[int],__A      :       Optional[List[int]] = None     ):
          _lowerCamelCase				:   Optional[Any]								=   [self.sep_token_id]
          _lowerCamelCase				:   Union[str, Any]								=   [self.cls_token_id]

          if token_ids_a is None:
                return len(cls + token_ids_a + sep     ) * [0]
          return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]



    def   lowerCamelCase_  (	self      :       str,__A      :       str,__A      :       Optional[str] = None     ):
          if not self.can_save_slow_tokenizer:
                raise ValueError(
                    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                    "tokenizer."     )

          if not os.path.isdir(__A     ):
                logger.error(f'Vocabulary path ({save_directory}) should be a directory'     )
                return
          _lowerCamelCase				:   Union[str, Any]								=   os.path.join(
              __A,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )

          if os.path.abspath(self.vocab_file     ) != os.path.abspath(__A     ):
                copyfile(self.vocab_file,__A     )

          return (out_vocab_file,)