

import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
  UpperCAmelCase__ :     Dict				    =    """pt"""
elif is_tf_available():
  UpperCAmelCase__ :     Tuple				    =    """tf"""
else:
  UpperCAmelCase__ :     int				    =    """jax"""







class        a__       (   UpperCAmelCase	,			unittest.TestCase       ):







  """simple docstring"""




  UpperCAmelCase__      : Optional[int]			       =ByTaTokenizer
  UpperCAmelCase__      : str			       =False


  def  _lowercase      (			self    : str		)					->Optional[int]:
        """simple docstring"""







        super().setUp()
        SCREAMING_SNAKE_CASE :      List[str]       =   ByTaTokenizer()
        tokenizer.save_pretrained(self.tmpdirname		)


  @cached_property
  def  _lowercase      (			self    : List[Any]		)					->List[str]:
        """simple docstring"""







        return ByTaTokenizer.from_pretrained("""google/byt5-small"""		)


  def  _lowercase      (			self    : List[Any]				,				**UpperCAmelCase__    : int		)					->ByTaTokenizer:
        """simple docstring"""







        return self.tokenizer_class.from_pretrained(self.tmpdirname				,				**UpperCAmelCase__		)


  def  _lowercase      (			self    : Optional[Any]				,				UpperCAmelCase__    : Optional[int]				,				UpperCAmelCase__    : int=False				,				UpperCAmelCase__    : Dict=2_0				,				UpperCAmelCase__    : Tuple=5		)					->Tuple[str, list]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      List[Any]       =   []
        for i in range(len(UpperCAmelCase__		)		):
              try:
                    SCREAMING_SNAKE_CASE :      Any       =   tokenizer.decode([i]				,				clean_up_tokenization_spaces=UpperCAmelCase__		)
              except UnicodeDecodeError:
                    pass
              toks.append((i, tok)		)

        SCREAMING_SNAKE_CASE :      Any       =   list(filter(lambda UpperCAmelCase__		: re.match(R"""^[ a-zA-Z]+$"""				,				t[1]		)				,				UpperCAmelCase__		)		)
        SCREAMING_SNAKE_CASE :      Optional[int]       =   list(filter(lambda UpperCAmelCase__		: [t[0]] == tokenizer.encode(t[1]				,				add_special_tokens=UpperCAmelCase__		)				,				UpperCAmelCase__		)		)
        if max_length is not None and len(UpperCAmelCase__		) > max_length:
              SCREAMING_SNAKE_CASE :      List[Any]       =   toks[:max_length]
        if min_length is not None and len(UpperCAmelCase__		) < min_length and len(UpperCAmelCase__		) > 0:
              while len(UpperCAmelCase__		) < min_length:
                    SCREAMING_SNAKE_CASE :      List[str]       =   toks + toks
        # toks_str = [t[1] for t in toks]
        SCREAMING_SNAKE_CASE :      Optional[Any]       =   [t[0] for t in toks]

        # Ensure consistency
        SCREAMING_SNAKE_CASE :      List[str]       =   tokenizer.decode(UpperCAmelCase__				,				clean_up_tokenization_spaces=UpperCAmelCase__		)
        if " " not in output_txt and len(UpperCAmelCase__		) > 1:
              SCREAMING_SNAKE_CASE :      Optional[int]       =   (
                  tokenizer.decode([toks_ids[0]]				,				clean_up_tokenization_spaces=UpperCAmelCase__		)
                  + """ """
                  + tokenizer.decode(toks_ids[1:]				,				clean_up_tokenization_spaces=UpperCAmelCase__		)
              )
        if with_prefix_space:
              SCREAMING_SNAKE_CASE :      str       =   """ """ + output_txt
        SCREAMING_SNAKE_CASE :      Optional[int]       =   tokenizer.encode(UpperCAmelCase__				,				add_special_tokens=UpperCAmelCase__		)
        return output_txt, output_ids


  def  _lowercase      (			self    : Tuple		)					->List[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Any       =   self.ta_base_tokenizer
        SCREAMING_SNAKE_CASE :      List[Any]       =   tokenizer(["""hi</s>""", """I went to the gym</s>""", """</s>"""]		)
        SCREAMING_SNAKE_CASE :      Tuple       =   tokenizer(["""hi""", """I went to the gym""", """"""]		)
        self.assertListEqual(batch_with_eos_added["""input_ids"""]				,				batch_without_eos_added["""input_ids"""]		)


  def  _lowercase      (			self    : str		)					->List[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      str       =   self.ta_base_tokenizer
        SCREAMING_SNAKE_CASE :      Dict       =   """Unicode €."""
        SCREAMING_SNAKE_CASE :      Any       =   tokenizer(UpperCAmelCase__		)
        SCREAMING_SNAKE_CASE :      Optional[int]       =   [8_8, 1_1_3, 1_0_8, 1_0_2, 1_1_4, 1_0_3, 1_0_4, 3_5, 2_2_9, 1_3_3, 1_7_5, 4_9, 1]
        self.assertEqual(encoded["""input_ids"""]				,				UpperCAmelCase__		)

        # decoding
        SCREAMING_SNAKE_CASE :      Union[str, Any]       =   tokenizer.decode(UpperCAmelCase__		)
        self.assertEqual(UpperCAmelCase__				,				"""Unicode €.</s>"""		)

        SCREAMING_SNAKE_CASE :      str       =   tokenizer("""e è é ê ë"""		)
        SCREAMING_SNAKE_CASE :      int       =   [1_0_4, 3_5, 1_9_8, 1_7_1, 3_5, 1_9_8, 1_7_2, 3_5, 1_9_8, 1_7_3, 3_5, 1_9_8, 1_7_4, 1]
        self.assertEqual(encoded["""input_ids"""]				,				UpperCAmelCase__		)
        # decoding
        SCREAMING_SNAKE_CASE :      Optional[int]       =   tokenizer.decode(UpperCAmelCase__		)
        self.assertEqual(UpperCAmelCase__				,				"""e è é ê ë</s>"""		)

        # encode/decode, but with `encode` instead of `__call__`
        self.assertEqual(tokenizer.decode(tokenizer.encode("""e è é ê ë"""		)		)				,				"""e è é ê ë</s>"""		)


  def  _lowercase      (			self    : Union[str, Any]		)					->List[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      int       =   self.ta_base_tokenizer
        SCREAMING_SNAKE_CASE :      Optional[Any]       =   ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
        # fmt: off
        SCREAMING_SNAKE_CASE :      Union[str, Any]       =   [6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 1, 0]
        # fmt: on
        SCREAMING_SNAKE_CASE :      str       =   tokenizer(UpperCAmelCase__				,				padding=UpperCAmelCase__				,				return_tensors=UpperCAmelCase__		)
        self.assertIsInstance(UpperCAmelCase__				,				UpperCAmelCase__		)

        if FRAMEWORK != "jax":
              SCREAMING_SNAKE_CASE :      Any       =   list(batch.input_ids.numpy()[0]		)
        else:
              SCREAMING_SNAKE_CASE :      int       =   list(batch.input_ids.tolist()[0]		)

        self.assertListEqual(UpperCAmelCase__				,				UpperCAmelCase__		)

        self.assertEqual((2, 3_7)				,				batch.input_ids.shape		)
        self.assertEqual((2, 3_7)				,				batch.attention_mask.shape		)


  def  _lowercase      (			self    : Union[str, Any]		)					->List[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Dict       =   self.ta_base_tokenizer
        SCREAMING_SNAKE_CASE :      List[Any]       =   ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
        SCREAMING_SNAKE_CASE :      Dict       =   tokenizer(UpperCAmelCase__				,				padding=UpperCAmelCase__				,				return_tensors=UpperCAmelCase__		)
        # check if input_ids are returned and no decoder_input_ids
        self.assertIn("""input_ids"""				,				UpperCAmelCase__		)
        self.assertIn("""attention_mask"""				,				UpperCAmelCase__		)
        self.assertNotIn("""decoder_input_ids"""				,				UpperCAmelCase__		)
        self.assertNotIn("""decoder_attention_mask"""				,				UpperCAmelCase__		)


  def  _lowercase      (			self    : Optional[Any]		)					->List[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Optional[int]       =   self.ta_base_tokenizer
        SCREAMING_SNAKE_CASE :      Union[str, Any]       =   [
            """Summary of the text.""",
            """Another summary.""",
        ]
        SCREAMING_SNAKE_CASE :      Dict       =   tokenizer(
            text_target=UpperCAmelCase__				,				max_length=3_2				,				padding="""max_length"""				,				truncation=UpperCAmelCase__				,				return_tensors=UpperCAmelCase__		)
        self.assertEqual(3_2				,				targets["""input_ids"""].shape[1]		)


  def  _lowercase      (			self    : int		)					->Optional[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Tuple       =   self.ta_base_tokenizer
        SCREAMING_SNAKE_CASE :      Optional[Any]       =   ["""A long paragraph for summarization. </s>"""]
        SCREAMING_SNAKE_CASE :      Any       =   ["""Summary of the text. </s>"""]
        # fmt: off
        SCREAMING_SNAKE_CASE :      Tuple       =   [6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 3_5, 1]
        SCREAMING_SNAKE_CASE :      str       =   [8_6, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_2_4, 3_5, 1_1_4, 1_0_5, 3_5, 1_1_9, 1_0_7, 1_0_4, 3_5, 1_1_9, 1_0_4, 1_2_3, 1_1_9, 4_9, 3_5, 1]
        # fmt: on

        SCREAMING_SNAKE_CASE :      int       =   tokenizer(UpperCAmelCase__				,				text_target=UpperCAmelCase__		)

        self.assertEqual(UpperCAmelCase__				,				batch["""input_ids"""][0]		)
        self.assertEqual(UpperCAmelCase__				,				batch["""labels"""][0]		)


  def  _lowercase      (			self    : str		)					->str:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Optional[Any]       =   self.get_tokenizers()
        for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"		):
                    self.assertNotEqual(tokenizer.model_max_length				,				4_2		)

        # Now let's start the test
        SCREAMING_SNAKE_CASE :      Any       =   self.get_tokenizers()
        for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"		):
                    # Isolate this from the other tests because we save additional tokens/etc
                    SCREAMING_SNAKE_CASE :      str       =   tempfile.mkdtemp()

                    SCREAMING_SNAKE_CASE :      Optional[int]       =   """ He is very happy, UNwant\u00E9d,running"""
                    SCREAMING_SNAKE_CASE :      List[Any]       =   tokenizer.encode(UpperCAmelCase__				,				add_special_tokens=UpperCAmelCase__		)
                    tokenizer.save_pretrained(UpperCAmelCase__		)

                    SCREAMING_SNAKE_CASE :      Union[str, Any]       =   tokenizer.__class__.from_pretrained(UpperCAmelCase__		)
                    SCREAMING_SNAKE_CASE :      Any       =   after_tokenizer.encode(UpperCAmelCase__				,				add_special_tokens=UpperCAmelCase__		)
                    self.assertListEqual(UpperCAmelCase__				,				UpperCAmelCase__		)

                    shutil.rmtree(UpperCAmelCase__		)

        SCREAMING_SNAKE_CASE :      int       =   self.get_tokenizers(model_max_length=4_2		)
        for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"		):
                    # Isolate this from the other tests because we save additional tokens/etc
                    SCREAMING_SNAKE_CASE :      List[str]       =   tempfile.mkdtemp()

                    SCREAMING_SNAKE_CASE :      Optional[Any]       =   """ He is very happy, UNwant\u00E9d,running"""
                    tokenizer.add_tokens(["""bim""", """bambam"""]		)
                    SCREAMING_SNAKE_CASE :      Any       =   tokenizer.additional_special_tokens
                    additional_special_tokens.append("""new_additional_special_token"""		)
                    tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}		)
                    SCREAMING_SNAKE_CASE :      List[str]       =   tokenizer.encode(UpperCAmelCase__				,				add_special_tokens=UpperCAmelCase__		)
                    tokenizer.save_pretrained(UpperCAmelCase__		)

                    SCREAMING_SNAKE_CASE :      Any       =   tokenizer.__class__.from_pretrained(UpperCAmelCase__		)
                    SCREAMING_SNAKE_CASE :      List[str]       =   after_tokenizer.encode(UpperCAmelCase__				,				add_special_tokens=UpperCAmelCase__		)
                    self.assertListEqual(UpperCAmelCase__				,				UpperCAmelCase__		)
                    self.assertIn("""new_additional_special_token"""				,				after_tokenizer.additional_special_tokens		)
                    self.assertEqual(after_tokenizer.model_max_length				,				4_2		)

                    SCREAMING_SNAKE_CASE :      Tuple       =   tokenizer.__class__.from_pretrained(UpperCAmelCase__				,				model_max_length=4_3		)
                    self.assertEqual(tokenizer.model_max_length				,				4_3		)

                    shutil.rmtree(UpperCAmelCase__		)


  def  _lowercase      (			self    : List[str]		)					->Union[str, Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Optional[Any]       =   []
        if self.test_slow_tokenizer:
              tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())		)

        if self.test_rust_tokenizer:
              tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())		)

        for tokenizer_class, tokenizer_utils in tokenizer_list:
              with tempfile.TemporaryDirectory() as tmp_dir:
                    tokenizer_utils.save_pretrained(UpperCAmelCase__		)

                    with open(os.path.join(UpperCAmelCase__				,				"""special_tokens_map.json"""		)				,				encoding="""utf-8"""		) as json_file:
                          SCREAMING_SNAKE_CASE :      List[Any]       =   json.load(UpperCAmelCase__		)

                    with open(os.path.join(UpperCAmelCase__				,				"""tokenizer_config.json"""		)				,				encoding="""utf-8"""		) as json_file:
                          SCREAMING_SNAKE_CASE :      List[str]       =   json.load(UpperCAmelCase__		)

                    SCREAMING_SNAKE_CASE :      List[Any]       =   [f"<extra_id_{i}>" for i in range(1_2_5		)]

                    SCREAMING_SNAKE_CASE :      Optional[int]       =   added_tokens_extra_ids + [
                        """an_additional_special_token"""
                    ]
                    SCREAMING_SNAKE_CASE :      int       =   added_tokens_extra_ids + [
                        """an_additional_special_token"""
                    ]

                    with open(os.path.join(UpperCAmelCase__				,				"""special_tokens_map.json"""		)				,				"""w"""				,				encoding="""utf-8"""		) as outfile:
                          json.dump(UpperCAmelCase__				,				UpperCAmelCase__		)
                    with open(os.path.join(UpperCAmelCase__				,				"""tokenizer_config.json"""		)				,				"""w"""				,				encoding="""utf-8"""		) as outfile:
                          json.dump(UpperCAmelCase__				,				UpperCAmelCase__		)

                    # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                    # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                    # "special_tokens_map.json" files
                    SCREAMING_SNAKE_CASE :      Optional[Any]       =   tokenizer_class.from_pretrained(
                        UpperCAmelCase__				,				)
                    self.assertIn(
                        """an_additional_special_token"""				,				tokenizer_without_change_in_init.additional_special_tokens		)
                    # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                    self.assertEqual(
                        ["""an_additional_special_token"""]				,				tokenizer_without_change_in_init.convert_ids_to_tokens(
                            tokenizer_without_change_in_init.convert_tokens_to_ids(["""an_additional_special_token"""]		)		)				,				)

                    # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                    SCREAMING_SNAKE_CASE :      Optional[Any]       =   added_tokens_extra_ids + [AddedToken("""a_new_additional_special_token"""				,				lstrip=UpperCAmelCase__		)]
                    SCREAMING_SNAKE_CASE :      List[Any]       =   tokenizer_class.from_pretrained(
                        UpperCAmelCase__				,				additional_special_tokens=UpperCAmelCase__				,				)

                    self.assertIn("""a_new_additional_special_token"""				,				tokenizer.additional_special_tokens		)
                    self.assertEqual(
                        ["""a_new_additional_special_token"""]				,				tokenizer.convert_ids_to_tokens(
                            tokenizer.convert_tokens_to_ids(["""a_new_additional_special_token"""]		)		)				,				)


  def  _lowercase      (			self    : Tuple		)					->List[str]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      List[Any]       =   []
        if self.test_slow_tokenizer:
              tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())		)

        if self.test_rust_tokenizer:
              tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())		)

        for tokenizer_class, tokenizer_utils in tokenizer_list:
              with tempfile.TemporaryDirectory() as tmp_dir:
                    tokenizer_utils.save_pretrained(UpperCAmelCase__		)

                    SCREAMING_SNAKE_CASE :      Union[str, Any]       =   tokenizer_class.from_pretrained(UpperCAmelCase__		)

                    self.assertTrue(tokenizer.decode([2_5_5]		) == """"""		)


  def  _lowercase      (			self    : Optional[int]		)					->Optional[int]:
        """simple docstring"""







        pass


  def  _lowercase      (			self    : Tuple		)					->List[str]:
        """simple docstring"""







        pass


  def  _lowercase      (			self    : Tuple		)					->Optional[int]:
        """simple docstring"""







        pass


  def  _lowercase      (			self    : int		)					->Optional[int]:
        """simple docstring"""







        pass


  def  _lowercase      (			self    : int		)					->Union[str, Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      str       =   self.get_tokenizers(fast=UpperCAmelCase__				,				do_lower_case=UpperCAmelCase__		)
        for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"		):
                    SCREAMING_SNAKE_CASE :      List[str]       =   ["""t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """x""", """t""", """</s>"""]
                    SCREAMING_SNAKE_CASE :      Any       =   tokenizer.convert_tokens_to_string(UpperCAmelCase__		)

                    self.assertIsInstance(UpperCAmelCase__				,				UpperCAmelCase__		)




  def  _lowercase      (			self    : Any		)					->Tuple:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Any       =   self.get_tokenizers()
        for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"		):
                    SCREAMING_SNAKE_CASE :      str       =   [
                        """bos_token""",
                        """eos_token""",
                        """unk_token""",
                        """sep_token""",
                        """pad_token""",
                        """cls_token""",
                        """mask_token""",
                    ]

                    SCREAMING_SNAKE_CASE :      Tuple       =   0
                    SCREAMING_SNAKE_CASE :      Optional[int]       =   tokenizer.convert_ids_to_tokens(
                        UpperCAmelCase__				,				skip_special_tokens=UpperCAmelCase__		)

                    for attr in attributes_list:
                          setattr(UpperCAmelCase__				,				attr + """_id"""				,				UpperCAmelCase__		)
                          self.assertEqual(getattr(UpperCAmelCase__				,				UpperCAmelCase__		)				,				UpperCAmelCase__		)
                          self.assertEqual(getattr(UpperCAmelCase__				,				attr + """_id"""		)				,				UpperCAmelCase__		)

                          setattr(UpperCAmelCase__				,				attr + """_id"""				,				UpperCAmelCase__		)
                          self.assertEqual(getattr(UpperCAmelCase__				,				UpperCAmelCase__		)				,				UpperCAmelCase__		)
                          self.assertEqual(getattr(UpperCAmelCase__				,				attr + """_id"""		)				,				UpperCAmelCase__		)

                    setattr(UpperCAmelCase__				,				"""additional_special_tokens_ids"""				,				[]		)
                    self.assertListEqual(getattr(UpperCAmelCase__				,				"""additional_special_tokens"""		)				,				[]		)
                    self.assertListEqual(getattr(UpperCAmelCase__				,				"""additional_special_tokens_ids"""		)				,				[]		)

                    setattr(UpperCAmelCase__				,				"""additional_special_tokens_ids"""				,				[token_id_to_test_setters]		)
                    self.assertListEqual(getattr(UpperCAmelCase__				,				"""additional_special_tokens"""		)				,				[token_to_test_setters]		)
                    self.assertListEqual(getattr(UpperCAmelCase__				,				"""additional_special_tokens_ids"""		)				,				[token_id_to_test_setters]		)







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


UpperCAmelCase__ :     str				    =    {"""processing_layoutxlm""": ["""LayoutXLMProcessor"""]}

try:
  if not is_sentencepiece_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase__ :     Any				    =    ["""LayoutXLMTokenizer"""]

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase__ :     List[str]				    =    ["""LayoutXLMTokenizerFast"""]

if TYPE_CHECKING:
  from .processing_layoutxlm import LayoutXLMProcessor

  try:
    if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_layoutxlm import LayoutXLMTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_layoutxlm_fast import LayoutXLMTokenizerFast

else:
  import sys

  UpperCAmelCase__ :     List[str]				    =    _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




