





"""simple docstring"""


import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XGLMTokenizer, XGLMTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase__				  =   get_tests_dir("fixtures/test_sentencepiece.model")



@require_sentencepiece
@require_tokenizers
class      __SCREAMING_SNAKE_CASE    (						_UpperCamelCase		,  unittest.TestCase   ):







  '''simple docstring'''


  SCREAMING_SNAKE_CASE__							:Optional[int]	     =					XGLMTokenizer
  SCREAMING_SNAKE_CASE__							:Union[str, Any]	     =					XGLMTokenizerFast
  SCREAMING_SNAKE_CASE__							:Dict	     =					True
  SCREAMING_SNAKE_CASE__							:List[Any]	     =					True

  def    __SCREAMING_SNAKE_CASE				(		self   :       int	)		->				Any:
        super().setUp()

        # We have a SentencePiece fixture for testing
        _UpperCamelCase      :	Tuple						=       XGLMTokenizer(__a					,   keep_accents=__a	)
        tokenizer.save_pretrained(self.tmpdirname	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       Any	)		->				str:
        _UpperCamelCase      :	str						=       "<pad>"
        _UpperCamelCase      :	Optional[int]						=       1

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(__a	)					,   __a	)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(__a	)					,   __a	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       Optional[Any]	)		->				Dict:
        _UpperCamelCase      :	int						=       list(self.get_tokenizer().get_vocab().keys()	)

        self.assertEqual(vocab_keys[0]					,   "<s>"	)
        self.assertEqual(vocab_keys[1]					,   "<pad>"	)
        self.assertEqual(len(__a	)					,   1008	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       List[Any]	)		->				Optional[int]:
        self.assertEqual(self.get_tokenizer().vocab_size					,   1008	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       str	)		->				Any:
        _UpperCamelCase      :	Tuple						=       XGLMTokenizer(__a					,   keep_accents=__a	)

        _UpperCamelCase      :	Union[str, Any]						=       tokenizer.tokenize("This is a test"	)
        self.assertListEqual(__a					,   ["▁This", "▁is", "▁a", "▁t", "est"]	)

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(__a	)					,   [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]					,   )

        _UpperCamelCase      :	Union[str, Any]						=       tokenizer.tokenize("I was born in 92000, and this is falsé."	)
        self.assertListEqual(
            __a					,   [
                SPIECE_UNDERLINE + "I",
                SPIECE_UNDERLINE + "was",
                SPIECE_UNDERLINE + "b",
                "or",
                "n",
                SPIECE_UNDERLINE + "in",
                SPIECE_UNDERLINE + "",
                "9",
                "2",
                "0",
                "0",
                "0",
                ",",
                SPIECE_UNDERLINE + "and",
                SPIECE_UNDERLINE + "this",
                SPIECE_UNDERLINE + "is",
                SPIECE_UNDERLINE + "f",
                "al",
                "s",
                "é",
                ".",
            ]					,   )
        _UpperCamelCase      :	Dict						=       tokenizer.convert_tokens_to_ids(__a	)
        self.assertListEqual(
            __a					,   [
                value + tokenizer.fairseq_offset
                for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
            ]					,   )

        _UpperCamelCase      :	Optional[int]						=       tokenizer.convert_ids_to_tokens(__a	)
        self.assertListEqual(
            __a					,   [
                SPIECE_UNDERLINE + "I",
                SPIECE_UNDERLINE + "was",
                SPIECE_UNDERLINE + "b",
                "or",
                "n",
                SPIECE_UNDERLINE + "in",
                SPIECE_UNDERLINE + "",
                "<unk>",
                "2",
                "0",
                "0",
                "0",
                ",",
                SPIECE_UNDERLINE + "and",
                SPIECE_UNDERLINE + "this",
                SPIECE_UNDERLINE + "is",
                SPIECE_UNDERLINE + "f",
                "al",
                "s",
                "<unk>",
                ".",
            ]					,   )

  @cached_property
  def    __SCREAMING_SNAKE_CASE				(		self   :       Optional[int]	)		->				Optional[int]:
        return XGLMTokenizer.from_pretrained("facebook/xglm-564M"	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       int	)		->				Dict:
        with tempfile.NamedTemporaryFile() as f:
              shutil.copyfile(__a					,   f.name	)
              _UpperCamelCase      :	str						=       XGLMTokenizer(f.name					,   keep_accents=__a	)
              _UpperCamelCase      :	Tuple						=       pickle.dumps(__a	)
        pickle.loads(__a	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       Any	)		->				Optional[Any]:
        if not self.test_rust_tokenizer:
              return

        _UpperCamelCase      :	Optional[int]						=       self.get_tokenizer()
        _UpperCamelCase      :	Dict						=       self.get_rust_tokenizer()

        _UpperCamelCase      :	Optional[int]						=       "I was born in 92000, and this is falsé."

        _UpperCamelCase      :	List[str]						=       tokenizer.tokenize(__a	)
        _UpperCamelCase      :	Union[str, Any]						=       rust_tokenizer.tokenize(__a	)
        self.assertListEqual(__a					,   __a	)

        _UpperCamelCase      :	int						=       tokenizer.encode(__a					,   add_special_tokens=__a	)
        _UpperCamelCase      :	Optional[int]						=       rust_tokenizer.encode(__a					,   add_special_tokens=__a	)
        self.assertListEqual(__a					,   __a	)

        _UpperCamelCase      :	List[Any]						=       self.get_rust_tokenizer()
        _UpperCamelCase      :	str						=       tokenizer.encode(__a	)
        _UpperCamelCase      :	Optional[Any]						=       rust_tokenizer.encode(__a	)
        self.assertListEqual(__a					,   __a	)

  @slow
  def    __SCREAMING_SNAKE_CASE				(		self   :       str	)		->				Optional[Any]:
        _UpperCamelCase      :	List[Any]						=       "Hello World!"
        _UpperCamelCase      :	Optional[int]						=       [2, 3_1227, 4447, 35]

        self.assertListEqual(__a					,   self.big_tokenizer.encode(__a	)	)

  @slow
  def    __SCREAMING_SNAKE_CASE				(		self   :       str	)		->				Optional[int]:
        _UpperCamelCase      :	Optional[Any]						=       (
            "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
            " add words that should not exsist and be tokenized to unk, such as saoneuhaoesuth"
        )
        # fmt: off
        _UpperCamelCase      :	Tuple						=       [2, 1018, 67, 11, 1988, 2617, 5631, 278, 11, 3407, 48, 7_1630, 2_8085, 4, 3234, 157, 13, 6, 5, 6, 4, 3526, 768, 15, 659, 57, 298, 3983, 864, 129, 21, 6, 5, 1_3675, 377, 652, 7580, 1_0341, 155, 2817, 422, 1666, 7, 1674, 53, 113, 20_2277, 1_7892, 33, 60, 87, 4, 3234, 157, 61, 2667, 5_2376, 19, 88, 23, 735]
        # fmt: on

        self.assertListEqual(__a					,   self.big_tokenizer.encode(__a	)	)


  @slow
  def    __SCREAMING_SNAKE_CASE				(		self   :       str	)		->				Optional[int]:
        # fmt: off
        _UpperCamelCase      :	List[Any]						=       {
            "input_ids": [[2, 10_8825, 1163, 15, 8_8010, 473, 1_5898, 157, 1_3672, 1857, 312, 8, 23_8021, 1163, 53, 1_3672, 1857, 312, 8, 5_3283, 18_2396, 8, 1_8566, 16, 3_6733, 4101, 8, 230, 24_4017, 12_2553, 7, 15, 13_2597, 4, 293, 1_2511, 7610, 4, 3414, 13_2597, 9, 4, 3_2361, 362, 4, 734, 2_8512, 3_2569, 18, 4, 3_2361, 2_6096, 1_4982, 73, 1_8715, 2_1433, 23_5261, 15, 492, 1_2427, 16, 53, 1_8715, 2_1433, 6_5454, 15, 2_3659, 563, 16, 278, 597, 2843, 595, 7931, 18_2396, 6_4186, 22, 886, 595, 13_2981, 53, 2_5540, 3449, 4_3982, 3_9901, 5951, 878, 330, 4, 2_7694, 8_0269, 312, 53, 6517, 1_1780, 611, 2_0408, 5], [2, 6, 13_2597, 67, 4_2897, 33, 592, 8, 16_3729, 2_5540, 361, 13_6997, 10_9514, 17_3230, 7, 501, 60, 10_2913, 196, 5631, 235, 6_3243, 473, 6, 23_1757, 74, 5277, 7905, 53, 3095, 3_7317, 22, 454, 18_3874, 5], [2, 268, 3_1298, 4_6530, 6, 13_2935, 4_3831, 7, 597, 32, 24, 3688, 9865, 5]],
            "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]
        }  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=__a					,   model_name="facebook/xglm-564M"					,   padding=__a					,   )







"""simple docstring"""


import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class      __SCREAMING_SNAKE_CASE    (						_UpperCamelCase		,  unittest.TestCase   ):







  '''simple docstring'''


  SCREAMING_SNAKE_CASE__							:Union[str, Any]	     =					RoCBertTokenizer
  SCREAMING_SNAKE_CASE__							:Dict	     =					None
  SCREAMING_SNAKE_CASE__							:List[Any]	     =					False
  SCREAMING_SNAKE_CASE__							:Union[str, Any]	     =					True
  SCREAMING_SNAKE_CASE__							:Union[str, Any]	     =					filter_non_english

  def    __SCREAMING_SNAKE_CASE				(		self   :       str	)		->				Dict:
        super().setUp()

        _UpperCamelCase      :	Any						=       ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "你", "好", "是", "谁", "a", "b", "c", "d"]
        _UpperCamelCase      :	List[str]						=       {}
        _UpperCamelCase      :	Tuple						=       {}
        for i, value in enumerate(__a	):
              _UpperCamelCase      :	List[str]						=       i
              _UpperCamelCase      :	Optional[Any]						=       i
        _UpperCamelCase      :	Dict						=       os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES["vocab_file"]	)
        _UpperCamelCase      :	Optional[Any]						=       os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES["word_shape_file"]	)
        _UpperCamelCase      :	Any						=       os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES["word_pronunciation_file"]	)
        with open(self.vocab_file					,   "w"					,   encoding="utf-8"	) as vocab_writer:
              vocab_writer.write("".join([x + "\n" for x in vocab_tokens]	)	)
        with open(self.word_shape_file					,   "w"					,   encoding="utf-8"	) as word_shape_writer:
              json.dump(__a					,   __a					,   ensure_ascii=__a	)
        with open(self.word_pronunciation_file					,   "w"					,   encoding="utf-8"	) as word_pronunciation_writer:
              json.dump(__a					,   __a					,   ensure_ascii=__a	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       int	)		->				Optional[int]:
        _UpperCamelCase      :	Tuple						=       self.tokenizer_class(self.vocab_file					,   self.word_shape_file					,   self.word_pronunciation_file	)

        _UpperCamelCase      :	int						=       tokenizer.tokenize("你好[SEP]你是谁"	)
        self.assertListEqual(__a					,   ["你", "好", "[SEP]", "你", "是", "谁"]	)
        self.assertListEqual(tokenizer.convert_tokens_to_ids(__a	)					,   [5, 6, 2, 5, 7, 8]	)
        self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(__a	)					,   [5, 6, 2, 5, 7, 8]	)
        self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(__a	)					,   [5, 6, 2, 5, 7, 8]	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       Dict	)		->				Dict:
        _UpperCamelCase      :	Dict						=       RoCBertBasicTokenizer()

        self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"	)					,   ["ah", "\u535A", "\u63A8", "zz"]	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       List[str]	)		->				Any:
        _UpperCamelCase      :	List[Any]						=       RoCBertBasicTokenizer(do_lower_case=__a	)

        self.assertListEqual(
            tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)					,   ["hello", "!", "how", "are", "you", "?"]	)
        self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)					,   ["hello"]	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       Optional[Any]	)		->				Tuple:
        _UpperCamelCase      :	Optional[Any]						=       RoCBertBasicTokenizer(do_lower_case=__a					,   strip_accents=__a	)

        self.assertListEqual(
            tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)					,   ["hällo", "!", "how", "are", "you", "?"]	)
        self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)					,   ["h\u00E9llo"]	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       Dict	)		->				str:
        _UpperCamelCase      :	Dict						=       RoCBertBasicTokenizer(do_lower_case=__a					,   strip_accents=__a	)

        self.assertListEqual(
            tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)					,   ["hallo", "!", "how", "are", "you", "?"]	)
        self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)					,   ["hello"]	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       Union[str, Any]	)		->				Union[str, Any]:
        _UpperCamelCase      :	List[str]						=       RoCBertBasicTokenizer(do_lower_case=__a	)

        self.assertListEqual(
            tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)					,   ["hallo", "!", "how", "are", "you", "?"]	)
        self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)					,   ["hello"]	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       str	)		->				Optional[int]:
        _UpperCamelCase      :	Tuple						=       RoCBertBasicTokenizer(do_lower_case=__a	)

        self.assertListEqual(
            tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)					,   ["HeLLo", "!", "how", "Are", "yoU", "?"]	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       List[str]	)		->				List[str]:
        _UpperCamelCase      :	Union[str, Any]						=       RoCBertBasicTokenizer(do_lower_case=__a					,   strip_accents=__a	)

        self.assertListEqual(
            tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)					,   ["HäLLo", "!", "how", "Are", "yoU", "?"]	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       Tuple	)		->				List[Any]:
        _UpperCamelCase      :	Tuple						=       RoCBertBasicTokenizer(do_lower_case=__a					,   strip_accents=__a	)

        self.assertListEqual(
            tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)					,   ["HaLLo", "!", "how", "Are", "yoU", "?"]	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       Optional[Any]	)		->				int:
        _UpperCamelCase      :	int						=       RoCBertBasicTokenizer(do_lower_case=__a					,   never_split=["[UNK]"]	)

        self.assertListEqual(
            tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"	)					,   ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       List[str]	)		->				Optional[Any]:
        _UpperCamelCase      :	Optional[int]						=       ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

        _UpperCamelCase      :	Any						=       {}
        for i, token in enumerate(__a	):
              _UpperCamelCase      :	str						=       i
        _UpperCamelCase      :	Optional[int]						=       RoCBertWordpieceTokenizer(vocab=__a					,   unk_token="[UNK]"	)

        self.assertListEqual(tokenizer.tokenize(""	)					,   []	)

        self.assertListEqual(tokenizer.tokenize("unwanted running"	)					,   ["un", "##want", "##ed", "runn", "##ing"]	)

        self.assertListEqual(tokenizer.tokenize("unwantedX running"	)					,   ["[UNK]", "runn", "##ing"]	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       Dict	)		->				Optional[Any]:
        self.assertTrue(_is_whitespace(" "	)	)
        self.assertTrue(_is_whitespace("\t"	)	)
        self.assertTrue(_is_whitespace("\r"	)	)
        self.assertTrue(_is_whitespace("\n"	)	)
        self.assertTrue(_is_whitespace("\u00A0"	)	)

        self.assertFalse(_is_whitespace("A"	)	)
        self.assertFalse(_is_whitespace("-"	)	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       Any	)		->				Dict:
        self.assertTrue(_is_control("\u0005"	)	)

        self.assertFalse(_is_control("A"	)	)
        self.assertFalse(_is_control(" "	)	)
        self.assertFalse(_is_control("\t"	)	)
        self.assertFalse(_is_control("\r"	)	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       Optional[int]	)		->				Optional[int]:
        self.assertTrue(_is_punctuation("-"	)	)
        self.assertTrue(_is_punctuation("$"	)	)
        self.assertTrue(_is_punctuation("`"	)	)
        self.assertTrue(_is_punctuation("."	)	)

        self.assertFalse(_is_punctuation("A"	)	)
        self.assertFalse(_is_punctuation(" "	)	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       int	)		->				List[Any]:
        _UpperCamelCase      :	Optional[Any]						=       self.get_tokenizer()

        # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
        self.assertListEqual([tokenizer.tokenize(__a	) for t in ["Test", "\xad", "test"]]					,   [["[UNK]"], [], ["[UNK]"]]	)

        if self.test_rust_tokenizer:
              _UpperCamelCase      :	Tuple						=       self.get_rust_tokenizer()
              self.assertListEqual(
                  [rust_tokenizer.tokenize(__a	) for t in ["Test", "\xad", "test"]]					,   [["[UNK]"], [], ["[UNK]"]]	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       Tuple	)		->				Optional[Any]:
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
                    _UpperCamelCase      :	int						=       self.rust_tokenizer_class.from_pretrained(__a					,   **__a	)

                    _UpperCamelCase      :	Union[str, Any]						=       F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                    _UpperCamelCase      :	Optional[Any]						=       tokenizer_r.encode_plus(
                        __a					,   return_attention_mask=__a					,   return_token_type_ids=__a					,   return_offsets_mapping=__a					,   add_special_tokens=__a					,   )

                    _UpperCamelCase      :	List[Any]						=       tokenizer_r.do_lower_case if hasattr(__a					,   "do_lower_case"	) else False
                    _UpperCamelCase      :	Dict						=       (
                        [
                            ((0, 0), tokenizer_r.cls_token),
                            ((0, 1), "A"),
                            ((1, 2), ","),
                            ((3, 5), "na"),
                            ((5, 6), "##ï"),
                            ((6, 8), "##ve"),
                            ((9, 15), tokenizer_r.mask_token),
                            ((16, 21), "Allen"),
                            ((21, 23), "##NL"),
                            ((23, 24), "##P"),
                            ((25, 33), "sentence"),
                            ((33, 34), "."),
                            ((0, 0), tokenizer_r.sep_token),
                        ]
                        if not do_lower_case
                        else [
                            ((0, 0), tokenizer_r.cls_token),
                            ((0, 1), "a"),
                            ((1, 2), ","),
                            ((3, 8), "naive"),
                            ((9, 15), tokenizer_r.mask_token),
                            ((16, 21), "allen"),
                            ((21, 23), "##nl"),
                            ((23, 24), "##p"),
                            ((25, 33), "sentence"),
                            ((33, 34), "."),
                            ((0, 0), tokenizer_r.sep_token),
                        ]
                    )

                    self.assertEqual(
                        [e[1] for e in expected_results]					,   tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]	)	)
                    self.assertEqual([e[0] for e in expected_results]					,   tokens["offset_mapping"]	)

  def    __SCREAMING_SNAKE_CASE				(		self   :       int	)		->				List[Any]:
        _UpperCamelCase      :	Optional[Any]						=       ["的", "人", "有"]
        _UpperCamelCase      :	int						=       "".join(__a	)
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
                    _UpperCamelCase      :	int						=       True
                    _UpperCamelCase      :	Any						=       self.tokenizer_class.from_pretrained(__a					,   **__a	)
                    _UpperCamelCase      :	Optional[Any]						=       self.rust_tokenizer_class.from_pretrained(__a					,   **__a	)

                    _UpperCamelCase      :	int						=       tokenizer_p.encode(__a					,   add_special_tokens=__a	)
                    _UpperCamelCase      :	int						=       tokenizer_r.encode(__a					,   add_special_tokens=__a	)

                    _UpperCamelCase      :	List[Any]						=       tokenizer_r.convert_ids_to_tokens(__a	)
                    _UpperCamelCase      :	Union[str, Any]						=       tokenizer_p.convert_ids_to_tokens(__a	)

                    # it is expected that each Chinese character is not preceded by "##"
                    self.assertListEqual(__a					,   __a	)
                    self.assertListEqual(__a					,   __a	)

                    _UpperCamelCase      :	Any						=       False
                    _UpperCamelCase      :	Dict						=       self.rust_tokenizer_class.from_pretrained(__a					,   **__a	)
                    _UpperCamelCase      :	Any						=       self.tokenizer_class.from_pretrained(__a					,   **__a	)

                    _UpperCamelCase      :	Any						=       tokenizer_r.encode(__a					,   add_special_tokens=__a	)
                    _UpperCamelCase      :	Any						=       tokenizer_p.encode(__a					,   add_special_tokens=__a	)

                    _UpperCamelCase      :	Union[str, Any]						=       tokenizer_r.convert_ids_to_tokens(__a	)
                    _UpperCamelCase      :	Dict						=       tokenizer_p.convert_ids_to_tokens(__a	)

                    # it is expected that only the first Chinese character is not preceded by "##".
                    _UpperCamelCase      :	Any						=       [
                        F'''##{token}''' if idx != 0 else token for idx, token in enumerate(__a	)
                    ]
                    self.assertListEqual(__a					,   __a	)
                    self.assertListEqual(__a					,   __a	)

  @slow
  def    __SCREAMING_SNAKE_CASE				(		self   :       Union[str, Any]	)		->				Any:
        _UpperCamelCase      :	Dict						=       self.tokenizer_class(self.vocab_file					,   self.word_shape_file					,   self.word_pronunciation_file	)

        _UpperCamelCase      :	Optional[int]						=       tokenizer.encode("你好"					,   add_special_tokens=__a	)
        _UpperCamelCase      :	Dict						=       tokenizer.encode("你是谁"					,   add_special_tokens=__a	)

        _UpperCamelCase      :	Union[str, Any]						=       tokenizer.build_inputs_with_special_tokens(__a	)
        _UpperCamelCase      :	Tuple						=       tokenizer.build_inputs_with_special_tokens(__a					,   __a	)

        assert encoded_sentence == [1] + text + [2]
        assert encoded_pair == [1] + text + [2] + text_a + [2]


  def    __SCREAMING_SNAKE_CASE				(		self   :       Optional[Any]	)		->				int:
        _UpperCamelCase      :	Optional[Any]						=       self.get_tokenizers(do_lower_case=__a	)
        for tokenizer in tokenizers:
              with self.subTest(F'''{tokenizer.__class__.__name__}'''	):
                    _UpperCamelCase      :	int						=       "你好，你是谁"
                    _UpperCamelCase      :	Any						=       tokenizer.tokenize(__a	)
                    _UpperCamelCase      :	Optional[Any]						=       tokenizer.convert_tokens_to_ids(__a	)
                    _UpperCamelCase      :	List[str]						=       tokenizer.convert_tokens_to_shape_ids(__a	)
                    _UpperCamelCase      :	Any						=       tokenizer.convert_tokens_to_pronunciation_ids(__a	)
                    _UpperCamelCase      :	Optional[int]						=       tokenizer.prepare_for_model(
                        __a					,   __a					,   __a					,   add_special_tokens=__a	)

                    _UpperCamelCase      :	Tuple						=       tokenizer.encode_plus(__a					,   add_special_tokens=__a	)

                    self.assertEqual(__a					,   __a	)
