





import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartaaTokenizer, MBartaaTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCamelCase   				=			get_tests_dir('fixtures/test_sentencepiece.model')

if is_torch_available():
	from transformers.models.mbart.modeling_mbart import shift_tokens_right

_lowerCamelCase   				=			250004
_lowerCamelCase   				=			250020




@require_sentencepiece
@require_tokenizers
class   UpperCamelCase_       (     UpperCamelCase__				,  unittest.TestCase    ):
							lowerCamelCase_           =					MBartaaTokenizer
							lowerCamelCase_           =					MBartaaTokenizerFast
							lowerCamelCase_           =					True
							lowerCamelCase_           =					True


							def      _snake_case  (							self    :str		)   ->   Any:



									"""simple docstring"""



									super().setUp()

									# We have a SentencePiece fixture for testing
									SCREAMING_SNAKE_CASE__			=      MBartaaTokenizer(__A							,				src_lang="""en_XX"""							,				tgt_lang="""ro_RO"""							,				keep_accents=__A		)
									tokenizer.save_pretrained(self.tmpdirname		)


							def      _snake_case  (							self    :Optional[int]		)   ->   Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      """<s>"""
									SCREAMING_SNAKE_CASE__			=      0

									self.assertEqual(self.get_tokenizer()._convert_token_to_id(__A		)							,				__A		)
									self.assertEqual(self.get_tokenizer()._convert_id_to_token(__A		)							,				__A		)


							def      _snake_case  (							self    :Union[str, Any]		)   ->   int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      list(self.get_tokenizer().get_vocab().keys()		)

									self.assertEqual(vocab_keys[0]							,				"""<s>"""		)
									self.assertEqual(vocab_keys[1]							,				"""<pad>"""		)
									self.assertEqual(vocab_keys[-1]							,				"""<mask>"""		)
									self.assertEqual(len(__A		)							,				1054		)


							def      _snake_case  (							self    :Optional[Any]		)   ->   int:



									"""simple docstring"""



									self.assertEqual(self.get_tokenizer().vocab_size							,				1054		)


							def      _snake_case  (							self    :int		)   ->   Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      MBartaaTokenizer(__A							,				src_lang="""en_XX"""							,				tgt_lang="""ro_RO"""							,				keep_accents=__A		)

									SCREAMING_SNAKE_CASE__			=      tokenizer.tokenize("""This is a test"""		)
									self.assertListEqual(__A							,				["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]		)

									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(__A		)							,				[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]							,				)

									SCREAMING_SNAKE_CASE__			=      tokenizer.tokenize("""I was born in 92000, and this is falsé."""		)
									self.assertListEqual(
									    __A							,				[SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """9""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """é""", """."""]							,				)
									SCREAMING_SNAKE_CASE__			=      tokenizer.convert_tokens_to_ids(__A		)
									self.assertListEqual(
									    __A							,				[
									        value + tokenizer.fairseq_offset
									        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
									    ]							,				)

									SCREAMING_SNAKE_CASE__			=      tokenizer.convert_ids_to_tokens(__A		)
									self.assertListEqual(
									    __A							,				[SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """<unk>""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """<unk>""", """."""]							,				)


							@slow
							def      _snake_case  (							self    :Union[str, Any]		)   ->   List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      {"""input_ids""": [[25_0004, 1_1062, 8_2772, 7, 15, 8_2772, 538, 5_1529, 237, 1_7198, 1290, 206, 9, 21_5175, 1314, 136, 1_7198, 1290, 206, 9, 5_6359, 42, 12_2009, 9, 1_6466, 16, 8_7344, 4537, 9, 4717, 7_8381, 6, 15_9958, 7, 15, 2_4480, 618, 4, 527, 2_2693, 5428, 4, 2777, 2_4480, 9874, 4, 4_3523, 594, 4, 803, 1_8392, 3_3189, 18, 4, 4_3523, 2_4447, 1_2399, 100, 2_4955, 8_3658, 9626, 14_4057, 15, 839, 2_2335, 16, 136, 2_4955, 8_3658, 8_3479, 15, 3_9102, 724, 16, 678, 645, 2789, 1328, 4589, 42, 12_2009, 11_5774, 23, 805, 1328, 4_6876, 7, 136, 5_3894, 1940, 4_2227, 4_1159, 1_7721, 823, 425, 4, 2_7512, 9_8722, 206, 136, 5531, 4970, 919, 1_7336, 5, 2], [25_0004, 2_0080, 618, 83, 8_2775, 47, 479, 9, 1517, 73, 5_3894, 333, 8_0581, 11_0117, 1_8811, 5256, 1295, 51, 15_2526, 297, 7986, 390, 12_4416, 538, 3_5431, 214, 98, 1_5044, 2_5737, 136, 7108, 4_3701, 23, 756, 13_5355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [25_0004, 581, 6_3773, 11_9455, 6, 14_7797, 8_8203, 7, 645, 70, 21, 3285, 1_0269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
									# fmt: on

									self.tokenizer_integration_test_util(
									    expected_encoding=__A							,				model_name="""facebook/mbart-large-50"""							,				revision="""d3913889c59cd5c9e456b269c376325eabad57e2"""							,				)


							def      _snake_case  (							self    :Optional[int]		)   ->   int:



									"""simple docstring"""



									if not self.test_slow_tokenizer:
											# as we don't have a slow version, we can't compare the outputs between slow and fast versions
											return

									SCREAMING_SNAKE_CASE__			=      (self.rust_tokenizer_class, """hf-internal-testing/tiny-random-mbart50""", {})
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
													SCREAMING_SNAKE_CASE__			=      self.rust_tokenizer_class.from_pretrained(__A							,				**__A		)
													SCREAMING_SNAKE_CASE__			=      self.tokenizer_class.from_pretrained(__A							,				**__A		)

													SCREAMING_SNAKE_CASE__			=      tempfile.mkdtemp()

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.save_pretrained(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_p.save_pretrained(__A		)

													# Checks it save with the same files + the tokenizer.json file for the fast one
													self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files		)		)
													SCREAMING_SNAKE_CASE__			=      tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f		)
													self.assertSequenceEqual(__A							,				__A		)

													# Checks everything loads correctly in the same way
													SCREAMING_SNAKE_CASE__			=      tokenizer_r.from_pretrained(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_p.from_pretrained(__A		)

													# Check special tokens are set accordingly on Rust and Python
													for key in tokenizer_pp.special_tokens_map:
															self.assertTrue(hasattr(__A							,				__A		)		)
															# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
															# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

													shutil.rmtree(__A		)

													# Save tokenizer rust, legacy_format=True
													SCREAMING_SNAKE_CASE__			=      tempfile.mkdtemp()

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.save_pretrained(__A							,				legacy_format=__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_p.save_pretrained(__A		)

													# Checks it save with the same files
													self.assertSequenceEqual(__A							,				__A		)

													# Checks everything loads correctly in the same way
													SCREAMING_SNAKE_CASE__			=      tokenizer_r.from_pretrained(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_p.from_pretrained(__A		)

													# Check special tokens are set accordingly on Rust and Python
													for key in tokenizer_pp.special_tokens_map:
															self.assertTrue(hasattr(__A							,				__A		)		)

													shutil.rmtree(__A		)

													# Save tokenizer rust, legacy_format=False
													SCREAMING_SNAKE_CASE__			=      tempfile.mkdtemp()

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.save_pretrained(__A							,				legacy_format=__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_p.save_pretrained(__A		)

													# Checks it saved the tokenizer.json file
													self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files		)		)

													# Checks everything loads correctly in the same way
													SCREAMING_SNAKE_CASE__			=      tokenizer_r.from_pretrained(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_p.from_pretrained(__A		)

													# Check special tokens are set accordingly on Rust and Python
													for key in tokenizer_pp.special_tokens_map:
															self.assertTrue(hasattr(__A							,				__A		)		)

													shutil.rmtree(__A		)




@require_torch
@require_sentencepiece
@require_tokenizers
class   UpperCamelCase_       (     unittest.TestCase    ):
							lowerCamelCase_           =					"facebook/mbart-large-50-one-to-many-mmt"
							lowerCamelCase_           =					[
							    " UN Chief Says There Is No Military Solution in Syria",
							    " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
							]
							lowerCamelCase_           =					[
							    "Şeful ONU declară că nu există o soluţie militară în Siria",
							    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
							    " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
							    " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
							]
							lowerCamelCase_           =					[EN_CODE, 82_74, 12_78_73, 2_59_16, 7, 86_22, 20_71, 4_38, 6_74_85, 53, 18_78_95, 23, 5_17_12, 2]


							@classmethod
							def      _snake_case  (							cls    :Tuple		)   ->   Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      MBartaaTokenizer.from_pretrained(
									    cls.checkpoint_name							,				src_lang="""en_XX"""							,				tgt_lang="""ro_RO"""		)
									SCREAMING_SNAKE_CASE__			=      1
									return cls


							def      _snake_case  (							self    :Any		)   ->   str:



									"""simple docstring"""



									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ar_AR"""]							,				25_0001		)
									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""en_EN"""]							,				25_0004		)
									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ro_RO"""]							,				25_0020		)
									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""mr_IN"""]							,				25_0038		)


							def      _snake_case  (							self    :Dict		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.tokenizer.batch_encode_plus(self.src_text		).input_ids[0]
									self.assertListEqual(self.expected_src_tokens							,				__A		)


							def      _snake_case  (							self    :int		)   ->   Tuple:



									"""simple docstring"""



									self.assertIn(__A							,				self.tokenizer.all_special_ids		)
									SCREAMING_SNAKE_CASE__			=      [RO_CODE, 884, 9019, 96, 9, 916, 8_6792, 36, 1_8743, 1_5596, 5, 2]
									SCREAMING_SNAKE_CASE__			=      self.tokenizer.decode(__A							,				skip_special_tokens=__A		)
									SCREAMING_SNAKE_CASE__			=      self.tokenizer.decode(generated_ids[1:]							,				skip_special_tokens=__A		)
									self.assertEqual(__A							,				__A		)
									self.assertNotIn(self.tokenizer.eos_token							,				__A		)


							def      _snake_case  (							self    :Tuple		)   ->   Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      ["""this is gunna be a long sentence """ * 20]
									assert isinstance(src_text[0]							,				__A		)
									SCREAMING_SNAKE_CASE__			=      10
									SCREAMING_SNAKE_CASE__			=      self.tokenizer(__A							,				max_length=__A							,				truncation=__A		).input_ids[0]
									self.assertEqual(ids[0]							,				__A		)
									self.assertEqual(ids[-1]							,				2		)
									self.assertEqual(len(__A		)							,				__A		)


							def      _snake_case  (							self    :List[Any]		)   ->   List[str]:



									"""simple docstring"""



									self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]		)							,				[25_0053, 25_0001]		)


							def      _snake_case  (							self    :Optional[Any]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      tempfile.mkdtemp()
									SCREAMING_SNAKE_CASE__			=      self.tokenizer.fairseq_tokens_to_ids
									self.tokenizer.save_pretrained(__A		)
									SCREAMING_SNAKE_CASE__			=      MBartaaTokenizer.from_pretrained(__A		)
									self.assertDictEqual(new_tok.fairseq_tokens_to_ids							,				__A		)


							@require_torch
							def      _snake_case  (							self    :str		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.tokenizer(self.src_text							,				text_target=self.tgt_text							,				padding=__A							,				return_tensors="""pt"""		)
									SCREAMING_SNAKE_CASE__			=      shift_tokens_right(batch["""labels"""]							,				self.tokenizer.pad_token_id		)

									# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
									assert batch.input_ids[1][0] == EN_CODE
									assert batch.input_ids[1][-1] == 2
									assert batch.labels[1][0] == RO_CODE
									assert batch.labels[1][-1] == 2
									assert batch.decoder_input_ids[1][:2].tolist() == [2, RO_CODE]


							@require_torch
							def      _snake_case  (							self    :Optional[int]		)   ->   Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.tokenizer(
									    self.src_text							,				text_target=self.tgt_text							,				padding=__A							,				truncation=__A							,				max_length=len(self.expected_src_tokens		)							,				return_tensors="""pt"""							,				)
									SCREAMING_SNAKE_CASE__			=      shift_tokens_right(batch["""labels"""]							,				self.tokenizer.pad_token_id		)

									self.assertIsInstance(__A							,				__A		)

									self.assertEqual((2, 14)							,				batch.input_ids.shape		)
									self.assertEqual((2, 14)							,				batch.attention_mask.shape		)
									SCREAMING_SNAKE_CASE__			=      batch.input_ids.tolist()[0]
									self.assertListEqual(self.expected_src_tokens							,				__A		)
									self.assertEqual(2							,				batch.decoder_input_ids[0, 0]		)  # decoder_start_token_id
									# Test that special tokens are reset
									self.assertEqual(self.tokenizer.prefix_tokens							,				[EN_CODE]		)
									self.assertEqual(self.tokenizer.suffix_tokens							,				[self.tokenizer.eos_token_id]		)


							def      _snake_case  (							self    :Dict		)   ->   Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.tokenizer(self.src_text							,				padding=__A							,				truncation=__A							,				max_length=3							,				return_tensors="""pt"""		)
									SCREAMING_SNAKE_CASE__			=      self.tokenizer(
									    text_target=self.tgt_text							,				padding=__A							,				truncation=__A							,				max_length=10							,				return_tensors="""pt"""		)
									SCREAMING_SNAKE_CASE__			=      targets["""input_ids"""]
									SCREAMING_SNAKE_CASE__			=      shift_tokens_right(__A							,				self.tokenizer.pad_token_id		)

									self.assertEqual(batch.input_ids.shape[1]							,				3		)
									self.assertEqual(batch.decoder_input_ids.shape[1]							,				10		)


							@require_torch
							def      _snake_case  (							self    :Dict		)   ->   int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.tokenizer._build_translation_inputs(
									    """A test"""							,				return_tensors="""pt"""							,				src_lang="""en_XX"""							,				tgt_lang="""ar_AR"""		)

									self.assertEqual(
									    nested_simplify(__A		)							,				{
									        # en_XX, A, test, EOS
									        """input_ids""": [[25_0004, 62, 3034, 2]],
									        """attention_mask""": [[1, 1, 1, 1]],
									        # ar_AR
									        """forced_bos_token_id""": 25_0001,
									    }							,				)






import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class   UpperCamelCase_       (     UpperCamelCase__				,  unittest.TestCase    ):
							lowerCamelCase_           =					RoCBertTokenizer
							lowerCamelCase_           =					None
							lowerCamelCase_           =					False
							lowerCamelCase_           =					True
							lowerCamelCase_           =					filter_non_english


							def      _snake_case  (							self    :List[Any]		)   ->   List[Any]:



									"""simple docstring"""



									super().setUp()

									SCREAMING_SNAKE_CASE__			=      ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """你""", """好""", """是""", """谁""", """a""", """b""", """c""", """d"""]
									SCREAMING_SNAKE_CASE__			=      {}
									SCREAMING_SNAKE_CASE__			=      {}
									for i, value in enumerate(__A		):
											SCREAMING_SNAKE_CASE__			=      i
											SCREAMING_SNAKE_CASE__			=      i
									SCREAMING_SNAKE_CASE__			=      os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""vocab_file"""]		)
									SCREAMING_SNAKE_CASE__			=      os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""word_shape_file"""]		)
									SCREAMING_SNAKE_CASE__			=      os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""word_pronunciation_file"""]		)
									with open(self.vocab_file							,				"""w"""							,				encoding="""utf-8"""		) as vocab_writer:
											vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]		)		)
									with open(self.word_shape_file							,				"""w"""							,				encoding="""utf-8"""		) as word_shape_writer:
											json.dump(__A							,				__A							,				ensure_ascii=__A		)
									with open(self.word_pronunciation_file							,				"""w"""							,				encoding="""utf-8"""		) as word_pronunciation_writer:
											json.dump(__A							,				__A							,				ensure_ascii=__A		)


							def      _snake_case  (							self    :List[Any]		)   ->   Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.tokenizer_class(self.vocab_file							,				self.word_shape_file							,				self.word_pronunciation_file		)

									SCREAMING_SNAKE_CASE__			=      tokenizer.tokenize("""你好[SEP]你是谁"""		)
									self.assertListEqual(__A							,				["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""]		)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(__A		)							,				[5, 6, 2, 5, 7, 8]		)
									self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(__A		)							,				[5, 6, 2, 5, 7, 8]		)
									self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(__A		)							,				[5, 6, 2, 5, 7, 8]		)


							def      _snake_case  (							self    :List[Any]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""		)							,				["""ah""", """\u535A""", """\u63A8""", """zz"""]		)


							def      _snake_case  (							self    :List[str]		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """		)							,				["""hello""", """!""", """how""", """are""", """you""", """?"""]		)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		)							,				["""hello"""]		)


							def      _snake_case  (							self    :str		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				strip_accents=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""hällo""", """!""", """how""", """are""", """you""", """?"""]		)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		)							,				["""h\u00E9llo"""]		)


							def      _snake_case  (							self    :Any		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				strip_accents=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]		)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		)							,				["""hello"""]		)


							def      _snake_case  (							self    :List[str]		)   ->   int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]		)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		)							,				["""hello"""]		)


							def      _snake_case  (							self    :Union[str, Any]		)   ->   Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """		)							,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]		)


							def      _snake_case  (							self    :int		)   ->   Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				strip_accents=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]		)


							def      _snake_case  (							self    :Union[str, Any]		)   ->   List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				strip_accents=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]		)


							def      _snake_case  (							self    :List[Any]		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				never_split=["""[UNK]"""]		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""		)							,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]		)


							def      _snake_case  (							self    :Any		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

									SCREAMING_SNAKE_CASE__			=      {}
									for i, token in enumerate(__A		):
											SCREAMING_SNAKE_CASE__			=      i
									SCREAMING_SNAKE_CASE__			=      RoCBertWordpieceTokenizer(vocab=__A							,				unk_token="""[UNK]"""		)

									self.assertListEqual(tokenizer.tokenize(""""""		)							,				[]		)

									self.assertListEqual(tokenizer.tokenize("""unwanted running"""		)							,				["""un""", """##want""", """##ed""", """runn""", """##ing"""]		)

									self.assertListEqual(tokenizer.tokenize("""unwantedX running"""		)							,				["""[UNK]""", """runn""", """##ing"""]		)


							def      _snake_case  (							self    :Any		)   ->   str:



									"""simple docstring"""



									self.assertTrue(_is_whitespace(""" """		)		)
									self.assertTrue(_is_whitespace("""\t"""		)		)
									self.assertTrue(_is_whitespace("""\r"""		)		)
									self.assertTrue(_is_whitespace("""\n"""		)		)
									self.assertTrue(_is_whitespace("""\u00A0"""		)		)

									self.assertFalse(_is_whitespace("""A"""		)		)
									self.assertFalse(_is_whitespace("""-"""		)		)


							def      _snake_case  (							self    :int		)   ->   str:



									"""simple docstring"""



									self.assertTrue(_is_control("""\u0005"""		)		)

									self.assertFalse(_is_control("""A"""		)		)
									self.assertFalse(_is_control(""" """		)		)
									self.assertFalse(_is_control("""\t"""		)		)
									self.assertFalse(_is_control("""\r"""		)		)


							def      _snake_case  (							self    :List[str]		)   ->   List[str]:



									"""simple docstring"""



									self.assertTrue(_is_punctuation("""-"""		)		)
									self.assertTrue(_is_punctuation("""$"""		)		)
									self.assertTrue(_is_punctuation("""`"""		)		)
									self.assertTrue(_is_punctuation("""."""		)		)

									self.assertFalse(_is_punctuation("""A"""		)		)
									self.assertFalse(_is_punctuation(""" """		)		)


							def      _snake_case  (							self    :str		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.get_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(__A		) for t in ["""Test""", """\xad""", """test"""]]							,				[["""[UNK]"""], [], ["""[UNK]"""]]		)

									if self.test_rust_tokenizer:
											SCREAMING_SNAKE_CASE__			=      self.get_rust_tokenizer()
											self.assertListEqual(
											    [rust_tokenizer.tokenize(__A		) for t in ["""Test""", """\xad""", """test"""]]							,				[["""[UNK]"""], [], ["""[UNK]"""]]		)


							def      _snake_case  (							self    :int		)   ->   Any:



									"""simple docstring"""



									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
													SCREAMING_SNAKE_CASE__			=      self.rust_tokenizer_class.from_pretrained(__A							,				**__A		)

													SCREAMING_SNAKE_CASE__			=      f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
													SCREAMING_SNAKE_CASE__			=      tokenizer_r.encode_plus(
													    __A							,				return_attention_mask=__A							,				return_token_type_ids=__A							,				return_offsets_mapping=__A							,				add_special_tokens=__A							,				)

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.do_lower_case if hasattr(__A							,				"""do_lower_case"""		) else False
													SCREAMING_SNAKE_CASE__			=      (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), """A"""),
													        ((1, 2), ""","""),
													        ((3, 5), """na"""),
													        ((5, 6), """##ï"""),
													        ((6, 8), """##ve"""),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), """Allen"""),
													        ((21, 23), """##NL"""),
													        ((23, 24), """##P"""),
													        ((25, 33), """sentence"""),
													        ((33, 34), """."""),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), """a"""),
													        ((1, 2), ""","""),
													        ((3, 8), """naive"""),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), """allen"""),
													        ((21, 23), """##nl"""),
													        ((23, 24), """##p"""),
													        ((25, 33), """sentence"""),
													        ((33, 34), """."""),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]							,				tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]		)		)
													self.assertEqual([e[0] for e in expected_results]							,				tokens["""offset_mapping"""]		)


							def      _snake_case  (							self    :Optional[int]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      ["""的""", """人""", """有"""]
									SCREAMING_SNAKE_CASE__			=      """""".join(__A		)
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
													SCREAMING_SNAKE_CASE__			=      True
													SCREAMING_SNAKE_CASE__			=      self.tokenizer_class.from_pretrained(__A							,				**__A		)
													SCREAMING_SNAKE_CASE__			=      self.rust_tokenizer_class.from_pretrained(__A							,				**__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer_p.encode(__A							,				add_special_tokens=__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_r.encode(__A							,				add_special_tokens=__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.convert_ids_to_tokens(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_p.convert_ids_to_tokens(__A		)

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(__A							,				__A		)
													self.assertListEqual(__A							,				__A		)

													SCREAMING_SNAKE_CASE__			=      False
													SCREAMING_SNAKE_CASE__			=      self.rust_tokenizer_class.from_pretrained(__A							,				**__A		)
													SCREAMING_SNAKE_CASE__			=      self.tokenizer_class.from_pretrained(__A							,				**__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.encode(__A							,				add_special_tokens=__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_p.encode(__A							,				add_special_tokens=__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.convert_ids_to_tokens(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_p.convert_ids_to_tokens(__A		)

													# it is expected that only the first Chinese character is not preceded by "##".
													SCREAMING_SNAKE_CASE__			=      [
													    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(__A		)
													]
													self.assertListEqual(__A							,				__A		)
													self.assertListEqual(__A							,				__A		)


							@slow
							def      _snake_case  (							self    :Union[str, Any]		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.tokenizer_class(self.vocab_file							,				self.word_shape_file							,				self.word_pronunciation_file		)

									SCREAMING_SNAKE_CASE__			=      tokenizer.encode("""你好"""							,				add_special_tokens=__A		)
									SCREAMING_SNAKE_CASE__			=      tokenizer.encode("""你是谁"""							,				add_special_tokens=__A		)

									SCREAMING_SNAKE_CASE__			=      tokenizer.build_inputs_with_special_tokens(__A		)
									SCREAMING_SNAKE_CASE__			=      tokenizer.build_inputs_with_special_tokens(__A							,				__A		)

									assert encoded_sentence == [1] + text + [2]
									assert encoded_pair == [1] + text + [2] + text_a + [2]


							def      _snake_case  (							self    :List[str]		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.get_tokenizers(do_lower_case=__A		)
									for tokenizer in tokenizers:
											with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
													SCREAMING_SNAKE_CASE__			=      """你好，你是谁"""
													SCREAMING_SNAKE_CASE__			=      tokenizer.tokenize(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer.convert_tokens_to_ids(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer.convert_tokens_to_shape_ids(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer.convert_tokens_to_pronunciation_ids(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer.prepare_for_model(
													    __A							,				__A							,				__A							,				add_special_tokens=__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer.encode_plus(__A							,				add_special_tokens=__A		)

													self.assertEqual(__A							,				__A		)