
'''simple docstring'''







import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 			lowerCAmelCase (				__UpperCAmelCase		,     unittest.TestCase			):
		a							:					Any       =					MobileBertTokenizer
		a							:					Any       =					MobileBertTokenizerFast
		a							:					int       =					True
		a							:					int       =					True
		a							:					Dict       =					filter_non_english
		a							:					Union[str, Any]       =					'''google/mobilebert-uncased'''




		def     lowercase       (				self      ):
								super().setUp()

								_SCREAMING_SNAKE_CASE				= [
								    """[UNK]""",
								    """[CLS]""",
								    """[SEP]""",
								    """[PAD]""",
								    """[MASK]""",
								    """want""",
								    """##want""",
								    """##ed""",
								    """wa""",
								    """un""",
								    """runn""",
								    """##ing""",
								    """,""",
								    """low""",
								    """lowest""",
								]
								_SCREAMING_SNAKE_CASE				= os.path.join(self.tmpdirname					,						VOCAB_FILES_NAMES["vocab_file"]      )
								with open(self.vocab_file					,						"w"					,						encoding="utf-8"      ) as vocab_writer:
														vocab_writer.write("".join([x + "\n" for x in vocab_tokens]      )      )

								_SCREAMING_SNAKE_CASE				= [
								    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
								    for tokenizer_def in self.tokenizers_list
								]




		def     lowercase       (				self					,						UpperCamelCase      ):
								_SCREAMING_SNAKE_CASE				= """UNwant\u00E9d,running"""
								_SCREAMING_SNAKE_CASE				= """unwanted, running"""
								return input_text, output_text




		def     lowercase       (				self      ):
								_SCREAMING_SNAKE_CASE				= self.tokenizer_class(self.vocab_file      )

								_SCREAMING_SNAKE_CASE				= tokenizer.tokenize("UNwant\u00E9d,running"      )
								self.assertListEqual(A__					,						["un", "##want", "##ed", ",", "runn", "##ing"]      )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(A__      )					,						[9, 6, 7, 12, 10, 11]      )




		def     lowercase       (				self      ):
								if not self.test_rust_tokenizer:
														return

								_SCREAMING_SNAKE_CASE				= self.get_tokenizer()
								_SCREAMING_SNAKE_CASE				= self.get_rust_tokenizer()

								_SCREAMING_SNAKE_CASE				= """UNwant\u00E9d,running"""

								_SCREAMING_SNAKE_CASE				= tokenizer.tokenize(A__      )
								_SCREAMING_SNAKE_CASE				= rust_tokenizer.tokenize(A__      )
								self.assertListEqual(A__					,						A__      )

								_SCREAMING_SNAKE_CASE				= tokenizer.encode(A__					,						add_special_tokens=A__      )
								_SCREAMING_SNAKE_CASE				= rust_tokenizer.encode(A__					,						add_special_tokens=A__      )
								self.assertListEqual(A__					,						A__      )

								_SCREAMING_SNAKE_CASE				= self.get_rust_tokenizer()
								_SCREAMING_SNAKE_CASE				= tokenizer.encode(A__      )
								_SCREAMING_SNAKE_CASE				= rust_tokenizer.encode(A__      )
								self.assertListEqual(A__					,						A__      )

								# With lower casing
								_SCREAMING_SNAKE_CASE				= self.get_tokenizer(do_lower_case=A__      )
								_SCREAMING_SNAKE_CASE				= self.get_rust_tokenizer(do_lower_case=A__      )

								_SCREAMING_SNAKE_CASE				= """UNwant\u00E9d,running"""

								_SCREAMING_SNAKE_CASE				= tokenizer.tokenize(A__      )
								_SCREAMING_SNAKE_CASE				= rust_tokenizer.tokenize(A__      )
								self.assertListEqual(A__					,						A__      )

								_SCREAMING_SNAKE_CASE				= tokenizer.encode(A__					,						add_special_tokens=A__      )
								_SCREAMING_SNAKE_CASE				= rust_tokenizer.encode(A__					,						add_special_tokens=A__      )
								self.assertListEqual(A__					,						A__      )

								_SCREAMING_SNAKE_CASE				= self.get_rust_tokenizer()
								_SCREAMING_SNAKE_CASE				= tokenizer.encode(A__      )
								_SCREAMING_SNAKE_CASE				= rust_tokenizer.encode(A__      )
								self.assertListEqual(A__					,						A__      )




		def     lowercase       (				self      ):
								_SCREAMING_SNAKE_CASE				= BasicTokenizer()

								self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"      )					,						["ah", "\u535A", "\u63A8", "zz"]      )




		def     lowercase       (				self      ):
								_SCREAMING_SNAKE_CASE				= BasicTokenizer(do_lower_case=A__      )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "      )					,						["hello", "!", "how", "are", "you", "?"]      )
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )					,						["hello"]      )




		def     lowercase       (				self      ):
								_SCREAMING_SNAKE_CASE				= BasicTokenizer(do_lower_case=A__					,						strip_accents=A__      )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )					,						["hällo", "!", "how", "are", "you", "?"]      )
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )					,						["h\u00E9llo"]      )




		def     lowercase       (				self      ):
								_SCREAMING_SNAKE_CASE				= BasicTokenizer(do_lower_case=A__					,						strip_accents=A__      )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )					,						["hallo", "!", "how", "are", "you", "?"]      )
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )					,						["hello"]      )




		def     lowercase       (				self      ):
								_SCREAMING_SNAKE_CASE				= BasicTokenizer(do_lower_case=A__      )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )					,						["hallo", "!", "how", "are", "you", "?"]      )
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )					,						["hello"]      )




		def     lowercase       (				self      ):
								_SCREAMING_SNAKE_CASE				= BasicTokenizer(do_lower_case=A__      )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "      )					,						["HeLLo", "!", "how", "Are", "yoU", "?"]      )




		def     lowercase       (				self      ):
								_SCREAMING_SNAKE_CASE				= BasicTokenizer(do_lower_case=A__					,						strip_accents=A__      )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )					,						["HäLLo", "!", "how", "Are", "yoU", "?"]      )




		def     lowercase       (				self      ):
								_SCREAMING_SNAKE_CASE				= BasicTokenizer(do_lower_case=A__					,						strip_accents=A__      )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )					,						["HaLLo", "!", "how", "Are", "yoU", "?"]      )




		def     lowercase       (				self      ):
								_SCREAMING_SNAKE_CASE				= BasicTokenizer(do_lower_case=A__					,						never_split=["[UNK]"]      )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"      )					,						["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]      )




		def     lowercase       (				self      ):
								_SCREAMING_SNAKE_CASE				= ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

								_SCREAMING_SNAKE_CASE				= {}
								for i, token in enumerate(A__      ):
														_SCREAMING_SNAKE_CASE				= i
								_SCREAMING_SNAKE_CASE				= WordpieceTokenizer(vocab=A__					,						unk_token="[UNK]"      )

								self.assertListEqual(tokenizer.tokenize(""      )					,						[]      )

								self.assertListEqual(tokenizer.tokenize("unwanted running"      )					,						["un", "##want", "##ed", "runn", "##ing"]      )

								self.assertListEqual(tokenizer.tokenize("unwantedX running"      )					,						["[UNK]", "runn", "##ing"]      )




		def     lowercase       (				self      ):
								self.assertTrue(_is_whitespace(" "      )      )
								self.assertTrue(_is_whitespace("\t"      )      )
								self.assertTrue(_is_whitespace("\r"      )      )
								self.assertTrue(_is_whitespace("\n"      )      )
								self.assertTrue(_is_whitespace("\u00A0"      )      )

								self.assertFalse(_is_whitespace("A"      )      )
								self.assertFalse(_is_whitespace("-"      )      )




		def     lowercase       (				self      ):
								self.assertTrue(_is_control("\u0005"      )      )

								self.assertFalse(_is_control("A"      )      )
								self.assertFalse(_is_control(" "      )      )
								self.assertFalse(_is_control("\t"      )      )
								self.assertFalse(_is_control("\r"      )      )




		def     lowercase       (				self      ):
								self.assertTrue(_is_punctuation("-"      )      )
								self.assertTrue(_is_punctuation("$"      )      )
								self.assertTrue(_is_punctuation("`"      )      )
								self.assertTrue(_is_punctuation("."      )      )

								self.assertFalse(_is_punctuation("A"      )      )
								self.assertFalse(_is_punctuation(" "      )      )




		def     lowercase       (				self      ):
								_SCREAMING_SNAKE_CASE				= self.get_tokenizer()
								_SCREAMING_SNAKE_CASE				= self.get_rust_tokenizer()

								# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
								self.assertListEqual([tokenizer.tokenize(A__      ) for t in ["Test", "\xad", "test"]]					,						[["[UNK]"], [], ["[UNK]"]]      )

								self.assertListEqual(
								    [rust_tokenizer.tokenize(A__      ) for t in ["Test", "\xad", "test"]]					,						[["[UNK]"], [], ["[UNK]"]]      )




		@slow
		def     lowercase       (				self      ):
								_SCREAMING_SNAKE_CASE				= self.tokenizer_class.from_pretrained("google/mobilebert-uncased"      )

								_SCREAMING_SNAKE_CASE				= tokenizer.encode("sequence builders"					,						add_special_tokens=A__      )
								_SCREAMING_SNAKE_CASE				= tokenizer.encode("multi-sequence build"					,						add_special_tokens=A__      )

								_SCREAMING_SNAKE_CASE				= tokenizer.build_inputs_with_special_tokens(A__      )
								_SCREAMING_SNAKE_CASE				= tokenizer.build_inputs_with_special_tokens(A__					,						A__      )

								assert encoded_sentence == [101] + text + [102]
								assert encoded_pair == [101] + text + [102] + text_a + [102]




		def     lowercase       (				self      ):
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
																				_SCREAMING_SNAKE_CASE				= self.rust_tokenizer_class.from_pretrained(A__					,						**A__      )

																				_SCREAMING_SNAKE_CASE				= F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
																				_SCREAMING_SNAKE_CASE				= tokenizer_r.encode_plus(
																				    A__					,						return_attention_mask=A__					,						return_token_type_ids=A__					,						return_offsets_mapping=A__					,						add_special_tokens=A__					,						)

																				_SCREAMING_SNAKE_CASE				= tokenizer_r.do_lower_case if hasattr(A__					,						"do_lower_case"      ) else False
																				_SCREAMING_SNAKE_CASE				= (
																				    [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), """A"""),
																				        ((1, 2), ""","""),
																				        ((3, 5), """na"""),
																				        ((5, 6), """##ï"""),
																				        ((6, 8), """##ve"""),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), """Allen"""),
																				        ((21, 23), """##NL"""),
																				        ((23, 24), """##P"""),
																				        ((25, 33), """sentence"""),
																				        ((33, 34), """."""),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				    if not do_lower_case
																				    else [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), """a"""),
																				        ((1, 2), ""","""),
																				        ((3, 8), """naive"""),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), """allen"""),
																				        ((21, 23), """##nl"""),
																				        ((23, 24), """##p"""),
																				        ((25, 33), """sentence"""),
																				        ((33, 34), """."""),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				)

																				self.assertEqual(
																				    [e[1] for e in expected_results]					,						tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]      )      )
																				self.assertEqual([e[0] for e in expected_results]					,						tokens["offset_mapping"]      )




		def     lowercase       (				self      ):
								_SCREAMING_SNAKE_CASE				= ["""的""", """人""", """有"""]
								_SCREAMING_SNAKE_CASE				= """""".join(A__      )
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
																				_SCREAMING_SNAKE_CASE				= True
																				_SCREAMING_SNAKE_CASE				= self.tokenizer_class.from_pretrained(A__					,						**A__      )
																				_SCREAMING_SNAKE_CASE				= self.rust_tokenizer_class.from_pretrained(A__					,						**A__      )

																				_SCREAMING_SNAKE_CASE				= tokenizer_p.encode(A__					,						add_special_tokens=A__      )
																				_SCREAMING_SNAKE_CASE				= tokenizer_r.encode(A__					,						add_special_tokens=A__      )

																				_SCREAMING_SNAKE_CASE				= tokenizer_r.convert_ids_to_tokens(A__      )
																				_SCREAMING_SNAKE_CASE				= tokenizer_p.convert_ids_to_tokens(A__      )

																				# it is expected that each Chinese character is not preceded by "##"
																				self.assertListEqual(A__					,						A__      )
																				self.assertListEqual(A__					,						A__      )

																				_SCREAMING_SNAKE_CASE				= False
																				_SCREAMING_SNAKE_CASE				= self.rust_tokenizer_class.from_pretrained(A__					,						**A__      )
																				_SCREAMING_SNAKE_CASE				= self.tokenizer_class.from_pretrained(A__					,						**A__      )

																				_SCREAMING_SNAKE_CASE				= tokenizer_r.encode(A__					,						add_special_tokens=A__      )
																				_SCREAMING_SNAKE_CASE				= tokenizer_p.encode(A__					,						add_special_tokens=A__      )

																				_SCREAMING_SNAKE_CASE				= tokenizer_r.convert_ids_to_tokens(A__      )
																				_SCREAMING_SNAKE_CASE				= tokenizer_p.convert_ids_to_tokens(A__      )

																				# it is expected that only the first Chinese character is not preceded by "##".
																				_SCREAMING_SNAKE_CASE				= [
																				    F'##{token}' if idx != 0 else token for idx, token in enumerate(A__      )
																				]
																				self.assertListEqual(A__					,						A__      )
																				self.assertListEqual(A__					,						A__      )

'''simple docstring'''







import os
import unittest

from transformers import LayoutLMTokenizer, LayoutLMTokenizerFast
from transformers.models.layoutlm.tokenization_layoutlm import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 			lowerCAmelCase (				__UpperCAmelCase		,     unittest.TestCase			):
		a							:					int       =					LayoutLMTokenizer
		a							:					Optional[int]       =					LayoutLMTokenizerFast
		a							:					Optional[int]       =					True
		a							:					Any       =					True




		def     lowercase       (				self      ):
								super().setUp()

								_SCREAMING_SNAKE_CASE				= [
								    "[UNK]",
								    "[CLS]",
								    "[SEP]",
								    "want",
								    "##want",
								    "##ed",
								    "wa",
								    "un",
								    "runn",
								    "##ing",
								    ",",
								    "low",
								    "lowest",
								]
								_SCREAMING_SNAKE_CASE				= os.path.join(self.tmpdirname					,						VOCAB_FILES_NAMES["vocab_file"]      )
								with open(self.vocab_file					,						"w"					,						encoding="utf-8"      ) as vocab_writer:
														vocab_writer.write("".join([x + "\n" for x in vocab_tokens]      )      )




		def     lowercase       (				self					,						**UpperCamelCase      ):
								return LayoutLMTokenizer.from_pretrained(self.tmpdirname					,						**UpperCamelCase      )




		def     lowercase       (				self					,						UpperCamelCase      ):
								_SCREAMING_SNAKE_CASE				= "UNwant\u00E9d,running"
								_SCREAMING_SNAKE_CASE				= "unwanted, running"
								return input_text, output_text




		def     lowercase       (				self      ):
								_SCREAMING_SNAKE_CASE				= self.tokenizer_class(self.vocab_file      )

								_SCREAMING_SNAKE_CASE				= tokenizer.tokenize("UNwant\u00E9d,running"      )
								self.assertListEqual(UpperCamelCase					,						["un", "##want", "##ed", ",", "runn", "##ing"]      )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase      )					,						[7, 4, 5, 10, 8, 9]      )




		def     lowercase       (				self      ):
								pass