




import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class    __lowerCAmelCase       (							lowerCAmelCase      , unittest.TestCase):
       _a 					=						BertTokenizer
       _a 					=						BertTokenizerFast
       _a 					=						True
       _a 					=						True
       _a 					=						filter_non_english






       def 					SCREAMING_SNAKE_CASE    (   self:					Union[str, Any]				):
         super().setUp()

         lowercase    :Dict      	=						[
             "[UNK]",
             "[CLS]",
             "[SEP]",
             "[PAD]",
             "[MASK]",
             "want",
             "##want",
             "##ed",
             "wa",
             "un",
             "runn",
             "##ing",
             ",",
             "low",
             "lowest",
         ]
         lowercase    :List[str]      	=						os.path.join(self.tmpdirname       ,					VOCAB_FILES_NAMES["vocab_file"]				)
         with open(self.vocab_file       ,					"w"       ,					encoding="utf-8"				) as vocab_writer:
           vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)






       def 					SCREAMING_SNAKE_CASE    (   self:					str       ,					_lowerCAmelCase:					int				):
         lowercase    :Any      	=						"UNwant\u00E9d,running"
         lowercase    :Any      	=						"unwanted, running"
         return input_text, output_text






       def 					SCREAMING_SNAKE_CASE    (   self:					Union[str, Any]				):
         lowercase    :Tuple      	=						self.tokenizer_class(self.vocab_file				)

         lowercase    :List[str]      	=						tokenizer.tokenize("UNwant\u00E9d,running"				)
         self.assertListEqual(_lowerCAmelCase       ,					["un", "##want", "##ed", ",", "runn", "##ing"]				)
         self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase				)       ,					[9, 6, 7, 12, 10, 11]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					int				):
         if not self.test_rust_tokenizer:
           return

         lowercase    :Tuple      	=						self.get_tokenizer()
         lowercase    :List[Any]      	=						self.get_rust_tokenizer()

         lowercase    :Tuple      	=						"UNwant\u00E9d,running"

         lowercase    :str      	=						tokenizer.tokenize(_lowerCAmelCase				)
         lowercase    :List[str]      	=						rust_tokenizer.tokenize(_lowerCAmelCase				)
         self.assertListEqual(_lowerCAmelCase       ,					_lowerCAmelCase				)

         lowercase    :List[Any]      	=						tokenizer.encode(_lowerCAmelCase       ,					add_special_tokens=_lowerCAmelCase				)
         lowercase    :List[Any]      	=						rust_tokenizer.encode(_lowerCAmelCase       ,					add_special_tokens=_lowerCAmelCase				)
         self.assertListEqual(_lowerCAmelCase       ,					_lowerCAmelCase				)

         lowercase    :Union[str, Any]      	=						self.get_rust_tokenizer()
         lowercase    :Optional[int]      	=						tokenizer.encode(_lowerCAmelCase				)
         lowercase    :Tuple      	=						rust_tokenizer.encode(_lowerCAmelCase				)
         self.assertListEqual(_lowerCAmelCase       ,					_lowerCAmelCase				)

         # With lower casing
         lowercase    :Any      	=						self.get_tokenizer(do_lower_case=_lowerCAmelCase				)
         lowercase    :List[Any]      	=						self.get_rust_tokenizer(do_lower_case=_lowerCAmelCase				)

         lowercase    :Union[str, Any]      	=						"UNwant\u00E9d,running"

         lowercase    :str      	=						tokenizer.tokenize(_lowerCAmelCase				)
         lowercase    :List[str]      	=						rust_tokenizer.tokenize(_lowerCAmelCase				)
         self.assertListEqual(_lowerCAmelCase       ,					_lowerCAmelCase				)

         lowercase    :Tuple      	=						tokenizer.encode(_lowerCAmelCase       ,					add_special_tokens=_lowerCAmelCase				)
         lowercase    :List[Any]      	=						rust_tokenizer.encode(_lowerCAmelCase       ,					add_special_tokens=_lowerCAmelCase				)
         self.assertListEqual(_lowerCAmelCase       ,					_lowerCAmelCase				)

         lowercase    :List[str]      	=						self.get_rust_tokenizer()
         lowercase    :List[str]      	=						tokenizer.encode(_lowerCAmelCase				)
         lowercase    :Tuple      	=						rust_tokenizer.encode(_lowerCAmelCase				)
         self.assertListEqual(_lowerCAmelCase       ,					_lowerCAmelCase				)






       def 					SCREAMING_SNAKE_CASE    (   self:					int				):
         lowercase    :List[Any]      	=						BasicTokenizer()

         self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"				)       ,					["ah", "\u535A", "\u63A8", "zz"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Tuple				):
         lowercase    :Optional[int]      	=						BasicTokenizer(do_lower_case=_lowerCAmelCase				)

         self.assertListEqual(
             tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)       ,					["hello", "!", "how", "are", "you", "?"]				)
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)       ,					["hello"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[Any]				):
         lowercase    :Union[str, Any]      	=						BasicTokenizer(do_lower_case=_lowerCAmelCase       ,					strip_accents=_lowerCAmelCase				)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)       ,					["hällo", "!", "how", "are", "you", "?"]				)
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)       ,					["h\u00E9llo"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					List[str]				):
         lowercase    :List[Any]      	=						BasicTokenizer(do_lower_case=_lowerCAmelCase       ,					strip_accents=_lowerCAmelCase				)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)       ,					["hallo", "!", "how", "are", "you", "?"]				)
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)       ,					["hello"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Union[str, Any]				):
         lowercase    :Optional[int]      	=						BasicTokenizer(do_lower_case=_lowerCAmelCase				)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)       ,					["hallo", "!", "how", "are", "you", "?"]				)
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)       ,					["hello"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Union[str, Any]				):
         lowercase    :List[Any]      	=						BasicTokenizer(do_lower_case=_lowerCAmelCase				)

         self.assertListEqual(
             tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)       ,					["HeLLo", "!", "how", "Are", "yoU", "?"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					str				):
         lowercase    :List[Any]      	=						BasicTokenizer(do_lower_case=_lowerCAmelCase       ,					strip_accents=_lowerCAmelCase				)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)       ,					["HäLLo", "!", "how", "Are", "yoU", "?"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[Any]				):
         lowercase    :Optional[Any]      	=						BasicTokenizer(do_lower_case=_lowerCAmelCase       ,					strip_accents=_lowerCAmelCase				)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)       ,					["HaLLo", "!", "how", "Are", "yoU", "?"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					List[str]				):
         lowercase    :Optional[Any]      	=						BasicTokenizer(do_lower_case=_lowerCAmelCase       ,					never_split=["[UNK]"]				)

         self.assertListEqual(
             tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"				)       ,					["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Any				):
         lowercase    :Optional[Any]      	=						BasicTokenizer()
         lowercase    :Tuple      	=						"a\n'll !!to?'d of, can't."
         lowercase    :Tuple      	=						["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
         self.assertListEqual(tokenizer.tokenize(_lowerCAmelCase				)       ,					_lowerCAmelCase				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Any				):
         lowercase    :Any      	=						["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

         lowercase    :Dict      	=						{}
         for i, token in enumerate(_lowerCAmelCase				):
           lowercase    :Any      	=						i
         lowercase    :List[str]      	=						WordpieceTokenizer(vocab=_lowerCAmelCase       ,					unk_token="[UNK]"				)

         self.assertListEqual(tokenizer.tokenize(""				)       ,					[]				)

         self.assertListEqual(tokenizer.tokenize("unwanted running"				)       ,					["un", "##want", "##ed", "runn", "##ing"]				)

         self.assertListEqual(tokenizer.tokenize("unwantedX running"				)       ,					["[UNK]", "runn", "##ing"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Tuple				):
         self.assertTrue(_is_whitespace(" "				)				)
         self.assertTrue(_is_whitespace("\t"				)				)
         self.assertTrue(_is_whitespace("\r"				)				)
         self.assertTrue(_is_whitespace("\n"				)				)
         self.assertTrue(_is_whitespace("\u00A0"				)				)

         self.assertFalse(_is_whitespace("A"				)				)
         self.assertFalse(_is_whitespace("-"				)				)






       def 					SCREAMING_SNAKE_CASE    (   self:					int				):
         self.assertTrue(_is_control("\u0005"				)				)

         self.assertFalse(_is_control("A"				)				)
         self.assertFalse(_is_control(" "				)				)
         self.assertFalse(_is_control("\t"				)				)
         self.assertFalse(_is_control("\r"				)				)






       def 					SCREAMING_SNAKE_CASE    (   self:					List[Any]				):
         self.assertTrue(_is_punctuation("-"				)				)
         self.assertTrue(_is_punctuation("$"				)				)
         self.assertTrue(_is_punctuation("`"				)				)
         self.assertTrue(_is_punctuation("."				)				)

         self.assertFalse(_is_punctuation("A"				)				)
         self.assertFalse(_is_punctuation(" "				)				)






       def 					SCREAMING_SNAKE_CASE    (   self:					List[Any]				):
         lowercase    :str      	=						self.get_tokenizer()
         lowercase    :Dict      	=						self.get_rust_tokenizer()

         # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
         self.assertListEqual([tokenizer.tokenize(_lowerCAmelCase				) for t in ["Test", "\xad", "test"]]       ,					[["[UNK]"], [], ["[UNK]"]]				)

         self.assertListEqual(
             [rust_tokenizer.tokenize(_lowerCAmelCase				) for t in ["Test", "\xad", "test"]]       ,					[["[UNK]"], [], ["[UNK]"]]				)






       @slow
       def 					SCREAMING_SNAKE_CASE    (   self:					str				):
         lowercase    :List[Any]      	=						self.tokenizer_class.from_pretrained("bert-base-uncased"				)

         lowercase    :List[Any]      	=						tokenizer.encode("sequence builders"       ,					add_special_tokens=_lowerCAmelCase				)
         lowercase    :Union[str, Any]      	=						tokenizer.encode("multi-sequence build"       ,					add_special_tokens=_lowerCAmelCase				)

         lowercase    :List[Any]      	=						tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase				)
         lowercase    :List[Any]      	=						tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase       ,					_lowerCAmelCase				)

         assert encoded_sentence == [1_01] + text + [1_02]
         assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]






       def 					SCREAMING_SNAKE_CASE    (   self:					str				):
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"				):
             lowercase    :Union[str, Any]      	=						self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase       ,					**_lowerCAmelCase				)

             lowercase    :str      	=						F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
             lowercase    :str      	=						tokenizer_r.encode_plus(
                 _lowerCAmelCase       ,					return_attention_mask=_lowerCAmelCase       ,					return_token_type_ids=_lowerCAmelCase       ,					return_offsets_mapping=_lowerCAmelCase       ,					add_special_tokens=_lowerCAmelCase       ,					)

             lowercase    :Any      	=						tokenizer_r.do_lower_case if hasattr(_lowerCAmelCase       ,					"do_lower_case"				) else False
             lowercase    :Union[str, Any]      	=						(
                 [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "A"),
                     ((1, 2), ","),
                     ((3, 5), "na"),
                     ((5, 6), "##ï"),
                     ((6, 8), "##ve"),
                     ((9, 15), tokenizer_r.mask_token),
                     ((16, 21), "Allen"),
                     ((21, 23), "##NL"),
                     ((23, 24), "##P"),
                     ((25, 33), "sentence"),
                     ((33, 34), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
                 if not do_lower_case
                 else [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "a"),
                     ((1, 2), ","),
                     ((3, 8), "naive"),
                     ((9, 15), tokenizer_r.mask_token),
                     ((16, 21), "allen"),
                     ((21, 23), "##nl"),
                     ((23, 24), "##p"),
                     ((25, 33), "sentence"),
                     ((33, 34), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
             )

             self.assertEqual(
                 [e[1] for e in expected_results]       ,					tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]				)				)
             self.assertEqual([e[0] for e in expected_results]       ,					tokens["offset_mapping"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[Any]				):
         lowercase    :str      	=						["的", "人", "有"]
         lowercase    :Union[str, Any]      	=						"".join(_lowerCAmelCase				)
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"				):
             lowercase    :List[Any]      	=						True
             lowercase    :Optional[Any]      	=						self.tokenizer_class.from_pretrained(_lowerCAmelCase       ,					**_lowerCAmelCase				)
             lowercase    :List[Any]      	=						self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase       ,					**_lowerCAmelCase				)

             lowercase    :str      	=						tokenizer_p.encode(_lowerCAmelCase       ,					add_special_tokens=_lowerCAmelCase				)
             lowercase    :int      	=						tokenizer_r.encode(_lowerCAmelCase       ,					add_special_tokens=_lowerCAmelCase				)

             lowercase    :Union[str, Any]      	=						tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase				)
             lowercase    :Tuple      	=						tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase				)

             # it is expected that each Chinese character is not preceded by "##"
             self.assertListEqual(_lowerCAmelCase       ,					_lowerCAmelCase				)
             self.assertListEqual(_lowerCAmelCase       ,					_lowerCAmelCase				)

             lowercase    :List[str]      	=						False
             lowercase    :Optional[int]      	=						self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase       ,					**_lowerCAmelCase				)
             lowercase    :Optional[Any]      	=						self.tokenizer_class.from_pretrained(_lowerCAmelCase       ,					**_lowerCAmelCase				)

             lowercase    :str      	=						tokenizer_r.encode(_lowerCAmelCase       ,					add_special_tokens=_lowerCAmelCase				)
             lowercase    :str      	=						tokenizer_p.encode(_lowerCAmelCase       ,					add_special_tokens=_lowerCAmelCase				)

             lowercase    :Any      	=						tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase				)
             lowercase    :Union[str, Any]      	=						tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase				)

             # it is expected that only the first Chinese character is not preceded by "##".
             lowercase    :Dict      	=						[
                 F"##{token}" if idx != 0 else token for idx, token in enumerate(_lowerCAmelCase				)
             ]
             self.assertListEqual(_lowerCAmelCase       ,					_lowerCAmelCase				)
             self.assertListEqual(_lowerCAmelCase       ,					_lowerCAmelCase				)









import string



def       UpperCAmelCase__  (     lowerCamelCase   ):
  lowercase    :Optional[int]      	=						""
  for i in sequence:
    lowercase    :Optional[Any]      	=						ord(lowerCamelCase   )
    if 65 <= extract <= 90:
      output += chr(155 - extract   )
    elif 97 <= extract <= 122:
      output += chr(219 - extract   )
    else:
      output += i
  return output



def       UpperCAmelCase__  (     lowerCamelCase   ):
  lowercase    :Union[str, Any]      	=						string.ascii_letters
  lowercase    :Dict      	=						string.ascii_lowercase[::-1] + string.ascii_uppercase[::-1]
  return "".join(
      letters_reversed[letters.index(lowerCamelCase   )] if c in letters else c for c in sequence   )



def       UpperCAmelCase__  (     ):
  from timeit import timeit

  print("Running performance benchmarks..."   )
  lowercase    :Dict      	=						"from string import printable ; from __main__ import atbash, atbash_slow"
  print(F"> atbash_slow(): {timeit('atbash_slow(printable)',			setup=lowerCamelCase   )} seconds"   )
  print(F">      atbash(): {timeit('atbash(printable)',			setup=lowerCamelCase   )} seconds"   )


if __name__ == "__main__":
     for example in ("ABCDEFGH", "123GGjj", "testStringtest", "with space"):
          print(f'''{example} encrypted in atbash: {atbash(example)}''')
     benchmark()



