




import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class    __lowerCAmelCase       (							_UpperCamelCase      , unittest.TestCase):
       _a 					=						BertTokenizer
       _a 					=						BertTokenizerFast
       _a 					=						True
       _a 					=						True
       _a 					=						filter_non_english






       def 					SCREAMING_SNAKE_CASE    (   self:					List[Any]				):
         super().setUp()

         lowercase    :str      	=						[
             '''[UNK]''',
             '''[CLS]''',
             '''[SEP]''',
             '''[PAD]''',
             '''[MASK]''',
             '''want''',
             '''##want''',
             '''##ed''',
             '''wa''',
             '''un''',
             '''runn''',
             '''##ing''',
             ''',''',
             '''low''',
             '''lowest''',
         ]
         lowercase    :List[str]      	=						os.path.join(self.tmpdirname       ,					VOCAB_FILES_NAMES["vocab_file"]				)
         with open(self.vocab_file       ,					"w"       ,					encoding="utf-8"				) as vocab_writer:
           vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)






       def 					SCREAMING_SNAKE_CASE    (   self:					List[Any]       ,					_lowerCAmelCase:					Union[str, Any]				):
         lowercase    :Any      	=						'''UNwant\u00E9d,running'''
         lowercase    :Tuple      	=						'''unwanted, running'''
         return input_text, output_text






       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[int]				):
         lowercase    :str      	=						self.tokenizer_class(self.vocab_file				)

         lowercase    :Any      	=						tokenizer.tokenize("UNwant\u00E9d,running"				)
         self.assertListEqual(_UpperCAmelCase       ,					["un", "##want", "##ed", ",", "runn", "##ing"]				)
         self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase				)       ,					[9, 6, 7, 12, 10, 11]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[Any]				):
         if not self.test_rust_tokenizer:
           return

         lowercase    :Optional[Any]      	=						self.get_tokenizer()
         lowercase    :Dict      	=						self.get_rust_tokenizer()

         lowercase    :Union[str, Any]      	=						'''UNwant\u00E9d,running'''

         lowercase    :Optional[Any]      	=						tokenizer.tokenize(_UpperCAmelCase				)
         lowercase    :Optional[Any]      	=						rust_tokenizer.tokenize(_UpperCAmelCase				)
         self.assertListEqual(_UpperCAmelCase       ,					_UpperCAmelCase				)

         lowercase    :Union[str, Any]      	=						tokenizer.encode(_UpperCAmelCase       ,					add_special_tokens=_UpperCAmelCase				)
         lowercase    :List[str]      	=						rust_tokenizer.encode(_UpperCAmelCase       ,					add_special_tokens=_UpperCAmelCase				)
         self.assertListEqual(_UpperCAmelCase       ,					_UpperCAmelCase				)

         lowercase    :str      	=						self.get_rust_tokenizer()
         lowercase    :str      	=						tokenizer.encode(_UpperCAmelCase				)
         lowercase    :Any      	=						rust_tokenizer.encode(_UpperCAmelCase				)
         self.assertListEqual(_UpperCAmelCase       ,					_UpperCAmelCase				)

         # With lower casing
         lowercase    :Dict      	=						self.get_tokenizer(do_lower_case=_UpperCAmelCase				)
         lowercase    :List[str]      	=						self.get_rust_tokenizer(do_lower_case=_UpperCAmelCase				)

         lowercase    :Optional[Any]      	=						'''UNwant\u00E9d,running'''

         lowercase    :str      	=						tokenizer.tokenize(_UpperCAmelCase				)
         lowercase    :List[str]      	=						rust_tokenizer.tokenize(_UpperCAmelCase				)
         self.assertListEqual(_UpperCAmelCase       ,					_UpperCAmelCase				)

         lowercase    :Tuple      	=						tokenizer.encode(_UpperCAmelCase       ,					add_special_tokens=_UpperCAmelCase				)
         lowercase    :Union[str, Any]      	=						rust_tokenizer.encode(_UpperCAmelCase       ,					add_special_tokens=_UpperCAmelCase				)
         self.assertListEqual(_UpperCAmelCase       ,					_UpperCAmelCase				)

         lowercase    :Optional[Any]      	=						self.get_rust_tokenizer()
         lowercase    :Optional[Any]      	=						tokenizer.encode(_UpperCAmelCase				)
         lowercase    :Tuple      	=						rust_tokenizer.encode(_UpperCAmelCase				)
         self.assertListEqual(_UpperCAmelCase       ,					_UpperCAmelCase				)






       def 					SCREAMING_SNAKE_CASE    (   self:					List[Any]				):
         lowercase    :Tuple      	=						BasicTokenizer()

         self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"				)       ,					["ah", "\u535A", "\u63A8", "zz"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					List[str]				):
         lowercase    :Optional[int]      	=						BasicTokenizer(do_lower_case=_UpperCAmelCase				)

         self.assertListEqual(
             tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)       ,					["hello", "!", "how", "are", "you", "?"]				)
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)       ,					["hello"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Dict				):
         lowercase    :List[Any]      	=						BasicTokenizer(do_lower_case=_UpperCAmelCase       ,					strip_accents=_UpperCAmelCase				)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)       ,					["hällo", "!", "how", "are", "you", "?"]				)
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)       ,					["h\u00E9llo"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[int]				):
         lowercase    :Dict      	=						BasicTokenizer(do_lower_case=_UpperCAmelCase       ,					strip_accents=_UpperCAmelCase				)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)       ,					["hallo", "!", "how", "are", "you", "?"]				)
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)       ,					["hello"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					List[Any]				):
         lowercase    :List[str]      	=						BasicTokenizer(do_lower_case=_UpperCAmelCase				)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)       ,					["hallo", "!", "how", "are", "you", "?"]				)
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)       ,					["hello"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					List[Any]				):
         lowercase    :int      	=						BasicTokenizer(do_lower_case=_UpperCAmelCase				)

         self.assertListEqual(
             tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)       ,					["HeLLo", "!", "how", "Are", "yoU", "?"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Dict				):
         lowercase    :List[str]      	=						BasicTokenizer(do_lower_case=_UpperCAmelCase       ,					strip_accents=_UpperCAmelCase				)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)       ,					["HäLLo", "!", "how", "Are", "yoU", "?"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					int				):
         lowercase    :Union[str, Any]      	=						BasicTokenizer(do_lower_case=_UpperCAmelCase       ,					strip_accents=_UpperCAmelCase				)

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)       ,					["HaLLo", "!", "how", "Are", "yoU", "?"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[Any]				):
         lowercase    :List[str]      	=						BasicTokenizer(do_lower_case=_UpperCAmelCase       ,					never_split=["[UNK]"]				)

         self.assertListEqual(
             tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"				)       ,					["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[Any]				):
         lowercase    :List[Any]      	=						BasicTokenizer()
         lowercase    :Optional[Any]      	=						'''a\n\'ll !!to?\'d of, can\'t.'''
         lowercase    :List[Any]      	=						['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
         self.assertListEqual(tokenizer.tokenize(_UpperCAmelCase				)       ,					_UpperCAmelCase				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Tuple				):
         lowercase    :int      	=						['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

         lowercase    :List[str]      	=						{}
         for i, token in enumerate(_UpperCAmelCase				):
           lowercase    :str      	=						i
         lowercase    :Optional[int]      	=						WordpieceTokenizer(vocab=_UpperCAmelCase       ,					unk_token="[UNK]"				)

         self.assertListEqual(tokenizer.tokenize(""				)       ,					[]				)

         self.assertListEqual(tokenizer.tokenize("unwanted running"				)       ,					["un", "##want", "##ed", "runn", "##ing"]				)

         self.assertListEqual(tokenizer.tokenize("unwantedX running"				)       ,					["[UNK]", "runn", "##ing"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					int				):
         self.assertTrue(_is_whitespace(" "				)				)
         self.assertTrue(_is_whitespace("\t"				)				)
         self.assertTrue(_is_whitespace("\r"				)				)
         self.assertTrue(_is_whitespace("\n"				)				)
         self.assertTrue(_is_whitespace("\u00A0"				)				)

         self.assertFalse(_is_whitespace("A"				)				)
         self.assertFalse(_is_whitespace("-"				)				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Any				):
         self.assertTrue(_is_control("\u0005"				)				)

         self.assertFalse(_is_control("A"				)				)
         self.assertFalse(_is_control(" "				)				)
         self.assertFalse(_is_control("\t"				)				)
         self.assertFalse(_is_control("\r"				)				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Tuple				):
         self.assertTrue(_is_punctuation("-"				)				)
         self.assertTrue(_is_punctuation("$"				)				)
         self.assertTrue(_is_punctuation("`"				)				)
         self.assertTrue(_is_punctuation("."				)				)

         self.assertFalse(_is_punctuation("A"				)				)
         self.assertFalse(_is_punctuation(" "				)				)






       def 					SCREAMING_SNAKE_CASE    (   self:					List[Any]				):
         lowercase    :Optional[Any]      	=						self.get_tokenizer()
         lowercase    :List[str]      	=						self.get_rust_tokenizer()

         # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
         self.assertListEqual([tokenizer.tokenize(_UpperCAmelCase				) for t in ["Test", "\xad", "test"]]       ,					[["[UNK]"], [], ["[UNK]"]]				)

         self.assertListEqual(
             [rust_tokenizer.tokenize(_UpperCAmelCase				) for t in ["Test", "\xad", "test"]]       ,					[["[UNK]"], [], ["[UNK]"]]				)






       @slow
       def 					SCREAMING_SNAKE_CASE    (   self:					Tuple				):
         lowercase    :List[Any]      	=						self.tokenizer_class.from_pretrained("bert-base-uncased"				)

         lowercase    :Optional[int]      	=						tokenizer.encode("sequence builders"       ,					add_special_tokens=_UpperCAmelCase				)
         lowercase    :Optional[int]      	=						tokenizer.encode("multi-sequence build"       ,					add_special_tokens=_UpperCAmelCase				)

         lowercase    :List[Any]      	=						tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase				)
         lowercase    :Union[str, Any]      	=						tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase       ,					_UpperCAmelCase				)

         assert encoded_sentence == [1_01] + text + [1_02]
         assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]






       def 					SCREAMING_SNAKE_CASE    (   self:					Tuple				):
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"				):
             lowercase    :List[str]      	=						self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase       ,					**_UpperCAmelCase				)

             lowercase    :Tuple      	=						F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
             lowercase    :List[str]      	=						tokenizer_r.encode_plus(
                 _UpperCAmelCase       ,					return_attention_mask=_UpperCAmelCase       ,					return_token_type_ids=_UpperCAmelCase       ,					return_offsets_mapping=_UpperCAmelCase       ,					add_special_tokens=_UpperCAmelCase       ,					)

             lowercase    :Tuple      	=						tokenizer_r.do_lower_case if hasattr(_UpperCAmelCase       ,					"do_lower_case"				) else False
             lowercase    :Dict      	=						(
                 [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), '''A'''),
                     ((1, 2), ''','''),
                     ((3, 5), '''na'''),
                     ((5, 6), '''##ï'''),
                     ((6, 8), '''##ve'''),
                     ((9, 15), tokenizer_r.mask_token),
                     ((16, 21), '''Allen'''),
                     ((21, 23), '''##NL'''),
                     ((23, 24), '''##P'''),
                     ((25, 33), '''sentence'''),
                     ((33, 34), '''.'''),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
                 if not do_lower_case
                 else [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), '''a'''),
                     ((1, 2), ''','''),
                     ((3, 8), '''naive'''),
                     ((9, 15), tokenizer_r.mask_token),
                     ((16, 21), '''allen'''),
                     ((21, 23), '''##nl'''),
                     ((23, 24), '''##p'''),
                     ((25, 33), '''sentence'''),
                     ((33, 34), '''.'''),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
             )

             self.assertEqual(
                 [e[1] for e in expected_results]       ,					tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]				)				)
             self.assertEqual([e[0] for e in expected_results]       ,					tokens["offset_mapping"]				)






       def 					SCREAMING_SNAKE_CASE    (   self:					int				):
         lowercase    :Dict      	=						['''的''', '''人''', '''有''']
         lowercase    :Any      	=						''''''.join(_UpperCAmelCase				)
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"				):
             lowercase    :int      	=						True
             lowercase    :List[str]      	=						self.tokenizer_class.from_pretrained(_UpperCAmelCase       ,					**_UpperCAmelCase				)
             lowercase    :Optional[int]      	=						self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase       ,					**_UpperCAmelCase				)

             lowercase    :str      	=						tokenizer_p.encode(_UpperCAmelCase       ,					add_special_tokens=_UpperCAmelCase				)
             lowercase    :Union[str, Any]      	=						tokenizer_r.encode(_UpperCAmelCase       ,					add_special_tokens=_UpperCAmelCase				)

             lowercase    :Optional[Any]      	=						tokenizer_r.convert_ids_to_tokens(_UpperCAmelCase				)
             lowercase    :Any      	=						tokenizer_p.convert_ids_to_tokens(_UpperCAmelCase				)

             # it is expected that each Chinese character is not preceded by "##"
             self.assertListEqual(_UpperCAmelCase       ,					_UpperCAmelCase				)
             self.assertListEqual(_UpperCAmelCase       ,					_UpperCAmelCase				)

             lowercase    :Optional[Any]      	=						False
             lowercase    :Union[str, Any]      	=						self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase       ,					**_UpperCAmelCase				)
             lowercase    :Tuple      	=						self.tokenizer_class.from_pretrained(_UpperCAmelCase       ,					**_UpperCAmelCase				)

             lowercase    :int      	=						tokenizer_r.encode(_UpperCAmelCase       ,					add_special_tokens=_UpperCAmelCase				)
             lowercase    :List[str]      	=						tokenizer_p.encode(_UpperCAmelCase       ,					add_special_tokens=_UpperCAmelCase				)

             lowercase    :List[Any]      	=						tokenizer_r.convert_ids_to_tokens(_UpperCAmelCase				)
             lowercase    :List[Any]      	=						tokenizer_p.convert_ids_to_tokens(_UpperCAmelCase				)

             # it is expected that only the first Chinese character is not preceded by "##".
             lowercase    :str      	=						[
                 F"##{token}" if idx != 0 else token for idx, token in enumerate(_UpperCAmelCase				)
             ]
             self.assertListEqual(_UpperCAmelCase       ,					_UpperCAmelCase				)
             self.assertListEqual(_UpperCAmelCase       ,					_UpperCAmelCase				)









def       UpperCAmelCase__  (     lowerCamelCase = "The quick brown fox jumps over the lazy dog",			):
  lowercase    :Dict      	=						set()

  # Replace all the whitespace in our sentence
  lowercase    :Optional[int]      	=						input_str.replace(" ",			""   )
  for alpha in input_str:
    if "a" <= alpha.lower() <= "z":
      frequency.add(alpha.lower()   )
  return len(lowerCamelCase   ) == 26



def       UpperCAmelCase__  (     lowerCamelCase = "The quick brown fox jumps over the lazy dog",			):
  lowercase    :Optional[Any]      	=						[False] * 26
  for char in input_str:
    if char.islower():
      lowercase    :Union[str, Any]      	=						True
    elif char.isupper():
      lowercase    :int      	=						True
  return all(lowerCamelCase   )



def       UpperCAmelCase__  (     lowerCamelCase = "The quick brown fox jumps over the lazy dog",			):
  return len({char for char in input_str.lower() if char.isalpha()}   ) == 26



def       UpperCAmelCase__  (     ):
  from timeit import timeit

  lowercase    :Optional[Any]      	=						"from __main__ import is_pangram, is_pangram_faster, is_pangram_fastest"
  print(timeit("is_pangram()",			setup=lowerCamelCase   )   )
  print(timeit("is_pangram_faster()",			setup=lowerCamelCase   )   )
  print(timeit("is_pangram_fastest()",			setup=lowerCamelCase   )   )
  # 5.348480500048026, 2.6477354579837993, 1.8470395830227062
  # 5.036091582966037, 2.644472333951853,  1.8869528750656173


if __name__ == "__main__":
     import doctest

     doctest.testmod()
     benchmark()



