


'''simple docstring'''



import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class  UpperCamelCase__(     lowerCAmelCase		,       unittest.TestCase ):
				__magic_name__       :   Any							= BertTokenizer
				__magic_name__       :   str							= BertTokenizerFast
				__magic_name__       :   List[Any]							= True
				__magic_name__       :   int							= True
				__magic_name__       :   Optional[Any]							= filter_non_english




				def   a__(	self    :				Optional[Any]   )->  str:

										"""simple docstring"""




										super().setUp()

										UpperCAmelCase         =			[
										    '''[UNK]''',
										    '''[CLS]''',
										    '''[SEP]''',
										    '''[PAD]''',
										    '''[MASK]''',
										    '''want''',
										    '''##want''',
										    '''##ed''',
										    '''wa''',
										    '''un''',
										    '''runn''',
										    '''##ing''',
										    ''',''',
										    '''low''',
										    '''lowest''',
										]
										UpperCAmelCase         =			os.path.join(self.tmpdirname   ,					VOCAB_FILES_NAMES['''vocab_file''']   )
										with open(self.vocab_file   ,					'''w'''   ,					encoding='''utf-8'''   ) as vocab_writer:
																vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )




				def   a__(	self    :				Any   ,					lowerCAmelCase    :				Any   )->  Tuple:

										"""simple docstring"""




										UpperCAmelCase         =			'''UNwant\u00E9d,running'''
										UpperCAmelCase         =			'''unwanted, running'''
										return input_text, output_text




				def   a__(	self    :				List[str]   )->  Tuple:

										"""simple docstring"""




										UpperCAmelCase         =			self.tokenizer_class(self.vocab_file   )

										UpperCAmelCase         =			tokenizer.tokenize('''UNwant\u00E9d,running'''   )
										self.assertListEqual(UpperCAmelCase__   ,					['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']   )
										self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__   )   ,					[9, 6, 7, 12, 10, 11]   )




				def   a__(	self    :				Optional[int]   )->  List[Any]:

										"""simple docstring"""




										if not self.test_rust_tokenizer:
																return

										UpperCAmelCase         =			self.get_tokenizer()
										UpperCAmelCase         =			self.get_rust_tokenizer()

										UpperCAmelCase         =			'''UNwant\u00E9d,running'''

										UpperCAmelCase         =			tokenizer.tokenize(UpperCAmelCase__   )
										UpperCAmelCase         =			rust_tokenizer.tokenize(UpperCAmelCase__   )
										self.assertListEqual(UpperCAmelCase__   ,					UpperCAmelCase__   )

										UpperCAmelCase         =			tokenizer.encode(UpperCAmelCase__   ,					add_special_tokens=UpperCAmelCase__   )
										UpperCAmelCase         =			rust_tokenizer.encode(UpperCAmelCase__   ,					add_special_tokens=UpperCAmelCase__   )
										self.assertListEqual(UpperCAmelCase__   ,					UpperCAmelCase__   )

										UpperCAmelCase         =			self.get_rust_tokenizer()
										UpperCAmelCase         =			tokenizer.encode(UpperCAmelCase__   )
										UpperCAmelCase         =			rust_tokenizer.encode(UpperCAmelCase__   )
										self.assertListEqual(UpperCAmelCase__   ,					UpperCAmelCase__   )

										# With lower casing
										UpperCAmelCase         =			self.get_tokenizer(do_lower_case=UpperCAmelCase__   )
										UpperCAmelCase         =			self.get_rust_tokenizer(do_lower_case=UpperCAmelCase__   )

										UpperCAmelCase         =			'''UNwant\u00E9d,running'''

										UpperCAmelCase         =			tokenizer.tokenize(UpperCAmelCase__   )
										UpperCAmelCase         =			rust_tokenizer.tokenize(UpperCAmelCase__   )
										self.assertListEqual(UpperCAmelCase__   ,					UpperCAmelCase__   )

										UpperCAmelCase         =			tokenizer.encode(UpperCAmelCase__   ,					add_special_tokens=UpperCAmelCase__   )
										UpperCAmelCase         =			rust_tokenizer.encode(UpperCAmelCase__   ,					add_special_tokens=UpperCAmelCase__   )
										self.assertListEqual(UpperCAmelCase__   ,					UpperCAmelCase__   )

										UpperCAmelCase         =			self.get_rust_tokenizer()
										UpperCAmelCase         =			tokenizer.encode(UpperCAmelCase__   )
										UpperCAmelCase         =			rust_tokenizer.encode(UpperCAmelCase__   )
										self.assertListEqual(UpperCAmelCase__   ,					UpperCAmelCase__   )




				def   a__(	self    :				Dict   )->  Any:

										"""simple docstring"""




										UpperCAmelCase         =			BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''   )   ,					['''ah''', '''\u535A''', '''\u63A8''', '''zz''']   )




				def   a__(	self    :				Dict   )->  Union[str, Any]:

										"""simple docstring"""




										UpperCAmelCase         =			BasicTokenizer(do_lower_case=UpperCAmelCase__   )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )   ,					['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )   ,					['''hello''']   )




				def   a__(	self    :				List[str]   )->  Any:

										"""simple docstring"""




										UpperCAmelCase         =			BasicTokenizer(do_lower_case=UpperCAmelCase__   ,					strip_accents=UpperCAmelCase__   )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )   ,					['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )   ,					['''h\u00E9llo''']   )




				def   a__(	self    :				List[str]   )->  Tuple:

										"""simple docstring"""




										UpperCAmelCase         =			BasicTokenizer(do_lower_case=UpperCAmelCase__   ,					strip_accents=UpperCAmelCase__   )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )   ,					['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )   ,					['''hello''']   )




				def   a__(	self    :				int   )->  Any:

										"""simple docstring"""




										UpperCAmelCase         =			BasicTokenizer(do_lower_case=UpperCAmelCase__   )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )   ,					['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   )   ,					['''hello''']   )




				def   a__(	self    :				Dict   )->  Union[str, Any]:

										"""simple docstring"""




										UpperCAmelCase         =			BasicTokenizer(do_lower_case=UpperCAmelCase__   )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   )   ,					['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )




				def   a__(	self    :				List[str]   )->  List[Any]:

										"""simple docstring"""




										UpperCAmelCase         =			BasicTokenizer(do_lower_case=UpperCAmelCase__   ,					strip_accents=UpperCAmelCase__   )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )   ,					['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )




				def   a__(	self    :				Optional[Any]   )->  Optional[int]:

										"""simple docstring"""




										UpperCAmelCase         =			BasicTokenizer(do_lower_case=UpperCAmelCase__   ,					strip_accents=UpperCAmelCase__   )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   )   ,					['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )




				def   a__(	self    :				Any   )->  List[str]:

										"""simple docstring"""




										UpperCAmelCase         =			BasicTokenizer(do_lower_case=UpperCAmelCase__   ,					never_split=['''[UNK]''']   )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''   )   ,					['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']   )




				def   a__(	self    :				Union[str, Any]   )->  List[str]:

										"""simple docstring"""




										UpperCAmelCase         =			BasicTokenizer()
										UpperCAmelCase         =			'''a\n\'ll !!to?\'d of, can\'t.'''
										UpperCAmelCase         =			['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
										self.assertListEqual(tokenizer.tokenize(UpperCAmelCase__   )   ,					UpperCAmelCase__   )




				def   a__(	self    :				Optional[int]   )->  Dict:

										"""simple docstring"""




										UpperCAmelCase         =			['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

										UpperCAmelCase         =			{}
										for i, token in enumerate(UpperCAmelCase__   ):
																UpperCAmelCase         =			i
										UpperCAmelCase         =			WordpieceTokenizer(vocab=UpperCAmelCase__   ,					unk_token='''[UNK]'''   )

										self.assertListEqual(tokenizer.tokenize(''''''   )   ,					[]   )

										self.assertListEqual(tokenizer.tokenize('''unwanted running'''   )   ,					['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']   )

										self.assertListEqual(tokenizer.tokenize('''unwantedX running'''   )   ,					['''[UNK]''', '''runn''', '''##ing''']   )




				def   a__(	self    :				Optional[int]   )->  Union[str, Any]:

										"""simple docstring"""




										self.assertTrue(_is_whitespace(''' '''   )   )
										self.assertTrue(_is_whitespace('''\t'''   )   )
										self.assertTrue(_is_whitespace('''\r'''   )   )
										self.assertTrue(_is_whitespace('''\n'''   )   )
										self.assertTrue(_is_whitespace('''\u00A0'''   )   )

										self.assertFalse(_is_whitespace('''A'''   )   )
										self.assertFalse(_is_whitespace('''-'''   )   )




				def   a__(	self    :				List[Any]   )->  Optional[int]:

										"""simple docstring"""




										self.assertTrue(_is_control('''\u0005'''   )   )

										self.assertFalse(_is_control('''A'''   )   )
										self.assertFalse(_is_control(''' '''   )   )
										self.assertFalse(_is_control('''\t'''   )   )
										self.assertFalse(_is_control('''\r'''   )   )




				def   a__(	self    :				Union[str, Any]   )->  int:

										"""simple docstring"""




										self.assertTrue(_is_punctuation('''-'''   )   )
										self.assertTrue(_is_punctuation('''$'''   )   )
										self.assertTrue(_is_punctuation('''`'''   )   )
										self.assertTrue(_is_punctuation('''.'''   )   )

										self.assertFalse(_is_punctuation('''A'''   )   )
										self.assertFalse(_is_punctuation(''' '''   )   )




				def   a__(	self    :				List[Any]   )->  List[str]:

										"""simple docstring"""




										UpperCAmelCase         =			self.get_tokenizer()
										UpperCAmelCase         =			self.get_rust_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(UpperCAmelCase__   ) for t in ['''Test''', '''\xad''', '''test''']]   ,					[['''[UNK]'''], [], ['''[UNK]''']]   )

										self.assertListEqual(
										    [rust_tokenizer.tokenize(UpperCAmelCase__   ) for t in ['''Test''', '''\xad''', '''test''']]   ,					[['''[UNK]'''], [], ['''[UNK]''']]   )




				@slow
				def   a__(	self    :				Any   )->  Union[str, Any]:

										"""simple docstring"""




										UpperCAmelCase         =			self.tokenizer_class.from_pretrained('''bert-base-uncased'''   )

										UpperCAmelCase         =			tokenizer.encode('''sequence builders'''   ,					add_special_tokens=UpperCAmelCase__   )
										UpperCAmelCase         =			tokenizer.encode('''multi-sequence build'''   ,					add_special_tokens=UpperCAmelCase__   )

										UpperCAmelCase         =			tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__   )
										UpperCAmelCase         =			tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__   ,					UpperCAmelCase__   )

										assert encoded_sentence == [101] + text + [102]
										assert encoded_pair == [101] + text + [102] + text_a + [102]




				def   a__(	self    :				Tuple   )->  Union[str, Any]:

										"""simple docstring"""




										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
																						UpperCAmelCase         =			self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__   ,					**UpperCAmelCase__   )

																						UpperCAmelCase         =			F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																						UpperCAmelCase         =			tokenizer_r.encode_plus(
																						    UpperCAmelCase__   ,					return_attention_mask=UpperCAmelCase__   ,					return_token_type_ids=UpperCAmelCase__   ,					return_offsets_mapping=UpperCAmelCase__   ,					add_special_tokens=UpperCAmelCase__   ,					)

																						UpperCAmelCase         =			tokenizer_r.do_lower_case if hasattr(UpperCAmelCase__   ,					'''do_lower_case'''   ) else False
																						UpperCAmelCase         =			(
																						    [
																						        ((0, 0), tokenizer_r.cls_token),
																						        ((0, 1), '''A'''),
																						        ((1, 2), ''','''),
																						        ((3, 5), '''na'''),
																						        ((5, 6), '''##ï'''),
																						        ((6, 8), '''##ve'''),
																						        ((9, 15), tokenizer_r.mask_token),
																						        ((16, 21), '''Allen'''),
																						        ((21, 23), '''##NL'''),
																						        ((23, 24), '''##P'''),
																						        ((25, 33), '''sentence'''),
																						        ((33, 34), '''.'''),
																						        ((0, 0), tokenizer_r.sep_token),
																						    ]
																						    if not do_lower_case
																						    else [
																						        ((0, 0), tokenizer_r.cls_token),
																						        ((0, 1), '''a'''),
																						        ((1, 2), ''','''),
																						        ((3, 8), '''naive'''),
																						        ((9, 15), tokenizer_r.mask_token),
																						        ((16, 21), '''allen'''),
																						        ((21, 23), '''##nl'''),
																						        ((23, 24), '''##p'''),
																						        ((25, 33), '''sentence'''),
																						        ((33, 34), '''.'''),
																						        ((0, 0), tokenizer_r.sep_token),
																						    ]
																						)

																						self.assertEqual(
																						    [e[1] for e in expected_results]   ,					tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']   )   )
																						self.assertEqual([e[0] for e in expected_results]   ,					tokens['''offset_mapping''']   )





				def   a__(	self    :				Union[str, Any]   )->  int:

										"""simple docstring"""




										UpperCAmelCase         =			['''的''', '''人''', '''有''']
										UpperCAmelCase         =			''''''.join(UpperCAmelCase__   )
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
																						UpperCAmelCase         =			True
																						UpperCAmelCase         =			self.tokenizer_class.from_pretrained(UpperCAmelCase__   ,					**UpperCAmelCase__   )
																						UpperCAmelCase         =			self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__   ,					**UpperCAmelCase__   )

																						UpperCAmelCase         =			tokenizer_p.encode(UpperCAmelCase__   ,					add_special_tokens=UpperCAmelCase__   )
																						UpperCAmelCase         =			tokenizer_r.encode(UpperCAmelCase__   ,					add_special_tokens=UpperCAmelCase__   )

																						UpperCAmelCase         =			tokenizer_r.convert_ids_to_tokens(UpperCAmelCase__   )
																						UpperCAmelCase         =			tokenizer_p.convert_ids_to_tokens(UpperCAmelCase__   )

																						# it is expected that each Chinese character is not preceded by "##"
																						self.assertListEqual(UpperCAmelCase__   ,					UpperCAmelCase__   )
																						self.assertListEqual(UpperCAmelCase__   ,					UpperCAmelCase__   )

																						UpperCAmelCase         =			False
																						UpperCAmelCase         =			self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__   ,					**UpperCAmelCase__   )
																						UpperCAmelCase         =			self.tokenizer_class.from_pretrained(UpperCAmelCase__   ,					**UpperCAmelCase__   )

																						UpperCAmelCase         =			tokenizer_r.encode(UpperCAmelCase__   ,					add_special_tokens=UpperCAmelCase__   )
																						UpperCAmelCase         =			tokenizer_p.encode(UpperCAmelCase__   ,					add_special_tokens=UpperCAmelCase__   )

																						UpperCAmelCase         =			tokenizer_r.convert_ids_to_tokens(UpperCAmelCase__   )
																						UpperCAmelCase         =			tokenizer_p.convert_ids_to_tokens(UpperCAmelCase__   )

																						# it is expected that only the first Chinese character is not preceded by "##".
																						UpperCAmelCase         =			[
																						    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(UpperCAmelCase__   )
																						]
																						self.assertListEqual(UpperCAmelCase__   ,					UpperCAmelCase__   )
																						self.assertListEqual(UpperCAmelCase__   ,					UpperCAmelCase__   )





'''simple docstring'''



def    lowerCamelCase__   ( A     :    str  ):




						'''simple docstring'''






						assert column_title.isupper()
						UpperCAmelCase         =			0
						UpperCAmelCase         =			len(A  ) - 1
						UpperCAmelCase         =			0

						while index >= 0:
												UpperCAmelCase         =			(ord(column_title[index]  ) - 64) * pow(26   ,     A  )
												answer += value
												power += 1
												index -= 1

						return answer


if __name__ == "__main__":
	from doctest import testmod

	testmod()





