




import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class    lowerCamelCase    (		_lowerCamelCase				,unittest.TestCase						):






			'''simple docstring'''





			UpperCamelCase__								=MobileBertTokenizer
			UpperCamelCase__								=MobileBertTokenizerFast
			UpperCamelCase__								=True
			UpperCamelCase__								=True
			UpperCamelCase__								=filter_non_english
			UpperCamelCase__								='''google/mobilebert-uncased'''

			def 							UpperCAmelCase__       ( self			:   Optional[int]  )						->   Any:
								super().setUp()

								__magic_name__							:	Optional[Any]    							=							[
								    '''[UNK]''',
								    '''[CLS]''',
								    '''[SEP]''',
								    '''[PAD]''',
								    '''[MASK]''',
								    '''want''',
								    '''##want''',
								    '''##ed''',
								    '''wa''',
								    '''un''',
								    '''runn''',
								    '''##ing''',
								    ''',''',
								    '''low''',
								    '''lowest''',
								]
								__magic_name__							:	Optional[Any]    							=							os.path.join(self.tmpdirname       ,						VOCAB_FILES_NAMES['''vocab_file''']  )
								with open(self.vocab_file       ,						'''w'''       ,						encoding='''utf-8'''  ) as vocab_writer:
													vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )

								__magic_name__							:	Optional[Any]    							=							[
								    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
								    for tokenizer_def in self.tokenizers_list
								]

			def 							UpperCAmelCase__       ( self			:   List[str]       ,						lowerCamelCase_			:   str  )						->   Any:
								__magic_name__							:	Optional[int]    							=							'''UNwant\u00E9d,running'''
								__magic_name__							:	Tuple    							=							'''unwanted, running'''
								return input_text, output_text

			def 							UpperCAmelCase__       ( self			:   int  )						->   Tuple:
								__magic_name__							:	int    							=							self.tokenizer_class(self.vocab_file  )

								__magic_name__							:	Dict    							=							tokenizer.tokenize('''UNwant\u00E9d,running'''  )
								self.assertListEqual(lowerCamelCase_       ,						['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']  )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_  )       ,						[9, 6, 7, 12, 10, 11]  )

			def 							UpperCAmelCase__       ( self			:   List[str]  )						->   Optional[int]:
								if not self.test_rust_tokenizer:
													return

								__magic_name__							:	Optional[Any]    							=							self.get_tokenizer()
								__magic_name__							:	int    							=							self.get_rust_tokenizer()

								__magic_name__							:	Tuple    							=							'''UNwant\u00E9d,running'''

								__magic_name__							:	Optional[int]    							=							tokenizer.tokenize(lowerCamelCase_  )
								__magic_name__							:	Union[str, Any]    							=							rust_tokenizer.tokenize(lowerCamelCase_  )
								self.assertListEqual(lowerCamelCase_       ,						lowerCamelCase_  )

								__magic_name__							:	Optional[int]    							=							tokenizer.encode(lowerCamelCase_       ,						add_special_tokens=lowerCamelCase_  )
								__magic_name__							:	Optional[Any]    							=							rust_tokenizer.encode(lowerCamelCase_       ,						add_special_tokens=lowerCamelCase_  )
								self.assertListEqual(lowerCamelCase_       ,						lowerCamelCase_  )

								__magic_name__							:	Any    							=							self.get_rust_tokenizer()
								__magic_name__							:	Union[str, Any]    							=							tokenizer.encode(lowerCamelCase_  )
								__magic_name__							:	Optional[Any]    							=							rust_tokenizer.encode(lowerCamelCase_  )
								self.assertListEqual(lowerCamelCase_       ,						lowerCamelCase_  )

								# With lower casing
								__magic_name__							:	Tuple    							=							self.get_tokenizer(do_lower_case=lowerCamelCase_  )
								__magic_name__							:	int    							=							self.get_rust_tokenizer(do_lower_case=lowerCamelCase_  )

								__magic_name__							:	Optional[int]    							=							'''UNwant\u00E9d,running'''

								__magic_name__							:	List[Any]    							=							tokenizer.tokenize(lowerCamelCase_  )
								__magic_name__							:	List[Any]    							=							rust_tokenizer.tokenize(lowerCamelCase_  )
								self.assertListEqual(lowerCamelCase_       ,						lowerCamelCase_  )

								__magic_name__							:	Union[str, Any]    							=							tokenizer.encode(lowerCamelCase_       ,						add_special_tokens=lowerCamelCase_  )
								__magic_name__							:	Dict    							=							rust_tokenizer.encode(lowerCamelCase_       ,						add_special_tokens=lowerCamelCase_  )
								self.assertListEqual(lowerCamelCase_       ,						lowerCamelCase_  )

								__magic_name__							:	str    							=							self.get_rust_tokenizer()
								__magic_name__							:	Dict    							=							tokenizer.encode(lowerCamelCase_  )
								__magic_name__							:	Any    							=							rust_tokenizer.encode(lowerCamelCase_  )
								self.assertListEqual(lowerCamelCase_       ,						lowerCamelCase_  )

			def 							UpperCAmelCase__       ( self			:   Union[str, Any]  )						->   Dict:
								__magic_name__							:	Optional[int]    							=							BasicTokenizer()

								self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''  )       ,						['''ah''', '''\u535A''', '''\u63A8''', '''zz''']  )

			def 							UpperCAmelCase__       ( self			:   Union[str, Any]  )						->   Union[str, Any]:
								__magic_name__							:	str    							=							BasicTokenizer(do_lower_case=lowerCamelCase_  )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  )       ,						['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )       ,						['''hello''']  )

			def 							UpperCAmelCase__       ( self			:   Union[str, Any]  )						->   Optional[int]:
								__magic_name__							:	Optional[int]    							=							BasicTokenizer(do_lower_case=lowerCamelCase_       ,						strip_accents=lowerCamelCase_  )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )       ,						['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )       ,						['''h\u00E9llo''']  )

			def 							UpperCAmelCase__       ( self			:   Optional[int]  )						->   Union[str, Any]:
								__magic_name__							:	List[Any]    							=							BasicTokenizer(do_lower_case=lowerCamelCase_       ,						strip_accents=lowerCamelCase_  )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )       ,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )       ,						['''hello''']  )

			def 							UpperCAmelCase__       ( self			:   List[Any]  )						->   List[Any]:
								__magic_name__							:	List[Any]    							=							BasicTokenizer(do_lower_case=lowerCamelCase_  )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )       ,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )       ,						['''hello''']  )

			def 							UpperCAmelCase__       ( self			:   Optional[int]  )						->   str:
								__magic_name__							:	Optional[Any]    							=							BasicTokenizer(do_lower_case=lowerCamelCase_  )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  )       ,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )

			def 							UpperCAmelCase__       ( self			:   Tuple  )						->   str:
								__magic_name__							:	Optional[int]    							=							BasicTokenizer(do_lower_case=lowerCamelCase_       ,						strip_accents=lowerCamelCase_  )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )       ,						['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )

			def 							UpperCAmelCase__       ( self			:   Optional[Any]  )						->   Dict:
								__magic_name__							:	Any    							=							BasicTokenizer(do_lower_case=lowerCamelCase_       ,						strip_accents=lowerCamelCase_  )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )       ,						['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )

			def 							UpperCAmelCase__       ( self			:   Any  )						->   Any:
								__magic_name__							:	int    							=							BasicTokenizer(do_lower_case=lowerCamelCase_       ,						never_split=['''[UNK]''']  )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''  )       ,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']  )

			def 							UpperCAmelCase__       ( self			:   Tuple  )						->   Union[str, Any]:
								__magic_name__							:	Tuple    							=							['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

								__magic_name__							:	List[str]    							=							{}
								for i, token in enumerate(lowerCamelCase_  ):
													__magic_name__							:	int    							=							i
								__magic_name__							:	List[Any]    							=							WordpieceTokenizer(vocab=lowerCamelCase_       ,						unk_token='''[UNK]'''  )

								self.assertListEqual(tokenizer.tokenize(''''''  )       ,						[]  )

								self.assertListEqual(tokenizer.tokenize('''unwanted running'''  )       ,						['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']  )

								self.assertListEqual(tokenizer.tokenize('''unwantedX running'''  )       ,						['''[UNK]''', '''runn''', '''##ing''']  )

			def 							UpperCAmelCase__       ( self			:   List[Any]  )						->   Tuple:
								self.assertTrue(_is_whitespace(''' '''  )  )
								self.assertTrue(_is_whitespace('''\t'''  )  )
								self.assertTrue(_is_whitespace('''\r'''  )  )
								self.assertTrue(_is_whitespace('''\n'''  )  )
								self.assertTrue(_is_whitespace('''\u00A0'''  )  )

								self.assertFalse(_is_whitespace('''A'''  )  )
								self.assertFalse(_is_whitespace('''-'''  )  )

			def 							UpperCAmelCase__       ( self			:   Optional[int]  )						->   int:
								self.assertTrue(_is_control('''\u0005'''  )  )

								self.assertFalse(_is_control('''A'''  )  )
								self.assertFalse(_is_control(''' '''  )  )
								self.assertFalse(_is_control('''\t'''  )  )
								self.assertFalse(_is_control('''\r'''  )  )

			def 							UpperCAmelCase__       ( self			:   Optional[int]  )						->   Optional[Any]:
								self.assertTrue(_is_punctuation('''-'''  )  )
								self.assertTrue(_is_punctuation('''$'''  )  )
								self.assertTrue(_is_punctuation('''`'''  )  )
								self.assertTrue(_is_punctuation('''.'''  )  )

								self.assertFalse(_is_punctuation('''A'''  )  )
								self.assertFalse(_is_punctuation(''' '''  )  )

			def 							UpperCAmelCase__       ( self			:   Union[str, Any]  )						->   List[Any]:
								__magic_name__							:	Dict    							=							self.get_tokenizer()
								__magic_name__							:	List[str]    							=							self.get_rust_tokenizer()

								# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
								self.assertListEqual([tokenizer.tokenize(lowerCamelCase_  ) for t in ['''Test''', '''\xad''', '''test''']]       ,						[['''[UNK]'''], [], ['''[UNK]''']]  )

								self.assertListEqual(
								    [rust_tokenizer.tokenize(lowerCamelCase_  ) for t in ['''Test''', '''\xad''', '''test''']]       ,						[['''[UNK]'''], [], ['''[UNK]''']]  )

			@slow
			def 							UpperCAmelCase__       ( self			:   Optional[int]  )						->   Any:
								__magic_name__							:	List[Any]    							=							self.tokenizer_class.from_pretrained('''google/mobilebert-uncased'''  )

								__magic_name__							:	Dict    							=							tokenizer.encode('''sequence builders'''       ,						add_special_tokens=lowerCamelCase_  )
								__magic_name__							:	List[Any]    							=							tokenizer.encode('''multi-sequence build'''       ,						add_special_tokens=lowerCamelCase_  )

								__magic_name__							:	Optional[int]    							=							tokenizer.build_inputs_with_special_tokens(lowerCamelCase_  )
								__magic_name__							:	Union[str, Any]    							=							tokenizer.build_inputs_with_special_tokens(lowerCamelCase_       ,						lowerCamelCase_  )

								assert encoded_sentence == [101] + text + [102]
								assert encoded_pair == [101] + text + [102] + text_a + [102]

			def 							UpperCAmelCase__       ( self			:   List[Any]  )						->   int:
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																		__magic_name__							:	Any    							=							self.rust_tokenizer_class.from_pretrained(lowerCamelCase_       ,						**lowerCamelCase_  )

																		__magic_name__							:	Tuple    							=							F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																		__magic_name__							:	Tuple    							=							tokenizer_r.encode_plus(
																		    lowerCamelCase_       ,						return_attention_mask=lowerCamelCase_       ,						return_token_type_ids=lowerCamelCase_       ,						return_offsets_mapping=lowerCamelCase_       ,						add_special_tokens=lowerCamelCase_       ,						)

																		__magic_name__							:	Union[str, Any]    							=							tokenizer_r.do_lower_case if hasattr(lowerCamelCase_       ,						'''do_lower_case'''  ) else False
																		__magic_name__							:	Optional[int]    							=							(
																		    [
																		        ((0, 0), tokenizer_r.cls_token),
																		        ((0, 1), '''A'''),
																		        ((1, 2), ''','''),
																		        ((3, 5), '''na'''),
																		        ((5, 6), '''##ï'''),
																		        ((6, 8), '''##ve'''),
																		        ((9, 15), tokenizer_r.mask_token),
																		        ((16, 21), '''Allen'''),
																		        ((21, 23), '''##NL'''),
																		        ((23, 24), '''##P'''),
																		        ((25, 33), '''sentence'''),
																		        ((33, 34), '''.'''),
																		        ((0, 0), tokenizer_r.sep_token),
																		    ]
																		    if not do_lower_case
																		    else [
																		        ((0, 0), tokenizer_r.cls_token),
																		        ((0, 1), '''a'''),
																		        ((1, 2), ''','''),
																		        ((3, 8), '''naive'''),
																		        ((9, 15), tokenizer_r.mask_token),
																		        ((16, 21), '''allen'''),
																		        ((21, 23), '''##nl'''),
																		        ((23, 24), '''##p'''),
																		        ((25, 33), '''sentence'''),
																		        ((33, 34), '''.'''),
																		        ((0, 0), tokenizer_r.sep_token),
																		    ]
																		)

																		self.assertEqual(
																		    [e[1] for e in expected_results]       ,						tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']  )  )
																		self.assertEqual([e[0] for e in expected_results]       ,						tokens['''offset_mapping''']  )


			def 							UpperCAmelCase__       ( self			:   Union[str, Any]  )						->   Optional[Any]:
								__magic_name__							:	Optional[Any]    							=							['''的''', '''人''', '''有''']
								__magic_name__							:	Dict    							=							''''''.join(lowerCamelCase_  )
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																		__magic_name__							:	int    							=							True
																		__magic_name__							:	Union[str, Any]    							=							self.tokenizer_class.from_pretrained(lowerCamelCase_       ,						**lowerCamelCase_  )
																		__magic_name__							:	Dict    							=							self.rust_tokenizer_class.from_pretrained(lowerCamelCase_       ,						**lowerCamelCase_  )

																		__magic_name__							:	Union[str, Any]    							=							tokenizer_p.encode(lowerCamelCase_       ,						add_special_tokens=lowerCamelCase_  )
																		__magic_name__							:	List[str]    							=							tokenizer_r.encode(lowerCamelCase_       ,						add_special_tokens=lowerCamelCase_  )

																		__magic_name__							:	Tuple    							=							tokenizer_r.convert_ids_to_tokens(lowerCamelCase_  )
																		__magic_name__							:	Union[str, Any]    							=							tokenizer_p.convert_ids_to_tokens(lowerCamelCase_  )

																		# it is expected that each Chinese character is not preceded by "##"
																		self.assertListEqual(lowerCamelCase_       ,						lowerCamelCase_  )
																		self.assertListEqual(lowerCamelCase_       ,						lowerCamelCase_  )

																		__magic_name__							:	str    							=							False
																		__magic_name__							:	Any    							=							self.rust_tokenizer_class.from_pretrained(lowerCamelCase_       ,						**lowerCamelCase_  )
																		__magic_name__							:	Any    							=							self.tokenizer_class.from_pretrained(lowerCamelCase_       ,						**lowerCamelCase_  )

																		__magic_name__							:	List[str]    							=							tokenizer_r.encode(lowerCamelCase_       ,						add_special_tokens=lowerCamelCase_  )
																		__magic_name__							:	List[Any]    							=							tokenizer_p.encode(lowerCamelCase_       ,						add_special_tokens=lowerCamelCase_  )

																		__magic_name__							:	Dict    							=							tokenizer_r.convert_ids_to_tokens(lowerCamelCase_  )
																		__magic_name__							:	Optional[Any]    							=							tokenizer_p.convert_ids_to_tokens(lowerCamelCase_  )

																		# it is expected that only the first Chinese character is not preceded by "##".
																		__magic_name__							:	List[Any]    							=							[
																		    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(lowerCamelCase_  )
																		]
																		self.assertListEqual(lowerCamelCase_       ,						lowerCamelCase_  )
																		self.assertListEqual(lowerCamelCase_       ,						lowerCamelCase_  )









from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCamelCase   :						Optional[int]				=			logging.get_logger(__name__)

__lowerCamelCase   :						Dict				=			{}


class    lowerCamelCase    (		_lowerCamelCase						):






			'''simple docstring'''





			UpperCamelCase__								='''llama'''
			UpperCamelCase__								=['''past_key_values''']

			def __init__( self			:   Dict       ,						lowerCamelCase_			:   List[str]=32000       ,						lowerCamelCase_			:   List[Any]=4096       ,						lowerCamelCase_			:   Any=11008       ,						lowerCamelCase_			:   Optional[Any]=32       ,						lowerCamelCase_			:   Dict=32       ,						lowerCamelCase_			:   Optional[Any]=None       ,						lowerCamelCase_			:   Dict="silu"       ,						lowerCamelCase_			:   Optional[int]=2048       ,						lowerCamelCase_			:   int=0.0_2       ,						lowerCamelCase_			:   Tuple=1E-6       ,						lowerCamelCase_			:   Tuple=True       ,						lowerCamelCase_			:   List[str]=0       ,						lowerCamelCase_			:   List[str]=1       ,						lowerCamelCase_			:   Optional[int]=2       ,						lowerCamelCase_			:   List[str]=1       ,						lowerCamelCase_			:   Dict=False       ,						lowerCamelCase_			:   int=None       ,						**lowerCamelCase_			:   Tuple       ,						)						->   Tuple:
								__magic_name__							:	Any    							=							vocab_size
								__magic_name__							:	Optional[int]    							=							max_position_embeddings
								__magic_name__							:	Union[str, Any]    							=							hidden_size
								__magic_name__							:	Dict    							=							intermediate_size
								__magic_name__							:	Union[str, Any]    							=							num_hidden_layers
								__magic_name__							:	Tuple    							=							num_attention_heads

								# for backward compatibility
								if num_key_value_heads is None:
													__magic_name__							:	Optional[int]    							=							num_attention_heads

								__magic_name__							:	Optional[int]    							=							num_key_value_heads
								__magic_name__							:	List[str]    							=							hidden_act
								__magic_name__							:	List[str]    							=							initializer_range
								__magic_name__							:	Tuple    							=							rms_norm_eps
								__magic_name__							:	Optional[int]    							=							pretraining_tp
								__magic_name__							:	Any    							=							use_cache
								__magic_name__							:	int    							=							rope_scaling
								self._rope_scaling_validation()

								super().__init__(
								    pad_token_id=lowerCamelCase_       ,						bos_token_id=lowerCamelCase_       ,						eos_token_id=lowerCamelCase_       ,						tie_word_embeddings=lowerCamelCase_       ,						**lowerCamelCase_       ,						)


			def 							UpperCAmelCase__       ( self			:   str  )						->   Tuple:
								if self.rope_scaling is None:
													return

								if not isinstance(self.rope_scaling       ,						lowerCamelCase_  ) or len(self.rope_scaling  ) != 2:
													raise ValueError(
													    '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
													    F'''got {self.rope_scaling}'''  )
								__magic_name__							:	List[str]    							=							self.rope_scaling.get('''type'''       ,						lowerCamelCase_  )
								__magic_name__							:	str    							=							self.rope_scaling.get('''factor'''       ,						lowerCamelCase_  )
								if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
													raise ValueError(
													    F'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'''  )
								if rope_scaling_factor is None or not isinstance(lowerCamelCase_       ,						lowerCamelCase_  ) or rope_scaling_factor <= 1.0:
													raise ValueError(F'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'''  )



