'''simple docstring'''





import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__UpperCamelCase	: int    		=  get_tests_dir('fixtures/test_sentencepiece.model')





@require_sentencepiece
@require_tokenizers
class       lowercase__		(     __a			,							unittest.TestCase):
						UpperCamelCase_  =	XLNetTokenizer
						UpperCamelCase_  =	XLNetTokenizerFast
						UpperCamelCase_  =	True
						UpperCamelCase_  =	True


						def  __A ( self			:       Any		):





												'''simple docstring'''





												super().setUp()

												# We have a SentencePiece fixture for testing
												SCREAMING_SNAKE_CASE							:					List[Any]           =							XLNetTokenizer(lowerCAmelCase_					,	keep_accents=lowerCAmelCase_		)
												tokenizer.sanitize_special_tokens()
												tokenizer.save_pretrained(self.tmpdirname		)


						def  __A ( self			:       str		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Optional[int]           =							'''<s>'''
												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							1

												self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase_		)					,	lowerCAmelCase_		)
												self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase_		)					,	lowerCAmelCase_		)


						def  __A ( self			:       Union[str, Any]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							list(self.get_tokenizer().get_vocab().keys()		)

												self.assertEqual(vocab_keys[0]					,	'''<unk>'''		)
												self.assertEqual(vocab_keys[1]					,	'''<s>'''		)
												self.assertEqual(vocab_keys[-1]					,	'''<eod>'''		)
												self.assertEqual(len(lowerCAmelCase_		)					,	1006		)


						def  __A ( self			:       List[Any]		):





												'''simple docstring'''





												self.assertEqual(self.get_tokenizer().vocab_size					,	1000		)


						def  __A ( self			:       Union[str, Any]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							XLNetTokenizer(lowerCAmelCase_					,	keep_accents=lowerCAmelCase_		)

												SCREAMING_SNAKE_CASE							:					List[Any]           =							tokenizer.tokenize('''This is a test'''		)
												self.assertListEqual(lowerCAmelCase_					,	['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']		)

												self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_		)					,	[285, 46, 10, 170, 382]		)

												SCREAMING_SNAKE_CASE							:					List[str]           =							tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
												self.assertListEqual(
												    lowerCAmelCase_					,	[
												        SPIECE_UNDERLINE + '''I''',
												        SPIECE_UNDERLINE + '''was''',
												        SPIECE_UNDERLINE + '''b''',
												        '''or''',
												        '''n''',
												        SPIECE_UNDERLINE + '''in''',
												        SPIECE_UNDERLINE + '''''',
												        '''9''',
												        '''2''',
												        '''0''',
												        '''0''',
												        '''0''',
												        ''',''',
												        SPIECE_UNDERLINE + '''and''',
												        SPIECE_UNDERLINE + '''this''',
												        SPIECE_UNDERLINE + '''is''',
												        SPIECE_UNDERLINE + '''f''',
												        '''al''',
												        '''s''',
												        '''é''',
												        '''.''',
												    ]					,	)
												SCREAMING_SNAKE_CASE							:					Tuple           =							tokenizer.convert_tokens_to_ids(lowerCAmelCase_		)
												self.assertListEqual(lowerCAmelCase_					,	[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]		)

												SCREAMING_SNAKE_CASE							:					str           =							tokenizer.convert_ids_to_tokens(lowerCAmelCase_		)
												self.assertListEqual(
												    lowerCAmelCase_					,	[
												        SPIECE_UNDERLINE + '''I''',
												        SPIECE_UNDERLINE + '''was''',
												        SPIECE_UNDERLINE + '''b''',
												        '''or''',
												        '''n''',
												        SPIECE_UNDERLINE + '''in''',
												        SPIECE_UNDERLINE + '''''',
												        '''<unk>''',
												        '''2''',
												        '''0''',
												        '''0''',
												        '''0''',
												        ''',''',
												        SPIECE_UNDERLINE + '''and''',
												        SPIECE_UNDERLINE + '''this''',
												        SPIECE_UNDERLINE + '''is''',
												        SPIECE_UNDERLINE + '''f''',
												        '''al''',
												        '''s''',
												        '''<unk>''',
												        '''.''',
												    ]					,	)


						def  __A ( self			:       str		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					str           =							XLNetTokenizer(lowerCAmelCase_					,	do_lower_case=lowerCAmelCase_		)
												SCREAMING_SNAKE_CASE							:					int           =							tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
												self.assertListEqual(
												    lowerCAmelCase_					,	[
												        SPIECE_UNDERLINE + '''''',
												        '''i''',
												        SPIECE_UNDERLINE + '''was''',
												        SPIECE_UNDERLINE + '''b''',
												        '''or''',
												        '''n''',
												        SPIECE_UNDERLINE + '''in''',
												        SPIECE_UNDERLINE + '''''',
												        '''9''',
												        '''2''',
												        '''0''',
												        '''0''',
												        '''0''',
												        ''',''',
												        SPIECE_UNDERLINE + '''and''',
												        SPIECE_UNDERLINE + '''this''',
												        SPIECE_UNDERLINE + '''is''',
												        SPIECE_UNDERLINE + '''f''',
												        '''al''',
												        '''se''',
												        '''.''',
												    ]					,	)
												self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)					,	['''▁he''', '''ll''', '''o''']		)


						def  __A ( self			:       int		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					int           =							XLNetTokenizer(lowerCAmelCase_					,	do_lower_case=lowerCAmelCase_		)
												SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
												self.assertListEqual(
												    lowerCAmelCase_					,	[
												        SPIECE_UNDERLINE + '''I''',
												        SPIECE_UNDERLINE + '''was''',
												        SPIECE_UNDERLINE + '''b''',
												        '''or''',
												        '''n''',
												        SPIECE_UNDERLINE + '''in''',
												        SPIECE_UNDERLINE + '''''',
												        '''9''',
												        '''2''',
												        '''0''',
												        '''0''',
												        '''0''',
												        ''',''',
												        SPIECE_UNDERLINE + '''and''',
												        SPIECE_UNDERLINE + '''this''',
												        SPIECE_UNDERLINE + '''is''',
												        SPIECE_UNDERLINE + '''f''',
												        '''al''',
												        '''se''',
												        '''.''',
												    ]					,	)


						@slow
						def  __A ( self			:       List[str]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							XLNetTokenizer.from_pretrained('''xlnet-base-cased'''		)

												SCREAMING_SNAKE_CASE							:					Optional[int]           =							tokenizer.encode('''sequence builders'''					,	add_special_tokens=lowerCAmelCase_		)
												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							tokenizer.encode('''multi-sequence build'''					,	add_special_tokens=lowerCAmelCase_		)

												SCREAMING_SNAKE_CASE							:					List[str]           =							tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_		)
												SCREAMING_SNAKE_CASE							:					List[Any]           =							tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_					,	lowerCAmelCase_		)

												assert encoded_sentence == text + [4, 3]
												assert encoded_pair == text + [4] + text_a + [4, 3]



						@slow
						def  __A ( self			:       Optional[Any]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Tuple           =							{'''input_ids''': [[17, 2_1442, 270, 17, 10, 1_4645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 2_2018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 1_4431, 13, 5500, 11, 1176, 580, 13, 1_6819, 4797, 23, 17, 10, 1_7135, 658, 19, 457, 7932, 13, 184, 19, 3154, 1_7135, 6468, 19, 1404, 1_2269, 19, 4229, 5356, 1_6264, 46, 19, 17, 2_0545, 1_0395, 9, 9, 9, 11, 28, 6421, 9531, 2_0729, 17, 10, 353, 1_7022, 11, 21, 6421, 9531, 1_6949, 17, 10, 1_1509, 753, 11, 33, 95, 2421, 7385, 956, 1_4431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 2_4738, 19, 1_3203, 658, 218, 787, 21, 430, 1_8482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 2_2178, 27, 1064, 22, 956, 13, 1_1101, 1429, 5854, 2_4313, 1_8953, 40, 422, 2_4366, 68, 1758, 37, 1_0483, 1_4257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 1_3894, 3380, 23, 95, 18, 1_7634, 2288, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
												# fmt: on

												self.tokenizer_integration_test_util(
												    expected_encoding=lowerCAmelCase_					,	model_name='''xlnet-base-cased'''					,	revision='''c841166438c31ec7ca9a106dee7bb312b73ae511'''					,	)






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


__UpperCamelCase	: Tuple    		=  {
    'configuration_ctrl': ['CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP', 'CTRLConfig'],
    'tokenization_ctrl': ['CTRLTokenizer'],
}

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					__UpperCamelCase	: Dict    		=  [
					    'CTRL_PRETRAINED_MODEL_ARCHIVE_LIST',
					    'CTRLForSequenceClassification',
					    'CTRLLMHeadModel',
					    'CTRLModel',
					    'CTRLPreTrainedModel',
					]

try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					__UpperCamelCase	: Any    		=  [
					    'TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST',
					    'TFCTRLForSequenceClassification',
					    'TFCTRLLMHeadModel',
					    'TFCTRLModel',
					    'TFCTRLPreTrainedModel',
					]


if TYPE_CHECKING:
					from .configuration_ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, CTRLConfig
					from .tokenization_ctrl import CTRLTokenizer

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_ctrl import (
										    CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
										    CTRLForSequenceClassification,
										    CTRLLMHeadModel,
										    CTRLModel,
										    CTRLPreTrainedModel,
										)

					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tf_ctrl import (
										    TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TFCTRLForSequenceClassification,
										    TFCTRLLMHeadModel,
										    TFCTRLModel,
										    TFCTRLPreTrainedModel,
										)

else:
					import sys

					__UpperCamelCase	: Optional[Any]    		=  _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





