



"""simple docstring"""





import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowercase  							=					get_tests_dir('''fixtures/test_sentencepiece.model''')




@require_sentencepiece
@require_tokenizers
class 			lowercase__			(      lowercase__,					unittest.TestCase    ):
							'''simple docstring'''







							_UpperCAmelCase      				=					XLNetTokenizer
							_UpperCAmelCase      				=					XLNetTokenizerFast
							_UpperCAmelCase      				=					True
							_UpperCAmelCase      				=					True






							def        lowerCamelCase_   (  self						)			->							Optional[int]:
									super().setUp()

									# We have a SentencePiece fixture for testing
									_UpperCAmelCase							 =		XLNetTokenizer(UpperCAmelCase__	,       keep_accents=UpperCAmelCase__						)
									tokenizer.sanitize_special_tokens()
									tokenizer.save_pretrained(self.tmpdirname						)






							def        lowerCamelCase_   (  self						)			->							Optional[Any]:
									_UpperCAmelCase							 =		'''<s>'''
									_UpperCAmelCase							 =		1

									self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase__						)	,       UpperCAmelCase__						)
									self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase__						)	,       UpperCAmelCase__						)






							def        lowerCamelCase_   (  self						)			->							Union[str, Any]:
									_UpperCAmelCase							 =		list(self.get_tokenizer().get_vocab().keys()						)

									self.assertEqual(vocab_keys[0]	,       '<unk>'						)
									self.assertEqual(vocab_keys[1]	,       '<s>'						)
									self.assertEqual(vocab_keys[-1]	,       '<eod>'						)
									self.assertEqual(len(UpperCAmelCase__						)	,       1006						)






							def        lowerCamelCase_   (  self						)			->							Any:
									self.assertEqual(self.get_tokenizer().vocab_size	,       1000						)






							def        lowerCamelCase_   (  self						)			->							Any:
									_UpperCAmelCase							 =		XLNetTokenizer(UpperCAmelCase__	,       keep_accents=UpperCAmelCase__						)

									_UpperCAmelCase							 =		tokenizer.tokenize('This is a test'						)
									self.assertListEqual(UpperCAmelCase__	,       ['▁This', '▁is', '▁a', '▁t', 'est']						)

									self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__						)	,       [285, 46, 10, 170, 382]						)

									_UpperCAmelCase							 =		tokenizer.tokenize('I was born in 92000, and this is falsé.'						)
									self.assertListEqual(
									    UpperCAmelCase__	,       [
									        SPIECE_UNDERLINE + 'I',
									        SPIECE_UNDERLINE + 'was',
									        SPIECE_UNDERLINE + 'b',
									        'or',
									        'n',
									        SPIECE_UNDERLINE + 'in',
									        SPIECE_UNDERLINE + '',
									        '9',
									        '2',
									        '0',
									        '0',
									        '0',
									        ',',
									        SPIECE_UNDERLINE + 'and',
									        SPIECE_UNDERLINE + 'this',
									        SPIECE_UNDERLINE + 'is',
									        SPIECE_UNDERLINE + 'f',
									        'al',
									        's',
									        'é',
									        '.',
									    ]	,       )
									_UpperCAmelCase							 =		tokenizer.convert_tokens_to_ids(UpperCAmelCase__						)
									self.assertListEqual(UpperCAmelCase__	,       [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]						)

									_UpperCAmelCase							 =		tokenizer.convert_ids_to_tokens(UpperCAmelCase__						)
									self.assertListEqual(
									    UpperCAmelCase__	,       [
									        SPIECE_UNDERLINE + 'I',
									        SPIECE_UNDERLINE + 'was',
									        SPIECE_UNDERLINE + 'b',
									        'or',
									        'n',
									        SPIECE_UNDERLINE + 'in',
									        SPIECE_UNDERLINE + '',
									        '<unk>',
									        '2',
									        '0',
									        '0',
									        '0',
									        ',',
									        SPIECE_UNDERLINE + 'and',
									        SPIECE_UNDERLINE + 'this',
									        SPIECE_UNDERLINE + 'is',
									        SPIECE_UNDERLINE + 'f',
									        'al',
									        's',
									        '<unk>',
									        '.',
									    ]	,       )






							def        lowerCamelCase_   (  self						)			->							int:
									_UpperCAmelCase							 =		XLNetTokenizer(UpperCAmelCase__	,       do_lower_case=UpperCAmelCase__						)
									_UpperCAmelCase							 =		tokenizer.tokenize('I was born in 92000, and this is falsé.'						)
									self.assertListEqual(
									    UpperCAmelCase__	,       [
									        SPIECE_UNDERLINE + '',
									        'i',
									        SPIECE_UNDERLINE + 'was',
									        SPIECE_UNDERLINE + 'b',
									        'or',
									        'n',
									        SPIECE_UNDERLINE + 'in',
									        SPIECE_UNDERLINE + '',
									        '9',
									        '2',
									        '0',
									        '0',
									        '0',
									        ',',
									        SPIECE_UNDERLINE + 'and',
									        SPIECE_UNDERLINE + 'this',
									        SPIECE_UNDERLINE + 'is',
									        SPIECE_UNDERLINE + 'f',
									        'al',
									        'se',
									        '.',
									    ]	,       )
									self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)	,       ['▁he', 'll', 'o']						)






							def        lowerCamelCase_   (  self						)			->							Any:
									_UpperCAmelCase							 =		XLNetTokenizer(UpperCAmelCase__	,       do_lower_case=UpperCAmelCase__						)
									_UpperCAmelCase							 =		tokenizer.tokenize('I was born in 92000, and this is falsé.'						)
									self.assertListEqual(
									    UpperCAmelCase__	,       [
									        SPIECE_UNDERLINE + 'I',
									        SPIECE_UNDERLINE + 'was',
									        SPIECE_UNDERLINE + 'b',
									        'or',
									        'n',
									        SPIECE_UNDERLINE + 'in',
									        SPIECE_UNDERLINE + '',
									        '9',
									        '2',
									        '0',
									        '0',
									        '0',
									        ',',
									        SPIECE_UNDERLINE + 'and',
									        SPIECE_UNDERLINE + 'this',
									        SPIECE_UNDERLINE + 'is',
									        SPIECE_UNDERLINE + 'f',
									        'al',
									        'se',
									        '.',
									    ]	,       )






							@slow
							def        lowerCamelCase_   (  self						)			->							Optional[int]:
									_UpperCAmelCase							 =		XLNetTokenizer.from_pretrained('xlnet-base-cased'						)

									_UpperCAmelCase							 =		tokenizer.encode('sequence builders'	,       add_special_tokens=UpperCAmelCase__						)
									_UpperCAmelCase							 =		tokenizer.encode('multi-sequence build'	,       add_special_tokens=UpperCAmelCase__						)

									_UpperCAmelCase							 =		tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__						)
									_UpperCAmelCase							 =		tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__	,       UpperCAmelCase__						)

									assert encoded_sentence == text + [4, 3]
									assert encoded_pair == text + [4] + text_a + [4, 3]






							@slow
							def        lowerCamelCase_   (  self						)			->							Optional[int]:
									_UpperCAmelCase							 =		{'''input_ids''': [[17, 21442, 270, 17, 10, 14645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 22018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 14431, 13, 5500, 11, 1176, 580, 13, 16819, 4797, 23, 17, 10, 17135, 658, 19, 457, 7932, 13, 184, 19, 3154, 17135, 6468, 19, 1404, 12269, 19, 4229, 5356, 16264, 46, 19, 17, 20545, 10395, 9, 9, 9, 11, 28, 6421, 9531, 20729, 17, 10, 353, 17022, 11, 21, 6421, 9531, 16949, 17, 10, 11509, 753, 11, 33, 95, 2421, 7385, 956, 14431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 24738, 19, 13203, 658, 218, 787, 21, 430, 18482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22178, 27, 1064, 22, 956, 13, 11101, 1429, 5854, 24313, 18953, 40, 422, 24366, 68, 1758, 37, 10483, 14257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 13894, 3380, 23, 95, 18, 17634, 2288, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
									# fmt: on

									self.tokenizer_integration_test_util(
									    expected_encoding=UpperCAmelCase__	,       model_name='xlnet-base-cased'	,       revision='c841166438c31ec7ca9a106dee7bb312b73ae511'	,       )







"""simple docstring"""





import warnings

from ...utils import logging
from .image_processing_yolos import YolosImageProcessor


lowercase  							=					logging.get_logger(__name__)




class 			lowercase__			(      A    ):
							'''simple docstring'''







							def __init__(  self	,       *snake_case	,       **snake_case						)			->							None:
									warnings.warn(
									    'The class YolosFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please'
									    ' use YolosImageProcessor instead.'	,       snake_case	,       )
									super().__init__(*snake_case	,       **snake_case						)


