



"""simple docstring"""





import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowercase  							=					get_tests_dir('''fixtures/spiece.model''')




@require_sentencepiece
@require_tokenizers
class 			lowercase__			(      A,					unittest.TestCase    ):
							'''simple docstring'''







							_UpperCAmelCase      				=					DebertaVaTokenizer
							_UpperCAmelCase      				=					DebertaVaTokenizerFast
							_UpperCAmelCase      				=					True
							_UpperCAmelCase      				=					True






							def        lowerCamelCase_   (  self						)			->							Union[str, Any]:
									super().setUp()

									# We have a SentencePiece fixture for testing
									_UpperCAmelCase							 =		DebertaVaTokenizer(UpperCamelCase__	,       unk_token='<unk>'						)
									tokenizer.save_pretrained(self.tmpdirname						)






							def        lowerCamelCase_   (  self	,       snake_case						)			->							str:
									_UpperCAmelCase							 =		'this is a test'
									_UpperCAmelCase							 =		'this is a test'
									return input_text, output_text






							def        lowerCamelCase_   (  self						)			->							List[str]:
									_UpperCAmelCase							 =		'<pad>'
									_UpperCAmelCase							 =		0

									self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase__						)	,       UpperCamelCase__						)
									self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase__						)	,       UpperCamelCase__						)






							def        lowerCamelCase_   (  self						)			->							Any:
									_UpperCAmelCase							 =		list(self.get_tokenizer().get_vocab().keys()						)
									self.assertEqual(vocab_keys[0]	,       '<pad>'						)
									self.assertEqual(vocab_keys[1]	,       '<unk>'						)
									self.assertEqual(vocab_keys[-1]	,       '[PAD]'						)
									self.assertEqual(len(UpperCamelCase__						)	,       30001						)






							def        lowerCamelCase_   (  self						)			->							Any:
									self.assertEqual(self.get_tokenizer().vocab_size	,       30000						)






							def        lowerCamelCase_   (  self						)			->							Union[str, Any]:
									_UpperCAmelCase							 =		' \tHeLLo!how  \n Are yoU?  '
									_UpperCAmelCase							 =		['▁hello', '!', 'how', '▁are', '▁you', '?']
									# fmt: on

									_UpperCAmelCase							 =		DebertaVaTokenizer(UpperCamelCase__	,       do_lower_case=UpperCamelCase__						)
									_UpperCAmelCase							 =		tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase__	,       add_special_tokens=UpperCamelCase__						)						)

									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)

									_UpperCAmelCase							 =		DebertaVaTokenizerFast(UpperCamelCase__	,       do_lower_case=UpperCamelCase__						)
									_UpperCAmelCase							 =		rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase__	,       add_special_tokens=UpperCamelCase__						)						)

									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)






							@unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'						)
							def        lowerCamelCase_   (  self						)			->							List[Any]:
									pass






							@unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'						)
							def        lowerCamelCase_   (  self						)			->							Optional[int]:
									pass






							def        lowerCamelCase_   (  self						)			->							int:
									_UpperCAmelCase							 =		'I was born in 92000, and this is falsé.'
									_UpperCAmelCase							 =		['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
									# fmt: on

									_UpperCAmelCase							 =		DebertaVaTokenizer(UpperCamelCase__	,       split_by_punct=UpperCamelCase__						)
									_UpperCAmelCase							 =		tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase__	,       add_special_tokens=UpperCamelCase__						)						)

									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)

									_UpperCAmelCase							 =		DebertaVaTokenizerFast(UpperCamelCase__	,       split_by_punct=UpperCamelCase__						)
									_UpperCAmelCase							 =		rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase__	,       add_special_tokens=UpperCamelCase__						)						)

									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)






							def        lowerCamelCase_   (  self						)			->							str:
									_UpperCAmelCase							 =		'I was born in 92000, and this is falsé.'
									_UpperCAmelCase							 =		['▁i', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
									# fmt: on

									_UpperCAmelCase							 =		DebertaVaTokenizer(UpperCamelCase__	,       do_lower_case=UpperCamelCase__	,       split_by_punct=UpperCamelCase__						)
									_UpperCAmelCase							 =		tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase__	,       add_special_tokens=UpperCamelCase__						)						)
									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)

									_UpperCAmelCase							 =		DebertaVaTokenizerFast(UpperCamelCase__	,       do_lower_case=UpperCamelCase__	,       split_by_punct=UpperCamelCase__						)
									_UpperCAmelCase							 =		rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase__	,       add_special_tokens=UpperCamelCase__						)						)
									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)






							def        lowerCamelCase_   (  self						)			->							Any:
									_UpperCAmelCase							 =		'I was born in 92000, and this is falsé.'
									_UpperCAmelCase							 =		['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
									# fmt: on

									_UpperCAmelCase							 =		DebertaVaTokenizer(UpperCamelCase__	,       do_lower_case=UpperCamelCase__	,       split_by_punct=UpperCamelCase__						)
									_UpperCAmelCase							 =		tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase__	,       add_special_tokens=UpperCamelCase__						)						)

									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)

									_UpperCAmelCase							 =		DebertaVaTokenizerFast(UpperCamelCase__	,       do_lower_case=UpperCamelCase__	,       split_by_punct=UpperCamelCase__						)
									_UpperCAmelCase							 =		rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase__	,       add_special_tokens=UpperCamelCase__						)						)

									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)






							def        lowerCamelCase_   (  self						)			->							str:
									_UpperCAmelCase							 =		'I was born in 92000, and this is falsé.'
									_UpperCAmelCase							 =		['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
									# fmt: on

									_UpperCAmelCase							 =		DebertaVaTokenizer(UpperCamelCase__	,       do_lower_case=UpperCamelCase__	,       split_by_punct=UpperCamelCase__						)
									_UpperCAmelCase							 =		tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase__	,       add_special_tokens=UpperCamelCase__						)						)

									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)

									_UpperCAmelCase							 =		DebertaVaTokenizerFast(UpperCamelCase__	,       do_lower_case=UpperCamelCase__	,       split_by_punct=UpperCamelCase__						)
									_UpperCAmelCase							 =		rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase__	,       add_special_tokens=UpperCamelCase__						)						)

									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)






							def        lowerCamelCase_   (  self						)			->							int:
									_UpperCAmelCase							 =		' \tHeLLo!how  \n Are yoU?  '
									_UpperCAmelCase							 =		['▁', '<unk>', 'e', '<unk>', 'o', '!', 'how', '▁', '<unk>', 're', '▁yo', '<unk>', '?']
									# fmt: on

									_UpperCAmelCase							 =		DebertaVaTokenizer(UpperCamelCase__	,       do_lower_case=UpperCamelCase__	,       split_by_punct=UpperCamelCase__						)
									_UpperCAmelCase							 =		tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase__	,       add_special_tokens=UpperCamelCase__						)						)

									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)

									_UpperCAmelCase							 =		DebertaVaTokenizerFast(UpperCamelCase__	,       do_lower_case=UpperCamelCase__	,       split_by_punct=UpperCamelCase__						)
									_UpperCAmelCase							 =		rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase__	,       add_special_tokens=UpperCamelCase__						)						)

									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)






							def        lowerCamelCase_   (  self						)			->							Optional[int]:
									_UpperCAmelCase							 =		self.get_tokenizer()
									_UpperCAmelCase							 =		self.get_rust_tokenizer()

									_UpperCAmelCase							 =		'I was born in 92000, and this is falsé.'

									_UpperCAmelCase							 =		tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase__	,       add_special_tokens=UpperCamelCase__						)						)
									_UpperCAmelCase							 =		rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase__	,       add_special_tokens=UpperCamelCase__						)						)
									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)

									_UpperCAmelCase							 =		tokenizer.encode(UpperCamelCase__	,       add_special_tokens=UpperCamelCase__						)
									_UpperCAmelCase							 =		rust_tokenizer.encode(UpperCamelCase__	,       add_special_tokens=UpperCamelCase__						)
									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)

									_UpperCAmelCase							 =		self.get_rust_tokenizer()
									_UpperCAmelCase							 =		tokenizer.encode(UpperCamelCase__						)
									_UpperCAmelCase							 =		rust_tokenizer.encode(UpperCamelCase__						)
									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)






							def        lowerCamelCase_   (  self						)			->							Dict:
									_UpperCAmelCase							 =		'This is a test'
									_UpperCAmelCase							 =		[13, 1, 4398, 25, 21, 1289]
									_UpperCAmelCase							 =		['▁', 'T', 'his', '▁is', '▁a', '▁test']
									_UpperCAmelCase							 =		['▁', '<unk>', 'his', '▁is', '▁a', '▁test']

									_UpperCAmelCase							 =		DebertaVaTokenizer(UpperCamelCase__	,       keep_accents=UpperCamelCase__						)
									_UpperCAmelCase							 =		DebertaVaTokenizerFast(UpperCamelCase__	,       keep_accents=UpperCamelCase__						)

									_UpperCAmelCase							 =		tokenizer.encode(UpperCamelCase__	,       add_special_tokens=UpperCamelCase__						)
									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)
									_UpperCAmelCase							 =		tokenizer.tokenize(UpperCamelCase__						)
									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)
									_UpperCAmelCase							 =		tokenizer.convert_ids_to_tokens(UpperCamelCase__						)
									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)

									_UpperCAmelCase							 =		rust_tokenizer.encode(UpperCamelCase__	,       add_special_tokens=UpperCamelCase__						)
									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)
									_UpperCAmelCase							 =		rust_tokenizer.tokenize(UpperCamelCase__						)
									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)
									_UpperCAmelCase							 =		rust_tokenizer.convert_ids_to_tokens(UpperCamelCase__						)
									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)

									# fmt: off
									_UpperCAmelCase							 =		'I was born in 92000, and this is falsé.'
									_UpperCAmelCase							 =		[13, 1, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]
									_UpperCAmelCase							 =		['▁', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.', ]
									_UpperCAmelCase							 =		['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
									# fmt: on

									_UpperCAmelCase							 =		tokenizer.encode(UpperCamelCase__	,       add_special_tokens=UpperCamelCase__						)
									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)
									_UpperCAmelCase							 =		tokenizer.tokenize(UpperCamelCase__						)
									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)
									_UpperCAmelCase							 =		tokenizer.convert_ids_to_tokens(UpperCamelCase__						)
									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)

									_UpperCAmelCase							 =		rust_tokenizer.encode(UpperCamelCase__	,       add_special_tokens=UpperCamelCase__						)
									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)
									_UpperCAmelCase							 =		rust_tokenizer.tokenize(UpperCamelCase__						)
									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)
									_UpperCAmelCase							 =		rust_tokenizer.convert_ids_to_tokens(UpperCamelCase__						)
									self.assertListEqual(UpperCamelCase__	,       UpperCamelCase__						)






							def        lowerCamelCase_   (  self						)			->							Union[str, Any]:
									_UpperCAmelCase							 =		DebertaVaTokenizer(UpperCamelCase__						)

									_UpperCAmelCase							 =		tokenizer.encode('sequence builders'						)
									_UpperCAmelCase							 =		tokenizer.encode('multi-sequence build'						)

									_UpperCAmelCase							 =		tokenizer.build_inputs_with_special_tokens(UpperCamelCase__						)
									_UpperCAmelCase							 =		tokenizer.build_inputs_with_special_tokens(UpperCamelCase__	,       UpperCamelCase__						)

									self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]	,       UpperCamelCase__						)
									self.assertEqual(
									    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]	,       UpperCamelCase__	,       )






							@slow
							def        lowerCamelCase_   (  self						)			->							Any:
									_UpperCAmelCase							 =		{'input_ids': [[1, 39867, 36, 19390, 486, 27, 35052, 81436, 18, 60685, 1225, 7, 35052, 81436, 18, 9367, 16899, 18, 15937, 53, 594, 773, 18, 16287, 30465, 36, 15937, 6, 41139, 38, 36979, 60763, 191, 6, 34132, 99, 6, 50538, 390, 43230, 6, 34132, 2779, 20850, 14, 699, 1072, 1194, 36, 382, 10901, 53, 7, 699, 1072, 2084, 36, 20422, 630, 53, 19, 105, 3049, 1896, 1053, 16899, 1506, 11, 37978, 4243, 7, 1237, 31869, 200, 16566, 654, 6, 35052, 81436, 7, 55630, 13593, 4, 2], [1, 26, 15011, 13, 667, 8, 1053, 18, 23611, 1237, 72356, 12820, 34, 104134, 1209, 35, 13313, 6627, 21, 202, 347, 7, 164, 2399, 11, 46, 4485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1232, 2864, 15785, 14951, 105, 5, 8581, 1250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
									# fmt: on

									self.tokenizer_integration_test_util(
									    expected_encoding=UpperCamelCase__	,       model_name='microsoft/deberta-v2-xlarge'	,       revision='ad6e42c1532ddf3a15c39246b63f5559d558b670'	,       )







"""simple docstring"""





import warnings

from ...utils import logging
from .image_processing_yolos import YolosImageProcessor


lowercase  							=					logging.get_logger(__name__)




class 			lowercase__			(      A    ):
							'''simple docstring'''







							def __init__(  self	,       *snake_case	,       **snake_case						)			->							None:
									warnings.warn(
									    'The class YolosFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please'
									    ' use YolosImageProcessor instead.'	,       snake_case	,       )
									super().__init__(*snake_case	,       **snake_case						)


