



'''simple docstring'''



import argparse

from ...utils.dataclasses import (
    ComputeEnvironment,
    DistributedType,
    DynamoBackend,
    PrecisionType,
    SageMakerDistributedType,
)
from ..menu import BulletMenu


lowerCAmelCase_     :       int								=						[
    'EAGER',
    'AOT_EAGER',
    'INDUCTOR',
    'NVFUSER',
    'AOT_NVFUSER',
    'AOT_CUDAGRAPHS',
    'OFI',
    'FX2TRT',
    'ONNXRT',
    'IPEX',
]
def 					UpperCAmelCase   (						A  :				Optional[Any]					,		A  :				Optional[Any]=None					,		A  :				Optional[int]=None					,		A  :				Optional[int]=None				):
			SCREAMING_SNAKE_CASE					:      Optional[int]     					=			True
			while ask_again:
						SCREAMING_SNAKE_CASE					:      Tuple     					=			input(A				)
						try:
									if default is not None and len(A				) == 0:
												return default
									return convert_value(A				) if convert_value is not None else result
						except Exception:
									if error_message is not None:
												print(A				)
def 					UpperCAmelCase   (						A  :				Optional[int]					,		A  :				Tuple=[]					,		A  :				str=None					,		A  :				Optional[int]=0				):
			SCREAMING_SNAKE_CASE					:      Optional[int]     					=			BulletMenu(A					,		A				)
			SCREAMING_SNAKE_CASE					:      Optional[int]     					=			menu.run(default_choice=A				)
			return convert_value(A				) if convert_value is not None else result
def 					UpperCAmelCase   (						A  :				Union[str, Any]				):
			SCREAMING_SNAKE_CASE					:      Dict     					=			int(A				)
			return ComputeEnvironment(['''LOCAL_MACHINE''', '''AMAZON_SAGEMAKER'''][value]				)
def 					UpperCAmelCase   (						A  :				Optional[Any]				):
			SCREAMING_SNAKE_CASE					:      int     					=			int(A				)
			return DistributedType(['''NO''', '''MULTI_CPU''', '''MULTI_XPU''', '''MULTI_GPU''', '''MULTI_NPU''', '''TPU'''][value]				)
def 					UpperCAmelCase   (						A  :				List[Any]				):
			SCREAMING_SNAKE_CASE					:      Dict     					=			int(A				)
			return DynamoBackend(DYNAMO_BACKENDS[value]				).value
def 					UpperCAmelCase   (						A  :				Optional[Any]				):
			SCREAMING_SNAKE_CASE					:      List[str]     					=			int(A				)
			return PrecisionType(['''no''', '''fp16''', '''bf16''', '''fp8'''][value]				)
def 					UpperCAmelCase   (						A  :				Dict				):
			SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			int(A				)
			return SageMakerDistributedType(['''NO''', '''DATA_PARALLEL''', '''MODEL_PARALLEL'''][value]				)
def 					UpperCAmelCase   (						A  :				Union[str, Any]				):
			return {"yes": True, "no": False}[value.lower()]






class 	lowerCamelCase_							(					argparse.RawDescriptionHelpFormatter			):







						def 					__lowercase    (  self   :      Optional[int]				,    lowerCAmelCase__   :      Optional[int]				,    lowerCAmelCase__   :      int				,    lowerCAmelCase__   :      Union[str, Any]				,    lowerCAmelCase__   :      List[Any]				):

									"""simple docstring"""
									SCREAMING_SNAKE_CASE					:      List[str]     					=			super()._format_usage(lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				)
									SCREAMING_SNAKE_CASE					:      int     					=			usage.replace('''<command> [<args>] '''				,    ''''''				)
									return usage









'''simple docstring'''



import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase_     :       Dict								=						get_tests_dir('fixtures/spiece.model')
@require_sentencepiece
@require_tokenizers
class 	lowerCamelCase_							(					snake_case_    ,				unittest.TestCase			):
						_lowerCAmelCase     :   Union[str, Any]	     =				DebertaVaTokenizer
						_lowerCAmelCase     :   List[Any]	     =				DebertaVaTokenizerFast
						_lowerCAmelCase     :   Optional[Any]	     =				True
						_lowerCAmelCase     :   int	     =				True







						def 					__lowercase    (  self   :      Optional[Any]				):

													"""simple docstring"""
													super().setUp()

													# We have a SentencePiece fixture for testing
													SCREAMING_SNAKE_CASE					:      Tuple     					=			DebertaVaTokenizer(lowerCAmelCase__				,    unk_token='''<unk>'''				)
													tokenizer.save_pretrained(self.tmpdirname				)







						def 					__lowercase    (  self   :      List[str]				,    lowerCAmelCase__   :      Any				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      List[str]     					=			'''this is a test'''
													SCREAMING_SNAKE_CASE					:      Tuple     					=			'''this is a test'''
													return input_text, output_text







						def 					__lowercase    (  self   :      int				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			'''<pad>'''
													SCREAMING_SNAKE_CASE					:      Tuple     					=			0

													self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase__				)				,    lowerCAmelCase__				)
													self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase__				)				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Optional[int]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Tuple     					=			list(self.get_tokenizer().get_vocab().keys()				)
													self.assertEqual(vocab_keys[0]				,    '''<pad>'''				)
													self.assertEqual(vocab_keys[1]				,    '''<unk>'''				)
													self.assertEqual(vocab_keys[-1]				,    '''[PAD]'''				)
													self.assertEqual(len(lowerCAmelCase__				)				,    3_00_01				)







						def 					__lowercase    (  self   :      str				):

													"""simple docstring"""
													self.assertEqual(self.get_tokenizer().vocab_size				,    3_00_00				)







						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													# fmt: off
													SCREAMING_SNAKE_CASE					:      int     					=			''' \tHeLLo!how  \n Are yoU?  '''
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			['''▁hello''', '''!''', '''how''', '''▁are''', '''▁you''', '''?''']
													# fmt: on

													SCREAMING_SNAKE_CASE					:      Any     					=			DebertaVaTokenizer(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Tuple     					=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Dict     					=			DebertaVaTokenizerFast(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						@unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''				)
						def 					__lowercase    (  self   :      Dict				):

													"""simple docstring"""
													pass







						@unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''				)
						def 					__lowercase    (  self   :      str				):

													"""simple docstring"""
													pass







						def 					__lowercase    (  self   :      int				):

													"""simple docstring"""
													# fmt: off
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			'''I was born in 92000, and this is falsé.'''
													SCREAMING_SNAKE_CASE					:      Any     					=			['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
													# fmt: on

													SCREAMING_SNAKE_CASE					:      Dict     					=			DebertaVaTokenizer(lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			DebertaVaTokenizerFast(lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      str     					=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Optional[Any]				):

													"""simple docstring"""
													# fmt: off
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			'''I was born in 92000, and this is falsé.'''
													SCREAMING_SNAKE_CASE					:      List[str]     					=			['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
													# fmt: on

													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			DebertaVaTokenizer(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      int     					=			DebertaVaTokenizerFast(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Any     					=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Any				):

													"""simple docstring"""
													# fmt: off
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			'''I was born in 92000, and this is falsé.'''
													SCREAMING_SNAKE_CASE					:      Dict     					=			['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
													# fmt: on

													SCREAMING_SNAKE_CASE					:      Tuple     					=			DebertaVaTokenizer(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Any     					=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Any     					=			DebertaVaTokenizerFast(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Dict				):

													"""simple docstring"""
													# fmt: off
													SCREAMING_SNAKE_CASE					:      str     					=			'''I was born in 92000, and this is falsé.'''
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
													# fmt: on

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			DebertaVaTokenizer(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      str     					=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			DebertaVaTokenizerFast(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      List[str]     					=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Union[str, Any]				):

													"""simple docstring"""
													# fmt: off
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			''' \tHeLLo!how  \n Are yoU?  '''
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			['''▁''', '''<unk>''', '''e''', '''<unk>''', '''o''', '''!''', '''how''', '''▁''', '''<unk>''', '''re''', '''▁yo''', '''<unk>''', '''?''']
													# fmt: on

													SCREAMING_SNAKE_CASE					:      List[str]     					=			DebertaVaTokenizer(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Any     					=			DebertaVaTokenizerFast(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      List[str]     					=			self.get_tokenizer()
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			self.get_rust_tokenizer()

													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			'''I was born in 92000, and this is falsé.'''

													SCREAMING_SNAKE_CASE					:      Tuple     					=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)
													SCREAMING_SNAKE_CASE					:      List[str]     					=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      str     					=			rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Tuple     					=			self.get_rust_tokenizer()
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			tokenizer.encode(lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Any     					=			rust_tokenizer.encode(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Tuple     					=			'''This is a test'''
													SCREAMING_SNAKE_CASE					:      List[str]     					=			[13, 1, 43_98, 25, 21, 12_89]
													SCREAMING_SNAKE_CASE					:      Tuple     					=			['''▁''', '''T''', '''his''', '''▁is''', '''▁a''', '''▁test''']
													SCREAMING_SNAKE_CASE					:      Any     					=			['''▁''', '''<unk>''', '''his''', '''▁is''', '''▁a''', '''▁test''']

													SCREAMING_SNAKE_CASE					:      List[str]     					=			DebertaVaTokenizer(lowerCAmelCase__				,    keep_accents=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      str     					=			DebertaVaTokenizerFast(lowerCAmelCase__				,    keep_accents=lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			tokenizer.tokenize(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Any     					=			tokenizer.convert_ids_to_tokens(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      List[str]     					=			rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      str     					=			rust_tokenizer.tokenize(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Any     					=			rust_tokenizer.convert_ids_to_tokens(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													# fmt: off
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			'''I was born in 92000, and this is falsé.'''
													SCREAMING_SNAKE_CASE					:      int     					=			[13, 1, 23, 3_86, 19, 5_61, 30_50, 15, 17, 48, 25, 82_56, 18, 1, 9]
													SCREAMING_SNAKE_CASE					:      Dict     					=			['''▁''', '''I''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''', ]
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
													# fmt: on

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Tuple     					=			tokenizer.tokenize(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			tokenizer.convert_ids_to_tokens(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      int     					=			rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Tuple     					=			rust_tokenizer.tokenize(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			rust_tokenizer.convert_ids_to_tokens(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Any				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      str     					=			DebertaVaTokenizer(lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      List[str]     					=			tokenizer.encode('''sequence builders'''				)
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			tokenizer.encode('''multi-sequence build'''				)

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Tuple     					=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__				,    lowerCAmelCase__				)

													self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]				,    lowerCAmelCase__				)
													self.assertEqual(
													    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]				,    lowerCAmelCase__				,    )







						@slow
						def 					__lowercase    (  self   :      Optional[Any]				):

													"""simple docstring"""
													# fmt: off
													SCREAMING_SNAKE_CASE					:      str     					=			{'''input_ids''': [[1, 3_98_67, 36, 1_93_90, 4_86, 27, 3_50_52, 8_14_36, 18, 6_06_85, 12_25, 7, 3_50_52, 8_14_36, 18, 93_67, 1_68_99, 18, 1_59_37, 53, 5_94, 7_73, 18, 1_62_87, 3_04_65, 36, 1_59_37, 6, 4_11_39, 38, 3_69_79, 6_07_63, 1_91, 6, 3_41_32, 99, 6, 5_05_38, 3_90, 4_32_30, 6, 3_41_32, 27_79, 2_08_50, 14, 6_99, 10_72, 11_94, 36, 3_82, 1_09_01, 53, 7, 6_99, 10_72, 20_84, 36, 2_04_22, 6_30, 53, 19, 1_05, 30_49, 18_96, 10_53, 1_68_99, 15_06, 11, 3_79_78, 42_43, 7, 12_37, 3_18_69, 2_00, 1_65_66, 6_54, 6, 3_50_52, 8_14_36, 7, 5_56_30, 1_35_93, 4, 2], [1, 26, 1_50_11, 13, 6_67, 8, 10_53, 18, 2_36_11, 12_37, 7_23_56, 1_28_20, 34, 10_41_34, 12_09, 35, 1_33_13, 66_27, 21, 2_02, 3_47, 7, 1_64, 23_99, 11, 46, 44_85, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 12_32, 28_64, 1_57_85, 1_49_51, 1_05, 5, 85_81, 12_50, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
													# fmt: on

													self.tokenizer_integration_test_util(
													    expected_encoding=lowerCAmelCase__				,    model_name='''microsoft/deberta-v2-xlarge'''				,    revision='''ad6e42c1532ddf3a15c39246b63f5559d558b670'''				,    )




