



'''simple docstring'''



import shutil
import tempfile
import unittest
from unittest.mock import patch

from transformers import (
    DefaultFlowCallback,
    IntervalStrategy,
    PrinterCallback,
    ProgressCallback,
    Trainer,
    TrainerCallback,
    TrainingArguments,
    is_torch_available,
)
from transformers.testing_utils import require_torch


if is_torch_available():
							from transformers.trainer import DEFAULT_CALLBACKS

							from .test_trainer import RegressionDataset, RegressionModelConfig, RegressionPreTrainedModel
class 	lowerCamelCase_							(					snake_case_			):







						def __init__(  self   :      Dict				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      str     					=			[]







						def 					__lowercase    (  self   :      Union[str, Any]				,    lowerCAmelCase__   :      Tuple				,    lowerCAmelCase__   :      List[Any]				,    lowerCAmelCase__   :      Optional[int]				,    **lowerCAmelCase__   :      Optional[Any]				):

													"""simple docstring"""
													self.events.append('''on_init_end'''				)







						def 					__lowercase    (  self   :      Dict				,    lowerCAmelCase__   :      Optional[int]				,    lowerCAmelCase__   :      Union[str, Any]				,    lowerCAmelCase__   :      Any				,    **lowerCAmelCase__   :      List[str]				):

													"""simple docstring"""
													self.events.append('''on_train_begin'''				)







						def 					__lowercase    (  self   :      str				,    lowerCAmelCase__   :      Optional[Any]				,    lowerCAmelCase__   :      Optional[Any]				,    lowerCAmelCase__   :      List[str]				,    **lowerCAmelCase__   :      Optional[Any]				):

													"""simple docstring"""
													self.events.append('''on_train_end'''				)







						def 					__lowercase    (  self   :      int				,    lowerCAmelCase__   :      Any				,    lowerCAmelCase__   :      str				,    lowerCAmelCase__   :      int				,    **lowerCAmelCase__   :      List[str]				):

													"""simple docstring"""
													self.events.append('''on_epoch_begin'''				)







						def 					__lowercase    (  self   :      int				,    lowerCAmelCase__   :      List[str]				,    lowerCAmelCase__   :      Dict				,    lowerCAmelCase__   :      int				,    **lowerCAmelCase__   :      List[str]				):

													"""simple docstring"""
													self.events.append('''on_epoch_end'''				)







						def 					__lowercase    (  self   :      int				,    lowerCAmelCase__   :      List[Any]				,    lowerCAmelCase__   :      Dict				,    lowerCAmelCase__   :      List[Any]				,    **lowerCAmelCase__   :      List[str]				):

													"""simple docstring"""
													self.events.append('''on_step_begin'''				)







						def 					__lowercase    (  self   :      int				,    lowerCAmelCase__   :      Any				,    lowerCAmelCase__   :      Optional[int]				,    lowerCAmelCase__   :      str				,    **lowerCAmelCase__   :      Optional[Any]				):

													"""simple docstring"""
													self.events.append('''on_step_end'''				)







						def 					__lowercase    (  self   :      int				,    lowerCAmelCase__   :      Union[str, Any]				,    lowerCAmelCase__   :      Union[str, Any]				,    lowerCAmelCase__   :      Any				,    **lowerCAmelCase__   :      Dict				):

													"""simple docstring"""
													self.events.append('''on_evaluate'''				)







						def 					__lowercase    (  self   :      str				,    lowerCAmelCase__   :      List[Any]				,    lowerCAmelCase__   :      Union[str, Any]				,    lowerCAmelCase__   :      Any				,    **lowerCAmelCase__   :      List[str]				):

													"""simple docstring"""
													self.events.append('''on_predict'''				)







						def 					__lowercase    (  self   :      Union[str, Any]				,    lowerCAmelCase__   :      str				,    lowerCAmelCase__   :      Any				,    lowerCAmelCase__   :      Dict				,    **lowerCAmelCase__   :      Dict				):

													"""simple docstring"""
													self.events.append('''on_save'''				)







						def 					__lowercase    (  self   :      Dict				,    lowerCAmelCase__   :      Optional[Any]				,    lowerCAmelCase__   :      Tuple				,    lowerCAmelCase__   :      Optional[Any]				,    **lowerCAmelCase__   :      Optional[Any]				):

													"""simple docstring"""
													self.events.append('''on_log'''				)







						def 					__lowercase    (  self   :      Tuple				,    lowerCAmelCase__   :      Optional[Any]				,    lowerCAmelCase__   :      int				,    lowerCAmelCase__   :      Optional[int]				,    **lowerCAmelCase__   :      Union[str, Any]				):

													"""simple docstring"""
													self.events.append('''on_prediction_step'''				)


@require_torch
class 	lowerCamelCase_							(					unittest.TestCase			):







						def 					__lowercase    (  self   :      Optional[int]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      List[str]     					=			tempfile.mkdtemp()







						def 					__lowercase    (  self   :      Optional[int]				):

													"""simple docstring"""
													shutil.rmtree(self.output_dir				)







						def 					__lowercase    (  self   :      Tuple				,    lowerCAmelCase__   :      Any=0				,    lowerCAmelCase__   :      str=0				,    lowerCAmelCase__   :      List[str]=64				,    lowerCAmelCase__   :      Union[str, Any]=64				,    lowerCAmelCase__   :      Optional[int]=None				,    lowerCAmelCase__   :      Tuple=False				,    **lowerCAmelCase__   :      List[Any]				):

													"""simple docstring"""
													# disable_tqdm in TrainingArguments has a flaky default since it depends on the level of logging. We make sure
													# its set to False since the tests later on depend on its value.
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			RegressionDataset(length=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			RegressionDataset(length=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      int     					=			RegressionModelConfig(a=lowerCAmelCase__				,    b=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			RegressionPreTrainedModel(lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			TrainingArguments(self.output_dir				,    disable_tqdm=lowerCAmelCase__				,    report_to=[]				,    **lowerCAmelCase__				)
													return Trainer(
													    lowerCAmelCase__				,    lowerCAmelCase__				,    train_dataset=lowerCAmelCase__				,    eval_dataset=lowerCAmelCase__				,    callbacks=lowerCAmelCase__				,    )







						def 					__lowercase    (  self   :      List[str]				,    lowerCAmelCase__   :      Dict				,    lowerCAmelCase__   :      Union[str, Any]				):

													"""simple docstring"""
													self.assertEqual(len(lowerCAmelCase__				)				,    len(lowerCAmelCase__				)				)

													# Order doesn't matter
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			sorted(lowerCAmelCase__				,    key=lambda lowerCAmelCase__				: cb.__name__ if isinstance(lowerCAmelCase__				,    lowerCAmelCase__				) else cb.__class__.__name__				)
													SCREAMING_SNAKE_CASE					:      Dict     					=			sorted(lowerCAmelCase__				,    key=lambda lowerCAmelCase__				: cb.__name__ if isinstance(lowerCAmelCase__				,    lowerCAmelCase__				) else cb.__class__.__name__				)

													for cba, cba in zip(lowerCAmelCase__				,    lowerCAmelCase__				):
																				if isinstance(lowerCAmelCase__				,    lowerCAmelCase__				) and isinstance(lowerCAmelCase__				,    lowerCAmelCase__				):
																											self.assertEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
																				elif isinstance(lowerCAmelCase__				,    lowerCAmelCase__				) and not isinstance(lowerCAmelCase__				,    lowerCAmelCase__				):
																											self.assertEqual(lowerCAmelCase__				,    cba.__class__				)
																				elif not isinstance(lowerCAmelCase__				,    lowerCAmelCase__				) and isinstance(lowerCAmelCase__				,    lowerCAmelCase__				):
																											self.assertEqual(cba.__class__				,    lowerCAmelCase__				)
																				else:
																											self.assertEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Dict				,    lowerCAmelCase__   :      Dict				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Any     					=			['''on_init_end''', '''on_train_begin''']
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			0
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			len(trainer.get_eval_dataloader()				)
													SCREAMING_SNAKE_CASE					:      Tuple     					=			['''on_prediction_step'''] * len(trainer.get_eval_dataloader()				) + ['''on_log''', '''on_evaluate''']
													for _ in range(trainer.state.num_train_epochs				):
																				expected_events.append('''on_epoch_begin'''				)
																				for _ in range(lowerCAmelCase__				):
																											step += 1
																											expected_events += ["on_step_begin", "on_step_end"]
																											if step % trainer.args.logging_steps == 0:
																																		expected_events.append('''on_log'''				)
																											if trainer.args.evaluation_strategy == IntervalStrategy.STEPS and step % trainer.args.eval_steps == 0:
																																		expected_events += evaluation_events.copy()
																											if step % trainer.args.save_steps == 0:
																																		expected_events.append('''on_save'''				)
																				expected_events.append('''on_epoch_end'''				)
																				if trainer.args.evaluation_strategy == IntervalStrategy.EPOCH:
																											expected_events += evaluation_events.copy()
													expected_events += ["on_log", "on_train_end"]
													return expected_events







						def 					__lowercase    (  self   :      Optional[Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			self.get_trainer()
													SCREAMING_SNAKE_CASE					:      Any     					=			DEFAULT_CALLBACKS.copy() + [ProgressCallback]
													self.check_callbacks_equality(trainer.callback_handler.callbacks				,    lowerCAmelCase__				)

													# Callbacks passed at init are added to the default callbacks
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			self.get_trainer(callbacks=[MyTestTrainerCallback]				)
													expected_callbacks.append(lowerCAmelCase__				)
													self.check_callbacks_equality(trainer.callback_handler.callbacks				,    lowerCAmelCase__				)

													# TrainingArguments.disable_tqdm controls if use ProgressCallback or PrinterCallback
													SCREAMING_SNAKE_CASE					:      List[str]     					=			self.get_trainer(disable_tqdm=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      List[str]     					=			DEFAULT_CALLBACKS.copy() + [PrinterCallback]
													self.check_callbacks_equality(trainer.callback_handler.callbacks				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Dict				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      List[str]     					=			DEFAULT_CALLBACKS.copy() + [ProgressCallback]
													SCREAMING_SNAKE_CASE					:      int     					=			self.get_trainer()

													# We can add, pop, or remove by class name
													trainer.remove_callback(lowerCAmelCase__				)
													expected_callbacks.remove(lowerCAmelCase__				)
													self.check_callbacks_equality(trainer.callback_handler.callbacks				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			self.get_trainer()
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			trainer.pop_callback(lowerCAmelCase__				)
													self.assertEqual(cb.__class__				,    lowerCAmelCase__				)
													self.check_callbacks_equality(trainer.callback_handler.callbacks				,    lowerCAmelCase__				)

													trainer.add_callback(lowerCAmelCase__				)
													expected_callbacks.insert(0				,    lowerCAmelCase__				)
													self.check_callbacks_equality(trainer.callback_handler.callbacks				,    lowerCAmelCase__				)

													# We can also add, pop, or remove by instance
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			self.get_trainer()
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			trainer.callback_handler.callbacks[0]
													trainer.remove_callback(lowerCAmelCase__				)
													expected_callbacks.remove(lowerCAmelCase__				)
													self.check_callbacks_equality(trainer.callback_handler.callbacks				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      List[str]     					=			self.get_trainer()
													SCREAMING_SNAKE_CASE					:      Dict     					=			trainer.callback_handler.callbacks[0]
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			trainer.pop_callback(lowerCAmelCase__				)
													self.assertEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													self.check_callbacks_equality(trainer.callback_handler.callbacks				,    lowerCAmelCase__				)

													trainer.add_callback(lowerCAmelCase__				)
													expected_callbacks.insert(0				,    lowerCAmelCase__				)
													self.check_callbacks_equality(trainer.callback_handler.callbacks				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													import warnings

													# XXX: for now ignore scatter_gather warnings in this test since it's not relevant to what's being tested
													warnings.simplefilter(action='''ignore'''				,    category=lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Dict     					=			self.get_trainer(callbacks=[MyTestTrainerCallback]				)
													trainer.train()
													SCREAMING_SNAKE_CASE					:      Dict     					=			trainer.callback_handler.callbacks[-2].events
													self.assertEqual(lowerCAmelCase__				,    self.get_expected_events(lowerCAmelCase__				)				)

													# Independent log/save/eval
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			self.get_trainer(callbacks=[MyTestTrainerCallback]				,    logging_steps=5				)
													trainer.train()
													SCREAMING_SNAKE_CASE					:      Tuple     					=			trainer.callback_handler.callbacks[-2].events
													self.assertEqual(lowerCAmelCase__				,    self.get_expected_events(lowerCAmelCase__				)				)

													SCREAMING_SNAKE_CASE					:      str     					=			self.get_trainer(callbacks=[MyTestTrainerCallback]				,    save_steps=5				)
													trainer.train()
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			trainer.callback_handler.callbacks[-2].events
													self.assertEqual(lowerCAmelCase__				,    self.get_expected_events(lowerCAmelCase__				)				)

													SCREAMING_SNAKE_CASE					:      Dict     					=			self.get_trainer(callbacks=[MyTestTrainerCallback]				,    eval_steps=5				,    evaluation_strategy='''steps'''				)
													trainer.train()
													SCREAMING_SNAKE_CASE					:      str     					=			trainer.callback_handler.callbacks[-2].events
													self.assertEqual(lowerCAmelCase__				,    self.get_expected_events(lowerCAmelCase__				)				)

													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			self.get_trainer(callbacks=[MyTestTrainerCallback]				,    evaluation_strategy='''epoch'''				)
													trainer.train()
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			trainer.callback_handler.callbacks[-2].events
													self.assertEqual(lowerCAmelCase__				,    self.get_expected_events(lowerCAmelCase__				)				)

													# A bit of everything
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			self.get_trainer(
													    callbacks=[MyTestTrainerCallback]				,    logging_steps=3				,    save_steps=10				,    eval_steps=5				,    evaluation_strategy='''steps'''				,    )
													trainer.train()
													SCREAMING_SNAKE_CASE					:      List[str]     					=			trainer.callback_handler.callbacks[-2].events
													self.assertEqual(lowerCAmelCase__				,    self.get_expected_events(lowerCAmelCase__				)				)

													# warning should be emitted for duplicated callbacks
													with patch('''transformers.trainer_callback.logger.warning'''				) as warn_mock:
																				SCREAMING_SNAKE_CASE					:      Dict     					=			self.get_trainer(
																				    callbacks=[MyTestTrainerCallback, MyTestTrainerCallback]				,    )
																				assert str(lowerCAmelCase__				) in warn_mock.call_args[0][0]









'''simple docstring'''



import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase_     :       Dict								=						get_tests_dir('fixtures/spiece.model')
@require_sentencepiece
@require_tokenizers
class 	lowerCamelCase_							(					snake_case_    ,				unittest.TestCase			):
						_lowerCAmelCase     :   Union[str, Any]	     =				DebertaVaTokenizer
						_lowerCAmelCase     :   List[Any]	     =				DebertaVaTokenizerFast
						_lowerCAmelCase     :   Optional[Any]	     =				True
						_lowerCAmelCase     :   int	     =				True







						def 					__lowercase    (  self   :      Optional[Any]				):

													"""simple docstring"""
													super().setUp()

													# We have a SentencePiece fixture for testing
													SCREAMING_SNAKE_CASE					:      Tuple     					=			DebertaVaTokenizer(lowerCAmelCase__				,    unk_token='''<unk>'''				)
													tokenizer.save_pretrained(self.tmpdirname				)







						def 					__lowercase    (  self   :      List[str]				,    lowerCAmelCase__   :      Any				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      List[str]     					=			'''this is a test'''
													SCREAMING_SNAKE_CASE					:      Tuple     					=			'''this is a test'''
													return input_text, output_text







						def 					__lowercase    (  self   :      int				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			'''<pad>'''
													SCREAMING_SNAKE_CASE					:      Tuple     					=			0

													self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase__				)				,    lowerCAmelCase__				)
													self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase__				)				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Optional[int]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Tuple     					=			list(self.get_tokenizer().get_vocab().keys()				)
													self.assertEqual(vocab_keys[0]				,    '''<pad>'''				)
													self.assertEqual(vocab_keys[1]				,    '''<unk>'''				)
													self.assertEqual(vocab_keys[-1]				,    '''[PAD]'''				)
													self.assertEqual(len(lowerCAmelCase__				)				,    3_00_01				)







						def 					__lowercase    (  self   :      str				):

													"""simple docstring"""
													self.assertEqual(self.get_tokenizer().vocab_size				,    3_00_00				)







						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													# fmt: off
													SCREAMING_SNAKE_CASE					:      int     					=			''' \tHeLLo!how  \n Are yoU?  '''
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			['''▁hello''', '''!''', '''how''', '''▁are''', '''▁you''', '''?''']
													# fmt: on

													SCREAMING_SNAKE_CASE					:      Any     					=			DebertaVaTokenizer(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Tuple     					=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Dict     					=			DebertaVaTokenizerFast(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						@unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''				)
						def 					__lowercase    (  self   :      Dict				):

													"""simple docstring"""
													pass







						@unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''				)
						def 					__lowercase    (  self   :      str				):

													"""simple docstring"""
													pass







						def 					__lowercase    (  self   :      int				):

													"""simple docstring"""
													# fmt: off
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			'''I was born in 92000, and this is falsé.'''
													SCREAMING_SNAKE_CASE					:      Any     					=			['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
													# fmt: on

													SCREAMING_SNAKE_CASE					:      Dict     					=			DebertaVaTokenizer(lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			DebertaVaTokenizerFast(lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      str     					=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Optional[Any]				):

													"""simple docstring"""
													# fmt: off
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			'''I was born in 92000, and this is falsé.'''
													SCREAMING_SNAKE_CASE					:      List[str]     					=			['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
													# fmt: on

													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			DebertaVaTokenizer(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      int     					=			DebertaVaTokenizerFast(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Any     					=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Any				):

													"""simple docstring"""
													# fmt: off
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			'''I was born in 92000, and this is falsé.'''
													SCREAMING_SNAKE_CASE					:      Dict     					=			['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
													# fmt: on

													SCREAMING_SNAKE_CASE					:      Tuple     					=			DebertaVaTokenizer(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Any     					=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Any     					=			DebertaVaTokenizerFast(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Dict				):

													"""simple docstring"""
													# fmt: off
													SCREAMING_SNAKE_CASE					:      str     					=			'''I was born in 92000, and this is falsé.'''
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
													# fmt: on

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			DebertaVaTokenizer(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      str     					=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			DebertaVaTokenizerFast(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      List[str]     					=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Union[str, Any]				):

													"""simple docstring"""
													# fmt: off
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			''' \tHeLLo!how  \n Are yoU?  '''
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			['''▁''', '''<unk>''', '''e''', '''<unk>''', '''o''', '''!''', '''how''', '''▁''', '''<unk>''', '''re''', '''▁yo''', '''<unk>''', '''?''']
													# fmt: on

													SCREAMING_SNAKE_CASE					:      List[str]     					=			DebertaVaTokenizer(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Any     					=			DebertaVaTokenizerFast(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      List[str]     					=			self.get_tokenizer()
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			self.get_rust_tokenizer()

													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			'''I was born in 92000, and this is falsé.'''

													SCREAMING_SNAKE_CASE					:      Tuple     					=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)
													SCREAMING_SNAKE_CASE					:      List[str]     					=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      str     					=			rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Tuple     					=			self.get_rust_tokenizer()
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			tokenizer.encode(lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Any     					=			rust_tokenizer.encode(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Tuple     					=			'''This is a test'''
													SCREAMING_SNAKE_CASE					:      List[str]     					=			[13, 1, 43_98, 25, 21, 12_89]
													SCREAMING_SNAKE_CASE					:      Tuple     					=			['''▁''', '''T''', '''his''', '''▁is''', '''▁a''', '''▁test''']
													SCREAMING_SNAKE_CASE					:      Any     					=			['''▁''', '''<unk>''', '''his''', '''▁is''', '''▁a''', '''▁test''']

													SCREAMING_SNAKE_CASE					:      List[str]     					=			DebertaVaTokenizer(lowerCAmelCase__				,    keep_accents=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      str     					=			DebertaVaTokenizerFast(lowerCAmelCase__				,    keep_accents=lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			tokenizer.tokenize(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Any     					=			tokenizer.convert_ids_to_tokens(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      List[str]     					=			rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      str     					=			rust_tokenizer.tokenize(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Any     					=			rust_tokenizer.convert_ids_to_tokens(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													# fmt: off
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			'''I was born in 92000, and this is falsé.'''
													SCREAMING_SNAKE_CASE					:      int     					=			[13, 1, 23, 3_86, 19, 5_61, 30_50, 15, 17, 48, 25, 82_56, 18, 1, 9]
													SCREAMING_SNAKE_CASE					:      Dict     					=			['''▁''', '''I''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''', ]
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
													# fmt: on

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Tuple     					=			tokenizer.tokenize(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			tokenizer.convert_ids_to_tokens(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      int     					=			rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Tuple     					=			rust_tokenizer.tokenize(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			rust_tokenizer.convert_ids_to_tokens(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Any				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      str     					=			DebertaVaTokenizer(lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      List[str]     					=			tokenizer.encode('''sequence builders'''				)
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			tokenizer.encode('''multi-sequence build'''				)

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Tuple     					=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__				,    lowerCAmelCase__				)

													self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]				,    lowerCAmelCase__				)
													self.assertEqual(
													    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]				,    lowerCAmelCase__				,    )







						@slow
						def 					__lowercase    (  self   :      Optional[Any]				):

													"""simple docstring"""
													# fmt: off
													SCREAMING_SNAKE_CASE					:      str     					=			{'''input_ids''': [[1, 3_98_67, 36, 1_93_90, 4_86, 27, 3_50_52, 8_14_36, 18, 6_06_85, 12_25, 7, 3_50_52, 8_14_36, 18, 93_67, 1_68_99, 18, 1_59_37, 53, 5_94, 7_73, 18, 1_62_87, 3_04_65, 36, 1_59_37, 6, 4_11_39, 38, 3_69_79, 6_07_63, 1_91, 6, 3_41_32, 99, 6, 5_05_38, 3_90, 4_32_30, 6, 3_41_32, 27_79, 2_08_50, 14, 6_99, 10_72, 11_94, 36, 3_82, 1_09_01, 53, 7, 6_99, 10_72, 20_84, 36, 2_04_22, 6_30, 53, 19, 1_05, 30_49, 18_96, 10_53, 1_68_99, 15_06, 11, 3_79_78, 42_43, 7, 12_37, 3_18_69, 2_00, 1_65_66, 6_54, 6, 3_50_52, 8_14_36, 7, 5_56_30, 1_35_93, 4, 2], [1, 26, 1_50_11, 13, 6_67, 8, 10_53, 18, 2_36_11, 12_37, 7_23_56, 1_28_20, 34, 10_41_34, 12_09, 35, 1_33_13, 66_27, 21, 2_02, 3_47, 7, 1_64, 23_99, 11, 46, 44_85, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 12_32, 28_64, 1_57_85, 1_49_51, 1_05, 5, 85_81, 12_50, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
													# fmt: on

													self.tokenizer_integration_test_util(
													    expected_encoding=lowerCAmelCase__				,    model_name='''microsoft/deberta-v2-xlarge'''				,    revision='''ad6e42c1532ddf3a15c39246b63f5559d558b670'''				,    )




