



'''simple docstring'''



import json
import os
import tempfile
from unittest.mock import patch

import torch
from torch.utils.data import DataLoader, TensorDataset

from accelerate import DistributedType, infer_auto_device_map, init_empty_weights
from accelerate.accelerator import Accelerator
from accelerate.state import GradientState, PartialState
from accelerate.test_utils import require_bnb, require_multi_gpu, slow
from accelerate.test_utils.testing import AccelerateTestCase, require_cuda
from accelerate.utils import patch_environment
def 					UpperCAmelCase   (						):
			SCREAMING_SNAKE_CASE					:      Optional[int]     					=			torch.nn.Linear(2					,		4				)
			SCREAMING_SNAKE_CASE					:      int     					=			torch.optim.AdamW(model.parameters()					,		lr=1.0				)
			SCREAMING_SNAKE_CASE					:      Any     					=			torch.optim.lr_scheduler.OneCycleLR(__UpperCamelCase					,		max_lr=0.01					,		steps_per_epoch=2					,		epochs=1				)
			SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			DataLoader(TensorDataset(torch.tensor([1, 2, 3]				)				)				)
			SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			DataLoader(TensorDataset(torch.tensor([4, 5, 6]				)				)				)

			return model, optimizer, scheduler, train_dl, valid_dl
def 					UpperCAmelCase   (						A  :				Union[str, Any]				):
			return (model.weight.abs().sum() + model.bias.abs().sum()).item()
def 					UpperCAmelCase   (						A  :				List[str]				):
			SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			torch.nn.Linear(*tuple(model.weight.T.shape				)				).state_dict()
			model.load_state_dict(__UpperCamelCase				)






class 	lowerCamelCase_							(					SCREAMING_SNAKE_CASE_			):







						@require_cuda
						def 					__lowercase    (  self   :      List[str]				):

									"""simple docstring"""
									SCREAMING_SNAKE_CASE					:      Any     					=			Accelerator()
									assert PartialState._shared_state["_cpu"] is False
									assert PartialState._shared_state["device"].type == "cuda"
									with self.assertRaises(UpperCamelCase__				):
												SCREAMING_SNAKE_CASE					:      Dict     					=			Accelerator(cpu=UpperCamelCase__				)







						def 					__lowercase    (  self   :      List[str]				):

									"""simple docstring"""
									SCREAMING_SNAKE_CASE					:      Optional[int]     					=			Accelerator()
									SCREAMING_SNAKE_CASE					:      str     					=			GradientState()
									assert state.num_steps == 1
									SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			4
									assert state.num_steps == 4

									assert state.sync_gradients is True
									SCREAMING_SNAKE_CASE					:      int     					=			False
									assert state.sync_gradients is False
									GradientState._reset_state()







						def 					__lowercase    (  self   :      Any				):

									"""simple docstring"""
									SCREAMING_SNAKE_CASE					:      Optional[int]     					=			Accelerator()
									SCREAMING_SNAKE_CASE					:      Dict     					=			create_components()

									(
									    SCREAMING_SNAKE_CASE							
									)					:      Any     					=			accelerator.prepare(UpperCamelCase__				,    UpperCamelCase__				,    UpperCamelCase__				,    UpperCamelCase__				,    UpperCamelCase__				)

									self.assertTrue(prepared_model in accelerator._models				)
									self.assertTrue(prepared_optimizer in accelerator._optimizers				)
									self.assertTrue(prepared_scheduler in accelerator._schedulers				)
									self.assertTrue(prepared_train_dl in accelerator._dataloaders				)
									self.assertTrue(prepared_valid_dl in accelerator._dataloaders				)







						def 					__lowercase    (  self   :      Any				):

									"""simple docstring"""
									SCREAMING_SNAKE_CASE					:      Dict     					=			Accelerator()
									SCREAMING_SNAKE_CASE					:      str     					=			create_components()
									accelerator.prepare(UpperCamelCase__				,    UpperCamelCase__				,    UpperCamelCase__				,    UpperCamelCase__				,    UpperCamelCase__				)
									accelerator.free_memory()

									self.assertTrue(len(accelerator._models				) == 0				)
									self.assertTrue(len(accelerator._optimizers				) == 0				)
									self.assertTrue(len(accelerator._schedulers				) == 0				)
									self.assertTrue(len(accelerator._dataloaders				) == 0				)







						def 					__lowercase    (  self   :      Optional[int]				):

									"""simple docstring"""
									PartialState._reset_state()

									# Mock torch.cuda.set_device to avoid an exception as the device doesn't exist
									def noop(*lowerCAmelCase__   :      Dict				,    **lowerCAmelCase__   :      Dict				):
												pass

									with patch('''torch.cuda.set_device'''				,    UpperCamelCase__				), patch_environment(ACCELERATE_TORCH_DEVICE='''cuda:64'''				):
												SCREAMING_SNAKE_CASE					:      Optional[int]     					=			Accelerator()
												self.assertEqual(str(accelerator.state.device				)				,    '''cuda:64'''				)







						def 					__lowercase    (  self   :      int				):

									"""simple docstring"""
									SCREAMING_SNAKE_CASE					:      Tuple     					=			Accelerator()
									SCREAMING_SNAKE_CASE					:      Any     					=			create_components()
									accelerator.prepare(UpperCamelCase__				,    UpperCamelCase__				,    UpperCamelCase__				,    UpperCamelCase__				,    UpperCamelCase__				)

									SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			get_signature(UpperCamelCase__				)

									with tempfile.TemporaryDirectory() as tmpdirname:
												accelerator.save_state(UpperCamelCase__				)

												# make sure random weights don't match
												load_random_weights(UpperCamelCase__				)
												self.assertTrue(abs(model_signature - get_signature(UpperCamelCase__				)				) > 1e-3				)

												# make sure loaded weights match
												accelerator.load_state(UpperCamelCase__				)
												self.assertTrue(abs(model_signature - get_signature(UpperCamelCase__				)				) < 1e-3				)







						def 					__lowercase    (  self   :      Optional[int]				):

									"""simple docstring"""
									SCREAMING_SNAKE_CASE					:      List[Any]     					=			Accelerator()
									SCREAMING_SNAKE_CASE					:      Optional[int]     					=			create_components()
									accelerator.prepare(UpperCamelCase__				,    UpperCamelCase__				,    UpperCamelCase__				,    UpperCamelCase__				,    UpperCamelCase__				)

									SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			get_signature(UpperCamelCase__				)

									# saving hook
									def save_config(lowerCAmelCase__   :      Dict				,    lowerCAmelCase__   :      int				,    lowerCAmelCase__   :      Union[str, Any]				):
												SCREAMING_SNAKE_CASE					:      Tuple     					=			{'''class_name''': models[0].__class__.__name__}

												with open(os.path.join(UpperCamelCase__				,    '''data.json'''				)				,    '''w'''				) as f:
															json.dump(UpperCamelCase__				,    UpperCamelCase__				)

        # loading hook
									def load_config(lowerCAmelCase__   :      str				,    lowerCAmelCase__   :      Dict				):
												with open(os.path.join(UpperCamelCase__				,    '''data.json'''				)				,    '''r'''				) as f:
															SCREAMING_SNAKE_CASE					:      Optional[int]     					=			json.load(UpperCamelCase__				)

												SCREAMING_SNAKE_CASE					:      List[Any]     					=			config['''class_name''']

									SCREAMING_SNAKE_CASE					:      Dict     					=			accelerator.register_save_state_pre_hook(UpperCamelCase__				)
									SCREAMING_SNAKE_CASE					:      Optional[int]     					=			accelerator.register_load_state_pre_hook(UpperCamelCase__				)

									with tempfile.TemporaryDirectory() as tmpdirname:
												accelerator.save_state(UpperCamelCase__				)

												# make sure random weights don't match with hooks
												load_random_weights(UpperCamelCase__				)
												self.assertTrue(abs(model_signature - get_signature(UpperCamelCase__				)				) > 1e-3				)

												# random class name to verify correct one is loaded
												SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			'''random'''

												# make sure loaded weights match with hooks
												accelerator.load_state(UpperCamelCase__				)
												self.assertTrue(abs(model_signature - get_signature(UpperCamelCase__				)				) < 1e-3				)

												# mode.class_name is loaded from config
												self.assertTrue(model.class_name == model.__class__.__name__				)

									# remove hooks
									save_hook.remove()
									load_hook.remove()

									with tempfile.TemporaryDirectory() as tmpdirname:
												accelerator.save_state(UpperCamelCase__				)

												# make sure random weights don't match with hooks removed
												load_random_weights(UpperCamelCase__				)
												self.assertTrue(abs(model_signature - get_signature(UpperCamelCase__				)				) > 1e-3				)

												# random class name to verify correct one is loaded
												SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			'''random'''

												# make sure loaded weights match with hooks removed
												accelerator.load_state(UpperCamelCase__				)
												self.assertTrue(abs(model_signature - get_signature(UpperCamelCase__				)				) < 1e-3				)

												# mode.class_name is NOT loaded from config
												self.assertTrue(model.class_name != model.__class__.__name__				)







						def 					__lowercase    (  self   :      int				):

									"""simple docstring"""
									SCREAMING_SNAKE_CASE					:      int     					=			Accelerator()
									SCREAMING_SNAKE_CASE					:      Tuple     					=			create_components()
									SCREAMING_SNAKE_CASE					:      List[Any]     					=			None

									# This should work
									SCREAMING_SNAKE_CASE					:      Tuple     					=			accelerator.prepare(
									    UpperCamelCase__				,    UpperCamelCase__				,    UpperCamelCase__				,    UpperCamelCase__				,    UpperCamelCase__				,    UpperCamelCase__				)
									self.assertTrue(dummy_obj is None				)







						def 					__lowercase    (  self   :      Optional[Any]				):

									"""simple docstring"""
									SCREAMING_SNAKE_CASE					:      str     					=			Accelerator()
									SCREAMING_SNAKE_CASE					:      int     					=			create_components()
									SCREAMING_SNAKE_CASE					:      Any     					=			[1, 2, 3]

									# This should work
									SCREAMING_SNAKE_CASE					:      List[Any]     					=			accelerator.prepare(
									    UpperCamelCase__				,    UpperCamelCase__				,    UpperCamelCase__				,    UpperCamelCase__				,    UpperCamelCase__				,    UpperCamelCase__				)
									self.assertEqual(
									    getattr(UpperCamelCase__				,    '''_is_accelerate_prepared'''				,    UpperCamelCase__				)				,    UpperCamelCase__				,    '''Dummy object should have `_is_accelerate_prepared` set to `True`'''				,    )
									self.assertEqual(
									    getattr(UpperCamelCase__				,    '''_is_accelerate_prepared'''				,    UpperCamelCase__				)				,    UpperCamelCase__				,    '''Model is missing `_is_accelerator_prepared` or is set to `False`'''				,    )
									self.assertEqual(
									    getattr(UpperCamelCase__				,    '''_is_accelerate_prepared'''				,    UpperCamelCase__				)				,    UpperCamelCase__				,    '''Optimizer is missing `_is_accelerator_prepared` or is set to `False`'''				,    )
									self.assertEqual(
									    getattr(UpperCamelCase__				,    '''_is_accelerate_prepared'''				,    UpperCamelCase__				)				,    UpperCamelCase__				,    '''Scheduler is missing `_is_accelerator_prepared` or is set to `False`'''				,    )
									self.assertEqual(
									    getattr(UpperCamelCase__				,    '''_is_accelerate_prepared'''				,    UpperCamelCase__				)				,    UpperCamelCase__				,    '''Train Dataloader is missing `_is_accelerator_prepared` or is set to `False`'''				,    )
									self.assertEqual(
									    getattr(UpperCamelCase__				,    '''_is_accelerate_prepared'''				,    UpperCamelCase__				)				,    UpperCamelCase__				,    '''Valid Dataloader is missing `_is_accelerator_prepared` or is set to `False`'''				,    )







						@slow
						@require_bnb
						def 					__lowercase    (  self   :      List[Any]				):

									"""simple docstring"""
									from transformers import AutoModelForCausalLM

									SCREAMING_SNAKE_CASE					:      Tuple     					=			AutoModelForCausalLM.from_pretrained(
									    '''EleutherAI/gpt-neo-125m'''				,    load_in_abit=UpperCamelCase__				,    device_map={'''''': 0}				,    )
									SCREAMING_SNAKE_CASE					:      Optional[int]     					=			Accelerator()

									# This should work
									SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			accelerator.prepare(UpperCamelCase__				)







						@slow
						@require_bnb
						def 					__lowercase    (  self   :      Optional[int]				):

									"""simple docstring"""
									from transformers import AutoModelForCausalLM

									SCREAMING_SNAKE_CASE					:      List[str]     					=			Accelerator()

									with init_empty_weights():
												SCREAMING_SNAKE_CASE					:      List[Any]     					=			AutoModelForCausalLM.from_pretrained(
												    '''EleutherAI/gpt-neo-125m'''				,    )
												model.tie_weights()
												SCREAMING_SNAKE_CASE					:      Any     					=			infer_auto_device_map(UpperCamelCase__				)
												SCREAMING_SNAKE_CASE					:      Any     					=			'''cpu'''

									SCREAMING_SNAKE_CASE					:      Dict     					=			AutoModelForCausalLM.from_pretrained(
									    '''EleutherAI/gpt-neo-125m'''				,    device_map=UpperCamelCase__				,    load_in_abit=UpperCamelCase__				,    llm_inta_enable_fpaa_cpu_offload=UpperCamelCase__				)

									# This should not work and get value error
									with self.assertRaises(UpperCamelCase__				):
												SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			accelerator.prepare(UpperCamelCase__				)







						@slow
						@require_bnb
						@require_multi_gpu
						def 					__lowercase    (  self   :      Any				):

									"""simple docstring"""
									from transformers import AutoModelForCausalLM

									SCREAMING_SNAKE_CASE					:      Any     					=			{'''distributed_type''': DistributedType.MULTI_GPU}

									with init_empty_weights():
												SCREAMING_SNAKE_CASE					:      Tuple     					=			AutoModelForCausalLM.from_pretrained(
												    '''EleutherAI/gpt-neo-125m'''				,    )
												model.tie_weights()
												SCREAMING_SNAKE_CASE					:      Dict     					=			infer_auto_device_map(UpperCamelCase__				)
												SCREAMING_SNAKE_CASE					:      Dict     					=			1

									SCREAMING_SNAKE_CASE					:      Dict     					=			AutoModelForCausalLM.from_pretrained(
									    '''EleutherAI/gpt-neo-125m'''				,    load_in_abit=UpperCamelCase__				,    device_map=UpperCamelCase__				,    )
									SCREAMING_SNAKE_CASE					:      Any     					=			Accelerator()

									# This should not work and get value error
									with self.assertRaises(UpperCamelCase__				):
												SCREAMING_SNAKE_CASE					:      Dict     					=			accelerator.prepare(UpperCamelCase__				)

									PartialState._reset_state()







						@slow
						@require_bnb
						@require_multi_gpu
						def 					__lowercase    (  self   :      Optional[int]				):

									"""simple docstring"""
									from transformers import AutoModelForCausalLM

									with init_empty_weights():
												SCREAMING_SNAKE_CASE					:      str     					=			AutoModelForCausalLM.from_pretrained(
												    '''EleutherAI/gpt-neo-125m'''				,    )
												SCREAMING_SNAKE_CASE					:      Tuple     					=			infer_auto_device_map(UpperCamelCase__				)
												SCREAMING_SNAKE_CASE					:      Optional[int]     					=			1

									SCREAMING_SNAKE_CASE					:      str     					=			AutoModelForCausalLM.from_pretrained(
									    '''EleutherAI/gpt-neo-125m'''				,    load_in_abit=UpperCamelCase__				,    device_map=UpperCamelCase__				,    )
									SCREAMING_SNAKE_CASE					:      List[Any]     					=			Accelerator()

									# This should work
									SCREAMING_SNAKE_CASE					:      List[Any]     					=			accelerator.prepare(UpperCamelCase__				)







						@require_cuda
						def 					__lowercase    (  self   :      Optional[int]				):

									"""simple docstring"""
									SCREAMING_SNAKE_CASE					:      int     					=			torch.nn.Linear(10				,    10				)
									SCREAMING_SNAKE_CASE					:      int     					=			torch.optim.SGD(model.parameters()				,    lr=0.01				)
									SCREAMING_SNAKE_CASE					:      str     					=			Accelerator(cpu=UpperCamelCase__				)
									SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			accelerator.prepare(UpperCamelCase__				)









'''simple docstring'''



import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase_     :       Dict								=						get_tests_dir('fixtures/spiece.model')
@require_sentencepiece
@require_tokenizers
class 	lowerCamelCase_							(					snake_case_    ,				unittest.TestCase			):
						_lowerCAmelCase     :   Union[str, Any]	     =				DebertaVaTokenizer
						_lowerCAmelCase     :   List[Any]	     =				DebertaVaTokenizerFast
						_lowerCAmelCase     :   Optional[Any]	     =				True
						_lowerCAmelCase     :   int	     =				True







						def 					__lowercase    (  self   :      Optional[Any]				):

													"""simple docstring"""
													super().setUp()

													# We have a SentencePiece fixture for testing
													SCREAMING_SNAKE_CASE					:      Tuple     					=			DebertaVaTokenizer(lowerCAmelCase__				,    unk_token='''<unk>'''				)
													tokenizer.save_pretrained(self.tmpdirname				)







						def 					__lowercase    (  self   :      List[str]				,    lowerCAmelCase__   :      Any				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      List[str]     					=			'''this is a test'''
													SCREAMING_SNAKE_CASE					:      Tuple     					=			'''this is a test'''
													return input_text, output_text







						def 					__lowercase    (  self   :      int				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			'''<pad>'''
													SCREAMING_SNAKE_CASE					:      Tuple     					=			0

													self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase__				)				,    lowerCAmelCase__				)
													self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase__				)				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Optional[int]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Tuple     					=			list(self.get_tokenizer().get_vocab().keys()				)
													self.assertEqual(vocab_keys[0]				,    '''<pad>'''				)
													self.assertEqual(vocab_keys[1]				,    '''<unk>'''				)
													self.assertEqual(vocab_keys[-1]				,    '''[PAD]'''				)
													self.assertEqual(len(lowerCAmelCase__				)				,    3_00_01				)







						def 					__lowercase    (  self   :      str				):

													"""simple docstring"""
													self.assertEqual(self.get_tokenizer().vocab_size				,    3_00_00				)







						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													# fmt: off
													SCREAMING_SNAKE_CASE					:      int     					=			''' \tHeLLo!how  \n Are yoU?  '''
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			['''▁hello''', '''!''', '''how''', '''▁are''', '''▁you''', '''?''']
													# fmt: on

													SCREAMING_SNAKE_CASE					:      Any     					=			DebertaVaTokenizer(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Tuple     					=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Dict     					=			DebertaVaTokenizerFast(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						@unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''				)
						def 					__lowercase    (  self   :      Dict				):

													"""simple docstring"""
													pass







						@unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''				)
						def 					__lowercase    (  self   :      str				):

													"""simple docstring"""
													pass







						def 					__lowercase    (  self   :      int				):

													"""simple docstring"""
													# fmt: off
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			'''I was born in 92000, and this is falsé.'''
													SCREAMING_SNAKE_CASE					:      Any     					=			['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
													# fmt: on

													SCREAMING_SNAKE_CASE					:      Dict     					=			DebertaVaTokenizer(lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			DebertaVaTokenizerFast(lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      str     					=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Optional[Any]				):

													"""simple docstring"""
													# fmt: off
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			'''I was born in 92000, and this is falsé.'''
													SCREAMING_SNAKE_CASE					:      List[str]     					=			['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
													# fmt: on

													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			DebertaVaTokenizer(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      int     					=			DebertaVaTokenizerFast(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Any     					=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Any				):

													"""simple docstring"""
													# fmt: off
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			'''I was born in 92000, and this is falsé.'''
													SCREAMING_SNAKE_CASE					:      Dict     					=			['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
													# fmt: on

													SCREAMING_SNAKE_CASE					:      Tuple     					=			DebertaVaTokenizer(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Any     					=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Any     					=			DebertaVaTokenizerFast(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Dict				):

													"""simple docstring"""
													# fmt: off
													SCREAMING_SNAKE_CASE					:      str     					=			'''I was born in 92000, and this is falsé.'''
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
													# fmt: on

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			DebertaVaTokenizer(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      str     					=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			DebertaVaTokenizerFast(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      List[str]     					=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Union[str, Any]				):

													"""simple docstring"""
													# fmt: off
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			''' \tHeLLo!how  \n Are yoU?  '''
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			['''▁''', '''<unk>''', '''e''', '''<unk>''', '''o''', '''!''', '''how''', '''▁''', '''<unk>''', '''re''', '''▁yo''', '''<unk>''', '''?''']
													# fmt: on

													SCREAMING_SNAKE_CASE					:      List[str]     					=			DebertaVaTokenizer(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Any     					=			DebertaVaTokenizerFast(lowerCAmelCase__				,    do_lower_case=lowerCAmelCase__				,    split_by_punct=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)

													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      List[str]     					=			self.get_tokenizer()
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			self.get_rust_tokenizer()

													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			'''I was born in 92000, and this is falsé.'''

													SCREAMING_SNAKE_CASE					:      Tuple     					=			tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)
													SCREAMING_SNAKE_CASE					:      List[str]     					=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      str     					=			rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      Tuple     					=			self.get_rust_tokenizer()
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			tokenizer.encode(lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Any     					=			rust_tokenizer.encode(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Tuple     					=			'''This is a test'''
													SCREAMING_SNAKE_CASE					:      List[str]     					=			[13, 1, 43_98, 25, 21, 12_89]
													SCREAMING_SNAKE_CASE					:      Tuple     					=			['''▁''', '''T''', '''his''', '''▁is''', '''▁a''', '''▁test''']
													SCREAMING_SNAKE_CASE					:      Any     					=			['''▁''', '''<unk>''', '''his''', '''▁is''', '''▁a''', '''▁test''']

													SCREAMING_SNAKE_CASE					:      List[str]     					=			DebertaVaTokenizer(lowerCAmelCase__				,    keep_accents=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      str     					=			DebertaVaTokenizerFast(lowerCAmelCase__				,    keep_accents=lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			tokenizer.tokenize(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Any     					=			tokenizer.convert_ids_to_tokens(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      List[str]     					=			rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      str     					=			rust_tokenizer.tokenize(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Any     					=			rust_tokenizer.convert_ids_to_tokens(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													# fmt: off
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			'''I was born in 92000, and this is falsé.'''
													SCREAMING_SNAKE_CASE					:      int     					=			[13, 1, 23, 3_86, 19, 5_61, 30_50, 15, 17, 48, 25, 82_56, 18, 1, 9]
													SCREAMING_SNAKE_CASE					:      Dict     					=			['''▁''', '''I''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''', ]
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
													# fmt: on

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Tuple     					=			tokenizer.tokenize(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			tokenizer.convert_ids_to_tokens(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      int     					=			rust_tokenizer.encode(lowerCAmelCase__				,    add_special_tokens=lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Tuple     					=			rust_tokenizer.tokenize(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			rust_tokenizer.convert_ids_to_tokens(lowerCAmelCase__				)
													self.assertListEqual(lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Any				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      str     					=			DebertaVaTokenizer(lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      List[str]     					=			tokenizer.encode('''sequence builders'''				)
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			tokenizer.encode('''multi-sequence build'''				)

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Tuple     					=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__				,    lowerCAmelCase__				)

													self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]				,    lowerCAmelCase__				)
													self.assertEqual(
													    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]				,    lowerCAmelCase__				,    )







						@slow
						def 					__lowercase    (  self   :      Optional[Any]				):

													"""simple docstring"""
													# fmt: off
													SCREAMING_SNAKE_CASE					:      str     					=			{'''input_ids''': [[1, 3_98_67, 36, 1_93_90, 4_86, 27, 3_50_52, 8_14_36, 18, 6_06_85, 12_25, 7, 3_50_52, 8_14_36, 18, 93_67, 1_68_99, 18, 1_59_37, 53, 5_94, 7_73, 18, 1_62_87, 3_04_65, 36, 1_59_37, 6, 4_11_39, 38, 3_69_79, 6_07_63, 1_91, 6, 3_41_32, 99, 6, 5_05_38, 3_90, 4_32_30, 6, 3_41_32, 27_79, 2_08_50, 14, 6_99, 10_72, 11_94, 36, 3_82, 1_09_01, 53, 7, 6_99, 10_72, 20_84, 36, 2_04_22, 6_30, 53, 19, 1_05, 30_49, 18_96, 10_53, 1_68_99, 15_06, 11, 3_79_78, 42_43, 7, 12_37, 3_18_69, 2_00, 1_65_66, 6_54, 6, 3_50_52, 8_14_36, 7, 5_56_30, 1_35_93, 4, 2], [1, 26, 1_50_11, 13, 6_67, 8, 10_53, 18, 2_36_11, 12_37, 7_23_56, 1_28_20, 34, 10_41_34, 12_09, 35, 1_33_13, 66_27, 21, 2_02, 3_47, 7, 1_64, 23_99, 11, 46, 44_85, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 12_32, 28_64, 1_57_85, 1_49_51, 1_05, 5, 85_81, 12_50, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
													# fmt: on

													self.tokenizer_integration_test_util(
													    expected_encoding=lowerCAmelCase__				,    model_name='''microsoft/deberta-v2-xlarge'''				,    revision='''ad6e42c1532ddf3a15c39246b63f5559d558b670'''				,    )




