from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
			import tensorflow as tf

			from transformers import (
			    TFConvBertForMaskedLM,
			    TFConvBertForMultipleChoice,
			    TFConvBertForQuestionAnswering,
			    TFConvBertForSequenceClassification,
			    TFConvBertForTokenClassification,
			    TFConvBertModel,
			)
class   _lowerCAmelCase		:







							def __init__(				self     :    str	,					__snake_case     :    Optional[int]	,					__snake_case     :    Optional[int]=13	,					__snake_case     :    Dict=7	,					__snake_case     :    List[Any]=True	,					__snake_case     :    str=True	,					__snake_case     :    Union[str, Any]=True	,					__snake_case     :    Optional[int]=True	,					__snake_case     :    int=99	,					__snake_case     :    Union[str, Any]=32	,					__snake_case     :    Tuple=2	,					__snake_case     :    Optional[int]=4	,					__snake_case     :    int=37	,					__snake_case     :    Optional[int]="gelu"	,					__snake_case     :    Optional[Any]=0.1	,					__snake_case     :    Optional[int]=0.1	,					__snake_case     :    str=512	,					__snake_case     :    Optional[int]=16	,					__snake_case     :    Optional[int]=2	,					__snake_case     :    List[str]=0.0_2	,					__snake_case     :    Optional[Any]=3	,					__snake_case     :    Any=4	,					__snake_case     :    Optional[Any]=None	,					):
										lowerCamelCase						:List[Any]						  =     parent
										lowerCamelCase						:Optional[Any]						  =     13
										lowerCamelCase						:Union[str, Any]						  =     7
										lowerCamelCase						:str						  =     True
										lowerCamelCase						:Optional[int]						  =     True
										lowerCamelCase						:Optional[int]						  =     True
										lowerCamelCase						:List[str]						  =     True
										lowerCamelCase						:Union[str, Any]						  =     99
										lowerCamelCase						:str						  =     384
										lowerCamelCase						:List[str]						  =     2
										lowerCamelCase						:Optional[Any]						  =     4
										lowerCamelCase						:Any						  =     37
										lowerCamelCase						:List[Any]						  =     '''gelu'''
										lowerCamelCase						:Dict						  =     0.1
										lowerCamelCase						:Optional[int]						  =     0.1
										lowerCamelCase						:List[Any]						  =     512
										lowerCamelCase						:List[Any]						  =     16
										lowerCamelCase						:Union[str, Any]						  =     2
										lowerCamelCase						:Any						  =     0.0_2
										lowerCamelCase						:Union[str, Any]						  =     3
										lowerCamelCase						:Optional[int]						  =     4
										lowerCamelCase						:int						  =     128
										lowerCamelCase						:List[Any]						  =     2
										lowerCamelCase						:Optional[int]						  =     9
										lowerCamelCase						:List[Any]						  =     1
										lowerCamelCase						:Dict						  =     None







							def 				snake_case  (				self     :    Optional[Any]		):
										lowerCamelCase						:int						  =     ids_tensor([self.batch_size, self.seq_length]	,					self.vocab_size		)

										lowerCamelCase						:Optional[Any]						  =     None
										if self.use_input_mask:
													lowerCamelCase						:int						  =     random_attention_mask([self.batch_size, self.seq_length]		)

										lowerCamelCase						:int						  =     None
										if self.use_token_type_ids:
													lowerCamelCase						:Dict						  =     ids_tensor([self.batch_size, self.seq_length]	,					self.type_vocab_size		)

										lowerCamelCase						:Optional[int]						  =     None
										lowerCamelCase						:List[Any]						  =     None
										lowerCamelCase						:Union[str, Any]						  =     None
										if self.use_labels:
													lowerCamelCase						:Dict						  =     ids_tensor([self.batch_size]	,					self.type_sequence_label_size		)
													lowerCamelCase						:Dict						  =     ids_tensor([self.batch_size, self.seq_length]	,					self.num_labels		)
													lowerCamelCase						:int						  =     ids_tensor([self.batch_size]	,					self.num_choices		)

										lowerCamelCase						:List[str]						  =     ConvBertConfig(
										    vocab_size=self.vocab_size	,					hidden_size=self.hidden_size	,					num_hidden_layers=self.num_hidden_layers	,					num_attention_heads=self.num_attention_heads	,					intermediate_size=self.intermediate_size	,					hidden_act=self.hidden_act	,					hidden_dropout_prob=self.hidden_dropout_prob	,					attention_probs_dropout_prob=self.attention_probs_dropout_prob	,					max_position_embeddings=self.max_position_embeddings	,					type_vocab_size=self.type_vocab_size	,					initializer_range=self.initializer_range	,					return_dict=__snake_case	,					)

										return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







							def 				snake_case  (				self     :    Dict	,					__snake_case     :    Optional[int]	,					__snake_case     :    List[Any]	,					__snake_case     :    Optional[int]	,					__snake_case     :    List[Any]	,					__snake_case     :    Union[str, Any]	,					__snake_case     :    List[Any]	,					__snake_case     :    Optional[Any]		):
										lowerCamelCase						:int						  =     TFConvBertModel(config=__snake_case		)
										lowerCamelCase						:int						  =     {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}

										lowerCamelCase						:Any						  =     [input_ids, input_mask]
										lowerCamelCase						:Dict						  =     model(__snake_case		)

										lowerCamelCase						:List[str]						  =     model(__snake_case		)

										self.parent.assertEqual(result.last_hidden_state.shape	,					(self.batch_size, self.seq_length, self.hidden_size)		)







							def 				snake_case  (				self     :    Optional[Any]	,					__snake_case     :    Union[str, Any]	,					__snake_case     :    int	,					__snake_case     :    Union[str, Any]	,					__snake_case     :    int	,					__snake_case     :    Any	,					__snake_case     :    Tuple	,					__snake_case     :    List[str]		):
										lowerCamelCase						:str						  =     TFConvBertForMaskedLM(config=__snake_case		)
										lowerCamelCase						:Dict						  =     {
										    '''input_ids''': input_ids,
										    '''attention_mask''': input_mask,
										    '''token_type_ids''': token_type_ids,
										}
										lowerCamelCase						:List[Any]						  =     model(__snake_case		)
										self.parent.assertEqual(result.logits.shape	,					(self.batch_size, self.seq_length, self.vocab_size)		)







							def 				snake_case  (				self     :    List[Any]	,					__snake_case     :    List[Any]	,					__snake_case     :    Optional[int]	,					__snake_case     :    Optional[int]	,					__snake_case     :    Any	,					__snake_case     :    str	,					__snake_case     :    Dict	,					__snake_case     :    Tuple		):
										lowerCamelCase						:Any						  =     self.num_labels
										lowerCamelCase						:List[str]						  =     TFConvBertForSequenceClassification(config=__snake_case		)
										lowerCamelCase						:int						  =     {
										    '''input_ids''': input_ids,
										    '''attention_mask''': input_mask,
										    '''token_type_ids''': token_type_ids,
										}

										lowerCamelCase						:str						  =     model(__snake_case		)
										self.parent.assertEqual(result.logits.shape	,					(self.batch_size, self.num_labels)		)







							def 				snake_case  (				self     :    Union[str, Any]	,					__snake_case     :    Union[str, Any]	,					__snake_case     :    List[Any]	,					__snake_case     :    Optional[Any]	,					__snake_case     :    str	,					__snake_case     :    Dict	,					__snake_case     :    Optional[Any]	,					__snake_case     :    int		):
										lowerCamelCase						:Union[str, Any]						  =     self.num_choices
										lowerCamelCase						:int						  =     TFConvBertForMultipleChoice(config=__snake_case		)
										lowerCamelCase						:int						  =     tf.tile(tf.expand_dims(__snake_case	,					1		)	,					(1, self.num_choices, 1)		)
										lowerCamelCase						:Optional[Any]						  =     tf.tile(tf.expand_dims(__snake_case	,					1		)	,					(1, self.num_choices, 1)		)
										lowerCamelCase						:Union[str, Any]						  =     tf.tile(tf.expand_dims(__snake_case	,					1		)	,					(1, self.num_choices, 1)		)
										lowerCamelCase						:List[Any]						  =     {
										    '''input_ids''': multiple_choice_inputs_ids,
										    '''attention_mask''': multiple_choice_input_mask,
										    '''token_type_ids''': multiple_choice_token_type_ids,
										}
										lowerCamelCase						:Optional[int]						  =     model(__snake_case		)
										self.parent.assertEqual(result.logits.shape	,					(self.batch_size, self.num_choices)		)







							def 				snake_case  (				self     :    Union[str, Any]	,					__snake_case     :    List[Any]	,					__snake_case     :    Dict	,					__snake_case     :    Tuple	,					__snake_case     :    Union[str, Any]	,					__snake_case     :    Any	,					__snake_case     :    List[Any]	,					__snake_case     :    Optional[int]		):
										lowerCamelCase						:List[Any]						  =     self.num_labels
										lowerCamelCase						:Any						  =     TFConvBertForTokenClassification(config=__snake_case		)
										lowerCamelCase						:List[str]						  =     {
										    '''input_ids''': input_ids,
										    '''attention_mask''': input_mask,
										    '''token_type_ids''': token_type_ids,
										}
										lowerCamelCase						:Any						  =     model(__snake_case		)
										self.parent.assertEqual(result.logits.shape	,					(self.batch_size, self.seq_length, self.num_labels)		)







							def 				snake_case  (				self     :    Any	,					__snake_case     :    Optional[Any]	,					__snake_case     :    Dict	,					__snake_case     :    Any	,					__snake_case     :    Optional[Any]	,					__snake_case     :    Union[str, Any]	,					__snake_case     :    Any	,					__snake_case     :    List[str]		):
										lowerCamelCase						:Dict						  =     TFConvBertForQuestionAnswering(config=__snake_case		)
										lowerCamelCase						:str						  =     {
										    '''input_ids''': input_ids,
										    '''attention_mask''': input_mask,
										    '''token_type_ids''': token_type_ids,
										}

										lowerCamelCase						:List[Any]						  =     model(__snake_case		)
										self.parent.assertEqual(result.start_logits.shape	,					(self.batch_size, self.seq_length)		)
										self.parent.assertEqual(result.end_logits.shape	,					(self.batch_size, self.seq_length)		)







							def 				snake_case  (				self     :    Optional[Any]		):
										lowerCamelCase						:List[Any]						  =     self.prepare_config_and_inputs()
										(
										    (
										    lowerCamelCase
										)  ,    (
										    lowerCamelCase
										)  ,    (
										    lowerCamelCase
										)  ,    (
										    lowerCamelCase
										)  ,    (
										    lowerCamelCase
										)  ,    (
										    lowerCamelCase
										)  ,    (
										    lowerCamelCase
										)  ,    
										)						:str						  =     config_and_inputs
										lowerCamelCase						:Dict						  =     {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
										return config, inputs_dict


@require_tf
class   _lowerCAmelCase		(     __SCREAMING_SNAKE_CASE      ,   __SCREAMING_SNAKE_CASE      ,   unittest.TestCase				):
							_UpperCAmelCase						    =     (
							    (
							        TFConvBertModel,
							        TFConvBertForMaskedLM,
							        TFConvBertForQuestionAnswering,
							        TFConvBertForSequenceClassification,
							        TFConvBertForTokenClassification,
							        TFConvBertForMultipleChoice,
							    )
							    if is_tf_available()
							    else ()
							)
							_UpperCAmelCase						    =     (
							    {
							        'feature-extraction': TFConvBertModel,
							        'fill-mask': TFConvBertForMaskedLM,
							        'question-answering': TFConvBertForQuestionAnswering,
							        'text-classification': TFConvBertForSequenceClassification,
							        'token-classification': TFConvBertForTokenClassification,
							        'zero-shot': TFConvBertForSequenceClassification,
							    }
							    if is_tf_available()
							    else {}
							)
							_UpperCAmelCase						    =     False
							_UpperCAmelCase						    =     False
							_UpperCAmelCase						    =     False







							def 				snake_case  (				self     :    int		):
										lowerCamelCase						:str						  =     TFConvBertModelTester(self		)
										lowerCamelCase						:str						  =     ConfigTester(self	,					config_class=__snake_case	,					hidden_size=37		)







							def 				snake_case  (				self     :    Optional[Any]		):
										self.config_tester.run_common_tests()







							def 				snake_case  (				self     :    Optional[int]		):
										lowerCamelCase						:Union[str, Any]						  =     self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*__snake_case		)







							def 				snake_case  (				self     :    List[Any]		):
										lowerCamelCase						:Optional[Any]						  =     self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_masked_lm(*__snake_case		)







							def 				snake_case  (				self     :    Any		):
										lowerCamelCase						:Tuple						  =     self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_multiple_choice(*__snake_case		)







							def 				snake_case  (				self     :    Tuple		):
										lowerCamelCase						:List[Any]						  =     self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_question_answering(*__snake_case		)







							def 				snake_case  (				self     :    Any		):
										lowerCamelCase						:Optional[Any]						  =     self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_sequence_classification(*__snake_case		)







							def 				snake_case  (				self     :    str		):
										lowerCamelCase						:int						  =     self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_token_classification(*__snake_case		)







							@slow
							def 				snake_case  (				self     :    int		):
										lowerCamelCase  ,    lowerCamelCase						:int						  =     self.model_tester.prepare_config_and_inputs_for_common()
										lowerCamelCase						:List[str]						  =     True
										lowerCamelCase						:List[Any]						  =     True

										if hasattr(__snake_case	,					'''use_cache'''		):
													lowerCamelCase						:List[Any]						  =     True

										lowerCamelCase						:int						  =     getattr(self.model_tester	,					'''encoder_seq_length'''	,					self.model_tester.seq_length		)
										lowerCamelCase						:Any						  =     getattr(self.model_tester	,					'''key_length'''	,					__snake_case		)

										for model_class in self.all_model_classes:
													lowerCamelCase						:Dict						  =     self._prepare_for_class(__snake_case	,					__snake_case		)
													lowerCamelCase						:Dict						  =     model_class(__snake_case		)
													lowerCamelCase						:Optional[int]						  =     len(model(__snake_case		)		)

													with tempfile.TemporaryDirectory() as tmpdirname:
																model.save_pretrained(__snake_case	,					saved_model=__snake_case		)
																lowerCamelCase						:Optional[Any]						  =     os.path.join(__snake_case	,					'''saved_model'''	,					'''1'''		)
																lowerCamelCase						:str						  =     tf.keras.models.load_model(__snake_case		)
																lowerCamelCase						:List[Any]						  =     model(__snake_case		)

																if self.is_encoder_decoder:
																			lowerCamelCase						:List[Any]						  =     outputs['''encoder_hidden_states''']
																			lowerCamelCase						:Optional[int]						  =     outputs['''encoder_attentions''']
																else:
																			lowerCamelCase						:Any						  =     outputs['''hidden_states''']
																			lowerCamelCase						:Union[str, Any]						  =     outputs['''attentions''']

																self.assertEqual(len(__snake_case		)	,					__snake_case		)

																lowerCamelCase						:Optional[int]						  =     getattr(
																    self.model_tester	,					'''expected_num_hidden_layers'''	,					self.model_tester.num_hidden_layers + 1		)

																self.assertEqual(len(__snake_case		)	,					__snake_case		)
																self.assertListEqual(
																    list(output_hidden_states[0].shape[-2:]		)	,					[self.model_tester.seq_length, self.model_tester.hidden_size]	,					)

																self.assertEqual(len(__snake_case		)	,					self.model_tester.num_hidden_layers		)
																self.assertListEqual(
																    list(output_attentions[0].shape[-3:]		)	,					[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]	,					)







							@slow
							def 				snake_case  (				self     :    Union[str, Any]		):
										lowerCamelCase						:int						  =     TFConvBertModel.from_pretrained('''YituTech/conv-bert-base'''		)
										self.assertIsNotNone(__snake_case		)







							def 				snake_case  (				self     :    Union[str, Any]		):
										lowerCamelCase  ,    lowerCamelCase						:Optional[Any]						  =     self.model_tester.prepare_config_and_inputs_for_common()
										lowerCamelCase						:Optional[int]						  =     True
										lowerCamelCase						:Optional[Any]						  =     getattr(self.model_tester	,					'''decoder_seq_length'''	,					self.model_tester.seq_length		)
										lowerCamelCase						:List[Any]						  =     getattr(self.model_tester	,					'''encoder_seq_length'''	,					self.model_tester.seq_length		)
										lowerCamelCase						:int						  =     getattr(self.model_tester	,					'''key_length'''	,					__snake_case		)
										lowerCamelCase						:List[Any]						  =     getattr(self.model_tester	,					'''key_length'''	,					__snake_case		)

										def check_decoder_attentions_output(__snake_case     :    Union[str, Any]		):
													lowerCamelCase						:Optional[Any]						  =     len(__snake_case		)
													self.assertEqual(out_len % 2	,					0		)
													lowerCamelCase						:Dict						  =     outputs.decoder_attentions
													self.assertEqual(len(__snake_case		)	,					self.model_tester.num_hidden_layers		)
													self.assertListEqual(
													    list(decoder_attentions[0].shape[-3:]		)	,					[self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length]	,					)

										def check_encoder_attentions_output(__snake_case     :    List[Any]		):
													lowerCamelCase						:Dict						  =     [
													    t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
													]
													self.assertEqual(len(__snake_case		)	,					self.model_tester.num_hidden_layers		)
													self.assertListEqual(
													    list(attentions[0].shape[-3:]		)	,					[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]	,					)

										for model_class in self.all_model_classes:
													lowerCamelCase						:Tuple						  =     True
													lowerCamelCase						:Dict						  =     False
													lowerCamelCase						:str						  =     model_class(__snake_case		)
													lowerCamelCase						:Any						  =     model(self._prepare_for_class(__snake_case	,					__snake_case		)		)
													lowerCamelCase						:str						  =     len(__snake_case		)
													self.assertEqual(config.output_hidden_states	,					__snake_case		)
													check_encoder_attentions_output(__snake_case		)

													if self.is_encoder_decoder:
																lowerCamelCase						:int						  =     model_class(__snake_case		)
																lowerCamelCase						:str						  =     model(self._prepare_for_class(__snake_case	,					__snake_case		)		)
																self.assertEqual(config.output_hidden_states	,					__snake_case		)
																check_decoder_attentions_output(__snake_case		)

													# Check that output attentions can also be changed via the config
													del inputs_dict["output_attentions"]
													lowerCamelCase						:int						  =     True
													lowerCamelCase						:Optional[int]						  =     model_class(__snake_case		)
													lowerCamelCase						:List[Any]						  =     model(self._prepare_for_class(__snake_case	,					__snake_case		)		)
													self.assertEqual(config.output_hidden_states	,					__snake_case		)
													check_encoder_attentions_output(__snake_case		)

													# Check attention is always last and order is fine
													lowerCamelCase						:Any						  =     True
													lowerCamelCase						:Dict						  =     True
													lowerCamelCase						:str						  =     model_class(__snake_case		)
													lowerCamelCase						:int						  =     model(self._prepare_for_class(__snake_case	,					__snake_case		)		)

													self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)	,					len(__snake_case		)		)
													self.assertEqual(model.config.output_hidden_states	,					__snake_case		)
													check_encoder_attentions_output(__snake_case		)


@require_tf
class   _lowerCAmelCase		(     unittest.TestCase				):







							@slow
							def 				snake_case  (				self     :    List[Any]		):
										lowerCamelCase						:Dict						  =     TFConvBertModel.from_pretrained('''YituTech/conv-bert-base'''		)
										lowerCamelCase						:Any						  =     tf.constant([[0, 1, 2, 3, 4, 5]]		)
										lowerCamelCase						:Optional[int]						  =     model(__snake_case		)[0]

										lowerCamelCase						:Optional[int]						  =     [1, 6, 768]
										self.assertEqual(output.shape	,					__snake_case		)

										lowerCamelCase						:Optional[Any]						  =     tf.constant(
										    [
										        [
										            [-0.0_3_4_7_5_4_9_3, -0.4_6_8_6_0_3_4, -0.3_0_6_3_8_8_3_2],
										            [0.2_2_6_3_7_2_4_8, -0.2_6_9_8_8_6_4_6, -0.7_4_2_3_4_2_4],
										            [0.1_0_3_2_4_8_6_8, -0.4_5_0_1_3_5_0_8, -0.5_8_2_8_0_7_8_4],
										        ]
										    ]		)
										tf.debugging.assert_near(output[:, :3, :3]	,					__snake_case	,					atol=1e-4		)




import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin
class   _lowerCAmelCase		(     __SCREAMING_SNAKE_CASE      ,   unittest.TestCase				):
							_UpperCAmelCase						    =     DebertaTokenizer
							_UpperCAmelCase						    =     True
							_UpperCAmelCase						    =     DebertaTokenizerFast







							def 				snake_case  (				self     :    List[str]		):
										super().setUp()

										# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
										lowerCamelCase						:Dict						  =     [
										    '''l''',
										    '''o''',
										    '''w''',
										    '''e''',
										    '''r''',
										    '''s''',
										    '''t''',
										    '''i''',
										    '''d''',
										    '''n''',
										    '''\u0120''',
										    '''\u0120l''',
										    '''\u0120n''',
										    '''\u0120lo''',
										    '''\u0120low''',
										    '''er''',
										    '''\u0120lowest''',
										    '''\u0120newer''',
										    '''\u0120wider''',
										    '''[UNK]''',
										]
										lowerCamelCase						:List[str]						  =     dict(zip(__snake_case	,					range(len(__snake_case		)		)		)		)
										lowerCamelCase						:Any						  =     ['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
										lowerCamelCase						:Dict						  =     {'''unk_token''': '''[UNK]'''}

										lowerCamelCase						:Any						  =     os.path.join(self.tmpdirname	,					VOCAB_FILES_NAMES['''vocab_file''']		)
										lowerCamelCase						:List[str]						  =     os.path.join(self.tmpdirname	,					VOCAB_FILES_NAMES['''merges_file''']		)
										with open(self.vocab_file	,					'''w'''	,					encoding='''utf-8'''		) as fp:
													fp.write(json.dumps(__snake_case		) + '''\n'''		)
										with open(self.merges_file	,					'''w'''	,					encoding='''utf-8'''		) as fp:
													fp.write('''\n'''.join(__snake_case		)		)







							def 				snake_case  (				self     :    str	,					**__snake_case     :    Dict		):
										kwargs.update(self.special_tokens_map		)
										return self.tokenizer_class.from_pretrained(self.tmpdirname	,					**__snake_case		)







							def 				snake_case  (				self     :    Optional[Any]	,					__snake_case     :    int		):
										lowerCamelCase						:List[Any]						  =     '''lower newer'''
										lowerCamelCase						:List[str]						  =     '''lower newer'''
										return input_text, output_text







							def 				snake_case  (				self     :    str		):
										lowerCamelCase						:Optional[int]						  =     self.get_tokenizer()
										lowerCamelCase						:Union[str, Any]						  =     '''lower newer'''
										lowerCamelCase						:str						  =     ['''l''', '''o''', '''w''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
										lowerCamelCase						:Optional[int]						  =     tokenizer.tokenize(__snake_case		)
										self.assertListEqual(__snake_case	,					__snake_case		)

										lowerCamelCase						:List[str]						  =     tokens + [tokenizer.unk_token]
										lowerCamelCase						:Optional[int]						  =     [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
										self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case		)	,					__snake_case		)







							def 				snake_case  (				self     :    Optional[int]		):
										lowerCamelCase						:List[str]						  =     self.get_tokenizer()
										lowerCamelCase						:Optional[int]						  =     tokenizer('''Hello'''	,					'''World'''		)
										lowerCamelCase						:List[str]						  =     [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
										self.assertListEqual(tokd['''token_type_ids''']	,					__snake_case		)







							@slow
							def 				snake_case  (				self     :    str		):
										lowerCamelCase						:Union[str, Any]						  =     self.tokenizer_class.from_pretrained('''microsoft/deberta-base'''		)

										lowerCamelCase						:Optional[Any]						  =     tokenizer.encode('''sequence builders'''	,					add_special_tokens=__snake_case		)
										lowerCamelCase						:Tuple						  =     tokenizer.encode('''multi-sequence build'''	,					add_special_tokens=__snake_case		)

										lowerCamelCase						:Union[str, Any]						  =     tokenizer.encode(
										    '''sequence builders'''	,					add_special_tokens=__snake_case	,					add_prefix_space=__snake_case		)
										lowerCamelCase						:str						  =     tokenizer.encode(
										    '''sequence builders'''	,					'''multi-sequence build'''	,					add_special_tokens=__snake_case	,					add_prefix_space=__snake_case		)

										lowerCamelCase						:Any						  =     tokenizer.build_inputs_with_special_tokens(__snake_case		)
										lowerCamelCase						:Dict						  =     tokenizer.build_inputs_with_special_tokens(__snake_case	,					__snake_case		)

										assert encoded_sentence == encoded_text_from_decode
										assert encoded_pair == encoded_pair_from_decode







							@slow
							def 				snake_case  (				self     :    str		):
										lowerCamelCase						:List[str]						  =     [self.tokenizer_class]
										if self.test_rust_tokenizer:
													tokenizer_classes.append(self.rust_tokenizer_class		)

										for tokenizer_class in tokenizer_classes:
													lowerCamelCase						:int						  =     tokenizer_class.from_pretrained('''microsoft/deberta-base'''		)

													lowerCamelCase						:Tuple						  =     [
													    '''ALBERT: A Lite BERT for Self-supervised Learning of Language Representations''',
													    '''ALBERT incorporates two parameter reduction techniques''',
													    '''The first one is a factorized embedding parameterization. By decomposing the large vocabulary'''
													    ''' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'''
													    ''' vocabulary embedding.''',
													]

													lowerCamelCase						:List[Any]						  =     tokenizer(__snake_case	,					padding=__snake_case		)
													lowerCamelCase						:Union[str, Any]						  =     [tokenizer.decode(__snake_case	,					skip_special_tokens=__snake_case		) for seq in encoding['''input_ids''']]

													# fmt: off
													lowerCamelCase						:Any						  =     {
													    '''input_ids''': [
													        [1, 2118, 11126, 565, 35, 83, 25191, 163, 18854, 13, 12156, 12, 16101, 25376, 13807, 9, 22205, 27893, 1635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
													        [1, 2118, 11126, 565, 24536, 80, 43797, 4878, 7373, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
													        [1, 133, 78, 65, 16, 10, 3724, 1538, 33183, 11303, 43797, 1938, 4, 870, 24165, 29105, 5, 739, 32644, 33183, 11303, 36173, 88, 80, 650, 7821, 45940, 6, 52, 2559, 5, 1836, 9, 5, 7397, 13171, 31, 5, 1836, 9, 32644, 33183, 11303, 4, 2]
													    ],
													    '''token_type_ids''': [
													        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
													        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
													        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
													    ],
													    '''attention_mask''': [
													        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
													        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
													        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
													    ]
													}
													# fmt: on

													lowerCamelCase						:Optional[int]						  =     [
													    '''ALBERT: A Lite BERT for Self-supervised Learning of Language Representations''',
													    '''ALBERT incorporates two parameter reduction techniques''',
													    '''The first one is a factorized embedding parameterization. By decomposing the large vocabulary'''
													    ''' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'''
													    ''' vocabulary embedding.''',
													]

													self.assertDictEqual(encoding.data	,					__snake_case		)

													for expected, decoded in zip(__snake_case	,					__snake_case		):
																self.assertEqual(__snake_case	,					__snake_case		)



