
"""simple docstring"""



import inspect
import tempfile
import unittest

from huggingface_hub import hf_hub_download

from transformers import is_torch_available
from transformers.testing_utils import is_flaky, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


_snake_case          =						1E-4

if is_torch_available():
						import torch

						from transformers import AutoformerConfig, AutoformerForPrediction, AutoformerModel
						from transformers.models.autoformer.modeling_autoformer import AutoformerDecoder, AutoformerEncoder
@require_torch
class        _SCREAMING_SNAKE_CASE  :






				'''simple docstring'''



				def __init__(					self	:			str				,     UpperCAmelCase_	:			Optional[Any]				,     UpperCAmelCase_	:			Any=16				,     UpperCAmelCase_	:			Optional[int]=13				,     UpperCAmelCase_	:			str=7				,     UpperCAmelCase_	:			str=14				,     UpperCAmelCase_	:			int=10				,     UpperCAmelCase_	:			int=19				,     UpperCAmelCase_	:			int=5				,     UpperCAmelCase_	:			Any=4				,     UpperCAmelCase_	:			Optional[Any]=True				,     UpperCAmelCase_	:			List[Any]=16				,     UpperCAmelCase_	:			List[Any]=2				,     UpperCAmelCase_	:			Optional[Any]=4				,     UpperCAmelCase_	:			Optional[int]=4				,     UpperCAmelCase_	:			Optional[Any]="gelu"				,     UpperCAmelCase_	:			Optional[Any]=0.1				,     UpperCAmelCase_	:			int=0.1				,     UpperCAmelCase_	:			Any=[1, 2, 3, 4, 5]				,     UpperCAmelCase_	:			Any=25				,     UpperCAmelCase_	:			Optional[Any]=5				,     )		->  Optional[Any]:







											"""simple docstring"""




											_lowerCAmelCase  	=			d_model
											_lowerCAmelCase  	=			parent
											_lowerCAmelCase  	=			batch_size
											_lowerCAmelCase  	=			prediction_length
											_lowerCAmelCase  	=			context_length
											_lowerCAmelCase  	=			cardinality
											_lowerCAmelCase  	=			num_time_features
											_lowerCAmelCase  	=			lags_sequence
											_lowerCAmelCase  	=			embedding_dimension
											_lowerCAmelCase  	=			is_training
											_lowerCAmelCase  	=			hidden_size
											_lowerCAmelCase  	=			num_hidden_layers
											_lowerCAmelCase  	=			num_attention_heads
											_lowerCAmelCase  	=			intermediate_size
											_lowerCAmelCase  	=			hidden_act
											_lowerCAmelCase  	=			hidden_dropout_prob
											_lowerCAmelCase  	=			attention_probs_dropout_prob

											_lowerCAmelCase  	=			context_length
											_lowerCAmelCase  	=			prediction_length + label_length
											_lowerCAmelCase  	=			label_length

											_lowerCAmelCase  	=			moving_average
											_lowerCAmelCase  	=			autocorrelation_factor


				def 				__lowerCamelCase	(					self	:			Union[str, Any]  )		->  Optional[int]:







											"""simple docstring"""




											return AutoformerConfig(
											    d_model=self.d_model				,     encoder_layers=self.num_hidden_layers				,     decoder_layers=self.num_hidden_layers				,     encoder_attention_heads=self.num_attention_heads				,     decoder_attention_heads=self.num_attention_heads				,     encoder_ffn_dim=self.intermediate_size				,     decoder_ffn_dim=self.intermediate_size				,     dropout=self.hidden_dropout_prob				,     attention_dropout=self.attention_probs_dropout_prob				,     prediction_length=self.prediction_length				,     context_length=self.context_length				,     label_length=self.label_length				,     lags_sequence=self.lags_sequence				,     num_time_features=self.num_time_features				,     num_static_categorical_features=1				,     cardinality=[self.cardinality]				,     embedding_dimension=[self.embedding_dimension]				,     moving_average=self.moving_average				,     )


				def 				__lowerCamelCase	(					self	:			List[str]				,     UpperCAmelCase_	:			Tuple  )		->  Tuple:







											"""simple docstring"""




											_lowerCAmelCase  	=			config.context_length + max(config.lags_sequence  )

											_lowerCAmelCase  	=			ids_tensor([self.batch_size, 1]				,     config.cardinality[0]  )
											_lowerCAmelCase  	=			floats_tensor([self.batch_size, _past_length, config.num_time_features]  )
											_lowerCAmelCase  	=			floats_tensor([self.batch_size, _past_length]  )
											_lowerCAmelCase  	=			floats_tensor([self.batch_size, _past_length]  ) > 0.5

											# decoder inputs
											_lowerCAmelCase  	=			floats_tensor([self.batch_size, config.prediction_length, config.num_time_features]  )
											_lowerCAmelCase  	=			floats_tensor([self.batch_size, config.prediction_length]  )

											_lowerCAmelCase  	=			{
											    'past_values': past_values,
											    'static_categorical_features': static_categorical_features,
											    'past_time_features': past_time_features,
											    'past_observed_mask': past_observed_mask,
											    'future_time_features': future_time_features,
											    'future_values': future_values,
											}
											return inputs_dict


				def 				__lowerCamelCase	(					self	:			List[str]  )		->  int:







											"""simple docstring"""




											_lowerCAmelCase  	=			self.get_config()
											_lowerCAmelCase  	=			self.prepare_autoformer_inputs_dict(_A  )
											return config, inputs_dict


				def 				__lowerCamelCase	(					self	:			List[Any]  )		->  List[str]:







											"""simple docstring"""




											_lowerCAmelCase  	=			self.prepare_config_and_inputs()
											return config, inputs_dict







				def 				__lowerCamelCase	(					self	:			Union[str, Any]				,     UpperCAmelCase_	:			Tuple				,     UpperCAmelCase_	:			Union[str, Any]  )		->  Dict:







											"""simple docstring"""




											_lowerCAmelCase  	=			AutoformerModel(config=_A  ).to(_A  ).eval()
											_lowerCAmelCase  	=			model(**_A  )

											_lowerCAmelCase  	=			outputs.encoder_last_hidden_state
											_lowerCAmelCase  	=			outputs.last_hidden_state

											with tempfile.TemporaryDirectory() as tmpdirname:
																		_lowerCAmelCase  	=			model.get_encoder()
																		encoder.save_pretrained(_A  )
																		_lowerCAmelCase  	=			AutoformerEncoder.from_pretrained(_A  ).to(_A  )

											_lowerCAmelCase  	=			model.create_network_inputs(**_A  )
											_lowerCAmelCase  	=			model.decomposition_layer(transformer_inputs[:, : config.context_length, ...]  )

											_lowerCAmelCase  	=			torch.cat(
											    (transformer_inputs[:, : config.context_length, ...], feature[:, : config.context_length, ...])				,     dim=-1				,     )
											_lowerCAmelCase  	=			encoder(inputs_embeds=_A  )[0]
											self.parent.assertTrue((encoder_last_hidden_state_a - encoder_last_hidden_state).abs().max().item() < 1E-3  )

											_lowerCAmelCase  	=			(
											    torch.mean(transformer_inputs[:, : config.context_length, ...]				,     dim=1  )
											    .unsqueeze(1  )
											    .repeat(1				,     config.prediction_length				,     1  )
											)
											_lowerCAmelCase  	=			torch.zeros(
											    [transformer_inputs.shape[0], config.prediction_length, transformer_inputs.shape[2]]				,     device=enc_input.device				,     )

											_lowerCAmelCase  	=			torch.cat(
											    (
											        torch.cat((seasonal_input[:, -config.label_length :, ...], zeros)				,     dim=1  ),
											        feature[:, config.context_length - config.label_length :, ...],
											    )				,     dim=-1				,     )
											_lowerCAmelCase  	=			torch.cat(
											    (
											        torch.cat((trend_input[:, -config.label_length :, ...], mean)				,     dim=1  ),
											        feature[:, config.context_length - config.label_length :, ...],
											    )				,     dim=-1				,     )

											with tempfile.TemporaryDirectory() as tmpdirname:
																		_lowerCAmelCase  	=			model.get_decoder()
																		decoder.save_pretrained(_A  )
																		_lowerCAmelCase  	=			AutoformerDecoder.from_pretrained(_A  ).to(_A  )

											_lowerCAmelCase  	=			decoder(
											    trend=_A				,     inputs_embeds=_A				,     encoder_hidden_states=_A				,     )[0]

											self.parent.assertTrue((last_hidden_state_a - last_hidden_state).abs().max().item() < 1E-3  )






@require_torch
class        _SCREAMING_SNAKE_CASE  (  a__      ,				a__      ,				unittest.TestCase							):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE_:							str							=					(AutoformerModel, AutoformerForPrediction) if is_torch_available() else ()
				SCREAMING_SNAKE_CASE_:							List[Any]							=					(AutoformerForPrediction,) if is_torch_available() else ()
				SCREAMING_SNAKE_CASE_:							Union[str, Any]							=					{"feature-extraction": AutoformerModel} if is_torch_available() else {}
				SCREAMING_SNAKE_CASE_:							Optional[int]							=					False
				SCREAMING_SNAKE_CASE_:							Tuple							=					False
				SCREAMING_SNAKE_CASE_:							Dict							=					False
				SCREAMING_SNAKE_CASE_:							Optional[Any]							=					False
				SCREAMING_SNAKE_CASE_:							Optional[Any]							=					False
				SCREAMING_SNAKE_CASE_:							int							=					False


				def 				__lowerCamelCase	(					self	:			Optional[Any]  )		->  Union[str, Any]:







											"""simple docstring"""




											_lowerCAmelCase  	=			AutoformerModelTester(self  )
											_lowerCAmelCase  	=			ConfigTester(self				,     config_class=_A				,     has_text_modality=_A  )


				def 				__lowerCamelCase	(					self	:			Union[str, Any]  )		->  Optional[int]:







											"""simple docstring"""




											self.config_tester.run_common_tests()


				def 				__lowerCamelCase	(					self	:			Any  )		->  Optional[int]:







											"""simple docstring"""




											_lowerCAmelCase  	=			self.model_tester.prepare_config_and_inputs()
											for model_class in self.all_model_classes:
																		_lowerCAmelCase  	=			model_class(_A  )

																		with tempfile.TemporaryDirectory() as tmpdirname:
																									model.save_pretrained(_A  )
																									_lowerCAmelCase  	=			model_class.from_pretrained(_A				,     output_loading_info=_A  )
																		self.assertEqual(info['missing_keys']				,     []  )


				def 				__lowerCamelCase	(					self	:			int  )		->  Optional[Any]:







											"""simple docstring"""




											_lowerCAmelCase  	=			self.model_tester.prepare_config_and_inputs_for_common()
											self.model_tester.check_encoder_decoder_model_standalone(*_A  )


				@unittest.skip(reason='Model has no tokens embeddings'  )
				def 				__lowerCamelCase	(					self	:			Optional[Any]  )		->  Dict:







											"""simple docstring"""




											pass


				def 				__lowerCamelCase	(					self	:			Dict  )		->  List[Any]:







											"""simple docstring"""




											_lowerCAmelCase  	=			inspect.signature(getattr(_A				,     'forward'  )  )
											# The main input is the name of the argument after `self`
											_lowerCAmelCase  	=			list(model_signature.parameters.keys()  )[1]
											self.assertEqual(AutoformerModel.main_input_name				,     _A  )


				def 				__lowerCamelCase	(					self	:			Union[str, Any]  )		->  str:







											"""simple docstring"""




											_lowerCAmelCase  	=			self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
																		_lowerCAmelCase  	=			model_class(_A  )
																		_lowerCAmelCase  	=			inspect.signature(model.forward  )
																		# signature.parameters is an OrderedDict => so arg_names order is deterministic
																		_lowerCAmelCase  	=			[*signature.parameters.keys()]

																		_lowerCAmelCase  	=			[
																		    'past_values',
																		    'past_time_features',
																		    'past_observed_mask',
																		    'static_categorical_features',
																		    'static_real_features',
																		    'future_values',
																		    'future_time_features',
																		]

																		if model.__class__.__name__ in ["AutoformerForPrediction"]:
																									expected_arg_names.append('future_observed_mask'  )

																		expected_arg_names.extend(
																		    [
																		        'decoder_attention_mask',
																		        'head_mask',
																		        'decoder_head_mask',
																		        'cross_attn_head_mask',
																		        'encoder_outputs',
																		        'past_key_values',
																		        'output_hidden_states',
																		        'output_attentions',
																		        'use_cache',
																		        'return_dict',
																		    ]  )

																		self.assertListEqual(arg_names[: len(_A  )]				,     _A  )


				def 				__lowerCamelCase	(					self	:			List[Any]  )		->  Any:







											"""simple docstring"""




											_lowerCAmelCase  	=			self.model_tester.prepare_config_and_inputs_for_common()
											_lowerCAmelCase  	=			True

											_lowerCAmelCase  	=			getattr(self.model_tester				,     'seq_length'				,     _A  )
											_lowerCAmelCase  	=			getattr(self.model_tester				,     'decoder_seq_length'				,     _A  )
											_lowerCAmelCase  	=			getattr(self.model_tester				,     'encoder_seq_length'				,     _A  )
											_lowerCAmelCase  	=			getattr(self.model_tester				,     'd_model'				,     _A  )
											_lowerCAmelCase  	=			getattr(self.model_tester				,     'num_attention_heads'				,     _A  )
											_lowerCAmelCase  	=			d_model // num_attention_heads

											for model_class in self.all_model_classes:
																		_lowerCAmelCase  	=			True
																		_lowerCAmelCase  	=			False
																		_lowerCAmelCase  	=			True
																		_lowerCAmelCase  	=			model_class(_A  )
																		model.to(_A  )
																		model.eval()
																		with torch.no_grad():
																									_lowerCAmelCase  	=			model(**self._prepare_for_class(_A				,     _A  )  )
																		_lowerCAmelCase  	=			outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
																		self.assertEqual(len(_A  )				,     self.model_tester.num_hidden_layers  )

																		# check that output_attentions also work using config
																		del inputs_dict["output_attentions"]
																		_lowerCAmelCase  	=			True
																		_lowerCAmelCase  	=			model_class(_A  )
																		model.to(_A  )
																		model.eval()
																		with torch.no_grad():
																									_lowerCAmelCase  	=			model(**self._prepare_for_class(_A				,     _A  )  )
																		_lowerCAmelCase  	=			outputs.encoder_attentions
																		self.assertEqual(len(_A  )				,     self.model_tester.num_hidden_layers  )

																		self.assertListEqual(
																		    list(attentions[0].shape[-3:]  )				,     [self.model_tester.num_attention_heads, encoder_seq_length, dim]				,     )
																		_lowerCAmelCase  	=			len(_A  )

																		_lowerCAmelCase  	=			7

																		if "last_hidden_state" in outputs:
																									correct_outlen += 1

																		if "trend" in outputs:
																									correct_outlen += 1

																		if "past_key_values" in outputs:
																									correct_outlen += 1  # past_key_values have been returned

																		if "loss" in outputs:
																									correct_outlen += 1

																		if "params" in outputs:
																									correct_outlen += 1

																		self.assertEqual(_A				,     _A  )

																		# decoder attentions
																		_lowerCAmelCase  	=			outputs.decoder_attentions
																		self.assertIsInstance(_A				,     (list, tuple)  )
																		self.assertEqual(len(_A  )				,     self.model_tester.num_hidden_layers  )
																		self.assertListEqual(
																		    list(decoder_attentions[0].shape[-3:]  )				,     [self.model_tester.num_attention_heads, decoder_seq_length, dim]				,     )

																		# cross attentions
																		_lowerCAmelCase  	=			outputs.cross_attentions
																		self.assertIsInstance(_A				,     (list, tuple)  )
																		self.assertEqual(len(_A  )				,     self.model_tester.num_hidden_layers  )
																		self.assertListEqual(
																		    list(cross_attentions[0].shape[-3:]  )				,     [self.model_tester.num_attention_heads, decoder_seq_length, dim]				,     )

											# Check attention is always last and order is fine
											_lowerCAmelCase  	=			True
											_lowerCAmelCase  	=			True
											_lowerCAmelCase  	=			model_class(_A  )
											model.to(_A  )
											model.eval()
											with torch.no_grad():
																		_lowerCAmelCase  	=			model(**self._prepare_for_class(_A				,     _A  )  )

											self.assertEqual(out_len + 2				,     len(_A  )  )

											_lowerCAmelCase  	=			outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions

											self.assertEqual(len(_A  )				,     self.model_tester.num_hidden_layers  )
											self.assertListEqual(
											    list(self_attentions[0].shape[-3:]  )				,     [self.model_tester.num_attention_heads, encoder_seq_length, dim]				,     )







				@is_flaky()
				def 				__lowerCamelCase	(					self	:			int  )		->  Optional[Any]:







											"""simple docstring"""




											super().test_retain_grad_hidden_states_attentions()






def 	__snake_case  (    SCREAMING_SNAKE_CASE:					List[str]="train-batch.pt"						):






							"""simple docstring"""



							_lowerCAmelCase  	=			hf_hub_download(repo_id='hf-internal-testing/tourism-monthly-batch'     ,  filename=_lowerCamelCase     ,  repo_type='dataset'						)
							_lowerCAmelCase  	=			torch.load(_lowerCamelCase     ,  map_location=_lowerCamelCase						)
							return batch

@require_torch
@slow
class        _SCREAMING_SNAKE_CASE  (  unittest.TestCase							):






				'''simple docstring'''



				def 				__lowerCamelCase	(					self	:			Union[str, Any]  )		->  List[Any]:







											"""simple docstring"""




											_lowerCAmelCase  	=			AutoformerModel.from_pretrained('huggingface/autoformer-tourism-monthly'  ).to(_A  )
											_lowerCAmelCase  	=			prepare_batch()

											with torch.no_grad():
																		_lowerCAmelCase  	=			model(
																		    past_values=batch['past_values']				,     past_time_features=batch['past_time_features']				,     past_observed_mask=batch['past_observed_mask']				,     static_categorical_features=batch['static_categorical_features']				,     future_values=batch['future_values']				,     future_time_features=batch['future_time_features']				,     )[0]

											_lowerCAmelCase  	=			torch.Size(
											    (64, model.config.prediction_length + model.config.label_length, model.config.feature_size)  )
											self.assertEqual(output.shape				,     _A  )

											_lowerCAmelCase  	=			torch.tensor(
											    [[0.3593, -1.3398, 0.6330], [0.2279, 1.5396, -0.1792], [0.0450, 1.3225, -0.2335]]				,     device=_A  )
											self.assertTrue(torch.allclose(output[0, :3, :3]				,     _A				,     atol=_A  )  )


				def 				__lowerCamelCase	(					self	:			Any  )		->  Any:







											"""simple docstring"""




											_lowerCAmelCase  	=			AutoformerForPrediction.from_pretrained('huggingface/autoformer-tourism-monthly'  ).to(_A  )
											_lowerCAmelCase  	=			prepare_batch('val-batch.pt'  )
											with torch.no_grad():
																		_lowerCAmelCase  	=			model(
																		    past_values=batch['past_values']				,     past_time_features=batch['past_time_features']				,     past_observed_mask=batch['past_observed_mask']				,     static_categorical_features=batch['static_categorical_features']				,     ).encoder_last_hidden_state
											_lowerCAmelCase  	=			torch.Size((64, model.config.context_length, model.config.d_model)  )
											self.assertEqual(output.shape				,     _A  )

											_lowerCAmelCase  	=			torch.tensor(
											    [[-0.0734, -0.9036, 0.8358], [4.7186, 2.4113, 1.9581], [1.7953, 2.3558, 1.2970]]				,     device=_A  )
											self.assertTrue(torch.allclose(output[0, :3, :3]				,     _A				,     atol=_A  )  )







				def 				__lowerCamelCase	(					self	:			Dict  )		->  List[str]:







											"""simple docstring"""




											_lowerCAmelCase  	=			AutoformerForPrediction.from_pretrained('huggingface/autoformer-tourism-monthly'  ).to(_A  )
											_lowerCAmelCase  	=			prepare_batch('val-batch.pt'  )
											with torch.no_grad():
																		_lowerCAmelCase  	=			model.generate(
																		    static_categorical_features=batch['static_categorical_features']				,     past_time_features=batch['past_time_features']				,     past_values=batch['past_values']				,     future_time_features=batch['future_time_features']				,     past_observed_mask=batch['past_observed_mask']				,     )
											_lowerCAmelCase  	=			torch.Size((64, model.config.num_parallel_samples, model.config.prediction_length)  )
											self.assertEqual(outputs.sequences.shape				,     _A  )

											_lowerCAmelCase  	=			torch.tensor([3_130.6_763, 4_056.5_293, 7_053.0_786]				,     device=_A  )
											_lowerCAmelCase  	=			outputs.sequences.mean(dim=1  )
											self.assertTrue(torch.allclose(mean_prediction[0, -3:]				,     _A				,     rtol=1E-1  )  )








"""simple docstring"""



import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class        _SCREAMING_SNAKE_CASE  (  UpperCAmelCase      ,				unittest.TestCase							):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE_:							str							=					None
				SCREAMING_SNAKE_CASE_:							List[str]							=					BloomTokenizerFast
				SCREAMING_SNAKE_CASE_:							List[str]							=					BloomTokenizerFast
				SCREAMING_SNAKE_CASE_:							Any							=					True
				SCREAMING_SNAKE_CASE_:							List[Any]							=					False
				SCREAMING_SNAKE_CASE_:							Optional[int]							=					"tokenizer_file"
				SCREAMING_SNAKE_CASE_:							str							=					{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}


				def 				__lowerCamelCase	(					self	:			Tuple  )		->  str:







											"""simple docstring"""




											super().setUp()
											_lowerCAmelCase  	=			BloomTokenizerFast.from_pretrained('bigscience/tokenizer'  )
											tokenizer.save_pretrained(self.tmpdirname  )


				def 				__lowerCamelCase	(					self	:			List[str]				,     **UpperCAmelCase_	:			Any  )		->  Optional[Any]:







											"""simple docstring"""




											kwargs.update(self.special_tokens_map  )
											return BloomTokenizerFast.from_pretrained(self.tmpdirname				,     **UpperCAmelCase_  )


				def 				__lowerCamelCase	(					self	:			List[Any]  )		->  str:







											"""simple docstring"""




											_lowerCAmelCase  	=			self.get_rust_tokenizer()

											_lowerCAmelCase  	=			['The quick brown fox</s>', 'jumps over the lazy dog</s>']
											_lowerCAmelCase  	=			[[2_175, 23_714, 73_173, 144_252, 2], [77, 132_619, 3_478, 368, 109_586, 35_433, 2]]

											_lowerCAmelCase  	=			tokenizer.batch_encode_plus(UpperCAmelCase_  )['input_ids']
											self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_  )

											_lowerCAmelCase  	=			tokenizer.batch_decode(UpperCAmelCase_  )
											self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_  )


				def 				__lowerCamelCase	(					self	:			Any				,     UpperCAmelCase_	:			List[str]=6  )		->  Any:







											"""simple docstring"""




											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
																									_lowerCAmelCase  	=			self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_				,     **UpperCAmelCase_  )
																									# tokenizer_r.pad_token = None # Hotfixing padding = None
																									# Simple input
																									_lowerCAmelCase  	=			'This is a simple input'
																									_lowerCAmelCase  	=			['This is a simple input 1', 'This is a simple input 2']
																									_lowerCAmelCase  	=			('This is a simple input', 'This is a pair')
																									_lowerCAmelCase  	=			[
																									    ('This is a simple input 1', 'This is a simple input 2'),
																									    ('This is a simple pair 1', 'This is a simple pair 2'),
																									]

																									# Simple input tests
																									try:
																																tokenizer_r.encode(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																																tokenizer_r.encode_plus(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )

																																tokenizer_r.batch_encode_plus(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																																tokenizer_r.encode(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																																tokenizer_r.batch_encode_plus(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																									except ValueError:
																																self.fail('Bloom Tokenizer should be able to deal with padding'  )

																									_lowerCAmelCase  	=			None  # Hotfixing padding = None
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Simple input
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Simple input
																									self.assertRaises(
																									    UpperCAmelCase_				,     tokenizer_r.batch_encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'				,     )

																									# Pair input
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Pair input
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Pair input
																									self.assertRaises(
																									    UpperCAmelCase_				,     tokenizer_r.batch_encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'				,     )


				def 				__lowerCamelCase	(					self	:			str  )		->  Any:







											"""simple docstring"""




											_lowerCAmelCase  	=			self.get_rust_tokenizer()
											_lowerCAmelCase  	=			load_dataset('xnli'				,     'all_languages'				,     split='test'				,     streaming=UpperCAmelCase_  )

											_lowerCAmelCase  	=			next(iter(UpperCAmelCase_  )  )['premise']  # pick up one data
											_lowerCAmelCase  	=			list(sample_data.values()  )

											_lowerCAmelCase  	=			list(map(tokenizer.encode				,     UpperCAmelCase_  )  )
											_lowerCAmelCase  	=			[tokenizer.decode(UpperCAmelCase_				,     clean_up_tokenization_spaces=UpperCAmelCase_  ) for x in output_tokens]
											self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_  )







				def 				__lowerCamelCase	(					self	:			Tuple  )		->  List[Any]:







											"""simple docstring"""




											self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map  )				,     1  )
											self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()  )[0]  )				,     1  )






