
"""simple docstring"""



import copy
import inspect
import unittest

import numpy as np
from huggingface_hub import hf_hub_download

from transformers import VideoMAEConfig
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch
	from torch import nn

	from transformers import (
	    MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING,
	    VideoMAEForPreTraining,
	    VideoMAEForVideoClassification,
	    VideoMAEModel,
	)
	from transformers.models.videomae.modeling_videomae import VIDEOMAE_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
	from transformers import VideoMAEImageProcessor
class        _SCREAMING_SNAKE_CASE  :






				'''simple docstring'''



				def __init__(					self	:			str				,     UpperCAmelCase_	:			Optional[Any]				,     UpperCAmelCase_	:			List[str]=13				,     UpperCAmelCase_	:			Tuple=10				,     UpperCAmelCase_	:			List[str]=3				,     UpperCAmelCase_	:			List[str]=2				,     UpperCAmelCase_	:			Union[str, Any]=2				,     UpperCAmelCase_	:			List[Any]=2				,     UpperCAmelCase_	:			List[str]=True				,     UpperCAmelCase_	:			Tuple=True				,     UpperCAmelCase_	:			List[str]=32				,     UpperCAmelCase_	:			Dict=5				,     UpperCAmelCase_	:			Dict=4				,     UpperCAmelCase_	:			Tuple=37				,     UpperCAmelCase_	:			Dict="gelu"				,     UpperCAmelCase_	:			Union[str, Any]=0.1				,     UpperCAmelCase_	:			List[Any]=0.1				,     UpperCAmelCase_	:			List[Any]=10				,     UpperCAmelCase_	:			Union[str, Any]=0.02				,     UpperCAmelCase_	:			List[Any]=0.9				,     UpperCAmelCase_	:			List[Any]=None				,     )		->  List[Any]:







											"""simple docstring"""




											_lowerCAmelCase  	=			parent
											_lowerCAmelCase  	=			batch_size
											_lowerCAmelCase  	=			image_size
											_lowerCAmelCase  	=			num_channels
											_lowerCAmelCase  	=			patch_size
											_lowerCAmelCase  	=			tubelet_size
											_lowerCAmelCase  	=			num_frames
											_lowerCAmelCase  	=			is_training
											_lowerCAmelCase  	=			use_labels
											_lowerCAmelCase  	=			hidden_size
											_lowerCAmelCase  	=			num_hidden_layers
											_lowerCAmelCase  	=			num_attention_heads
											_lowerCAmelCase  	=			intermediate_size
											_lowerCAmelCase  	=			hidden_act
											_lowerCAmelCase  	=			hidden_dropout_prob
											_lowerCAmelCase  	=			attention_probs_dropout_prob
											_lowerCAmelCase  	=			type_sequence_label_size
											_lowerCAmelCase  	=			initializer_range
											_lowerCAmelCase  	=			mask_ratio
											_lowerCAmelCase  	=			scope

											# in VideoMAE, the number of tokens equals num_frames/tubelet_size * num_patches per frame
											_lowerCAmelCase  	=			(image_size // patch_size) ** 2
											_lowerCAmelCase  	=			(num_frames // tubelet_size) * self.num_patches_per_frame

											# use this variable to define bool_masked_pos
											_lowerCAmelCase  	=			int(mask_ratio * self.seq_length  )


				def 				__lowerCamelCase	(					self	:			int  )		->  Tuple:







											"""simple docstring"""




											_lowerCAmelCase  	=			floats_tensor(
											    [self.batch_size, self.num_frames, self.num_channels, self.image_size, self.image_size]  )

											_lowerCAmelCase  	=			None
											if self.use_labels:
																		_lowerCAmelCase  	=			ids_tensor([self.batch_size]				,     self.type_sequence_label_size  )

											_lowerCAmelCase  	=			self.get_config()

											return config, pixel_values, labels


				def 				__lowerCamelCase	(					self	:			Optional[int]  )		->  Dict:







											"""simple docstring"""




											return VideoMAEConfig(
											    image_size=self.image_size				,     patch_size=self.patch_size				,     num_channels=self.num_channels				,     num_frames=self.num_frames				,     tubelet_size=self.tubelet_size				,     hidden_size=self.hidden_size				,     num_hidden_layers=self.num_hidden_layers				,     num_attention_heads=self.num_attention_heads				,     intermediate_size=self.intermediate_size				,     hidden_act=self.hidden_act				,     hidden_dropout_prob=self.hidden_dropout_prob				,     attention_probs_dropout_prob=self.attention_probs_dropout_prob				,     is_decoder=UpperCAmelCase_				,     initializer_range=self.initializer_range				,     )


				def 				__lowerCamelCase	(					self	:			Optional[int]				,     UpperCAmelCase_	:			Dict				,     UpperCAmelCase_	:			List[str]				,     UpperCAmelCase_	:			Dict  )		->  List[str]:







											"""simple docstring"""




											_lowerCAmelCase  	=			VideoMAEModel(config=UpperCAmelCase_  )
											model.to(UpperCAmelCase_  )
											model.eval()
											_lowerCAmelCase  	=			model(UpperCAmelCase_  )
											self.parent.assertEqual(result.last_hidden_state.shape				,     (self.batch_size, self.seq_length, self.hidden_size)  )


				def 				__lowerCamelCase	(					self	:			Dict				,     UpperCAmelCase_	:			Any				,     UpperCAmelCase_	:			Dict				,     UpperCAmelCase_	:			str  )		->  Dict:







											"""simple docstring"""




											_lowerCAmelCase  	=			VideoMAEForPreTraining(UpperCAmelCase_  )
											model.to(UpperCAmelCase_  )
											model.eval()
											# important: each video needs to have the same number of masked patches
											# hence we define a single mask, which we then repeat for each example in the batch
											_lowerCAmelCase  	=			torch.ones((self.num_masks,)  )
											_lowerCAmelCase  	=			torch.cat([mask, torch.zeros(self.seq_length - mask.size(0  )  )]  )
											_lowerCAmelCase  	=			mask.expand(self.batch_size				,     -1  ).bool()

											_lowerCAmelCase  	=			model(UpperCAmelCase_				,     UpperCAmelCase_  )
											# model only returns predictions for masked patches
											_lowerCAmelCase  	=			mask.sum().item()
											_lowerCAmelCase  	=			3 * self.tubelet_size * self.patch_size**2
											self.parent.assertEqual(result.logits.shape				,     (self.batch_size, num_masked_patches, decoder_num_labels)  )







				def 				__lowerCamelCase	(					self	:			int  )		->  Union[str, Any]:







											"""simple docstring"""




											_lowerCAmelCase  	=			self.prepare_config_and_inputs()
											_lowerCAmelCase	,		_lowerCAmelCase	,		_lowerCAmelCase  	=			config_and_inputs
											_lowerCAmelCase  	=			{'pixel_values': pixel_values}
											return config, inputs_dict






@require_torch
class        _SCREAMING_SNAKE_CASE  (  UpperCAmelCase      ,				UpperCAmelCase      ,				unittest.TestCase							):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE_:							str							=					(
				    (VideoMAEModel, VideoMAEForPreTraining, VideoMAEForVideoClassification) if is_torch_available() else ()
				)
				SCREAMING_SNAKE_CASE_:							Union[str, Any]							=					(
				    {"feature-extraction": VideoMAEModel, "video-classification": VideoMAEForVideoClassification}
				    if is_torch_available()
				    else {}
				)

				SCREAMING_SNAKE_CASE_:							Any							=					False
				SCREAMING_SNAKE_CASE_:							str							=					False
				SCREAMING_SNAKE_CASE_:							List[Any]							=					False
				SCREAMING_SNAKE_CASE_:							Tuple							=					False


				def 				__lowerCamelCase	(					self	:			List[Any]  )		->  Optional[Any]:







											"""simple docstring"""




											_lowerCAmelCase  	=			VideoMAEModelTester(self  )
											_lowerCAmelCase  	=			ConfigTester(self				,     config_class=UpperCAmelCase_				,     has_text_modality=UpperCAmelCase_				,     hidden_size=37  )


				def 				__lowerCamelCase	(					self	:			List[str]				,     UpperCAmelCase_	:			Tuple				,     UpperCAmelCase_	:			List[str]				,     UpperCAmelCase_	:			Dict=False  )		->  Optional[Any]:







											"""simple docstring"""




											_lowerCAmelCase  	=			copy.deepcopy(UpperCAmelCase_  )

											if model_class == VideoMAEForPreTraining:
																		# important: each video needs to have the same number of masked patches
																		# hence we define a single mask, which we then repeat for each example in the batch
																		_lowerCAmelCase  	=			torch.ones((self.model_tester.num_masks,)  )
																		_lowerCAmelCase  	=			torch.cat([mask, torch.zeros(self.model_tester.seq_length - mask.size(0  )  )]  )
																		_lowerCAmelCase  	=			mask.expand(self.model_tester.batch_size				,     -1  ).bool()
																		_lowerCAmelCase  	=			bool_masked_pos.to(UpperCAmelCase_  )

											if return_labels:
																		if model_class in [
																		    *get_values(UpperCAmelCase_  ),
																		]:
																									_lowerCAmelCase  	=			torch.zeros(
																									    self.model_tester.batch_size				,     dtype=torch.long				,     device=UpperCAmelCase_  )

											return inputs_dict


				def 				__lowerCamelCase	(					self	:			Union[str, Any]  )		->  int:







											"""simple docstring"""




											self.config_tester.run_common_tests()


				@unittest.skip(reason='VideoMAE does not use inputs_embeds'  )
				def 				__lowerCamelCase	(					self	:			Any  )		->  Tuple:







											"""simple docstring"""




											pass


				def 				__lowerCamelCase	(					self	:			List[Any]  )		->  Optional[Any]:







											"""simple docstring"""




											_lowerCAmelCase	,		_lowerCAmelCase  	=			self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
																		_lowerCAmelCase  	=			model_class(UpperCAmelCase_  )
																		self.assertIsInstance(model.get_input_embeddings()				,     (nn.Module)  )
																		_lowerCAmelCase  	=			model.get_output_embeddings()
																		self.assertTrue(x is None or isinstance(UpperCAmelCase_				,     nn.Linear  )  )


				def 				__lowerCamelCase	(					self	:			Union[str, Any]  )		->  List[str]:







											"""simple docstring"""




											_lowerCAmelCase	,		_lowerCAmelCase  	=			self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
																		_lowerCAmelCase  	=			model_class(UpperCAmelCase_  )
																		_lowerCAmelCase  	=			inspect.signature(model.forward  )
																		# signature.parameters is an OrderedDict => so arg_names order is deterministic
																		_lowerCAmelCase  	=			[*signature.parameters.keys()]

																		_lowerCAmelCase  	=			['pixel_values']
																		self.assertListEqual(arg_names[:1]				,     UpperCAmelCase_  )


				def 				__lowerCamelCase	(					self	:			Any  )		->  Any:







											"""simple docstring"""




											_lowerCAmelCase  	=			self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*UpperCAmelCase_  )


				def 				__lowerCamelCase	(					self	:			Any  )		->  Union[str, Any]:







											"""simple docstring"""




											_lowerCAmelCase  	=			self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_pretraining(*UpperCAmelCase_  )


				@slow
				def 				__lowerCamelCase	(					self	:			Dict  )		->  Tuple:







											"""simple docstring"""




											for model_name in VIDEOMAE_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																		_lowerCAmelCase  	=			VideoMAEModel.from_pretrained(UpperCAmelCase_  )
																		self.assertIsNotNone(UpperCAmelCase_  )


				def 				__lowerCamelCase	(					self	:			str  )		->  Any:







											"""simple docstring"""




											if not self.has_attentions:
																		pass

											else:
																		_lowerCAmelCase	,		_lowerCAmelCase  	=			self.model_tester.prepare_config_and_inputs_for_common()
																		_lowerCAmelCase  	=			True

																		for model_class in self.all_model_classes:
																									_lowerCAmelCase  	=			self.model_tester.seq_length - self.model_tester.num_masks
																									_lowerCAmelCase  	=			(
																									    num_visible_patches if model_class == VideoMAEForPreTraining else self.model_tester.seq_length
																									)

																									_lowerCAmelCase  	=			True
																									_lowerCAmelCase  	=			False
																									_lowerCAmelCase  	=			True
																									_lowerCAmelCase  	=			model_class(UpperCAmelCase_  )
																									model.to(UpperCAmelCase_  )
																									model.eval()
																									with torch.no_grad():
																																_lowerCAmelCase  	=			model(**self._prepare_for_class(UpperCAmelCase_				,     UpperCAmelCase_  )  )
																									_lowerCAmelCase  	=			outputs.attentions
																									self.assertEqual(len(UpperCAmelCase_  )				,     self.model_tester.num_hidden_layers  )

																									# check that output_attentions also work using config
																									del inputs_dict["output_attentions"]
																									_lowerCAmelCase  	=			True
																									_lowerCAmelCase  	=			model_class(UpperCAmelCase_  )
																									model.to(UpperCAmelCase_  )
																									model.eval()
																									with torch.no_grad():
																																_lowerCAmelCase  	=			model(**self._prepare_for_class(UpperCAmelCase_				,     UpperCAmelCase_  )  )
																									_lowerCAmelCase  	=			outputs.attentions
																									self.assertEqual(len(UpperCAmelCase_  )				,     self.model_tester.num_hidden_layers  )

																									self.assertListEqual(
																									    list(attentions[0].shape[-3:]  )				,     [self.model_tester.num_attention_heads, seq_len, seq_len]				,     )
																									_lowerCAmelCase  	=			len(UpperCAmelCase_  )

																									# Check attention is always last and order is fine
																									_lowerCAmelCase  	=			True
																									_lowerCAmelCase  	=			True
																									_lowerCAmelCase  	=			model_class(UpperCAmelCase_  )
																									model.to(UpperCAmelCase_  )
																									model.eval()
																									with torch.no_grad():
																																_lowerCAmelCase  	=			model(**self._prepare_for_class(UpperCAmelCase_				,     UpperCAmelCase_  )  )

																									self.assertEqual(out_len + 1				,     len(UpperCAmelCase_  )  )

																									_lowerCAmelCase  	=			outputs.attentions

																									self.assertEqual(len(UpperCAmelCase_  )				,     self.model_tester.num_hidden_layers  )
																									self.assertListEqual(
																									    list(self_attentions[0].shape[-3:]  )				,     [self.model_tester.num_attention_heads, seq_len, seq_len]				,     )


				def 				__lowerCamelCase	(					self	:			Tuple  )		->  List[str]:







											"""simple docstring"""




											def check_hidden_states_output(UpperCAmelCase_	:			Tuple				,     UpperCAmelCase_	:			Optional[Any]				,     UpperCAmelCase_	:			Tuple  ):
																		_lowerCAmelCase  	=			model_class(UpperCAmelCase_  )
																		model.to(UpperCAmelCase_  )
																		model.eval()

																		with torch.no_grad():
																									_lowerCAmelCase  	=			model(**self._prepare_for_class(UpperCAmelCase_				,     UpperCAmelCase_  )  )

																		_lowerCAmelCase  	=			outputs.hidden_states
																		_lowerCAmelCase  	=			self.model_tester.num_hidden_layers + 1
																		self.assertEqual(len(UpperCAmelCase_  )				,     UpperCAmelCase_  )

																		_lowerCAmelCase  	=			self.model_tester.seq_length - self.model_tester.num_masks
																		_lowerCAmelCase  	=			num_visible_patches if model_class == VideoMAEForPreTraining else self.model_tester.seq_length

																		self.assertListEqual(
																		    list(hidden_states[0].shape[-2:]  )				,     [seq_length, self.model_tester.hidden_size]				,     )

											_lowerCAmelCase	,		_lowerCAmelCase  	=			self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
																		_lowerCAmelCase  	=			True
																		check_hidden_states_output(UpperCAmelCase_				,     UpperCAmelCase_				,     UpperCAmelCase_  )

																		# check that output_hidden_states also work using config
																		del inputs_dict["output_hidden_states"]
																		_lowerCAmelCase  	=			True

																		check_hidden_states_output(UpperCAmelCase_				,     UpperCAmelCase_				,     UpperCAmelCase_  )







				@unittest.skip('Will be fixed soon by reducing the size of the model used for common tests.'  )
				def 				__lowerCamelCase	(					self	:			Dict  )		->  List[Any]:







											"""simple docstring"""




											pass






def 	__snake_case  (    ):






							"""simple docstring"""



							_lowerCAmelCase  	=			hf_hub_download(
							    repo_id='hf-internal-testing/spaghetti-video'     ,  filename='eating_spaghetti.npy'     ,  repo_type='dataset'						)
							_lowerCAmelCase  	=			np.load(SCREAMING_SNAKE_CASE						)
							return list(SCREAMING_SNAKE_CASE						)

@require_torch
@require_vision
class        _SCREAMING_SNAKE_CASE  (  unittest.TestCase							):






				'''simple docstring'''



				@cached_property
				def 				__lowerCamelCase	(					self	:			Any  )		->  Any:







											"""simple docstring"""




											return (
											    VideoMAEImageProcessor(image_mean=[0.5, 0.5, 0.5]				,     image_std=[0.5, 0.5, 0.5]  )
											    if is_vision_available()
											    else None
											)


				@slow
				def 				__lowerCamelCase	(					self	:			Optional[Any]  )		->  Dict:







											"""simple docstring"""




											_lowerCAmelCase  	=			VideoMAEForVideoClassification.from_pretrained('MCG-NJU/videomae-base-finetuned-kinetics'  ).to(
											    UpperCAmelCase_  )

											_lowerCAmelCase  	=			self.default_image_processor
											_lowerCAmelCase  	=			prepare_video()
											_lowerCAmelCase  	=			image_processor(UpperCAmelCase_				,     return_tensors='pt'  ).to(UpperCAmelCase_  )

											# forward pass
											with torch.no_grad():
																		_lowerCAmelCase  	=			model(**UpperCAmelCase_  )

											# verify the logits
											_lowerCAmelCase  	=			torch.Size((1, 400)  )
											self.assertEqual(outputs.logits.shape				,     UpperCAmelCase_  )

											_lowerCAmelCase  	=			torch.tensor([0.3669, -0.0688, -0.2421]  ).to(UpperCAmelCase_  )

											self.assertTrue(torch.allclose(outputs.logits[0, :3]				,     UpperCAmelCase_				,     atol=1E-4  )  )







				@slow
				def 				__lowerCamelCase	(					self	:			List[str]  )		->  Union[str, Any]:







											"""simple docstring"""




											_lowerCAmelCase  	=			VideoMAEForPreTraining.from_pretrained('MCG-NJU/videomae-base-short'  ).to(UpperCAmelCase_  )

											_lowerCAmelCase  	=			self.default_image_processor
											_lowerCAmelCase  	=			prepare_video()
											_lowerCAmelCase  	=			image_processor(UpperCAmelCase_				,     return_tensors='pt'  ).to(UpperCAmelCase_  )

											# add boolean mask, indicating which patches to mask
											_lowerCAmelCase  	=			hf_hub_download(repo_id='hf-internal-testing/bool-masked-pos'				,     filename='bool_masked_pos.pt'  )
											_lowerCAmelCase  	=			torch.load(UpperCAmelCase_  )

											# forward pass
											with torch.no_grad():
																		_lowerCAmelCase  	=			model(**UpperCAmelCase_  )

											# verify the logits
											_lowerCAmelCase  	=			torch.Size([1, 1_408, 1_536]  )
											_lowerCAmelCase  	=			torch.tensor(
											    [[0.7994, 0.9612, 0.8508], [0.7401, 0.8958, 0.8302], [0.5862, 0.7468, 0.7325]]				,     device=UpperCAmelCase_  )
											self.assertEqual(outputs.logits.shape				,     UpperCAmelCase_  )
											self.assertTrue(torch.allclose(outputs.logits[0, :3, :3]				,     UpperCAmelCase_				,     atol=1E-4  )  )

											# verify the loss (`config.norm_pix_loss` = `True`)
											_lowerCAmelCase  	=			torch.tensor([0.5142]				,     device=UpperCAmelCase_  )
											self.assertTrue(torch.allclose(outputs.loss				,     UpperCAmelCase_				,     atol=1E-4  )  )

											# verify the loss (`config.norm_pix_loss` = `False`)
											_lowerCAmelCase  	=			VideoMAEForPreTraining.from_pretrained('MCG-NJU/videomae-base-short'				,     norm_pix_loss=UpperCAmelCase_  ).to(
											    UpperCAmelCase_  )

											with torch.no_grad():
																		_lowerCAmelCase  	=			model(**UpperCAmelCase_  )

											_lowerCAmelCase  	=			torch.tensor(torch.tensor([0.6469]  )				,     device=UpperCAmelCase_  )
											self.assertTrue(torch.allclose(outputs.loss				,     UpperCAmelCase_				,     atol=1E-4  )  )








"""simple docstring"""



import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class        _SCREAMING_SNAKE_CASE  (  UpperCAmelCase      ,				unittest.TestCase							):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE_:							str							=					None
				SCREAMING_SNAKE_CASE_:							List[str]							=					BloomTokenizerFast
				SCREAMING_SNAKE_CASE_:							List[str]							=					BloomTokenizerFast
				SCREAMING_SNAKE_CASE_:							Any							=					True
				SCREAMING_SNAKE_CASE_:							List[Any]							=					False
				SCREAMING_SNAKE_CASE_:							Optional[int]							=					"tokenizer_file"
				SCREAMING_SNAKE_CASE_:							str							=					{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}


				def 				__lowerCamelCase	(					self	:			Tuple  )		->  str:







											"""simple docstring"""




											super().setUp()
											_lowerCAmelCase  	=			BloomTokenizerFast.from_pretrained('bigscience/tokenizer'  )
											tokenizer.save_pretrained(self.tmpdirname  )


				def 				__lowerCamelCase	(					self	:			List[str]				,     **UpperCAmelCase_	:			Any  )		->  Optional[Any]:







											"""simple docstring"""




											kwargs.update(self.special_tokens_map  )
											return BloomTokenizerFast.from_pretrained(self.tmpdirname				,     **UpperCAmelCase_  )


				def 				__lowerCamelCase	(					self	:			List[Any]  )		->  str:







											"""simple docstring"""




											_lowerCAmelCase  	=			self.get_rust_tokenizer()

											_lowerCAmelCase  	=			['The quick brown fox</s>', 'jumps over the lazy dog</s>']
											_lowerCAmelCase  	=			[[2_175, 23_714, 73_173, 144_252, 2], [77, 132_619, 3_478, 368, 109_586, 35_433, 2]]

											_lowerCAmelCase  	=			tokenizer.batch_encode_plus(UpperCAmelCase_  )['input_ids']
											self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_  )

											_lowerCAmelCase  	=			tokenizer.batch_decode(UpperCAmelCase_  )
											self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_  )


				def 				__lowerCamelCase	(					self	:			Any				,     UpperCAmelCase_	:			List[str]=6  )		->  Any:







											"""simple docstring"""




											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
																									_lowerCAmelCase  	=			self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_				,     **UpperCAmelCase_  )
																									# tokenizer_r.pad_token = None # Hotfixing padding = None
																									# Simple input
																									_lowerCAmelCase  	=			'This is a simple input'
																									_lowerCAmelCase  	=			['This is a simple input 1', 'This is a simple input 2']
																									_lowerCAmelCase  	=			('This is a simple input', 'This is a pair')
																									_lowerCAmelCase  	=			[
																									    ('This is a simple input 1', 'This is a simple input 2'),
																									    ('This is a simple pair 1', 'This is a simple pair 2'),
																									]

																									# Simple input tests
																									try:
																																tokenizer_r.encode(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																																tokenizer_r.encode_plus(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )

																																tokenizer_r.batch_encode_plus(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																																tokenizer_r.encode(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																																tokenizer_r.batch_encode_plus(UpperCAmelCase_				,     max_length=UpperCAmelCase_  )
																									except ValueError:
																																self.fail('Bloom Tokenizer should be able to deal with padding'  )

																									_lowerCAmelCase  	=			None  # Hotfixing padding = None
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Simple input
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Simple input
																									self.assertRaises(
																									    UpperCAmelCase_				,     tokenizer_r.batch_encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'				,     )

																									# Pair input
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Pair input
																									self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'  )

																									# Pair input
																									self.assertRaises(
																									    UpperCAmelCase_				,     tokenizer_r.batch_encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding='max_length'				,     )


				def 				__lowerCamelCase	(					self	:			str  )		->  Any:







											"""simple docstring"""




											_lowerCAmelCase  	=			self.get_rust_tokenizer()
											_lowerCAmelCase  	=			load_dataset('xnli'				,     'all_languages'				,     split='test'				,     streaming=UpperCAmelCase_  )

											_lowerCAmelCase  	=			next(iter(UpperCAmelCase_  )  )['premise']  # pick up one data
											_lowerCAmelCase  	=			list(sample_data.values()  )

											_lowerCAmelCase  	=			list(map(tokenizer.encode				,     UpperCAmelCase_  )  )
											_lowerCAmelCase  	=			[tokenizer.decode(UpperCAmelCase_				,     clean_up_tokenization_spaces=UpperCAmelCase_  ) for x in output_tokens]
											self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_  )







				def 				__lowerCamelCase	(					self	:			Tuple  )		->  List[Any]:







											"""simple docstring"""




											self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map  )				,     1  )
											self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()  )[0]  )				,     1  )






