



"""simple docstring"""

import os
import tempfile
import unittest

from transformers import FlaubertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    FlaubertForMultipleChoice,
			    FlaubertForQuestionAnswering,
			    FlaubertForQuestionAnsweringSimple,
			    FlaubertForSequenceClassification,
			    FlaubertForTokenClassification,
			    FlaubertModel,
			    FlaubertWithLMHeadModel,
			)
			from transformers.models.flaubert.modeling_flaubert import FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST







class    __lowercase     (					_UpperCAmelCase):







						"""simple docstring"""







						def __init__(self   ,		lowercase__   ,		lowercase__=13   ,		lowercase__=7   ,		lowercase__=True   ,		lowercase__=True   ,		lowercase__=True   ,		lowercase__=True   ,		lowercase__=True   ,		lowercase__=False   ,		lowercase__=False   ,		lowercase__=False   ,		lowercase__=2   ,		lowercase__=99   ,		lowercase__=0   ,		lowercase__=32   ,		lowercase__=5   ,		lowercase__=4   ,		lowercase__=0.1   ,		lowercase__=0.1   ,		lowercase__=5_12   ,		lowercase__=12   ,		lowercase__=2   ,		lowercase__=0.02   ,		lowercase__=3   ,		lowercase__=4   ,		lowercase__="last"   ,		lowercase__=None   ,		lowercase__=None   ,		):
											snake_case_					:		str 					=					parent
											snake_case_					:		Tuple 					=					batch_size
											snake_case_					:		Any 					=					seq_length
											snake_case_					:		Any 					=					is_training
											snake_case_					:		int 					=					use_input_lengths
											snake_case_					:		str 					=					use_token_type_ids
											snake_case_					:		Tuple 					=					use_labels
											snake_case_					:		List[str] 					=					gelu_activation
											snake_case_					:		Optional[Any] 					=					sinusoidal_embeddings
											snake_case_					:		List[str] 					=					causal
											snake_case_					:		Any 					=					asm
											snake_case_					:		int 					=					n_langs
											snake_case_					:		Optional[Any] 					=					vocab_size
											snake_case_					:		Any 					=					n_special
											snake_case_					:		Tuple 					=					hidden_size
											snake_case_					:		Tuple 					=					num_hidden_layers
											snake_case_					:		Dict 					=					num_attention_heads
											snake_case_					:		Optional[int] 					=					hidden_dropout_prob
											snake_case_					:		Any 					=					attention_probs_dropout_prob
											snake_case_					:		List[str] 					=					max_position_embeddings
											snake_case_					:		Optional[int] 					=					type_vocab_size
											snake_case_					:		List[Any] 					=					type_sequence_label_size
											snake_case_					:		int 					=					initializer_range
											snake_case_					:		str 					=					num_labels
											snake_case_					:		str 					=					num_choices
											snake_case_					:		List[Any] 					=					summary_type
											snake_case_					:		Any 					=					use_proj
											snake_case_					:		int 					=					scope







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		int 					=					ids_tensor([self.batch_size, self.seq_length]   ,		self.vocab_size       )
											snake_case_					:		List[str] 					=					random_attention_mask([self.batch_size, self.seq_length]       )

											snake_case_					:		Any 					=					None
											if self.use_input_lengths:
																snake_case_					:		str 					=					(
																    ids_tensor([self.batch_size]   ,		vocab_size=2       ) + self.seq_length - 2
																)  # small variation of seq_length

											snake_case_					:		List[str] 					=					None
											if self.use_token_type_ids:
																snake_case_					:		List[str] 					=					ids_tensor([self.batch_size, self.seq_length]   ,		self.n_langs       )

											snake_case_					:		Optional[Any] 					=					None
											snake_case_					:		Optional[int] 					=					None
											snake_case_					:		List[Any] 					=					None
											if self.use_labels:
																snake_case_					:		Tuple 					=					ids_tensor([self.batch_size]   ,		self.type_sequence_label_size       )
																snake_case_					:		Optional[int] 					=					ids_tensor([self.batch_size, self.seq_length]   ,		self.num_labels       )
																snake_case_					:		str 					=					ids_tensor([self.batch_size]   ,		2       ).float()
																snake_case_					:		Dict 					=					ids_tensor([self.batch_size]   ,		self.num_choices       )

											snake_case_					:		List[Any] 					=					self.get_config()

											return (
											    config,
											    input_ids,
											    token_type_ids,
											    input_lengths,
											    sequence_labels,
											    token_labels,
											    is_impossible_labels,
											    choice_labels,
											    input_mask,
											)







						def 						__UpperCamelCase							(self       ):
											return FlaubertConfig(
											    vocab_size=self.vocab_size   ,		n_special=self.n_special   ,		emb_dim=self.hidden_size   ,		n_layers=self.num_hidden_layers   ,		n_heads=self.num_attention_heads   ,		dropout=self.hidden_dropout_prob   ,		attention_dropout=self.attention_probs_dropout_prob   ,		gelu_activation=self.gelu_activation   ,		sinusoidal_embeddings=self.sinusoidal_embeddings   ,		asm=self.asm   ,		causal=self.causal   ,		n_langs=self.n_langs   ,		max_position_embeddings=self.max_position_embeddings   ,		initializer_range=self.initializer_range   ,		summary_type=self.summary_type   ,		use_proj=self.use_proj   ,		)







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		):
											snake_case_					:		int 					=					FlaubertModel(config=__UpperCamelCase       )
											model.to(__UpperCamelCase       )
											model.eval()
											snake_case_					:		List[Any] 					=					model(__UpperCamelCase   ,		lengths=__UpperCamelCase   ,		langs=__UpperCamelCase       )
											snake_case_					:		Optional[Any] 					=					model(__UpperCamelCase   ,		langs=__UpperCamelCase       )
											snake_case_					:		Any 					=					model(__UpperCamelCase       )
											self.parent.assertEqual(result.last_hidden_state.shape   ,		(self.batch_size, self.seq_length, self.hidden_size)       )







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		):
											snake_case_					:		str 					=					FlaubertWithLMHeadModel(__UpperCamelCase       )
											model.to(__UpperCamelCase       )
											model.eval()

											snake_case_					:		Any 					=					model(__UpperCamelCase   ,		token_type_ids=__UpperCamelCase   ,		labels=__UpperCamelCase       )
											self.parent.assertEqual(result.loss.shape   ,		()       )
											self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.seq_length, self.vocab_size)       )







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		):
											snake_case_					:		Union[str, Any] 					=					FlaubertForQuestionAnsweringSimple(__UpperCamelCase       )
											model.to(__UpperCamelCase       )
											model.eval()

											snake_case_					:		Any 					=					model(__UpperCamelCase       )

											snake_case_					:		List[Any] 					=					model(__UpperCamelCase   ,		start_positions=__UpperCamelCase   ,		end_positions=__UpperCamelCase       )
											self.parent.assertEqual(result.start_logits.shape   ,		(self.batch_size, self.seq_length)       )
											self.parent.assertEqual(result.end_logits.shape   ,		(self.batch_size, self.seq_length)       )







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		):
											snake_case_					:		str 					=					FlaubertForQuestionAnswering(__UpperCamelCase       )
											model.to(__UpperCamelCase       )
											model.eval()

											snake_case_					:		List[str] 					=					model(__UpperCamelCase       )

											snake_case_					:		Optional[Any] 					=					model(
											    __UpperCamelCase   ,		start_positions=__UpperCamelCase   ,		end_positions=__UpperCamelCase   ,		cls_index=__UpperCamelCase   ,		is_impossible=__UpperCamelCase   ,		p_mask=__UpperCamelCase   ,		)

											snake_case_					:		int 					=					model(
											    __UpperCamelCase   ,		start_positions=__UpperCamelCase   ,		end_positions=__UpperCamelCase   ,		cls_index=__UpperCamelCase   ,		is_impossible=__UpperCamelCase   ,		)

											((snake_case_   )					,			   )					:		List[str] 					=					result_with_labels.to_tuple()

											snake_case_					:		str 					=					model(__UpperCamelCase   ,		start_positions=__UpperCamelCase   ,		end_positions=__UpperCamelCase       )

											((snake_case_   )					,			   )					:		Dict 					=					result_with_labels.to_tuple()

											self.parent.assertEqual(result_with_labels.loss.shape   ,		()       )
											self.parent.assertEqual(result.start_top_log_probs.shape   ,		(self.batch_size, model.config.start_n_top)       )
											self.parent.assertEqual(result.start_top_index.shape   ,		(self.batch_size, model.config.start_n_top)       )
											self.parent.assertEqual(
											    result.end_top_log_probs.shape   ,		(self.batch_size, model.config.start_n_top * model.config.end_n_top)       )
											self.parent.assertEqual(
											    result.end_top_index.shape   ,		(self.batch_size, model.config.start_n_top * model.config.end_n_top)       )
											self.parent.assertEqual(result.cls_logits.shape   ,		(self.batch_size,)       )







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		):
											snake_case_					:		Tuple 					=					FlaubertForSequenceClassification(__UpperCamelCase       )
											model.to(__UpperCamelCase       )
											model.eval()

											snake_case_					:		Any 					=					model(__UpperCamelCase       )
											snake_case_					:		Union[str, Any] 					=					model(__UpperCamelCase   ,		labels=__UpperCamelCase       )

											self.parent.assertEqual(result.loss.shape   ,		()       )
											self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.type_sequence_label_size)       )







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		):
											snake_case_					:		Union[str, Any] 					=					self.num_labels
											snake_case_					:		str 					=					FlaubertForTokenClassification(__UpperCamelCase       )
											model.to(__UpperCamelCase       )
											model.eval()

											snake_case_					:		str 					=					model(__UpperCamelCase   ,		attention_mask=__UpperCamelCase   ,		labels=__UpperCamelCase       )
											self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.seq_length, self.num_labels)       )







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		):
											snake_case_					:		Optional[int] 					=					self.num_choices
											snake_case_					:		Tuple 					=					FlaubertForMultipleChoice(config=__UpperCamelCase       )
											model.to(__UpperCamelCase       )
											model.eval()
											snake_case_					:		Any 					=					input_ids.unsqueeze(1       ).expand(-1   ,		self.num_choices   ,		-1       ).contiguous()
											snake_case_					:		List[str] 					=					token_type_ids.unsqueeze(1       ).expand(-1   ,		self.num_choices   ,		-1       ).contiguous()
											snake_case_					:		Any 					=					input_mask.unsqueeze(1       ).expand(-1   ,		self.num_choices   ,		-1       ).contiguous()
											snake_case_					:		List[Any] 					=					model(
											    __UpperCamelCase   ,		attention_mask=__UpperCamelCase   ,		token_type_ids=__UpperCamelCase   ,		labels=__UpperCamelCase   ,		)
											self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.num_choices)       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Tuple 					=					self.prepare_config_and_inputs()
											(
											    (
											    snake_case_   
											)					,			(
											    snake_case_   
											)					,			(
											    snake_case_   
											)					,			(
											    snake_case_   
											)					,			(
											    snake_case_   
											)					,			(
											    snake_case_   
											)					,			(
											    snake_case_   
											)					,			(
											    snake_case_   
											)					,			(
											    snake_case_   
											)					,			   
											)					:		List[Any] 					=					config_and_inputs
											snake_case_					:		int 					=					{
											    """input_ids""": input_ids,
											    """token_type_ids""": token_type_ids,
											    """lengths""": input_lengths,
											    """attention_mask""": input_mask,
											}
											return config, inputs_dict







@require_torch
class    __lowercase     (					_UpperCAmelCase					,    _UpperCAmelCase					,    unittest.TestCase):







						"""simple docstring"""







						_A		:							List[str]							       =		(
						    (
						        FlaubertModel,
						        FlaubertWithLMHeadModel,
						        FlaubertForQuestionAnswering,
						        FlaubertForQuestionAnsweringSimple,
						        FlaubertForSequenceClassification,
						        FlaubertForTokenClassification,
						        FlaubertForMultipleChoice,
						    )
						    if is_torch_available()
						    else ()
						)
						_A		:							Tuple							       =		(
						    {
						        """feature-extraction""": FlaubertModel,
						        """fill-mask""": FlaubertWithLMHeadModel,
						        """question-answering""": FlaubertForQuestionAnsweringSimple,
						        """text-classification""": FlaubertForSequenceClassification,
						        """token-classification""": FlaubertForTokenClassification,
						        """zero-shot""": FlaubertForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__       ):
											if (
											    pipeline_test_casse_name == "QAPipelineTests"
											    and tokenizer_name is not None
											    and not tokenizer_name.endswith("""Fast"""       )
											):
																# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
																# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
																# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
																return True

											return False







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__   ,		lowercase__=False       ):
											snake_case_					:		Union[str, Any] 					=					super()._prepare_for_class(__UpperCamelCase   ,		__UpperCamelCase   ,		return_labels=__UpperCamelCase       )

											if return_labels:
																if model_class.__name__ == "FlaubertForQuestionAnswering":
																					snake_case_					:		Optional[int] 					=					torch.zeros(
																					    self.model_tester.batch_size   ,		dtype=torch.long   ,		device=__UpperCamelCase       )
																					snake_case_					:		List[str] 					=					torch.zeros(
																					    self.model_tester.batch_size   ,		dtype=torch.long   ,		device=__UpperCamelCase       )

											return inputs_dict







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Optional[int] 					=					FlaubertModelTester(self       )
											snake_case_					:		str 					=					ConfigTester(self   ,		config_class=__UpperCamelCase   ,		emb_dim=37       )







						def 						__UpperCamelCase							(self       ):
											self.config_tester.run_common_tests()







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		str 					=					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_flaubert_model(*__UpperCamelCase       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Any 					=					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_flaubert_lm_head(*__UpperCamelCase       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Dict 					=					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_flaubert_simple_qa(*__UpperCamelCase       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Tuple 					=					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_flaubert_qa(*__UpperCamelCase       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Tuple 					=					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_flaubert_sequence_classif(*__UpperCamelCase       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Tuple 					=					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_flaubert_token_classif(*__UpperCamelCase       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Union[str, Any] 					=					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_flaubert_multiple_choice(*__UpperCamelCase       )







						@slow
						def 						__UpperCamelCase							(self       ):
											for model_name in FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																snake_case_					:		Dict 					=					FlaubertModel.from_pretrained(__UpperCamelCase       )
																self.assertIsNotNone(__UpperCamelCase       )







						@slow
						@require_torch_gpu
						def 						__UpperCamelCase							(self       ):
											snake_case_					,			snake_case_					:		Dict 					=					self.model_tester.prepare_config_and_inputs_for_common()
											for model_class in self.all_model_classes:
																# FlauBertForMultipleChoice behaves incorrectly in JIT environments.
																if model_class == FlaubertForMultipleChoice:
																					return

																snake_case_					:		int 					=					True
																snake_case_					:		Dict 					=					model_class(config=__UpperCamelCase       )

																snake_case_					:		Any 					=					self._prepare_for_class(__UpperCamelCase   ,		__UpperCamelCase       )
																snake_case_					:		int 					=					torch.jit.trace(
																    __UpperCamelCase   ,		(inputs_dict["""input_ids"""].to("""cpu"""       ), inputs_dict["""attention_mask"""].to("""cpu"""       ))       )

																with tempfile.TemporaryDirectory() as tmp:
																					torch.jit.save(__UpperCamelCase   ,		os.path.join(__UpperCamelCase   ,		"""traced_model.pt"""       )       )
																					snake_case_					:		Dict 					=					torch.jit.load(os.path.join(__UpperCamelCase   ,		"""traced_model.pt"""       )   ,		map_location=__UpperCamelCase       )
																					loaded(inputs_dict["""input_ids"""].to(__UpperCamelCase       )   ,		inputs_dict["""attention_mask"""].to(__UpperCamelCase       )       )







@require_torch
class    __lowercase     (					unittest.TestCase):







						"""simple docstring"""







						@slow
						def 						__UpperCamelCase							(self       ):
											snake_case_					:		List[str] 					=					FlaubertModel.from_pretrained("""flaubert/flaubert_base_cased"""       )
											snake_case_					:		Tuple 					=					torch.tensor([[0, 3_45, 2_32, 3_28, 7_40, 1_40, 16_95, 69, 60_78, 15_88, 2]]       )
											with torch.no_grad():
																snake_case_					:		List[str] 					=					model(__UpperCamelCase       )[0]
											snake_case_					:		str 					=					torch.Size((1, 11, 7_68)       )
											self.assertEqual(output.shape   ,		__UpperCamelCase       )
											snake_case_					:		str 					=					torch.tensor(
											    [[[-2.6251, -1.4298, -0.0227], [-2.8510, -1.6387, 0.2258], [-2.8114, -1.1832, -0.3066]]]       )

											self.assertTrue(torch.allclose(output[:, :3, :3]   ,		__UpperCamelCase   ,		atol=1e-4       )       )









"""simple docstring"""

import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import BatchEncoding, MarianTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import is_sentencepiece_available, is_tf_available, is_torch_available


if is_sentencepiece_available():
			from transformers.models.marian.tokenization_marian import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


a_			   =					get_tests_dir('''fixtures/test_sentencepiece.model''')

a_			   =					{'''target_lang''': '''fi''', '''source_lang''': '''en'''}
a_			   =					'''>>zh<<'''
a_			   =					'''Helsinki-NLP/'''

if is_torch_available():
			a_			   =					'''pt'''
elif is_tf_available():
			a_			   =					'''tf'''
else:
			a_			   =					'''jax'''







@require_sentencepiece
class    __lowercase     (					_UpperCAmelCase					,    unittest.TestCase):







						"""simple docstring"""







						_A		:							str							       =		MarianTokenizer
						_A		:							List[str]							       =		False
						_A		:							List[str]							       =		True







						def 						__UpperCamelCase							(self       ):
											super().setUp()
											snake_case_					:		Optional[int] 					=					["""</s>""", """<unk>""", """▁This""", """▁is""", """▁a""", """▁t""", """est""", """\u0120""", """<pad>"""]
											snake_case_					:		Any 					=					dict(zip(lowercase__   ,		range(len(lowercase__       )       )       )       )
											snake_case_					:		Any 					=					Path(self.tmpdirname       )
											save_json(lowercase__   ,		save_dir / VOCAB_FILES_NAMES["""vocab"""]       )
											save_json(lowercase__   ,		save_dir / VOCAB_FILES_NAMES["""tokenizer_config_file"""]       )
											if not (save_dir / VOCAB_FILES_NAMES["source_spm"]).exists():
																copyfile(lowercase__   ,		save_dir / VOCAB_FILES_NAMES["""source_spm"""]       )
																copyfile(lowercase__   ,		save_dir / VOCAB_FILES_NAMES["""target_spm"""]       )

											snake_case_					:		Optional[Any] 					=					MarianTokenizer.from_pretrained(self.tmpdirname       )
											tokenizer.save_pretrained(self.tmpdirname       )







						def 						__UpperCamelCase							(self   ,		**lowercase__       ):
											return MarianTokenizer.from_pretrained(self.tmpdirname   ,		**lowercase__       )







						def 						__UpperCamelCase							(self   ,		lowercase__       ):
											return (
											    "This is a test",
											    "This is a test",
											)







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Union[str, Any] 					=					"""</s>"""
											snake_case_					:		Tuple 					=					0

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase__       )   ,		lowercase__       )
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase__       )   ,		lowercase__       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		List[str] 					=					list(self.get_tokenizer().get_vocab().keys()       )

											self.assertEqual(vocab_keys[0]   ,		"""</s>"""       )
											self.assertEqual(vocab_keys[1]   ,		"""<unk>"""       )
											self.assertEqual(vocab_keys[-1]   ,		"""<pad>"""       )
											self.assertEqual(len(lowercase__       )   ,		9       )







						def 						__UpperCamelCase							(self       ):
											self.assertEqual(self.get_tokenizer().vocab_size   ,		9       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Any 					=					MarianTokenizer.from_pretrained(f'{ORG_NAME}opus-mt-en-de'       )
											snake_case_					:		Tuple 					=					en_de_tokenizer(["""I am a small frog"""]   ,		return_tensors=lowercase__       )
											self.assertIsInstance(lowercase__   ,		lowercase__       )
											snake_case_					:		Dict 					=					[38, 1_21, 14, 6_97, 3_88_48, 0]
											self.assertListEqual(lowercase__   ,		batch.input_ids[0]       )

											snake_case_					:		Tuple 					=					tempfile.mkdtemp()
											en_de_tokenizer.save_pretrained(lowercase__       )
											snake_case_					:		str 					=					[x.name for x in Path(lowercase__       ).glob("""*"""       )]
											self.assertIn("""source.spm"""   ,		lowercase__       )
											MarianTokenizer.from_pretrained(lowercase__       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Union[str, Any] 					=					self.get_tokenizer()

											snake_case_					:		List[str] 					=					tok(
											    ["""I am a small frog""" * 10_00, """I am a small frog"""]   ,		padding=lowercase__   ,		truncation=lowercase__   ,		return_tensors=lowercase__       )
											self.assertIsInstance(lowercase__   ,		lowercase__       )
											self.assertEqual(batch.input_ids.shape   ,		(2, 5_12)       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Tuple 					=					self.get_tokenizer()
											snake_case_					:		Tuple 					=					tok(["""I am a tiny frog""", """I am a small frog"""]   ,		padding=lowercase__   ,		return_tensors=lowercase__       )
											self.assertIsInstance(lowercase__   ,		lowercase__       )
											self.assertEqual(batch_smaller.input_ids.shape   ,		(2, 10)       )







						@slow
						def 						__UpperCamelCase							(self       ):
											# fmt: off
											snake_case_					:		str 					=					{"""input_ids""": [[4_34_95, 4_62, 20, 4_21_64, 13_69, 52, 4_64, 1_32, 17_03, 4_92, 13, 74_91, 3_89_99, 6, 8, 4_64, 1_32, 17_03, 4_92, 13, 46_69, 3_78_67, 13, 75_25, 27, 15_93, 9_88, 13, 3_39_72, 70_29, 6, 20, 82_51, 3_83, 2, 2_70, 58_66, 37_88, 2, 23_53, 82_51, 1_23_38, 2, 1_39_58, 3_87, 2, 36_29, 69_53, 1_88, 29_00, 2, 1_39_58, 80_11, 1_15_01, 23, 84_60, 40_73, 3_40_09, 20, 4_35, 1_14_39, 27, 8, 84_60, 40_73, 60_04, 20, 99_88, 3_75, 27, 33, 2_66, 19_45, 10_76, 13_50, 3_78_67, 32_88, 5, 5_77, 10_76, 43_74, 8, 50_82, 5, 2_64_53, 2_57, 5_56, 4_03, 2, 2_42, 1_32, 3_83, 3_16, 4_92, 8, 1_07_67, 6, 3_16, 3_04, 42_39, 3, 0], [1_48, 1_57_22, 19, 18_39, 12, 13_50, 13, 2_23_27, 50_82, 54_18, 4_75_67, 3_59_38, 59, 3_18, 1_95_52, 1_08, 21_83, 54, 1_49_76, 48_35, 32, 5_47, 11_14, 8, 3_15, 24_17, 5, 92, 1_90_88, 3, 0, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00], [36, 63_95, 1_25_70, 3_91_47, 1_15_97, 6, 2_66, 4, 4_54_05, 72_96, 3, 0, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
											# fmt: on

											self.tokenizer_integration_test_util(
											    expected_encoding=lowercase__   ,		model_name="""Helsinki-NLP/opus-mt-en-de"""   ,		revision="""1a8c2263da11e68e50938f97e10cd57820bd504c"""   ,		decode_kwargs={"""use_source_tokenizer""": True}   ,		)







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Any 					=					MarianTokenizer.from_pretrained("""hf-internal-testing/test-marian-two-vocabs"""       )

											snake_case_					:		Dict 					=					"""Tämä on testi"""
											snake_case_					:		List[Any] 					=					"""This is a test"""

											snake_case_					:		Optional[int] 					=					[76, 7, 20_47, 2]
											snake_case_					:		List[str] 					=					[69, 12, 11, 9_40, 2]

											snake_case_					:		Any 					=					tokenizer(lowercase__       ).input_ids
											self.assertListEqual(lowercase__   ,		lowercase__       )

											snake_case_					:		str 					=					tokenizer(text_target=lowercase__       ).input_ids
											self.assertListEqual(lowercase__   ,		lowercase__       )

											snake_case_					:		int 					=					tokenizer.decode(lowercase__   ,		skip_special_tokens=lowercase__       )
											self.assertEqual(lowercase__   ,		lowercase__       )




