



"""simple docstring"""





import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartTokenizer, MBartTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCAmelCase       :					Any      					=					get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
					from transformers.models.mbart.modeling_mbart import shift_tokens_right

__lowerCAmelCase       :					int      					=					25_0004
__lowerCAmelCase       :					Union[str, Any]      					=					25_0020



@require_sentencepiece
@require_tokenizers
class    _lowerCAmelCase    (				SCREAMING_SNAKE_CASE__  ,				unittest.TestCase		):





		"""simple docstring"""






		_lowerCamelCase			  =       MBartTokenizer
		_lowerCamelCase			  =       MBartTokenizerFast
		_lowerCamelCase			  =       True
		_lowerCamelCase			  =       True





		def    UpperCAmelCase__							(					self   ) ->						int:







								'''simple docstring'''

								super().setUp()

								# We have a SentencePiece fixture for testing
								snake_case_   : List[str]						=			MBartTokenizer(_lowercase							,    keep_accents=_lowercase   )
								tokenizer.save_pretrained(self.tmpdirname   )





		def    UpperCAmelCase__							(					self   ) ->						Any:







								'''simple docstring'''

								snake_case_   : Tuple						=			MBartTokenizer(_lowercase							,    keep_accents=_lowercase   )

								snake_case_   : Any						=			tokenizer.tokenize("""This is a test"""   )
								self.assertListEqual(_lowercase							,    ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]   )

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(_lowercase   )							,    [value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]							,    )

								snake_case_   : Optional[Any]						=			tokenizer.tokenize("""I was born in 92000, and this is falsé."""   )
								self.assertListEqual(
								    _lowercase							,    [
								        SPIECE_UNDERLINE + """I""",
								        SPIECE_UNDERLINE + """was""",
								        SPIECE_UNDERLINE + """b""",
								        """or""",
								        """n""",
								        SPIECE_UNDERLINE + """in""",
								        SPIECE_UNDERLINE + """""",
								        """9""",
								        """2""",
								        """0""",
								        """0""",
								        """0""",
								        """,""",
								        SPIECE_UNDERLINE + """and""",
								        SPIECE_UNDERLINE + """this""",
								        SPIECE_UNDERLINE + """is""",
								        SPIECE_UNDERLINE + """f""",
								        """al""",
								        """s""",
								        """é""",
								        """.""",
								    ]							,    )
								snake_case_   : Optional[Any]						=			tokenizer.convert_tokens_to_ids(_lowercase   )
								self.assertListEqual(
								    _lowercase							,    [
								        value + tokenizer.fairseq_offset
								        for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
								        #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
								    ]							,    )

								snake_case_   : Tuple						=			tokenizer.convert_ids_to_tokens(_lowercase   )
								self.assertListEqual(
								    _lowercase							,    [
								        SPIECE_UNDERLINE + """I""",
								        SPIECE_UNDERLINE + """was""",
								        SPIECE_UNDERLINE + """b""",
								        """or""",
								        """n""",
								        SPIECE_UNDERLINE + """in""",
								        SPIECE_UNDERLINE + """""",
								        """<unk>""",
								        """2""",
								        """0""",
								        """0""",
								        """0""",
								        """,""",
								        SPIECE_UNDERLINE + """and""",
								        SPIECE_UNDERLINE + """this""",
								        SPIECE_UNDERLINE + """is""",
								        SPIECE_UNDERLINE + """f""",
								        """al""",
								        """s""",
								        """<unk>""",
								        """.""",
								    ]							,    )





		def    UpperCAmelCase__							(					self   ) ->						Dict:







								'''simple docstring'''

								if not self.test_slow_tokenizer:
														# as we don't have a slow version, we can't compare the outputs between slow and fast versions
														return

								snake_case_   : Any						=			(self.rust_tokenizer_class, """hf-internal-testing/tiny-random-mbart""", {})
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'   ):
																				snake_case_   : Tuple						=			self.rust_tokenizer_class.from_pretrained(_lowercase							,    **_lowercase   )
																				snake_case_   : Optional[int]						=			self.tokenizer_class.from_pretrained(_lowercase							,    **_lowercase   )

																				snake_case_   : Union[str, Any]						=			tempfile.mkdtemp()

																				snake_case_   : str						=			tokenizer_r.save_pretrained(_lowercase   )
																				snake_case_   : Optional[Any]						=			tokenizer_p.save_pretrained(_lowercase   )

																				# Checks it save with the same files + the tokenizer.json file for the fast one
																				self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files   )   )
																				snake_case_   : int						=			tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f   )
																				self.assertSequenceEqual(_lowercase							,    _lowercase   )

																				# Checks everything loads correctly in the same way
																				snake_case_   : List[str]						=			tokenizer_r.from_pretrained(_lowercase   )
																				snake_case_   : List[str]						=			tokenizer_p.from_pretrained(_lowercase   )

																				# Check special tokens are set accordingly on Rust and Python
																				for key in tokenizer_pp.special_tokens_map:
																										self.assertTrue(hasattr(_lowercase							,    _lowercase   )   )
																										# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
																										# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

																				shutil.rmtree(_lowercase   )

																				# Save tokenizer rust, legacy_format=True
																				snake_case_   : List[Any]						=			tempfile.mkdtemp()

																				snake_case_   : str						=			tokenizer_r.save_pretrained(_lowercase							,    legacy_format=_lowercase   )
																				snake_case_   : List[Any]						=			tokenizer_p.save_pretrained(_lowercase   )

																				# Checks it save with the same files
																				self.assertSequenceEqual(_lowercase							,    _lowercase   )

																				# Checks everything loads correctly in the same way
																				snake_case_   : str						=			tokenizer_r.from_pretrained(_lowercase   )
																				snake_case_   : List[Any]						=			tokenizer_p.from_pretrained(_lowercase   )

																				# Check special tokens are set accordingly on Rust and Python
																				for key in tokenizer_pp.special_tokens_map:
																										self.assertTrue(hasattr(_lowercase							,    _lowercase   )   )

																				shutil.rmtree(_lowercase   )

																				# Save tokenizer rust, legacy_format=False
																				snake_case_   : List[Any]						=			tempfile.mkdtemp()

																				snake_case_   : Optional[Any]						=			tokenizer_r.save_pretrained(_lowercase							,    legacy_format=_lowercase   )
																				snake_case_   : str						=			tokenizer_p.save_pretrained(_lowercase   )

																				# Checks it saved the tokenizer.json file
																				self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files   )   )

																				# Checks everything loads correctly in the same way
																				snake_case_   : Dict						=			tokenizer_r.from_pretrained(_lowercase   )
																				snake_case_   : Optional[Any]						=			tokenizer_p.from_pretrained(_lowercase   )

																				# Check special tokens are set accordingly on Rust and Python
																				for key in tokenizer_pp.special_tokens_map:
																										self.assertTrue(hasattr(_lowercase							,    _lowercase   )   )

																				shutil.rmtree(_lowercase   )




@require_torch
@require_sentencepiece
@require_tokenizers
class    _lowerCAmelCase    (				unittest.TestCase		):





		"""simple docstring"""






		_lowerCamelCase			  =       '''facebook/mbart-large-en-ro'''
		_lowerCamelCase			  =       [
		    ''' UN Chief Says There Is No Military Solution in Syria''',
		    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
		]
		_lowerCamelCase			  =       [
		    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
		    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
		    ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
		    ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
		]
		_lowerCamelCase			  =       [8_274, 127_873, 25_916, 7, 8_622, 2_071, 438, 67_485, 53, 187_895, 23, 51_712, 2, EN_CODE]





		@classmethod
		def    UpperCAmelCase__							(					cls   ) ->						Union[str, Any]:







								'''simple docstring'''

								snake_case_   : MBartTokenizer						=			MBartTokenizer.from_pretrained(
								    cls.checkpoint_name							,    src_lang="""en_XX"""							,    tgt_lang="""ro_RO"""   )
								snake_case_   : Union[str, Any]						=			1
								return cls





		def    UpperCAmelCase__							(					self   ) ->						int:







								'''simple docstring'''

								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ar_AR"""]							,    2_5_0_0_0_1   )
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""en_EN"""]							,    2_5_0_0_0_4   )
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ro_RO"""]							,    2_5_0_0_2_0   )





		def    UpperCAmelCase__							(					self   ) ->						List[Any]:







								'''simple docstring'''

								snake_case_   : Union[str, Any]						=			self.tokenizer.batch_encode_plus(self.src_text   ).input_ids[0]
								self.assertListEqual(self.expected_src_tokens							,    _lowercase   )





		def    UpperCAmelCase__							(					self   ) ->						str:







								'''simple docstring'''

								self.assertIn(_lowercase							,    self.tokenizer.all_special_ids   )
								snake_case_   : Tuple						=			[RO_CODE, 8_8_4, 9_0_1_9, 9_6, 9, 9_1_6, 8_6_7_9_2, 3_6, 1_8_7_4_3, 1_5_5_9_6, 5, 2]
								snake_case_   : Optional[int]						=			self.tokenizer.decode(_lowercase							,    skip_special_tokens=_lowercase   )
								snake_case_   : Union[str, Any]						=			self.tokenizer.decode(generated_ids[1:]							,    skip_special_tokens=_lowercase   )
								self.assertEqual(_lowercase							,    _lowercase   )
								self.assertNotIn(self.tokenizer.eos_token							,    _lowercase   )





		def    UpperCAmelCase__							(					self   ) ->						str:







								'''simple docstring'''

								snake_case_   : Union[str, Any]						=			["""this is gunna be a long sentence """ * 2_0]
								assert isinstance(src_text[0]							,    _lowercase   )
								snake_case_   : int						=			1_0
								snake_case_   : str						=			self.tokenizer(_lowercase							,    max_length=_lowercase							,    truncation=_lowercase   ).input_ids[0]
								self.assertEqual(ids[-2]							,    2   )
								self.assertEqual(ids[-1]							,    _lowercase   )
								self.assertEqual(len(_lowercase   )							,    _lowercase   )





		def    UpperCAmelCase__							(					self   ) ->						Dict:







								'''simple docstring'''

								self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]   )							,    [2_5_0_0_2_6, 2_5_0_0_0_1]   )





		def    UpperCAmelCase__							(					self   ) ->						Optional[int]:







								'''simple docstring'''

								snake_case_   : str						=			tempfile.mkdtemp()
								snake_case_   : Tuple						=			self.tokenizer.fairseq_tokens_to_ids
								self.tokenizer.save_pretrained(_lowercase   )
								snake_case_   : str						=			MBartTokenizer.from_pretrained(_lowercase   )
								self.assertDictEqual(new_tok.fairseq_tokens_to_ids							,    _lowercase   )





		@require_torch
		def    UpperCAmelCase__							(					self   ) ->						Union[str, Any]:







								'''simple docstring'''

								snake_case_   : Optional[int]						=			self.tokenizer(self.src_text							,    text_target=self.tgt_text							,    padding=_lowercase							,    return_tensors="""pt"""   )
								snake_case_   : Any						=			shift_tokens_right(batch["""labels"""]							,    self.tokenizer.pad_token_id   )

								# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
								assert batch.input_ids[1][-2:].tolist() == [2, EN_CODE]
								assert batch.decoder_input_ids[1][0].tolist() == RO_CODE
								assert batch.decoder_input_ids[1][-1] == 2
								assert batch.labels[1][-2:].tolist() == [2, RO_CODE]





		@require_torch
		def    UpperCAmelCase__							(					self   ) ->						int:







								'''simple docstring'''

								snake_case_   : Optional[Any]						=			self.tokenizer(
								    self.src_text							,    text_target=self.tgt_text							,    padding=_lowercase							,    truncation=_lowercase							,    max_length=len(self.expected_src_tokens   )							,    return_tensors="""pt"""							,    )

								snake_case_   : Dict						=			shift_tokens_right(batch["""labels"""]							,    self.tokenizer.pad_token_id   )

								self.assertIsInstance(_lowercase							,    _lowercase   )

								self.assertEqual((2, 1_4)							,    batch.input_ids.shape   )
								self.assertEqual((2, 1_4)							,    batch.attention_mask.shape   )
								snake_case_   : Dict						=			batch.input_ids.tolist()[0]
								self.assertListEqual(self.expected_src_tokens							,    _lowercase   )
								self.assertEqual(2							,    batch.decoder_input_ids[0, -1]   )  # EOS
								# Test that special tokens are reset
								self.assertEqual(self.tokenizer.prefix_tokens							,    []   )
								self.assertEqual(self.tokenizer.suffix_tokens							,    [self.tokenizer.eos_token_id, EN_CODE]   )





		def    UpperCAmelCase__							(					self   ) ->						Any:







								'''simple docstring'''

								snake_case_   : Optional[Any]						=			self.tokenizer(self.src_text							,    padding=_lowercase							,    truncation=_lowercase							,    max_length=3							,    return_tensors="""pt"""   )
								snake_case_   : Dict						=			self.tokenizer(
								    text_target=self.tgt_text							,    padding=_lowercase							,    truncation=_lowercase							,    max_length=1_0							,    return_tensors="""pt"""   )
								snake_case_   : Tuple						=			targets["""input_ids"""]
								snake_case_   : Optional[Any]						=			shift_tokens_right(_lowercase							,    self.tokenizer.pad_token_id   )

								self.assertEqual(batch.input_ids.shape[1]							,    3   )
								self.assertEqual(batch.decoder_input_ids.shape[1]							,    1_0   )





		@require_torch
		def    UpperCAmelCase__							(					self   ) ->						Tuple:







								'''simple docstring'''

								snake_case_   : Optional[int]						=			self.tokenizer._build_translation_inputs(
								    """A test"""							,    return_tensors="""pt"""							,    src_lang="""en_XX"""							,    tgt_lang="""ar_AR"""   )

								self.assertEqual(
								    nested_simplify(_lowercase   )							,    {
								        # A, test, EOS, en_XX
								        """input_ids""": [[6_2, 3_0_3_4, 2, 2_5_0_0_0_4]],
								        """attention_mask""": [[1, 1, 1, 1]],
								        # ar_AR
								        """forced_bos_token_id""": 2_5_0_0_0_1,
								    }							,    )






"""simple docstring"""





from .data_collator import (
    DataCollatorForLanguageModeling,
    DataCollatorForPermutationLanguageModeling,
    DataCollatorForSeqaSeq,
    DataCollatorForSOP,
    DataCollatorForTokenClassification,
    DataCollatorForWholeWordMask,
    DataCollatorWithPadding,
    DefaultDataCollator,
    default_data_collator,
)
from .metrics import glue_compute_metrics, xnli_compute_metrics
from .processors import (
    DataProcessor,
    InputExample,
    InputFeatures,
    SingleSentenceClassificationProcessor,
    SquadExample,
    SquadFeatures,
    SquadVaProcessor,
    SquadVaProcessor,
    glue_convert_examples_to_features,
    glue_output_modes,
    glue_processors,
    glue_tasks_num_labels,
    squad_convert_examples_to_features,
    xnli_output_modes,
    xnli_processors,
    xnli_tasks_num_labels,
)

