





'''simple docstring'''





import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartTokenizer, MBartTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


_a						=   get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
			from transformers.models.mbart.modeling_mbart import shift_tokens_right

_a						=   250_004
_a						=   250_020






@require_sentencepiece
@require_tokenizers
class       __A					(					__A    ,       unittest.TestCase				):

			'''simple docstring'''







			lowerCAmelCase_							  =		MBartTokenizer
			lowerCAmelCase_							  =		MBartTokenizerFast
			lowerCAmelCase_							  =		True
			lowerCAmelCase_							  =		True





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				super().setUp()

				# We have a SentencePiece fixture for testing
				lowerCamelCase__              =	MBartTokenizer(__lowerCAmelCase ,						keep_accents=__lowerCAmelCase							)
				tokenizer.save_pretrained(self.tmpdirname							)





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	MBartTokenizer(__lowerCAmelCase ,						keep_accents=__lowerCAmelCase							)

				lowerCamelCase__              =	tokenizer.tokenize('''This is a test'''							)
				self.assertListEqual(__lowerCAmelCase ,						['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']							)

				self.assertListEqual(
				    tokenizer.convert_tokens_to_ids(__lowerCAmelCase							) ,						[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]] ,						)

				lowerCamelCase__              =	tokenizer.tokenize('''I was born in 92000, and this is falsé.'''							)
				self.assertListEqual(
				    __lowerCAmelCase ,						[
				        SPIECE_UNDERLINE + '''I''',
				        SPIECE_UNDERLINE + '''was''',
				        SPIECE_UNDERLINE + '''b''',
				        '''or''',
				        '''n''',
				        SPIECE_UNDERLINE + '''in''',
				        SPIECE_UNDERLINE + '''''',
				        '''9''',
				        '''2''',
				        '''0''',
				        '''0''',
				        '''0''',
				        ''',''',
				        SPIECE_UNDERLINE + '''and''',
				        SPIECE_UNDERLINE + '''this''',
				        SPIECE_UNDERLINE + '''is''',
				        SPIECE_UNDERLINE + '''f''',
				        '''al''',
				        '''s''',
				        '''é''',
				        '''.''',
				    ] ,						)
				lowerCamelCase__              =	tokenizer.convert_tokens_to_ids(__lowerCAmelCase							)
				self.assertListEqual(
				    __lowerCAmelCase ,						[
				        value + tokenizer.fairseq_offset
				        for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
				        #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
				    ] ,						)

				lowerCamelCase__              =	tokenizer.convert_ids_to_tokens(__lowerCAmelCase							)
				self.assertListEqual(
				    __lowerCAmelCase ,						[
				        SPIECE_UNDERLINE + '''I''',
				        SPIECE_UNDERLINE + '''was''',
				        SPIECE_UNDERLINE + '''b''',
				        '''or''',
				        '''n''',
				        SPIECE_UNDERLINE + '''in''',
				        SPIECE_UNDERLINE + '''''',
				        '''<unk>''',
				        '''2''',
				        '''0''',
				        '''0''',
				        '''0''',
				        ''',''',
				        SPIECE_UNDERLINE + '''and''',
				        SPIECE_UNDERLINE + '''this''',
				        SPIECE_UNDERLINE + '''is''',
				        SPIECE_UNDERLINE + '''f''',
				        '''al''',
				        '''s''',
				        '''<unk>''',
				        '''.''',
				    ] ,						)





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				if not self.test_slow_tokenizer:
					# as we don't have a slow version, we can't compare the outputs between slow and fast versions
					return

				lowerCamelCase__              =	(self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-mbart', {})
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'							):
						lowerCamelCase__              =	self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase ,						**__lowerCAmelCase							)
						lowerCamelCase__              =	self.tokenizer_class.from_pretrained(__lowerCAmelCase ,						**__lowerCAmelCase							)

						lowerCamelCase__              =	tempfile.mkdtemp()

						lowerCamelCase__              =	tokenizer_r.save_pretrained(__lowerCAmelCase							)
						lowerCamelCase__              =	tokenizer_p.save_pretrained(__lowerCAmelCase							)

						# Checks it save with the same files + the tokenizer.json file for the fast one
						self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files							)							)
						lowerCamelCase__              =	tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f							)
						self.assertSequenceEqual(__lowerCAmelCase ,						__lowerCAmelCase							)

						# Checks everything loads correctly in the same way
						lowerCamelCase__              =	tokenizer_r.from_pretrained(__lowerCAmelCase							)
						lowerCamelCase__              =	tokenizer_p.from_pretrained(__lowerCAmelCase							)

						# Check special tokens are set accordingly on Rust and Python
						for key in tokenizer_pp.special_tokens_map:
							self.assertTrue(hasattr(__lowerCAmelCase ,						__lowerCAmelCase							)							)
							# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
							# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

						shutil.rmtree(__lowerCAmelCase							)

						# Save tokenizer rust, legacy_format=True
						lowerCamelCase__              =	tempfile.mkdtemp()

						lowerCamelCase__              =	tokenizer_r.save_pretrained(__lowerCAmelCase ,						legacy_format=__lowerCAmelCase							)
						lowerCamelCase__              =	tokenizer_p.save_pretrained(__lowerCAmelCase							)

						# Checks it save with the same files
						self.assertSequenceEqual(__lowerCAmelCase ,						__lowerCAmelCase							)

						# Checks everything loads correctly in the same way
						lowerCamelCase__              =	tokenizer_r.from_pretrained(__lowerCAmelCase							)
						lowerCamelCase__              =	tokenizer_p.from_pretrained(__lowerCAmelCase							)

						# Check special tokens are set accordingly on Rust and Python
						for key in tokenizer_pp.special_tokens_map:
							self.assertTrue(hasattr(__lowerCAmelCase ,						__lowerCAmelCase							)							)

						shutil.rmtree(__lowerCAmelCase							)

						# Save tokenizer rust, legacy_format=False
						lowerCamelCase__              =	tempfile.mkdtemp()

						lowerCamelCase__              =	tokenizer_r.save_pretrained(__lowerCAmelCase ,						legacy_format=__lowerCAmelCase							)
						lowerCamelCase__              =	tokenizer_p.save_pretrained(__lowerCAmelCase							)

						# Checks it saved the tokenizer.json file
						self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files							)							)

						# Checks everything loads correctly in the same way
						lowerCamelCase__              =	tokenizer_r.from_pretrained(__lowerCAmelCase							)
						lowerCamelCase__              =	tokenizer_p.from_pretrained(__lowerCAmelCase							)

						# Check special tokens are set accordingly on Rust and Python
						for key in tokenizer_pp.special_tokens_map:
							self.assertTrue(hasattr(__lowerCAmelCase ,						__lowerCAmelCase							)							)

						shutil.rmtree(__lowerCAmelCase							)






@require_torch
@require_sentencepiece
@require_tokenizers
class       __A					(					unittest.TestCase				):

			'''simple docstring'''







			lowerCAmelCase_							  =		"""facebook/mbart-large-en-ro"""
			lowerCAmelCase_							  =		[
			    """ UN Chief Says There Is No Military Solution in Syria""",
			    """ Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
			]
			lowerCAmelCase_							  =		[
			    """Şeful ONU declară că nu există o soluţie militară în Siria""",
			    """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"""
			    """ pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"""
			    """ face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
			]
			lowerCAmelCase_							  =		[8274, 127873, 25916, 7, 8622, 2071, 438, 67485, 53, 187895, 23, 51712, 2, EN_CODE]





			@classmethod
			def 	__lowerCamelCase       (					cls							):



				'''simple docstring'''







				lowerCamelCase__              =	MBartTokenizer.from_pretrained(
				    cls.checkpoint_name ,						src_lang='''en_XX''' ,						tgt_lang='''ro_RO'''							)
				lowerCamelCase__              =	1
				return cls





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ar_AR'''] ,						2_5_0_0_0_1							)
				self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''en_EN'''] ,						2_5_0_0_0_4							)
				self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ro_RO'''] ,						2_5_0_0_2_0							)





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	self.tokenizer.batch_encode_plus(self.src_text							).input_ids[0]
				self.assertListEqual(self.expected_src_tokens ,						__lowerCAmelCase							)





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				self.assertIn(__lowerCAmelCase ,						self.tokenizer.all_special_ids							)
				lowerCamelCase__              =	[RO_CODE, 8_8_4, 9_0_1_9, 9_6, 9, 9_1_6, 8_6_7_9_2, 3_6, 1_8_7_4_3, 1_5_5_9_6, 5, 2]
				lowerCamelCase__              =	self.tokenizer.decode(__lowerCAmelCase ,						skip_special_tokens=__lowerCAmelCase							)
				lowerCamelCase__              =	self.tokenizer.decode(generated_ids[1:] ,						skip_special_tokens=__lowerCAmelCase							)
				self.assertEqual(__lowerCAmelCase ,						__lowerCAmelCase							)
				self.assertNotIn(self.tokenizer.eos_token ,						__lowerCAmelCase							)





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	['this is gunna be a long sentence ' * 2_0]
				assert isinstance(src_text[0] ,						__lowerCAmelCase							)
				lowerCamelCase__              =	1_0
				lowerCamelCase__              =	self.tokenizer(__lowerCAmelCase ,						max_length=__lowerCAmelCase ,						truncation=__lowerCAmelCase							).input_ids[0]
				self.assertEqual(ids[-2] ,						2							)
				self.assertEqual(ids[-1] ,						__lowerCAmelCase							)
				self.assertEqual(len(__lowerCAmelCase							) ,						__lowerCAmelCase							)





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']							) ,						[2_5_0_0_2_6, 2_5_0_0_0_1]							)





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	tempfile.mkdtemp()
				lowerCamelCase__              =	self.tokenizer.fairseq_tokens_to_ids
				self.tokenizer.save_pretrained(__lowerCAmelCase							)
				lowerCamelCase__              =	MBartTokenizer.from_pretrained(__lowerCAmelCase							)
				self.assertDictEqual(new_tok.fairseq_tokens_to_ids ,						__lowerCAmelCase							)





			@require_torch
			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	self.tokenizer(self.src_text ,						text_target=self.tgt_text ,						padding=__lowerCAmelCase ,						return_tensors='''pt'''							)
				lowerCamelCase__              =	shift_tokens_right(batch['''labels'''] ,						self.tokenizer.pad_token_id							)

				# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
				assert batch.input_ids[1][-2:].tolist() == [2, EN_CODE]
				assert batch.decoder_input_ids[1][0].tolist() == RO_CODE
				assert batch.decoder_input_ids[1][-1] == 2
				assert batch.labels[1][-2:].tolist() == [2, RO_CODE]





			@require_torch
			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	self.tokenizer(
				    self.src_text ,						text_target=self.tgt_text ,						padding=__lowerCAmelCase ,						truncation=__lowerCAmelCase ,						max_length=len(self.expected_src_tokens							) ,						return_tensors='''pt''' ,						)

				lowerCamelCase__              =	shift_tokens_right(batch['''labels'''] ,						self.tokenizer.pad_token_id							)

				self.assertIsInstance(__lowerCAmelCase ,						__lowerCAmelCase							)

				self.assertEqual((2, 1_4) ,						batch.input_ids.shape							)
				self.assertEqual((2, 1_4) ,						batch.attention_mask.shape							)
				lowerCamelCase__              =	batch.input_ids.tolist()[0]
				self.assertListEqual(self.expected_src_tokens ,						__lowerCAmelCase							)
				self.assertEqual(2 ,						batch.decoder_input_ids[0, -1]							)  # EOS
				# Test that special tokens are reset
				self.assertEqual(self.tokenizer.prefix_tokens ,						[]							)
				self.assertEqual(self.tokenizer.suffix_tokens ,						[self.tokenizer.eos_token_id, EN_CODE]							)





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	self.tokenizer(self.src_text ,						padding=__lowerCAmelCase ,						truncation=__lowerCAmelCase ,						max_length=3 ,						return_tensors='''pt'''							)
				lowerCamelCase__              =	self.tokenizer(
				    text_target=self.tgt_text ,						padding=__lowerCAmelCase ,						truncation=__lowerCAmelCase ,						max_length=1_0 ,						return_tensors='''pt'''							)
				lowerCamelCase__              =	targets['input_ids']
				lowerCamelCase__              =	shift_tokens_right(__lowerCAmelCase ,						self.tokenizer.pad_token_id							)

				self.assertEqual(batch.input_ids.shape[1] ,						3							)
				self.assertEqual(batch.decoder_input_ids.shape[1] ,						1_0							)





			@require_torch
			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				lowerCamelCase__              =	self.tokenizer._build_translation_inputs(
				    '''A test''' ,						return_tensors='''pt''' ,						src_lang='''en_XX''' ,						tgt_lang='''ar_AR'''							)

				self.assertEqual(
				    nested_simplify(__lowerCAmelCase							) ,						{
				        # A, test, EOS, en_XX
				        '''input_ids''': [[6_2, 3_0_3_4, 2, 2_5_0_0_0_4]],
				        '''attention_mask''': [[1, 1, 1, 1]],
				        # ar_AR
				        '''forced_bos_token_id''': 2_5_0_0_0_1,
				    } ,						)










from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_a						=   {
    "configuration_funnel": ["FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP", "FunnelConfig"],
    "convert_funnel_original_tf_checkpoint_to_pytorch": [],
    "tokenization_funnel": ["FunnelTokenizer"],
}

try:
			if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_a						=   ["FunnelTokenizerFast"]

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_a						=   [
			    "FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "FunnelBaseModel",
			    "FunnelForMaskedLM",
			    "FunnelForMultipleChoice",
			    "FunnelForPreTraining",
			    "FunnelForQuestionAnswering",
			    "FunnelForSequenceClassification",
			    "FunnelForTokenClassification",
			    "FunnelModel",
			    "FunnelPreTrainedModel",
			    "load_tf_weights_in_funnel",
			]

try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_a						=   [
			    "TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "TFFunnelBaseModel",
			    "TFFunnelForMaskedLM",
			    "TFFunnelForMultipleChoice",
			    "TFFunnelForPreTraining",
			    "TFFunnelForQuestionAnswering",
			    "TFFunnelForSequenceClassification",
			    "TFFunnelForTokenClassification",
			    "TFFunnelModel",
			    "TFFunnelPreTrainedModel",
			]


if TYPE_CHECKING:
			from .configuration_funnel import FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP, FunnelConfig
			from .tokenization_funnel import FunnelTokenizer

			try:
						if not is_tokenizers_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_funnel_fast import FunnelTokenizerFast

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_funnel import (
						    FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
						    FunnelBaseModel,
						    FunnelForMaskedLM,
						    FunnelForMultipleChoice,
						    FunnelForPreTraining,
						    FunnelForQuestionAnswering,
						    FunnelForSequenceClassification,
						    FunnelForTokenClassification,
						    FunnelModel,
						    FunnelPreTrainedModel,
						    load_tf_weights_in_funnel,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_funnel import (
						    TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFFunnelBaseModel,
						    TFFunnelForMaskedLM,
						    TFFunnelForMultipleChoice,
						    TFFunnelForPreTraining,
						    TFFunnelForQuestionAnswering,
						    TFFunnelForSequenceClassification,
						    TFFunnelForTokenClassification,
						    TFFunnelModel,
						    TFFunnelPreTrainedModel,
						)

else:
			import sys

			_a						=   _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



