






'''simple docstring'''

import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import MaMaaaTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)
from transformers.utils import is_sentencepiece_available


if is_sentencepiece_available():
			from transformers.models.mam_aaa.tokenization_mam_aaa import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


if is_sentencepiece_available():
			__lowerCamelCase : Union[str, Any]											=			get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
			from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

__lowerCamelCase : Tuple											=			12_8022
__lowerCamelCase : Optional[int]											=			12_8028

@require_sentencepiece
class       UpperCAmelCase (						_lowercase			,       unittest.TestCase  ):
		UpperCAmelCase					:	Optional[Any] 			=		MaMaaaTokenizer
		UpperCAmelCase					:	Dict 			=		False
		UpperCAmelCase					:	List[str] 			=		False
		UpperCAmelCase					:	Tuple 			=		True




		def 				UpperCAmelCase__   (self      :	Tuple				)					-> int:
						super().setUp()

						lowercase         = ["</s>", "<unk>", "▁This", "▁is", "▁a", "▁t", "est", "\u0120", "<pad>"]
						lowercase         = dict(zip(A__							,			range(len(A__				)				)				)				)
						lowercase         = Path(self.tmpdirname				)
						save_json(A__							,			save_dir / VOCAB_FILES_NAMES["vocab_file"]				)
						if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
										copyfile(A__							,			save_dir / VOCAB_FILES_NAMES["spm_file"]				)

						lowercase         = MaMaaaTokenizer.from_pretrained(self.tmpdirname				)
						tokenizer.save_pretrained(self.tmpdirname				)




		def 				UpperCAmelCase__   (self      :	List[Any]							,			**A__      :	List[str]				)					-> Optional[Any]:
						return MaMaaaTokenizer.from_pretrained(self.tmpdirname							,			**A__				)




		def 				UpperCAmelCase__   (self      :	Dict							,			A__      :	Dict				)					-> List[str]:
						return (
						    "This is a test",
						    "This is a test",
						)




		def 				UpperCAmelCase__   (self      :	Tuple				)					-> Tuple:
						lowercase         = "</s>"
						lowercase         = 0

						self.assertEqual(self.get_tokenizer()._convert_token_to_id(A__				)							,			A__				)
						self.assertEqual(self.get_tokenizer()._convert_id_to_token(A__				)							,			A__				)




		def 				UpperCAmelCase__   (self      :	Dict				)					-> Optional[int]:
						lowercase         = self.get_tokenizer()
						lowercase         = list(tokenizer.get_vocab().keys()				)

						self.assertEqual(vocab_keys[0]							,			"</s>"				)
						self.assertEqual(vocab_keys[1]							,			"<unk>"				)
						self.assertEqual(vocab_keys[-1]							,			"<s>"				)
						self.assertEqual(len(A__				)							,			tokenizer.vocab_size + len(tokenizer.get_added_vocab()				)				)




		@unittest.skip("Skip this test while all models are still to be uploaded."				)
		def 				UpperCAmelCase__   (self      :	str				)					-> List[Any]:
						pass




		def 				UpperCAmelCase__   (self      :	Optional[Any]				)					-> Tuple:
						lowercase         = self.get_tokenizer()

						lowercase         = tokenizer.tokenize("This is a test"				)
						self.assertListEqual(A__							,			["▁This", "▁is", "▁a", "▁t", "est"]				)

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(A__				)							,			[2, 3, 4, 5, 6]							,			)

						lowercase         = tokenizer.convert_ids_to_tokens([2, 3, 4, 5, 6]				)
						self.assertListEqual(A__							,			["▁This", "▁is", "▁a", "▁t", "est"]				)

						lowercase         = tokenizer.convert_tokens_to_string(A__				)
						self.assertEqual(A__							,			"This is a test"				)





		@slow
		def 				UpperCAmelCase__   (self      :	List[str]				)					-> Union[str, Any]:
						# fmt: off
						lowercase         = {"input_ids": [[1_2_8_0_2_2, 1_1_0_1_0_8, 3_9_7, 1_1, 3_8_2_7_2, 2_2_4_7, 1_2_4_8_1_1, 2_8_5, 1_8_1_0_5, 1_5_8_6, 2_0_7, 7, 3_9_5_3_4, 4_4_2_8, 3_9_7, 1_0_1_9, 1_8_1_0_5, 1_5_8_6, 2_0_7, 7, 4_1_3_3_7, 1_6_7_8_6, 2_4_1, 7, 2_0_2_1_4, 1_7, 1_2_5_6_9_0, 1_0_3_9_8, 7, 4_4_3_7_8, 5_8_0_6_9, 6_8_3_4_2, 7_7_9_8, 7_3_4_3, 1_1, 2_9_9, 3_3_3_1_0, 4, 1_5_8, 3_7_3_5_0, 9_4_0_7_7, 4_5_6_9, 2_9_9, 3_3_3_1_0, 9_0, 4, 5_2_8_4_0, 2_9_0, 4, 3_1_2_7_0, 1_1_2, 2_9_9, 6_8_2, 4, 5_2_8_4_0, 3_9_9_5_3, 1_4_0_7_9, 1_9_3, 5_2_5_1_9, 9_0_8_9_4, 1_7_8_9_4, 1_2_0_6_9_7, 1_1, 4_0_4_4_5, 5_5_1, 1_7, 1_0_1_9, 5_2_5_1_9, 9_0_8_9_4, 1_7_7_5_6, 9_6_3, 1_1, 4_0_4_4_5, 4_8_0, 1_7, 9_7_9_2, 1_1_2_0, 5_1_7_3, 1_3_9_3, 6_2_4_0, 1_6_7_8_6, 2_4_1, 1_2_0_9_9_6, 2_8, 1_2_4_5, 1_3_9_3, 1_1_8_2_4_0, 1_1_1_2_3, 1_0_1_9, 9_3_6_1_2, 2_6_9_1, 1_0_6_1_8, 9_8_0_5_8, 1_2_0_4_0_9, 1_9_2_8, 2_7_9, 4, 4_0_6_8_3, 3_6_7, 1_7_8, 2_0_7, 1_0_1_9, 1_0_3, 1_0_3_1_2_1, 5_0_6, 6_5_2_9_6, 5, 2], [1_2_8_0_2_2, 2_1_2_1_7, 3_6_7, 1_1_7, 1_2_5_4_5_0, 1_2_8, 7_1_9, 7, 7_3_0_8, 4_0, 9_3_6_1_2, 1_2_6_6_9, 1_1_1_6, 1_6_7_0_4, 7_1, 1_7_7_8_5, 3_6_9_9, 1_5_5_9_2, 3_5, 1_4_4, 9_5_8_4, 2_4_1, 1_1_9_4_3, 7_1_3, 9_5_0, 7_9_9, 2_2_4_7, 8_8_4_2_7, 1_5_0, 1_4_9, 1_1_8_8_1_3, 1_2_0_7_0_6, 1_0_1_9, 1_0_6_9_0_6, 8_1_5_1_8, 2_8, 1_2_2_4, 2_2_7_9_9, 3_9_7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1_2_8_0_2_2, 1_6_5_8, 1_2_3_3_1_1, 5_1_5_5, 5_5_7_8, 4_7_2_2, 2_7_9, 1_4_9_4_7, 2_3_6_6, 1_1_2_0, 1_1_9_7, 1_4, 1_3_4_8, 9_2_3_2, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
						# fmt: on

						self.tokenizer_integration_test_util(
						    expected_encoding=A__							,			model_name="facebook/m2m100_418M"							,			revision="c168bae485c864188cf9aa0e4108b0b6934dc91e"							,			)




@require_torch
@require_sentencepiece
@require_tokenizers
class       UpperCAmelCase (						unittest.TestCase  ):
		UpperCAmelCase					:	Any 			=		'''facebook/m2m100_418M'''
		UpperCAmelCase					:	Tuple 			=		[
		    '''In my opinion, there are two levels of response from the French government.''',
		    '''NSA Affair Emphasizes Complete Lack of Debate on Intelligence''',
		]
		UpperCAmelCase					:	List[Any] 			=		[
		    '''Selon moi, il y a deux niveaux de réponse de la part du gouvernement français.''',
		    '''L\'affaire NSA souligne l\'absence totale de débat sur le renseignement''',
		]

		# fmt: off
		UpperCAmelCase					:	str 			=		[EN_CODE, 593, 1949, 11_5781, 4, 7_1586, 4234, 6_0633, 12_6233, 432, 12_3808, 1_5592, 1197, 11_7132, 12_0618, 5, 2]




		@classmethod
		def 				UpperCAmelCase__   (cls      :	Dict				)					-> Optional[Any]:
						lowercase         = MaMaaaTokenizer.from_pretrained(
						    cls.checkpoint_name							,			src_lang="en"							,			tgt_lang="fr"				)
						lowercase         = 1
						return cls




		def 				UpperCAmelCase__   (self      :	List[Any]				)					-> Union[str, Any]:
						self.assertEqual(self.tokenizer.get_lang_id("ar"				)							,			1_2_8_0_0_6				)
						self.assertEqual(self.tokenizer.get_lang_id("en"				)							,			1_2_8_0_2_2				)
						self.assertEqual(self.tokenizer.get_lang_id("ro"				)							,			1_2_8_0_7_6				)
						self.assertEqual(self.tokenizer.get_lang_id("mr"				)							,			1_2_8_0_6_3				)




		def 				UpperCAmelCase__   (self      :	Optional[Any]				)					-> List[str]:
						lowercase         = self.tokenizer.get_vocab()
						self.assertEqual(len(A__				)							,			self.tokenizer.vocab_size				)
						self.assertEqual(vocab["<unk>"]							,			3				)
						self.assertIn(self.tokenizer.get_lang_token("en"				)							,			A__				)




		def 				UpperCAmelCase__   (self      :	Optional[Any]				)					-> Union[str, Any]:
						lowercase         = "en"
						lowercase         = self.tokenizer.batch_encode_plus(self.src_text				).input_ids[0]
						self.assertListEqual(self.expected_src_tokens							,			A__				)




		def 				UpperCAmelCase__   (self      :	Tuple				)					-> List[Any]:
						self.assertIn(A__							,			self.tokenizer.all_special_ids				)
						# fmt: off
						lowercase         = [FR_CODE, 5_3_6_4, 8_2, 8_6_4_2, 4, 2_9_4, 4_7, 8, 1_4_0_2_8, 1_3_6, 3_2_8_6, 9_7_0_6, 6, 9_0_7_9_7, 6, 1_4_4_0_1_2, 1_6_2, 8_8_1_2_8, 3_0_0_6_1, 5, 2]
						# fmt: on
						lowercase         = self.tokenizer.decode(A__							,			skip_special_tokens=A__				)
						lowercase         = self.tokenizer.decode(generated_ids[1:]							,			skip_special_tokens=A__				)
						self.assertEqual(A__							,			A__				)
						self.assertNotIn(self.tokenizer.eos_token							,			A__				)




		def 				UpperCAmelCase__   (self      :	List[Any]				)					-> int:
						lowercase         = tempfile.mkdtemp()
						lowercase         = self.tokenizer.lang_token_to_id
						self.tokenizer.save_pretrained(A__				)
						lowercase         = MaMaaaTokenizer.from_pretrained(A__				)
						self.assertDictEqual(new_tok.lang_token_to_id							,			A__				)




		@require_torch
		def 				UpperCAmelCase__   (self      :	Tuple				)					-> Union[str, Any]:
						lowercase         = "en"
						lowercase         = "fr"

						lowercase         = self.tokenizer(self.src_text							,			text_target=self.tgt_text							,			padding=A__							,			return_tensors="pt"				)

						lowercase         = shift_tokens_right(
						    batch["labels"]							,			self.tokenizer.pad_token_id							,			self.tokenizer.eos_token_id				)

						for k in batch:
										lowercase         = batch[k].tolist()
						# batch = {k: v.tolist() for k,v in batch.items()}
						# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
						# batch.decoder_inputs_ids[0][0] ==
						assert batch.input_ids[1][0] == EN_CODE
						assert batch.input_ids[1][-1] == 2
						assert batch.labels[1][0] == FR_CODE
						assert batch.labels[1][-1] == 2
						assert batch.decoder_input_ids[1][:2] == [2, FR_CODE]




		@require_torch
		def 				UpperCAmelCase__   (self      :	int				)					-> List[str]:
						lowercase         = "mr"
						self.assertListEqual(self.tokenizer.prefix_tokens							,			[self.tokenizer.get_lang_id("mr"				)]				)
						self.assertListEqual(self.tokenizer.suffix_tokens							,			[self.tokenizer.eos_token_id]				)

						lowercase         = "zh"
						self.assertListEqual(self.tokenizer.prefix_tokens							,			[self.tokenizer.get_lang_id("zh"				)]				)
						self.assertListEqual(self.tokenizer.suffix_tokens							,			[self.tokenizer.eos_token_id]				)




		@require_torch
		def 				UpperCAmelCase__   (self      :	Union[str, Any]				)					-> Any:
						lowercase         = "mr"
						self.tokenizer._switch_to_target_mode()
						self.assertListEqual(self.tokenizer.prefix_tokens							,			[self.tokenizer.get_lang_id("mr"				)]				)
						self.assertListEqual(self.tokenizer.suffix_tokens							,			[self.tokenizer.eos_token_id]				)
						self.tokenizer._switch_to_input_mode()
						self.assertListEqual(self.tokenizer.prefix_tokens							,			[self.tokenizer.get_lang_id(self.tokenizer.src_lang				)]				)

						lowercase         = "zh"
						self.tokenizer._switch_to_target_mode()
						self.assertListEqual(self.tokenizer.prefix_tokens							,			[self.tokenizer.get_lang_id("zh"				)]				)
						self.assertListEqual(self.tokenizer.suffix_tokens							,			[self.tokenizer.eos_token_id]				)
						self.tokenizer._switch_to_input_mode()
						self.assertListEqual(self.tokenizer.prefix_tokens							,			[self.tokenizer.get_lang_id(self.tokenizer.src_lang				)]				)





		@require_torch
		def 				UpperCAmelCase__   (self      :	int				)					-> Optional[Any]:
						lowercase         = self.tokenizer._build_translation_inputs("A test"							,			return_tensors="pt"							,			src_lang="en"							,			tgt_lang="ar"				)

						self.assertEqual(
						    nested_simplify(A__				)							,			{
						        # en_XX, A, test, EOS
						        "input_ids": [[1_2_8_0_2_2, 5_8, 4_1_8_3, 2]],
						        "attention_mask": [[1, 1, 1, 1]],
						        # ar_AR
						        "forced_bos_token_id": 1_2_8_0_0_6,
						    }							,			)











'''simple docstring'''

import warnings
from contextlib import contextmanager

from ....processing_utils import ProcessorMixin

class       UpperCAmelCase (						_lowercase  ):
		UpperCAmelCase					:	Optional[Any] 			=		'''MCTCTFeatureExtractor'''
		UpperCAmelCase					:	Tuple 			=		'''AutoTokenizer'''




		def __init__(self      :	int							,			A__      :	Tuple							,			A__      :	Union[str, Any]				)					-> Dict:
						super().__init__(A__							,			A__				)
						lowercase         = self.feature_extractor
						lowercase         = False




		def __call__(self      :	Tuple							,			*A__      :	str							,			**A__      :	Dict				)					-> int:
						# For backward compatibility
						if self._in_target_context_manager:
										return self.current_processor(*A__							,			**A__				)

						if "raw_speech" in kwargs:
										warnings.warn("Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead."				)
										lowercase         = kwargs.pop("raw_speech"				)
						else:
										lowercase         = kwargs.pop("audio"							,			A__				)
						lowercase         = kwargs.pop("sampling_rate"							,			A__				)
						lowercase         = kwargs.pop("text"							,			A__				)
						if len(A__				) > 0:
										lowercase         = args[0]
										lowercase         = args[1:]

						if audio is None and text is None:
										raise ValueError("You need to specify either an `audio` or `text` input to process."				)

						if audio is not None:
										lowercase         = self.feature_extractor(A__							,			*A__							,			sampling_rate=A__							,			**A__				)
						if text is not None:
										lowercase         = self.tokenizer(A__							,			**A__				)

						if text is None:
										return inputs
						elif audio is None:
										return encodings
						else:
										lowercase         = encodings["input_ids"]
										return inputs




		def 				UpperCAmelCase__   (self      :	Tuple							,			*A__      :	str							,			**A__      :	str				)					-> str:
						return self.tokenizer.batch_decode(*A__							,			**A__				)




		def 				UpperCAmelCase__   (self      :	Any							,			*A__      :	List[Any]							,			**A__      :	List[str]				)					-> Tuple:
						# For backward compatibility
						if self._in_target_context_manager:
										return self.current_processor.pad(*A__							,			**A__				)

						lowercase         = kwargs.pop("input_features"							,			A__				)
						lowercase         = kwargs.pop("labels"							,			A__				)
						if len(A__				) > 0:
										lowercase         = args[0]
										lowercase         = args[1:]

						if input_features is not None:
										lowercase         = self.feature_extractor.pad(A__							,			*A__							,			**A__				)
						if labels is not None:
										lowercase         = self.tokenizer.pad(A__							,			**A__				)

						if labels is None:
										return input_features
						elif input_features is None:
										return labels
						else:
										lowercase         = labels["input_ids"]
										return input_features




		def 				UpperCAmelCase__   (self      :	Tuple							,			*A__      :	Optional[int]							,			**A__      :	Optional[int]				)					-> Tuple:
						return self.tokenizer.decode(*A__							,			**A__				)





		@contextmanager
		def 				UpperCAmelCase__   (self      :	Optional[Any]				)					-> Union[str, Any]:
						warnings.warn(
						    "`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your "
						    "labels by using the argument `text` of the regular `__call__` method (either in the same call as "
						    "your audio inputs, or in a separate call."				)
						lowercase         = True
						lowercase         = self.tokenizer
						yield
						lowercase         = self.feature_extractor
						lowercase         = False




