




import uuid
from typing import Any, Dict, List, Optional, Union

from ..utils import add_end_docstrings, is_tf_available, is_torch_available, logging
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_tf_available():
		import tensorflow as tf

if is_torch_available():
		import torch


__A						:							Optional[int]							=  logging.get_logger(__name__)




class 			UpperCAmelCase_      :




							'''simple docstring'''



							def __init__(					self       :	Tuple       ,     a       :	str = None       ,     a       :	uuid.UUID = None       ,     a       :	Optional[Any]=None       ,     a       :	Any=None			)			->					Optional[Any]:
										if not conversation_id:
													SCREAMING_SNAKE_CASE           =					uuid.uuida()
										if past_user_inputs is None:
													SCREAMING_SNAKE_CASE           =					[]
										if generated_responses is None:
													SCREAMING_SNAKE_CASE           =					[]

										SCREAMING_SNAKE_CASE           =					conversation_id
										SCREAMING_SNAKE_CASE           =					past_user_inputs
										SCREAMING_SNAKE_CASE           =					generated_responses
										SCREAMING_SNAKE_CASE           =					text





							def __eq__(					self       :	Optional[Any]       ,     a       :	int			)			->					Union[str, Any]:
										if not isinstance(a       ,     a			):
													return False
										if self.uuid == other.uuid:
													return True
										return (
										    self.new_user_input == other.new_user_input
										    and self.past_user_inputs == other.past_user_inputs
										    and self.generated_responses == other.generated_responses
										)





							def    _UpperCAmelCase     (					self       :	Tuple       ,     a       :	str       ,     a       :	bool = False			)			->					Tuple:
										if self.new_user_input:
													if overwrite:
																logger.warning(
																    f"""User input added while unprocessed input was existing: \"{self.new_user_input}\" was overwritten """
																    f"""with: \"{text}\"."""			)
																SCREAMING_SNAKE_CASE           =					text
													else:
																logger.warning(
																    f"""User input added while unprocessed input was existing: \"{self.new_user_input}\" new input """
																    f"""ignored: \"{text}\". Set `overwrite` to True to overwrite unprocessed user input"""			)
										else:
													SCREAMING_SNAKE_CASE           =					text





							def    _UpperCAmelCase     (					self       :	Optional[Any]			)			->					Any:
										if self.new_user_input:
													self.past_user_inputs.append(self.new_user_input			)
										SCREAMING_SNAKE_CASE           =					None





							def    _UpperCAmelCase     (					self       :	Union[str, Any]       ,     a       :	str			)			->					Any:
										self.generated_responses.append(a			)





							def    _UpperCAmelCase     (					self       :	List[Any]			)			->					List[str]:
										for user_input, generated_response in zip(self.past_user_inputs       ,     self.generated_responses			):
													yield True, user_input
													yield False, generated_response
										if self.new_user_input:
													yield True, self.new_user_input







							def __repr__(					self       :	Any			)			->					int:
										SCREAMING_SNAKE_CASE           =					f"""Conversation id: {self.uuid} \n"""
										for is_user, text in self.iter_texts():
													SCREAMING_SNAKE_CASE           =					"""user""" if is_user else """bot"""
													output += f"""{name} >> {text} \n"""
										return output





@add_end_docstrings(
    A		,      R'''
        min_length_for_response (`int`, *optional*, defaults to 32):
            The minimum length (in number of tokens) for a response.
        minimum_tokens (`int`, *optional*, defaults to 10):
            The minimum length of tokens to leave for a response.
    '''		,      )
class 			UpperCAmelCase_      (   A       ):




							'''simple docstring'''



							def __init__(					self       :	List[Any]       ,     *a       :	List[str]       ,     **a       :	Optional[Any]			)			->					Optional[Any]:
										super().__init__(*a       ,     **a			)
										if self.tokenizer.pad_token_id is None:
													SCREAMING_SNAKE_CASE           =					self.tokenizer.eos_token





							def    _UpperCAmelCase     (					self       :	Union[str, Any]       ,     a       :	Any=None       ,     a       :	List[Any]=None       ,     a       :	str=None       ,     **a       :	Union[str, Any]			)			->					str:
										SCREAMING_SNAKE_CASE           =					{}
										SCREAMING_SNAKE_CASE           =					{}
										SCREAMING_SNAKE_CASE           =					{}

										if min_length_for_response is not None:
													SCREAMING_SNAKE_CASE           =					min_length_for_response
										if minimum_tokens is not None:
													SCREAMING_SNAKE_CASE           =					minimum_tokens

										if "max_length" in generate_kwargs:
													SCREAMING_SNAKE_CASE           =					generate_kwargs["""max_length"""]
													# self.max_length = generate_kwargs.get("max_length", self.model.config.max_length)
										if clean_up_tokenization_spaces is not None:
													SCREAMING_SNAKE_CASE           =					clean_up_tokenization_spaces

										if generate_kwargs:
													forward_params.update(a			)
										return preprocess_params, forward_params, postprocess_params





							def __call__(					self       :	List[str]       ,     a       :	Union[Conversation, List[Conversation]]       ,     a       :	int=0       ,     **a       :	Any			)			->					Any:
										SCREAMING_SNAKE_CASE           =					super().__call__(a       ,     num_workers=a       ,     **a			)
										if isinstance(a       ,     a			) and len(a			) == 1:
													return outputs[0]
										return outputs





							def    _UpperCAmelCase     (					self       :	Tuple       ,     a       :	Conversation       ,     a       :	Optional[int]=32			)			->					Dict[str, Any]:
										if not isinstance(a       ,     a			):
													raise ValueError("""ConversationalPipeline, expects Conversation as inputs"""			)
										if conversation.new_user_input is None:
													raise ValueError(
													    f"""Conversation with UUID {type(conversation.uuid			)} does not contain new user input to process. """
													    """Add user inputs with the conversation's `add_user_input` method"""			)
										if hasattr(self.tokenizer       ,     """_build_conversation_input_ids"""			):
													SCREAMING_SNAKE_CASE           =					self.tokenizer._build_conversation_input_ids(a			)
										else:
													# If the tokenizer cannot handle conversations, we default to only the old version
													SCREAMING_SNAKE_CASE           =					self._legacy_parse_and_tokenize(a			)

										if self.framework == "pt":
													SCREAMING_SNAKE_CASE           =					torch.LongTensor([input_ids]			)
										elif self.framework == "tf":
													SCREAMING_SNAKE_CASE           =					tf.constant([input_ids]			)
										return {"input_ids": input_ids, "conversation": conversation}





							def    _UpperCAmelCase     (					self       :	List[str]       ,     a       :	Optional[Any]       ,     a       :	str=10       ,     **a       :	Tuple			)			->					Tuple:
										SCREAMING_SNAKE_CASE           =					generate_kwargs.get("""max_length"""       ,     self.model.config.max_length			)

										SCREAMING_SNAKE_CASE           =					model_inputs["""input_ids"""].shape[1]
										if max_length - minimum_tokens < n:
													logger.warning(f"""Conversation input is to long ({n}), trimming it to ({max_length} - {minimum_tokens})"""			)
													SCREAMING_SNAKE_CASE           =					max_length - minimum_tokens
													SCREAMING_SNAKE_CASE           =					model_inputs["""input_ids"""][:, -trim:]
													if "attention_mask" in model_inputs:
																SCREAMING_SNAKE_CASE           =					model_inputs["""attention_mask"""][:, -trim:]
										SCREAMING_SNAKE_CASE           =					model_inputs.pop("""conversation"""			)
										SCREAMING_SNAKE_CASE           =					max_length
										SCREAMING_SNAKE_CASE           =					self.model.generate(**a       ,     **a			)
										if self.model.config.is_encoder_decoder:
													SCREAMING_SNAKE_CASE           =					1
										else:
													SCREAMING_SNAKE_CASE           =					n
										return {"output_ids": output_ids[:, start_position:], "conversation": conversation}





							def    _UpperCAmelCase     (					self       :	List[Any]       ,     a       :	Optional[int]       ,     a       :	Optional[int]=True			)			->					str:
										SCREAMING_SNAKE_CASE           =					model_outputs["""output_ids"""]
										SCREAMING_SNAKE_CASE           =					self.tokenizer.decode(
										    output_ids[0]       ,     skip_special_tokens=a       ,     clean_up_tokenization_spaces=a       ,     )
										SCREAMING_SNAKE_CASE           =					model_outputs["""conversation"""]
										conversation.mark_processed()
										conversation.append_response(a			)
										return conversation







							def    _UpperCAmelCase     (					self       :	List[Any]       ,     a       :	Conversation			)			->					Dict:
										SCREAMING_SNAKE_CASE           =					self.tokenizer.eos_token_id
										SCREAMING_SNAKE_CASE           =					[]
										for is_user, text in conversation.iter_texts():
													if eos_token_id is not None:
																input_ids.extend(self.tokenizer.encode(a       ,     add_special_tokens=a			) + [eos_token_id]			)
													else:
																input_ids.extend(self.tokenizer.encode(a       ,     add_special_tokens=a			)			)

										if len(a			) > self.tokenizer.model_max_length:
													SCREAMING_SNAKE_CASE           =					input_ids[-self.tokenizer.model_max_length :]
										return input_ids












import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


__A						:							Any							=  get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
		from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

__A						:							Optional[int]							=  2_5_6_0_4_7
__A						:							Dict							=  2_5_6_1_4_5




@require_sentencepiece
@require_tokenizers
class 			UpperCAmelCase_      (   A		,      unittest.TestCase       ):




							'''simple docstring'''



							a__						  =     NllbTokenizer
							a__						  =     NllbTokenizerFast
							a__						  =     True
							a__						  =     True
							a__						  =     {}





							def    _UpperCAmelCase     (					self       :	Any			)			->					int:
										super().setUp()

										# We have a SentencePiece fixture for testing
										SCREAMING_SNAKE_CASE           =					NllbTokenizer(a       ,     keep_accents=a			)
										tokenizer.save_pretrained(self.tmpdirname			)





							def    _UpperCAmelCase     (					self       :	int			)			->					str:
										SCREAMING_SNAKE_CASE           =					NllbTokenizer(a       ,     keep_accents=a			)

										SCREAMING_SNAKE_CASE           =					tokenizer.tokenize("""This is a test"""			)
										self.assertListEqual(a       ,     ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]			)

										self.assertListEqual(
										    tokenizer.convert_tokens_to_ids(a			)       ,     [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]       ,     )

										SCREAMING_SNAKE_CASE           =					tokenizer.tokenize("""I was born in 92000, and this is falsé."""			)
										self.assertListEqual(
										    a       ,     [
										        SPIECE_UNDERLINE + """I""",
										        SPIECE_UNDERLINE + """was""",
										        SPIECE_UNDERLINE + """b""",
										        """or""",
										        """n""",
										        SPIECE_UNDERLINE + """in""",
										        SPIECE_UNDERLINE + """""",
										        """9""",
										        """2""",
										        """0""",
										        """0""",
										        """0""",
										        """,""",
										        SPIECE_UNDERLINE + """and""",
										        SPIECE_UNDERLINE + """this""",
										        SPIECE_UNDERLINE + """is""",
										        SPIECE_UNDERLINE + """f""",
										        """al""",
										        """s""",
										        """é""",
										        """.""",
										    ]       ,     )
										SCREAMING_SNAKE_CASE           =					tokenizer.convert_tokens_to_ids(a			)
										self.assertListEqual(
										    a       ,     [
										        value + tokenizer.fairseq_offset
										        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
										    ]       ,     )

										SCREAMING_SNAKE_CASE           =					tokenizer.convert_ids_to_tokens(a			)
										self.assertListEqual(
										    a       ,     [
										        SPIECE_UNDERLINE + """I""",
										        SPIECE_UNDERLINE + """was""",
										        SPIECE_UNDERLINE + """b""",
										        """or""",
										        """n""",
										        SPIECE_UNDERLINE + """in""",
										        SPIECE_UNDERLINE + """""",
										        """<unk>""",
										        """2""",
										        """0""",
										        """0""",
										        """0""",
										        """,""",
										        SPIECE_UNDERLINE + """and""",
										        SPIECE_UNDERLINE + """this""",
										        SPIECE_UNDERLINE + """is""",
										        SPIECE_UNDERLINE + """f""",
										        """al""",
										        """s""",
										        """<unk>""",
										        """.""",
										    ]       ,     )





							def    _UpperCAmelCase     (					self       :	Optional[Any]			)			->					Any:
										SCREAMING_SNAKE_CASE           =					(self.rust_tokenizer_class, """hf-internal-testing/tiny-random-nllb""", {})
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
																SCREAMING_SNAKE_CASE           =					self.rust_tokenizer_class.from_pretrained(a       ,     **a			)
																SCREAMING_SNAKE_CASE           =					self.tokenizer_class.from_pretrained(a       ,     **a			)

																SCREAMING_SNAKE_CASE           =					tempfile.mkdtemp()

																SCREAMING_SNAKE_CASE           =					tokenizer_r.save_pretrained(a			)
																SCREAMING_SNAKE_CASE           =					tokenizer_p.save_pretrained(a			)

																# Checks it save with the same files + the tokenizer.json file for the fast one
																self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files			)			)
																SCREAMING_SNAKE_CASE           =					tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f			)
																self.assertSequenceEqual(a       ,     a			)

																# Checks everything loads correctly in the same way
																SCREAMING_SNAKE_CASE           =					tokenizer_r.from_pretrained(a			)
																SCREAMING_SNAKE_CASE           =					tokenizer_p.from_pretrained(a			)

																# Check special tokens are set accordingly on Rust and Python
																for key in tokenizer_pp.special_tokens_map:
																			self.assertTrue(hasattr(a       ,     a			)			)

																shutil.rmtree(a			)

																# Save tokenizer rust, legacy_format=True
																SCREAMING_SNAKE_CASE           =					tempfile.mkdtemp()

																SCREAMING_SNAKE_CASE           =					tokenizer_r.save_pretrained(a       ,     legacy_format=a			)
																SCREAMING_SNAKE_CASE           =					tokenizer_p.save_pretrained(a			)

																# Checks it save with the same files
																self.assertSequenceEqual(a       ,     a			)

																# Checks everything loads correctly in the same way
																SCREAMING_SNAKE_CASE           =					tokenizer_r.from_pretrained(a			)
																SCREAMING_SNAKE_CASE           =					tokenizer_p.from_pretrained(a			)

																# Check special tokens are set accordingly on Rust and Python
																for key in tokenizer_pp.special_tokens_map:
																			self.assertTrue(hasattr(a       ,     a			)			)

																shutil.rmtree(a			)

																# Save tokenizer rust, legacy_format=False
																SCREAMING_SNAKE_CASE           =					tempfile.mkdtemp()

																SCREAMING_SNAKE_CASE           =					tokenizer_r.save_pretrained(a       ,     legacy_format=a			)
																SCREAMING_SNAKE_CASE           =					tokenizer_p.save_pretrained(a			)

																# Checks it saved the tokenizer.json file
																self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files			)			)

																# Checks everything loads correctly in the same way
																SCREAMING_SNAKE_CASE           =					tokenizer_r.from_pretrained(a			)
																SCREAMING_SNAKE_CASE           =					tokenizer_p.from_pretrained(a			)

																# Check special tokens are set accordingly on Rust and Python
																for key in tokenizer_pp.special_tokens_map:
																			self.assertTrue(hasattr(a       ,     a			)			)

																shutil.rmtree(a			)





							@require_torch
							def    _UpperCAmelCase     (					self       :	Dict			)			->					List[str]:
										if not self.test_seqaseq:
													return

										SCREAMING_SNAKE_CASE           =					self.get_tokenizers()
										for tokenizer in tokenizers:
													with self.subTest(f"""{tokenizer.__class__.__name__}"""			):
																# Longer text that will definitely require truncation.
																SCREAMING_SNAKE_CASE           =					[
																    """ UN Chief Says There Is No Military Solution in Syria""",
																    """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"""
																    """ Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"""
																    """ will only worsen the violence and misery for millions of people.""",
																]
																SCREAMING_SNAKE_CASE           =					[
																    """Şeful ONU declară că nu există o soluţie militară în Siria""",
																    """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"""
																    """ Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"""
																    """ că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
																]
																try:
																			SCREAMING_SNAKE_CASE           =					tokenizer.prepare_seqaseq_batch(
																			    src_texts=a       ,     tgt_texts=a       ,     max_length=3       ,     max_target_length=10       ,     return_tensors="""pt"""       ,     src_lang="""eng_Latn"""       ,     tgt_lang="""ron_Latn"""       ,     )
																except NotImplementedError:
																			return
																self.assertEqual(batch.input_ids.shape[1]       ,     3			)
																self.assertEqual(batch.labels.shape[1]       ,     10			)
																# max_target_length will default to max_length if not specified
																SCREAMING_SNAKE_CASE           =					tokenizer.prepare_seqaseq_batch(
																    a       ,     tgt_texts=a       ,     max_length=3       ,     return_tensors="""pt"""			)
																self.assertEqual(batch.input_ids.shape[1]       ,     3			)
																self.assertEqual(batch.labels.shape[1]       ,     3			)

																SCREAMING_SNAKE_CASE           =					tokenizer.prepare_seqaseq_batch(
																    src_texts=a       ,     max_length=3       ,     max_target_length=10       ,     return_tensors="""pt"""			)
																self.assertEqual(batch_encoder_only.input_ids.shape[1]       ,     3			)
																self.assertEqual(batch_encoder_only.attention_mask.shape[1]       ,     3			)
																self.assertNotIn("""decoder_input_ids"""       ,     a			)





							@unittest.skip("""Unfortunately way too slow to build a BPE with SentencePiece."""			)
							def    _UpperCAmelCase     (					self       :	int			)			->					Optional[int]:
										pass







							def    _UpperCAmelCase     (					self       :	Optional[int]			)			->					Optional[int]:
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
																SCREAMING_SNAKE_CASE           =					[AddedToken("""<special>"""       ,     lstrip=a			)]

																SCREAMING_SNAKE_CASE           =					self.rust_tokenizer_class.from_pretrained(
																    a       ,     additional_special_tokens=a       ,     **a			)
																SCREAMING_SNAKE_CASE           =					tokenizer_r.encode("""Hey this is a <special> token"""			)

																SCREAMING_SNAKE_CASE           =					tokenizer_r.encode("""<special>"""       ,     add_special_tokens=a			)[0]

																self.assertTrue(special_token_id in r_output			)

																if self.test_slow_tokenizer:
																			SCREAMING_SNAKE_CASE           =					self.rust_tokenizer_class.from_pretrained(
																			    a       ,     additional_special_tokens=a       ,     **a       ,     )
																			SCREAMING_SNAKE_CASE           =					self.tokenizer_class.from_pretrained(
																			    a       ,     additional_special_tokens=a       ,     **a			)

																			SCREAMING_SNAKE_CASE           =					tokenizer_p.encode("""Hey this is a <special> token"""			)

																			SCREAMING_SNAKE_CASE           =					tokenizer_cr.encode("""Hey this is a <special> token"""			)

																			self.assertEqual(a       ,     a			)
																			self.assertEqual(a       ,     a			)
																			self.assertTrue(special_token_id in p_output			)
																			self.assertTrue(special_token_id in cr_output			)





@require_torch
@require_sentencepiece
@require_tokenizers
class 			UpperCAmelCase_      (   unittest.TestCase       ):




							'''simple docstring'''



							a__						  =     '''facebook/nllb-200-distilled-600M'''
							a__						  =     [
							    ''' UN Chief Says There Is No Military Solution in Syria''',
							    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
							]
							a__						  =     [
							    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
							    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
							    ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
							    ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
							]
							a__						  =     [
							    2_5_6_0_4_7,
							    1_6_2_9_7,
							    1_3_4_4_0_8,
							    8_1_6_5,
							    2_4_8_0_6_6,
							    1_4_7_3_4,
							    9_5_0,
							    1_1_3_5,
							    1_0_5_7_2_1,
							    3_5_7_3,
							    8_3,
							    2_7_3_5_2,
							    1_0_8,
							    4_9_4_8_6,
							    2,
							]





							@classmethod
							def    _UpperCAmelCase     (					cls       :	List[Any]			)			->					Optional[Any]:
										SCREAMING_SNAKE_CASE           =					NllbTokenizer.from_pretrained(
										    cls.checkpoint_name       ,     src_lang="""eng_Latn"""       ,     tgt_lang="""ron_Latn"""			)
										SCREAMING_SNAKE_CASE           =					1
										return cls





							def    _UpperCAmelCase     (					self       :	Dict			)			->					Dict:
										self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Arab"""]       ,     256_001			)
										self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Latn"""]       ,     256_002			)
										self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""fra_Latn"""]       ,     256_057			)





							def    _UpperCAmelCase     (					self       :	Optional[int]			)			->					Optional[Any]:
										SCREAMING_SNAKE_CASE           =					self.tokenizer.batch_encode_plus(self.src_text			).input_ids[0]
										self.assertListEqual(self.expected_src_tokens       ,     a			)





							def    _UpperCAmelCase     (					self       :	Union[str, Any]			)			->					Tuple:
										self.assertIn(a       ,     self.tokenizer.all_special_ids			)
										# fmt: off
										SCREAMING_SNAKE_CASE           =					[RO_CODE, 4_254, 98_068, 112_923, 39_072, 3_909, 713, 102_767, 26, 17_314, 35_642, 14_683, 33_118, 2_022, 66_987, 2, 256_047]
										# fmt: on

										SCREAMING_SNAKE_CASE           =					self.tokenizer.decode(a       ,     skip_special_tokens=a			)
										SCREAMING_SNAKE_CASE           =					self.tokenizer.decode(generated_ids[1:]       ,     skip_special_tokens=a			)
										self.assertEqual(a       ,     a			)
										self.assertNotIn(self.tokenizer.eos_token       ,     a			)





							def    _UpperCAmelCase     (					self       :	Optional[Any]			)			->					List[Any]:
										SCREAMING_SNAKE_CASE           =					["""this is gunna be a long sentence """ * 20]
										assert isinstance(src_text[0]       ,     a			)
										SCREAMING_SNAKE_CASE           =					10
										SCREAMING_SNAKE_CASE           =					self.tokenizer(a       ,     max_length=a       ,     truncation=a			).input_ids[0]
										self.assertEqual(ids[-1]       ,     2			)
										self.assertEqual(ids[0]       ,     a			)
										self.assertEqual(len(a			)       ,     a			)





							def    _UpperCAmelCase     (					self       :	Dict			)			->					Any:
										self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]			)       ,     [256_203, 3]			)





							def    _UpperCAmelCase     (					self       :	str			)			->					Optional[int]:
										SCREAMING_SNAKE_CASE           =					tempfile.mkdtemp()
										SCREAMING_SNAKE_CASE           =					self.tokenizer.fairseq_tokens_to_ids
										self.tokenizer.save_pretrained(a			)
										SCREAMING_SNAKE_CASE           =					NllbTokenizer.from_pretrained(a			)
										self.assertDictEqual(new_tok.fairseq_tokens_to_ids       ,     a			)





							@require_torch
							def    _UpperCAmelCase     (					self       :	Optional[int]			)			->					Union[str, Any]:
										SCREAMING_SNAKE_CASE           =					self.tokenizer(
										    self.src_text       ,     text_target=self.tgt_text       ,     padding=a       ,     truncation=a       ,     max_length=len(self.expected_src_tokens			)       ,     return_tensors="""pt"""       ,     )
										SCREAMING_SNAKE_CASE           =					shift_tokens_right(
										    batch["""labels"""]       ,     self.tokenizer.pad_token_id       ,     self.tokenizer.lang_code_to_id["""ron_Latn"""]			)

										self.assertIsInstance(a       ,     a			)

										self.assertEqual((2, 15)       ,     batch.input_ids.shape			)
										self.assertEqual((2, 15)       ,     batch.attention_mask.shape			)
										SCREAMING_SNAKE_CASE           =					batch.input_ids.tolist()[0]
										self.assertListEqual(self.expected_src_tokens       ,     a			)
										self.assertEqual(a       ,     batch.decoder_input_ids[0, 0]			)  # EOS
										# Test that special tokens are reset
										self.assertEqual(self.tokenizer.prefix_tokens       ,     [EN_CODE]			)
										self.assertEqual(self.tokenizer.suffix_tokens       ,     [self.tokenizer.eos_token_id]			)





							def    _UpperCAmelCase     (					self       :	str			)			->					Dict:
										SCREAMING_SNAKE_CASE           =					self.tokenizer(self.src_text       ,     padding=a       ,     truncation=a       ,     max_length=3       ,     return_tensors="""pt"""			)
										SCREAMING_SNAKE_CASE           =					self.tokenizer(
										    text_target=self.tgt_text       ,     padding=a       ,     truncation=a       ,     max_length=10       ,     return_tensors="""pt"""			)
										SCREAMING_SNAKE_CASE           =					targets["""input_ids"""]
										SCREAMING_SNAKE_CASE           =					shift_tokens_right(
										    a       ,     self.tokenizer.pad_token_id       ,     decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]       ,     )

										self.assertEqual(batch.input_ids.shape[1]       ,     3			)
										self.assertEqual(batch.decoder_input_ids.shape[1]       ,     10			)





							@require_torch
							def    _UpperCAmelCase     (					self       :	int			)			->					Optional[int]:
										SCREAMING_SNAKE_CASE           =					self.tokenizer._build_translation_inputs(
										    """A test"""       ,     return_tensors="""pt"""       ,     src_lang="""eng_Latn"""       ,     tgt_lang="""fra_Latn"""			)

										self.assertEqual(
										    nested_simplify(a			)       ,     {
										        # A, test, EOS, en_XX
										        """input_ids""": [[256_047, 70, 7_356, 2]],
										        """attention_mask""": [[1, 1, 1, 1]],
										        # ar_AR
										        """forced_bos_token_id""": 256_057,
										    }       ,     )







							@require_torch
							def    _UpperCAmelCase     (					self       :	str			)			->					int:
										SCREAMING_SNAKE_CASE           =					True
										SCREAMING_SNAKE_CASE           =					self.tokenizer(
										    """UN Chief says there is no military solution in Syria"""       ,     src_lang="""eng_Latn"""       ,     tgt_lang="""fra_Latn"""			)
										self.assertEqual(
										    inputs.input_ids       ,     [16_297, 134_408, 25_653, 6_370, 248, 254, 103_929, 94_995, 108, 49_486, 2, 256_047]			)

										SCREAMING_SNAKE_CASE           =					False
										SCREAMING_SNAKE_CASE           =					self.tokenizer(
										    """UN Chief says there is no military solution in Syria"""       ,     src_lang="""eng_Latn"""       ,     tgt_lang="""fra_Latn"""			)
										self.assertEqual(
										    inputs.input_ids       ,     [256_047, 16_297, 134_408, 25_653, 6_370, 248, 254, 103_929, 94_995, 108, 49_486, 2]			)






