


'''simple docstring'''


import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase__    	=     get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
						from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

lowerCamelCase__    	=     256_047
lowerCamelCase__    	=     256_145






@require_sentencepiece
@require_tokenizers
class 	lowerCAmelCase__			(			UpperCAmelCase__    ,       unittest.TestCase	):
			lowerCAmelCase :	Optional[Any]          =     NllbTokenizer
			lowerCAmelCase :	Tuple          =     NllbTokenizerFast
			lowerCAmelCase :	str          =     True
			lowerCAmelCase :	str          =     True
			lowerCAmelCase :	int          =     {}




			def 							lowerCAmelCase__    (		self  :     Union[str, Any] )   ->Optional[int]:



					'''simple docstring'''







					super().setUp()

					# We have a SentencePiece fixture for testing
					_UpperCAmelCase		:       Union[str, Any]								=      NllbTokenizer(lowerCamelCase__	,  keep_accents=lowerCamelCase__ )
					tokenizer.save_pretrained(self.tmpdirname )




			def 							lowerCAmelCase__    (		self  :     Optional[int] )   ->Optional[int]:



					'''simple docstring'''







					_UpperCAmelCase		:       Dict								=      NllbTokenizer(lowerCamelCase__	,  keep_accents=lowerCamelCase__ )

					_UpperCAmelCase		:       Tuple								=      tokenizer.tokenize("This is a test" )
					self.assertListEqual(lowerCamelCase__	,  ["▁This", "▁is", "▁a", "▁t", "est"] )

					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(lowerCamelCase__ )	,  [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]	,  )

					_UpperCAmelCase		:       str								=      tokenizer.tokenize("I was born in 92000, and this is falsé." )
					self.assertListEqual(
					    lowerCamelCase__	,  [
					        SPIECE_UNDERLINE + "I",
					        SPIECE_UNDERLINE + "was",
					        SPIECE_UNDERLINE + "b",
					        "or",
					        "n",
					        SPIECE_UNDERLINE + "in",
					        SPIECE_UNDERLINE + "",
					        "9",
					        "2",
					        "0",
					        "0",
					        "0",
					        ",",
					        SPIECE_UNDERLINE + "and",
					        SPIECE_UNDERLINE + "this",
					        SPIECE_UNDERLINE + "is",
					        SPIECE_UNDERLINE + "f",
					        "al",
					        "s",
					        "é",
					        ".",
					    ]	,  )
					_UpperCAmelCase		:       Any								=      tokenizer.convert_tokens_to_ids(lowerCamelCase__ )
					self.assertListEqual(
					    lowerCamelCase__	,  [
					        value + tokenizer.fairseq_offset
					        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
					    ]	,  )

					_UpperCAmelCase		:       Optional[Any]								=      tokenizer.convert_ids_to_tokens(lowerCamelCase__ )
					self.assertListEqual(
					    lowerCamelCase__	,  [
					        SPIECE_UNDERLINE + "I",
					        SPIECE_UNDERLINE + "was",
					        SPIECE_UNDERLINE + "b",
					        "or",
					        "n",
					        SPIECE_UNDERLINE + "in",
					        SPIECE_UNDERLINE + "",
					        "<unk>",
					        "2",
					        "0",
					        "0",
					        "0",
					        ",",
					        SPIECE_UNDERLINE + "and",
					        SPIECE_UNDERLINE + "this",
					        SPIECE_UNDERLINE + "is",
					        SPIECE_UNDERLINE + "f",
					        "al",
					        "s",
					        "<unk>",
					        ".",
					    ]	,  )




			def 							lowerCAmelCase__    (		self  :     Any )   ->Tuple:



					'''simple docstring'''







					_UpperCAmelCase		:       Dict								=      (self.rust_tokenizer_class, "hf-internal-testing/tiny-random-nllb", {})
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
									_UpperCAmelCase		:       Any								=      self.rust_tokenizer_class.from_pretrained(lowerCamelCase__	,  **lowerCamelCase__ )
									_UpperCAmelCase		:       Any								=      self.tokenizer_class.from_pretrained(lowerCamelCase__	,  **lowerCamelCase__ )

									_UpperCAmelCase		:       int								=      tempfile.mkdtemp()

									_UpperCAmelCase		:       Tuple								=      tokenizer_r.save_pretrained(lowerCamelCase__ )
									_UpperCAmelCase		:       Any								=      tokenizer_p.save_pretrained(lowerCamelCase__ )

									# Checks it save with the same files + the tokenizer.json file for the fast one
									self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files ) )
									_UpperCAmelCase		:       List[Any]								=      tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f )
									self.assertSequenceEqual(lowerCamelCase__	,  lowerCamelCase__ )

									# Checks everything loads correctly in the same way
									_UpperCAmelCase		:       Tuple								=      tokenizer_r.from_pretrained(lowerCamelCase__ )
									_UpperCAmelCase		:       Optional[Any]								=      tokenizer_p.from_pretrained(lowerCamelCase__ )

									# Check special tokens are set accordingly on Rust and Python
									for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(lowerCamelCase__	,  lowerCamelCase__ ) )

									shutil.rmtree(lowerCamelCase__ )

									# Save tokenizer rust, legacy_format=True
									_UpperCAmelCase		:       Optional[Any]								=      tempfile.mkdtemp()

									_UpperCAmelCase		:       List[str]								=      tokenizer_r.save_pretrained(lowerCamelCase__	,  legacy_format=lowerCamelCase__ )
									_UpperCAmelCase		:       List[str]								=      tokenizer_p.save_pretrained(lowerCamelCase__ )

									# Checks it save with the same files
									self.assertSequenceEqual(lowerCamelCase__	,  lowerCamelCase__ )

									# Checks everything loads correctly in the same way
									_UpperCAmelCase		:       Union[str, Any]								=      tokenizer_r.from_pretrained(lowerCamelCase__ )
									_UpperCAmelCase		:       Optional[Any]								=      tokenizer_p.from_pretrained(lowerCamelCase__ )

									# Check special tokens are set accordingly on Rust and Python
									for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(lowerCamelCase__	,  lowerCamelCase__ ) )

									shutil.rmtree(lowerCamelCase__ )

									# Save tokenizer rust, legacy_format=False
									_UpperCAmelCase		:       Optional[int]								=      tempfile.mkdtemp()

									_UpperCAmelCase		:       Dict								=      tokenizer_r.save_pretrained(lowerCamelCase__	,  legacy_format=lowerCamelCase__ )
									_UpperCAmelCase		:       List[Any]								=      tokenizer_p.save_pretrained(lowerCamelCase__ )

									# Checks it saved the tokenizer.json file
									self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files ) )

									# Checks everything loads correctly in the same way
									_UpperCAmelCase		:       int								=      tokenizer_r.from_pretrained(lowerCamelCase__ )
									_UpperCAmelCase		:       int								=      tokenizer_p.from_pretrained(lowerCamelCase__ )

									# Check special tokens are set accordingly on Rust and Python
									for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(lowerCamelCase__	,  lowerCamelCase__ ) )

									shutil.rmtree(lowerCamelCase__ )




			@require_torch
			def 							lowerCAmelCase__    (		self  :     Optional[int] )   ->List[str]:



					'''simple docstring'''







					if not self.test_seqaseq:
							return

					_UpperCAmelCase		:       int								=      self.get_tokenizers()
					for tokenizer in tokenizers:
							with self.subTest(F"""{tokenizer.__class__.__name__}""" ):
									# Longer text that will definitely require truncation.
									_UpperCAmelCase		:       Tuple								=      [
									    " UN Chief Says There Is No Military Solution in Syria",
									    " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"
									    " Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"
									    " will only worsen the violence and misery for millions of people.",
									]
									_UpperCAmelCase		:       Dict								=      [
									    "Şeful ONU declară că nu există o soluţie militară în Siria",
									    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"
									    " Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"
									    " că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
									]
									try:
											_UpperCAmelCase		:       str								=      tokenizer.prepare_seqaseq_batch(
											    src_texts=lowerCamelCase__	,  tgt_texts=lowerCamelCase__	,  max_length=3	,  max_target_length=10	,  return_tensors="pt"	,  src_lang="eng_Latn"	,  tgt_lang="ron_Latn"	,  )
									except NotImplementedError:
											return
									self.assertEqual(batch.input_ids.shape[1]	,  3 )
									self.assertEqual(batch.labels.shape[1]	,  10 )
									# max_target_length will default to max_length if not specified
									_UpperCAmelCase		:       Union[str, Any]								=      tokenizer.prepare_seqaseq_batch(
									    lowerCamelCase__	,  tgt_texts=lowerCamelCase__	,  max_length=3	,  return_tensors="pt" )
									self.assertEqual(batch.input_ids.shape[1]	,  3 )
									self.assertEqual(batch.labels.shape[1]	,  3 )

									_UpperCAmelCase		:       Union[str, Any]								=      tokenizer.prepare_seqaseq_batch(
									    src_texts=lowerCamelCase__	,  max_length=3	,  max_target_length=10	,  return_tensors="pt" )
									self.assertEqual(batch_encoder_only.input_ids.shape[1]	,  3 )
									self.assertEqual(batch_encoder_only.attention_mask.shape[1]	,  3 )
									self.assertNotIn("decoder_input_ids"	,  lowerCamelCase__ )




			@unittest.skip("Unfortunately way too slow to build a BPE with SentencePiece." )
			def 							lowerCAmelCase__    (		self  :     Any )   ->Tuple:



					'''simple docstring'''







					pass






			def 							lowerCAmelCase__    (		self  :     List[str] )   ->str:



					'''simple docstring'''







					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
									_UpperCAmelCase		:       str								=      [AddedToken("<special>"	,  lstrip=lowerCamelCase__ )]

									_UpperCAmelCase		:       int								=      self.rust_tokenizer_class.from_pretrained(
									    lowerCamelCase__	,  additional_special_tokens=lowerCamelCase__	,  **lowerCamelCase__ )
									_UpperCAmelCase		:       str								=      tokenizer_r.encode("Hey this is a <special> token" )

									_UpperCAmelCase		:       str								=      tokenizer_r.encode("<special>"	,  add_special_tokens=lowerCamelCase__ )[0]

									self.assertTrue(special_token_id in r_output )

									if self.test_slow_tokenizer:
											_UpperCAmelCase		:       Optional[int]								=      self.rust_tokenizer_class.from_pretrained(
											    lowerCamelCase__	,  additional_special_tokens=lowerCamelCase__	,  **lowerCamelCase__	,  )
											_UpperCAmelCase		:       Optional[int]								=      self.tokenizer_class.from_pretrained(
											    lowerCamelCase__	,  additional_special_tokens=lowerCamelCase__	,  **lowerCamelCase__ )

											_UpperCAmelCase		:       Tuple								=      tokenizer_p.encode("Hey this is a <special> token" )

											_UpperCAmelCase		:       Dict								=      tokenizer_cr.encode("Hey this is a <special> token" )

											self.assertEqual(lowerCamelCase__	,  lowerCamelCase__ )
											self.assertEqual(lowerCamelCase__	,  lowerCamelCase__ )
											self.assertTrue(special_token_id in p_output )
											self.assertTrue(special_token_id in cr_output )






@require_torch
@require_sentencepiece
@require_tokenizers
class 	lowerCAmelCase__			(			unittest.TestCase	):
			lowerCAmelCase :	Union[str, Any]          =     "facebook/nllb-200-distilled-600M"
			lowerCAmelCase :	Dict          =     [
			    " UN Chief Says There Is No Military Solution in Syria",
			    " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
			]
			lowerCAmelCase :	Optional[int]          =     [
			    "Şeful ONU declară că nu există o soluţie militară în Siria",
			    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
			    " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
			    " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
			]
			lowerCAmelCase :	Dict          =     [
			    256_047,
			    16_297,
			    134_408,
			    8_165,
			    248_066,
			    14_734,
			    950,
			    1_135,
			    105_721,
			    3_573,
			    83,
			    27_352,
			    108,
			    49_486,
			    2,
			]




			@classmethod
			def 							lowerCAmelCase__    (		cls  :     Tuple )   ->Union[str, Any]:



					'''simple docstring'''







					_UpperCAmelCase		:       NllbTokenizer								=      NllbTokenizer.from_pretrained(
					    cls.checkpoint_name	,  src_lang="eng_Latn"	,  tgt_lang="ron_Latn" )
					_UpperCAmelCase		:       str								=      1
					return cls




			def 							lowerCAmelCase__    (		self  :     Dict )   ->Optional[Any]:



					'''simple docstring'''







					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Arab"]	,  25_60_01 )
					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Latn"]	,  25_60_02 )
					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["fra_Latn"]	,  25_60_57 )




			def 							lowerCAmelCase__    (		self  :     int )   ->Optional[int]:



					'''simple docstring'''







					_UpperCAmelCase		:       Optional[int]								=      self.tokenizer.batch_encode_plus(self.src_text ).input_ids[0]
					self.assertListEqual(self.expected_src_tokens	,  lowerCamelCase__ )




			def 							lowerCAmelCase__    (		self  :     List[Any] )   ->List[str]:



					'''simple docstring'''







					self.assertIn(lowerCamelCase__	,  self.tokenizer.all_special_ids )
					# fmt: off
					_UpperCAmelCase		:       List[Any]								=      [RO_CODE, 42_54, 9_80_68, 11_29_23, 3_90_72, 39_09, 7_13, 10_27_67, 26, 1_73_14, 3_56_42, 1_46_83, 3_31_18, 20_22, 6_69_87, 2, 25_60_47]
					# fmt: on

					_UpperCAmelCase		:       Optional[Any]								=      self.tokenizer.decode(lowerCamelCase__	,  skip_special_tokens=lowerCamelCase__ )
					_UpperCAmelCase		:       Dict								=      self.tokenizer.decode(generated_ids[1:]	,  skip_special_tokens=lowerCamelCase__ )
					self.assertEqual(lowerCamelCase__	,  lowerCamelCase__ )
					self.assertNotIn(self.tokenizer.eos_token	,  lowerCamelCase__ )




			def 							lowerCAmelCase__    (		self  :     Dict )   ->Any:



					'''simple docstring'''







					_UpperCAmelCase		:       Optional[int]								=      ["this is gunna be a long sentence " * 20]
					assert isinstance(src_text[0]	,  lowerCamelCase__ )
					_UpperCAmelCase		:       Tuple								=      10
					_UpperCAmelCase		:       Union[str, Any]								=      self.tokenizer(lowerCamelCase__	,  max_length=lowerCamelCase__	,  truncation=lowerCamelCase__ ).input_ids[0]
					self.assertEqual(ids[-1]	,  2 )
					self.assertEqual(ids[0]	,  lowerCamelCase__ )
					self.assertEqual(len(lowerCamelCase__ )	,  lowerCamelCase__ )




			def 							lowerCAmelCase__    (		self  :     Dict )   ->Dict:



					'''simple docstring'''







					self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "ar_AR"] )	,  [25_62_03, 3] )




			def 							lowerCAmelCase__    (		self  :     Tuple )   ->int:



					'''simple docstring'''







					_UpperCAmelCase		:       Optional[int]								=      tempfile.mkdtemp()
					_UpperCAmelCase		:       Any								=      self.tokenizer.fairseq_tokens_to_ids
					self.tokenizer.save_pretrained(lowerCamelCase__ )
					_UpperCAmelCase		:       int								=      NllbTokenizer.from_pretrained(lowerCamelCase__ )
					self.assertDictEqual(new_tok.fairseq_tokens_to_ids	,  lowerCamelCase__ )




			@require_torch
			def 							lowerCAmelCase__    (		self  :     List[Any] )   ->Union[str, Any]:



					'''simple docstring'''







					_UpperCAmelCase		:       List[Any]								=      self.tokenizer(
					    self.src_text	,  text_target=self.tgt_text	,  padding=lowerCamelCase__	,  truncation=lowerCamelCase__	,  max_length=len(self.expected_src_tokens )	,  return_tensors="pt"	,  )
					_UpperCAmelCase		:       Optional[int]								=      shift_tokens_right(
					    batch["labels"]	,  self.tokenizer.pad_token_id	,  self.tokenizer.lang_code_to_id["ron_Latn"] )

					self.assertIsInstance(lowerCamelCase__	,  lowerCamelCase__ )

					self.assertEqual((2, 15)	,  batch.input_ids.shape )
					self.assertEqual((2, 15)	,  batch.attention_mask.shape )
					_UpperCAmelCase		:       str								=      batch.input_ids.tolist()[0]
					self.assertListEqual(self.expected_src_tokens	,  lowerCamelCase__ )
					self.assertEqual(lowerCamelCase__	,  batch.decoder_input_ids[0, 0] )  # EOS
					# Test that special tokens are reset
					self.assertEqual(self.tokenizer.prefix_tokens	,  [EN_CODE] )
					self.assertEqual(self.tokenizer.suffix_tokens	,  [self.tokenizer.eos_token_id] )




			def 							lowerCAmelCase__    (		self  :     List[Any] )   ->Optional[int]:



					'''simple docstring'''







					_UpperCAmelCase		:       Optional[Any]								=      self.tokenizer(self.src_text	,  padding=lowerCamelCase__	,  truncation=lowerCamelCase__	,  max_length=3	,  return_tensors="pt" )
					_UpperCAmelCase		:       List[Any]								=      self.tokenizer(
					    text_target=self.tgt_text	,  padding=lowerCamelCase__	,  truncation=lowerCamelCase__	,  max_length=10	,  return_tensors="pt" )
					_UpperCAmelCase		:       List[Any]								=      targets["input_ids"]
					_UpperCAmelCase		:       Tuple								=      shift_tokens_right(
					    lowerCamelCase__	,  self.tokenizer.pad_token_id	,  decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]	,  )

					self.assertEqual(batch.input_ids.shape[1]	,  3 )
					self.assertEqual(batch.decoder_input_ids.shape[1]	,  10 )




			@require_torch
			def 							lowerCAmelCase__    (		self  :     List[Any] )   ->Any:



					'''simple docstring'''







					_UpperCAmelCase		:       Optional[Any]								=      self.tokenizer._build_translation_inputs(
					    "A test"	,  return_tensors="pt"	,  src_lang="eng_Latn"	,  tgt_lang="fra_Latn" )

					self.assertEqual(
					    nested_simplify(lowerCamelCase__ )	,  {
					        # A, test, EOS, en_XX
					        "input_ids": [[25_60_47, 70, 73_56, 2]],
					        "attention_mask": [[1, 1, 1, 1]],
					        # ar_AR
					        "forced_bos_token_id": 25_60_57,
					    }	,  )






			@require_torch
			def 							lowerCAmelCase__    (		self  :     Optional[Any] )   ->Dict:



					'''simple docstring'''







					_UpperCAmelCase		:       Union[str, Any]								=      True
					_UpperCAmelCase		:       Optional[Any]								=      self.tokenizer(
					    "UN Chief says there is no military solution in Syria"	,  src_lang="eng_Latn"	,  tgt_lang="fra_Latn" )
					self.assertEqual(
					    inputs.input_ids	,  [1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2, 25_60_47] )

					_UpperCAmelCase		:       Union[str, Any]								=      False
					_UpperCAmelCase		:       Dict								=      self.tokenizer(
					    "UN Chief says there is no military solution in Syria"	,  src_lang="eng_Latn"	,  tgt_lang="fra_Latn" )
					self.assertEqual(
					    inputs.input_ids	,  [25_60_47, 1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2] )







'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowerCamelCase__    	=     {
    'configuration_instructblip': [
        'INSTRUCTBLIP_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'InstructBlipConfig',
        'InstructBlipQFormerConfig',
        'InstructBlipVisionConfig',
    ],
    'processing_instructblip': ['InstructBlipProcessor'],
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						lowerCamelCase__    	=     [
						    'INSTRUCTBLIP_PRETRAINED_MODEL_ARCHIVE_LIST',
						    'InstructBlipQFormerModel',
						    'InstructBlipPreTrainedModel',
						    'InstructBlipForConditionalGeneration',
						    'InstructBlipVisionModel',
						]

if TYPE_CHECKING:
						from .configuration_instructblip import (
						    INSTRUCTBLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
						    InstructBlipConfig,
						    InstructBlipQFormerConfig,
						    InstructBlipVisionConfig,
						)
						from .processing_instructblip import InstructBlipProcessor

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_instructblip import (
												    INSTRUCTBLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
												    InstructBlipForConditionalGeneration,
												    InstructBlipPreTrainedModel,
												    InstructBlipQFormerModel,
												    InstructBlipVisionModel,
												)

else:
						import sys

						lowerCamelCase__    	=     _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)



