
'''simple docstring'''
import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


A__   :		Tuple 		=get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
     from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

A__   :		Optional[Any] 		=256_047
A__   :		Dict 		=256_145







@require_sentencepiece
@require_tokenizers
class  __A				( _SCREAMING_SNAKE_CASE    ,						unittest.TestCase			):
 lowerCamelCase				       =NllbTokenizer
 lowerCamelCase				       =NllbTokenizerFast
 lowerCamelCase				       =True
 lowerCamelCase				       =True
 lowerCamelCase				       ={}
 def 		lowercase_(   self			:    Optional[Any]       ):



   """simple docstring"""





   super().setUp()

   # We have a SentencePiece fixture for testing
   __A		:     List[Any]												=		NllbTokenizer(lowerCamelCase				,   keep_accents=lowerCamelCase       )
   tokenizer.save_pretrained(self.tmpdirname       )
 def 		lowercase_(   self			:    Any       ):



   """simple docstring"""





   __A		:     Union[str, Any]												=		NllbTokenizer(lowerCamelCase				,   keep_accents=lowerCamelCase       )

   __A		:     str												=		tokenizer.tokenize("""This is a test"""       )
   self.assertListEqual(lowerCamelCase				,   ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]       )

   self.assertListEqual(
       tokenizer.convert_tokens_to_ids(lowerCamelCase       )				,   [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]				,   )

   __A		:     Tuple												=		tokenizer.tokenize("""I was born in 92000, and this is falsé."""       )
   self.assertListEqual(
       lowerCamelCase				,   [
           SPIECE_UNDERLINE + """I""",
           SPIECE_UNDERLINE + """was""",
           SPIECE_UNDERLINE + """b""",
           """or""",
           """n""",
           SPIECE_UNDERLINE + """in""",
           SPIECE_UNDERLINE + """""",
           """9""",
           """2""",
           """0""",
           """0""",
           """0""",
           """,""",
           SPIECE_UNDERLINE + """and""",
           SPIECE_UNDERLINE + """this""",
           SPIECE_UNDERLINE + """is""",
           SPIECE_UNDERLINE + """f""",
           """al""",
           """s""",
           """é""",
           """.""",
       ]				,   )
   __A		:     Dict												=		tokenizer.convert_tokens_to_ids(lowerCamelCase       )
   self.assertListEqual(
       lowerCamelCase				,   [
           value + tokenizer.fairseq_offset
           for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
       ]				,   )

   __A		:     Tuple												=		tokenizer.convert_ids_to_tokens(lowerCamelCase       )
   self.assertListEqual(
       lowerCamelCase				,   [
           SPIECE_UNDERLINE + """I""",
           SPIECE_UNDERLINE + """was""",
           SPIECE_UNDERLINE + """b""",
           """or""",
           """n""",
           SPIECE_UNDERLINE + """in""",
           SPIECE_UNDERLINE + """""",
           """<unk>""",
           """2""",
           """0""",
           """0""",
           """0""",
           """,""",
           SPIECE_UNDERLINE + """and""",
           SPIECE_UNDERLINE + """this""",
           SPIECE_UNDERLINE + """is""",
           SPIECE_UNDERLINE + """f""",
           """al""",
           """s""",
           """<unk>""",
           """.""",
       ]				,   )
 def 		lowercase_(   self			:    Optional[Any]       ):



   """simple docstring"""





   __A		:     Optional[int]												=		(self.rust_tokenizer_class, """hf-internal-testing/tiny-random-nllb""", {})
   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
       __A		:     Tuple												=		self.rust_tokenizer_class.from_pretrained(lowerCamelCase				,   **lowerCamelCase       )
       __A		:     Any												=		self.tokenizer_class.from_pretrained(lowerCamelCase				,   **lowerCamelCase       )

       __A		:     str												=		tempfile.mkdtemp()

       __A		:     Optional[Any]												=		tokenizer_r.save_pretrained(lowerCamelCase       )
       __A		:     List[Any]												=		tokenizer_p.save_pretrained(lowerCamelCase       )

       # Checks it save with the same files + the tokenizer.json file for the fast one
       self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files       )       )
       __A		:     Dict												=		tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f       )
       self.assertSequenceEqual(lowerCamelCase				,   lowerCamelCase       )

       # Checks everything loads correctly in the same way
       __A		:     Optional[int]												=		tokenizer_r.from_pretrained(lowerCamelCase       )
       __A		:     int												=		tokenizer_p.from_pretrained(lowerCamelCase       )

       # Check special tokens are set accordingly on Rust and Python
       for key in tokenizer_pp.special_tokens_map:
         self.assertTrue(hasattr(lowerCamelCase				,   lowerCamelCase       )       )

       shutil.rmtree(lowerCamelCase       )

       # Save tokenizer rust, legacy_format=True
       __A		:     Optional[Any]												=		tempfile.mkdtemp()

       __A		:     List[Any]												=		tokenizer_r.save_pretrained(lowerCamelCase				,   legacy_format=lowerCamelCase       )
       __A		:     Any												=		tokenizer_p.save_pretrained(lowerCamelCase       )

       # Checks it save with the same files
       self.assertSequenceEqual(lowerCamelCase				,   lowerCamelCase       )

       # Checks everything loads correctly in the same way
       __A		:     Dict												=		tokenizer_r.from_pretrained(lowerCamelCase       )
       __A		:     Optional[int]												=		tokenizer_p.from_pretrained(lowerCamelCase       )

       # Check special tokens are set accordingly on Rust and Python
       for key in tokenizer_pp.special_tokens_map:
         self.assertTrue(hasattr(lowerCamelCase				,   lowerCamelCase       )       )

       shutil.rmtree(lowerCamelCase       )

       # Save tokenizer rust, legacy_format=False
       __A		:     Optional[Any]												=		tempfile.mkdtemp()

       __A		:     Any												=		tokenizer_r.save_pretrained(lowerCamelCase				,   legacy_format=lowerCamelCase       )
       __A		:     Dict												=		tokenizer_p.save_pretrained(lowerCamelCase       )

       # Checks it saved the tokenizer.json file
       self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files       )       )

       # Checks everything loads correctly in the same way
       __A		:     int												=		tokenizer_r.from_pretrained(lowerCamelCase       )
       __A		:     Optional[Any]												=		tokenizer_p.from_pretrained(lowerCamelCase       )

       # Check special tokens are set accordingly on Rust and Python
       for key in tokenizer_pp.special_tokens_map:
         self.assertTrue(hasattr(lowerCamelCase				,   lowerCamelCase       )       )

       shutil.rmtree(lowerCamelCase       )
 @require_torch
 def 		lowercase_(   self			:    Dict       ):



   """simple docstring"""





   if not self.test_seqaseq:
     return

   __A		:     Optional[int]												=		self.get_tokenizers()
   for tokenizer in tokenizers:
     with self.subTest(f"{tokenizer.__class__.__name__}"       ):
       # Longer text that will definitely require truncation.
       __A		:     int												=		[
           """ UN Chief Says There Is No Military Solution in Syria""",
           """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"""
           """ Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"""
           """ will only worsen the violence and misery for millions of people.""",
       ]
       __A		:     int												=		[
           """Şeful ONU declară că nu există o soluţie militară în Siria""",
           """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"""
           """ Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"""
           """ că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
       ]
       try:
         __A		:     Optional[int]												=		tokenizer.prepare_seqaseq_batch(
             src_texts=lowerCamelCase				,   tgt_texts=lowerCamelCase				,   max_length=3				,   max_target_length=10				,   return_tensors="""pt"""				,   src_lang="""eng_Latn"""				,   tgt_lang="""ron_Latn"""				,   )
       except NotImplementedError:
         return
       self.assertEqual(batch.input_ids.shape[1]				,   3       )
       self.assertEqual(batch.labels.shape[1]				,   10       )
       # max_target_length will default to max_length if not specified
       __A		:     List[Any]												=		tokenizer.prepare_seqaseq_batch(
           lowerCamelCase				,   tgt_texts=lowerCamelCase				,   max_length=3				,   return_tensors="""pt"""       )
       self.assertEqual(batch.input_ids.shape[1]				,   3       )
       self.assertEqual(batch.labels.shape[1]				,   3       )

       __A		:     Optional[int]												=		tokenizer.prepare_seqaseq_batch(
           src_texts=lowerCamelCase				,   max_length=3				,   max_target_length=10				,   return_tensors="""pt"""       )
       self.assertEqual(batch_encoder_only.input_ids.shape[1]				,   3       )
       self.assertEqual(batch_encoder_only.attention_mask.shape[1]				,   3       )
       self.assertNotIn("""decoder_input_ids"""				,   lowerCamelCase       )
 @unittest.skip("""Unfortunately way too slow to build a BPE with SentencePiece."""       )
 def 		lowercase_(   self			:    Any       ):



   """simple docstring"""





   pass






 def 		lowercase_(   self			:    List[Any]       ):



   """simple docstring"""





   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
       __A		:     Any												=		[AddedToken("""<special>"""				,   lstrip=lowerCamelCase       )]

       __A		:     int												=		self.rust_tokenizer_class.from_pretrained(
           lowerCamelCase				,   additional_special_tokens=lowerCamelCase				,   **lowerCamelCase       )
       __A		:     Union[str, Any]												=		tokenizer_r.encode("""Hey this is a <special> token"""       )

       __A		:     Optional[int]												=		tokenizer_r.encode("""<special>"""				,   add_special_tokens=lowerCamelCase       )[0]

       self.assertTrue(special_token_id in r_output       )

       if self.test_slow_tokenizer:
         __A		:     Optional[Any]												=		self.rust_tokenizer_class.from_pretrained(
             lowerCamelCase				,   additional_special_tokens=lowerCamelCase				,   **lowerCamelCase				,   )
         __A		:     int												=		self.tokenizer_class.from_pretrained(
             lowerCamelCase				,   additional_special_tokens=lowerCamelCase				,   **lowerCamelCase       )

         __A		:     List[str]												=		tokenizer_p.encode("""Hey this is a <special> token"""       )

         __A		:     Dict												=		tokenizer_cr.encode("""Hey this is a <special> token"""       )

         self.assertEqual(lowerCamelCase				,   lowerCamelCase       )
         self.assertEqual(lowerCamelCase				,   lowerCamelCase       )
         self.assertTrue(special_token_id in p_output       )
         self.assertTrue(special_token_id in cr_output       )







@require_torch
@require_sentencepiece
@require_tokenizers
class  __A				( unittest.TestCase			):
 lowerCamelCase				       ='''facebook/nllb-200-distilled-600M'''
 lowerCamelCase				       =[
     ''' UN Chief Says There Is No Military Solution in Syria''',
     ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
 ]
 lowerCamelCase				       =[
     '''Şeful ONU declară că nu există o soluţie militară în Siria''',
     '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
     ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
     ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
 ]
 lowerCamelCase				       =[
     256047,
     16297,
     134408,
     8165,
     248066,
     14734,
     950,
     1135,
     105721,
     3573,
     83,
     27352,
     108,
     49486,
     2,
 ]
 @classmethod
 def 		lowercase_(   cls			:    int       ):



   """simple docstring"""





   __A		:     NllbTokenizer												=		NllbTokenizer.from_pretrained(
       cls.checkpoint_name				,   src_lang="""eng_Latn"""				,   tgt_lang="""ron_Latn"""       )
   __A		:     List[str]												=		1
   return cls
 def 		lowercase_(   self			:    List[str]       ):



   """simple docstring"""





   self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Arab"""]				,   25_60_01       )
   self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Latn"""]				,   25_60_02       )
   self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""fra_Latn"""]				,   25_60_57       )
 def 		lowercase_(   self			:    List[str]       ):



   """simple docstring"""





   __A		:     Union[str, Any]												=		self.tokenizer.batch_encode_plus(self.src_text       ).input_ids[0]
   self.assertListEqual(self.expected_src_tokens				,   lowerCamelCase       )
 def 		lowercase_(   self			:    Dict       ):



   """simple docstring"""





   self.assertIn(lowerCamelCase				,   self.tokenizer.all_special_ids       )
   # fmt: off
   __A		:     Dict												=		[RO_CODE, 42_54, 9_80_68, 11_29_23, 3_90_72, 39_09, 7_13, 10_27_67, 26, 1_73_14, 3_56_42, 1_46_83, 3_31_18, 20_22, 6_69_87, 2, 25_60_47]
   # fmt: on

   __A		:     Optional[Any]												=		self.tokenizer.decode(lowerCamelCase				,   skip_special_tokens=lowerCamelCase       )
   __A		:     str												=		self.tokenizer.decode(generated_ids[1:]				,   skip_special_tokens=lowerCamelCase       )
   self.assertEqual(lowerCamelCase				,   lowerCamelCase       )
   self.assertNotIn(self.tokenizer.eos_token				,   lowerCamelCase       )
 def 		lowercase_(   self			:    int       ):



   """simple docstring"""





   __A		:     str												=		["""this is gunna be a long sentence """ * 20]
   assert isinstance(src_text[0]				,   lowerCamelCase       )
   __A		:     Optional[int]												=		10
   __A		:     Optional[int]												=		self.tokenizer(lowerCamelCase				,   max_length=lowerCamelCase				,   truncation=lowerCamelCase       ).input_ids[0]
   self.assertEqual(ids[-1]				,   2       )
   self.assertEqual(ids[0]				,   lowerCamelCase       )
   self.assertEqual(len(lowerCamelCase       )				,   lowerCamelCase       )
 def 		lowercase_(   self			:    int       ):



   """simple docstring"""





   self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]       )				,   [25_62_03, 3]       )
 def 		lowercase_(   self			:    Optional[int]       ):



   """simple docstring"""





   __A		:     Tuple												=		tempfile.mkdtemp()
   __A		:     Tuple												=		self.tokenizer.fairseq_tokens_to_ids
   self.tokenizer.save_pretrained(lowerCamelCase       )
   __A		:     Tuple												=		NllbTokenizer.from_pretrained(lowerCamelCase       )
   self.assertDictEqual(new_tok.fairseq_tokens_to_ids				,   lowerCamelCase       )
 @require_torch
 def 		lowercase_(   self			:    Optional[Any]       ):



   """simple docstring"""





   __A		:     List[Any]												=		self.tokenizer(
       self.src_text				,   text_target=self.tgt_text				,   padding=lowerCamelCase				,   truncation=lowerCamelCase				,   max_length=len(self.expected_src_tokens       )				,   return_tensors="""pt"""				,   )
   __A		:     List[str]												=		shift_tokens_right(
       batch["""labels"""]				,   self.tokenizer.pad_token_id				,   self.tokenizer.lang_code_to_id["""ron_Latn"""]       )

   self.assertIsInstance(lowerCamelCase				,   lowerCamelCase       )

   self.assertEqual((2, 15)				,   batch.input_ids.shape       )
   self.assertEqual((2, 15)				,   batch.attention_mask.shape       )
   __A		:     Union[str, Any]												=		batch.input_ids.tolist()[0]
   self.assertListEqual(self.expected_src_tokens				,   lowerCamelCase       )
   self.assertEqual(lowerCamelCase				,   batch.decoder_input_ids[0, 0]       )  # EOS
   # Test that special tokens are reset
   self.assertEqual(self.tokenizer.prefix_tokens				,   [EN_CODE]       )
   self.assertEqual(self.tokenizer.suffix_tokens				,   [self.tokenizer.eos_token_id]       )
 def 		lowercase_(   self			:    Tuple       ):



   """simple docstring"""





   __A		:     Dict												=		self.tokenizer(self.src_text				,   padding=lowerCamelCase				,   truncation=lowerCamelCase				,   max_length=3				,   return_tensors="""pt"""       )
   __A		:     Optional[int]												=		self.tokenizer(
       text_target=self.tgt_text				,   padding=lowerCamelCase				,   truncation=lowerCamelCase				,   max_length=10				,   return_tensors="""pt"""       )
   __A		:     int												=		targets["""input_ids"""]
   __A		:     Union[str, Any]												=		shift_tokens_right(
       lowerCamelCase				,   self.tokenizer.pad_token_id				,   decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]				,   )

   self.assertEqual(batch.input_ids.shape[1]				,   3       )
   self.assertEqual(batch.decoder_input_ids.shape[1]				,   10       )
 @require_torch
 def 		lowercase_(   self			:    Optional[Any]       ):



   """simple docstring"""





   __A		:     Tuple												=		self.tokenizer._build_translation_inputs(
       """A test"""				,   return_tensors="""pt"""				,   src_lang="""eng_Latn"""				,   tgt_lang="""fra_Latn"""       )

   self.assertEqual(
       nested_simplify(lowerCamelCase       )				,   {
           # A, test, EOS, en_XX
           """input_ids""": [[25_60_47, 70, 73_56, 2]],
           """attention_mask""": [[1, 1, 1, 1]],
           # ar_AR
           """forced_bos_token_id""": 25_60_57,
       }				,   )






 @require_torch
 def 		lowercase_(   self			:    str       ):



   """simple docstring"""





   __A		:     Dict												=		True
   __A		:     int												=		self.tokenizer(
       """UN Chief says there is no military solution in Syria"""				,   src_lang="""eng_Latn"""				,   tgt_lang="""fra_Latn"""       )
   self.assertEqual(
       inputs.input_ids				,   [1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2, 25_60_47]       )

   __A		:     Any												=		False
   __A		:     List[Any]												=		self.tokenizer(
       """UN Chief says there is no military solution in Syria"""				,   src_lang="""eng_Latn"""				,   tgt_lang="""fra_Latn"""       )
   self.assertEqual(
       inputs.input_ids				,   [25_60_47, 1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2]       )





'''simple docstring'''
from typing import Union

from ..utils import add_end_docstrings, is_torch_available, is_vision_available, logging
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_vision_available():
     from PIL import Image

     from ..image_utils import load_image

if is_torch_available():
     from ..models.auto.modeling_auto import MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING

A__   :		Any 		=logging.get_logger(__name__)







@add_end_docstrings(_SCREAMING_SNAKE_CASE			)
class  __A				( _SCREAMING_SNAKE_CASE			):
 def __init__(   self			:    Union[str, Any]				,   *lowerCamelCase			:    int				,   **lowerCamelCase			:    Optional[int]       ):



   """simple docstring"""





   super().__init__(*lowerCamelCase				,   **lowerCamelCase       )
   self.check_model_type(lowerCamelCase       )
 def 		lowercase_(   self			:    Any				,   lowerCamelCase			:    Dict=None				,   lowerCamelCase			:    Union[str, Any]=None				,   lowerCamelCase			:    int=None				,   **lowerCamelCase			:    int       ):



   """simple docstring"""





   __A							,					__A		:     Tuple												=		{}, {}
   if padding is not None:
     __A		:     Any												=		padding
   if truncation is not None:
     __A		:     Optional[Any]												=		truncation
   if top_k is not None:
     __A		:     List[str]												=		top_k
   return preprocess_params, {}, postprocess_params
 def __call__(   self			:    Any				,   lowerCamelCase			:    Union["Image.Image", str]				,   lowerCamelCase			:    str = None				,   **lowerCamelCase			:    Tuple       ):



   """simple docstring"""





   if isinstance(lowerCamelCase				,   (Image.Image, str)       ) and isinstance(lowerCamelCase				,   lowerCamelCase       ):
     __A		:     Tuple												=		{"""image""": image, """question""": question}
   else:
     __A		:     List[Any]												=		image
   __A		:     Any												=		super().__call__(lowerCamelCase				,   **lowerCamelCase       )
   return results
 def 		lowercase_(   self			:    int				,   lowerCamelCase			:    Optional[int]				,   lowerCamelCase			:    Dict=False				,   lowerCamelCase			:    str=False       ):



   """simple docstring"""





   __A		:     List[str]												=		load_image(inputs["""image"""]       )
   __A		:     Optional[Any]												=		self.tokenizer(
       inputs["""question"""]				,   return_tensors=self.framework				,   padding=lowerCamelCase				,   truncation=lowerCamelCase       )
   __A		:     Union[str, Any]												=		self.image_processor(images=lowerCamelCase				,   return_tensors=self.framework       )
   model_inputs.update(lowerCamelCase       )
   return model_inputs
 def 		lowercase_(   self			:    List[str]				,   lowerCamelCase			:    Tuple       ):



   """simple docstring"""





   __A		:     List[str]												=		self.model(**lowerCamelCase       )
   return model_outputs






 def 		lowercase_(   self			:    str				,   lowerCamelCase			:    Optional[Any]				,   lowerCamelCase			:    List[Any]=5       ):



   """simple docstring"""





   if top_k > self.model.config.num_labels:
     __A		:     str												=		self.model.config.num_labels

   if self.framework == "pt":
     __A		:     Optional[Any]												=		model_outputs.logits.sigmoid()[0]
     __A							,					__A		:     int												=		probs.topk(lowerCamelCase       )
   else:
     raise ValueError(f"Unsupported framework: {self.framework}"       )

   __A		:     Optional[Any]												=		scores.tolist()
   __A		:     Dict												=		ids.tolist()
   return [{"score": score, "answer": self.model.config.idalabel[_id]} for score, _id in zip(lowerCamelCase				,   lowerCamelCase       )]



