

"""simple docstring"""



import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE_					:	Tuple							  =						get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
 from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

SCREAMING_SNAKE_CASE_					:	Optional[int]							  =						2_5_6_0_4_7
SCREAMING_SNAKE_CASE_					:	List[Any]							  =						2_5_6_1_4_5



@require_sentencepiece
@require_tokenizers
class 					a						(       _lowerCamelCase,	unittest.TestCase				):



    """simple docstring"""



    UpperCAmelCase						 =					NllbTokenizer
    UpperCAmelCase						 =					NllbTokenizerFast
    UpperCAmelCase						 =					True
    UpperCAmelCase						 =					True
    UpperCAmelCase						 =					{}

    def 				UpperCamelCase	(     self: Union[str, Any]       ):






         """simple docstring"""


         super().setUp()

         # We have a SentencePiece fixture for testing
         A__										=  NllbTokenizer(UpperCamelCase    ,						keep_accents=UpperCamelCase       )
         tokenizer.save_pretrained(self.tmpdirname       )

    def 				UpperCamelCase	(     self: str       ):






         """simple docstring"""


         A__										=  NllbTokenizer(UpperCamelCase    ,						keep_accents=UpperCamelCase       )

         A__										=  tokenizer.tokenize("""This is a test"""       )
         self.assertListEqual(UpperCamelCase    ,						["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]       )

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(UpperCamelCase       )    ,						[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]    ,						)

         A__										=  tokenizer.tokenize("""I was born in 92000, and this is falsé."""       )
         self.assertListEqual(
             UpperCamelCase    ,						[
                 SPIECE_UNDERLINE + """I""",
                 SPIECE_UNDERLINE + """was""",
                 SPIECE_UNDERLINE + """b""",
                 """or""",
                 """n""",
                 SPIECE_UNDERLINE + """in""",
                 SPIECE_UNDERLINE + """""",
                 """9""",
                 """2""",
                 """0""",
                 """0""",
                 """0""",
                 """,""",
                 SPIECE_UNDERLINE + """and""",
                 SPIECE_UNDERLINE + """this""",
                 SPIECE_UNDERLINE + """is""",
                 SPIECE_UNDERLINE + """f""",
                 """al""",
                 """s""",
                 """é""",
                 """.""",
             ]    ,						)
         A__										=  tokenizer.convert_tokens_to_ids(UpperCamelCase       )
         self.assertListEqual(
             UpperCamelCase    ,						[
                 value + tokenizer.fairseq_offset
                 for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
             ]    ,						)

         A__										=  tokenizer.convert_ids_to_tokens(UpperCamelCase       )
         self.assertListEqual(
             UpperCamelCase    ,						[
                 SPIECE_UNDERLINE + """I""",
                 SPIECE_UNDERLINE + """was""",
                 SPIECE_UNDERLINE + """b""",
                 """or""",
                 """n""",
                 SPIECE_UNDERLINE + """in""",
                 SPIECE_UNDERLINE + """""",
                 """<unk>""",
                 """2""",
                 """0""",
                 """0""",
                 """0""",
                 """,""",
                 SPIECE_UNDERLINE + """and""",
                 SPIECE_UNDERLINE + """this""",
                 SPIECE_UNDERLINE + """is""",
                 SPIECE_UNDERLINE + """f""",
                 """al""",
                 """s""",
                 """<unk>""",
                 """.""",
             ]    ,						)

    def 				UpperCamelCase	(     self: Optional[int]       ):






         """simple docstring"""


         A__										=  (self.rust_tokenizer_class, """hf-internal-testing/tiny-random-nllb""", {})
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
                   A__										=  self.rust_tokenizer_class.from_pretrained(UpperCamelCase    ,						**UpperCamelCase       )
                   A__										=  self.tokenizer_class.from_pretrained(UpperCamelCase    ,						**UpperCamelCase       )

                   A__										=  tempfile.mkdtemp()

                   A__										=  tokenizer_r.save_pretrained(UpperCamelCase       )
                   A__										=  tokenizer_p.save_pretrained(UpperCamelCase       )

                   # Checks it save with the same files + the tokenizer.json file for the fast one
                   self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files       )       )
                   A__										=  tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f       )
                   self.assertSequenceEqual(UpperCamelCase    ,						UpperCamelCase       )

                   # Checks everything loads correctly in the same way
                   A__										=  tokenizer_r.from_pretrained(UpperCamelCase       )
                   A__										=  tokenizer_p.from_pretrained(UpperCamelCase       )

                   # Check special tokens are set accordingly on Rust and Python
                   for key in tokenizer_pp.special_tokens_map:
                        self.assertTrue(hasattr(UpperCamelCase    ,						UpperCamelCase       )       )

                   shutil.rmtree(UpperCamelCase       )

                   # Save tokenizer rust, legacy_format=True
                   A__										=  tempfile.mkdtemp()

                   A__										=  tokenizer_r.save_pretrained(UpperCamelCase    ,						legacy_format=UpperCamelCase       )
                   A__										=  tokenizer_p.save_pretrained(UpperCamelCase       )

                   # Checks it save with the same files
                   self.assertSequenceEqual(UpperCamelCase    ,						UpperCamelCase       )

                   # Checks everything loads correctly in the same way
                   A__										=  tokenizer_r.from_pretrained(UpperCamelCase       )
                   A__										=  tokenizer_p.from_pretrained(UpperCamelCase       )

                   # Check special tokens are set accordingly on Rust and Python
                   for key in tokenizer_pp.special_tokens_map:
                        self.assertTrue(hasattr(UpperCamelCase    ,						UpperCamelCase       )       )

                   shutil.rmtree(UpperCamelCase       )

                   # Save tokenizer rust, legacy_format=False
                   A__										=  tempfile.mkdtemp()

                   A__										=  tokenizer_r.save_pretrained(UpperCamelCase    ,						legacy_format=UpperCamelCase       )
                   A__										=  tokenizer_p.save_pretrained(UpperCamelCase       )

                   # Checks it saved the tokenizer.json file
                   self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files       )       )

                   # Checks everything loads correctly in the same way
                   A__										=  tokenizer_r.from_pretrained(UpperCamelCase       )
                   A__										=  tokenizer_p.from_pretrained(UpperCamelCase       )

                   # Check special tokens are set accordingly on Rust and Python
                   for key in tokenizer_pp.special_tokens_map:
                        self.assertTrue(hasattr(UpperCamelCase    ,						UpperCamelCase       )       )

                   shutil.rmtree(UpperCamelCase       )

    @require_torch
    def 				UpperCamelCase	(     self: Union[str, Any]       ):






         """simple docstring"""


         if not self.test_seqaseq:
              return

         A__										=  self.get_tokenizers()
         for tokenizer in tokenizers:
              with self.subTest(f"""{tokenizer.__class__.__name__}"""       ):
                   # Longer text that will definitely require truncation.
                   A__										=  [
                       """ UN Chief Says There Is No Military Solution in Syria""",
                       """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"""
                       """ Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"""
                       """ will only worsen the violence and misery for millions of people.""",
                   ]
                   A__										=  [
                       """Şeful ONU declară că nu există o soluţie militară în Siria""",
                       """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"""
                       """ Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"""
                       """ că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
                   ]
                   try:
                        A__										=  tokenizer.prepare_seqaseq_batch(
                            src_texts=UpperCamelCase    ,						tgt_texts=UpperCamelCase    ,						max_length=3    ,						max_target_length=10    ,						return_tensors="""pt"""    ,						src_lang="""eng_Latn"""    ,						tgt_lang="""ron_Latn"""    ,						)
                   except NotImplementedError:
                        return
                   self.assertEqual(batch.input_ids.shape[1]    ,						3       )
                   self.assertEqual(batch.labels.shape[1]    ,						10       )
                   # max_target_length will default to max_length if not specified
                   A__										=  tokenizer.prepare_seqaseq_batch(
                       UpperCamelCase    ,						tgt_texts=UpperCamelCase    ,						max_length=3    ,						return_tensors="""pt"""       )
                   self.assertEqual(batch.input_ids.shape[1]    ,						3       )
                   self.assertEqual(batch.labels.shape[1]    ,						3       )

                   A__										=  tokenizer.prepare_seqaseq_batch(
                       src_texts=UpperCamelCase    ,						max_length=3    ,						max_target_length=10    ,						return_tensors="""pt"""       )
                   self.assertEqual(batch_encoder_only.input_ids.shape[1]    ,						3       )
                   self.assertEqual(batch_encoder_only.attention_mask.shape[1]    ,						3       )
                   self.assertNotIn("""decoder_input_ids"""    ,						UpperCamelCase       )

    @unittest.skip("""Unfortunately way too slow to build a BPE with SentencePiece."""       )
    def 				UpperCamelCase	(     self: Tuple       ):






         """simple docstring"""


         pass





    def 				UpperCamelCase	(     self: Union[str, Any]       ):






         """simple docstring"""


         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
                   A__										=  [AddedToken("""<special>"""    ,						lstrip=UpperCamelCase       )]

                   A__										=  self.rust_tokenizer_class.from_pretrained(
                       UpperCamelCase    ,						additional_special_tokens=UpperCamelCase    ,						**UpperCamelCase       )
                   A__										=  tokenizer_r.encode("""Hey this is a <special> token"""       )

                   A__										=  tokenizer_r.encode("""<special>"""    ,						add_special_tokens=UpperCamelCase       )[0]

                   self.assertTrue(special_token_id in r_output       )

                   if self.test_slow_tokenizer:
                        A__										=  self.rust_tokenizer_class.from_pretrained(
                            UpperCamelCase    ,						additional_special_tokens=UpperCamelCase    ,						**UpperCamelCase    ,						)
                        A__										=  self.tokenizer_class.from_pretrained(
                            UpperCamelCase    ,						additional_special_tokens=UpperCamelCase    ,						**UpperCamelCase       )

                        A__										=  tokenizer_p.encode("""Hey this is a <special> token"""       )

                        A__										=  tokenizer_cr.encode("""Hey this is a <special> token"""       )

                        self.assertEqual(UpperCamelCase    ,						UpperCamelCase       )
                        self.assertEqual(UpperCamelCase    ,						UpperCamelCase       )
                        self.assertTrue(special_token_id in p_output       )
                        self.assertTrue(special_token_id in cr_output       )



@require_torch
@require_sentencepiece
@require_tokenizers
class 					a						(       unittest.TestCase				):



    """simple docstring"""



    UpperCAmelCase						 =					"facebook/nllb-200-distilled-600M"
    UpperCAmelCase						 =					[
        " UN Chief Says There Is No Military Solution in Syria",
        " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
    ]
    UpperCAmelCase						 =					[
        "Şeful ONU declară că nu există o soluţie militară în Siria",
        "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
        " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
        " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
    ]
    UpperCAmelCase						 =					[
        2_5_6_0_4_7,
        1_6_2_9_7,
        1_3_4_4_0_8,
        8_1_6_5,
        2_4_8_0_6_6,
        1_4_7_3_4,
        9_5_0,
        1_1_3_5,
        1_0_5_7_2_1,
        3_5_7_3,
        8_3,
        2_7_3_5_2,
        1_0_8,
        4_9_4_8_6,
        2,
    ]

    @classmethod
    def 				UpperCamelCase	(     cls: Dict       ):






         """simple docstring"""


         A__										=  NllbTokenizer.from_pretrained(
             cls.checkpoint_name    ,						src_lang="""eng_Latn"""    ,						tgt_lang="""ron_Latn"""       )
         A__										=  1
         return cls

    def 				UpperCamelCase	(     self: Tuple       ):






         """simple docstring"""


         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Arab"""]    ,						25_60_01       )
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Latn"""]    ,						25_60_02       )
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""fra_Latn"""]    ,						25_60_57       )

    def 				UpperCamelCase	(     self: Tuple       ):






         """simple docstring"""


         A__										=  self.tokenizer.batch_encode_plus(self.src_text       ).input_ids[0]
         self.assertListEqual(self.expected_src_tokens    ,						UpperCamelCase       )

    def 				UpperCamelCase	(     self: Optional[Any]       ):






         """simple docstring"""


         self.assertIn(UpperCamelCase    ,						self.tokenizer.all_special_ids       )
         # fmt: off
         A__										=  [RO_CODE, 42_54, 9_80_68, 11_29_23, 3_90_72, 39_09, 7_13, 10_27_67, 26, 1_73_14, 3_56_42, 1_46_83, 3_31_18, 20_22, 6_69_87, 2, 25_60_47]
         # fmt: on

         A__										=  self.tokenizer.decode(UpperCamelCase    ,						skip_special_tokens=UpperCamelCase       )
         A__										=  self.tokenizer.decode(generated_ids[1:]    ,						skip_special_tokens=UpperCamelCase       )
         self.assertEqual(UpperCamelCase    ,						UpperCamelCase       )
         self.assertNotIn(self.tokenizer.eos_token    ,						UpperCamelCase       )

    def 				UpperCamelCase	(     self: Optional[int]       ):






         """simple docstring"""


         A__										=  ["""this is gunna be a long sentence """ * 20]
         assert isinstance(src_text[0]    ,						UpperCamelCase       )
         A__										=  10
         A__										=  self.tokenizer(UpperCamelCase    ,						max_length=UpperCamelCase    ,						truncation=UpperCamelCase       ).input_ids[0]
         self.assertEqual(ids[-1]    ,						2       )
         self.assertEqual(ids[0]    ,						UpperCamelCase       )
         self.assertEqual(len(UpperCamelCase       )    ,						UpperCamelCase       )

    def 				UpperCamelCase	(     self: Optional[Any]       ):






         """simple docstring"""


         self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]       )    ,						[25_62_03, 3]       )

    def 				UpperCamelCase	(     self: str       ):






         """simple docstring"""


         A__										=  tempfile.mkdtemp()
         A__										=  self.tokenizer.fairseq_tokens_to_ids
         self.tokenizer.save_pretrained(UpperCamelCase       )
         A__										=  NllbTokenizer.from_pretrained(UpperCamelCase       )
         self.assertDictEqual(new_tok.fairseq_tokens_to_ids    ,						UpperCamelCase       )

    @require_torch
    def 				UpperCamelCase	(     self: Any       ):






         """simple docstring"""


         A__										=  self.tokenizer(
             self.src_text    ,						text_target=self.tgt_text    ,						padding=UpperCamelCase    ,						truncation=UpperCamelCase    ,						max_length=len(self.expected_src_tokens       )    ,						return_tensors="""pt"""    ,						)
         A__										=  shift_tokens_right(
             batch["""labels"""]    ,						self.tokenizer.pad_token_id    ,						self.tokenizer.lang_code_to_id["""ron_Latn"""]       )

         self.assertIsInstance(UpperCamelCase    ,						UpperCamelCase       )

         self.assertEqual((2, 15)    ,						batch.input_ids.shape       )
         self.assertEqual((2, 15)    ,						batch.attention_mask.shape       )
         A__										=  batch.input_ids.tolist()[0]
         self.assertListEqual(self.expected_src_tokens    ,						UpperCamelCase       )
         self.assertEqual(UpperCamelCase    ,						batch.decoder_input_ids[0, 0]       )  # EOS
         # Test that special tokens are reset
         self.assertEqual(self.tokenizer.prefix_tokens    ,						[EN_CODE]       )
         self.assertEqual(self.tokenizer.suffix_tokens    ,						[self.tokenizer.eos_token_id]       )

    def 				UpperCamelCase	(     self: List[Any]       ):






         """simple docstring"""


         A__										=  self.tokenizer(self.src_text    ,						padding=UpperCamelCase    ,						truncation=UpperCamelCase    ,						max_length=3    ,						return_tensors="""pt"""       )
         A__										=  self.tokenizer(
             text_target=self.tgt_text    ,						padding=UpperCamelCase    ,						truncation=UpperCamelCase    ,						max_length=10    ,						return_tensors="""pt"""       )
         A__										=  targets["""input_ids"""]
         A__										=  shift_tokens_right(
             UpperCamelCase    ,						self.tokenizer.pad_token_id    ,						decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]    ,						)

         self.assertEqual(batch.input_ids.shape[1]    ,						3       )
         self.assertEqual(batch.decoder_input_ids.shape[1]    ,						10       )

    @require_torch
    def 				UpperCamelCase	(     self: int       ):






         """simple docstring"""


         A__										=  self.tokenizer._build_translation_inputs(
             """A test"""    ,						return_tensors="""pt"""    ,						src_lang="""eng_Latn"""    ,						tgt_lang="""fra_Latn"""       )

         self.assertEqual(
             nested_simplify(UpperCamelCase       )    ,						{
                 # A, test, EOS, en_XX
                 """input_ids""": [[25_60_47, 70, 73_56, 2]],
                 """attention_mask""": [[1, 1, 1, 1]],
                 # ar_AR
                 """forced_bos_token_id""": 25_60_57,
             }    ,						)





    @require_torch
    def 				UpperCamelCase	(     self: Dict       ):






         """simple docstring"""


         A__										=  True
         A__										=  self.tokenizer(
             """UN Chief says there is no military solution in Syria"""    ,						src_lang="""eng_Latn"""    ,						tgt_lang="""fra_Latn"""       )
         self.assertEqual(
             inputs.input_ids    ,						[1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2, 25_60_47]       )

         A__										=  False
         A__										=  self.tokenizer(
             """UN Chief says there is no military solution in Syria"""    ,						src_lang="""eng_Latn"""    ,						tgt_lang="""fra_Latn"""       )
         self.assertEqual(
             inputs.input_ids    ,						[25_60_47, 1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2]       )









"""simple docstring"""



from itertools import count





def  _snake_case     (    UpperCAmelCase_  :					int = 50			):
     A__										=  [1] * min_block_length

     for n in count(UpperCAmelCase_			):
          fill_count_functions.append(1			)

          for block_length in range(UpperCAmelCase_					,   n + 1			):
               for block_start in range(n - block_length			):
                    fill_count_functions[n] += fill_count_functions[
                        n - block_start - block_length - 1
                    ]

               fill_count_functions[n] += 1

          if fill_count_functions[n] > 100_0000:
               break

     return n


if __name__ == "__main__":
 print(f"""{solution() = }""")






