


"""simple docstring"""

import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase__  							=get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
       from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

UpperCAmelCase__  							=25_6047
UpperCAmelCase__  							=25_6145





@require_sentencepiece
@require_tokenizers
class    lowerCamelCase__						(		_a			,   unittest.TestCase    ):
      a :     List[Any]											=		NllbTokenizer
      a :     int											=		NllbTokenizerFast
      a :     str											=		True
      a :     Optional[Any]											=		True
      a :     str											=		{}
      def SCREAMING_SNAKE_CASE_	(		self	:	Any							):






       '''simple docstring'''


       super().setUp()

       # We have a SentencePiece fixture for testing
       __lowercase									=	NllbTokenizer(A_      ,       keep_accents=A_							)
       tokenizer.save_pretrained(self.tmpdirname							)
      def SCREAMING_SNAKE_CASE_	(		self	:	List[Any]							):






       '''simple docstring'''


       __lowercase									=	NllbTokenizer(A_      ,       keep_accents=A_							)

       __lowercase									=	tokenizer.tokenize("""This is a test"""							)
       self.assertListEqual(A_      ,       ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]							)

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(A_							)      ,       [value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]      ,       )

       __lowercase									=	tokenizer.tokenize("""I was born in 92000, and this is falsé."""							)
       self.assertListEqual(
           A_      ,       [
               SPIECE_UNDERLINE + """I""",
               SPIECE_UNDERLINE + """was""",
               SPIECE_UNDERLINE + """b""",
               """or""",
               """n""",
               SPIECE_UNDERLINE + """in""",
               SPIECE_UNDERLINE + """""",
               """9""",
               """2""",
               """0""",
               """0""",
               """0""",
               """,""",
               SPIECE_UNDERLINE + """and""",
               SPIECE_UNDERLINE + """this""",
               SPIECE_UNDERLINE + """is""",
               SPIECE_UNDERLINE + """f""",
               """al""",
               """s""",
               """é""",
               """.""",
           ]      ,       )
       __lowercase									=	tokenizer.convert_tokens_to_ids(A_							)
       self.assertListEqual(
           A_      ,       [
               value + tokenizer.fairseq_offset
               for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 2, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 2, 4]
           ]      ,       )

       __lowercase									=	tokenizer.convert_ids_to_tokens(A_							)
       self.assertListEqual(
           A_      ,       [
               SPIECE_UNDERLINE + """I""",
               SPIECE_UNDERLINE + """was""",
               SPIECE_UNDERLINE + """b""",
               """or""",
               """n""",
               SPIECE_UNDERLINE + """in""",
               SPIECE_UNDERLINE + """""",
               """<unk>""",
               """2""",
               """0""",
               """0""",
               """0""",
               """,""",
               SPIECE_UNDERLINE + """and""",
               SPIECE_UNDERLINE + """this""",
               SPIECE_UNDERLINE + """is""",
               SPIECE_UNDERLINE + """f""",
               """al""",
               """s""",
               """<unk>""",
               """.""",
           ]      ,       )
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[Any]							):






       '''simple docstring'''


       __lowercase									=	(self.rust_tokenizer_class, """hf-internal-testing/tiny-random-nllb""", {})
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
         __lowercase									=	self.rust_tokenizer_class.from_pretrained(A_      ,       **A_							)
         __lowercase									=	self.tokenizer_class.from_pretrained(A_      ,       **A_							)

         __lowercase									=	tempfile.mkdtemp()

         __lowercase									=	tokenizer_r.save_pretrained(A_							)
         __lowercase									=	tokenizer_p.save_pretrained(A_							)

         # Checks it save with the same files + the tokenizer.json file for the fast one
         self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files							)							)
         __lowercase									=	tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f							)
         self.assertSequenceEqual(A_      ,       A_							)

         # Checks everything loads correctly in the same way
         __lowercase									=	tokenizer_r.from_pretrained(A_							)
         __lowercase									=	tokenizer_p.from_pretrained(A_							)

         # Check special tokens are set accordingly on Rust and Python
         for key in tokenizer_pp.special_tokens_map:
          self.assertTrue(hasattr(A_      ,       A_							)							)

         shutil.rmtree(A_							)

         # Save tokenizer rust, legacy_format=True
         __lowercase									=	tempfile.mkdtemp()

         __lowercase									=	tokenizer_r.save_pretrained(A_      ,       legacy_format=A_							)
         __lowercase									=	tokenizer_p.save_pretrained(A_							)

         # Checks it save with the same files
         self.assertSequenceEqual(A_      ,       A_							)

         # Checks everything loads correctly in the same way
         __lowercase									=	tokenizer_r.from_pretrained(A_							)
         __lowercase									=	tokenizer_p.from_pretrained(A_							)

         # Check special tokens are set accordingly on Rust and Python
         for key in tokenizer_pp.special_tokens_map:
          self.assertTrue(hasattr(A_      ,       A_							)							)

         shutil.rmtree(A_							)

         # Save tokenizer rust, legacy_format=False
         __lowercase									=	tempfile.mkdtemp()

         __lowercase									=	tokenizer_r.save_pretrained(A_      ,       legacy_format=A_							)
         __lowercase									=	tokenizer_p.save_pretrained(A_							)

         # Checks it saved the tokenizer.json file
         self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files							)							)

         # Checks everything loads correctly in the same way
         __lowercase									=	tokenizer_r.from_pretrained(A_							)
         __lowercase									=	tokenizer_p.from_pretrained(A_							)

         # Check special tokens are set accordingly on Rust and Python
         for key in tokenizer_pp.special_tokens_map:
          self.assertTrue(hasattr(A_      ,       A_							)							)

         shutil.rmtree(A_							)
      @require_torch
      def SCREAMING_SNAKE_CASE_	(		self	:	List[str]							):






       '''simple docstring'''


       if not self.test_seqaseq:
        return

       __lowercase									=	self.get_tokenizers()
       for tokenizer in tokenizers:
        with self.subTest(F'''{tokenizer.__class__.__name__}'''							):
         # Longer text that will definitely require truncation.
         __lowercase									=	[
             """ UN Chief Says There Is No Military Solution in Syria""",
             """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"""
             """ Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"""
             """ will only worsen the violence and misery for millions of people.""",
         ]
         __lowercase									=	[
             """Şeful ONU declară că nu există o soluţie militară în Siria""",
             """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"""
             """ Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"""
             """ că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
         ]
         try:
          __lowercase									=	tokenizer.prepare_seqaseq_batch(
              src_texts=A_      ,       tgt_texts=A_      ,       max_length=3      ,       max_target_length=1_0      ,       return_tensors="""pt"""      ,       src_lang="""eng_Latn"""      ,       tgt_lang="""ron_Latn"""      ,       )
         except NotImplementedError:
          return
         self.assertEqual(batch.input_ids.shape[1]      ,       3							)
         self.assertEqual(batch.labels.shape[1]      ,       1_0							)
         # max_target_length will default to max_length if not specified
         __lowercase									=	tokenizer.prepare_seqaseq_batch(
             A_      ,       tgt_texts=A_      ,       max_length=3      ,       return_tensors="""pt"""							)
         self.assertEqual(batch.input_ids.shape[1]      ,       3							)
         self.assertEqual(batch.labels.shape[1]      ,       3							)

         __lowercase									=	tokenizer.prepare_seqaseq_batch(
             src_texts=A_      ,       max_length=3      ,       max_target_length=1_0      ,       return_tensors="""pt"""							)
         self.assertEqual(batch_encoder_only.input_ids.shape[1]      ,       3							)
         self.assertEqual(batch_encoder_only.attention_mask.shape[1]      ,       3							)
         self.assertNotIn("""decoder_input_ids"""      ,       A_							)
      @unittest.skip("""Unfortunately way too slow to build a BPE with SentencePiece."""							)
      def SCREAMING_SNAKE_CASE_	(		self	:	List[Any]							):






       '''simple docstring'''


       pass






      def SCREAMING_SNAKE_CASE_	(		self	:	Tuple							):






       '''simple docstring'''


       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
         __lowercase									=	[AddedToken("""<special>"""      ,       lstrip=A_							)]

         __lowercase									=	self.rust_tokenizer_class.from_pretrained(
             A_      ,       additional_special_tokens=A_      ,       **A_							)
         __lowercase									=	tokenizer_r.encode("""Hey this is a <special> token"""							)

         __lowercase									=	tokenizer_r.encode("""<special>"""      ,       add_special_tokens=A_							)[0]

         self.assertTrue(special_token_id in r_output							)

         if self.test_slow_tokenizer:
          __lowercase									=	self.rust_tokenizer_class.from_pretrained(
              A_      ,       additional_special_tokens=A_      ,       **A_      ,       )
          __lowercase									=	self.tokenizer_class.from_pretrained(
              A_      ,       additional_special_tokens=A_      ,       **A_							)

          __lowercase									=	tokenizer_p.encode("""Hey this is a <special> token"""							)

          __lowercase									=	tokenizer_cr.encode("""Hey this is a <special> token"""							)

          self.assertEqual(A_      ,       A_							)
          self.assertEqual(A_      ,       A_							)
          self.assertTrue(special_token_id in p_output							)
          self.assertTrue(special_token_id in cr_output							)






@require_torch
@require_sentencepiece
@require_tokenizers
class    lowerCamelCase__						(		unittest.TestCase    ):
      a :     Union[str, Any]											=		"""facebook/nllb-200-distilled-600M"""
      a :     int											=		[
          """ UN Chief Says There Is No Military Solution in Syria""",
          """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
      ]
      a :     Dict											=		[
          """Şeful ONU declară că nu există o soluţie militară în Siria""",
          """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"""
          """ pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"""
          """ face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
      ]
      a :     List[Any]											=		[
          25_60_47,
          1_62_97,
          13_44_08,
          81_65,
          24_80_66,
          1_47_34,
          9_50,
          11_35,
          10_57_21,
          35_73,
          83,
          2_73_52,
          1_08,
          4_94_86,
          2,
      ]
      @classmethod
      def SCREAMING_SNAKE_CASE_	(		cls	:	Dict							):






       '''simple docstring'''


       __lowercase									=	NllbTokenizer.from_pretrained(
           cls.checkpoint_name      ,       src_lang="""eng_Latn"""      ,       tgt_lang="""ron_Latn"""							)
       __lowercase									=	1
       return cls
      def SCREAMING_SNAKE_CASE_	(		self	:	Union[str, Any]							):






       '''simple docstring'''


       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Arab"""]      ,       2_5_6_0_0_1							)
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Latn"""]      ,       2_5_6_0_0_2							)
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""fra_Latn"""]      ,       2_5_6_0_5_7							)
      def SCREAMING_SNAKE_CASE_	(		self	:	Any							):






       '''simple docstring'''


       __lowercase									=	self.tokenizer.batch_encode_plus(self.src_text							).input_ids[0]
       self.assertListEqual(self.expected_src_tokens      ,       A_							)
      def SCREAMING_SNAKE_CASE_	(		self	:	int							):






       '''simple docstring'''


       self.assertIn(A_      ,       self.tokenizer.all_special_ids							)
       # fmt: off
       __lowercase									=	[RO_CODE, 4_2_5_4, 9_8_0_6_8, 1_1_2_9_2_3, 3_9_0_7_2, 3_9_0_9, 7_1_3, 1_0_2_7_6_7, 2_6, 1_7_3_1_4, 3_5_6_4_2, 1_4_6_8_3, 3_3_1_1_8, 2_0_2_2, 6_6_9_8_7, 2, 2_5_6_0_4_7]
       # fmt: on

       __lowercase									=	self.tokenizer.decode(A_      ,       skip_special_tokens=A_							)
       __lowercase									=	self.tokenizer.decode(generated_ids[1:]      ,       skip_special_tokens=A_							)
       self.assertEqual(A_      ,       A_							)
       self.assertNotIn(self.tokenizer.eos_token      ,       A_							)
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[Any]							):






       '''simple docstring'''


       __lowercase									=	["""this is gunna be a long sentence """ * 2_0]
       assert isinstance(src_text[0]      ,       A_							)
       __lowercase									=	1_0
       __lowercase									=	self.tokenizer(A_      ,       max_length=A_      ,       truncation=A_							).input_ids[0]
       self.assertEqual(ids[-1]      ,       2							)
       self.assertEqual(ids[0]      ,       A_							)
       self.assertEqual(len(A_							)      ,       A_							)
      def SCREAMING_SNAKE_CASE_	(		self	:	List[Any]							):






       '''simple docstring'''


       self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]							)      ,       [2_5_6_2_0_3, 3]							)
      def SCREAMING_SNAKE_CASE_	(		self	:	Tuple							):






       '''simple docstring'''


       __lowercase									=	tempfile.mkdtemp()
       __lowercase									=	self.tokenizer.fairseq_tokens_to_ids
       self.tokenizer.save_pretrained(A_							)
       __lowercase									=	NllbTokenizer.from_pretrained(A_							)
       self.assertDictEqual(new_tok.fairseq_tokens_to_ids      ,       A_							)
      @require_torch
      def SCREAMING_SNAKE_CASE_	(		self	:	List[str]							):






       '''simple docstring'''


       __lowercase									=	self.tokenizer(
           self.src_text      ,       text_target=self.tgt_text      ,       padding=A_      ,       truncation=A_      ,       max_length=len(self.expected_src_tokens							)      ,       return_tensors="""pt"""      ,       )
       __lowercase									=	shift_tokens_right(
           batch["""labels"""]      ,       self.tokenizer.pad_token_id      ,       self.tokenizer.lang_code_to_id["""ron_Latn"""]							)

       self.assertIsInstance(A_      ,       A_							)

       self.assertEqual((2, 1_5)      ,       batch.input_ids.shape							)
       self.assertEqual((2, 1_5)      ,       batch.attention_mask.shape							)
       __lowercase									=	batch.input_ids.tolist()[0]
       self.assertListEqual(self.expected_src_tokens      ,       A_							)
       self.assertEqual(A_      ,       batch.decoder_input_ids[0, 0]							)  # EOS
       # Test that special tokens are reset
       self.assertEqual(self.tokenizer.prefix_tokens      ,       [EN_CODE]							)
       self.assertEqual(self.tokenizer.suffix_tokens      ,       [self.tokenizer.eos_token_id]							)
      def SCREAMING_SNAKE_CASE_	(		self	:	str							):






       '''simple docstring'''


       __lowercase									=	self.tokenizer(self.src_text      ,       padding=A_      ,       truncation=A_      ,       max_length=3      ,       return_tensors="""pt"""							)
       __lowercase									=	self.tokenizer(
           text_target=self.tgt_text      ,       padding=A_      ,       truncation=A_      ,       max_length=1_0      ,       return_tensors="""pt"""							)
       __lowercase									=	targets["""input_ids"""]
       __lowercase									=	shift_tokens_right(
           A_      ,       self.tokenizer.pad_token_id      ,       decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]      ,       )

       self.assertEqual(batch.input_ids.shape[1]      ,       3							)
       self.assertEqual(batch.decoder_input_ids.shape[1]      ,       1_0							)
      @require_torch
      def SCREAMING_SNAKE_CASE_	(		self	:	Tuple							):






       '''simple docstring'''


       __lowercase									=	self.tokenizer._build_translation_inputs(
           """A test"""      ,       return_tensors="""pt"""      ,       src_lang="""eng_Latn"""      ,       tgt_lang="""fra_Latn"""							)

       self.assertEqual(
           nested_simplify(A_							)      ,       {
               # A, test, EOS, en_XX
               """input_ids""": [[2_5_6_0_4_7, 7_0, 7_3_5_6, 2]],
               """attention_mask""": [[1, 1, 1, 1]],
               # ar_AR
               """forced_bos_token_id""": 2_5_6_0_5_7,
           }      ,       )






      @require_torch
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[Any]							):






       '''simple docstring'''


       __lowercase									=	True
       __lowercase									=	self.tokenizer(
           """UN Chief says there is no military solution in Syria"""      ,       src_lang="""eng_Latn"""      ,       tgt_lang="""fra_Latn"""							)
       self.assertEqual(
           inputs.input_ids      ,       [1_6_2_9_7, 1_3_4_4_0_8, 2_5_6_5_3, 6_3_7_0, 2_4_8, 2_5_4, 1_0_3_9_2_9, 9_4_9_9_5, 1_0_8, 4_9_4_8_6, 2, 2_5_6_0_4_7]							)

       __lowercase									=	False
       __lowercase									=	self.tokenizer(
           """UN Chief says there is no military solution in Syria"""      ,       src_lang="""eng_Latn"""      ,       tgt_lang="""fra_Latn"""							)
       self.assertEqual(
           inputs.input_ids      ,       [2_5_6_0_4_7, 1_6_2_9_7, 1_3_4_4_0_8, 2_5_6_5_3, 6_3_7_0, 2_4_8, 2_5_4, 1_0_3_9_2_9, 9_4_9_9_5, 1_0_8, 4_9_4_8_6, 2]							)




"""simple docstring"""

def lowerCAmelCase_      (				UpperCamelCase__	: str				,   UpperCamelCase__	: str ):






 """simple docstring"""







 def get_matched_characters(UpperCamelCase__	: str				,   UpperCamelCase__	: str ) -> str:
  __lowercase									=	[]
  __lowercase									=	min(len(_stra )				,   len(_stra ) ) // 2
  for i, l in enumerate(_stra ):
   __lowercase									=	int(max(0				,   i - limit ) )
   __lowercase									=	int(min(i + limit + 1				,   len(_stra ) ) )
   if l in _stra[left:right]:
    matched.append(UpperCamelCase__ )
    __lowercase									=	f'''{_stra[0:_stra.index(UpperCamelCase__ )]} {_stra[_stra.index(UpperCamelCase__ ) + 1:]}'''

  return "".join(UpperCamelCase__ )

 # matching characters
 __lowercase									=	get_matched_characters(UpperCamelCase__				,   UpperCamelCase__ )
 __lowercase									=	get_matched_characters(UpperCamelCase__				,   UpperCamelCase__ )
 __lowercase									=	len(UpperCamelCase__ )

 # transposition
 __lowercase									=	(
     len([(ca, ca) for ca, ca in zip(UpperCamelCase__				,   UpperCamelCase__ ) if ca != ca] ) // 2
 )

 if not match_count:
  __lowercase									=	0.0
 else:
  __lowercase									=	(
      1
      / 3
      * (
          match_count / len(UpperCamelCase__ )
          + match_count / len(UpperCamelCase__ )
          + (match_count - transpositions) / match_count
      )
  )

 # common prefix up to 4 characters
 __lowercase									=	0
 for ca, ca in zip(stra[:4]				,   stra[:4] ):
  if ca == ca:
   prefix_len += 1
  else:
   break

 return jaro + 0.1 * prefix_len * (1 - jaro)


if __name__ == "__main__":
       import doctest

       doctest.testmod()
       print(jaro_winkler("hello", "world"))
