




'''simple docstring'''


import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase_ 		=      get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
		from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

UpperCAmelCase_ 		=      25_60_47
UpperCAmelCase_ 		=      25_61_45





@require_sentencepiece
@require_tokenizers
class    __lowercase						(  __magic_name__       ,							unittest.TestCase ):
			_a							=      NllbTokenizer
			_a							=      NllbTokenizerFast
			_a							=      True
			_a							=      True
			_a							=      {}







			def 		UpperCamelCase__			(	self      )				->    Union[str, Any]:
					super().setUp()

					# We have a SentencePiece fixture for testing
					__a			=					NllbTokenizer(UpperCamelCase      ,							keep_accents=UpperCamelCase      )
					tokenizer.save_pretrained(self.tmpdirname      )







			def 		UpperCamelCase__			(	self      )				->    Union[str, Any]:
					__a			=					NllbTokenizer(UpperCamelCase      ,							keep_accents=UpperCamelCase      )

					__a			=					tokenizer.tokenize('This is a test'      )
					self.assertListEqual(UpperCamelCase      ,							['▁This', '▁is', '▁a', '▁t', 'est']      )

					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(UpperCamelCase      )      ,							[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]      ,							)

					__a			=					tokenizer.tokenize('I was born in 92000, and this is falsé.'      )
					self.assertListEqual(
					    UpperCamelCase      ,							[
					        SPIECE_UNDERLINE + 'I',
					        SPIECE_UNDERLINE + 'was',
					        SPIECE_UNDERLINE + 'b',
					        'or',
					        'n',
					        SPIECE_UNDERLINE + 'in',
					        SPIECE_UNDERLINE + '',
					        '9',
					        '2',
					        '0',
					        '0',
					        '0',
					        ',',
					        SPIECE_UNDERLINE + 'and',
					        SPIECE_UNDERLINE + 'this',
					        SPIECE_UNDERLINE + 'is',
					        SPIECE_UNDERLINE + 'f',
					        'al',
					        's',
					        'é',
					        '.',
					    ]      ,							)
					__a			=					tokenizer.convert_tokens_to_ids(UpperCamelCase      )
					self.assertListEqual(
					    UpperCamelCase      ,							[
					        value + tokenizer.fairseq_offset
					        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
					    ]      ,							)

					__a			=					tokenizer.convert_ids_to_tokens(UpperCamelCase      )
					self.assertListEqual(
					    UpperCamelCase      ,							[
					        SPIECE_UNDERLINE + 'I',
					        SPIECE_UNDERLINE + 'was',
					        SPIECE_UNDERLINE + 'b',
					        'or',
					        'n',
					        SPIECE_UNDERLINE + 'in',
					        SPIECE_UNDERLINE + '',
					        '<unk>',
					        '2',
					        '0',
					        '0',
					        '0',
					        ',',
					        SPIECE_UNDERLINE + 'and',
					        SPIECE_UNDERLINE + 'this',
					        SPIECE_UNDERLINE + 'is',
					        SPIECE_UNDERLINE + 'f',
					        'al',
					        's',
					        '<unk>',
					        '.',
					    ]      ,							)







			def 		UpperCamelCase__			(	self      )				->    str:
					__a			=					(self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-nllb', {})
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"      ):
									__a			=					self.rust_tokenizer_class.from_pretrained(UpperCamelCase      ,							**UpperCamelCase      )
									__a			=					self.tokenizer_class.from_pretrained(UpperCamelCase      ,							**UpperCamelCase      )

									__a			=					tempfile.mkdtemp()

									__a			=					tokenizer_r.save_pretrained(UpperCamelCase      )
									__a			=					tokenizer_p.save_pretrained(UpperCamelCase      )

									# Checks it save with the same files + the tokenizer.json file for the fast one
									self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files      )      )
									__a			=					tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f      )
									self.assertSequenceEqual(UpperCamelCase      ,							UpperCamelCase      )

									# Checks everything loads correctly in the same way
									__a			=					tokenizer_r.from_pretrained(UpperCamelCase      )
									__a			=					tokenizer_p.from_pretrained(UpperCamelCase      )

									# Check special tokens are set accordingly on Rust and Python
									for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(UpperCamelCase      ,							UpperCamelCase      )      )

									shutil.rmtree(UpperCamelCase      )

									# Save tokenizer rust, legacy_format=True
									__a			=					tempfile.mkdtemp()

									__a			=					tokenizer_r.save_pretrained(UpperCamelCase      ,							legacy_format=UpperCamelCase      )
									__a			=					tokenizer_p.save_pretrained(UpperCamelCase      )

									# Checks it save with the same files
									self.assertSequenceEqual(UpperCamelCase      ,							UpperCamelCase      )

									# Checks everything loads correctly in the same way
									__a			=					tokenizer_r.from_pretrained(UpperCamelCase      )
									__a			=					tokenizer_p.from_pretrained(UpperCamelCase      )

									# Check special tokens are set accordingly on Rust and Python
									for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(UpperCamelCase      ,							UpperCamelCase      )      )

									shutil.rmtree(UpperCamelCase      )

									# Save tokenizer rust, legacy_format=False
									__a			=					tempfile.mkdtemp()

									__a			=					tokenizer_r.save_pretrained(UpperCamelCase      ,							legacy_format=UpperCamelCase      )
									__a			=					tokenizer_p.save_pretrained(UpperCamelCase      )

									# Checks it saved the tokenizer.json file
									self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files      )      )

									# Checks everything loads correctly in the same way
									__a			=					tokenizer_r.from_pretrained(UpperCamelCase      )
									__a			=					tokenizer_p.from_pretrained(UpperCamelCase      )

									# Check special tokens are set accordingly on Rust and Python
									for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(UpperCamelCase      ,							UpperCamelCase      )      )

									shutil.rmtree(UpperCamelCase      )







			@require_torch
			def 		UpperCamelCase__			(	self      )				->    Optional[Any]:
					if not self.test_seqaseq:
							return

					__a			=					self.get_tokenizers()
					for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"      ):
									# Longer text that will definitely require truncation.
									__a			=					[
									    ' UN Chief Says There Is No Military Solution in Syria',
									    ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'
									    ' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'
									    ' will only worsen the violence and misery for millions of people.',
									]
									__a			=					[
									    'Şeful ONU declară că nu există o soluţie militară în Siria',
									    'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'
									    ' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'
									    ' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
									]
									try:
											__a			=					tokenizer.prepare_seqaseq_batch(
											    src_texts=UpperCamelCase      ,							tgt_texts=UpperCamelCase      ,							max_length=3      ,							max_target_length=10      ,							return_tensors='pt'      ,							src_lang='eng_Latn'      ,							tgt_lang='ron_Latn'      ,							)
									except NotImplementedError:
											return
									self.assertEqual(batch.input_ids.shape[1]      ,							3      )
									self.assertEqual(batch.labels.shape[1]      ,							10      )
									# max_target_length will default to max_length if not specified
									__a			=					tokenizer.prepare_seqaseq_batch(
									    UpperCamelCase      ,							tgt_texts=UpperCamelCase      ,							max_length=3      ,							return_tensors='pt'      )
									self.assertEqual(batch.input_ids.shape[1]      ,							3      )
									self.assertEqual(batch.labels.shape[1]      ,							3      )

									__a			=					tokenizer.prepare_seqaseq_batch(
									    src_texts=UpperCamelCase      ,							max_length=3      ,							max_target_length=10      ,							return_tensors='pt'      )
									self.assertEqual(batch_encoder_only.input_ids.shape[1]      ,							3      )
									self.assertEqual(batch_encoder_only.attention_mask.shape[1]      ,							3      )
									self.assertNotIn('decoder_input_ids'      ,							UpperCamelCase      )







			@unittest.skip('Unfortunately way too slow to build a BPE with SentencePiece.'      )
			def 		UpperCamelCase__			(	self      )				->    Optional[int]:
					pass







			def 		UpperCamelCase__			(	self      )				->    List[str]:
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"      ):
									__a			=					[AddedToken('<special>'      ,							lstrip=UpperCamelCase      )]

									__a			=					self.rust_tokenizer_class.from_pretrained(
									    UpperCamelCase      ,							additional_special_tokens=UpperCamelCase      ,							**UpperCamelCase      )
									__a			=					tokenizer_r.encode('Hey this is a <special> token'      )

									__a			=					tokenizer_r.encode('<special>'      ,							add_special_tokens=UpperCamelCase      )[0]

									self.assertTrue(special_token_id in r_output      )

									if self.test_slow_tokenizer:
											__a			=					self.rust_tokenizer_class.from_pretrained(
											    UpperCamelCase      ,							additional_special_tokens=UpperCamelCase      ,							**UpperCamelCase      ,							)
											__a			=					self.tokenizer_class.from_pretrained(
											    UpperCamelCase      ,							additional_special_tokens=UpperCamelCase      ,							**UpperCamelCase      )

											__a			=					tokenizer_p.encode('Hey this is a <special> token'      )

											__a			=					tokenizer_cr.encode('Hey this is a <special> token'      )

											self.assertEqual(UpperCamelCase      ,							UpperCamelCase      )
											self.assertEqual(UpperCamelCase      ,							UpperCamelCase      )
											self.assertTrue(special_token_id in p_output      )
											self.assertTrue(special_token_id in cr_output      )





@require_torch
@require_sentencepiece
@require_tokenizers
class    __lowercase						(  unittest.TestCase ):
			_a							=      """facebook/nllb-200-distilled-600M"""
			_a							=      [
			    """ UN Chief Says There Is No Military Solution in Syria""",
			    """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
			]
			_a							=      [
			    """Şeful ONU declară că nu există o soluţie militară în Siria""",
			    """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"""
			    """ pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"""
			    """ face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
			]
			_a							=      [
			    256047,
			    16297,
			    134408,
			    8165,
			    248066,
			    14734,
			    950,
			    1135,
			    105721,
			    3573,
			    83,
			    27352,
			    108,
			    49486,
			    2,
			]







			@classmethod
			def 		UpperCamelCase__			(	cls      )				->    str:
					__a			=					NllbTokenizer.from_pretrained(
					    cls.checkpoint_name      ,							src_lang='eng_Latn'      ,							tgt_lang='ron_Latn'      )
					__a			=					1
					return cls







			def 		UpperCamelCase__			(	self      )				->    Optional[int]:
					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Arab']      ,							25_6001      )
					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Latn']      ,							25_6002      )
					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['fra_Latn']      ,							25_6057      )







			def 		UpperCamelCase__			(	self      )				->    int:
					__a			=					self.tokenizer.batch_encode_plus(self.src_text      ).input_ids[0]
					self.assertListEqual(self.expected_src_tokens      ,							UpperCamelCase      )







			def 		UpperCamelCase__			(	self      )				->    Union[str, Any]:
					self.assertIn(UpperCamelCase      ,							self.tokenizer.all_special_ids      )
					# fmt: off
					__a			=					[RO_CODE, 4254, 9_8068, 11_2923, 3_9072, 3909, 713, 10_2767, 26, 1_7314, 3_5642, 1_4683, 3_3118, 2022, 6_6987, 2, 25_6047]
					# fmt: on

					__a			=					self.tokenizer.decode(UpperCamelCase      ,							skip_special_tokens=UpperCamelCase      )
					__a			=					self.tokenizer.decode(generated_ids[1:]      ,							skip_special_tokens=UpperCamelCase      )
					self.assertEqual(UpperCamelCase      ,							UpperCamelCase      )
					self.assertNotIn(self.tokenizer.eos_token      ,							UpperCamelCase      )







			def 		UpperCamelCase__			(	self      )				->    str:
					__a			=					['this is gunna be a long sentence ' * 20]
					assert isinstance(src_text[0]      ,							UpperCamelCase      )
					__a			=					10
					__a			=					self.tokenizer(UpperCamelCase      ,							max_length=UpperCamelCase      ,							truncation=UpperCamelCase      ).input_ids[0]
					self.assertEqual(ids[-1]      ,							2      )
					self.assertEqual(ids[0]      ,							UpperCamelCase      )
					self.assertEqual(len(UpperCamelCase      )      ,							UpperCamelCase      )







			def 		UpperCamelCase__			(	self      )				->    Optional[Any]:
					self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR']      )      ,							[25_6203, 3]      )







			def 		UpperCamelCase__			(	self      )				->    Optional[int]:
					__a			=					tempfile.mkdtemp()
					__a			=					self.tokenizer.fairseq_tokens_to_ids
					self.tokenizer.save_pretrained(UpperCamelCase      )
					__a			=					NllbTokenizer.from_pretrained(UpperCamelCase      )
					self.assertDictEqual(new_tok.fairseq_tokens_to_ids      ,							UpperCamelCase      )







			@require_torch
			def 		UpperCamelCase__			(	self      )				->    Any:
					__a			=					self.tokenizer(
					    self.src_text      ,							text_target=self.tgt_text      ,							padding=UpperCamelCase      ,							truncation=UpperCamelCase      ,							max_length=len(self.expected_src_tokens      )      ,							return_tensors='pt'      ,							)
					__a			=					shift_tokens_right(
					    batch['labels']      ,							self.tokenizer.pad_token_id      ,							self.tokenizer.lang_code_to_id['ron_Latn']      )

					self.assertIsInstance(UpperCamelCase      ,							UpperCamelCase      )

					self.assertEqual((2, 15)      ,							batch.input_ids.shape      )
					self.assertEqual((2, 15)      ,							batch.attention_mask.shape      )
					__a			=					batch.input_ids.tolist()[0]
					self.assertListEqual(self.expected_src_tokens      ,							UpperCamelCase      )
					self.assertEqual(UpperCamelCase      ,							batch.decoder_input_ids[0, 0]      )  # EOS
					# Test that special tokens are reset
					self.assertEqual(self.tokenizer.prefix_tokens      ,							[EN_CODE]      )
					self.assertEqual(self.tokenizer.suffix_tokens      ,							[self.tokenizer.eos_token_id]      )







			def 		UpperCamelCase__			(	self      )				->    str:
					__a			=					self.tokenizer(self.src_text      ,							padding=UpperCamelCase      ,							truncation=UpperCamelCase      ,							max_length=3      ,							return_tensors='pt'      )
					__a			=					self.tokenizer(
					    text_target=self.tgt_text      ,							padding=UpperCamelCase      ,							truncation=UpperCamelCase      ,							max_length=10      ,							return_tensors='pt'      )
					__a			=					targets['input_ids']
					__a			=					shift_tokens_right(
					    UpperCamelCase      ,							self.tokenizer.pad_token_id      ,							decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]      ,							)

					self.assertEqual(batch.input_ids.shape[1]      ,							3      )
					self.assertEqual(batch.decoder_input_ids.shape[1]      ,							10      )







			@require_torch
			def 		UpperCamelCase__			(	self      )				->    Tuple:
					__a			=					self.tokenizer._build_translation_inputs(
					    'A test'      ,							return_tensors='pt'      ,							src_lang='eng_Latn'      ,							tgt_lang='fra_Latn'      )

					self.assertEqual(
					    nested_simplify(UpperCamelCase      )      ,							{
					        # A, test, EOS, en_XX
					        'input_ids': [[25_6047, 70, 7356, 2]],
					        'attention_mask': [[1, 1, 1, 1]],
					        # ar_AR
					        'forced_bos_token_id': 25_6057,
					    }      ,							)







			@require_torch
			def 		UpperCamelCase__			(	self      )				->    Any:
					__a			=					True
					__a			=					self.tokenizer(
					    'UN Chief says there is no military solution in Syria'      ,							src_lang='eng_Latn'      ,							tgt_lang='fra_Latn'      )
					self.assertEqual(
					    inputs.input_ids      ,							[1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2, 25_6047]      )

					__a			=					False
					__a			=					self.tokenizer(
					    'UN Chief says there is no military solution in Syria'      ,							src_lang='eng_Latn'      ,							tgt_lang='fra_Latn'      )
					self.assertEqual(
					    inputs.input_ids      ,							[25_6047, 1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2]      )









'''simple docstring'''


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_ 		=      logging.get_logger(__name__)

UpperCAmelCase_ 		=      "▁"

UpperCAmelCase_ 		=      {"vocab_file": "sentencepiece.bpe.model"}

UpperCAmelCase_ 		=      {
    "vocab_file": {
        "facebook/nllb-200-distilled-600M": (
            "https://huggingface.co/facebook/nllb-200-distilled-600M/blob/main/sentencepiece.bpe.model"
        ),
    }
}

UpperCAmelCase_ 		=      {
    "facebook/nllb-200-distilled-600M": 10_24,
}

# fmt: off
UpperCAmelCase_ 		=      ["ace_Arab", "ace_Latn", "acm_Arab", "acq_Arab", "aeb_Arab", "afr_Latn", "ajp_Arab", "aka_Latn", "amh_Ethi", "apc_Arab", "arb_Arab", "ars_Arab", "ary_Arab", "arz_Arab", "asm_Beng", "ast_Latn", "awa_Deva", "ayr_Latn", "azb_Arab", "azj_Latn", "bak_Cyrl", "bam_Latn", "ban_Latn", "bel_Cyrl", "bem_Latn", "ben_Beng", "bho_Deva", "bjn_Arab", "bjn_Latn", "bod_Tibt", "bos_Latn", "bug_Latn", "bul_Cyrl", "cat_Latn", "ceb_Latn", "ces_Latn", "cjk_Latn", "ckb_Arab", "crh_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "dik_Latn", "dyu_Latn", "dzo_Tibt", "ell_Grek", "eng_Latn", "epo_Latn", "est_Latn", "eus_Latn", "ewe_Latn", "fao_Latn", "pes_Arab", "fij_Latn", "fin_Latn", "fon_Latn", "fra_Latn", "fur_Latn", "fuv_Latn", "gla_Latn", "gle_Latn", "glg_Latn", "grn_Latn", "guj_Gujr", "hat_Latn", "hau_Latn", "heb_Hebr", "hin_Deva", "hne_Deva", "hrv_Latn", "hun_Latn", "hye_Armn", "ibo_Latn", "ilo_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jav_Latn", "jpn_Jpan", "kab_Latn", "kac_Latn", "kam_Latn", "kan_Knda", "kas_Arab", "kas_Deva", "kat_Geor", "knc_Arab", "knc_Latn", "kaz_Cyrl", "kbp_Latn", "kea_Latn", "khm_Khmr", "kik_Latn", "kin_Latn", "kir_Cyrl", "kmb_Latn", "kon_Latn", "kor_Hang", "kmr_Latn", "lao_Laoo", "lvs_Latn", "lij_Latn", "lim_Latn", "lin_Latn", "lit_Latn", "lmo_Latn", "ltg_Latn", "ltz_Latn", "lua_Latn", "lug_Latn", "luo_Latn", "lus_Latn", "mag_Deva", "mai_Deva", "mal_Mlym", "mar_Deva", "min_Latn", "mkd_Cyrl", "plt_Latn", "mlt_Latn", "mni_Beng", "khk_Cyrl", "mos_Latn", "mri_Latn", "zsm_Latn", "mya_Mymr", "nld_Latn", "nno_Latn", "nob_Latn", "npi_Deva", "nso_Latn", "nus_Latn", "nya_Latn", "oci_Latn", "gaz_Latn", "ory_Orya", "pag_Latn", "pan_Guru", "pap_Latn", "pol_Latn", "por_Latn", "prs_Arab", "pbt_Arab", "quy_Latn", "ron_Latn", "run_Latn", "rus_Cyrl", "sag_Latn", "san_Deva", "sat_Beng", "scn_Latn", "shn_Mymr", "sin_Sinh", "slk_Latn", "slv_Latn", "smo_Latn", "sna_Latn", "snd_Arab", "som_Latn", "sot_Latn", "spa_Latn", "als_Latn", "srd_Latn", "srp_Cyrl", "ssw_Latn", "sun_Latn", "swe_Latn", "swh_Latn", "szl_Latn", "tam_Taml", "tat_Cyrl", "tel_Telu", "tgk_Cyrl", "tgl_Latn", "tha_Thai", "tir_Ethi", "taq_Latn", "taq_Tfng", "tpi_Latn", "tsn_Latn", "tso_Latn", "tuk_Latn", "tum_Latn", "tur_Latn", "twi_Latn", "tzm_Tfng", "uig_Arab", "ukr_Cyrl", "umb_Latn", "urd_Arab", "uzn_Latn", "vec_Latn", "vie_Latn", "war_Latn", "wol_Latn", "xho_Latn", "ydd_Hebr", "yor_Latn", "yue_Hant", "zho_Hans", "zho_Hant", "zul_Latn"]





class    __lowercase						(  __magic_name__ ):
   _a							=      VOCAB_FILES_NAMES
   _a							=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _a							=      PRETRAINED_VOCAB_FILES_MAP
   _a							=      ["""input_ids""", """attention_mask"""]

   _a							=      []
   _a							=      []







   def __init__(	self      ,							UpperCamelCase      ,							UpperCamelCase="<s>"      ,							UpperCamelCase="</s>"      ,							UpperCamelCase="</s>"      ,							UpperCamelCase="<s>"      ,							UpperCamelCase="<unk>"      ,							UpperCamelCase="<pad>"      ,							UpperCamelCase="<mask>"      ,							UpperCamelCase=None      ,							UpperCamelCase=None      ,							UpperCamelCase=None      ,							UpperCamelCase = None      ,							UpperCamelCase=None      ,							UpperCamelCase=False      ,							**UpperCamelCase      ,							)				->    Tuple:
     # Mask token behave like a normal word, i.e. include the space before it
     __a			=					AddedToken(UpperCamelCase      ,							lstrip=UpperCamelCase      ,							rstrip=UpperCamelCase      ) if isinstance(UpperCamelCase      ,							UpperCamelCase      ) else mask_token

     __a			=					{} if sp_model_kwargs is None else sp_model_kwargs
     __a			=					legacy_behaviour

     super().__init__(
         bos_token=UpperCamelCase      ,							eos_token=UpperCamelCase      ,							unk_token=UpperCamelCase      ,							sep_token=UpperCamelCase      ,							cls_token=UpperCamelCase      ,							pad_token=UpperCamelCase      ,							mask_token=UpperCamelCase      ,							tokenizer_file=UpperCamelCase      ,							src_lang=UpperCamelCase      ,							tgt_lang=UpperCamelCase      ,							additional_special_tokens=UpperCamelCase      ,							sp_model_kwargs=self.sp_model_kwargs      ,							legacy_behaviour=UpperCamelCase      ,							**UpperCamelCase      ,							)

     __a			=					spm.SentencePieceProcessor(**self.sp_model_kwargs      )
     self.sp_model.Load(str(UpperCamelCase      )      )
     __a			=					vocab_file
     # Original fairseq vocab and spm vocab must be "aligned":
     # Vocab    |    0    |    1    |   2    |    3    |  4   |  5   |  6   |   7  |   8  |  9
     # -------- | ------- | ------- | ------ | ------- | ---- | ---- | ---- | ---- | ---- | ----
     # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | 'an' | '▁n' | '▁m' | '▁t' | '▁k' | '▁a'
     # spm      | '<unk>' | '<s>'   | '</s>' | 'an'    | '▁n' | '▁m' | '▁t' | '▁k' | '▁a' | '▁s'

     # Mimic fairseq token-to-id alignment for the first 4 token
     __a			=					{'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

     # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
     __a			=					1

     __a			=					len(self.sp_model      )
     __a			=					{
         code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(UpperCamelCase      )
     }
     __a			=					{v: k for k, v in self.lang_code_to_id.items()}
     __a			=					len(self.sp_model      ) + len(self.lang_code_to_id      ) + self.fairseq_offset

     self.fairseq_tokens_to_ids.update(self.lang_code_to_id      )
     __a			=					{v: k for k, v in self.fairseq_tokens_to_ids.items()}
     __a			=					list(self.lang_code_to_id.keys()      )

     if additional_special_tokens is not None:
       # Only add those special tokens if they are not already there.
       self._additional_special_tokens.extend(
           [t for t in additional_special_tokens if t not in self._additional_special_tokens]      )

     __a			=					src_lang if src_lang is not None else 'eng_Latn'
     __a			=					self.lang_code_to_id[self._src_lang]
     __a			=					tgt_lang
     self.set_src_lang_special_tokens(self._src_lang      )







   def __getstate__(	self      )				->    Union[str, Any]:
     __a			=					self.__dict__.copy()
     __a			=					None
     __a			=					self.sp_model.serialized_model_proto()
     return state







   def __setstate__(	self      ,							UpperCamelCase      )				->    str:
     __a			=					d

     # for backward compatibility
     if not hasattr(self      ,							'sp_model_kwargs'      ):
       __a			=					{}

     __a			=					spm.SentencePieceProcessor(**self.sp_model_kwargs      )
     self.sp_model.LoadFromSerializedProto(self.sp_model_proto      )







   @property
   def 		UpperCamelCase__			(	self      )				->    str:
     return len(self.sp_model      ) + len(self.lang_code_to_id      ) + self.fairseq_offset + 1  # Plus 1 for the mask token







   @property
   def 		UpperCamelCase__			(	self      )				->    str:
     return self._src_lang







   @src_lang.setter
   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      )				->    None:
     __a			=					new_src_lang
     self.set_src_lang_special_tokens(self._src_lang      )







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase = None      ,							UpperCamelCase = False      )				->    List[int]:

     if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=UpperCamelCase      ,							token_ids_a=UpperCamelCase      ,							already_has_special_tokens=UpperCamelCase      )

     __a			=					[1] * len(self.prefix_tokens      )
     __a			=					[1] * len(self.suffix_tokens      )
     if token_ids_a is None:
       return prefix_ones + ([0] * len(UpperCamelCase      )) + suffix_ones
     return prefix_ones + ([0] * len(UpperCamelCase      )) + ([0] * len(UpperCamelCase      )) + suffix_ones







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase = None      )				->    List[int]:
     if token_ids_a is None:
       return self.prefix_tokens + token_ids_a + self.suffix_tokens
     # We don't expect to process pairs, but leave the pair logic for API consistency
     return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase = None      )				->    List[int]:
     __a			=					[self.sep_token_id]
     __a			=					[self.cls_token_id]

     if token_ids_a is None:
       return len(cls + token_ids_a + sep      ) * [0]
     return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase      ,							UpperCamelCase      ,							UpperCamelCase      ,							**UpperCamelCase      )				->    Dict:
     if src_lang is None or tgt_lang is None:
       raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'      )
     __a			=					src_lang
     __a			=					self(UpperCamelCase      ,							add_special_tokens=UpperCamelCase      ,							return_tensors=UpperCamelCase      ,							**UpperCamelCase      )
     __a			=					self.convert_tokens_to_ids(UpperCamelCase      )
     __a			=					tgt_lang_id
     return inputs







   def 		UpperCamelCase__			(	self      )				->    Union[str, Any]:
     __a			=					{self.convert_ids_to_tokens(UpperCamelCase      ): i for i in range(self.vocab_size      )}
     vocab.update(self.added_tokens_encoder      )
     return vocab







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      )				->    List[str]:
     return self.sp_model.encode(UpperCamelCase      ,							out_type=UpperCamelCase      )







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      )				->    Optional[Any]:
     if token in self.fairseq_tokens_to_ids:
       return self.fairseq_tokens_to_ids[token]
     __a			=					self.sp_model.PieceToId(UpperCamelCase      )

     # Need to return unknown token if the SP model returned 0
     return spm_id + self.fairseq_offset if spm_id else self.unk_token_id







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      )				->    Any:
     if index in self.fairseq_ids_to_tokens:
       return self.fairseq_ids_to_tokens[index]
     return self.sp_model.IdToPiece(index - self.fairseq_offset      )







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      )				->    str:
     __a			=					''.join(UpperCamelCase      ).replace(UpperCamelCase      ,							' '      ).strip()
     return out_string







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase = None      )				->    Tuple[str]:
     if not os.path.isdir(UpperCamelCase      ):
       logger.error(f"Vocabulary path ({save_directory}) should be a directory"      )
       return
     __a			=					os.path.join(
         UpperCamelCase      ,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )

     if os.path.abspath(self.vocab_file      ) != os.path.abspath(UpperCamelCase      ) and os.path.isfile(self.vocab_file      ):
       copyfile(self.vocab_file      ,							UpperCamelCase      )
     elif not os.path.isfile(self.vocab_file      ):
       with open(UpperCamelCase      ,							'wb'      ) as fi:
         __a			=					self.sp_model.serialized_model_proto()
         fi.write(UpperCamelCase      )

     return (out_vocab_file,)







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase = "eng_Latn"      ,							UpperCamelCase = None      ,							UpperCamelCase = "fra_Latn"      ,							**UpperCamelCase      ,							)				->    BatchEncoding:
     __a			=					src_lang
     __a			=					tgt_lang
     return super().prepare_seqaseq_batch(UpperCamelCase      ,							UpperCamelCase      ,							**UpperCamelCase      )







   def 		UpperCamelCase__			(	self      )				->    List[str]:
     return self.set_src_lang_special_tokens(self.src_lang      )







   def 		UpperCamelCase__			(	self      )				->    int:
     return self.set_tgt_lang_special_tokens(self.tgt_lang      )







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      )				->    None:
     __a			=					self.lang_code_to_id[src_lang]
     if self.legacy_behaviour:
       __a			=					[]
       __a			=					[self.eos_token_id, self.cur_lang_code]
     else:
       __a			=					[self.cur_lang_code]
       __a			=					[self.eos_token_id]







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      )				->    None:
     __a			=					self.lang_code_to_id[lang]
     if self.legacy_behaviour:
       __a			=					[]
       __a			=					[self.eos_token_id, self.cur_lang_code]
     else:
       __a			=					[self.cur_lang_code]
       __a			=					[self.eos_token_id]



