
'''simple docstring'''



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
     from .tokenization_mbart import MBartTokenizer
else:
     UpperCamelCase__      :		List[Any]       							=  None


UpperCamelCase__      :		List[str]       							=  logging.get_logger(__name__)


UpperCamelCase__      :		Optional[Any]       							=  {"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

UpperCamelCase__      :		Tuple       							=  {
    "vocab_file": {
        "facebook/mbart-large-en-ro": (
            "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"
        ),
        "facebook/mbart-large-cc25": (
            "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/mbart-large-en-ro": "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json",
        "facebook/mbart-large-cc25": "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json",
    },
}

UpperCamelCase__      :		Optional[int]       							=  {
    "facebook/mbart-large-en-ro": 1_024,
    "facebook/mbart-large-cc25": 1_024,
}

# fmt: off
UpperCamelCase__      :		Optional[int]       							=  ["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN"]

class    _a				(_lowerCamelCase):





       """simple docstring"""




       SCREAMING_SNAKE_CASE      =  VOCAB_FILES_NAMES
       SCREAMING_SNAKE_CASE      =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       SCREAMING_SNAKE_CASE      =  PRETRAINED_VOCAB_FILES_MAP
       SCREAMING_SNAKE_CASE      =  ['input_ids', 'attention_mask']
       SCREAMING_SNAKE_CASE      =  MBartTokenizer

       SCREAMING_SNAKE_CASE      =  []
       SCREAMING_SNAKE_CASE      =  []






       def __init__(							self       ,							A__=None       ,							A__=None       ,							A__="<s>"       ,							A__="</s>"       ,							A__="</s>"       ,							A__="<s>"       ,							A__="<unk>"       ,							A__="<pad>"       ,							A__="<mask>"       ,							A__=None       ,							A__=None       ,							A__=None       ,							**A__       ,							)    ->			str:
          # Mask token behave like a normal word, i.e. include the space before it
          _SCREAMING_SNAKE_CASE      =				AddedToken(A__       ,							lstrip=A__       ,							rstrip=A__      ) if isinstance(A__       ,							A__      ) else mask_token

          super().__init__(
              vocab_file=A__       ,							tokenizer_file=A__       ,							bos_token=A__       ,							eos_token=A__       ,							sep_token=A__       ,							cls_token=A__       ,							unk_token=A__       ,							pad_token=A__       ,							mask_token=A__       ,							src_lang=A__       ,							tgt_lang=A__       ,							additional_special_tokens=A__       ,							**A__       ,							)

          _SCREAMING_SNAKE_CASE      =				vocab_file
          _SCREAMING_SNAKE_CASE      =				False if not self.vocab_file else True

          _SCREAMING_SNAKE_CASE      =				FAIRSEQ_LANGUAGE_CODES.copy()

          if additional_special_tokens is not None:
             # Only add those special tokens if they are not already there.
             _additional_special_tokens.extend(
                 [t for t in additional_special_tokens if t not in _additional_special_tokens]      )

          self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}      )
          _SCREAMING_SNAKE_CASE      =				{
              lang_code: self.convert_tokens_to_ids(A__      ) for lang_code in FAIRSEQ_LANGUAGE_CODES
          }

          _SCREAMING_SNAKE_CASE      =				src_lang if src_lang is not None else """en_XX"""
          _SCREAMING_SNAKE_CASE      =				self.convert_tokens_to_ids(self._src_lang      )
          _SCREAMING_SNAKE_CASE      =				tgt_lang
          self.set_src_lang_special_tokens(self._src_lang      )






       @property
       def        UpperCamelCase				(							self      )    ->			str:
          return self._src_lang






       @src_lang.setter
       def        UpperCamelCase				(							self       ,							A__      )    ->			None:
          _SCREAMING_SNAKE_CASE      =				new_src_lang
          self.set_src_lang_special_tokens(self._src_lang      )






       def        UpperCamelCase				(							self       ,							A__       ,							A__ = None      )    ->			List[int]:
          if token_ids_a is None:
             return self.prefix_tokens + token_ids_a + self.suffix_tokens
          # We don't expect to process pairs, but leave the pair logic for API consistency
          return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






       def        UpperCamelCase				(							self       ,							A__       ,							A__ = None      )    ->			List[int]:
          _SCREAMING_SNAKE_CASE      =				[self.sep_token_id]
          _SCREAMING_SNAKE_CASE      =				[self.cls_token_id]

          if token_ids_a is None:
             return len(cls + token_ids_a + sep      ) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]






       def        UpperCamelCase				(							self       ,							A__       ,							A__       ,							A__       ,							A__       ,							**A__      )    ->			Tuple:
          if src_lang is None or tgt_lang is None:
             raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""      )
          _SCREAMING_SNAKE_CASE      =				src_lang
          _SCREAMING_SNAKE_CASE      =				self(A__       ,							add_special_tokens=A__       ,							return_tensors=A__       ,							**A__      )
          _SCREAMING_SNAKE_CASE      =				self.convert_tokens_to_ids(A__      )
          _SCREAMING_SNAKE_CASE      =				tgt_lang_id
          return inputs






       def        UpperCamelCase				(							self       ,							A__       ,							A__ = "en_XX"       ,							A__ = None       ,							A__ = "ro_RO"       ,							**A__       ,							)    ->			BatchEncoding:
          _SCREAMING_SNAKE_CASE      =				src_lang
          _SCREAMING_SNAKE_CASE      =				tgt_lang
          return super().prepare_seqaseq_batch(A__       ,							A__       ,							**A__      )






       def        UpperCamelCase				(							self      )    ->			Any:
          return self.set_src_lang_special_tokens(self.src_lang      )






       def        UpperCamelCase				(							self      )    ->			Tuple:
          return self.set_tgt_lang_special_tokens(self.tgt_lang      )






       def        UpperCamelCase				(							self       ,							A__      )    ->			None:
          _SCREAMING_SNAKE_CASE      =				self.convert_tokens_to_ids(A__      )
          _SCREAMING_SNAKE_CASE      =				[]
          _SCREAMING_SNAKE_CASE      =				[self.eos_token_id, self.cur_lang_code]

          _SCREAMING_SNAKE_CASE      =				self.convert_ids_to_tokens(self.prefix_tokens      )
          _SCREAMING_SNAKE_CASE      =				self.convert_ids_to_tokens(self.suffix_tokens      )

          _SCREAMING_SNAKE_CASE      =				processors.TemplateProcessing(
              single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str       ,							pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str       ,							special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str       ,							self.prefix_tokens + self.suffix_tokens      )      )       ,							)






       def        UpperCamelCase				(							self       ,							A__      )    ->			None:
          _SCREAMING_SNAKE_CASE      =				self.convert_tokens_to_ids(A__      )
          _SCREAMING_SNAKE_CASE      =				[]
          _SCREAMING_SNAKE_CASE      =				[self.eos_token_id, self.cur_lang_code]

          _SCREAMING_SNAKE_CASE      =				self.convert_ids_to_tokens(self.prefix_tokens      )
          _SCREAMING_SNAKE_CASE      =				self.convert_ids_to_tokens(self.suffix_tokens      )

          _SCREAMING_SNAKE_CASE      =				processors.TemplateProcessing(
              single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str       ,							pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str       ,							special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str       ,							self.prefix_tokens + self.suffix_tokens      )      )       ,							)






       def        UpperCamelCase				(							self       ,							A__       ,							A__ = None      )    ->			Tuple[str]:
          if not self.can_save_slow_tokenizer:
             raise ValueError(
                 """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                 """tokenizer."""      )

          if not os.path.isdir(A__      ):
             logger.error(F"Vocabulary path ({save_directory}) should be a directory."      )
             return
          _SCREAMING_SNAKE_CASE      =				os.path.join(
              A__       ,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]      )

          if os.path.abspath(self.vocab_file      ) != os.path.abspath(A__      ):
             copyfile(self.vocab_file       ,							A__      )

          return (out_vocab_file,)








'''simple docstring'''



import unittest

import numpy as np

from transformers.testing_utils import require_flax, require_tf, require_torch
from transformers.utils import (
    expand_dims,
    flatten_dict,
    is_flax_available,
    is_tf_available,
    is_torch_available,
    reshape,
    squeeze,
    transpose,
)


if is_flax_available():
     import jax.numpy as jnp

if is_tf_available():
     import tensorflow as tf

if is_torch_available():
     import torch

class    _a				(unittest.TestCase):





       """simple docstring"""




       def        UpperCamelCase				(							self      )    ->			Optional[Any]:
          _SCREAMING_SNAKE_CASE      =				{
              """task_specific_params""": {
                  """summarization""": {"""length_penalty""": 1.0, """max_length""": 1_28, """min_length""": 12, """num_beams""": 4},
                  """summarization_cnn""": {"""length_penalty""": 2.0, """max_length""": 1_42, """min_length""": 56, """num_beams""": 4},
                  """summarization_xsum""": {"""length_penalty""": 1.0, """max_length""": 62, """min_length""": 11, """num_beams""": 6},
              }
          }
          _SCREAMING_SNAKE_CASE      =				{
              """task_specific_params.summarization.length_penalty""": 1.0,
              """task_specific_params.summarization.max_length""": 1_28,
              """task_specific_params.summarization.min_length""": 12,
              """task_specific_params.summarization.num_beams""": 4,
              """task_specific_params.summarization_cnn.length_penalty""": 2.0,
              """task_specific_params.summarization_cnn.max_length""": 1_42,
              """task_specific_params.summarization_cnn.min_length""": 56,
              """task_specific_params.summarization_cnn.num_beams""": 4,
              """task_specific_params.summarization_xsum.length_penalty""": 1.0,
              """task_specific_params.summarization_xsum.max_length""": 62,
              """task_specific_params.summarization_xsum.min_length""": 11,
              """task_specific_params.summarization_xsum.num_beams""": 6,
          }

          self.assertEqual(flatten_dict(A__      )       ,							A__      )






       def        UpperCamelCase				(							self      )    ->			int:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          self.assertTrue(np.allclose(transpose(A__      )       ,							x.transpose()      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4       ,							5      )
          self.assertTrue(np.allclose(transpose(A__       ,							axes=(1, 2, 0)      )       ,							x.transpose((1, 2, 0)      )      )      )






       @require_torch
       def        UpperCamelCase				(							self      )    ->			Optional[Any]:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				torch.tensor(A__      )
          self.assertTrue(np.allclose(transpose(A__      )       ,							transpose(A__      ).numpy()      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4       ,							5      )
          _SCREAMING_SNAKE_CASE      =				torch.tensor(A__      )
          self.assertTrue(np.allclose(transpose(A__       ,							axes=(1, 2, 0)      )       ,							transpose(A__       ,							axes=(1, 2, 0)      ).numpy()      )      )






       @require_tf
       def        UpperCamelCase				(							self      )    ->			Optional[int]:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				tf.constant(A__      )
          self.assertTrue(np.allclose(transpose(A__      )       ,							transpose(A__      ).numpy()      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4       ,							5      )
          _SCREAMING_SNAKE_CASE      =				tf.constant(A__      )
          self.assertTrue(np.allclose(transpose(A__       ,							axes=(1, 2, 0)      )       ,							transpose(A__       ,							axes=(1, 2, 0)      ).numpy()      )      )






       @require_flax
       def        UpperCamelCase				(							self      )    ->			List[str]:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				jnp.array(A__      )
          self.assertTrue(np.allclose(transpose(A__      )       ,							np.asarray(transpose(A__      )      )      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4       ,							5      )
          _SCREAMING_SNAKE_CASE      =				jnp.array(A__      )
          self.assertTrue(np.allclose(transpose(A__       ,							axes=(1, 2, 0)      )       ,							np.asarray(transpose(A__       ,							axes=(1, 2, 0)      )      )      )      )






       def        UpperCamelCase				(							self      )    ->			Optional[int]:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          self.assertTrue(np.allclose(reshape(A__       ,							(4, 3)      )       ,							np.reshape(A__       ,							(4, 3)      )      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4       ,							5      )
          self.assertTrue(np.allclose(reshape(A__       ,							(12, 5)      )       ,							np.reshape(A__       ,							(12, 5)      )      )      )






       @require_torch
       def        UpperCamelCase				(							self      )    ->			Optional[int]:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				torch.tensor(A__      )
          self.assertTrue(np.allclose(reshape(A__       ,							(4, 3)      )       ,							reshape(A__       ,							(4, 3)      ).numpy()      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4       ,							5      )
          _SCREAMING_SNAKE_CASE      =				torch.tensor(A__      )
          self.assertTrue(np.allclose(reshape(A__       ,							(12, 5)      )       ,							reshape(A__       ,							(12, 5)      ).numpy()      )      )






       @require_tf
       def        UpperCamelCase				(							self      )    ->			Tuple:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				tf.constant(A__      )
          self.assertTrue(np.allclose(reshape(A__       ,							(4, 3)      )       ,							reshape(A__       ,							(4, 3)      ).numpy()      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4       ,							5      )
          _SCREAMING_SNAKE_CASE      =				tf.constant(A__      )
          self.assertTrue(np.allclose(reshape(A__       ,							(12, 5)      )       ,							reshape(A__       ,							(12, 5)      ).numpy()      )      )






       @require_flax
       def        UpperCamelCase				(							self      )    ->			List[Any]:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				jnp.array(A__      )
          self.assertTrue(np.allclose(reshape(A__       ,							(4, 3)      )       ,							np.asarray(reshape(A__       ,							(4, 3)      )      )      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4       ,							5      )
          _SCREAMING_SNAKE_CASE      =				jnp.array(A__      )
          self.assertTrue(np.allclose(reshape(A__       ,							(12, 5)      )       ,							np.asarray(reshape(A__       ,							(12, 5)      )      )      )      )






       def        UpperCamelCase				(							self      )    ->			Any:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(1       ,							3       ,							4      )
          self.assertTrue(np.allclose(squeeze(A__      )       ,							np.squeeze(A__      )      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(1       ,							4       ,							1       ,							5      )
          self.assertTrue(np.allclose(squeeze(A__       ,							axis=2      )       ,							np.squeeze(A__       ,							axis=2      )      )      )






       @require_torch
       def        UpperCamelCase				(							self      )    ->			Any:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(1       ,							3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				torch.tensor(A__      )
          self.assertTrue(np.allclose(squeeze(A__      )       ,							squeeze(A__      ).numpy()      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(1       ,							4       ,							1       ,							5      )
          _SCREAMING_SNAKE_CASE      =				torch.tensor(A__      )
          self.assertTrue(np.allclose(squeeze(A__       ,							axis=2      )       ,							squeeze(A__       ,							axis=2      ).numpy()      )      )






       @require_tf
       def        UpperCamelCase				(							self      )    ->			List[str]:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(1       ,							3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				tf.constant(A__      )
          self.assertTrue(np.allclose(squeeze(A__      )       ,							squeeze(A__      ).numpy()      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(1       ,							4       ,							1       ,							5      )
          _SCREAMING_SNAKE_CASE      =				tf.constant(A__      )
          self.assertTrue(np.allclose(squeeze(A__       ,							axis=2      )       ,							squeeze(A__       ,							axis=2      ).numpy()      )      )






       @require_flax
       def        UpperCamelCase				(							self      )    ->			Optional[int]:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(1       ,							3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				jnp.array(A__      )
          self.assertTrue(np.allclose(squeeze(A__      )       ,							np.asarray(squeeze(A__      )      )      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(1       ,							4       ,							1       ,							5      )
          _SCREAMING_SNAKE_CASE      =				jnp.array(A__      )
          self.assertTrue(np.allclose(squeeze(A__       ,							axis=2      )       ,							np.asarray(squeeze(A__       ,							axis=2      )      )      )      )






       def        UpperCamelCase				(							self      )    ->			Optional[Any]:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          self.assertTrue(np.allclose(expand_dims(A__       ,							axis=1      )       ,							np.expand_dims(A__       ,							axis=1      )      )      )






       @require_torch
       def        UpperCamelCase				(							self      )    ->			Optional[int]:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				torch.tensor(A__      )
          self.assertTrue(np.allclose(expand_dims(A__       ,							axis=1      )       ,							expand_dims(A__       ,							axis=1      ).numpy()      )      )






       @require_tf
       def        UpperCamelCase				(							self      )    ->			str:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				tf.constant(A__      )
          self.assertTrue(np.allclose(expand_dims(A__       ,							axis=1      )       ,							expand_dims(A__       ,							axis=1      ).numpy()      )      )






       @require_flax
       def        UpperCamelCase				(							self      )    ->			Any:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				jnp.array(A__      )
          self.assertTrue(np.allclose(expand_dims(A__       ,							axis=1      )       ,							np.asarray(expand_dims(A__       ,							axis=1      )      )      )      )






