


'''simple docstring'''


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
						from .tokenization_mbart import MBartTokenizer
else:
						lowerCamelCase__    	=     None


lowerCamelCase__    	=     logging.get_logger(__name__)


lowerCamelCase__    	=     {'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

lowerCamelCase__    	=     {
    'vocab_file': {
        'facebook/mbart-large-en-ro': (
            'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'
        ),
        'facebook/mbart-large-cc25': (
            'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/mbart-large-en-ro': 'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json',
        'facebook/mbart-large-cc25': 'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json',
    },
}

lowerCamelCase__    	=     {
    'facebook/mbart-large-en-ro': 1_024,
    'facebook/mbart-large-cc25': 1_024,
}

# fmt: off
lowerCamelCase__    	=     ['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN']






class 	lowerCAmelCase__			(			UpperCAmelCase__	):
			lowerCAmelCase :	List[str]          =     VOCAB_FILES_NAMES
			lowerCAmelCase :	Tuple          =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase :	str          =     PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase :	Any          =     ["input_ids", "attention_mask"]
			lowerCAmelCase :	int          =     MBartTokenizer

			lowerCAmelCase :	List[int]          =     []
			lowerCAmelCase :	List[int]          =     []




			def __init__(		self  :     int	,  lowerCamelCase__  :     Any=None	,  lowerCamelCase__  :     List[Any]=None	,  lowerCamelCase__  :     int="<s>"	,  lowerCamelCase__  :     str="</s>"	,  lowerCamelCase__  :     Tuple="</s>"	,  lowerCamelCase__  :     str="<s>"	,  lowerCamelCase__  :     List[Any]="<unk>"	,  lowerCamelCase__  :     Dict="<pad>"	,  lowerCamelCase__  :     List[str]="<mask>"	,  lowerCamelCase__  :     Optional[int]=None	,  lowerCamelCase__  :     int=None	,  lowerCamelCase__  :     List[str]=None	,  **lowerCamelCase__  :     str	,  )   ->Optional[Any]:



					'''simple docstring'''







					_UpperCAmelCase		:       List[Any]								=      AddedToken(lowerCamelCase__	,  lstrip=lowerCamelCase__	,  rstrip=lowerCamelCase__ ) if isinstance(lowerCamelCase__	,  lowerCamelCase__ ) else mask_token

					super().__init__(
					    vocab_file=lowerCamelCase__	,  tokenizer_file=lowerCamelCase__	,  bos_token=lowerCamelCase__	,  eos_token=lowerCamelCase__	,  sep_token=lowerCamelCase__	,  cls_token=lowerCamelCase__	,  unk_token=lowerCamelCase__	,  pad_token=lowerCamelCase__	,  mask_token=lowerCamelCase__	,  src_lang=lowerCamelCase__	,  tgt_lang=lowerCamelCase__	,  additional_special_tokens=lowerCamelCase__	,  **lowerCamelCase__	,  )

					_UpperCAmelCase		:       Tuple								=      vocab_file
					_UpperCAmelCase		:       str								=      False if not self.vocab_file else True

					_UpperCAmelCase		:       Union[str, Any]								=      FAIRSEQ_LANGUAGE_CODES.copy()

					if additional_special_tokens is not None:
							# Only add those special tokens if they are not already there.
							_additional_special_tokens.extend(
							    [t for t in additional_special_tokens if t not in _additional_special_tokens] )

					self.add_special_tokens({"additional_special_tokens": _additional_special_tokens} )
					_UpperCAmelCase		:       Union[str, Any]								=      {
					    lang_code: self.convert_tokens_to_ids(lowerCamelCase__ ) for lang_code in FAIRSEQ_LANGUAGE_CODES
					}

					_UpperCAmelCase		:       int								=      src_lang if src_lang is not None else "en_XX"
					_UpperCAmelCase		:       List[str]								=      self.convert_tokens_to_ids(self._src_lang )
					_UpperCAmelCase		:       Union[str, Any]								=      tgt_lang
					self.set_src_lang_special_tokens(self._src_lang )




			@property
			def 							lowerCAmelCase__    (		self  :     str )   ->str:



					'''simple docstring'''







					return self._src_lang




			@src_lang.setter
			def 							lowerCAmelCase__    (		self  :     Union[str, Any]	,  lowerCamelCase__  :     str )   ->None:



					'''simple docstring'''







					_UpperCAmelCase		:       List[str]								=      new_src_lang
					self.set_src_lang_special_tokens(self._src_lang )




			def 							lowerCAmelCase__    (		self  :     str	,  lowerCamelCase__  :     List[int]	,  lowerCamelCase__  :     Optional[List[int]] = None )   ->List[int]:



					'''simple docstring'''







					if token_ids_a is None:
							return self.prefix_tokens + token_ids_a + self.suffix_tokens
					# We don't expect to process pairs, but leave the pair logic for API consistency
					return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




			def 							lowerCAmelCase__    (		self  :     Tuple	,  lowerCamelCase__  :     List[int]	,  lowerCamelCase__  :     Optional[List[int]] = None )   ->List[int]:



					'''simple docstring'''







					_UpperCAmelCase		:       Tuple								=      [self.sep_token_id]
					_UpperCAmelCase		:       Optional[int]								=      [self.cls_token_id]

					if token_ids_a is None:
							return len(cls + token_ids_a + sep ) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]




			def 							lowerCAmelCase__    (		self  :     Any	,  lowerCamelCase__  :     Union[str, Any]	,  lowerCamelCase__  :     str	,  lowerCamelCase__  :     Optional[str]	,  lowerCamelCase__  :     Optional[str]	,  **lowerCamelCase__  :     str )   ->Dict:



					'''simple docstring'''







					if src_lang is None or tgt_lang is None:
							raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model" )
					_UpperCAmelCase		:       List[str]								=      src_lang
					_UpperCAmelCase		:       Any								=      self(lowerCamelCase__	,  add_special_tokens=lowerCamelCase__	,  return_tensors=lowerCamelCase__	,  **lowerCamelCase__ )
					_UpperCAmelCase		:       Any								=      self.convert_tokens_to_ids(lowerCamelCase__ )
					_UpperCAmelCase		:       List[str]								=      tgt_lang_id
					return inputs




			def 							lowerCAmelCase__    (		self  :     int	,  lowerCamelCase__  :     List[str]	,  lowerCamelCase__  :     str = "en_XX"	,  lowerCamelCase__  :     Optional[List[str]] = None	,  lowerCamelCase__  :     str = "ro_RO"	,  **lowerCamelCase__  :     List[Any]	,  )   ->BatchEncoding:



					'''simple docstring'''







					_UpperCAmelCase		:       int								=      src_lang
					_UpperCAmelCase		:       Optional[int]								=      tgt_lang
					return super().prepare_seqaseq_batch(lowerCamelCase__	,  lowerCamelCase__	,  **lowerCamelCase__ )




			def 							lowerCAmelCase__    (		self  :     Optional[Any] )   ->List[str]:



					'''simple docstring'''







					return self.set_src_lang_special_tokens(self.src_lang )




			def 							lowerCAmelCase__    (		self  :     Union[str, Any] )   ->str:



					'''simple docstring'''







					return self.set_tgt_lang_special_tokens(self.tgt_lang )




			def 							lowerCAmelCase__    (		self  :     Dict	,  lowerCamelCase__  :     int )   ->None:



					'''simple docstring'''







					_UpperCAmelCase		:       Tuple								=      self.convert_tokens_to_ids(lowerCamelCase__ )
					_UpperCAmelCase		:       str								=      []
					_UpperCAmelCase		:       List[Any]								=      [self.eos_token_id, self.cur_lang_code]

					_UpperCAmelCase		:       Optional[int]								=      self.convert_ids_to_tokens(self.prefix_tokens )
					_UpperCAmelCase		:       List[str]								=      self.convert_ids_to_tokens(self.suffix_tokens )

					_UpperCAmelCase		:       List[str]								=      processors.TemplateProcessing(
					    single=prefix_tokens_str + ["$A"] + suffix_tokens_str	,  pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str	,  special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str	,  self.prefix_tokens + self.suffix_tokens ) )	,  )




			def 							lowerCAmelCase__    (		self  :     str	,  lowerCamelCase__  :     str )   ->None:



					'''simple docstring'''







					_UpperCAmelCase		:       Union[str, Any]								=      self.convert_tokens_to_ids(lowerCamelCase__ )
					_UpperCAmelCase		:       List[str]								=      []
					_UpperCAmelCase		:       List[str]								=      [self.eos_token_id, self.cur_lang_code]

					_UpperCAmelCase		:       Any								=      self.convert_ids_to_tokens(self.prefix_tokens )
					_UpperCAmelCase		:       Union[str, Any]								=      self.convert_ids_to_tokens(self.suffix_tokens )

					_UpperCAmelCase		:       List[str]								=      processors.TemplateProcessing(
					    single=prefix_tokens_str + ["$A"] + suffix_tokens_str	,  pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str	,  special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str	,  self.prefix_tokens + self.suffix_tokens ) )	,  )






			def 							lowerCAmelCase__    (		self  :     Union[str, Any]	,  lowerCamelCase__  :     str	,  lowerCamelCase__  :     Optional[str] = None )   ->Tuple[str]:



					'''simple docstring'''







					if not self.can_save_slow_tokenizer:
							raise ValueError(
							    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
							    "tokenizer." )

					if not os.path.isdir(lowerCamelCase__ ):
							logger.error(F"""Vocabulary path ({save_directory}) should be a directory.""" )
							return
					_UpperCAmelCase		:       Dict								=      os.path.join(
					    lowerCamelCase__	,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )

					if os.path.abspath(self.vocab_file ) != os.path.abspath(lowerCamelCase__ ):
							copyfile(self.vocab_file	,  lowerCamelCase__ )

					return (out_vocab_file,)







'''simple docstring'''


# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


lowerCamelCase__    	=     {'configuration_mra': ['MRA_PRETRAINED_CONFIG_ARCHIVE_MAP', 'MraConfig']}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						lowerCamelCase__    	=     [
						    'MRA_PRETRAINED_MODEL_ARCHIVE_LIST',
						    'MraForMaskedLM',
						    'MraForMultipleChoice',
						    'MraForQuestionAnswering',
						    'MraForSequenceClassification',
						    'MraForTokenClassification',
						    'MraLayer',
						    'MraModel',
						    'MraPreTrainedModel',
						]


if TYPE_CHECKING:
						from .configuration_mra import MRA_PRETRAINED_CONFIG_ARCHIVE_MAP, MraConfig

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_mra import (
												    MRA_PRETRAINED_MODEL_ARCHIVE_LIST,
												    MraForMaskedLM,
												    MraForMultipleChoice,
												    MraForQuestionAnswering,
												    MraForSequenceClassification,
												    MraForTokenClassification,
												    MraLayer,
												    MraModel,
												    MraPreTrainedModel,
												)
else:
						import sys

						lowerCamelCase__    	=     _LazyModule(__name__, globals()['__file__'], _import_structure)



