




"""simple docstring"""





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


A   =       logging.get_logger(__name__)

A   =       """▁"""

A   =       {"""vocab_file""": """sentencepiece.bpe.model"""}

A   =       {
    """vocab_file""": {
        """facebook/mbart-large-en-ro""": (
            """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"""
        ),
        """facebook/mbart-large-cc25""": (
            """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"""
        ),
    }
}

A   =       {
    """facebook/mbart-large-en-ro""": 1_024,
    """facebook/mbart-large-cc25""": 1_024,
}

# fmt: off
A   =       ["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN"""]



class      a__				(	__magic_name__   ):
			lowercase_    =	VOCAB_FILES_NAMES
			lowercase_    =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowercase_    =	PRETRAINED_VOCAB_FILES_MAP
			lowercase_    =	["input_ids", "attention_mask"]

			lowercase_    =	[]
			lowercase_    =	[]


			def __init__(      self      :    str      ,  UpperCamelCase_      :    Optional[Any]      ,  UpperCamelCase_      :    List[Any]="<s>"      ,  UpperCamelCase_      :    List[Any]="</s>"      ,  UpperCamelCase_      :    Optional[Any]="</s>"      ,  UpperCamelCase_      :    List[Any]="<s>"      ,  UpperCamelCase_      :    int="<unk>"      ,  UpperCamelCase_      :    Union[str, Any]="<pad>"      ,  UpperCamelCase_      :    Tuple="<mask>"      ,  UpperCamelCase_      :    List[str]=None      ,  UpperCamelCase_      :    List[Any]=None      ,  UpperCamelCase_      :    Dict=None      ,  UpperCamelCase_      :    Optional[Dict[str, Any]] = None      ,  UpperCamelCase_      :    int=None      ,  **UpperCamelCase_      :    Optional[Any]      ,  ):


						"""simple docstring"""




						__UpperCAmelCase	: List[Any]		=					AddedToken(UpperCamelCase_      ,  lstrip=UpperCamelCase_      ,  rstrip=UpperCamelCase_) if isinstance(UpperCamelCase_      ,  UpperCamelCase_) else mask_token

						__UpperCAmelCase	: Optional[int]		=					{} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    bos_token=UpperCamelCase_      ,  eos_token=UpperCamelCase_      ,  unk_token=UpperCamelCase_      ,  sep_token=UpperCamelCase_      ,  cls_token=UpperCamelCase_      ,  pad_token=UpperCamelCase_      ,  mask_token=UpperCamelCase_      ,  tokenizer_file=UpperCamelCase_      ,  src_lang=UpperCamelCase_      ,  tgt_lang=UpperCamelCase_      ,  additional_special_tokens=UpperCamelCase_      ,  sp_model_kwargs=self.sp_model_kwargs      ,  **UpperCamelCase_      ,  )

						__UpperCAmelCase	: Dict		=					spm.SentencePieceProcessor(**self.sp_model_kwargs)
						self.sp_model.Load(str(UpperCamelCase_))
						__UpperCAmelCase	: int		=					vocab_file

						# Original fairseq vocab and spm vocab must be "aligned":
						# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
						# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
						# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
						# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

						# Mimic fairseq token-to-id alignment for the first 4 token
						__UpperCAmelCase	: Optional[Any]		=					{"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

						# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
						__UpperCAmelCase	: Union[str, Any]		=					1

						__UpperCAmelCase	: Optional[Any]		=					len(self.sp_model)
						__UpperCAmelCase	: int		=					{
						    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(UpperCamelCase_)
						}
						__UpperCAmelCase	: str		=					{v: k for k, v in self.lang_code_to_id.items()}
						__UpperCAmelCase	: Union[str, Any]		=					len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset

						self.fairseq_tokens_to_ids.update(self.lang_code_to_id)
						__UpperCAmelCase	: List[Any]		=					{v: k for k, v in self.fairseq_tokens_to_ids.items()}
						__UpperCAmelCase	: Optional[Any]		=					list(self.lang_code_to_id.keys())

						if additional_special_tokens is not None:
									# Only add those special tokens if they are not already there.
									self._additional_special_tokens.extend(
									    [t for t in additional_special_tokens if t not in self._additional_special_tokens])

						__UpperCAmelCase	: Dict		=					src_lang if src_lang is not None else "en_XX"
						__UpperCAmelCase	: Tuple		=					self.lang_code_to_id[self._src_lang]
						__UpperCAmelCase	: List[str]		=					tgt_lang
						self.set_src_lang_special_tokens(self._src_lang)


			def __getstate__(      self      :    List[Any]):


						"""simple docstring"""




						__UpperCAmelCase	: List[str]		=					self.__dict__.copy()
						__UpperCAmelCase	: Dict		=					None
						__UpperCAmelCase	: Dict		=					self.sp_model.serialized_model_proto()
						return state


			def __setstate__(      self      :    List[Any]      ,  UpperCamelCase_      :    Any):


						"""simple docstring"""




						__UpperCAmelCase	: Optional[int]		=					d

						# for backward compatibility
						if not hasattr(self      ,  "sp_model_kwargs"):
									__UpperCAmelCase	: Any		=					{}

						__UpperCAmelCase	: Any		=					spm.SentencePieceProcessor(**self.sp_model_kwargs)
						self.sp_model.LoadFromSerializedProto(self.sp_model_proto)


			@property
			def       a_     (      self      :    Tuple):


						"""simple docstring"""




						return len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset + 1  # Plus 1 for the mask token


			@property
			def       a_     (      self      :    List[str]):


						"""simple docstring"""




						return self._src_lang


			@src_lang.setter
			def       a_     (      self      :    Optional[Any]      ,  UpperCamelCase_      :    str):


						"""simple docstring"""




						__UpperCAmelCase	: Optional[int]		=					new_src_lang
						self.set_src_lang_special_tokens(self._src_lang)


			def       a_     (      self      :    Optional[int]      ,  UpperCamelCase_      :    List[int]      ,  UpperCamelCase_      :    Optional[List[int]] = None      ,  UpperCamelCase_      :    bool = False):


						"""simple docstring"""




						if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=UpperCamelCase_      ,  token_ids_a=UpperCamelCase_      ,  already_has_special_tokens=UpperCamelCase_)

						__UpperCAmelCase	: Optional[int]		=					[1] * len(self.prefix_tokens)
						__UpperCAmelCase	: Optional[int]		=					[1] * len(self.suffix_tokens)
						if token_ids_a is None:
									return prefix_ones + ([0] * len(UpperCamelCase_)) + suffix_ones
						return prefix_ones + ([0] * len(UpperCamelCase_)) + ([0] * len(UpperCamelCase_)) + suffix_ones


			def       a_     (      self      :    str      ,  UpperCamelCase_      :    List[int]      ,  UpperCamelCase_      :    Optional[List[int]] = None):


						"""simple docstring"""




						if token_ids_a is None:
									return self.prefix_tokens + token_ids_a + self.suffix_tokens
						# We don't expect to process pairs, but leave the pair logic for API consistency
						return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


			def       a_     (      self      :    Optional[int]      ,  UpperCamelCase_      :    List[int]      ,  UpperCamelCase_      :    Optional[List[int]] = None):


						"""simple docstring"""




						__UpperCAmelCase	: List[Any]		=					[self.sep_token_id]
						__UpperCAmelCase	: Tuple		=					[self.cls_token_id]

						if token_ids_a is None:
									return len(cls + token_ids_a + sep) * [0]
						return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]


			def       a_     (      self      :    str      ,  UpperCamelCase_      :    Union[str, Any]      ,  UpperCamelCase_      :    str      ,  UpperCamelCase_      :    Optional[str]      ,  UpperCamelCase_      :    Optional[str]      ,  **UpperCamelCase_      :    Any):


						"""simple docstring"""




						if src_lang is None or tgt_lang is None:
									raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model")
						__UpperCAmelCase	: Optional[int]		=					src_lang
						__UpperCAmelCase	: Tuple		=					self(UpperCamelCase_      ,  add_special_tokens=UpperCamelCase_      ,  return_tensors=UpperCamelCase_      ,  **UpperCamelCase_)
						__UpperCAmelCase	: Union[str, Any]		=					self.convert_tokens_to_ids(UpperCamelCase_)
						__UpperCAmelCase	: str		=					tgt_lang_id
						return inputs


			def       a_     (      self      :    Union[str, Any]):


						"""simple docstring"""




						__UpperCAmelCase	: Any		=					{self.convert_ids_to_tokens(UpperCamelCase_): i for i in range(self.vocab_size)}
						vocab.update(self.added_tokens_encoder)
						return vocab


			def       a_     (      self      :    int      ,  UpperCamelCase_      :    str):


						"""simple docstring"""




						return self.sp_model.encode(UpperCamelCase_      ,  out_type=UpperCamelCase_)


			def       a_     (      self      :    Any      ,  UpperCamelCase_      :    Union[str, Any]):


						"""simple docstring"""




						if token in self.fairseq_tokens_to_ids:
									return self.fairseq_tokens_to_ids[token]
						__UpperCAmelCase	: Optional[Any]		=					self.sp_model.PieceToId(UpperCamelCase_)

						# Need to return unknown token if the SP model returned 0
						return spm_id + self.fairseq_offset if spm_id else self.unk_token_id


			def       a_     (      self      :    Optional[int]      ,  UpperCamelCase_      :    Optional[int]):


						"""simple docstring"""




						if index in self.fairseq_ids_to_tokens:
									return self.fairseq_ids_to_tokens[index]
						return self.sp_model.IdToPiece(index - self.fairseq_offset)


			def       a_     (      self      :    Dict      ,  UpperCamelCase_      :    Any):


						"""simple docstring"""




						__UpperCAmelCase	: Union[str, Any]		=					"".join(UpperCamelCase_).replace(UpperCamelCase_      ,  " ").strip()
						return out_string


			def       a_     (      self      :    int      ,  UpperCamelCase_      :    str      ,  UpperCamelCase_      :    Optional[str] = None):


						"""simple docstring"""




						if not os.path.isdir(UpperCamelCase_):
									logger.error(F"Vocabulary path ({save_directory}) should be a directory")
									return
						__UpperCAmelCase	: int		=					os.path.join(
						    UpperCamelCase_      ,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])

						if os.path.abspath(self.vocab_file) != os.path.abspath(UpperCamelCase_) and os.path.isfile(self.vocab_file):
									copyfile(self.vocab_file      ,  UpperCamelCase_)
						elif not os.path.isfile(self.vocab_file):
									with open(UpperCamelCase_      ,  "wb") as fi:
												__UpperCAmelCase	: Dict		=					self.sp_model.serialized_model_proto()
												fi.write(UpperCamelCase_)

						return (out_vocab_file,)


			def       a_     (      self      :    Tuple      ,  UpperCamelCase_      :    List[str]      ,  UpperCamelCase_      :    str = "en_XX"      ,  UpperCamelCase_      :    Optional[List[str]] = None      ,  UpperCamelCase_      :    str = "ro_RO"      ,  **UpperCamelCase_      :    Any      ,  ):


						"""simple docstring"""




						__UpperCAmelCase	: List[Any]		=					src_lang
						__UpperCAmelCase	: int		=					tgt_lang
						return super().prepare_seqaseq_batch(UpperCamelCase_      ,  UpperCamelCase_      ,  **UpperCamelCase_)


			def       a_     (      self      :    Optional[Any]):


						"""simple docstring"""




						return self.set_src_lang_special_tokens(self.src_lang)


			def       a_     (      self      :    Optional[int]):


						"""simple docstring"""




						return self.set_tgt_lang_special_tokens(self.tgt_lang)


			def       a_     (      self      :    Optional[Any]      ,  UpperCamelCase_      :    Union[str, Any]):


						"""simple docstring"""




						__UpperCAmelCase	: Any		=					self.lang_code_to_id[src_lang]
						__UpperCAmelCase	: List[str]		=					[]
						__UpperCAmelCase	: Tuple		=					[self.eos_token_id, self.cur_lang_code]


			def       a_     (      self      :    Optional[int]      ,  UpperCamelCase_      :    str):


						"""simple docstring"""




						__UpperCAmelCase	: str		=					self.lang_code_to_id[lang]
						__UpperCAmelCase	: List[Any]		=					[]
						__UpperCAmelCase	: Union[str, Any]		=					[self.eos_token_id, self.cur_lang_code]








"""simple docstring"""





from ...processing_utils import ProcessorMixin



class      a__				(	__magic_name__   ):
			lowercase_    =	["image_processor", "feature_extractor"]
			lowercase_    =	"TvltImageProcessor"
			lowercase_    =	"TvltFeatureExtractor"


			def __init__(      self      :    Tuple      ,  UpperCamelCase_      :    Optional[int]      ,  UpperCamelCase_      :    Dict):


						"""simple docstring"""




						super().__init__(image_processor=UpperCamelCase_      ,  feature_extractor=UpperCamelCase_)

						__UpperCAmelCase	: Optional[int]		=					image_processor
						__UpperCAmelCase	: Dict		=					feature_extractor


			def __call__(      self      :    Tuple      ,  UpperCamelCase_      :    str=None      ,  UpperCamelCase_      :    Optional[int]=None      ,  UpperCamelCase_      :    Optional[int]=None      ,  UpperCamelCase_      :    str=None      ,  UpperCamelCase_      :    List[str]=False      ,  UpperCamelCase_      :    Optional[int]=False      ,  *UpperCamelCase_      :    List[str]      ,  **UpperCamelCase_      :    str      ,  ):


						"""simple docstring"""




						if images is None and audio is None:
									raise ValueError("You need to specify either an `images` or `audio` input to process.")

						__UpperCAmelCase	: Optional[Any]		=					None
						if images is not None:
									__UpperCAmelCase	: int		=					self.image_processor(UpperCamelCase_      ,  mask_pixel=UpperCamelCase_      ,  *UpperCamelCase_      ,  **UpperCamelCase_)
						if images_mixed is not None:
									__UpperCAmelCase	: int		=					self.image_processor(UpperCamelCase_      ,  is_mixed=UpperCamelCase_      ,  *UpperCamelCase_      ,  **UpperCamelCase_)
						if audio is not None:
									__UpperCAmelCase	: List[Any]		=					self.feature_extractor(
									    UpperCamelCase_      ,  *UpperCamelCase_      ,  sampling_rate=UpperCamelCase_      ,  mask_audio=UpperCamelCase_      ,  **UpperCamelCase_)

						__UpperCAmelCase	: List[str]		=					{}
						if audio is not None:
									output_dict.update(UpperCamelCase_)
						if images is not None:
									output_dict.update(UpperCamelCase_)
						if images_mixed_dict is not None:
									output_dict.update(UpperCamelCase_)
						return output_dict


			@property
			def       a_     (      self      :    str):


						"""simple docstring"""




						__UpperCAmelCase	: List[Any]		=					self.image_processor.model_input_names
						__UpperCAmelCase	: List[Any]		=					self.feature_extractor.model_input_names
						return list(dict.fromkeys(image_processor_input_names + feature_extractor_input_names))


