



'''simple docstring'''

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_A				:    Any     						=logging.get_logger(__name__)

_A				:    Optional[Any]     						='''▁'''

_A				:    Optional[Any]     						={'''vocab_file''': '''sentencepiece.bpe.model'''}

_A				:    Optional[Any]     						={
    '''vocab_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

_A				:    Any     						={
    '''facebook/mbart-large-50-one-to-many-mmt''': 1_0_2_4,
}

# fmt: off
_A				:    Optional[Any]     						=['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''', '''af_ZA''', '''az_AZ''', '''bn_IN''', '''fa_IR''', '''he_IL''', '''hr_HR''', '''id_ID''', '''ka_GE''', '''km_KH''', '''mk_MK''', '''ml_IN''', '''mn_MN''', '''mr_IN''', '''pl_PL''', '''ps_AF''', '''pt_XX''', '''sv_SE''', '''sw_KE''', '''ta_IN''', '''te_IN''', '''th_TH''', '''tl_XX''', '''uk_UA''', '''ur_PK''', '''xh_ZA''', '''gl_ES''', '''sl_SI''']



class 		lowerCamelCase__      (						A     ):




	'''simple docstring'''



	A_						    =						VOCAB_FILES_NAMES
	A_						    =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	A_						    =						PRETRAINED_VOCAB_FILES_MAP
	A_						    =						["""input_ids""", """attention_mask"""]

	A_						    =						[]
	A_						    =						[]




	def __init__(  self				:	Union[str, Any]     ,  UpperCamelCase_				:	Optional[Any]     ,  UpperCamelCase_				:	Optional[int]=None     ,  UpperCamelCase_				:	Any=None     ,  UpperCamelCase_				:	List[str]="</s>"     ,  UpperCamelCase_				:	Union[str, Any]="</s>"     ,  UpperCamelCase_				:	List[Any]="<s>"     ,  UpperCamelCase_				:	str="<unk>"     ,  UpperCamelCase_				:	List[str]="<pad>"     ,  UpperCamelCase_				:	List[str]="<mask>"     ,  UpperCamelCase_				:	Optional[Dict[str, Any]] = None     ,  **UpperCamelCase_				:	int     ,  )  ->       None:



		'''simple docstring'''





		_lowercase					:       int       =		AddedToken(UpperCamelCase_     ,  lstrip=UpperCamelCase_     ,  rstrip=UpperCamelCase_					) if isinstance(UpperCamelCase_     ,  UpperCamelCase_					) else mask_token

		_lowercase					:       List[Any]       =		{} if sp_model_kwargs is None else sp_model_kwargs

		_lowercase					:       List[str]       =		kwargs.get('additional_special_tokens'     ,  []					)
		kwargs["additional_special_tokens"] += [
		    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
		]

		super().__init__(
		    src_lang=UpperCamelCase_     ,  tgt_lang=UpperCamelCase_     ,  eos_token=UpperCamelCase_     ,  unk_token=UpperCamelCase_     ,  sep_token=UpperCamelCase_     ,  cls_token=UpperCamelCase_     ,  pad_token=UpperCamelCase_     ,  mask_token=UpperCamelCase_     ,  sp_model_kwargs=self.sp_model_kwargs     ,  **UpperCamelCase_     ,  )

		_lowercase					:       int       =		spm.SentencePieceProcessor(**self.sp_model_kwargs					)
		self.sp_model.Load(str(UpperCamelCase_					)					)
		_lowercase					:       Optional[Any]       =		vocab_file

		# Original fairseq vocab and spm vocab must be "aligned":
		# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
		# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
		# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
		# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

		# Mimic fairseq token-to-id alignment for the first 4 token
		_lowercase					:       Union[str, Any]       =		{'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

		# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
		_lowercase					:       List[str]       =		1

		_lowercase					:       str       =		len(self.sp_model					)
		_lowercase					:       List[Any]       =		{
		    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(UpperCamelCase_					)
		}
		_lowercase					:       Dict       =		{v: k for k, v in self.lang_code_to_id.items()}
		_lowercase					:       Tuple       =		len(self.sp_model					) + len(self.lang_code_to_id					) + self.fairseq_offset

		self.fairseq_tokens_to_ids.update(self.lang_code_to_id					)
		_lowercase					:       Optional[int]       =		{v: k for k, v in self.fairseq_tokens_to_ids.items()}

		_lowercase					:       Any       =		src_lang if src_lang is not None else 'en_XX'
		_lowercase					:       str       =		self.lang_code_to_id[self._src_lang]
		_lowercase					:       Optional[Any]       =		tgt_lang
		self.set_src_lang_special_tokens(self._src_lang					)




	@property
	def 						__UpperCAmelCase     (  self				:	List[str]					)  ->       int:



		'''simple docstring'''





		return len(self.sp_model					) + len(self.lang_code_to_id					) + self.fairseq_offset + 1  # Plus 1 for the mask token




	@property
	def 						__UpperCAmelCase     (  self				:	str					)  ->       str:



		'''simple docstring'''





		return self._src_lang




	@src_lang.setter
	def 						__UpperCAmelCase     (  self				:	Dict     ,  UpperCamelCase_				:	str					)  ->       None:



		'''simple docstring'''





		_lowercase					:       List[Any]       =		new_src_lang
		self.set_src_lang_special_tokens(self._src_lang					)




	def __getstate__(  self				:	str					)  ->       Dict:



		'''simple docstring'''





		_lowercase					:       int       =		self.__dict__.copy()
		_lowercase					:       Tuple       =		None
		return state




	def __setstate__(  self				:	List[str]     ,  UpperCamelCase_				:	Dict					)  ->       None:



		'''simple docstring'''





		_lowercase					:       Any       =		d

		# for backward compatibility
		if not hasattr(self     ,  'sp_model_kwargs'					):
			_lowercase					:       Optional[int]       =		{}

		_lowercase					:       Dict       =		spm.SentencePieceProcessor(**self.sp_model_kwargs					)
		self.sp_model.Load(self.vocab_file					)




	def 						__UpperCAmelCase     (  self				:	Dict					)  ->       Dict:



		'''simple docstring'''





		_lowercase					:       str       =		{self.convert_ids_to_tokens(UpperCamelCase_					): i for i in range(self.vocab_size					)}
		vocab.update(self.added_tokens_encoder					)
		return vocab




	def 						__UpperCAmelCase     (  self				:	int     ,  UpperCamelCase_				:	str					)  ->       List[str]:



		'''simple docstring'''





		return self.sp_model.encode(UpperCamelCase_     ,  out_type=UpperCamelCase_					)




	def 						__UpperCAmelCase     (  self				:	Optional[Any]     ,  UpperCamelCase_				:	str					)  ->       int:



		'''simple docstring'''





		if token in self.fairseq_tokens_to_ids:
			return self.fairseq_tokens_to_ids[token]
		_lowercase					:       Dict       =		self.sp_model.PieceToId(UpperCamelCase_					)

		# Need to return unknown token if the SP model returned 0
		return spm_id + self.fairseq_offset if spm_id else self.unk_token_id




	def 						__UpperCAmelCase     (  self				:	List[str]     ,  UpperCamelCase_				:	int					)  ->       str:



		'''simple docstring'''





		if index in self.fairseq_ids_to_tokens:
			return self.fairseq_ids_to_tokens[index]
		return self.sp_model.IdToPiece(index - self.fairseq_offset					)




	def 						__UpperCAmelCase     (  self				:	Optional[Any]     ,  UpperCamelCase_				:	str					)  ->       Dict:



		'''simple docstring'''





		_lowercase					:       Tuple       =		[]
		_lowercase					:       Tuple       =		''
		_lowercase					:       Optional[Any]       =		False
		for token in tokens:
			# make sure that special tokens are not decoded using sentencepiece model
			if token in self.all_special_tokens:
				if not prev_is_special:
					out_string += " "
				out_string += self.sp_model.decode(UpperCamelCase_					) + token
				_lowercase					:       str       =		True
				_lowercase					:       Tuple       =		[]
			else:
				current_sub_tokens.append(UpperCamelCase_					)
				_lowercase					:       Any       =		False
		out_string += self.sp_model.decode(UpperCamelCase_					)
		return out_string.strip()




	def 						__UpperCAmelCase     (  self				:	Union[str, Any]     ,  UpperCamelCase_				:	str     ,  UpperCamelCase_				:	Optional[str] = None					)  ->       Tuple[str]:



		'''simple docstring'''





		if not os.path.isdir(UpperCamelCase_					):
			logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''					)
			return
		_lowercase					:       Optional[Any]       =		os.path.join(
		    UpperCamelCase_     ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)

		if os.path.abspath(self.vocab_file					) != os.path.abspath(UpperCamelCase_					) and os.path.isfile(self.vocab_file					):
			copyfile(self.vocab_file     ,  UpperCamelCase_					)
		elif not os.path.isfile(self.vocab_file					):
			with open(UpperCamelCase_     ,  'wb'					) as fi:
				_lowercase					:       int       =		self.sp_model.serialized_model_proto()
				fi.write(UpperCamelCase_					)

		return (out_vocab_file,)




	def 						__UpperCAmelCase     (  self				:	Tuple     ,  UpperCamelCase_				:	List[int]     ,  UpperCamelCase_				:	Optional[List[int]] = None     ,  UpperCamelCase_				:	bool = False					)  ->       List[int]:



		'''simple docstring'''





		if already_has_special_tokens:
			return super().get_special_tokens_mask(
			    token_ids_a=UpperCamelCase_     ,  token_ids_a=UpperCamelCase_     ,  already_has_special_tokens=UpperCamelCase_					)

		_lowercase					:       Tuple       =		[1] * len(self.prefix_tokens					)
		_lowercase					:       List[str]       =		[1] * len(self.suffix_tokens					)
		if token_ids_a is None:
			return prefix_ones + ([0] * len(UpperCamelCase_					)) + suffix_ones
		return prefix_ones + ([0] * len(UpperCamelCase_					)) + ([0] * len(UpperCamelCase_					)) + suffix_ones




	def 						__UpperCAmelCase     (  self				:	Union[str, Any]     ,  UpperCamelCase_				:	List[int]     ,  UpperCamelCase_				:	Optional[List[int]] = None					)  ->       List[int]:



		'''simple docstring'''





		if token_ids_a is None:
			return self.prefix_tokens + token_ids_a + self.suffix_tokens
		# We don't expect to process pairs, but leave the pair logic for API consistency
		return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




	def 						__UpperCAmelCase     (  self				:	List[Any]     ,  UpperCamelCase_				:	List[str]     ,  UpperCamelCase_				:	str     ,  UpperCamelCase_				:	Optional[str]     ,  UpperCamelCase_				:	Optional[str]     ,  **UpperCamelCase_				:	Optional[int]					)  ->       Any:



		'''simple docstring'''





		if src_lang is None or tgt_lang is None:
			raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'					)
		_lowercase					:       Optional[Any]       =		src_lang
		_lowercase					:       Optional[Any]       =		self(UpperCamelCase_     ,  add_special_tokens=UpperCamelCase_     ,  return_tensors=UpperCamelCase_     ,  **UpperCamelCase_					)
		_lowercase					:       Optional[Any]       =		self.convert_tokens_to_ids(UpperCamelCase_					)
		_lowercase					:       int       =		tgt_lang_id
		return inputs




	def 						__UpperCAmelCase     (  self				:	List[str]     ,  UpperCamelCase_				:	List[str]     ,  UpperCamelCase_				:	str = "en_XX"     ,  UpperCamelCase_				:	Optional[List[str]] = None     ,  UpperCamelCase_				:	str = "ro_RO"     ,  **UpperCamelCase_				:	Dict     ,  )  ->       BatchEncoding:



		'''simple docstring'''





		_lowercase					:       Any       =		src_lang
		_lowercase					:       int       =		tgt_lang
		return super().prepare_seqaseq_batch(UpperCamelCase_     ,  UpperCamelCase_     ,  **UpperCamelCase_					)




	def 						__UpperCAmelCase     (  self				:	List[str]					)  ->       List[Any]:



		'''simple docstring'''





		return self.set_src_lang_special_tokens(self.src_lang					)




	def 						__UpperCAmelCase     (  self				:	Optional[int]					)  ->       Union[str, Any]:



		'''simple docstring'''





		return self.set_tgt_lang_special_tokens(self.tgt_lang					)




	def 						__UpperCAmelCase     (  self				:	Any     ,  UpperCamelCase_				:	str					)  ->       None:



		'''simple docstring'''





		_lowercase					:       List[Any]       =		self.lang_code_to_id[src_lang]
		_lowercase					:       Dict       =		[self.cur_lang_code_id]
		_lowercase					:       Optional[int]       =		[self.eos_token_id]




	def 						__UpperCAmelCase     (  self				:	Optional[int]     ,  UpperCamelCase_				:	str					)  ->       None:



		'''simple docstring'''





		_lowercase					:       List[Any]       =		self.lang_code_to_id[tgt_lang]
		_lowercase					:       Union[str, Any]       =		[self.cur_lang_code_id]
		_lowercase					:       Any       =		[self.eos_token_id]





'''simple docstring'''

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


_A				:    Union[str, Any]     						={'''configuration_reformer''': ['''REFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''ReformerConfig''']}

try:
		if not is_sentencepiece_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		_A				:    Dict     						=['''ReformerTokenizer''']

try:
		if not is_tokenizers_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		_A				:    Union[str, Any]     						=['''ReformerTokenizerFast''']

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		_A				:    str     						=[
		    '''REFORMER_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''ReformerAttention''',
		    '''ReformerForMaskedLM''',
		    '''ReformerForQuestionAnswering''',
		    '''ReformerForSequenceClassification''',
		    '''ReformerLayer''',
		    '''ReformerModel''',
		    '''ReformerModelWithLMHead''',
		    '''ReformerPreTrainedModel''',
		]


if TYPE_CHECKING:
		from .configuration_reformer import REFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, ReformerConfig

		try:
				if not is_sentencepiece_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_reformer import ReformerTokenizer

		try:
				if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_reformer_fast import ReformerTokenizerFast

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_reformer import (
				    REFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
				    ReformerAttention,
				    ReformerForMaskedLM,
				    ReformerForQuestionAnswering,
				    ReformerForSequenceClassification,
				    ReformerLayer,
				    ReformerModel,
				    ReformerModelWithLMHead,
				    ReformerPreTrainedModel,
				)

else:
		import sys

		_A				:    Union[str, Any]     						=_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
