



'''simple docstring'''

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_A				:    Any     						=logging.get_logger(__name__)

_A				:    Optional[Any]     						='''▁'''

_A				:    Optional[Any]     						={'''vocab_file''': '''sentencepiece.bpe.model'''}

_A				:    Optional[Any]     						={
    '''vocab_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

_A				:    Any     						={
    '''facebook/mbart-large-50-one-to-many-mmt''': 1_0_2_4,
}

# fmt: off
_A				:    Optional[Any]     						=['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''', '''af_ZA''', '''az_AZ''', '''bn_IN''', '''fa_IR''', '''he_IL''', '''hr_HR''', '''id_ID''', '''ka_GE''', '''km_KH''', '''mk_MK''', '''ml_IN''', '''mn_MN''', '''mr_IN''', '''pl_PL''', '''ps_AF''', '''pt_XX''', '''sv_SE''', '''sw_KE''', '''ta_IN''', '''te_IN''', '''th_TH''', '''tl_XX''', '''uk_UA''', '''ur_PK''', '''xh_ZA''', '''gl_ES''', '''sl_SI''']



class 		lowerCamelCase__      (						A     ):




					'''simple docstring'''



					A_						    =						VOCAB_FILES_NAMES
					A_						    =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					A_						    =						PRETRAINED_VOCAB_FILES_MAP
					A_						    =						["""input_ids""", """attention_mask"""]

					A_						    =						[]
					A_						    =						[]




					def __init__(  self				:	Union[str, Any]     ,  UpperCamelCase_				:	Optional[Any]     ,  UpperCamelCase_				:	Optional[int]=None     ,  UpperCamelCase_				:	Any=None     ,  UpperCamelCase_				:	List[str]="</s>"     ,  UpperCamelCase_				:	Union[str, Any]="</s>"     ,  UpperCamelCase_				:	List[Any]="<s>"     ,  UpperCamelCase_				:	str="<unk>"     ,  UpperCamelCase_				:	List[str]="<pad>"     ,  UpperCamelCase_				:	List[str]="<mask>"     ,  UpperCamelCase_				:	Optional[Dict[str, Any]] = None     ,  **UpperCamelCase_				:	int     ,  )  ->       None:



						'''simple docstring'''





						_lowercase					:       int       =		AddedToken(UpperCamelCase_     ,  lstrip=UpperCamelCase_     ,  rstrip=UpperCamelCase_					) if isinstance(UpperCamelCase_     ,  UpperCamelCase_					) else mask_token

						_lowercase					:       List[Any]       =		{} if sp_model_kwargs is None else sp_model_kwargs

						_lowercase					:       List[str]       =		kwargs.get('additional_special_tokens'     ,  []					)
						kwargs["additional_special_tokens"] += [
						    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
						]

						super().__init__(
						    src_lang=UpperCamelCase_     ,  tgt_lang=UpperCamelCase_     ,  eos_token=UpperCamelCase_     ,  unk_token=UpperCamelCase_     ,  sep_token=UpperCamelCase_     ,  cls_token=UpperCamelCase_     ,  pad_token=UpperCamelCase_     ,  mask_token=UpperCamelCase_     ,  sp_model_kwargs=self.sp_model_kwargs     ,  **UpperCamelCase_     ,  )

						_lowercase					:       int       =		spm.SentencePieceProcessor(**self.sp_model_kwargs					)
						self.sp_model.Load(str(UpperCamelCase_					)					)
						_lowercase					:       Optional[Any]       =		vocab_file

						# Original fairseq vocab and spm vocab must be "aligned":
						# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
						# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
						# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
						# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

						# Mimic fairseq token-to-id alignment for the first 4 token
						_lowercase					:       Union[str, Any]       =		{'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

						# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
						_lowercase					:       List[str]       =		1

						_lowercase					:       str       =		len(self.sp_model					)
						_lowercase					:       List[Any]       =		{
						    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(UpperCamelCase_					)
						}
						_lowercase					:       Dict       =		{v: k for k, v in self.lang_code_to_id.items()}
						_lowercase					:       Tuple       =		len(self.sp_model					) + len(self.lang_code_to_id					) + self.fairseq_offset

						self.fairseq_tokens_to_ids.update(self.lang_code_to_id					)
						_lowercase					:       Optional[int]       =		{v: k for k, v in self.fairseq_tokens_to_ids.items()}

						_lowercase					:       Any       =		src_lang if src_lang is not None else 'en_XX'
						_lowercase					:       str       =		self.lang_code_to_id[self._src_lang]
						_lowercase					:       Optional[Any]       =		tgt_lang
						self.set_src_lang_special_tokens(self._src_lang					)




					@property
					def 						__UpperCAmelCase     (  self				:	List[str]					)  ->       int:



						'''simple docstring'''





						return len(self.sp_model					) + len(self.lang_code_to_id					) + self.fairseq_offset + 1  # Plus 1 for the mask token




					@property
					def 						__UpperCAmelCase     (  self				:	str					)  ->       str:



						'''simple docstring'''





						return self._src_lang




					@src_lang.setter
					def 						__UpperCAmelCase     (  self				:	Dict     ,  UpperCamelCase_				:	str					)  ->       None:



						'''simple docstring'''





						_lowercase					:       List[Any]       =		new_src_lang
						self.set_src_lang_special_tokens(self._src_lang					)




					def __getstate__(  self				:	str					)  ->       Dict:



						'''simple docstring'''





						_lowercase					:       int       =		self.__dict__.copy()
						_lowercase					:       Tuple       =		None
						return state




					def __setstate__(  self				:	List[str]     ,  UpperCamelCase_				:	Dict					)  ->       None:



						'''simple docstring'''





						_lowercase					:       Any       =		d

						# for backward compatibility
						if not hasattr(self     ,  'sp_model_kwargs'					):
							_lowercase					:       Optional[int]       =		{}

						_lowercase					:       Dict       =		spm.SentencePieceProcessor(**self.sp_model_kwargs					)
						self.sp_model.Load(self.vocab_file					)




					def 						__UpperCAmelCase     (  self				:	Dict					)  ->       Dict:



						'''simple docstring'''





						_lowercase					:       str       =		{self.convert_ids_to_tokens(UpperCamelCase_					): i for i in range(self.vocab_size					)}
						vocab.update(self.added_tokens_encoder					)
						return vocab




					def 						__UpperCAmelCase     (  self				:	int     ,  UpperCamelCase_				:	str					)  ->       List[str]:



						'''simple docstring'''





						return self.sp_model.encode(UpperCamelCase_     ,  out_type=UpperCamelCase_					)




					def 						__UpperCAmelCase     (  self				:	Optional[Any]     ,  UpperCamelCase_				:	str					)  ->       int:



						'''simple docstring'''





						if token in self.fairseq_tokens_to_ids:
							return self.fairseq_tokens_to_ids[token]
						_lowercase					:       Dict       =		self.sp_model.PieceToId(UpperCamelCase_					)

						# Need to return unknown token if the SP model returned 0
						return spm_id + self.fairseq_offset if spm_id else self.unk_token_id




					def 						__UpperCAmelCase     (  self				:	List[str]     ,  UpperCamelCase_				:	int					)  ->       str:



						'''simple docstring'''





						if index in self.fairseq_ids_to_tokens:
							return self.fairseq_ids_to_tokens[index]
						return self.sp_model.IdToPiece(index - self.fairseq_offset					)




					def 						__UpperCAmelCase     (  self				:	Optional[Any]     ,  UpperCamelCase_				:	str					)  ->       Dict:



						'''simple docstring'''





						_lowercase					:       Tuple       =		[]
						_lowercase					:       Tuple       =		''
						_lowercase					:       Optional[Any]       =		False
						for token in tokens:
							# make sure that special tokens are not decoded using sentencepiece model
							if token in self.all_special_tokens:
								if not prev_is_special:
									out_string += " "
								out_string += self.sp_model.decode(UpperCamelCase_					) + token
								_lowercase					:       str       =		True
								_lowercase					:       Tuple       =		[]
							else:
								current_sub_tokens.append(UpperCamelCase_					)
								_lowercase					:       Any       =		False
						out_string += self.sp_model.decode(UpperCamelCase_					)
						return out_string.strip()




					def 						__UpperCAmelCase     (  self				:	Union[str, Any]     ,  UpperCamelCase_				:	str     ,  UpperCamelCase_				:	Optional[str] = None					)  ->       Tuple[str]:



						'''simple docstring'''





						if not os.path.isdir(UpperCamelCase_					):
							logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''					)
							return
						_lowercase					:       Optional[Any]       =		os.path.join(
						    UpperCamelCase_     ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)

						if os.path.abspath(self.vocab_file					) != os.path.abspath(UpperCamelCase_					) and os.path.isfile(self.vocab_file					):
							copyfile(self.vocab_file     ,  UpperCamelCase_					)
						elif not os.path.isfile(self.vocab_file					):
							with open(UpperCamelCase_     ,  'wb'					) as fi:
								_lowercase					:       int       =		self.sp_model.serialized_model_proto()
								fi.write(UpperCamelCase_					)

						return (out_vocab_file,)




					def 						__UpperCAmelCase     (  self				:	Tuple     ,  UpperCamelCase_				:	List[int]     ,  UpperCamelCase_				:	Optional[List[int]] = None     ,  UpperCamelCase_				:	bool = False					)  ->       List[int]:



						'''simple docstring'''





						if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=UpperCamelCase_     ,  token_ids_a=UpperCamelCase_     ,  already_has_special_tokens=UpperCamelCase_					)

						_lowercase					:       Tuple       =		[1] * len(self.prefix_tokens					)
						_lowercase					:       List[str]       =		[1] * len(self.suffix_tokens					)
						if token_ids_a is None:
							return prefix_ones + ([0] * len(UpperCamelCase_					)) + suffix_ones
						return prefix_ones + ([0] * len(UpperCamelCase_					)) + ([0] * len(UpperCamelCase_					)) + suffix_ones




					def 						__UpperCAmelCase     (  self				:	Union[str, Any]     ,  UpperCamelCase_				:	List[int]     ,  UpperCamelCase_				:	Optional[List[int]] = None					)  ->       List[int]:



						'''simple docstring'''





						if token_ids_a is None:
							return self.prefix_tokens + token_ids_a + self.suffix_tokens
						# We don't expect to process pairs, but leave the pair logic for API consistency
						return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




					def 						__UpperCAmelCase     (  self				:	List[Any]     ,  UpperCamelCase_				:	List[str]     ,  UpperCamelCase_				:	str     ,  UpperCamelCase_				:	Optional[str]     ,  UpperCamelCase_				:	Optional[str]     ,  **UpperCamelCase_				:	Optional[int]					)  ->       Any:



						'''simple docstring'''





						if src_lang is None or tgt_lang is None:
							raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'					)
						_lowercase					:       Optional[Any]       =		src_lang
						_lowercase					:       Optional[Any]       =		self(UpperCamelCase_     ,  add_special_tokens=UpperCamelCase_     ,  return_tensors=UpperCamelCase_     ,  **UpperCamelCase_					)
						_lowercase					:       Optional[Any]       =		self.convert_tokens_to_ids(UpperCamelCase_					)
						_lowercase					:       int       =		tgt_lang_id
						return inputs




					def 						__UpperCAmelCase     (  self				:	List[str]     ,  UpperCamelCase_				:	List[str]     ,  UpperCamelCase_				:	str = "en_XX"     ,  UpperCamelCase_				:	Optional[List[str]] = None     ,  UpperCamelCase_				:	str = "ro_RO"     ,  **UpperCamelCase_				:	Dict     ,  )  ->       BatchEncoding:



						'''simple docstring'''





						_lowercase					:       Any       =		src_lang
						_lowercase					:       int       =		tgt_lang
						return super().prepare_seqaseq_batch(UpperCamelCase_     ,  UpperCamelCase_     ,  **UpperCamelCase_					)




					def 						__UpperCAmelCase     (  self				:	List[str]					)  ->       List[Any]:



						'''simple docstring'''





						return self.set_src_lang_special_tokens(self.src_lang					)




					def 						__UpperCAmelCase     (  self				:	Optional[int]					)  ->       Union[str, Any]:



						'''simple docstring'''





						return self.set_tgt_lang_special_tokens(self.tgt_lang					)




					def 						__UpperCAmelCase     (  self				:	Any     ,  UpperCamelCase_				:	str					)  ->       None:



						'''simple docstring'''





						_lowercase					:       List[Any]       =		self.lang_code_to_id[src_lang]
						_lowercase					:       Dict       =		[self.cur_lang_code_id]
						_lowercase					:       Optional[int]       =		[self.eos_token_id]




					def 						__UpperCAmelCase     (  self				:	Optional[int]     ,  UpperCamelCase_				:	str					)  ->       None:



						'''simple docstring'''





						_lowercase					:       List[Any]       =		self.lang_code_to_id[tgt_lang]
						_lowercase					:       Union[str, Any]       =		[self.cur_lang_code_id]
						_lowercase					:       Any       =		[self.eos_token_id]





'''simple docstring'''

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_A				:    Optional[Any]     						=logging.get_logger(__name__)

_A				:    Optional[int]     						={
    '''microsoft/markuplm-base''': '''https://huggingface.co/microsoft/markuplm-base/resolve/main/config.json''',
    '''microsoft/markuplm-large''': '''https://huggingface.co/microsoft/markuplm-large/resolve/main/config.json''',
}



class 		lowerCamelCase__      (						A     ):




					'''simple docstring'''



					A_						    =						"""markuplm"""




					def __init__(  self				:	int     ,  UpperCamelCase_				:	Optional[Any]=3_0522     ,  UpperCamelCase_				:	Optional[Any]=768     ,  UpperCamelCase_				:	Tuple=12     ,  UpperCamelCase_				:	Union[str, Any]=12     ,  UpperCamelCase_				:	Tuple=3072     ,  UpperCamelCase_				:	Union[str, Any]="gelu"     ,  UpperCamelCase_				:	List[Any]=0.1     ,  UpperCamelCase_				:	List[Any]=0.1     ,  UpperCamelCase_				:	Dict=512     ,  UpperCamelCase_				:	Optional[int]=2     ,  UpperCamelCase_				:	Any=0.02     ,  UpperCamelCase_				:	Optional[Any]=1E-12     ,  UpperCamelCase_				:	List[str]=0     ,  UpperCamelCase_				:	Optional[int]=0     ,  UpperCamelCase_				:	Tuple=2     ,  UpperCamelCase_				:	str=256     ,  UpperCamelCase_				:	Optional[Any]=1024     ,  UpperCamelCase_				:	Union[str, Any]=216     ,  UpperCamelCase_				:	int=1001     ,  UpperCamelCase_				:	int=32     ,  UpperCamelCase_				:	int=50     ,  UpperCamelCase_				:	str="absolute"     ,  UpperCamelCase_				:	Union[str, Any]=True     ,  UpperCamelCase_				:	Optional[int]=None     ,  **UpperCamelCase_				:	Any     ,  )  ->       Optional[int]:



						'''simple docstring'''





						super().__init__(
						    pad_token_id=UpperCamelCase_     ,  bos_token_id=UpperCamelCase_     ,  eos_token_id=UpperCamelCase_     ,  **UpperCamelCase_     ,  )
						_lowercase					:       List[Any]       =		vocab_size
						_lowercase					:       Union[str, Any]       =		hidden_size
						_lowercase					:       Dict       =		num_hidden_layers
						_lowercase					:       Optional[Any]       =		num_attention_heads
						_lowercase					:       Dict       =		hidden_act
						_lowercase					:       Optional[Any]       =		intermediate_size
						_lowercase					:       Optional[int]       =		hidden_dropout_prob
						_lowercase					:       Union[str, Any]       =		attention_probs_dropout_prob
						_lowercase					:       Any       =		max_position_embeddings
						_lowercase					:       List[Any]       =		type_vocab_size
						_lowercase					:       Union[str, Any]       =		initializer_range
						_lowercase					:       Optional[int]       =		layer_norm_eps
						_lowercase					:       Optional[Any]       =		position_embedding_type
						_lowercase					:       str       =		use_cache
						_lowercase					:       str       =		classifier_dropout
						# additional properties
						_lowercase					:       int       =		max_depth
						_lowercase					:       Dict       =		max_xpath_tag_unit_embeddings
						_lowercase					:       str       =		max_xpath_subs_unit_embeddings
						_lowercase					:       List[str]       =		tag_pad_id
						_lowercase					:       Optional[int]       =		subs_pad_id
						_lowercase					:       Any       =		xpath_unit_hidden_size
