



import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__lowerCamelCase     :							Optional[int]  							=				logging.get_logger(__name__)

__lowerCamelCase     :							str  							=				"▁"

__lowerCamelCase     :							List[Any]  							=				{
    "vocab_file": "vocab.json",
    "spm_file": "sentencepiece.bpe.model",
    "tokenizer_config_file": "tokenizer_config.json",
}

__lowerCamelCase     :							Tuple  							=				{
    "vocab_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json",
    },
    "spm_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model",
    },
    "tokenizer_config_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json",
    },
}

__lowerCamelCase     :							Any  							=				{
    "facebook/m2m100_418M": 1_024,
}

# fmt: off
__lowerCamelCase     :							List[Any]  							=				{
    "m2m100": ["af", "am", "ar", "ast", "az", "ba", "be", "bg", "bn", "br", "bs", "ca", "ceb", "cs", "cy", "da", "de", "el", "en", "es", "et", "fa", "ff", "fi", "fr", "fy", "ga", "gd", "gl", "gu", "ha", "he", "hi", "hr", "ht", "hu", "hy", "id", "ig", "ilo", "is", "it", "ja", "jv", "ka", "kk", "km", "kn", "ko", "lb", "lg", "ln", "lo", "lt", "lv", "mg", "mk", "ml", "mn", "mr", "ms", "my", "ne", "nl", "no", "ns", "oc", "or", "pa", "pl", "ps", "pt", "ro", "ru", "sd", "si", "sk", "sl", "so", "sq", "sr", "ss", "su", "sv", "sw", "ta", "th", "tl", "tn", "tr", "uk", "ur", "uz", "vi", "wo", "xh", "yi", "yo", "zh", "zu"],
    "wmt21": ["en", "ha", "is", "ja", "cs", "ru", "zh", "de"]
}





class  __magic_name__   (							A__						):
		lowercase      :       int          =VOCAB_FILES_NAMES
		lowercase      :       Tuple          =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		lowercase      :       str          =PRETRAINED_VOCAB_FILES_MAP
		lowercase      :       Optional[int]          =['''input_ids''', '''attention_mask''']

		lowercase      :       List[int]          =[]
		lowercase      :       List[int]          =[]






		def __init__(						self      :     Tuple				,		UpperCamelCase__      :     Optional[int]				,		UpperCamelCase__      :     int				,		UpperCamelCase__      :     int=None				,		UpperCamelCase__      :     int=None				,		UpperCamelCase__      :     int="<s>"				,		UpperCamelCase__      :     Dict="</s>"				,		UpperCamelCase__      :     Any="</s>"				,		UpperCamelCase__      :     Union[str, Any]="<pad>"				,		UpperCamelCase__      :     List[str]="<unk>"				,		UpperCamelCase__      :     str="m2m100"				,		UpperCamelCase__      :     Optional[Dict[str, Any]] = None				,		UpperCamelCase__      :     Any=8				,		**UpperCamelCase__      :     int				,		)				->					None:
								'''simple docstring'''







								UpperCAmelCase         =     {} if sp_model_kwargs is None else sp_model_kwargs

								UpperCAmelCase         =     language_codes
								UpperCAmelCase         =     FAIRSEQ_LANGUAGE_CODES[language_codes]
								UpperCAmelCase         =     {lang_code: F'__{lang_code}__' for lang_code in fairseq_language_code}

								UpperCAmelCase         =     kwargs.get("additional_special_tokens"				,		[]      )
								kwargs["additional_special_tokens"] += [
								    self.get_lang_token(UpperCamelCase__      )
								    for lang_code in fairseq_language_code
								    if self.get_lang_token(UpperCamelCase__      ) not in kwargs["additional_special_tokens"]
								]

								super().__init__(
								    src_lang=UpperCamelCase__				,		tgt_lang=UpperCamelCase__				,		bos_token=UpperCamelCase__				,		eos_token=UpperCamelCase__				,		sep_token=UpperCamelCase__				,		unk_token=UpperCamelCase__				,		pad_token=UpperCamelCase__				,		language_codes=UpperCamelCase__				,		sp_model_kwargs=self.sp_model_kwargs				,		num_madeup_words=UpperCamelCase__				,		**UpperCamelCase__				,		)

								UpperCAmelCase         =     vocab_file
								UpperCAmelCase         =     load_json(UpperCamelCase__      )
								UpperCAmelCase         =     {v: k for k, v in self.encoder.items()}
								UpperCAmelCase         =     spm_file
								UpperCAmelCase         =     load_spm(UpperCamelCase__				,		self.sp_model_kwargs      )

								UpperCAmelCase         =     len(self.encoder      )

								UpperCAmelCase         =     {
								    self.get_lang_token(UpperCamelCase__      ): self.encoder_size + i for i, lang_code in enumerate(UpperCamelCase__      )
								}
								UpperCAmelCase         =     {lang_code: self.encoder_size + i for i, lang_code in enumerate(UpperCamelCase__      )}
								UpperCAmelCase         =     {v: k for k, v in self.lang_token_to_id.items()}

								UpperCAmelCase         =     src_lang if src_lang is not None else "en"
								UpperCAmelCase         =     tgt_lang
								UpperCAmelCase         =     self.get_lang_id(self._src_lang      )
								self.set_src_lang_special_tokens(self._src_lang      )

								UpperCAmelCase         =     num_madeup_words






		@property
		def SCREAMING_SNAKE_CASE_			(						self      :     Tuple      )				->					int:
								'''simple docstring'''







								return len(self.encoder      ) + len(self.lang_token_to_id      )






		@property
		def SCREAMING_SNAKE_CASE_			(						self      :     Tuple      )				->					str:
								'''simple docstring'''







								return self._src_lang






		@src_lang.setter
		def SCREAMING_SNAKE_CASE_			(						self      :     Union[str, Any]				,		UpperCamelCase__      :     str      )				->					None:
								'''simple docstring'''







								UpperCAmelCase         =     new_src_lang
								self.set_src_lang_special_tokens(self._src_lang      )






		def SCREAMING_SNAKE_CASE_			(						self      :     List[Any]				,		UpperCamelCase__      :     str      )				->					List[str]:
								'''simple docstring'''







								return self.sp_model.encode(UpperCamelCase__				,		out_type=UpperCamelCase__      )






		def SCREAMING_SNAKE_CASE_			(						self      :     int				,		UpperCamelCase__      :     Union[str, Any]      )				->					Tuple:
								'''simple docstring'''







								if token in self.lang_token_to_id:
														return self.lang_token_to_id[token]
								return self.encoder.get(UpperCamelCase__				,		self.encoder[self.unk_token]      )






		def SCREAMING_SNAKE_CASE_			(						self      :     Union[str, Any]				,		UpperCamelCase__      :     int      )				->					str:
								'''simple docstring'''







								if index in self.id_to_lang_token:
														return self.id_to_lang_token[index]
								return self.decoder.get(UpperCamelCase__				,		self.unk_token      )






		def SCREAMING_SNAKE_CASE_			(						self      :     int				,		UpperCamelCase__      :     List[Any]      )				->					int:
								'''simple docstring'''







								UpperCAmelCase         =     []
								UpperCAmelCase         =     ""
								for token in tokens:
														# make sure that special tokens are not decoded using sentencepiece model
														if token in self.all_special_tokens:
																				out_string += self.sp_model.decode(UpperCamelCase__      ) + token
																				UpperCAmelCase         =     []
														else:
																				current_sub_tokens.append(UpperCamelCase__      )
								out_string += self.sp_model.decode(UpperCamelCase__      )
								return out_string.strip()






		def SCREAMING_SNAKE_CASE_			(						self      :     Any				,		UpperCamelCase__      :     List[int]				,		UpperCamelCase__      :     Optional[List[int]] = None				,		UpperCamelCase__      :     bool = False      )				->					List[int]:
								'''simple docstring'''







								if already_has_special_tokens:
														return super().get_special_tokens_mask(
														    token_ids_a=UpperCamelCase__				,		token_ids_a=UpperCamelCase__				,		already_has_special_tokens=UpperCamelCase__      )

								UpperCAmelCase         =     [1] * len(self.prefix_tokens      )
								UpperCAmelCase         =     [1] * len(self.suffix_tokens      )
								if token_ids_a is None:
														return prefix_ones + ([0] * len(UpperCamelCase__      )) + suffix_ones
								return prefix_ones + ([0] * len(UpperCamelCase__      )) + ([0] * len(UpperCamelCase__      )) + suffix_ones






		def SCREAMING_SNAKE_CASE_			(						self      :     int				,		UpperCamelCase__      :     List[int]				,		UpperCamelCase__      :     Optional[List[int]] = None      )				->					List[int]:
								'''simple docstring'''







								if token_ids_a is None:
														return self.prefix_tokens + token_ids_a + self.suffix_tokens
								# We don't expect to process pairs, but leave the pair logic for API consistency
								return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






		def SCREAMING_SNAKE_CASE_			(						self      :     Tuple      )				->					Dict:
								'''simple docstring'''







								UpperCAmelCase         =     {self.convert_ids_to_tokens(UpperCamelCase__      ): i for i in range(self.vocab_size      )}
								vocab.update(self.added_tokens_encoder      )
								return vocab






		def __getstate__(						self      :     Union[str, Any]      )				->					Dict:
								'''simple docstring'''







								UpperCAmelCase         =     self.__dict__.copy()
								UpperCAmelCase         =     None
								return state






		def __setstate__(						self      :     List[str]				,		UpperCamelCase__      :     Dict      )				->					None:
								'''simple docstring'''







								UpperCAmelCase         =     d

								# for backward compatibility
								if not hasattr(self				,		"sp_model_kwargs"      ):
														UpperCAmelCase         =     {}

								UpperCAmelCase         =     load_spm(self.spm_file				,		self.sp_model_kwargs      )






		def SCREAMING_SNAKE_CASE_			(						self      :     Tuple				,		UpperCamelCase__      :     str				,		UpperCamelCase__      :     Optional[str] = None      )				->					Tuple[str]:
								'''simple docstring'''







								UpperCAmelCase         =     Path(UpperCamelCase__      )
								if not save_dir.is_dir():
														raise OSError(F'{save_directory} should be a directory'      )
								UpperCAmelCase         =     save_dir / (
								    (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["vocab_file"]
								)
								UpperCAmelCase         =     save_dir / (
								    (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["spm_file"]
								)

								save_json(self.encoder				,		UpperCamelCase__      )

								if os.path.abspath(self.spm_file      ) != os.path.abspath(UpperCamelCase__      ) and os.path.isfile(self.spm_file      ):
														copyfile(self.spm_file				,		UpperCamelCase__      )
								elif not os.path.isfile(self.spm_file      ):
														with open(UpperCamelCase__				,		"wb"      ) as fi:
																				UpperCAmelCase         =     self.sp_model.serialized_model_proto()
																				fi.write(UpperCamelCase__      )

								return (str(UpperCamelCase__      ), str(UpperCamelCase__      ))






		def SCREAMING_SNAKE_CASE_			(						self      :     str				,		UpperCamelCase__      :     List[str]				,		UpperCamelCase__      :     str = "en"				,		UpperCamelCase__      :     Optional[List[str]] = None				,		UpperCamelCase__      :     str = "ro"				,		**UpperCamelCase__      :     Tuple				,		)				->					BatchEncoding:
								'''simple docstring'''







								UpperCAmelCase         =     src_lang
								UpperCAmelCase         =     tgt_lang
								self.set_src_lang_special_tokens(self.src_lang      )
								return super().prepare_seqaseq_batch(UpperCamelCase__				,		UpperCamelCase__				,		**UpperCamelCase__      )






		def SCREAMING_SNAKE_CASE_			(						self      :     str				,		UpperCamelCase__      :     Union[str, Any]				,		UpperCamelCase__      :     Optional[str]				,		UpperCamelCase__      :     Optional[str]				,		**UpperCamelCase__      :     int      )				->					Dict:
								'''simple docstring'''







								if src_lang is None or tgt_lang is None:
														raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"      )
								UpperCAmelCase         =     src_lang
								UpperCAmelCase         =     self(UpperCamelCase__				,		add_special_tokens=UpperCamelCase__				,		**UpperCamelCase__      )
								UpperCAmelCase         =     self.get_lang_id(UpperCamelCase__      )
								UpperCAmelCase         =     tgt_lang_id
								return inputs






		def SCREAMING_SNAKE_CASE_			(						self      :     Optional[int]      )				->					Optional[int]:
								'''simple docstring'''







								self.set_src_lang_special_tokens(self.src_lang      )






		def SCREAMING_SNAKE_CASE_			(						self      :     List[str]      )				->					Optional[int]:
								'''simple docstring'''







								self.set_tgt_lang_special_tokens(self.tgt_lang      )






		def SCREAMING_SNAKE_CASE_			(						self      :     Union[str, Any]				,		UpperCamelCase__      :     str      )				->					None:
								'''simple docstring'''







								UpperCAmelCase         =     self.get_lang_token(UpperCamelCase__      )
								UpperCAmelCase         =     self.lang_token_to_id[lang_token]
								UpperCAmelCase         =     [self.cur_lang_id]
								UpperCAmelCase         =     [self.eos_token_id]






		def SCREAMING_SNAKE_CASE_			(						self      :     List[str]				,		UpperCamelCase__      :     str      )				->					None:
								'''simple docstring'''







								UpperCAmelCase         =     self.get_lang_token(UpperCamelCase__      )
								UpperCAmelCase         =     self.lang_token_to_id[lang_token]
								UpperCAmelCase         =     [self.cur_lang_id]
								UpperCAmelCase         =     [self.eos_token_id]






		def SCREAMING_SNAKE_CASE_			(						self      :     List[Any]				,		UpperCamelCase__      :     str      )				->					str:
								'''simple docstring'''







								return self.lang_code_to_token[lang]







		def SCREAMING_SNAKE_CASE_			(						self      :     Union[str, Any]				,		UpperCamelCase__      :     str      )				->					int:
								'''simple docstring'''







								UpperCAmelCase         =     self.get_lang_token(UpperCamelCase__      )
								return self.lang_token_to_id[lang_token]

def    lowerCamelCase_(lowerCamelCase_    ,   lowerCamelCase_    ) ->       sentencepiece.SentencePieceProcessor:
						UpperCAmelCase         =     sentencepiece.SentencePieceProcessor(**lowerCamelCase_    )
						spm.Load(str(lowerCamelCase_    )    )
						return spm

def    lowerCamelCase_(lowerCamelCase_    ) ->       Union[Dict, List]:
						with open(lowerCamelCase_    ,   "r"    ) as f:
												return json.load(lowerCamelCase_    )





def    lowerCamelCase_(lowerCamelCase_    ,   lowerCamelCase_    ) ->       None:
						with open(lowerCamelCase_    ,   "w"    ) as f:
												json.dump(lowerCamelCase_    ,   lowerCamelCase_    ,   indent=2    )








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__lowerCamelCase     :							Optional[int]  							=				{
    "configuration_roc_bert": ["ROC_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "RoCBertConfig"],
    "tokenization_roc_bert": ["RoCBertTokenizer"],
}

try:
		if not is_tokenizers_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		pass

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__lowerCamelCase     :							List[Any]  							=				[
		    "ROC_BERT_PRETRAINED_MODEL_ARCHIVE_LIST",
		    "RoCBertForCausalLM",
		    "RoCBertForMaskedLM",
		    "RoCBertForMultipleChoice",
		    "RoCBertForPreTraining",
		    "RoCBertForQuestionAnswering",
		    "RoCBertForSequenceClassification",
		    "RoCBertForTokenClassification",
		    "RoCBertLayer",
		    "RoCBertModel",
		    "RoCBertPreTrainedModel",
		    "load_tf_weights_in_roc_bert",
		]

if TYPE_CHECKING:
		from .configuration_roc_bert import ROC_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, RoCBertConfig
		from .tokenization_roc_bert import RoCBertTokenizer

		try:
				if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				raise OptionalDependencyNotAvailable()

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_roc_bert import (
				    ROC_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    RoCBertForCausalLM,
				    RoCBertForMaskedLM,
				    RoCBertForMultipleChoice,
				    RoCBertForPreTraining,
				    RoCBertForQuestionAnswering,
				    RoCBertForSequenceClassification,
				    RoCBertForTokenClassification,
				    RoCBertLayer,
				    RoCBertModel,
				    RoCBertPreTrainedModel,
				    load_tf_weights_in_roc_bert,
				)


else:
		import sys

		__lowerCamelCase     :							Optional[int]  							=				_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



