


'''simple docstring'''



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowercase    :    Any			     =  logging.get_logger(__name__)

_lowercase    :    str			     =  """▁"""

_lowercase    :    Dict			     =  {"""vocab_file""": """sentencepiece.bpe.model""", """monolingual_vocab_file""": """dict.txt"""}

_lowercase    :    Dict			     =  {
    """vocab_file""": {
        """vinai/bartpho-syllable""": """https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model""",
    },
    """monolingual_vocab_file""": {
        """vinai/bartpho-syllable""": """https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt""",
    },
}

_lowercase    :    Optional[int]			     =  {"""vinai/bartpho-syllable""": 1024}






class  UpperCamelCase__(     lowerCAmelCase ):
				__magic_name__       :   Tuple							= VOCAB_FILES_NAMES
				__magic_name__       :   Tuple							= PRETRAINED_VOCAB_FILES_MAP
				__magic_name__       :   Union[str, Any]							= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__magic_name__       :   List[Any]							= ["input_ids", "attention_mask"]




				def __init__(	self    :				Optional[Any]   ,					lowerCAmelCase    :				Union[str, Any]   ,					lowerCAmelCase    :				Optional[int]   ,					lowerCAmelCase    :				int="<s>"   ,					lowerCAmelCase    :				Dict="</s>"   ,					lowerCAmelCase    :				Dict="</s>"   ,					lowerCAmelCase    :				str="<s>"   ,					lowerCAmelCase    :				Optional[int]="<unk>"   ,					lowerCAmelCase    :				str="<pad>"   ,					lowerCAmelCase    :				List[Any]="<mask>"   ,					lowerCAmelCase    :				Optional[Dict[str, Any]] = None   ,					**lowerCAmelCase    :				Optional[int]   ,					)->  None:

										"""simple docstring"""




										UpperCAmelCase         =			AddedToken(lowerCAmelCase   ,					lstrip=lowerCAmelCase   ,					rstrip=lowerCAmelCase   ) if isinstance(lowerCAmelCase   ,					lowerCAmelCase   ) else mask_token

										UpperCAmelCase         =			{} if sp_model_kwargs is None else sp_model_kwargs

										super().__init__(
										    bos_token=lowerCAmelCase   ,					eos_token=lowerCAmelCase   ,					unk_token=lowerCAmelCase   ,					sep_token=lowerCAmelCase   ,					cls_token=lowerCAmelCase   ,					pad_token=lowerCAmelCase   ,					mask_token=lowerCAmelCase   ,					sp_model_kwargs=self.sp_model_kwargs   ,					**lowerCAmelCase   ,					)

										UpperCAmelCase         =			vocab_file
										UpperCAmelCase         =			monolingual_vocab_file
										UpperCAmelCase         =			spm.SentencePieceProcessor(**self.sp_model_kwargs   )
										self.sp_model.Load(str(lowerCAmelCase   )   )

										# Load the reduced vocab

										# Keep order of special tokens for backward compatibility
										UpperCAmelCase         =			{}
										UpperCAmelCase         =			0
										for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
																if str(lowerCAmelCase   ) not in self.fairseq_tokens_to_ids:
																						UpperCAmelCase         =			cnt
																						cnt += 1
										with open(lowerCAmelCase   ,					'''r'''   ,					encoding='''utf-8'''   ) as f:
																for line in f.readlines():
																						UpperCAmelCase         =			line.strip().split()[0]
																						UpperCAmelCase         =			len(self.fairseq_tokens_to_ids   )
										if str(lowerCAmelCase   ) not in self.fairseq_tokens_to_ids:
																UpperCAmelCase         =			len(self.fairseq_tokens_to_ids   )

										UpperCAmelCase         =			{v: k for k, v in self.fairseq_tokens_to_ids.items()}




				def __getstate__(	self    :				Union[str, Any]   )->  List[str]:

										"""simple docstring"""




										UpperCAmelCase         =			self.__dict__.copy()
										UpperCAmelCase         =			None
										UpperCAmelCase         =			self.sp_model.serialized_model_proto()
										return state




				def __setstate__(	self    :				int   ,					lowerCAmelCase    :				str   )->  List[Any]:

										"""simple docstring"""




										UpperCAmelCase         =			d

										# for backward compatibility
										if not hasattr(self   ,					'''sp_model_kwargs'''   ):
																UpperCAmelCase         =			{}

										UpperCAmelCase         =			spm.SentencePieceProcessor(**self.sp_model_kwargs   )
										self.sp_model.LoadFromSerializedProto(self.sp_model_proto   )




				def   a__(	self    :				List[str]   ,					lowerCAmelCase    :				List[int]   ,					lowerCAmelCase    :				Optional[List[int]] = None   )->  List[int]:

										"""simple docstring"""




										if token_ids_a is None:
																return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
										UpperCAmelCase         =			[self.cls_token_id]
										UpperCAmelCase         =			[self.sep_token_id]
										return cls + token_ids_a + sep + sep + token_ids_a + sep




				def   a__(	self    :				str   ,					lowerCAmelCase    :				List[int]   ,					lowerCAmelCase    :				Optional[List[int]] = None   ,					lowerCAmelCase    :				bool = False   )->  List[int]:

										"""simple docstring"""




										if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=lowerCAmelCase   ,					token_ids_a=lowerCAmelCase   ,					already_has_special_tokens=lowerCAmelCase   )

										if token_ids_a is None:
																return [1] + ([0] * len(lowerCAmelCase   )) + [1]
										return [1] + ([0] * len(lowerCAmelCase   )) + [1, 1] + ([0] * len(lowerCAmelCase   )) + [1]




				def   a__(	self    :				Tuple   ,					lowerCAmelCase    :				List[int]   ,					lowerCAmelCase    :				Optional[List[int]] = None   )->  List[int]:

										"""simple docstring"""




										UpperCAmelCase         =			[self.sep_token_id]
										UpperCAmelCase         =			[self.cls_token_id]

										if token_ids_a is None:
																return len(cls + token_ids_a + sep   ) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]




				@property
				def   a__(	self    :				Any   )->  Optional[int]:

										"""simple docstring"""




										return len(self.fairseq_ids_to_tokens   )




				def   a__(	self    :				Dict   )->  int:

										"""simple docstring"""




										UpperCAmelCase         =			{self.convert_ids_to_tokens(lowerCAmelCase   ): i for i in range(self.vocab_size   )}
										vocab.update(self.added_tokens_encoder   )
										return vocab




				def   a__(	self    :				Any   ,					lowerCAmelCase    :				str   )->  List[str]:

										"""simple docstring"""




										return self.sp_model.encode(lowerCAmelCase   ,					out_type=lowerCAmelCase   )




				def   a__(	self    :				Union[str, Any]   ,					lowerCAmelCase    :				Union[str, Any]   )->  List[Any]:

										"""simple docstring"""




										if token in self.fairseq_tokens_to_ids:
																return self.fairseq_tokens_to_ids[token]
										else:
																return self.unk_token_id




				def   a__(	self    :				Tuple   ,					lowerCAmelCase    :				List[str]   )->  Union[str, Any]:

										"""simple docstring"""




										return self.fairseq_ids_to_tokens[index]




				def   a__(	self    :				List[Any]   ,					lowerCAmelCase    :				Optional[Any]   )->  Optional[Any]:

										"""simple docstring"""




										UpperCAmelCase         =			''''''.join(lowerCAmelCase   ).replace(lowerCAmelCase   ,					''' '''   ).strip()
										return out_string





				def   a__(	self    :				List[Any]   ,					lowerCAmelCase    :				str   ,					lowerCAmelCase    :				Optional[str] = None   )->  Tuple[str]:

										"""simple docstring"""




										if not os.path.isdir(lowerCAmelCase   ):
																logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
																return
										UpperCAmelCase         =			os.path.join(
										    lowerCAmelCase   ,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )
										UpperCAmelCase         =			os.path.join(
										    lowerCAmelCase   ,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''monolingual_vocab_file''']   ,					)

										if os.path.abspath(self.vocab_file   ) != os.path.abspath(lowerCAmelCase   ) and os.path.isfile(self.vocab_file   ):
																copyfile(self.vocab_file   ,					lowerCAmelCase   )
										elif not os.path.isfile(self.vocab_file   ):
																with open(lowerCAmelCase   ,					'''wb'''   ) as fi:
																						UpperCAmelCase         =			self.sp_model.serialized_model_proto()
																						fi.write(lowerCAmelCase   )

										if os.path.abspath(self.monolingual_vocab_file   ) != os.path.abspath(
										    lowerCAmelCase   ) and os.path.isfile(self.monolingual_vocab_file   ):
																copyfile(self.monolingual_vocab_file   ,					lowerCAmelCase   )
										elif not os.path.isfile(self.monolingual_vocab_file   ):
																with open(lowerCAmelCase   ,					'''w'''   ,					encoding='''utf-8'''   ) as fp:
																						for token in self.fairseq_tokens_to_ids:
																												if token not in self.all_special_tokens:
																																		fp.write(F"""{str(lowerCAmelCase   )} \n"""   )

										return out_vocab_file, out_monolingual_vocab_file









'''simple docstring'''



def    lowerCamelCase__   ( A     :    str  ):




						'''simple docstring'''






						assert column_title.isupper()
						UpperCAmelCase         =			0
						UpperCAmelCase         =			len(A  ) - 1
						UpperCAmelCase         =			0

						while index >= 0:
												UpperCAmelCase         =			(ord(column_title[index]  ) - 64) * pow(26   ,     A  )
												answer += value
												power += 1
												index -= 1

						return answer


if __name__ == "__main__":
	from doctest import testmod

	testmod()





