'''simple docstring'''





import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A__           =							logging.get_logger(__name__)

A__           =							'''▁'''

A__           =							{'''vocab_file''': '''vocab.txt''', '''sentencepiece_model_ckpt''': '''sentencepiece.bpe.model'''}

A__           =							{
    '''sentencepiece_model_file''': '''sentencepiece.bpe.model''',
    '''vocab_file''': '''vocab.txt''',
}

A__           =							{
    '''vocab_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
    },
    '''sentencepiece_model_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
    },
}

A__           =							{
    '''ernie-m-base''': 514,
    '''ernie-m-large''': 514,
}

A__           =							{
    '''ernie-m-base''': {'''do_lower_case''': False},
    '''ernie-m-large''': {'''do_lower_case''': False},
}
class   _lowerCAmelCase		(     __SCREAMING_SNAKE_CASE				):
							_UpperCAmelCase						    =     ['input_ids']

							_UpperCAmelCase						    =     VOCAB_FILES_NAMES
							_UpperCAmelCase						    =     PRETRAINED_INIT_CONFIGURATION
							_UpperCAmelCase						    =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_UpperCAmelCase						    =     PRETRAINED_VOCAB_FILES_MAP
							_UpperCAmelCase						    =     RESOURCE_FILES_NAMES







							def __init__(				self     :    Any	,					__snake_case     :    Tuple	,					__snake_case     :    Optional[int]=None	,					__snake_case     :    Any=False	,					__snake_case     :    List[str]="utf8"	,					__snake_case     :    Optional[int]="[UNK]"	,					__snake_case     :    Dict="[SEP]"	,					__snake_case     :    List[str]="[PAD]"	,					__snake_case     :    Dict="[CLS]"	,					__snake_case     :    Optional[int]="[MASK]"	,					__snake_case     :    Optional[Dict[str, Any]] = None	,					**__snake_case     :    Optional[int]	,					):
										lowerCamelCase						:Optional[int]						  =     {} if sp_model_kwargs is None else sp_model_kwargs
										super().__init__(
										    do_lower_case=UpperCamelCase_	,					unk_token=UpperCamelCase_	,					sep_token=UpperCamelCase_	,					pad_token=UpperCamelCase_	,					cls_token=UpperCamelCase_	,					mask_token=UpperCamelCase_	,					vocab_file=UpperCamelCase_	,					encoding=UpperCamelCase_	,					sp_model_kwargs=self.sp_model_kwargs	,					**UpperCamelCase_	,					)
										lowerCamelCase						:Dict						  =     do_lower_case
										lowerCamelCase						:Optional[Any]						  =     sentencepiece_model_ckpt
										lowerCamelCase						:Tuple						  =     spm.SentencePieceProcessor(**self.sp_model_kwargs		)
										self.sp_model.Load(UpperCamelCase_		)

										# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
										if vocab_file is not None:
													lowerCamelCase						:List[Any]						  =     self.load_vocab(filepath=UpperCamelCase_		)
										else:
													lowerCamelCase						:str						  =     {self.sp_model.id_to_piece(UpperCamelCase_		): id for id in range(self.sp_model.get_piece_size()		)}
										lowerCamelCase						:List[str]						  =     {v: k for k, v in self.vocab.items()}







							def 				snake_case  (				self     :    List[str]	,					__snake_case     :    Dict		):





										if text is None:
													return None

										lowerCamelCase						:Optional[Any]						  =     self.tokenize(UpperCamelCase_		)
										lowerCamelCase						:int						  =     '', []

										for i, ch in enumerate(UpperCamelCase_		):
													if ch in self.SP_CHAR_MAPPING:
																lowerCamelCase						:str						  =     self.SP_CHAR_MAPPING.get(UpperCamelCase_		)
													else:
																lowerCamelCase						:Optional[int]						  =     unicodedata.normalize('''NFKC'''	,					UpperCamelCase_		)
													if self.is_whitespace(UpperCamelCase_		):
																continue
													normalized_text += ch
													char_mapping.extend([i] * len(UpperCamelCase_		)		)

										lowerCamelCase						:Any						  =     normalized_text, [], 0

										if self.do_lower_case:
													lowerCamelCase						:Union[str, Any]						  =     text.lower()

										for token in split_tokens:
													if token[:1] == "▁":
																lowerCamelCase						:Tuple						  =     token[1:]
													lowerCamelCase						:Optional[Any]						  =     text[offset:].index(UpperCamelCase_		) + offset
													lowerCamelCase						:Any						  =     start + len(UpperCamelCase_		)

													token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)		)
													lowerCamelCase						:Tuple						  =     end
										return token_mapping







							@property
							def 				snake_case  (				self     :    Dict		):
										return len(self.vocab		)







							def 				snake_case  (				self     :    Tuple		):
										return dict(self.vocab	,					**self.added_tokens_encoder		)







							def __getstate__(				self     :    List[str]		):
										lowerCamelCase						:Optional[int]						  =     self.__dict__.copy()
										lowerCamelCase						:List[str]						  =     None
										return state







							def __setstate__(				self     :    List[Any]	,					__snake_case     :    Dict		):
										lowerCamelCase						:Any						  =     d

										# for backward compatibility
										if not hasattr(self	,					'''sp_model_kwargs'''		):
													lowerCamelCase						:Tuple						  =     {}

										lowerCamelCase						:Tuple						  =     spm.SentencePieceProcessor(**self.sp_model_kwargs		)
										self.sp_model.Load(self.sentencepiece_model_ckpt		)







							def 				snake_case  (				self     :    Dict	,					__snake_case     :    Dict		):
										return "".join((self.SP_CHAR_MAPPING.get(UpperCamelCase_	,					UpperCamelCase_		) for c in text)		)







							def 				snake_case  (				self     :    Union[str, Any]	,					__snake_case     :    int	,					__snake_case     :    Dict=False	,					__snake_case     :    Optional[int]=64	,					__snake_case     :    Tuple=0.1		):





										if self.sp_model_kwargs.get('''enable_sampling'''		) is True:
													lowerCamelCase						:List[Any]						  =     True
										if self.sp_model_kwargs.get('''alpha'''		) is not None:
													lowerCamelCase						:Optional[int]						  =     self.sp_model_kwargs.get('''alpha'''		)
										if self.sp_model_kwargs.get('''nbest_size'''		) is not None:
													lowerCamelCase						:List[Any]						  =     self.sp_model_kwargs.get('''nbest_size'''		)

										if not enable_sampling:
													lowerCamelCase						:Any						  =     self.sp_model.EncodeAsPieces(UpperCamelCase_		)
										else:
													lowerCamelCase						:Tuple						  =     self.sp_model.SampleEncodeAsPieces(UpperCamelCase_	,					UpperCamelCase_	,					UpperCamelCase_		)
										lowerCamelCase						:Any						  =     []
										for pi, piece in enumerate(UpperCamelCase_		):
													if piece == SPIECE_UNDERLINE:
																if not pieces[pi + 1].startswith(UpperCamelCase_		) and pi != 0:
																			new_pieces.append(UpperCamelCase_		)
																			continue
																else:
																			continue
													lowerCamelCase						:str						  =     0
													for i, chunk in enumerate(UpperCamelCase_		):
																if chunk == SPIECE_UNDERLINE:
																			continue
																if self.is_ch_char(UpperCamelCase_		) or self.is_punct(UpperCamelCase_		):
																			if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																						new_pieces.append(piece[lst_i:i]		)
																			new_pieces.append(UpperCamelCase_		)
																			lowerCamelCase						:Tuple						  =     i + 1
																elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																			if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																						new_pieces.append(piece[lst_i:i]		)
																			lowerCamelCase						:Any						  =     i
																elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																			if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																						new_pieces.append(piece[lst_i:i]		)
																			lowerCamelCase						:Any						  =     i
													if len(UpperCamelCase_		) > lst_i:
																new_pieces.append(piece[lst_i:]		)
										return new_pieces







							def 				snake_case  (				self     :    Tuple	,					__snake_case     :    Dict		):
										lowerCamelCase						:str						  =     ''.join(UpperCamelCase_		).replace(UpperCamelCase_	,					''' '''		).strip()
										return out_string







							def 				snake_case  (				self     :    str	,					__snake_case     :    str		):
										lowerCamelCase						:str						  =     self.convert_ids_to_tokens(UpperCamelCase_		)
										lowerCamelCase						:int						  =     ''.join(UpperCamelCase_		).replace(UpperCamelCase_	,					''' '''		).strip()
										return out_string







							def 				snake_case  (				self     :    Optional[Any]	,					__snake_case     :    Any		):
										return self.vocab.get(UpperCamelCase_	,					self.vocab.get(self.unk_token		)		)







							def 				snake_case  (				self     :    Optional[int]	,					__snake_case     :    List[str]		):
										return self.reverse_vocab.get(UpperCamelCase_	,					self.unk_token		)







							def 				snake_case  (				self     :    Optional[int]	,					__snake_case     :    str	,					__snake_case     :    int=None		):





										if token_ids_a is None:
													return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
										lowerCamelCase						:List[Any]						  =     [self.cls_token_id]
										lowerCamelCase						:Tuple						  =     [self.sep_token_id]
										return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep







							def 				snake_case  (				self     :    Dict	,					__snake_case     :    Optional[Any]	,					__snake_case     :    str=None		):





										if offset_mapping_a is None:
													return [(0, 0)] + offset_mapping_a + [(0, 0)]

										return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]







							def 				snake_case  (				self     :    Optional[Any]	,					__snake_case     :    Optional[int]	,					__snake_case     :    Optional[int]=None	,					__snake_case     :    Any=False		):





										if already_has_special_tokens:
													if token_ids_a is not None:
																raise ValueError(
																    '''You should not supply a second sequence if the provided sequence of '''
																    '''ids is already formatted with special tokens for the model.'''		)
													return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

										if token_ids_a is not None:
													return [1] + ([0] * len(UpperCamelCase_		)) + [1, 1] + ([0] * len(UpperCamelCase_		)) + [1]
										return [1] + ([0] * len(UpperCamelCase_		)) + [1]







							def 				snake_case  (				self     :    Union[str, Any]	,					__snake_case     :    List[int]	,					__snake_case     :    Optional[List[int]] = None		):





										if token_ids_a is None:
													# [CLS] X [SEP]
													return (len(UpperCamelCase_		) + 2) * [0]

										# [CLS] A [SEP] [SEP] B [SEP]
										return [0] * (len(UpperCamelCase_		) + 1) + [1] * (len(UpperCamelCase_		) + 3)







							def 				snake_case  (				self     :    str	,					__snake_case     :    Union[str, Any]		):





										if "\u4e00" <= char <= "\u9fff":
													return True
										return False







							def 				snake_case  (				self     :    Optional[int]	,					__snake_case     :    List[Any]		):





										if ("a" <= char <= "z") or ("A" <= char <= "Z"):
													return True
										return False







							def 				snake_case  (				self     :    Union[str, Any]	,					__snake_case     :    List[str]		):





										if char in ",;:.?!~，；：。？！《》【】":
													return True
										return False







							def 				snake_case  (				self     :    Any	,					__snake_case     :    Optional[Any]		):





										if char == " " or char == "\t" or char == "\n" or char == "\r":
													return True
										if len(UpperCamelCase_		) == 1:
													lowerCamelCase						:Any						  =     unicodedata.category(UpperCamelCase_		)
													if cat == "Zs":
																return True
										return False







							def 				snake_case  (				self     :    Union[str, Any]	,					__snake_case     :    Any		):
										lowerCamelCase						:List[str]						  =     {}
										with io.open(UpperCamelCase_	,					'''r'''	,					encoding='''utf-8'''		) as f:
													for index, line in enumerate(UpperCamelCase_		):
																lowerCamelCase						:List[str]						  =     line.rstrip('''\n'''		)
																lowerCamelCase						:Optional[Any]						  =     int(UpperCamelCase_		)

										return token_to_idx







							def 				snake_case  (				self     :    Dict	,					__snake_case     :    str	,					__snake_case     :    Optional[str] = None		):
										lowerCamelCase						:Optional[int]						  =     0
										if os.path.isdir(UpperCamelCase_		):
													lowerCamelCase						:Any						  =     os.path.join(
													    UpperCamelCase_	,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)
										else:
													lowerCamelCase						:Union[str, Any]						  =     (filename_prefix + '-' if filename_prefix else '') + save_directory
										with open(UpperCamelCase_	,					'''w'''	,					encoding='''utf-8'''		) as writer:
													for token, token_index in sorted(self.vocab.items()	,					key=lambda __snake_case		: kv[1]		):
																if index != token_index:
																			logger.warning(
																			    F"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
																			    ''' Please check that the vocabulary is not corrupted!'''		)
																			lowerCamelCase						:Optional[Any]						  =     token_index
																writer.write(token + '''\n'''		)
																index += 1

										lowerCamelCase						:int						  =     os.path.join(UpperCamelCase_	,					'''sentencepiece.bpe.model'''		)
										with open(UpperCamelCase_	,					'''wb'''		) as fi:
													lowerCamelCase						:int						  =     self.sp_model.serialized_model_proto()
													fi.write(UpperCamelCase_		)

										return (vocab_file,)




def  _lowerCamelCase					(   a_				: list):

			if not grid or not grid[0]:
						raise TypeError('''The grid does not contain the appropriate information''')

			for cell_n in range(1						,			len(grid[0])):
						grid[0][cell_n] += grid[0][cell_n - 1]
			lowerCamelCase						:Any						  =     grid[0]

			for row_n in range(1						,			len(a_)):
						lowerCamelCase						:List[str]						  =     grid[row_n]
						lowerCamelCase						:Union[str, Any]						  =     fill_row(a_						,			a_)
						lowerCamelCase						:List[Any]						  =     grid[row_n]

			return grid[-1][-1]





def  _lowerCamelCase					(   a_				: list						,			a_				: list):
			current_row[0] += row_above[0]
			for cell_n in range(1						,			len(a_)):
						current_row[cell_n] += min(current_row[cell_n - 1]						,			row_above[cell_n])

			return current_row


if __name__ == "__main__":
			import doctest

			doctest.testmod()



