





import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_a						=   logging.get_logger(__name__)

_a						=   "▁"

_a						=   {"vocab_file": "vocab.txt", "sentencepiece_model_ckpt": "sentencepiece.bpe.model"}

_a						=   {
    "sentencepiece_model_file": "sentencepiece.bpe.model",
    "vocab_file": "vocab.txt",
}

_a						=   {
    "vocab_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
    },
    "sentencepiece_model_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
    },
}

_a						=   {
    "ernie-m-base": 514,
    "ernie-m-large": 514,
}

_a						=   {
    "ernie-m-base": {"do_lower_case": False},
    "ernie-m-large": {"do_lower_case": False},
}






class       __A					(					lowerCAmelCase				):

			'''simple docstring'''







			lowerCAmelCase_							  =		["input_ids"]

			lowerCAmelCase_							  =		VOCAB_FILES_NAMES
			lowerCAmelCase_							  =		PRETRAINED_INIT_CONFIGURATION
			lowerCAmelCase_							  =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase_							  =		PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase_							  =		RESOURCE_FILES_NAMES





			def __init__(					self ,						__lowerCAmelCase ,						__lowerCAmelCase=None ,						__lowerCAmelCase=False ,						__lowerCAmelCase="utf8" ,						__lowerCAmelCase="[UNK]" ,						__lowerCAmelCase="[SEP]" ,						__lowerCAmelCase="[PAD]" ,						__lowerCAmelCase="[CLS]" ,						__lowerCAmelCase="[MASK]" ,						__lowerCAmelCase = None ,						**__lowerCAmelCase ,						):



				'''simple docstring'''







				lowerCamelCase__              =	{} if sp_model_kwargs is None else sp_model_kwargs
				super().__init__(
				    do_lower_case=__lowerCAmelCase ,						unk_token=__lowerCAmelCase ,						sep_token=__lowerCAmelCase ,						pad_token=__lowerCAmelCase ,						cls_token=__lowerCAmelCase ,						mask_token=__lowerCAmelCase ,						vocab_file=__lowerCAmelCase ,						encoding=__lowerCAmelCase ,						sp_model_kwargs=self.sp_model_kwargs ,						**__lowerCAmelCase ,						)
				lowerCamelCase__              =	do_lower_case
				lowerCamelCase__              =	sentencepiece_model_ckpt
				lowerCamelCase__              =	spm.SentencePieceProcessor(**self.sp_model_kwargs							)
				self.sp_model.Load(__lowerCAmelCase							)

				# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
				if vocab_file is not None:
					lowerCamelCase__              =	self.load_vocab(filepath=__lowerCAmelCase							)
				else:
					lowerCamelCase__              =	{self.sp_model.id_to_piece(__lowerCAmelCase							): id for id in range(self.sp_model.get_piece_size()							)}
				lowerCamelCase__              =	{v: k for k, v in self.vocab.items()}





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase							):



				'''simple docstring'''







				if text is None:
					return None

				lowerCamelCase__              =	self.tokenize(__lowerCAmelCase							)
				lowerCamelCase__       ,			lowerCamelCase__              =	'''''', []

				for i, ch in enumerate(__lowerCAmelCase							):
					if ch in self.SP_CHAR_MAPPING:
						lowerCamelCase__              =	self.SP_CHAR_MAPPING.get(__lowerCAmelCase							)
					else:
						lowerCamelCase__              =	unicodedata.normalize('''NFKC''' ,						__lowerCAmelCase							)
					if self.is_whitespace(__lowerCAmelCase							):
						continue
					normalized_text += ch
					char_mapping.extend([i] * len(__lowerCAmelCase							)							)

				lowerCamelCase__       ,			lowerCamelCase__       ,			lowerCamelCase__              =	normalized_text, [], 0

				if self.do_lower_case:
					lowerCamelCase__              =	text.lower()

				for token in split_tokens:
					if token[:1] == "▁":
						lowerCamelCase__              =	token[1:]
					lowerCamelCase__              =	text[offset:].index(__lowerCAmelCase							) + offset
					lowerCamelCase__              =	start + len(__lowerCAmelCase							)

					token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)							)
					lowerCamelCase__              =	end
				return token_mapping





			@property
			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				return len(self.vocab							)





			def 	__lowerCamelCase       (					self							):



				'''simple docstring'''







				return dict(self.vocab ,						**self.added_tokens_encoder							)





			def __getstate__(					self							):



				'''simple docstring'''







				lowerCamelCase__              =	self.__dict__.copy()
				lowerCamelCase__              =	None
				return state





			def __setstate__(					self ,						__lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__              =	d

				# for backward compatibility
				if not hasattr(self ,						'''sp_model_kwargs'''							):
					lowerCamelCase__              =	{}

				lowerCamelCase__              =	spm.SentencePieceProcessor(**self.sp_model_kwargs							)
				self.sp_model.Load(self.sentencepiece_model_ckpt							)





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase							):



				'''simple docstring'''







				return "".join((self.SP_CHAR_MAPPING.get(__lowerCAmelCase ,						__lowerCAmelCase							) for c in text)							)





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase=False ,						__lowerCAmelCase=6_4 ,						__lowerCAmelCase=0.1							):



				'''simple docstring'''







				if self.sp_model_kwargs.get('''enable_sampling'''							) is True:
					lowerCamelCase__              =	True
				if self.sp_model_kwargs.get('''alpha'''							) is not None:
					lowerCamelCase__              =	self.sp_model_kwargs.get('''alpha'''							)
				if self.sp_model_kwargs.get('''nbest_size'''							) is not None:
					lowerCamelCase__              =	self.sp_model_kwargs.get('''nbest_size'''							)

				if not enable_sampling:
					lowerCamelCase__              =	self.sp_model.EncodeAsPieces(__lowerCAmelCase							)
				else:
					lowerCamelCase__              =	self.sp_model.SampleEncodeAsPieces(__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase							)
				lowerCamelCase__              =	[]
				for pi, piece in enumerate(__lowerCAmelCase							):
					if piece == SPIECE_UNDERLINE:
						if not pieces[pi + 1].startswith(__lowerCAmelCase							) and pi != 0:
							new_pieces.append(__lowerCAmelCase							)
							continue
						else:
							continue
					lowerCamelCase__              =	0
					for i, chunk in enumerate(__lowerCAmelCase							):
						if chunk == SPIECE_UNDERLINE:
							continue
						if self.is_ch_char(__lowerCAmelCase							) or self.is_punct(__lowerCAmelCase							):
							if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
								new_pieces.append(piece[lst_i:i]							)
							new_pieces.append(__lowerCAmelCase							)
							lowerCamelCase__              =	i + 1
						elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
							if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
								new_pieces.append(piece[lst_i:i]							)
							lowerCamelCase__              =	i
						elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
							if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
								new_pieces.append(piece[lst_i:i]							)
							lowerCamelCase__              =	i
					if len(__lowerCAmelCase							) > lst_i:
						new_pieces.append(piece[lst_i:]							)
				return new_pieces





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__              =	''''''.join(__lowerCAmelCase							).replace(__lowerCAmelCase ,						''' '''							).strip()
				return out_string





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__              =	self.convert_ids_to_tokens(__lowerCAmelCase							)
				lowerCamelCase__              =	''''''.join(__lowerCAmelCase							).replace(__lowerCAmelCase ,						''' '''							).strip()
				return out_string





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase							):



				'''simple docstring'''







				return self.vocab.get(__lowerCAmelCase ,						self.vocab.get(self.unk_token							)							)





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase							):



				'''simple docstring'''







				return self.reverse_vocab.get(__lowerCAmelCase ,						self.unk_token							)





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase=None							):



				'''simple docstring'''







				if token_ids_a is None:
					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
				lowerCamelCase__              =	[self.cls_token_id]
				lowerCamelCase__              =	[self.sep_token_id]
				return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase=None							):



				'''simple docstring'''







				if offset_mapping_a is None:
					return [(0, 0)] + offset_mapping_a + [(0, 0)]

				return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase=None ,						__lowerCAmelCase=False							):



				'''simple docstring'''







				if already_has_special_tokens:
					if token_ids_a is not None:
						raise ValueError(
						    '''You should not supply a second sequence if the provided sequence of '''
						    '''ids is already formatted with special tokens for the model.'''							)
					return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

				if token_ids_a is not None:
					return [1] + ([0] * len(__lowerCAmelCase							)) + [1, 1] + ([0] * len(__lowerCAmelCase							)) + [1]
				return [1] + ([0] * len(__lowerCAmelCase							)) + [1]





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase = None							):



				'''simple docstring'''







				if token_ids_a is None:
					# [CLS] X [SEP]
					return (len(__lowerCAmelCase							) + 2) * [0]

				# [CLS] A [SEP] [SEP] B [SEP]
				return [0] * (len(__lowerCAmelCase							) + 1) + [1] * (len(__lowerCAmelCase							) + 3)





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase							):



				'''simple docstring'''







				if "\u4e00" <= char <= "\u9fff":
					return True
				return False





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase							):



				'''simple docstring'''







				if ("a" <= char <= "z") or ("A" <= char <= "Z"):
					return True
				return False





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase							):



				'''simple docstring'''







				if char in ",;:.?!~，；：。？！《》【】":
					return True
				return False





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase							):



				'''simple docstring'''







				if char == " " or char == "\t" or char == "\n" or char == "\r":
					return True
				if len(__lowerCAmelCase							) == 1:
					lowerCamelCase__              =	unicodedata.category(__lowerCAmelCase							)
					if cat == "Zs":
						return True
				return False





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__              =	{}
				with io.open(__lowerCAmelCase ,						'''r''' ,						encoding='''utf-8'''							) as f:
					for index, line in enumerate(__lowerCAmelCase							):
						lowerCamelCase__              =	line.rstrip('''\n'''							)
						lowerCamelCase__              =	int(__lowerCAmelCase							)

				return token_to_idx





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase = None							):



				'''simple docstring'''







				lowerCamelCase__              =	0
				if os.path.isdir(__lowerCAmelCase							):
					lowerCamelCase__              =	os.path.join(
					    __lowerCAmelCase ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)
				else:
					lowerCamelCase__              =	(filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
				with open(__lowerCAmelCase ,						'''w''' ,						encoding='''utf-8'''							) as writer:
					for token, token_index in sorted(self.vocab.items() ,						key=lambda __lowerCAmelCase							: kv[1]							):
						if index != token_index:
							logger.warning(
							    F'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
							    ''' Please check that the vocabulary is not corrupted!'''							)
							lowerCamelCase__              =	token_index
						writer.write(token + '''\n'''							)
						index += 1

				lowerCamelCase__              =	os.path.join(__lowerCAmelCase ,						'''sentencepiece.bpe.model'''							)
				with open(__lowerCAmelCase ,						'''wb'''							) as fi:
					lowerCamelCase__              =	self.sp_model.serialized_model_proto()
					fi.write(__lowerCAmelCase							)

				return (vocab_file,)










# This is the module that test_patching.py uses to test patch_submodule()

import os  # noqa: this is just for tests
import os as renamed_os  # noqa: this is just for tests
from os import path  # noqa: this is just for tests
from os import path as renamed_path  # noqa: this is just for tests
from os.path import join  # noqa: this is just for tests
from os.path import join as renamed_join  # noqa: this is just for tests


_a						=   open  # noqa: we just need to have a builtin inside this module to test it properly



