



'''simple docstring'''

import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_A				:    Dict     						=logging.get_logger(__name__)

_A				:    Optional[int]     						='''▁'''

_A				:    str     						={'''vocab_file''': '''vocab.txt''', '''sentencepiece_model_ckpt''': '''sentencepiece.bpe.model'''}

_A				:    Any     						={
    '''sentencepiece_model_file''': '''sentencepiece.bpe.model''',
    '''vocab_file''': '''vocab.txt''',
}

_A				:    Optional[int]     						={
    '''vocab_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
    },
    '''sentencepiece_model_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
    },
}

_A				:    Dict     						={
    '''ernie-m-base''': 5_1_4,
    '''ernie-m-large''': 5_1_4,
}

_A				:    Optional[Any]     						={
    '''ernie-m-base''': {'''do_lower_case''': False},
    '''ernie-m-large''': {'''do_lower_case''': False},
}



class 		lowerCamelCase__      (						A     ):




					'''simple docstring'''



					A_						    =						["input_ids"]

					A_						    =						VOCAB_FILES_NAMES
					A_						    =						PRETRAINED_INIT_CONFIGURATION
					A_						    =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					A_						    =						PRETRAINED_VOCAB_FILES_MAP
					A_						    =						RESOURCE_FILES_NAMES




					def __init__(  self				:	Any     ,  UpperCamelCase_				:	Tuple     ,  UpperCamelCase_				:	Optional[int]=None     ,  UpperCamelCase_				:	Any=False     ,  UpperCamelCase_				:	List[str]="utf8"     ,  UpperCamelCase_				:	Optional[int]="[UNK]"     ,  UpperCamelCase_				:	Dict="[SEP]"     ,  UpperCamelCase_				:	List[str]="[PAD]"     ,  UpperCamelCase_				:	Dict="[CLS]"     ,  UpperCamelCase_				:	Optional[int]="[MASK]"     ,  UpperCamelCase_				:	Optional[Dict[str, Any]] = None     ,  **UpperCamelCase_				:	Optional[int]     ,  )  ->       None:



						'''simple docstring'''





						_lowercase					:       Optional[int]       =		{} if sp_model_kwargs is None else sp_model_kwargs
						super().__init__(
						    do_lower_case=UpperCamelCase_     ,  unk_token=UpperCamelCase_     ,  sep_token=UpperCamelCase_     ,  pad_token=UpperCamelCase_     ,  cls_token=UpperCamelCase_     ,  mask_token=UpperCamelCase_     ,  vocab_file=UpperCamelCase_     ,  encoding=UpperCamelCase_     ,  sp_model_kwargs=self.sp_model_kwargs     ,  **UpperCamelCase_     ,  )
						_lowercase					:       Dict       =		do_lower_case
						_lowercase					:       Optional[Any]       =		sentencepiece_model_ckpt
						_lowercase					:       Tuple       =		spm.SentencePieceProcessor(**self.sp_model_kwargs					)
						self.sp_model.Load(UpperCamelCase_					)

						# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
						if vocab_file is not None:
							_lowercase					:       List[Any]       =		self.load_vocab(filepath=UpperCamelCase_					)
						else:
							_lowercase					:       str       =		{self.sp_model.id_to_piece(UpperCamelCase_					): id for id in range(self.sp_model.get_piece_size()					)}
						_lowercase					:       List[str]       =		{v: k for k, v in self.vocab.items()}




					def 						__UpperCAmelCase     (  self				:	List[str]     ,  UpperCamelCase_				:	Dict					)  ->       Union[str, Any]:



						'''simple docstring'''





						if text is None:
							return None

						_lowercase					:       Optional[Any]       =		self.tokenize(UpperCamelCase_					)
						_lowercase  ,    _lowercase					:       int       =		'', []

						for i, ch in enumerate(UpperCamelCase_					):
							if ch in self.SP_CHAR_MAPPING:
								_lowercase					:       str       =		self.SP_CHAR_MAPPING.get(UpperCamelCase_					)
							else:
								_lowercase					:       Optional[int]       =		unicodedata.normalize('NFKC'     ,  UpperCamelCase_					)
							if self.is_whitespace(UpperCamelCase_					):
								continue
							normalized_text += ch
							char_mapping.extend([i] * len(UpperCamelCase_					)					)

						_lowercase  ,    _lowercase  ,    _lowercase					:       Any       =		normalized_text, [], 0

						if self.do_lower_case:
							_lowercase					:       Union[str, Any]       =		text.lower()

						for token in split_tokens:
							if token[:1] == "▁":
								_lowercase					:       Tuple       =		token[1:]
							_lowercase					:       Optional[Any]       =		text[offset:].index(UpperCamelCase_					) + offset
							_lowercase					:       Any       =		start + len(UpperCamelCase_					)

							token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)					)
							_lowercase					:       Tuple       =		end
						return token_mapping




					@property
					def 						__UpperCAmelCase     (  self				:	Dict					)  ->       Tuple:



						'''simple docstring'''





						return len(self.vocab					)




					def 						__UpperCAmelCase     (  self				:	Tuple					)  ->       Optional[Any]:



						'''simple docstring'''





						return dict(self.vocab     ,  **self.added_tokens_encoder					)




					def __getstate__(  self				:	List[str]					)  ->       List[Any]:



						'''simple docstring'''





						_lowercase					:       Optional[int]       =		self.__dict__.copy()
						_lowercase					:       List[str]       =		None
						return state




					def __setstate__(  self				:	List[Any]     ,  UpperCamelCase_				:	Dict					)  ->       Optional[Any]:



						'''simple docstring'''





						_lowercase					:       Any       =		d

						# for backward compatibility
						if not hasattr(self     ,  'sp_model_kwargs'					):
							_lowercase					:       Tuple       =		{}

						_lowercase					:       Tuple       =		spm.SentencePieceProcessor(**self.sp_model_kwargs					)
						self.sp_model.Load(self.sentencepiece_model_ckpt					)




					def 						__UpperCAmelCase     (  self				:	Dict     ,  UpperCamelCase_				:	Dict					)  ->       Tuple:



						'''simple docstring'''





						return "".join((self.SP_CHAR_MAPPING.get(UpperCamelCase_     ,  UpperCamelCase_					) for c in text)					)




					def 						__UpperCAmelCase     (  self				:	Union[str, Any]     ,  UpperCamelCase_				:	int     ,  UpperCamelCase_				:	Dict=False     ,  UpperCamelCase_				:	Optional[int]=64     ,  UpperCamelCase_				:	Tuple=0.1					)  ->       Union[str, Any]:



						'''simple docstring'''





						if self.sp_model_kwargs.get('enable_sampling'					) is True:
							_lowercase					:       List[Any]       =		True
						if self.sp_model_kwargs.get('alpha'					) is not None:
							_lowercase					:       Optional[int]       =		self.sp_model_kwargs.get('alpha'					)
						if self.sp_model_kwargs.get('nbest_size'					) is not None:
							_lowercase					:       List[Any]       =		self.sp_model_kwargs.get('nbest_size'					)

						if not enable_sampling:
							_lowercase					:       Any       =		self.sp_model.EncodeAsPieces(UpperCamelCase_					)
						else:
							_lowercase					:       Tuple       =		self.sp_model.SampleEncodeAsPieces(UpperCamelCase_     ,  UpperCamelCase_     ,  UpperCamelCase_					)
						_lowercase					:       Any       =		[]
						for pi, piece in enumerate(UpperCamelCase_					):
							if piece == SPIECE_UNDERLINE:
								if not pieces[pi + 1].startswith(UpperCamelCase_					) and pi != 0:
									new_pieces.append(UpperCamelCase_					)
									continue
								else:
									continue
							_lowercase					:       str       =		0
							for i, chunk in enumerate(UpperCamelCase_					):
								if chunk == SPIECE_UNDERLINE:
									continue
								if self.is_ch_char(UpperCamelCase_					) or self.is_punct(UpperCamelCase_					):
									if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
										new_pieces.append(piece[lst_i:i]					)
									new_pieces.append(UpperCamelCase_					)
									_lowercase					:       Tuple       =		i + 1
								elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
									if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
										new_pieces.append(piece[lst_i:i]					)
									_lowercase					:       Any       =		i
								elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
									if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
										new_pieces.append(piece[lst_i:i]					)
									_lowercase					:       Any       =		i
							if len(UpperCamelCase_					) > lst_i:
								new_pieces.append(piece[lst_i:]					)
						return new_pieces




					def 						__UpperCAmelCase     (  self				:	Tuple     ,  UpperCamelCase_				:	Dict					)  ->       Optional[int]:



						'''simple docstring'''





						_lowercase					:       str       =		''.join(UpperCamelCase_					).replace(UpperCamelCase_     ,  ' '					).strip()
						return out_string




					def 						__UpperCAmelCase     (  self				:	str     ,  UpperCamelCase_				:	str					)  ->       Any:



						'''simple docstring'''





						_lowercase					:       str       =		self.convert_ids_to_tokens(UpperCamelCase_					)
						_lowercase					:       int       =		''.join(UpperCamelCase_					).replace(UpperCamelCase_     ,  ' '					).strip()
						return out_string




					def 						__UpperCAmelCase     (  self				:	Optional[Any]     ,  UpperCamelCase_				:	Any					)  ->       int:



						'''simple docstring'''





						return self.vocab.get(UpperCamelCase_     ,  self.vocab.get(self.unk_token					)					)




					def 						__UpperCAmelCase     (  self				:	Optional[int]     ,  UpperCamelCase_				:	List[str]					)  ->       str:



						'''simple docstring'''





						return self.reverse_vocab.get(UpperCamelCase_     ,  self.unk_token					)




					def 						__UpperCAmelCase     (  self				:	Optional[int]     ,  UpperCamelCase_				:	str     ,  UpperCamelCase_				:	int=None					)  ->       str:



						'''simple docstring'''





						if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
						_lowercase					:       List[Any]       =		[self.cls_token_id]
						_lowercase					:       Tuple       =		[self.sep_token_id]
						return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep




					def 						__UpperCAmelCase     (  self				:	Dict     ,  UpperCamelCase_				:	Optional[Any]     ,  UpperCamelCase_				:	str=None					)  ->       int:



						'''simple docstring'''





						if offset_mapping_a is None:
							return [(0, 0)] + offset_mapping_a + [(0, 0)]

						return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]




					def 						__UpperCAmelCase     (  self				:	Optional[Any]     ,  UpperCamelCase_				:	Optional[int]     ,  UpperCamelCase_				:	Optional[int]=None     ,  UpperCamelCase_				:	Any=False					)  ->       Tuple:



						'''simple docstring'''





						if already_has_special_tokens:
							if token_ids_a is not None:
								raise ValueError(
								    'You should not supply a second sequence if the provided sequence of '
								    'ids is already formatted with special tokens for the model.'					)
							return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

						if token_ids_a is not None:
							return [1] + ([0] * len(UpperCamelCase_					)) + [1, 1] + ([0] * len(UpperCamelCase_					)) + [1]
						return [1] + ([0] * len(UpperCamelCase_					)) + [1]




					def 						__UpperCAmelCase     (  self				:	Union[str, Any]     ,  UpperCamelCase_				:	List[int]     ,  UpperCamelCase_				:	Optional[List[int]] = None					)  ->       List[int]:



						'''simple docstring'''





						if token_ids_a is None:
							# [CLS] X [SEP]
							return (len(UpperCamelCase_					) + 2) * [0]

						# [CLS] A [SEP] [SEP] B [SEP]
						return [0] * (len(UpperCamelCase_					) + 1) + [1] * (len(UpperCamelCase_					) + 3)




					def 						__UpperCAmelCase     (  self				:	str     ,  UpperCamelCase_				:	Union[str, Any]					)  ->       List[Any]:



						'''simple docstring'''





						if "\u4e00" <= char <= "\u9fff":
							return True
						return False




					def 						__UpperCAmelCase     (  self				:	Optional[int]     ,  UpperCamelCase_				:	List[Any]					)  ->       List[str]:



						'''simple docstring'''





						if ("a" <= char <= "z") or ("A" <= char <= "Z"):
							return True
						return False




					def 						__UpperCAmelCase     (  self				:	Union[str, Any]     ,  UpperCamelCase_				:	List[str]					)  ->       List[str]:



						'''simple docstring'''





						if char in ",;:.?!~，；：。？！《》【】":
							return True
						return False




					def 						__UpperCAmelCase     (  self				:	Any     ,  UpperCamelCase_				:	Optional[Any]					)  ->       int:



						'''simple docstring'''





						if char == " " or char == "\t" or char == "\n" or char == "\r":
							return True
						if len(UpperCamelCase_					) == 1:
							_lowercase					:       Any       =		unicodedata.category(UpperCamelCase_					)
							if cat == "Zs":
								return True
						return False




					def 						__UpperCAmelCase     (  self				:	Union[str, Any]     ,  UpperCamelCase_				:	Any					)  ->       Optional[int]:



						'''simple docstring'''





						_lowercase					:       List[str]       =		{}
						with io.open(UpperCamelCase_     ,  'r'     ,  encoding='utf-8'					) as f:
							for index, line in enumerate(UpperCamelCase_					):
								_lowercase					:       List[str]       =		line.rstrip('\n'					)
								_lowercase					:       Optional[Any]       =		int(UpperCamelCase_					)

						return token_to_idx




					def 						__UpperCAmelCase     (  self				:	Dict     ,  UpperCamelCase_				:	str     ,  UpperCamelCase_				:	Optional[str] = None					)  ->       Tuple[str]:



						'''simple docstring'''





						_lowercase					:       Optional[int]       =		0
						if os.path.isdir(UpperCamelCase_					):
							_lowercase					:       Any       =		os.path.join(
							    UpperCamelCase_     ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)
						else:
							_lowercase					:       Union[str, Any]       =		(filename_prefix + '-' if filename_prefix else '') + save_directory
						with open(UpperCamelCase_     ,  'w'     ,  encoding='utf-8'					) as writer:
							for token, token_index in sorted(self.vocab.items()     ,  key=lambda UpperCamelCase_					: kv[1]					):
								if index != token_index:
									logger.warning(
									    F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
									    ' Please check that the vocabulary is not corrupted!'					)
									_lowercase					:       Optional[Any]       =		token_index
								writer.write(token + '\n'					)
								index += 1

						_lowercase					:       int       =		os.path.join(UpperCamelCase_     ,  'sentencepiece.bpe.model'					)
						with open(UpperCamelCase_     ,  'wb'					) as fi:
							_lowercase					:       int       =		self.sp_model.serialized_model_proto()
							fi.write(UpperCamelCase_					)

						return (vocab_file,)





'''simple docstring'''

import unittest

from transformers import AutoTokenizer, is_flax_available
from transformers.testing_utils import require_flax, require_sentencepiece, require_tokenizers, slow


if is_flax_available():
		import jax.numpy as jnp

		from transformers import FlaxXLMRobertaModel



@require_sentencepiece
@require_tokenizers
@require_flax
class 		lowerCamelCase__      (						unittest.TestCase     ):




					'''simple docstring'''



					@slow
					def 						__UpperCAmelCase     (  self				:	int					)  ->       Union[str, Any]:



						'''simple docstring'''





						_lowercase					:       List[Any]       =		FlaxXLMRobertaModel.from_pretrained('xlm-roberta-base'					)
						_lowercase					:       str       =		AutoTokenizer.from_pretrained('xlm-roberta-base'					)
						_lowercase					:       List[Any]       =		'The dog is cute and lives in the garden house'
						_lowercase					:       Optional[int]       =		jnp.array([tokenizer.encode(UpperCamelCase_					)]					)

						_lowercase					:       int       =		(1, 12, 768)  # batch_size, sequence_length, embedding_vector_dim
						_lowercase					:       Tuple       =		jnp.array(
						    [[-0.01_01, 0.12_18, -0.08_03, 0.08_01, 0.13_27, 0.07_76, -0.12_15, 0.23_83, 0.33_38, 0.31_06, 0.03_00, 0.02_52]]					)

						_lowercase					:       List[str]       =		model(UpperCamelCase_					)['last_hidden_state']
						self.assertEqual(output.shape     ,  UpperCamelCase_					)
						# compare the actual values for a slice of last dim
						self.assertTrue(jnp.allclose(output[:, :, -1]     ,  UpperCamelCase_     ,  atol=1E-3					)					)
