



'''simple docstring'''



import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase	 =       logging.get_logger(__name__)

lowerCamelCase	 =       """▁"""

lowerCamelCase	 =       {"""vocab_file""": """vocab.txt""", """sentencepiece_model_ckpt""": """sentencepiece.bpe.model"""}

lowerCamelCase	 =       {
    """sentencepiece_model_file""": """sentencepiece.bpe.model""",
    """vocab_file""": """vocab.txt""",
}

lowerCamelCase	 =       {
    """vocab_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
    },
    """sentencepiece_model_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
    },
}

lowerCamelCase	 =       {
    """ernie-m-base""": 514,
    """ernie-m-large""": 514,
}

lowerCamelCase	 =       {
    """ernie-m-base""": {"""do_lower_case""": False},
    """ernie-m-large""": {"""do_lower_case""": False},
}





class _UpperCamelCase		(     A		):





				'''simple docstring'''





				lowerCAmelCase__					    =    ["input_ids"]

				lowerCAmelCase__					    =    VOCAB_FILES_NAMES
				lowerCAmelCase__					    =    PRETRAINED_INIT_CONFIGURATION
				lowerCAmelCase__					    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCAmelCase__					    =    PRETRAINED_VOCAB_FILES_MAP
				lowerCAmelCase__					    =    RESOURCE_FILES_NAMES

				def __init__(							self  :  Optional[Any]  ,       _lowerCAmelCase  :  List[Any]  ,       _lowerCAmelCase  :  Optional[int]=None  ,       _lowerCAmelCase  :  Optional[int]=False  ,       _lowerCAmelCase  :  List[str]="utf8"  ,       _lowerCAmelCase  :  Any="[UNK]"  ,       _lowerCAmelCase  :  Union[str, Any]="[SEP]"  ,       _lowerCAmelCase  :  Optional[int]="[PAD]"  ,       _lowerCAmelCase  :  List[str]="[CLS]"  ,       _lowerCAmelCase  :  int="[MASK]"  ,       _lowerCAmelCase  :  Optional[Dict[str, Any]] = None  ,       **_lowerCAmelCase  :  str  ,       ):







											'''simple docstring'''

											__lowercase       	={} if sp_model_kwargs is None else sp_model_kwargs
											super().__init__(
											    do_lower_case=_lowerCAmelCase  ,       unk_token=_lowerCAmelCase  ,       sep_token=_lowerCAmelCase  ,       pad_token=_lowerCAmelCase  ,       cls_token=_lowerCAmelCase  ,       mask_token=_lowerCAmelCase  ,       vocab_file=_lowerCAmelCase  ,       encoding=_lowerCAmelCase  ,       sp_model_kwargs=self.sp_model_kwargs  ,       **_lowerCAmelCase  ,       )
											__lowercase       	=do_lower_case
											__lowercase       	=sentencepiece_model_ckpt
											__lowercase       	=spm.SentencePieceProcessor(**self.sp_model_kwargs)
											self.sp_model.Load(_lowerCAmelCase)

											# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
											if vocab_file is not None:
																		__lowercase       	=self.load_vocab(filepath=_lowerCAmelCase)
											else:
																		__lowercase       	={self.sp_model.id_to_piece(_lowerCAmelCase): id for id in range(self.sp_model.get_piece_size())}
											__lowercase       	={v: k for k, v in self.vocab.items()}

				def 				__lowerCamelCase     (							self  :  Any  ,       _lowerCAmelCase  :  str):







											'''simple docstring'''

											if text is None:
																		return None

											__lowercase       	=self.tokenize(_lowerCAmelCase)
											__lowercase	,				__lowercase       	='', []

											for i, ch in enumerate(_lowerCAmelCase):
																		if ch in self.SP_CHAR_MAPPING:
																									__lowercase       	=self.SP_CHAR_MAPPING.get(_lowerCAmelCase)
																		else:
																									__lowercase       	=unicodedata.normalize('NFKC'  ,       _lowerCAmelCase)
																		if self.is_whitespace(_lowerCAmelCase):
																									continue
																		normalized_text += ch
																		char_mapping.extend([i] * len(_lowerCAmelCase))

											__lowercase	,				__lowercase	,				__lowercase       	=normalized_text, [], 0

											if self.do_lower_case:
																		__lowercase       	=text.lower()

											for token in split_tokens:
																		if token[:1] == "▁":
																									__lowercase       	=token[1:]
																		__lowercase       	=text[offset:].index(_lowerCAmelCase) + offset
																		__lowercase       	=start + len(_lowerCAmelCase)

																		token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1))
																		__lowercase       	=end
											return token_mapping

				@property
				def 				__lowerCamelCase     (							self  :  List[str]):







											'''simple docstring'''

											return len(self.vocab)

				def 				__lowerCamelCase     (							self  :  List[str]):







											'''simple docstring'''

											return dict(self.vocab  ,       **self.added_tokens_encoder)

				def __getstate__(							self  :  Union[str, Any]):







											'''simple docstring'''

											__lowercase       	=self.__dict__.copy()
											__lowercase       	=None
											return state

				def __setstate__(							self  :  Any  ,       _lowerCAmelCase  :  Any):







											'''simple docstring'''

											__lowercase       	=d

											# for backward compatibility
											if not hasattr(self  ,       'sp_model_kwargs'):
																		__lowercase       	={}

											__lowercase       	=spm.SentencePieceProcessor(**self.sp_model_kwargs)
											self.sp_model.Load(self.sentencepiece_model_ckpt)

				def 				__lowerCamelCase     (							self  :  Optional[int]  ,       _lowerCAmelCase  :  Optional[int]):







											'''simple docstring'''

											return "".join((self.SP_CHAR_MAPPING.get(_lowerCAmelCase  ,       _lowerCAmelCase) for c in text))

				def 				__lowerCamelCase     (							self  :  Tuple  ,       _lowerCAmelCase  :  int  ,       _lowerCAmelCase  :  List[Any]=False  ,       _lowerCAmelCase  :  Union[str, Any]=6_4  ,       _lowerCAmelCase  :  int=0.1):







											'''simple docstring'''

											if self.sp_model_kwargs.get('enable_sampling') is True:
																		__lowercase       	=True
											if self.sp_model_kwargs.get('alpha') is not None:
																		__lowercase       	=self.sp_model_kwargs.get('alpha')
											if self.sp_model_kwargs.get('nbest_size') is not None:
																		__lowercase       	=self.sp_model_kwargs.get('nbest_size')

											if not enable_sampling:
																		__lowercase       	=self.sp_model.EncodeAsPieces(_lowerCAmelCase)
											else:
																		__lowercase       	=self.sp_model.SampleEncodeAsPieces(_lowerCAmelCase  ,       _lowerCAmelCase  ,       _lowerCAmelCase)
											__lowercase       	=[]
											for pi, piece in enumerate(_lowerCAmelCase):
																		if piece == SPIECE_UNDERLINE:
																									if not pieces[pi + 1].startswith(_lowerCAmelCase) and pi != 0:
																																new_pieces.append(_lowerCAmelCase)
																																continue
																									else:
																																continue
																		__lowercase       	=0
																		for i, chunk in enumerate(_lowerCAmelCase):
																									if chunk == SPIECE_UNDERLINE:
																																continue
																									if self.is_ch_char(_lowerCAmelCase) or self.is_punct(_lowerCAmelCase):
																																if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																							new_pieces.append(piece[lst_i:i])
																																new_pieces.append(_lowerCAmelCase)
																																__lowercase       	=i + 1
																									elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																																if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																							new_pieces.append(piece[lst_i:i])
																																__lowercase       	=i
																									elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																																if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																							new_pieces.append(piece[lst_i:i])
																																__lowercase       	=i
																		if len(_lowerCAmelCase) > lst_i:
																									new_pieces.append(piece[lst_i:])
											return new_pieces

				def 				__lowerCamelCase     (							self  :  Dict  ,       _lowerCAmelCase  :  Dict):







											'''simple docstring'''

											__lowercase       	=''.join(_lowerCAmelCase).replace(_lowerCAmelCase  ,       ' ').strip()
											return out_string

				def 				__lowerCamelCase     (							self  :  Optional[Any]  ,       _lowerCAmelCase  :  List[Any]):







											'''simple docstring'''

											__lowercase       	=self.convert_ids_to_tokens(_lowerCAmelCase)
											__lowercase       	=''.join(_lowerCAmelCase).replace(_lowerCAmelCase  ,       ' ').strip()
											return out_string

				def 				__lowerCamelCase     (							self  :  Optional[int]  ,       _lowerCAmelCase  :  List[Any]):







											'''simple docstring'''

											return self.vocab.get(_lowerCAmelCase  ,       self.vocab.get(self.unk_token))

				def 				__lowerCamelCase     (							self  :  int  ,       _lowerCAmelCase  :  Any):







											'''simple docstring'''

											return self.reverse_vocab.get(_lowerCAmelCase  ,       self.unk_token)

				def 				__lowerCamelCase     (							self  :  Any  ,       _lowerCAmelCase  :  str  ,       _lowerCAmelCase  :  int=None):







											'''simple docstring'''

											if token_ids_a is None:
																		return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
											__lowercase       	=[self.cls_token_id]
											__lowercase       	=[self.sep_token_id]
											return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep

				def 				__lowerCamelCase     (							self  :  Optional[Any]  ,       _lowerCAmelCase  :  Tuple  ,       _lowerCAmelCase  :  Optional[int]=None):







											'''simple docstring'''

											if offset_mapping_a is None:
																		return [(0, 0)] + offset_mapping_a + [(0, 0)]

											return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]

				def 				__lowerCamelCase     (							self  :  Optional[int]  ,       _lowerCAmelCase  :  Dict  ,       _lowerCAmelCase  :  Optional[Any]=None  ,       _lowerCAmelCase  :  Optional[Any]=False):







											'''simple docstring'''

											if already_has_special_tokens:
																		if token_ids_a is not None:
																									raise ValueError(
																									    'You should not supply a second sequence if the provided sequence of '
																									    'ids is already formatted with special tokens for the model.')
																		return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

											if token_ids_a is not None:
																		return [1] + ([0] * len(_lowerCAmelCase)) + [1, 1] + ([0] * len(_lowerCAmelCase)) + [1]
											return [1] + ([0] * len(_lowerCAmelCase)) + [1]

				def 				__lowerCamelCase     (							self  :  Any  ,       _lowerCAmelCase  :  List[int]  ,       _lowerCAmelCase  :  Optional[List[int]] = None):







											'''simple docstring'''

											if token_ids_a is None:
																		# [CLS] X [SEP]
																		return (len(_lowerCAmelCase) + 2) * [0]

											# [CLS] A [SEP] [SEP] B [SEP]
											return [0] * (len(_lowerCAmelCase) + 1) + [1] * (len(_lowerCAmelCase) + 3)

				def 				__lowerCamelCase     (							self  :  Dict  ,       _lowerCAmelCase  :  List[str]):







											'''simple docstring'''

											if "\u4e00" <= char <= "\u9fff":
																		return True
											return False

				def 				__lowerCamelCase     (							self  :  Optional[Any]  ,       _lowerCAmelCase  :  Dict):







											'''simple docstring'''

											if ("a" <= char <= "z") or ("A" <= char <= "Z"):
																		return True
											return False

				def 				__lowerCamelCase     (							self  :  str  ,       _lowerCAmelCase  :  Dict):







											'''simple docstring'''

											if char in ",;:.?!~，；：。？！《》【】":
																		return True
											return False

				def 				__lowerCamelCase     (							self  :  Union[str, Any]  ,       _lowerCAmelCase  :  int):







											'''simple docstring'''

											if char == " " or char == "\t" or char == "\n" or char == "\r":
																		return True
											if len(_lowerCAmelCase) == 1:
																		__lowercase       	=unicodedata.category(_lowerCAmelCase)
																		if cat == "Zs":
																									return True
											return False

				def 				__lowerCamelCase     (							self  :  int  ,       _lowerCAmelCase  :  int):







											'''simple docstring'''

											__lowercase       	={}
											with io.open(_lowerCAmelCase  ,       'r'  ,       encoding='utf-8') as f:
																		for index, line in enumerate(_lowerCAmelCase):
																									__lowercase       	=line.rstrip('\n')
																									__lowercase       	=int(_lowerCAmelCase)

											return token_to_idx

				def 				__lowerCamelCase     (							self  :  Any  ,       _lowerCAmelCase  :  str  ,       _lowerCAmelCase  :  Optional[str] = None):







											'''simple docstring'''

											__lowercase       	=0
											if os.path.isdir(_lowerCAmelCase):
																		__lowercase       	=os.path.join(
																		    _lowerCAmelCase  ,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])
											else:
																		__lowercase       	=(filename_prefix + '-' if filename_prefix else '') + save_directory
											with open(_lowerCAmelCase  ,       'w'  ,       encoding='utf-8') as writer:
																		for token, token_index in sorted(self.vocab.items()  ,       key=lambda _lowerCAmelCase: kv[1]):
																									if index != token_index:
																																logger.warning(
																																    f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
																																    ' Please check that the vocabulary is not corrupted!')
																																__lowercase       	=token_index
																									writer.write(token + '\n')
																									index += 1

											__lowercase       	=os.path.join(_lowerCAmelCase  ,       'sentencepiece.bpe.model')
											with open(_lowerCAmelCase  ,       'wb') as fi:
																		__lowercase       	=self.sp_model.serialized_model_proto()
																		fi.write(_lowerCAmelCase)

											return (vocab_file,)








'''simple docstring'''



import json
import os
import unittest

from transformers.models.blenderbot_small.tokenization_blenderbot_small import (
    VOCAB_FILES_NAMES,
    BlenderbotSmallTokenizer,
)

from ...test_tokenization_common import TokenizerTesterMixin





class _UpperCamelCase		(     A      ,       unittest.TestCase		):





				'''simple docstring'''





				lowerCAmelCase__					    =    BlenderbotSmallTokenizer
				lowerCAmelCase__					    =    False

				def 				__lowerCamelCase     (							self  :  str):







											'''simple docstring'''

											super().setUp()

											__lowercase       	=['__start__', 'adapt', 'act', 'ap@@', 'te', '__end__', '__unk__']
											__lowercase       	=dict(zip(_lowerCAmelCase  ,       range(len(_lowerCAmelCase))))

											__lowercase       	=['#version: 0.2', 'a p', 't e</w>', 'ap t</w>', 'a d', 'ad apt</w>', 'a c', 'ac t</w>', '']
											__lowercase       	={'unk_token': '__unk__', 'bos_token': '__start__', 'eos_token': '__end__'}

											__lowercase       	=os.path.join(self.tmpdirname  ,       VOCAB_FILES_NAMES['vocab_file'])
											__lowercase       	=os.path.join(self.tmpdirname  ,       VOCAB_FILES_NAMES['merges_file'])
											with open(self.vocab_file  ,       'w'  ,       encoding='utf-8') as fp:
																		fp.write(json.dumps(_lowerCAmelCase) + '\n')
											with open(self.merges_file  ,       'w'  ,       encoding='utf-8') as fp:
																		fp.write('\n'.join(_lowerCAmelCase))

				def 				__lowerCamelCase     (							self  :  int  ,       **_lowerCAmelCase  :  Tuple):







											'''simple docstring'''

											kwargs.update(self.special_tokens_map)
											return BlenderbotSmallTokenizer.from_pretrained(self.tmpdirname  ,       **_lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  Tuple  ,       _lowerCAmelCase  :  Any):







											'''simple docstring'''

											__lowercase       	='adapt act apte'
											__lowercase       	='adapt act apte'
											return input_text, output_text

				def 				__lowerCamelCase     (							self  :  Union[str, Any]):







											'''simple docstring'''

											__lowercase       	=BlenderbotSmallTokenizer(self.vocab_file  ,       self.merges_file  ,       **self.special_tokens_map)
											__lowercase       	='adapt act apte'
											__lowercase       	=['adapt', 'act', 'ap@@', 'te']
											__lowercase       	=tokenizer.tokenize(_lowerCAmelCase)
											self.assertListEqual(_lowerCAmelCase  ,       _lowerCAmelCase)

											__lowercase       	=[tokenizer.bos_token] + tokens + [tokenizer.eos_token]

											__lowercase       	=[0, 1, 2, 3, 4, 5]
											self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase)  ,       _lowerCAmelCase)

				def 				__lowerCamelCase     (							self  :  Union[str, Any]):







											'''simple docstring'''

											__lowercase       	=BlenderbotSmallTokenizer.from_pretrained('facebook/blenderbot-90M')
											assert tok('sam').input_ids == [1_3_8_4]
											__lowercase       	='I am a small frog.'
											__lowercase       	=tok([src_text]  ,       padding=_lowerCAmelCase  ,       truncation=_lowerCAmelCase)['input_ids']
											__lowercase       	=tok.batch_decode(_lowerCAmelCase  ,       skip_special_tokens=_lowerCAmelCase  ,       clean_up_tokenization_spaces=_lowerCAmelCase)[0]
											assert src_text != decoded  # I wish it did!
											assert decoded == "i am a small frog ."

				def 				__lowerCamelCase     (							self  :  str):







											'''simple docstring'''

											__lowercase       	=BlenderbotSmallTokenizer.from_pretrained('facebook/blenderbot-90M')
											__lowercase       	='I am a small frog .'
											__lowercase       	='.'
											__lowercase       	=tok(_lowerCAmelCase)['input_ids']
											__lowercase       	=tok(_lowerCAmelCase)['input_ids']

											assert encoded[-1] == encoded_dot[0]



