





'''simple docstring'''

import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase					:		List[Any]		=							logging.get_logger(__name__)

UpperCAmelCase					:		Union[str, Any]		=							'▁'

UpperCAmelCase					:		Dict		=							{'vocab_file': 'vocab.txt', 'sentencepiece_model_ckpt': 'sentencepiece.bpe.model'}

UpperCAmelCase					:		Optional[Any]		=							{
    'sentencepiece_model_file': 'sentencepiece.bpe.model',
    'vocab_file': 'vocab.txt',
}

UpperCAmelCase					:		Any		=							{
    'vocab_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
    },
    'sentencepiece_model_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
    },
}

UpperCAmelCase					:		Dict		=							{
    'ernie-m-base': 5_1_4,
    'ernie-m-large': 5_1_4,
}

UpperCAmelCase					:		List[str]		=							{
    'ernie-m-base': {'do_lower_case': False},
    'ernie-m-large': {'do_lower_case': False},
}





class lowerCamelCase						(a__					):
  _lowercase  :	List[str]       					=				["input_ids"]

  _lowercase  :	Any       					=				VOCAB_FILES_NAMES
  _lowercase  :	Optional[int]       					=				PRETRAINED_INIT_CONFIGURATION
  _lowercase  :	Dict       					=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _lowercase  :	Dict       					=				PRETRAINED_VOCAB_FILES_MAP
  _lowercase  :	Union[str, Any]       					=				RESOURCE_FILES_NAMES







  def __init__(		self     ,     lowercase__     ,     lowercase__=None     ,     lowercase__=False     ,     lowercase__="utf8"     ,     lowercase__="[UNK]"     ,     lowercase__="[SEP]"     ,     lowercase__="[PAD]"     ,     lowercase__="[CLS]"     ,     lowercase__="[MASK]"     ,     lowercase__ = None     ,     **lowercase__     ,     )		->						None:






        """simple docstring"""





        _snake_case		:	List[str]					     =     {} if sp_model_kwargs is None else sp_model_kwargs
        super().__init__(
            do_lower_case=lowercase__     ,     unk_token=lowercase__     ,     sep_token=lowercase__     ,     pad_token=lowercase__     ,     cls_token=lowercase__     ,     mask_token=lowercase__     ,     vocab_file=lowercase__     ,     encoding=lowercase__     ,     sp_model_kwargs=self.sp_model_kwargs     ,     **lowercase__     ,     )
        _snake_case		:	Union[str, Any]					     =     do_lower_case
        _snake_case		:	List[Any]					     =     sentencepiece_model_ckpt
        _snake_case		:	Tuple					     =     spm.SentencePieceProcessor(**self.sp_model_kwargs		)
        self.sp_model.Load(lowercase__		)

        # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
        if vocab_file is not None:
              _snake_case		:	Union[str, Any]					     =     self.load_vocab(filepath=lowercase__		)
        else:
              _snake_case		:	int					     =     {self.sp_model.id_to_piece(lowercase__		): id for id in range(self.sp_model.get_piece_size()		)}
        _snake_case		:	Any					     =     {v: k for k, v in self.vocab.items()}







  def 				UpperCAmelCase_				(		self     ,     lowercase__		)		->						Optional[Any]:






        """simple docstring"""





        if text is None:
              return None

        _snake_case		:	List[str]					     =     self.tokenize(lowercase__		)
        _snake_case     ,							_snake_case		:	Union[str, Any]					     =     '''''', []

        for i, ch in enumerate(lowercase__		):
              if ch in self.SP_CHAR_MAPPING:
                    _snake_case		:	Optional[Any]					     =     self.SP_CHAR_MAPPING.get(lowercase__		)
              else:
                    _snake_case		:	Tuple					     =     unicodedata.normalize('''NFKC'''     ,     lowercase__		)
              if self.is_whitespace(lowercase__		):
                    continue
              normalized_text += ch
              char_mapping.extend([i] * len(lowercase__		)		)

        _snake_case     ,							_snake_case     ,							_snake_case		:	Dict					     =     normalized_text, [], 0

        if self.do_lower_case:
              _snake_case		:	Optional[int]					     =     text.lower()

        for token in split_tokens:
              if token[:1] == "▁":
                    _snake_case		:	List[Any]					     =     token[1:]
              _snake_case		:	int					     =     text[offset:].index(lowercase__		) + offset
              _snake_case		:	Optional[Any]					     =     start + len(lowercase__		)

              token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)		)
              _snake_case		:	str					     =     end
        return token_mapping







  @property
  def 				UpperCAmelCase_				(		self		)		->						Union[str, Any]:






        """simple docstring"""





        return len(self.vocab		)







  def 				UpperCAmelCase_				(		self		)		->						str:






        """simple docstring"""





        return dict(self.vocab     ,     **self.added_tokens_encoder		)







  def __getstate__(		self		)		->						Optional[Any]:






        """simple docstring"""





        _snake_case		:	str					     =     self.__dict__.copy()
        _snake_case		:	Optional[Any]					     =     None
        return state







  def __setstate__(		self     ,     lowercase__		)		->						Tuple:






        """simple docstring"""





        _snake_case		:	List[Any]					     =     d

        # for backward compatibility
        if not hasattr(self     ,     '''sp_model_kwargs'''		):
              _snake_case		:	str					     =     {}

        _snake_case		:	Optional[Any]					     =     spm.SentencePieceProcessor(**self.sp_model_kwargs		)
        self.sp_model.Load(self.sentencepiece_model_ckpt		)







  def 				UpperCAmelCase_				(		self     ,     lowercase__		)		->						Union[str, Any]:






        """simple docstring"""





        return "".join((self.SP_CHAR_MAPPING.get(lowercase__     ,     lowercase__		) for c in text)		)







  def 				UpperCAmelCase_				(		self     ,     lowercase__     ,     lowercase__=False     ,     lowercase__=64     ,     lowercase__=0.1		)		->						Any:






        """simple docstring"""





        if self.sp_model_kwargs.get('''enable_sampling'''		) is True:
              _snake_case		:	int					     =     True
        if self.sp_model_kwargs.get('''alpha'''		) is not None:
              _snake_case		:	Dict					     =     self.sp_model_kwargs.get('''alpha'''		)
        if self.sp_model_kwargs.get('''nbest_size'''		) is not None:
              _snake_case		:	Union[str, Any]					     =     self.sp_model_kwargs.get('''nbest_size'''		)

        if not enable_sampling:
              _snake_case		:	str					     =     self.sp_model.EncodeAsPieces(lowercase__		)
        else:
              _snake_case		:	Optional[int]					     =     self.sp_model.SampleEncodeAsPieces(lowercase__     ,     lowercase__     ,     lowercase__		)
        _snake_case		:	Tuple					     =     []
        for pi, piece in enumerate(lowercase__		):
              if piece == SPIECE_UNDERLINE:
                    if not pieces[pi + 1].startswith(lowercase__		) and pi != 0:
                          new_pieces.append(lowercase__		)
                          continue
                    else:
                          continue
              _snake_case		:	List[Any]					     =     0
              for i, chunk in enumerate(lowercase__		):
                    if chunk == SPIECE_UNDERLINE:
                          continue
                    if self.is_ch_char(lowercase__		) or self.is_punct(lowercase__		):
                          if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                new_pieces.append(piece[lst_i:i]		)
                          new_pieces.append(lowercase__		)
                          _snake_case		:	List[str]					     =     i + 1
                    elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                          if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                new_pieces.append(piece[lst_i:i]		)
                          _snake_case		:	Optional[int]					     =     i
                    elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                          if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                new_pieces.append(piece[lst_i:i]		)
                          _snake_case		:	Any					     =     i
              if len(lowercase__		) > lst_i:
                    new_pieces.append(piece[lst_i:]		)
        return new_pieces







  def 				UpperCAmelCase_				(		self     ,     lowercase__		)		->						int:






        """simple docstring"""





        _snake_case		:	Union[str, Any]					     =     ''''''.join(lowercase__		).replace(lowercase__     ,     ''' '''		).strip()
        return out_string







  def 				UpperCAmelCase_				(		self     ,     lowercase__		)		->						str:






        """simple docstring"""





        _snake_case		:	List[Any]					     =     self.convert_ids_to_tokens(lowercase__		)
        _snake_case		:	Dict					     =     ''''''.join(lowercase__		).replace(lowercase__     ,     ''' '''		).strip()
        return out_string







  def 				UpperCAmelCase_				(		self     ,     lowercase__		)		->						Union[str, Any]:






        """simple docstring"""





        return self.vocab.get(lowercase__     ,     self.vocab.get(self.unk_token		)		)







  def 				UpperCAmelCase_				(		self     ,     lowercase__		)		->						Optional[Any]:






        """simple docstring"""





        return self.reverse_vocab.get(lowercase__     ,     self.unk_token		)







  def 				UpperCAmelCase_				(		self     ,     lowercase__     ,     lowercase__=None		)		->						str:






        """simple docstring"""





        if token_ids_a is None:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
        _snake_case		:	List[str]					     =     [self.cls_token_id]
        _snake_case		:	Tuple					     =     [self.sep_token_id]
        return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep







  def 				UpperCAmelCase_				(		self     ,     lowercase__     ,     lowercase__=None		)		->						Union[str, Any]:






        """simple docstring"""





        if offset_mapping_a is None:
              return [(0, 0)] + offset_mapping_a + [(0, 0)]

        return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]







  def 				UpperCAmelCase_				(		self     ,     lowercase__     ,     lowercase__=None     ,     lowercase__=False		)		->						Optional[int]:






        """simple docstring"""





        if already_has_special_tokens:
              if token_ids_a is not None:
                    raise ValueError(
                        '''You should not supply a second sequence if the provided sequence of '''
                        '''ids is already formatted with special tokens for the model.'''		)
              return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

        if token_ids_a is not None:
              return [1] + ([0] * len(lowercase__		)) + [1, 1] + ([0] * len(lowercase__		)) + [1]
        return [1] + ([0] * len(lowercase__		)) + [1]







  def 				UpperCAmelCase_				(		self     ,     lowercase__     ,     lowercase__ = None		)		->						List[int]:






        """simple docstring"""





        if token_ids_a is None:
              # [CLS] X [SEP]
              return (len(lowercase__		) + 2) * [0]

        # [CLS] A [SEP] [SEP] B [SEP]
        return [0] * (len(lowercase__		) + 1) + [1] * (len(lowercase__		) + 3)







  def 				UpperCAmelCase_				(		self     ,     lowercase__		)		->						Dict:






        """simple docstring"""





        if "\u4e00" <= char <= "\u9fff":
              return True
        return False







  def 				UpperCAmelCase_				(		self     ,     lowercase__		)		->						Any:






        """simple docstring"""





        if ("a" <= char <= "z") or ("A" <= char <= "Z"):
              return True
        return False







  def 				UpperCAmelCase_				(		self     ,     lowercase__		)		->						str:






        """simple docstring"""





        if char in ",;:.?!~，；：。？！《》【】":
              return True
        return False







  def 				UpperCAmelCase_				(		self     ,     lowercase__		)		->						Tuple:






        """simple docstring"""





        if char == " " or char == "\t" or char == "\n" or char == "\r":
              return True
        if len(lowercase__		) == 1:
              _snake_case		:	Union[str, Any]					     =     unicodedata.category(lowercase__		)
              if cat == "Zs":
                    return True
        return False







  def 				UpperCAmelCase_				(		self     ,     lowercase__		)		->						Any:






        """simple docstring"""





        _snake_case		:	Union[str, Any]					     =     {}
        with io.open(lowercase__     ,     '''r'''     ,     encoding='''utf-8'''		) as f:
              for index, line in enumerate(lowercase__		):
                    _snake_case		:	Optional[int]					     =     line.rstrip('''\n'''		)
                    _snake_case		:	List[Any]					     =     int(lowercase__		)

        return token_to_idx







  def 				UpperCAmelCase_				(		self     ,     lowercase__     ,     lowercase__ = None		)		->						Tuple[str]:






        """simple docstring"""





        _snake_case		:	Dict					     =     0
        if os.path.isdir(lowercase__		):
              _snake_case		:	Tuple					     =     os.path.join(
                  lowercase__     ,     (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)
        else:
              _snake_case		:	Union[str, Any]					     =     (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
        with open(lowercase__     ,     '''w'''     ,     encoding='''utf-8'''		) as writer:
              for token, token_index in sorted(self.vocab.items()     ,     key=lambda lowercase__		: kv[1]		):
                    if index != token_index:
                          logger.warning(
                              F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
                              ''' Please check that the vocabulary is not corrupted!'''		)
                          _snake_case		:	Optional[int]					     =     token_index
                    writer.write(token + '''\n'''		)
                    index += 1

        _snake_case		:	Dict					     =     os.path.join(lowercase__     ,     '''sentencepiece.bpe.model'''		)
        with open(lowercase__     ,     '''wb'''		) as fi:
              _snake_case		:	Dict					     =     self.sp_model.serialized_model_proto()
              fi.write(lowercase__		)

        return (vocab_file,)









'''simple docstring'''

import math

from numpy import inf
from scipy.integrate import quad




def    _a    (							lowerCAmelCase_	):





      """simple docstring"""



      if num <= 0:
            raise ValueError('''math domain error'''	)

      return quad(lowerCAmelCase_			,      0			,      lowerCAmelCase_			,      args=(lowerCAmelCase_)	)[0]




def    _a    (							lowerCAmelCase_			,      lowerCAmelCase_	):





      """simple docstring"""



      return math.pow(lowerCAmelCase_			,      z - 1	) * math.exp(-x	)


if __name__ == "__main__":
   from doctest import testmod

   testmod()


