





import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__lowerCamelCase :  List[Any]										=  logging.get_logger(__name__)

__lowerCamelCase :  Tuple										=  """▁"""

__lowerCamelCase :  str										=  {"""vocab_file""": """vocab.txt""", """sentencepiece_model_ckpt""": """sentencepiece.bpe.model"""}

__lowerCamelCase :  Tuple										=  {
    """sentencepiece_model_file""": """sentencepiece.bpe.model""",
    """vocab_file""": """vocab.txt""",
}

__lowerCamelCase :  Any										=  {
    """vocab_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
    },
    """sentencepiece_model_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
    },
}

__lowerCamelCase :  List[str]										=  {
    """ernie-m-base""": 514,
    """ernie-m-large""": 514,
}

__lowerCamelCase :  Dict										=  {
    """ernie-m-base""": {"""do_lower_case""": False},
    """ernie-m-large""": {"""do_lower_case""": False},
}



class       A__     ( __A    ):
      _UpperCAmelCase			:List[str]							=     ["input_ids"]

      _UpperCAmelCase			:Dict							=     VOCAB_FILES_NAMES
      _UpperCAmelCase			:Tuple							=     PRETRAINED_INIT_CONFIGURATION
      _UpperCAmelCase			:List[str]							=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _UpperCAmelCase			:List[str]							=     PRETRAINED_VOCAB_FILES_MAP
      _UpperCAmelCase			:Tuple							=     RESOURCE_FILES_NAMES





      def __init__(				self ,							A_ ,							A_=None ,							A_=False ,							A_="utf8" ,							A_="[UNK]" ,							A_="[SEP]" ,							A_="[PAD]" ,							A_="[CLS]" ,							A_="[MASK]" ,							A_ = None ,							**A_ ,							):
       '''simple docstring'''




       UpperCamelCase			:      Optional[int] 			=     {} if sp_model_kwargs is None else sp_model_kwargs
       super().__init__(
           do_lower_case=UpperCamelCase__ ,							unk_token=UpperCamelCase__ ,							sep_token=UpperCamelCase__ ,							pad_token=UpperCamelCase__ ,							cls_token=UpperCamelCase__ ,							mask_token=UpperCamelCase__ ,							vocab_file=UpperCamelCase__ ,							encoding=UpperCamelCase__ ,							sp_model_kwargs=self.sp_model_kwargs ,							**UpperCamelCase__ ,							)
       UpperCamelCase			:      str 			=     do_lower_case
       UpperCamelCase			:      Optional[Any] 			=     sentencepiece_model_ckpt
       UpperCamelCase			:      List[Any] 			=     spm.SentencePieceProcessor(**self.sp_model_kwargs     )
       self.sp_model.Load(UpperCamelCase__     )

       # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
       if vocab_file is not None:
        UpperCamelCase			:      Optional[int] 			=     self.load_vocab(filepath=UpperCamelCase__     )
       else:
        UpperCamelCase			:      Dict 			=     {self.sp_model.id_to_piece(UpperCamelCase__     ): id for id in range(self.sp_model.get_piece_size()     )}
       UpperCamelCase			:      Tuple 			=     {v: k for k, v in self.vocab.items()}





      def  __UpperCamelCase(				self ,							A_     ):
       '''simple docstring'''




       if text is None:
        return None

       UpperCamelCase			:      Optional[Any] 			=     self.tokenize(UpperCamelCase__     )
       UpperCamelCase			:      Dict 			=     '', []

       for i, ch in enumerate(UpperCamelCase__     ):
        if ch in self.SP_CHAR_MAPPING:
         UpperCamelCase			:      Any 			=     self.SP_CHAR_MAPPING.get(UpperCamelCase__     )
        else:
         UpperCamelCase			:      Tuple 			=     unicodedata.normalize("NFKC" ,							UpperCamelCase__     )
        if self.is_whitespace(UpperCamelCase__     ):
         continue
        normalized_text += ch
        char_mapping.extend([i] * len(UpperCamelCase__     )     )

       UpperCamelCase			:      Tuple 			=     normalized_text, [], 0

       if self.do_lower_case:
        UpperCamelCase			:      int 			=     text.lower()

       for token in split_tokens:
        if token[:1] == "▁":
         UpperCamelCase			:      Dict 			=     token[1:]
        UpperCamelCase			:      List[str] 			=     text[offset:].index(UpperCamelCase__     ) + offset
        UpperCamelCase			:      Tuple 			=     start + len(UpperCamelCase__     )

        token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)     )
        UpperCamelCase			:      List[Any] 			=     end
       return token_mapping





      @property
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       return len(self.vocab     )





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       return dict(self.vocab ,							**self.added_tokens_encoder     )





      def __getstate__(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      Optional[int] 			=     self.__dict__.copy()
       UpperCamelCase			:      int 			=     None
       return state





      def __setstate__(				self ,							A_     ):
       '''simple docstring'''




       UpperCamelCase			:      List[Any] 			=     d

       # for backward compatibility
       if not hasattr(self ,							"sp_model_kwargs"     ):
        UpperCamelCase			:      Tuple 			=     {}

       UpperCamelCase			:      List[str] 			=     spm.SentencePieceProcessor(**self.sp_model_kwargs     )
       self.sp_model.Load(self.sentencepiece_model_ckpt     )





      def  __UpperCamelCase(				self ,							A_     ):
       '''simple docstring'''




       return "".join((self.SP_CHAR_MAPPING.get(UpperCamelCase__ ,							UpperCamelCase__     ) for c in text)     )





      def  __UpperCamelCase(				self ,							A_ ,							A_=False ,							A_=64 ,							A_=0.1     ):
       '''simple docstring'''




       if self.sp_model_kwargs.get("enable_sampling"     ) is True:
        UpperCamelCase			:      int 			=     True
       if self.sp_model_kwargs.get("alpha"     ) is not None:
        UpperCamelCase			:      Tuple 			=     self.sp_model_kwargs.get("alpha"     )
       if self.sp_model_kwargs.get("nbest_size"     ) is not None:
        UpperCamelCase			:      Dict 			=     self.sp_model_kwargs.get("nbest_size"     )

       if not enable_sampling:
        UpperCamelCase			:      Optional[int] 			=     self.sp_model.EncodeAsPieces(UpperCamelCase__     )
       else:
        UpperCamelCase			:      Dict 			=     self.sp_model.SampleEncodeAsPieces(UpperCamelCase__ ,							UpperCamelCase__ ,							UpperCamelCase__     )
       UpperCamelCase			:      Any 			=     []
       for pi, piece in enumerate(UpperCamelCase__     ):
        if piece == SPIECE_UNDERLINE:
         if not pieces[pi + 1].startswith(UpperCamelCase__     ) and pi != 0:
          new_pieces.append(UpperCamelCase__     )
          continue
         else:
          continue
        UpperCamelCase			:      Optional[Any] 			=     0
        for i, chunk in enumerate(UpperCamelCase__     ):
         if chunk == SPIECE_UNDERLINE:
          continue
         if self.is_ch_char(UpperCamelCase__     ) or self.is_punct(UpperCamelCase__     ):
          if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
           new_pieces.append(piece[lst_i:i]     )
          new_pieces.append(UpperCamelCase__     )
          UpperCamelCase			:      List[Any] 			=     i + 1
         elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
          if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
           new_pieces.append(piece[lst_i:i]     )
          UpperCamelCase			:      str 			=     i
         elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
          if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
           new_pieces.append(piece[lst_i:i]     )
          UpperCamelCase			:      List[Any] 			=     i
        if len(UpperCamelCase__     ) > lst_i:
         new_pieces.append(piece[lst_i:]     )
       return new_pieces





      def  __UpperCamelCase(				self ,							A_     ):
       '''simple docstring'''




       UpperCamelCase			:      Optional[int] 			=     ''.join(UpperCamelCase__     ).replace(UpperCamelCase__ ,							" "     ).strip()
       return out_string





      def  __UpperCamelCase(				self ,							A_     ):
       '''simple docstring'''




       UpperCamelCase			:      Dict 			=     self.convert_ids_to_tokens(UpperCamelCase__     )
       UpperCamelCase			:      int 			=     ''.join(UpperCamelCase__     ).replace(UpperCamelCase__ ,							" "     ).strip()
       return out_string





      def  __UpperCamelCase(				self ,							A_     ):
       '''simple docstring'''




       return self.vocab.get(UpperCamelCase__ ,							self.vocab.get(self.unk_token     )     )





      def  __UpperCamelCase(				self ,							A_     ):
       '''simple docstring'''




       return self.reverse_vocab.get(UpperCamelCase__ ,							self.unk_token     )





      def  __UpperCamelCase(				self ,							A_ ,							A_=None     ):
       '''simple docstring'''




       if token_ids_a is None:
        return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       UpperCamelCase			:      Optional[int] 			=     [self.cls_token_id]
       UpperCamelCase			:      Optional[Any] 			=     [self.sep_token_id]
       return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep





      def  __UpperCamelCase(				self ,							A_ ,							A_=None     ):
       '''simple docstring'''




       if offset_mapping_a is None:
        return [(0, 0)] + offset_mapping_a + [(0, 0)]

       return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]





      def  __UpperCamelCase(				self ,							A_ ,							A_=None ,							A_=False     ):
       '''simple docstring'''




       if already_has_special_tokens:
        if token_ids_a is not None:
         raise ValueError(
             "You should not supply a second sequence if the provided sequence of "
             "ids is already formatted with special tokens for the model."     )
        return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

       if token_ids_a is not None:
        return [1] + ([0] * len(UpperCamelCase__     )) + [1, 1] + ([0] * len(UpperCamelCase__     )) + [1]
       return [1] + ([0] * len(UpperCamelCase__     )) + [1]





      def  __UpperCamelCase(				self ,							A_ ,							A_ = None     ):
       '''simple docstring'''




       if token_ids_a is None:
        # [CLS] X [SEP]
        return (len(UpperCamelCase__     ) + 2) * [0]

       # [CLS] A [SEP] [SEP] B [SEP]
       return [0] * (len(UpperCamelCase__     ) + 1) + [1] * (len(UpperCamelCase__     ) + 3)





      def  __UpperCamelCase(				self ,							A_     ):
       '''simple docstring'''




       if "\u4e00" <= char <= "\u9fff":
        return True
       return False





      def  __UpperCamelCase(				self ,							A_     ):
       '''simple docstring'''




       if ("a" <= char <= "z") or ("A" <= char <= "Z"):
        return True
       return False





      def  __UpperCamelCase(				self ,							A_     ):
       '''simple docstring'''




       if char in ",;:.?!~，；：。？！《》【】":
        return True
       return False





      def  __UpperCamelCase(				self ,							A_     ):
       '''simple docstring'''




       if char == " " or char == "\t" or char == "\n" or char == "\r":
        return True
       if len(UpperCamelCase__     ) == 1:
        UpperCamelCase			:      int 			=     unicodedata.category(UpperCamelCase__     )
        if cat == "Zs":
         return True
       return False





      def  __UpperCamelCase(				self ,							A_     ):
       '''simple docstring'''




       UpperCamelCase			:      Optional[Any] 			=     {}
       with io.open(UpperCamelCase__ ,							"r" ,							encoding="utf-8"     ) as f:
        for index, line in enumerate(UpperCamelCase__     ):
         UpperCamelCase			:      Union[str, Any] 			=     line.rstrip("\n"     )
         UpperCamelCase			:      Tuple 			=     int(UpperCamelCase__     )

       return token_to_idx





      def  __UpperCamelCase(				self ,							A_ ,							A_ = None     ):
       '''simple docstring'''




       UpperCamelCase			:      Union[str, Any] 			=     0
       if os.path.isdir(UpperCamelCase__     ):
        UpperCamelCase			:      int 			=     os.path.join(
            UpperCamelCase__ ,							(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )
       else:
        UpperCamelCase			:      List[Any] 			=     (filename_prefix + '-' if filename_prefix else '') + save_directory
       with open(UpperCamelCase__ ,							"w" ,							encoding="utf-8"     ) as writer:
        for token, token_index in sorted(self.vocab.items() ,							key=lambda A_     : kv[1]     ):
         if index != token_index:
          logger.warning(
              F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
              " Please check that the vocabulary is not corrupted!"     )
          UpperCamelCase			:      str 			=     token_index
         writer.write(token + "\n"     )
         index += 1

       UpperCamelCase			:      Optional[Any] 			=     os.path.join(UpperCamelCase__ ,							"sentencepiece.bpe.model"     )
       with open(UpperCamelCase__ ,							"wb"     ) as fi:
        UpperCamelCase			:      int 			=     self.sp_model.serialized_model_proto()
        fi.write(UpperCamelCase__     )

       return (vocab_file,)









from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__lowerCamelCase :  Tuple										=  logging.get_logger(__name__)

__lowerCamelCase :  str										=  {
    """camembert-base""": """https://huggingface.co/camembert-base/resolve/main/config.json""",
    """umberto-commoncrawl-cased-v1""": (
        """https://huggingface.co/Musixmatch/umberto-commoncrawl-cased-v1/resolve/main/config.json"""
    ),
    """umberto-wikipedia-uncased-v1""": (
        """https://huggingface.co/Musixmatch/umberto-wikipedia-uncased-v1/resolve/main/config.json"""
    ),
}



class       A__     ( __snake_case    ):
      _UpperCAmelCase			:Union[str, Any]							=     'camembert'





      def __init__(				self ,							A_=3_0522 ,							A_=768 ,							A_=12 ,							A_=12 ,							A_=3072 ,							A_="gelu" ,							A_=0.1 ,							A_=0.1 ,							A_=512 ,							A_=2 ,							A_=0.02 ,							A_=1e-12 ,							A_=1 ,							A_=0 ,							A_=2 ,							A_="absolute" ,							A_=True ,							A_=None ,							**A_ ,							):
       '''simple docstring'''




       super().__init__(pad_token_id=A_ ,							bos_token_id=A_ ,							eos_token_id=A_ ,							**A_     )

       UpperCamelCase			:      List[str] 			=     vocab_size
       UpperCamelCase			:      Union[str, Any] 			=     hidden_size
       UpperCamelCase			:      Any 			=     num_hidden_layers
       UpperCamelCase			:      Union[str, Any] 			=     num_attention_heads
       UpperCamelCase			:      Dict 			=     hidden_act
       UpperCamelCase			:      str 			=     intermediate_size
       UpperCamelCase			:      str 			=     hidden_dropout_prob
       UpperCamelCase			:      Dict 			=     attention_probs_dropout_prob
       UpperCamelCase			:      Union[str, Any] 			=     max_position_embeddings
       UpperCamelCase			:      Optional[Any] 			=     type_vocab_size
       UpperCamelCase			:      int 			=     initializer_range
       UpperCamelCase			:      List[str] 			=     layer_norm_eps
       UpperCamelCase			:      Dict 			=     position_embedding_type
       UpperCamelCase			:      int 			=     use_cache
       UpperCamelCase			:      List[str] 			=     classifier_dropout



class       A__     ( __snake_case    ):





      @property
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       if self.task == "multiple-choice":
        UpperCamelCase			:      Optional[int] 			=     {0: "batch", 1: "choice", 2: "sequence"}
       else:
        UpperCamelCase			:      str 			=     {0: "batch", 1: "sequence"}
       return OrderedDict(
           [
               ("input_ids", dynamic_axis),
               ("attention_mask", dynamic_axis),
           ]     )
