





import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase 				=     logging.get_logger(__name__)

_lowerCAmelCase 				=     """▁"""

_lowerCAmelCase 				=     {"""vocab_file""": """vocab.txt""", """sentencepiece_model_ckpt""": """sentencepiece.bpe.model"""}

_lowerCAmelCase 				=     {
    """sentencepiece_model_file""": """sentencepiece.bpe.model""",
    """vocab_file""": """vocab.txt""",
}

_lowerCAmelCase 				=     {
    """vocab_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
    },
    """sentencepiece_model_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
    },
}

_lowerCAmelCase 				=     {
    """ernie-m-base""": 514,
    """ernie-m-large""": 514,
}

_lowerCAmelCase 				=     {
    """ernie-m-base""": {"""do_lower_case""": False},
    """ernie-m-large""": {"""do_lower_case""": False},
}

class    _UpperCAmelCase							(		_lowerCamelCase     ):
      a						       =							["input_ids"]

      a						       =							VOCAB_FILES_NAMES
      a						       =							PRETRAINED_INIT_CONFIGURATION
      a						       =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      a						       =							PRETRAINED_VOCAB_FILES_MAP
      a						       =							RESOURCE_FILES_NAMES
      def __init__(		self      ,       a__      ,       a__=None      ,       a__=False      ,       a__="utf8"      ,       a__="[UNK]"      ,       a__="[SEP]"      ,       a__="[PAD]"      ,       a__="[CLS]"      ,       a__="[MASK]"      ,       a__ = None      ,       **a__      ,       ):
         # Mask token behave like a normal word, i.e. include the space before it and
         # is included in the raw text, there should be a match in a non-normalized sentence.

         A_     :				List[str]			      =	{} if sp_model_kwargs is None else sp_model_kwargs
         super().__init__(
             do_lower_case=a__      ,       unk_token=a__      ,       sep_token=a__      ,       pad_token=a__      ,       cls_token=a__      ,       mask_token=a__      ,       vocab_file=a__      ,       encoding=a__      ,       sp_model_kwargs=self.sp_model_kwargs      ,       **a__      ,       )
         A_     :				Union[str, Any]			      =	do_lower_case
         A_     :				List[Any]			      =	sentencepiece_model_ckpt
         A_     :				Optional[int]			      =	spm.SentencePieceProcessor(**self.sp_model_kwargs       )
         self.sp_model.Load(a__       )

         # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
         if vocab_file is not None:
            A_     :				Union[str, Any]			      =	self.load_vocab(filepath=a__       )
         else:
            A_     :				Tuple			      =	{self.sp_model.id_to_piece(a__       ): id for id in range(self.sp_model.get_piece_size()       )}
         A_     :				List[str]			      =	{v: k for k, v in self.vocab.items()}
      def 		_lowerCamelCase      (		self      ,       a__       ):
         if text is None:
            return None

         A_     :				int			      =	self.tokenize(a__       )
         A_      ,				A_     :				Dict			      =	"""""", []

         for i, ch in enumerate(a__       ):
            if ch in self.SP_CHAR_MAPPING:
               A_     :				Any			      =	self.SP_CHAR_MAPPING.get(a__       )
            else:
               A_     :				int			      =	unicodedata.normalize("""NFKC"""      ,       a__       )
            if self.is_whitespace(a__       ):
               continue
            normalized_text += ch
            char_mapping.extend([i] * len(a__       )       )

         A_      ,				A_      ,				A_     :				List[str]			      =	normalized_text, [], 0

         if self.do_lower_case:
            A_     :				Any			      =	text.lower()

         for token in split_tokens:
            if token[:1] == "▁":
               A_     :				int			      =	token[1:]
            A_     :				List[Any]			      =	text[offset:].index(a__       ) + offset
            A_     :				Optional[Any]			      =	start + len(a__       )

            token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)       )
            A_     :				int			      =	end
         return token_mapping
      @property
      def 		_lowerCamelCase      (		self       ):
         return len(self.vocab       )
      def 		_lowerCamelCase      (		self       ):
         return dict(self.vocab      ,       **self.added_tokens_encoder       )
      def __getstate__(		self       ):
         A_     :				Optional[Any]			      =	self.__dict__.copy()
         A_     :				Optional[Any]			      =	None
         return state
      def __setstate__(		self      ,       a__       ):
         A_     :				Dict			      =	d

         # for backward compatibility
         if not hasattr(self      ,       """sp_model_kwargs"""       ):
            A_     :				Any			      =	{}

         A_     :				Optional[int]			      =	spm.SentencePieceProcessor(**self.sp_model_kwargs       )
         self.sp_model.Load(self.sentencepiece_model_ckpt       )
      def 		_lowerCamelCase      (		self      ,       a__       ):
         return "".join((self.SP_CHAR_MAPPING.get(a__      ,       a__       ) for c in text)       )
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__=False      ,       a__=64      ,       a__=0.1       ):

         if self.sp_model_kwargs.get("""enable_sampling"""       ) is True:
            A_     :				int			      =	True
         if self.sp_model_kwargs.get("""alpha"""       ) is not None:
            A_     :				Optional[Any]			      =	self.sp_model_kwargs.get("""alpha"""       )
         if self.sp_model_kwargs.get("""nbest_size"""       ) is not None:
            A_     :				Optional[int]			      =	self.sp_model_kwargs.get("""nbest_size"""       )

         if not enable_sampling:
            A_     :				Union[str, Any]			      =	self.sp_model.EncodeAsPieces(a__       )
         else:
            A_     :				Optional[Any]			      =	self.sp_model.SampleEncodeAsPieces(a__      ,       a__      ,       a__       )
         A_     :				Optional[int]			      =	[]
         for pi, piece in enumerate(a__       ):
            if piece == SPIECE_UNDERLINE:
               if not pieces[pi + 1].startswith(a__       ) and pi != 0:
                  new_pieces.append(a__       )
                  continue
               else:
                  continue
            A_     :				List[Any]			      =	0
            for i, chunk in enumerate(a__       ):
               if chunk == SPIECE_UNDERLINE:
                  continue
               if self.is_ch_char(a__       ) or self.is_punct(a__       ):
                  if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                     new_pieces.append(piece[lst_i:i]       )
                  new_pieces.append(a__       )
                  A_     :				List[str]			      =	i + 1
               elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                  if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                     new_pieces.append(piece[lst_i:i]       )
                  A_     :				Any			      =	i
               elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                  if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                     new_pieces.append(piece[lst_i:i]       )
                  A_     :				Tuple			      =	i
            if len(a__       ) > lst_i:
               new_pieces.append(piece[lst_i:]       )
         return new_pieces
      def 		_lowerCamelCase      (		self      ,       a__       ):
         A_     :				Dict			      =	"""""".join(a__       ).replace(a__      ,       """ """       ).strip()
         return out_string
      def 		_lowerCamelCase      (		self      ,       a__       ):
         A_     :				int			      =	self.convert_ids_to_tokens(a__       )
         A_     :				Tuple			      =	"""""".join(a__       ).replace(a__      ,       """ """       ).strip()
         return out_string
      def 		_lowerCamelCase      (		self      ,       a__       ):
         return self.vocab.get(a__      ,       self.vocab.get(self.unk_token       )       )
      def 		_lowerCamelCase      (		self      ,       a__       ):
         return self.reverse_vocab.get(a__      ,       self.unk_token       )
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__=None       ):
         if token_ids_a is None:
            return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         A_     :				Any			      =	[self.cls_token_id]
         A_     :				Optional[Any]			      =	[self.sep_token_id]
         return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__=None       ):
         if offset_mapping_a is None:
            return [(0, 0)] + offset_mapping_a + [(0, 0)]

         return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__=None      ,       a__=False       ):

         if already_has_special_tokens:
            if token_ids_a is not None:
               raise ValueError(
                   """You should not supply a second sequence if the provided sequence of """
                   """ids is already formatted with special tokens for the model."""       )
            return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

         if token_ids_a is not None:
            return [1] + ([0] * len(a__       )) + [1, 1] + ([0] * len(a__       )) + [1]
         return [1] + ([0] * len(a__       )) + [1]
      def 		_lowerCamelCase      (		self      ,       a__      ,       a__ = None       ):
         # called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
         if token_ids_a is None:
            # [CLS] X [SEP]
            return (len(a__       ) + 2) * [0]

         # [CLS] A [SEP] [SEP] B [SEP]
         return [0] * (len(a__       ) + 1) + [1] * (len(a__       ) + 3)
      def 		_lowerCamelCase      (		self      ,       a__       ):
         if "\u4e00" <= char <= "\u9fff":
            return True
         return False
      def 		_lowerCamelCase      (		self      ,       a__       ):
         if ("a" <= char <= "z") or ("A" <= char <= "Z"):
            return True
         return False
      def 		_lowerCamelCase      (		self      ,       a__       ):
         if char in ",;:.?!~，；：。？！《》【】":
            return True
         return False
      def 		_lowerCamelCase      (		self      ,       a__       ):
         if char == " " or char == "\t" or char == "\n" or char == "\r":
            return True
         if len(a__       ) == 1:
            A_     :				Dict			      =	unicodedata.category(a__       )
            if cat == "Zs":
               return True
         return False
      def 		_lowerCamelCase      (		self      ,       a__       ):
         A_     :				int			      =	{}
         with io.open(a__      ,       """r"""      ,       encoding="""utf-8"""       ) as f:
            for index, line in enumerate(a__       ):
               A_     :				str			      =	line.rstrip("""\n"""       )
               A_     :				str			      =	int(a__       )

         return token_to_idx






      def 		_lowerCamelCase      (		self      ,       a__      ,       a__ = None       ):
         A_     :				Tuple			      =	0
         if os.path.isdir(a__       ):
            A_     :				Union[str, Any]			      =	os.path.join(
                a__      ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )
         else:
            A_     :				Dict			      =	(filename_prefix + """-""" if filename_prefix else """""") + save_directory
         with open(a__      ,       """w"""      ,       encoding="""utf-8"""       ) as writer:
            for token, token_index in sorted(self.vocab.items()      ,       key=lambda a__       : kv[1]       ):
               if index != token_index:
                  logger.warning(
                      F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                      """ Please check that the vocabulary is not corrupted!"""       )
                  A_     :				Union[str, Any]			      =	token_index
               writer.write(token + """\n"""       )
               index += 1

         A_     :				str			      =	os.path.join(a__      ,       """sentencepiece.bpe.model"""       )
         with open(a__      ,       """wb"""       ) as fi:
            A_     :				Union[str, Any]			      =	self.sp_model.serialized_model_proto()
            fi.write(a__       )

         return (vocab_file,)













from ....configuration_utils import PretrainedConfig
from ....utils import logging


_lowerCAmelCase 				=     logging.get_logger(__name__)

# TODO: upload to AWS
_lowerCAmelCase 				=     {
    """yjernite/retribert-base-uncased""": (
        """https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/config.json"""
    ),
}

class    _UpperCAmelCase							(		_lowerCamelCase     ):
      a						       =							'''retribert'''
      def __init__(		self      ,       a__=30522      ,       a__=768      ,       a__=8      ,       a__=12      ,       a__=3072      ,       a__="gelu"      ,       a__=0.1      ,       a__=0.1      ,       a__=512      ,       a__=2      ,       a__=0.02      ,       a__=1E-12      ,       a__=True      ,       a__=128      ,       a__=0      ,       **a__      ,       ):
         super().__init__(pad_token_id=a__      ,       **a__       )

         A_     :				Union[str, Any]			      =	vocab_size
         A_     :				Optional[Any]			      =	hidden_size
         A_     :				Optional[Any]			      =	num_hidden_layers
         A_     :				List[str]			      =	num_attention_heads
         A_     :				Optional[int]			      =	hidden_act
         A_     :				Any			      =	intermediate_size
         A_     :				Tuple			      =	hidden_dropout_prob
         A_     :				Optional[Any]			      =	attention_probs_dropout_prob
         A_     :				Dict			      =	max_position_embeddings
         A_     :				Dict			      =	type_vocab_size
         A_     :				List[str]			      =	initializer_range
         A_     :				Optional[int]			      =	layer_norm_eps
         A_     :				Optional[Any]			      =	share_encoders
         A_     :				Dict			      =	projection_dim






