





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


a        =						logging.get_logger(__name__)

a        =						{"vocab_file": "vocab.txt"}

a        =						{
    "vocab_file": {
        "YituTech/conv-bert-base": "https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt",
        "YituTech/conv-bert-medium-small": (
            "https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt"
        ),
        "YituTech/conv-bert-small": "https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt",
    }
}

a        =						{
    "YituTech/conv-bert-base": 512,
    "YituTech/conv-bert-medium-small": 512,
    "YituTech/conv-bert-small": 512,
}


a        =						{
    "YituTech/conv-bert-base": {"do_lower_case": True},
    "YituTech/conv-bert-medium-small": {"do_lower_case": True},
    "YituTech/conv-bert-small": {"do_lower_case": True},
}







class __a (   _snake_case			):
   __UpperCamelCase   : List[Any]	      =	VOCAB_FILES_NAMES
   __UpperCamelCase   : str	      =	PRETRAINED_VOCAB_FILES_MAP
   __UpperCamelCase   : Dict	      =	PRETRAINED_INIT_CONFIGURATION
   __UpperCamelCase   : Optional[int]	      =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCamelCase   : List[Any]	      =	ConvBertTokenizer





   def __init__(					self				:			List[Any]				,lowerCamelCase				:			Dict=None				,lowerCamelCase				:			str=None				,lowerCamelCase				:			str=True				,lowerCamelCase				:			Any="[UNK]"				,lowerCamelCase				:			List[str]="[SEP]"				,lowerCamelCase				:			Union[str, Any]="[PAD]"				,lowerCamelCase				:			List[str]="[CLS]"				,lowerCamelCase				:			Union[str, Any]="[MASK]"				,lowerCamelCase				:			Optional[Any]=True				,lowerCamelCase				:			Dict=None				,**lowerCamelCase				:			Union[str, Any]				,):

      '''simple docstring'''

      super().__init__(
          lowerCamelCase				,tokenizer_file=lowerCamelCase				,do_lower_case=lowerCamelCase				,unk_token=lowerCamelCase				,sep_token=lowerCamelCase				,pad_token=lowerCamelCase				,cls_token=lowerCamelCase				,mask_token=lowerCamelCase				,tokenize_chinese_chars=lowerCamelCase				,strip_accents=lowerCamelCase				,**lowerCamelCase				,)

      __SCREAMING_SNAKE_CASE      							=       json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
      if (
          normalizer_state.get("""lowercase"""				,lowerCamelCase    ) != do_lower_case
          or normalizer_state.get("""strip_accents"""				,lowerCamelCase    ) != strip_accents
          or normalizer_state.get("""handle_chinese_chars"""				,lowerCamelCase    ) != tokenize_chinese_chars
      ):
         __SCREAMING_SNAKE_CASE      							=       getattr(lowerCamelCase				,normalizer_state.pop("""type"""    )    )
         __SCREAMING_SNAKE_CASE      							=       do_lower_case
         __SCREAMING_SNAKE_CASE      							=       strip_accents
         __SCREAMING_SNAKE_CASE      							=       tokenize_chinese_chars
         __SCREAMING_SNAKE_CASE      							=       normalizer_class(**lowerCamelCase    )

      __SCREAMING_SNAKE_CASE      							=       do_lower_case





   def      UpperCAmelCase__     (					self				:			Tuple				,lowerCamelCase				:			List[str]				,lowerCamelCase				:			List[Any]=None    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

      return output





   def      UpperCAmelCase__     (					self				:			Union[str, Any]				,lowerCamelCase				:			List[int]				,lowerCamelCase				:			Optional[List[int]] = None    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       [self.sep_token_id]
      __SCREAMING_SNAKE_CASE      							=       [self.cls_token_id]
      if token_ids_a is None:
         return len(cls + token_ids_a + sep    ) * [0]
      return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]





   def      UpperCAmelCase__     (					self				:			Any				,lowerCamelCase				:			str				,lowerCamelCase				:			Optional[str] = None    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       self._tokenizer.model.save(lowerCamelCase				,name=lowerCamelCase    )
      return tuple(lowerCamelCase    )












'''simple docstring'''

from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


a        =						logging.get_logger(__name__)

a        =						{
    "camembert-base": "https://huggingface.co/camembert-base/resolve/main/config.json",
    "umberto-commoncrawl-cased-v1": (
        "https://huggingface.co/Musixmatch/umberto-commoncrawl-cased-v1/resolve/main/config.json"
    ),
    "umberto-wikipedia-uncased-v1": (
        "https://huggingface.co/Musixmatch/umberto-wikipedia-uncased-v1/resolve/main/config.json"
    ),
}







class __a (   _snake_case			):
   __UpperCamelCase   : Tuple	      =	'camembert'





   def __init__(					self				:			int				,lowerCamelCase				:			List[Any]=3_0522				,lowerCamelCase				:			List[Any]=768				,lowerCamelCase				:			str=12				,lowerCamelCase				:			List[str]=12				,lowerCamelCase				:			Optional[Any]=3072				,lowerCamelCase				:			Tuple="gelu"				,lowerCamelCase				:			List[str]=0.1				,lowerCamelCase				:			Tuple=0.1				,lowerCamelCase				:			Union[str, Any]=512				,lowerCamelCase				:			Dict=2				,lowerCamelCase				:			Tuple=0.02				,lowerCamelCase				:			List[Any]=1E-1_2				,lowerCamelCase				:			Union[str, Any]=1				,lowerCamelCase				:			Optional[Any]=0				,lowerCamelCase				:			List[Any]=2				,lowerCamelCase				:			List[str]="absolute"				,lowerCamelCase				:			int=True				,lowerCamelCase				:			Any=None				,**lowerCamelCase				:			Optional[Any]				,):

      '''simple docstring'''

      super().__init__(pad_token_id=lowerCamelCase				,bos_token_id=lowerCamelCase				,eos_token_id=lowerCamelCase				,**lowerCamelCase    )

      __SCREAMING_SNAKE_CASE      							=       vocab_size
      __SCREAMING_SNAKE_CASE      							=       hidden_size
      __SCREAMING_SNAKE_CASE      							=       num_hidden_layers
      __SCREAMING_SNAKE_CASE      							=       num_attention_heads
      __SCREAMING_SNAKE_CASE      							=       hidden_act
      __SCREAMING_SNAKE_CASE      							=       intermediate_size
      __SCREAMING_SNAKE_CASE      							=       hidden_dropout_prob
      __SCREAMING_SNAKE_CASE      							=       attention_probs_dropout_prob
      __SCREAMING_SNAKE_CASE      							=       max_position_embeddings
      __SCREAMING_SNAKE_CASE      							=       type_vocab_size
      __SCREAMING_SNAKE_CASE      							=       initializer_range
      __SCREAMING_SNAKE_CASE      							=       layer_norm_eps
      __SCREAMING_SNAKE_CASE      							=       position_embedding_type
      __SCREAMING_SNAKE_CASE      							=       use_cache
      __SCREAMING_SNAKE_CASE      							=       classifier_dropout







class __a (   _snake_case			):





   @property
   def      UpperCAmelCase__     (					self				:			Optional[Any]    ):

      '''simple docstring'''

      if self.task == "multiple-choice":
         __SCREAMING_SNAKE_CASE      							=       {0: """batch""", 1: """choice""", 2: """sequence"""}
      else:
         __SCREAMING_SNAKE_CASE      							=       {0: """batch""", 1: """sequence"""}
      return OrderedDict(
          [
              ("""input_ids""", dynamic_axis),
              ("""attention_mask""", dynamic_axis),
          ]    )





