




'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCAmelCase_ 		=      {
    "configuration_distilbert": [
        "DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "DistilBertConfig",
        "DistilBertOnnxConfig",
    ],
    "tokenization_distilbert": ["DistilBertTokenizer"],
}

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase_ 		=      ["DistilBertTokenizerFast"]

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase_ 		=      [
      "DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
      "DistilBertForMaskedLM",
      "DistilBertForMultipleChoice",
      "DistilBertForQuestionAnswering",
      "DistilBertForSequenceClassification",
      "DistilBertForTokenClassification",
      "DistilBertModel",
      "DistilBertPreTrainedModel",
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase_ 		=      [
      "TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
      "TFDistilBertForMaskedLM",
      "TFDistilBertForMultipleChoice",
      "TFDistilBertForQuestionAnswering",
      "TFDistilBertForSequenceClassification",
      "TFDistilBertForTokenClassification",
      "TFDistilBertMainLayer",
      "TFDistilBertModel",
      "TFDistilBertPreTrainedModel",
  ]

try:
  if not is_flax_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase_ 		=      [
      "FlaxDistilBertForMaskedLM",
      "FlaxDistilBertForMultipleChoice",
      "FlaxDistilBertForQuestionAnswering",
      "FlaxDistilBertForSequenceClassification",
      "FlaxDistilBertForTokenClassification",
      "FlaxDistilBertModel",
      "FlaxDistilBertPreTrainedModel",
  ]


if TYPE_CHECKING:
  from .configuration_distilbert import (
      DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
      DistilBertConfig,
      DistilBertOnnxConfig,
  )
  from .tokenization_distilbert import DistilBertTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_distilbert_fast import DistilBertTokenizerFast

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_distilbert import (
        DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        DistilBertForMaskedLM,
        DistilBertForMultipleChoice,
        DistilBertForQuestionAnswering,
        DistilBertForSequenceClassification,
        DistilBertForTokenClassification,
        DistilBertModel,
        DistilBertPreTrainedModel,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_distilbert import (
        TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFDistilBertForMaskedLM,
        TFDistilBertForMultipleChoice,
        TFDistilBertForQuestionAnswering,
        TFDistilBertForSequenceClassification,
        TFDistilBertForTokenClassification,
        TFDistilBertMainLayer,
        TFDistilBertModel,
        TFDistilBertPreTrainedModel,
    )

  try:
    if not is_flax_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_flax_distilbert import (
        FlaxDistilBertForMaskedLM,
        FlaxDistilBertForMultipleChoice,
        FlaxDistilBertForQuestionAnswering,
        FlaxDistilBertForSequenceClassification,
        FlaxDistilBertForTokenClassification,
        FlaxDistilBertModel,
        FlaxDistilBertPreTrainedModel,
    )

else:
  import sys

  UpperCAmelCase_ 		=      _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)









'''simple docstring'''


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


UpperCAmelCase_ 		=      logging.get_logger(__name__)

UpperCAmelCase_ 		=      {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase_ 		=      {
    "vocab_file": {
        "junnyu/roformer_chinese_small": "https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_base": "https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_char_small": (
            "https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_chinese_char_base": (
            "https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_discriminator": (
            "https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_generator": (
            "https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"
        ),
    }
}

UpperCAmelCase_ 		=      {
    "junnyu/roformer_chinese_small": 15_36,
    "junnyu/roformer_chinese_base": 15_36,
    "junnyu/roformer_chinese_char_small": 5_12,
    "junnyu/roformer_chinese_char_base": 5_12,
    "junnyu/roformer_small_discriminator": 1_28,
    "junnyu/roformer_small_generator": 1_28,
}


UpperCAmelCase_ 		=      {
    "junnyu/roformer_chinese_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_base": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_base": {"do_lower_case": True},
    "junnyu/roformer_small_discriminator": {"do_lower_case": True},
    "junnyu/roformer_small_generator": {"do_lower_case": True},
}





class    __lowercase						(  __magic_name__ ):
   _a							=      VOCAB_FILES_NAMES
   _a							=      PRETRAINED_VOCAB_FILES_MAP
   _a							=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _a							=      PRETRAINED_INIT_CONFIGURATION
   _a							=      RoFormerTokenizer







   def __init__(	self      ,							UpperCamelCase=None      ,							UpperCamelCase=None      ,							UpperCamelCase=True      ,							UpperCamelCase="[UNK]"      ,							UpperCamelCase="[SEP]"      ,							UpperCamelCase="[PAD]"      ,							UpperCamelCase="[CLS]"      ,							UpperCamelCase="[MASK]"      ,							UpperCamelCase=True      ,							UpperCamelCase=None      ,							**UpperCamelCase      ,							)				->    int:
     super().__init__(
         UpperCamelCase      ,							tokenizer_file=UpperCamelCase      ,							do_lower_case=UpperCamelCase      ,							unk_token=UpperCamelCase      ,							sep_token=UpperCamelCase      ,							pad_token=UpperCamelCase      ,							cls_token=UpperCamelCase      ,							mask_token=UpperCamelCase      ,							tokenize_chinese_chars=UpperCamelCase      ,							strip_accents=UpperCamelCase      ,							**UpperCamelCase      ,							)

     __a			=					json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
     if (
         pre_tok_state.get('lowercase'      ,							UpperCamelCase      ) != do_lower_case
         or pre_tok_state.get('strip_accents'      ,							UpperCamelCase      ) != strip_accents
     ):
       __a			=					getattr(UpperCamelCase      ,							pre_tok_state.pop('type'      )      )
       __a			=					do_lower_case
       __a			=					strip_accents
       __a			=					pre_tok_class(**UpperCamelCase      )

     __a			=					do_lower_case







   def __getstate__(	self      )				->    List[Any]:
     __a			=					self.__dict__.copy()
     __a			=					BertPreTokenizer()
     return state







   def __setstate__(	self      ,							UpperCamelCase      )				->    Optional[int]:
     __a			=					d
     __a			=					self.__dict__['_tokenizer'].get_vocab()
     __a			=					PreTokenizer.custom(JiebaPreTokenizer(UpperCamelCase      )      )







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase=None      )				->    int:
     __a			=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

     return output







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase = None      )				->    List[int]:
     __a			=					[self.sep_token_id]
     __a			=					[self.cls_token_id]
     if token_ids_a is None:
       return len(cls + token_ids_a + sep      ) * [0]
     return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase = None      )				->    Tuple[str]:
     __a			=					self._tokenizer.model.save(UpperCamelCase      ,							name=UpperCamelCase      )
     return tuple(UpperCamelCase      )







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase=None      ,							UpperCamelCase=None      ,							UpperCamelCase=False      ,							**UpperCamelCase      ,							)				->    List[Any]:
     __a			=					BertPreTokenizer()
     return super().save_pretrained(UpperCamelCase      ,							UpperCamelCase      ,							UpperCamelCase      ,							UpperCamelCase      ,							**UpperCamelCase      )



