import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


UpperCAmelCase_								=							logging.get_logger(__name__)

UpperCAmelCase_								=							{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCAmelCase_								=							{
    '''vocab_file''': {
        '''junnyu/roformer_chinese_small''': '''https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_base''': '''https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_char_small''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_chinese_char_base''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_discriminator''': (
            '''https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_generator''': (
            '''https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'''
        ),
    }
}

UpperCAmelCase_								=							{
    '''junnyu/roformer_chinese_small''': 1_536,
    '''junnyu/roformer_chinese_base''': 1_536,
    '''junnyu/roformer_chinese_char_small''': 512,
    '''junnyu/roformer_chinese_char_base''': 512,
    '''junnyu/roformer_small_discriminator''': 128,
    '''junnyu/roformer_small_generator''': 128,
}


UpperCAmelCase_								=							{
    '''junnyu/roformer_chinese_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_discriminator''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_generator''': {'''do_lower_case''': True},
}





class      lowerCAmelCase						(					_a			):
     _SCREAMING_SNAKE_CASE     :     Optional[int]		=VOCAB_FILES_NAMES
     _SCREAMING_SNAKE_CASE     :     Any		=PRETRAINED_VOCAB_FILES_MAP
     _SCREAMING_SNAKE_CASE     :     int		=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _SCREAMING_SNAKE_CASE     :     Optional[Any]		=PRETRAINED_INIT_CONFIGURATION
     _SCREAMING_SNAKE_CASE     :     Tuple		=RoFormerTokenizer



     def __init__(					self				,      lowerCAmelCase__=None				,      lowerCAmelCase__=None				,      lowerCAmelCase__=True				,      lowerCAmelCase__="[UNK]"				,      lowerCAmelCase__="[SEP]"				,      lowerCAmelCase__="[PAD]"				,      lowerCAmelCase__="[CLS]"				,      lowerCAmelCase__="[MASK]"				,      lowerCAmelCase__=True				,      lowerCAmelCase__=None				,      **lowerCAmelCase__				,      ):
            super().__init__(
                lowerCAmelCase__				,      tokenizer_file=lowerCAmelCase__				,      do_lower_case=lowerCAmelCase__				,      unk_token=lowerCAmelCase__				,      sep_token=lowerCAmelCase__				,      pad_token=lowerCAmelCase__				,      cls_token=lowerCAmelCase__				,      mask_token=lowerCAmelCase__				,      tokenize_chinese_chars=lowerCAmelCase__				,      strip_accents=lowerCAmelCase__				,      **lowerCAmelCase__				,      )

            _A=	json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
            if (
                pre_tok_state.get('lowercase'				,      lowerCAmelCase__       ) != do_lower_case
                or pre_tok_state.get('strip_accents'				,      lowerCAmelCase__       ) != strip_accents
            ):
                   _A=	getattr(lowerCAmelCase__				,      pre_tok_state.pop('type'       )       )
                   _A=	do_lower_case
                   _A=	strip_accents
                   _A=	pre_tok_class(**lowerCAmelCase__       )

            _A=	do_lower_case



     def __getstate__(					self       ):
            _A=	self.__dict__.copy()
            _A=	BertPreTokenizer()
            return state



     def __setstate__(					self				,      lowerCAmelCase__       ):
            _A=	d
            _A=	self.__dict__['_tokenizer'].get_vocab()
            _A=	PreTokenizer.custom(JiebaPreTokenizer(lowerCAmelCase__       )       )



     def 			a__  (					self				,      lowerCAmelCase__				,      lowerCAmelCase__=None       ):
            _A=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

            if token_ids_a:
                   output += token_ids_a + [self.sep_token_id]

            return output



     def 			a__  (					self				,      lowerCAmelCase__				,      lowerCAmelCase__ = None       ):
            _A=	[self.sep_token_id]
            _A=	[self.cls_token_id]
            if token_ids_a is None:
                   return len(cls + token_ids_a + sep       ) * [0]
            return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]



     def 			a__  (					self				,      lowerCAmelCase__				,      lowerCAmelCase__ = None       ):
            _A=	self._tokenizer.model.save(lowerCAmelCase__				,      name=lowerCAmelCase__       )
            return tuple(lowerCAmelCase__       )





     def 			a__  (					self				,      lowerCAmelCase__				,      lowerCAmelCase__=None				,      lowerCAmelCase__=None				,      lowerCAmelCase__=False				,      **lowerCAmelCase__				,      ):
            _A=	BertPreTokenizer()
            return super().save_pretrained(lowerCAmelCase__				,      lowerCAmelCase__				,      lowerCAmelCase__				,      lowerCAmelCase__				,      **lowerCAmelCase__       )
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tensorflow_text_available, is_torch_available


UpperCAmelCase_								=							{
    '''configuration_ernie''': ['''ERNIE_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''ErnieConfig''', '''ErnieOnnxConfig'''],
}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      UpperCAmelCase_								=							[
          '''ERNIE_PRETRAINED_MODEL_ARCHIVE_LIST''',
          '''ErnieForCausalLM''',
          '''ErnieForMaskedLM''',
          '''ErnieForMultipleChoice''',
          '''ErnieForNextSentencePrediction''',
          '''ErnieForPreTraining''',
          '''ErnieForQuestionAnswering''',
          '''ErnieForSequenceClassification''',
          '''ErnieForTokenClassification''',
          '''ErnieModel''',
          '''ErniePreTrainedModel''',
      ]

if TYPE_CHECKING:
      from .configuration_ernie import ERNIE_PRETRAINED_CONFIG_ARCHIVE_MAP, ErnieConfig, ErnieOnnxConfig

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_ernie import (
                ERNIE_PRETRAINED_MODEL_ARCHIVE_LIST,
                ErnieForCausalLM,
                ErnieForMaskedLM,
                ErnieForMultipleChoice,
                ErnieForNextSentencePrediction,
                ErnieForPreTraining,
                ErnieForQuestionAnswering,
                ErnieForSequenceClassification,
                ErnieForTokenClassification,
                ErnieModel,
                ErniePreTrainedModel,
            )

else:
      import sys

      UpperCAmelCase_								=							_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)