import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


A_      =      logging.get_logger(__name__)

A_      =      {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

A_      =      {
    "vocab_file": {
        "junnyu/roformer_chinese_small": "https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_base": "https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_char_small": (
            "https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_chinese_char_base": (
            "https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_discriminator": (
            "https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_generator": (
            "https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"
        ),
    }
}

A_      =      {
    "junnyu/roformer_chinese_small": 1536,
    "junnyu/roformer_chinese_base": 1536,
    "junnyu/roformer_chinese_char_small": 512,
    "junnyu/roformer_chinese_char_base": 512,
    "junnyu/roformer_small_discriminator": 128,
    "junnyu/roformer_small_generator": 128,
}


A_      =      {
    "junnyu/roformer_chinese_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_base": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_base": {"do_lower_case": True},
    "junnyu/roformer_small_discriminator": {"do_lower_case": True},
    "junnyu/roformer_small_generator": {"do_lower_case": True},
}






class      __lowercase					(    _A    ):
						lowercase					 =     VOCAB_FILES_NAMES
						lowercase					 =     PRETRAINED_VOCAB_FILES_MAP
						lowercase					 =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						lowercase					 =     PRETRAINED_INIT_CONFIGURATION
						lowercase					 =     RoFormerTokenizer

						def __init__(	self				:      Union[str, Any]							,							__lowerCamelCase				:      Any=None							,							__lowerCamelCase				:      Dict=None							,							__lowerCamelCase				:      int=True							,							__lowerCamelCase				:      str="[UNK]"							,							__lowerCamelCase				:      Optional[Any]="[SEP]"							,							__lowerCamelCase				:      int="[PAD]"							,							__lowerCamelCase				:      List[Any]="[CLS]"							,							__lowerCamelCase				:      Optional[Any]="[MASK]"							,							__lowerCamelCase				:      Optional[Any]=True							,							__lowerCamelCase				:      Any=None							,							**__lowerCamelCase				:      str							,							)							->		int:


									'''simple docstring'''

									super().__init__(
									    __lowerCamelCase							,							tokenizer_file=__lowerCamelCase							,							do_lower_case=__lowerCamelCase							,							unk_token=__lowerCamelCase							,							sep_token=__lowerCamelCase							,							pad_token=__lowerCamelCase							,							cls_token=__lowerCamelCase							,							mask_token=__lowerCamelCase							,							tokenize_chinese_chars=__lowerCamelCase							,							strip_accents=__lowerCamelCase							,							**__lowerCamelCase							,							)

									lowercase							=		json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
									if (
									    pre_tok_state.get('''lowercase'''							,							__lowerCamelCase   ) != do_lower_case
									    or pre_tok_state.get('''strip_accents'''							,							__lowerCamelCase   ) != strip_accents
									):
												lowercase							=		getattr(__lowerCamelCase							,							pre_tok_state.pop('''type'''   )   )
												lowercase							=		do_lower_case
												lowercase							=		strip_accents
												lowercase							=		pre_tok_class(**__lowerCamelCase   )

									lowercase							=		do_lower_case

						def __getstate__(	self				:      Dict   )							->		int:


									'''simple docstring'''

									lowercase							=		self.__dict__.copy()
									lowercase							=		BertPreTokenizer()
									return state

						def __setstate__(	self				:      List[str]							,							__lowerCamelCase				:      str   )							->		int:


									'''simple docstring'''

									lowercase							=		d
									lowercase							=		self.__dict__['''_tokenizer'''].get_vocab()
									lowercase							=		PreTokenizer.custom(JiebaPreTokenizer(__lowerCamelCase   )   )

						def        __a		(	self				:      Optional[int]							,							__lowerCamelCase				:      List[Any]							,							__lowerCamelCase				:      Optional[int]=None   )							->		Tuple:


									'''simple docstring'''

									lowercase							=		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
												output += token_ids_a + [self.sep_token_id]

									return output

						def        __a		(	self				:      Tuple							,							__lowerCamelCase				:      List[int]							,							__lowerCamelCase				:      Optional[List[int]] = None   )							->		List[int]:


									'''simple docstring'''

									lowercase							=		[self.sep_token_id]
									lowercase							=		[self.cls_token_id]
									if token_ids_a is None:
												return len(cls + token_ids_a + sep   ) * [0]
									return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]

						def        __a		(	self				:      Optional[Any]							,							__lowerCamelCase				:      str							,							__lowerCamelCase				:      Optional[str] = None   )							->		Tuple[str]:


									'''simple docstring'''

									lowercase							=		self._tokenizer.model.save(__lowerCamelCase							,							name=__lowerCamelCase   )
									return tuple(__lowerCamelCase   )



						def        __a		(	self				:      Tuple							,							__lowerCamelCase				:      List[str]							,							__lowerCamelCase				:      Union[str, Any]=None							,							__lowerCamelCase				:      Optional[int]=None							,							__lowerCamelCase				:      List[str]=False							,							**__lowerCamelCase				:      List[Any]							,							)							->		List[str]:


									'''simple docstring'''

									lowercase							=		BertPreTokenizer()
									return super().save_pretrained(__lowerCamelCase							,							__lowerCamelCase							,							__lowerCamelCase							,							__lowerCamelCase							,							**__lowerCamelCase   )






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


A_      =      {
    "configuration_gpt_bigcode": ["GPT_BIGCODE_PRETRAINED_CONFIG_ARCHIVE_MAP", "GPTBigCodeConfig"],
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A_      =      [
						    "GPT_BIGCODE_PRETRAINED_MODEL_ARCHIVE_LIST",
						    "GPTBigCodeForSequenceClassification",
						    "GPTBigCodeForTokenClassification",
						    "GPTBigCodeForCausalLM",
						    "GPTBigCodeModel",
						    "GPTBigCodePreTrainedModel",
						]

if TYPE_CHECKING:
						from .configuration_gpt_bigcode import GPT_BIGCODE_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTBigCodeConfig

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_gpt_bigcode import (
												    GPT_BIGCODE_PRETRAINED_MODEL_ARCHIVE_LIST,
												    GPTBigCodeForCausalLM,
												    GPTBigCodeForSequenceClassification,
												    GPTBigCodeForTokenClassification,
												    GPTBigCodeModel,
												    GPTBigCodePreTrainedModel,
												)


else:
						import sys

						A_      =      _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





