'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


SCREAMING_SNAKE_CASE_	   =			logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_	   =			{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE_	   =			[
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

SCREAMING_SNAKE_CASE_	   =			{
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
SCREAMING_SNAKE_CASE_	   =			{F"""funnel-transformer/{name}""": 5_12 for name in _model_names}
SCREAMING_SNAKE_CASE_	   =			{F"""funnel-transformer/{name}""": {'do_lower_case': True} for name in _model_names}




class    a (       __lowerCAmelCase				):



		"""simple docstring"""
		__lowerCAmelCase				      =					VOCAB_FILES_NAMES
		__lowerCAmelCase				      =					PRETRAINED_VOCAB_FILES_MAP
		__lowerCAmelCase				      =					PRETRAINED_INIT_CONFIGURATION
		__lowerCAmelCase				      =					FunnelTokenizer
		__lowerCAmelCase				      =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__lowerCAmelCase				      =					2







		def __init__(							self   ,    snake_case_=None   ,    snake_case_=None   ,    snake_case_=True   ,    snake_case_="<unk>"   ,    snake_case_="<sep>"   ,    snake_case_="<pad>"   ,    snake_case_="<cls>"   ,    snake_case_="<mask>"   ,    snake_case_="<s>"   ,    snake_case_="</s>"   ,    snake_case_=True   ,    snake_case_=True   ,    snake_case_=None   ,    snake_case_="##"   ,    **snake_case_   ,    ):

				'''simple docstring'''







				super().__init__(
				    snake_case_   ,    tokenizer_file=snake_case_   ,    do_lower_case=snake_case_   ,    unk_token=snake_case_   ,    sep_token=snake_case_   ,    pad_token=snake_case_   ,    cls_token=snake_case_   ,    mask_token=snake_case_   ,    bos_token=snake_case_   ,    eos_token=snake_case_   ,    clean_text=snake_case_   ,    tokenize_chinese_chars=snake_case_   ,    strip_accents=snake_case_   ,    wordpieces_prefix=snake_case_   ,    **snake_case_   ,    )

				__UpperCAmelCase:					int		= json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
				if (
				    normalizer_state.get("""lowercase"""   ,    snake_case_					) != do_lower_case
				    or normalizer_state.get("""strip_accents"""   ,    snake_case_					) != strip_accents
				    or normalizer_state.get("""handle_chinese_chars"""   ,    snake_case_					) != tokenize_chinese_chars
				):
						__UpperCAmelCase:					Union[str, Any]		= getattr(snake_case_   ,    normalizer_state.pop("""type"""					)					)
						__UpperCAmelCase:					Dict		= do_lower_case
						__UpperCAmelCase:					List[Any]		= strip_accents
						__UpperCAmelCase:					Optional[Any]		= tokenize_chinese_chars
						__UpperCAmelCase:					Optional[int]		= normalizer_class(**snake_case_					)

				__UpperCAmelCase:					Optional[int]		= do_lower_case







		def    lowercase_  (							self   ,    snake_case_   ,    snake_case_=None					):

				'''simple docstring'''







				__UpperCAmelCase:					str		= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
						output += token_ids_a + [self.sep_token_id]

				return output







		def    lowercase_  (							self   ,    snake_case_   ,    snake_case_ = None					):

				'''simple docstring'''







				__UpperCAmelCase:					List[Any]		= [self.sep_token_id]
				__UpperCAmelCase:					List[Any]		= [self.cls_token_id]
				if token_ids_a is None:
						return len(cls					) * [self.cls_token_type_id] + len(token_ids_a + sep					) * [0]
				return len(cls					) * [self.cls_token_type_id] + len(token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]







		def    lowercase_  (							self   ,    snake_case_   ,    snake_case_ = None					):

				'''simple docstring'''







				__UpperCAmelCase:					int		= self._tokenizer.model.save(snake_case_   ,    name=snake_case_					)
				return tuple(snake_case_					)
'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


SCREAMING_SNAKE_CASE_	   =			{
    'configuration_electra': ['ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP', 'ElectraConfig', 'ElectraOnnxConfig'],
    'tokenization_electra': ['ElectraTokenizer'],
}

try:
    if not is_tokenizers_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    SCREAMING_SNAKE_CASE_	   =			['ElectraTokenizerFast']

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    SCREAMING_SNAKE_CASE_	   =			[
        'ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST',
        'ElectraForCausalLM',
        'ElectraForMaskedLM',
        'ElectraForMultipleChoice',
        'ElectraForPreTraining',
        'ElectraForQuestionAnswering',
        'ElectraForSequenceClassification',
        'ElectraForTokenClassification',
        'ElectraModel',
        'ElectraPreTrainedModel',
        'load_tf_weights_in_electra',
    ]

try:
    if not is_tf_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    SCREAMING_SNAKE_CASE_	   =			[
        'TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST',
        'TFElectraForMaskedLM',
        'TFElectraForMultipleChoice',
        'TFElectraForPreTraining',
        'TFElectraForQuestionAnswering',
        'TFElectraForSequenceClassification',
        'TFElectraForTokenClassification',
        'TFElectraModel',
        'TFElectraPreTrainedModel',
    ]

try:
    if not is_flax_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    SCREAMING_SNAKE_CASE_	   =			[
        'FlaxElectraForCausalLM',
        'FlaxElectraForMaskedLM',
        'FlaxElectraForMultipleChoice',
        'FlaxElectraForPreTraining',
        'FlaxElectraForQuestionAnswering',
        'FlaxElectraForSequenceClassification',
        'FlaxElectraForTokenClassification',
        'FlaxElectraModel',
        'FlaxElectraPreTrainedModel',
    ]


if TYPE_CHECKING:
    from .configuration_electra import ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP, ElectraConfig, ElectraOnnxConfig
    from .tokenization_electra import ElectraTokenizer

    try:
        if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_electra_fast import ElectraTokenizerFast

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_electra import (
            ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
            ElectraForCausalLM,
            ElectraForMaskedLM,
            ElectraForMultipleChoice,
            ElectraForPreTraining,
            ElectraForQuestionAnswering,
            ElectraForSequenceClassification,
            ElectraForTokenClassification,
            ElectraModel,
            ElectraPreTrainedModel,
            load_tf_weights_in_electra,
        )

    try:
        if not is_tf_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_tf_electra import (
            TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
            TFElectraForMaskedLM,
            TFElectraForMultipleChoice,
            TFElectraForPreTraining,
            TFElectraForQuestionAnswering,
            TFElectraForSequenceClassification,
            TFElectraForTokenClassification,
            TFElectraModel,
            TFElectraPreTrainedModel,
        )

    try:
        if not is_flax_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_flax_electra import (
            FlaxElectraForCausalLM,
            FlaxElectraForMaskedLM,
            FlaxElectraForMultipleChoice,
            FlaxElectraForPreTraining,
            FlaxElectraForQuestionAnswering,
            FlaxElectraForSequenceClassification,
            FlaxElectraForTokenClassification,
            FlaxElectraModel,
            FlaxElectraPreTrainedModel,
        )

else:
    import sys

    SCREAMING_SNAKE_CASE_	   =			_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)