


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


A   :					Union[str, Any]	=      logging.get_logger(__name__)

A   :					List[Any]	=      {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

A   :					List[Any]	=      [
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

A   :					Optional[int]	=      {
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
A   :					Union[str, Any]	=      {f'funnel-transformer/{name}': 512 for name in _model_names}
A   :					Union[str, Any]	=      {f'funnel-transformer/{name}': {"do_lower_case": True} for name in _model_names}







class     _lowercase				(							lowercase__):


							"""simple docstring"""




							A__      		=    VOCAB_FILES_NAMES
							A__      		=    PRETRAINED_VOCAB_FILES_MAP
							A__      		=    PRETRAINED_INIT_CONFIGURATION
							A__      		=    FunnelTokenizer
							A__      		=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							A__      		=    2





							def __init__(     self							:    Optional[Any]      ,   __lowerCamelCase							:    int=None      ,   __lowerCamelCase							:    Optional[int]=None      ,   __lowerCamelCase							:    Union[str, Any]=True      ,   __lowerCamelCase							:    Optional[int]="<unk>"      ,   __lowerCamelCase							:    Optional[Any]="<sep>"      ,   __lowerCamelCase							:    Tuple="<pad>"      ,   __lowerCamelCase							:    str="<cls>"      ,   __lowerCamelCase							:    List[str]="<mask>"      ,   __lowerCamelCase							:    Dict="<s>"      ,   __lowerCamelCase							:    List[str]="</s>"      ,   __lowerCamelCase							:    Dict=True      ,   __lowerCamelCase							:    List[Any]=True      ,   __lowerCamelCase							:    Optional[Any]=None      ,   __lowerCamelCase							:    List[str]="##"      ,   **__lowerCamelCase							:    Optional[int]      ,   ):


													'''simple docstring'''



													super().__init__(
													    __lowerCamelCase      ,   tokenizer_file=__lowerCamelCase      ,   do_lower_case=__lowerCamelCase      ,   unk_token=__lowerCamelCase      ,   sep_token=__lowerCamelCase      ,   pad_token=__lowerCamelCase      ,   cls_token=__lowerCamelCase      ,   mask_token=__lowerCamelCase      ,   bos_token=__lowerCamelCase      ,   eos_token=__lowerCamelCase      ,   clean_text=__lowerCamelCase      ,   tokenize_chinese_chars=__lowerCamelCase      ,   strip_accents=__lowerCamelCase      ,   wordpieces_prefix=__lowerCamelCase      ,   **__lowerCamelCase      ,   )

													lowerCamelCase__	:       str					=						json.loads(self.backend_tokenizer.normalizer.__getstate__() )
													if (
													    normalizer_state.get("lowercase"      ,   __lowerCamelCase ) != do_lower_case
													    or normalizer_state.get("strip_accents"      ,   __lowerCamelCase ) != strip_accents
													    or normalizer_state.get("handle_chinese_chars"      ,   __lowerCamelCase ) != tokenize_chinese_chars
													):
																			lowerCamelCase__	:       int					=						getattr(__lowerCamelCase      ,   normalizer_state.pop("type" ) )
																			lowerCamelCase__	:       Dict					=						do_lower_case
																			lowerCamelCase__	:       Union[str, Any]					=						strip_accents
																			lowerCamelCase__	:       Dict					=						tokenize_chinese_chars
																			lowerCamelCase__	:       Optional[int]					=						normalizer_class(**__lowerCamelCase )

													lowerCamelCase__	:       Optional[Any]					=						do_lower_case





							def 		lowerCAmelCase  (     self							:    Tuple      ,   __lowerCamelCase							:    Dict      ,   __lowerCamelCase							:    Union[str, Any]=None ):


													'''simple docstring'''



													lowerCamelCase__	:       Tuple					=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

													if token_ids_a:
																			output += token_ids_a + [self.sep_token_id]

													return output





							def 		lowerCAmelCase  (     self							:    Dict      ,   __lowerCamelCase							:    List[int]      ,   __lowerCamelCase							:    Optional[List[int]] = None ):


													'''simple docstring'''



													lowerCamelCase__	:       List[str]					=						[self.sep_token_id]
													lowerCamelCase__	:       Optional[int]					=						[self.cls_token_id]
													if token_ids_a is None:
																			return len(cls ) * [self.cls_token_type_id] + len(token_ids_a + sep ) * [0]
													return len(cls ) * [self.cls_token_type_id] + len(token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]





							def 		lowerCAmelCase  (     self							:    List[str]      ,   __lowerCamelCase							:    str      ,   __lowerCamelCase							:    Optional[str] = None ):


													'''simple docstring'''



													lowerCamelCase__	:       List[Any]					=						self._tokenizer.model.save(__lowerCamelCase      ,   name=__lowerCamelCase )
													return tuple(__lowerCamelCase )






from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


A   :					int	=      logging.get_logger(__name__)

A   :					Optional[int]	=      {
    "facebook/xmod-base": "https://huggingface.co/facebook/xmod-base/resolve/main/config.json",
    "facebook/xmod-large-prenorm": "https://huggingface.co/facebook/xmod-large-prenorm/resolve/main/config.json",
    "facebook/xmod-base-13-125k": "https://huggingface.co/facebook/xmod-base-13-125k/resolve/main/config.json",
    "facebook/xmod-base-30-125k": "https://huggingface.co/facebook/xmod-base-30-125k/resolve/main/config.json",
    "facebook/xmod-base-30-195k": "https://huggingface.co/facebook/xmod-base-30-195k/resolve/main/config.json",
    "facebook/xmod-base-60-125k": "https://huggingface.co/facebook/xmod-base-60-125k/resolve/main/config.json",
    "facebook/xmod-base-60-265k": "https://huggingface.co/facebook/xmod-base-60-265k/resolve/main/config.json",
    "facebook/xmod-base-75-125k": "https://huggingface.co/facebook/xmod-base-75-125k/resolve/main/config.json",
    "facebook/xmod-base-75-269k": "https://huggingface.co/facebook/xmod-base-75-269k/resolve/main/config.json",
}







class     _lowercase				(							lowercase__):


       """simple docstring"""




       A__      		=    "xmod"





       def __init__(     self							:    int      ,   __lowerCamelCase							:    Any=30522      ,   __lowerCamelCase							:    Any=768      ,   __lowerCamelCase							:    str=12      ,   __lowerCamelCase							:    Any=12      ,   __lowerCamelCase							:    List[str]=3072      ,   __lowerCamelCase							:    List[Any]="gelu"      ,   __lowerCamelCase							:    Union[str, Any]=0.1      ,   __lowerCamelCase							:    int=0.1      ,   __lowerCamelCase							:    Tuple=512      ,   __lowerCamelCase							:    str=2      ,   __lowerCamelCase							:    List[str]=0.0_2      ,   __lowerCamelCase							:    List[str]=1E-1_2      ,   __lowerCamelCase							:    str=1      ,   __lowerCamelCase							:    Optional[int]=0      ,   __lowerCamelCase							:    Optional[Any]=2      ,   __lowerCamelCase							:    str="absolute"      ,   __lowerCamelCase							:    List[str]=True      ,   __lowerCamelCase							:    Dict=None      ,   __lowerCamelCase							:    Optional[Any]=False      ,   __lowerCamelCase							:    Optional[Any]=2      ,   __lowerCamelCase							:    Tuple=False      ,   __lowerCamelCase							:    Tuple=True      ,   __lowerCamelCase							:    Union[str, Any]=True      ,   __lowerCamelCase							:    str=("en_XX",)      ,   __lowerCamelCase							:    Union[str, Any]=None      ,   **__lowerCamelCase							:    Optional[int]      ,   ):


             '''simple docstring'''



             super().__init__(pad_token_id=__lowerCamelCase      ,   bos_token_id=__lowerCamelCase      ,   eos_token_id=__lowerCamelCase      ,   **__lowerCamelCase )

             lowerCamelCase__	:       Union[str, Any]					=						vocab_size
             lowerCamelCase__	:       Union[str, Any]					=						hidden_size
             lowerCamelCase__	:       Optional[int]					=						num_hidden_layers
             lowerCamelCase__	:       List[Any]					=						num_attention_heads
             lowerCamelCase__	:       Union[str, Any]					=						hidden_act
             lowerCamelCase__	:       Optional[int]					=						intermediate_size
             lowerCamelCase__	:       Optional[int]					=						hidden_dropout_prob
             lowerCamelCase__	:       List[Any]					=						attention_probs_dropout_prob
             lowerCamelCase__	:       Any					=						max_position_embeddings
             lowerCamelCase__	:       List[Any]					=						type_vocab_size
             lowerCamelCase__	:       int					=						initializer_range
             lowerCamelCase__	:       Tuple					=						layer_norm_eps
             lowerCamelCase__	:       Union[str, Any]					=						position_embedding_type
             lowerCamelCase__	:       str					=						use_cache
             lowerCamelCase__	:       Union[str, Any]					=						classifier_dropout
             lowerCamelCase__	:       Any					=						pre_norm
             lowerCamelCase__	:       Tuple					=						adapter_reduction_factor
             lowerCamelCase__	:       Tuple					=						adapter_layer_norm
             lowerCamelCase__	:       List[Any]					=						adapter_reuse_layer_norm
             lowerCamelCase__	:       Dict					=						ln_before_adapter
             lowerCamelCase__	:       List[Any]					=						list(__lowerCamelCase )
             lowerCamelCase__	:       Optional[Any]					=						default_language







class     _lowercase				(							lowercase__):


       """simple docstring"""




       @property
       def 		lowerCAmelCase  (     self							:    Tuple ):


             '''simple docstring'''



             if self.task == "multiple-choice":
                   lowerCamelCase__	:       Dict					=						{0: "batch", 1: "choice", 2: "sequence"}
             else:
                   lowerCamelCase__	:       List[str]					=						{0: "batch", 1: "sequence"}
             return OrderedDict(
                 [
                     ("input_ids", dynamic_axis),
                     ("attention_mask", dynamic_axis),
                 ] )


