'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


SCREAMING_SNAKE_CASE_	   =			logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_	   =			{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE_	   =			[
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

SCREAMING_SNAKE_CASE_	   =			{
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
SCREAMING_SNAKE_CASE_	   =			{F"""funnel-transformer/{name}""": 5_12 for name in _model_names}
SCREAMING_SNAKE_CASE_	   =			{F"""funnel-transformer/{name}""": {'do_lower_case': True} for name in _model_names}




class    a (       __lowerCAmelCase				):



  """simple docstring"""
  __lowerCAmelCase				      =					VOCAB_FILES_NAMES
  __lowerCAmelCase				      =					PRETRAINED_VOCAB_FILES_MAP
  __lowerCAmelCase				      =					PRETRAINED_INIT_CONFIGURATION
  __lowerCAmelCase				      =					FunnelTokenizer
  __lowerCAmelCase				      =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  __lowerCAmelCase				      =					2







  def __init__(							self   ,    snake_case_=None   ,    snake_case_=None   ,    snake_case_=True   ,    snake_case_="<unk>"   ,    snake_case_="<sep>"   ,    snake_case_="<pad>"   ,    snake_case_="<cls>"   ,    snake_case_="<mask>"   ,    snake_case_="<s>"   ,    snake_case_="</s>"   ,    snake_case_=True   ,    snake_case_=True   ,    snake_case_=None   ,    snake_case_="##"   ,    **snake_case_   ,    ):

    '''simple docstring'''







    super().__init__(
        snake_case_   ,    tokenizer_file=snake_case_   ,    do_lower_case=snake_case_   ,    unk_token=snake_case_   ,    sep_token=snake_case_   ,    pad_token=snake_case_   ,    cls_token=snake_case_   ,    mask_token=snake_case_   ,    bos_token=snake_case_   ,    eos_token=snake_case_   ,    clean_text=snake_case_   ,    tokenize_chinese_chars=snake_case_   ,    strip_accents=snake_case_   ,    wordpieces_prefix=snake_case_   ,    **snake_case_   ,    )

    __UpperCAmelCase:					int		= json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
    if (
        normalizer_state.get("""lowercase"""   ,    snake_case_					) != do_lower_case
        or normalizer_state.get("""strip_accents"""   ,    snake_case_					) != strip_accents
        or normalizer_state.get("""handle_chinese_chars"""   ,    snake_case_					) != tokenize_chinese_chars
    ):
      __UpperCAmelCase:					Union[str, Any]		= getattr(snake_case_   ,    normalizer_state.pop("""type"""					)					)
      __UpperCAmelCase:					Dict		= do_lower_case
      __UpperCAmelCase:					List[Any]		= strip_accents
      __UpperCAmelCase:					Optional[Any]		= tokenize_chinese_chars
      __UpperCAmelCase:					Optional[int]		= normalizer_class(**snake_case_					)

    __UpperCAmelCase:					Optional[int]		= do_lower_case







  def    lowercase_  (							self   ,    snake_case_   ,    snake_case_=None					):

    '''simple docstring'''







    __UpperCAmelCase:					str		= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

    if token_ids_a:
      output += token_ids_a + [self.sep_token_id]

    return output







  def    lowercase_  (							self   ,    snake_case_   ,    snake_case_ = None					):

    '''simple docstring'''







    __UpperCAmelCase:					List[Any]		= [self.sep_token_id]
    __UpperCAmelCase:					List[Any]		= [self.cls_token_id]
    if token_ids_a is None:
      return len(cls					) * [self.cls_token_type_id] + len(token_ids_a + sep					) * [0]
    return len(cls					) * [self.cls_token_type_id] + len(token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]







  def    lowercase_  (							self   ,    snake_case_   ,    snake_case_ = None					):

    '''simple docstring'''







    __UpperCAmelCase:					int		= self._tokenizer.model.save(snake_case_   ,    name=snake_case_					)
    return tuple(snake_case_					)
'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


SCREAMING_SNAKE_CASE_	   =			{
    'configuration_rembert': ['REMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'RemBertConfig', 'RemBertOnnxConfig']
}

try:
    if not is_sentencepiece_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    SCREAMING_SNAKE_CASE_	   =			['RemBertTokenizer']

try:
    if not is_tokenizers_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    SCREAMING_SNAKE_CASE_	   =			['RemBertTokenizerFast']

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    SCREAMING_SNAKE_CASE_	   =			[
        'REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
        'RemBertForCausalLM',
        'RemBertForMaskedLM',
        'RemBertForMultipleChoice',
        'RemBertForQuestionAnswering',
        'RemBertForSequenceClassification',
        'RemBertForTokenClassification',
        'RemBertLayer',
        'RemBertModel',
        'RemBertPreTrainedModel',
        'load_tf_weights_in_rembert',
    ]


try:
    if not is_tf_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    SCREAMING_SNAKE_CASE_	   =			[
        'TF_REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
        'TFRemBertForCausalLM',
        'TFRemBertForMaskedLM',
        'TFRemBertForMultipleChoice',
        'TFRemBertForQuestionAnswering',
        'TFRemBertForSequenceClassification',
        'TFRemBertForTokenClassification',
        'TFRemBertLayer',
        'TFRemBertModel',
        'TFRemBertPreTrainedModel',
    ]


if TYPE_CHECKING:
    from .configuration_rembert import REMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, RemBertConfig, RemBertOnnxConfig

    try:
        if not is_sentencepiece_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_rembert import RemBertTokenizer

    try:
        if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_rembert_fast import RemBertTokenizerFast

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_rembert import (
            REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
            RemBertForCausalLM,
            RemBertForMaskedLM,
            RemBertForMultipleChoice,
            RemBertForQuestionAnswering,
            RemBertForSequenceClassification,
            RemBertForTokenClassification,
            RemBertLayer,
            RemBertModel,
            RemBertPreTrainedModel,
            load_tf_weights_in_rembert,
        )

    try:
        if not is_tf_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_tf_rembert import (
            TF_REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
            TFRemBertForCausalLM,
            TFRemBertForMaskedLM,
            TFRemBertForMultipleChoice,
            TFRemBertForQuestionAnswering,
            TFRemBertForSequenceClassification,
            TFRemBertForTokenClassification,
            TFRemBertLayer,
            TFRemBertModel,
            TFRemBertPreTrainedModel,
        )


else:
    import sys

    SCREAMING_SNAKE_CASE_	   =			_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)