

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__UpperCAmelCase    				=     logging.get_logger(__name__)

__UpperCAmelCase    				=     {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__UpperCAmelCase    				=     [
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

__UpperCAmelCase    				=     {
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
__UpperCAmelCase    				=     {f'''funnel-transformer/{name}''': 5_1_2 for name in _model_names}
__UpperCAmelCase    				=     {f'''funnel-transformer/{name}''': {'do_lower_case': True} for name in _model_names}





class A__    ( A ):



 """simple docstring"""







 _lowercase     :					List[str]      =						VOCAB_FILES_NAMES
 _lowercase     :					Optional[int]      =						PRETRAINED_VOCAB_FILES_MAP
 _lowercase     :					Tuple      =						PRETRAINED_INIT_CONFIGURATION
 _lowercase     :					Optional[Any]      =						FunnelTokenizer
 _lowercase     :					str      =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowercase     :					int      =						2

 def __init__(		self :	int     ,						A_ :	List[str]=None     ,						A_ :	Tuple=None     ,						A_ :	List[Any]=True     ,						A_ :	Union[str, Any]="<unk>"     ,						A_ :	List[str]="<sep>"     ,						A_ :	Union[str, Any]="<pad>"     ,						A_ :	Union[str, Any]="<cls>"     ,						A_ :	Optional[Any]="<mask>"     ,						A_ :	int="<s>"     ,						A_ :	str="</s>"     ,						A_ :	Union[str, Any]=True     ,						A_ :	str=True     ,						A_ :	str=None     ,						A_ :	int="##"     ,						**A_ :	List[str]     ,						):






      '''simple docstring'''




      super().__init__(
          A_     ,						tokenizer_file=A_     ,						do_lower_case=A_     ,						unk_token=A_     ,						sep_token=A_     ,						pad_token=A_     ,						cls_token=A_     ,						mask_token=A_     ,						bos_token=A_     ,						eos_token=A_     ,						clean_text=A_     ,						tokenize_chinese_chars=A_     ,						strip_accents=A_     ,						wordpieces_prefix=A_     ,						**A_     ,						)

      _lowerCAmelCase   :					Any        =    json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
      if (
          normalizer_state.get("lowercase"     ,						A_       ) != do_lower_case
          or normalizer_state.get("strip_accents"     ,						A_       ) != strip_accents
          or normalizer_state.get("handle_chinese_chars"     ,						A_       ) != tokenize_chinese_chars
      ):
           _lowerCAmelCase   :					Dict        =    getattr(A_     ,						normalizer_state.pop("type"       )       )
           _lowerCAmelCase   :					Any        =    do_lower_case
           _lowerCAmelCase   :					int        =    strip_accents
           _lowerCAmelCase   :					Any        =    tokenize_chinese_chars
           _lowerCAmelCase   :					str        =    normalizer_class(**A_       )

      _lowerCAmelCase   :					List[Any]        =    do_lower_case

 def    __magic_name__       (		self :	Any     ,						A_ :	Optional[Any]     ,						A_ :	List[str]=None       ):






      '''simple docstring'''




      _lowerCAmelCase   :					Optional[int]        =    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
           output += token_ids_a + [self.sep_token_id]

      return output

 def    __magic_name__       (		self :	List[str]     ,						A_ :	List[int]     ,						A_ :	Optional[List[int]] = None       ):






      '''simple docstring'''




      _lowerCAmelCase   :					Tuple        =    [self.sep_token_id]
      _lowerCAmelCase   :					Optional[Any]        =    [self.cls_token_id]
      if token_ids_a is None:
           return len(cls       ) * [self.cls_token_type_id] + len(token_ids_a + sep       ) * [0]
      return len(cls       ) * [self.cls_token_type_id] + len(token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]


 def    __magic_name__       (		self :	List[str]     ,						A_ :	str     ,						A_ :	Optional[str] = None       ):






      '''simple docstring'''




      _lowerCAmelCase   :					Tuple        =    self._tokenizer.model.save(A_     ,						name=A_       )
      return tuple(A_       )









import warnings

from ...utils import logging
from .image_processing_chinese_clip import ChineseCLIPImageProcessor


__UpperCAmelCase    				=     logging.get_logger(__name__)





class A__    ( A ):



 """simple docstring"""







 def __init__(		self :	Tuple     ,						*A_ :	Optional[int]     ,						**A_ :	int       ):






      '''simple docstring'''




      warnings.warn(
          "The class ChineseCLIPFeatureExtractor is deprecated and will be removed in version 5 of Transformers."
          " Please use ChineseCLIPImageProcessor instead."     ,						A_     ,						)
      super().__init__(*A_     ,						**A_       )






