





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


_SCREAMING_SNAKE_CASE					=    logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE					=    {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_SCREAMING_SNAKE_CASE					=    [
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

_SCREAMING_SNAKE_CASE					=    {
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
_SCREAMING_SNAKE_CASE					=    {f'''funnel-transformer/{name}''': 5_1_2 for name in _model_names}
_SCREAMING_SNAKE_CASE					=    {f'''funnel-transformer/{name}''': {'''do_lower_case''': True} for name in _model_names}






class 							__lowercase       (      lowerCAmelCase__						):







	'''simple docstring'''

	a    :				Any											=       VOCAB_FILES_NAMES
	a    :				Optional[Any]											=       PRETRAINED_VOCAB_FILES_MAP
	a    :				str											=       PRETRAINED_INIT_CONFIGURATION
	a    :				List[Any]											=       FunnelTokenizer
	a    :				Optional[Any]											=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a    :				int											=       2


	def __init__(self     ,_lowerCamelCase=None     ,_lowerCamelCase=None     ,_lowerCamelCase=True     ,_lowerCamelCase="<unk>"     ,_lowerCamelCase="<sep>"     ,_lowerCamelCase="<pad>"     ,_lowerCamelCase="<cls>"     ,_lowerCamelCase="<mask>"     ,_lowerCamelCase="<s>"     ,_lowerCamelCase="</s>"     ,_lowerCamelCase=True     ,_lowerCamelCase=True     ,_lowerCamelCase=None     ,_lowerCamelCase="##"     ,**_lowerCamelCase     ,)							->							Optional[int]:







				'''simple docstring'''





				super().__init__(
				    _lowerCamelCase     ,tokenizer_file=_lowerCamelCase     ,do_lower_case=_lowerCamelCase     ,unk_token=_lowerCamelCase     ,sep_token=_lowerCamelCase     ,pad_token=_lowerCamelCase     ,cls_token=_lowerCamelCase     ,mask_token=_lowerCamelCase     ,bos_token=_lowerCamelCase     ,eos_token=_lowerCamelCase     ,clean_text=_lowerCamelCase     ,tokenize_chinese_chars=_lowerCamelCase     ,strip_accents=_lowerCamelCase     ,wordpieces_prefix=_lowerCamelCase     ,**_lowerCamelCase     ,)

				__lowercase										=							json.loads(self.backend_tokenizer.normalizer.__getstate__() )
				if (
				    normalizer_state.get('''lowercase'''     ,_lowerCamelCase ) != do_lower_case
				    or normalizer_state.get('''strip_accents'''     ,_lowerCamelCase ) != strip_accents
				    or normalizer_state.get('''handle_chinese_chars'''     ,_lowerCamelCase ) != tokenize_chinese_chars
				):
							__lowercase										=							getattr(_lowerCamelCase     ,normalizer_state.pop('''type''' ) )
							__lowercase										=							do_lower_case
							__lowercase										=							strip_accents
							__lowercase										=							tokenize_chinese_chars
							__lowercase										=							normalizer_class(**_lowerCamelCase )

				__lowercase										=							do_lower_case


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase=None )							->							int:







				'''simple docstring'''





				__lowercase										=							[self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

				return output


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase = None )							->							List[int]:







				'''simple docstring'''





				__lowercase										=							[self.sep_token_id]
				__lowercase										=							[self.cls_token_id]
				if token_ids_a is None:
							return len(cls ) * [self.cls_token_type_id] + len(token_ids_a + sep ) * [0]
				return len(cls ) * [self.cls_token_type_id] + len(token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase = None )							->							Tuple[str]:







				'''simple docstring'''





				__lowercase										=							self._tokenizer.model.save(_lowerCamelCase     ,name=_lowerCamelCase )
				return tuple(_lowerCamelCase )












'''simple docstring'''



import math




def    _lowerCAmelCase						(				lowerCamelCase_			:  int				):
			assert isinstance(lowerCamelCase_		,       lowerCamelCase_				) and (
			    number >= 0
			), "'number' must been an int and positive"

			if 1 < number < 4:
						# 2 and 3 are primes
						return True
			elif number < 2 or not number % 2:
						# Negatives, 0, 1 and all even numbers are not primes
						return False

			__lowercase										=							range(3		,       int(math.sqrt(lowerCamelCase_				) + 1				)		,       2				)
			return not any(not number % i for i in odd_numbers				)




def    _lowerCAmelCase						(				lowerCamelCase_			:  Dict		,       lowerCamelCase_			:  Any=1		,       **lowerCamelCase_			:  Tuple				):
			__lowercase										=							factor * value
			__lowercase										=							value

			while not is_prime(lowerCamelCase_				):
						value += 1 if not ("desc" in kwargs and kwargs["desc"] is True) else -1

			if value == first_value_val:
						return next_prime(value + 1		,       **lowerCamelCase_				)
			return value





