





'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_squeezebert import SqueezeBertTokenizer


_SCREAMING_SNAKE_CASE					=    logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE					=    {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_SCREAMING_SNAKE_CASE					=    {
    '''vocab_file''': {
        '''squeezebert/squeezebert-uncased''': (
            '''https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/vocab.txt'''
        ),
        '''squeezebert/squeezebert-mnli''': '''https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/vocab.txt''',
        '''squeezebert/squeezebert-mnli-headless''': (
            '''https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''squeezebert/squeezebert-uncased''': (
            '''https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/tokenizer.json'''
        ),
        '''squeezebert/squeezebert-mnli''': (
            '''https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/tokenizer.json'''
        ),
        '''squeezebert/squeezebert-mnli-headless''': (
            '''https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/tokenizer.json'''
        ),
    },
}

_SCREAMING_SNAKE_CASE					=    {
    '''squeezebert/squeezebert-uncased''': 5_1_2,
    '''squeezebert/squeezebert-mnli''': 5_1_2,
    '''squeezebert/squeezebert-mnli-headless''': 5_1_2,
}


_SCREAMING_SNAKE_CASE					=    {
    '''squeezebert/squeezebert-uncased''': {'''do_lower_case''': True},
    '''squeezebert/squeezebert-mnli''': {'''do_lower_case''': True},
    '''squeezebert/squeezebert-mnli-headless''': {'''do_lower_case''': True},
}






class 							__lowercase       (      lowerCAmelCase__						):







	'''simple docstring'''

	a    :				Optional[int]											=       VOCAB_FILES_NAMES
	a    :				int											=       PRETRAINED_VOCAB_FILES_MAP
	a    :				Any											=       PRETRAINED_INIT_CONFIGURATION
	a    :				Optional[int]											=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a    :				Union[str, Any]											=       SqueezeBertTokenizer


	def __init__(self     ,_lowerCamelCase=None     ,_lowerCamelCase=None     ,_lowerCamelCase=True     ,_lowerCamelCase="[UNK]"     ,_lowerCamelCase="[SEP]"     ,_lowerCamelCase="[PAD]"     ,_lowerCamelCase="[CLS]"     ,_lowerCamelCase="[MASK]"     ,_lowerCamelCase=True     ,_lowerCamelCase=None     ,**_lowerCamelCase     ,)							->							str:







				'''simple docstring'''





				super().__init__(
				    _lowerCamelCase     ,tokenizer_file=_lowerCamelCase     ,do_lower_case=_lowerCamelCase     ,unk_token=_lowerCamelCase     ,sep_token=_lowerCamelCase     ,pad_token=_lowerCamelCase     ,cls_token=_lowerCamelCase     ,mask_token=_lowerCamelCase     ,tokenize_chinese_chars=_lowerCamelCase     ,strip_accents=_lowerCamelCase     ,**_lowerCamelCase     ,)

				__lowercase										=							json.loads(self.backend_tokenizer.normalizer.__getstate__() )
				if (
				    normalizer_state.get('''lowercase'''     ,_lowerCamelCase ) != do_lower_case
				    or normalizer_state.get('''strip_accents'''     ,_lowerCamelCase ) != strip_accents
				    or normalizer_state.get('''handle_chinese_chars'''     ,_lowerCamelCase ) != tokenize_chinese_chars
				):
							__lowercase										=							getattr(_lowerCamelCase     ,normalizer_state.pop('''type''' ) )
							__lowercase										=							do_lower_case
							__lowercase										=							strip_accents
							__lowercase										=							tokenize_chinese_chars
							__lowercase										=							normalizer_class(**_lowerCamelCase )

				__lowercase										=							do_lower_case


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase=None )							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase										=							[self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

				return output


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase = None )							->							List[int]:







				'''simple docstring'''





				__lowercase										=							[self.sep_token_id]
				__lowercase										=							[self.cls_token_id]
				if token_ids_a is None:
							return len(cls + token_ids_a + sep ) * [0]
				return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase = None )							->							Tuple[str]:







				'''simple docstring'''





				__lowercase										=							self._tokenizer.model.save(_lowerCamelCase     ,name=_lowerCamelCase )
				return tuple(_lowerCamelCase )












'''simple docstring'''



# flake8: noqa
# Lint as: python3
_SCREAMING_SNAKE_CASE					=    [
    '''VerificationMode''',
    '''Version''',
    '''disable_progress_bar''',
    '''enable_progress_bar''',
    '''is_progress_bar_enabled''',
    '''experimental''',
]

from .info_utils import VerificationMode
from .logging import disable_progress_bar, enable_progress_bar, is_progress_bar_enabled
from .version import Version
from .experimental import experimental





