




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


__A						:							Optional[Any]							=  logging.get_logger(__name__)

__A						:							List[str]							=  {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

__A						:							int							=  {
    """vocab_file""": {
        """distilbert-base-uncased""": """https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt""",
        """distilbert-base-uncased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"""
        ),
        """distilbert-base-cased""": """https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt""",
        """distilbert-base-cased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"""
        ),
        """distilbert-base-german-cased""": """https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt""",
        """distilbert-base-multilingual-cased""": (
            """https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """distilbert-base-uncased""": """https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json""",
        """distilbert-base-uncased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-cased""": """https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json""",
        """distilbert-base-cased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-german-cased""": (
            """https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-multilingual-cased""": (
            """https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"""
        ),
    },
}

__A						:							Optional[int]							=  {
    """distilbert-base-uncased""": 5_1_2,
    """distilbert-base-uncased-distilled-squad""": 5_1_2,
    """distilbert-base-cased""": 5_1_2,
    """distilbert-base-cased-distilled-squad""": 5_1_2,
    """distilbert-base-german-cased""": 5_1_2,
    """distilbert-base-multilingual-cased""": 5_1_2,
}


__A						:							int							=  {
    """distilbert-base-uncased""": {"""do_lower_case""": True},
    """distilbert-base-uncased-distilled-squad""": {"""do_lower_case""": True},
    """distilbert-base-cased""": {"""do_lower_case""": False},
    """distilbert-base-cased-distilled-squad""": {"""do_lower_case""": False},
    """distilbert-base-german-cased""": {"""do_lower_case""": False},
    """distilbert-base-multilingual-cased""": {"""do_lower_case""": False},
}




class 			UpperCAmelCase_      (   A       ):




							'''simple docstring'''



							a__						  =     VOCAB_FILES_NAMES
							a__						  =     PRETRAINED_VOCAB_FILES_MAP
							a__						  =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							a__						  =     PRETRAINED_INIT_CONFIGURATION
							a__						  =     ['''input_ids''', '''attention_mask''']
							a__						  =     DistilBertTokenizer





							def __init__(					self       :	List[str]       ,     a       :	Tuple=None       ,     a       :	Tuple=None       ,     a       :	List[str]=True       ,     a       :	Union[str, Any]="[UNK]"       ,     a       :	Optional[Any]="[SEP]"       ,     a       :	Any="[PAD]"       ,     a       :	List[Any]="[CLS]"       ,     a       :	List[str]="[MASK]"       ,     a       :	Union[str, Any]=True       ,     a       :	Tuple=None       ,     **a       :	List[str]       ,     )			->					Optional[Any]:
										super().__init__(
										    a       ,     tokenizer_file=a       ,     do_lower_case=a       ,     unk_token=a       ,     sep_token=a       ,     pad_token=a       ,     cls_token=a       ,     mask_token=a       ,     tokenize_chinese_chars=a       ,     strip_accents=a       ,     **a       ,     )

										SCREAMING_SNAKE_CASE           =					json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
										if (
										    normalizer_state.get("""lowercase"""       ,     a			) != do_lower_case
										    or normalizer_state.get("""strip_accents"""       ,     a			) != strip_accents
										    or normalizer_state.get("""handle_chinese_chars"""       ,     a			) != tokenize_chinese_chars
										):
													SCREAMING_SNAKE_CASE           =					getattr(a       ,     normalizer_state.pop("""type"""			)			)
													SCREAMING_SNAKE_CASE           =					do_lower_case
													SCREAMING_SNAKE_CASE           =					strip_accents
													SCREAMING_SNAKE_CASE           =					tokenize_chinese_chars
													SCREAMING_SNAKE_CASE           =					normalizer_class(**a			)

										SCREAMING_SNAKE_CASE           =					do_lower_case





							def    _UpperCAmelCase     (					self       :	int       ,     a       :	int       ,     a       :	List[str]=None			)			->					List[Any]:
										SCREAMING_SNAKE_CASE           =					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

										if token_ids_a:
													output += token_ids_a + [self.sep_token_id]

										return output





							def    _UpperCAmelCase     (					self       :	int       ,     a       :	List[int]       ,     a       :	Optional[List[int]] = None			)			->					List[int]:
										SCREAMING_SNAKE_CASE           =					[self.sep_token_id]
										SCREAMING_SNAKE_CASE           =					[self.cls_token_id]
										if token_ids_a is None:
													return len(cls + token_ids_a + sep			) * [0]
										return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]







							def    _UpperCAmelCase     (					self       :	Any       ,     a       :	str       ,     a       :	Optional[str] = None			)			->					Tuple[str]:
										SCREAMING_SNAKE_CASE           =					self._tokenizer.model.save(a       ,     name=a			)
										return tuple(a			)












def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE   ):
			'''simple docstring'''
			if not all(char in """01""" for char in bin_string   ):
						raise ValueError("""Non-binary value was passed to the function"""   )
			if not bin_string:
						raise ValueError("""Empty string was passed to the function"""   )
			SCREAMING_SNAKE_CASE           =					""""""
			while len(SCREAMING_SNAKE_CASE   ) % 3 != 0:
						SCREAMING_SNAKE_CASE           =					"""0""" + bin_string
			SCREAMING_SNAKE_CASE           =					[
			    bin_string[index : index + 3]
			    for index in range(len(SCREAMING_SNAKE_CASE   )   )
			    if index % 3 == 0
			]
			for bin_group in bin_string_in_3_list:
						SCREAMING_SNAKE_CASE           =					0
						for index, val in enumerate(SCREAMING_SNAKE_CASE   ):
									oct_val += int(2 ** (2 - index) * int(SCREAMING_SNAKE_CASE   )   )
						oct_string += str(SCREAMING_SNAKE_CASE   )
			return oct_string


if __name__ == "__main__":
		from doctest import testmod

		testmod()






