




"""simple docstring"""


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


A_      			=       logging.get_logger(__name__)

A_      			=       {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

A_      			=       {
    '''vocab_file''': {
        '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt''',
        '''distilbert-base-uncased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'''
        ),
        '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt''',
        '''distilbert-base-cased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'''
        ),
        '''distilbert-base-german-cased''': '''https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt''',
        '''distilbert-base-multilingual-cased''': (
            '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json''',
        '''distilbert-base-uncased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json''',
        '''distilbert-base-cased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-german-cased''': (
            '''https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-multilingual-cased''': (
            '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'''
        ),
    },
}

A_      			=       {
    '''distilbert-base-uncased''': 5_12,
    '''distilbert-base-uncased-distilled-squad''': 5_12,
    '''distilbert-base-cased''': 5_12,
    '''distilbert-base-cased-distilled-squad''': 5_12,
    '''distilbert-base-german-cased''': 5_12,
    '''distilbert-base-multilingual-cased''': 5_12,
}


A_      			=       {
    '''distilbert-base-uncased''': {'''do_lower_case''': True},
    '''distilbert-base-uncased-distilled-squad''': {'''do_lower_case''': True},
    '''distilbert-base-cased''': {'''do_lower_case''': False},
    '''distilbert-base-cased-distilled-squad''': {'''do_lower_case''': False},
    '''distilbert-base-german-cased''': {'''do_lower_case''': False},
    '''distilbert-base-multilingual-cased''': {'''do_lower_case''': False},
}






class       lowercase(  __a				):
 lowercase__				      =      VOCAB_FILES_NAMES
 lowercase__				      =      PRETRAINED_VOCAB_FILES_MAP
 lowercase__				      =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowercase__				      =      PRETRAINED_INIT_CONFIGURATION
 lowercase__				      =      ["input_ids", "attention_mask"]
 lowercase__				      =      DistilBertTokenizer


 def __init__(       self:	int,   a_:	Union[str, Any]=None,   a_:	int=None,   a_:	str=True,   a_:	Dict="[UNK]",   a_:	List[str]="[SEP]",   a_:	Dict="[PAD]",   a_:	Union[str, Any]="[CLS]",   a_:	Dict="[MASK]",   a_:	Optional[Any]=True,   a_:	List[Any]=None,   **a_:	Optional[int],   ):




   '''simple docstring'''



   super().__init__(
       a_,   tokenizer_file=a_,   do_lower_case=a_,   unk_token=a_,   sep_token=a_,   pad_token=a_,   cls_token=a_,   mask_token=a_,   tokenize_chinese_chars=a_,   strip_accents=a_,   **a_,   )

   _snake_case							:	int			=						json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
   if (
       normalizer_state.get("""lowercase""",   a_	) != do_lower_case
       or normalizer_state.get("""strip_accents""",   a_	) != strip_accents
       or normalizer_state.get("""handle_chinese_chars""",   a_	) != tokenize_chinese_chars
   ):
     _snake_case							:	Optional[Any]			=						getattr(a_,   normalizer_state.pop("""type"""	)	)
     _snake_case							:	List[Any]			=						do_lower_case
     _snake_case							:	Optional[Any]			=						strip_accents
     _snake_case							:	Optional[int]			=						tokenize_chinese_chars
     _snake_case							:	Dict			=						normalizer_class(**a_	)

   _snake_case							:	int			=						do_lower_case


 def 				UpperCamelCase_     (       self:	Tuple,   a_:	Union[str, Any],   a_:	str=None	):




   '''simple docstring'''



   _snake_case							:	Tuple			=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

   if token_ids_a:
     output += token_ids_a + [self.sep_token_id]

   return output


 def 				UpperCamelCase_     (       self:	Optional[int],   a_:	List[int],   a_:	Optional[List[int]] = None	):




   '''simple docstring'''



   _snake_case							:	List[Any]			=						[self.sep_token_id]
   _snake_case							:	Dict			=						[self.cls_token_id]
   if token_ids_a is None:
     return len(cls + token_ids_a + sep	) * [0]
   return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]


 def 				UpperCamelCase_     (       self:	Optional[Any],   a_:	str,   a_:	Optional[str] = None	):




   '''simple docstring'''



   _snake_case							:	Optional[Any]			=						self._tokenizer.model.save(a_,   name=a_	)
   return tuple(a_	)








"""simple docstring"""


from abc import ABC, abstractmethod
from argparse import ArgumentParser






class       lowercase(  __a				):







 '''simple docstring'''





 @staticmethod
 @abstractmethod
 def 				UpperCamelCase_     (       a_:	ArgumentParser	):




   '''simple docstring'''



   raise NotImplementedError()


 @abstractmethod
 def 				UpperCamelCase_     (       self:	List[str]	):




   '''simple docstring'''



   raise NotImplementedError()


