






'''simple docstring'''

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


_A				:    Optional[Any]     						=logging.get_logger(__name__)

_A				:    Optional[int]     						={'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_A				:    int     						={
    '''vocab_file''': {
        '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt''',
        '''distilbert-base-uncased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'''
        ),
        '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt''',
        '''distilbert-base-cased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'''
        ),
        '''distilbert-base-german-cased''': '''https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt''',
        '''distilbert-base-multilingual-cased''': (
            '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json''',
        '''distilbert-base-uncased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json''',
        '''distilbert-base-cased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-german-cased''': (
            '''https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-multilingual-cased''': (
            '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'''
        ),
    },
}

_A				:    Any     						={
    '''distilbert-base-uncased''': 5_1_2,
    '''distilbert-base-uncased-distilled-squad''': 5_1_2,
    '''distilbert-base-cased''': 5_1_2,
    '''distilbert-base-cased-distilled-squad''': 5_1_2,
    '''distilbert-base-german-cased''': 5_1_2,
    '''distilbert-base-multilingual-cased''': 5_1_2,
}


_A				:    List[str]     						={
    '''distilbert-base-uncased''': {'''do_lower_case''': True},
    '''distilbert-base-uncased-distilled-squad''': {'''do_lower_case''': True},
    '''distilbert-base-cased''': {'''do_lower_case''': False},
    '''distilbert-base-cased-distilled-squad''': {'''do_lower_case''': False},
    '''distilbert-base-german-cased''': {'''do_lower_case''': False},
    '''distilbert-base-multilingual-cased''': {'''do_lower_case''': False},
}



class 		lowerCamelCase__      (						A     ):




					'''simple docstring'''



					A_						    =						VOCAB_FILES_NAMES
					A_						    =						PRETRAINED_VOCAB_FILES_MAP
					A_						    =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					A_						    =						PRETRAINED_INIT_CONFIGURATION
					A_						    =						["""input_ids""", """attention_mask"""]
					A_						    =						DistilBertTokenizer




					def __init__(  self				:	int     ,  UpperCamelCase_				:	Union[str, Any]=None     ,  UpperCamelCase_				:	Optional[Any]=None     ,  UpperCamelCase_				:	int=True     ,  UpperCamelCase_				:	Any="[UNK]"     ,  UpperCamelCase_				:	List[Any]="[SEP]"     ,  UpperCamelCase_				:	Optional[Any]="[PAD]"     ,  UpperCamelCase_				:	List[Any]="[CLS]"     ,  UpperCamelCase_				:	List[Any]="[MASK]"     ,  UpperCamelCase_				:	List[Any]=True     ,  UpperCamelCase_				:	Any=None     ,  **UpperCamelCase_				:	Optional[int]     ,  )  ->       Dict:



						'''simple docstring'''





						super().__init__(
						    UpperCamelCase_     ,  tokenizer_file=UpperCamelCase_     ,  do_lower_case=UpperCamelCase_     ,  unk_token=UpperCamelCase_     ,  sep_token=UpperCamelCase_     ,  pad_token=UpperCamelCase_     ,  cls_token=UpperCamelCase_     ,  mask_token=UpperCamelCase_     ,  tokenize_chinese_chars=UpperCamelCase_     ,  strip_accents=UpperCamelCase_     ,  **UpperCamelCase_     ,  )

						_lowercase					:       Any       =		json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
						if (
						    normalizer_state.get('lowercase'     ,  UpperCamelCase_					) != do_lower_case
						    or normalizer_state.get('strip_accents'     ,  UpperCamelCase_					) != strip_accents
						    or normalizer_state.get('handle_chinese_chars'     ,  UpperCamelCase_					) != tokenize_chinese_chars
						):
							_lowercase					:       int       =		getattr(UpperCamelCase_     ,  normalizer_state.pop('type'					)					)
							_lowercase					:       str       =		do_lower_case
							_lowercase					:       Dict       =		strip_accents
							_lowercase					:       Optional[Any]       =		tokenize_chinese_chars
							_lowercase					:       Any       =		normalizer_class(**UpperCamelCase_					)

						_lowercase					:       Optional[Any]       =		do_lower_case




					def 						__UpperCAmelCase     (  self				:	int     ,  UpperCamelCase_				:	Any     ,  UpperCamelCase_				:	List[str]=None					)  ->       Optional[int]:



						'''simple docstring'''





						_lowercase					:       str       =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

						if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

						return output




					def 						__UpperCAmelCase     (  self				:	Optional[Any]     ,  UpperCamelCase_				:	List[int]     ,  UpperCamelCase_				:	Optional[List[int]] = None					)  ->       List[int]:



						'''simple docstring'''





						_lowercase					:       List[str]       =		[self.sep_token_id]
						_lowercase					:       int       =		[self.cls_token_id]
						if token_ids_a is None:
							return len(cls + token_ids_a + sep					) * [0]
						return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]




					def 						__UpperCAmelCase     (  self				:	Tuple     ,  UpperCamelCase_				:	str     ,  UpperCamelCase_				:	Optional[str] = None					)  ->       Tuple[str]:



						'''simple docstring'''





						_lowercase					:       Optional[Any]       =		self._tokenizer.model.save(UpperCamelCase_     ,  name=UpperCamelCase_					)
						return tuple(UpperCamelCase_					)





'''simple docstring'''

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_A				:    Dict     						=logging.get_logger(__name__)

_A				:    Dict     						={
    # See all MEGATRON_BERT models at https://huggingface.co/models?filter=bert
}



class 		lowerCamelCase__      (						A     ):




					'''simple docstring'''



					A_						    =						"""megatron-bert"""




					def __init__(  self				:	int     ,  UpperCamelCase_				:	int=2_9056     ,  UpperCamelCase_				:	Optional[int]=1024     ,  UpperCamelCase_				:	Optional[Any]=24     ,  UpperCamelCase_				:	List[Any]=16     ,  UpperCamelCase_				:	Optional[int]=4096     ,  UpperCamelCase_				:	Optional[Any]="gelu"     ,  UpperCamelCase_				:	List[Any]=0.1     ,  UpperCamelCase_				:	Union[str, Any]=0.1     ,  UpperCamelCase_				:	int=512     ,  UpperCamelCase_				:	Dict=2     ,  UpperCamelCase_				:	Union[str, Any]=0.02     ,  UpperCamelCase_				:	Any=1E-12     ,  UpperCamelCase_				:	Tuple=0     ,  UpperCamelCase_				:	Optional[int]="absolute"     ,  UpperCamelCase_				:	Optional[Any]=True     ,  **UpperCamelCase_				:	Any     ,  )  ->       List[Any]:



						'''simple docstring'''





						super().__init__(pad_token_id=UpperCamelCase_     ,  **UpperCamelCase_					)

						_lowercase					:       Dict       =		vocab_size
						_lowercase					:       Any       =		hidden_size
						_lowercase					:       Union[str, Any]       =		num_hidden_layers
						_lowercase					:       Dict       =		num_attention_heads
						_lowercase					:       Dict       =		hidden_act
						_lowercase					:       Optional[Any]       =		intermediate_size
						_lowercase					:       Optional[int]       =		hidden_dropout_prob
						_lowercase					:       Optional[Any]       =		attention_probs_dropout_prob
						_lowercase					:       Any       =		max_position_embeddings
						_lowercase					:       str       =		type_vocab_size
						_lowercase					:       Optional[Any]       =		initializer_range
						_lowercase					:       List[str]       =		layer_norm_eps
						_lowercase					:       List[Any]       =		position_embedding_type
						_lowercase					:       Optional[Any]       =		use_cache
