





"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


lowerCAmelCase_			=							logging.get_logger(__name__)

lowerCAmelCase_			=							{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCAmelCase_			=							{
    '''vocab_file''': {
        '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt''',
        '''distilbert-base-uncased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'''
        ),
        '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt''',
        '''distilbert-base-cased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'''
        ),
        '''distilbert-base-german-cased''': '''https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt''',
        '''distilbert-base-multilingual-cased''': (
            '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json''',
        '''distilbert-base-uncased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json''',
        '''distilbert-base-cased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-german-cased''': (
            '''https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-multilingual-cased''': (
            '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'''
        ),
    },
}

lowerCAmelCase_			=							{
    '''distilbert-base-uncased''': 512,
    '''distilbert-base-uncased-distilled-squad''': 512,
    '''distilbert-base-cased''': 512,
    '''distilbert-base-cased-distilled-squad''': 512,
    '''distilbert-base-german-cased''': 512,
    '''distilbert-base-multilingual-cased''': 512,
}


lowerCAmelCase_			=							{
    '''distilbert-base-uncased''': {'''do_lower_case''': True},
    '''distilbert-base-uncased-distilled-squad''': {'''do_lower_case''': True},
    '''distilbert-base-cased''': {'''do_lower_case''': False},
    '''distilbert-base-cased-distilled-squad''': {'''do_lower_case''': False},
    '''distilbert-base-german-cased''': {'''do_lower_case''': False},
    '''distilbert-base-multilingual-cased''': {'''do_lower_case''': False},
}



class 	lowerCAmelCase					(      snake_case							):
       lowerCAmelCase__						 =  VOCAB_FILES_NAMES
       lowerCAmelCase__						 =  PRETRAINED_VOCAB_FILES_MAP
       lowerCAmelCase__						 =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       lowerCAmelCase__						 =  PRETRAINED_INIT_CONFIGURATION
       lowerCAmelCase__						 =  ["""input_ids""", """attention_mask"""]
       lowerCAmelCase__						 =  DistilBertTokenizer






       def __init__(   self    ,  a__=None    ,  a__=None    ,  a__=True    ,  a__="[UNK]"    ,  a__="[SEP]"    ,  a__="[PAD]"    ,  a__="[CLS]"    ,  a__="[MASK]"    ,  a__=True    ,  a__=None    ,  **a__    ,  ):
              super().__init__(
                  a__    ,  tokenizer_file=a__    ,  do_lower_case=a__    ,  unk_token=a__    ,  sep_token=a__    ,  pad_token=a__    ,  cls_token=a__    ,  mask_token=a__    ,  tokenize_chinese_chars=a__    ,  strip_accents=a__    ,  **a__    ,  )

              _UpperCAmelCase     =     json.loads(self.backend_tokenizer.normalizer.__getstate__()     )
              if (
                  normalizer_state.get('lowercase'    ,  a__     ) != do_lower_case
                  or normalizer_state.get('strip_accents'    ,  a__     ) != strip_accents
                  or normalizer_state.get('handle_chinese_chars'    ,  a__     ) != tokenize_chinese_chars
              ):
                     _UpperCAmelCase     =     getattr(a__    ,  normalizer_state.pop('type'     )     )
                     _UpperCAmelCase     =     do_lower_case
                     _UpperCAmelCase     =     strip_accents
                     _UpperCAmelCase     =     tokenize_chinese_chars
                     _UpperCAmelCase     =     normalizer_class(**a__     )

              _UpperCAmelCase     =     do_lower_case






       def 							__A (   self    ,  a__    ,  a__=None     ):
              _UpperCAmelCase     =     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

              if token_ids_a:
                     output += token_ids_a + [self.sep_token_id]

              return output






       def 							__A (   self    ,  a__    ,  a__ = None     ):
              _UpperCAmelCase     =     [self.sep_token_id]
              _UpperCAmelCase     =     [self.cls_token_id]
              if token_ids_a is None:
                     return len(cls + token_ids_a + sep     ) * [0]
              return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]






       def 							__A (   self    ,  a__    ,  a__ = None     ):
              _UpperCAmelCase     =     self._tokenizer.model.save(a__    ,  name=a__     )
              return tuple(a__     )













"""simple docstring"""






def       __lowerCamelCase    (       SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE							)		->       str:






       """simple docstring"""




       if a < 0 or b < 0:
              raise ValueError('the value of both inputs must be positive'							)

       _UpperCAmelCase     =     str(bin(SCREAMING_SNAKE_CASE							)							)[2:]  # remove the leading "0b"
       _UpperCAmelCase     =     str(bin(SCREAMING_SNAKE_CASE							)							)[2:]  # remove the leading "0b"

       _UpperCAmelCase     =     max(len(SCREAMING_SNAKE_CASE							),len(SCREAMING_SNAKE_CASE							)							)

       return "0b" + "".join(
           str(int(char_a == '1' and char_b == '1'							)							)
           for char_a, char_b in zip(a_binary.zfill(SCREAMING_SNAKE_CASE							),b_binary.zfill(SCREAMING_SNAKE_CASE							)							)							)


if __name__ == "__main__":
    import doctest

    doctest.testmod()






