import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


lowerCAmelCase			:							str     =logging.get_logger(__name__)

lowerCAmelCase			:							Dict     ={"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

lowerCAmelCase			:							int     ={
    "vocab_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt",
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-german-cased": (
            "https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"
        ),
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
    },
}

lowerCAmelCase			:							Any     ={
    "distilbert-base-uncased": 512,
    "distilbert-base-uncased-distilled-squad": 512,
    "distilbert-base-cased": 512,
    "distilbert-base-cased-distilled-squad": 512,
    "distilbert-base-german-cased": 512,
    "distilbert-base-multilingual-cased": 512,
}


lowerCAmelCase			:							Optional[Any]     ={
    "distilbert-base-uncased": {"do_lower_case": True},
    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
    "distilbert-base-cased": {"do_lower_case": False},
    "distilbert-base-cased-distilled-squad": {"do_lower_case": False},
    "distilbert-base-german-cased": {"do_lower_case": False},
    "distilbert-base-multilingual-cased": {"do_lower_case": False},
}




class 	__snake_case	( __lowerCAmelCase		):





      '''simple docstring'''






      _snake_case    		=       VOCAB_FILES_NAMES
      _snake_case    		=       PRETRAINED_VOCAB_FILES_MAP
      _snake_case    		=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _snake_case    		=       PRETRAINED_INIT_CONFIGURATION
      _snake_case    		=       ['input_ids', 'attention_mask']
      _snake_case    		=       DistilBertTokenizer



      def __init__(					self     :				Tuple    ,  _UpperCamelCase     :				int=None    ,  _UpperCamelCase     :				int=None    ,  _UpperCamelCase     :				Any=True    ,  _UpperCamelCase     :				Optional[Any]="[UNK]"    ,  _UpperCamelCase     :				int="[SEP]"    ,  _UpperCamelCase     :				Optional[int]="[PAD]"    ,  _UpperCamelCase     :				List[Any]="[CLS]"    ,  _UpperCamelCase     :				int="[MASK]"    ,  _UpperCamelCase     :				Tuple=True    ,  _UpperCamelCase     :				Optional[Any]=None    ,  **_UpperCamelCase     :				str    ,  )				->str:


          """simple docstring"""


          super().__init__(
              _UpperCamelCase    ,  tokenizer_file=_UpperCamelCase    ,  do_lower_case=_UpperCamelCase    ,  unk_token=_UpperCamelCase    ,  sep_token=_UpperCamelCase    ,  pad_token=_UpperCamelCase    ,  cls_token=_UpperCamelCase    ,  mask_token=_UpperCamelCase    ,  tokenize_chinese_chars=_UpperCamelCase    ,  strip_accents=_UpperCamelCase    ,  **_UpperCamelCase    ,  )

          _lowerCamelCase     :		Optional[Any] 						=						json.loads(self.backend_tokenizer.normalizer.__getstate__())
          if (
              normalizer_state.get("""lowercase"""    ,  _UpperCamelCase) != do_lower_case
              or normalizer_state.get("""strip_accents"""    ,  _UpperCamelCase) != strip_accents
              or normalizer_state.get("""handle_chinese_chars"""    ,  _UpperCamelCase) != tokenize_chinese_chars
          ):
              _lowerCamelCase     :		Dict 						=						getattr(_UpperCamelCase    ,  normalizer_state.pop("""type"""))
              _lowerCamelCase     :		int 						=						do_lower_case
              _lowerCamelCase     :		Optional[Any] 						=						strip_accents
              _lowerCamelCase     :		Dict 						=						tokenize_chinese_chars
              _lowerCamelCase     :		Optional[int] 						=						normalizer_class(**_UpperCamelCase)

          _lowerCamelCase     :		List[str] 						=						do_lower_case



      def 	_SCREAMING_SNAKE_CASE    (					self     :				List[str]    ,  _UpperCamelCase     :				List[Any]    ,  _UpperCamelCase     :				str=None)				->str:


          """simple docstring"""


          _lowerCamelCase     :		Optional[Any] 						=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
              output += token_ids_a + [self.sep_token_id]

          return output



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Union[str, Any]    ,  _UpperCamelCase     :				List[int]    ,  _UpperCamelCase     :				Optional[List[int]] = None)				->List[int]:


          """simple docstring"""


          _lowerCamelCase     :		Union[str, Any] 						=						[self.sep_token_id]
          _lowerCamelCase     :		Optional[Any] 						=						[self.cls_token_id]
          if token_ids_a is None:
              return len(cls + token_ids_a + sep) * [0]
          return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]




      def 	_SCREAMING_SNAKE_CASE    (					self     :				Union[str, Any]    ,  _UpperCamelCase     :				str    ,  _UpperCamelCase     :				Optional[str] = None)				->Tuple[str]:


          """simple docstring"""


          _lowerCamelCase     :		Any 						=						self._tokenizer.model.save(_UpperCamelCase    ,  name=_UpperCamelCase)
          return tuple(_UpperCamelCase)





import unittest

from transformers.utils.backbone_utils import (
    BackboneMixin,
    get_aligned_output_features_output_indices,
    verify_out_features_out_indices,
)




class 	__snake_case	( unittest.TestCase		):





      '''simple docstring'''






      def 	_SCREAMING_SNAKE_CASE    (					self     :				Union[str, Any])				->Optional[Any]:


          """simple docstring"""


          _lowerCamelCase     :		Tuple 						=						["""a""", """b""", """c"""]

          # Defaults to last layer if both are None
          _lowerCamelCase						,							_lowerCamelCase     :		int 						=						get_aligned_output_features_output_indices(_UpperCamelCase    ,  _UpperCamelCase    ,  _UpperCamelCase)
          self.assertEqual(_UpperCamelCase    ,  ["""c"""])
          self.assertEqual(_UpperCamelCase    ,  [2])

          # Out indices set to match out features
          _lowerCamelCase						,							_lowerCamelCase     :		int 						=						get_aligned_output_features_output_indices(["""a""", """c"""]    ,  _UpperCamelCase    ,  _UpperCamelCase)
          self.assertEqual(_UpperCamelCase    ,  ["""a""", """c"""])
          self.assertEqual(_UpperCamelCase    ,  [0, 2])

          # Out features set to match out indices
          _lowerCamelCase						,							_lowerCamelCase     :		Tuple 						=						get_aligned_output_features_output_indices(_UpperCamelCase    ,  [0, 2]    ,  _UpperCamelCase)
          self.assertEqual(_UpperCamelCase    ,  ["""a""", """c"""])
          self.assertEqual(_UpperCamelCase    ,  [0, 2])

          # Out features selected from negative indices
          _lowerCamelCase						,							_lowerCamelCase     :		str 						=						get_aligned_output_features_output_indices(_UpperCamelCase    ,  [-3, -1]    ,  _UpperCamelCase)
          self.assertEqual(_UpperCamelCase    ,  ["""a""", """c"""])
          self.assertEqual(_UpperCamelCase    ,  [-3, -1])



      def 	_SCREAMING_SNAKE_CASE    (					self     :				int)				->int:


          """simple docstring"""


          with self.assertRaises(_UpperCamelCase):
              verify_out_features_out_indices(["""a""", """b"""]    ,  (0, 1)    ,  _UpperCamelCase)

          # Out features must be a list
          with self.assertRaises(_UpperCamelCase):
              verify_out_features_out_indices(("""a""", """b""")    ,  (0, 1)    ,  ["""a""", """b"""])

          # Out features must be a subset of stage names
          with self.assertRaises(_UpperCamelCase):
              verify_out_features_out_indices(["""a""", """b"""]    ,  (0, 1)    ,  ["""a"""])

          # Out indices must be a list or tuple
          with self.assertRaises(_UpperCamelCase):
              verify_out_features_out_indices(_UpperCamelCase    ,  0    ,  ["""a""", """b"""])

          # Out indices must be a subset of stage names
          with self.assertRaises(_UpperCamelCase):
              verify_out_features_out_indices(_UpperCamelCase    ,  (0, 1)    ,  ["""a"""])

          # Out features and out indices must be the same length
          with self.assertRaises(_UpperCamelCase):
              verify_out_features_out_indices(["""a""", """b"""]    ,  (0,)    ,  ["""a""", """b""", """c"""])

          # Out features should match out indices
          with self.assertRaises(_UpperCamelCase):
              verify_out_features_out_indices(["""a""", """b"""]    ,  (0, 2)    ,  ["""a""", """b""", """c"""])

          # Out features and out indices should be in order
          with self.assertRaises(_UpperCamelCase):
              verify_out_features_out_indices(["""b""", """a"""]    ,  (0, 1)    ,  ["""a""", """b"""])

          # Check passes with valid inputs
          verify_out_features_out_indices(["""a""", """b""", """d"""]    ,  (0, 1, -1)    ,  ["""a""", """b""", """c""", """d"""])




      def 	_SCREAMING_SNAKE_CASE    (					self     :				int)				->List[Any]:


          """simple docstring"""


          _lowerCamelCase     :		int 						=						BackboneMixin()

          _lowerCamelCase     :		Union[str, Any] 						=						["""a""", """b""", """c"""]
          _lowerCamelCase     :		Tuple 						=						["""a""", """c"""]
          _lowerCamelCase     :		List[Any] 						=						[0, 2]

          # Check that the output features and indices are set correctly
          self.assertEqual(backbone.out_features    ,  ["""a""", """c"""])
          self.assertEqual(backbone.out_indices    ,  [0, 2])

          # Check out features and indices are updated correctly
          _lowerCamelCase     :		str 						=						["""a""", """b"""]
          self.assertEqual(backbone.out_features    ,  ["""a""", """b"""])
          self.assertEqual(backbone.out_indices    ,  [0, 1])

          _lowerCamelCase     :		Optional[int] 						=						[-3, -1]
          self.assertEqual(backbone.out_features    ,  ["""a""", """c"""])
          self.assertEqual(backbone.out_indices    ,  [-3, -1])




