




'''simple docstring'''


import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation


UpperCAmelCase_ 		=      logging.get_logger(__name__)

UpperCAmelCase_ 		=      {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase_ 		=      {
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/tokenizer.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/tokenizer.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/tokenizer.json",
    },
}

UpperCAmelCase_ 		=      {
    "gpt2": 10_24,
    "gpt2-medium": 10_24,
    "gpt2-large": 10_24,
    "gpt2-xl": 10_24,
    "distilgpt2": 10_24,
}





class    __lowercase						(  __magic_name__ ):
   _a							=      VOCAB_FILES_NAMES
   _a							=      PRETRAINED_VOCAB_FILES_MAP
   _a							=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _a							=      ["""input_ids""", """attention_mask"""]
   _a							=      GPTaTokenizer







   def __init__(	self      ,							UpperCamelCase=None      ,							UpperCamelCase=None      ,							UpperCamelCase=None      ,							UpperCamelCase="<|endoftext|>"      ,							UpperCamelCase="<|endoftext|>"      ,							UpperCamelCase="<|endoftext|>"      ,							UpperCamelCase=False      ,							**UpperCamelCase      ,							)				->    Union[str, Any]:
     super().__init__(
         UpperCamelCase      ,							UpperCamelCase      ,							tokenizer_file=UpperCamelCase      ,							unk_token=UpperCamelCase      ,							bos_token=UpperCamelCase      ,							eos_token=UpperCamelCase      ,							add_prefix_space=UpperCamelCase      ,							**UpperCamelCase      ,							)

     __a			=					kwargs.pop('add_bos_token'      ,							UpperCamelCase      )

     __a			=					json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()      )
     if pre_tok_state.get('add_prefix_space'      ,							UpperCamelCase      ) != add_prefix_space:
       __a			=					getattr(UpperCamelCase      ,							pre_tok_state.pop('type'      )      )
       __a			=					add_prefix_space
       __a			=					pre_tok_class(**UpperCamelCase      )

     __a			=					add_prefix_space







   def 		UpperCamelCase__			(	self      ,							*UpperCamelCase      ,							**UpperCamelCase      )				->    BatchEncoding:
     __a			=					kwargs.get('is_split_into_words'      ,							UpperCamelCase      )
     assert self.add_prefix_space or not is_split_into_words, (
         f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
         "to use it with pretokenized inputs."
     )

     return super()._batch_encode_plus(*UpperCamelCase      ,							**UpperCamelCase      )







   def 		UpperCamelCase__			(	self      ,							*UpperCamelCase      ,							**UpperCamelCase      )				->    BatchEncoding:
     __a			=					kwargs.get('is_split_into_words'      ,							UpperCamelCase      )

     assert self.add_prefix_space or not is_split_into_words, (
         f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
         "to use it with pretokenized inputs."
     )

     return super()._encode_plus(*UpperCamelCase      ,							**UpperCamelCase      )







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase = None      )				->    Tuple[str]:
     __a			=					self._tokenizer.model.save(UpperCamelCase      ,							name=UpperCamelCase      )
     return tuple(UpperCamelCase      )







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      )				->    List[int]:
     __a			=					[]
     for is_user, text in conversation.iter_texts():
       input_ids.extend(self.encode(UpperCamelCase      ,							add_special_tokens=UpperCamelCase      ) + [self.eos_token_id]      )

     if len(UpperCamelCase      ) > self.model_max_length:
       __a			=					input_ids[-self.model_max_length :]
     return input_ids









'''simple docstring'''


from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCAmelCase_ 		=      logging.get_logger(__name__)

UpperCAmelCase_ 		=      {
    "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/config.json",
    "distilbert-base-uncased-distilled-squad": (
        "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/config.json"
    ),
    "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/config.json",
    "distilbert-base-cased-distilled-squad": (
        "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/config.json"
    ),
    "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/config.json",
    "distilbert-base-multilingual-cased": (
        "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/config.json"
    ),
    "distilbert-base-uncased-finetuned-sst-2-english": (
        "https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english/resolve/main/config.json"
    ),
}





class    __lowercase						(  __magic_name__ ):
   _a							=      """distilbert"""
   _a							=      {
       """hidden_size""": """dim""",
       """num_attention_heads""": """n_heads""",
       """num_hidden_layers""": """n_layers""",
   }







   def __init__(	self      ,							UpperCamelCase=3_0522      ,							UpperCamelCase=512      ,							UpperCamelCase=False      ,							UpperCamelCase=6      ,							UpperCamelCase=12      ,							UpperCamelCase=768      ,							UpperCamelCase=4 * 768      ,							UpperCamelCase=0.1      ,							UpperCamelCase=0.1      ,							UpperCamelCase="gelu"      ,							UpperCamelCase=0.02      ,							UpperCamelCase=0.1      ,							UpperCamelCase=0.2      ,							UpperCamelCase=0      ,							**UpperCamelCase      ,							)				->    int:
     __a			=					vocab_size
     __a			=					max_position_embeddings
     __a			=					sinusoidal_pos_embds
     __a			=					n_layers
     __a			=					n_heads
     __a			=					dim
     __a			=					hidden_dim
     __a			=					dropout
     __a			=					attention_dropout
     __a			=					activation
     __a			=					initializer_range
     __a			=					qa_dropout
     __a			=					seq_classif_dropout
     super().__init__(**UpperCamelCase      ,							pad_token_id=UpperCamelCase      )





class    __lowercase						(  __magic_name__ ):







   @property
   def 		UpperCamelCase__			(	self      )				->    Mapping[str, Mapping[int, str]]:
     if self.task == "multiple-choice":
       __a			=					{0: 'batch', 1: 'choice', 2: 'sequence'}
     else:
       __a			=					{0: 'batch', 1: 'sequence'}
     return OrderedDict(
         [
             ('input_ids', dynamic_axis),
             ('attention_mask', dynamic_axis),
         ]      )



