

'''simple docstring'''

import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation


A_       						=							logging.get_logger(__name__)

A_       						=							{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

A_       						=							{
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/tokenizer.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/tokenizer.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/tokenizer.json",
    },
}

A_       						=							{
    "gpt2": 10_24,
    "gpt2-medium": 10_24,
    "gpt2-large": 10_24,
    "gpt2-xl": 10_24,
    "distilgpt2": 10_24,
}





class 							_snake_case (       _a						):
   _A       :     str       =					VOCAB_FILES_NAMES
   _A       :     Optional[Any]       =					PRETRAINED_VOCAB_FILES_MAP
   _A       :     List[str]       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _A       :     List[str]       =					['''input_ids''', '''attention_mask''']
   _A       :     int       =					GPTaTokenizer
   def __init__( self			:      Union[str, Any]     ,SCREAMING_SNAKE_CASE__			:      Union[str, Any]=None     ,SCREAMING_SNAKE_CASE__			:      Tuple=None     ,SCREAMING_SNAKE_CASE__			:      Dict=None     ,SCREAMING_SNAKE_CASE__			:      Dict="<|endoftext|>"     ,SCREAMING_SNAKE_CASE__			:      Dict="<|endoftext|>"     ,SCREAMING_SNAKE_CASE__			:      Dict="<|endoftext|>"     ,SCREAMING_SNAKE_CASE__			:      List[str]=False     ,**SCREAMING_SNAKE_CASE__			:      Optional[int]     ,):
     super().__init__(
         SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__     ,tokenizer_file=SCREAMING_SNAKE_CASE__     ,unk_token=SCREAMING_SNAKE_CASE__     ,bos_token=SCREAMING_SNAKE_CASE__     ,eos_token=SCREAMING_SNAKE_CASE__     ,add_prefix_space=SCREAMING_SNAKE_CASE__     ,**SCREAMING_SNAKE_CASE__     ,)

     SCREAMING_SNAKE_CASE:Dict						      =					kwargs.pop("add_bos_token"     ,SCREAMING_SNAKE_CASE__							)

     SCREAMING_SNAKE_CASE:Union[str, Any]						      =					json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
     if pre_tok_state.get("add_prefix_space"     ,SCREAMING_SNAKE_CASE__							) != add_prefix_space:
       SCREAMING_SNAKE_CASE:Dict						      =					getattr(SCREAMING_SNAKE_CASE__     ,pre_tok_state.pop("type"							)							)
       SCREAMING_SNAKE_CASE:Dict						      =					add_prefix_space
       SCREAMING_SNAKE_CASE:Any						      =					pre_tok_class(**SCREAMING_SNAKE_CASE__							)

     SCREAMING_SNAKE_CASE:Optional[int]						      =					add_prefix_space
   def     __UpperCamelCase						( self			:      Dict     ,*SCREAMING_SNAKE_CASE__			:      List[str]     ,**SCREAMING_SNAKE_CASE__			:      Dict							):
     SCREAMING_SNAKE_CASE:List[str]						      =					kwargs.get("is_split_into_words"     ,SCREAMING_SNAKE_CASE__							)
     assert self.add_prefix_space or not is_split_into_words, (
         F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
         "to use it with pretokenized inputs."
     )

     return super()._batch_encode_plus(*SCREAMING_SNAKE_CASE__     ,**SCREAMING_SNAKE_CASE__							)
   def     __UpperCamelCase						( self			:      Union[str, Any]     ,*SCREAMING_SNAKE_CASE__			:      Optional[int]     ,**SCREAMING_SNAKE_CASE__			:      Tuple							):
     SCREAMING_SNAKE_CASE:Dict						      =					kwargs.get("is_split_into_words"     ,SCREAMING_SNAKE_CASE__							)

     assert self.add_prefix_space or not is_split_into_words, (
         F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
         "to use it with pretokenized inputs."
     )

     return super()._encode_plus(*SCREAMING_SNAKE_CASE__     ,**SCREAMING_SNAKE_CASE__							)
   def     __UpperCamelCase						( self			:      List[str]     ,SCREAMING_SNAKE_CASE__			:      str     ,SCREAMING_SNAKE_CASE__			:      Optional[str] = None							):
     SCREAMING_SNAKE_CASE:int						      =					self._tokenizer.model.save(SCREAMING_SNAKE_CASE__     ,name=SCREAMING_SNAKE_CASE__							)
     return tuple(SCREAMING_SNAKE_CASE__							)







   def     __UpperCamelCase						( self			:      Union[str, Any]     ,SCREAMING_SNAKE_CASE__			:      "Conversation"							):
     SCREAMING_SNAKE_CASE:Dict						      =					[]
     for is_user, text in conversation.iter_texts():
       input_ids.extend(self.encode(SCREAMING_SNAKE_CASE__     ,add_special_tokens=SCREAMING_SNAKE_CASE__							) + [self.eos_token_id]							)

     if len(SCREAMING_SNAKE_CASE__							) > self.model_max_length:
       SCREAMING_SNAKE_CASE:Any						      =					input_ids[-self.model_max_length :]
     return input_ids









'''simple docstring'''

from importlib import import_module

from .logging import get_logger


A_       						=							get_logger(__name__)





class 							_snake_case :
   def __init__( self			:      Dict     ,SCREAMING_SNAKE_CASE__			:      Any     ,SCREAMING_SNAKE_CASE__			:      Dict=None							):
     SCREAMING_SNAKE_CASE:Any						      =					attrs or []
     if module is not None:
       for key in module.__dict__:
         if key in attrs or not key.startswith("__"							):
           setattr(self     ,SCREAMING_SNAKE_CASE__     ,getattr(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)							)
     SCREAMING_SNAKE_CASE:Any						      =					module._original_module if isinstance(SCREAMING_SNAKE_CASE__     ,_PatchedModuleObj							) else module





class 							_snake_case :
   _A       :     Any       =					[]
   def __init__( self			:      Any     ,SCREAMING_SNAKE_CASE__			:      List[str]     ,SCREAMING_SNAKE_CASE__			:      str     ,SCREAMING_SNAKE_CASE__			:      str     ,SCREAMING_SNAKE_CASE__			:      Optional[Any]=None							):
     SCREAMING_SNAKE_CASE:Any						      =					obj
     SCREAMING_SNAKE_CASE:Any						      =					target
     SCREAMING_SNAKE_CASE:Union[str, Any]						      =					new
     SCREAMING_SNAKE_CASE:Any						      =					target.split("."							)[0]
     SCREAMING_SNAKE_CASE:str						      =					{}
     SCREAMING_SNAKE_CASE:List[Any]						      =					attrs or []
   def __enter__( self			:      Tuple							):
     *SCREAMING_SNAKE_CASE      ,    SCREAMING_SNAKE_CASE:int						      =					self.target.split("."							)

     # Patch modules:
     # it's used to patch attributes of submodules like "os.path.join";
     # in this case we need to patch "os" and "os.path"

     for i in range(len(SCREAMING_SNAKE_CASE__							)							):
       try:
         SCREAMING_SNAKE_CASE:int						      =					import_module(".".join(submodules[: i + 1]							)							)
       except ModuleNotFoundError:
         continue
       # We iterate over all the globals in self.obj in case we find "os" or "os.path"
       for attr in self.obj.__dir__():
         SCREAMING_SNAKE_CASE:str						      =					getattr(self.obj     ,SCREAMING_SNAKE_CASE__							)
         # We don't check for the name of the global, but rather if its value *is* "os" or "os.path".
         # This allows to patch renamed modules like "from os import path as ospath".
         if obj_attr is submodule or (
             (isinstance(SCREAMING_SNAKE_CASE__     ,_PatchedModuleObj							) and obj_attr._original_module is submodule)
         ):
           SCREAMING_SNAKE_CASE:Union[str, Any]						      =					obj_attr
           # patch at top level
           setattr(self.obj     ,SCREAMING_SNAKE_CASE__     ,_PatchedModuleObj(SCREAMING_SNAKE_CASE__     ,attrs=self.attrs							)							)
           SCREAMING_SNAKE_CASE:int						      =					getattr(self.obj     ,SCREAMING_SNAKE_CASE__							)
           # construct lower levels patches
           for key in submodules[i + 1 :]:
             setattr(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__     ,_PatchedModuleObj(getattr(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)     ,attrs=self.attrs							)							)
             SCREAMING_SNAKE_CASE:Dict						      =					getattr(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)
           # finally set the target attribute
           setattr(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__     ,self.new							)

        # Patch attribute itself:
        # it's used for builtins like "open",
        # and also to patch "os.path.join" we may also need to patch "join"
        # itself if it was imported as "from os.path import join".

     if submodules:  # if it's an attribute of a submodule like "os.path.join"
       try:
         SCREAMING_SNAKE_CASE:Any						      =					getattr(import_module(".".join(SCREAMING_SNAKE_CASE__							)							)     ,SCREAMING_SNAKE_CASE__							)
       except (AttributeError, ModuleNotFoundError):
         return
       # We iterate over all the globals in self.obj in case we find "os.path.join"
       for attr in self.obj.__dir__():
         # We don't check for the name of the global, but rather if its value *is* "os.path.join".
         # This allows to patch renamed attributes like "from os.path import join as pjoin".
         if getattr(self.obj     ,SCREAMING_SNAKE_CASE__							) is attr_value:
           SCREAMING_SNAKE_CASE:Union[str, Any]						      =					getattr(self.obj     ,SCREAMING_SNAKE_CASE__							)
           setattr(self.obj     ,SCREAMING_SNAKE_CASE__     ,self.new							)
     elif target_attr in globals()["__builtins__"]:  # if it'a s builtin like "open"
       SCREAMING_SNAKE_CASE:int						      =					globals()["__builtins__"][target_attr]
       setattr(self.obj     ,SCREAMING_SNAKE_CASE__     ,self.new							)
     else:
       raise RuntimeError(F'''Tried to patch attribute {target_attr} instead of a submodule.'''							)
   def __exit__( self			:      List[Any]     ,*SCREAMING_SNAKE_CASE__			:      Union[str, Any]							):
     for attr in list(self.original							):
       setattr(self.obj     ,SCREAMING_SNAKE_CASE__     ,self.original.pop(SCREAMING_SNAKE_CASE__							)							)
   def     __UpperCamelCase						( self			:      Optional[int]							):
     self.__enter__()
     self._active_patches.append(self							)







   def     __UpperCamelCase						( self			:      Dict							):
     try:
       self._active_patches.remove(self							)
     except ValueError:
       # If the patch hasn't been started this will fail
       return None

     return self.__exit__()






