





'''simple docstring'''

from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin







class __a (   _snake_case,   _snake_case,   _snake_case			):
   __UpperCamelCase   : List[Any]	      =	[R'h\.\d+\.attn\.bias', R'h\.\d+\.attn\.masked_bias']





   @register_to_config
   def __init__(					self				:			List[str]				,lowerCamelCase				:			int				,lowerCamelCase				:			int				,lowerCamelCase				:			Optional[int] = None				,lowerCamelCase				:			int = 5_0257				,lowerCamelCase				:			int = 1024				,lowerCamelCase				:			int = 768				,lowerCamelCase				:			int = 12				,lowerCamelCase				:			int = 12				,lowerCamelCase				:			Optional[int] = None				,lowerCamelCase				:			str = "gelu_new"				,lowerCamelCase				:			float = 0.1				,lowerCamelCase				:			float = 0.1				,lowerCamelCase				:			float = 0.1				,lowerCamelCase				:			float = 1E-5				,lowerCamelCase				:			float = 0.02				,lowerCamelCase				:			bool = True				,lowerCamelCase				:			bool = True				,lowerCamelCase				:			bool = False				,lowerCamelCase				:			bool = False				,):

      '''simple docstring'''

      super().__init__()

      __SCREAMING_SNAKE_CASE      							=       prefix_length

      if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
         raise ValueError(
             f"""`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and"""
             f""" `n_embd`: {n_embd} are not equal."""    )

      __SCREAMING_SNAKE_CASE      							=       prefix_inner_dim
      __SCREAMING_SNAKE_CASE      							=       prefix_hidden_dim

      __SCREAMING_SNAKE_CASE      							=       (
          nn.Linear(self.prefix_inner_dim				,self.prefix_hidden_dim    )
          if self.prefix_hidden_dim is not None
          else nn.Identity()
      )
      __SCREAMING_SNAKE_CASE      							=       (
          nn.Linear(self.prefix_hidden_dim				,lowerCamelCase    ) if self.prefix_hidden_dim is not None else nn.Identity()
      )

      __SCREAMING_SNAKE_CASE      							=       GPTaConfig(
          vocab_size=lowerCamelCase				,n_positions=lowerCamelCase				,n_embd=lowerCamelCase				,n_layer=lowerCamelCase				,n_head=lowerCamelCase				,n_inner=lowerCamelCase				,activation_function=lowerCamelCase				,resid_pdrop=lowerCamelCase				,embd_pdrop=lowerCamelCase				,attn_pdrop=lowerCamelCase				,layer_norm_epsilon=lowerCamelCase				,initializer_range=lowerCamelCase				,scale_attn_weights=lowerCamelCase				,use_cache=lowerCamelCase				,scale_attn_by_inverse_layer_idx=lowerCamelCase				,reorder_and_upcast_attn=lowerCamelCase				,)
      __SCREAMING_SNAKE_CASE      							=       GPTaLMHeadModel(lowerCamelCase    )





   def      UpperCAmelCase__     (					self				:			Union[str, Any]				,lowerCamelCase				:			torch.Tensor				,lowerCamelCase				:			torch.Tensor				,lowerCamelCase				:			Optional[torch.Tensor] = None				,lowerCamelCase				:			Optional[torch.Tensor] = None				,):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       self.transformer.transformer.wte(lowerCamelCase    )
      __SCREAMING_SNAKE_CASE      							=       self.encode_prefix(lowerCamelCase    )
      __SCREAMING_SNAKE_CASE      							=       self.decode_prefix(lowerCamelCase    )
      __SCREAMING_SNAKE_CASE      							=       torch.cat((prefix_embeds, embedding_text)				,dim=1    )

      if labels is not None:
         __SCREAMING_SNAKE_CASE      							=       self.get_dummy_token(input_ids.shape[0]				,input_ids.device    )
         __SCREAMING_SNAKE_CASE      							=       torch.cat((dummy_token, input_ids)				,dim=1    )
      __SCREAMING_SNAKE_CASE      							=       self.transformer(inputs_embeds=lowerCamelCase				,labels=lowerCamelCase				,attention_mask=lowerCamelCase    )
      if self.prefix_hidden_dim is not None:
         return out, hidden
      else:
         return out





   def      UpperCAmelCase__     (					self				:			int				,lowerCamelCase				:			int				,lowerCamelCase				:			torch.device    ):

      '''simple docstring'''

      return torch.zeros(lowerCamelCase				,self.prefix_length				,dtype=torch.intaa				,device=lowerCamelCase    )





   def      UpperCAmelCase__     (					self				:			str				,lowerCamelCase				:			Optional[Any]    ):

      '''simple docstring'''

      return self.encode_prefix(lowerCamelCase    )





   @torch.no_grad()
   def      UpperCAmelCase__     (					self				:			Any				,lowerCamelCase				:			int				,lowerCamelCase				:			Union[str, Any]				,lowerCamelCase				:			str    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       torch.split(lowerCamelCase				,1				,dim=0    )
      __SCREAMING_SNAKE_CASE      							=       []
      __SCREAMING_SNAKE_CASE      							=       []
      for feature in features:
         __SCREAMING_SNAKE_CASE      							=       self.decode_prefix(feature.to(lowerCamelCase    )    )  # back to the clip feature
         # Only support beam search for now
         __SCREAMING_SNAKE_CASE							,     __SCREAMING_SNAKE_CASE      							=       self.generate_beam(
             input_embeds=lowerCamelCase				,device=lowerCamelCase				,eos_token_id=lowerCamelCase    )
         generated_tokens.append(output_tokens[0]    )
         generated_seq_lengths.append(seq_lengths[0]    )
      __SCREAMING_SNAKE_CASE      							=       torch.stack(lowerCamelCase    )
      __SCREAMING_SNAKE_CASE      							=       torch.stack(lowerCamelCase    )
      return generated_tokens, generated_seq_lengths





   @torch.no_grad()
   def      UpperCAmelCase__     (					self				:			str				,lowerCamelCase				:			int=None				,lowerCamelCase				:			Union[str, Any]=None				,lowerCamelCase				:			List[str]=None				,lowerCamelCase				:			int = 5				,lowerCamelCase				:			int = 67				,lowerCamelCase				:			float = 1.0				,lowerCamelCase				:			Optional[int] = None				,):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       eos_token_id
      __SCREAMING_SNAKE_CASE      							=       None
      __SCREAMING_SNAKE_CASE      							=       None
      __SCREAMING_SNAKE_CASE      							=       torch.ones(lowerCamelCase				,device=lowerCamelCase				,dtype=torch.int    )
      __SCREAMING_SNAKE_CASE      							=       torch.zeros(lowerCamelCase				,device=lowerCamelCase				,dtype=torch.bool    )

      if input_embeds is not None:
         __SCREAMING_SNAKE_CASE      							=       input_embeds
      else:
         __SCREAMING_SNAKE_CASE      							=       self.transformer.transformer.wte(lowerCamelCase    )

      for i in range(lowerCamelCase    ):
         __SCREAMING_SNAKE_CASE      							=       self.transformer(inputs_embeds=lowerCamelCase    )
         __SCREAMING_SNAKE_CASE      							=       outputs.logits
         __SCREAMING_SNAKE_CASE      							=       logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
         __SCREAMING_SNAKE_CASE      							=       logits.softmax(-1    ).log()

         if scores is None:
            __SCREAMING_SNAKE_CASE							,     __SCREAMING_SNAKE_CASE      							=       logits.topk(lowerCamelCase				,-1    )
            __SCREAMING_SNAKE_CASE      							=       generated.expand(lowerCamelCase				,*generated.shape[1:]    )
            __SCREAMING_SNAKE_CASE							,     __SCREAMING_SNAKE_CASE      							=       next_tokens.permute(1				,0    ), scores.squeeze(0    )
            if tokens is None:
               __SCREAMING_SNAKE_CASE      							=       next_tokens
            else:
               __SCREAMING_SNAKE_CASE      							=       tokens.expand(lowerCamelCase				,*tokens.shape[1:]    )
               __SCREAMING_SNAKE_CASE      							=       torch.cat((tokens, next_tokens)				,dim=1    )
         else:
            __SCREAMING_SNAKE_CASE      							=       -float(np.inf    )
            __SCREAMING_SNAKE_CASE      							=       0
            __SCREAMING_SNAKE_CASE      							=       scores[:, None] + logits
            seq_lengths[~is_stopped] += 1
            __SCREAMING_SNAKE_CASE      							=       scores_sum / seq_lengths[:, None]
            __SCREAMING_SNAKE_CASE							,     __SCREAMING_SNAKE_CASE      							=       scores_sum_average.view(-1    ).topk(lowerCamelCase				,-1    )
            __SCREAMING_SNAKE_CASE      							=       next_tokens // scores_sum.shape[1]
            __SCREAMING_SNAKE_CASE      							=       seq_lengths[next_tokens_source]
            __SCREAMING_SNAKE_CASE      							=       next_tokens % scores_sum.shape[1]
            __SCREAMING_SNAKE_CASE      							=       next_tokens.unsqueeze(1    )
            __SCREAMING_SNAKE_CASE      							=       tokens[next_tokens_source]
            __SCREAMING_SNAKE_CASE      							=       torch.cat((tokens, next_tokens)				,dim=1    )
            __SCREAMING_SNAKE_CASE      							=       generated[next_tokens_source]
            __SCREAMING_SNAKE_CASE      							=       scores_sum_average * seq_lengths
            __SCREAMING_SNAKE_CASE      							=       is_stopped[next_tokens_source]

         __SCREAMING_SNAKE_CASE      							=       self.transformer.transformer.wte(next_tokens.squeeze()    ).view(generated.shape[0]				,1				,-1    )
         __SCREAMING_SNAKE_CASE      							=       torch.cat((generated, next_token_embed)				,dim=1    )
         __SCREAMING_SNAKE_CASE      							=       is_stopped + next_tokens.eq(lowerCamelCase    ).squeeze()
         if is_stopped.all():
            break

      __SCREAMING_SNAKE_CASE      							=       scores / seq_lengths
      __SCREAMING_SNAKE_CASE      							=       scores.argsort(descending=lowerCamelCase    )
      # tokens tensors are already padded to max_seq_length
      __SCREAMING_SNAKE_CASE      							=       [tokens[i] for i in order]
      __SCREAMING_SNAKE_CASE      							=       torch.stack(lowerCamelCase				,dim=0    )
      __SCREAMING_SNAKE_CASE      							=       torch.tensor([seq_lengths[i] for i in order]				,dtype=seq_lengths.dtype    )
      return output_texts, seq_lengths












'''simple docstring'''

from .dependency_versions_table import deps
from .utils.versions import require_version, require_version_core


# define which module versions we always want to check at run time
# (usually the ones defined in `install_requires` in setup.py)
#
# order specific notes:
# - tqdm must be checked before tokenizers

a        =						[
    "python",
    "tqdm",
    "regex",
    "requests",
    "packaging",
    "filelock",
    "numpy",
    "tokenizers",
    "huggingface-hub",
    "safetensors",
    "accelerate",
    "pyyaml",
]

for pkg in pkgs_to_check_at_runtime:
    if pkg in deps:
        if pkg == "tokenizers":
            # must be loaded here, or else tqdm check may fail
            from .utils import is_tokenizers_available

            if not is_tokenizers_available():
                continue  # not required, check version only if installed
        elif pkg == "accelerate":
            # must be loaded here, or else tqdm check may fail
            from .utils import is_accelerate_available

            # Maybe switch to is_torch_available in the future here so that Accelerate is hard dep of
            # Transformers with PyTorch
            if not is_accelerate_available():
                continue  # not required, check version only if installed

        require_version_core(deps[pkg])
    else:
        raise ValueError(F'''can\'t find {pkg} in {deps.keys()}, check dependency_versions_table.py''')

def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase=None      )						->					Optional[Any]:

   '''simple docstring'''







   require_version(deps[pkg]							,	__UpperCAmelCase      )





