

'''simple docstring'''

from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin





class 							_snake_case (       __lowerCAmelCase	,  __lowerCAmelCase	,  __lowerCAmelCase						):
   _A       :     List[str]       =					[r"h\.\d+\.attn\.bias", r"h\.\d+\.attn\.masked_bias"]
   @register_to_config
   def __init__( self			:      Tuple     ,SCREAMING_SNAKE_CASE__			:      int     ,SCREAMING_SNAKE_CASE__			:      int     ,SCREAMING_SNAKE_CASE__			:      Optional[int] = None     ,SCREAMING_SNAKE_CASE__			:      int = 50_257     ,SCREAMING_SNAKE_CASE__			:      int = 1_024     ,SCREAMING_SNAKE_CASE__			:      int = 768     ,SCREAMING_SNAKE_CASE__			:      int = 12     ,SCREAMING_SNAKE_CASE__			:      int = 12     ,SCREAMING_SNAKE_CASE__			:      Optional[int] = None     ,SCREAMING_SNAKE_CASE__			:      str = "gelu_new"     ,SCREAMING_SNAKE_CASE__			:      float = 0.1     ,SCREAMING_SNAKE_CASE__			:      float = 0.1     ,SCREAMING_SNAKE_CASE__			:      float = 0.1     ,SCREAMING_SNAKE_CASE__			:      float = 1e-5     ,SCREAMING_SNAKE_CASE__			:      float = 0.02     ,SCREAMING_SNAKE_CASE__			:      bool = True     ,SCREAMING_SNAKE_CASE__			:      bool = True     ,SCREAMING_SNAKE_CASE__			:      bool = False     ,SCREAMING_SNAKE_CASE__			:      bool = False     ,):
     super().__init__()

     SCREAMING_SNAKE_CASE:int						      =					prefix_length

     if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
       raise ValueError(
           F'''`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and'''
           F''' `n_embd`: {n_embd} are not equal.'''							)

     SCREAMING_SNAKE_CASE:str						      =					prefix_inner_dim
     SCREAMING_SNAKE_CASE:Optional[int]						      =					prefix_hidden_dim

     SCREAMING_SNAKE_CASE:Optional[int]						      =					(
         nn.Linear(self.prefix_inner_dim     ,self.prefix_hidden_dim							)
         if self.prefix_hidden_dim is not None
         else nn.Identity()
     )
     SCREAMING_SNAKE_CASE:int						      =					(
         nn.Linear(self.prefix_hidden_dim     ,lowerCamelCase__							) if self.prefix_hidden_dim is not None else nn.Identity()
     )

     SCREAMING_SNAKE_CASE:Optional[Any]						      =					GPTaConfig(
         vocab_size=lowerCamelCase__     ,n_positions=lowerCamelCase__     ,n_embd=lowerCamelCase__     ,n_layer=lowerCamelCase__     ,n_head=lowerCamelCase__     ,n_inner=lowerCamelCase__     ,activation_function=lowerCamelCase__     ,resid_pdrop=lowerCamelCase__     ,embd_pdrop=lowerCamelCase__     ,attn_pdrop=lowerCamelCase__     ,layer_norm_epsilon=lowerCamelCase__     ,initializer_range=lowerCamelCase__     ,scale_attn_weights=lowerCamelCase__     ,use_cache=lowerCamelCase__     ,scale_attn_by_inverse_layer_idx=lowerCamelCase__     ,reorder_and_upcast_attn=lowerCamelCase__     ,)
     SCREAMING_SNAKE_CASE:Union[str, Any]						      =					GPTaLMHeadModel(lowerCamelCase__							)
   def     __UpperCamelCase						( self			:      Optional[int]     ,SCREAMING_SNAKE_CASE__			:      torch.Tensor     ,SCREAMING_SNAKE_CASE__			:      torch.Tensor     ,SCREAMING_SNAKE_CASE__			:      Optional[torch.Tensor] = None     ,SCREAMING_SNAKE_CASE__			:      Optional[torch.Tensor] = None     ,):
     SCREAMING_SNAKE_CASE:Optional[Any]						      =					self.transformer.transformer.wte(lowerCamelCase__							)
     SCREAMING_SNAKE_CASE:Union[str, Any]						      =					self.encode_prefix(lowerCamelCase__							)
     SCREAMING_SNAKE_CASE:Tuple						      =					self.decode_prefix(lowerCamelCase__							)
     SCREAMING_SNAKE_CASE:int						      =					torch.cat((prefix_embeds, embedding_text)     ,dim=1							)

     if labels is not None:
       SCREAMING_SNAKE_CASE:Tuple						      =					self.get_dummy_token(input_ids.shape[0]     ,input_ids.device							)
       SCREAMING_SNAKE_CASE:int						      =					torch.cat((dummy_token, input_ids)     ,dim=1							)
     SCREAMING_SNAKE_CASE:Tuple						      =					self.transformer(inputs_embeds=lowerCamelCase__     ,labels=lowerCamelCase__     ,attention_mask=lowerCamelCase__							)
     if self.prefix_hidden_dim is not None:
       return out, hidden
     else:
       return out
   def     __UpperCamelCase						( self			:      Any     ,SCREAMING_SNAKE_CASE__			:      int     ,SCREAMING_SNAKE_CASE__			:      torch.device							):
     return torch.zeros(lowerCamelCase__     ,self.prefix_length     ,dtype=torch.intaa     ,device=lowerCamelCase__							)
   def     __UpperCamelCase						( self			:      str     ,SCREAMING_SNAKE_CASE__			:      int							):
     return self.encode_prefix(lowerCamelCase__							)
   @torch.no_grad()
   def     __UpperCamelCase						( self			:      str     ,SCREAMING_SNAKE_CASE__			:      List[Any]     ,SCREAMING_SNAKE_CASE__			:      Union[str, Any]     ,SCREAMING_SNAKE_CASE__			:      Optional[Any]							):
     SCREAMING_SNAKE_CASE:List[str]						      =					torch.split(lowerCamelCase__     ,1     ,dim=0							)
     SCREAMING_SNAKE_CASE:List[Any]						      =					[]
     SCREAMING_SNAKE_CASE:Optional[int]						      =					[]
     for feature in features:
       SCREAMING_SNAKE_CASE:List[str]						      =					self.decode_prefix(feature.to(lowerCamelCase__							)							)  # back to the clip feature
       # Only support beam search for now
       SCREAMING_SNAKE_CASE:List[str]						      =					self.generate_beam(
           input_embeds=lowerCamelCase__     ,device=lowerCamelCase__     ,eos_token_id=lowerCamelCase__							)
       generated_tokens.append(output_tokens[0]							)
       generated_seq_lengths.append(seq_lengths[0]							)
     SCREAMING_SNAKE_CASE:Union[str, Any]						      =					torch.stack(lowerCamelCase__							)
     SCREAMING_SNAKE_CASE:str						      =					torch.stack(lowerCamelCase__							)
     return generated_tokens, generated_seq_lengths







   @torch.no_grad()
   def     __UpperCamelCase						( self			:      int     ,SCREAMING_SNAKE_CASE__			:      Optional[Any]=None     ,SCREAMING_SNAKE_CASE__			:      Tuple=None     ,SCREAMING_SNAKE_CASE__			:      Dict=None     ,SCREAMING_SNAKE_CASE__			:      int = 5     ,SCREAMING_SNAKE_CASE__			:      int = 67     ,SCREAMING_SNAKE_CASE__			:      float = 1.0     ,SCREAMING_SNAKE_CASE__			:      Optional[int] = None     ,):
     SCREAMING_SNAKE_CASE:Union[str, Any]						      =					eos_token_id
     SCREAMING_SNAKE_CASE:int						      =					None
     SCREAMING_SNAKE_CASE:Any						      =					None
     SCREAMING_SNAKE_CASE:int						      =					torch.ones(lowerCamelCase__     ,device=lowerCamelCase__     ,dtype=torch.int							)
     SCREAMING_SNAKE_CASE:Optional[Any]						      =					torch.zeros(lowerCamelCase__     ,device=lowerCamelCase__     ,dtype=torch.bool							)

     if input_embeds is not None:
       SCREAMING_SNAKE_CASE:Optional[int]						      =					input_embeds
     else:
       SCREAMING_SNAKE_CASE:str						      =					self.transformer.transformer.wte(lowerCamelCase__							)

     for i in range(lowerCamelCase__							):
       SCREAMING_SNAKE_CASE:Dict						      =					self.transformer(inputs_embeds=lowerCamelCase__							)
       SCREAMING_SNAKE_CASE:Any						      =					outputs.logits
       SCREAMING_SNAKE_CASE:Optional[int]						      =					logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
       SCREAMING_SNAKE_CASE:Optional[Any]						      =					logits.softmax(-1							).log()

       if scores is None:
         SCREAMING_SNAKE_CASE:Dict						      =					logits.topk(lowerCamelCase__     ,-1							)
         SCREAMING_SNAKE_CASE:List[Any]						      =					generated.expand(lowerCamelCase__     ,*generated.shape[1:]							)
         SCREAMING_SNAKE_CASE:Union[str, Any]						      =					next_tokens.permute(1     ,0							), scores.squeeze(0							)
         if tokens is None:
           SCREAMING_SNAKE_CASE:Dict						      =					next_tokens
         else:
           SCREAMING_SNAKE_CASE:Dict						      =					tokens.expand(lowerCamelCase__     ,*tokens.shape[1:]							)
           SCREAMING_SNAKE_CASE:Union[str, Any]						      =					torch.cat((tokens, next_tokens)     ,dim=1							)
       else:
         SCREAMING_SNAKE_CASE:List[Any]						      =					-float(np.inf							)
         SCREAMING_SNAKE_CASE:List[str]						      =					0
         SCREAMING_SNAKE_CASE:Union[str, Any]						      =					scores[:, None] + logits
         seq_lengths[~is_stopped] += 1
         SCREAMING_SNAKE_CASE:Tuple						      =					scores_sum / seq_lengths[:, None]
         SCREAMING_SNAKE_CASE:Optional[Any]						      =					scores_sum_average.view(-1							).topk(lowerCamelCase__     ,-1							)
         SCREAMING_SNAKE_CASE:Union[str, Any]						      =					next_tokens // scores_sum.shape[1]
         SCREAMING_SNAKE_CASE:Any						      =					seq_lengths[next_tokens_source]
         SCREAMING_SNAKE_CASE:Union[str, Any]						      =					next_tokens % scores_sum.shape[1]
         SCREAMING_SNAKE_CASE:Tuple						      =					next_tokens.unsqueeze(1							)
         SCREAMING_SNAKE_CASE:Any						      =					tokens[next_tokens_source]
         SCREAMING_SNAKE_CASE:Optional[Any]						      =					torch.cat((tokens, next_tokens)     ,dim=1							)
         SCREAMING_SNAKE_CASE:str						      =					generated[next_tokens_source]
         SCREAMING_SNAKE_CASE:Any						      =					scores_sum_average * seq_lengths
         SCREAMING_SNAKE_CASE:Any						      =					is_stopped[next_tokens_source]

       SCREAMING_SNAKE_CASE:List[str]						      =					self.transformer.transformer.wte(next_tokens.squeeze()							).view(generated.shape[0]     ,1     ,-1							)
       SCREAMING_SNAKE_CASE:str						      =					torch.cat((generated, next_token_embed)     ,dim=1							)
       SCREAMING_SNAKE_CASE:str						      =					is_stopped + next_tokens.eq(lowerCamelCase__							).squeeze()
       if is_stopped.all():
         break

     SCREAMING_SNAKE_CASE:Optional[Any]						      =					scores / seq_lengths
     SCREAMING_SNAKE_CASE:Optional[int]						      =					scores.argsort(descending=lowerCamelCase__							)
     # tokens tensors are already padded to max_seq_length
     SCREAMING_SNAKE_CASE:str						      =					[tokens[i] for i in order]
     SCREAMING_SNAKE_CASE:List[str]						      =					torch.stack(lowerCamelCase__     ,dim=0							)
     SCREAMING_SNAKE_CASE:Dict						      =					torch.tensor([seq_lengths[i] for i in order]     ,dtype=seq_lengths.dtype							)
     return output_texts, seq_lengths









'''simple docstring'''

import inspect
import re
from hashlib import shaaaa
from typing import Dict, List

from .arrow import arrow
from .audiofolder import audiofolder
from .csv import csv
from .imagefolder import imagefolder
from .json import json
from .pandas import pandas
from .parquet import parquet
from .sql import sql  # noqa F401
from .text import text





def    A_  ( snake_case ):
  SCREAMING_SNAKE_CASE:List[Any]						      =					[]
  for line in lines:
    SCREAMING_SNAKE_CASE:Any						      =					re.sub(r"#.*"      ,       ""      ,       snake_case )  # remove comments
    if line:
      filtered_lines.append(snake_case )
  SCREAMING_SNAKE_CASE:int						      =					"\n".join(snake_case )

  # Make a hash from all this code
  SCREAMING_SNAKE_CASE:str						      =					full_str.encode("utf-8" )
  return shaaaa(snake_case ).hexdigest()


# get importable module names and hash for caching
A_       						=							{
    "csv": (csv.__name__, _hash_python_lines(inspect.getsource(csv).splitlines())),
    "json": (json.__name__, _hash_python_lines(inspect.getsource(json).splitlines())),
    "pandas": (pandas.__name__, _hash_python_lines(inspect.getsource(pandas).splitlines())),
    "parquet": (parquet.__name__, _hash_python_lines(inspect.getsource(parquet).splitlines())),
    "arrow": (arrow.__name__, _hash_python_lines(inspect.getsource(arrow).splitlines())),
    "text": (text.__name__, _hash_python_lines(inspect.getsource(text).splitlines())),
    "imagefolder": (imagefolder.__name__, _hash_python_lines(inspect.getsource(imagefolder).splitlines())),
    "audiofolder": (audiofolder.__name__, _hash_python_lines(inspect.getsource(audiofolder).splitlines())),
}

# Used to infer the module to use based on the data files extensions
A_       						=							{
    ".csv": ("csv", {}),
    ".tsv": ("csv", {"sep": "\t"}),
    ".json": ("json", {}),
    ".jsonl": ("json", {}),
    ".parquet": ("parquet", {}),
    ".arrow": ("arrow", {}),
    ".txt": ("text", {}),
}
_EXTENSION_TO_MODULE.update({ext: ("imagefolder", {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ("imagefolder", {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext: ("audiofolder", {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ("audiofolder", {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
A_       						=							{"imagefolder", "audiofolder"}

# Used to filter data files based on extensions given a module name
A_       						=							{}
for _ext, (_module, _) in _EXTENSION_TO_MODULE.items():
    _MODULE_TO_EXTENSIONS.setdefault(_module, []).append(_ext)

_MODULE_TO_EXTENSIONS["imagefolder"].append(".zip")


_MODULE_TO_EXTENSIONS["audiofolder"].append(".zip")






