




"""simple docstring"""





from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin



class      a__				(	__magic_name__							,    __magic_name__							,    __magic_name__   ):
			lowercase_    =	[R"h\.\d+\.attn\.bias", R"h\.\d+\.attn\.masked_bias"]


			@register_to_config
			def __init__(      self      :    List[str]      ,  UpperCamelCase_      :    int      ,  UpperCamelCase_      :    int      ,  UpperCamelCase_      :    Optional[int] = None      ,  UpperCamelCase_      :    int = 50257      ,  UpperCamelCase_      :    int = 1024      ,  UpperCamelCase_      :    int = 768      ,  UpperCamelCase_      :    int = 12      ,  UpperCamelCase_      :    int = 12      ,  UpperCamelCase_      :    Optional[int] = None      ,  UpperCamelCase_      :    str = "gelu_new"      ,  UpperCamelCase_      :    float = 0.1      ,  UpperCamelCase_      :    float = 0.1      ,  UpperCamelCase_      :    float = 0.1      ,  UpperCamelCase_      :    float = 1e-5      ,  UpperCamelCase_      :    float = 0.02      ,  UpperCamelCase_      :    bool = True      ,  UpperCamelCase_      :    bool = True      ,  UpperCamelCase_      :    bool = False      ,  UpperCamelCase_      :    bool = False      ,  ):


						"""simple docstring"""




						super().__init__()

						__UpperCAmelCase	: str		=					prefix_length

						if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
									raise ValueError(
									    F"`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and"
									    F" `n_embd`: {n_embd} are not equal.")

						__UpperCAmelCase	: Any		=					prefix_inner_dim
						__UpperCAmelCase	: str		=					prefix_hidden_dim

						__UpperCAmelCase	: List[Any]		=					(
						    nn.Linear(self.prefix_inner_dim      ,  self.prefix_hidden_dim)
						    if self.prefix_hidden_dim is not None
						    else nn.Identity()
						)
						__UpperCAmelCase	: Any		=					(
						    nn.Linear(self.prefix_hidden_dim      ,  UpperCamelCase_) if self.prefix_hidden_dim is not None else nn.Identity()
						)

						__UpperCAmelCase	: str		=					GPTaConfig(
						    vocab_size=UpperCamelCase_      ,  n_positions=UpperCamelCase_      ,  n_embd=UpperCamelCase_      ,  n_layer=UpperCamelCase_      ,  n_head=UpperCamelCase_      ,  n_inner=UpperCamelCase_      ,  activation_function=UpperCamelCase_      ,  resid_pdrop=UpperCamelCase_      ,  embd_pdrop=UpperCamelCase_      ,  attn_pdrop=UpperCamelCase_      ,  layer_norm_epsilon=UpperCamelCase_      ,  initializer_range=UpperCamelCase_      ,  scale_attn_weights=UpperCamelCase_      ,  use_cache=UpperCamelCase_      ,  scale_attn_by_inverse_layer_idx=UpperCamelCase_      ,  reorder_and_upcast_attn=UpperCamelCase_      ,  )
						__UpperCAmelCase	: Union[str, Any]		=					GPTaLMHeadModel(UpperCamelCase_)


			def       a_     (      self      :    Any      ,  UpperCamelCase_      :    torch.Tensor      ,  UpperCamelCase_      :    torch.Tensor      ,  UpperCamelCase_      :    Optional[torch.Tensor] = None      ,  UpperCamelCase_      :    Optional[torch.Tensor] = None      ,  ):


						"""simple docstring"""




						__UpperCAmelCase	: List[str]		=					self.transformer.transformer.wte(UpperCamelCase_)
						__UpperCAmelCase	: Union[str, Any]		=					self.encode_prefix(UpperCamelCase_)
						__UpperCAmelCase	: Dict		=					self.decode_prefix(UpperCamelCase_)
						__UpperCAmelCase	: Optional[int]		=					torch.cat((prefix_embeds, embedding_text)      ,  dim=1)

						if labels is not None:
									__UpperCAmelCase	: List[str]		=					self.get_dummy_token(input_ids.shape[0]      ,  input_ids.device)
									__UpperCAmelCase	: Dict		=					torch.cat((dummy_token, input_ids)      ,  dim=1)
						__UpperCAmelCase	: Union[str, Any]		=					self.transformer(inputs_embeds=UpperCamelCase_      ,  labels=UpperCamelCase_      ,  attention_mask=UpperCamelCase_)
						if self.prefix_hidden_dim is not None:
									return out, hidden
						else:
									return out


			def       a_     (      self      :    Tuple      ,  UpperCamelCase_      :    int      ,  UpperCamelCase_      :    torch.device):


						"""simple docstring"""




						return torch.zeros(UpperCamelCase_      ,  self.prefix_length      ,  dtype=torch.intaa      ,  device=UpperCamelCase_)


			def       a_     (      self      :    Union[str, Any]      ,  UpperCamelCase_      :    Tuple):


						"""simple docstring"""




						return self.encode_prefix(UpperCamelCase_)


			@torch.no_grad()
			def       a_     (      self      :    Dict      ,  UpperCamelCase_      :    Optional[Any]      ,  UpperCamelCase_      :    Tuple      ,  UpperCamelCase_      :    List[Any]):


						"""simple docstring"""




						__UpperCAmelCase	: Any		=					torch.split(UpperCamelCase_      ,  1      ,  dim=0)
						__UpperCAmelCase	: str		=					[]
						__UpperCAmelCase	: List[Any]		=					[]
						for feature in features:
									__UpperCAmelCase	: Optional[int]		=					self.decode_prefix(feature.to(UpperCamelCase_))  # back to the clip feature
									# Only support beam search for now
									__UpperCAmelCase	,     __UpperCAmelCase	: Dict		=					self.generate_beam(
									    input_embeds=UpperCamelCase_      ,  device=UpperCamelCase_      ,  eos_token_id=UpperCamelCase_)
									generated_tokens.append(output_tokens[0])
									generated_seq_lengths.append(seq_lengths[0])
						__UpperCAmelCase	: Dict		=					torch.stack(UpperCamelCase_)
						__UpperCAmelCase	: Any		=					torch.stack(UpperCamelCase_)
						return generated_tokens, generated_seq_lengths


			@torch.no_grad()
			def       a_     (      self      :    List[Any]      ,  UpperCamelCase_      :    int=None      ,  UpperCamelCase_      :    str=None      ,  UpperCamelCase_      :    Optional[int]=None      ,  UpperCamelCase_      :    int = 5      ,  UpperCamelCase_      :    int = 67      ,  UpperCamelCase_      :    float = 1.0      ,  UpperCamelCase_      :    Optional[int] = None      ,  ):


						"""simple docstring"""




						__UpperCAmelCase	: int		=					eos_token_id
						__UpperCAmelCase	: List[Any]		=					None
						__UpperCAmelCase	: Optional[Any]		=					None
						__UpperCAmelCase	: Optional[int]		=					torch.ones(UpperCamelCase_      ,  device=UpperCamelCase_      ,  dtype=torch.int)
						__UpperCAmelCase	: Dict		=					torch.zeros(UpperCamelCase_      ,  device=UpperCamelCase_      ,  dtype=torch.bool)

						if input_embeds is not None:
									__UpperCAmelCase	: Optional[int]		=					input_embeds
						else:
									__UpperCAmelCase	: Optional[Any]		=					self.transformer.transformer.wte(UpperCamelCase_)

						for i in range(UpperCamelCase_):
									__UpperCAmelCase	: Any		=					self.transformer(inputs_embeds=UpperCamelCase_)
									__UpperCAmelCase	: Dict		=					outputs.logits
									__UpperCAmelCase	: List[str]		=					logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
									__UpperCAmelCase	: Any		=					logits.softmax(-1).log()

									if scores is None:
												__UpperCAmelCase	,     __UpperCAmelCase	: List[str]		=					logits.topk(UpperCamelCase_      ,  -1)
												__UpperCAmelCase	: Dict		=					generated.expand(UpperCamelCase_      ,  *generated.shape[1:])
												__UpperCAmelCase	,     __UpperCAmelCase	: Any		=					next_tokens.permute(1      ,  0), scores.squeeze(0)
												if tokens is None:
															__UpperCAmelCase	: Tuple		=					next_tokens
												else:
															__UpperCAmelCase	: Optional[int]		=					tokens.expand(UpperCamelCase_      ,  *tokens.shape[1:])
															__UpperCAmelCase	: Any		=					torch.cat((tokens, next_tokens)      ,  dim=1)
									else:
												__UpperCAmelCase	: Dict		=					-float(np.inf)
												__UpperCAmelCase	: Any		=					0
												__UpperCAmelCase	: Tuple		=					scores[:, None] + logits
												seq_lengths[~is_stopped] += 1
												__UpperCAmelCase	: int		=					scores_sum / seq_lengths[:, None]
												__UpperCAmelCase	,     __UpperCAmelCase	: int		=					scores_sum_average.view(-1).topk(UpperCamelCase_      ,  -1)
												__UpperCAmelCase	: Optional[int]		=					next_tokens // scores_sum.shape[1]
												__UpperCAmelCase	: Optional[int]		=					seq_lengths[next_tokens_source]
												__UpperCAmelCase	: Optional[Any]		=					next_tokens % scores_sum.shape[1]
												__UpperCAmelCase	: List[Any]		=					next_tokens.unsqueeze(1)
												__UpperCAmelCase	: Optional[Any]		=					tokens[next_tokens_source]
												__UpperCAmelCase	: Optional[int]		=					torch.cat((tokens, next_tokens)      ,  dim=1)
												__UpperCAmelCase	: str		=					generated[next_tokens_source]
												__UpperCAmelCase	: Any		=					scores_sum_average * seq_lengths
												__UpperCAmelCase	: Optional[int]		=					is_stopped[next_tokens_source]

									__UpperCAmelCase	: Optional[Any]		=					self.transformer.transformer.wte(next_tokens.squeeze()).view(generated.shape[0]      ,  1      ,  -1)
									__UpperCAmelCase	: List[Any]		=					torch.cat((generated, next_token_embed)      ,  dim=1)
									__UpperCAmelCase	: int		=					is_stopped + next_tokens.eq(UpperCamelCase_).squeeze()
									if is_stopped.all():
												break

						__UpperCAmelCase	: Optional[int]		=					scores / seq_lengths
						__UpperCAmelCase	: Any		=					scores.argsort(descending=UpperCamelCase_)
						# tokens tensors are already padded to max_seq_length
						__UpperCAmelCase	: int		=					[tokens[i] for i in order]
						__UpperCAmelCase	: str		=					torch.stack(UpperCamelCase_      ,  dim=0)
						__UpperCAmelCase	: Optional[int]		=					torch.tensor([seq_lengths[i] for i in order]      ,  dtype=seq_lengths.dtype)
						return output_texts, seq_lengths








"""simple docstring"""





import json
import logging
import os
import socket

import git
import numpy as np
import torch


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - PID: %(process)d -  %(message)s""",
    datefmt="""%m/%d/%Y %H:%M:%S""",
    level=logging.INFO,
)
A   =       logging.getLogger(__name__)

def 					_UpperCamelCase    (			UpperCamelCase							)			->			Any:


			"""simple docstring"""
			__UpperCAmelCase	: Optional[Any]		=					git.Repo(search_parent_directories=UpperCamelCase							)
			__UpperCAmelCase	: Any		=					{
			    "repo_id": str(UpperCamelCase							),
			    "repo_sha": str(repo.head.object.hexsha							),
			    "repo_branch": str(repo.active_branch							),
			}

			with open(os.path.join(UpperCamelCase				,       "git_log.json"							)				,       "w"							) as f:
						json.dump(UpperCamelCase				,       UpperCamelCase				,       indent=4							)

def 					_UpperCamelCase    (			UpperCamelCase							)			->			str:


			"""simple docstring"""
			if params.n_gpu <= 0:
						__UpperCAmelCase	: str		=					0
						__UpperCAmelCase	: Dict		=					-1
						__UpperCAmelCase	: Tuple		=					True
						__UpperCAmelCase	: List[str]		=					False
						return

			assert torch.cuda.is_available()

			logger.info("Initializing GPUs"							)
			if params.n_gpu > 1:
						assert params.local_rank != -1

						__UpperCAmelCase	: Optional[int]		=					int(os.environ["WORLD_SIZE"]							)
						__UpperCAmelCase	: Union[str, Any]		=					int(os.environ["N_GPU_NODE"]							)
						__UpperCAmelCase	: Optional[int]		=					int(os.environ["RANK"]							)

						# number of nodes / node ID
						__UpperCAmelCase	: int		=					params.world_size // params.n_gpu_per_node
						__UpperCAmelCase	: Optional[int]		=					params.global_rank // params.n_gpu_per_node
						__UpperCAmelCase	: Tuple		=					True

						assert params.n_nodes == int(os.environ["N_NODES"]							)
						assert params.node_id == int(os.environ["NODE_RANK"]							)

			# local job (single GPU)
			else:
						assert params.local_rank == -1

						__UpperCAmelCase	: Union[str, Any]		=					1
						__UpperCAmelCase	: Dict		=					0
						__UpperCAmelCase	: List[Any]		=					0
						__UpperCAmelCase	: Tuple		=					0
						__UpperCAmelCase	: Any		=					1
						__UpperCAmelCase	: str		=					1
						__UpperCAmelCase	: Dict		=					False

			# sanity checks
			assert params.n_nodes >= 1
			assert 0 <= params.node_id < params.n_nodes
			assert 0 <= params.local_rank <= params.global_rank < params.world_size
			assert params.world_size == params.n_nodes * params.n_gpu_per_node

			# define whether this is the master process / if we are in multi-node distributed mode
			__UpperCAmelCase	: Union[str, Any]		=					params.node_id == 0 and params.local_rank == 0
			__UpperCAmelCase	: Dict		=					params.n_nodes > 1

			# summary
			__UpperCAmelCase	: Any		=					f"--- Global rank: {params.global_rank} - "
			logger.info(PREFIX + "Number of nodes: %i" % params.n_nodes							)
			logger.info(PREFIX + "Node ID        : %i" % params.node_id							)
			logger.info(PREFIX + "Local rank     : %i" % params.local_rank							)
			logger.info(PREFIX + "World size     : %i" % params.world_size							)
			logger.info(PREFIX + "GPUs per node  : %i" % params.n_gpu_per_node							)
			logger.info(PREFIX + "Master         : %s" % str(params.is_master							)							)
			logger.info(PREFIX + "Multi-node     : %s" % str(params.multi_node							)							)
			logger.info(PREFIX + "Multi-GPU      : %s" % str(params.multi_gpu							)							)
			logger.info(PREFIX + "Hostname       : %s" % socket.gethostname()							)

			# set GPU device
			torch.cuda.set_device(params.local_rank							)

			# initialize multi-GPU
			if params.multi_gpu:
						logger.info("Initializing PyTorch distributed"							)
						torch.distributed.init_process_group(
						    init_method="env://"				,       backend="nccl"				,       )





def 					_UpperCamelCase    (			UpperCamelCase							)			->			Tuple:


			"""simple docstring"""
			np.random.seed(args.seed							)
			torch.manual_seed(args.seed							)
			if args.n_gpu > 0:
						torch.cuda.manual_seed_all(args.seed							)


