


'''simple docstring'''

from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin


class       a__				( __A							,						__A							,						__A      ):







				"""simple docstring"""



				__UpperCamelCase    :		List[Any]	=	[r'h\.\d+\.attn\.bias', r'h\.\d+\.attn\.masked_bias']




				@register_to_config
				def __init__(self       ,			__lowercase       ,			__lowercase       ,			__lowercase = None       ,			__lowercase = 5_02_57       ,			__lowercase = 10_24       ,			__lowercase = 7_68       ,			__lowercase = 12       ,			__lowercase = 12       ,			__lowercase = None       ,			__lowercase = "gelu_new"       ,			__lowercase = 0.1       ,			__lowercase = 0.1       ,			__lowercase = 0.1       ,			__lowercase = 1e-5       ,			__lowercase = 0.0_2       ,			__lowercase = True       ,			__lowercase = True       ,			__lowercase = False       ,			__lowercase = False       ,			):
										super().__init__()

										__lowerCAmelCase							       =				prefix_length

										if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
																raise ValueError(
																    F"""`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and"""
																    F""" `n_embd`: {n_embd} are not equal."""		)

										__lowerCAmelCase							       =				prefix_inner_dim
										__lowerCAmelCase							       =				prefix_hidden_dim

										__lowerCAmelCase							       =				(
										    nn.Linear(self.prefix_inner_dim       ,			self.prefix_hidden_dim		)
										    if self.prefix_hidden_dim is not None
										    else nn.Identity()
										)
										__lowerCAmelCase							       =				(
										    nn.Linear(self.prefix_hidden_dim       ,			__lowercase		) if self.prefix_hidden_dim is not None else nn.Identity()
										)

										__lowerCAmelCase							       =				GPTaConfig(
										    vocab_size=__lowercase       ,			n_positions=__lowercase       ,			n_embd=__lowercase       ,			n_layer=__lowercase       ,			n_head=__lowercase       ,			n_inner=__lowercase       ,			activation_function=__lowercase       ,			resid_pdrop=__lowercase       ,			embd_pdrop=__lowercase       ,			attn_pdrop=__lowercase       ,			layer_norm_epsilon=__lowercase       ,			initializer_range=__lowercase       ,			scale_attn_weights=__lowercase       ,			use_cache=__lowercase       ,			scale_attn_by_inverse_layer_idx=__lowercase       ,			reorder_and_upcast_attn=__lowercase       ,			)
										__lowerCAmelCase							       =				GPTaLMHeadModel(__lowercase		)




				def 					_snake_case  (self       ,			__lowercase       ,			__lowercase       ,			__lowercase = None       ,			__lowercase = None       ,			):
										__lowerCAmelCase							       =				self.transformer.transformer.wte(__lowercase		)
										__lowerCAmelCase							       =				self.encode_prefix(__lowercase		)
										__lowerCAmelCase							       =				self.decode_prefix(__lowercase		)
										__lowerCAmelCase							       =				torch.cat((prefix_embeds, embedding_text)       ,			dim=1		)

										if labels is not None:
																__lowerCAmelCase							       =				self.get_dummy_token(input_ids.shape[0]       ,			input_ids.device		)
																__lowerCAmelCase							       =				torch.cat((dummy_token, input_ids)       ,			dim=1		)
										__lowerCAmelCase							       =				self.transformer(inputs_embeds=__lowercase       ,			labels=__lowercase       ,			attention_mask=__lowercase		)
										if self.prefix_hidden_dim is not None:
																return out, hidden
										else:
																return out




				def 					_snake_case  (self       ,			__lowercase       ,			__lowercase		):
										return torch.zeros(__lowercase       ,			self.prefix_length       ,			dtype=torch.intaa       ,			device=__lowercase		)




				def 					_snake_case  (self       ,			__lowercase		):
										return self.encode_prefix(__lowercase		)




				@torch.no_grad()
				def 					_snake_case  (self       ,			__lowercase       ,			__lowercase       ,			__lowercase		):
										__lowerCAmelCase							       =				torch.split(__lowercase       ,			1       ,			dim=0		)
										__lowerCAmelCase							       =				[]
										__lowerCAmelCase							       =				[]
										for feature in features:
																__lowerCAmelCase							       =				self.decode_prefix(feature.to(__lowercase		)		)  # back to the clip feature
																# Only support beam search for now
																__lowerCAmelCase       ,					__lowerCAmelCase							       =				self.generate_beam(
																    input_embeds=__lowercase       ,			device=__lowercase       ,			eos_token_id=__lowercase		)
																generated_tokens.append(output_tokens[0]		)
																generated_seq_lengths.append(seq_lengths[0]		)
										__lowerCAmelCase							       =				torch.stack(__lowercase		)
										__lowerCAmelCase							       =				torch.stack(__lowercase		)
										return generated_tokens, generated_seq_lengths




				@torch.no_grad()
				def 					_snake_case  (self       ,			__lowercase=None       ,			__lowercase=None       ,			__lowercase=None       ,			__lowercase = 5       ,			__lowercase = 67       ,			__lowercase = 1.0       ,			__lowercase = None       ,			):
										__lowerCAmelCase							       =				eos_token_id
										__lowerCAmelCase							       =				None
										__lowerCAmelCase							       =				None
										__lowerCAmelCase							       =				torch.ones(__lowercase       ,			device=__lowercase       ,			dtype=torch.int		)
										__lowerCAmelCase							       =				torch.zeros(__lowercase       ,			device=__lowercase       ,			dtype=torch.bool		)

										if input_embeds is not None:
																__lowerCAmelCase							       =				input_embeds
										else:
																__lowerCAmelCase							       =				self.transformer.transformer.wte(__lowercase		)

										for i in range(__lowercase		):
																__lowerCAmelCase							       =				self.transformer(inputs_embeds=__lowercase		)
																__lowerCAmelCase							       =				outputs.logits
																__lowerCAmelCase							       =				logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
																__lowerCAmelCase							       =				logits.softmax(-1		).log()

																if scores is None:
																						__lowerCAmelCase       ,					__lowerCAmelCase							       =				logits.topk(__lowercase       ,			-1		)
																						__lowerCAmelCase							       =				generated.expand(__lowercase       ,			*generated.shape[1:]		)
																						__lowerCAmelCase       ,					__lowerCAmelCase							       =				next_tokens.permute(1       ,			0		), scores.squeeze(0		)
																						if tokens is None:
																												__lowerCAmelCase							       =				next_tokens
																						else:
																												__lowerCAmelCase							       =				tokens.expand(__lowercase       ,			*tokens.shape[1:]		)
																												__lowerCAmelCase							       =				torch.cat((tokens, next_tokens)       ,			dim=1		)
																else:
																						__lowerCAmelCase							       =				-float(np.inf		)
																						__lowerCAmelCase							       =				0
																						__lowerCAmelCase							       =				scores[:, None] + logits
																						seq_lengths[~is_stopped] += 1
																						__lowerCAmelCase							       =				scores_sum / seq_lengths[:, None]
																						__lowerCAmelCase       ,					__lowerCAmelCase							       =				scores_sum_average.view(-1		).topk(__lowercase       ,			-1		)
																						__lowerCAmelCase							       =				next_tokens // scores_sum.shape[1]
																						__lowerCAmelCase							       =				seq_lengths[next_tokens_source]
																						__lowerCAmelCase							       =				next_tokens % scores_sum.shape[1]
																						__lowerCAmelCase							       =				next_tokens.unsqueeze(1		)
																						__lowerCAmelCase							       =				tokens[next_tokens_source]
																						__lowerCAmelCase							       =				torch.cat((tokens, next_tokens)       ,			dim=1		)
																						__lowerCAmelCase							       =				generated[next_tokens_source]
																						__lowerCAmelCase							       =				scores_sum_average * seq_lengths
																						__lowerCAmelCase							       =				is_stopped[next_tokens_source]

																__lowerCAmelCase							       =				self.transformer.transformer.wte(next_tokens.squeeze()		).view(generated.shape[0]       ,			1       ,			-1		)
																__lowerCAmelCase							       =				torch.cat((generated, next_token_embed)       ,			dim=1		)
																__lowerCAmelCase							       =				is_stopped + next_tokens.eq(__lowercase		).squeeze()
																if is_stopped.all():
																						break

										__lowerCAmelCase							       =				scores / seq_lengths
										__lowerCAmelCase							       =				scores.argsort(descending=__lowercase		)
										# tokens tensors are already padded to max_seq_length
										__lowerCAmelCase							       =				[tokens[i] for i in order]
										__lowerCAmelCase							       =				torch.stack(__lowercase       ,			dim=0		)
										__lowerCAmelCase							       =				torch.tensor([seq_lengths[i] for i in order]       ,			dtype=seq_lengths.dtype		)
										return output_texts, seq_lengths





'''simple docstring'''

import sys

from .dependency_versions_table import deps
from .utils.versions import require_version, require_version_core


# define which module versions we always want to check at run time
# (usually the ones defined in `install_requires` in setup.py)
#
# order specific notes:
# - tqdm must be checked before tokenizers

_UpperCAmelCase				: Tuple							    =				"""python tqdm regex requests packaging filelock numpy tokenizers""".split()
if sys.version_info < (3, 7):
	pkgs_to_check_at_runtime.append("""dataclasses""")
if sys.version_info < (3, 8):
	pkgs_to_check_at_runtime.append("""importlib_metadata""")

for pkg in pkgs_to_check_at_runtime:
	if pkg in deps:
		if pkg == "tokenizers":
			# must be loaded here, or else tqdm check may fail
			from .utils import is_tokenizers_available

			if not is_tokenizers_available():
				continue  # not required, check version only if installed

		require_version_core(deps[pkg])
	else:
		raise ValueError(f"""can't find {pkg} in {deps.keys()}, check dependency_versions_table.py""")

def 							__magic_name__(						lowerCamelCase,				lowerCamelCase=None):
						require_version(deps[pkg],				lowerCamelCase)

