





import torch
from torch import nn






class       __A					(					nn.Module				):

			'''simple docstring'''







			def __init__(					self ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase=1 ,						__lowerCAmelCase=False							):



				'''simple docstring'''







				super().__init__()

				lowerCamelCase__              =	n_token
				lowerCamelCase__              =	d_embed
				lowerCamelCase__              =	d_proj

				lowerCamelCase__              =	cutoffs + [n_token]
				lowerCamelCase__              =	[0] + self.cutoffs
				lowerCamelCase__              =	div_val

				lowerCamelCase__              =	self.cutoffs[0]
				lowerCamelCase__              =	len(self.cutoffs							) - 1
				lowerCamelCase__              =	self.shortlist_size + self.n_clusters

				if self.n_clusters > 0:
					lowerCamelCase__              =	nn.Parameter(torch.zeros(self.n_clusters ,						self.d_embed							)							)
					lowerCamelCase__              =	nn.Parameter(torch.zeros(self.n_clusters							)							)

				lowerCamelCase__              =	nn.ModuleList()
				lowerCamelCase__              =	nn.ParameterList()

				if div_val == 1:
					for i in range(len(self.cutoffs							)							):
						if d_proj != d_embed:
							self.out_projs.append(nn.Parameter(torch.FloatTensor(__lowerCAmelCase ,						__lowerCAmelCase							)							)							)
						else:
							self.out_projs.append(__lowerCAmelCase							)

					self.out_layers.append(nn.Linear(__lowerCAmelCase ,						__lowerCAmelCase							)							)
				else:
					for i in range(len(self.cutoffs							)							):
						lowerCamelCase__       ,			lowerCamelCase__              =	self.cutoff_ends[i], self.cutoff_ends[i + 1]
						lowerCamelCase__              =	d_embed // (div_val**i)

						self.out_projs.append(nn.Parameter(torch.FloatTensor(__lowerCAmelCase ,						__lowerCAmelCase							)							)							)

						self.out_layers.append(nn.Linear(__lowerCAmelCase ,						r_idx - l_idx							)							)

				lowerCamelCase__              =	keep_order





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase							):



				'''simple docstring'''







				if proj is None:
					lowerCamelCase__              =	nn.functional.linear(__lowerCAmelCase ,						__lowerCAmelCase ,						bias=__lowerCAmelCase							)
				else:
					# if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
					lowerCamelCase__              =	nn.functional.linear(__lowerCAmelCase ,						proj.t().contiguous()							)
					lowerCamelCase__              =	nn.functional.linear(__lowerCAmelCase ,						__lowerCAmelCase ,						bias=__lowerCAmelCase							)
					# else:
					#     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
					#     if bias is not None:
					#         logit = logit + bias

				return logit





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase ,						__lowerCAmelCase=None ,						__lowerCAmelCase=False							):



				'''simple docstring'''







				if labels is not None:
					# Shift so that tokens < n predict n
					lowerCamelCase__              =	hidden[..., :-1, :].contiguous()
					lowerCamelCase__              =	labels[..., 1:].contiguous()
					lowerCamelCase__              =	hidden.view(-1 ,						hidden.size(-1							)							)
					lowerCamelCase__              =	labels.view(-1							)
					if hidden.size(0							) != labels.size(0							):
						raise RuntimeError('''Input and labels should have the same size in the batch dimension.'''							)
				else:
					lowerCamelCase__              =	hidden.view(-1 ,						hidden.size(-1							)							)

				if self.n_clusters == 0:
					lowerCamelCase__              =	self._compute_logit(__lowerCAmelCase ,						self.out_layers[0].weight ,						self.out_layers[0].bias ,						self.out_projs[0]							)
					if labels is not None:
						lowerCamelCase__              =	labels != -1_0_0
						lowerCamelCase__              =	torch.zeros_like(__lowerCAmelCase ,						dtype=hidden.dtype ,						device=hidden.device							)
						lowerCamelCase__              =	(
						    -nn.functional.log_softmax(__lowerCAmelCase ,						dim=-1							)[mask].gather(1 ,						labels[mask].unsqueeze(1							)							).squeeze(1							)
						)
					else:
						lowerCamelCase__              =	nn.functional.log_softmax(__lowerCAmelCase ,						dim=-1							)
				else:
					# construct weights and biases
					lowerCamelCase__       ,			lowerCamelCase__              =	[], []
					for i in range(len(self.cutoffs							)							):
						if self.div_val == 1:
							lowerCamelCase__       ,			lowerCamelCase__              =	self.cutoff_ends[i], self.cutoff_ends[i + 1]
							lowerCamelCase__              =	self.out_layers[0].weight[l_idx:r_idx]
							lowerCamelCase__              =	self.out_layers[0].bias[l_idx:r_idx]
						else:
							lowerCamelCase__              =	self.out_layers[i].weight
							lowerCamelCase__              =	self.out_layers[i].bias

						if i == 0:
							lowerCamelCase__              =	torch.cat([weight_i, self.cluster_weight] ,						dim=0							)
							lowerCamelCase__              =	torch.cat([bias_i, self.cluster_bias] ,						dim=0							)

						weights.append(__lowerCAmelCase							)
						biases.append(__lowerCAmelCase							)

					lowerCamelCase__       ,			lowerCamelCase__       ,			lowerCamelCase__              =	weights[0], biases[0], self.out_projs[0]

					lowerCamelCase__              =	self._compute_logit(__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase							)
					lowerCamelCase__              =	nn.functional.log_softmax(__lowerCAmelCase ,						dim=1							)

					if labels is None:
						lowerCamelCase__              =	hidden.new_empty((head_logit.size(0							), self.n_token)							)
					else:
						lowerCamelCase__              =	torch.zeros_like(__lowerCAmelCase ,						dtype=hidden.dtype ,						device=hidden.device							)

					lowerCamelCase__              =	0
					lowerCamelCase__              =	[0] + self.cutoffs
					for i in range(len(__lowerCAmelCase							) - 1							):
						lowerCamelCase__       ,			lowerCamelCase__              =	cutoff_values[i], cutoff_values[i + 1]

						if labels is not None:
							lowerCamelCase__              =	(labels >= l_idx) & (labels < r_idx)
							lowerCamelCase__              =	mask_i.nonzero().squeeze()

							if indices_i.numel() == 0:
								continue

							lowerCamelCase__              =	labels.index_select(0 ,						__lowerCAmelCase							) - l_idx
							lowerCamelCase__              =	head_logprob.index_select(0 ,						__lowerCAmelCase							)
							lowerCamelCase__              =	hidden.index_select(0 ,						__lowerCAmelCase							)
						else:
							lowerCamelCase__              =	hidden

						if i == 0:
							if labels is not None:
								lowerCamelCase__              =	head_logprob_i.gather(1 ,						target_i[:, None]							).squeeze(1							)
							else:
								lowerCamelCase__              =	head_logprob[:, : self.cutoffs[0]]
						else:
							lowerCamelCase__       ,			lowerCamelCase__       ,			lowerCamelCase__              =	weights[i], biases[i], self.out_projs[i]

							lowerCamelCase__              =	self._compute_logit(__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase							)
							lowerCamelCase__              =	nn.functional.log_softmax(__lowerCAmelCase ,						dim=1							)
							lowerCamelCase__              =	self.cutoffs[0] + i - 1  # No probability for the head cluster
							if labels is not None:
								lowerCamelCase__              =	head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
								    1 ,						target_i[:, None]							).squeeze(1							)
							else:
								lowerCamelCase__              =	head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
								lowerCamelCase__              =	logprob_i

						if labels is not None:
							if (hasattr(self ,						'''keep_order'''							) and self.keep_order) or keep_order:
								out.index_copy_(0 ,						__lowerCAmelCase ,						-logprob_i							)
							else:
								out[offset : offset + logprob_i.size(0							)].copy_(-logprob_i							)
							offset += logprob_i.size(0							)

				return out





			def 	__lowerCamelCase       (					self ,						__lowerCAmelCase							):



				'''simple docstring'''







				if self.n_clusters == 0:
					lowerCamelCase__              =	self._compute_logit(__lowerCAmelCase ,						self.out_layers[0].weight ,						self.out_layers[0].bias ,						self.out_projs[0]							)
					return nn.functional.log_softmax(__lowerCAmelCase ,						dim=-1							)
				else:
					# construct weights and biases
					lowerCamelCase__       ,			lowerCamelCase__              =	[], []
					for i in range(len(self.cutoffs							)							):
						if self.div_val == 1:
							lowerCamelCase__       ,			lowerCamelCase__              =	self.cutoff_ends[i], self.cutoff_ends[i + 1]
							lowerCamelCase__              =	self.out_layers[0].weight[l_idx:r_idx]
							lowerCamelCase__              =	self.out_layers[0].bias[l_idx:r_idx]
						else:
							lowerCamelCase__              =	self.out_layers[i].weight
							lowerCamelCase__              =	self.out_layers[i].bias

						if i == 0:
							lowerCamelCase__              =	torch.cat([weight_i, self.cluster_weight] ,						dim=0							)
							lowerCamelCase__              =	torch.cat([bias_i, self.cluster_bias] ,						dim=0							)

						weights.append(__lowerCAmelCase							)
						biases.append(__lowerCAmelCase							)

					lowerCamelCase__       ,			lowerCamelCase__       ,			lowerCamelCase__              =	weights[0], biases[0], self.out_projs[0]
					lowerCamelCase__              =	self._compute_logit(__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase							)

					lowerCamelCase__              =	hidden.new_empty((head_logit.size(0							), self.n_token)							)
					lowerCamelCase__              =	nn.functional.log_softmax(__lowerCAmelCase ,						dim=1							)

					lowerCamelCase__              =	[0] + self.cutoffs
					for i in range(len(__lowerCAmelCase							) - 1							):
						lowerCamelCase__       ,			lowerCamelCase__              =	cutoff_values[i], cutoff_values[i + 1]

						if i == 0:
							lowerCamelCase__              =	head_logprob[:, : self.cutoffs[0]]
						else:
							lowerCamelCase__       ,			lowerCamelCase__       ,			lowerCamelCase__              =	weights[i], biases[i], self.out_projs[i]

							lowerCamelCase__              =	self._compute_logit(__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase ,						__lowerCAmelCase							)
							lowerCamelCase__              =	nn.functional.log_softmax(__lowerCAmelCase ,						dim=1							)

							lowerCamelCase__              =	head_logprob[:, -i] + tail_logprob_i
							lowerCamelCase__              =	logprob_i

					return out










_a						=   "\n# Transformers 설치 방법\n! pip install transformers datasets\n# 마지막 릴리스 대신 소스에서 설치하려면, 위 명령을 주석으로 바꾸고 아래 명령을 해제하세요.\n# ! pip install git+https://github.com/huggingface/transformers.git\n"

_a						=   [{"type": "code", "content": INSTALL_CONTENT}]
_a						=   {
    "{processor_class}": "FakeProcessorClass",
    "{model_class}": "FakeModelClass",
    "{object_class}": "FakeObjectClass",
}



