


'''simple docstring'''

import torch
from torch import nn


class       a__				( nn.Module      ):







				"""simple docstring"""



				def __init__(self       ,			__lowercase       ,			__lowercase       ,			__lowercase       ,			__lowercase       ,			__lowercase=1       ,			__lowercase=False		):
										super().__init__()

										__lowerCAmelCase							       =				n_token
										__lowerCAmelCase							       =				d_embed
										__lowerCAmelCase							       =				d_proj

										__lowerCAmelCase							       =				cutoffs + [n_token]
										__lowerCAmelCase							       =				[0] + self.cutoffs
										__lowerCAmelCase							       =				div_val

										__lowerCAmelCase							       =				self.cutoffs[0]
										__lowerCAmelCase							       =				len(self.cutoffs		) - 1
										__lowerCAmelCase							       =				self.shortlist_size + self.n_clusters

										if self.n_clusters > 0:
																__lowerCAmelCase							       =				nn.Parameter(torch.zeros(self.n_clusters       ,			self.d_embed		)		)
																__lowerCAmelCase							       =				nn.Parameter(torch.zeros(self.n_clusters		)		)

										__lowerCAmelCase							       =				nn.ModuleList()
										__lowerCAmelCase							       =				nn.ParameterList()

										if div_val == 1:
																for i in range(len(self.cutoffs		)		):
																						if d_proj != d_embed:
																												self.out_projs.append(nn.Parameter(torch.FloatTensor(__lowercase       ,			__lowercase		)		)		)
																						else:
																												self.out_projs.append(__lowercase		)

																self.out_layers.append(nn.Linear(__lowercase       ,			__lowercase		)		)
										else:
																for i in range(len(self.cutoffs		)		):
																						__lowerCAmelCase       ,					__lowerCAmelCase							       =				self.cutoff_ends[i], self.cutoff_ends[i + 1]
																						__lowerCAmelCase							       =				d_embed // (div_val**i)

																						self.out_projs.append(nn.Parameter(torch.FloatTensor(__lowercase       ,			__lowercase		)		)		)

																						self.out_layers.append(nn.Linear(__lowercase       ,			r_idx - l_idx		)		)

										__lowerCAmelCase							       =				keep_order




				def 					_snake_case  (self       ,			__lowercase       ,			__lowercase       ,			__lowercase       ,			__lowercase		):
										if proj is None:
																__lowerCAmelCase							       =				nn.functional.linear(__lowercase       ,			__lowercase       ,			bias=__lowercase		)
										else:
																# if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
																__lowerCAmelCase							       =				nn.functional.linear(__lowercase       ,			proj.t().contiguous()		)
																__lowerCAmelCase							       =				nn.functional.linear(__lowercase       ,			__lowercase       ,			bias=__lowercase		)
																# else:
																#     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
																#     if bias is not None:
																#         logit = logit + bias

										return logit




				def 					_snake_case  (self       ,			__lowercase       ,			__lowercase=None       ,			__lowercase=False		):

										if labels is not None:
																# Shift so that tokens < n predict n
																__lowerCAmelCase							       =				hidden[..., :-1, :].contiguous()
																__lowerCAmelCase							       =				labels[..., 1:].contiguous()
																__lowerCAmelCase							       =				hidden.view(-1       ,			hidden.size(-1		)		)
																__lowerCAmelCase							       =				labels.view(-1		)
																if hidden.size(0		) != labels.size(0		):
																						raise RuntimeError('''Input and labels should have the same size in the batch dimension.'''		)
										else:
																__lowerCAmelCase							       =				hidden.view(-1       ,			hidden.size(-1		)		)

										if self.n_clusters == 0:
																__lowerCAmelCase							       =				self._compute_logit(__lowercase       ,			self.out_layers[0].weight       ,			self.out_layers[0].bias       ,			self.out_projs[0]		)
																if labels is not None:
																						__lowerCAmelCase							       =				labels != -1_00
																						__lowerCAmelCase							       =				torch.zeros_like(__lowercase       ,			dtype=hidden.dtype       ,			device=hidden.device		)
																						__lowerCAmelCase							       =				(
																						    -nn.functional.log_softmax(__lowercase       ,			dim=-1		)[mask].gather(1       ,			labels[mask].unsqueeze(1		)		).squeeze(1		)
																						)
																else:
																						__lowerCAmelCase							       =				nn.functional.log_softmax(__lowercase       ,			dim=-1		)
										else:
																# construct weights and biases
																__lowerCAmelCase       ,					__lowerCAmelCase							       =				[], []
																for i in range(len(self.cutoffs		)		):
																						if self.div_val == 1:
																												__lowerCAmelCase       ,					__lowerCAmelCase							       =				self.cutoff_ends[i], self.cutoff_ends[i + 1]
																												__lowerCAmelCase							       =				self.out_layers[0].weight[l_idx:r_idx]
																												__lowerCAmelCase							       =				self.out_layers[0].bias[l_idx:r_idx]
																						else:
																												__lowerCAmelCase							       =				self.out_layers[i].weight
																												__lowerCAmelCase							       =				self.out_layers[i].bias

																						if i == 0:
																												__lowerCAmelCase							       =				torch.cat([weight_i, self.cluster_weight]       ,			dim=0		)
																												__lowerCAmelCase							       =				torch.cat([bias_i, self.cluster_bias]       ,			dim=0		)

																						weights.append(__lowercase		)
																						biases.append(__lowercase		)

																__lowerCAmelCase       ,					__lowerCAmelCase       ,					__lowerCAmelCase							       =				weights[0], biases[0], self.out_projs[0]

																__lowerCAmelCase							       =				self._compute_logit(__lowercase       ,			__lowercase       ,			__lowercase       ,			__lowercase		)
																__lowerCAmelCase							       =				nn.functional.log_softmax(__lowercase       ,			dim=1		)

																if labels is None:
																						__lowerCAmelCase							       =				hidden.new_empty((head_logit.size(0		), self.n_token)		)
																else:
																						__lowerCAmelCase							       =				torch.zeros_like(__lowercase       ,			dtype=hidden.dtype       ,			device=hidden.device		)

																__lowerCAmelCase							       =				0
																__lowerCAmelCase							       =				[0] + self.cutoffs
																for i in range(len(__lowercase		) - 1		):
																						__lowerCAmelCase       ,					__lowerCAmelCase							       =				cutoff_values[i], cutoff_values[i + 1]

																						if labels is not None:
																												__lowerCAmelCase							       =				(labels >= l_idx) & (labels < r_idx)
																												__lowerCAmelCase							       =				mask_i.nonzero().squeeze()

																												if indices_i.numel() == 0:
																																		continue

																												__lowerCAmelCase							       =				labels.index_select(0       ,			__lowercase		) - l_idx
																												__lowerCAmelCase							       =				head_logprob.index_select(0       ,			__lowercase		)
																												__lowerCAmelCase							       =				hidden.index_select(0       ,			__lowercase		)
																						else:
																												__lowerCAmelCase							       =				hidden

																						if i == 0:
																												if labels is not None:
																																		__lowerCAmelCase							       =				head_logprob_i.gather(1       ,			target_i[:, None]		).squeeze(1		)
																												else:
																																		__lowerCAmelCase							       =				head_logprob[:, : self.cutoffs[0]]
																						else:
																												__lowerCAmelCase       ,					__lowerCAmelCase       ,					__lowerCAmelCase							       =				weights[i], biases[i], self.out_projs[i]

																												__lowerCAmelCase							       =				self._compute_logit(__lowercase       ,			__lowercase       ,			__lowercase       ,			__lowercase		)
																												__lowerCAmelCase							       =				nn.functional.log_softmax(__lowercase       ,			dim=1		)
																												__lowerCAmelCase							       =				self.cutoffs[0] + i - 1  # No probability for the head cluster
																												if labels is not None:
																																		__lowerCAmelCase							       =				head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
																																		    1       ,			target_i[:, None]		).squeeze(1		)
																												else:
																																		__lowerCAmelCase							       =				head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
																																		__lowerCAmelCase							       =				logprob_i

																						if labels is not None:
																												if (hasattr(self       ,			'''keep_order'''		) and self.keep_order) or keep_order:
																																		out.index_copy_(0       ,			__lowercase       ,			-logprob_i		)
																												else:
																																		out[offset : offset + logprob_i.size(0		)].copy_(-logprob_i		)
																												offset += logprob_i.size(0		)

										return out




				def 					_snake_case  (self       ,			__lowercase		):
										if self.n_clusters == 0:
																__lowerCAmelCase							       =				self._compute_logit(__lowercase       ,			self.out_layers[0].weight       ,			self.out_layers[0].bias       ,			self.out_projs[0]		)
																return nn.functional.log_softmax(__lowercase       ,			dim=-1		)
										else:
																# construct weights and biases
																__lowerCAmelCase       ,					__lowerCAmelCase							       =				[], []
																for i in range(len(self.cutoffs		)		):
																						if self.div_val == 1:
																												__lowerCAmelCase       ,					__lowerCAmelCase							       =				self.cutoff_ends[i], self.cutoff_ends[i + 1]
																												__lowerCAmelCase							       =				self.out_layers[0].weight[l_idx:r_idx]
																												__lowerCAmelCase							       =				self.out_layers[0].bias[l_idx:r_idx]
																						else:
																												__lowerCAmelCase							       =				self.out_layers[i].weight
																												__lowerCAmelCase							       =				self.out_layers[i].bias

																						if i == 0:
																												__lowerCAmelCase							       =				torch.cat([weight_i, self.cluster_weight]       ,			dim=0		)
																												__lowerCAmelCase							       =				torch.cat([bias_i, self.cluster_bias]       ,			dim=0		)

																						weights.append(__lowercase		)
																						biases.append(__lowercase		)

																__lowerCAmelCase       ,					__lowerCAmelCase       ,					__lowerCAmelCase							       =				weights[0], biases[0], self.out_projs[0]
																__lowerCAmelCase							       =				self._compute_logit(__lowercase       ,			__lowercase       ,			__lowercase       ,			__lowercase		)

																__lowerCAmelCase							       =				hidden.new_empty((head_logit.size(0		), self.n_token)		)
																__lowerCAmelCase							       =				nn.functional.log_softmax(__lowercase       ,			dim=1		)

																__lowerCAmelCase							       =				[0] + self.cutoffs
																for i in range(len(__lowercase		) - 1		):
																						__lowerCAmelCase       ,					__lowerCAmelCase							       =				cutoff_values[i], cutoff_values[i + 1]

																						if i == 0:
																												__lowerCAmelCase							       =				head_logprob[:, : self.cutoffs[0]]
																						else:
																												__lowerCAmelCase       ,					__lowerCAmelCase       ,					__lowerCAmelCase							       =				weights[i], biases[i], self.out_projs[i]

																												__lowerCAmelCase							       =				self._compute_logit(__lowercase       ,			__lowercase       ,			__lowercase       ,			__lowercase		)
																												__lowerCAmelCase							       =				nn.functional.log_softmax(__lowercase       ,			dim=1		)

																												__lowerCAmelCase							       =				head_logprob[:, -i] + tail_logprob_i
																												__lowerCAmelCase							       =				logprob_i

																return out





'''simple docstring'''

from .glue import GlueDataset, GlueDataTrainingArguments
from .language_modeling import (
    LineByLineTextDataset,
    LineByLineWithRefDataset,
    LineByLineWithSOPTextDataset,
    TextDataset,
    TextDatasetForNextSentencePrediction,
)
from .squad import SquadDataset, SquadDataTrainingArguments

