
'''simple docstring'''



import torch
from torch import nn

class    _a				(nn.Module):





       """simple docstring"""




       def __init__(							self       ,							A__       ,							A__       ,							A__       ,							A__       ,							A__=1       ,							A__=False      )    ->			Any:
          super().__init__()

          _SCREAMING_SNAKE_CASE      =				n_token
          _SCREAMING_SNAKE_CASE      =				d_embed
          _SCREAMING_SNAKE_CASE      =				d_proj

          _SCREAMING_SNAKE_CASE      =				cutoffs + [n_token]
          _SCREAMING_SNAKE_CASE      =				[0] + self.cutoffs
          _SCREAMING_SNAKE_CASE      =				div_val

          _SCREAMING_SNAKE_CASE      =				self.cutoffs[0]
          _SCREAMING_SNAKE_CASE      =				len(self.cutoffs      ) - 1
          _SCREAMING_SNAKE_CASE      =				self.shortlist_size + self.n_clusters

          if self.n_clusters > 0:
             _SCREAMING_SNAKE_CASE      =				nn.Parameter(torch.zeros(self.n_clusters       ,							self.d_embed      )      )
             _SCREAMING_SNAKE_CASE      =				nn.Parameter(torch.zeros(self.n_clusters      )      )

          _SCREAMING_SNAKE_CASE      =				nn.ModuleList()
          _SCREAMING_SNAKE_CASE      =				nn.ParameterList()

          if div_val == 1:
             for i in range(len(self.cutoffs      )      ):
                if d_proj != d_embed:
                   self.out_projs.append(nn.Parameter(torch.FloatTensor(A__       ,							A__      )      )      )
                else:
                   self.out_projs.append(A__      )

             self.out_layers.append(nn.Linear(A__       ,							A__      )      )
          else:
             for i in range(len(self.cutoffs      )      ):
                _SCREAMING_SNAKE_CASE ,							_SCREAMING_SNAKE_CASE      =				self.cutoff_ends[i], self.cutoff_ends[i + 1]
                _SCREAMING_SNAKE_CASE      =				d_embed // (div_val**i)

                self.out_projs.append(nn.Parameter(torch.FloatTensor(A__       ,							A__      )      )      )

                self.out_layers.append(nn.Linear(A__       ,							r_idx - l_idx      )      )

          _SCREAMING_SNAKE_CASE      =				keep_order






       def        UpperCamelCase				(							self       ,							A__       ,							A__       ,							A__       ,							A__      )    ->			Tuple:
          if proj is None:
             _SCREAMING_SNAKE_CASE      =				nn.functional.linear(A__       ,							A__       ,							bias=A__      )
          else:
             # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
             _SCREAMING_SNAKE_CASE      =				nn.functional.linear(A__       ,							proj.t().contiguous()      )
             _SCREAMING_SNAKE_CASE      =				nn.functional.linear(A__       ,							A__       ,							bias=A__      )
             # else:
             #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
             #     if bias is not None:
             #         logit = logit + bias

          return logit






       def        UpperCamelCase				(							self       ,							A__       ,							A__=None       ,							A__=False      )    ->			Any:

          if labels is not None:
             # Shift so that tokens < n predict n
             _SCREAMING_SNAKE_CASE      =				hidden[..., :-1, :].contiguous()
             _SCREAMING_SNAKE_CASE      =				labels[..., 1:].contiguous()
             _SCREAMING_SNAKE_CASE      =				hidden.view(-1       ,							hidden.size(-1      )      )
             _SCREAMING_SNAKE_CASE      =				labels.view(-1      )
             if hidden.size(0      ) != labels.size(0      ):
                raise RuntimeError("""Input and labels should have the same size in the batch dimension."""      )
          else:
             _SCREAMING_SNAKE_CASE      =				hidden.view(-1       ,							hidden.size(-1      )      )

          if self.n_clusters == 0:
             _SCREAMING_SNAKE_CASE      =				self._compute_logit(A__       ,							self.out_layers[0].weight       ,							self.out_layers[0].bias       ,							self.out_projs[0]      )
             if labels is not None:
                _SCREAMING_SNAKE_CASE      =				labels != -1_00
                _SCREAMING_SNAKE_CASE      =				torch.zeros_like(A__       ,							dtype=hidden.dtype       ,							device=hidden.device      )
                _SCREAMING_SNAKE_CASE      =				(
                    -nn.functional.log_softmax(A__       ,							dim=-1      )[mask].gather(1       ,							labels[mask].unsqueeze(1      )      ).squeeze(1      )
                )
             else:
                _SCREAMING_SNAKE_CASE      =				nn.functional.log_softmax(A__       ,							dim=-1      )
          else:
             # construct weights and biases
             _SCREAMING_SNAKE_CASE ,							_SCREAMING_SNAKE_CASE      =				[], []
             for i in range(len(self.cutoffs      )      ):
                if self.div_val == 1:
                   _SCREAMING_SNAKE_CASE ,							_SCREAMING_SNAKE_CASE      =				self.cutoff_ends[i], self.cutoff_ends[i + 1]
                   _SCREAMING_SNAKE_CASE      =				self.out_layers[0].weight[l_idx:r_idx]
                   _SCREAMING_SNAKE_CASE      =				self.out_layers[0].bias[l_idx:r_idx]
                else:
                   _SCREAMING_SNAKE_CASE      =				self.out_layers[i].weight
                   _SCREAMING_SNAKE_CASE      =				self.out_layers[i].bias

                if i == 0:
                   _SCREAMING_SNAKE_CASE      =				torch.cat([weight_i, self.cluster_weight]       ,							dim=0      )
                   _SCREAMING_SNAKE_CASE      =				torch.cat([bias_i, self.cluster_bias]       ,							dim=0      )

                weights.append(A__      )
                biases.append(A__      )

             _SCREAMING_SNAKE_CASE ,							_SCREAMING_SNAKE_CASE ,							_SCREAMING_SNAKE_CASE      =				weights[0], biases[0], self.out_projs[0]

             _SCREAMING_SNAKE_CASE      =				self._compute_logit(A__       ,							A__       ,							A__       ,							A__      )
             _SCREAMING_SNAKE_CASE      =				nn.functional.log_softmax(A__       ,							dim=1      )

             if labels is None:
                _SCREAMING_SNAKE_CASE      =				hidden.new_empty((head_logit.size(0      ), self.n_token)      )
             else:
                _SCREAMING_SNAKE_CASE      =				torch.zeros_like(A__       ,							dtype=hidden.dtype       ,							device=hidden.device      )

             _SCREAMING_SNAKE_CASE      =				0
             _SCREAMING_SNAKE_CASE      =				[0] + self.cutoffs
             for i in range(len(A__      ) - 1      ):
                _SCREAMING_SNAKE_CASE ,							_SCREAMING_SNAKE_CASE      =				cutoff_values[i], cutoff_values[i + 1]

                if labels is not None:
                   _SCREAMING_SNAKE_CASE      =				(labels >= l_idx) & (labels < r_idx)
                   _SCREAMING_SNAKE_CASE      =				mask_i.nonzero().squeeze()

                   if indices_i.numel() == 0:
                      continue

                   _SCREAMING_SNAKE_CASE      =				labels.index_select(0       ,							A__      ) - l_idx
                   _SCREAMING_SNAKE_CASE      =				head_logprob.index_select(0       ,							A__      )
                   _SCREAMING_SNAKE_CASE      =				hidden.index_select(0       ,							A__      )
                else:
                   _SCREAMING_SNAKE_CASE      =				hidden

                if i == 0:
                   if labels is not None:
                      _SCREAMING_SNAKE_CASE      =				head_logprob_i.gather(1       ,							target_i[:, None]      ).squeeze(1      )
                   else:
                      _SCREAMING_SNAKE_CASE      =				head_logprob[:, : self.cutoffs[0]]
                else:
                   _SCREAMING_SNAKE_CASE ,							_SCREAMING_SNAKE_CASE ,							_SCREAMING_SNAKE_CASE      =				weights[i], biases[i], self.out_projs[i]

                   _SCREAMING_SNAKE_CASE      =				self._compute_logit(A__       ,							A__       ,							A__       ,							A__      )
                   _SCREAMING_SNAKE_CASE      =				nn.functional.log_softmax(A__       ,							dim=1      )
                   _SCREAMING_SNAKE_CASE      =				self.cutoffs[0] + i - 1  # No probability for the head cluster
                   if labels is not None:
                      _SCREAMING_SNAKE_CASE      =				head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                          1       ,							target_i[:, None]      ).squeeze(1      )
                   else:
                      _SCREAMING_SNAKE_CASE      =				head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
                      _SCREAMING_SNAKE_CASE      =				logprob_i

                if labels is not None:
                   if (hasattr(self       ,							"""keep_order"""      ) and self.keep_order) or keep_order:
                      out.index_copy_(0       ,							A__       ,							-logprob_i      )
                   else:
                      out[offset : offset + logprob_i.size(0      )].copy_(-logprob_i      )
                   offset += logprob_i.size(0      )

          return out






       def        UpperCamelCase				(							self       ,							A__      )    ->			Dict:
          if self.n_clusters == 0:
             _SCREAMING_SNAKE_CASE      =				self._compute_logit(A__       ,							self.out_layers[0].weight       ,							self.out_layers[0].bias       ,							self.out_projs[0]      )
             return nn.functional.log_softmax(A__       ,							dim=-1      )
          else:
             # construct weights and biases
             _SCREAMING_SNAKE_CASE ,							_SCREAMING_SNAKE_CASE      =				[], []
             for i in range(len(self.cutoffs      )      ):
                if self.div_val == 1:
                   _SCREAMING_SNAKE_CASE ,							_SCREAMING_SNAKE_CASE      =				self.cutoff_ends[i], self.cutoff_ends[i + 1]
                   _SCREAMING_SNAKE_CASE      =				self.out_layers[0].weight[l_idx:r_idx]
                   _SCREAMING_SNAKE_CASE      =				self.out_layers[0].bias[l_idx:r_idx]
                else:
                   _SCREAMING_SNAKE_CASE      =				self.out_layers[i].weight
                   _SCREAMING_SNAKE_CASE      =				self.out_layers[i].bias

                if i == 0:
                   _SCREAMING_SNAKE_CASE      =				torch.cat([weight_i, self.cluster_weight]       ,							dim=0      )
                   _SCREAMING_SNAKE_CASE      =				torch.cat([bias_i, self.cluster_bias]       ,							dim=0      )

                weights.append(A__      )
                biases.append(A__      )

             _SCREAMING_SNAKE_CASE ,							_SCREAMING_SNAKE_CASE ,							_SCREAMING_SNAKE_CASE      =				weights[0], biases[0], self.out_projs[0]
             _SCREAMING_SNAKE_CASE      =				self._compute_logit(A__       ,							A__       ,							A__       ,							A__      )

             _SCREAMING_SNAKE_CASE      =				hidden.new_empty((head_logit.size(0      ), self.n_token)      )
             _SCREAMING_SNAKE_CASE      =				nn.functional.log_softmax(A__       ,							dim=1      )

             _SCREAMING_SNAKE_CASE      =				[0] + self.cutoffs
             for i in range(len(A__      ) - 1      ):
                _SCREAMING_SNAKE_CASE ,							_SCREAMING_SNAKE_CASE      =				cutoff_values[i], cutoff_values[i + 1]

                if i == 0:
                   _SCREAMING_SNAKE_CASE      =				head_logprob[:, : self.cutoffs[0]]
                else:
                   _SCREAMING_SNAKE_CASE ,							_SCREAMING_SNAKE_CASE ,							_SCREAMING_SNAKE_CASE      =				weights[i], biases[i], self.out_projs[i]

                   _SCREAMING_SNAKE_CASE      =				self._compute_logit(A__       ,							A__       ,							A__       ,							A__      )
                   _SCREAMING_SNAKE_CASE      =				nn.functional.log_softmax(A__       ,							dim=1      )

                   _SCREAMING_SNAKE_CASE      =				head_logprob[:, -i] + tail_logprob_i
                   _SCREAMING_SNAKE_CASE      =				logprob_i

             return out








'''simple docstring'''



import unittest

import numpy as np

from transformers.testing_utils import require_flax, require_tf, require_torch
from transformers.utils import (
    expand_dims,
    flatten_dict,
    is_flax_available,
    is_tf_available,
    is_torch_available,
    reshape,
    squeeze,
    transpose,
)


if is_flax_available():
     import jax.numpy as jnp

if is_tf_available():
     import tensorflow as tf

if is_torch_available():
     import torch

class    _a				(unittest.TestCase):





       """simple docstring"""




       def        UpperCamelCase				(							self      )    ->			Optional[Any]:
          _SCREAMING_SNAKE_CASE      =				{
              """task_specific_params""": {
                  """summarization""": {"""length_penalty""": 1.0, """max_length""": 1_28, """min_length""": 12, """num_beams""": 4},
                  """summarization_cnn""": {"""length_penalty""": 2.0, """max_length""": 1_42, """min_length""": 56, """num_beams""": 4},
                  """summarization_xsum""": {"""length_penalty""": 1.0, """max_length""": 62, """min_length""": 11, """num_beams""": 6},
              }
          }
          _SCREAMING_SNAKE_CASE      =				{
              """task_specific_params.summarization.length_penalty""": 1.0,
              """task_specific_params.summarization.max_length""": 1_28,
              """task_specific_params.summarization.min_length""": 12,
              """task_specific_params.summarization.num_beams""": 4,
              """task_specific_params.summarization_cnn.length_penalty""": 2.0,
              """task_specific_params.summarization_cnn.max_length""": 1_42,
              """task_specific_params.summarization_cnn.min_length""": 56,
              """task_specific_params.summarization_cnn.num_beams""": 4,
              """task_specific_params.summarization_xsum.length_penalty""": 1.0,
              """task_specific_params.summarization_xsum.max_length""": 62,
              """task_specific_params.summarization_xsum.min_length""": 11,
              """task_specific_params.summarization_xsum.num_beams""": 6,
          }

          self.assertEqual(flatten_dict(A__      )       ,							A__      )






       def        UpperCamelCase				(							self      )    ->			int:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          self.assertTrue(np.allclose(transpose(A__      )       ,							x.transpose()      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4       ,							5      )
          self.assertTrue(np.allclose(transpose(A__       ,							axes=(1, 2, 0)      )       ,							x.transpose((1, 2, 0)      )      )      )






       @require_torch
       def        UpperCamelCase				(							self      )    ->			Optional[Any]:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				torch.tensor(A__      )
          self.assertTrue(np.allclose(transpose(A__      )       ,							transpose(A__      ).numpy()      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4       ,							5      )
          _SCREAMING_SNAKE_CASE      =				torch.tensor(A__      )
          self.assertTrue(np.allclose(transpose(A__       ,							axes=(1, 2, 0)      )       ,							transpose(A__       ,							axes=(1, 2, 0)      ).numpy()      )      )






       @require_tf
       def        UpperCamelCase				(							self      )    ->			Optional[int]:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				tf.constant(A__      )
          self.assertTrue(np.allclose(transpose(A__      )       ,							transpose(A__      ).numpy()      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4       ,							5      )
          _SCREAMING_SNAKE_CASE      =				tf.constant(A__      )
          self.assertTrue(np.allclose(transpose(A__       ,							axes=(1, 2, 0)      )       ,							transpose(A__       ,							axes=(1, 2, 0)      ).numpy()      )      )






       @require_flax
       def        UpperCamelCase				(							self      )    ->			List[str]:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				jnp.array(A__      )
          self.assertTrue(np.allclose(transpose(A__      )       ,							np.asarray(transpose(A__      )      )      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4       ,							5      )
          _SCREAMING_SNAKE_CASE      =				jnp.array(A__      )
          self.assertTrue(np.allclose(transpose(A__       ,							axes=(1, 2, 0)      )       ,							np.asarray(transpose(A__       ,							axes=(1, 2, 0)      )      )      )      )






       def        UpperCamelCase				(							self      )    ->			Optional[int]:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          self.assertTrue(np.allclose(reshape(A__       ,							(4, 3)      )       ,							np.reshape(A__       ,							(4, 3)      )      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4       ,							5      )
          self.assertTrue(np.allclose(reshape(A__       ,							(12, 5)      )       ,							np.reshape(A__       ,							(12, 5)      )      )      )






       @require_torch
       def        UpperCamelCase				(							self      )    ->			Optional[int]:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				torch.tensor(A__      )
          self.assertTrue(np.allclose(reshape(A__       ,							(4, 3)      )       ,							reshape(A__       ,							(4, 3)      ).numpy()      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4       ,							5      )
          _SCREAMING_SNAKE_CASE      =				torch.tensor(A__      )
          self.assertTrue(np.allclose(reshape(A__       ,							(12, 5)      )       ,							reshape(A__       ,							(12, 5)      ).numpy()      )      )






       @require_tf
       def        UpperCamelCase				(							self      )    ->			Tuple:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				tf.constant(A__      )
          self.assertTrue(np.allclose(reshape(A__       ,							(4, 3)      )       ,							reshape(A__       ,							(4, 3)      ).numpy()      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4       ,							5      )
          _SCREAMING_SNAKE_CASE      =				tf.constant(A__      )
          self.assertTrue(np.allclose(reshape(A__       ,							(12, 5)      )       ,							reshape(A__       ,							(12, 5)      ).numpy()      )      )






       @require_flax
       def        UpperCamelCase				(							self      )    ->			List[Any]:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				jnp.array(A__      )
          self.assertTrue(np.allclose(reshape(A__       ,							(4, 3)      )       ,							np.asarray(reshape(A__       ,							(4, 3)      )      )      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4       ,							5      )
          _SCREAMING_SNAKE_CASE      =				jnp.array(A__      )
          self.assertTrue(np.allclose(reshape(A__       ,							(12, 5)      )       ,							np.asarray(reshape(A__       ,							(12, 5)      )      )      )      )






       def        UpperCamelCase				(							self      )    ->			Any:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(1       ,							3       ,							4      )
          self.assertTrue(np.allclose(squeeze(A__      )       ,							np.squeeze(A__      )      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(1       ,							4       ,							1       ,							5      )
          self.assertTrue(np.allclose(squeeze(A__       ,							axis=2      )       ,							np.squeeze(A__       ,							axis=2      )      )      )






       @require_torch
       def        UpperCamelCase				(							self      )    ->			Any:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(1       ,							3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				torch.tensor(A__      )
          self.assertTrue(np.allclose(squeeze(A__      )       ,							squeeze(A__      ).numpy()      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(1       ,							4       ,							1       ,							5      )
          _SCREAMING_SNAKE_CASE      =				torch.tensor(A__      )
          self.assertTrue(np.allclose(squeeze(A__       ,							axis=2      )       ,							squeeze(A__       ,							axis=2      ).numpy()      )      )






       @require_tf
       def        UpperCamelCase				(							self      )    ->			List[str]:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(1       ,							3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				tf.constant(A__      )
          self.assertTrue(np.allclose(squeeze(A__      )       ,							squeeze(A__      ).numpy()      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(1       ,							4       ,							1       ,							5      )
          _SCREAMING_SNAKE_CASE      =				tf.constant(A__      )
          self.assertTrue(np.allclose(squeeze(A__       ,							axis=2      )       ,							squeeze(A__       ,							axis=2      ).numpy()      )      )






       @require_flax
       def        UpperCamelCase				(							self      )    ->			Optional[int]:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(1       ,							3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				jnp.array(A__      )
          self.assertTrue(np.allclose(squeeze(A__      )       ,							np.asarray(squeeze(A__      )      )      )      )

          _SCREAMING_SNAKE_CASE      =				np.random.randn(1       ,							4       ,							1       ,							5      )
          _SCREAMING_SNAKE_CASE      =				jnp.array(A__      )
          self.assertTrue(np.allclose(squeeze(A__       ,							axis=2      )       ,							np.asarray(squeeze(A__       ,							axis=2      )      )      )      )






       def        UpperCamelCase				(							self      )    ->			Optional[Any]:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          self.assertTrue(np.allclose(expand_dims(A__       ,							axis=1      )       ,							np.expand_dims(A__       ,							axis=1      )      )      )






       @require_torch
       def        UpperCamelCase				(							self      )    ->			Optional[int]:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				torch.tensor(A__      )
          self.assertTrue(np.allclose(expand_dims(A__       ,							axis=1      )       ,							expand_dims(A__       ,							axis=1      ).numpy()      )      )






       @require_tf
       def        UpperCamelCase				(							self      )    ->			str:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				tf.constant(A__      )
          self.assertTrue(np.allclose(expand_dims(A__       ,							axis=1      )       ,							expand_dims(A__       ,							axis=1      ).numpy()      )      )






       @require_flax
       def        UpperCamelCase				(							self      )    ->			Any:
          _SCREAMING_SNAKE_CASE      =				np.random.randn(3       ,							4      )
          _SCREAMING_SNAKE_CASE      =				jnp.array(A__      )
          self.assertTrue(np.allclose(expand_dims(A__       ,							axis=1      )       ,							np.asarray(expand_dims(A__       ,							axis=1      )      )      )      )






