






import torch
from torch import nn





class      lowercase__    (							nn.Module  ):







  def __init__(	self    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase=1    ,						_lowercase=False						):
       super().__init__()

       lowerCAmelCase_					:       Optional[Any]    =		n_token
       lowerCAmelCase_					:       List[Any]    =		d_embed
       lowerCAmelCase_					:       Dict    =		d_proj

       lowerCAmelCase_					:       Dict    =		cutoffs + [n_token]
       lowerCAmelCase_					:       List[str]    =		[0] + self.cutoffs
       lowerCAmelCase_					:       Union[str, Any]    =		div_val

       lowerCAmelCase_					:       int    =		self.cutoffs[0]
       lowerCAmelCase_					:       Dict    =		len(self.cutoffs						) - 1
       lowerCAmelCase_					:       Dict    =		self.shortlist_size + self.n_clusters

       if self.n_clusters > 0:
            lowerCAmelCase_					:       Any    =		nn.Parameter(torch.zeros(self.n_clusters    ,						self.d_embed						)						)
            lowerCAmelCase_					:       Any    =		nn.Parameter(torch.zeros(self.n_clusters						)						)

       lowerCAmelCase_					:       int    =		nn.ModuleList()
       lowerCAmelCase_					:       str    =		nn.ParameterList()

       if div_val == 1:
            for i in range(len(self.cutoffs						)						):
                 if d_proj != d_embed:
                      self.out_projs.append(nn.Parameter(torch.FloatTensor(_lowercase    ,						_lowercase						)						)						)
                 else:
                      self.out_projs.append(_lowercase						)

            self.out_layers.append(nn.Linear(_lowercase    ,						_lowercase						)						)
       else:
            for i in range(len(self.cutoffs						)						):
                 lowerCAmelCase_					:       Union[str, Any]    =		self.cutoff_ends[i], self.cutoff_ends[i + 1]
                 lowerCAmelCase_					:       Optional[int]    =		d_embed // (div_val**i)

                 self.out_projs.append(nn.Parameter(torch.FloatTensor(_lowercase    ,						_lowercase						)						)						)

                 self.out_layers.append(nn.Linear(_lowercase    ,						r_idx - l_idx						)						)

       lowerCAmelCase_					:       Optional[Any]    =		keep_order







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase						):

       if proj is None:
            lowerCAmelCase_					:       Any    =		nn.functional.linear(_lowercase    ,						_lowercase    ,						bias=_lowercase						)
       else:
            # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
            lowerCAmelCase_					:       Dict    =		nn.functional.linear(_lowercase    ,						proj.t().contiguous()						)
            lowerCAmelCase_					:       Dict    =		nn.functional.linear(_lowercase    ,						_lowercase    ,						bias=_lowercase						)
            # else:
            #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
            #     if bias is not None:
            #         logit = logit + bias

       return logit







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase=None    ,						_lowercase=False						):

       if labels is not None:
            # Shift so that tokens < n predict n
            lowerCAmelCase_					:       Dict    =		hidden[..., :-1, :].contiguous()
            lowerCAmelCase_					:       Dict    =		labels[..., 1:].contiguous()
            lowerCAmelCase_					:       int    =		hidden.view(-1    ,						hidden.size(-1						)						)
            lowerCAmelCase_					:       Optional[Any]    =		labels.view(-1						)
            if hidden.size(0						) != labels.size(0						):
                 raise RuntimeError("""Input and labels should have the same size in the batch dimension."""						)
       else:
            lowerCAmelCase_					:       Tuple    =		hidden.view(-1    ,						hidden.size(-1						)						)

       if self.n_clusters == 0:
            lowerCAmelCase_					:       Any    =		self._compute_logit(_lowercase    ,						self.out_layers[0].weight    ,						self.out_layers[0].bias    ,						self.out_projs[0]						)
            if labels is not None:
                 lowerCAmelCase_					:       List[str]    =		labels != -100
                 lowerCAmelCase_					:       List[Any]    =		torch.zeros_like(_lowercase    ,						dtype=hidden.dtype    ,						device=hidden.device						)
                 lowerCAmelCase_					:       Optional[int]    =		(
                     -nn.functional.log_softmax(_lowercase    ,						dim=-1						)[mask].gather(1    ,						labels[mask].unsqueeze(1						)						).squeeze(1						)
                 )
            else:
                 lowerCAmelCase_					:       Tuple    =		nn.functional.log_softmax(_lowercase    ,						dim=-1						)
       else:
            # construct weights and biases
            lowerCAmelCase_					:       List[str]    =		[], []
            for i in range(len(self.cutoffs						)						):
                 if self.div_val == 1:
                      lowerCAmelCase_					:       Any    =		self.cutoff_ends[i], self.cutoff_ends[i + 1]
                      lowerCAmelCase_					:       Optional[Any]    =		self.out_layers[0].weight[l_idx:r_idx]
                      lowerCAmelCase_					:       str    =		self.out_layers[0].bias[l_idx:r_idx]
                 else:
                      lowerCAmelCase_					:       Union[str, Any]    =		self.out_layers[i].weight
                      lowerCAmelCase_					:       Dict    =		self.out_layers[i].bias

                 if i == 0:
                      lowerCAmelCase_					:       List[str]    =		torch.cat([weight_i, self.cluster_weight]    ,						dim=0						)
                      lowerCAmelCase_					:       List[str]    =		torch.cat([bias_i, self.cluster_bias]    ,						dim=0						)

                 weights.append(_lowercase						)
                 biases.append(_lowercase						)

            lowerCAmelCase_					:       Any    =		weights[0], biases[0], self.out_projs[0]

            lowerCAmelCase_					:       Optional[int]    =		self._compute_logit(_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase						)
            lowerCAmelCase_					:       str    =		nn.functional.log_softmax(_lowercase    ,						dim=1						)

            if labels is None:
                 lowerCAmelCase_					:       Any    =		hidden.new_empty((head_logit.size(0						), self.n_token)						)
            else:
                 lowerCAmelCase_					:       List[Any]    =		torch.zeros_like(_lowercase    ,						dtype=hidden.dtype    ,						device=hidden.device						)

            lowerCAmelCase_					:       Dict    =		0
            lowerCAmelCase_					:       List[Any]    =		[0] + self.cutoffs
            for i in range(len(_lowercase						) - 1						):
                 lowerCAmelCase_					:       List[Any]    =		cutoff_values[i], cutoff_values[i + 1]

                 if labels is not None:
                      lowerCAmelCase_					:       List[str]    =		(labels >= l_idx) & (labels < r_idx)
                      lowerCAmelCase_					:       Optional[Any]    =		mask_i.nonzero().squeeze()

                      if indices_i.numel() == 0:
                           continue

                      lowerCAmelCase_					:       Any    =		labels.index_select(0    ,						_lowercase						) - l_idx
                      lowerCAmelCase_					:       str    =		head_logprob.index_select(0    ,						_lowercase						)
                      lowerCAmelCase_					:       int    =		hidden.index_select(0    ,						_lowercase						)
                 else:
                      lowerCAmelCase_					:       Dict    =		hidden

                 if i == 0:
                      if labels is not None:
                           lowerCAmelCase_					:       Optional[int]    =		head_logprob_i.gather(1    ,						target_i[:, None]						).squeeze(1						)
                      else:
                           lowerCAmelCase_					:       Union[str, Any]    =		head_logprob[:, : self.cutoffs[0]]
                 else:
                      lowerCAmelCase_					:       int    =		weights[i], biases[i], self.out_projs[i]

                      lowerCAmelCase_					:       Union[str, Any]    =		self._compute_logit(_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase						)
                      lowerCAmelCase_					:       Union[str, Any]    =		nn.functional.log_softmax(_lowercase    ,						dim=1						)
                      lowerCAmelCase_					:       Any    =		self.cutoffs[0] + i - 1  # No probability for the head cluster
                      if labels is not None:
                           lowerCAmelCase_					:       Any    =		head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                               1    ,						target_i[:, None]						).squeeze(1						)
                      else:
                           lowerCAmelCase_					:       int    =		head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
                           lowerCAmelCase_					:       Union[str, Any]    =		logprob_i

                 if labels is not None:
                      if (hasattr(self    ,						"""keep_order"""						) and self.keep_order) or keep_order:
                           out.index_copy_(0    ,						_lowercase    ,						-logprob_i						)
                      else:
                           out[offset : offset + logprob_i.size(0						)].copy_(-logprob_i						)
                      offset += logprob_i.size(0						)

       return out







  def 	UpperCAmelCase__		(	self    ,						_lowercase						):

       if self.n_clusters == 0:
            lowerCAmelCase_					:       str    =		self._compute_logit(_lowercase    ,						self.out_layers[0].weight    ,						self.out_layers[0].bias    ,						self.out_projs[0]						)
            return nn.functional.log_softmax(_lowercase    ,						dim=-1						)
       else:
            # construct weights and biases
            lowerCAmelCase_					:       Union[str, Any]    =		[], []
            for i in range(len(self.cutoffs						)						):
                 if self.div_val == 1:
                      lowerCAmelCase_					:       Union[str, Any]    =		self.cutoff_ends[i], self.cutoff_ends[i + 1]
                      lowerCAmelCase_					:       int    =		self.out_layers[0].weight[l_idx:r_idx]
                      lowerCAmelCase_					:       int    =		self.out_layers[0].bias[l_idx:r_idx]
                 else:
                      lowerCAmelCase_					:       str    =		self.out_layers[i].weight
                      lowerCAmelCase_					:       Union[str, Any]    =		self.out_layers[i].bias

                 if i == 0:
                      lowerCAmelCase_					:       str    =		torch.cat([weight_i, self.cluster_weight]    ,						dim=0						)
                      lowerCAmelCase_					:       Dict    =		torch.cat([bias_i, self.cluster_bias]    ,						dim=0						)

                 weights.append(_lowercase						)
                 biases.append(_lowercase						)

            lowerCAmelCase_					:       Optional[int]    =		weights[0], biases[0], self.out_projs[0]
            lowerCAmelCase_					:       str    =		self._compute_logit(_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase						)

            lowerCAmelCase_					:       int    =		hidden.new_empty((head_logit.size(0						), self.n_token)						)
            lowerCAmelCase_					:       List[str]    =		nn.functional.log_softmax(_lowercase    ,						dim=1						)

            lowerCAmelCase_					:       str    =		[0] + self.cutoffs
            for i in range(len(_lowercase						) - 1						):
                 lowerCAmelCase_					:       Any    =		cutoff_values[i], cutoff_values[i + 1]

                 if i == 0:
                      lowerCAmelCase_					:       Optional[int]    =		head_logprob[:, : self.cutoffs[0]]
                 else:
                      lowerCAmelCase_					:       List[str]    =		weights[i], biases[i], self.out_projs[i]

                      lowerCAmelCase_					:       str    =		self._compute_logit(_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase						)
                      lowerCAmelCase_					:       Dict    =		nn.functional.log_softmax(_lowercase    ,						dim=1						)

                      lowerCAmelCase_					:       Optional[int]    =		head_logprob[:, -i] + tail_logprob_i
                      lowerCAmelCase_					:       str    =		logprob_i

            return out







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


UpperCAmelCase_					:   Dict 							=						{
    """configuration_poolformer""": [
        """POOLFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """PoolFormerConfig""",
        """PoolFormerOnnxConfig""",
    ]
}

try:
      if not is_vision_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      UpperCAmelCase_					:   Union[str, Any] 							=						["""PoolFormerFeatureExtractor"""]
      UpperCAmelCase_					:   str 							=						["""PoolFormerImageProcessor"""]

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      UpperCAmelCase_					:   List[str] 							=						[
          """POOLFORMER_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """PoolFormerForImageClassification""",
          """PoolFormerModel""",
          """PoolFormerPreTrainedModel""",
      ]


if TYPE_CHECKING:
      from .configuration_poolformer import (
          POOLFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
          PoolFormerConfig,
          PoolFormerOnnxConfig,
      )

      try:
            if not is_vision_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .feature_extraction_poolformer import PoolFormerFeatureExtractor
            from .image_processing_poolformer import PoolFormerImageProcessor

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_poolformer import (
                POOLFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
                PoolFormerForImageClassification,
                PoolFormerModel,
                PoolFormerPreTrainedModel,
            )


else:
      import sys

      UpperCAmelCase_					:   Optional[Any] 							=						_LazyModule(__name__, globals()["""__file__"""], _import_structure)
