


'''simple docstring'''

import tensorflow as tf

from ...tf_utils import shape_list


class       a__				( tf.keras.layers.Layer      ):







    """simple docstring"""



    def __init__(self       ,			__lowercase       ,			__lowercase       ,			__lowercase       ,			__lowercase       ,			__lowercase=1       ,			__lowercase=False       ,			**__lowercase		):
          super().__init__(**__lowercase		)

          __lowerCAmelCase							       =				vocab_size
          __lowerCAmelCase							       =				d_embed
          __lowerCAmelCase							       =				d_proj

          __lowerCAmelCase							       =				cutoffs + [vocab_size]
          __lowerCAmelCase							       =				[0] + self.cutoffs
          __lowerCAmelCase							       =				div_val

          __lowerCAmelCase							       =				self.cutoffs[0]
          __lowerCAmelCase							       =				len(self.cutoffs		) - 1
          __lowerCAmelCase							       =				self.shortlist_size + self.n_clusters
          __lowerCAmelCase							       =				keep_order

          __lowerCAmelCase							       =				[]
          __lowerCAmelCase							       =				[]




    def 					_snake_case  (self       ,			__lowercase		):
          if self.n_clusters > 0:
                __lowerCAmelCase							       =				self.add_weight(
                    shape=(self.n_clusters, self.d_embed)       ,			initializer='''zeros'''       ,			trainable=__lowercase       ,			name='''cluster_weight'''		)
                __lowerCAmelCase							       =				self.add_weight(
                    shape=(self.n_clusters,)       ,			initializer='''zeros'''       ,			trainable=__lowercase       ,			name='''cluster_bias'''		)

          if self.div_val == 1:
                for i in range(len(self.cutoffs		)		):
                      if self.d_proj != self.d_embed:
                            __lowerCAmelCase							       =				self.add_weight(
                                shape=(self.d_embed, self.d_proj)       ,			initializer='''zeros'''       ,			trainable=__lowercase       ,			name=F"""out_projs_._{i}"""       ,			)
                            self.out_projs.append(__lowercase		)
                      else:
                            self.out_projs.append(__lowercase		)
                      __lowerCAmelCase							       =				self.add_weight(
                          shape=(self.vocab_size, self.d_embed)       ,			initializer='''zeros'''       ,			trainable=__lowercase       ,			name=F"""out_layers_._{i}_._weight"""       ,			)
                      __lowerCAmelCase							       =				self.add_weight(
                          shape=(self.vocab_size,)       ,			initializer='''zeros'''       ,			trainable=__lowercase       ,			name=F"""out_layers_._{i}_._bias"""       ,			)
                      self.out_layers.append((weight, bias)		)
          else:
                for i in range(len(self.cutoffs		)		):
                      __lowerCAmelCase       ,					__lowerCAmelCase							       =				self.cutoff_ends[i], self.cutoff_ends[i + 1]
                      __lowerCAmelCase							       =				self.d_embed // (self.div_val**i)

                      __lowerCAmelCase							       =				self.add_weight(
                          shape=(d_emb_i, self.d_proj)       ,			initializer='''zeros'''       ,			trainable=__lowercase       ,			name=F"""out_projs_._{i}"""		)
                      self.out_projs.append(__lowercase		)
                      __lowerCAmelCase							       =				self.add_weight(
                          shape=(r_idx - l_idx, d_emb_i)       ,			initializer='''zeros'''       ,			trainable=__lowercase       ,			name=F"""out_layers_._{i}_._weight"""       ,			)
                      __lowerCAmelCase							       =				self.add_weight(
                          shape=(r_idx - l_idx,)       ,			initializer='''zeros'''       ,			trainable=__lowercase       ,			name=F"""out_layers_._{i}_._bias"""       ,			)
                      self.out_layers.append((weight, bias)		)
          super().build(__lowercase		)




    @staticmethod
    def 					_snake_case  (__lowercase       ,			__lowercase       ,			__lowercase       ,			__lowercase=None		):
          __lowerCAmelCase							       =				x
          if proj is not None:
                __lowerCAmelCase							       =				tf.einsum('''ibd,ed->ibe'''       ,			__lowercase       ,			__lowercase		)
          return tf.einsum('''ibd,nd->ibn'''       ,			__lowercase       ,			__lowercase		) + b




    @staticmethod
    def 					_snake_case  (__lowercase       ,			__lowercase		):
          __lowerCAmelCase							       =				shape_list(__lowercase		)
          __lowerCAmelCase							       =				tf.range(lp_size[0]       ,			dtype=target.dtype		)
          __lowerCAmelCase							       =				tf.stack([r, target]       ,			1		)
          return tf.gather_nd(__lowercase       ,			__lowercase		)




    def 					_snake_case  (self       ,			__lowercase       ,			__lowercase       ,			__lowercase=True       ,			__lowercase=False		):
          __lowerCAmelCase							       =				0
          if self.n_clusters == 0:
                __lowerCAmelCase							       =				self._logit(__lowercase       ,			self.out_layers[0][0]       ,			self.out_layers[0][1]       ,			self.out_projs[0]		)
                if target is not None:
                      __lowerCAmelCase							       =				tf.nn.sparse_softmax_cross_entropy_with_logits(labels=__lowercase       ,			logits=__lowercase		)
                __lowerCAmelCase							       =				tf.nn.log_softmax(__lowercase       ,			axis=-1		)
          else:
                __lowerCAmelCase							       =				shape_list(__lowercase		)
                __lowerCAmelCase							       =				[]
                __lowerCAmelCase							       =				tf.zeros(hidden_sizes[:2]		)
                for i in range(len(self.cutoffs		)		):
                      __lowerCAmelCase       ,					__lowerCAmelCase							       =				self.cutoff_ends[i], self.cutoff_ends[i + 1]
                      if target is not None:
                            __lowerCAmelCase							       =				(target >= l_idx) & (target < r_idx)
                            __lowerCAmelCase							       =				tf.where(__lowercase		)
                            __lowerCAmelCase							       =				tf.boolean_mask(__lowercase       ,			__lowercase		) - l_idx

                      if self.div_val == 1:
                            __lowerCAmelCase							       =				self.out_layers[0][0][l_idx:r_idx]
                            __lowerCAmelCase							       =				self.out_layers[0][1][l_idx:r_idx]
                      else:
                            __lowerCAmelCase							       =				self.out_layers[i][0]
                            __lowerCAmelCase							       =				self.out_layers[i][1]

                      if i == 0:
                            __lowerCAmelCase							       =				tf.concat([cur_W, self.cluster_weight]       ,			0		)
                            __lowerCAmelCase							       =				tf.concat([cur_b, self.cluster_bias]       ,			0		)

                            __lowerCAmelCase							       =				self._logit(__lowercase       ,			__lowercase       ,			__lowercase       ,			self.out_projs[0]		)
                            __lowerCAmelCase							       =				tf.nn.log_softmax(__lowercase		)
                            out.append(head_logprob[..., : self.cutoffs[0]]		)
                            if target is not None:
                                  __lowerCAmelCase							       =				tf.boolean_mask(__lowercase       ,			__lowercase		)
                                  __lowerCAmelCase							       =				self._gather_logprob(__lowercase       ,			__lowercase		)
                      else:
                            __lowerCAmelCase							       =				self._logit(__lowercase       ,			__lowercase       ,			__lowercase       ,			self.out_projs[i]		)
                            __lowerCAmelCase							       =				tf.nn.log_softmax(__lowercase		)
                            __lowerCAmelCase							       =				self.cutoffs[0] + i - 1  # No probability for the head cluster
                            __lowerCAmelCase							       =				head_logprob[..., cluster_prob_idx, None] + tail_logprob
                            out.append(__lowercase		)
                            if target is not None:
                                  __lowerCAmelCase							       =				tf.boolean_mask(__lowercase       ,			__lowercase		)
                                  __lowerCAmelCase							       =				tf.boolean_mask(__lowercase       ,			__lowercase		)
                                  __lowerCAmelCase							       =				self._gather_logprob(__lowercase       ,			__lowercase		)
                                  cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
                      if target is not None:
                            loss += tf.scatter_nd(__lowercase       ,			-cur_logprob       ,			shape_list(__lowercase		)		)
                __lowerCAmelCase							       =				tf.concat(__lowercase       ,			axis=-1		)

          if target is not None:
                if return_mean:
                      __lowerCAmelCase							       =				tf.reduce_mean(__lowercase		)
                # Add the training-time loss value to the layer using `self.add_loss()`.
                self.add_loss(__lowercase		)

                # Log the loss as a metric (we could log arbitrary metrics,
                # including different metrics for training and inference.
                self.add_metric(__lowercase       ,			name=self.name       ,			aggregation='''mean''' if return_mean else ''''''		)

          return out





'''simple docstring'''

from typing import TYPE_CHECKING

from ...file_utils import _LazyModule, is_torch_available
from ...utils import OptionalDependencyNotAvailable


_UpperCAmelCase				: str							    =				{
    """configuration_gpt_neox_japanese""": ["""GPT_NEOX_JAPANESE_PRETRAINED_CONFIG_ARCHIVE_MAP""", """GPTNeoXJapaneseConfig"""],
    """tokenization_gpt_neox_japanese""": ["""GPTNeoXJapaneseTokenizer"""],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_UpperCAmelCase				: Union[str, Any]							    =				[
	    """GPT_NEOX_JAPANESE_PRETRAINED_MODEL_ARCHIVE_LIST""",
	    """GPTNeoXJapaneseForCausalLM""",
	    """GPTNeoXJapaneseLayer""",
	    """GPTNeoXJapaneseModel""",
	    """GPTNeoXJapanesePreTrainedModel""",
	]


if TYPE_CHECKING:
	from .configuration_gpt_neox_japanese import GPT_NEOX_JAPANESE_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTNeoXJapaneseConfig
	from .tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_gpt_neox_japanese import (
		    GPT_NEOX_JAPANESE_PRETRAINED_MODEL_ARCHIVE_LIST,
		    GPTNeoXJapaneseForCausalLM,
		    GPTNeoXJapaneseLayer,
		    GPTNeoXJapaneseModel,
		    GPTNeoXJapanesePreTrainedModel,
		)


else:
	import sys

	_UpperCAmelCase				: int							    =				_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

