



import tensorflow as tf

from ...tf_utils import shape_list


class      __lowerCAmelCase     (		tf.keras.layers.Layer      ):
					"""simple docstring"""






					def __init__(     self					:       Union[str, Any]				,					_snake_case					:       Optional[int]				,					_snake_case					:       Union[str, Any]				,					_snake_case					:       Tuple				,					_snake_case					:       Optional[int]				,					_snake_case					:       Any=1				,					_snake_case					:       Optional[Any]=False				,					**_snake_case					:       List[Any]					):






								"""simple docstring"""


								super().__init__(**_snake_case					)

								A__       =    vocab_size
								A__       =    d_embed
								A__       =    d_proj

								A__       =    cutoffs + [vocab_size]
								A__       =    [0] + self.cutoffs
								A__       =    div_val

								A__       =    self.cutoffs[0]
								A__       =    len(self.cutoffs					) - 1
								A__       =    self.shortlist_size + self.n_clusters
								A__       =    keep_order

								A__       =    []
								A__       =    []



					def     _a							(     self					:       Optional[int]				,					_snake_case					:       Any					):






								"""simple docstring"""


								if self.n_clusters > 0:
											A__       =    self.add_weight(
											    shape=(self.n_clusters, self.d_embed)				,					initializer='zeros'				,					trainable=_snake_case				,					name='cluster_weight'					)
											A__       =    self.add_weight(
											    shape=(self.n_clusters,)				,					initializer='zeros'				,					trainable=_snake_case				,					name='cluster_bias'					)

								if self.div_val == 1:
											for i in range(len(self.cutoffs					)					):
														if self.d_proj != self.d_embed:
																	A__       =    self.add_weight(
																	    shape=(self.d_embed, self.d_proj)				,					initializer='zeros'				,					trainable=_snake_case				,					name=F'''out_projs_._{i}'''				,					)
																	self.out_projs.append(_snake_case					)
														else:
																	self.out_projs.append(_snake_case					)
														A__       =    self.add_weight(
														    shape=(self.vocab_size, self.d_embed)				,					initializer='zeros'				,					trainable=_snake_case				,					name=F'''out_layers_._{i}_._weight'''				,					)
														A__       =    self.add_weight(
														    shape=(self.vocab_size,)				,					initializer='zeros'				,					trainable=_snake_case				,					name=F'''out_layers_._{i}_._bias'''				,					)
														self.out_layers.append((weight, bias)					)
								else:
											for i in range(len(self.cutoffs					)					):
														A__     , A__       =    self.cutoff_ends[i], self.cutoff_ends[i + 1]
														A__       =    self.d_embed // (self.div_val**i)

														A__       =    self.add_weight(
														    shape=(d_emb_i, self.d_proj)				,					initializer='zeros'				,					trainable=_snake_case				,					name=F'''out_projs_._{i}'''					)
														self.out_projs.append(_snake_case					)
														A__       =    self.add_weight(
														    shape=(r_idx - l_idx, d_emb_i)				,					initializer='zeros'				,					trainable=_snake_case				,					name=F'''out_layers_._{i}_._weight'''				,					)
														A__       =    self.add_weight(
														    shape=(r_idx - l_idx,)				,					initializer='zeros'				,					trainable=_snake_case				,					name=F'''out_layers_._{i}_._bias'''				,					)
														self.out_layers.append((weight, bias)					)
								super().build(_snake_case					)



					@staticmethod
					def     _a							(     _snake_case					:       List[Any]				,					_snake_case					:       str				,					_snake_case					:       Any				,					_snake_case					:       Union[str, Any]=None					):






								"""simple docstring"""


								A__       =    x
								if proj is not None:
											A__       =    tf.einsum('ibd,ed->ibe'				,					_snake_case				,					_snake_case					)
								return tf.einsum('ibd,nd->ibn'				,					_snake_case				,					_snake_case					) + b



					@staticmethod
					def     _a							(     _snake_case					:       Optional[int]				,					_snake_case					:       Optional[Any]					):






								"""simple docstring"""


								A__       =    shape_list(_snake_case					)
								A__       =    tf.range(lp_size[0]				,					dtype=target.dtype					)
								A__       =    tf.stack([r, target]				,					1					)
								return tf.gather_nd(_snake_case				,					_snake_case					)



					def     _a							(     self					:       Optional[int]				,					_snake_case					:       Any				,					_snake_case					:       Optional[Any]				,					_snake_case					:       Any=True				,					_snake_case					:       Union[str, Any]=False					):






								"""simple docstring"""


								A__       =    0
								if self.n_clusters == 0:
											A__       =    self._logit(_snake_case				,					self.out_layers[0][0]				,					self.out_layers[0][1]				,					self.out_projs[0]					)
											if target is not None:
														A__       =    tf.nn.sparse_softmax_cross_entropy_with_logits(labels=_snake_case				,					logits=_snake_case					)
											A__       =    tf.nn.log_softmax(_snake_case				,					axis=-1					)
								else:
											A__       =    shape_list(_snake_case					)
											A__       =    []
											A__       =    tf.zeros(hidden_sizes[:2]					)
											for i in range(len(self.cutoffs					)					):
														A__     , A__       =    self.cutoff_ends[i], self.cutoff_ends[i + 1]
														if target is not None:
																	A__       =    (target >= l_idx) & (target < r_idx)
																	A__       =    tf.where(_snake_case					)
																	A__       =    tf.boolean_mask(_snake_case				,					_snake_case					) - l_idx

														if self.div_val == 1:
																	A__       =    self.out_layers[0][0][l_idx:r_idx]
																	A__       =    self.out_layers[0][1][l_idx:r_idx]
														else:
																	A__       =    self.out_layers[i][0]
																	A__       =    self.out_layers[i][1]

														if i == 0:
																	A__       =    tf.concat([cur_W, self.cluster_weight]				,					0					)
																	A__       =    tf.concat([cur_b, self.cluster_bias]				,					0					)

																	A__       =    self._logit(_snake_case				,					_snake_case				,					_snake_case				,					self.out_projs[0]					)
																	A__       =    tf.nn.log_softmax(_snake_case					)
																	out.append(head_logprob[..., : self.cutoffs[0]]					)
																	if target is not None:
																				A__       =    tf.boolean_mask(_snake_case				,					_snake_case					)
																				A__       =    self._gather_logprob(_snake_case				,					_snake_case					)
														else:
																	A__       =    self._logit(_snake_case				,					_snake_case				,					_snake_case				,					self.out_projs[i]					)
																	A__       =    tf.nn.log_softmax(_snake_case					)
																	A__       =    self.cutoffs[0] + i - 1  # No probability for the head cluster
																	A__       =    head_logprob[..., cluster_prob_idx, None] + tail_logprob
																	out.append(_snake_case					)
																	if target is not None:
																				A__       =    tf.boolean_mask(_snake_case				,					_snake_case					)
																				A__       =    tf.boolean_mask(_snake_case				,					_snake_case					)
																				A__       =    self._gather_logprob(_snake_case				,					_snake_case					)
																				cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
														if target is not None:
																	loss += tf.scatter_nd(_snake_case				,					-cur_logprob				,					shape_list(_snake_case					)					)
											A__       =    tf.concat(_snake_case				,					axis=-1					)

								if target is not None:
											if return_mean:
														A__       =    tf.reduce_mean(_snake_case					)
											# Add the training-time loss value to the layer using `self.add_loss()`.
											self.add_loss(_snake_case					)

											# Log the loss as a metric (we could log arbitrary metrics,
											# including different metrics for training and inference.
											self.add_metric(_snake_case				,					name=self.name				,					aggregation='mean' if return_mean else ''					)

								return out









def A						(       __UpperCamelCase					)						->      bool:
			return number & 1 == 0


if __name__ == "__main__":
							import doctest

							doctest.testmod()




