




import tensorflow as tf

from ...tf_utils import shape_list




class    __lowerCAmelCase       (							tf.keras.layers.Layer):






       def __init__(   self:					Optional[int]       ,					_lowerCAmelCase:					Dict       ,					_lowerCAmelCase:					Optional[int]       ,					_lowerCAmelCase:					Optional[Any]       ,					_lowerCAmelCase:					str       ,					_lowerCAmelCase:					Union[str, Any]=1       ,					_lowerCAmelCase:					Any=False       ,					**_lowerCAmelCase:					List[Any]				):
         super().__init__(**_lowerCAmelCase				)

         lowercase    :Tuple      	=						vocab_size
         lowercase    :str      	=						d_embed
         lowercase    :str      	=						d_proj

         lowercase    :Union[str, Any]      	=						cutoffs + [vocab_size]
         lowercase    :List[Any]      	=						[0] + self.cutoffs
         lowercase    :List[str]      	=						div_val

         lowercase    :Optional[Any]      	=						self.cutoffs[0]
         lowercase    :str      	=						len(self.cutoffs				) - 1
         lowercase    :List[str]      	=						self.shortlist_size + self.n_clusters
         lowercase    :Tuple      	=						keep_order

         lowercase    :Optional[int]      	=						[]
         lowercase    :Optional[int]      	=						[]






       def 					SCREAMING_SNAKE_CASE    (   self:					Union[str, Any]       ,					_lowerCAmelCase:					Tuple				):
         if self.n_clusters > 0:
           lowercase    :Tuple      	=						self.add_weight(
               shape=(self.n_clusters, self.d_embed)       ,					initializer="zeros"       ,					trainable=_lowerCAmelCase       ,					name="cluster_weight"				)
           lowercase    :Optional[int]      	=						self.add_weight(
               shape=(self.n_clusters,)       ,					initializer="zeros"       ,					trainable=_lowerCAmelCase       ,					name="cluster_bias"				)

         if self.div_val == 1:
           for i in range(len(self.cutoffs				)				):
             if self.d_proj != self.d_embed:
               lowercase    :Optional[Any]      	=						self.add_weight(
                   shape=(self.d_embed, self.d_proj)       ,					initializer="zeros"       ,					trainable=_lowerCAmelCase       ,					name=F"out_projs_._{i}"       ,					)
               self.out_projs.append(_lowerCAmelCase				)
             else:
               self.out_projs.append(_lowerCAmelCase				)
             lowercase    :Union[str, Any]      	=						self.add_weight(
                 shape=(self.vocab_size, self.d_embed)       ,					initializer="zeros"       ,					trainable=_lowerCAmelCase       ,					name=F"out_layers_._{i}_._weight"       ,					)
             lowercase    :Optional[int]      	=						self.add_weight(
                 shape=(self.vocab_size,)       ,					initializer="zeros"       ,					trainable=_lowerCAmelCase       ,					name=F"out_layers_._{i}_._bias"       ,					)
             self.out_layers.append((weight, bias)				)
         else:
           for i in range(len(self.cutoffs				)				):
             lowercase	,    lowercase    :Tuple      	=						self.cutoff_ends[i], self.cutoff_ends[i + 1]
             lowercase    :List[str]      	=						self.d_embed // (self.div_val**i)

             lowercase    :Any      	=						self.add_weight(
                 shape=(d_emb_i, self.d_proj)       ,					initializer="zeros"       ,					trainable=_lowerCAmelCase       ,					name=F"out_projs_._{i}"				)
             self.out_projs.append(_lowerCAmelCase				)
             lowercase    :Any      	=						self.add_weight(
                 shape=(r_idx - l_idx, d_emb_i)       ,					initializer="zeros"       ,					trainable=_lowerCAmelCase       ,					name=F"out_layers_._{i}_._weight"       ,					)
             lowercase    :Union[str, Any]      	=						self.add_weight(
                 shape=(r_idx - l_idx,)       ,					initializer="zeros"       ,					trainable=_lowerCAmelCase       ,					name=F"out_layers_._{i}_._bias"       ,					)
             self.out_layers.append((weight, bias)				)
         super().build(_lowerCAmelCase				)






       @staticmethod
       def 					SCREAMING_SNAKE_CASE    (   _lowerCAmelCase:					Union[str, Any]       ,					_lowerCAmelCase:					Optional[Any]       ,					_lowerCAmelCase:					List[str]       ,					_lowerCAmelCase:					Any=None				):
         lowercase    :str      	=						x
         if proj is not None:
           lowercase    :Any      	=						tf.einsum("ibd,ed->ibe"       ,					_lowerCAmelCase       ,					_lowerCAmelCase				)
         return tf.einsum("ibd,nd->ibn"       ,					_lowerCAmelCase       ,					_lowerCAmelCase				) + b






       @staticmethod
       def 					SCREAMING_SNAKE_CASE    (   _lowerCAmelCase:					Tuple       ,					_lowerCAmelCase:					Tuple				):
         lowercase    :Optional[int]      	=						shape_list(_lowerCAmelCase				)
         lowercase    :int      	=						tf.range(lp_size[0]       ,					dtype=target.dtype				)
         lowercase    :str      	=						tf.stack([r, target]       ,					1				)
         return tf.gather_nd(_lowerCAmelCase       ,					_lowerCAmelCase				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[Any]       ,					_lowerCAmelCase:					List[str]       ,					_lowerCAmelCase:					List[Any]       ,					_lowerCAmelCase:					int=True       ,					_lowerCAmelCase:					Optional[Any]=False				):
         lowercase    :Union[str, Any]      	=						0
         if self.n_clusters == 0:
           lowercase    :List[Any]      	=						self._logit(_lowerCAmelCase       ,					self.out_layers[0][0]       ,					self.out_layers[0][1]       ,					self.out_projs[0]				)
           if target is not None:
             lowercase    :Dict      	=						tf.nn.sparse_softmax_cross_entropy_with_logits(labels=_lowerCAmelCase       ,					logits=_lowerCAmelCase				)
           lowercase    :List[Any]      	=						tf.nn.log_softmax(_lowerCAmelCase       ,					axis=-1				)
         else:
           lowercase    :Union[str, Any]      	=						shape_list(_lowerCAmelCase				)
           lowercase    :Any      	=						[]
           lowercase    :Optional[int]      	=						tf.zeros(hidden_sizes[:2]				)
           for i in range(len(self.cutoffs				)				):
             lowercase	,    lowercase    :Optional[int]      	=						self.cutoff_ends[i], self.cutoff_ends[i + 1]
             if target is not None:
               lowercase    :int      	=						(target >= l_idx) & (target < r_idx)
               lowercase    :Any      	=						tf.where(_lowerCAmelCase				)
               lowercase    :List[str]      	=						tf.boolean_mask(_lowerCAmelCase       ,					_lowerCAmelCase				) - l_idx

             if self.div_val == 1:
               lowercase    :int      	=						self.out_layers[0][0][l_idx:r_idx]
               lowercase    :List[str]      	=						self.out_layers[0][1][l_idx:r_idx]
             else:
               lowercase    :str      	=						self.out_layers[i][0]
               lowercase    :str      	=						self.out_layers[i][1]

             if i == 0:
               lowercase    :Union[str, Any]      	=						tf.concat([cur_W, self.cluster_weight]       ,					0				)
               lowercase    :List[Any]      	=						tf.concat([cur_b, self.cluster_bias]       ,					0				)

               lowercase    :Tuple      	=						self._logit(_lowerCAmelCase       ,					_lowerCAmelCase       ,					_lowerCAmelCase       ,					self.out_projs[0]				)
               lowercase    :Tuple      	=						tf.nn.log_softmax(_lowerCAmelCase				)
               out.append(head_logprob[..., : self.cutoffs[0]]				)
               if target is not None:
                 lowercase    :List[str]      	=						tf.boolean_mask(_lowerCAmelCase       ,					_lowerCAmelCase				)
                 lowercase    :List[Any]      	=						self._gather_logprob(_lowerCAmelCase       ,					_lowerCAmelCase				)
             else:
               lowercase    :Optional[int]      	=						self._logit(_lowerCAmelCase       ,					_lowerCAmelCase       ,					_lowerCAmelCase       ,					self.out_projs[i]				)
               lowercase    :Any      	=						tf.nn.log_softmax(_lowerCAmelCase				)
               lowercase    :int      	=						self.cutoffs[0] + i - 1  # No probability for the head cluster
               lowercase    :int      	=						head_logprob[..., cluster_prob_idx, None] + tail_logprob
               out.append(_lowerCAmelCase				)
               if target is not None:
                 lowercase    :Union[str, Any]      	=						tf.boolean_mask(_lowerCAmelCase       ,					_lowerCAmelCase				)
                 lowercase    :Tuple      	=						tf.boolean_mask(_lowerCAmelCase       ,					_lowerCAmelCase				)
                 lowercase    :List[Any]      	=						self._gather_logprob(_lowerCAmelCase       ,					_lowerCAmelCase				)
                 cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
             if target is not None:
               loss += tf.scatter_nd(_lowerCAmelCase       ,					-cur_logprob       ,					shape_list(_lowerCAmelCase				)				)
           lowercase    :int      	=						tf.concat(_lowerCAmelCase       ,					axis=-1				)

         if target is not None:
           if return_mean:
             lowercase    :Dict      	=						tf.reduce_mean(_lowerCAmelCase				)
           # Add the training-time loss value to the layer using `self.add_loss()`.
           self.add_loss(_lowerCAmelCase				)

           # Log the loss as a metric (we could log arbitrary metrics,
           # including different metrics for training and inference.
           self.add_metric(_lowerCAmelCase       ,					name=self.name       ,					aggregation="mean" if return_mean else ""				)

         return out









def       UpperCAmelCase__  (     lowerCamelCase   ):
  if divisor % 5 == 0 or divisor % 2 == 0:
    return 0
  lowercase    :str      	=						1
  lowercase    :Tuple      	=						1
  while repunit:
    lowercase    :Dict      	=						(10 * repunit + 1) % divisor
    repunit_index += 1
  return repunit_index



def       UpperCAmelCase__  (     lowerCamelCase = 1000000   ):
  lowercase    :List[Any]      	=						limit - 1
  if divisor % 2 == 0:
    divisor += 1
  while least_divisible_repunit(lowerCamelCase   ) <= limit:
    divisor += 2
  return divisor


if __name__ == "__main__":
     print(f'''{solution() = }''')



