






"""simple docstring"""





import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import TaBlock, TaConfig, TaLayerNorm

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin
class    __lowerCamelCase    (  snake_case__							,	snake_case__							,	snake_case__  ):

   @register_to_config
   def __init__(					self ,				snake_case_ ,				snake_case_ ,				snake_case_ ,				snake_case_ ,				snake_case_ ,				snake_case_ ,				snake_case_ ,				snake_case_ ,				snake_case_ ,				snake_case_ = False ,				)		->       List[str]:
       super().__init__()

       UpperCamelCase__         =    nn.Embedding(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE      )

       UpperCamelCase__         =    nn.Embedding(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE      )
       UpperCamelCase__         =    False

       UpperCamelCase__         =    nn.Dropout(p=_SCREAMING_SNAKE_CASE      )

       UpperCamelCase__         =    TaConfig(
           vocab_size=_SCREAMING_SNAKE_CASE ,				d_model=_SCREAMING_SNAKE_CASE ,				num_heads=_SCREAMING_SNAKE_CASE ,				d_kv=_SCREAMING_SNAKE_CASE ,				d_ff=_SCREAMING_SNAKE_CASE ,				dropout_rate=_SCREAMING_SNAKE_CASE ,				feed_forward_proj=_SCREAMING_SNAKE_CASE ,				is_decoder=_SCREAMING_SNAKE_CASE ,				is_encoder_decoder=_SCREAMING_SNAKE_CASE ,				)

       UpperCamelCase__         =    nn.ModuleList()
       for lyr_num in range(_SCREAMING_SNAKE_CASE      ):
           UpperCamelCase__         =    TaBlock(_SCREAMING_SNAKE_CASE      )
           self.encoders.append(_SCREAMING_SNAKE_CASE      )

       UpperCamelCase__         =    TaLayerNorm(_SCREAMING_SNAKE_CASE      )
       UpperCamelCase__         =    nn.Dropout(p=_SCREAMING_SNAKE_CASE      )







   def 						SCREAMING_SNAKE_CASE__ (					self ,				snake_case_ ,				snake_case_      )		->       Optional[int]:
       UpperCamelCase__         =    self.token_embedder(_SCREAMING_SNAKE_CASE      )

       UpperCamelCase__         =    encoder_input_tokens.shape[1]
       UpperCamelCase__         =    torch.arange(_SCREAMING_SNAKE_CASE ,				device=encoder_input_tokens.device      )
       x += self.position_encoding(_SCREAMING_SNAKE_CASE      )

       UpperCamelCase__         =    self.dropout_pre(_SCREAMING_SNAKE_CASE      )

       # inverted the attention mask
       UpperCamelCase__         =    encoder_input_tokens.size()
       UpperCamelCase__         =    self.get_extended_attention_mask(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE      )

       for lyr in self.encoders:
           UpperCamelCase__         =    lyr(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE      )[0]
       UpperCamelCase__         =    self.layer_norm(_SCREAMING_SNAKE_CASE      )

       return self.dropout_post(_SCREAMING_SNAKE_CASE      ), encoder_inputs_mask












"""simple docstring"""





def   lowerCAmelCase_( SCREAMING_SNAKE_CASE					)					->	bool:
    """simple docstring"""

    UpperCamelCase__         =    (1 + 24 * n) ** 0.5
    return ((1 + root) / 6) % 1 == 0







def   lowerCAmelCase_( SCREAMING_SNAKE_CASE = 50_00					)					->	int:
    """simple docstring"""

    UpperCamelCase__         =    [(i * (3 * i - 1)) // 2 for i in range(1 ,	SCREAMING_SNAKE_CASE					)]
    for i, pentagonal_i in enumerate(SCREAMING_SNAKE_CASE					):
        for j in range(SCREAMING_SNAKE_CASE ,	len(SCREAMING_SNAKE_CASE					)					):
            UpperCamelCase__         =    pentagonal_nums[j]
            UpperCamelCase__         =    pentagonal_i + pentagonal_j
            UpperCamelCase__         =    pentagonal_j - pentagonal_i
            if is_pentagonal(SCREAMING_SNAKE_CASE					) and is_pentagonal(SCREAMING_SNAKE_CASE					):
                return b

    return -1


if __name__ == "__main__":
  print(F"""{solution() = }""")




