


from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE       :		Optional[int]         =				logging.get_logger(__name__)

SCREAMING_SNAKE_CASE       :		Dict         =				{}






class 							UpperCamelCase					(		UpperCAmelCase_     ):


   '''simple docstring'''







   lowercase					:	Optional[Any]									="llama"
   lowercase					:	str									=["past_key_values"]
   def __init__(     self   ,							UpperCamelCase_=3_2000   ,							UpperCamelCase_=4096   ,							UpperCamelCase_=1_1008   ,							UpperCamelCase_=32   ,							UpperCamelCase_=32   ,							UpperCamelCase_=None   ,							UpperCamelCase_="silu"   ,							UpperCamelCase_=2048   ,							UpperCamelCase_=0.02   ,							UpperCamelCase_=1E-6   ,							UpperCamelCase_=True   ,							UpperCamelCase_=0   ,							UpperCamelCase_=1   ,							UpperCamelCase_=2   ,							UpperCamelCase_=1   ,							UpperCamelCase_=False   ,							UpperCamelCase_=None   ,							**UpperCamelCase_   ,							):
        lowercase_       :List[str]     =       vocab_size
        lowercase_       :Dict     =       max_position_embeddings
        lowercase_       :str     =       hidden_size
        lowercase_       :Optional[int]     =       intermediate_size
        lowercase_       :Any     =       num_hidden_layers
        lowercase_       :List[Any]     =       num_attention_heads

        # for backward compatibility
        if num_key_value_heads is None:
             lowercase_       :str     =       num_attention_heads

        lowercase_       :List[str]     =       num_key_value_heads
        lowercase_       :Union[str, Any]     =       hidden_act
        lowercase_       :Tuple     =       initializer_range
        lowercase_       :List[Any]     =       rms_norm_eps
        lowercase_       :Optional[Any]     =       pretraining_tp
        lowercase_       :Tuple     =       use_cache
        lowercase_       :Any     =       rope_scaling
        self._rope_scaling_validation()

        super().__init__(
            pad_token_id=_snake_case   ,							bos_token_id=_snake_case   ,							eos_token_id=_snake_case   ,							tie_word_embeddings=_snake_case   ,							**_snake_case   ,							)




   def 		UpperCamelCase  (     self		):


        if self.rope_scaling is None:
             return

        if not isinstance(self.rope_scaling   ,							_snake_case		) or len(self.rope_scaling		) != 2:
             raise ValueError(
                 '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
                 f"got {self.rope_scaling}"		)
        lowercase_       :Union[str, Any]     =       self.rope_scaling.get('''type'''   ,							_snake_case		)
        lowercase_       :str     =       self.rope_scaling.get('''factor'''   ,							_snake_case		)
        if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
             raise ValueError(
                 f"`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}"		)
        if rope_scaling_factor is None or not isinstance(_snake_case   ,							_snake_case		) or rope_scaling_factor <= 1.0:
             raise ValueError(f"`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}"		)





from torch import nn




def     UpperCamelCase    (      _a		)					->							str:
     '''simple docstring'''





     if act_fn in ["swish", "silu"]:
          return nn.SiLU()
     elif act_fn == "mish":
          return nn.Mish()
     elif act_fn == "gelu":
          return nn.GELU()
     else:
          raise ValueError(f"Unsupported activation function: {act_fn}"		)

