



from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase_								=     logging.get_logger(__name__)

lowercase_								=     {}
class 				_snake_case			(					lowercase__):
				UpperCamelCase__				:					Optional[int]     		="""llama"""
				UpperCamelCase__				:					str     		=["""past_key_values"""]




				def __init__(						self					:						Optional[int],					__lowercase					:						str=3_2000,					__lowercase					:						Union[str, Any]=4096,					__lowercase					:						str=1_1008,					__lowercase					:						List[str]=32,					__lowercase					:						Optional[int]=32,					__lowercase					:						Tuple=None,					__lowercase					:						str="silu",					__lowercase					:						Optional[int]=2048,					__lowercase					:						Any=0.02,					__lowercase					:						Optional[Any]=1e-6,					__lowercase					:						Optional[int]=True,					__lowercase					:						int=0,					__lowercase					:						Optional[int]=1,					__lowercase					:						str=2,					__lowercase					:						List[Any]=1,					__lowercase					:						Any=False,					__lowercase					:						int=None,					**__lowercase					:						Optional[int],					):
							lowercase__					  =				vocab_size
							lowercase__					  =				max_position_embeddings
							lowercase__					  =				hidden_size
							lowercase__					  =				intermediate_size
							lowercase__					  =				num_hidden_layers
							lowercase__					  =				num_attention_heads

							# for backward compatibility
							if num_key_value_heads is None:
										lowercase__					  =				num_attention_heads

							lowercase__					  =				num_key_value_heads
							lowercase__					  =				hidden_act
							lowercase__					  =				initializer_range
							lowercase__					  =				rms_norm_eps
							lowercase__					  =				pretraining_tp
							lowercase__					  =				use_cache
							lowercase__					  =				rope_scaling
							self._rope_scaling_validation()

							super().__init__(
							    pad_token_id=__lowercase,					bos_token_id=__lowercase,					eos_token_id=__lowercase,					tie_word_embeddings=__lowercase,					**__lowercase,					)




				def 							A__      (						self					:						List[Any]						):
							if self.rope_scaling is None:
										return

							if not isinstance(self.rope_scaling,					__lowercase						) or len(self.rope_scaling						) != 2:
										raise ValueError(
										    "`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, "
										    F'''got {self.rope_scaling}'''						)
							lowercase__					  =				self.rope_scaling.get("type",					__lowercase						)
							lowercase__					  =				self.rope_scaling.get("factor",					__lowercase						)
							if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
										raise ValueError(
										    F'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'''						)
							if rope_scaling_factor is None or not isinstance(__lowercase,					__lowercase						) or rope_scaling_factor <= 1.0:
										raise ValueError(F'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'''						)









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


lowercase_								=     {
    """configuration_roberta_prelayernorm""": [
        """ROBERTA_PRELAYERNORM_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """RobertaPreLayerNormConfig""",
        """RobertaPreLayerNormOnnxConfig""",
    ],
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowercase_								=     [
							    """ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """RobertaPreLayerNormForCausalLM""",
							    """RobertaPreLayerNormForMaskedLM""",
							    """RobertaPreLayerNormForMultipleChoice""",
							    """RobertaPreLayerNormForQuestionAnswering""",
							    """RobertaPreLayerNormForSequenceClassification""",
							    """RobertaPreLayerNormForTokenClassification""",
							    """RobertaPreLayerNormModel""",
							    """RobertaPreLayerNormPreTrainedModel""",
							]

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowercase_								=     [
							    """TF_ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """TFRobertaPreLayerNormForCausalLM""",
							    """TFRobertaPreLayerNormForMaskedLM""",
							    """TFRobertaPreLayerNormForMultipleChoice""",
							    """TFRobertaPreLayerNormForQuestionAnswering""",
							    """TFRobertaPreLayerNormForSequenceClassification""",
							    """TFRobertaPreLayerNormForTokenClassification""",
							    """TFRobertaPreLayerNormMainLayer""",
							    """TFRobertaPreLayerNormModel""",
							    """TFRobertaPreLayerNormPreTrainedModel""",
							]

try:
							if not is_flax_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowercase_								=     [
							    """FlaxRobertaPreLayerNormForCausalLM""",
							    """FlaxRobertaPreLayerNormForMaskedLM""",
							    """FlaxRobertaPreLayerNormForMultipleChoice""",
							    """FlaxRobertaPreLayerNormForQuestionAnswering""",
							    """FlaxRobertaPreLayerNormForSequenceClassification""",
							    """FlaxRobertaPreLayerNormForTokenClassification""",
							    """FlaxRobertaPreLayerNormModel""",
							    """FlaxRobertaPreLayerNormPreTrainedModel""",
							]


if TYPE_CHECKING:
							from .configuration_roberta_prelayernorm import (
							    ROBERTA_PRELAYERNORM_PRETRAINED_CONFIG_ARCHIVE_MAP,
							    RobertaPreLayerNormConfig,
							    RobertaPreLayerNormOnnxConfig,
							)

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_roberta_prelayernorm import (
														    ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST,
														    RobertaPreLayerNormForCausalLM,
														    RobertaPreLayerNormForMaskedLM,
														    RobertaPreLayerNormForMultipleChoice,
														    RobertaPreLayerNormForQuestionAnswering,
														    RobertaPreLayerNormForSequenceClassification,
														    RobertaPreLayerNormForTokenClassification,
														    RobertaPreLayerNormModel,
														    RobertaPreLayerNormPreTrainedModel,
														)

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_roberta_prelayernorm import (
														    TF_ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFRobertaPreLayerNormForCausalLM,
														    TFRobertaPreLayerNormForMaskedLM,
														    TFRobertaPreLayerNormForMultipleChoice,
														    TFRobertaPreLayerNormForQuestionAnswering,
														    TFRobertaPreLayerNormForSequenceClassification,
														    TFRobertaPreLayerNormForTokenClassification,
														    TFRobertaPreLayerNormMainLayer,
														    TFRobertaPreLayerNormModel,
														    TFRobertaPreLayerNormPreTrainedModel,
														)

							try:
														if not is_flax_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_flax_roberta_prelayernorm import (
														    FlaxRobertaPreLayerNormForCausalLM,
														    FlaxRobertaPreLayerNormForMaskedLM,
														    FlaxRobertaPreLayerNormForMultipleChoice,
														    FlaxRobertaPreLayerNormForQuestionAnswering,
														    FlaxRobertaPreLayerNormForSequenceClassification,
														    FlaxRobertaPreLayerNormForTokenClassification,
														    FlaxRobertaPreLayerNormModel,
														    FlaxRobertaPreLayerNormPreTrainedModel,
														)

else:
							import sys

							lowercase_								=     _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




