






from ...configuration_utils import PretrainedConfig
from ...utils import logging


__A      =				logging.get_logger(__name__)

__A      =				{
    's-JoL/Open-Llama-V1': 'https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json',
}





class 						SCREAMING_SNAKE_CASE				( snake_case					):





							"""simple docstring"""

							A_						  =					"open-llama"

							def __init__(  self:   int		,     __A:   Optional[Any]=10_00_00		,     __A:   Union[str, Any]=40_96		,     __A:   List[str]=1_10_08		,     __A:   Union[str, Any]=32		,     __A:   Any=32		,     __A:   str="silu"		,     __A:   str=20_48		,     __A:   Any=0.02		,     __A:   Optional[Any]=1e-6		,     __A:   str=True		,     __A:   Optional[Any]=0		,     __A:   List[Any]=1		,     __A:   int=2		,     __A:   str=False		,     __A:   List[str]=True		,     __A:   List[str]=0.1		,     __A:   str=0.1		,     __A:   str=True		,     __A:   List[str]=True		,     __A:   List[str]=None		,     **__A:   List[str]		,     )    ->				str:
												_A								= vocab_size
												_A								= max_position_embeddings
												_A								= hidden_size
												_A								= intermediate_size
												_A								= num_hidden_layers
												_A								= num_attention_heads
												_A								= hidden_act
												_A								= initializer_range
												_A								= rms_norm_eps
												_A								= use_cache
												_A								= kwargs.pop(
												    '''use_memorry_efficient_attention'''		,     __A   )
												_A								= hidden_dropout_prob
												_A								= attention_dropout_prob
												_A								= use_stable_embedding
												_A								= shared_input_output_embedding
												_A								= rope_scaling
												self._rope_scaling_validation()

												super().__init__(
												    pad_token_id=__A		,     bos_token_id=__A		,     eos_token_id=__A		,     tie_word_embeddings=__A		,     **__A		,     )



							def       __A (  self:   List[Any]   )    ->				Optional[Any]:
												if self.rope_scaling is None:
																	return

												if not isinstance(self.rope_scaling		,     __A   ) or len(self.rope_scaling   ) != 2:
																	raise ValueError(
																	    '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
																	    f"""got {self.rope_scaling}"""   )
												_A								= self.rope_scaling.get('''type'''		,     __A   )
												_A								= self.rope_scaling.get('''factor'''		,     __A   )
												if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
																	raise ValueError(
																	    f"""`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"""   )
												if rope_scaling_factor is None or not isinstance(__A		,     __A   ) or rope_scaling_factor <= 1.0:
																	raise ValueError(f"""`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"""   )











from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


__A      =				{
    'configuration_mega': ['MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP', 'MegaConfig', 'MegaOnnxConfig'],
}

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				__A      =				[
				    'MEGA_PRETRAINED_MODEL_ARCHIVE_LIST',
				    'MegaForCausalLM',
				    'MegaForMaskedLM',
				    'MegaForMultipleChoice',
				    'MegaForQuestionAnswering',
				    'MegaForSequenceClassification',
				    'MegaForTokenClassification',
				    'MegaModel',
				    'MegaPreTrainedModel',
				]

if TYPE_CHECKING:
				from .configuration_mega import MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP, MegaConfig, MegaOnnxConfig

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_mega import (
								    MEGA_PRETRAINED_MODEL_ARCHIVE_LIST,
								    MegaForCausalLM,
								    MegaForMaskedLM,
								    MegaForMultipleChoice,
								    MegaForQuestionAnswering,
								    MegaForSequenceClassification,
								    MegaForTokenClassification,
								    MegaModel,
								    MegaPreTrainedModel,
								)

else:
				import sys

				__A      =				_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)



