




from ...configuration_utils import PretrainedConfig
from ...utils import logging


_UpperCAmelCase	:						List[Any]			=     logging.get_logger(__name__)

_UpperCAmelCase	:						str			=     {
    "s-JoL/Open-Llama-V1": "https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json",
}




class    __lowerCAmelCase       (							lowerCAmelCase):
       _a 					=						'''open-llama'''






       def __init__(   self:					Optional[Any]       ,					_lowerCAmelCase:					Optional[int]=10_00_00       ,					_lowerCAmelCase:					Union[str, Any]=40_96       ,					_lowerCAmelCase:					Union[str, Any]=1_10_08       ,					_lowerCAmelCase:					Optional[int]=32       ,					_lowerCAmelCase:					Dict=32       ,					_lowerCAmelCase:					Tuple="silu"       ,					_lowerCAmelCase:					List[str]=20_48       ,					_lowerCAmelCase:					Optional[int]=0.02       ,					_lowerCAmelCase:					Tuple=1e-6       ,					_lowerCAmelCase:					List[Any]=True       ,					_lowerCAmelCase:					List[Any]=0       ,					_lowerCAmelCase:					Tuple=1       ,					_lowerCAmelCase:					int=2       ,					_lowerCAmelCase:					str=False       ,					_lowerCAmelCase:					Optional[int]=True       ,					_lowerCAmelCase:					Tuple=0.1       ,					_lowerCAmelCase:					Optional[Any]=0.1       ,					_lowerCAmelCase:					List[str]=True       ,					_lowerCAmelCase:					List[Any]=True       ,					_lowerCAmelCase:					Optional[Any]=None       ,					**_lowerCAmelCase:					Dict       ,					):
         lowercase    :Tuple      	=						vocab_size
         lowercase    :Optional[Any]      	=						max_position_embeddings
         lowercase    :Optional[int]      	=						hidden_size
         lowercase    :Optional[int]      	=						intermediate_size
         lowercase    :Tuple      	=						num_hidden_layers
         lowercase    :List[Any]      	=						num_attention_heads
         lowercase    :Tuple      	=						hidden_act
         lowercase    :Any      	=						initializer_range
         lowercase    :List[Any]      	=						rms_norm_eps
         lowercase    :Dict      	=						use_cache
         lowercase    :List[Any]      	=						kwargs.pop(
             "use_memorry_efficient_attention"       ,					_lowerCAmelCase				)
         lowercase    :int      	=						hidden_dropout_prob
         lowercase    :Optional[int]      	=						attention_dropout_prob
         lowercase    :Any      	=						use_stable_embedding
         lowercase    :int      	=						shared_input_output_embedding
         lowercase    :List[Any]      	=						rope_scaling
         self._rope_scaling_validation()

         super().__init__(
             pad_token_id=_lowerCAmelCase       ,					bos_token_id=_lowerCAmelCase       ,					eos_token_id=_lowerCAmelCase       ,					tie_word_embeddings=_lowerCAmelCase       ,					**_lowerCAmelCase       ,					)






       def 					SCREAMING_SNAKE_CASE    (   self:					List[Any]				):
         if self.rope_scaling is None:
           return

         if not isinstance(self.rope_scaling       ,					_lowerCAmelCase				) or len(self.rope_scaling				) != 2:
           raise ValueError(
               "`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, "
               F"got {self.rope_scaling}"				)
         lowercase    :Dict      	=						self.rope_scaling.get("type"       ,					_lowerCAmelCase				)
         lowercase    :Optional[int]      	=						self.rope_scaling.get("factor"       ,					_lowerCAmelCase				)
         if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
           raise ValueError(
               F"`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"				)
         if rope_scaling_factor is None or not isinstance(_lowerCAmelCase       ,					_lowerCAmelCase				) or rope_scaling_factor <= 1.0:
           raise ValueError(F"`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"				)






import inspect
import os
import unittest

import torch

import accelerate
from accelerate import debug_launcher
from accelerate.test_utils import (
    execute_subprocess_async,
    require_cpu,
    require_huggingface_suite,
    require_multi_gpu,
    require_single_gpu,
)
from accelerate.utils import patch_environment




@require_huggingface_suite
class    __lowerCAmelCase       (							unittest.TestCase):






       def 					SCREAMING_SNAKE_CASE    (   self:					List[str]				):
         lowercase    :List[str]      	=						inspect.getfile(accelerate.test_utils				)
         lowercase    :List[Any]      	=						os.path.sep.join(
             mod_file.split(os.path.sep				)[:-1] + ["scripts", "external_deps", "test_metrics.py"]				)

         from accelerate.test_utils.scripts.external_deps import test_metrics  # noqa: F401

         lowercase    :str      	=						test_metrics






       @require_cpu
       def 					SCREAMING_SNAKE_CASE    (   self:					Tuple				):
         debug_launcher(self.test_metrics.main       ,					num_processes=1				)






       @require_cpu
       def 					SCREAMING_SNAKE_CASE    (   self:					int				):
         debug_launcher(self.test_metrics.main				)






       @require_single_gpu
       def 					SCREAMING_SNAKE_CASE    (   self:					List[Any]				):
         self.test_metrics.main()






       @require_multi_gpu
       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[Any]				):
         print(F"Found {torch.cuda.device_count()} devices."				)
         lowercase    :str      	=						["torchrun", F"--nproc_per_node={torch.cuda.device_count()}", self.test_file_path]
         with patch_environment(omp_num_threads=1				):
           execute_subprocess_async(_lowerCAmelCase       ,					env=os.environ.copy()				)



