



import importlib.util
import json
import os
import warnings
from dataclasses import dataclass, field

import torch

from ..training_args import TrainingArguments
from ..utils import cached_property, is_sagemaker_dp_enabled, logging


A_		:     Tuple        =     logging.get_logger(__name__)


def 	UpperCAmelCase__      (      ):
 a						=		os.getenv("SM_HP_MP_PARAMETERS" ,					"{}"					)
 try:
  # Parse it and check the field "partitions" is included, it is required for model parallel.
  a						=		json.loads(UpperCAmelCase__					)
  if "partitions" not in smp_options:
   return False
 except json.JSONDecodeError:
  return False

 # Get the sagemaker specific framework parameters from mpi_options variable.
 a						=		os.getenv("SM_FRAMEWORK_PARAMS" ,					"{}"					)
 try:
  # Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
  a						=		json.loads(UpperCAmelCase__					)
  if not mpi_options.get("sagemaker_mpi_enabled" ,					UpperCAmelCase__					):
   return False
 except json.JSONDecodeError:
  return False
 # Lastly, check if the `smdistributed` module is present.
 return importlib.util.find_spec("smdistributed"					) is not None


if is_sagemaker_model_parallel_available():
     import smdistributed.modelparallel.torch as smp

     smp.init()




@dataclass
class 		_lowercase		(       __lowerCAmelCase	):
   _UpperCAmelCase     	=			field(
       default='''''',   metadata={'''help''': '''Used by the SageMaker launcher to send mp-specific args. Ignored in SageMakerTrainer'''},   )

   def 		A							(      self			:       List[Any]							)       ->				str:


    """simple docstring"""






    super().__post_init__()
    warnings.warn(
        "`SageMakerTrainingArguments` is deprecated and will be removed in v5 of Transformers. You can use "
        "`TrainingArguments` instead."							,   lowerCamelCase__							,   )

   @cached_property
   def 		A							(      self			:       Dict							)       ->				"torch.device":


    """simple docstring"""






    logger.info("PyTorch: setting up devices"							)
    if torch.distributed.is_available() and torch.distributed.is_initialized() and self.local_rank == -1:
     logger.warning(
         "torch.distributed process group is initialized, but local_rank == -1. "
         "In order to use Torch DDP, launch your script with `python -m torch.distributed.launch"							)
    if self.no_cuda:
     a						=		torch.device("cpu"							)
     a						=		0
    elif is_sagemaker_model_parallel_available():
     a						=		smp.local_rank()
     a						=		torch.device("cuda"							,   lowerCamelCase__							)
     a						=		1
    elif is_sagemaker_dp_enabled():
     import smdistributed.dataparallel.torch.torch_smddp  # noqa: F401

     torch.distributed.init_process_group(backend="smddp"							,   timeout=self.ddp_timeout_delta							)
     a						=		int(os.getenv("SMDATAPARALLEL_LOCAL_RANK"							)							)
     a						=		torch.device("cuda"							,   self.local_rank							)
     a						=		1
    elif self.local_rank == -1:
     # if n_gpu is > 1 we'll use nn.DataParallel.
     # If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
     # Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
     # trigger an error that a device index is missing. Index 0 takes into account the
     # GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
     # will use the first GPU in that env, i.e. GPU#1
     a						=		torch.device("cuda:0" if torch.cuda.is_available() else "cpu"							)
     # Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
     # the default value.
     a						=		torch.cuda.device_count()
    else:
     # Here, we'll use torch.distributed.
     # Initializes the distributed backend which will take care of synchronizing nodes/GPUs
     if not torch.distributed.is_initialized():
      torch.distributed.init_process_group(backend="nccl"							,   timeout=self.ddp_timeout_delta							)
     a						=		torch.device("cuda"							,   self.local_rank							)
     a						=		1

    if device.type == "cuda":
     torch.cuda.set_device(lowerCamelCase__							)

    return device

   @property
   def 		A							(      self			:       List[Any]							)       ->				int:


    """simple docstring"""






    if is_sagemaker_model_parallel_available():
     return smp.dp_size()

    return super().world_size

   @property
   def 		A							(      self			:       List[Any]							)       ->				Any:


    """simple docstring"""






    return not is_sagemaker_model_parallel_available()




   @property
   def 		A							(      self			:       Union[str, Any]							)       ->				List[Any]:


    """simple docstring"""






    return False






from ...configuration_utils import PretrainedConfig
from ...utils import logging


A_		:     str        =     logging.get_logger(__name__)

A_		:     List[Any]        =     {
    '''RWKV/rwkv-4-169m-pile''': '''https://huggingface.co/RWKV/rwkv-4-169m-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-430m-pile''': '''https://huggingface.co/RWKV/rwkv-4-430m-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-1b5-pile''': '''https://huggingface.co/RWKV/rwkv-4-1b5-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-3b-pile''': '''https://huggingface.co/RWKV/rwkv-4-3b-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-7b-pile''': '''https://huggingface.co/RWKV/rwkv-4-7b-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-14b-pile''': '''https://huggingface.co/RWKV/rwkv-4-14b-pile/resolve/main/config.json''',
    '''RWKV/rwkv-raven-1b5''': '''https://huggingface.co/RWKV/rwkv-raven-1b5/resolve/main/config.json''',
    '''RWKV/rwkv-raven-3b''': '''https://huggingface.co/RWKV/rwkv-raven-3b/resolve/main/config.json''',
    '''RWKV/rwkv-raven-7b''': '''https://huggingface.co/RWKV/rwkv-raven-7b/resolve/main/config.json''',
    '''RWKV/rwkv-raven-14b''': '''https://huggingface.co/RWKV/rwkv-raven-14b/resolve/main/config.json''',
}



class 		_lowercase		(       UpperCAmelCase__	):
   _UpperCAmelCase     	=			'''rwkv'''
   _UpperCAmelCase     	=			{'''max_position_embeddings''': '''context_length'''}

   def __init__(      self			:       List[str]							,   __lowerCAmelCase			:       Union[str, Any]=5_0277							,   __lowerCAmelCase			:       str=1024							,   __lowerCAmelCase			:       Union[str, Any]=4096							,   __lowerCAmelCase			:       Optional[int]=32							,   __lowerCAmelCase			:       Optional[int]=None							,   __lowerCAmelCase			:       Optional[Any]=None							,   __lowerCAmelCase			:       List[Any]=1E-5							,   __lowerCAmelCase			:       Union[str, Any]=0							,   __lowerCAmelCase			:       Optional[Any]=0							,   __lowerCAmelCase			:       Dict=6							,   __lowerCAmelCase			:       int=False							,   __lowerCAmelCase			:       Tuple=True							,   **__lowerCAmelCase			:       List[str]							,   )       ->				List[Any]:


    """simple docstring"""






    a						=		vocab_size
    a						=		context_length
    a						=		hidden_size
    a						=		num_hidden_layers
    a						=		attention_hidden_size if attention_hidden_size is not None else hidden_size
    a						=		intermediate_size if intermediate_size is not None else 4 * hidden_size
    a						=		layer_norm_epsilon
    a						=		rescale_every
    a						=		use_cache

    a						=		bos_token_id
    a						=		eos_token_id

    super().__init__(
        tie_word_embeddings=__lowerCAmelCase							,   bos_token_id=__lowerCAmelCase							,   eos_token_id=__lowerCAmelCase							,   **__lowerCAmelCase							)

