
import importlib.util
import json
import os
import warnings
from dataclasses import dataclass, field

import torch

from ..training_args import TrainingArguments
from ..utils import cached_property, is_sagemaker_dp_enabled, logging


__snake_case:  List[str]								=			logging.get_logger(__name__)






def 	_snake_case						(      ):
  """simple docstring"""







  a_      :	Tuple      =							os.getenv("""SM_HP_MP_PARAMETERS"""     ,		"""{}"""   )
  try:
    # Parse it and check the field "partitions" is included, it is required for model parallel.
    a_      :	int      =							json.loads(A_   )
    if "partitions" not in smp_options:
      return False
  except json.JSONDecodeError:
    return False

  # Get the sagemaker specific framework parameters from mpi_options variable.
  a_      :	Optional[Any]      =							os.getenv("""SM_FRAMEWORK_PARAMS"""     ,		"""{}"""   )
  try:
    # Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
    a_      :	str      =							json.loads(A_   )
    if not mpi_options.get("""sagemaker_mpi_enabled"""     ,		A_   ):
      return False
  except json.JSONDecodeError:
    return False
  # Lastly, check if the `smdistributed` module is present.
  return importlib.util.find_spec("""smdistributed"""   ) is not None


if is_sagemaker_model_parallel_available():
    import smdistributed.modelparallel.torch as smp

    smp.init()




@dataclass
class  _UpperCAmelCase ( lowerCAmelCase__    ):

     """simple docstring"""

     a_    	=					field(
         default=""	,metadata={"help": "Used by the SageMaker launcher to send mp-specific args. Ignored in SageMakerTrainer"}	,)


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       super().__post_init__()
       warnings.warn(
           """`SageMakerTrainingArguments` is deprecated and will be removed in v5 of Transformers. You can use """
           """`TrainingArguments` instead."""							,		lowerCAmelCase_							,		)


     @cached_property
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       logger.info("""PyTorch: setting up devices""" )
       if torch.distributed.is_available() and torch.distributed.is_initialized() and self.local_rank == -1:
         logger.warning(
             """torch.distributed process group is initialized, but local_rank == -1. """
             """In order to use Torch DDP, launch your script with `python -m torch.distributed.launch""" )
       if self.no_cuda:
         a_      :	List[str]      =							torch.device("""cpu""" )
         a_      :	str      =							0
       elif is_sagemaker_model_parallel_available():
         a_      :	List[Any]      =							smp.local_rank()
         a_      :	int      =							torch.device("""cuda"""							,		lowerCAmelCase_ )
         a_      :	Optional[Any]      =							1
       elif is_sagemaker_dp_enabled():
         import smdistributed.dataparallel.torch.torch_smddp  # noqa: F401

         torch.distributed.init_process_group(backend="""smddp"""							,		timeout=self.ddp_timeout_delta )
         a_      :	str      =							int(os.getenv("""SMDATAPARALLEL_LOCAL_RANK""" ) )
         a_      :	str      =							torch.device("""cuda"""							,		self.local_rank )
         a_      :	int      =							1
       elif self.local_rank == -1:
         # if n_gpu is > 1 we'll use nn.DataParallel.
         # If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
         # Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
         # trigger an error that a device index is missing. Index 0 takes into account the
         # GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
         # will use the first GPU in that env, i.e. GPU#1
         a_      :	Union[str, Any]      =							torch.device("""cuda:0""" if torch.cuda.is_available() else """cpu""" )
         # Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
         # the default value.
         a_      :	Tuple      =							torch.cuda.device_count()
       else:
         # Here, we'll use torch.distributed.
         # Initializes the distributed backend which will take care of synchronizing nodes/GPUs
         if not torch.distributed.is_initialized():
           torch.distributed.init_process_group(backend="""nccl"""							,		timeout=self.ddp_timeout_delta )
         a_      :	Optional[Any]      =							torch.device("""cuda"""							,		self.local_rank )
         a_      :	Union[str, Any]      =							1

       if device.type == "cuda":
         torch.cuda.set_device(lowerCAmelCase_ )

       return device


     @property
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       if is_sagemaker_model_parallel_available():
         return smp.dp_size()

       return super().world_size


     @property
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       return not is_sagemaker_model_parallel_available()






     @property
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       return False




'''simple docstring'''

from torch import nn






def 	_snake_case						(      A_       :    Dict   ):
  """simple docstring"""







  if act_fn in ["swish", "silu"]:
    return nn.SiLU()
  elif act_fn == "mish":
    return nn.Mish()
  elif act_fn == "gelu":
    return nn.GELU()
  else:
    raise ValueError(f'''Unsupported activation function: {act_fn}'''   )


