




"""simple docstring"""





from ...configuration_utils import PretrainedConfig
from ...utils import logging


A   =       logging.get_logger(__name__)

A   =       {
    "google/switch-base-8": "https://huggingface.co/google/switch-base-8/blob/main/config.json",
}



class      A				(	_UpperCAmelCase   ):
			lowercase_    =	'''switch_transformers'''
			lowercase_    =	['''past_key_values''']
			lowercase_    =	{'''hidden_size''': '''d_model''', '''num_attention_heads''': '''num_heads''', '''num_hidden_layers''': '''num_layers'''}


			def __init__(      self      :    Optional[int]      ,  UpperCamelCase_      :    List[str]=32128      ,  UpperCamelCase_      :    Tuple=768      ,  UpperCamelCase_      :    List[str]=64      ,  UpperCamelCase_      :    Optional[Any]=2048      ,  UpperCamelCase_      :    Dict=64      ,  UpperCamelCase_      :    Optional[int]=12      ,  UpperCamelCase_      :    int=3      ,  UpperCamelCase_      :    Tuple=12      ,  UpperCamelCase_      :    List[str]=3      ,  UpperCamelCase_      :    List[str]=12      ,  UpperCamelCase_      :    List[Any]=8      ,  UpperCamelCase_      :    List[str]=False      ,  UpperCamelCase_      :    Dict=0.01      ,  UpperCamelCase_      :    Dict="float32"      ,  UpperCamelCase_      :    int=False      ,  UpperCamelCase_      :    Union[str, Any]=32      ,  UpperCamelCase_      :    Optional[int]=128      ,  UpperCamelCase_      :    Union[str, Any]=0.1      ,  UpperCamelCase_      :    Any=1e-6      ,  UpperCamelCase_      :    int=0.001      ,  UpperCamelCase_      :    Union[str, Any]=0.001      ,  UpperCamelCase_      :    Dict=1.0      ,  UpperCamelCase_      :    List[str]="relu"      ,  UpperCamelCase_      :    List[str]=True      ,  UpperCamelCase_      :    Optional[Any]=False      ,  UpperCamelCase_      :    Union[str, Any]=True      ,  UpperCamelCase_      :    Dict=0      ,  UpperCamelCase_      :    List[Any]=1      ,  **UpperCamelCase_      :    Dict      ,  ):


						"""simple docstring"""




						__UpperCAmelCase	: int		=					vocab_size
						__UpperCAmelCase	: Any		=					d_model
						__UpperCAmelCase	: int		=					d_kv
						__UpperCAmelCase	: List[str]		=					d_ff

						__UpperCAmelCase	: str		=					num_sparse_encoder_layers

						__UpperCAmelCase	: str		=					num_layers
						__UpperCAmelCase	: Dict		=					(
						    num_decoder_layers if num_decoder_layers is not None else self.num_layers
						)  # default = symmetry
						__UpperCAmelCase	: Tuple		=					num_sparse_decoder_layers

						# This tells us, each how many encoder layer we'll have to set a sparse layer.
						if self.num_sparse_encoder_layers > 0:
									__UpperCAmelCase	: List[Any]		=					self.num_layers // self.num_sparse_encoder_layers
						else:
									__UpperCAmelCase	: List[str]		=					self.num_layers  # HACK: this will create 0 sparse layers

						# This tells us, each how many encoder layer we'll have to set a sparse layer.
						if self.num_sparse_decoder_layers > 0:
									__UpperCAmelCase	: Any		=					self.num_decoder_layers // self.num_sparse_decoder_layers
						else:
									__UpperCAmelCase	: Dict		=					self.num_decoder_layers  # HACK: this will create 0 sparse layers

						__UpperCAmelCase	: Dict		=					num_heads
						__UpperCAmelCase	: List[str]		=					num_experts
						__UpperCAmelCase	: int		=					expert_capacity
						__UpperCAmelCase	: List[Any]		=					router_bias
						__UpperCAmelCase	: Any		=					router_jitter_noise
						if router_dtype not in ["float32", "float16", "bfloat16"]:
									raise ValueError(F"`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}")
						__UpperCAmelCase	: Optional[int]		=					router_dtype

						__UpperCAmelCase	: str		=					router_ignore_padding_tokens
						__UpperCAmelCase	: int		=					relative_attention_num_buckets
						__UpperCAmelCase	: Union[str, Any]		=					relative_attention_max_distance

						__UpperCAmelCase	: List[str]		=					dropout_rate
						__UpperCAmelCase	: Union[str, Any]		=					layer_norm_epsilon
						__UpperCAmelCase	: Union[str, Any]		=					initializer_factor
						__UpperCAmelCase	: str		=					feed_forward_proj
						__UpperCAmelCase	: List[Any]		=					use_cache
						__UpperCAmelCase	: Union[str, Any]		=					add_router_probs

						__UpperCAmelCase	: Optional[int]		=					router_z_loss_coef
						__UpperCAmelCase	: List[Any]		=					router_aux_loss_coef

						__UpperCAmelCase	: Union[str, Any]		=					self.feed_forward_proj.split("-")
						__UpperCAmelCase	: str		=					act_info[-1]
						__UpperCAmelCase	: Union[str, Any]		=					act_info[0] == "gated"

						if len(A_) > 1 and act_info[0] != "gated" or len(A_) > 2:
									raise ValueError(
									    F"`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."
									    "Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. "
									    "\'gated-gelu\' or \'relu\'")

						# for backwards compatibility
						if feed_forward_proj == "gated-gelu":
									__UpperCAmelCase	: Optional[int]		=					"gelu_new"

						super().__init__(
						    pad_token_id=A_      ,  eos_token_id=A_      ,  is_encoder_decoder=A_      ,  **A_      ,  )








"""simple docstring"""





import json
import logging
import os
import socket

import git
import numpy as np
import torch


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - PID: %(process)d -  %(message)s""",
    datefmt="""%m/%d/%Y %H:%M:%S""",
    level=logging.INFO,
)
A   =       logging.getLogger(__name__)

def 					_UpperCamelCase    (			UpperCamelCase							)			->			Any:


			"""simple docstring"""
			__UpperCAmelCase	: Optional[Any]		=					git.Repo(search_parent_directories=UpperCamelCase							)
			__UpperCAmelCase	: Any		=					{
			    "repo_id": str(UpperCamelCase							),
			    "repo_sha": str(repo.head.object.hexsha							),
			    "repo_branch": str(repo.active_branch							),
			}

			with open(os.path.join(UpperCamelCase				,       "git_log.json"							)				,       "w"							) as f:
						json.dump(UpperCamelCase				,       UpperCamelCase				,       indent=4							)

def 					_UpperCamelCase    (			UpperCamelCase							)			->			str:


			"""simple docstring"""
			if params.n_gpu <= 0:
						__UpperCAmelCase	: str		=					0
						__UpperCAmelCase	: Dict		=					-1
						__UpperCAmelCase	: Tuple		=					True
						__UpperCAmelCase	: List[str]		=					False
						return

			assert torch.cuda.is_available()

			logger.info("Initializing GPUs"							)
			if params.n_gpu > 1:
						assert params.local_rank != -1

						__UpperCAmelCase	: Optional[int]		=					int(os.environ["WORLD_SIZE"]							)
						__UpperCAmelCase	: Union[str, Any]		=					int(os.environ["N_GPU_NODE"]							)
						__UpperCAmelCase	: Optional[int]		=					int(os.environ["RANK"]							)

						# number of nodes / node ID
						__UpperCAmelCase	: int		=					params.world_size // params.n_gpu_per_node
						__UpperCAmelCase	: Optional[int]		=					params.global_rank // params.n_gpu_per_node
						__UpperCAmelCase	: Tuple		=					True

						assert params.n_nodes == int(os.environ["N_NODES"]							)
						assert params.node_id == int(os.environ["NODE_RANK"]							)

			# local job (single GPU)
			else:
						assert params.local_rank == -1

						__UpperCAmelCase	: Union[str, Any]		=					1
						__UpperCAmelCase	: Dict		=					0
						__UpperCAmelCase	: List[Any]		=					0
						__UpperCAmelCase	: Tuple		=					0
						__UpperCAmelCase	: Any		=					1
						__UpperCAmelCase	: str		=					1
						__UpperCAmelCase	: Dict		=					False

			# sanity checks
			assert params.n_nodes >= 1
			assert 0 <= params.node_id < params.n_nodes
			assert 0 <= params.local_rank <= params.global_rank < params.world_size
			assert params.world_size == params.n_nodes * params.n_gpu_per_node

			# define whether this is the master process / if we are in multi-node distributed mode
			__UpperCAmelCase	: Union[str, Any]		=					params.node_id == 0 and params.local_rank == 0
			__UpperCAmelCase	: Dict		=					params.n_nodes > 1

			# summary
			__UpperCAmelCase	: Any		=					f"--- Global rank: {params.global_rank} - "
			logger.info(PREFIX + "Number of nodes: %i" % params.n_nodes							)
			logger.info(PREFIX + "Node ID        : %i" % params.node_id							)
			logger.info(PREFIX + "Local rank     : %i" % params.local_rank							)
			logger.info(PREFIX + "World size     : %i" % params.world_size							)
			logger.info(PREFIX + "GPUs per node  : %i" % params.n_gpu_per_node							)
			logger.info(PREFIX + "Master         : %s" % str(params.is_master							)							)
			logger.info(PREFIX + "Multi-node     : %s" % str(params.multi_node							)							)
			logger.info(PREFIX + "Multi-GPU      : %s" % str(params.multi_gpu							)							)
			logger.info(PREFIX + "Hostname       : %s" % socket.gethostname()							)

			# set GPU device
			torch.cuda.set_device(params.local_rank							)

			# initialize multi-GPU
			if params.multi_gpu:
						logger.info("Initializing PyTorch distributed"							)
						torch.distributed.init_process_group(
						    init_method="env://"				,       backend="nccl"				,       )





def 					_UpperCamelCase    (			UpperCamelCase							)			->			Tuple:


			"""simple docstring"""
			np.random.seed(args.seed							)
			torch.manual_seed(args.seed							)
			if args.n_gpu > 0:
						torch.cuda.manual_seed_all(args.seed							)


