






from ...configuration_utils import PretrainedConfig
from ...utils import logging


A:      Tuple  				=				logging.get_logger(__name__)

A:      Optional[int]  				=				{
    "transfo-xl-wt103": "https://huggingface.co/transfo-xl-wt103/resolve/main/config.json",
}






class 							__magic_name__     (	UpperCAmelCase_				):







	"""simple docstring"""



	SCREAMING_SNAKE_CASE_					:      Any        =  'transfo-xl'
	SCREAMING_SNAKE_CASE_					:      int        =  ['mems']
	SCREAMING_SNAKE_CASE_					:      Tuple        =  {
	    'n_token': 'vocab_size',
	    'hidden_size': 'd_model',
	    'num_attention_heads': 'n_head',
	    'num_hidden_layers': 'n_layer',
	}
	def __init__(							self  ,      _lowercase=26_7735  ,      _lowercase=[2_0000, 4_0000, 20_0000]  ,      _lowercase=1024  ,      _lowercase=1024  ,      _lowercase=16  ,      _lowercase=64  ,      _lowercase=4096  ,      _lowercase=4  ,      _lowercase=False  ,      _lowercase=18  ,      _lowercase=1600  ,      _lowercase=1000  ,      _lowercase=True  ,      _lowercase=True  ,      _lowercase=0  ,      _lowercase=-1  ,      _lowercase=True  ,      _lowercase=0.1  ,      _lowercase=0.0  ,      _lowercase=True  ,      _lowercase="normal"  ,      _lowercase=0.01  ,      _lowercase=0.01  ,      _lowercase=0.02  ,      _lowercase=1E-5  ,      _lowercase=0  ,      **_lowercase  ,      )						->  Optional[Any]:
							lowercase_      :   Union[str, Any]        =			vocab_size
							lowercase_      :   str        =			[]
							self.cutoffs.extend(_lowercase       )
							if proj_share_all_but_first:
													lowercase_      :   List[Any]        =			[False] + [True] * len(self.cutoffs       )
							else:
													lowercase_      :   List[str]        =			[False] + [False] * len(self.cutoffs       )
							lowercase_      :   str        =			d_model
							lowercase_      :   int        =			d_embed
							lowercase_      :   Dict        =			d_head
							lowercase_      :   Optional[int]        =			d_inner
							lowercase_      :   int        =			div_val
							lowercase_      :   List[str]        =			pre_lnorm
							lowercase_      :   Any        =			n_layer
							lowercase_      :   Dict        =			n_head
							lowercase_      :   Tuple        =			mem_len
							lowercase_      :   Optional[int]        =			same_length
							lowercase_      :   List[str]        =			attn_type
							lowercase_      :   Optional[int]        =			clamp_len
							lowercase_      :   int        =			sample_softmax
							lowercase_      :   List[str]        =			adaptive
							lowercase_      :   List[str]        =			dropout
							lowercase_      :   Union[str, Any]        =			dropatt
							lowercase_      :   int        =			untie_r
							lowercase_      :   Optional[Any]        =			init
							lowercase_      :   Union[str, Any]        =			init_range
							lowercase_      :   Any        =			proj_init_std
							lowercase_      :   List[Any]        =			init_std
							lowercase_      :   Optional[Any]        =			layer_norm_epsilon
							super().__init__(eos_token_id=_lowercase  ,      **_lowercase       )
	@property
	def        lowerCamelCase__		(							self       )						->  int:
							# Message copied from Transformer-XL documentation
							logger.info(f"The model {self.model_type} is one of the few models that has no sequence length limit."       )
							return -1







	@max_position_embeddings.setter
	def        lowerCamelCase__		(							self  ,      _lowercase       )						->  Tuple:
							# Message copied from Transformer-XL documentation
							raise NotImplementedError(
							    f"The model {self.model_type} is one of the few models that has no sequence length limit."       )








'''simple docstring'''

import json
import logging
import os
import socket

import git
import numpy as np
import torch


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - PID: %(process)d -  %(message)s",
    datefmt="%m/%d/%Y %H:%M:%S",
    level=logging.INFO,
)
A:      Tuple  				=				logging.getLogger(__name__)






def 	_UpperCAmelCase    (  a		:	str							)			->			List[Any]:





						"""simple docstring"""
						lowercase_      :   List[str]        =			git.Repo(search_parent_directories=a							)
						lowercase_      :   Union[str, Any]        =			{
						    'repo_id': str(a							),
						    'repo_sha': str(repo.head.object.hexsha							),
						    'repo_branch': str(repo.active_branch							),
						}

						with open(os.path.join(a						,			'git_log.json'							)						,			'w'							) as f:
												json.dump(a						,			a						,			indent=4							)






def 	_UpperCAmelCase    (  a		:	str							)			->			Union[str, Any]:





						"""simple docstring"""
						if params.n_gpu <= 0:
												lowercase_      :   int        =			0
												lowercase_      :   Union[str, Any]        =			-1
												lowercase_      :   List[str]        =			True
												lowercase_      :   Optional[Any]        =			False
												return

						assert torch.cuda.is_available()

						logger.info('Initializing GPUs'							)
						if params.n_gpu > 1:
												assert params.local_rank != -1

												lowercase_      :   Dict        =			int(os.environ['WORLD_SIZE']							)
												lowercase_      :   Union[str, Any]        =			int(os.environ['N_GPU_NODE']							)
												lowercase_      :   Optional[int]        =			int(os.environ['RANK']							)

												# number of nodes / node ID
												lowercase_      :   int        =			params.world_size // params.n_gpu_per_node
												lowercase_      :   str        =			params.global_rank // params.n_gpu_per_node
												lowercase_      :   Dict        =			True

												assert params.n_nodes == int(os.environ['N_NODES']							)
												assert params.node_id == int(os.environ['NODE_RANK']							)

						# local job (single GPU)
						else:
												assert params.local_rank == -1

												lowercase_      :   str        =			1
												lowercase_      :   Dict        =			0
												lowercase_      :   Tuple        =			0
												lowercase_      :   List[Any]        =			0
												lowercase_      :   int        =			1
												lowercase_      :   Tuple        =			1
												lowercase_      :   str        =			False

						# sanity checks
						assert params.n_nodes >= 1
						assert 0 <= params.node_id < params.n_nodes
						assert 0 <= params.local_rank <= params.global_rank < params.world_size
						assert params.world_size == params.n_nodes * params.n_gpu_per_node

						# define whether this is the master process / if we are in multi-node distributed mode
						lowercase_      :   List[str]        =			params.node_id == 0 and params.local_rank == 0
						lowercase_      :   Optional[Any]        =			params.n_nodes > 1

						# summary
						lowercase_      :   int        =			f"--- Global rank: {params.global_rank} - "
						logger.info(PREFIX + 'Number of nodes: %i' % params.n_nodes							)
						logger.info(PREFIX + 'Node ID        : %i' % params.node_id							)
						logger.info(PREFIX + 'Local rank     : %i' % params.local_rank							)
						logger.info(PREFIX + 'World size     : %i' % params.world_size							)
						logger.info(PREFIX + 'GPUs per node  : %i' % params.n_gpu_per_node							)
						logger.info(PREFIX + 'Master         : %s' % str(params.is_master							)							)
						logger.info(PREFIX + 'Multi-node     : %s' % str(params.multi_node							)							)
						logger.info(PREFIX + 'Multi-GPU      : %s' % str(params.multi_gpu							)							)
						logger.info(PREFIX + 'Hostname       : %s' % socket.gethostname()							)

						# set GPU device
						torch.cuda.set_device(params.local_rank							)

						# initialize multi-GPU
						if params.multi_gpu:
												logger.info('Initializing PyTorch distributed'							)
												torch.distributed.init_process_group(
												    init_method='env://'						,			backend='nccl'						,			)






def 	_UpperCAmelCase    (  a		:	Dict							)			->			Optional[int]:





						"""simple docstring"""
						np.random.seed(args.seed							)
						torch.manual_seed(args.seed							)
						if args.n_gpu > 0:
												torch.cuda.manual_seed_all(args.seed							)
