

import gc
import threading
import time

import psutil
import torch

class 				a  :


						def __init__(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Tuple 		=    psutil.Process()
										_UpperCAmelCase       :		Optional[Any] 		=    False


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		int 		=    -1

										while True:
														_UpperCAmelCase       :		Optional[Any] 		=    max(self.process.memory_info().rss						,					self.cpu_memory_peak	)

														# can't sleep or will not catch the peak right (this comment is here on purpose)
														if not self.peak_monitoring:
																		break


						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Union[str, Any] 		=    True
										_UpperCAmelCase       :		Optional[Any] 		=    threading.Thread(target=self.peak_monitor	)
										_UpperCAmelCase       :		str 		=    True
										self.thread.start()





						def        _UpperCAmelCase				(       self	):







										'''simple docstring'''


										_UpperCAmelCase       :		Tuple 		=    False
										self.thread.join()
										return self.cpu_memory_peak







SCREAMING_SNAKE_CASE_        =      PeakCPUMemory()

def 							__SCREAMING_SNAKE_CASE							(				)   ->       Optional[Any]:
				# Time
				_UpperCAmelCase       :		Any 		=    {"time": time.time()}

				gc.collect()
				torch.cuda.empty_cache()

				# CPU mem
				_UpperCAmelCase       :		Union[str, Any] 		=    psutil.Process().memory_info().rss
				cpu_peak_tracker.start()

				# GPU mem
				for i in range(torch.cuda.device_count() ):
								_UpperCAmelCase       :		int 		=    torch.cuda.memory_allocated(lowerCAmelCase )
				torch.cuda.reset_peak_memory_stats()

				return measures

def 							__SCREAMING_SNAKE_CASE							(				lowerCAmelCase:  Union[str, Any] )   ->       int:
				# Time
				_UpperCAmelCase       :		List[str] 		=    {"time": time.time() - start_measures["time"]}

				gc.collect()
				torch.cuda.empty_cache()

				# CPU mem
				_UpperCAmelCase       :		List[str] 		=    (psutil.Process().memory_info().rss - start_measures["cpu"]) / 2**20
				_UpperCAmelCase       :		Optional[Any] 		=    (cpu_peak_tracker.stop() - start_measures["cpu"]) / 2**20

				# GPU mem
				for i in range(torch.cuda.device_count() ):
								_UpperCAmelCase       :		Union[str, Any] 		=    (torch.cuda.memory_allocated(lowerCAmelCase ) - start_measures[str(lowerCAmelCase )]) / 2**20
								_UpperCAmelCase       :		Dict 		=    (torch.cuda.max_memory_allocated(lowerCAmelCase ) - start_measures[str(lowerCAmelCase )]) / 2**20

				return measures





def 							__SCREAMING_SNAKE_CASE							(				lowerCAmelCase:  Optional[Any]  ,      lowerCAmelCase:  List[str] )   ->       Tuple:
				print(F'{description}:' )
				print(F'- Time: {measures["time"]:.2f}s' )
				for i in range(torch.cuda.device_count() ):
								print(F'- GPU {i} allocated: {measures[str(lowerCAmelCase )]:.2f}MiB' )
								_UpperCAmelCase       :		Optional[Any] 		=    measures[F'{i}-peak']
								print(F'- GPU {i} peak: {peak:.2f}MiB' )
				print(F'- CPU RAM allocated: {measures["cpu"]:.2f}MiB' )
				print(F'- CPU RAM peak: {measures["cpu-peak"]:.2f}MiB' )





from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_        =      logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_        =      {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/config.json'}

class 				a  (	UpperCAmelCase				):
						_lowercase    						=			"openai-gpt"
						_lowercase    						=			{
						    "max_position_embeddings": "n_positions",
						    "hidden_size": "n_embd",
						    "num_attention_heads": "n_head",
						    "num_hidden_layers": "n_layer",
						}


						def __init__(       self						,					A_=40478						,					A_=512						,					A_=768						,					A_=12						,					A_=12						,					A_="gelu"						,					A_=0.1						,					A_=0.1						,					A_=0.1						,					A_=1e-5						,					A_=0.02						,					A_="cls_index"						,					A_=True						,					A_=None						,					A_=True						,					A_=0.1						,					**A_						,					):







										'''simple docstring'''


										_UpperCAmelCase       :		Any 		=    vocab_size
										_UpperCAmelCase       :		str 		=    n_positions
										_UpperCAmelCase       :		List[Any] 		=    n_embd
										_UpperCAmelCase       :		Dict 		=    n_layer
										_UpperCAmelCase       :		Any 		=    n_head
										_UpperCAmelCase       :		int 		=    afn
										_UpperCAmelCase       :		int 		=    resid_pdrop
										_UpperCAmelCase       :		Tuple 		=    embd_pdrop
										_UpperCAmelCase       :		Optional[Any] 		=    attn_pdrop
										_UpperCAmelCase       :		Any 		=    layer_norm_epsilon
										_UpperCAmelCase       :		List[str] 		=    initializer_range
										_UpperCAmelCase       :		Union[str, Any] 		=    summary_type
										_UpperCAmelCase       :		List[Any] 		=    summary_use_proj
										_UpperCAmelCase       :		Optional[Any] 		=    summary_activation
										_UpperCAmelCase       :		int 		=    summary_first_dropout
										_UpperCAmelCase       :		List[str] 		=    summary_proj_to_labels
										super().__init__(**A_	)


