
"""simple docstring"""





import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
		from transformers import LlamaTokenizerFast
except ImportError as e:
		warnings.warn(e)
		warnings.warn(
		    '''The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'''
		)
		UpperCamelCase_					:						int				= None
UpperCamelCase_					:						Dict				= {
    '''7B''': 1_1008,
    '''13B''': 1_3824,
    '''30B''': 1_7920,
    '''65B''': 2_2016,
    '''70B''': 2_8672,
}
UpperCamelCase_					:						Any				= {
    '''7B''': 1,
    '''7Bf''': 1,
    '''13B''': 2,
    '''13Bf''': 2,
    '''30B''': 4,
    '''65B''': 8,
    '''70B''': 8,
    '''70Bf''': 8,
}
def 							A_     (__a				,   __a=1				,   __a=256				):

					'''simple docstring'''





					return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3				)				) + multiple_of - 1) // multiple_of)
def 							A_     (__a				):

					'''simple docstring'''





					with open(__a				,   "r"				) as f:
										return json.load(__a				)
def 							A_     (__a				,   __a				):

					'''simple docstring'''





					with open(__a				,   "w"				) as f:
										json.dump(__a				,   __a				)
def 							A_     (__a				,   __a				,   __a				,   __a=True				):

					'''simple docstring'''





					os.makedirs(__a				,   exist_ok=__a				)
					A_				 =						os.path.join(__a				,   "tmp"				)
					os.makedirs(__a				,   exist_ok=__a				)

					A_				 =						read_json(os.path.join(__a				,   "params.json"				)				)
					A_				 =						NUM_SHARDS[model_size]
					A_				 =						params["n_layers"]
					A_				 =						params["n_heads"]
					A_				 =						n_heads // num_shards
					A_				 =						params["dim"]
					A_				 =						dim // n_heads
					A_				 =						10000.0
					A_				 =						1.0 / (base ** (torch.arange(0				,   __a				,   2				).float() / dims_per_head))

					if "n_kv_heads" in params:
										A_				 =						params["n_kv_heads"]  # for GQA / MQA
										A_				 =						n_heads_per_shard // num_key_value_heads
										A_				 =						dim // num_key_value_heads
					else:  # compatibility with other checkpoints
										A_				 =						n_heads
										A_				 =						n_heads_per_shard
										A_				 =						dim

					# permute for sliced rotary
					def permute(__a				,   __a=n_heads				,   __a=dim				,   __a=dim				):
										return w.view(__a				,   dima // n_heads // 2				,   2				,   __a				).transpose(1				,   2				).reshape(__a				,   __a				)

					print(f'Fetching all parameters from the checkpoint at {input_base_path}.'				)
					# Load weights
					if model_size == "7B":
										# Not sharded
										# (The sharded implementation would also work, but this is simpler.)
										A_				 =						torch.load(os.path.join(__a				,   "consolidated.00.pth"				)				,   map_location="cpu"				)
					else:
										# Sharded
										A_				 =						[
										    torch.load(os.path.join(__a				,   f'consolidated.{i:02d}.pth'				)				,   map_location="cpu"				)
										    for i in range(__a				)
										]
					A_				 =						0
					A_				 =						{"weight_map": {}}
					for layer_i in range(__a				):
										A_				 =						f'pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'
										if model_size == "7B":
															# Unsharded
															A_				 =						{
															    f'model.layers.{layer_i}.self_attn.q_proj.weight': permute(
															        loaded[f'layers.{layer_i}.attention.wq.weight']				),
															    f'model.layers.{layer_i}.self_attn.k_proj.weight': permute(
															        loaded[f'layers.{layer_i}.attention.wk.weight']				),
															    f'model.layers.{layer_i}.self_attn.v_proj.weight': loaded[f'layers.{layer_i}.attention.wv.weight'],
															    f'model.layers.{layer_i}.self_attn.o_proj.weight': loaded[f'layers.{layer_i}.attention.wo.weight'],
															    f'model.layers.{layer_i}.mlp.gate_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w1.weight'],
															    f'model.layers.{layer_i}.mlp.down_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w2.weight'],
															    f'model.layers.{layer_i}.mlp.up_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w3.weight'],
															    f'model.layers.{layer_i}.input_layernorm.weight': loaded[f'layers.{layer_i}.attention_norm.weight'],
															    f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[f'layers.{layer_i}.ffn_norm.weight'],
															}
										else:
															# Sharded
															# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
															# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
															# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

															A_				 =						{
															    f'model.layers.{layer_i}.input_layernorm.weight': loaded[0][
															        f'layers.{layer_i}.attention_norm.weight'
															    ].clone(),
															    f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[0][
															        f'layers.{layer_i}.ffn_norm.weight'
															    ].clone(),
															}
															A_				 =						permute(
															    torch.cat(
															        [
															            loaded[i][f'layers.{layer_i}.attention.wq.weight'].view(__a				,   __a				,   __a				)
															            for i in range(__a				)
															        ]				,   dim=0				,   ).reshape(__a				,   __a				)				)
															A_				 =						permute(
															    torch.cat(
															        [
															            loaded[i][f'layers.{layer_i}.attention.wk.weight'].view(
															                __a				,   __a				,   __a				)
															            for i in range(__a				)
															        ]				,   dim=0				,   ).reshape(__a				,   __a				)				,   __a				,   __a				,   __a				,   )
															A_				 =						torch.cat(
															    [
															        loaded[i][f'layers.{layer_i}.attention.wv.weight'].view(
															            __a				,   __a				,   __a				)
															        for i in range(__a				)
															    ]				,   dim=0				,   ).reshape(__a				,   __a				)

															A_				 =						torch.cat(
															    [loaded[i][f'layers.{layer_i}.attention.wo.weight'] for i in range(__a				)]				,   dim=1				)
															A_				 =						torch.cat(
															    [loaded[i][f'layers.{layer_i}.feed_forward.w1.weight'] for i in range(__a				)]				,   dim=0				)
															A_				 =						torch.cat(
															    [loaded[i][f'layers.{layer_i}.feed_forward.w2.weight'] for i in range(__a				)]				,   dim=1				)
															A_				 =						torch.cat(
															    [loaded[i][f'layers.{layer_i}.feed_forward.w3.weight'] for i in range(__a				)]				,   dim=0				)

										A_				 =						inv_freq
										for k, v in state_dict.items():
															A_				 =						filename
															param_count += v.numel()
										torch.save(__a				,   os.path.join(__a				,   __a				)				)

					A_				 =						f'pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'
					if model_size == "7B":
										# Unsharded
										A_				 =						{
										    "model.embed_tokens.weight": loaded["tok_embeddings.weight"],
										    "model.norm.weight": loaded["norm.weight"],
										    "lm_head.weight": loaded["output.weight"],
										}
					else:
										A_				 =						{
										    "model.norm.weight": loaded[0]["norm.weight"],
										    "model.embed_tokens.weight": torch.cat(
										        [loaded[i]["tok_embeddings.weight"] for i in range(__a				)]				,   dim=1				),
										    "lm_head.weight": torch.cat([loaded[i]["output.weight"] for i in range(__a				)]				,   dim=0				),
										}

					for k, v in state_dict.items():
										A_				 =						filename
										param_count += v.numel()
					torch.save(__a				,   os.path.join(__a				,   __a				)				)

					# Write configs
					A_				 =						{"total_size": param_count * 2}
					write_json(__a				,   os.path.join(__a				,   "pytorch_model.bin.index.json"				)				)
					A_				 =						params["ffn_dim_multiplier"] if "ffn_dim_multiplier" in params else 1
					A_				 =						params["multiple_of"] if "multiple_of" in params else 256
					A_				 =						LlamaConfig(
					    hidden_size=__a				,   intermediate_size=compute_intermediate_size(__a				,   __a				,   __a				)				,   num_attention_heads=params["n_heads"]				,   num_hidden_layers=params["n_layers"]				,   rms_norm_eps=params["norm_eps"]				,   num_key_value_heads=__a				,   )
					config.save_pretrained(__a				)

					# Make space so we can load the model properly now.
					del state_dict
					del loaded
					gc.collect()

					print("Loading the checkpoint in a Llama model."				)
					A_				 =						LlamaForCausalLM.from_pretrained(__a				,   torch_dtype=torch.floataa				,   low_cpu_mem_usage=__a				)
					# Avoid saving this as part of the config.
					del model.config._name_or_path

					print("Saving in the Transformers format."				)
					model.save_pretrained(__a				,   safe_serialization=__a				)
					shutil.rmtree(__a				)
def 							A_     (__a				,   __a				):

					'''simple docstring'''





					A_				 =						LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
					print(f'Saving a {tokenizer_class.__name__} to {tokenizer_path}.'				)
					A_				 =						tokenizer_class(__a				)
					tokenizer.save_pretrained(__a				)
def 							A_     ():

					'''simple docstring'''





					A_				 =						argparse.ArgumentParser()
					parser.add_argument(
					    "--input_dir"				,   help="Location of LLaMA weights, which contains tokenizer.model and model folders"				,   )
					parser.add_argument(
					    "--model_size"				,   choices=["7B", "7Bf", "13B", "13Bf", "30B", "65B", "70B", "70Bf", "tokenizer_only"]				,   )
					parser.add_argument(
					    "--output_dir"				,   help="Location to write HF model and tokenizer"				,   )
					parser.add_argument("--safe_serialization"				,   type=__a				,   help="Whether or not to save using `safetensors`."				)
					A_				 =						parser.parse_args()
					if args.model_size != "tokenizer_only":
										write_model(
										    model_path=args.output_dir				,   input_base_path=os.path.join(args.input_dir				,   args.model_size				)				,   model_size=args.model_size				,   safe_serialization=args.safe_serialization				,   )
					A_				 =						os.path.join(args.input_dir				,   "tokenizer.model"				)
					write_tokenizer(args.output_dir				,   __a				)


if __name__ == "__main__":
		main()






"""simple docstring"""





from collections import deque



class __lowerCAmelCase				:




			"""simple docstring"""







			def __init__(				self						:					Optional[Any]       ,   _snake_case						:					str       ,   _snake_case						:					int       ,   _snake_case						:					int	)      -> None:







								"""simple docstring"""
								A_				 =						process_name  # process name
								A_				 =						arrival_time  # arrival time of the process
								# completion time of finished process or last interrupted time
								A_				 =						arrival_time
								A_				 =						burst_time  # remaining burst time
								A_				 =						0  # total time of the process wait in ready queue
								A_				 =						0  # time from arrival time to completion time





class __lowerCAmelCase				:




			"""simple docstring"""







			def __init__(				self						:					str       ,   _snake_case						:					int       ,   _snake_case						:					list[int]       ,   _snake_case						:					deque[Process]       ,   _snake_case						:					int       ,   )      -> None:







								"""simple docstring"""
								# total number of mlfq's queues
								A_				 =						number_of_queues
								# time slice of queues that round robin algorithm applied
								A_				 =						time_slices
								# unfinished process is in this ready_queue
								A_				 =						queue
								# current time
								A_				 =						current_time
								# finished process is in this sequence queue
								A_				 =						deque()




			def      lowerCamelCase__     (				self						:					List[str]	)      -> list[str]:







								"""simple docstring"""
								A_				 =						[]
								for i in range(len(self.finish_queue	)	):
													sequence.append(self.finish_queue[i].process_name	)
								return sequence




			def      lowerCamelCase__     (				self						:					str       ,   _snake_case						:					list[Process]	)      -> list[int]:







								"""simple docstring"""
								A_				 =						[]
								for i in range(len(_snake_case	)	):
													waiting_times.append(queue[i].waiting_time	)
								return waiting_times




			def      lowerCamelCase__     (				self						:					Tuple       ,   _snake_case						:					list[Process]	)      -> list[int]:







								"""simple docstring"""
								A_				 =						[]
								for i in range(len(_snake_case	)	):
													turnaround_times.append(queue[i].turnaround_time	)
								return turnaround_times




			def      lowerCamelCase__     (				self						:					Tuple       ,   _snake_case						:					list[Process]	)      -> list[int]:







								"""simple docstring"""
								A_				 =						[]
								for i in range(len(_snake_case	)	):
													completion_times.append(queue[i].stop_time	)
								return completion_times




			def      lowerCamelCase__     (				self						:					Optional[int]       ,   _snake_case						:					deque[Process]	)      -> list[int]:







								"""simple docstring"""
								return [q.burst_time for q in queue]




			def      lowerCamelCase__     (				self						:					int       ,   _snake_case						:					Process	)      -> int:







								"""simple docstring"""
								process.waiting_time += self.current_time - process.stop_time
								return process.waiting_time




			def      lowerCamelCase__     (				self						:					Dict       ,   _snake_case						:					deque[Process]	)      -> deque[Process]:







								"""simple docstring"""
								A_				 =						deque()  # sequence deque of finished process
								while len(_snake_case	) != 0:
													A_				 =						ready_queue.popleft()  # current process

													# if process's arrival time is later than current time, update current time
													if self.current_time < cp.arrival_time:
																		self.current_time += cp.arrival_time

													# update waiting time of current process
													self.update_waiting_time(_snake_case	)
													# update current time
													self.current_time += cp.burst_time
													# finish the process and set the process's burst-time 0
													A_				 =						0
													# set the process's turnaround time because it is finished
													A_				 =						self.current_time - cp.arrival_time
													# set the completion time
													A_				 =						self.current_time
													# add the process to queue that has finished queue
													finished.append(_snake_case	)

								self.finish_queue.extend(_snake_case	)  # add finished process to finish queue
								# FCFS will finish all remaining processes
								return finished




			def      lowerCamelCase__     (				self						:					List[str]       ,   _snake_case						:					deque[Process]       ,   _snake_case						:					int	)      -> tuple[deque[Process], deque[Process]]:







								"""simple docstring"""
								A_				 =						deque()  # sequence deque of terminated process
								# just for 1 cycle and unfinished processes will go back to queue
								for _ in range(len(_snake_case	)	):
													A_				 =						ready_queue.popleft()  # current process

													# if process's arrival time is later than current time, update current time
													if self.current_time < cp.arrival_time:
																		self.current_time += cp.arrival_time

													# update waiting time of unfinished processes
													self.update_waiting_time(_snake_case	)
													# if the burst time of process is bigger than time-slice
													if cp.burst_time > time_slice:
																		# use CPU for only time-slice
																		self.current_time += time_slice
																		# update remaining burst time
																		cp.burst_time -= time_slice
																		# update end point time
																		A_				 =						self.current_time
																		# locate the process behind the queue because it is not finished
																		ready_queue.append(_snake_case	)
													else:
																		# use CPU for remaining burst time
																		self.current_time += cp.burst_time
																		# set burst time 0 because the process is finished
																		A_				 =						0
																		# set the finish time
																		A_				 =						self.current_time
																		# update the process' turnaround time because it is finished
																		A_				 =						self.current_time - cp.arrival_time
																		# add the process to queue that has finished queue
																		finished.append(_snake_case	)

								self.finish_queue.extend(_snake_case	)  # add finished process to finish queue
								# return finished processes queue and remaining processes queue
								return finished, ready_queue






			def      lowerCamelCase__     (				self						:					Any	)      -> deque[Process]:







								"""simple docstring"""

								#  all queues except last one have round_robin algorithm
								for i in range(self.number_of_queues - 1	):
													A_ ,    A_				 =						self.round_robin(
													    self.ready_queue       ,   self.time_slices[i]	)
								#  the last queue has first_come_first_served algorithm
								self.first_come_first_served(self.ready_queue	)

								return self.finish_queue


if __name__ == "__main__":
		import doctest

		UpperCamelCase_					:						Tuple				= Process('''P1''', 0, 53)
		UpperCamelCase_					:						List[Any]				= Process('''P2''', 0, 17)
		UpperCamelCase_					:						Optional[int]				= Process('''P3''', 0, 68)
		UpperCamelCase_					:						Union[str, Any]				= Process('''P4''', 0, 24)
		UpperCamelCase_					:						Any				= 3
		UpperCamelCase_					:						List[str]				= [17, 25]
		UpperCamelCase_					:						List[str]				= deque([Pa, Pa, Pa, Pa])

		if len(time_slices) != number_of_queues - 1:
				raise SystemExit(0)

		doctest.testmod(extraglobs={'''queue''': deque([Pa, Pa, Pa, Pa])})

		UpperCamelCase_					:						str				= Process('''P1''', 0, 53)
		UpperCamelCase_					:						Dict				= Process('''P2''', 0, 17)
		UpperCamelCase_					:						List[Any]				= Process('''P3''', 0, 68)
		UpperCamelCase_					:						int				= Process('''P4''', 0, 24)
		UpperCamelCase_					:						Dict				= 3
		UpperCamelCase_					:						str				= [17, 25]
		UpperCamelCase_					:						Optional[int]				= deque([Pa, Pa, Pa, Pa])
		UpperCamelCase_					:						str				= MLFQ(number_of_queues, time_slices, queue, 0)
		UpperCamelCase_					:						int				= mlfq.multi_level_feedback_queue()

		# print total waiting times of processes(P1, P2, P3, P4)
		print(
		    F"""waiting time:\
        \t\t\t{MLFQ.calculate_waiting_time(mlfq, [Pa, Pa, Pa, Pa])}"""
		)
		# print completion times of processes(P1, P2, P3, P4)
		print(
		    F"""completion time:\
        \t\t{MLFQ.calculate_completion_time(mlfq, [Pa, Pa, Pa, Pa])}"""
		)
		# print total turnaround times of processes(P1, P2, P3, P4)
		print(
		    F"""turnaround time:\
        \t\t{MLFQ.calculate_turnaround_time(mlfq, [Pa, Pa, Pa, Pa])}"""
		)
		# print sequence of finished processes
		print(
		    F"""sequence of finished processes:\
        {mlfq.calculate_sequence_of_finish_queue()}"""
		)




