





import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
			from transformers import LlamaTokenizerFast
except ImportError as e:
			warnings.warn(e)
			warnings.warn(
			    "The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"
			)
			_a						=   None
_a						=   {
    "7B": 11_008,
    "13B": 13_824,
    "30B": 17_920,
    "65B": 22_016,
    "70B": 28_672,
}
_a						=   {
    "7B": 1,
    "7Bf": 1,
    "13B": 2,
    "13Bf": 2,
    "30B": 4,
    "65B": 8,
    "70B": 8,
    "70Bf": 8,
}





def lowerCAmelCase__(__snake_case ,__snake_case=1 ,__snake_case=256							)       ->   Union[str, Any]:







	'''simple docstring'''







	return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3							)							) + multiple_of - 1) // multiple_of)





def lowerCAmelCase__(__snake_case							)       ->   List[str]:







	'''simple docstring'''







	with open(__snake_case ,'''r'''							) as f:
		return json.load(__snake_case							)





def lowerCAmelCase__(__snake_case ,__snake_case							)       ->   int:







	'''simple docstring'''







	with open(__snake_case ,'''w'''							) as f:
		json.dump(__snake_case ,__snake_case							)





def lowerCAmelCase__(__snake_case ,__snake_case ,__snake_case ,__snake_case=True							)       ->   List[Any]:







	'''simple docstring'''







	os.makedirs(__snake_case ,exist_ok=__snake_case							)
	lowerCamelCase__              =	os.path.join(__snake_case ,'''tmp'''							)
	os.makedirs(__snake_case ,exist_ok=__snake_case							)

	lowerCamelCase__              =	read_json(os.path.join(__snake_case ,'''params.json'''							)							)
	lowerCamelCase__              =	NUM_SHARDS[model_size]
	lowerCamelCase__              =	params['''n_layers''']
	lowerCamelCase__              =	params['''n_heads''']
	lowerCamelCase__              =	n_heads // num_shards
	lowerCamelCase__              =	params['''dim''']
	lowerCamelCase__              =	dim // n_heads
	lowerCamelCase__              =	1_0_0_0_0.0
	lowerCamelCase__              =	1.0 / (base ** (torch.arange(0 ,__snake_case ,2							).float() / dims_per_head))

	if "n_kv_heads" in params:
		lowerCamelCase__              =	params['''n_kv_heads''']  # for GQA / MQA
		lowerCamelCase__              =	n_heads_per_shard // num_key_value_heads
		lowerCamelCase__              =	dim // num_key_value_heads
	else:  # compatibility with other checkpoints
		lowerCamelCase__              =	n_heads
		lowerCamelCase__              =	n_heads_per_shard
		lowerCamelCase__              =	dim

	# permute for sliced rotary
	def permute(__snake_case ,__snake_case=n_heads ,__snake_case=dim ,__snake_case=dim							):
		return w.view(__snake_case ,dima // n_heads // 2 ,2 ,__snake_case							).transpose(1 ,2							).reshape(__snake_case ,__snake_case							)

	print(F'Fetching all parameters from the checkpoint at {input_base_path}.'							)
	# Load weights
	if model_size == "7B":
		# Not sharded
		# (The sharded implementation would also work, but this is simpler.)
		lowerCamelCase__              =	torch.load(os.path.join(__snake_case ,'''consolidated.00.pth'''							) ,map_location='''cpu'''							)
	else:
		# Sharded
		lowerCamelCase__              =	[
		    torch.load(os.path.join(__snake_case ,F'consolidated.{i:02d}.pth'							) ,map_location='''cpu'''							)
		    for i in range(__snake_case							)
		]
	lowerCamelCase__              =	0
	lowerCamelCase__              =	{'''weight_map''': {}}
	for layer_i in range(__snake_case							):
		lowerCamelCase__              =	F'pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'
		if model_size == "7B":
			# Unsharded
			lowerCamelCase__              =	{
			    F'model.layers.{layer_i}.self_attn.q_proj.weight': permute(
			        loaded[F'layers.{layer_i}.attention.wq.weight']							),
			    F'model.layers.{layer_i}.self_attn.k_proj.weight': permute(
			        loaded[F'layers.{layer_i}.attention.wk.weight']							),
			    F'model.layers.{layer_i}.self_attn.v_proj.weight': loaded[F'layers.{layer_i}.attention.wv.weight'],
			    F'model.layers.{layer_i}.self_attn.o_proj.weight': loaded[F'layers.{layer_i}.attention.wo.weight'],
			    F'model.layers.{layer_i}.mlp.gate_proj.weight': loaded[F'layers.{layer_i}.feed_forward.w1.weight'],
			    F'model.layers.{layer_i}.mlp.down_proj.weight': loaded[F'layers.{layer_i}.feed_forward.w2.weight'],
			    F'model.layers.{layer_i}.mlp.up_proj.weight': loaded[F'layers.{layer_i}.feed_forward.w3.weight'],
			    F'model.layers.{layer_i}.input_layernorm.weight': loaded[F'layers.{layer_i}.attention_norm.weight'],
			    F'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[F'layers.{layer_i}.ffn_norm.weight'],
			}
		else:
			# Sharded
			# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
			# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
			# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

			lowerCamelCase__              =	{
			    F'model.layers.{layer_i}.input_layernorm.weight': loaded[0][
			        F'layers.{layer_i}.attention_norm.weight'
			    ].clone(),
			    F'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[0][
			        F'layers.{layer_i}.ffn_norm.weight'
			    ].clone(),
			}
			lowerCamelCase__              =	permute(
			    torch.cat(
			        [
			            loaded[i][F'layers.{layer_i}.attention.wq.weight'].view(__snake_case ,__snake_case ,__snake_case							)
			            for i in range(__snake_case							)
			        ] ,dim=0 ,).reshape(__snake_case ,__snake_case							)							)
			lowerCamelCase__              =	permute(
			    torch.cat(
			        [
			            loaded[i][F'layers.{layer_i}.attention.wk.weight'].view(
			                __snake_case ,__snake_case ,__snake_case							)
			            for i in range(__snake_case							)
			        ] ,dim=0 ,).reshape(__snake_case ,__snake_case							) ,__snake_case ,__snake_case ,__snake_case ,)
			lowerCamelCase__              =	torch.cat(
			    [
			        loaded[i][F'layers.{layer_i}.attention.wv.weight'].view(
			            __snake_case ,__snake_case ,__snake_case							)
			        for i in range(__snake_case							)
			    ] ,dim=0 ,).reshape(__snake_case ,__snake_case							)

			lowerCamelCase__              =	torch.cat(
			    [loaded[i][F'layers.{layer_i}.attention.wo.weight'] for i in range(__snake_case							)] ,dim=1							)
			lowerCamelCase__              =	torch.cat(
			    [loaded[i][F'layers.{layer_i}.feed_forward.w1.weight'] for i in range(__snake_case							)] ,dim=0							)
			lowerCamelCase__              =	torch.cat(
			    [loaded[i][F'layers.{layer_i}.feed_forward.w2.weight'] for i in range(__snake_case							)] ,dim=1							)
			lowerCamelCase__              =	torch.cat(
			    [loaded[i][F'layers.{layer_i}.feed_forward.w3.weight'] for i in range(__snake_case							)] ,dim=0							)

		lowerCamelCase__              =	inv_freq
		for k, v in state_dict.items():
			lowerCamelCase__              =	filename
			param_count += v.numel()
		torch.save(__snake_case ,os.path.join(__snake_case ,__snake_case							)							)

	lowerCamelCase__              =	F'pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'
	if model_size == "7B":
		# Unsharded
		lowerCamelCase__              =	{
		    '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
		    '''model.norm.weight''': loaded['''norm.weight'''],
		    '''lm_head.weight''': loaded['''output.weight'''],
		}
	else:
		lowerCamelCase__              =	{
		    '''model.norm.weight''': loaded[0]['''norm.weight'''],
		    '''model.embed_tokens.weight''': torch.cat(
		        [loaded[i]['''tok_embeddings.weight'''] for i in range(__snake_case							)] ,dim=1							),
		    '''lm_head.weight''': torch.cat([loaded[i]['''output.weight'''] for i in range(__snake_case							)] ,dim=0							),
		}

	for k, v in state_dict.items():
		lowerCamelCase__              =	filename
		param_count += v.numel()
	torch.save(__snake_case ,os.path.join(__snake_case ,__snake_case							)							)

	# Write configs
	lowerCamelCase__              =	{'''total_size''': param_count * 2}
	write_json(__snake_case ,os.path.join(__snake_case ,'''pytorch_model.bin.index.json'''							)							)
	lowerCamelCase__              =	params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
	lowerCamelCase__              =	params['''multiple_of'''] if '''multiple_of''' in params else 256
	lowerCamelCase__              =	LlamaConfig(
	    hidden_size=__snake_case ,intermediate_size=compute_intermediate_size(__snake_case ,__snake_case ,__snake_case							) ,num_attention_heads=params['''n_heads'''] ,num_hidden_layers=params['''n_layers'''] ,rms_norm_eps=params['''norm_eps'''] ,num_key_value_heads=__snake_case ,)
	config.save_pretrained(__snake_case							)

	# Make space so we can load the model properly now.
	del state_dict
	del loaded
	gc.collect()

	print('''Loading the checkpoint in a Llama model.'''							)
	lowerCamelCase__              =	LlamaForCausalLM.from_pretrained(__snake_case ,torch_dtype=torch.floataa ,low_cpu_mem_usage=__snake_case							)
	# Avoid saving this as part of the config.
	del model.config._name_or_path

	print('''Saving in the Transformers format.'''							)
	model.save_pretrained(__snake_case ,safe_serialization=__snake_case							)
	shutil.rmtree(__snake_case							)





def lowerCAmelCase__(__snake_case ,__snake_case							)       ->   List[Any]:







	'''simple docstring'''







	lowerCamelCase__              =	LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
	print(F'Saving a {tokenizer_class.__name__} to {tokenizer_path}.'							)
	lowerCamelCase__              =	tokenizer_class(__snake_case							)
	tokenizer.save_pretrained(__snake_case							)





def lowerCAmelCase__()       ->   Optional[int]:







	'''simple docstring'''







	lowerCamelCase__              =	argparse.ArgumentParser()
	parser.add_argument(
	    '''--input_dir''' ,help='''Location of LLaMA weights, which contains tokenizer.model and model folders''' ,)
	parser.add_argument(
	    '''--model_size''' ,choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only'''] ,)
	parser.add_argument(
	    '''--output_dir''' ,help='''Location to write HF model and tokenizer''' ,)
	parser.add_argument('''--safe_serialization''' ,type=__snake_case ,help='''Whether or not to save using `safetensors`.'''							)
	lowerCamelCase__              =	parser.parse_args()
	if args.model_size != "tokenizer_only":
		write_model(
		    model_path=args.output_dir ,input_base_path=os.path.join(args.input_dir ,args.model_size							) ,model_size=args.model_size ,safe_serialization=args.safe_serialization ,)
	lowerCamelCase__              =	os.path.join(args.input_dir ,'''tokenizer.model'''							)
	write_tokenizer(args.output_dir ,__snake_case							)


if __name__ == "__main__":
			main()










def lowerCAmelCase__(__snake_case							)       ->   int:







	'''simple docstring'''







	if not grid or not grid[0]:
		raise TypeError('''The grid does not contain the appropriate information'''							)

	for cell_n in range(1 ,len(grid[0]							)							):
		grid[0][cell_n] += grid[0][cell_n - 1]
	lowerCamelCase__              =	grid[0]

	for row_n in range(1 ,len(__snake_case							)							):
		lowerCamelCase__              =	grid[row_n]
		lowerCamelCase__              =	fill_row(__snake_case ,__snake_case							)
		lowerCamelCase__              =	grid[row_n]

	return grid[-1][-1]





def lowerCAmelCase__(__snake_case ,__snake_case							)       ->   list:







	'''simple docstring'''







	current_row[0] += row_above[0]
	for cell_n in range(1 ,len(__snake_case							)							):
		current_row[cell_n] += min(current_row[cell_n - 1] ,row_above[cell_n]							)

	return current_row


if __name__ == "__main__":
			import doctest

			doctest.testmod()



