





import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
   from transformers import LlamaTokenizerFast
except ImportError as e:
   warnings.warn(e)
   warnings.warn(
       "The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"
   )
   _a						=   None
_a						=   {
    "7B": 11_008,
    "13B": 13_824,
    "30B": 17_920,
    "65B": 22_016,
    "70B": 28_672,
}
_a						=   {
    "7B": 1,
    "7Bf": 1,
    "13B": 2,
    "13Bf": 2,
    "30B": 4,
    "65B": 8,
    "70B": 8,
    "70Bf": 8,
}





def lowerCAmelCase__(__snake_case ,__snake_case=1 ,__snake_case=256							)       ->   Union[str, Any]:







 '''simple docstring'''







 return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3							)							) + multiple_of - 1) // multiple_of)





def lowerCAmelCase__(__snake_case							)       ->   List[str]:







 '''simple docstring'''







 with open(__snake_case ,'''r'''							) as f:
  return json.load(__snake_case							)





def lowerCAmelCase__(__snake_case ,__snake_case							)       ->   int:







 '''simple docstring'''







 with open(__snake_case ,'''w'''							) as f:
  json.dump(__snake_case ,__snake_case							)





def lowerCAmelCase__(__snake_case ,__snake_case ,__snake_case ,__snake_case=True							)       ->   List[Any]:







 '''simple docstring'''







 os.makedirs(__snake_case ,exist_ok=__snake_case							)
 lowerCamelCase__              =	os.path.join(__snake_case ,'''tmp'''							)
 os.makedirs(__snake_case ,exist_ok=__snake_case							)

 lowerCamelCase__              =	read_json(os.path.join(__snake_case ,'''params.json'''							)							)
 lowerCamelCase__              =	NUM_SHARDS[model_size]
 lowerCamelCase__              =	params['''n_layers''']
 lowerCamelCase__              =	params['''n_heads''']
 lowerCamelCase__              =	n_heads // num_shards
 lowerCamelCase__              =	params['''dim''']
 lowerCamelCase__              =	dim // n_heads
 lowerCamelCase__              =	10000.0
 lowerCamelCase__              =	1.0 / (base ** (torch.arange(0 ,__snake_case ,2							).float() / dims_per_head))

 if "n_kv_heads" in params:
  lowerCamelCase__              =	params['''n_kv_heads''']  # for GQA / MQA
  lowerCamelCase__              =	n_heads_per_shard // num_key_value_heads
  lowerCamelCase__              =	dim // num_key_value_heads
 else:  # compatibility with other checkpoints
  lowerCamelCase__              =	n_heads
  lowerCamelCase__              =	n_heads_per_shard
  lowerCamelCase__              =	dim

 # permute for sliced rotary
 def permute(__snake_case ,__snake_case=n_heads ,__snake_case=dim ,__snake_case=dim							):
  return w.view(__snake_case ,dima // n_heads // 2 ,2 ,__snake_case							).transpose(1 ,2							).reshape(__snake_case ,__snake_case							)

 print(F'Fetching all parameters from the checkpoint at {input_base_path}.'							)
 # Load weights
 if model_size == "7B":
  # Not sharded
  # (The sharded implementation would also work, but this is simpler.)
  lowerCamelCase__              =	torch.load(os.path.join(__snake_case ,'''consolidated.00.pth'''							) ,map_location='''cpu'''							)
 else:
  # Sharded
  lowerCamelCase__              =	[
      torch.load(os.path.join(__snake_case ,F'consolidated.{i:02d}.pth'							) ,map_location='''cpu'''							)
      for i in range(__snake_case							)
  ]
 lowerCamelCase__              =	0
 lowerCamelCase__              =	{'''weight_map''': {}}
 for layer_i in range(__snake_case							):
  lowerCamelCase__              =	F'pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'
  if model_size == "7B":
   # Unsharded
   lowerCamelCase__              =	{
       F'model.layers.{layer_i}.self_attn.q_proj.weight': permute(
           loaded[F'layers.{layer_i}.attention.wq.weight']							),
       F'model.layers.{layer_i}.self_attn.k_proj.weight': permute(
           loaded[F'layers.{layer_i}.attention.wk.weight']							),
       F'model.layers.{layer_i}.self_attn.v_proj.weight': loaded[F'layers.{layer_i}.attention.wv.weight'],
       F'model.layers.{layer_i}.self_attn.o_proj.weight': loaded[F'layers.{layer_i}.attention.wo.weight'],
       F'model.layers.{layer_i}.mlp.gate_proj.weight': loaded[F'layers.{layer_i}.feed_forward.w1.weight'],
       F'model.layers.{layer_i}.mlp.down_proj.weight': loaded[F'layers.{layer_i}.feed_forward.w2.weight'],
       F'model.layers.{layer_i}.mlp.up_proj.weight': loaded[F'layers.{layer_i}.feed_forward.w3.weight'],
       F'model.layers.{layer_i}.input_layernorm.weight': loaded[F'layers.{layer_i}.attention_norm.weight'],
       F'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[F'layers.{layer_i}.ffn_norm.weight'],
   }
  else:
   # Sharded
   # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
   # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
   # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

   lowerCamelCase__              =	{
       F'model.layers.{layer_i}.input_layernorm.weight': loaded[0][
           F'layers.{layer_i}.attention_norm.weight'
       ].clone(),
       F'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[0][
           F'layers.{layer_i}.ffn_norm.weight'
       ].clone(),
   }
   lowerCamelCase__              =	permute(
       torch.cat(
           [
               loaded[i][F'layers.{layer_i}.attention.wq.weight'].view(__snake_case ,__snake_case ,__snake_case							)
               for i in range(__snake_case							)
           ] ,dim=0 ,).reshape(__snake_case ,__snake_case							)							)
   lowerCamelCase__              =	permute(
       torch.cat(
           [
               loaded[i][F'layers.{layer_i}.attention.wk.weight'].view(
                   __snake_case ,__snake_case ,__snake_case							)
               for i in range(__snake_case							)
           ] ,dim=0 ,).reshape(__snake_case ,__snake_case							) ,__snake_case ,__snake_case ,__snake_case ,)
   lowerCamelCase__              =	torch.cat(
       [
           loaded[i][F'layers.{layer_i}.attention.wv.weight'].view(
               __snake_case ,__snake_case ,__snake_case							)
           for i in range(__snake_case							)
       ] ,dim=0 ,).reshape(__snake_case ,__snake_case							)

   lowerCamelCase__              =	torch.cat(
       [loaded[i][F'layers.{layer_i}.attention.wo.weight'] for i in range(__snake_case							)] ,dim=1							)
   lowerCamelCase__              =	torch.cat(
       [loaded[i][F'layers.{layer_i}.feed_forward.w1.weight'] for i in range(__snake_case							)] ,dim=0							)
   lowerCamelCase__              =	torch.cat(
       [loaded[i][F'layers.{layer_i}.feed_forward.w2.weight'] for i in range(__snake_case							)] ,dim=1							)
   lowerCamelCase__              =	torch.cat(
       [loaded[i][F'layers.{layer_i}.feed_forward.w3.weight'] for i in range(__snake_case							)] ,dim=0							)

  lowerCamelCase__              =	inv_freq
  for k, v in state_dict.items():
   lowerCamelCase__              =	filename
   param_count += v.numel()
  torch.save(__snake_case ,os.path.join(__snake_case ,__snake_case							)							)

 lowerCamelCase__              =	F'pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'
 if model_size == "7B":
  # Unsharded
  lowerCamelCase__              =	{
      '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
      '''model.norm.weight''': loaded['''norm.weight'''],
      '''lm_head.weight''': loaded['''output.weight'''],
  }
 else:
  lowerCamelCase__              =	{
      '''model.norm.weight''': loaded[0]['''norm.weight'''],
      '''model.embed_tokens.weight''': torch.cat(
          [loaded[i]['''tok_embeddings.weight'''] for i in range(__snake_case							)] ,dim=1							),
      '''lm_head.weight''': torch.cat([loaded[i]['''output.weight'''] for i in range(__snake_case							)] ,dim=0							),
  }

 for k, v in state_dict.items():
  lowerCamelCase__              =	filename
  param_count += v.numel()
 torch.save(__snake_case ,os.path.join(__snake_case ,__snake_case							)							)

 # Write configs
 lowerCamelCase__              =	{'''total_size''': param_count * 2}
 write_json(__snake_case ,os.path.join(__snake_case ,'''pytorch_model.bin.index.json'''							)							)
 lowerCamelCase__              =	params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
 lowerCamelCase__              =	params['''multiple_of'''] if '''multiple_of''' in params else 256
 lowerCamelCase__              =	LlamaConfig(
     hidden_size=__snake_case ,intermediate_size=compute_intermediate_size(__snake_case ,__snake_case ,__snake_case							) ,num_attention_heads=params['''n_heads'''] ,num_hidden_layers=params['''n_layers'''] ,rms_norm_eps=params['''norm_eps'''] ,num_key_value_heads=__snake_case ,)
 config.save_pretrained(__snake_case							)

 # Make space so we can load the model properly now.
 del state_dict
 del loaded
 gc.collect()

 print('''Loading the checkpoint in a Llama model.'''							)
 lowerCamelCase__              =	LlamaForCausalLM.from_pretrained(__snake_case ,torch_dtype=torch.floataa ,low_cpu_mem_usage=__snake_case							)
 # Avoid saving this as part of the config.
 del model.config._name_or_path

 print('''Saving in the Transformers format.'''							)
 model.save_pretrained(__snake_case ,safe_serialization=__snake_case							)
 shutil.rmtree(__snake_case							)





def lowerCAmelCase__(__snake_case ,__snake_case							)       ->   List[Any]:







 '''simple docstring'''







 lowerCamelCase__              =	LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
 print(F'Saving a {tokenizer_class.__name__} to {tokenizer_path}.'							)
 lowerCamelCase__              =	tokenizer_class(__snake_case							)
 tokenizer.save_pretrained(__snake_case							)





def lowerCAmelCase__()       ->   Optional[int]:







 '''simple docstring'''







 lowerCamelCase__              =	argparse.ArgumentParser()
 parser.add_argument(
     '''--input_dir''' ,help='''Location of LLaMA weights, which contains tokenizer.model and model folders''' ,)
 parser.add_argument(
     '''--model_size''' ,choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only'''] ,)
 parser.add_argument(
     '''--output_dir''' ,help='''Location to write HF model and tokenizer''' ,)
 parser.add_argument('''--safe_serialization''' ,type=__snake_case ,help='''Whether or not to save using `safetensors`.'''							)
 lowerCamelCase__              =	parser.parse_args()
 if args.model_size != "tokenizer_only":
  write_model(
      model_path=args.output_dir ,input_base_path=os.path.join(args.input_dir ,args.model_size							) ,model_size=args.model_size ,safe_serialization=args.safe_serialization ,)
 lowerCamelCase__              =	os.path.join(args.input_dir ,'''tokenizer.model'''							)
 write_tokenizer(args.output_dir ,__snake_case							)


if __name__ == "__main__":
   main()










import warnings

from ...utils import logging
from .image_processing_segformer import SegformerImageProcessor


_a						=   logging.get_logger(__name__)






class       __A					(					lowerCAmelCase				):

			'''simple docstring'''







			def __init__(					self ,						*__lowerCAmelCase ,						**__lowerCAmelCase							):



				'''simple docstring'''







				warnings.warn(
				    '''The class SegformerFeatureExtractor is deprecated and will be removed in version 5 of Transformers.'''
				    ''' Please use SegformerImageProcessor instead.''' ,						__lowerCAmelCase ,						)
				super().__init__(*__lowerCAmelCase ,						**__lowerCAmelCase							)



