



import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
    from transformers import LlamaTokenizerFast
except ImportError as e:
    warnings.warn(e)
    warnings.warn(
        '''The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'''
    )
    lowercase_	       =			None
lowercase_	       =			{
    '''7B''': 1_10_08,
    '''13B''': 1_38_24,
    '''30B''': 1_79_20,
    '''65B''': 2_20_16,
    '''70B''': 2_86_72,
}
lowercase_	       =			{
    '''7B''': 1,
    '''7Bf''': 1,
    '''13B''': 2,
    '''13Bf''': 2,
    '''30B''': 4,
    '''65B''': 8,
    '''70B''': 8,
    '''70Bf''': 8,
}






def 					__lowerCAmelCase		(  __lowerCamelCase	:    List[str]				,   __lowerCamelCase	:    Optional[int]=1				,   __lowerCamelCase	:    int=256	)  ->						Union[str, Any]:
     return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3	)	) + multiple_of - 1) // multiple_of)






def 					__lowerCAmelCase		(  __lowerCamelCase	:    Dict	)  ->						List[str]:





     with open(__snake_case				,   """r"""	) as f:
          return json.load(__snake_case	)






def 					__lowerCAmelCase		(  __lowerCamelCase	:    int				,   __lowerCamelCase	:    Optional[int]	)  ->						Optional[Any]:





     with open(__snake_case				,   """w"""	) as f:
          json.dump(__snake_case				,   __snake_case	)






def 					__lowerCAmelCase		(  __lowerCamelCase	:    List[Any]				,   __lowerCamelCase	:    str				,   __lowerCamelCase	:    str				,   __lowerCamelCase	:    List[str]=True	)  ->						Optional[Any]:
     os.makedirs(__snake_case				,   exist_ok=__snake_case	)
     __lowerCAmelCase	   =os.path.join(__snake_case				,   """tmp"""	)
     os.makedirs(__snake_case				,   exist_ok=__snake_case	)

     __lowerCAmelCase	   =read_json(os.path.join(__snake_case				,   """params.json"""	)	)
     __lowerCAmelCase	   =NUM_SHARDS[model_size]
     __lowerCAmelCase	   =params["""n_layers"""]
     __lowerCAmelCase	   =params["""n_heads"""]
     __lowerCAmelCase	   =n_heads // num_shards
     __lowerCAmelCase	   =params["""dim"""]
     __lowerCAmelCase	   =dim // n_heads
     __lowerCAmelCase	   =1_0000.0
     __lowerCAmelCase	   =1.0 / (base ** (torch.arange(0				,   __snake_case				,   2	).float() / dims_per_head))

     if "n_kv_heads" in params:
          __lowerCAmelCase	   =params["""n_kv_heads"""]  # for GQA / MQA
          __lowerCAmelCase	   =n_heads_per_shard // num_key_value_heads
          __lowerCAmelCase	   =dim // num_key_value_heads
     else:  # compatibility with other checkpoints
          __lowerCAmelCase	   =n_heads
          __lowerCAmelCase	   =n_heads_per_shard
          __lowerCAmelCase	   =dim

     # permute for sliced rotary
     def permute(__lowerCamelCase	:    int				,   __lowerCamelCase	:    Any=n_heads				,   __lowerCamelCase	:    int=dim				,   __lowerCamelCase	:    Union[str, Any]=dim	):
          return w.view(__snake_case				,   dima // n_heads // 2				,   2				,   __snake_case	).transpose(1				,   2	).reshape(__snake_case				,   __snake_case	)

     print(f"""Fetching all parameters from the checkpoint at {input_base_path}."""	)
     # Load weights
     if model_size == "7B":
          # Not sharded
          # (The sharded implementation would also work, but this is simpler.)
          __lowerCAmelCase	   =torch.load(os.path.join(__snake_case				,   """consolidated.00.pth"""	)				,   map_location="""cpu"""	)
     else:
          # Sharded
          __lowerCAmelCase	   =[
              torch.load(os.path.join(__snake_case				,   f"""consolidated.{i:02d}.pth"""	)				,   map_location="""cpu"""	)
              for i in range(__snake_case	)
          ]
     __lowerCAmelCase	   =0
     __lowerCAmelCase	   ={"""weight_map""": {}}
     for layer_i in range(__snake_case	):
          __lowerCAmelCase	   =f"""pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"""
          if model_size == "7B":
               # Unsharded
               __lowerCAmelCase	   ={
                   f"""model.layers.{layer_i}.self_attn.q_proj.weight""": permute(
                       loaded[f"""layers.{layer_i}.attention.wq.weight"""]	),
                   f"""model.layers.{layer_i}.self_attn.k_proj.weight""": permute(
                       loaded[f"""layers.{layer_i}.attention.wk.weight"""]	),
                   f"""model.layers.{layer_i}.self_attn.v_proj.weight""": loaded[f"""layers.{layer_i}.attention.wv.weight"""],
                   f"""model.layers.{layer_i}.self_attn.o_proj.weight""": loaded[f"""layers.{layer_i}.attention.wo.weight"""],
                   f"""model.layers.{layer_i}.mlp.gate_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w1.weight"""],
                   f"""model.layers.{layer_i}.mlp.down_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w2.weight"""],
                   f"""model.layers.{layer_i}.mlp.up_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w3.weight"""],
                   f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[f"""layers.{layer_i}.attention_norm.weight"""],
                   f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[f"""layers.{layer_i}.ffn_norm.weight"""],
               }
          else:
               # Sharded
               # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
               # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
               # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

               __lowerCAmelCase	   ={
                   f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[0][
                       f"""layers.{layer_i}.attention_norm.weight"""
                   ].clone(),
                   f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[0][
                       f"""layers.{layer_i}.ffn_norm.weight"""
                   ].clone(),
               }
               __lowerCAmelCase	   =permute(
                   torch.cat(
                       [
                           loaded[i][f"""layers.{layer_i}.attention.wq.weight"""].view(__snake_case				,   __snake_case				,   __snake_case	)
                           for i in range(__snake_case	)
                       ]				,   dim=0				,   ).reshape(__snake_case				,   __snake_case	)	)
               __lowerCAmelCase	   =permute(
                   torch.cat(
                       [
                           loaded[i][f"""layers.{layer_i}.attention.wk.weight"""].view(
                               __snake_case				,   __snake_case				,   __snake_case	)
                           for i in range(__snake_case	)
                       ]				,   dim=0				,   ).reshape(__snake_case				,   __snake_case	)				,   __snake_case				,   __snake_case				,   __snake_case				,   )
               __lowerCAmelCase	   =torch.cat(
                   [
                       loaded[i][f"""layers.{layer_i}.attention.wv.weight"""].view(
                           __snake_case				,   __snake_case				,   __snake_case	)
                       for i in range(__snake_case	)
                   ]				,   dim=0				,   ).reshape(__snake_case				,   __snake_case	)

               __lowerCAmelCase	   =torch.cat(
                   [loaded[i][f"""layers.{layer_i}.attention.wo.weight"""] for i in range(__snake_case	)]				,   dim=1	)
               __lowerCAmelCase	   =torch.cat(
                   [loaded[i][f"""layers.{layer_i}.feed_forward.w1.weight"""] for i in range(__snake_case	)]				,   dim=0	)
               __lowerCAmelCase	   =torch.cat(
                   [loaded[i][f"""layers.{layer_i}.feed_forward.w2.weight"""] for i in range(__snake_case	)]				,   dim=1	)
               __lowerCAmelCase	   =torch.cat(
                   [loaded[i][f"""layers.{layer_i}.feed_forward.w3.weight"""] for i in range(__snake_case	)]				,   dim=0	)

          __lowerCAmelCase	   =inv_freq
          for k, v in state_dict.items():
               __lowerCAmelCase	   =filename
               param_count += v.numel()
          torch.save(__snake_case				,   os.path.join(__snake_case				,   __snake_case	)	)

     __lowerCAmelCase	   =f"""pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"""
     if model_size == "7B":
          # Unsharded
          __lowerCAmelCase	   ={
              """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
              """model.norm.weight""": loaded["""norm.weight"""],
              """lm_head.weight""": loaded["""output.weight"""],
          }
     else:
          __lowerCAmelCase	   ={
              """model.norm.weight""": loaded[0]["""norm.weight"""],
              """model.embed_tokens.weight""": torch.cat(
                  [loaded[i]["""tok_embeddings.weight"""] for i in range(__snake_case	)]				,   dim=1	),
              """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(__snake_case	)]				,   dim=0	),
          }

     for k, v in state_dict.items():
          __lowerCAmelCase	   =filename
          param_count += v.numel()
     torch.save(__snake_case				,   os.path.join(__snake_case				,   __snake_case	)	)

     # Write configs
     __lowerCAmelCase	   ={"""total_size""": param_count * 2}
     write_json(__snake_case				,   os.path.join(__snake_case				,   """pytorch_model.bin.index.json"""	)	)
     __lowerCAmelCase	   =params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
     __lowerCAmelCase	   =params["""multiple_of"""] if """multiple_of""" in params else 256
     __lowerCAmelCase	   =LlamaConfig(
         hidden_size=__snake_case				,   intermediate_size=compute_intermediate_size(__snake_case				,   __snake_case				,   __snake_case	)				,   num_attention_heads=params["""n_heads"""]				,   num_hidden_layers=params["""n_layers"""]				,   rms_norm_eps=params["""norm_eps"""]				,   num_key_value_heads=__snake_case				,   )
     config.save_pretrained(__snake_case	)

     # Make space so we can load the model properly now.
     del state_dict
     del loaded
     gc.collect()

     print("""Loading the checkpoint in a Llama model."""	)
     __lowerCAmelCase	   =LlamaForCausalLM.from_pretrained(__snake_case				,   torch_dtype=torch.floataa				,   low_cpu_mem_usage=__snake_case	)
     # Avoid saving this as part of the config.
     del model.config._name_or_path

     print("""Saving in the Transformers format."""	)
     model.save_pretrained(__snake_case				,   safe_serialization=__snake_case	)
     shutil.rmtree(__snake_case	)






def 					__lowerCAmelCase		(  __lowerCamelCase	:    List[Any]				,   __lowerCamelCase	:    Optional[int]	)  ->						Any:
     __lowerCAmelCase	   =LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
     print(f"""Saving a {tokenizer_class.__name__} to {tokenizer_path}."""	)
     __lowerCAmelCase	   =tokenizer_class(__snake_case	)
     tokenizer.save_pretrained(__snake_case	)






def 					__lowerCAmelCase		(  )  ->						int:
     __lowerCAmelCase	   =argparse.ArgumentParser()
     parser.add_argument(
         """--input_dir"""				,   help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""				,   )
     parser.add_argument(
         """--model_size"""				,   choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]				,   )
     parser.add_argument(
         """--output_dir"""				,   help="""Location to write HF model and tokenizer"""				,   )
     parser.add_argument("""--safe_serialization"""				,   type=__snake_case				,   help="""Whether or not to save using `safetensors`."""	)
     __lowerCAmelCase	   =parser.parse_args()
     if args.model_size != "tokenizer_only":
          write_model(
              model_path=args.output_dir				,   input_base_path=os.path.join(args.input_dir				,   args.model_size	)				,   model_size=args.model_size				,   safe_serialization=args.safe_serialization				,   )
     __lowerCAmelCase	   =os.path.join(args.input_dir				,   """tokenizer.model"""	)
     write_tokenizer(args.output_dir				,   __snake_case	)


if __name__ == "__main__":
    main()









import warnings

from ...utils import logging
from .image_processing_poolformer import PoolFormerImageProcessor


lowercase_	       =			logging.get_logger(__name__)






class 						__a (     SCREAMING_SNAKE_CASE				):



	def __init__(						self							:    Optional[Any]				,	*snake_case_							:    List[str]				,	**snake_case_							:    Union[str, Any])->							None:
						warnings.warn(
						    """The class PoolFormerFeatureExtractor is deprecated and will be removed in version 5 of Transformers."""
						    """ Please use PoolFormerImageProcessor instead."""				,	snake_case_				,	)
						super().__init__(*snake_case_				,	**snake_case_)




