






'''simple docstring'''






import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
      from transformers import LlamaTokenizerFast
except ImportError as e:
      warnings.warn(e)
      warnings.warn(
          '''The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'''
      )
      __A    	=   None
__A    	=   {
    '''7B''': 1_1_0_0_8,
    '''13B''': 1_3_8_2_4,
    '''30B''': 1_7_9_2_0,
    '''65B''': 2_2_0_1_6,
    '''70B''': 2_8_6_7_2,
}
__A    	=   {
    '''7B''': 1,
    '''7Bf''': 1,
    '''13B''': 2,
    '''13Bf''': 2,
    '''30B''': 4,
    '''65B''': 8,
    '''70B''': 8,
    '''70Bf''': 8,
}




def        _SCREAMING_SNAKE_CASE  (				A  :       Optional[int]  ,		A  :       Tuple=1  ,		A  :       Union[str, Any]=2_56  )    -> int:







   """simple docstring"""
   return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3  )  ) + multiple_of - 1) // multiple_of)




def        _SCREAMING_SNAKE_CASE  (				A  :       int  )    -> List[str]:







   """simple docstring"""
   with open(A  ,		'r'  ) as f:
      return json.load(A  )




def        _SCREAMING_SNAKE_CASE  (				A  :       Union[str, Any]  ,		A  :       Any  )    -> str:







   """simple docstring"""
   with open(A  ,		'w'  ) as f:
      json.dump(A  ,		A  )




def        _SCREAMING_SNAKE_CASE  (				A  :       Union[str, Any]  ,		A  :       Union[str, Any]  ,		A  :       Union[str, Any]  ,		A  :       Optional[int]=True  )    -> List[str]:







   """simple docstring"""
   os.makedirs(A  ,		exist_ok=A  )
   __snake_case				:      str     							= os.path.join(A  ,		'tmp'  )
   os.makedirs(A  ,		exist_ok=A  )

   __snake_case				:      int     							= read_json(os.path.join(A  ,		'params.json'  )  )
   __snake_case				:      int     							= NUM_SHARDS[model_size]
   __snake_case				:      Any     							= params['n_layers']
   __snake_case				:      Any     							= params['n_heads']
   __snake_case				:      int     							= n_heads // num_shards
   __snake_case				:      Optional[int]     							= params['dim']
   __snake_case				:      str     							= dim // n_heads
   __snake_case				:      Dict     							= 10000.0
   __snake_case				:      List[Any]     							= 1.0 / (base ** (torch.arange(0  ,		A  ,		2  ).float() / dims_per_head))

   if "n_kv_heads" in params:
      __snake_case				:      Tuple     							= params['n_kv_heads']  # for GQA / MQA
      __snake_case				:      Any     							= n_heads_per_shard // num_key_value_heads
      __snake_case				:      Any     							= dim // num_key_value_heads
   else:  # compatibility with other checkpoints
      __snake_case				:      Any     							= n_heads
      __snake_case				:      Union[str, Any]     							= n_heads_per_shard
      __snake_case				:      Union[str, Any]     							= dim

   # permute for sliced rotary
   def permute(A  :       Tuple  ,		A  :       List[str]=n_heads  ,		A  :       Dict=dim  ,		A  :       Union[str, Any]=dim  ):
      return w.view(A  ,		dima // n_heads // 2  ,		2  ,		A  ).transpose(1  ,		2  ).reshape(A  ,		A  )

   print(F"""Fetching all parameters from the checkpoint at {input_base_path}."""  )
   # Load weights
   if model_size == "7B":
      # Not sharded
      # (The sharded implementation would also work, but this is simpler.)
      __snake_case				:      str     							= torch.load(os.path.join(A  ,		'consolidated.00.pth'  )  ,		map_location='cpu'  )
   else:
      # Sharded
      __snake_case				:      int     							= [
          torch.load(os.path.join(A  ,		F"""consolidated.{i:02d}.pth"""  )  ,		map_location='cpu'  )
          for i in range(A  )
      ]
   __snake_case				:      Dict     							= 0
   __snake_case				:      Any     							= {'weight_map': {}}
   for layer_i in range(A  ):
      __snake_case				:      str     							= F"""pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"""
      if model_size == "7B":
         # Unsharded
         __snake_case				:      List[str]     							= {
             F"""model.layers.{layer_i}.self_attn.q_proj.weight""": permute(
                 loaded[F"""layers.{layer_i}.attention.wq.weight"""]  ),
             F"""model.layers.{layer_i}.self_attn.k_proj.weight""": permute(
                 loaded[F"""layers.{layer_i}.attention.wk.weight"""]  ),
             F"""model.layers.{layer_i}.self_attn.v_proj.weight""": loaded[F"""layers.{layer_i}.attention.wv.weight"""],
             F"""model.layers.{layer_i}.self_attn.o_proj.weight""": loaded[F"""layers.{layer_i}.attention.wo.weight"""],
             F"""model.layers.{layer_i}.mlp.gate_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w1.weight"""],
             F"""model.layers.{layer_i}.mlp.down_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w2.weight"""],
             F"""model.layers.{layer_i}.mlp.up_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w3.weight"""],
             F"""model.layers.{layer_i}.input_layernorm.weight""": loaded[F"""layers.{layer_i}.attention_norm.weight"""],
             F"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[F"""layers.{layer_i}.ffn_norm.weight"""],
         }
      else:
         # Sharded
         # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
         # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
         # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

         __snake_case				:      Optional[int]     							= {
             F"""model.layers.{layer_i}.input_layernorm.weight""": loaded[0][
                 F"""layers.{layer_i}.attention_norm.weight"""
             ].clone(),
             F"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[0][
                 F"""layers.{layer_i}.ffn_norm.weight"""
             ].clone(),
         }
         __snake_case				:      Optional[int]     							= permute(
             torch.cat(
                 [
                     loaded[i][F"""layers.{layer_i}.attention.wq.weight"""].view(A  ,		A  ,		A  )
                     for i in range(A  )
                 ]  ,		dim=0  ,		).reshape(A  ,		A  )  )
         __snake_case				:      Any     							= permute(
             torch.cat(
                 [
                     loaded[i][F"""layers.{layer_i}.attention.wk.weight"""].view(
                         A  ,		A  ,		A  )
                     for i in range(A  )
                 ]  ,		dim=0  ,		).reshape(A  ,		A  )  ,		A  ,		A  ,		A  ,		)
         __snake_case				:      Tuple     							= torch.cat(
             [
                 loaded[i][F"""layers.{layer_i}.attention.wv.weight"""].view(
                     A  ,		A  ,		A  )
                 for i in range(A  )
             ]  ,		dim=0  ,		).reshape(A  ,		A  )

         __snake_case				:      Tuple     							= torch.cat(
             [loaded[i][F"""layers.{layer_i}.attention.wo.weight"""] for i in range(A  )]  ,		dim=1  )
         __snake_case				:      Union[str, Any]     							= torch.cat(
             [loaded[i][F"""layers.{layer_i}.feed_forward.w1.weight"""] for i in range(A  )]  ,		dim=0  )
         __snake_case				:      int     							= torch.cat(
             [loaded[i][F"""layers.{layer_i}.feed_forward.w2.weight"""] for i in range(A  )]  ,		dim=1  )
         __snake_case				:      Any     							= torch.cat(
             [loaded[i][F"""layers.{layer_i}.feed_forward.w3.weight"""] for i in range(A  )]  ,		dim=0  )

      __snake_case				:      str     							= inv_freq
      for k, v in state_dict.items():
         __snake_case				:      Union[str, Any]     							= filename
         param_count += v.numel()
      torch.save(A  ,		os.path.join(A  ,		A  )  )

   __snake_case				:      Optional[int]     							= F"""pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"""
   if model_size == "7B":
      # Unsharded
      __snake_case				:      List[str]     							= {
          'model.embed_tokens.weight': loaded['tok_embeddings.weight'],
          'model.norm.weight': loaded['norm.weight'],
          'lm_head.weight': loaded['output.weight'],
      }
   else:
      __snake_case				:      str     							= {
          'model.norm.weight': loaded[0]['norm.weight'],
          'model.embed_tokens.weight': torch.cat(
              [loaded[i]['tok_embeddings.weight'] for i in range(A  )]  ,		dim=1  ),
          'lm_head.weight': torch.cat([loaded[i]['output.weight'] for i in range(A  )]  ,		dim=0  ),
      }

   for k, v in state_dict.items():
      __snake_case				:      Union[str, Any]     							= filename
      param_count += v.numel()
   torch.save(A  ,		os.path.join(A  ,		A  )  )

   # Write configs
   __snake_case				:      Optional[int]     							= {'total_size': param_count * 2}
   write_json(A  ,		os.path.join(A  ,		'pytorch_model.bin.index.json'  )  )
   __snake_case				:      List[str]     							= params['ffn_dim_multiplier'] if 'ffn_dim_multiplier' in params else 1
   __snake_case				:      Tuple     							= params['multiple_of'] if 'multiple_of' in params else 2_56
   __snake_case				:      Optional[Any]     							= LlamaConfig(
       hidden_size=A  ,		intermediate_size=compute_intermediate_size(A  ,		A  ,		A  )  ,		num_attention_heads=params['n_heads']  ,		num_hidden_layers=params['n_layers']  ,		rms_norm_eps=params['norm_eps']  ,		num_key_value_heads=A  ,		)
   config.save_pretrained(A  )

   # Make space so we can load the model properly now.
   del state_dict
   del loaded
   gc.collect()

   print('Loading the checkpoint in a Llama model.'  )
   __snake_case				:      List[str]     							= LlamaForCausalLM.from_pretrained(A  ,		torch_dtype=torch.floataa  ,		low_cpu_mem_usage=A  )
   # Avoid saving this as part of the config.
   del model.config._name_or_path

   print('Saving in the Transformers format.'  )
   model.save_pretrained(A  ,		safe_serialization=A  )
   shutil.rmtree(A  )




def        _SCREAMING_SNAKE_CASE  (				A  :       Dict  ,		A  :       Dict  )    -> Dict:







   """simple docstring"""
   __snake_case				:      Tuple     							= LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
   print(F"""Saving a {tokenizer_class.__name__} to {tokenizer_path}."""  )
   __snake_case				:      str     							= tokenizer_class(A  )
   tokenizer.save_pretrained(A  )




def        _SCREAMING_SNAKE_CASE  (				)    -> int:







   """simple docstring"""
   __snake_case				:      Optional[Any]     							= argparse.ArgumentParser()
   parser.add_argument(
       '--input_dir'  ,		help='Location of LLaMA weights, which contains tokenizer.model and model folders'  ,		)
   parser.add_argument(
       '--model_size'  ,		choices=['7B', '7Bf', '13B', '13Bf', '30B', '65B', '70B', '70Bf', 'tokenizer_only']  ,		)
   parser.add_argument(
       '--output_dir'  ,		help='Location to write HF model and tokenizer'  ,		)
   parser.add_argument('--safe_serialization'  ,		type=A  ,		help='Whether or not to save using `safetensors`.'  )
   __snake_case				:      Tuple     							= parser.parse_args()
   if args.model_size != "tokenizer_only":
      write_model(
          model_path=args.output_dir  ,		input_base_path=os.path.join(args.input_dir  ,		args.model_size  )  ,		model_size=args.model_size  ,		safe_serialization=args.safe_serialization  ,		)
   __snake_case				:      Optional[int]     							= os.path.join(args.input_dir  ,		'tokenizer.model'  )
   write_tokenizer(args.output_dir  ,		A  )


if __name__ == "__main__":
      main()







'''simple docstring'''






from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
      import PIL


__A    	=   logging.get_logger(__name__)




class a_    (    UpperCamelCase_							):
   _snake_case        =							["""pixel_values"""]







   def __init__(self					,   __a = True					,   __a = None					,   __a = None					,   __a = PILImageResampling.BILINEAR					,   __a = True					,   __a = 1 / 2_5_5					,   __a = True					,   __a = None					,   __a = None					,   **__a					,   )			->      None:
      """simple docstring"""






      super().__init__(**__a)
      __snake_case				:      Tuple     							= size if size is not None else {'shortest_edge': 3_8_4}
      __snake_case				:      List[Any]     							= get_size_dict(__a					,   default_to_square=__a)

      __snake_case				:      int     							= do_resize
      __snake_case				:      List[str]     							= size
      # Default value set here for backwards compatibility where the value in config is None
      __snake_case				:      Any     							= crop_pct if crop_pct is not None else 2_2_4 / 2_5_6
      __snake_case				:      Tuple     							= resample
      __snake_case				:      Dict     							= do_rescale
      __snake_case				:      Any     							= rescale_factor
      __snake_case				:      str     							= do_normalize
      __snake_case				:      Union[str, Any]     							= image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
      __snake_case				:      Dict     							= image_std if image_std is not None else IMAGENET_STANDARD_STD







   def     SCREAMING_SNAKE_CASE__					(self					,   __a					,   __a					,   __a					,   __a = PILImageResampling.BICUBIC					,   __a = None					,   **__a					,   )			->      np.ndarray:
      """simple docstring"""






      __snake_case				:      Dict     							= get_size_dict(__a					,   default_to_square=__a)
      if "shortest_edge" not in size:
         raise ValueError(F"""Size dictionary must contain 'shortest_edge' key. Got {size.keys()}""")
      __snake_case				:      List[str]     							= size['shortest_edge']

      if shortest_edge < 3_8_4:
         # maintain same ratio, resizing shortest edge to shortest_edge/crop_pct
         __snake_case				:      Any     							= int(shortest_edge / crop_pct)
         __snake_case				:      Any     							= get_resize_output_image_size(__a					,   size=__a					,   default_to_square=__a)
         __snake_case				:      int     							= resize(image=__a					,   size=__a					,   resample=__a					,   data_format=__a					,   **__a)
         # then crop to (shortest_edge, shortest_edge)
         return center_crop(image=__a					,   size=(shortest_edge, shortest_edge)					,   data_format=__a					,   **__a)
      else:
         # warping (no cropping) when evaluated at 384 or larger
         return resize(
             __a					,   size=(shortest_edge, shortest_edge)					,   resample=__a					,   data_format=__a					,   **__a)







   def     SCREAMING_SNAKE_CASE__					(self					,   __a					,   __a					,   __a = None					,   **__a					,   )			->      Any:
      """simple docstring"""






      return rescale(__a					,   scale=__a					,   data_format=__a					,   **__a)







   def     SCREAMING_SNAKE_CASE__					(self					,   __a					,   __a					,   __a					,   __a = None					,   **__a					,   )			->      np.ndarray:
      """simple docstring"""






      return normalize(__a					,   mean=__a					,   std=__a					,   data_format=__a					,   **__a)







   def     SCREAMING_SNAKE_CASE__					(self					,   __a					,   __a = None					,   __a = None					,   __a = None					,   __a = None					,   __a = None					,   __a = None					,   __a = None					,   __a = None					,   __a = None					,   __a = None					,   __a = ChannelDimension.FIRST					,   **__a					,   )			->      PIL.Image.Image:
      """simple docstring"""






      __snake_case				:      Optional[int]     							= do_resize if do_resize is not None else self.do_resize
      __snake_case				:      Dict     							= crop_pct if crop_pct is not None else self.crop_pct
      __snake_case				:      Tuple     							= resample if resample is not None else self.resample
      __snake_case				:      Any     							= do_rescale if do_rescale is not None else self.do_rescale
      __snake_case				:      Any     							= rescale_factor if rescale_factor is not None else self.rescale_factor
      __snake_case				:      Union[str, Any]     							= do_normalize if do_normalize is not None else self.do_normalize
      __snake_case				:      Optional[int]     							= image_mean if image_mean is not None else self.image_mean
      __snake_case				:      Optional[Any]     							= image_std if image_std is not None else self.image_std

      __snake_case				:      List[str]     							= size if size is not None else self.size
      __snake_case				:      Any     							= get_size_dict(__a					,   default_to_square=__a)

      __snake_case				:      Dict     							= make_list_of_images(__a)

      if not valid_images(__a):
         raise ValueError(
             'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
             'torch.Tensor, tf.Tensor or jax.ndarray.')

      if do_resize and size is None or resample is None:
         raise ValueError('Size and resample must be specified if do_resize is True.')

      if do_resize and size["shortest_edge"] < 3_8_4 and crop_pct is None:
         raise ValueError('crop_pct must be specified if size < 384.')

      if do_rescale and rescale_factor is None:
         raise ValueError('Rescale factor must be specified if do_rescale is True.')

      if do_normalize and (image_mean is None or image_std is None):
         raise ValueError('Image mean and std must be specified if do_normalize is True.')

      # All transformations expect numpy arrays.
      __snake_case				:      Tuple     							= [to_numpy_array(__a) for image in images]

      if do_resize:
         __snake_case				:      Optional[int]     							= [self.resize(image=__a					,   size=__a					,   crop_pct=__a					,   resample=__a) for image in images]

      if do_rescale:
         __snake_case				:      Optional[int]     							= [self.rescale(image=__a					,   scale=__a) for image in images]

      if do_normalize:
         __snake_case				:      Any     							= [self.normalize(image=__a					,   mean=__a					,   std=__a) for image in images]

      __snake_case				:      Dict     							= [to_channel_dimension_format(__a					,   __a) for image in images]

      __snake_case				:      Union[str, Any]     							= {'pixel_values': images}
      return BatchFeature(data=__a					,   tensor_type=__a)