

import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

SCREAMING_SNAKE_CASE_        =      [
    """word_embeddings_layernorm.weight""",
    """word_embeddings_layernorm.bias""",
    """input_layernorm.weight""",
    """input_layernorm.bias""",
    """post_attention_layernorm.weight""",
    """post_attention_layernorm.bias""",
    """self_attention.dense.bias""",
    """mlp.dense_4h_to_h.bias""",
    """ln_f.weight""",
    """ln_f.bias""",
]

SCREAMING_SNAKE_CASE_        =      [
    """mlp.dense_4h_to_h.weight""",
    """self_attention.dense.weight""",
]

def 							__SCREAMING_SNAKE_CASE							(				lowerCAmelCase:  Union[str, Any]  ,      lowerCAmelCase:  str )   ->       Dict:
				_UpperCAmelCase       :		Any 		=    {
				    "word_embeddings.weight": "word_embeddings.weight",
				    "word_embeddings.norm.weight": "word_embeddings_layernorm.weight",
				    "word_embeddings.norm.bias": "word_embeddings_layernorm.bias",
				    "weight": "ln_f.weight",
				    "bias": "ln_f.bias",
				}

				if key in layer_rename_map:
								return layer_rename_map[key]

				# Handle transformer blocks
				_UpperCAmelCase       :		Optional[Any] 		=    int(re.match(R".*layer_(\d*).*"  ,      lowerCAmelCase )[1] )
				layer_number -= 3
				return F'h.{layer_number}.' + key

def 							__SCREAMING_SNAKE_CASE							(				lowerCAmelCase:  Union[str, Any] )   ->       List[str]:







				if dtype == torch.bool:
								return 1 / 8
				_UpperCAmelCase       :		str 		=    re.search(R"[^\d](\d+)$"  ,      str(lowerCAmelCase ) )
				if bit_search is None:
								raise ValueError(F'`dtype` is not a valid dtype: {dtype}.' )
				_UpperCAmelCase       :		Union[str, Any] 		=    int(bit_search.groups()[0] )
				return bit_size // 8

def 							__SCREAMING_SNAKE_CASE							(				lowerCAmelCase:  Optional[Any]  ,      lowerCAmelCase:  List[str]  ,      lowerCAmelCase:  Any  ,      lowerCAmelCase:  Optional[int]  ,      lowerCAmelCase:  Any )   ->       Union[str, Any]:







				if bloom_config_file == "":
								_UpperCAmelCase       :		str 		=    BloomConfig()
				else:
								_UpperCAmelCase       :		Dict 		=    BloomConfig.from_json_file(lowerCAmelCase )

				if shard_model:
								_UpperCAmelCase       :		Dict 		=    os.listdir(lowerCAmelCase )
								_UpperCAmelCase       :		Any 		=    sorted(filter(lambda lowerCAmelCase : s.startswith("layer" ) and "model_00" in s  ,      lowerCAmelCase ) )

								_UpperCAmelCase       :		Optional[int] 		=    {"weight_map": {}, "metadata": {}}
								_UpperCAmelCase       :		Optional[int] 		=    0

								_UpperCAmelCase       :		str 		=    None

								_UpperCAmelCase       :		Optional[Any] 		=    BloomConfig()

								for j, file in enumerate(lowerCAmelCase ):
												print("Processing file: {}".format(lowerCAmelCase ) )
												_UpperCAmelCase       :		Optional[int] 		=    None

												for i in range(lowerCAmelCase ):
																# load all TP files
																_UpperCAmelCase       :		List[Any] 		=    file.replace("model_00"  ,      F'model_0{i}' )
																_UpperCAmelCase       :		Tuple 		=    torch.load(os.path.join(lowerCAmelCase  ,      lowerCAmelCase )  ,      map_location="cpu" )

																# Rename keys in the transformers names
																_UpperCAmelCase       :		Optional[int] 		=    list(temp.keys() )
																for key in keys:
																				_UpperCAmelCase       :		Optional[Any] 		=    temp.pop(lowerCAmelCase )

																if tensors is None:
																				_UpperCAmelCase       :		Any 		=    temp
																else:
																				for key in tensors.keys():
																								if any(key.endswith(lowerCAmelCase ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH ):
																												# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																												tensors[key] += temp[key]
																								else:
																												# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																												_UpperCAmelCase       :		Dict 		=    1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN ) else 0
																												# We concatenate these weights accross TP ranks
																												_UpperCAmelCase       :		Any 		=    torch.cat([tensors[key], temp[key]]  ,      dim=lowerCAmelCase )

																								# Divide by the number of TP the weights we want to average
												for key in tensors.keys():
																if any(key.endswith(lowerCAmelCase ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH ):
																				_UpperCAmelCase       :		List[str] 		=    tensors[key] / pretraining_tp
												torch.save(
												    lowerCAmelCase  ,      os.path.join(
												        lowerCAmelCase  ,      "pytorch_model_{}-of-{}.bin".format(str(j + 1 ).zfill(5 )  ,      str(len(lowerCAmelCase ) ).zfill(5 ) )  ,      )  ,      )

												for key in tensors.keys():
																_UpperCAmelCase       :		Dict 		=    tensors[key]
																total_size += value.numel() * get_dtype_size(value.dtype )
																if key not in index_dict["weight_map"]:
																				_UpperCAmelCase       :		List[str] 		=    "pytorch_model_{}-of-{}.bin".format(
																				    str(j + 1 ).zfill(5 )  ,      str(len(lowerCAmelCase ) ).zfill(5 ) )

								_UpperCAmelCase       :		Optional[Any] 		=    BloomConfig()
								_UpperCAmelCase       :		Tuple 		=    pytorch_dump_folder_path + "/" + CONFIG_NAME
								_UpperCAmelCase       :		Optional[int] 		=    total_size
								with open(lowerCAmelCase  ,      "w"  ,      encoding="utf-8" ) as f:
												f.write(config.to_json_string() )
								with open(os.path.join(lowerCAmelCase  ,      WEIGHTS_NAME + ".index.json" )  ,      "w"  ,      encoding="utf-8" ) as f:
												_UpperCAmelCase       :		Union[str, Any] 		=    json.dumps(lowerCAmelCase  ,      indent=2  ,      sort_keys=lowerCAmelCase ) + "\n"
												f.write(lowerCAmelCase )
				else:
								_UpperCAmelCase       :		List[str] 		=    BloomModel(lowerCAmelCase )

								_UpperCAmelCase       :		Optional[int] 		=    os.listdir(lowerCAmelCase )
								_UpperCAmelCase       :		int 		=    sorted(filter(lambda lowerCAmelCase : s.startswith("layer" ) and "model_00" in s  ,      lowerCAmelCase ) )

								_UpperCAmelCase       :		Tuple 		=    None
								for i, file in enumerate(lowerCAmelCase ):
												_UpperCAmelCase       :		Optional[Any] 		=    None
												for i in range(lowerCAmelCase ):
																# load all TP files
																_UpperCAmelCase       :		Tuple 		=    file.replace("model_00"  ,      F'model_0{i}' )
																_UpperCAmelCase       :		Union[str, Any] 		=    torch.load(os.path.join(lowerCAmelCase  ,      lowerCAmelCase )  ,      map_location="cpu" )

																# Rename keys in the transformers names
																_UpperCAmelCase       :		Optional[Any] 		=    list(temp.keys() )
																for key in keys:
																				_UpperCAmelCase       :		List[Any] 		=    temp.pop(lowerCAmelCase )

																if tensors is None:
																				_UpperCAmelCase       :		Optional[Any] 		=    temp
																else:
																				for key in tensors.keys():
																								# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																								if any(key.endswith(lowerCAmelCase ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH ):
																												tensors[key] += temp[key]
																								else:
																												# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																												_UpperCAmelCase       :		str 		=    1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN ) else 0
																												# We concatenate these weights accross TP ranks
																												_UpperCAmelCase       :		Any 		=    torch.cat([tensors[key], temp[key]]  ,      dim=lowerCAmelCase )

																								# Divide by the number of TP the weights we want to average
												for key in tensors.keys():
																if any(key.endswith(lowerCAmelCase ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH ):
																				_UpperCAmelCase       :		Dict 		=    tensors[key] / pretraining_tp

												_UpperCAmelCase       :		Dict 		=    model.load_state_dict(lowerCAmelCase  ,      strict=lowerCAmelCase )
												assert not other_keys.unexpected_keys, F'The keys {other_keys.unexpected_keys} are unexpected'
												if missing_keys is None:
																_UpperCAmelCase       :		Optional[Any] 		=    set(other_keys.missing_keys )
												else:
																_UpperCAmelCase       :		str 		=    missing_keys.intersection(set(other_keys.missing_keys ) )

								assert not missing_keys, F'The keys {missing_keys} are missing'

								# Save pytorch-model
								os.makedirs(lowerCAmelCase  ,      exist_ok=lowerCAmelCase )
								_UpperCAmelCase       :		Tuple 		=    pytorch_dump_folder_path + "/" + WEIGHTS_NAME
								_UpperCAmelCase       :		int 		=    pytorch_dump_folder_path + "/" + CONFIG_NAME
								print(F'Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}' )
								if config.torch_dtype is not None:
												_UpperCAmelCase       :		List[Any] 		=    model.to(config.torch_dtype )
								torch.save(model.state_dict()  ,      lowerCAmelCase )
								print(F'Save configuration file to {pytorch_config_dump_path}' )
								with open(lowerCAmelCase  ,      "w"  ,      encoding="utf-8" ) as f:
												f.write(config.to_json_string() )


if __name__ == "__main__":
					SCREAMING_SNAKE_CASE_        =      argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    '--bloom_checkpoint_path',
					    default=None,
					    type=str,
					    required=True,
					    help='Path to the Megatron-LM checkpoint path.',
					)
					parser.add_argument(
					    '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
					)
					parser.add_argument(
					    '--bloom_config_file',
					    default='',
					    type=str,
					    help=(
					        'An optional config json file corresponding to the pre-trained model. \n'
					        'This specifies the model architecture.'
					    ),
					)
					parser.add_argument(
					    '--shard_model',
					    action='store_true',
					    help='An optional setting to shard the output model \nThis enables sharding the converted checkpoint',
					)
					parser.add_argument(
					    '--pretraining_tp',
					    default=4,
					    type=int,
					    help='Pretraining TP rank that has been used when training the model in Megatron-LM \n',
					)
					SCREAMING_SNAKE_CASE_        =      parser.parse_args()
					convert_bloom_checkpoint_to_pytorch(
					    args.bloom_checkpoint_path,
					    args.bloom_config_file,
					    args.pytorch_dump_folder_path,
					    args.shard_model,
					    args.pretraining_tp,
					)







from __future__ import annotations

import numpy as np

def 							__SCREAMING_SNAKE_CASE							(				lowerCAmelCase:  list[float] )   ->       Dict:
				return np.maximum(0  ,      lowerCAmelCase )


if __name__ == "__main__":
					print(np.array(relu([-1, 0, 5])))  # --> [0, 0, 5]


