'''simple docstring'''




import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

_lowercase		:				Tuple	       =		[
    "word_embeddings_layernorm.weight",
    "word_embeddings_layernorm.bias",
    "input_layernorm.weight",
    "input_layernorm.bias",
    "post_attention_layernorm.weight",
    "post_attention_layernorm.bias",
    "self_attention.dense.bias",
    "mlp.dense_4h_to_h.bias",
    "ln_f.weight",
    "ln_f.bias",
]

_lowercase		:				str	       =		[
    "mlp.dense_4h_to_h.weight",
    "self_attention.dense.weight",
]




def 							lowerCamelCase			(   UpperCAmelCase__ :      Optional[int]   ,   UpperCAmelCase__ :      Optional[int]				)    ->   str:
							lowercase_      :    List[str]       		= {
							    """word_embeddings.weight""": """word_embeddings.weight""",
							    """word_embeddings.norm.weight""": """word_embeddings_layernorm.weight""",
							    """word_embeddings.norm.bias""": """word_embeddings_layernorm.bias""",
							    """weight""": """ln_f.weight""",
							    """bias""": """ln_f.bias""",
							}

							if key in layer_rename_map:
														return layer_rename_map[key]

							# Handle transformer blocks
							lowercase_      :    Dict       		= int(re.match(R""".*layer_(\d*).*"""   ,   UpperCAmelCase__				)[1]				)
							layer_number -= 3
							return F'''h.{layer_number}.''' + key




def 							lowerCamelCase			(   UpperCAmelCase__ :      Any				)    ->   Union[str, Any]:
							if dtype == torch.bool:
														return 1 / 8
							lowercase_      :    Tuple       		= re.search(R"""[^\d](\d+)$"""   ,   str(UpperCAmelCase__				)				)
							if bit_search is None:
														raise ValueError(F'''`dtype` is not a valid dtype: {dtype}.'''				)
							lowercase_      :    Optional[int]       		= int(bit_search.groups()[0]				)
							return bit_size // 8




def 							lowerCamelCase			(   UpperCAmelCase__ :      Dict   ,   UpperCAmelCase__ :      Optional[Any]   ,   UpperCAmelCase__ :      Optional[int]   ,   UpperCAmelCase__ :      Union[str, Any]   ,   UpperCAmelCase__ :      Optional[int]				)    ->   Optional[Any]:
							# Construct model
							if bloom_config_file == "":
														lowercase_      :    Union[str, Any]       		= BloomConfig()
							else:
														lowercase_      :    int       		= BloomConfig.from_json_file(UpperCAmelCase__				)

							if shard_model:
														lowercase_      :    Optional[Any]       		= os.listdir(UpperCAmelCase__				)
														lowercase_      :    Union[str, Any]       		= sorted(filter(lambda UpperCAmelCase__				: s.startswith("""layer"""				) and "model_00" in s   ,   UpperCAmelCase__				)				)

														lowercase_      :    Union[str, Any]       		= {"""weight_map""": {}, """metadata""": {}}
														lowercase_      :    Optional[int]       		= 0

														lowercase_      :    int       		= None

														lowercase_      :    str       		= BloomConfig()

														for j, file in enumerate(UpperCAmelCase__				):
																					print("""Processing file: {}""".format(UpperCAmelCase__				)				)
																					lowercase_      :    Any       		= None

																					for i in range(UpperCAmelCase__				):
																												# load all TP files
																												lowercase_      :    Any       		= file.replace("""model_00"""   ,   F'''model_0{i}'''				)
																												lowercase_      :    Any       		= torch.load(os.path.join(UpperCAmelCase__   ,   UpperCAmelCase__				)   ,   map_location="""cpu"""				)

																												# Rename keys in the transformers names
																												lowercase_      :    int       		= list(temp.keys()				)
																												for key in keys:
																																			lowercase_      :    Any       		= temp.pop(UpperCAmelCase__				)

																												if tensors is None:
																																			lowercase_      :    str       		= temp
																												else:
																																			for key in tensors.keys():
																																										if any(key.endswith(UpperCAmelCase__				) for end in WEIGHTS_TO_AVERAGE_ENDSWITH				):
																																																	# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																																																	tensors[key] += temp[key]
																																										else:
																																																	# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																																																	lowercase_      :    Union[str, Any]       		= 1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN				) else 0
																																																	# We concatenate these weights accross TP ranks
																																																	lowercase_      :    Any       		= torch.cat([tensors[key], temp[key]]   ,   dim=UpperCAmelCase__				)

            # Divide by the number of TP the weights we want to average
																					for key in tensors.keys():
																												if any(key.endswith(UpperCAmelCase__				) for end in WEIGHTS_TO_AVERAGE_ENDSWITH				):
																																			lowercase_      :    Dict       		= tensors[key] / pretraining_tp
																					torch.save(
																					    UpperCAmelCase__   ,   os.path.join(
																					        UpperCAmelCase__   ,   """pytorch_model_{}-of-{}.bin""".format(str(j + 1				).zfill(5				)   ,   str(len(UpperCAmelCase__				)				).zfill(5				)				)   ,   )   ,   )

																					for key in tensors.keys():
																												lowercase_      :    Optional[int]       		= tensors[key]
																												total_size += value.numel() * get_dtype_size(value.dtype				)
																												if key not in index_dict["weight_map"]:
																																			lowercase_      :    Optional[int]       		= """pytorch_model_{}-of-{}.bin""".format(
																																			    str(j + 1				).zfill(5				)   ,   str(len(UpperCAmelCase__				)				).zfill(5				)				)

														lowercase_      :    Dict       		= BloomConfig()
														lowercase_      :    int       		= pytorch_dump_folder_path + """/""" + CONFIG_NAME
														lowercase_      :    List[Any]       		= total_size
														with open(UpperCAmelCase__   ,   """w"""   ,   encoding="""utf-8"""				) as f:
																					f.write(config.to_json_string()				)
														with open(os.path.join(UpperCAmelCase__   ,   WEIGHTS_NAME + """.index.json"""				)   ,   """w"""   ,   encoding="""utf-8"""				) as f:
																					lowercase_      :    List[Any]       		= json.dumps(UpperCAmelCase__   ,   indent=2   ,   sort_keys=UpperCAmelCase__				) + """\n"""
																					f.write(UpperCAmelCase__				)
							else:
														lowercase_      :    Optional[int]       		= BloomModel(UpperCAmelCase__				)

														lowercase_      :    int       		= os.listdir(UpperCAmelCase__				)
														lowercase_      :    Optional[int]       		= sorted(filter(lambda UpperCAmelCase__				: s.startswith("""layer"""				) and "model_00" in s   ,   UpperCAmelCase__				)				)

														lowercase_      :    Union[str, Any]       		= None
														for i, file in enumerate(UpperCAmelCase__				):
																					lowercase_      :    Tuple       		= None
																					for i in range(UpperCAmelCase__				):
																												# load all TP files
																												lowercase_      :    List[str]       		= file.replace("""model_00"""   ,   F'''model_0{i}'''				)
																												lowercase_      :    List[Any]       		= torch.load(os.path.join(UpperCAmelCase__   ,   UpperCAmelCase__				)   ,   map_location="""cpu"""				)

																												# Rename keys in the transformers names
																												lowercase_      :    int       		= list(temp.keys()				)
																												for key in keys:
																																			lowercase_      :    int       		= temp.pop(UpperCAmelCase__				)

																												if tensors is None:
																																			lowercase_      :    Dict       		= temp
																												else:
																																			for key in tensors.keys():
																																										# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																																										if any(key.endswith(UpperCAmelCase__				) for end in WEIGHTS_TO_AVERAGE_ENDSWITH				):
																																																	tensors[key] += temp[key]
																																										else:
																																																	# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																																																	lowercase_      :    Dict       		= 1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN				) else 0
																																																	# We concatenate these weights accross TP ranks
																																																	lowercase_      :    Optional[Any]       		= torch.cat([tensors[key], temp[key]]   ,   dim=UpperCAmelCase__				)

            # Divide by the number of TP the weights we want to average
																					for key in tensors.keys():
																												if any(key.endswith(UpperCAmelCase__				) for end in WEIGHTS_TO_AVERAGE_ENDSWITH				):
																																			lowercase_      :    int       		= tensors[key] / pretraining_tp

																					lowercase_      :    Optional[int]       		= model.load_state_dict(UpperCAmelCase__   ,   strict=UpperCAmelCase__				)
																					assert not other_keys.unexpected_keys, F'''The keys {other_keys.unexpected_keys} are unexpected'''
																					if missing_keys is None:
																												lowercase_      :    Union[str, Any]       		= set(other_keys.missing_keys				)
																					else:
																												lowercase_      :    str       		= missing_keys.intersection(set(other_keys.missing_keys				)				)

														assert not missing_keys, F'''The keys {missing_keys} are missing'''

														# Save pytorch-model
														os.makedirs(UpperCAmelCase__   ,   exist_ok=UpperCAmelCase__				)
														lowercase_      :    Union[str, Any]       		= pytorch_dump_folder_path + """/""" + WEIGHTS_NAME
														lowercase_      :    Any       		= pytorch_dump_folder_path + """/""" + CONFIG_NAME
														print(F'''Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'''				)
														if config.torch_dtype is not None:
																					lowercase_      :    Union[str, Any]       		= model.to(config.torch_dtype				)
														torch.save(model.state_dict()   ,   UpperCAmelCase__				)
														print(F'''Save configuration file to {pytorch_config_dump_path}'''				)
														with open(UpperCAmelCase__   ,   """w"""   ,   encoding="""utf-8"""				) as f:
																					f.write(config.to_json_string()				)


if __name__ == "__main__":
			_lowercase		:				Union[str, Any]	       =		argparse.ArgumentParser()
			# Required parameters
			parser.add_argument(
			    "--bloom_checkpoint_path",
			    default=None,
			    type=str,
			    required=True,
			    help="Path to the Megatron-LM checkpoint path.",
			)
			parser.add_argument(
			    "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
			)
			parser.add_argument(
			    "--bloom_config_file",
			    default="",
			    type=str,
			    help=(
			        "An optional config json file corresponding to the pre-trained model. \n"
			        "This specifies the model architecture."
			    ),
			)
			parser.add_argument(
			    "--shard_model",
			    action="store_true",
			    help="An optional setting to shard the output model \nThis enables sharding the converted checkpoint",
			)
			parser.add_argument(
			    "--pretraining_tp",
			    default=4,
			    type=int,
			    help="Pretraining TP rank that has been used when training the model in Megatron-LM \n",
			)
			_lowercase		:				List[str]	       =		parser.parse_args()
			convert_bloom_checkpoint_to_pytorch(
			    args.bloom_checkpoint_path,
			    args.bloom_config_file,
			    args.pytorch_dump_folder_path,
			    args.shard_model,
			    args.pretraining_tp,
			)






'''simple docstring'''




import pytest

import datasets.config
from datasets.utils.info_utils import is_small_dataset




@pytest.mark.parametrize("""dataset_size"""   ,   [None, 400 * 2**20, 600 * 2**20]				)
@pytest.mark.parametrize("""input_in_memory_max_size"""   ,   ["""default""", 0, 100 * 2**20, 900 * 2**20]				)
def 							lowerCamelCase			(   UpperCAmelCase__ :      Union[str, Any]   ,   UpperCAmelCase__ :      List[Any]   ,   UpperCAmelCase__ :      str				)    ->   Any:
							if input_in_memory_max_size != "default":
														monkeypatch.setattr(datasets.config   ,   """IN_MEMORY_MAX_SIZE"""   ,   UpperCAmelCase__				)
							lowercase_      :    List[Any]       		= datasets.config.IN_MEMORY_MAX_SIZE
							if input_in_memory_max_size == "default":
														assert in_memory_max_size == 0
							else:
														assert in_memory_max_size == input_in_memory_max_size
							if dataset_size and in_memory_max_size:
														lowercase_      :    str       		= dataset_size < in_memory_max_size
							else:
														lowercase_      :    List[Any]       		= False
							lowercase_      :    Any       		= is_small_dataset(UpperCAmelCase__				)
							assert result == expected





