





import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

__lowercase       :Union[str, Any]       =  [
    """word_embeddings_layernorm.weight""",
    """word_embeddings_layernorm.bias""",
    """input_layernorm.weight""",
    """input_layernorm.bias""",
    """post_attention_layernorm.weight""",
    """post_attention_layernorm.bias""",
    """self_attention.dense.bias""",
    """mlp.dense_4h_to_h.bias""",
    """ln_f.weight""",
    """ln_f.bias""",
]

__lowercase       :str       =  [
    """mlp.dense_4h_to_h.weight""",
    """self_attention.dense.weight""",
]







def       UpperCAmelCase							(     _lowerCamelCase      :  Union[str, Any]			,   _lowerCamelCase      :  str     ):





						'''simple docstring'''


						SCREAMING_SNAKE_CASE__     :				Any   =					{
						    "word_embeddings.weight": "word_embeddings.weight",
						    "word_embeddings.norm.weight": "word_embeddings_layernorm.weight",
						    "word_embeddings.norm.bias": "word_embeddings_layernorm.bias",
						    "weight": "ln_f.weight",
						    "bias": "ln_f.bias",
						}

						if key in layer_rename_map:
												return layer_rename_map[key]

						# Handle transformer blocks
						SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					int(re.match(r".*layer_(\d*).*"			,   _lowerCamelCase     )[1]     )
						layer_number -= 3
						return f"""h.{layer_number}.""" + key







def       UpperCAmelCase							(     _lowerCamelCase      :  Union[str, Any]     ):





						'''simple docstring'''


						if dtype == torch.bool:
												return 1 / 8
						SCREAMING_SNAKE_CASE__     :				str   =					re.search(r"[^\d](\d+)$"			,   str(_lowerCamelCase     )     )
						if bit_search is None:
												raise ValueError(f"""`dtype` is not a valid dtype: {dtype}."""     )
						SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					int(bit_search.groups()[0]     )
						return bit_size // 8







def       UpperCAmelCase							(     _lowerCamelCase      :  Optional[Any]			,   _lowerCamelCase      :  List[str]			,   _lowerCamelCase      :  Any			,   _lowerCamelCase      :  Optional[int]			,   _lowerCamelCase      :  Any     ):





						'''simple docstring'''


						if bloom_config_file == "":
												SCREAMING_SNAKE_CASE__     :				str   =					BloomConfig()
						else:
												SCREAMING_SNAKE_CASE__     :				Dict   =					BloomConfig.from_json_file(_lowerCamelCase     )

						if shard_model:
												SCREAMING_SNAKE_CASE__     :				Dict   =					os.listdir(_lowerCamelCase     )
												SCREAMING_SNAKE_CASE__     :				Any   =					sorted(filter(lambda _lowerCamelCase     : s.startswith("layer"     ) and "model_00" in s			,   _lowerCamelCase     )     )

												SCREAMING_SNAKE_CASE__     :				Optional[int]   =					{"weight_map": {}, "metadata": {}}
												SCREAMING_SNAKE_CASE__     :				Optional[int]   =					0

												SCREAMING_SNAKE_CASE__     :				str   =					None

												SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					BloomConfig()

												for j, file in enumerate(_lowerCamelCase     ):
																		print("Processing file: {}".format(_lowerCamelCase     )     )
																		SCREAMING_SNAKE_CASE__     :				Optional[int]   =					None

																		for i in range(_lowerCamelCase     ):
																								# load all TP files
																								SCREAMING_SNAKE_CASE__     :				List[Any]   =					file.replace("model_00"			,   f"""model_0{i}"""     )
																								SCREAMING_SNAKE_CASE__     :				Tuple   =					torch.load(os.path.join(_lowerCamelCase			,   _lowerCamelCase     )			,   map_location="cpu"     )

																								# Rename keys in the transformers names
																								SCREAMING_SNAKE_CASE__     :				Optional[int]   =					list(temp.keys()     )
																								for key in keys:
																														SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					temp.pop(_lowerCamelCase     )

																								if tensors is None:
																														SCREAMING_SNAKE_CASE__     :				Any   =					temp
																								else:
																														for key in tensors.keys():
																																				if any(key.endswith(_lowerCamelCase     ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH     ):
																																										# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																																										tensors[key] += temp[key]
																																				else:
																																										# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																																										SCREAMING_SNAKE_CASE__     :				Dict   =					1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN     ) else 0
																																										# We concatenate these weights accross TP ranks
																																										SCREAMING_SNAKE_CASE__     :				Any   =					torch.cat([tensors[key], temp[key]]			,   dim=_lowerCamelCase     )

																																				# Divide by the number of TP the weights we want to average
																		for key in tensors.keys():
																								if any(key.endswith(_lowerCamelCase     ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH     ):
																														SCREAMING_SNAKE_CASE__     :				List[str]   =					tensors[key] / pretraining_tp
																		torch.save(
																		    _lowerCamelCase			,   os.path.join(
																		        _lowerCamelCase			,   "pytorch_model_{}-of-{}.bin".format(str(j + 1     ).zfill(5     )			,   str(len(_lowerCamelCase     )     ).zfill(5     )     )			,   )			,   )

																		for key in tensors.keys():
																								SCREAMING_SNAKE_CASE__     :				Dict   =					tensors[key]
																								total_size += value.numel() * get_dtype_size(value.dtype     )
																								if key not in index_dict["weight_map"]:
																														SCREAMING_SNAKE_CASE__     :				List[str]   =					"pytorch_model_{}-of-{}.bin".format(
																														    str(j + 1     ).zfill(5     )			,   str(len(_lowerCamelCase     )     ).zfill(5     )     )

												SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					BloomConfig()
												SCREAMING_SNAKE_CASE__     :				Tuple   =					pytorch_dump_folder_path + "/" + CONFIG_NAME
												SCREAMING_SNAKE_CASE__     :				Optional[int]   =					total_size
												with open(_lowerCamelCase			,   "w"			,   encoding="utf-8"     ) as f:
																		f.write(config.to_json_string()     )
												with open(os.path.join(_lowerCamelCase			,   WEIGHTS_NAME + ".index.json"     )			,   "w"			,   encoding="utf-8"     ) as f:
																		SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					json.dumps(_lowerCamelCase			,   indent=2			,   sort_keys=_lowerCamelCase     ) + "\n"
																		f.write(_lowerCamelCase     )
						else:
												SCREAMING_SNAKE_CASE__     :				List[str]   =					BloomModel(_lowerCamelCase     )

												SCREAMING_SNAKE_CASE__     :				Optional[int]   =					os.listdir(_lowerCamelCase     )
												SCREAMING_SNAKE_CASE__     :				int   =					sorted(filter(lambda _lowerCamelCase     : s.startswith("layer"     ) and "model_00" in s			,   _lowerCamelCase     )     )

												SCREAMING_SNAKE_CASE__     :				Tuple   =					None
												for i, file in enumerate(_lowerCamelCase     ):
																		SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					None
																		for i in range(_lowerCamelCase     ):
																								# load all TP files
																								SCREAMING_SNAKE_CASE__     :				Tuple   =					file.replace("model_00"			,   f"""model_0{i}"""     )
																								SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					torch.load(os.path.join(_lowerCamelCase			,   _lowerCamelCase     )			,   map_location="cpu"     )

																								# Rename keys in the transformers names
																								SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					list(temp.keys()     )
																								for key in keys:
																														SCREAMING_SNAKE_CASE__     :				List[Any]   =					temp.pop(_lowerCamelCase     )

																								if tensors is None:
																														SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					temp
																								else:
																														for key in tensors.keys():
																																				# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																																				if any(key.endswith(_lowerCamelCase     ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH     ):
																																										tensors[key] += temp[key]
																																				else:
																																										# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																																										SCREAMING_SNAKE_CASE__     :				str   =					1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN     ) else 0
																																										# We concatenate these weights accross TP ranks
																																										SCREAMING_SNAKE_CASE__     :				Any   =					torch.cat([tensors[key], temp[key]]			,   dim=_lowerCamelCase     )

																																				# Divide by the number of TP the weights we want to average
																		for key in tensors.keys():
																								if any(key.endswith(_lowerCamelCase     ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH     ):
																														SCREAMING_SNAKE_CASE__     :				Dict   =					tensors[key] / pretraining_tp

																		SCREAMING_SNAKE_CASE__     :				Dict   =					model.load_state_dict(_lowerCamelCase			,   strict=_lowerCamelCase     )
																		assert not other_keys.unexpected_keys, f"""The keys {other_keys.unexpected_keys} are unexpected"""
																		if missing_keys is None:
																								SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					set(other_keys.missing_keys     )
																		else:
																								SCREAMING_SNAKE_CASE__     :				str   =					missing_keys.intersection(set(other_keys.missing_keys     )     )

												assert not missing_keys, f"""The keys {missing_keys} are missing"""

												# Save pytorch-model
												os.makedirs(_lowerCamelCase			,   exist_ok=_lowerCamelCase     )
												SCREAMING_SNAKE_CASE__     :				Tuple   =					pytorch_dump_folder_path + "/" + WEIGHTS_NAME
												SCREAMING_SNAKE_CASE__     :				int   =					pytorch_dump_folder_path + "/" + CONFIG_NAME
												print(f"""Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}"""     )
												if config.torch_dtype is not None:
																		SCREAMING_SNAKE_CASE__     :				List[Any]   =					model.to(config.torch_dtype     )
												torch.save(model.state_dict()			,   _lowerCamelCase     )
												print(f"""Save configuration file to {pytorch_config_dump_path}"""     )
												with open(_lowerCamelCase			,   "w"			,   encoding="utf-8"     ) as f:
																		f.write(config.to_json_string()     )


if __name__ == "__main__":
				__lowercase       :Optional[Any]       =  argparse.ArgumentParser()
				# Required parameters
				parser.add_argument(
				    "--bloom_checkpoint_path",
				    default=None,
				    type=str,
				    required=True,
				    help="Path to the Megatron-LM checkpoint path.",
				)
				parser.add_argument(
				    "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
				)
				parser.add_argument(
				    "--bloom_config_file",
				    default="",
				    type=str,
				    help=(
				        "An optional config json file corresponding to the pre-trained model. \n"
				        "This specifies the model architecture."
				    ),
				)
				parser.add_argument(
				    "--shard_model",
				    action="store_true",
				    help="An optional setting to shard the output model \nThis enables sharding the converted checkpoint",
				)
				parser.add_argument(
				    "--pretraining_tp",
				    default=4,
				    type=int,
				    help="Pretraining TP rank that has been used when training the model in Megatron-LM \n",
				)
				__lowercase       :Dict       =  parser.parse_args()
				convert_bloom_checkpoint_to_pytorch(
				    args.bloom_checkpoint_path,
				    args.bloom_config_file,
				    args.pytorch_dump_folder_path,
				    args.shard_model,
				    args.pretraining_tp,
				)






from __future__ import annotations

from fractions import Fraction







def       UpperCAmelCase							(     _lowerCamelCase      :  int			,   _lowerCamelCase      :  int     ):





						'''simple docstring'''


						return (
						    num != den and num % 10 == den // 10 and (num // 10) / (den % 10) == num / den
						)







def       UpperCAmelCase							(     _lowerCamelCase      :  int     ):





						'''simple docstring'''


						SCREAMING_SNAKE_CASE__     :				int   =					[]
						SCREAMING_SNAKE_CASE__     :				str   =					11
						SCREAMING_SNAKE_CASE__     :				Any   =					int("1" + "0" * digit_len     )
						for num in range(_lowerCamelCase			,   _lowerCamelCase     ):
												while den <= 99:
																		if (num != den) and (num % 10 == den // 10) and (den % 10 != 0):
																								if is_digit_cancelling(_lowerCamelCase			,   _lowerCamelCase     ):
																														solutions.append(f"""{num}/{den}"""     )
																		den += 1
												num += 1
												SCREAMING_SNAKE_CASE__     :				str   =					10
						return solutions







def       UpperCAmelCase							(     _lowerCamelCase      :  int = 2     ):





						'''simple docstring'''


						SCREAMING_SNAKE_CASE__     :				Optional[int]   =					1.0
						for fraction in fraction_list(_lowerCamelCase     ):
												SCREAMING_SNAKE_CASE__     :				Any   =					Fraction(_lowerCamelCase     )
												result *= frac.denominator / frac.numerator
						return int(_lowerCamelCase     )


if __name__ == "__main__":
				print(solution())