





'''simple docstring'''

import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

a        =						[
    "word_embeddings_layernorm.weight",
    "word_embeddings_layernorm.bias",
    "input_layernorm.weight",
    "input_layernorm.bias",
    "post_attention_layernorm.weight",
    "post_attention_layernorm.bias",
    "self_attention.dense.bias",
    "mlp.dense_4h_to_h.bias",
    "ln_f.weight",
    "ln_f.bias",
]

a        =						[
    "mlp.dense_4h_to_h.weight",
    "self_attention.dense.weight",
]

def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase      )						->					Tuple:

   '''simple docstring'''







   __SCREAMING_SNAKE_CASE      							=       {
       """word_embeddings.weight""": """word_embeddings.weight""",
       """word_embeddings.norm.weight""": """word_embeddings_layernorm.weight""",
       """word_embeddings.norm.bias""": """word_embeddings_layernorm.bias""",
       """weight""": """ln_f.weight""",
       """bias""": """ln_f.bias""",
   }

   if key in layer_rename_map:
      return layer_rename_map[key]

   # Handle transformer blocks
   __SCREAMING_SNAKE_CASE      							=       int(re.match(R""".*layer_(\d*).*"""							,	__UpperCAmelCase      )[1]      )
   layer_number -= 3
   return f"""h.{layer_number}.""" + key

def 						__magic_name__       (							__UpperCAmelCase      )						->					Dict:

   '''simple docstring'''







   if dtype == torch.bool:
      return 1 / 8
   __SCREAMING_SNAKE_CASE      							=       re.search(R"""[^\d](\d+)$"""							,	str(__UpperCAmelCase      )      )
   if bit_search is None:
      raise ValueError(f"""`dtype` is not a valid dtype: {dtype}."""      )
   __SCREAMING_SNAKE_CASE      							=       int(bit_search.groups()[0]      )
   return bit_size // 8

def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase      )						->					Any:

   '''simple docstring'''







   if bloom_config_file == "":
      __SCREAMING_SNAKE_CASE      							=       BloomConfig()
   else:
      __SCREAMING_SNAKE_CASE      							=       BloomConfig.from_json_file(__UpperCAmelCase      )

   if shard_model:
      __SCREAMING_SNAKE_CASE      							=       os.listdir(__UpperCAmelCase      )
      __SCREAMING_SNAKE_CASE      							=       sorted(filter(lambda __UpperCAmelCase      : s.startswith("""layer"""      ) and "model_00" in s							,	__UpperCAmelCase      )      )

      __SCREAMING_SNAKE_CASE      							=       {"""weight_map""": {}, """metadata""": {}}
      __SCREAMING_SNAKE_CASE      							=       0

      __SCREAMING_SNAKE_CASE      							=       None

      __SCREAMING_SNAKE_CASE      							=       BloomConfig()

      for j, file in enumerate(__UpperCAmelCase      ):
         print("""Processing file: {}""".format(__UpperCAmelCase      )      )
         __SCREAMING_SNAKE_CASE      							=       None

         for i in range(__UpperCAmelCase      ):
            # load all TP files
            __SCREAMING_SNAKE_CASE      							=       file.replace("""model_00"""							,	f"""model_0{i}"""      )
            __SCREAMING_SNAKE_CASE      							=       torch.load(os.path.join(__UpperCAmelCase							,	__UpperCAmelCase      )							,	map_location="""cpu"""      )

            # Rename keys in the transformers names
            __SCREAMING_SNAKE_CASE      							=       list(temp.keys()      )
            for key in keys:
               __SCREAMING_SNAKE_CASE      							=       temp.pop(__UpperCAmelCase      )

            if tensors is None:
               __SCREAMING_SNAKE_CASE      							=       temp
            else:
               for key in tensors.keys():
                  if any(key.endswith(__UpperCAmelCase      ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH      ):
                     # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                     tensors[key] += temp[key]
                  else:
                     # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                     __SCREAMING_SNAKE_CASE      							=       1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN      ) else 0
                     # We concatenate these weights accross TP ranks
                     __SCREAMING_SNAKE_CASE      							=       torch.cat([tensors[key], temp[key]]							,	dim=__UpperCAmelCase      )

            # Divide by the number of TP the weights we want to average
         for key in tensors.keys():
            if any(key.endswith(__UpperCAmelCase      ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH      ):
               __SCREAMING_SNAKE_CASE      							=       tensors[key] / pretraining_tp
         torch.save(
             __UpperCAmelCase							,	os.path.join(
                 __UpperCAmelCase							,	"""pytorch_model_{}-of-{}.bin""".format(str(j + 1      ).zfill(5      )							,	str(len(__UpperCAmelCase      )      ).zfill(5      )      )							,	)							,	)

         for key in tensors.keys():
            __SCREAMING_SNAKE_CASE      							=       tensors[key]
            total_size += value.numel() * get_dtype_size(value.dtype      )
            if key not in index_dict["weight_map"]:
               __SCREAMING_SNAKE_CASE      							=       """pytorch_model_{}-of-{}.bin""".format(
                   str(j + 1      ).zfill(5      )							,	str(len(__UpperCAmelCase      )      ).zfill(5      )      )

      __SCREAMING_SNAKE_CASE      							=       BloomConfig()
      __SCREAMING_SNAKE_CASE      							=       pytorch_dump_folder_path + """/""" + CONFIG_NAME
      __SCREAMING_SNAKE_CASE      							=       total_size
      with open(__UpperCAmelCase							,	"""w"""							,	encoding="""utf-8"""      ) as f:
         f.write(config.to_json_string()      )
      with open(os.path.join(__UpperCAmelCase							,	WEIGHTS_NAME + """.index.json"""      )							,	"""w"""							,	encoding="""utf-8"""      ) as f:
         __SCREAMING_SNAKE_CASE      							=       json.dumps(__UpperCAmelCase							,	indent=2							,	sort_keys=__UpperCAmelCase      ) + """\n"""
         f.write(__UpperCAmelCase      )
   else:
      __SCREAMING_SNAKE_CASE      							=       BloomModel(__UpperCAmelCase      )

      __SCREAMING_SNAKE_CASE      							=       os.listdir(__UpperCAmelCase      )
      __SCREAMING_SNAKE_CASE      							=       sorted(filter(lambda __UpperCAmelCase      : s.startswith("""layer"""      ) and "model_00" in s							,	__UpperCAmelCase      )      )

      __SCREAMING_SNAKE_CASE      							=       None
      for i, file in enumerate(__UpperCAmelCase      ):
         __SCREAMING_SNAKE_CASE      							=       None
         for i in range(__UpperCAmelCase      ):
            # load all TP files
            __SCREAMING_SNAKE_CASE      							=       file.replace("""model_00"""							,	f"""model_0{i}"""      )
            __SCREAMING_SNAKE_CASE      							=       torch.load(os.path.join(__UpperCAmelCase							,	__UpperCAmelCase      )							,	map_location="""cpu"""      )

            # Rename keys in the transformers names
            __SCREAMING_SNAKE_CASE      							=       list(temp.keys()      )
            for key in keys:
               __SCREAMING_SNAKE_CASE      							=       temp.pop(__UpperCAmelCase      )

            if tensors is None:
               __SCREAMING_SNAKE_CASE      							=       temp
            else:
               for key in tensors.keys():
                  # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                  if any(key.endswith(__UpperCAmelCase      ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH      ):
                     tensors[key] += temp[key]
                  else:
                     # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                     __SCREAMING_SNAKE_CASE      							=       1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN      ) else 0
                     # We concatenate these weights accross TP ranks
                     __SCREAMING_SNAKE_CASE      							=       torch.cat([tensors[key], temp[key]]							,	dim=__UpperCAmelCase      )

            # Divide by the number of TP the weights we want to average
         for key in tensors.keys():
            if any(key.endswith(__UpperCAmelCase      ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH      ):
               __SCREAMING_SNAKE_CASE      							=       tensors[key] / pretraining_tp

         __SCREAMING_SNAKE_CASE      							=       model.load_state_dict(__UpperCAmelCase							,	strict=__UpperCAmelCase      )
         assert not other_keys.unexpected_keys, f"""The keys {other_keys.unexpected_keys} are unexpected"""
         if missing_keys is None:
            __SCREAMING_SNAKE_CASE      							=       set(other_keys.missing_keys      )
         else:
            __SCREAMING_SNAKE_CASE      							=       missing_keys.intersection(set(other_keys.missing_keys      )      )

      assert not missing_keys, f"""The keys {missing_keys} are missing"""

      # Save pytorch-model
      os.makedirs(__UpperCAmelCase							,	exist_ok=__UpperCAmelCase      )
      __SCREAMING_SNAKE_CASE      							=       pytorch_dump_folder_path + """/""" + WEIGHTS_NAME
      __SCREAMING_SNAKE_CASE      							=       pytorch_dump_folder_path + """/""" + CONFIG_NAME
      print(f"""Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}"""      )
      if config.torch_dtype is not None:
         __SCREAMING_SNAKE_CASE      							=       model.to(config.torch_dtype      )
      torch.save(model.state_dict()							,	__UpperCAmelCase      )
      print(f"""Save configuration file to {pytorch_config_dump_path}"""      )
      with open(__UpperCAmelCase							,	"""w"""							,	encoding="""utf-8"""      ) as f:
         f.write(config.to_json_string()      )


if __name__ == "__main__":
    a        =						argparse.ArgumentParser()
    # Required parameters
    parser.add_argument(
        "--bloom_checkpoint_path",
        default=None,
        type=str,
        required=True,
        help="Path to the Megatron-LM checkpoint path.",
    )
    parser.add_argument(
        "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
    )
    parser.add_argument(
        "--bloom_config_file",
        default="",
        type=str,
        help=(
            "An optional config json file corresponding to the pre-trained model. \n"
            "This specifies the model architecture."
        ),
    )
    parser.add_argument(
        "--shard_model",
        action="store_true",
        help="An optional setting to shard the output model \nThis enables sharding the converted checkpoint",
    )
    parser.add_argument(
        "--pretraining_tp",
        default=4,
        type=int,
        help="Pretraining TP rank that has been used when training the model in Megatron-LM \n",
    )
    a        =						parser.parse_args()
    convert_bloom_checkpoint_to_pytorch(
        args.bloom_checkpoint_path,
        args.bloom_config_file,
        args.pytorch_dump_folder_path,
        args.shard_model,
        args.pretraining_tp,
    )












'''simple docstring'''

import os
import string
import sys


a        =						1 << 8

a        =						{
    "tab": ord("\t"),
    "newline": ord("\r"),
    "esc": 27,
    "up": 65 + ARROW_KEY_FLAG,
    "down": 66 + ARROW_KEY_FLAG,
    "right": 67 + ARROW_KEY_FLAG,
    "left": 68 + ARROW_KEY_FLAG,
    "mod_int": 91,
    "undefined": sys.maxsize,
    "interrupt": 3,
    "insert": 50,
    "delete": 51,
    "pg_up": 53,
    "pg_down": 54,
}

a        =						KEYMAP["up"]
a        =						KEYMAP["left"]

if sys.platform == "win32":
    a        =						[]
    a        =						{
        b"\xe0H": KEYMAP["up"] - ARROW_KEY_FLAG,
        b"\x00H": KEYMAP["up"] - ARROW_KEY_FLAG,
        b"\xe0P": KEYMAP["down"] - ARROW_KEY_FLAG,
        b"\x00P": KEYMAP["down"] - ARROW_KEY_FLAG,
        b"\xe0M": KEYMAP["right"] - ARROW_KEY_FLAG,
        b"\x00M": KEYMAP["right"] - ARROW_KEY_FLAG,
        b"\xe0K": KEYMAP["left"] - ARROW_KEY_FLAG,
        b"\x00K": KEYMAP["left"] - ARROW_KEY_FLAG,
    }

for i in range(10):
    a        =						ord(str(i))

def 						__magic_name__       (							)						->					Union[str, Any]:

   '''simple docstring'''







   if os.name == "nt":
      import msvcrt

      __SCREAMING_SNAKE_CASE      							=       """mbcs"""
      # Flush the keyboard buffer
      while msvcrt.kbhit():
         msvcrt.getch()
      if len(__UpperCAmelCase      ) == 0:
         # Read the keystroke
         __SCREAMING_SNAKE_CASE      							=       msvcrt.getch()

         # If it is a prefix char, get second part
         if ch in (b"\x00", b"\xe0"):
            __SCREAMING_SNAKE_CASE      							=       ch + msvcrt.getch()
            # Translate actual Win chars to bullet char types
            try:
               __SCREAMING_SNAKE_CASE      							=       chr(WIN_KEYMAP[cha]      )
               WIN_CH_BUFFER.append(chr(KEYMAP["""mod_int"""]      )      )
               WIN_CH_BUFFER.append(__UpperCAmelCase      )
               if ord(__UpperCAmelCase      ) in (
                   KEYMAP["insert"] - 1 << 9,
                   KEYMAP["delete"] - 1 << 9,
                   KEYMAP["pg_up"] - 1 << 9,
                   KEYMAP["pg_down"] - 1 << 9,
               ):
                  WIN_CH_BUFFER.append(chr(126      )      )
               __SCREAMING_SNAKE_CASE      							=       chr(KEYMAP["""esc"""]      )
            except KeyError:
               __SCREAMING_SNAKE_CASE      							=       cha[1]
         else:
            __SCREAMING_SNAKE_CASE      							=       ch.decode(__UpperCAmelCase      )
      else:
         __SCREAMING_SNAKE_CASE      							=       WIN_CH_BUFFER.pop(0      )
   elif os.name == "posix":
      import termios
      import tty

      __SCREAMING_SNAKE_CASE      							=       sys.stdin.fileno()
      __SCREAMING_SNAKE_CASE      							=       termios.tcgetattr(__UpperCAmelCase      )
      try:
         tty.setraw(__UpperCAmelCase      )
         __SCREAMING_SNAKE_CASE      							=       sys.stdin.read(1      )
      finally:
         termios.tcsetattr(__UpperCAmelCase							,	termios.TCSADRAIN							,	__UpperCAmelCase      )
   return ch







def 						__magic_name__       (							)						->					List[str]:

   '''simple docstring'''







   __SCREAMING_SNAKE_CASE      							=       get_raw_chars()
   if ord(__UpperCAmelCase      ) in [KEYMAP["interrupt"], KEYMAP["newline"]]:
      return char

   elif ord(__UpperCAmelCase      ) == KEYMAP["esc"]:
      __SCREAMING_SNAKE_CASE      							=       get_raw_chars()
      if ord(__UpperCAmelCase      ) == KEYMAP["mod_int"]:
         __SCREAMING_SNAKE_CASE      							=       get_raw_chars()
         if ord(__UpperCAmelCase      ) >= KEYMAP["arrow_begin"] - ARROW_KEY_FLAG and ord(__UpperCAmelCase      ) <= KEYMAP["arrow_end"] - ARROW_KEY_FLAG:
            return chr(ord(__UpperCAmelCase      ) + ARROW_KEY_FLAG      )
         else:
            return KEYMAP["undefined"]
      else:
         return get_raw_chars()

   else:
      if char in string.printable:
         return char
      else:
         return KEYMAP["undefined"]





