
'''simple docstring'''

import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

__snake_case:  int								=			[
    "word_embeddings_layernorm.weight",
    "word_embeddings_layernorm.bias",
    "input_layernorm.weight",
    "input_layernorm.bias",
    "post_attention_layernorm.weight",
    "post_attention_layernorm.bias",
    "self_attention.dense.bias",
    "mlp.dense_4h_to_h.bias",
    "ln_f.weight",
    "ln_f.bias",
]

__snake_case:  Dict								=			[
    "mlp.dense_4h_to_h.weight",
    "self_attention.dense.weight",
]






def 	_snake_case						(      A_       :    Optional[Any]     ,		A_       :    Tuple   ):
  """simple docstring"""







  a_      :	Tuple      =							{
      """word_embeddings.weight""": """word_embeddings.weight""",
      """word_embeddings.norm.weight""": """word_embeddings_layernorm.weight""",
      """word_embeddings.norm.bias""": """word_embeddings_layernorm.bias""",
      """weight""": """ln_f.weight""",
      """bias""": """ln_f.bias""",
  }

  if key in layer_rename_map:
    return layer_rename_map[key]

  # Handle transformer blocks
  a_      :	Dict      =							int(re.match(R""".*layer_(\d*).*"""     ,		A_   )[1]   )
  layer_number -= 3
  return f'''h.{layer_number}.''' + key






def 	_snake_case						(      A_       :    Tuple   ):
  """simple docstring"""







  if dtype == torch.bool:
    return 1 / 8
  a_      :	Union[str, Any]      =							re.search(R"""[^\d](\d+)$"""     ,		str(A_   )   )
  if bit_search is None:
    raise ValueError(f'''`dtype` is not a valid dtype: {dtype}.'''   )
  a_      :	Optional[int]      =							int(bit_search.groups()[0]   )
  return bit_size // 8






def 	_snake_case						(      A_       :    Optional[int]     ,		A_       :    Optional[Any]     ,		A_       :    Dict     ,		A_       :    Dict     ,		A_       :    List[str]   ):
  """simple docstring"""







  if bloom_config_file == "":
    a_      :	Tuple      =							BloomConfig()
  else:
    a_      :	List[Any]      =							BloomConfig.from_json_file(A_   )

  if shard_model:
    a_      :	List[Any]      =							os.listdir(A_   )
    a_      :	Any      =							sorted(filter(lambda A_   : s.startswith("""layer"""   ) and "model_00" in s     ,		A_   )   )

    a_      :	Any      =							{"""weight_map""": {}, """metadata""": {}}
    a_      :	Union[str, Any]      =							0

    a_      :	int      =							None

    a_      :	List[Any]      =							BloomConfig()

    for j, file in enumerate(A_   ):
      print("""Processing file: {}""".format(A_   )   )
      a_      :	Optional[Any]      =							None

      for i in range(A_   ):
        # load all TP files
        a_      :	int      =							file.replace("""model_00"""     ,		f'''model_0{i}'''   )
        a_      :	Dict      =							torch.load(os.path.join(A_     ,		A_   )     ,		map_location="""cpu"""   )

        # Rename keys in the transformers names
        a_      :	Union[str, Any]      =							list(temp.keys()   )
        for key in keys:
          a_      :	Dict      =							temp.pop(A_   )

        if tensors is None:
          a_      :	List[str]      =							temp
        else:
          for key in tensors.keys():
            if any(key.endswith(A_   ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH   ):
              # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
              tensors[key] += temp[key]
            else:
              # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
              a_      :	int      =							1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN   ) else 0
              # We concatenate these weights accross TP ranks
              a_      :	List[str]      =							torch.cat([tensors[key], temp[key]]     ,		dim=A_   )

            # Divide by the number of TP the weights we want to average
      for key in tensors.keys():
        if any(key.endswith(A_   ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH   ):
          a_      :	List[Any]      =							tensors[key] / pretraining_tp
      torch.save(
          A_     ,		os.path.join(
              A_     ,		"""pytorch_model_{}-of-{}.bin""".format(str(j + 1   ).zfill(5   )     ,		str(len(A_   )   ).zfill(5   )   )     ,		)     ,		)

      for key in tensors.keys():
        a_      :	str      =							tensors[key]
        total_size += value.numel() * get_dtype_size(value.dtype   )
        if key not in index_dict["weight_map"]:
          a_      :	Optional[int]      =							"""pytorch_model_{}-of-{}.bin""".format(
              str(j + 1   ).zfill(5   )     ,		str(len(A_   )   ).zfill(5   )   )

    a_      :	int      =							BloomConfig()
    a_      :	Optional[int]      =							pytorch_dump_folder_path + """/""" + CONFIG_NAME
    a_      :	Union[str, Any]      =							total_size
    with open(A_     ,		"""w"""     ,		encoding="""utf-8"""   ) as f:
      f.write(config.to_json_string()   )
    with open(os.path.join(A_     ,		WEIGHTS_NAME + """.index.json"""   )     ,		"""w"""     ,		encoding="""utf-8"""   ) as f:
      a_      :	Union[str, Any]      =							json.dumps(A_     ,		indent=2     ,		sort_keys=A_   ) + """\n"""
      f.write(A_   )
  else:
    a_      :	Dict      =							BloomModel(A_   )

    a_      :	int      =							os.listdir(A_   )
    a_      :	Union[str, Any]      =							sorted(filter(lambda A_   : s.startswith("""layer"""   ) and "model_00" in s     ,		A_   )   )

    a_      :	List[str]      =							None
    for i, file in enumerate(A_   ):
      a_      :	List[str]      =							None
      for i in range(A_   ):
        # load all TP files
        a_      :	int      =							file.replace("""model_00"""     ,		f'''model_0{i}'''   )
        a_      :	str      =							torch.load(os.path.join(A_     ,		A_   )     ,		map_location="""cpu"""   )

        # Rename keys in the transformers names
        a_      :	Optional[int]      =							list(temp.keys()   )
        for key in keys:
          a_      :	Optional[int]      =							temp.pop(A_   )

        if tensors is None:
          a_      :	List[Any]      =							temp
        else:
          for key in tensors.keys():
            # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
            if any(key.endswith(A_   ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH   ):
              tensors[key] += temp[key]
            else:
              # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
              a_      :	Any      =							1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN   ) else 0
              # We concatenate these weights accross TP ranks
              a_      :	int      =							torch.cat([tensors[key], temp[key]]     ,		dim=A_   )

            # Divide by the number of TP the weights we want to average
      for key in tensors.keys():
        if any(key.endswith(A_   ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH   ):
          a_      :	Optional[int]      =							tensors[key] / pretraining_tp

      a_      :	Union[str, Any]      =							model.load_state_dict(A_     ,		strict=A_   )
      assert not other_keys.unexpected_keys, f'''The keys {other_keys.unexpected_keys} are unexpected'''
      if missing_keys is None:
        a_      :	str      =							set(other_keys.missing_keys   )
      else:
        a_      :	Dict      =							missing_keys.intersection(set(other_keys.missing_keys   )   )

    assert not missing_keys, f'''The keys {missing_keys} are missing'''

    # Save pytorch-model
    os.makedirs(A_     ,		exist_ok=A_   )
    a_      :	Optional[Any]      =							pytorch_dump_folder_path + """/""" + WEIGHTS_NAME
    a_      :	Optional[Any]      =							pytorch_dump_folder_path + """/""" + CONFIG_NAME
    print(f'''Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'''   )
    if config.torch_dtype is not None:
      a_      :	Optional[int]      =							model.to(config.torch_dtype   )
    torch.save(model.state_dict()     ,		A_   )
    print(f'''Save configuration file to {pytorch_config_dump_path}'''   )
    with open(A_     ,		"""w"""     ,		encoding="""utf-8"""   ) as f:
      f.write(config.to_json_string()   )


if __name__ == "__main__":
    __snake_case:  List[str]								=			argparse.ArgumentParser()
    # Required parameters
    parser.add_argument(
        "--bloom_checkpoint_path",
        default=None,
        type=str,
        required=True,
        help="Path to the Megatron-LM checkpoint path.",
    )
    parser.add_argument(
        "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
    )
    parser.add_argument(
        "--bloom_config_file",
        default="",
        type=str,
        help=(
            "An optional config json file corresponding to the pre-trained model. \n"
            "This specifies the model architecture."
        ),
    )
    parser.add_argument(
        "--shard_model",
        action="store_true",
        help="An optional setting to shard the output model \nThis enables sharding the converted checkpoint",
    )
    parser.add_argument(
        "--pretraining_tp",
        default=4,
        type=int,
        help="Pretraining TP rank that has been used when training the model in Megatron-LM \n",
    )
    __snake_case:  Union[str, Any]								=			parser.parse_args()
    convert_bloom_checkpoint_to_pytorch(
        args.bloom_checkpoint_path,
        args.bloom_config_file,
        args.pytorch_dump_folder_path,
        args.shard_model,
        args.pretraining_tp,
    )




'''simple docstring'''

def 	_snake_case						(      A_       :    Optional[int]   ):
  """simple docstring"""







  if not head:
    return True
  # split the list to two parts
  a_      , a_      :	Union[str, Any]      =							head.next, head
  while fast and fast.next:
    a_      :	Dict      =							fast.next.next
    a_      :	List[str]      =							slow.next
  a_      :	str      =							slow.next
  a_      :	Optional[int]      =							None  # Don't forget here! But forget still works!
  # reverse the second part
  a_      :	Dict      =							None
  while second:
    a_      :	Any      =							second.next
    a_      :	Any      =							node
    a_      :	Optional[Any]      =							second
    a_      :	Optional[Any]      =							nxt
  # compare two parts
  # second part has the same or one less node
  while node:
    if node.val != head.val:
      return False
    a_      :	List[str]      =							node.next
    a_      :	Dict      =							head.next
  return True






def 	_snake_case						(      A_       :    List[str]   ):
  """simple docstring"""







  if not head or not head.next:
    return True

  # 1. Get the midpoint (slow)
  a_      :	Optional[Any]      =							head
  while fast and fast.next:
    a_      , a_      :	Optional[int]      =							fast.next.next, slow.next

  # 2. Push the second half into the stack
  a_      :	List[str]      =							[slow.val]
  while slow.next:
    a_      :	List[str]      =							slow.next
    stack.append(slow.val   )

  # 3. Comparison
  while stack:
    if stack.pop() != cur.val:
      return False
    a_      :	Union[str, Any]      =							cur.next

  return True






def 	_snake_case						(      A_       :    Optional[int]   ):
  """simple docstring"""







  if not head or not head.next:
    return True
  a_      :	List[Any]      =							{}
  a_      :	int      =							0
  while head:
    if head.val in d:
      d[head.val].append(A_   )
    else:
      a_      :	int      =							[pos]
    a_      :	List[str]      =							head.next
    pos += 1
  a_      :	str      =							pos - 1
  a_      :	List[str]      =							0
  for v in d.values():
    if len(A_   ) % 2 != 0:
      middle += 1
    else:
      a_      :	Optional[int]      =							0
      for i in range(0     ,		len(A_   )   ):
        if v[i] + v[len(A_   ) - 1 - step] != checksum:
          return False
        step += 1
    if middle > 1:
      return False
  return True


