

# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig






def 					__lowercase      (     _A						,    _A						,    _A=0			)     ->						Optional[Any]:
      # Format the message.
      if name is None:
            SCREAMING_SNAKE_CASE :      Optional[int]       =   None
      else:
            SCREAMING_SNAKE_CASE :      Optional[int]       =   """.""" * max(0						,    spaces - 2			) + """# {:""" + str(50 - spaces			) + """s}"""
            SCREAMING_SNAKE_CASE :      List[Any]       =   fmt.format(_A			)

      # Print and recurse (if needed).
      if isinstance(_A						,    _A			):
            if msg is not None:
                  print(_A			)
            for k in val.keys():
                  recursive_print(_A						,    val[k]						,    spaces + 2			)
      elif isinstance(_A						,    torch.Tensor			):
            print(_A						,    """:"""						,    val.size()			)
      else:
            print(_A						,    """:"""						,    _A			)






def 					__lowercase      (     _A						,    _A						,    _A						,    _A						,    _A			)     ->						Tuple:
      # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
      # for compatibility with later versions of NVIDIA Megatron-LM.
      # The inverse operation is performed inside Megatron-LM to read checkpoints:
      # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
      # If param is the weight tensor of the self-attention block, the returned tensor
      # will have to be transposed one more time to be read by HuggingFace GPT2.
      SCREAMING_SNAKE_CASE :      List[Any]       =   param.size()
      if checkpoint_version == 1.0:
            # version 1.0 stores [num_heads * hidden_size * num_splits, :]
            SCREAMING_SNAKE_CASE :      Any       =   (num_heads, hidden_size, num_splits) + input_shape[1:]
            SCREAMING_SNAKE_CASE :      Union[str, Any]       =   param.view(*_A			)
            SCREAMING_SNAKE_CASE :      List[str]       =   param.transpose(0						,    2			)
            SCREAMING_SNAKE_CASE :      List[str]       =   param.transpose(1						,    2			).contiguous()
      elif checkpoint_version >= 2.0:
            # other versions store [num_heads * num_splits * hidden_size, :]
            SCREAMING_SNAKE_CASE :      Optional[int]       =   (num_heads, num_splits, hidden_size) + input_shape[1:]
            SCREAMING_SNAKE_CASE :      Optional[Any]       =   param.view(*_A			)
            SCREAMING_SNAKE_CASE :      str       =   param.transpose(0						,    1			).contiguous()
      SCREAMING_SNAKE_CASE :      str       =   param.view(*_A			)
      return param






def 					__lowercase      (     _A						,    _A						,    _A			)     ->						Union[str, Any]:
      # The converted output model.
      SCREAMING_SNAKE_CASE :      Optional[int]       =   {}

      # old versions did not store training args
      SCREAMING_SNAKE_CASE :      str       =   input_state_dict.get("""args"""						,    _A			)
      if ds_args is not None:
            # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
            # from pprint import pprint
            # pprint(vars(ds_args))

            SCREAMING_SNAKE_CASE :      List[str]       =   ds_args.padded_vocab_size
            SCREAMING_SNAKE_CASE :      Optional[Any]       =   ds_args.max_position_embeddings
            SCREAMING_SNAKE_CASE :      Any       =   ds_args.hidden_size
            SCREAMING_SNAKE_CASE :      List[str]       =   ds_args.num_layers
            SCREAMING_SNAKE_CASE :      int       =   ds_args.num_attention_heads
            SCREAMING_SNAKE_CASE :      Dict       =   ds_args.ffn_hidden_size
            # pprint(config)

      # The number of heads.
      SCREAMING_SNAKE_CASE :      List[Any]       =   config.n_head
      # The hidden_size per head.
      SCREAMING_SNAKE_CASE :      Optional[int]       =   config.n_embd // config.n_head
      # Megatron-LM checkpoint version
      if "checkpoint_version" in input_state_dict.keys():
            SCREAMING_SNAKE_CASE :      Optional[Any]       =   input_state_dict["""checkpoint_version"""]
      else:
            SCREAMING_SNAKE_CASE :      List[Any]       =   0.0

      # The model.
      SCREAMING_SNAKE_CASE :      List[str]       =   input_state_dict["""model"""]
      # The language model.
      SCREAMING_SNAKE_CASE :      List[str]       =   model["""language_model"""]
      # The embeddings.
      SCREAMING_SNAKE_CASE :      Any       =   lm["""embedding"""]

      # The word embeddings.
      SCREAMING_SNAKE_CASE :      int       =   embeddings["""word_embeddings"""]["""weight"""]
      # Truncate the embedding table to vocab_size rows.
      SCREAMING_SNAKE_CASE :      int       =   word_embeddings[: config.vocab_size, :]
      SCREAMING_SNAKE_CASE :      Optional[int]       =   word_embeddings

      # The position embeddings.
      SCREAMING_SNAKE_CASE :      Dict       =   embeddings["""position_embeddings"""]["""weight"""]
      # Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
      SCREAMING_SNAKE_CASE :      Dict       =   pos_embeddings.size(0			)
      if n_positions != config.n_positions:
            raise ValueError(
                F"pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don't match"			)
      # Store the position embeddings.
      SCREAMING_SNAKE_CASE :      Union[str, Any]       =   pos_embeddings

      # The transformer.
      SCREAMING_SNAKE_CASE :      int       =   lm["""transformer"""] if """transformer""" in lm.keys() else lm["""encoder"""]

      # The regex to extract layer names.
      SCREAMING_SNAKE_CASE :      List[Any]       =   re.compile(R"""layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"""			)

      # The simple map of names for "automated" rules.
      SCREAMING_SNAKE_CASE :      Any       =   {
          """attention.dense""": """.attn.c_proj.""",
          """self_attention.dense""": """.attn.c_proj.""",
          """mlp.dense_h_to_4h""": """.mlp.c_fc.""",
          """mlp.dense_4h_to_h""": """.mlp.c_proj.""",
      }

      # Extract the layers.
      for key, val in transformer.items():
            # Match the name.
            SCREAMING_SNAKE_CASE :      Union[str, Any]       =   layer_re.match(_A			)

            # Stop if that's not a layer
            if m is None:
                  break

            # The index of the layer.
            SCREAMING_SNAKE_CASE :      Any       =   int(m.group(1			)			)
            # The name of the operation.
            SCREAMING_SNAKE_CASE :      List[Any]       =   m.group(2			)
            # Is it a weight or a bias?
            SCREAMING_SNAKE_CASE :      int       =   m.group(3			)

            # The name of the layer.
            SCREAMING_SNAKE_CASE :      List[Any]       =   F"transformer.h.{layer_idx}"

            # For layernorm(s), simply store the layer norm.
            if op_name.endswith("""layernorm"""			):
                  SCREAMING_SNAKE_CASE :      List[Any]       =   """ln_1""" if op_name.startswith("""input"""			) else """ln_2"""
                  SCREAMING_SNAKE_CASE :      List[str]       =   val

            # Transpose the QKV matrix.
            elif (
                op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
            ) and weight_or_bias == "weight":
                  # Insert a tensor of 1x1xDxD bias.
                  SCREAMING_SNAKE_CASE :      List[Any]       =   torch.tril(torch.ones((n_positions, n_positions)						,    dtype=torch.floataa			)			).view(
                      1						,    1						,    _A						,    _A			)
                  SCREAMING_SNAKE_CASE :      str       =   causal_mask

                  # Insert a "dummy" tensor for masked_bias.
                  SCREAMING_SNAKE_CASE :      Dict       =   torch.tensor(-1e4						,    dtype=torch.floataa			)
                  SCREAMING_SNAKE_CASE :      Optional[Any]       =   masked_bias

                  SCREAMING_SNAKE_CASE :      Any       =   fix_query_key_value_ordering(_A						,    _A						,    3						,    _A						,    _A			)
                  # Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
                  SCREAMING_SNAKE_CASE :      Optional[Any]       =   out_val.transpose(0						,    1			).contiguous()
                  # Store.
                  SCREAMING_SNAKE_CASE :      List[Any]       =   out_val

            # Transpose the bias.
            elif (
                op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
            ) and weight_or_bias == "bias":
                  SCREAMING_SNAKE_CASE :      Dict       =   fix_query_key_value_ordering(_A						,    _A						,    3						,    _A						,    _A			)
                  # Store. No change of shape.
                  SCREAMING_SNAKE_CASE :      Optional[int]       =   out_val

            # Transpose the weights.
            elif weight_or_bias == "weight":
                  SCREAMING_SNAKE_CASE :      int       =   megatron_to_transformers[op_name]
                  SCREAMING_SNAKE_CASE :      Union[str, Any]       =   val.transpose(0						,    1			)

            # Copy the bias.
            elif weight_or_bias == "bias":
                  SCREAMING_SNAKE_CASE :      Union[str, Any]       =   megatron_to_transformers[op_name]
                  SCREAMING_SNAKE_CASE :      List[str]       =   val

    # DEBUG.
      assert config.n_layer == layer_idx + 1

      # The final layernorm.
      SCREAMING_SNAKE_CASE :      Optional[int]       =   transformer["""final_layernorm.weight"""]
      SCREAMING_SNAKE_CASE :      Any       =   transformer["""final_layernorm.bias"""]

      # For LM head, transformers' wants the matrix to weight embeddings.
      SCREAMING_SNAKE_CASE :      int       =   word_embeddings

      # It should be done!
      return output_state_dict






def 					__lowercase      (     )     ->						List[Any]:
      # Create the argument parser.
      SCREAMING_SNAKE_CASE :      List[str]       =   argparse.ArgumentParser()
      parser.add_argument("""--print-checkpoint-structure"""						,    action="""store_true"""			)
      parser.add_argument(
          """path_to_checkpoint"""						,    type=_A						,    help="""Path to the checkpoint file (.zip archive or direct .pt file)"""						,    )
      parser.add_argument(
          """--config_file"""						,    default=""""""						,    type=_A						,    help="""An optional config json file describing the pre-trained model."""						,    )
      SCREAMING_SNAKE_CASE :      Tuple       =   parser.parse_args()

      # Extract the basename.
      SCREAMING_SNAKE_CASE :      Dict       =   os.path.dirname(args.path_to_checkpoint			)

      # Load the model.
      # the .zip is very optional, let's keep it for backward compatibility
      print(F"Extracting PyTorch state dictionary from {args.path_to_checkpoint}"			)
      if args.path_to_checkpoint.endswith(""".zip"""			):
            with zipfile.ZipFile(args.path_to_checkpoint						,    """r"""			) as checkpoint:
                  with checkpoint.open("""release/mp_rank_00/model_optim_rng.pt"""			) as pytorch_dict:
                        SCREAMING_SNAKE_CASE :      Tuple       =   torch.load(_A						,    map_location="""cpu"""			)
      else:
            SCREAMING_SNAKE_CASE :      List[Any]       =   torch.load(args.path_to_checkpoint						,    map_location="""cpu"""			)

      SCREAMING_SNAKE_CASE :      Optional[Any]       =   input_state_dict.get("""args"""						,    _A			)

      # Read the config, or default to the model released by NVIDIA.
      if args.config_file == "":
            if ds_args is not None:
                  if ds_args.bias_gelu_fusion:
                        SCREAMING_SNAKE_CASE :      Any       =   """gelu_fast"""
                  elif ds_args.openai_gelu:
                        SCREAMING_SNAKE_CASE :      Optional[Any]       =   """gelu_new"""
                  else:
                        SCREAMING_SNAKE_CASE :      Tuple       =   """gelu"""
            else:
                  # in the very early days this used to be "gelu_new"
                  SCREAMING_SNAKE_CASE :      int       =   """gelu_new"""

            # Spell out all parameters in case the defaults change.
            SCREAMING_SNAKE_CASE :      Dict       =   GPTaConfig(
                vocab_size=50257						,    n_positions=1024						,    n_embd=1024						,    n_layer=24						,    n_head=16						,    n_inner=4096						,    activation_function=_A						,    resid_pdrop=0.1						,    embd_pdrop=0.1						,    attn_pdrop=0.1						,    layer_norm_epsilon=1e-5						,    initializer_range=0.02						,    summary_type="""cls_index"""						,    summary_use_proj=_A						,    summary_activation=_A						,    summary_proj_to_labels=_A						,    summary_first_dropout=0.1						,    scale_attn_weights=_A						,    use_cache=_A						,    bos_token_id=50256						,    eos_token_id=50256						,    )
      else:
            SCREAMING_SNAKE_CASE :      Union[str, Any]       =   GPTaConfig.from_json_file(args.config_file			)

      SCREAMING_SNAKE_CASE :      Tuple       =   ["""GPT2LMHeadModel"""]

      # Convert.
      print("""Converting"""			)
      SCREAMING_SNAKE_CASE :      Tuple       =   convert_megatron_checkpoint(_A						,    _A						,    _A			)

      # Print the structure of converted state dict.
      if args.print_checkpoint_structure:
            recursive_print(_A						,    _A			)

      # Add tokenizer class info to config
      # see https://github.com/huggingface/transformers/issues/13906)
      if ds_args is not None:
            SCREAMING_SNAKE_CASE :      List[str]       =   ds_args.tokenizer_type
            if tokenizer_type == "GPT2BPETokenizer":
                  SCREAMING_SNAKE_CASE :      Optional[Any]       =   """gpt2"""
            elif tokenizer_type == "PretrainedFromHF":
                  SCREAMING_SNAKE_CASE :      Optional[Any]       =   ds_args.tokenizer_name_or_path
            else:
                  raise ValueError(F"Unrecognized tokenizer_type {tokenizer_type}"			)
      else:
            SCREAMING_SNAKE_CASE :      Optional[Any]       =   """gpt2"""

      SCREAMING_SNAKE_CASE :      Optional[int]       =   AutoTokenizer.from_pretrained(_A			)
      SCREAMING_SNAKE_CASE :      Union[str, Any]       =   type(_A			).__name__
      SCREAMING_SNAKE_CASE :      Optional[Any]       =   tokenizer_class

      # Store the config to file.
      print("""Saving config"""			)
      config.save_pretrained(_A			)

      # Save tokenizer based on args
      print(F"Adding {tokenizer_class} tokenizer files"			)
      tokenizer.save_pretrained(_A			)

      # Store the state_dict to file.
      SCREAMING_SNAKE_CASE :      str       =   os.path.join(_A						,    """pytorch_model.bin"""			)
      print(F"Saving checkpoint to \"{output_checkpoint_file}\""			)
      torch.save(_A						,    _A			)


####################################################################################################

if __name__ == "__main__":
  main()

####################################################################################################







import argparse
import json
import os

import torch
from transformers.file_utils import has_file

from diffusers import UNetaDConditionModel, UNetaDModel


UpperCAmelCase__ :     Optional[Any]				    =    False
UpperCAmelCase__ :     Union[str, Any]				    =    True
UpperCAmelCase__ :     int				    =    False


if __name__ == "__main__":
  UpperCAmelCase__ :     List[Any]				    =    argparse.ArgumentParser()

  parser.add_argument(
      """--repo_path""",
      default=None,
      type=str,
      required=True,
      help="""The config json file corresponding to the architecture.""",
  )

  parser.add_argument("""--dump_path""", default=None, type=str, required=True, help="""Path to the output model.""")

  UpperCAmelCase__ :     List[str]				    =    parser.parse_args()

  UpperCAmelCase__ :     Dict				    =    {
      """image_size""": """sample_size""",
      """num_res_blocks""": """layers_per_block""",
      """block_channels""": """block_out_channels""",
      """down_blocks""": """down_block_types""",
      """up_blocks""": """up_block_types""",
      """downscale_freq_shift""": """freq_shift""",
      """resnet_num_groups""": """norm_num_groups""",
      """resnet_act_fn""": """act_fn""",
      """resnet_eps""": """norm_eps""",
      """num_head_channels""": """attention_head_dim""",
  }

  UpperCAmelCase__ :     str				    =    {
      """time_steps""": """time_proj""",
      """mid""": """mid_block""",
      """downsample_blocks""": """down_blocks""",
      """upsample_blocks""": """up_blocks""",
  }

  UpperCAmelCase__ :     Union[str, Any]				    =    """""" if has_file(args.repo_path, """config.json""") else """unet"""

  with open(os.path.join(args.repo_path, subfolder, """config.json"""), """r""", encoding="""utf-8""") as reader:
    UpperCAmelCase__ :     Union[str, Any]				    =    reader.read()
    UpperCAmelCase__ :     Any				    =    json.loads(text)

  if do_only_config:
    for key in config_parameters_to_change.keys():
      config.pop(key, None)

  if has_file(args.repo_path, """config.json"""):
    UpperCAmelCase__ :     int				    =    UNetaDModel(**config)
  else:
    UpperCAmelCase__ :     Optional[Any]				    =    UNetaDConditionModel if """ldm-text2im-large-256""" in args.repo_path else UNetaDModel
    UpperCAmelCase__ :     int				    =    class_name(**config)

  if do_only_config:
    model.save_config(os.path.join(args.repo_path, subfolder))

  UpperCAmelCase__ :     Optional[Any]				    =    dict(model.config)

  if do_only_renaming:
    for key, value in config_parameters_to_change.items():
      if key in config:
        UpperCAmelCase__ :     str				    =    config[key]
        del config[key]

    UpperCAmelCase__ :     Optional[int]				    =    [k.replace("""UNetRes""", """""") for k in config["""down_block_types"""]]
    UpperCAmelCase__ :     Optional[int]				    =    [k.replace("""UNetRes""", """""") for k in config["""up_block_types"""]]

  if do_only_weights:
    UpperCAmelCase__ :     Dict				    =    torch.load(os.path.join(args.repo_path, subfolder, """diffusion_pytorch_model.bin"""))

    UpperCAmelCase__ :     Optional[Any]				    =    {}
    for param_key, param_value in state_dict.items():
      if param_key.endswith(""".op.bias""") or param_key.endswith(""".op.weight"""):
        continue
      UpperCAmelCase__ :     Union[str, Any]				    =    False
      for key, new_key in key_parameters_to_change.items():
        if not has_changed and param_key.split(""".""")[0] == key:
          UpperCAmelCase__ :     List[Any]				    =    param_value
          UpperCAmelCase__ :     List[str]				    =    True
      if not has_changed:
        UpperCAmelCase__ :     Any				    =    param_value

    model.load_state_dict(new_state_dict)
    model.save_pretrained(os.path.join(args.repo_path, subfolder))




