

"""simple docstring"""
# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig

def        snake_case   (     A__			,A__			,A__=0		):




			if name is None:
						UpperCAmelCase_  :				List[Any]      =      None
			else:
						UpperCAmelCase_  :				Tuple      =      "." * max(0			,spaces - 2		) + "# {:" + str(50 - spaces		) + "s}"
						UpperCAmelCase_  :				Dict      =      fmt.format(__a		)

			# Print and recurse (if needed).
			if isinstance(__a			,__a		):
						if msg is not None:
									print(__a		)
						for k in val.keys():
									recursive_print(__a			,val[k]			,spaces + 2		)
			elif isinstance(__a			,torch.Tensor		):
						print(__a			,":"			,val.size()		)
			else:
						print(__a			,":"			,__a		)

def        snake_case   (     A__			,A__			,A__			,A__			,A__		):
			UpperCAmelCase_  :				Tuple      =      param.size()
			if checkpoint_version == 1.0:
						# version 1.0 stores [num_heads * hidden_size * num_splits, :]
						UpperCAmelCase_  :				Optional[Any]      =      (num_heads, hidden_size, num_splits) + input_shape[1:]
						UpperCAmelCase_  :				List[Any]      =      param.view(*__a		)
						UpperCAmelCase_  :				Dict      =      param.transpose(0			,2		)
						UpperCAmelCase_  :				List[str]      =      param.transpose(1			,2		).contiguous()
			elif checkpoint_version >= 2.0:
						# other versions store [num_heads * num_splits * hidden_size, :]
						UpperCAmelCase_  :				Optional[Any]      =      (num_heads, num_splits, hidden_size) + input_shape[1:]
						UpperCAmelCase_  :				List[Any]      =      param.view(*__a		)
						UpperCAmelCase_  :				Optional[int]      =      param.transpose(0			,1		).contiguous()
			UpperCAmelCase_  :				Optional[int]      =      param.view(*__a		)
			return param

def        snake_case   (     A__			,A__			,A__		):
			UpperCAmelCase_  :				Union[str, Any]      =      {}

			# old versions did not store training args
			UpperCAmelCase_  :				Tuple      =      input_state_dict.get("args"			,__a		)
			if ds_args is not None:
						# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
						# from pprint import pprint
						# pprint(vars(ds_args))

						UpperCAmelCase_  :				Tuple      =      ds_args.padded_vocab_size
						UpperCAmelCase_  :				Tuple      =      ds_args.max_position_embeddings
						UpperCAmelCase_  :				str      =      ds_args.hidden_size
						UpperCAmelCase_  :				Optional[int]      =      ds_args.num_layers
						UpperCAmelCase_  :				List[str]      =      ds_args.num_attention_heads
						UpperCAmelCase_  :				Tuple      =      ds_args.ffn_hidden_size
						# pprint(config)

			# The number of heads.
			UpperCAmelCase_  :				Tuple      =      config.n_head
			# The hidden_size per head.
			UpperCAmelCase_  :				Union[str, Any]      =      config.n_embd // config.n_head
			# Megatron-LM checkpoint version
			if "checkpoint_version" in input_state_dict.keys():
						UpperCAmelCase_  :				Dict      =      input_state_dict["checkpoint_version"]
			else:
						UpperCAmelCase_  :				Optional[Any]      =      0.0

			# The model.
			UpperCAmelCase_  :				Union[str, Any]      =      input_state_dict["model"]
			# The language model.
			UpperCAmelCase_  :				Optional[Any]      =      model["language_model"]
			# The embeddings.
			UpperCAmelCase_  :				int      =      lm["embedding"]

			# The word embeddings.
			UpperCAmelCase_  :				List[str]      =      embeddings["word_embeddings"]["weight"]
			# Truncate the embedding table to vocab_size rows.
			UpperCAmelCase_  :				Optional[int]      =      word_embeddings[: config.vocab_size, :]
			UpperCAmelCase_  :				Union[str, Any]      =      word_embeddings

			# The position embeddings.
			UpperCAmelCase_  :				List[Any]      =      embeddings["position_embeddings"]["weight"]
			# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
			UpperCAmelCase_  :				List[Any]      =      pos_embeddings.size(0		)
			if n_positions != config.n_positions:
						raise ValueError(
						    F"""pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don't match"""		)
			# Store the position embeddings.
			UpperCAmelCase_  :				str      =      pos_embeddings

			# The transformer.
			UpperCAmelCase_  :				Union[str, Any]      =      lm["transformer"] if "transformer" in lm.keys() else lm["encoder"]

			# The regex to extract layer names.
			UpperCAmelCase_  :				Tuple      =      re.compile(r"layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"		)

			# The simple map of names for "automated" rules.
			UpperCAmelCase_  :				Union[str, Any]      =      {
			    "attention.dense": ".attn.c_proj.",
			    "self_attention.dense": ".attn.c_proj.",
			    "mlp.dense_h_to_4h": ".mlp.c_fc.",
			    "mlp.dense_4h_to_h": ".mlp.c_proj.",
			}

			# Extract the layers.
			for key, val in transformer.items():
						# Match the name.
						UpperCAmelCase_  :				Any      =      layer_re.match(__a		)

						# Stop if that's not a layer
						if m is None:
									break

						# The index of the layer.
						UpperCAmelCase_  :				Any      =      int(m.group(1		)		)
						# The name of the operation.
						UpperCAmelCase_  :				List[Any]      =      m.group(2		)
						# Is it a weight or a bias?
						UpperCAmelCase_  :				int      =      m.group(3		)

						# The name of the layer.
						UpperCAmelCase_  :				int      =      F"""transformer.h.{layer_idx}"""

						# For layernorm(s), simply store the layer norm.
						if op_name.endswith("layernorm"		):
									UpperCAmelCase_  :				str      =      "ln_1" if op_name.startswith("input"		) else "ln_2"
									UpperCAmelCase_  :				List[str]      =      val

						# Transpose the QKV matrix.
						elif (
						    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
						) and weight_or_bias == "weight":
									# Insert a tensor of 1x1xDxD bias.
									UpperCAmelCase_  :				Optional[Any]      =      torch.tril(torch.ones((n_positions, n_positions)			,dtype=torch.floataa		)		).view(
									    1			,1			,__a			,__a		)
									UpperCAmelCase_  :				Union[str, Any]      =      causal_mask

									# Insert a "dummy" tensor for masked_bias.
									UpperCAmelCase_  :				Dict      =      torch.tensor(-1e4			,dtype=torch.floataa		)
									UpperCAmelCase_  :				List[Any]      =      masked_bias

									UpperCAmelCase_  :				Optional[int]      =      fix_query_key_value_ordering(__a			,__a			,3			,__a			,__a		)
									# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
									UpperCAmelCase_  :				Optional[Any]      =      out_val.transpose(0			,1		).contiguous()
									# Store.
									UpperCAmelCase_  :				List[Any]      =      out_val

						# Transpose the bias.
						elif (
						    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
						) and weight_or_bias == "bias":
									UpperCAmelCase_  :				Optional[Any]      =      fix_query_key_value_ordering(__a			,__a			,3			,__a			,__a		)
									# Store. No change of shape.
									UpperCAmelCase_  :				Optional[Any]      =      out_val

						# Transpose the weights.
						elif weight_or_bias == "weight":
									UpperCAmelCase_  :				Tuple      =      megatron_to_transformers[op_name]
									UpperCAmelCase_  :				Tuple      =      val.transpose(0			,1		)

						# Copy the bias.
						elif weight_or_bias == "bias":
									UpperCAmelCase_  :				List[str]      =      megatron_to_transformers[op_name]
									UpperCAmelCase_  :				Optional[int]      =      val

    # DEBUG.
			assert config.n_layer == layer_idx + 1

			# The final layernorm.
			UpperCAmelCase_  :				Optional[int]      =      transformer["final_layernorm.weight"]
			UpperCAmelCase_  :				List[str]      =      transformer["final_layernorm.bias"]

			# For LM head, transformers' wants the matrix to weight embeddings.
			UpperCAmelCase_  :				Union[str, Any]      =      word_embeddings

			# It should be done!
			return output_state_dict

def        snake_case   (     ):
			UpperCAmelCase_  :				Tuple      =      argparse.ArgumentParser()
			parser.add_argument("--print-checkpoint-structure"			,action="store_true"		)
			parser.add_argument(
			    "path_to_checkpoint"			,type=__a			,help="Path to the checkpoint file (.zip archive or direct .pt file)"			,)
			parser.add_argument(
			    "--config_file"			,default=""			,type=__a			,help="An optional config json file describing the pre-trained model."			,)
			UpperCAmelCase_  :				List[str]      =      parser.parse_args()

			# Extract the basename.
			UpperCAmelCase_  :				Optional[Any]      =      os.path.dirname(args.path_to_checkpoint		)

			# Load the model.
			# the .zip is very optional, let's keep it for backward compatibility
			print(F"""Extracting PyTorch state dictionary from {args.path_to_checkpoint}"""		)
			if args.path_to_checkpoint.endswith(".zip"		):
						with zipfile.ZipFile(args.path_to_checkpoint			,"r"		) as checkpoint:
									with checkpoint.open("release/mp_rank_00/model_optim_rng.pt"		) as pytorch_dict:
												UpperCAmelCase_  :				List[Any]      =      torch.load(__a			,map_location="cpu"		)
			else:
						UpperCAmelCase_  :				List[Any]      =      torch.load(args.path_to_checkpoint			,map_location="cpu"		)

			UpperCAmelCase_  :				List[Any]      =      input_state_dict.get("args"			,__a		)

			# Read the config, or default to the model released by NVIDIA.
			if args.config_file == "":
						if ds_args is not None:
									if ds_args.bias_gelu_fusion:
												UpperCAmelCase_  :				Optional[Any]      =      "gelu_fast"
									elif ds_args.openai_gelu:
												UpperCAmelCase_  :				int      =      "gelu_new"
									else:
												UpperCAmelCase_  :				List[str]      =      "gelu"
						else:
									# in the very early days this used to be "gelu_new"
									UpperCAmelCase_  :				List[Any]      =      "gelu_new"

						# Spell out all parameters in case the defaults change.
						UpperCAmelCase_  :				Optional[int]      =      GPTaConfig(
						    vocab_size=5_02_57			,n_positions=10_24			,n_embd=10_24			,n_layer=24			,n_head=16			,n_inner=40_96			,activation_function=__a			,resid_pdrop=0.1			,embd_pdrop=0.1			,attn_pdrop=0.1			,layer_norm_epsilon=1e-5			,initializer_range=0.02			,summary_type="cls_index"			,summary_use_proj=__a			,summary_activation=__a			,summary_proj_to_labels=__a			,summary_first_dropout=0.1			,scale_attn_weights=__a			,use_cache=__a			,bos_token_id=5_02_56			,eos_token_id=5_02_56			,)
			else:
						UpperCAmelCase_  :				List[str]      =      GPTaConfig.from_json_file(args.config_file		)

			UpperCAmelCase_  :				Tuple      =      ["GPT2LMHeadModel"]

			# Convert.
			print("Converting"		)
			UpperCAmelCase_  :				List[str]      =      convert_megatron_checkpoint(__a			,__a			,__a		)

			# Print the structure of converted state dict.
			if args.print_checkpoint_structure:
						recursive_print(__a			,__a		)

			# Add tokenizer class info to config
			# see https://github.com/huggingface/transformers/issues/13906)
			if ds_args is not None:
						UpperCAmelCase_  :				Dict      =      ds_args.tokenizer_type
						if tokenizer_type == "GPT2BPETokenizer":
									UpperCAmelCase_  :				Dict      =      "gpt2"
						elif tokenizer_type == "PretrainedFromHF":
									UpperCAmelCase_  :				Optional[int]      =      ds_args.tokenizer_name_or_path
						else:
									raise ValueError(F"""Unrecognized tokenizer_type {tokenizer_type}"""		)
			else:
						UpperCAmelCase_  :				Any      =      "gpt2"

			UpperCAmelCase_  :				List[Any]      =      AutoTokenizer.from_pretrained(__a		)
			UpperCAmelCase_  :				Any      =      type(__a		).__name__
			UpperCAmelCase_  :				Union[str, Any]      =      tokenizer_class

			# Store the config to file.
			print("Saving config"		)
			config.save_pretrained(__a		)

			# Save tokenizer based on args
			print(F"""Adding {tokenizer_class} tokenizer files"""		)
			tokenizer.save_pretrained(__a		)

			# Store the state_dict to file.
			UpperCAmelCase_  :				List[str]      =      os.path.join(__a			,"pytorch_model.bin"		)
			print(F"""Saving checkpoint to \"{output_checkpoint_file}\""""		)
			torch.save(__a			,__a		)


####################################################################################################

if __name__ == "__main__":
			main()

####################################################################################################











"""simple docstring"""
from __future__ import annotations

import time

lowerCamelCase_       			=   list[tuple[int, int]]

lowerCamelCase_       			=   [
    [0, 0, 0, 0, 0, 0, 0],
    [0, 1, 0, 0, 0, 0, 0],  # 0 are free path whereas 1's are obstacles
    [0, 0, 0, 0, 0, 0, 0],
    [0, 0, 1, 0, 0, 0, 0],
    [1, 0, 1, 0, 0, 0, 0],
    [0, 0, 0, 0, 0, 0, 0],
    [0, 0, 0, 0, 1, 0, 0],
]

lowerCamelCase_       			=   [[-1, 0], [0, -1], [1, 0], [0, 1]]  # up, left, down, right







class     UpperCamelCase_ :





	def __init__(   self      :				int			,			lowerCAmelCase_      :				int			,			lowerCAmelCase_      :				int			,			lowerCAmelCase_      :				int			,			lowerCAmelCase_      :				int			,			lowerCAmelCase_      :				Node | None		)		->    List[str]:
				UpperCAmelCase_  :				List[Any]      =      pos_x
				UpperCAmelCase_  :				Dict      =      pos_y
				UpperCAmelCase_  :				List[Any]      =      (pos_y, pos_x)
				UpperCAmelCase_  :				int      =      goal_x
				UpperCAmelCase_  :				int      =      goal_y
				UpperCAmelCase_  :				Union[str, Any]      =      parent







class     UpperCamelCase_ :





	def __init__(   self      :				Dict			,			lowerCAmelCase_      :				tuple[int, int]			,			lowerCAmelCase_      :				tuple[int, int]		)		->    int:
				UpperCAmelCase_  :				Dict      =      Node(start[1]			,			start[0]			,			goal[1]			,			goal[0]			,			lowerCAmelCase_		)
				UpperCAmelCase_  :				List[str]      =      Node(goal[1]			,			goal[0]			,			goal[1]			,			goal[0]			,			lowerCAmelCase_		)

				UpperCAmelCase_  :				List[str]      =      [self.start]
				UpperCAmelCase_  :				List[Any]      =      False





	def   _SCREAMING_SNAKE_CASE   (   self      :				int		)		->    Path | None:
				while self.node_queue:
							UpperCAmelCase_  :				Optional[int]      =      self.node_queue.pop(0		)

							if current_node.pos == self.target.pos:
										UpperCAmelCase_  :				int      =      True
										return self.retrace_path(lowerCAmelCase_		)

							UpperCAmelCase_  :				Union[str, Any]      =      self.get_successors(lowerCAmelCase_		)

							for node in successors:
										self.node_queue.append(lowerCAmelCase_		)

				if not self.reached:
							return [self.start.pos]
				return None





	def   _SCREAMING_SNAKE_CASE   (   self      :				Tuple			,			lowerCAmelCase_      :				Node		)		->    list[Node]:
				UpperCAmelCase_  :				Union[str, Any]      =      []
				for action in delta:
							UpperCAmelCase_  :				Optional[Any]      =      parent.pos_x + action[1]
							UpperCAmelCase_  :				Tuple      =      parent.pos_y + action[0]
							if not (0 <= pos_x <= len(grid[0]		) - 1 and 0 <= pos_y <= len(lowerCAmelCase_		) - 1):
										continue

							if grid[pos_y][pos_x] != 0:
										continue

							successors.append(
							    Node(lowerCAmelCase_			,			lowerCAmelCase_			,			self.target.pos_y			,			self.target.pos_x			,			lowerCAmelCase_		)		)
				return successors





	def   _SCREAMING_SNAKE_CASE   (   self      :				Union[str, Any]			,			lowerCAmelCase_      :				Node | None		)		->    Path:
				UpperCAmelCase_  :				Any      =      node
				UpperCAmelCase_  :				Any      =      []
				while current_node is not None:
							path.append((current_node.pos_y, current_node.pos_x)		)
							UpperCAmelCase_  :				int      =      current_node.parent
				path.reverse()
				return path







class     UpperCamelCase_ :





	def __init__(   self      :				str			,			lowerCAmelCase_      :				int			,			lowerCAmelCase_      :				Tuple		)		->    Tuple:
				UpperCAmelCase_  :				Union[str, Any]      =      BreadthFirstSearch(lowerCAmelCase_			,			lowerCAmelCase_		)
				UpperCAmelCase_  :				Any      =      BreadthFirstSearch(lowerCAmelCase_			,			lowerCAmelCase_		)
				UpperCAmelCase_  :				Any      =      False





	def   _SCREAMING_SNAKE_CASE   (   self      :				List[Any]		)		->    Path | None:
				while self.fwd_bfs.node_queue or self.bwd_bfs.node_queue:
							UpperCAmelCase_  :				List[Any]      =      self.fwd_bfs.node_queue.pop(0		)
							UpperCAmelCase_  :				Tuple      =      self.bwd_bfs.node_queue.pop(0		)

							if current_bwd_node.pos == current_fwd_node.pos:
										UpperCAmelCase_  :				Optional[int]      =      True
										return self.retrace_bidirectional_path(
										    lowerCAmelCase_			,			lowerCAmelCase_		)

							UpperCAmelCase_  :				Any      =      current_bwd_node
							UpperCAmelCase_  :				Optional[int]      =      current_fwd_node

							UpperCAmelCase_  :				Any      =      {
							    self.fwd_bfs: self.fwd_bfs.get_successors(lowerCAmelCase_		),
							    self.bwd_bfs: self.bwd_bfs.get_successors(lowerCAmelCase_		),
							}

							for bfs in [self.fwd_bfs, self.bwd_bfs]:
										for node in successors[bfs]:
													bfs.node_queue.append(lowerCAmelCase_		)

				if not self.reached:
							return [self.fwd_bfs.start.pos]
				return None





	def   _SCREAMING_SNAKE_CASE   (   self      :				int			,			lowerCAmelCase_      :				Node			,			lowerCAmelCase_      :				Node		)		->    Path:
				UpperCAmelCase_  :				Tuple      =      self.fwd_bfs.retrace_path(lowerCAmelCase_		)
				UpperCAmelCase_  :				str      =      self.bwd_bfs.retrace_path(lowerCAmelCase_		)
				bwd_path.pop()
				bwd_path.reverse()
				UpperCAmelCase_  :				int      =      fwd_path + bwd_path
				return path


if __name__ == "__main__":
			# all coordinates are given in format [y,x]
			import doctest

			doctest.testmod()
			lowerCamelCase_       			=   (0, 0)
			lowerCamelCase_       			=   (len(grid) - 1, len(grid[0]) - 1)
			for elem in grid:
						print(elem)

			lowerCamelCase_       			=   time.time()
			lowerCamelCase_       			=   BreadthFirstSearch(init, goal)
			lowerCamelCase_       			=   bfs.search()
			lowerCamelCase_       			=   time.time() - start_bfs_time

			print('''Unidirectional BFS computation time : ''', bfs_time)

			lowerCamelCase_       			=   time.time()
			lowerCamelCase_       			=   BidirectionalBreadthFirstSearch(init, goal)
			lowerCamelCase_       			=   bd_bfs.search()
			lowerCamelCase_       			=   time.time() - start_bd_bfs_time

			print('''Bidirectional BFS computation time : ''', bd_bfs_time)



