


'''simple docstring'''


import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig




def      lowerCamelCase					(      lowerCamelCase  :      Union[str, Any]							,						lowerCamelCase  :      Dict							,						lowerCamelCase  :      List[str]=0):




      if name is None:
            A_						:  Optional[Any]          =				None
      else:
            A_						:  int          =				'''.''' * max(0							,						spaces - 2) + '''# {:''' + str(50 - spaces) + '''s}'''
            A_						:  Optional[int]          =				fmt.format(__A)

      # Print and recurse (if needed).
      if isinstance(__A							,						__A):
            if msg is not None:
                  print(__A)
            for k in val.keys():
                  recursive_print(__A							,						val[k]							,						spaces + 2)
      elif isinstance(__A							,						torch.Tensor):
            print(__A							,						""":"""							,						val.size())
      else:
            print(__A							,						""":"""							,						__A)




def      lowerCamelCase					(      lowerCamelCase  :      Optional[Any]							,						lowerCamelCase  :      Optional[int]							,						lowerCamelCase  :      Optional[Any]							,						lowerCamelCase  :      Optional[int]							,						lowerCamelCase  :      Union[str, Any]):
      A_						:  Optional[int]          =				param.size()
      if checkpoint_version == 1.0:
            # version 1.0 stores [num_heads * hidden_size * num_splits, :]
            A_						:  int          =				(num_heads, hidden_size, num_splits) + input_shape[1:]
            A_						:  int          =				param.view(*__A)
            A_						:  Optional[Any]          =				param.transpose(0							,						2)
            A_						:  Union[str, Any]          =				param.transpose(1							,						2).contiguous()
      elif checkpoint_version >= 2.0:
            # other versions store [num_heads * num_splits * hidden_size, :]
            A_						:  List[Any]          =				(num_heads, num_splits, hidden_size) + input_shape[1:]
            A_						:  Tuple          =				param.view(*__A)
            A_						:  Any          =				param.transpose(0							,						1).contiguous()
      A_						:  str          =				param.view(*__A)
      return param




def      lowerCamelCase					(      lowerCamelCase  :      Dict							,						lowerCamelCase  :      Union[str, Any]							,						lowerCamelCase  :      List[Any]):
      A_						:  Union[str, Any]          =				{}

      # old versions did not store training args
      A_						:  int          =				input_state_dict.get("""args"""							,						__A)
      if ds_args is not None:
            # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
            # from pprint import pprint
            # pprint(vars(ds_args))

            A_						:  Tuple          =				ds_args.padded_vocab_size
            A_						:  Any          =				ds_args.max_position_embeddings
            A_						:  Dict          =				ds_args.hidden_size
            A_						:  Any          =				ds_args.num_layers
            A_						:  Dict          =				ds_args.num_attention_heads
            A_						:  int          =				ds_args.ffn_hidden_size
            # pprint(config)

      # The number of heads.
      A_						:  Any          =				config.n_head
      # The hidden_size per head.
      A_						:  Optional[int]          =				config.n_embd // config.n_head
      # Megatron-LM checkpoint version
      if "checkpoint_version" in input_state_dict.keys():
            A_						:  str          =				input_state_dict['''checkpoint_version''']
      else:
            A_						:  Any          =				0.0

      # The model.
      A_						:  Tuple          =				input_state_dict['''model''']
      # The language model.
      A_						:  Any          =				model['''language_model''']
      # The embeddings.
      A_						:  Union[str, Any]          =				lm['''embedding''']

      # The word embeddings.
      A_						:  Optional[Any]          =				embeddings['''word_embeddings''']['''weight''']
      # Truncate the embedding table to vocab_size rows.
      A_						:  Dict          =				word_embeddings[: config.vocab_size, :]
      A_						:  str          =				word_embeddings

      # The position embeddings.
      A_						:  int          =				embeddings['''position_embeddings''']['''weight''']
      # Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
      A_						:  List[str]          =				pos_embeddings.size(0)
      if n_positions != config.n_positions:
            raise ValueError(
                F'pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match')
      # Store the position embeddings.
      A_						:  List[Any]          =				pos_embeddings

      # The transformer.
      A_						:  Dict          =				lm['''transformer'''] if '''transformer''' in lm.keys() else lm['''encoder''']

      # The regex to extract layer names.
      A_						:  Dict          =				re.compile(r"""layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)""")

      # The simple map of names for "automated" rules.
      A_						:  List[str]          =				{
          '''attention.dense''': '''.attn.c_proj.''',
          '''self_attention.dense''': '''.attn.c_proj.''',
          '''mlp.dense_h_to_4h''': '''.mlp.c_fc.''',
          '''mlp.dense_4h_to_h''': '''.mlp.c_proj.''',
      }

      # Extract the layers.
      for key, val in transformer.items():
            # Match the name.
            A_						:  Optional[Any]          =				layer_re.match(__A)

            # Stop if that's not a layer
            if m is None:
                  break

            # The index of the layer.
            A_						:  List[Any]          =				int(m.group(1))
            # The name of the operation.
            A_						:  Dict          =				m.group(2)
            # Is it a weight or a bias?
            A_						:  Any          =				m.group(3)

            # The name of the layer.
            A_						:  int          =				F'transformer.h.{layer_idx}'

            # For layernorm(s), simply store the layer norm.
            if op_name.endswith("""layernorm"""):
                  A_						:  List[Any]          =				'''ln_1''' if op_name.startswith("""input""") else '''ln_2'''
                  A_						:  Any          =				val

            # Transpose the QKV matrix.
            elif (
                op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
            ) and weight_or_bias == "weight":
                  # Insert a tensor of 1x1xDxD bias.
                  A_						:  List[str]          =				torch.tril(torch.ones((n_positions, n_positions)							,						dtype=torch.floataa)).view(
                      1							,						1							,						__A							,						__A)
                  A_						:  Any          =				causal_mask

                  # Insert a "dummy" tensor for masked_bias.
                  A_						:  Tuple          =				torch.tensor(-1E4							,						dtype=torch.floataa)
                  A_						:  int          =				masked_bias

                  A_						:  Optional[Any]          =				fix_query_key_value_ordering(__A							,						__A							,						3							,						__A							,						__A)
                  # Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
                  A_						:  Tuple          =				out_val.transpose(0							,						1).contiguous()
                  # Store.
                  A_						:  Any          =				out_val

            # Transpose the bias.
            elif (
                op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
            ) and weight_or_bias == "bias":
                  A_						:  List[str]          =				fix_query_key_value_ordering(__A							,						__A							,						3							,						__A							,						__A)
                  # Store. No change of shape.
                  A_						:  Union[str, Any]          =				out_val

            # Transpose the weights.
            elif weight_or_bias == "weight":
                  A_						:  List[str]          =				megatron_to_transformers[op_name]
                  A_						:  Any          =				val.transpose(0							,						1)

            # Copy the bias.
            elif weight_or_bias == "bias":
                  A_						:  str          =				megatron_to_transformers[op_name]
                  A_						:  str          =				val

                   # DEBUG.
      assert config.n_layer == layer_idx + 1

      # The final layernorm.
      A_						:  List[Any]          =				transformer['''final_layernorm.weight''']
      A_						:  str          =				transformer['''final_layernorm.bias''']

      # For LM head, transformers' wants the matrix to weight embeddings.
      A_						:  List[str]          =				word_embeddings

      # It should be done!
      return output_state_dict




def      lowerCamelCase					(      ):
      A_						:  Optional[Any]          =				argparse.ArgumentParser()
      parser.add_argument("""--print-checkpoint-structure"""							,						action="""store_true""")
      parser.add_argument(
          """path_to_checkpoint"""							,						type=__A							,						help="""Path to the checkpoint file (.zip archive or direct .pt file)"""							,						)
      parser.add_argument(
          """--config_file"""							,						default=""""""							,						type=__A							,						help="""An optional config json file describing the pre-trained model."""							,						)
      A_						:  Dict          =				parser.parse_args()

      # Extract the basename.
      A_						:  Dict          =				os.path.dirname(args.path_to_checkpoint)

      # Load the model.
      # the .zip is very optional, let's keep it for backward compatibility
      print(F'Extracting PyTorch state dictionary from {args.path_to_checkpoint}')
      if args.path_to_checkpoint.endswith(""".zip"""):
            with zipfile.ZipFile(args.path_to_checkpoint							,						"""r""") as checkpoint:
                  with checkpoint.open("""release/mp_rank_00/model_optim_rng.pt""") as pytorch_dict:
                        A_						:  Optional[Any]          =				torch.load(__A							,						map_location="""cpu""")
      else:
            A_						:  str          =				torch.load(args.path_to_checkpoint							,						map_location="""cpu""")

      A_						:  List[Any]          =				input_state_dict.get("""args"""							,						__A)

      # Read the config, or default to the model released by NVIDIA.
      if args.config_file == "":
            if ds_args is not None:
                  if ds_args.bias_gelu_fusion:
                        A_						:  Optional[Any]          =				'''gelu_fast'''
                  elif ds_args.openai_gelu:
                        A_						:  Tuple          =				'''gelu_new'''
                  else:
                        A_						:  str          =				'''gelu'''
            else:
                  # in the very early days this used to be "gelu_new"
                  A_						:  Dict          =				'''gelu_new'''

            # Spell out all parameters in case the defaults change.
            A_						:  List[str]          =				GPTaConfig(
                vocab_size=5_0257							,						n_positions=1024							,						n_embd=1024							,						n_layer=24							,						n_head=16							,						n_inner=4096							,						activation_function=__A							,						resid_pdrop=0.1							,						embd_pdrop=0.1							,						attn_pdrop=0.1							,						layer_norm_epsilon=1E-5							,						initializer_range=0.02							,						summary_type="""cls_index"""							,						summary_use_proj=__A							,						summary_activation=__A							,						summary_proj_to_labels=__A							,						summary_first_dropout=0.1							,						scale_attn_weights=__A							,						use_cache=__A							,						bos_token_id=5_0256							,						eos_token_id=5_0256							,						)
      else:
            A_						:  List[Any]          =				GPTaConfig.from_json_file(args.config_file)

      A_						:  Any          =				['''GPT2LMHeadModel''']

      # Convert.
      print("""Converting""")
      A_						:  Dict          =				convert_megatron_checkpoint(__A							,						__A							,						__A)

      # Print the structure of converted state dict.
      if args.print_checkpoint_structure:
            recursive_print(__A							,						__A)

      # Add tokenizer class info to config
      # see https://github.com/huggingface/transformers/issues/13906)
      if ds_args is not None:
            A_						:  Tuple          =				ds_args.tokenizer_type
            if tokenizer_type == "GPT2BPETokenizer":
                  A_						:  Optional[int]          =				'''gpt2'''
            elif tokenizer_type == "PretrainedFromHF":
                  A_						:  Any          =				ds_args.tokenizer_name_or_path
            else:
                  raise ValueError(F'Unrecognized tokenizer_type {tokenizer_type}')
      else:
            A_						:  List[Any]          =				'''gpt2'''

      A_						:  Optional[int]          =				AutoTokenizer.from_pretrained(__A)
      A_						:  Any          =				type(__A).__name__
      A_						:  Union[str, Any]          =				tokenizer_class

      # Store the config to file.
      print("""Saving config""")
      config.save_pretrained(__A)

      # Save tokenizer based on args
      print(F'Adding {tokenizer_class} tokenizer files')
      tokenizer.save_pretrained(__A)

      # Store the state_dict to file.
      A_						:  Any          =				os.path.join(__A							,						"""pytorch_model.bin""")
      print(F'Saving checkpoint to "{output_checkpoint_file}"')
      torch.save(__A							,						__A)


####################################################################################################

if __name__ == "__main__":
     main()

####################################################################################################




'''simple docstring'''


from ... import PretrainedConfig


__magic_name__			=      {
    'sijunhe/nezha-cn-base': 'https://huggingface.co/sijunhe/nezha-cn-base/resolve/main/config.json',
}



class 	__lowerCAmelCase		(							__SCREAMING_SNAKE_CASE    ):


  '''simple docstring'''







  a_  					=							NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP
  a_  					=							"""nezha"""






  def __init__(							self :       int	,_a :       Union[str, Any]=21128	,_a :       int=768	,_a :       Any=12	,_a :       List[str]=12	,_a :       str=3072	,_a :       int="gelu"	,_a :       int=0.1	,_a :       str=0.1	,_a :       Tuple=512	,_a :       List[Any]=64	,_a :       Dict=2	,_a :       List[Any]=0.02	,_a :       Optional[Any]=1e-12	,_a :       List[Any]=0.1	,_a :       Union[str, Any]=0	,_a :       Any=2	,_a :       Union[str, Any]=3	,_a :       int=True	,**_a :       int	,):

        '''simple docstring'''




        super().__init__(pad_token_id=_a	,bos_token_id=_a	,eos_token_id=_a	,**_a )

        A_						:  Tuple          =				vocab_size
        A_						:  int          =				hidden_size
        A_						:  Any          =				num_hidden_layers
        A_						:  List[Any]          =				num_attention_heads
        A_						:  Tuple          =				hidden_act
        A_						:  List[Any]          =				intermediate_size
        A_						:  List[str]          =				hidden_dropout_prob
        A_						:  Tuple          =				attention_probs_dropout_prob
        A_						:  Dict          =				max_position_embeddings
        A_						:  Optional[Any]          =				max_relative_position
        A_						:  List[Any]          =				type_vocab_size
        A_						:  int          =				initializer_range
        A_						:  Tuple          =				layer_norm_eps
        A_						:  Dict          =				classifier_dropout
        A_						:  int          =				use_cache
