




import argparse
import json
import os
from collections import OrderedDict

import numpy as np
import tensorflow as tf
import torch



def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Dict:
  lowerCAmelCase__				:  List[str]		       =     os.path.join(args.tf_model_dir	,      '''parameters.json'''      )
  lowerCAmelCase__				:  Dict		       =     json.loads(open(UpperCamelCase      ).read()      )
  if not params:
    raise ValueError(
        F"""It seems that the json file at {parameter_file} is empty. Make sure you have a correct json file."""      )
  if not args.output.endswith('''.pt'''      ):
    lowerCAmelCase__				:  Union[str, Any]		       =     args.output + '''.pt'''
  lowerCAmelCase__				:  Dict		       =     OrderedDict()
  with tf.device('''/CPU:0'''      ):
    lowerCAmelCase__				:  Union[str, Any]		       =     tf.train.load_checkpoint(args.tf_model_dir      )
    lowerCAmelCase__				:  Optional[Any]		       =     reader.get_variable_to_shape_map()
    for key_name in shapes.keys():
      lowerCAmelCase__				:  Union[str, Any]		       =     reader.get_tensor(UpperCamelCase      ).astype(np.floataa      )
      if key_name.endswith('''/adam_m'''      ) or key_name.endswith('''/adam_v'''      ):
        continue
      if key_name.startswith('''pasts/'''      ):
        if key_name.startswith('''pasts/mlp'''      ):
          lowerCAmelCase__				:  Optional[Any]		       =     int(key_name[9]      )
        elif key_name.startswith('''pasts/out'''      ):
          lowerCAmelCase__				:  int		       =     8
        lowerCAmelCase__				:  Dict		       =     '''model.sqout.%d.weight''' % (player * 2)  # enter to nn.Sequencial with Tanh, so 2 at a time
        lowerCAmelCase__				:  Any		       =     vnp.transpose([1, 0]      ).copy()  # Mesh-Tensorflow is a diagonal matrix
        lowerCAmelCase__				:  Tuple		       =     torch.tensor(UpperCamelCase      )
      elif key_name.startswith('''model/moe'''      ):
        lowerCAmelCase__				:  int		       =     int(key_name[9:].split('''/'''      )[0]      )
        if key_name.endswith('''/switch_gating/kernel'''      ):
          lowerCAmelCase__				:  Any		       =     '''model.blocks.%d.feed_forward.mlp.router.classifier.weight''' % player
          lowerCAmelCase__				:  List[str]		       =     vnp.transpose([1, 0]      ).copy()  # Mesh-Tensorflow is a diagonal matrix
          lowerCAmelCase__				:  List[Any]		       =     torch.tensor(UpperCamelCase      )
        elif key_name.endswith('''/softmlp/kernel'''      ):
          lowerCAmelCase__				:  Optional[Any]		       =     '''model.blocks.%d.feed_forward.soft_bypass_mlp.weight''' % player
          lowerCAmelCase__				:  Union[str, Any]		       =     vnp.transpose([1, 0]      ).copy()  # Mesh-Tensorflow is a diagonal matrix
          lowerCAmelCase__				:  int		       =     torch.tensor(UpperCamelCase      )
        elif key_name.endswith('''/wo/kernel'''      ) or key_name.endswith('''/wi/kernel'''      ):
          lowerCAmelCase__				:  str		       =     key_name[-9:-7]
          for i in range(16      ):
            lowerCAmelCase__				:  List[Any]		       =     '''model.blocks.%d.feed_forward.mlp.experts.expert_%d.%s.weight''' % (player, i, nlayer)
            lowerCAmelCase__				:  str		       =     (
                vnp[i].transpose([1, 0]      ).copy()
            )  # In Mesh-Tensorflow, it is one array, so it is divided
            lowerCAmelCase__				:  Any		       =     torch.tensor(UpperCamelCase      )
      elif key_name.startswith('''model/mlp'''      ):
        lowerCAmelCase__				:  str		       =     int(key_name[9:].split('''/'''      )[0]      )
        if key_name.endswith('''/p1/kernel'''      ):
          lowerCAmelCase__				:  str		       =     '''model.blocks.%d.feed_forward.mlp.wi.weight''' % player
          lowerCAmelCase__				:  Dict		       =     vnp.transpose([1, 0]      ).copy()  # Mesh-Tensorflow is a diagonal matrix
          lowerCAmelCase__				:  Any		       =     torch.tensor(UpperCamelCase      )
        elif key_name.endswith('''/p1/bias'''      ):
          lowerCAmelCase__				:  Dict		       =     '''model.blocks.%d.feed_forward.mlp.wi.bias''' % player
          lowerCAmelCase__				:  Optional[Any]		       =     vnp.copy()  # same because it is one dimensional
          lowerCAmelCase__				:  Optional[Any]		       =     torch.tensor(UpperCamelCase      )
        elif key_name.endswith('''/p2/kernel'''      ):
          lowerCAmelCase__				:  str		       =     '''model.blocks.%d.feed_forward.mlp.wo.weight''' % player
          lowerCAmelCase__				:  List[Any]		       =     vnp.transpose([1, 0]      ).copy()  # Mesh-Tensorflow is a diagonal matrix
          lowerCAmelCase__				:  Any		       =     torch.tensor(UpperCamelCase      )
        elif key_name.endswith('''/p2/bias'''      ):
          lowerCAmelCase__				:  Union[str, Any]		       =     '''model.blocks.%d.feed_forward.mlp.wo.bias''' % player
          lowerCAmelCase__				:  List[str]		       =     vnp.copy()  # same because it is one dimensional
          lowerCAmelCase__				:  Optional[Any]		       =     torch.tensor(UpperCamelCase      )
      elif key_name.startswith('''model/ln'''      ):
        lowerCAmelCase__				:  str		       =     int(key_name[8:].split('''/'''      )[0]      )
        if key_name.endswith('''/b'''      ):
          lowerCAmelCase__				:  Tuple		       =     '''model.blocks.%d.feed_forward.norm.bias''' % player
          lowerCAmelCase__				:  Optional[int]		       =     vnp.copy()  # same because it is one dimensional
          lowerCAmelCase__				:  int		       =     torch.tensor(UpperCamelCase      )
        elif key_name.endswith('''/g'''      ):
          lowerCAmelCase__				:  Optional[int]		       =     '''model.blocks.%d.feed_forward.norm.weight''' % player
          lowerCAmelCase__				:  int		       =     vnp.copy()  # same because it is one dimensional
          lowerCAmelCase__				:  Dict		       =     torch.tensor(UpperCamelCase      )
      elif key_name.startswith('''model/att'''      ):
        lowerCAmelCase__				:  Tuple		       =     int(key_name[9:].split('''/'''      )[0]      )
        if key_name.endswith('''/qkv/kernel'''      ):
          lowerCAmelCase__				:  Any		       =     vnp.copy()  # Compute same dimension as Mesh-tensorflow using einsum
          lowerCAmelCase__				:  Union[str, Any]		       =     state[:, 0, :, :]
          lowerCAmelCase__				:  Optional[int]		       =     state[:, 1, :, :]
          lowerCAmelCase__				:  List[str]		       =     state[:, 2, :, :]
          lowerCAmelCase__				:  Optional[Any]		       =     (
              state_q.reshape([state_q.shape[0], state_q.shape[1] * state_q.shape[2]]      )
              .transpose([1, 0]      )
              .copy()
          )  # Mesh-Tensorflow is a diagonal matrix
          lowerCAmelCase__				:  List[str]		       =     (
              state_k.reshape([state_k.shape[0], state_k.shape[1] * state_k.shape[2]]      )
              .transpose([1, 0]      )
              .copy()
          )  # Mesh-Tensorflow is a diagonal matrix
          lowerCAmelCase__				:  str		       =     (
              state_v.reshape([state_v.shape[0], state_v.shape[1] * state_v.shape[2]]      )
              .transpose([1, 0]      )
              .copy()
          )  # Mesh-Tensorflow is a diagonal matrix
          lowerCAmelCase__				:  Any		       =     '''model.blocks.%d.self_attn.self_attn.q_proj.weight''' % player
          lowerCAmelCase__				:  List[str]		       =     torch.tensor(UpperCamelCase      )
          lowerCAmelCase__				:  Any		       =     '''model.blocks.%d.self_attn.self_attn.k_proj.weight''' % player
          lowerCAmelCase__				:  Optional[int]		       =     torch.tensor(UpperCamelCase      )
          lowerCAmelCase__				:  Any		       =     '''model.blocks.%d.self_attn.self_attn.v_proj.weight''' % player
          lowerCAmelCase__				:  List[str]		       =     torch.tensor(UpperCamelCase      )
        elif key_name.endswith('''/o/kernel'''      ):
          lowerCAmelCase__				:  Tuple		       =     '''model.blocks.%d.self_attn.self_attn.out_proj.weight''' % player
          lowerCAmelCase__				:  Optional[int]		       =     (
              vnp.reshape([vnp.shape[0] * vnp.shape[1], vnp.shape[2]]      ).transpose([1, 0]      ).copy()
          )  # Mesh-Tensorflow is a diagonal matrix
          lowerCAmelCase__				:  Union[str, Any]		       =     torch.tensor(UpperCamelCase      )
      elif key_name.startswith('''model/an'''      ):
        lowerCAmelCase__				:  Optional[Any]		       =     int(key_name[8:].split('''/'''      )[0]      )
        if key_name.endswith('''/b'''      ):
          lowerCAmelCase__				:  List[Any]		       =     '''model.blocks.%d.self_attn.norm.bias''' % player
          lowerCAmelCase__				:  Optional[Any]		       =     vnp.copy()  # same because it is one dimensional
          lowerCAmelCase__				:  Union[str, Any]		       =     torch.tensor(UpperCamelCase      )
        elif key_name.endswith('''/g'''      ):
          lowerCAmelCase__				:  Dict		       =     '''model.blocks.%d.self_attn.norm.weight''' % player
          lowerCAmelCase__				:  str		       =     vnp.copy()  # same because it is one dimensional
          lowerCAmelCase__				:  List[str]		       =     torch.tensor(UpperCamelCase      )
      elif (
          key_name.startswith('''model/wte'''      )
          or key_name.startswith('''model/wpe'''      )
          or key_name.startswith('''model/ete'''      )
      ):
        lowerCAmelCase__				:  Optional[Any]		       =     {'''wte''': '''embed_tokens''', '''wpe''': '''position_embeddings''', '''ete''': '''extra_position_embeddings'''}[
            key_name[-3:]
        ]
        lowerCAmelCase__				:  Any		       =     '''model.%s.weight''' % nlayer
        lowerCAmelCase__				:  str		       =     vnp.copy()  # same in embedded
        lowerCAmelCase__				:  Dict		       =     torch.tensor(UpperCamelCase      )
        if key_name.startswith('''model/wte'''      ):
          lowerCAmelCase__				:  Any		       =     '''lm_head.weight'''
          lowerCAmelCase__				:  List[Any]		       =     vnp.copy()  # same in embedded
          lowerCAmelCase__				:  List[Any]		       =     torch.tensor(UpperCamelCase      )
      elif key_name.startswith('''model/wob'''      ):
        lowerCAmelCase__				:  Optional[Any]		       =     '''final_logits_bias'''
        lowerCAmelCase__				:  Union[str, Any]		       =     vnp.copy()  # same in embedded
        lowerCAmelCase__				:  Any		       =     state.reshape((1, -1)      )
        lowerCAmelCase__				:  int		       =     torch.tensor(UpperCamelCase      )
      elif key_name == "model/dense/kernel":
        lowerCAmelCase__				:  str		       =     '''model.last_project.weight'''
        lowerCAmelCase__				:  List[str]		       =     vnp.transpose([1, 0]      ).copy()  # Mesh-Tensorflow is a diagonal matrix
        lowerCAmelCase__				:  List[Any]		       =     torch.tensor(UpperCamelCase      )
      elif key_name == "model/dense_1/bias":
        lowerCAmelCase__				:  List[str]		       =     '''model.last_project.bias'''
        lowerCAmelCase__				:  Optional[int]		       =     vnp.copy()  # same because it is one dimensional
        lowerCAmelCase__				:  str		       =     torch.tensor(UpperCamelCase      )
  torch.save(UpperCamelCase	,      args.output      )


if __name__ == "__main__":
     lowerCAmelCase_					    = argparse.ArgumentParser(
         description="""model converter.""", formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
     parser.add_argument("""--tf_model_dir""", metavar="""PATH""", type=str, required=True, help="""import model""")
     parser.add_argument("""--output""", metavar="""PATH""", type=str, required=True, help="""output model""")
     lowerCAmelCase_					    = parser.parse_args()
     convert_tf_gptsan_to_pt(args)







import baseaa
import io
import json
import os
from copy import deepcopy

from ..optimizer import AcceleratedOptimizer
from ..scheduler import AcceleratedScheduler






class  _lowerCAmelCase     :

      def __init__(   self      ,				__UpperCAmelCase		):
        if isinstance(__UpperCAmelCase      ,				__UpperCAmelCase		):
          # Don't modify user's data should they want to reuse it (e.g. in tests), because once we
          # modified it, it will not be accepted here again, since `auto` values would have been overridden
          lowerCAmelCase__				:  Union[str, Any]		       =     deepcopy(__UpperCAmelCase		)
        elif os.path.exists(__UpperCAmelCase		):
          with io.open(__UpperCAmelCase      ,				'''r'''      ,				encoding='''utf-8'''		) as f:
            lowerCAmelCase__				:  int		       =     json.load(__UpperCAmelCase		)
        else:
          try:
            lowerCAmelCase__				:  Union[str, Any]		       =     baseaa.urlsafe_baadecode(__UpperCAmelCase		).decode('''utf-8'''		)
            lowerCAmelCase__				:  Optional[int]		       =     json.loads(__UpperCAmelCase		)
          except (UnicodeDecodeError, AttributeError, ValueError):
            raise ValueError(
                f"""Expected a string path to an existing deepspeed config, or a dictionary, or a base64 encoded string. Received: {config_file_or_dict}"""		)

        lowerCAmelCase__				:  Optional[Any]		       =     config

        self.set_stage_and_offload()

      def 	__magic_name__(   self		):
        # zero stage - this is done as early as possible, before model is created, to allow
        # ``is_deepspeed_zero3_enabled`` query and getting to the early deepspeed config object
        # during ``zero.Init()`` which needs to know the dtype, and some other hparams.
        lowerCAmelCase__				:  Optional[Any]		       =     self.get_value('''zero_optimization.stage'''      ,				-1		)

        # offload
        lowerCAmelCase__				:  Dict		       =     False
        if self.is_zeroa() or self.is_zeroa():
          lowerCAmelCase__				:  Dict		       =     set(['''cpu''', '''nvme''']		)
          lowerCAmelCase__				:  Optional[Any]		       =     set(
              [
                  self.get_value('''zero_optimization.offload_optimizer.device'''		),
                  self.get_value('''zero_optimization.offload_param.device'''		),
              ]		)
          if len(offload_devices & offload_devices_valid		) > 0:
            lowerCAmelCase__				:  List[Any]		       =     True

      def 	__magic_name__(   self      ,				__UpperCAmelCase		):
        lowerCAmelCase__				:  int		       =     self.config

        # find the config node of interest if it exists
        lowerCAmelCase__				:  Tuple		       =     ds_key_long.split('''.'''		)
        lowerCAmelCase__				:  int		       =     nodes.pop()
        for node in nodes:
          lowerCAmelCase__				:  int		       =     config.get(__UpperCAmelCase		)
          if config is None:
            return None, ds_key

        return config, ds_key

      def 	__magic_name__(   self      ,				__UpperCAmelCase      ,				__UpperCAmelCase=None		):
        lowerCAmelCase__       ,			lowerCAmelCase__				:  Optional[int]		       =     self.find_config_node(__UpperCAmelCase		)
        if config is None:
          return default
        return config.get(__UpperCAmelCase      ,				__UpperCAmelCase		)

      def 	__magic_name__(   self      ,				__UpperCAmelCase      ,				__UpperCAmelCase=False		):
        lowerCAmelCase__				:  Dict		       =     self.config

        # find the config node of interest if it exists
        lowerCAmelCase__				:  str		       =     ds_key_long.split('''.'''		)
        for node in nodes:
          lowerCAmelCase__				:  Dict		       =     config
          lowerCAmelCase__				:  List[str]		       =     config.get(__UpperCAmelCase		)
          if config is None:
            if must_exist:
              raise ValueError(f"""Can't find {ds_key_long} entry in the config: {self.config}"""		)
            else:
              return

        # if found remove it
        if parent_config is not None:
          parent_config.pop(__UpperCAmelCase		)

      def 	__magic_name__(   self      ,				__UpperCAmelCase		):
        lowerCAmelCase__				:  Union[str, Any]		       =     self.get_value(__UpperCAmelCase		)
        return False if value is None else bool(__UpperCAmelCase		)

      def 	__magic_name__(   self      ,				__UpperCAmelCase		):
        lowerCAmelCase__				:  Optional[Any]		       =     self.get_value(__UpperCAmelCase		)
        return False if value is None else not bool(__UpperCAmelCase		)

      def 	__magic_name__(   self		):
        return self._stage == 2

      def 	__magic_name__(   self		):
        return self._stage == 3







      def 	__magic_name__(   self		):
        return self._offload






class  _lowerCAmelCase     :

      def __init__(   self      ,				__UpperCAmelCase		):
        lowerCAmelCase__				:  int		       =     engine







      def 	__magic_name__(   self      ,				__UpperCAmelCase      ,				**__UpperCAmelCase		):
        # runs backpropagation and handles mixed precision
        self.engine.backward(__UpperCAmelCase      ,				**__UpperCAmelCase		)

        # Deepspeed's `engine.step` performs the following operations:
        # - gradient accumulation check
        # - gradient clipping
        # - optimizer step
        # - zero grad
        # - checking overflow
        # - lr_scheduler step (only if engine.lr_scheduler is not None)
        self.engine.step()
        # and this plugin overrides the above calls with no-ops when Accelerate runs under
        # Deepspeed, but allows normal functionality for non-Deepspeed cases thus enabling a simple
        # training loop that works transparently under many training regimes.






class  _lowerCAmelCase     (  _lowercase	):

      def __init__(   self      ,				__UpperCAmelCase		):
        super().__init__(__UpperCAmelCase      ,				device_placement=__UpperCAmelCase      ,				scaler=__UpperCAmelCase		)
        lowerCAmelCase__				:  Optional[int]		       =     hasattr(self.optimizer      ,				'''overflow'''		)

      def 	__magic_name__(   self      ,				__UpperCAmelCase=None		):
        pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed

      def 	__magic_name__(   self		):
        pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed







      @property
      def 	__magic_name__(   self		):
        if self.__has_overflow__:
          return self.optimizer.overflow
        return False






class  _lowerCAmelCase     (  _lowercase	):

      def __init__(   self      ,				__UpperCAmelCase      ,				__UpperCAmelCase		):
        super().__init__(__UpperCAmelCase      ,				__UpperCAmelCase		)







      def 	__magic_name__(   self		):
        pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed






class  _lowerCAmelCase     :

      def __init__(   self      ,				__UpperCAmelCase      ,				__UpperCAmelCase=0.001      ,				__UpperCAmelCase=0      ,				**__UpperCAmelCase		):
        lowerCAmelCase__				:  Any		       =     params
        lowerCAmelCase__				:  str		       =     lr
        lowerCAmelCase__				:  List[str]		       =     weight_decay
        lowerCAmelCase__				:  Dict		       =     kwargs






class  _lowerCAmelCase     :

      def __init__(   self      ,				__UpperCAmelCase      ,				__UpperCAmelCase=None      ,				__UpperCAmelCase=0      ,				**__UpperCAmelCase		):
        lowerCAmelCase__				:  Union[str, Any]		       =     optimizer
        lowerCAmelCase__				:  Optional[Any]		       =     total_num_steps
        lowerCAmelCase__				:  str		       =     warmup_num_steps
        lowerCAmelCase__				:  Dict		       =     kwargs

