
"""simple docstring"""





import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version('''>=''', FSDP_PYTORCH_VERSION):
		import torch.distributed.checkpoint as dist_cp
		from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
		from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
		from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
		from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


UpperCamelCase_					:						List[Any]				= get_logger(__name__)
def 							A_     (__a				,   __a				,   __a				,   __a				,   __a=0				):
					os.makedirs(_A				,   exist_ok=_A				)
					with FSDP.state_dict_type(
					    _A				,   fsdp_plugin.state_dict_type				,   fsdp_plugin.state_dict_config				,   fsdp_plugin.optim_state_dict_config				):
										A_				 =						model.state_dict()
										if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
															A_				 =						f'{MODEL_NAME}.bin' if model_index == 0 else f'{MODEL_NAME}_{model_index}.bin'
															A_				 =						os.path.join(_A				,   _A				)
															if accelerator.process_index == 0:
																				logger.info(f'Saving model to {output_model_file}'				)
																				torch.save(_A				,   _A				)
																				logger.info(f'Model saved to {output_model_file}'				)
										elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
															A_				 =						(
															    f'{MODEL_NAME}_rank{accelerator.process_index}.bin'
															    if model_index == 0
															    else f'{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin'
															)
															A_				 =						os.path.join(_A				,   _A				)
															logger.info(f'Saving model to {output_model_file}'				)
															torch.save(_A				,   _A				)
															logger.info(f'Model saved to {output_model_file}'				)
										elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
															A_				 =						os.path.join(_A				,   f'{MODEL_NAME}_{model_index}'				)
															os.makedirs(_A				,   exist_ok=_A				)
															logger.info(f'Saving model to {ckpt_dir}'				)
															A_				 =						{"model": state_dict}

															dist_cp.save_state_dict(
															    state_dict=_A				,   storage_writer=dist_cp.FileSystemWriter(_A				)				,   planner=DefaultSavePlanner()				,   )
															logger.info(f'Model saved to {ckpt_dir}'				)
def 							A_     (__a				,   __a				,   __a				,   __a				,   __a=0				):
					accelerator.wait_for_everyone()
					with FSDP.state_dict_type(
					    _A				,   fsdp_plugin.state_dict_type				,   fsdp_plugin.state_dict_config				,   fsdp_plugin.optim_state_dict_config				):
										if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
															if type(_A				) != FSDP and accelerator.process_index != 0:
																				if not fsdp_plugin.sync_module_states:
																									raise ValueError(
																									    "Set the `sync_module_states` flag to `True` so that model states are synced across processes when "
																									    "initializing FSDP object"				)
																				return
															A_				 =						f'{MODEL_NAME}.bin' if model_index == 0 else f'{MODEL_NAME}_{model_index}.bin'
															A_				 =						os.path.join(_A				,   _A				)
															logger.info(f'Loading model from {input_model_file}'				)
															A_				 =						torch.load(_A				)
															logger.info(f'Model loaded from {input_model_file}'				)
										elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
															A_				 =						(
															    f'{MODEL_NAME}_rank{accelerator.process_index}.bin'
															    if model_index == 0
															    else f'{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin'
															)
															A_				 =						os.path.join(_A				,   _A				)
															logger.info(f'Loading model from {input_model_file}'				)
															A_				 =						torch.load(_A				)
															logger.info(f'Model loaded from {input_model_file}'				)
										elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
															A_				 =						(
															    os.path.join(_A				,   f'{MODEL_NAME}_{model_index}'				)
															    if f'{MODEL_NAME}' not in input_dir
															    else input_dir
															)
															logger.info(f'Loading model from {ckpt_dir}'				)
															A_				 =						{"model": model.state_dict()}
															dist_cp.load_state_dict(
															    state_dict=_A				,   storage_reader=dist_cp.FileSystemReader(_A				)				,   planner=DefaultLoadPlanner()				,   )
															A_				 =						state_dict["model"]
															logger.info(f'Model loaded from {ckpt_dir}'				)
										model.load_state_dict(_A				)
def 							A_     (__a				,   __a				,   __a				,   __a				,   __a				,   __a=0				):
					os.makedirs(_A				,   exist_ok=_A				)
					with FSDP.state_dict_type(
					    _A				,   fsdp_plugin.state_dict_type				,   fsdp_plugin.state_dict_config				,   fsdp_plugin.optim_state_dict_config				):
										A_				 =						FSDP.optim_state_dict(_A				,   _A				)
										if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
															if accelerator.process_index == 0:
																				A_				 =						(
																				    f'{OPTIMIZER_NAME}.bin' if optimizer_index == 0 else f'{OPTIMIZER_NAME}_{optimizer_index}.bin'
																				)
																				A_				 =						os.path.join(_A				,   _A				)
																				logger.info(f'Saving Optimizer state to {output_optimizer_file}'				)
																				torch.save(_A				,   _A				)
																				logger.info(f'Optimizer state saved in {output_optimizer_file}'				)
										else:
															A_				 =						os.path.join(_A				,   f'{OPTIMIZER_NAME}_{optimizer_index}'				)
															os.makedirs(_A				,   exist_ok=_A				)
															logger.info(f'Saving Optimizer state to {ckpt_dir}'				)
															dist_cp.save_state_dict(
															    state_dict={"optimizer": optim_state}				,   storage_writer=dist_cp.FileSystemWriter(_A				)				,   planner=DefaultSavePlanner()				,   )
															logger.info(f'Optimizer state saved in {ckpt_dir}'				)




def 							A_     (__a				,   __a				,   __a				,   __a				,   __a				,   __a=0				):
					accelerator.wait_for_everyone()
					with FSDP.state_dict_type(
					    _A				,   fsdp_plugin.state_dict_type				,   fsdp_plugin.state_dict_config				,   fsdp_plugin.optim_state_dict_config				):
										if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
															A_				 =						None
															# below check should work but currently it isn't working (mostly opytorch issue),
															# in the meantime disabling it at the cost of excess memory usage
															# if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
															A_				 =						(
															    f'{OPTIMIZER_NAME}.bin' if optimizer_index == 0 else f'{OPTIMIZER_NAME}_{optimizer_index}.bin'
															)
															A_				 =						os.path.join(_A				,   _A				)
															logger.info(f'Loading Optimizer state from {input_optimizer_file}'				)
															A_				 =						torch.load(_A				)
															logger.info(f'Optimizer state loaded from {input_optimizer_file}'				)
										else:
															A_				 =						(
															    os.path.join(_A				,   f'{OPTIMIZER_NAME}_{optimizer_index}'				)
															    if f'{OPTIMIZER_NAME}' not in input_dir
															    else input_dir
															)
															logger.info(f'Loading Optimizer from {ckpt_dir}'				)
															A_				 =						load_sharded_optimizer_state_dict(
															    model_state_dict=model.state_dict()				,   optimizer_key="optimizer"				,   storage_reader=dist_cp.FileSystemReader(_A				)				,   )
															A_				 =						optim_state["optimizer"]
															logger.info(f'Optimizer loaded from {ckpt_dir}'				)
										A_				 =						FSDP.optim_state_dict_to_load(_A				,   _A				,   _A				)
										optimizer.load_state_dict(_A				)






"""simple docstring"""





import baseaa
def 							A_     (__a				):

					'''simple docstring'''





					return baseaa.aaaencode(string.encode("utf-8"				)				)
def 							A_     (__a				):

					'''simple docstring'''





					return baseaa.aaadecode(__a				).decode("utf-8"				)


if __name__ == "__main__":
		import doctest

		doctest.testmod()




