






"""simple docstring"""
import argparse
import json
import os
import sys
import tempfile
import unittest
from argparse import Namespace
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import List, Literal, Optional

import yaml

from transformers import HfArgumentParser, TrainingArguments
from transformers.hf_argparser import make_choice_type_function, string_to_bool


# Since Python 3.10, we can use the builtin `|` operator for Union types
# See PEP 604: https://peps.python.org/pep-0604
_lowerCAmelCase         =							sys.version_info >= (3, 10)






def 			__UpperCamelCase	(		snake_case__=None   ,   snake_case__=None					):
	return field(default_factory=lambda: default   ,   metadata=snake_case__					)



@dataclass
class      SCREAMING_SNAKE_CASE  :





	"""simple docstring"""




	_A       :				int
	_A       :				float
	_A       :				str
	_A       :				bool






@dataclass
class      SCREAMING_SNAKE_CASE  :





	"""simple docstring"""




	_A       :				int    =				42
	_A       :				str    =				field(default="""toto"""       ,       metadata={"""help""": """help message"""}						)






@dataclass
class      SCREAMING_SNAKE_CASE  :





	"""simple docstring"""




	_A       :				bool    =				False
	_A       :				bool    =				True
	_A       :				Optional[bool]    =				None






class      SCREAMING_SNAKE_CASE  (				_SCREAMING_SNAKE_CASE						):





	"""simple docstring"""




	_A       :				str    =				"""titi"""
	_A       :				str    =				"""toto"""






class      SCREAMING_SNAKE_CASE  (				_SCREAMING_SNAKE_CASE						):





	"""simple docstring"""




	_A       :				Any    =				"""titi"""
	_A       :				Any    =				"""toto"""
	_A       :				List[str]    =				42






@dataclass
class      SCREAMING_SNAKE_CASE  :





	"""simple docstring"""




	_A       :				BasicEnum    =				"toto"





	def    lowerCamelCase(self							):
		A_		:						Any	    =   BasicEnum(self.foo							)






@dataclass
class      SCREAMING_SNAKE_CASE  :





	"""simple docstring"""




	_A       :				MixedTypeEnum    =				"toto"





	def    lowerCamelCase(self							):
		A_		:						Union[str, Any]	    =   MixedTypeEnum(self.foo							)






@dataclass
class      SCREAMING_SNAKE_CASE  :





	"""simple docstring"""




	_A       :				Optional[int]    =				None
	_A       :				Optional[float]    =				field(default=_SCREAMING_SNAKE_CASE       ,       metadata={"""help""": """help message"""}						)
	_A       :				Optional[str]    =				None
	_A       :				Optional[List[str]]    =				list_field(default=[]						)
	_A       :				Optional[List[int]]    =				list_field(default=[]						)






@dataclass
class      SCREAMING_SNAKE_CASE  :





	"""simple docstring"""




	_A       :				List[int]    =				list_field(default=[]						)
	_A       :				List[int]    =				list_field(default=[1, 2, 3]						)
	_A       :				List[str]    =				list_field(default=["""Hallo""", """Bonjour""", """Hello"""]						)
	_A       :				List[float]    =				list_field(default=[0.1, 0.2, 0.3]						)






@dataclass
class      SCREAMING_SNAKE_CASE  :





	"""simple docstring"""




	_A       :				List[int]    =				field()
	_A       :				str    =				field()
	_A       :				BasicEnum    =				field()





	def    lowerCamelCase(self							):
		A_		:						int	    =   BasicEnum(self.required_enum							)






@dataclass
class      SCREAMING_SNAKE_CASE  :





	"""simple docstring"""




	_A       :				int
	_A       :				"BasicEnum"    =				field()
	_A       :				"Optional[bool]"    =				None
	_A       :				"str"    =				field(default="""toto"""       ,       metadata={"""help""": """help message"""}						)
	_A       :				"List[str]"    =				list_field(default=["""Hallo""", """Bonjour""", """Hello"""]						)


if is_python_no_less_than_3_10:



							@dataclass
							class      SCREAMING_SNAKE_CASE  :





								"""simple docstring"""




								_A       :				bool    =				False
								_A       :				bool    =				True
								_A       :				bool | None    =				None






							@dataclass
							class      SCREAMING_SNAKE_CASE  :





								"""simple docstring"""




								_A       :				int | None    =				None
								_A       :				float | None    =				field(default=_SCREAMING_SNAKE_CASE       ,       metadata={"""help""": """help message"""}						)
								_A       :				str | None    =				None
								_A       :				list[str] | None    =				list_field(default=[]						)
								_A       :				list[int] | None    =				list_field(default=[]						)




class      SCREAMING_SNAKE_CASE  (				unittest.TestCase						):





	"""simple docstring"""




	def    lowerCamelCase(self     ,     lowerCAmelCase_     ,     lowerCAmelCase_							):
		self.assertEqual(len(a._actions							)     ,     len(b._actions							)							)
		for x, y in zip(a._actions     ,     b._actions							):
			A_		:						Optional[Any]	    =   {k: v for k, v in vars(lowerCAmelCase_							).items() if k != """container"""}
			A_		:						Tuple	    =   {k: v for k, v in vars(lowerCAmelCase_							).items() if k != """container"""}

			# Choices with mixed type have custom function as "type"
			# So we need to compare results directly for equality
			if xx.get("""choices"""     ,     lowerCAmelCase_							) and yy.get("""choices"""     ,     lowerCAmelCase_							):
				for expected_choice in yy["choices"] + xx["choices"]:
					self.assertEqual(xx["""type"""](lowerCAmelCase_							)     ,     yy["""type"""](lowerCAmelCase_							)							)
				del xx["type"], yy["type"]

			self.assertEqual(lowerCAmelCase_     ,     lowerCAmelCase_							)





	def    lowerCamelCase(self							):
		A_		:						Optional[Any]	    =   HfArgumentParser(lowerCAmelCase_							)

		A_		:						Optional[Any]	    =   argparse.ArgumentParser()
		expected.add_argument("""--foo"""     ,     type=lowerCAmelCase_     ,     required=lowerCAmelCase_							)
		expected.add_argument("""--bar"""     ,     type=lowerCAmelCase_     ,     required=lowerCAmelCase_							)
		expected.add_argument("""--baz"""     ,     type=lowerCAmelCase_     ,     required=lowerCAmelCase_							)
		expected.add_argument("""--flag"""     ,     type=lowerCAmelCase_     ,     default=lowerCAmelCase_     ,     const=lowerCAmelCase_     ,     nargs="""?"""							)
		self.argparsersEqual(lowerCAmelCase_     ,     lowerCAmelCase_							)

		A_		:						Optional[Any]	    =   ["""--foo""", """1""", """--baz""", """quux""", """--bar""", """0.5"""]
		((A_)    ,     )		:						List[str]	    =   parser.parse_args_into_dataclasses(lowerCAmelCase_     ,     look_for_args_file=lowerCAmelCase_							)
		self.assertFalse(example.flag							)





	def    lowerCamelCase(self							):
		A_		:						Dict	    =   HfArgumentParser(lowerCAmelCase_							)

		A_		:						str	    =   argparse.ArgumentParser()
		expected.add_argument("""--foo"""     ,     default=42     ,     type=lowerCAmelCase_							)
		expected.add_argument("""--baz"""     ,     default="""toto"""     ,     type=lowerCAmelCase_     ,     help="""help message"""							)
		self.argparsersEqual(lowerCAmelCase_     ,     lowerCAmelCase_							)





	def    lowerCamelCase(self							):
		A_		:						int	    =   argparse.ArgumentParser()
		expected.add_argument("""--foo"""     ,     type=lowerCAmelCase_     ,     default=lowerCAmelCase_     ,     const=lowerCAmelCase_     ,     nargs="""?"""							)
		expected.add_argument("""--baz"""     ,     type=lowerCAmelCase_     ,     default=lowerCAmelCase_     ,     const=lowerCAmelCase_     ,     nargs="""?"""							)
		# A boolean no_* argument always has to come after its "default: True" regular counter-part
		# and its default must be set to False
		expected.add_argument("""--no_baz"""     ,     action="""store_false"""     ,     default=lowerCAmelCase_     ,     dest="""baz"""							)
		expected.add_argument("""--opt"""     ,     type=lowerCAmelCase_     ,     default=lowerCAmelCase_							)

		A_		:						Dict	    =   [WithDefaultBoolExample]
		if is_python_no_less_than_3_10:
			dataclass_types.append(lowerCAmelCase_							)

		for dataclass_type in dataclass_types:
			A_		:						Tuple	    =   HfArgumentParser(lowerCAmelCase_							)
			self.argparsersEqual(lowerCAmelCase_     ,     lowerCAmelCase_							)

			A_		:						Dict	    =   parser.parse_args([]							)
			self.assertEqual(lowerCAmelCase_     ,     Namespace(foo=lowerCAmelCase_     ,     baz=lowerCAmelCase_     ,     opt=lowerCAmelCase_							)							)

			A_		:						Tuple	    =   parser.parse_args(["""--foo""", """--no_baz"""]							)
			self.assertEqual(lowerCAmelCase_     ,     Namespace(foo=lowerCAmelCase_     ,     baz=lowerCAmelCase_     ,     opt=lowerCAmelCase_							)							)

			A_		:						Optional[Any]	    =   parser.parse_args(["""--foo""", """--baz"""]							)
			self.assertEqual(lowerCAmelCase_     ,     Namespace(foo=lowerCAmelCase_     ,     baz=lowerCAmelCase_     ,     opt=lowerCAmelCase_							)							)

			A_		:						List[str]	    =   parser.parse_args(["""--foo""", """True""", """--baz""", """True""", """--opt""", """True"""]							)
			self.assertEqual(lowerCAmelCase_     ,     Namespace(foo=lowerCAmelCase_     ,     baz=lowerCAmelCase_     ,     opt=lowerCAmelCase_							)							)

			A_		:						str	    =   parser.parse_args(["""--foo""", """False""", """--baz""", """False""", """--opt""", """False"""]							)
			self.assertEqual(lowerCAmelCase_     ,     Namespace(foo=lowerCAmelCase_     ,     baz=lowerCAmelCase_     ,     opt=lowerCAmelCase_							)							)





	def    lowerCamelCase(self							):
		A_		:						Dict	    =   HfArgumentParser(lowerCAmelCase_							)

		A_		:						Optional[Any]	    =   argparse.ArgumentParser()
		expected.add_argument(
		    """--foo"""     ,     default="""toto"""     ,     choices=["""titi""", """toto""", 42]     ,     type=make_choice_type_function(["""titi""", """toto""", 42]							)     ,     )
		self.argparsersEqual(lowerCAmelCase_     ,     lowerCAmelCase_							)

		A_		:						str	    =   parser.parse_args([]							)
		self.assertEqual(args.foo     ,     """toto"""							)
		A_		:						Optional[Any]	    =   parser.parse_args_into_dataclasses([]							)[0]
		self.assertEqual(enum_ex.foo     ,     MixedTypeEnum.toto							)

		A_		:						int	    =   parser.parse_args(["""--foo""", """titi"""]							)
		self.assertEqual(args.foo     ,     """titi"""							)
		A_		:						Optional[int]	    =   parser.parse_args_into_dataclasses(["""--foo""", """titi"""]							)[0]
		self.assertEqual(enum_ex.foo     ,     MixedTypeEnum.titi							)

		A_		:						Optional[Any]	    =   parser.parse_args(["""--foo""", """42"""]							)
		self.assertEqual(args.foo     ,     42							)
		A_		:						Optional[int]	    =   parser.parse_args_into_dataclasses(["""--foo""", """42"""]							)[0]
		self.assertEqual(enum_ex.foo     ,     MixedTypeEnum.fourtytwo							)





	def    lowerCamelCase(self							):



		@dataclass
		class      SCREAMING_SNAKE_CASE  :





			"""simple docstring"""




			_A       :				Literal["titi", "toto", 42]    =				"toto"






		A_		:						List[Any]	    =   HfArgumentParser(lowerCAmelCase_							)

		A_		:						Dict	    =   argparse.ArgumentParser()
		expected.add_argument(
		    """--foo"""     ,     default="""toto"""     ,     choices=("""titi""", """toto""", 42)     ,     type=make_choice_type_function(["""titi""", """toto""", 42]							)     ,     )
		self.argparsersEqual(lowerCAmelCase_     ,     lowerCAmelCase_							)

		A_		:						List[Any]	    =   parser.parse_args([]							)
		self.assertEqual(args.foo     ,     """toto"""							)

		A_		:						Any	    =   parser.parse_args(["""--foo""", """titi"""]							)
		self.assertEqual(args.foo     ,     """titi"""							)

		A_		:						List[str]	    =   parser.parse_args(["""--foo""", """42"""]							)
		self.assertEqual(args.foo     ,     42							)





	def    lowerCamelCase(self							):
		A_		:						Union[str, Any]	    =   HfArgumentParser(lowerCAmelCase_							)

		A_		:						Tuple	    =   argparse.ArgumentParser()
		expected.add_argument("""--foo_int"""     ,     nargs="""+"""     ,     default=[]     ,     type=lowerCAmelCase_							)
		expected.add_argument("""--bar_int"""     ,     nargs="""+"""     ,     default=[1, 2, 3]     ,     type=lowerCAmelCase_							)
		expected.add_argument("""--foo_str"""     ,     nargs="""+"""     ,     default=["""Hallo""", """Bonjour""", """Hello"""]     ,     type=lowerCAmelCase_							)
		expected.add_argument("""--foo_float"""     ,     nargs="""+"""     ,     default=[0.1, 0.2, 0.3]     ,     type=lowerCAmelCase_							)

		self.argparsersEqual(lowerCAmelCase_     ,     lowerCAmelCase_							)

		A_		:						List[Any]	    =   parser.parse_args([]							)
		self.assertEqual(
		    lowerCAmelCase_     ,     Namespace(foo_int=[]     ,     bar_int=[1, 2, 3]     ,     foo_str=["""Hallo""", """Bonjour""", """Hello"""]     ,     foo_float=[0.1, 0.2, 0.3]							)     ,     )

		A_		:						Optional[int]	    =   parser.parse_args("""--foo_int 1 --bar_int 2 3 --foo_str a b c --foo_float 0.1 0.7""".split()							)
		self.assertEqual(lowerCAmelCase_     ,     Namespace(foo_int=[1]     ,     bar_int=[2, 3]     ,     foo_str=["""a""", """b""", """c"""]     ,     foo_float=[0.1, 0.7]							)							)





	def    lowerCamelCase(self							):
		A_		:						str	    =   argparse.ArgumentParser()
		expected.add_argument("""--foo"""     ,     default=lowerCAmelCase_     ,     type=lowerCAmelCase_							)
		expected.add_argument("""--bar"""     ,     default=lowerCAmelCase_     ,     type=lowerCAmelCase_     ,     help="""help message"""							)
		expected.add_argument("""--baz"""     ,     default=lowerCAmelCase_     ,     type=lowerCAmelCase_							)
		expected.add_argument("""--ces"""     ,     nargs="""+"""     ,     default=[]     ,     type=lowerCAmelCase_							)
		expected.add_argument("""--des"""     ,     nargs="""+"""     ,     default=[]     ,     type=lowerCAmelCase_							)

		A_		:						Dict	    =   [OptionalExample]
		if is_python_no_less_than_3_10:
			dataclass_types.append(lowerCAmelCase_							)

		for dataclass_type in dataclass_types:
			A_		:						Union[str, Any]	    =   HfArgumentParser(lowerCAmelCase_							)

			self.argparsersEqual(lowerCAmelCase_     ,     lowerCAmelCase_							)

			A_		:						Optional[int]	    =   parser.parse_args([]							)
			self.assertEqual(lowerCAmelCase_     ,     Namespace(foo=lowerCAmelCase_     ,     bar=lowerCAmelCase_     ,     baz=lowerCAmelCase_     ,     ces=[]     ,     des=[]							)							)

			A_		:						str	    =   parser.parse_args("""--foo 12 --bar 3.14 --baz 42 --ces a b c --des 1 2 3""".split()							)
			self.assertEqual(lowerCAmelCase_     ,     Namespace(foo=12     ,     bar=3.14     ,     baz="""42"""     ,     ces=["""a""", """b""", """c"""]     ,     des=[1, 2, 3]							)							)





	def    lowerCamelCase(self							):
		A_		:						List[str]	    =   HfArgumentParser(lowerCAmelCase_							)

		A_		:						int	    =   argparse.ArgumentParser()
		expected.add_argument("""--required_list"""     ,     nargs="""+"""     ,     type=lowerCAmelCase_     ,     required=lowerCAmelCase_							)
		expected.add_argument("""--required_str"""     ,     type=lowerCAmelCase_     ,     required=lowerCAmelCase_							)
		expected.add_argument(
		    """--required_enum"""     ,     type=make_choice_type_function(["""titi""", """toto"""]							)     ,     choices=["""titi""", """toto"""]     ,     required=lowerCAmelCase_     ,     )
		self.argparsersEqual(lowerCAmelCase_     ,     lowerCAmelCase_							)





	def    lowerCamelCase(self							):
		A_		:						Any	    =   HfArgumentParser(lowerCAmelCase_							)

		A_		:						Any	    =   argparse.ArgumentParser()
		expected.add_argument("""--foo"""     ,     type=lowerCAmelCase_     ,     required=lowerCAmelCase_							)
		expected.add_argument(
		    """--required_enum"""     ,     type=make_choice_type_function(["""titi""", """toto"""]							)     ,     choices=["""titi""", """toto"""]     ,     required=lowerCAmelCase_     ,     )
		expected.add_argument("""--opt"""     ,     type=lowerCAmelCase_     ,     default=lowerCAmelCase_							)
		expected.add_argument("""--baz"""     ,     default="""toto"""     ,     type=lowerCAmelCase_     ,     help="""help message"""							)
		expected.add_argument("""--foo_str"""     ,     nargs="""+"""     ,     default=["""Hallo""", """Bonjour""", """Hello"""]     ,     type=lowerCAmelCase_							)
		self.argparsersEqual(lowerCAmelCase_     ,     lowerCAmelCase_							)





	def    lowerCamelCase(self							):
		A_		:						Optional[int]	    =   HfArgumentParser(lowerCAmelCase_							)

		A_		:						List[Any]	    =   {
		    """foo""": 12,
		    """bar""": 3.14,
		    """baz""": """42""",
		    """flag""": True,
		}

		A_		:						Tuple	    =   parser.parse_dict(lowerCAmelCase_							)[0]
		A_		:						Dict	    =   BasicExample(**lowerCAmelCase_							)
		self.assertEqual(lowerCAmelCase_     ,     lowerCAmelCase_							)





	def    lowerCamelCase(self							):
		A_		:						Union[str, Any]	    =   HfArgumentParser(lowerCAmelCase_							)

		A_		:						int	    =   {
		    """foo""": 12,
		    """bar""": 3.14,
		    """baz""": """42""",
		    """flag""": True,
		    """extra""": 42,
		}

		self.assertRaises(lowerCAmelCase_     ,     parser.parse_dict     ,     lowerCAmelCase_     ,     allow_extra_keys=lowerCAmelCase_							)





	def    lowerCamelCase(self							):
		A_		:						int	    =   HfArgumentParser(lowerCAmelCase_							)

		A_		:						int	    =   {
		    """foo""": 12,
		    """bar""": 3.14,
		    """baz""": """42""",
		    """flag""": True,
		}
		with tempfile.TemporaryDirectory() as tmp_dir:
			A_		:						Union[str, Any]	    =   os.path.join(lowerCAmelCase_     ,     """temp_json"""							)
			os.mkdir(lowerCAmelCase_							)
			with open(temp_local_path + """.json"""     ,     """w+"""							) as f:
				json.dump(lowerCAmelCase_     ,     lowerCAmelCase_							)
			A_		:						int	    =   parser.parse_yaml_file(Path(temp_local_path + """.json"""							)							)[0]

		A_		:						List[str]	    =   BasicExample(**lowerCAmelCase_							)
		self.assertEqual(lowerCAmelCase_     ,     lowerCAmelCase_							)





	def    lowerCamelCase(self							):
		A_		:						Dict	    =   HfArgumentParser(lowerCAmelCase_							)

		A_		:						int	    =   {
		    """foo""": 12,
		    """bar""": 3.14,
		    """baz""": """42""",
		    """flag""": True,
		}
		with tempfile.TemporaryDirectory() as tmp_dir:
			A_		:						Union[str, Any]	    =   os.path.join(lowerCAmelCase_     ,     """temp_yaml"""							)
			os.mkdir(lowerCAmelCase_							)
			with open(temp_local_path + """.yaml"""     ,     """w+"""							) as f:
				yaml.dump(lowerCAmelCase_     ,     lowerCAmelCase_							)
			A_		:						Union[str, Any]	    =   parser.parse_yaml_file(Path(temp_local_path + """.yaml"""							)							)[0]
		A_		:						Optional[Any]	    =   BasicExample(**lowerCAmelCase_							)
		self.assertEqual(lowerCAmelCase_     ,     lowerCAmelCase_							)





	def    lowerCamelCase(self							):
		A_		:						Any	    =   HfArgumentParser(lowerCAmelCase_							)
		self.assertIsNotNone(lowerCAmelCase_							)









"""simple docstring"""
import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version(">=", FSDP_PYTORCH_VERSION):
							import torch.distributed.checkpoint as dist_cp
							from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
							from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
							from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
							from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


_lowerCAmelCase         =							get_logger(__name__)






def 			__UpperCamelCase	(		snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__=0					):
	os.makedirs(snake_case__   ,   exist_ok=snake_case__					)
	with FSDP.state_dict_type(
	    snake_case__   ,   fsdp_plugin.state_dict_type   ,   fsdp_plugin.state_dict_config   ,   fsdp_plugin.optim_state_dict_config					):
		A_		:						Optional[int]	    =   model.state_dict()
		if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
			A_		:						int	    =   F"""{MODEL_NAME}.bin""" if model_index == 0 else F"""{MODEL_NAME}_{model_index}.bin"""
			A_		:						Union[str, Any]	    =   os.path.join(snake_case__   ,   snake_case__					)
			if accelerator.process_index == 0:
				logger.info(F"""Saving model to {output_model_file}"""					)
				torch.save(snake_case__   ,   snake_case__					)
				logger.info(F"""Model saved to {output_model_file}"""					)
		elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
			A_		:						List[str]	    =   (
			    F"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
			    if model_index == 0
			    else F"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
			)
			A_		:						List[str]	    =   os.path.join(snake_case__   ,   snake_case__					)
			logger.info(F"""Saving model to {output_model_file}"""					)
			torch.save(snake_case__   ,   snake_case__					)
			logger.info(F"""Model saved to {output_model_file}"""					)
		elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
			A_		:						str	    =   os.path.join(snake_case__   ,   F"""{MODEL_NAME}_{model_index}"""					)
			os.makedirs(snake_case__   ,   exist_ok=snake_case__					)
			logger.info(F"""Saving model to {ckpt_dir}"""					)
			A_		:						Dict	    =   {"""model""": state_dict}

			dist_cp.save_state_dict(
			    state_dict=snake_case__   ,   storage_writer=dist_cp.FileSystemWriter(snake_case__					)   ,   planner=DefaultSavePlanner()   ,   )
			logger.info(F"""Model saved to {ckpt_dir}"""					)






def 			__UpperCamelCase	(		snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__=0					):
	accelerator.wait_for_everyone()
	with FSDP.state_dict_type(
	    snake_case__   ,   fsdp_plugin.state_dict_type   ,   fsdp_plugin.state_dict_config   ,   fsdp_plugin.optim_state_dict_config					):
		if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
			if type(snake_case__					) != FSDP and accelerator.process_index != 0:
				if not fsdp_plugin.sync_module_states:
					raise ValueError(
					    """Set the `sync_module_states` flag to `True` so that model states are synced across processes when """
					    """initializing FSDP object"""					)
				return
			A_		:						Tuple	    =   F"""{MODEL_NAME}.bin""" if model_index == 0 else F"""{MODEL_NAME}_{model_index}.bin"""
			A_		:						Any	    =   os.path.join(snake_case__   ,   snake_case__					)
			logger.info(F"""Loading model from {input_model_file}"""					)
			A_		:						List[Any]	    =   torch.load(snake_case__					)
			logger.info(F"""Model loaded from {input_model_file}"""					)
		elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
			A_		:						List[str]	    =   (
			    F"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
			    if model_index == 0
			    else F"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
			)
			A_		:						Any	    =   os.path.join(snake_case__   ,   snake_case__					)
			logger.info(F"""Loading model from {input_model_file}"""					)
			A_		:						List[str]	    =   torch.load(snake_case__					)
			logger.info(F"""Model loaded from {input_model_file}"""					)
		elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
			A_		:						Optional[Any]	    =   (
			    os.path.join(snake_case__   ,   F"""{MODEL_NAME}_{model_index}"""					)
			    if F"""{MODEL_NAME}""" not in input_dir
			    else input_dir
			)
			logger.info(F"""Loading model from {ckpt_dir}"""					)
			A_		:						List[str]	    =   {"""model""": model.state_dict()}
			dist_cp.load_state_dict(
			    state_dict=snake_case__   ,   storage_reader=dist_cp.FileSystemReader(snake_case__					)   ,   planner=DefaultLoadPlanner()   ,   )
			A_		:						Tuple	    =   state_dict["""model"""]
			logger.info(F"""Model loaded from {ckpt_dir}"""					)
		model.load_state_dict(snake_case__					)






def 			__UpperCamelCase	(		snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__=0					):
	os.makedirs(snake_case__   ,   exist_ok=snake_case__					)
	with FSDP.state_dict_type(
	    snake_case__   ,   fsdp_plugin.state_dict_type   ,   fsdp_plugin.state_dict_config   ,   fsdp_plugin.optim_state_dict_config					):
		A_		:						Optional[Any]	    =   FSDP.optim_state_dict(snake_case__   ,   snake_case__					)
		if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
			if accelerator.process_index == 0:
				A_		:						Any	    =   (
				    F"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else F"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
				)
				A_		:						Optional[int]	    =   os.path.join(snake_case__   ,   snake_case__					)
				logger.info(F"""Saving Optimizer state to {output_optimizer_file}"""					)
				torch.save(snake_case__   ,   snake_case__					)
				logger.info(F"""Optimizer state saved in {output_optimizer_file}"""					)
		else:
			A_		:						Union[str, Any]	    =   os.path.join(snake_case__   ,   F"""{OPTIMIZER_NAME}_{optimizer_index}"""					)
			os.makedirs(snake_case__   ,   exist_ok=snake_case__					)
			logger.info(F"""Saving Optimizer state to {ckpt_dir}"""					)
			dist_cp.save_state_dict(
			    state_dict={"""optimizer""": optim_state}   ,   storage_writer=dist_cp.FileSystemWriter(snake_case__					)   ,   planner=DefaultSavePlanner()   ,   )
			logger.info(F"""Optimizer state saved in {ckpt_dir}"""					)






def 			__UpperCamelCase	(		snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__   ,   snake_case__=0					):
	accelerator.wait_for_everyone()
	with FSDP.state_dict_type(
	    snake_case__   ,   fsdp_plugin.state_dict_type   ,   fsdp_plugin.state_dict_config   ,   fsdp_plugin.optim_state_dict_config					):
		if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
			A_		:						List[str]	    =   None
			# below check should work but currently it isn't working (mostly opytorch issue),
			# in the meantime disabling it at the cost of excess memory usage
			# if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
			A_		:						Tuple	    =   (
			    F"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else F"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
			)
			A_		:						int	    =   os.path.join(snake_case__   ,   snake_case__					)
			logger.info(F"""Loading Optimizer state from {input_optimizer_file}"""					)
			A_		:						Tuple	    =   torch.load(snake_case__					)
			logger.info(F"""Optimizer state loaded from {input_optimizer_file}"""					)
		else:
			A_		:						str	    =   (
			    os.path.join(snake_case__   ,   F"""{OPTIMIZER_NAME}_{optimizer_index}"""					)
			    if F"""{OPTIMIZER_NAME}""" not in input_dir
			    else input_dir
			)
			logger.info(F"""Loading Optimizer from {ckpt_dir}"""					)
			A_		:						Union[str, Any]	    =   load_sharded_optimizer_state_dict(
			    model_state_dict=model.state_dict()   ,   optimizer_key="""optimizer"""   ,   storage_reader=dist_cp.FileSystemReader(snake_case__					)   ,   )
			A_		:						Tuple	    =   optim_state["""optimizer"""]
			logger.info(F"""Optimizer loaded from {ckpt_dir}"""					)
		A_		:						List[Any]	    =   FSDP.optim_state_dict_to_load(snake_case__   ,   snake_case__   ,   snake_case__					)
		optimizer.load_state_dict(snake_case__					)

