
'''simple docstring'''






import inspect
import os

import torch
from transformers import AutoModel
from transformers.testing_utils import mockenv_context
from transformers.trainer_utils import set_seed

import accelerate
from accelerate.accelerator import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils.testing import (
    AccelerateTestCase,
    TempDirTestCase,
    execute_subprocess_async,
    require_cuda,
    require_fsdp,
    require_multi_gpu,
    slow,
)
from accelerate.utils.constants import (
    FSDP_AUTO_WRAP_POLICY,
    FSDP_BACKWARD_PREFETCH,
    FSDP_SHARDING_STRATEGY,
    FSDP_STATE_DICT_TYPE,
)
from accelerate.utils.dataclasses import FullyShardedDataParallelPlugin
from accelerate.utils.other import patch_environment


set_seed(42)

A			      =      'bert-base-cased'
A			      =      'fp16'
A			      =      'bf16'
A			      =      [FPaa, BFaa]




@require_fsdp
@require_cuda
class     __snake_case		(   a__):
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									super().setUp()

									lowerCamelCase					:			Optional[Any]									= dict(
									    ACCELERATE_USE_FSDP='true',    MASTER_ADDR='localhost',    MASTER_PORT='10999',    RANK='0',    LOCAL_RANK='0',    WORLD_SIZE='1',    )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy

									for i, strategy in enumerate(A      ):
													lowerCamelCase					:			int									= self.dist_env.copy()
													lowerCamelCase					:			str									= F'''{i + 1}'''
													lowerCamelCase					:			Optional[int]									= strategy
													with mockenv_context(**A      ):
																	lowerCamelCase					:			Any									= FullyShardedDataParallelPlugin()
																	self.assertEqual(fsdp_plugin.sharding_strategy,    ShardingStrategy(i + 1      )      )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									from torch.distributed.fsdp.fully_sharded_data_parallel import BackwardPrefetch

									for i, prefetch_policy in enumerate(A      ):
													lowerCamelCase					:			Tuple									= self.dist_env.copy()
													lowerCamelCase					:			Union[str, Any]									= prefetch_policy
													with mockenv_context(**A      ):
																	lowerCamelCase					:			List[Any]									= FullyShardedDataParallelPlugin()
																	if prefetch_policy == "NO_PREFETCH":
																					self.assertIsNone(fsdp_plugin.backward_prefetch      )
																	else:
																					self.assertEqual(fsdp_plugin.backward_prefetch,    BackwardPrefetch(i + 1      )      )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType

									for i, state_dict_type in enumerate(A      ):
													lowerCamelCase					:			Dict									= self.dist_env.copy()
													lowerCamelCase					:			Optional[int]									= state_dict_type
													with mockenv_context(**A      ):
																	lowerCamelCase					:			Any									= FullyShardedDataParallelPlugin()
																	self.assertEqual(fsdp_plugin.state_dict_type,    StateDictType(i + 1      )      )
																	if state_dict_type == "FULL_STATE_DICT":
																					self.assertTrue(fsdp_plugin.state_dict_config.offload_to_cpu      )
																					self.assertTrue(fsdp_plugin.state_dict_config.ranka_only      )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			Union[str, Any]									= AutoModel.from_pretrained(A      )
									for policy in FSDP_AUTO_WRAP_POLICY:
													lowerCamelCase					:			str									= self.dist_env.copy()
													lowerCamelCase					:			List[str]									= policy
													if policy == "TRANSFORMER_BASED_WRAP":
																	lowerCamelCase					:			Optional[int]									= 'BertLayer'
													elif policy == "SIZE_BASED_WRAP":
																	lowerCamelCase					:			Tuple									= '2000'
													with mockenv_context(**A      ):
																	lowerCamelCase					:			Any									= FullyShardedDataParallelPlugin()
																	fsdp_plugin.set_auto_wrap_policy(A      )
																	if policy == "NO_WRAP":
																					self.assertIsNone(fsdp_plugin.auto_wrap_policy      )
																	else:
																					self.assertIsNotNone(fsdp_plugin.auto_wrap_policy      )

									lowerCamelCase					:			Optional[Any]									= self.dist_env.copy()
									lowerCamelCase					:			Dict									= 'TRANSFORMER_BASED_WRAP'
									lowerCamelCase					:			int									= 'T5Layer'
									with mockenv_context(**A      ):
													lowerCamelCase					:			Union[str, Any]									= FullyShardedDataParallelPlugin()
													with self.assertRaises(A      ) as cm:
																	fsdp_plugin.set_auto_wrap_policy(A      )
													self.assertTrue('Could not find the transformer layer class to wrap in the model.' in str(cm.exception      )      )

									lowerCamelCase					:			Any									= self.dist_env.copy()
									lowerCamelCase					:			Any									= 'SIZE_BASED_WRAP'
									lowerCamelCase					:			Union[str, Any]									= '0'
									with mockenv_context(**A      ):
													lowerCamelCase					:			Optional[Any]									= FullyShardedDataParallelPlugin()
													fsdp_plugin.set_auto_wrap_policy(A      )
													self.assertIsNone(fsdp_plugin.auto_wrap_policy      )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
									from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler

									for mp_dtype in dtypes:
													lowerCamelCase					:			Optional[int]									= self.dist_env.copy()
													lowerCamelCase					:			Optional[int]									= mp_dtype
													with mockenv_context(**A      ):
																	lowerCamelCase					:			int									= Accelerator()
																	if mp_dtype == "fp16":
																					lowerCamelCase					:			List[Any]									= torch.floataa
																	elif mp_dtype == "bf16":
																					lowerCamelCase					:			Optional[int]									= torch.bfloataa
																	lowerCamelCase					:			str									= MixedPrecision(param_dtype=A,    reduce_dtype=A,    buffer_dtype=A      )
																	self.assertEqual(accelerator.state.fsdp_plugin.mixed_precision_policy,    A      )
																	if mp_dtype == FPaa:
																					self.assertTrue(isinstance(accelerator.scaler,    A      )      )
																	elif mp_dtype == BFaa:
																					self.assertIsNone(accelerator.scaler      )
																	AcceleratorState._reset_state(A      )




					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload

									for flag in [True, False]:
													lowerCamelCase					:			str									= self.dist_env.copy()
													lowerCamelCase					:			Dict									= str(A      ).lower()
													with mockenv_context(**A      ):
																	lowerCamelCase					:			Optional[Any]									= FullyShardedDataParallelPlugin()
																	self.assertEqual(fsdp_plugin.cpu_offload,    CPUOffload(offload_params=A      )      )





@require_fsdp
@require_multi_gpu
@slow
class     __snake_case		(   a__):
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									super().setUp()
									lowerCamelCase					:			List[str]									= 0.82
									lowerCamelCase					:			List[Any]									= [
									    'fsdp_shard_grad_op_transformer_based_wrap',
									    'fsdp_full_shard_transformer_based_wrap',
									]
									lowerCamelCase					:			Tuple									= {
									    'multi_gpu_fp16': 3200,
									    'fsdp_shard_grad_op_transformer_based_wrap_fp16': 2000,
									    'fsdp_full_shard_transformer_based_wrap_fp16': 1900,
									    # Disabling below test as it overwhelms the RAM memory usage
									    # on CI self-hosted runner leading to tests getting killed.
									    # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
									}
									lowerCamelCase					:			str									= 160
									lowerCamelCase					:			List[str]									= 160

									lowerCamelCase					:			Dict									= inspect.getfile(accelerate.test_utils      )
									lowerCamelCase					:			Union[str, Any]									= os.path.sep.join(mod_file.split(os.path.sep      )[:-1] + ['scripts', 'external_deps']      )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			Union[str, Any]									= os.path.join(self.test_scripts_folder,    'test_performance.py'      )
									lowerCamelCase					:			str									= ['accelerate', 'launch', '--num_processes=2', '--num_machines=1', '--machine_rank=0', '--use_fsdp']
									for config in self.performance_configs:
													lowerCamelCase					:			Dict									= cmd.copy()
													for i, strategy in enumerate(A      ):
																	if strategy.lower() in config:
																					cmd_config.append(F'''--fsdp_sharding_strategy={i+1}'''      )
																					break

													if "fp32" in config:
																	cmd_config.append('--mixed_precision=no'      )
													else:
																	cmd_config.append('--mixed_precision=fp16'      )

													if "cpu_offload" in config:
																	cmd_config.append('--fsdp_offload_params=True'      )

													for policy in FSDP_AUTO_WRAP_POLICY:
																	if policy.lower() in config:
																					cmd_config.append(F'''--fsdp_auto_wrap_policy={policy}'''      )
																					break

													if policy == "TRANSFORMER_BASED_WRAP":
																	cmd_config.append('--fsdp_transformer_layer_cls_to_wrap=BertLayer'      )
													elif policy == "SIZE_BASED_WRAP":
																	cmd_config.append('--fsdp_min_num_params=2000'      )

													cmd_config.extend(
													    [
													        self.test_file_path,
													        F'''--output_dir={self.tmpdir}''',
													        F'''--performance_lower_bound={self.performance_lower_bound}''',
													    ]      )
													with patch_environment(omp_num_threads=1      ):
																	execute_subprocess_async(A,    env=os.environ.copy()      )
					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			Optional[int]									= os.path.join(self.test_scripts_folder,    'test_checkpointing.py'      )
									lowerCamelCase					:			Tuple									= [
									    'accelerate',
									    'launch',
									    '--num_processes=2',
									    '--num_machines=1',
									    '--machine_rank=0',
									    '--use_fsdp',
									    '--mixed_precision=fp16',
									    '--fsdp_transformer_layer_cls_to_wrap=BertLayer',
									]

									for i, strategy in enumerate(A      ):
													lowerCamelCase					:			Any									= cmd.copy()
													cmd_config.append(F'''--fsdp_sharding_strategy={i+1}'''      )
													if strategy != "FULL_SHARD":
																	continue
													lowerCamelCase					:			List[str]									= len(A      )
													for state_dict_type in FSDP_STATE_DICT_TYPE:
																	lowerCamelCase					:			Union[str, Any]									= cmd_config[:state_dict_config_index]
																	cmd_config.append(F'''--fsdp_state_dict_type={state_dict_type}'''      )
																	cmd_config.extend(
																	    [
																	        self.test_file_path,
																	        F'''--output_dir={self.tmpdir}''',
																	        '--partial_train_epoch=1',
																	    ]      )
																	with patch_environment(omp_num_threads=1      ):
																					execute_subprocess_async(A,    env=os.environ.copy()      )

																	lowerCamelCase					:			Any									= cmd_config[:-1]
																	lowerCamelCase					:			Any									= os.path.join(self.tmpdir,    'epoch_0'      )
																	cmd_config.extend(
																	    [
																	        F'''--resume_from_checkpoint={resume_from_checkpoint}''',
																	    ]      )
																	with patch_environment(omp_num_threads=1      ):
																					execute_subprocess_async(A,    env=os.environ.copy()      )




					def     UpperCAmelCase_							(     self      ):

									"""simple docstring"""


									lowerCamelCase					:			int									= os.path.join(self.test_scripts_folder,    'test_peak_memory_usage.py'      )
									lowerCamelCase					:			List[Any]									= [
									    'accelerate',
									    'launch',
									    '--num_processes=2',
									    '--num_machines=1',
									    '--machine_rank=0',
									]
									for spec, peak_mem_upper_bound in self.peak_memory_usage_upper_bound.items():
													lowerCamelCase					:			int									= cmd.copy()
													if "fp16" in spec:
																	cmd_config.extend(['--mixed_precision=fp16']      )
													else:
																	cmd_config.extend(['--mixed_precision=no']      )

													if "multi_gpu" in spec:
																	continue
													else:
																	cmd_config.extend(['--use_fsdp']      )
																	for i, strategy in enumerate(A      ):
																					if strategy.lower() in spec:
																									cmd_config.append(F'''--fsdp_sharding_strategy={i+1}'''      )
																									break

																	if "cpu_offload" in spec:
																					cmd_config.append('--fsdp_offload_params=True'      )

																	for policy in FSDP_AUTO_WRAP_POLICY:
																					if policy.lower() in spec:
																									cmd_config.append(F'''--fsdp_auto_wrap_policy={policy}'''      )
																									break

																	if policy == "TRANSFORMER_BASED_WRAP":
																					cmd_config.append('--fsdp_transformer_layer_cls_to_wrap=BertLayer'      )
																	elif policy == "SIZE_BASED_WRAP":
																					cmd_config.append('--fsdp_min_num_params=2000'      )

													cmd_config.extend(
													    [
													        self.test_file_path,
													        F'''--output_dir={self.tmpdir}''',
													        F'''--peak_memory_upper_bound={peak_mem_upper_bound}''',
													        F'''--n_train={self.n_train}''',
													        F'''--n_val={self.n_val}''',
													    ]      )
													with patch_environment(omp_num_threads=1      ):
																	execute_subprocess_async(A,    env=os.environ.copy()      )


'''simple docstring'''






def    UpperCAmelCase							(  UpperCAmelCase__							:							int = 10_00):
				lowerCamelCase					:			Optional[Any]									= 2**power
				lowerCamelCase					:			str									= str(UpperCAmelCase__)
				lowerCamelCase					:			Union[str, Any]									= list(UpperCAmelCase__)
				lowerCamelCase					:			Optional[Any]									= 0

				for i in list_num:
								sum_of_num += int(UpperCAmelCase__)

				return sum_of_num


if __name__ == "__main__":
					A			      =      int(input('Enter the power of 2: ').strip())
					print('2 ^ ', power, ' = ', 2**power)
					A			      =      solution(power)
					print('Sum of the digits is: ', result)
