
import inspect
import os

import torch
from transformers import AutoModel
from transformers.testing_utils import mockenv_context
from transformers.trainer_utils import set_seed

import accelerate
from accelerate.accelerator import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils.testing import (
    AccelerateTestCase,
    TempDirTestCase,
    execute_subprocess_async,
    require_cuda,
    require_fsdp,
    require_multi_gpu,
    slow,
)
from accelerate.utils.constants import (
    FSDP_AUTO_WRAP_POLICY,
    FSDP_BACKWARD_PREFETCH,
    FSDP_SHARDING_STRATEGY,
    FSDP_STATE_DICT_TYPE,
)
from accelerate.utils.dataclasses import FullyShardedDataParallelPlugin
from accelerate.utils.other import patch_environment


set_seed(4_2)

lowerCamelCase      ="bert-base-cased"
lowerCamelCase      ="fp16"
lowerCamelCase      ="bf16"
lowerCamelCase      =[FPaa, BFaa]




@require_fsdp
@require_cuda
class 				_lowerCamelCase (      UpperCamelCase_ ):





     """simple docstring"""







     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Any:




           """simple docstring"""







           super().setUp()

           UpperCamelCase__    :   List[str]					=				dict(
               ACCELERATE_USE_FSDP='''true'''						,						MASTER_ADDR='''localhost'''						,						MASTER_PORT='''10999'''						,						RANK='''0'''						,						LOCAL_RANK='''0'''						,						WORLD_SIZE='''1'''						,						)






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   List[str]:




           """simple docstring"""







           from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy

           for i, strategy in enumerate(__SCREAMING_SNAKE_CASE  ):
                 UpperCamelCase__    :   Optional[int]					=				self.dist_env.copy()
                 UpperCamelCase__    :   List[str]					=				F'''{i + 1}'''
                 UpperCamelCase__    :   Any					=				strategy
                 with mockenv_context(**__SCREAMING_SNAKE_CASE  ):
                       UpperCamelCase__    :   List[str]					=				FullyShardedDataParallelPlugin()
                       self.assertEqual(fsdp_plugin.sharding_strategy						,						ShardingStrategy(i + 1  )  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Tuple:




           """simple docstring"""







           from torch.distributed.fsdp.fully_sharded_data_parallel import BackwardPrefetch

           for i, prefetch_policy in enumerate(__SCREAMING_SNAKE_CASE  ):
                 UpperCamelCase__    :   Any					=				self.dist_env.copy()
                 UpperCamelCase__    :   List[str]					=				prefetch_policy
                 with mockenv_context(**__SCREAMING_SNAKE_CASE  ):
                       UpperCamelCase__    :   int					=				FullyShardedDataParallelPlugin()
                       if prefetch_policy == "NO_PREFETCH":
                             self.assertIsNone(fsdp_plugin.backward_prefetch  )
                       else:
                             self.assertEqual(fsdp_plugin.backward_prefetch						,						BackwardPrefetch(i + 1  )  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   str:




           """simple docstring"""







           from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType

           for i, state_dict_type in enumerate(__SCREAMING_SNAKE_CASE  ):
                 UpperCamelCase__    :   str					=				self.dist_env.copy()
                 UpperCamelCase__    :   int					=				state_dict_type
                 with mockenv_context(**__SCREAMING_SNAKE_CASE  ):
                       UpperCamelCase__    :   Union[str, Any]					=				FullyShardedDataParallelPlugin()
                       self.assertEqual(fsdp_plugin.state_dict_type						,						StateDictType(i + 1  )  )
                       if state_dict_type == "FULL_STATE_DICT":
                             self.assertTrue(fsdp_plugin.state_dict_config.offload_to_cpu  )
                             self.assertTrue(fsdp_plugin.state_dict_config.ranka_only  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Dict:




           """simple docstring"""







           UpperCamelCase__    :   Optional[Any]					=				AutoModel.from_pretrained(__SCREAMING_SNAKE_CASE  )
           for policy in FSDP_AUTO_WRAP_POLICY:
                 UpperCamelCase__    :   Tuple					=				self.dist_env.copy()
                 UpperCamelCase__    :   List[Any]					=				policy
                 if policy == "TRANSFORMER_BASED_WRAP":
                       UpperCamelCase__    :   Optional[int]					=				'''BertLayer'''
                 elif policy == "SIZE_BASED_WRAP":
                       UpperCamelCase__    :   Optional[Any]					=				'''2000'''
                 with mockenv_context(**__SCREAMING_SNAKE_CASE  ):
                       UpperCamelCase__    :   Tuple					=				FullyShardedDataParallelPlugin()
                       fsdp_plugin.set_auto_wrap_policy(__SCREAMING_SNAKE_CASE  )
                       if policy == "NO_WRAP":
                             self.assertIsNone(fsdp_plugin.auto_wrap_policy  )
                       else:
                             self.assertIsNotNone(fsdp_plugin.auto_wrap_policy  )

           UpperCamelCase__    :   Tuple					=				self.dist_env.copy()
           UpperCamelCase__    :   List[Any]					=				'''TRANSFORMER_BASED_WRAP'''
           UpperCamelCase__    :   Any					=				'''T5Layer'''
           with mockenv_context(**__SCREAMING_SNAKE_CASE  ):
                 UpperCamelCase__    :   Optional[Any]					=				FullyShardedDataParallelPlugin()
                 with self.assertRaises(__SCREAMING_SNAKE_CASE  ) as cm:
                       fsdp_plugin.set_auto_wrap_policy(__SCREAMING_SNAKE_CASE  )
                 self.assertTrue('''Could not find the transformer layer class to wrap in the model.''' in str(cm.exception  )  )

           UpperCamelCase__    :   str					=				self.dist_env.copy()
           UpperCamelCase__    :   Optional[int]					=				'''SIZE_BASED_WRAP'''
           UpperCamelCase__    :   Optional[int]					=				'''0'''
           with mockenv_context(**__SCREAMING_SNAKE_CASE  ):
                 UpperCamelCase__    :   Any					=				FullyShardedDataParallelPlugin()
                 fsdp_plugin.set_auto_wrap_policy(__SCREAMING_SNAKE_CASE  )
                 self.assertIsNone(fsdp_plugin.auto_wrap_policy  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Tuple:




           """simple docstring"""







           from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
           from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler

           for mp_dtype in dtypes:
                 UpperCamelCase__    :   Optional[int]					=				self.dist_env.copy()
                 UpperCamelCase__    :   Tuple					=				mp_dtype
                 with mockenv_context(**__SCREAMING_SNAKE_CASE  ):
                       UpperCamelCase__    :   Optional[Any]					=				Accelerator()
                       if mp_dtype == "fp16":
                             UpperCamelCase__    :   List[Any]					=				torch.floataa
                       elif mp_dtype == "bf16":
                             UpperCamelCase__    :   str					=				torch.bfloataa
                       UpperCamelCase__    :   Optional[int]					=				MixedPrecision(param_dtype=__SCREAMING_SNAKE_CASE						,						reduce_dtype=__SCREAMING_SNAKE_CASE						,						buffer_dtype=__SCREAMING_SNAKE_CASE  )
                       self.assertEqual(accelerator.state.fsdp_plugin.mixed_precision_policy						,						__SCREAMING_SNAKE_CASE  )
                       if mp_dtype == FPaa:
                             self.assertTrue(isinstance(accelerator.scaler						,						__SCREAMING_SNAKE_CASE  )  )
                       elif mp_dtype == BFaa:
                             self.assertIsNone(accelerator.scaler  )
                       AcceleratorState._reset_state(__SCREAMING_SNAKE_CASE  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Union[str, Any]:




           """simple docstring"""







           from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload

           for flag in [True, False]:
                 UpperCamelCase__    :   Optional[int]					=				self.dist_env.copy()
                 UpperCamelCase__    :   Dict					=				str(__SCREAMING_SNAKE_CASE  ).lower()
                 with mockenv_context(**__SCREAMING_SNAKE_CASE  ):
                       UpperCamelCase__    :   Union[str, Any]					=				FullyShardedDataParallelPlugin()
                       self.assertEqual(fsdp_plugin.cpu_offload						,						CPUOffload(offload_params=__SCREAMING_SNAKE_CASE  )  )




@require_fsdp
@require_multi_gpu
@slow
class 				_lowerCamelCase (      UpperCamelCase_ ):





     """simple docstring"""







     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Tuple:




           """simple docstring"""







           super().setUp()
           UpperCamelCase__    :   Tuple					=				0.82
           UpperCamelCase__    :   Optional[Any]					=				[
               '''fsdp_shard_grad_op_transformer_based_wrap''',
               '''fsdp_full_shard_transformer_based_wrap''',
           ]
           UpperCamelCase__    :   Tuple					=				{
               '''multi_gpu_fp16''': 3_2_0_0,
               '''fsdp_shard_grad_op_transformer_based_wrap_fp16''': 2_0_0_0,
               '''fsdp_full_shard_transformer_based_wrap_fp16''': 1_9_0_0,
               # Disabling below test as it overwhelms the RAM memory usage
               # on CI self-hosted runner leading to tests getting killed.
               # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
           }
           UpperCamelCase__    :   Optional[Any]					=				1_6_0
           UpperCamelCase__    :   Optional[int]					=				1_6_0

           UpperCamelCase__    :   Union[str, Any]					=				inspect.getfile(accelerate.test_utils  )
           UpperCamelCase__    :   List[str]					=				os.path.sep.join(mod_file.split(os.path.sep  )[:-1] + ['''scripts''', '''external_deps''']  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   str:




           """simple docstring"""







           UpperCamelCase__    :   Optional[Any]					=				os.path.join(self.test_scripts_folder						,						'''test_performance.py'''  )
           UpperCamelCase__    :   int					=				['''accelerate''', '''launch''', '''--num_processes=2''', '''--num_machines=1''', '''--machine_rank=0''', '''--use_fsdp''']
           for config in self.performance_configs:
                 UpperCamelCase__    :   Optional[int]					=				cmd.copy()
                 for i, strategy in enumerate(__SCREAMING_SNAKE_CASE  ):
                       if strategy.lower() in config:
                             cmd_config.append(F'''--fsdp_sharding_strategy={i+1}'''  )
                             break

                 if "fp32" in config:
                       cmd_config.append('''--mixed_precision=no'''  )
                 else:
                       cmd_config.append('''--mixed_precision=fp16'''  )

                 if "cpu_offload" in config:
                       cmd_config.append('''--fsdp_offload_params=True'''  )

                 for policy in FSDP_AUTO_WRAP_POLICY:
                       if policy.lower() in config:
                             cmd_config.append(F'''--fsdp_auto_wrap_policy={policy}'''  )
                             break

                 if policy == "TRANSFORMER_BASED_WRAP":
                       cmd_config.append('''--fsdp_transformer_layer_cls_to_wrap=BertLayer'''  )
                 elif policy == "SIZE_BASED_WRAP":
                       cmd_config.append('''--fsdp_min_num_params=2000'''  )

                 cmd_config.extend(
                     [
                         self.test_file_path,
                         F'''--output_dir={self.tmpdir}''',
                         F'''--performance_lower_bound={self.performance_lower_bound}''',
                     ]  )
                 with patch_environment(omp_num_threads=1  ):
                       execute_subprocess_async(__SCREAMING_SNAKE_CASE						,						env=os.environ.copy()  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   Any:




           """simple docstring"""







           UpperCamelCase__    :   List[Any]					=				os.path.join(self.test_scripts_folder						,						'''test_checkpointing.py'''  )
           UpperCamelCase__    :   Dict					=				[
               '''accelerate''',
               '''launch''',
               '''--num_processes=2''',
               '''--num_machines=1''',
               '''--machine_rank=0''',
               '''--use_fsdp''',
               '''--mixed_precision=fp16''',
               '''--fsdp_transformer_layer_cls_to_wrap=BertLayer''',
           ]

           for i, strategy in enumerate(__SCREAMING_SNAKE_CASE  ):
                 UpperCamelCase__    :   List[str]					=				cmd.copy()
                 cmd_config.append(F'''--fsdp_sharding_strategy={i+1}'''  )
                 if strategy != "FULL_SHARD":
                       continue
                 UpperCamelCase__    :   int					=				len(__SCREAMING_SNAKE_CASE  )
                 for state_dict_type in FSDP_STATE_DICT_TYPE:
                       UpperCamelCase__    :   str					=				cmd_config[:state_dict_config_index]
                       cmd_config.append(F'''--fsdp_state_dict_type={state_dict_type}'''  )
                       cmd_config.extend(
                           [
                               self.test_file_path,
                               F'''--output_dir={self.tmpdir}''',
                               '''--partial_train_epoch=1''',
                           ]  )
                       with patch_environment(omp_num_threads=1  ):
                             execute_subprocess_async(__SCREAMING_SNAKE_CASE						,						env=os.environ.copy()  )

                       UpperCamelCase__    :   Any					=				cmd_config[:-1]
                       UpperCamelCase__    :   int					=				os.path.join(self.tmpdir						,						'''epoch_0'''  )
                       cmd_config.extend(
                           [
                               F'''--resume_from_checkpoint={resume_from_checkpoint}''',
                           ]  )
                       with patch_environment(omp_num_threads=1  ):
                             execute_subprocess_async(__SCREAMING_SNAKE_CASE						,						env=os.environ.copy()  )






     def    __SCREAMING_SNAKE_CASE			(							self  )  ->   List[Any]:




           """simple docstring"""







           UpperCamelCase__    :   Optional[Any]					=				os.path.join(self.test_scripts_folder						,						'''test_peak_memory_usage.py'''  )
           UpperCamelCase__    :   Any					=				[
               '''accelerate''',
               '''launch''',
               '''--num_processes=2''',
               '''--num_machines=1''',
               '''--machine_rank=0''',
           ]
           for spec, peak_mem_upper_bound in self.peak_memory_usage_upper_bound.items():
                 UpperCamelCase__    :   List[str]					=				cmd.copy()
                 if "fp16" in spec:
                       cmd_config.extend(['''--mixed_precision=fp16''']  )
                 else:
                       cmd_config.extend(['''--mixed_precision=no''']  )

                 if "multi_gpu" in spec:
                       continue
                 else:
                       cmd_config.extend(['''--use_fsdp''']  )
                       for i, strategy in enumerate(__SCREAMING_SNAKE_CASE  ):
                             if strategy.lower() in spec:
                                   cmd_config.append(F'''--fsdp_sharding_strategy={i+1}'''  )
                                   break

                       if "cpu_offload" in spec:
                             cmd_config.append('''--fsdp_offload_params=True'''  )

                       for policy in FSDP_AUTO_WRAP_POLICY:
                             if policy.lower() in spec:
                                   cmd_config.append(F'''--fsdp_auto_wrap_policy={policy}'''  )
                                   break

                       if policy == "TRANSFORMER_BASED_WRAP":
                             cmd_config.append('''--fsdp_transformer_layer_cls_to_wrap=BertLayer'''  )
                       elif policy == "SIZE_BASED_WRAP":
                             cmd_config.append('''--fsdp_min_num_params=2000'''  )

                 cmd_config.extend(
                     [
                         self.test_file_path,
                         F'''--output_dir={self.tmpdir}''',
                         F'''--peak_memory_upper_bound={peak_mem_upper_bound}''',
                         F'''--n_train={self.n_train}''',
                         F'''--n_val={self.n_val}''',
                     ]  )
                 with patch_environment(omp_num_threads=1  ):
                       execute_subprocess_async(__SCREAMING_SNAKE_CASE						,						env=os.environ.copy()  )








from __future__ import annotations

from collections.abc import Callable


def 							SCREAMING_SNAKE_CASE_      (			UpperCamelCase__ ,		UpperCamelCase__ ,		UpperCamelCase__ ,		UpperCamelCase__ = 1_0_0 ,		):
      UpperCamelCase__    :   Union[str, Any]					=				x_start
      UpperCamelCase__    :   List[Any]					=				fnc(UpperCamelCase__		)
      UpperCamelCase__    :   Any					=				0.0
      for _ in range(UpperCamelCase__		):
            # Approximates small segments of curve as linear and solve
            # for trapezoidal area
            UpperCamelCase__    :   str					=				(x_end - x_start) / steps + xa
            UpperCamelCase__    :   Dict					=				fnc(UpperCamelCase__		)
            area += abs(fxa + fxa		) * (xa - xa) / 2
            # Increment step
            UpperCamelCase__    :   Tuple					=				xa
            UpperCamelCase__    :   Union[str, Any]					=				fxa
      return area


if __name__ == "__main__":


    def 							SCREAMING_SNAKE_CASE_      (			UpperCamelCase__		):
          return x**3 + x**2

    print("f(x) = x^3 + x^2")
    print("The area between the curve, x = -5, x = 5 and the x axis is:")
    lowerCamelCase      =1_0
    while i <= 1_0_0_0_0_0:
        print(F'''with {i} steps: {trapezoidal_area(f, -5, 5, i)}''')
        i *= 1_0


