



"""simple docstring"""





import inspect
import os

import torch
from transformers import AutoModel
from transformers.testing_utils import mockenv_context
from transformers.trainer_utils import set_seed

import accelerate
from accelerate.accelerator import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils.testing import (
    AccelerateTestCase,
    TempDirTestCase,
    execute_subprocess_async,
    require_cuda,
    require_fsdp,
    require_multi_gpu,
    slow,
)
from accelerate.utils.constants import (
    FSDP_AUTO_WRAP_POLICY,
    FSDP_BACKWARD_PREFETCH,
    FSDP_SHARDING_STRATEGY,
    FSDP_STATE_DICT_TYPE,
)
from accelerate.utils.dataclasses import FullyShardedDataParallelPlugin
from accelerate.utils.other import patch_environment


set_seed(42)

lowerCamelCase      =		"""bert-base-cased"""
lowerCamelCase      =		"""fp16"""
lowerCamelCase      =		"""bf16"""
lowerCamelCase      =		[FPaa, BFaa]
@require_fsdp
@require_cuda
class    lowercase__     (					SCREAMING_SNAKE_CASE   ):







       '''simple docstring'''


       def   lowercase__       (							self    :						Any					)				->		str:


             '''simple docstring'''




             super().setUp()

             UpperCAmelCase_    				=							dict(
                 ACCELERATE_USE_FSDP="true"	,       MASTER_ADDR="localhost"	,       MASTER_PORT="10999"	,       RANK="0"	,       LOCAL_RANK="0"	,       WORLD_SIZE="1"	,       )



       def   lowercase__       (							self    :						str					)				->		Optional[Any]:


             '''simple docstring'''




             from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy

             for i, strategy in enumerate(_UpperCAmelCase					):
                   UpperCAmelCase_    				=							self.dist_env.copy()
                   UpperCAmelCase_    				=							F"""{i + 1}"""
                   UpperCAmelCase_    				=							strategy
                   with mockenv_context(**_UpperCAmelCase					):
                         UpperCAmelCase_    				=							FullyShardedDataParallelPlugin()
                         self.assertEqual(fsdp_plugin.sharding_strategy	,       ShardingStrategy(i + 1					)					)



       def   lowercase__       (							self    :						List[str]					)				->		Dict:


             '''simple docstring'''




             from torch.distributed.fsdp.fully_sharded_data_parallel import BackwardPrefetch

             for i, prefetch_policy in enumerate(_UpperCAmelCase					):
                   UpperCAmelCase_    				=							self.dist_env.copy()
                   UpperCAmelCase_    				=							prefetch_policy
                   with mockenv_context(**_UpperCAmelCase					):
                         UpperCAmelCase_    				=							FullyShardedDataParallelPlugin()
                         if prefetch_policy == "NO_PREFETCH":
                               self.assertIsNone(fsdp_plugin.backward_prefetch					)
                         else:
                               self.assertEqual(fsdp_plugin.backward_prefetch	,       BackwardPrefetch(i + 1					)					)



       def   lowercase__       (							self    :						Optional[Any]					)				->		Union[str, Any]:


             '''simple docstring'''




             from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType

             for i, state_dict_type in enumerate(_UpperCAmelCase					):
                   UpperCAmelCase_    				=							self.dist_env.copy()
                   UpperCAmelCase_    				=							state_dict_type
                   with mockenv_context(**_UpperCAmelCase					):
                         UpperCAmelCase_    				=							FullyShardedDataParallelPlugin()
                         self.assertEqual(fsdp_plugin.state_dict_type	,       StateDictType(i + 1					)					)
                         if state_dict_type == "FULL_STATE_DICT":
                               self.assertTrue(fsdp_plugin.state_dict_config.offload_to_cpu					)
                               self.assertTrue(fsdp_plugin.state_dict_config.ranka_only					)



       def   lowercase__       (							self    :						List[Any]					)				->		str:


             '''simple docstring'''




             UpperCAmelCase_    				=							AutoModel.from_pretrained(_UpperCAmelCase					)
             for policy in FSDP_AUTO_WRAP_POLICY:
                   UpperCAmelCase_    				=							self.dist_env.copy()
                   UpperCAmelCase_    				=							policy
                   if policy == "TRANSFORMER_BASED_WRAP":
                         UpperCAmelCase_    				=							"BertLayer"
                   elif policy == "SIZE_BASED_WRAP":
                         UpperCAmelCase_    				=							"2000"
                   with mockenv_context(**_UpperCAmelCase					):
                         UpperCAmelCase_    				=							FullyShardedDataParallelPlugin()
                         fsdp_plugin.set_auto_wrap_policy(_UpperCAmelCase					)
                         if policy == "NO_WRAP":
                               self.assertIsNone(fsdp_plugin.auto_wrap_policy					)
                         else:
                               self.assertIsNotNone(fsdp_plugin.auto_wrap_policy					)

             UpperCAmelCase_    				=							self.dist_env.copy()
             UpperCAmelCase_    				=							"TRANSFORMER_BASED_WRAP"
             UpperCAmelCase_    				=							"T5Layer"
             with mockenv_context(**_UpperCAmelCase					):
                   UpperCAmelCase_    				=							FullyShardedDataParallelPlugin()
                   with self.assertRaises(_UpperCAmelCase					) as cm:
                         fsdp_plugin.set_auto_wrap_policy(_UpperCAmelCase					)
                   self.assertTrue("Could not find the transformer layer class to wrap in the model." in str(cm.exception					)					)

             UpperCAmelCase_    				=							self.dist_env.copy()
             UpperCAmelCase_    				=							"SIZE_BASED_WRAP"
             UpperCAmelCase_    				=							"0"
             with mockenv_context(**_UpperCAmelCase					):
                   UpperCAmelCase_    				=							FullyShardedDataParallelPlugin()
                   fsdp_plugin.set_auto_wrap_policy(_UpperCAmelCase					)
                   self.assertIsNone(fsdp_plugin.auto_wrap_policy					)



       def   lowercase__       (							self    :						List[Any]					)				->		List[Any]:


             '''simple docstring'''




             from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
             from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler

             for mp_dtype in dtypes:
                   UpperCAmelCase_    				=							self.dist_env.copy()
                   UpperCAmelCase_    				=							mp_dtype
                   with mockenv_context(**_UpperCAmelCase					):
                         UpperCAmelCase_    				=							Accelerator()
                         if mp_dtype == "fp16":
                               UpperCAmelCase_    				=							torch.floataa
                         elif mp_dtype == "bf16":
                               UpperCAmelCase_    				=							torch.bfloataa
                         UpperCAmelCase_    				=							MixedPrecision(param_dtype=_UpperCAmelCase	,       reduce_dtype=_UpperCAmelCase	,       buffer_dtype=_UpperCAmelCase					)
                         self.assertEqual(accelerator.state.fsdp_plugin.mixed_precision_policy	,       _UpperCAmelCase					)
                         if mp_dtype == FPaa:
                               self.assertTrue(isinstance(accelerator.scaler	,       _UpperCAmelCase					)					)
                         elif mp_dtype == BFaa:
                               self.assertIsNone(accelerator.scaler					)
                         AcceleratorState._reset_state(_UpperCAmelCase					)







       def   lowercase__       (							self    :						Optional[Any]					)				->		Union[str, Any]:


             '''simple docstring'''




             from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload

             for flag in [True, False]:
                   UpperCAmelCase_    				=							self.dist_env.copy()
                   UpperCAmelCase_    				=							str(_UpperCAmelCase					).lower()
                   with mockenv_context(**_UpperCAmelCase					):
                         UpperCAmelCase_    				=							FullyShardedDataParallelPlugin()
                         self.assertEqual(fsdp_plugin.cpu_offload	,       CPUOffload(offload_params=_UpperCAmelCase					)					)



@require_fsdp
@require_multi_gpu
@slow
class    lowercase__     (					SCREAMING_SNAKE_CASE   ):







       '''simple docstring'''


       def   lowercase__       (							self    :						List[Any]					)				->		Optional[int]:


             '''simple docstring'''




             super().setUp()
             UpperCAmelCase_    				=							0.82
             UpperCAmelCase_    				=							[
                 "fsdp_shard_grad_op_transformer_based_wrap",
                 "fsdp_full_shard_transformer_based_wrap",
             ]
             UpperCAmelCase_    				=							{
                 "multi_gpu_fp16": 3200,
                 "fsdp_shard_grad_op_transformer_based_wrap_fp16": 2000,
                 "fsdp_full_shard_transformer_based_wrap_fp16": 1900,
                 # Disabling below test as it overwhelms the RAM memory usage
                 # on CI self-hosted runner leading to tests getting killed.
                 # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
             }
             UpperCAmelCase_    				=							160
             UpperCAmelCase_    				=							160

             UpperCAmelCase_    				=							inspect.getfile(accelerate.test_utils					)
             UpperCAmelCase_    				=							os.path.sep.join(mod_file.split(os.path.sep					)[:-1] + ["scripts", "external_deps"]					)



       def   lowercase__       (							self    :						Tuple					)				->		Dict:


             '''simple docstring'''




             UpperCAmelCase_    				=							os.path.join(self.test_scripts_folder	,       "test_performance.py"					)
             UpperCAmelCase_    				=							["accelerate", "launch", "--num_processes=2", "--num_machines=1", "--machine_rank=0", "--use_fsdp"]
             for config in self.performance_configs:
                   UpperCAmelCase_    				=							cmd.copy()
                   for i, strategy in enumerate(_UpperCAmelCase					):
                         if strategy.lower() in config:
                               cmd_config.append(F"""--fsdp_sharding_strategy={i+1}"""					)
                               break

                   if "fp32" in config:
                         cmd_config.append("--mixed_precision=no"					)
                   else:
                         cmd_config.append("--mixed_precision=fp16"					)

                   if "cpu_offload" in config:
                         cmd_config.append("--fsdp_offload_params=True"					)

                   for policy in FSDP_AUTO_WRAP_POLICY:
                         if policy.lower() in config:
                               cmd_config.append(F"""--fsdp_auto_wrap_policy={policy}"""					)
                               break

                   if policy == "TRANSFORMER_BASED_WRAP":
                         cmd_config.append("--fsdp_transformer_layer_cls_to_wrap=BertLayer"					)
                   elif policy == "SIZE_BASED_WRAP":
                         cmd_config.append("--fsdp_min_num_params=2000"					)

                   cmd_config.extend(
                       [
                           self.test_file_path,
                           F"""--output_dir={self.tmpdir}""",
                           F"""--performance_lower_bound={self.performance_lower_bound}""",
                       ]					)
                   with patch_environment(omp_num_threads=1					):
                         execute_subprocess_async(_UpperCAmelCase	,       env=os.environ.copy()					)



       def   lowercase__       (							self    :						Any					)				->		List[str]:


             '''simple docstring'''




             UpperCAmelCase_    				=							os.path.join(self.test_scripts_folder	,       "test_checkpointing.py"					)
             UpperCAmelCase_    				=							[
                 "accelerate",
                 "launch",
                 "--num_processes=2",
                 "--num_machines=1",
                 "--machine_rank=0",
                 "--use_fsdp",
                 "--mixed_precision=fp16",
                 "--fsdp_transformer_layer_cls_to_wrap=BertLayer",
             ]

             for i, strategy in enumerate(_UpperCAmelCase					):
                   UpperCAmelCase_    				=							cmd.copy()
                   cmd_config.append(F"""--fsdp_sharding_strategy={i+1}"""					)
                   if strategy != "FULL_SHARD":
                         continue
                   UpperCAmelCase_    				=							len(_UpperCAmelCase					)
                   for state_dict_type in FSDP_STATE_DICT_TYPE:
                         UpperCAmelCase_    				=							cmd_config[:state_dict_config_index]
                         cmd_config.append(F"""--fsdp_state_dict_type={state_dict_type}"""					)
                         cmd_config.extend(
                             [
                                 self.test_file_path,
                                 F"""--output_dir={self.tmpdir}""",
                                 "--partial_train_epoch=1",
                             ]					)
                         with patch_environment(omp_num_threads=1					):
                               execute_subprocess_async(_UpperCAmelCase	,       env=os.environ.copy()					)

                         UpperCAmelCase_    				=							cmd_config[:-1]
                         UpperCAmelCase_    				=							os.path.join(self.tmpdir	,       "epoch_0"					)
                         cmd_config.extend(
                             [
                                 F"""--resume_from_checkpoint={resume_from_checkpoint}""",
                             ]					)
                         with patch_environment(omp_num_threads=1					):
                               execute_subprocess_async(_UpperCAmelCase	,       env=os.environ.copy()					)







       def   lowercase__       (							self    :						Dict					)				->		List[str]:


             '''simple docstring'''




             UpperCAmelCase_    				=							os.path.join(self.test_scripts_folder	,       "test_peak_memory_usage.py"					)
             UpperCAmelCase_    				=							[
                 "accelerate",
                 "launch",
                 "--num_processes=2",
                 "--num_machines=1",
                 "--machine_rank=0",
             ]
             for spec, peak_mem_upper_bound in self.peak_memory_usage_upper_bound.items():
                   UpperCAmelCase_    				=							cmd.copy()
                   if "fp16" in spec:
                         cmd_config.extend(["--mixed_precision=fp16"]					)
                   else:
                         cmd_config.extend(["--mixed_precision=no"]					)

                   if "multi_gpu" in spec:
                         continue
                   else:
                         cmd_config.extend(["--use_fsdp"]					)
                         for i, strategy in enumerate(_UpperCAmelCase					):
                               if strategy.lower() in spec:
                                     cmd_config.append(F"""--fsdp_sharding_strategy={i+1}"""					)
                                     break

                         if "cpu_offload" in spec:
                               cmd_config.append("--fsdp_offload_params=True"					)

                         for policy in FSDP_AUTO_WRAP_POLICY:
                               if policy.lower() in spec:
                                     cmd_config.append(F"""--fsdp_auto_wrap_policy={policy}"""					)
                                     break

                         if policy == "TRANSFORMER_BASED_WRAP":
                               cmd_config.append("--fsdp_transformer_layer_cls_to_wrap=BertLayer"					)
                         elif policy == "SIZE_BASED_WRAP":
                               cmd_config.append("--fsdp_min_num_params=2000"					)

                   cmd_config.extend(
                       [
                           self.test_file_path,
                           F"""--output_dir={self.tmpdir}""",
                           F"""--peak_memory_upper_bound={peak_mem_upper_bound}""",
                           F"""--n_train={self.n_train}""",
                           F"""--n_val={self.n_val}""",
                       ]					)
                   with patch_environment(omp_num_threads=1					):
                         execute_subprocess_async(_UpperCAmelCase	,       env=os.environ.copy()					)









"""simple docstring"""





from __future__ import annotations

from decimal import Decimal

from numpy import array
def      a__			(   lowerCAmelCase__   ):
      UpperCAmelCase_    				=							Decimal

      # Check if the provided matrix has 2 rows and 2 columns
      # since this implementation only works for 2x2 matrices
      if len(lowerCAmelCase__   ) == 2 and len(matrix[0]   ) == 2 and len(matrix[1]   ) == 2:
            # Calculate the determinant of the matrix
            UpperCAmelCase_    				=							float(
                d(matrix[0][0]   ) * d(matrix[1][1]   ) - d(matrix[1][0]   ) * d(matrix[0][1]   )   )
            if determinant == 0:
                  raise ValueError("This matrix has no inverse."   )

            # Creates a copy of the matrix with swapped positions of the elements
            UpperCAmelCase_    				=							[[0.0, 0.0], [0.0, 0.0]]
            UpperCAmelCase_				,					UpperCAmelCase_    				=							matrix[1][1], matrix[0][0]
            UpperCAmelCase_				,					UpperCAmelCase_    				=							-matrix[1][0], -matrix[0][1]

            # Calculate the inverse of the matrix
            return [
                [(float(d(lowerCAmelCase__   )   ) / determinant) or 0.0 for n in row] for row in swapped_matrix
            ]
      elif (
          len(lowerCAmelCase__   ) == 3
          and len(matrix[0]   ) == 3
          and len(matrix[1]   ) == 3
          and len(matrix[2]   ) == 3
      ):
            # Calculate the determinant of the matrix using Sarrus rule
            UpperCAmelCase_    				=							float(
                (
                    (d(matrix[0][0]   ) * d(matrix[1][1]   ) * d(matrix[2][2]   ))
                    + (d(matrix[0][1]   ) * d(matrix[1][2]   ) * d(matrix[2][0]   ))
                    + (d(matrix[0][2]   ) * d(matrix[1][0]   ) * d(matrix[2][1]   ))
                )
                - (
                    (d(matrix[0][2]   ) * d(matrix[1][1]   ) * d(matrix[2][0]   ))
                    + (d(matrix[0][1]   ) * d(matrix[1][0]   ) * d(matrix[2][2]   ))
                    + (d(matrix[0][0]   ) * d(matrix[1][2]   ) * d(matrix[2][1]   ))
                )   )
            if determinant == 0:
                  raise ValueError("This matrix has no inverse."   )

            # Creating cofactor matrix
            UpperCAmelCase_    				=							[
                [d(0.0   ), d(0.0   ), d(0.0   )],
                [d(0.0   ), d(0.0   ), d(0.0   )],
                [d(0.0   ), d(0.0   ), d(0.0   )],
            ]
            UpperCAmelCase_    				=							(d(matrix[1][1]   ) * d(matrix[2][2]   )) - (
                d(matrix[1][2]   ) * d(matrix[2][1]   )
            )
            UpperCAmelCase_    				=							-(
                (d(matrix[1][0]   ) * d(matrix[2][2]   )) - (d(matrix[1][2]   ) * d(matrix[2][0]   ))
            )
            UpperCAmelCase_    				=							(d(matrix[1][0]   ) * d(matrix[2][1]   )) - (
                d(matrix[1][1]   ) * d(matrix[2][0]   )
            )
            UpperCAmelCase_    				=							-(
                (d(matrix[0][1]   ) * d(matrix[2][2]   )) - (d(matrix[0][2]   ) * d(matrix[2][1]   ))
            )
            UpperCAmelCase_    				=							(d(matrix[0][0]   ) * d(matrix[2][2]   )) - (
                d(matrix[0][2]   ) * d(matrix[2][0]   )
            )
            UpperCAmelCase_    				=							-(
                (d(matrix[0][0]   ) * d(matrix[2][1]   )) - (d(matrix[0][1]   ) * d(matrix[2][0]   ))
            )
            UpperCAmelCase_    				=							(d(matrix[0][1]   ) * d(matrix[1][2]   )) - (
                d(matrix[0][2]   ) * d(matrix[1][1]   )
            )
            UpperCAmelCase_    				=							-(
                (d(matrix[0][0]   ) * d(matrix[1][2]   )) - (d(matrix[0][2]   ) * d(matrix[1][0]   ))
            )
            UpperCAmelCase_    				=							(d(matrix[0][0]   ) * d(matrix[1][1]   )) - (
                d(matrix[0][1]   ) * d(matrix[1][0]   )
            )

            # Transpose the cofactor matrix (Adjoint matrix)
            UpperCAmelCase_    				=							array(lowerCAmelCase__   )
            for i in range(3   ):
                  for j in range(3   ):
                        UpperCAmelCase_    				=							cofactor_matrix[j][i]

        # Inverse of the matrix using the formula (1/determinant) * adjoint matrix
            UpperCAmelCase_    				=							array(lowerCAmelCase__   )
            for i in range(3   ):
                  for j in range(3   ):
                        inverse_matrix[i][j] /= d(lowerCAmelCase__   )

        # Calculate the inverse of the matrix
            return [[float(d(lowerCAmelCase__   )   ) or 0.0 for n in row] for row in inverse_matrix]
      raise ValueError("Please provide a matrix of size 2x2 or 3x3."   )




