'''simple docstring'''

import importlib.metadata
import warnings
from copy import deepcopy

from packaging import version

from ..utils import logging
from .import_utils import is_accelerate_available, is_bitsandbytes_available


if is_bitsandbytes_available():
       import bitsandbytes as bnb
       import torch
       import torch.nn as nn

       from ..pytorch_utils import ConvaD

if is_accelerate_available():
       from accelerate import init_empty_weights
       from accelerate.utils import find_tied_parameters

lowerCAmelCase_ : Optional[int]						  =      logging.get_logger(__name__)


def __a (	__lowerCamelCase			:						int ,		__lowerCamelCase			:						Optional[int] ,		__lowerCamelCase			:						Optional[int] ,		__lowerCamelCase			:						Optional[int]=None ,		__lowerCamelCase			:						List[str]=None				)	->							Any:
      '''simple docstring'''


      if "." in tensor_name:
            lowercase_     =      tensor_name.split("."				)
            for split in splits[:-1]:
                  lowercase_     =      getattr(__lowerCamelCase ,		__lowerCamelCase				)
                  if new_module is None:
                        raise ValueError(f'{module} has no attribute {split}.'				)
                  lowercase_     =      new_module
            lowercase_     =      splits[-1]

      if tensor_name not in module._parameters and tensor_name not in module._buffers:
            raise ValueError(f'{module} does not have a parameter or a buffer named {tensor_name}.'				)
      lowercase_     =      tensor_name in module._buffers
      lowercase_     =      getattr(__lowerCamelCase ,		__lowerCamelCase				)

      if old_value.device == torch.device("meta"				) and device not in ["meta", torch.device("meta"				)] and value is None:
            raise ValueError(f'{tensor_name} is on the meta device, we need a `value` to put in on {device}.'				)

      lowercase_     =      False
      lowercase_     =      False
      if is_buffer or not is_bitsandbytes_available():
            lowercase_     =      False
            lowercase_     =      False
      else:
            lowercase_     =      hasattr(bnb.nn ,		"Params4bit"				) and isinstance(module._parameters[tensor_name] ,		bnb.nn.Paramsabit				)
            lowercase_     =      isinstance(module._parameters[tensor_name] ,		bnb.nn.IntaParams				)

      if is_abit or is_abit:
            lowercase_     =      module._parameters[tensor_name]
            if param.device.type != "cuda":
                  if value is None:
                        lowercase_     =      old_value.to(__lowerCamelCase				)
                  elif isinstance(__lowerCamelCase ,		torch.Tensor				):
                        lowercase_     =      value.to("cpu"				)
                        if value.dtype == torch.inta:
                              lowercase_     =      version.parse(importlib.metadata.version("bitsandbytes"				)				) > version.parse(
                                  "0.37.2"				)
                              if not is_abit_serializable:
                                    raise ValueError(
                                        "Detected int8 weights but the version of bitsandbytes is not compatible with int8 serialization. "
                                        "Make sure to download the latest `bitsandbytes` version. `pip install --upgrade bitsandbytes`."				)
                  else:
                        lowercase_     =      torch.tensor(__lowerCamelCase ,		device="cpu"				)

                  # Support models using `Conv1D` in place of `nn.Linear` (e.g. gpt2) by transposing the weight matrix prior to quantization.
                  # Since weights are saved in the correct "orientation", we skip transposing when loading.
                  if issubclass(module.source_cls ,		__lowerCamelCase				) and fpaa_statistics is None:
                        lowercase_     =      new_value.T

                  lowercase_     =      old_value.__dict__
                  if is_abit:
                        lowercase_     =      bnb.nn.IntaParams(__lowerCamelCase ,		requires_grad=__lowerCamelCase ,		**__lowerCamelCase				).to(__lowerCamelCase				)
                  elif is_abit:
                        lowercase_     =      bnb.nn.Paramsabit(__lowerCamelCase ,		requires_grad=__lowerCamelCase ,		**__lowerCamelCase				).to(__lowerCamelCase				)

                  lowercase_     =      new_value
                  if fpaa_statistics is not None:
                        setattr(module.weight ,		"SCB" ,		fpaa_statistics.to(__lowerCamelCase				)				)

      else:
            if value is None:
                  lowercase_     =      old_value.to(__lowerCamelCase				)
            elif isinstance(__lowerCamelCase ,		torch.Tensor				):
                  lowercase_     =      value.to(__lowerCamelCase				)
            else:
                  lowercase_     =      torch.tensor(__lowerCamelCase ,		device=__lowerCamelCase				)

            if is_buffer:
                  lowercase_     =      new_value
            else:
                  lowercase_     =      nn.Parameter(__lowerCamelCase ,		requires_grad=old_value.requires_grad				)
                  lowercase_     =      new_value


def __a (	__lowerCamelCase			:						int ,		__lowerCamelCase			:						List[Any]=None ,		__lowerCamelCase			:						Optional[Any]=None ,		__lowerCamelCase			:						Optional[Any]=None ,		__lowerCamelCase			:						List[str]=False				)	->							Tuple:
      '''simple docstring'''


      for name, module in model.named_children():
            if current_key_name is None:
                  lowercase_     =      []
            current_key_name.append(__lowerCamelCase				)

            if (isinstance(__lowerCamelCase ,		nn.Linear				) or isinstance(__lowerCamelCase ,		__lowerCamelCase				)) and name not in modules_to_not_convert:
                  # Check if the current key is not in the `modules_to_not_convert`
                  if not any(key in ".".join(__lowerCamelCase				) for key in modules_to_not_convert				):
                        with init_empty_weights():
                              if isinstance(__lowerCamelCase ,		__lowerCamelCase				):
                                    lowercase_ ,							lowercase_     =      module.weight.shape
                              else:
                                    lowercase_     =      module.in_features
                                    lowercase_     =      module.out_features

                              if quantization_config.quantization_method() == "llm_int8":
                                    lowercase_     =      bnb.nn.LinearabitLt(
                                        __lowerCamelCase ,		__lowerCamelCase ,		module.bias is not None ,		has_fpaa_weights=quantization_config.llm_inta_has_fpaa_weight ,		threshold=quantization_config.llm_inta_threshold ,		)
                                    lowercase_     =      True
                              else:
                                    if (
                                        quantization_config.llm_inta_skip_modules is not None
                                        and name in quantization_config.llm_inta_skip_modules
                                    ):
                                          pass
                                    else:
                                          lowercase_     =      bnb.nn.Linearabit(
                                              __lowerCamelCase ,		__lowerCamelCase ,		module.bias is not None ,		quantization_config.bnb_abit_compute_dtype ,		compress_statistics=quantization_config.bnb_abit_use_double_quant ,		quant_type=quantization_config.bnb_abit_quant_type ,		)
                                          lowercase_     =      True
                    # Store the module class in case we need to transpose the weight later
                              lowercase_     =      type(__lowerCamelCase				)
                              # Force requires grad to False to avoid unexpected errors
                              model._modules[name].requires_grad_(__lowerCamelCase				)
            if len(list(module.children()				)				) > 0:
                  lowercase_ ,							lowercase_     =      _replace_with_bnb_linear(
                      __lowerCamelCase ,		__lowerCamelCase ,		__lowerCamelCase ,		__lowerCamelCase ,		has_been_replaced=__lowerCamelCase ,		)
            # Remove the last key for recursion
            current_key_name.pop(-1				)
      return model, has_been_replaced


def __a (	__lowerCamelCase			:						int ,		__lowerCamelCase			:						List[Any]=None ,		__lowerCamelCase			:						int=None ,		__lowerCamelCase			:						Tuple=None				)	->							Tuple:
      '''simple docstring'''


      lowercase_     =      ["lm_head"] if modules_to_not_convert is None else modules_to_not_convert
      lowercase_ ,							lowercase_     =      _replace_with_bnb_linear(
          __lowerCamelCase ,		__lowerCamelCase ,		__lowerCamelCase ,		__lowerCamelCase				)

      if not has_been_replaced:
            logger.warning(
                "You are loading your model in 8bit or 4bit but no linear modules were found in your model."
                " Please double check your model architecture, or submit an issue on github if you think this is"
                " a bug."				)

      return model


def __a (	*__lowerCamelCase			:						Optional[int] ,		**__lowerCamelCase			:						List[str]				)	->							Union[str, Any]:
      '''simple docstring'''


      warnings.warn(
          "`replace_8bit_linear` will be deprecated in a future version, please use `replace_with_bnb_linear` instead" ,		__lowerCamelCase ,		)
      return replace_with_bnb_linear(*__lowerCamelCase ,		**__lowerCamelCase				)


def __a (	*__lowerCamelCase			:						int ,		**__lowerCamelCase			:						Any				)	->							int:
      '''simple docstring'''


      warnings.warn(
          "`set_module_8bit_tensor_to_device` will be deprecated in a future version, please use `set_module_quantized_tensor_to_device` instead" ,		__lowerCamelCase ,		)
      return set_module_quantized_tensor_to_device(*__lowerCamelCase ,		**__lowerCamelCase				)






def __a (	__lowerCamelCase			:						Dict				)	->							str:
      '''simple docstring'''


      lowercase_     =      deepcopy(__lowerCamelCase				)  # this has 0 cost since it is done inside `init_empty_weights` context manager`
      tied_model.tie_weights()

      lowercase_     =      find_tied_parameters(__lowerCamelCase				)
      # For compatibility with Accelerate < 0.18
      if isinstance(__lowerCamelCase ,		__lowerCamelCase				):
            lowercase_     =      sum(list(tied_params.values()				) ,		[]				) + list(tied_params.keys()				)
      else:
            lowercase_     =      sum(__lowerCamelCase ,		[]				)
      lowercase_     =      len(__lowerCamelCase				) > 0

      # Check if it is a base model
      lowercase_     =      not hasattr(__lowerCamelCase ,		model.base_model_prefix				)

      # Ignore this for base models (BertModel, GPT2Model, etc.)
      if (not has_tied_params) and is_base_model:
            return []

      # otherwise they have an attached head
      lowercase_     =      list(model.named_children()				)
      lowercase_     =      [list_modules[-1][0]]

      # add last module together with tied weights
      lowercase_     =      set(__lowerCamelCase				) - set(__lowerCamelCase				)
      lowercase_     =      list(set(__lowerCamelCase				)				) + list(__lowerCamelCase				)

      # remove ".weight" from the keys
      lowercase_     =      [".weight", ".bias"]
      lowercase_     =      []
      for name in list_untouched:
            for name_to_remove in names_to_remove:
                  if name_to_remove in name:
                        lowercase_     =      name.replace(__lowerCamelCase ,		""				)
            filtered_module_names.append(__lowerCamelCase				)

      return filtered_module_names







'''simple docstring'''

import inspect
import os

import torch
from transformers import AutoModel
from transformers.testing_utils import mockenv_context
from transformers.trainer_utils import set_seed

import accelerate
from accelerate.accelerator import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils.testing import (
    AccelerateTestCase,
    TempDirTestCase,
    execute_subprocess_async,
    require_cuda,
    require_fsdp,
    require_multi_gpu,
    slow,
)
from accelerate.utils.constants import (
    FSDP_AUTO_WRAP_POLICY,
    FSDP_BACKWARD_PREFETCH,
    FSDP_SHARDING_STRATEGY,
    FSDP_STATE_DICT_TYPE,
)
from accelerate.utils.dataclasses import FullyShardedDataParallelPlugin
from accelerate.utils.other import patch_environment


set_seed(42)

lowerCAmelCase_ : Optional[Any]						  =      "bert-base-cased"
lowerCAmelCase_ : Any						  =      "fp16"
lowerCAmelCase_ : Union[str, Any]						  =      "bf16"
lowerCAmelCase_ : List[Any]						  =      [FPaa, BFaa]





@require_fsdp
@require_cuda
class 							lowercase						(						__lowerCamelCase ):


   def __UpperCAmelCase       (   self				:	str)      ->					Union[str, Any]:
         super().setUp()

         lowercase_     =      dict(
             ACCELERATE_USE_FSDP="true"   ,		MASTER_ADDR="localhost"   ,		MASTER_PORT="10999"   ,		RANK="0"   ,		LOCAL_RANK="0"   ,		WORLD_SIZE="1"   ,		)


   def __UpperCAmelCase       (   self				:	Optional[Any])      ->					List[Any]:
         from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy

         for i, strategy in enumerate(__lowerCAmelCase):
               lowercase_     =      self.dist_env.copy()
               lowercase_     =      F'{i + 1}'
               lowercase_     =      strategy
               with mockenv_context(**__lowerCAmelCase):
                     lowercase_     =      FullyShardedDataParallelPlugin()
                     self.assertEqual(fsdp_plugin.sharding_strategy   ,		ShardingStrategy(i + 1))


   def __UpperCAmelCase       (   self				:	Union[str, Any])      ->					List[str]:
         from torch.distributed.fsdp.fully_sharded_data_parallel import BackwardPrefetch

         for i, prefetch_policy in enumerate(__lowerCAmelCase):
               lowercase_     =      self.dist_env.copy()
               lowercase_     =      prefetch_policy
               with mockenv_context(**__lowerCAmelCase):
                     lowercase_     =      FullyShardedDataParallelPlugin()
                     if prefetch_policy == "NO_PREFETCH":
                           self.assertIsNone(fsdp_plugin.backward_prefetch)
                     else:
                           self.assertEqual(fsdp_plugin.backward_prefetch   ,		BackwardPrefetch(i + 1))


   def __UpperCAmelCase       (   self				:	Dict)      ->					List[str]:
         from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType

         for i, state_dict_type in enumerate(__lowerCAmelCase):
               lowercase_     =      self.dist_env.copy()
               lowercase_     =      state_dict_type
               with mockenv_context(**__lowerCAmelCase):
                     lowercase_     =      FullyShardedDataParallelPlugin()
                     self.assertEqual(fsdp_plugin.state_dict_type   ,		StateDictType(i + 1))
                     if state_dict_type == "FULL_STATE_DICT":
                           self.assertTrue(fsdp_plugin.state_dict_config.offload_to_cpu)
                           self.assertTrue(fsdp_plugin.state_dict_config.ranka_only)


   def __UpperCAmelCase       (   self				:	Optional[int])      ->					List[Any]:
         lowercase_     =      AutoModel.from_pretrained(__lowerCAmelCase)
         for policy in FSDP_AUTO_WRAP_POLICY:
               lowercase_     =      self.dist_env.copy()
               lowercase_     =      policy
               if policy == "TRANSFORMER_BASED_WRAP":
                     lowercase_     =      "BertLayer"
               elif policy == "SIZE_BASED_WRAP":
                     lowercase_     =      "2000"
               with mockenv_context(**__lowerCAmelCase):
                     lowercase_     =      FullyShardedDataParallelPlugin()
                     fsdp_plugin.set_auto_wrap_policy(__lowerCAmelCase)
                     if policy == "NO_WRAP":
                           self.assertIsNone(fsdp_plugin.auto_wrap_policy)
                     else:
                           self.assertIsNotNone(fsdp_plugin.auto_wrap_policy)

         lowercase_     =      self.dist_env.copy()
         lowercase_     =      "TRANSFORMER_BASED_WRAP"
         lowercase_     =      "T5Layer"
         with mockenv_context(**__lowerCAmelCase):
               lowercase_     =      FullyShardedDataParallelPlugin()
               with self.assertRaises(__lowerCAmelCase) as cm:
                     fsdp_plugin.set_auto_wrap_policy(__lowerCAmelCase)
               self.assertTrue("Could not find the transformer layer class to wrap in the model." in str(cm.exception))

         lowercase_     =      self.dist_env.copy()
         lowercase_     =      "SIZE_BASED_WRAP"
         lowercase_     =      "0"
         with mockenv_context(**__lowerCAmelCase):
               lowercase_     =      FullyShardedDataParallelPlugin()
               fsdp_plugin.set_auto_wrap_policy(__lowerCAmelCase)
               self.assertIsNone(fsdp_plugin.auto_wrap_policy)


   def __UpperCAmelCase       (   self				:	Union[str, Any])      ->					int:
         from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
         from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler

         for mp_dtype in dtypes:
               lowercase_     =      self.dist_env.copy()
               lowercase_     =      mp_dtype
               with mockenv_context(**__lowerCAmelCase):
                     lowercase_     =      Accelerator()
                     if mp_dtype == "fp16":
                           lowercase_     =      torch.floataa
                     elif mp_dtype == "bf16":
                           lowercase_     =      torch.bfloataa
                     lowercase_     =      MixedPrecision(param_dtype=__lowerCAmelCase   ,		reduce_dtype=__lowerCAmelCase   ,		buffer_dtype=__lowerCAmelCase)
                     self.assertEqual(accelerator.state.fsdp_plugin.mixed_precision_policy   ,		__lowerCAmelCase)
                     if mp_dtype == FPaa:
                           self.assertTrue(isinstance(accelerator.scaler   ,		__lowerCAmelCase))
                     elif mp_dtype == BFaa:
                           self.assertIsNone(accelerator.scaler)
                     AcceleratorState._reset_state(__lowerCAmelCase)


   def __UpperCAmelCase       (   self				:	List[str])      ->					Dict:
         from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload

         for flag in [True, False]:
               lowercase_     =      self.dist_env.copy()
               lowercase_     =      str(__lowerCAmelCase).lower()
               with mockenv_context(**__lowerCAmelCase):
                     lowercase_     =      FullyShardedDataParallelPlugin()
                     self.assertEqual(fsdp_plugin.cpu_offload   ,		CPUOffload(offload_params=__lowerCAmelCase))





@require_fsdp
@require_multi_gpu
@slow
class 							lowercase						(						__lowerCamelCase ):


   def __UpperCAmelCase       (   self				:	Optional[int])      ->					str:
         super().setUp()
         lowercase_     =      0.82
         lowercase_     =      [
             "fsdp_shard_grad_op_transformer_based_wrap",
             "fsdp_full_shard_transformer_based_wrap",
         ]
         lowercase_     =      {
             "multi_gpu_fp16": 3200,
             "fsdp_shard_grad_op_transformer_based_wrap_fp16": 2000,
             "fsdp_full_shard_transformer_based_wrap_fp16": 1900,
             # Disabling below test as it overwhelms the RAM memory usage
             # on CI self-hosted runner leading to tests getting killed.
             # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
         }
         lowercase_     =      160
         lowercase_     =      160

         lowercase_     =      inspect.getfile(accelerate.test_utils)
         lowercase_     =      os.path.sep.join(mod_file.split(os.path.sep)[:-1] + ["scripts", "external_deps"])


   def __UpperCAmelCase       (   self				:	Optional[Any])      ->					Optional[Any]:
         lowercase_     =      os.path.join(self.test_scripts_folder   ,		"test_performance.py")
         lowercase_     =      ["accelerate", "launch", "--num_processes=2", "--num_machines=1", "--machine_rank=0", "--use_fsdp"]
         for config in self.performance_configs:
               lowercase_     =      cmd.copy()
               for i, strategy in enumerate(__lowerCAmelCase):
                     if strategy.lower() in config:
                           cmd_config.append(F'--fsdp_sharding_strategy={i+1}')
                           break

               if "fp32" in config:
                     cmd_config.append("--mixed_precision=no")
               else:
                     cmd_config.append("--mixed_precision=fp16")

               if "cpu_offload" in config:
                     cmd_config.append("--fsdp_offload_params=True")

               for policy in FSDP_AUTO_WRAP_POLICY:
                     if policy.lower() in config:
                           cmd_config.append(F'--fsdp_auto_wrap_policy={policy}')
                           break

               if policy == "TRANSFORMER_BASED_WRAP":
                     cmd_config.append("--fsdp_transformer_layer_cls_to_wrap=BertLayer")
               elif policy == "SIZE_BASED_WRAP":
                     cmd_config.append("--fsdp_min_num_params=2000")

               cmd_config.extend(
                   [
                       self.test_file_path,
                       F'--output_dir={self.tmpdir}',
                       F'--performance_lower_bound={self.performance_lower_bound}',
                   ])
               with patch_environment(omp_num_threads=1):
                     execute_subprocess_async(__lowerCAmelCase   ,		env=os.environ.copy())


   def __UpperCAmelCase       (   self				:	Dict)      ->					Dict:
         lowercase_     =      os.path.join(self.test_scripts_folder   ,		"test_checkpointing.py")
         lowercase_     =      [
             "accelerate",
             "launch",
             "--num_processes=2",
             "--num_machines=1",
             "--machine_rank=0",
             "--use_fsdp",
             "--mixed_precision=fp16",
             "--fsdp_transformer_layer_cls_to_wrap=BertLayer",
         ]

         for i, strategy in enumerate(__lowerCAmelCase):
               lowercase_     =      cmd.copy()
               cmd_config.append(F'--fsdp_sharding_strategy={i+1}')
               if strategy != "FULL_SHARD":
                     continue
               lowercase_     =      len(__lowerCAmelCase)
               for state_dict_type in FSDP_STATE_DICT_TYPE:
                     lowercase_     =      cmd_config[:state_dict_config_index]
                     cmd_config.append(F'--fsdp_state_dict_type={state_dict_type}')
                     cmd_config.extend(
                         [
                             self.test_file_path,
                             F'--output_dir={self.tmpdir}',
                             "--partial_train_epoch=1",
                         ])
                     with patch_environment(omp_num_threads=1):
                           execute_subprocess_async(__lowerCAmelCase   ,		env=os.environ.copy())

                     lowercase_     =      cmd_config[:-1]
                     lowercase_     =      os.path.join(self.tmpdir   ,		"epoch_0")
                     cmd_config.extend(
                         [
                             F'--resume_from_checkpoint={resume_from_checkpoint}',
                         ])
                     with patch_environment(omp_num_threads=1):
                           execute_subprocess_async(__lowerCAmelCase   ,		env=os.environ.copy())


   def __UpperCAmelCase       (   self				:	Optional[int])      ->					int:
         lowercase_     =      os.path.join(self.test_scripts_folder   ,		"test_peak_memory_usage.py")
         lowercase_     =      [
             "accelerate",
             "launch",
             "--num_processes=2",
             "--num_machines=1",
             "--machine_rank=0",
         ]
         for spec, peak_mem_upper_bound in self.peak_memory_usage_upper_bound.items():
               lowercase_     =      cmd.copy()
               if "fp16" in spec:
                     cmd_config.extend(["--mixed_precision=fp16"])
               else:
                     cmd_config.extend(["--mixed_precision=no"])

               if "multi_gpu" in spec:
                     continue
               else:
                     cmd_config.extend(["--use_fsdp"])
                     for i, strategy in enumerate(__lowerCAmelCase):
                           if strategy.lower() in spec:
                                 cmd_config.append(F'--fsdp_sharding_strategy={i+1}')
                                 break

                     if "cpu_offload" in spec:
                           cmd_config.append("--fsdp_offload_params=True")

                     for policy in FSDP_AUTO_WRAP_POLICY:
                           if policy.lower() in spec:
                                 cmd_config.append(F'--fsdp_auto_wrap_policy={policy}')
                                 break

                     if policy == "TRANSFORMER_BASED_WRAP":
                           cmd_config.append("--fsdp_transformer_layer_cls_to_wrap=BertLayer")
                     elif policy == "SIZE_BASED_WRAP":
                           cmd_config.append("--fsdp_min_num_params=2000")

               cmd_config.extend(
                   [
                       self.test_file_path,
                       F'--output_dir={self.tmpdir}',
                       F'--peak_memory_upper_bound={peak_mem_upper_bound}',
                       F'--n_train={self.n_train}',
                       F'--n_val={self.n_val}',
                   ])
               with patch_environment(omp_num_threads=1):
                     execute_subprocess_async(__lowerCAmelCase   ,		env=os.environ.copy())






