


"""simple docstring"""


import argparse
import json
from collections import OrderedDict
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import PoolFormerConfig, PoolFormerForImageClassification, PoolFormerImageProcessor
from transformers.utils import logging


logging.set_verbosity_info()
__lowerCamelCase      :str	    =		logging.get_logger(__name__)




def   snake_case		(     UpperCamelCase__       :						List[str]			,				UpperCamelCase__       :						str			,				UpperCamelCase__       :						Any			,				UpperCamelCase__       :						Any    )   ->      int:
      lowerCamelCase					:					int        =	original_name.split("""."""    )[0]
      lowerCamelCase					:					int        =	key.split("""."""    )
      lowerCamelCase					:					Optional[int]        =	int(key_list[key_list.index(UpperCamelCase__    ) - 2]    )
      lowerCamelCase					:					List[Any]        =	int(key_list[key_list.index(UpperCamelCase__    ) - 1]    )
      lowerCamelCase					:					List[str]        =	orig_block_num - offset

      lowerCamelCase					:					Optional[int]        =	key.replace(F'{orig_block_num}.{layer_num}.{original_name}'			,				F'block.{new_block_num}.{layer_num}.{new_name}'    )
      return key




def   snake_case		(     UpperCamelCase__       :						Union[str, Any]    )   ->      str:
      lowerCamelCase					:					Union[str, Any]        =	OrderedDict()
      lowerCamelCase     ,			lowerCamelCase					:					Dict        =	0, 0
      for key, value in state_dict.items():
            if key.startswith("""network"""    ):
                  lowerCamelCase					:					Any        =	key.replace("""network"""			,				"""poolformer.encoder"""    )
            if "proj" in key:
                  # Works for the first embedding as well as the internal embedding layers
                  if key.endswith("""bias"""    ) and "patch_embed" not in key:
                        patch_emb_offset += 1
                  lowerCamelCase					:					List[Any]        =	key[: key.find("""proj"""    )]
                  lowerCamelCase					:					Optional[Any]        =	key.replace(UpperCamelCase__			,				F'patch_embeddings.{total_embed_found}.'    )
                  lowerCamelCase					:					str        =	key.replace("""proj"""			,				"""projection"""    )
                  if key.endswith("""bias"""    ):
                        total_embed_found += 1
            if "patch_embeddings" in key:
                  lowerCamelCase					:					Any        =	"""poolformer.encoder.""" + key
            if "mlp.fc1" in key:
                  lowerCamelCase					:					Optional[int]        =	replace_key_with_offset(UpperCamelCase__			,				UpperCamelCase__			,				"""mlp.fc1"""			,				"""output.conv1"""    )
            if "mlp.fc2" in key:
                  lowerCamelCase					:					str        =	replace_key_with_offset(UpperCamelCase__			,				UpperCamelCase__			,				"""mlp.fc2"""			,				"""output.conv2"""    )
            if "norm1" in key:
                  lowerCamelCase					:					List[Any]        =	replace_key_with_offset(UpperCamelCase__			,				UpperCamelCase__			,				"""norm1"""			,				"""before_norm"""    )
            if "norm2" in key:
                  lowerCamelCase					:					Dict        =	replace_key_with_offset(UpperCamelCase__			,				UpperCamelCase__			,				"""norm2"""			,				"""after_norm"""    )
            if "layer_scale_1" in key:
                  lowerCamelCase					:					str        =	replace_key_with_offset(UpperCamelCase__			,				UpperCamelCase__			,				"""layer_scale_1"""			,				"""layer_scale_1"""    )
            if "layer_scale_2" in key:
                  lowerCamelCase					:					Optional[int]        =	replace_key_with_offset(UpperCamelCase__			,				UpperCamelCase__			,				"""layer_scale_2"""			,				"""layer_scale_2"""    )
            if "head" in key:
                  lowerCamelCase					:					List[str]        =	key.replace("""head"""			,				"""classifier"""    )
            lowerCamelCase					:					Dict        =	value
      return new_state_dict




def   snake_case		(     )   ->      Tuple:
      lowerCamelCase					:					str        =	"""http://images.cocodataset.org/val2017/000000039769.jpg"""
      lowerCamelCase					:					Optional[Any]        =	Image.open(requests.get(UpperCamelCase__			,				stream=UpperCamelCase__    ).raw    )

      return image




@torch.no_grad()
def   snake_case		(     UpperCamelCase__       :						List[Any]			,				UpperCamelCase__       :						Union[str, Any]			,				UpperCamelCase__       :						List[str]    )   ->      Optional[Any]:
      lowerCamelCase					:					str        =	PoolFormerConfig()

      # set attributes based on model_name
      lowerCamelCase					:					int        =	"""huggingface/label-files"""
      lowerCamelCase					:					List[Any]        =	model_name[-3:]
      lowerCamelCase					:					Tuple        =	1000
      lowerCamelCase					:					Dict        =	"""imagenet-1k-id2label.json"""
      lowerCamelCase					:					List[str]        =	(1, 1000)

      # set config attributes
      lowerCamelCase					:					List[str]        =	json.load(open(hf_hub_download(UpperCamelCase__			,				UpperCamelCase__			,				repo_type="""dataset"""    )			,				"""r"""    )    )
      lowerCamelCase					:					List[Any]        =	{int(UpperCamelCase__    ): v for k, v in idalabel.items()}
      lowerCamelCase					:					Optional[Any]        =	idalabel
      lowerCamelCase					:					Optional[int]        =	{v: k for k, v in idalabel.items()}
      if size == "s12":
            lowerCamelCase					:					Dict        =	[2, 2, 6, 2]
            lowerCamelCase					:					List[str]        =	[64, 128, 320, 512]
            lowerCamelCase					:					List[str]        =	4.0
            lowerCamelCase					:					Optional[Any]        =	0.9
      elif size == "s24":
            lowerCamelCase					:					int        =	[4, 4, 12, 4]
            lowerCamelCase					:					Optional[int]        =	[64, 128, 320, 512]
            lowerCamelCase					:					Optional[Any]        =	4.0
            lowerCamelCase					:					List[Any]        =	0.9
      elif size == "s36":
            lowerCamelCase					:					Optional[Any]        =	[6, 6, 18, 6]
            lowerCamelCase					:					Dict        =	[64, 128, 320, 512]
            lowerCamelCase					:					Union[str, Any]        =	4.0
            lowerCamelCase					:					int        =	1E-6
            lowerCamelCase					:					Tuple        =	0.9
      elif size == "m36":
            lowerCamelCase					:					Tuple        =	[6, 6, 18, 6]
            lowerCamelCase					:					int        =	[96, 192, 384, 768]
            lowerCamelCase					:					Dict        =	4.0
            lowerCamelCase					:					str        =	1E-6
            lowerCamelCase					:					Tuple        =	0.9_5
      elif size == "m48":
            lowerCamelCase					:					Optional[Any]        =	[8, 8, 24, 8]
            lowerCamelCase					:					Optional[int]        =	[96, 192, 384, 768]
            lowerCamelCase					:					List[Any]        =	4.0
            lowerCamelCase					:					List[str]        =	1E-6
            lowerCamelCase					:					Tuple        =	0.9_5
      else:
            raise ValueError(F'Size {size} not supported'    )

      # load image processor
      lowerCamelCase					:					Tuple        =	PoolFormerImageProcessor(crop_pct=UpperCamelCase__    )

      # Prepare image
      lowerCamelCase					:					Tuple        =	prepare_img()
      lowerCamelCase					:					str        =	image_processor(images=UpperCamelCase__			,				return_tensors="""pt"""    ).pixel_values

      logger.info(F'Converting model {model_name}...'    )

      # load original state dict
      lowerCamelCase					:					Tuple        =	torch.load(UpperCamelCase__			,				map_location=torch.device("""cpu"""    )    )

      # rename keys
      lowerCamelCase					:					List[Any]        =	rename_keys(UpperCamelCase__    )

      # create HuggingFace model and load state dict
      lowerCamelCase					:					Optional[Any]        =	PoolFormerForImageClassification(UpperCamelCase__    )
      model.load_state_dict(UpperCamelCase__    )
      model.eval()

      # Define image processor
      lowerCamelCase					:					Any        =	PoolFormerImageProcessor(crop_pct=UpperCamelCase__    )
      lowerCamelCase					:					Dict        =	image_processor(images=prepare_img()			,				return_tensors="""pt"""    ).pixel_values

      # forward pass
      lowerCamelCase					:					List[Any]        =	model(UpperCamelCase__    )
      lowerCamelCase					:					Union[str, Any]        =	outputs.logits

      # define expected logit slices for different models
      if size == "s12":
            lowerCamelCase					:					Union[str, Any]        =	torch.tensor([-0.3_0_4_5, -0.6_7_5_8, -0.4_8_6_9]    )
      elif size == "s24":
            lowerCamelCase					:					Optional[Any]        =	torch.tensor([0.4_4_0_2, -0.1_3_7_4, -0.8_0_4_5]    )
      elif size == "s36":
            lowerCamelCase					:					Any        =	torch.tensor([-0.6_0_8_0, -0.5_1_3_3, -0.5_8_9_8]    )
      elif size == "m36":
            lowerCamelCase					:					Union[str, Any]        =	torch.tensor([0.3_9_5_2, 0.2_2_6_3, -1.2_6_6_8]    )
      elif size == "m48":
            lowerCamelCase					:					List[Any]        =	torch.tensor([0.1_1_6_7, -0.0_6_5_6, -0.3_4_2_3]    )
      else:
            raise ValueError(F'Size {size} not supported'    )

      # verify logits
      assert logits.shape == expected_shape
      assert torch.allclose(logits[0, :3]			,				UpperCamelCase__			,				atol=1E-2    )

      # finally, save model and image processor
      logger.info(F'Saving PyTorch model and image processor to {pytorch_dump_folder_path}...'    )
      Path(UpperCamelCase__    ).mkdir(exist_ok=UpperCamelCase__    )
      model.save_pretrained(UpperCamelCase__    )
      print(F'Saving image processor to {pytorch_dump_folder_path}'    )
      image_processor.save_pretrained(UpperCamelCase__    )


if __name__ == "__main__":
    __lowerCamelCase      :Dict	    =		argparse.ArgumentParser()

    parser.add_argument(
        '--model_name',
        default='poolformer_s12',
        type=str,
        help='Name of the model you\'d like to convert.',
    )
    parser.add_argument(
        '--checkpoint_path', default=None, type=str, help='Path to the original PyTorch checkpoint (.pth file).'
    )
    parser.add_argument(
        '--pytorch_dump_folder_path', default=None, type=str, help='Path to the folder to output PyTorch model.'
    )
    __lowerCamelCase      :Optional[int]	    =		parser.parse_args()
    convert_poolformer_checkpoint(args.model_name, args.checkpoint_path, args.pytorch_dump_folder_path)








"""simple docstring"""


import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu


class 			A__	(						nn.Module):


     """simple docstring"""

     def __init__(	self:						Dict	)->    Dict:
           super().__init__()
           lowerCamelCase					:					Tuple        =	nn.Linear(3     ,		4	)
           lowerCamelCase					:					Optional[Any]        =	nn.BatchNormad(4	)
           lowerCamelCase					:					Optional[Any]        =	nn.Linear(4     ,		5	)







     def 		a__						(	self:						List[str]     ,		__a:						List[Any]	)->    Optional[Any]:
           return self.lineara(self.batchnorm(self.lineara(__a	)	)	)


class 			A__	(						__lowercase):


     """simple docstring"""

     def 		a__						(	self:						Tuple     ,		__a:						int     ,		*__a:						Any     ,		**__a:						Tuple	)->    Tuple:
           return (args[0] + 1,) + args[1:], kwargs


class 			A__	(						__lowercase):


     """simple docstring"""

     def 		a__						(	self:						Optional[int]     ,		__a:						List[str]     ,		__a:						List[Any]	)->    List[str]:
           return output + 1


class 			A__	(						unittest.TestCase):


     """simple docstring"""

     def 		a__						(	self:						int	)->    str:
           lowerCamelCase					:					List[str]        =	ModelForTest()
           lowerCamelCase					:					Dict        =	ModelHook()

           add_hook_to_module(__a     ,		__a	)
           self.assertEqual(test_model._hf_hook     ,		__a	)
           self.assertTrue(hasattr(__a     ,		"""_old_forward"""	)	)

           # Check adding the hook did not change the name or the signature
           self.assertEqual(test_model.forward.__name__     ,		"""forward"""	)
           self.assertListEqual(list(inspect.signature(test_model.forward	).parameters	)     ,		["""x"""]	)

           remove_hook_from_module(__a	)
           self.assertFalse(hasattr(__a     ,		"""_hf_hook"""	)	)
           self.assertFalse(hasattr(__a     ,		"""_old_forward"""	)	)

     def 		a__						(	self:						int	)->    str:
           lowerCamelCase					:					List[str]        =	ModelForTest()
           lowerCamelCase					:					Union[str, Any]        =	ModelHook()

           add_hook_to_module(__a     ,		__a	)
           add_hook_to_module(__a     ,		__a     ,		append=__a	)

           self.assertEqual(isinstance(test_model._hf_hook     ,		__a	)     ,		__a	)
           self.assertEqual(len(test_model._hf_hook.hooks	)     ,		2	)
           self.assertTrue(hasattr(__a     ,		"""_old_forward"""	)	)

           # Check adding the hook did not change the name or the signature
           self.assertEqual(test_model.forward.__name__     ,		"""forward"""	)
           self.assertListEqual(list(inspect.signature(test_model.forward	).parameters	)     ,		["""x"""]	)

           remove_hook_from_module(__a	)
           self.assertFalse(hasattr(__a     ,		"""_hf_hook"""	)	)
           self.assertFalse(hasattr(__a     ,		"""_old_forward"""	)	)

     def 		a__						(	self:						List[Any]	)->    List[str]:
           lowerCamelCase					:					str        =	ModelForTest()
           lowerCamelCase					:					Dict        =	torch.randn(2     ,		3	)
           lowerCamelCase					:					Union[str, Any]        =	test_model(x + 1	)
           lowerCamelCase					:					Optional[int]        =	test_model(x + 2	)

           lowerCamelCase					:					List[Any]        =	PreForwardHook()
           add_hook_to_module(__a     ,		__a	)
           lowerCamelCase					:					Optional[int]        =	test_model(__a	)
           self.assertTrue(torch.allclose(__a     ,		__a     ,		atol=1e-5	)	)

           # Attaching a hook to a model when it already has one replaces, does not chain
           lowerCamelCase					:					Dict        =	PreForwardHook()
           add_hook_to_module(__a     ,		__a	)
           lowerCamelCase					:					Tuple        =	test_model(__a	)
           self.assertTrue(torch.allclose(__a     ,		__a     ,		atol=1e-5	)	)

           # You need to use the sequential hook to chain two or more hooks
           lowerCamelCase					:					Any        =	SequentialHook(PreForwardHook()     ,		PreForwardHook()	)
           add_hook_to_module(__a     ,		__a	)

           lowerCamelCase					:					Optional[Any]        =	test_model(__a	)
           assert torch.allclose(__a     ,		__a     ,		atol=1e-5	)

     def 		a__						(	self:						Any	)->    Optional[int]:
           lowerCamelCase					:					str        =	ModelForTest()
           lowerCamelCase					:					List[str]        =	torch.randn(2     ,		3	)
           lowerCamelCase					:					int        =	test_model(__a	)

           lowerCamelCase					:					Dict        =	PostForwardHook()
           add_hook_to_module(__a     ,		__a	)
           lowerCamelCase					:					Tuple        =	test_model(__a	)
           self.assertTrue(torch.allclose(__a     ,		output + 1     ,		atol=1e-5	)	)

           # Attaching a hook to a model when it already has one replaces, does not chain
           lowerCamelCase					:					str        =	PostForwardHook()
           add_hook_to_module(__a     ,		__a	)
           lowerCamelCase					:					Optional[Any]        =	test_model(__a	)
           self.assertTrue(torch.allclose(__a     ,		output + 1     ,		atol=1e-5	)	)

           # You need to use the sequential hook to chain two or more hooks
           lowerCamelCase					:					Union[str, Any]        =	SequentialHook(PostForwardHook()     ,		PostForwardHook()	)
           add_hook_to_module(__a     ,		__a	)

           lowerCamelCase					:					str        =	test_model(__a	)
           assert torch.allclose(__a     ,		output + 2     ,		atol=1e-5	)

     def 		a__						(	self:						int	)->    Dict:
           lowerCamelCase					:					List[Any]        =	ModelForTest()
           lowerCamelCase					:					Optional[int]        =	torch.randn(2     ,		3	)
           lowerCamelCase					:					List[str]        =	test_model(__a	)

           lowerCamelCase					:					Any        =	PostForwardHook()
           add_hook_to_module(__a     ,		__a	)
           lowerCamelCase					:					str        =	test_model(__a	)
           self.assertTrue(torch.allclose(__a     ,		output + 1	)	)
           self.assertTrue(outputa.requires_grad	)

           lowerCamelCase					:					Optional[int]        =	True
           lowerCamelCase					:					Optional[int]        =	test_model(__a	)
           self.assertFalse(outputa.requires_grad	)

     @require_multi_gpu
     def 		a__						(	self:						List[str]	)->    Union[str, Any]:
           lowerCamelCase					:					int        =	ModelForTest()
           # Everything is on CPU
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)

           # This will move each submodule on different devices
           add_hook_to_module(model.lineara     ,		AlignDevicesHook(execution_device=0	)	)
           add_hook_to_module(model.batchnorm     ,		AlignDevicesHook(execution_device=0	)	)
           add_hook_to_module(model.lineara     ,		AlignDevicesHook(execution_device=1	)	)

           self.assertEqual(model.lineara.weight.device     ,		torch.device(0	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device(0	)	)
           self.assertEqual(model.batchnorm.running_mean.device     ,		torch.device(0	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device(1	)	)

           # We can still make a forward pass. The input does not need to be on any particular device
           lowerCamelCase					:					str        =	torch.randn(2     ,		3	)
           lowerCamelCase					:					Dict        =	model(__a	)
           self.assertEqual(output.device     ,		torch.device(1	)	)

           # We can add a general hook to put back output on same device as input.
           add_hook_to_module(__a     ,		AlignDevicesHook(io_same_device=__a	)	)
           lowerCamelCase					:					Optional[int]        =	torch.randn(2     ,		3	).to(0	)
           lowerCamelCase					:					str        =	model(__a	)
           self.assertEqual(output.device     ,		torch.device(0	)	)

     def 		a__						(	self:						List[str]	)->    Tuple:
           lowerCamelCase					:					Union[str, Any]        =	ModelForTest()

           # Everything is on CPU
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)

           # This will move each submodule on different devices
           lowerCamelCase					:					Tuple        =	{"""execution_device""": 0 if torch.cuda.is_available() else """cpu""", """offload""": True}

           add_hook_to_module(model.lineara     ,		AlignDevicesHook(**__a	)	)
           add_hook_to_module(model.batchnorm     ,		AlignDevicesHook(**__a	)	)
           add_hook_to_module(model.lineara     ,		AlignDevicesHook(**__a	)	)

           # Parameters have been offloaded, so on the meta device
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           # Buffers are not included in the offload by default, so are on the execution device
           lowerCamelCase					:					List[Any]        =	torch.device(hook_kwargs["""execution_device"""]	)
           self.assertEqual(model.batchnorm.running_mean.device     ,		__a	)

           lowerCamelCase					:					Optional[Any]        =	torch.randn(2     ,		3	)
           lowerCamelCase					:					Optional[Any]        =	model(__a	)
           self.assertEqual(output.device     ,		__a	)

           # Removing hooks loads back the weights in the model.
           remove_hook_from_module(model.lineara	)
           remove_hook_from_module(model.batchnorm	)
           remove_hook_from_module(model.lineara	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)

           # Now test with buffers included in the offload
           lowerCamelCase					:					Any        =	{
               """execution_device""": 0 if torch.cuda.is_available() else """cpu""",
               """offload""": True,
               """offload_buffers""": True,
           }

           add_hook_to_module(model.lineara     ,		AlignDevicesHook(**__a	)	)
           add_hook_to_module(model.batchnorm     ,		AlignDevicesHook(**__a	)	)
           add_hook_to_module(model.lineara     ,		AlignDevicesHook(**__a	)	)

           # Parameters have been offloaded, so on the meta device, buffers included
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.batchnorm.running_mean.device     ,		torch.device("""meta"""	)	)

           lowerCamelCase					:					int        =	torch.randn(2     ,		3	)
           lowerCamelCase					:					Optional[int]        =	model(__a	)
           self.assertEqual(output.device     ,		__a	)

           # Removing hooks loads back the weights in the model.
           remove_hook_from_module(model.lineara	)
           remove_hook_from_module(model.batchnorm	)
           remove_hook_from_module(model.lineara	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)

     def 		a__						(	self:						Any	)->    List[str]:
           lowerCamelCase					:					int        =	ModelForTest()

           # Everything is on CPU
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)

           # This will move each submodule on different devices
           lowerCamelCase					:					int        =	0 if torch.cuda.is_available() else """cpu"""
           attach_align_device_hook(__a     ,		execution_device=__a     ,		offload=__a	)

           # Parameters have been offloaded, so on the meta device
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           # Buffers are not included in the offload by default, so are on the execution device
           lowerCamelCase					:					List[Any]        =	torch.device(__a	)
           self.assertEqual(model.batchnorm.running_mean.device     ,		__a	)

           lowerCamelCase					:					Dict        =	torch.randn(2     ,		3	)
           lowerCamelCase					:					Optional[Any]        =	model(__a	)
           self.assertEqual(output.device     ,		__a	)

           # Removing hooks loads back the weights in the model.
           remove_hook_from_submodules(__a	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)

           # Now test with buffers included in the offload
           attach_align_device_hook(__a     ,		execution_device=__a     ,		offload=__a     ,		offload_buffers=__a	)

           # Parameters have been offloaded, so on the meta device, buffers included
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.batchnorm.running_mean.device     ,		torch.device("""meta"""	)	)

           lowerCamelCase					:					Optional[int]        =	torch.randn(2     ,		3	)
           lowerCamelCase					:					int        =	model(__a	)
           self.assertEqual(output.device     ,		__a	)

           # Removing hooks loads back the weights in the model.
           remove_hook_from_submodules(__a	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)







     def 		a__						(	self:						Optional[Any]	)->    List[Any]:
           lowerCamelCase					:					List[Any]        =	ModelForTest()

           # Everything is on CPU
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)

           # This will move each submodule on different devices
           lowerCamelCase					:					Any        =	0 if torch.cuda.is_available() else """cpu"""
           attach_align_device_hook(
               __a     ,		execution_device=__a     ,		offload=__a     ,		weights_map=model.state_dict()	)

           # Parameters have been offloaded, so on the meta device
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           # Buffers are not included in the offload by default, so are on the execution device
           lowerCamelCase					:					List[Any]        =	torch.device(__a	)
           self.assertEqual(model.batchnorm.running_mean.device     ,		__a	)

           lowerCamelCase					:					Dict        =	torch.randn(2     ,		3	)
           lowerCamelCase					:					int        =	model(__a	)
           self.assertEqual(output.device     ,		__a	)

           # Removing hooks loads back the weights in the model.
           remove_hook_from_submodules(__a	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)

           # Now test with buffers included in the offload
           attach_align_device_hook(
               __a     ,		execution_device=__a     ,		offload=__a     ,		weights_map=model.state_dict()     ,		offload_buffers=__a     ,		)

           # Parameters have been offloaded, so on the meta device, buffers included
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.batchnorm.running_mean.device     ,		torch.device("""meta"""	)	)

           lowerCamelCase					:					Tuple        =	torch.randn(2     ,		3	)
           lowerCamelCase					:					Any        =	model(__a	)
           self.assertEqual(output.device     ,		__a	)

           # Removing hooks loads back the weights in the model.
           remove_hook_from_submodules(__a	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)




