
"""simple docstring"""





import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu




class 			lowercase__			(      nn.Module    ):
							'''simple docstring'''







							def __init__(  self						)			->							Any:
									super().__init__()
									_UpperCAmelCase							 =		nn.Linear(3	,       4						)
									_UpperCAmelCase							 =		nn.BatchNormad(4						)
									_UpperCAmelCase							 =		nn.Linear(4	,       5						)






							def        lowerCamelCase_   (  self	,       snake_case						)			->							Any:
									return self.lineara(self.batchnorm(self.lineara(snake_case						)						)						)







class 			lowercase__			(      A    ):
							'''simple docstring'''







							def        lowerCamelCase_   (  self	,       snake_case	,       *snake_case	,       **snake_case						)			->							int:
									return (args[0] + 1,) + args[1:], kwargs







class 			lowercase__			(      A    ):
							'''simple docstring'''







							def        lowerCamelCase_   (  self	,       snake_case	,       snake_case						)			->							str:
									return output + 1







class 			lowercase__			(      unittest.TestCase    ):
							'''simple docstring'''







							def        lowerCamelCase_   (  self						)			->							Any:
									_UpperCAmelCase							 =		ModelForTest()
									_UpperCAmelCase							 =		ModelHook()

									add_hook_to_module(snake_case	,       snake_case						)
									self.assertEqual(test_model._hf_hook	,       snake_case						)
									self.assertTrue(hasattr(snake_case	,       '_old_forward'						)						)

									# Check adding the hook did not change the name or the signature
									self.assertEqual(test_model.forward.__name__	,       'forward'						)
									self.assertListEqual(list(inspect.signature(test_model.forward						).parameters						)	,       ['x']						)

									remove_hook_from_module(snake_case						)
									self.assertFalse(hasattr(snake_case	,       '_hf_hook'						)						)
									self.assertFalse(hasattr(snake_case	,       '_old_forward'						)						)






							def        lowerCamelCase_   (  self						)			->							Optional[int]:
									_UpperCAmelCase							 =		ModelForTest()
									_UpperCAmelCase							 =		ModelHook()

									add_hook_to_module(snake_case	,       snake_case						)
									add_hook_to_module(snake_case	,       snake_case	,       append=snake_case						)

									self.assertEqual(isinstance(test_model._hf_hook	,       snake_case						)	,       snake_case						)
									self.assertEqual(len(test_model._hf_hook.hooks						)	,       2						)
									self.assertTrue(hasattr(snake_case	,       '_old_forward'						)						)

									# Check adding the hook did not change the name or the signature
									self.assertEqual(test_model.forward.__name__	,       'forward'						)
									self.assertListEqual(list(inspect.signature(test_model.forward						).parameters						)	,       ['x']						)

									remove_hook_from_module(snake_case						)
									self.assertFalse(hasattr(snake_case	,       '_hf_hook'						)						)
									self.assertFalse(hasattr(snake_case	,       '_old_forward'						)						)






							def        lowerCamelCase_   (  self						)			->							Tuple:
									_UpperCAmelCase							 =		ModelForTest()
									_UpperCAmelCase							 =		torch.randn(2	,       3						)
									_UpperCAmelCase							 =		test_model(x + 1						)
									_UpperCAmelCase							 =		test_model(x + 2						)

									_UpperCAmelCase							 =		PreForwardHook()
									add_hook_to_module(snake_case	,       snake_case						)
									_UpperCAmelCase							 =		test_model(snake_case						)
									self.assertTrue(torch.allclose(snake_case	,       snake_case	,       atol=1E-5						)						)

									# Attaching a hook to a model when it already has one replaces, does not chain
									_UpperCAmelCase							 =		PreForwardHook()
									add_hook_to_module(snake_case	,       snake_case						)
									_UpperCAmelCase							 =		test_model(snake_case						)
									self.assertTrue(torch.allclose(snake_case	,       snake_case	,       atol=1E-5						)						)

									# You need to use the sequential hook to chain two or more hooks
									_UpperCAmelCase							 =		SequentialHook(PreForwardHook()	,       PreForwardHook()						)
									add_hook_to_module(snake_case	,       snake_case						)

									_UpperCAmelCase							 =		test_model(snake_case						)
									assert torch.allclose(snake_case	,       snake_case	,       atol=1E-5						)






							def        lowerCamelCase_   (  self						)			->							Union[str, Any]:
									_UpperCAmelCase							 =		ModelForTest()
									_UpperCAmelCase							 =		torch.randn(2	,       3						)
									_UpperCAmelCase							 =		test_model(snake_case						)

									_UpperCAmelCase							 =		PostForwardHook()
									add_hook_to_module(snake_case	,       snake_case						)
									_UpperCAmelCase							 =		test_model(snake_case						)
									self.assertTrue(torch.allclose(snake_case	,       output + 1	,       atol=1E-5						)						)

									# Attaching a hook to a model when it already has one replaces, does not chain
									_UpperCAmelCase							 =		PostForwardHook()
									add_hook_to_module(snake_case	,       snake_case						)
									_UpperCAmelCase							 =		test_model(snake_case						)
									self.assertTrue(torch.allclose(snake_case	,       output + 1	,       atol=1E-5						)						)

									# You need to use the sequential hook to chain two or more hooks
									_UpperCAmelCase							 =		SequentialHook(PostForwardHook()	,       PostForwardHook()						)
									add_hook_to_module(snake_case	,       snake_case						)

									_UpperCAmelCase							 =		test_model(snake_case						)
									assert torch.allclose(snake_case	,       output + 2	,       atol=1E-5						)






							def        lowerCamelCase_   (  self						)			->							Union[str, Any]:
									_UpperCAmelCase							 =		ModelForTest()
									_UpperCAmelCase							 =		torch.randn(2	,       3						)
									_UpperCAmelCase							 =		test_model(snake_case						)

									_UpperCAmelCase							 =		PostForwardHook()
									add_hook_to_module(snake_case	,       snake_case						)
									_UpperCAmelCase							 =		test_model(snake_case						)
									self.assertTrue(torch.allclose(snake_case	,       output + 1						)						)
									self.assertTrue(outputa.requires_grad						)

									_UpperCAmelCase							 =		True
									_UpperCAmelCase							 =		test_model(snake_case						)
									self.assertFalse(outputa.requires_grad						)






							@require_multi_gpu
							def        lowerCamelCase_   (  self						)			->							str:
									_UpperCAmelCase							 =		ModelForTest()
									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)

									# This will move each submodule on different devices
									add_hook_to_module(model.lineara	,       AlignDevicesHook(execution_device=0						)						)
									add_hook_to_module(model.batchnorm	,       AlignDevicesHook(execution_device=0						)						)
									add_hook_to_module(model.lineara	,       AlignDevicesHook(execution_device=1						)						)

									self.assertEqual(model.lineara.weight.device	,       torch.device(0						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device(0						)						)
									self.assertEqual(model.batchnorm.running_mean.device	,       torch.device(0						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device(1						)						)

									# We can still make a forward pass. The input does not need to be on any particular device
									_UpperCAmelCase							 =		torch.randn(2	,       3						)
									_UpperCAmelCase							 =		model(snake_case						)
									self.assertEqual(output.device	,       torch.device(1						)						)

									# We can add a general hook to put back output on same device as input.
									add_hook_to_module(snake_case	,       AlignDevicesHook(io_same_device=snake_case						)						)
									_UpperCAmelCase							 =		torch.randn(2	,       3						).to(0						)
									_UpperCAmelCase							 =		model(snake_case						)
									self.assertEqual(output.device	,       torch.device(0						)						)






							def        lowerCamelCase_   (  self						)			->							int:
									_UpperCAmelCase							 =		ModelForTest()

									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)

									# This will move each submodule on different devices
									_UpperCAmelCase							 =		{'execution_device': 0 if torch.cuda.is_available() else 'cpu', 'offload': True}

									add_hook_to_module(model.lineara	,       AlignDevicesHook(**snake_case						)						)
									add_hook_to_module(model.batchnorm	,       AlignDevicesHook(**snake_case						)						)
									add_hook_to_module(model.lineara	,       AlignDevicesHook(**snake_case						)						)

									# Parameters have been offloaded, so on the meta device
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									# Buffers are not included in the offload by default, so are on the execution device
									_UpperCAmelCase							 =		torch.device(hook_kwargs['execution_device']						)
									self.assertEqual(model.batchnorm.running_mean.device	,       snake_case						)

									_UpperCAmelCase							 =		torch.randn(2	,       3						)
									_UpperCAmelCase							 =		model(snake_case						)
									self.assertEqual(output.device	,       snake_case						)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_module(model.lineara						)
									remove_hook_from_module(model.batchnorm						)
									remove_hook_from_module(model.lineara						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)

									# Now test with buffers included in the offload
									_UpperCAmelCase							 =		{
									    'execution_device': 0 if torch.cuda.is_available() else 'cpu',
									    'offload': True,
									    'offload_buffers': True,
									}

									add_hook_to_module(model.lineara	,       AlignDevicesHook(**snake_case						)						)
									add_hook_to_module(model.batchnorm	,       AlignDevicesHook(**snake_case						)						)
									add_hook_to_module(model.lineara	,       AlignDevicesHook(**snake_case						)						)

									# Parameters have been offloaded, so on the meta device, buffers included
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.batchnorm.running_mean.device	,       torch.device('meta'						)						)

									_UpperCAmelCase							 =		torch.randn(2	,       3						)
									_UpperCAmelCase							 =		model(snake_case						)
									self.assertEqual(output.device	,       snake_case						)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_module(model.lineara						)
									remove_hook_from_module(model.batchnorm						)
									remove_hook_from_module(model.lineara						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)






							def        lowerCamelCase_   (  self						)			->							List[Any]:
									_UpperCAmelCase							 =		ModelForTest()

									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)

									# This will move each submodule on different devices
									_UpperCAmelCase							 =		0 if torch.cuda.is_available() else 'cpu'
									attach_align_device_hook(snake_case	,       execution_device=snake_case	,       offload=snake_case						)

									# Parameters have been offloaded, so on the meta device
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									# Buffers are not included in the offload by default, so are on the execution device
									_UpperCAmelCase							 =		torch.device(snake_case						)
									self.assertEqual(model.batchnorm.running_mean.device	,       snake_case						)

									_UpperCAmelCase							 =		torch.randn(2	,       3						)
									_UpperCAmelCase							 =		model(snake_case						)
									self.assertEqual(output.device	,       snake_case						)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(snake_case						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)

									# Now test with buffers included in the offload
									attach_align_device_hook(snake_case	,       execution_device=snake_case	,       offload=snake_case	,       offload_buffers=snake_case						)

									# Parameters have been offloaded, so on the meta device, buffers included
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.batchnorm.running_mean.device	,       torch.device('meta'						)						)

									_UpperCAmelCase							 =		torch.randn(2	,       3						)
									_UpperCAmelCase							 =		model(snake_case						)
									self.assertEqual(output.device	,       snake_case						)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(snake_case						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)






							def        lowerCamelCase_   (  self						)			->							int:
									_UpperCAmelCase							 =		ModelForTest()

									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)

									# This will move each submodule on different devices
									_UpperCAmelCase							 =		0 if torch.cuda.is_available() else 'cpu'
									attach_align_device_hook(
									    snake_case	,       execution_device=snake_case	,       offload=snake_case	,       weights_map=model.state_dict()						)

									# Parameters have been offloaded, so on the meta device
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									# Buffers are not included in the offload by default, so are on the execution device
									_UpperCAmelCase							 =		torch.device(snake_case						)
									self.assertEqual(model.batchnorm.running_mean.device	,       snake_case						)

									_UpperCAmelCase							 =		torch.randn(2	,       3						)
									_UpperCAmelCase							 =		model(snake_case						)
									self.assertEqual(output.device	,       snake_case						)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(snake_case						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)

									# Now test with buffers included in the offload
									attach_align_device_hook(
									    snake_case	,       execution_device=snake_case	,       offload=snake_case	,       weights_map=model.state_dict()	,       offload_buffers=snake_case	,       )

									# Parameters have been offloaded, so on the meta device, buffers included
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.batchnorm.running_mean.device	,       torch.device('meta'						)						)

									_UpperCAmelCase							 =		torch.randn(2	,       3						)
									_UpperCAmelCase							 =		model(snake_case						)
									self.assertEqual(output.device	,       snake_case						)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(snake_case						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)







"""simple docstring"""





from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging
from ...utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices


lowercase  							=					logging.get_logger(__name__)

lowercase  							=					{
    '''microsoft/swin-tiny-patch4-window7-224''': (
        '''https://huggingface.co/microsoft/swin-tiny-patch4-window7-224/resolve/main/config.json'''
    ),
    # See all Swin models at https://huggingface.co/models?filter=swin
}




class 			lowercase__			(      A,					A    ):
							'''simple docstring'''







							_UpperCAmelCase      				=					'''swin'''

							_UpperCAmelCase      				=					{
							    '''num_attention_heads''': '''num_heads''',
							    '''num_hidden_layers''': '''num_layers''',
							}






							def __init__(  self	,       snake_case=224	,       snake_case=4	,       snake_case=3	,       snake_case=96	,       snake_case=[2, 2, 6, 2]	,       snake_case=[3, 6, 12, 24]	,       snake_case=7	,       snake_case=4.0	,       snake_case=True	,       snake_case=0.0	,       snake_case=0.0	,       snake_case=0.1	,       snake_case="gelu"	,       snake_case=False	,       snake_case=0.02	,       snake_case=1E-5	,       snake_case=32	,       snake_case=None	,       snake_case=None	,       **snake_case	,       )			->							List[Any]:
									super().__init__(**snake_case						)

									_UpperCAmelCase							 =		image_size
									_UpperCAmelCase							 =		patch_size
									_UpperCAmelCase							 =		num_channels
									_UpperCAmelCase							 =		embed_dim
									_UpperCAmelCase							 =		depths
									_UpperCAmelCase							 =		len(snake_case						)
									_UpperCAmelCase							 =		num_heads
									_UpperCAmelCase							 =		window_size
									_UpperCAmelCase							 =		mlp_ratio
									_UpperCAmelCase							 =		qkv_bias
									_UpperCAmelCase							 =		hidden_dropout_prob
									_UpperCAmelCase							 =		attention_probs_dropout_prob
									_UpperCAmelCase							 =		drop_path_rate
									_UpperCAmelCase							 =		hidden_act
									_UpperCAmelCase							 =		use_absolute_embeddings
									_UpperCAmelCase							 =		layer_norm_eps
									_UpperCAmelCase							 =		initializer_range
									_UpperCAmelCase							 =		encoder_stride
									# we set the hidden_size attribute in order to make Swin work with VisionEncoderDecoderModel
									# this indicates the channel dimension after the last stage of the model
									_UpperCAmelCase							 =		int(embed_dim * 2 ** (len(snake_case						) - 1)						)
									_UpperCAmelCase							 =		['stem'] + [f'stage{idx}' for idx in range(1	,       len(snake_case						) + 1						)]
									_UpperCAmelCase ,					_UpperCAmelCase							 =		get_aligned_output_features_output_indices(
									    out_features=snake_case	,       out_indices=snake_case	,       stage_names=self.stage_names						)







class 			lowercase__			(      A    ):
							'''simple docstring'''







							_UpperCAmelCase      				=					version.parse('''1.11'''    )






							@property
							def        lowerCamelCase_   (  self						)			->							Mapping[str, Mapping[int, str]]:
									return OrderedDict(
									    [
									        ('pixel_values', {0: 'batch', 1: 'num_channels', 2: 'height', 3: 'width'}),
									    ]						)






							@property
							def        lowerCamelCase_   (  self						)			->							float:
									return 1E-4


