



"""simple docstring"""





import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu




class 			lowercase__			(      nn.Module    ):
							'''simple docstring'''







							def __init__(  self						)			->							Any:
									super().__init__()
									_UpperCAmelCase							 =		nn.Linear(3	,       4						)
									_UpperCAmelCase							 =		nn.BatchNormad(4						)
									_UpperCAmelCase							 =		nn.Linear(4	,       5						)






							def        lowerCamelCase_   (  self	,       snake_case						)			->							Any:
									return self.lineara(self.batchnorm(self.lineara(snake_case						)						)						)







class 			lowercase__			(      A    ):
							'''simple docstring'''







							def        lowerCamelCase_   (  self	,       snake_case	,       *snake_case	,       **snake_case						)			->							int:
									return (args[0] + 1,) + args[1:], kwargs







class 			lowercase__			(      A    ):
							'''simple docstring'''







							def        lowerCamelCase_   (  self	,       snake_case	,       snake_case						)			->							str:
									return output + 1







class 			lowercase__			(      unittest.TestCase    ):
							'''simple docstring'''







							def        lowerCamelCase_   (  self						)			->							Any:
									_UpperCAmelCase							 =		ModelForTest()
									_UpperCAmelCase							 =		ModelHook()

									add_hook_to_module(snake_case	,       snake_case						)
									self.assertEqual(test_model._hf_hook	,       snake_case						)
									self.assertTrue(hasattr(snake_case	,       '_old_forward'						)						)

									# Check adding the hook did not change the name or the signature
									self.assertEqual(test_model.forward.__name__	,       'forward'						)
									self.assertListEqual(list(inspect.signature(test_model.forward						).parameters						)	,       ['x']						)

									remove_hook_from_module(snake_case						)
									self.assertFalse(hasattr(snake_case	,       '_hf_hook'						)						)
									self.assertFalse(hasattr(snake_case	,       '_old_forward'						)						)






							def        lowerCamelCase_   (  self						)			->							Optional[int]:
									_UpperCAmelCase							 =		ModelForTest()
									_UpperCAmelCase							 =		ModelHook()

									add_hook_to_module(snake_case	,       snake_case						)
									add_hook_to_module(snake_case	,       snake_case	,       append=snake_case						)

									self.assertEqual(isinstance(test_model._hf_hook	,       snake_case						)	,       snake_case						)
									self.assertEqual(len(test_model._hf_hook.hooks						)	,       2						)
									self.assertTrue(hasattr(snake_case	,       '_old_forward'						)						)

									# Check adding the hook did not change the name or the signature
									self.assertEqual(test_model.forward.__name__	,       'forward'						)
									self.assertListEqual(list(inspect.signature(test_model.forward						).parameters						)	,       ['x']						)

									remove_hook_from_module(snake_case						)
									self.assertFalse(hasattr(snake_case	,       '_hf_hook'						)						)
									self.assertFalse(hasattr(snake_case	,       '_old_forward'						)						)






							def        lowerCamelCase_   (  self						)			->							Tuple:
									_UpperCAmelCase							 =		ModelForTest()
									_UpperCAmelCase							 =		torch.randn(2	,       3						)
									_UpperCAmelCase							 =		test_model(x + 1						)
									_UpperCAmelCase							 =		test_model(x + 2						)

									_UpperCAmelCase							 =		PreForwardHook()
									add_hook_to_module(snake_case	,       snake_case						)
									_UpperCAmelCase							 =		test_model(snake_case						)
									self.assertTrue(torch.allclose(snake_case	,       snake_case	,       atol=1E-5						)						)

									# Attaching a hook to a model when it already has one replaces, does not chain
									_UpperCAmelCase							 =		PreForwardHook()
									add_hook_to_module(snake_case	,       snake_case						)
									_UpperCAmelCase							 =		test_model(snake_case						)
									self.assertTrue(torch.allclose(snake_case	,       snake_case	,       atol=1E-5						)						)

									# You need to use the sequential hook to chain two or more hooks
									_UpperCAmelCase							 =		SequentialHook(PreForwardHook()	,       PreForwardHook()						)
									add_hook_to_module(snake_case	,       snake_case						)

									_UpperCAmelCase							 =		test_model(snake_case						)
									assert torch.allclose(snake_case	,       snake_case	,       atol=1E-5						)






							def        lowerCamelCase_   (  self						)			->							Union[str, Any]:
									_UpperCAmelCase							 =		ModelForTest()
									_UpperCAmelCase							 =		torch.randn(2	,       3						)
									_UpperCAmelCase							 =		test_model(snake_case						)

									_UpperCAmelCase							 =		PostForwardHook()
									add_hook_to_module(snake_case	,       snake_case						)
									_UpperCAmelCase							 =		test_model(snake_case						)
									self.assertTrue(torch.allclose(snake_case	,       output + 1	,       atol=1E-5						)						)

									# Attaching a hook to a model when it already has one replaces, does not chain
									_UpperCAmelCase							 =		PostForwardHook()
									add_hook_to_module(snake_case	,       snake_case						)
									_UpperCAmelCase							 =		test_model(snake_case						)
									self.assertTrue(torch.allclose(snake_case	,       output + 1	,       atol=1E-5						)						)

									# You need to use the sequential hook to chain two or more hooks
									_UpperCAmelCase							 =		SequentialHook(PostForwardHook()	,       PostForwardHook()						)
									add_hook_to_module(snake_case	,       snake_case						)

									_UpperCAmelCase							 =		test_model(snake_case						)
									assert torch.allclose(snake_case	,       output + 2	,       atol=1E-5						)






							def        lowerCamelCase_   (  self						)			->							Union[str, Any]:
									_UpperCAmelCase							 =		ModelForTest()
									_UpperCAmelCase							 =		torch.randn(2	,       3						)
									_UpperCAmelCase							 =		test_model(snake_case						)

									_UpperCAmelCase							 =		PostForwardHook()
									add_hook_to_module(snake_case	,       snake_case						)
									_UpperCAmelCase							 =		test_model(snake_case						)
									self.assertTrue(torch.allclose(snake_case	,       output + 1						)						)
									self.assertTrue(outputa.requires_grad						)

									_UpperCAmelCase							 =		True
									_UpperCAmelCase							 =		test_model(snake_case						)
									self.assertFalse(outputa.requires_grad						)






							@require_multi_gpu
							def        lowerCamelCase_   (  self						)			->							str:
									_UpperCAmelCase							 =		ModelForTest()
									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)

									# This will move each submodule on different devices
									add_hook_to_module(model.lineara	,       AlignDevicesHook(execution_device=0						)						)
									add_hook_to_module(model.batchnorm	,       AlignDevicesHook(execution_device=0						)						)
									add_hook_to_module(model.lineara	,       AlignDevicesHook(execution_device=1						)						)

									self.assertEqual(model.lineara.weight.device	,       torch.device(0						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device(0						)						)
									self.assertEqual(model.batchnorm.running_mean.device	,       torch.device(0						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device(1						)						)

									# We can still make a forward pass. The input does not need to be on any particular device
									_UpperCAmelCase							 =		torch.randn(2	,       3						)
									_UpperCAmelCase							 =		model(snake_case						)
									self.assertEqual(output.device	,       torch.device(1						)						)

									# We can add a general hook to put back output on same device as input.
									add_hook_to_module(snake_case	,       AlignDevicesHook(io_same_device=snake_case						)						)
									_UpperCAmelCase							 =		torch.randn(2	,       3						).to(0						)
									_UpperCAmelCase							 =		model(snake_case						)
									self.assertEqual(output.device	,       torch.device(0						)						)






							def        lowerCamelCase_   (  self						)			->							int:
									_UpperCAmelCase							 =		ModelForTest()

									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)

									# This will move each submodule on different devices
									_UpperCAmelCase							 =		{'execution_device': 0 if torch.cuda.is_available() else 'cpu', 'offload': True}

									add_hook_to_module(model.lineara	,       AlignDevicesHook(**snake_case						)						)
									add_hook_to_module(model.batchnorm	,       AlignDevicesHook(**snake_case						)						)
									add_hook_to_module(model.lineara	,       AlignDevicesHook(**snake_case						)						)

									# Parameters have been offloaded, so on the meta device
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									# Buffers are not included in the offload by default, so are on the execution device
									_UpperCAmelCase							 =		torch.device(hook_kwargs['execution_device']						)
									self.assertEqual(model.batchnorm.running_mean.device	,       snake_case						)

									_UpperCAmelCase							 =		torch.randn(2	,       3						)
									_UpperCAmelCase							 =		model(snake_case						)
									self.assertEqual(output.device	,       snake_case						)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_module(model.lineara						)
									remove_hook_from_module(model.batchnorm						)
									remove_hook_from_module(model.lineara						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)

									# Now test with buffers included in the offload
									_UpperCAmelCase							 =		{
									    'execution_device': 0 if torch.cuda.is_available() else 'cpu',
									    'offload': True,
									    'offload_buffers': True,
									}

									add_hook_to_module(model.lineara	,       AlignDevicesHook(**snake_case						)						)
									add_hook_to_module(model.batchnorm	,       AlignDevicesHook(**snake_case						)						)
									add_hook_to_module(model.lineara	,       AlignDevicesHook(**snake_case						)						)

									# Parameters have been offloaded, so on the meta device, buffers included
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.batchnorm.running_mean.device	,       torch.device('meta'						)						)

									_UpperCAmelCase							 =		torch.randn(2	,       3						)
									_UpperCAmelCase							 =		model(snake_case						)
									self.assertEqual(output.device	,       snake_case						)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_module(model.lineara						)
									remove_hook_from_module(model.batchnorm						)
									remove_hook_from_module(model.lineara						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)






							def        lowerCamelCase_   (  self						)			->							List[Any]:
									_UpperCAmelCase							 =		ModelForTest()

									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)

									# This will move each submodule on different devices
									_UpperCAmelCase							 =		0 if torch.cuda.is_available() else 'cpu'
									attach_align_device_hook(snake_case	,       execution_device=snake_case	,       offload=snake_case						)

									# Parameters have been offloaded, so on the meta device
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									# Buffers are not included in the offload by default, so are on the execution device
									_UpperCAmelCase							 =		torch.device(snake_case						)
									self.assertEqual(model.batchnorm.running_mean.device	,       snake_case						)

									_UpperCAmelCase							 =		torch.randn(2	,       3						)
									_UpperCAmelCase							 =		model(snake_case						)
									self.assertEqual(output.device	,       snake_case						)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(snake_case						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)

									# Now test with buffers included in the offload
									attach_align_device_hook(snake_case	,       execution_device=snake_case	,       offload=snake_case	,       offload_buffers=snake_case						)

									# Parameters have been offloaded, so on the meta device, buffers included
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.batchnorm.running_mean.device	,       torch.device('meta'						)						)

									_UpperCAmelCase							 =		torch.randn(2	,       3						)
									_UpperCAmelCase							 =		model(snake_case						)
									self.assertEqual(output.device	,       snake_case						)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(snake_case						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)






							def        lowerCamelCase_   (  self						)			->							int:
									_UpperCAmelCase							 =		ModelForTest()

									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)

									# This will move each submodule on different devices
									_UpperCAmelCase							 =		0 if torch.cuda.is_available() else 'cpu'
									attach_align_device_hook(
									    snake_case	,       execution_device=snake_case	,       offload=snake_case	,       weights_map=model.state_dict()						)

									# Parameters have been offloaded, so on the meta device
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									# Buffers are not included in the offload by default, so are on the execution device
									_UpperCAmelCase							 =		torch.device(snake_case						)
									self.assertEqual(model.batchnorm.running_mean.device	,       snake_case						)

									_UpperCAmelCase							 =		torch.randn(2	,       3						)
									_UpperCAmelCase							 =		model(snake_case						)
									self.assertEqual(output.device	,       snake_case						)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(snake_case						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)

									# Now test with buffers included in the offload
									attach_align_device_hook(
									    snake_case	,       execution_device=snake_case	,       offload=snake_case	,       weights_map=model.state_dict()	,       offload_buffers=snake_case	,       )

									# Parameters have been offloaded, so on the meta device, buffers included
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('meta'						)						)
									self.assertEqual(model.batchnorm.running_mean.device	,       torch.device('meta'						)						)

									_UpperCAmelCase							 =		torch.randn(2	,       3						)
									_UpperCAmelCase							 =		model(snake_case						)
									self.assertEqual(output.device	,       snake_case						)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(snake_case						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.batchnorm.weight.device	,       torch.device('cpu'						)						)
									self.assertEqual(model.lineara.weight.device	,       torch.device('cpu'						)						)







"""simple docstring"""





from itertools import permutations




def 	UpperCAmelCase     (		A	:						tuple       ):

		'''simple docstring'''


		if num[3] % 2 != 0:
				return False

		if (num[2] + num[3] + num[4]) % 3 != 0:
				return False

		if num[5] % 5 != 0:
				return False

		_UpperCAmelCase							 =		[7, 11, 13, 17]
		for i, test in enumerate(A       ):
				if (num[i + 4] * 100 + num[i + 5] * 10 + num[i + 6]) % test != 0:
						return False
		return True




def 	UpperCAmelCase     (		A	:						int = 10       ):

		'''simple docstring'''


		return sum(
		    int(''.join(map(A     ,				A       )       )       )
		    for num in permutations(range(A       )       )
		    if is_substring_divisible(A       )       )


if __name__ == "__main__":
					print(F'''{solution() = }''')


