


"""simple docstring"""


import json
import os
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from requests.exceptions import HTTPError

from transformers.utils import (
    CONFIG_NAME,
    FLAX_WEIGHTS_NAME,
    TF2_WEIGHTS_NAME,
    TRANSFORMERS_CACHE,
    WEIGHTS_NAME,
    cached_file,
    get_file_from_repo,
    has_file,
)


__lowerCamelCase      :int	    =		'hf-internal-testing/tiny-random-bert'
__lowerCamelCase      :Tuple	    =		os.path.join(TRANSFORMERS_CACHE, 'models--hf-internal-testing--tiny-random-bert')
__lowerCamelCase      :List[str]	    =		'9b8c223d42b2188cb49d29af482996f9d0f3e5a6'


class 			A__	(						unittest.TestCase):


     """simple docstring"""

     def 		a__						(	self:						List[str]	)->    Optional[int]:
           lowerCamelCase					:					List[str]        =	cached_file(UpperCAmelCase_     ,		UpperCAmelCase_	)
           # Should have downloaded the file in here
           self.assertTrue(os.path.isdir(UpperCAmelCase_	)	)
           # Cache should contain at least those three subfolders:
           for subfolder in ["blobs", "refs", "snapshots"]:
                 self.assertTrue(os.path.isdir(os.path.join(UpperCAmelCase_     ,		UpperCAmelCase_	)	)	)
           with open(os.path.join(UpperCAmelCase_     ,		"""refs"""     ,		"""main"""	)	) as f:
                 lowerCamelCase					:					List[str]        =	f.read()
           self.assertEqual(UpperCAmelCase_     ,		os.path.join(UpperCAmelCase_     ,		"""snapshots"""     ,		UpperCAmelCase_     ,		UpperCAmelCase_	)	)
           self.assertTrue(os.path.isfile(UpperCAmelCase_	)	)

           # File is cached at the same place the second time.
           lowerCamelCase					:					str        =	cached_file(UpperCAmelCase_     ,		UpperCAmelCase_	)
           self.assertEqual(UpperCAmelCase_     ,		UpperCAmelCase_	)

           # Using a specific revision to test the full commit hash.
           lowerCamelCase					:					Dict        =	cached_file(UpperCAmelCase_     ,		UpperCAmelCase_     ,		revision="""9b8c223"""	)
           self.assertEqual(UpperCAmelCase_     ,		os.path.join(UpperCAmelCase_     ,		"""snapshots"""     ,		UpperCAmelCase_     ,		UpperCAmelCase_	)	)

     def 		a__						(	self:						Union[str, Any]	)->    Optional[int]:
           with self.assertRaisesRegex(UpperCAmelCase_     ,		"""is not a valid model identifier"""	):
                 lowerCamelCase					:					int        =	cached_file("""tiny-random-bert"""     ,		UpperCAmelCase_	)

           with self.assertRaisesRegex(UpperCAmelCase_     ,		"""is not a valid git identifier"""	):
                 lowerCamelCase					:					Tuple        =	cached_file(UpperCAmelCase_     ,		UpperCAmelCase_     ,		revision="""aaaa"""	)

           with self.assertRaisesRegex(UpperCAmelCase_     ,		"""does not appear to have a file named"""	):
                 lowerCamelCase					:					List[str]        =	cached_file(UpperCAmelCase_     ,		"""conf"""	)

     def 		a__						(	self:						Union[str, Any]	)->    Optional[int]:
           with self.assertRaisesRegex(UpperCAmelCase_     ,		"""does not appear to have a file named"""	):
                 lowerCamelCase					:					str        =	cached_file(UpperCAmelCase_     ,		"""conf"""	)

           with open(os.path.join(UpperCAmelCase_     ,		"""refs"""     ,		"""main"""	)	) as f:
                 lowerCamelCase					:					Tuple        =	f.read()
           self.assertTrue(os.path.isfile(os.path.join(UpperCAmelCase_     ,		""".no_exist"""     ,		UpperCAmelCase_     ,		"""conf"""	)	)	)

           lowerCamelCase					:					List[str]        =	cached_file(UpperCAmelCase_     ,		"""conf"""     ,		_raise_exceptions_for_missing_entries=UpperCAmelCase_	)
           self.assertIsNone(UpperCAmelCase_	)

           lowerCamelCase					:					Dict        =	cached_file(UpperCAmelCase_     ,		"""conf"""     ,		local_files_only=UpperCAmelCase_     ,		_raise_exceptions_for_missing_entries=UpperCAmelCase_	)
           self.assertIsNone(UpperCAmelCase_	)

           lowerCamelCase					:					Tuple        =	mock.Mock()
           lowerCamelCase					:					Any        =	500
           lowerCamelCase					:					Dict        =	{}
           lowerCamelCase					:					Optional[int]        =	HTTPError
           lowerCamelCase					:					Union[str, Any]        =	{}

           # Under the mock environment we get a 500 error when trying to reach the tokenizer.
           with mock.patch("""requests.Session.request"""     ,		return_value=UpperCAmelCase_	) as mock_head:
                 lowerCamelCase					:					str        =	cached_file(UpperCAmelCase_     ,		"""conf"""     ,		_raise_exceptions_for_connection_errors=UpperCAmelCase_	)
                 self.assertIsNone(UpperCAmelCase_	)
                 # This check we did call the fake head request
                 mock_head.assert_called()

     def 		a__						(	self:						str	)->    Any:
           self.assertTrue(has_file("""hf-internal-testing/tiny-bert-pt-only"""     ,		UpperCAmelCase_	)	)
           self.assertFalse(has_file("""hf-internal-testing/tiny-bert-pt-only"""     ,		UpperCAmelCase_	)	)
           self.assertFalse(has_file("""hf-internal-testing/tiny-bert-pt-only"""     ,		UpperCAmelCase_	)	)

     def 		a__						(	self:						int	)->    Dict:
           # `get_file_from_repo` returns None if the file does not exist
           self.assertIsNone(get_file_from_repo("""bert-base-cased"""     ,		"""ahah.txt"""	)	)

           # The function raises if the repository does not exist.
           with self.assertRaisesRegex(UpperCAmelCase_     ,		"""is not a valid model identifier"""	):
                 get_file_from_repo("""bert-base-case"""     ,		UpperCAmelCase_	)

           # The function raises if the revision does not exist.
           with self.assertRaisesRegex(UpperCAmelCase_     ,		"""is not a valid git identifier"""	):
                 get_file_from_repo("""bert-base-cased"""     ,		UpperCAmelCase_     ,		revision="""ahaha"""	)

           lowerCamelCase					:					str        =	get_file_from_repo("""bert-base-cased"""     ,		UpperCAmelCase_	)
           # The name is the cached name which is not very easy to test, so instead we load the content.
           lowerCamelCase					:					int        =	json.loads(open(UpperCAmelCase_     ,		"""r"""	).read()	)
           self.assertEqual(config["""hidden_size"""]     ,		768	)







     def 		a__						(	self:						Optional[Any]	)->    str:
           with tempfile.TemporaryDirectory() as tmp_dir:
                 lowerCamelCase					:					int        =	Path(UpperCAmelCase_	) / """a.txt"""
                 filename.touch()
                 self.assertEqual(get_file_from_repo(UpperCAmelCase_     ,		"""a.txt"""	)     ,		str(UpperCAmelCase_	)	)

                 self.assertIsNone(get_file_from_repo(UpperCAmelCase_     ,		"""b.txt"""	)	)








"""simple docstring"""


import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu


class 			A__	(						nn.Module):


     """simple docstring"""

     def __init__(	self:						Dict	)->    Dict:
           super().__init__()
           lowerCamelCase					:					Tuple        =	nn.Linear(3     ,		4	)
           lowerCamelCase					:					Optional[Any]        =	nn.BatchNormad(4	)
           lowerCamelCase					:					Optional[Any]        =	nn.Linear(4     ,		5	)







     def 		a__						(	self:						List[str]     ,		__a:						List[Any]	)->    Optional[Any]:
           return self.lineara(self.batchnorm(self.lineara(__a	)	)	)


class 			A__	(						__lowercase):


     """simple docstring"""

     def 		a__						(	self:						Tuple     ,		__a:						int     ,		*__a:						Any     ,		**__a:						Tuple	)->    Tuple:
           return (args[0] + 1,) + args[1:], kwargs


class 			A__	(						__lowercase):


     """simple docstring"""

     def 		a__						(	self:						Optional[int]     ,		__a:						List[str]     ,		__a:						List[Any]	)->    List[str]:
           return output + 1


class 			A__	(						unittest.TestCase):


     """simple docstring"""

     def 		a__						(	self:						int	)->    str:
           lowerCamelCase					:					List[str]        =	ModelForTest()
           lowerCamelCase					:					Dict        =	ModelHook()

           add_hook_to_module(__a     ,		__a	)
           self.assertEqual(test_model._hf_hook     ,		__a	)
           self.assertTrue(hasattr(__a     ,		"""_old_forward"""	)	)

           # Check adding the hook did not change the name or the signature
           self.assertEqual(test_model.forward.__name__     ,		"""forward"""	)
           self.assertListEqual(list(inspect.signature(test_model.forward	).parameters	)     ,		["""x"""]	)

           remove_hook_from_module(__a	)
           self.assertFalse(hasattr(__a     ,		"""_hf_hook"""	)	)
           self.assertFalse(hasattr(__a     ,		"""_old_forward"""	)	)

     def 		a__						(	self:						int	)->    str:
           lowerCamelCase					:					List[str]        =	ModelForTest()
           lowerCamelCase					:					Union[str, Any]        =	ModelHook()

           add_hook_to_module(__a     ,		__a	)
           add_hook_to_module(__a     ,		__a     ,		append=__a	)

           self.assertEqual(isinstance(test_model._hf_hook     ,		__a	)     ,		__a	)
           self.assertEqual(len(test_model._hf_hook.hooks	)     ,		2	)
           self.assertTrue(hasattr(__a     ,		"""_old_forward"""	)	)

           # Check adding the hook did not change the name or the signature
           self.assertEqual(test_model.forward.__name__     ,		"""forward"""	)
           self.assertListEqual(list(inspect.signature(test_model.forward	).parameters	)     ,		["""x"""]	)

           remove_hook_from_module(__a	)
           self.assertFalse(hasattr(__a     ,		"""_hf_hook"""	)	)
           self.assertFalse(hasattr(__a     ,		"""_old_forward"""	)	)

     def 		a__						(	self:						List[Any]	)->    List[str]:
           lowerCamelCase					:					str        =	ModelForTest()
           lowerCamelCase					:					Dict        =	torch.randn(2     ,		3	)
           lowerCamelCase					:					Union[str, Any]        =	test_model(x + 1	)
           lowerCamelCase					:					Optional[int]        =	test_model(x + 2	)

           lowerCamelCase					:					List[Any]        =	PreForwardHook()
           add_hook_to_module(__a     ,		__a	)
           lowerCamelCase					:					Optional[int]        =	test_model(__a	)
           self.assertTrue(torch.allclose(__a     ,		__a     ,		atol=1e-5	)	)

           # Attaching a hook to a model when it already has one replaces, does not chain
           lowerCamelCase					:					Dict        =	PreForwardHook()
           add_hook_to_module(__a     ,		__a	)
           lowerCamelCase					:					Tuple        =	test_model(__a	)
           self.assertTrue(torch.allclose(__a     ,		__a     ,		atol=1e-5	)	)

           # You need to use the sequential hook to chain two or more hooks
           lowerCamelCase					:					Any        =	SequentialHook(PreForwardHook()     ,		PreForwardHook()	)
           add_hook_to_module(__a     ,		__a	)

           lowerCamelCase					:					Optional[Any]        =	test_model(__a	)
           assert torch.allclose(__a     ,		__a     ,		atol=1e-5	)

     def 		a__						(	self:						Any	)->    Optional[int]:
           lowerCamelCase					:					str        =	ModelForTest()
           lowerCamelCase					:					List[str]        =	torch.randn(2     ,		3	)
           lowerCamelCase					:					int        =	test_model(__a	)

           lowerCamelCase					:					Dict        =	PostForwardHook()
           add_hook_to_module(__a     ,		__a	)
           lowerCamelCase					:					Tuple        =	test_model(__a	)
           self.assertTrue(torch.allclose(__a     ,		output + 1     ,		atol=1e-5	)	)

           # Attaching a hook to a model when it already has one replaces, does not chain
           lowerCamelCase					:					str        =	PostForwardHook()
           add_hook_to_module(__a     ,		__a	)
           lowerCamelCase					:					Optional[Any]        =	test_model(__a	)
           self.assertTrue(torch.allclose(__a     ,		output + 1     ,		atol=1e-5	)	)

           # You need to use the sequential hook to chain two or more hooks
           lowerCamelCase					:					Union[str, Any]        =	SequentialHook(PostForwardHook()     ,		PostForwardHook()	)
           add_hook_to_module(__a     ,		__a	)

           lowerCamelCase					:					str        =	test_model(__a	)
           assert torch.allclose(__a     ,		output + 2     ,		atol=1e-5	)

     def 		a__						(	self:						int	)->    Dict:
           lowerCamelCase					:					List[Any]        =	ModelForTest()
           lowerCamelCase					:					Optional[int]        =	torch.randn(2     ,		3	)
           lowerCamelCase					:					List[str]        =	test_model(__a	)

           lowerCamelCase					:					Any        =	PostForwardHook()
           add_hook_to_module(__a     ,		__a	)
           lowerCamelCase					:					str        =	test_model(__a	)
           self.assertTrue(torch.allclose(__a     ,		output + 1	)	)
           self.assertTrue(outputa.requires_grad	)

           lowerCamelCase					:					Optional[int]        =	True
           lowerCamelCase					:					Optional[int]        =	test_model(__a	)
           self.assertFalse(outputa.requires_grad	)

     @require_multi_gpu
     def 		a__						(	self:						List[str]	)->    Union[str, Any]:
           lowerCamelCase					:					int        =	ModelForTest()
           # Everything is on CPU
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)

           # This will move each submodule on different devices
           add_hook_to_module(model.lineara     ,		AlignDevicesHook(execution_device=0	)	)
           add_hook_to_module(model.batchnorm     ,		AlignDevicesHook(execution_device=0	)	)
           add_hook_to_module(model.lineara     ,		AlignDevicesHook(execution_device=1	)	)

           self.assertEqual(model.lineara.weight.device     ,		torch.device(0	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device(0	)	)
           self.assertEqual(model.batchnorm.running_mean.device     ,		torch.device(0	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device(1	)	)

           # We can still make a forward pass. The input does not need to be on any particular device
           lowerCamelCase					:					str        =	torch.randn(2     ,		3	)
           lowerCamelCase					:					Dict        =	model(__a	)
           self.assertEqual(output.device     ,		torch.device(1	)	)

           # We can add a general hook to put back output on same device as input.
           add_hook_to_module(__a     ,		AlignDevicesHook(io_same_device=__a	)	)
           lowerCamelCase					:					Optional[int]        =	torch.randn(2     ,		3	).to(0	)
           lowerCamelCase					:					str        =	model(__a	)
           self.assertEqual(output.device     ,		torch.device(0	)	)

     def 		a__						(	self:						List[str]	)->    Tuple:
           lowerCamelCase					:					Union[str, Any]        =	ModelForTest()

           # Everything is on CPU
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)

           # This will move each submodule on different devices
           lowerCamelCase					:					Tuple        =	{"""execution_device""": 0 if torch.cuda.is_available() else """cpu""", """offload""": True}

           add_hook_to_module(model.lineara     ,		AlignDevicesHook(**__a	)	)
           add_hook_to_module(model.batchnorm     ,		AlignDevicesHook(**__a	)	)
           add_hook_to_module(model.lineara     ,		AlignDevicesHook(**__a	)	)

           # Parameters have been offloaded, so on the meta device
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           # Buffers are not included in the offload by default, so are on the execution device
           lowerCamelCase					:					List[Any]        =	torch.device(hook_kwargs["""execution_device"""]	)
           self.assertEqual(model.batchnorm.running_mean.device     ,		__a	)

           lowerCamelCase					:					Optional[Any]        =	torch.randn(2     ,		3	)
           lowerCamelCase					:					Optional[Any]        =	model(__a	)
           self.assertEqual(output.device     ,		__a	)

           # Removing hooks loads back the weights in the model.
           remove_hook_from_module(model.lineara	)
           remove_hook_from_module(model.batchnorm	)
           remove_hook_from_module(model.lineara	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)

           # Now test with buffers included in the offload
           lowerCamelCase					:					Any        =	{
               """execution_device""": 0 if torch.cuda.is_available() else """cpu""",
               """offload""": True,
               """offload_buffers""": True,
           }

           add_hook_to_module(model.lineara     ,		AlignDevicesHook(**__a	)	)
           add_hook_to_module(model.batchnorm     ,		AlignDevicesHook(**__a	)	)
           add_hook_to_module(model.lineara     ,		AlignDevicesHook(**__a	)	)

           # Parameters have been offloaded, so on the meta device, buffers included
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.batchnorm.running_mean.device     ,		torch.device("""meta"""	)	)

           lowerCamelCase					:					int        =	torch.randn(2     ,		3	)
           lowerCamelCase					:					Optional[int]        =	model(__a	)
           self.assertEqual(output.device     ,		__a	)

           # Removing hooks loads back the weights in the model.
           remove_hook_from_module(model.lineara	)
           remove_hook_from_module(model.batchnorm	)
           remove_hook_from_module(model.lineara	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)

     def 		a__						(	self:						Any	)->    List[str]:
           lowerCamelCase					:					int        =	ModelForTest()

           # Everything is on CPU
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)

           # This will move each submodule on different devices
           lowerCamelCase					:					int        =	0 if torch.cuda.is_available() else """cpu"""
           attach_align_device_hook(__a     ,		execution_device=__a     ,		offload=__a	)

           # Parameters have been offloaded, so on the meta device
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           # Buffers are not included in the offload by default, so are on the execution device
           lowerCamelCase					:					List[Any]        =	torch.device(__a	)
           self.assertEqual(model.batchnorm.running_mean.device     ,		__a	)

           lowerCamelCase					:					Dict        =	torch.randn(2     ,		3	)
           lowerCamelCase					:					Optional[Any]        =	model(__a	)
           self.assertEqual(output.device     ,		__a	)

           # Removing hooks loads back the weights in the model.
           remove_hook_from_submodules(__a	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)

           # Now test with buffers included in the offload
           attach_align_device_hook(__a     ,		execution_device=__a     ,		offload=__a     ,		offload_buffers=__a	)

           # Parameters have been offloaded, so on the meta device, buffers included
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.batchnorm.running_mean.device     ,		torch.device("""meta"""	)	)

           lowerCamelCase					:					Optional[int]        =	torch.randn(2     ,		3	)
           lowerCamelCase					:					int        =	model(__a	)
           self.assertEqual(output.device     ,		__a	)

           # Removing hooks loads back the weights in the model.
           remove_hook_from_submodules(__a	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)







     def 		a__						(	self:						Optional[Any]	)->    List[Any]:
           lowerCamelCase					:					List[Any]        =	ModelForTest()

           # Everything is on CPU
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)

           # This will move each submodule on different devices
           lowerCamelCase					:					Any        =	0 if torch.cuda.is_available() else """cpu"""
           attach_align_device_hook(
               __a     ,		execution_device=__a     ,		offload=__a     ,		weights_map=model.state_dict()	)

           # Parameters have been offloaded, so on the meta device
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           # Buffers are not included in the offload by default, so are on the execution device
           lowerCamelCase					:					List[Any]        =	torch.device(__a	)
           self.assertEqual(model.batchnorm.running_mean.device     ,		__a	)

           lowerCamelCase					:					Dict        =	torch.randn(2     ,		3	)
           lowerCamelCase					:					int        =	model(__a	)
           self.assertEqual(output.device     ,		__a	)

           # Removing hooks loads back the weights in the model.
           remove_hook_from_submodules(__a	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)

           # Now test with buffers included in the offload
           attach_align_device_hook(
               __a     ,		execution_device=__a     ,		offload=__a     ,		weights_map=model.state_dict()     ,		offload_buffers=__a     ,		)

           # Parameters have been offloaded, so on the meta device, buffers included
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""meta"""	)	)
           self.assertEqual(model.batchnorm.running_mean.device     ,		torch.device("""meta"""	)	)

           lowerCamelCase					:					Tuple        =	torch.randn(2     ,		3	)
           lowerCamelCase					:					Any        =	model(__a	)
           self.assertEqual(output.device     ,		__a	)

           # Removing hooks loads back the weights in the model.
           remove_hook_from_submodules(__a	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.batchnorm.weight.device     ,		torch.device("""cpu"""	)	)
           self.assertEqual(model.lineara.weight.device     ,		torch.device("""cpu"""	)	)




