





'''simple docstring'''



import gc
import unittest

import torch
from parameterized import parameterized

from diffusers import AutoencoderKL
from diffusers.utils import floats_tensor, load_hf_numpy, require_torch_gpu, slow, torch_all_close, torch_device
from diffusers.utils.import_utils import is_xformers_available
from diffusers.utils.testing_utils import enable_full_determinism

from .test_modeling_common import ModelTesterMixin, UNetTesterMixin


enable_full_determinism()






class 							__lowercase       (      lowerCAmelCase__ ,      lowerCAmelCase__ ,      unittest.TestCase						):







	'''simple docstring'''

	a    :				str											=       AutoencoderKL
	a    :				Dict											=       "sample"
	a    :				List[str]											=       1e-2


	@property
	def 						_UpperCAmelCase     (self )							->							Optional[Any]:







				'''simple docstring'''





				__lowercase										=							4
				__lowercase										=							3
				__lowercase										=							(32, 32)

				__lowercase										=							floats_tensor((batch_size, num_channels) + sizes ).to(_lowerCamelCase )

				return {"sample": image}


	@property
	def 						_UpperCAmelCase     (self )							->							Tuple:







				'''simple docstring'''





				return (3, 32, 32)


	@property
	def 						_UpperCAmelCase     (self )							->							Any:







				'''simple docstring'''





				return (3, 32, 32)


	def 						_UpperCAmelCase     (self )							->							List[Any]:







				'''simple docstring'''





				__lowercase										=							{
				    '''block_out_channels''': [32, 64],
				    '''in_channels''': 3,
				    '''out_channels''': 3,
				    '''down_block_types''': ['''DownEncoderBlock2D''', '''DownEncoderBlock2D'''],
				    '''up_block_types''': ['''UpDecoderBlock2D''', '''UpDecoderBlock2D'''],
				    '''latent_channels''': 4,
				}
				__lowercase										=							self.dummy_input
				return init_dict, inputs_dict


	def 						_UpperCAmelCase     (self )							->							Any:







				'''simple docstring'''





				pass


	def 						_UpperCAmelCase     (self )							->							Tuple:







				'''simple docstring'''





				pass


	@unittest.skipIf(torch_device == '''mps'''     ,'''Gradient checkpointing skipped on MPS''' )
	def 						_UpperCAmelCase     (self )							->							Any:







				'''simple docstring'''





				__lowercase						,		__lowercase										=							self.prepare_init_args_and_inputs_for_common()
				__lowercase										=							self.model_class(**_lowerCamelCase )
				model.to(_lowerCamelCase )

				assert not model.is_gradient_checkpointing and model.training

				__lowercase										=							model(**_lowerCamelCase ).sample
				# run the backwards pass on the model. For backwards pass, for simplicity purpose,
				# we won't calculate the loss and rather backprop on out.sum()
				model.zero_grad()

				__lowercase										=							torch.randn_like(_lowerCamelCase )
				__lowercase										=							(out - labels).mean()
				loss.backward()

				# re-instantiate the model now enabling gradient checkpointing
				__lowercase										=							self.model_class(**_lowerCamelCase )
				# clone model
				model_a.load_state_dict(model.state_dict() )
				model_a.to(_lowerCamelCase )
				model_a.enable_gradient_checkpointing()

				assert model_a.is_gradient_checkpointing and model_a.training

				__lowercase										=							model_a(**_lowerCamelCase ).sample
				# run the backwards pass on the model. For backwards pass, for simplicity purpose,
				# we won't calculate the loss and rather backprop on out.sum()
				model_a.zero_grad()
				__lowercase										=							(out_a - labels).mean()
				loss_a.backward()

				# compare the output and parameters gradients
				self.assertTrue((loss - loss_a).abs() < 1E-5 )
				__lowercase										=							dict(model.named_parameters() )
				__lowercase										=							dict(model_a.named_parameters() )
				for name, param in named_params.items():
							self.assertTrue(torch_all_close(param.grad.data     ,named_params_a[name].grad.data     ,atol=5E-5 ) )


	def 						_UpperCAmelCase     (self )							->							List[str]:







				'''simple docstring'''





				__lowercase						,		__lowercase										=							AutoencoderKL.from_pretrained('''fusing/autoencoder-kl-dummy'''     ,output_loading_info=_lowerCamelCase )
				self.assertIsNotNone(_lowerCamelCase )
				self.assertEqual(len(loading_info['''missing_keys'''] )     ,0 )

				model.to(_lowerCamelCase )
				__lowercase										=							model(**self.dummy_input )

				assert image is not None, "Make sure output is not None"


	def 						_UpperCAmelCase     (self )							->							str:







				'''simple docstring'''





				__lowercase										=							AutoencoderKL.from_pretrained('''fusing/autoencoder-kl-dummy''' )
				__lowercase										=							model.to(_lowerCamelCase )
				model.eval()

				if torch_device == "mps":
							__lowercase										=							torch.manual_seed(0 )
				else:
							__lowercase										=							torch.Generator(device=_lowerCamelCase ).manual_seed(0 )

				__lowercase										=							torch.randn(
				    1     ,model.config.in_channels     ,model.config.sample_size     ,model.config.sample_size     ,generator=torch.manual_seed(0 )     ,)
				__lowercase										=							image.to(_lowerCamelCase )
				with torch.no_grad():
							__lowercase										=							model(_lowerCamelCase     ,sample_posterior=_lowerCamelCase     ,generator=_lowerCamelCase ).sample

				__lowercase										=							output[0, -1, -3:, -3:].flatten().cpu()

				# Since the VAE Gaussian prior's generator is seeded on the appropriate device,
				# the expected output slices are not the same for CPU and GPU.
				if torch_device == "mps":
							__lowercase										=							torch.tensor(
							    [
							        -4.0_0_7_8E-0_1,
							        -3.8_3_2_3E-0_4,
							        -1.2_6_8_1E-0_1,
							        -1.1_4_6_2E-0_1,
							        2.0_0_9_5E-0_1,
							        1.0_8_9_3E-0_1,
							        -8.8_2_4_7E-0_2,
							        -3.0_3_6_1E-0_1,
							        -9.8_6_4_4E-0_3,
							    ] )
				elif torch_device == "cpu":
							__lowercase										=							torch.tensor(
							    [-0.1_3_5_2, 0.0_8_7_8, 0.0_4_1_9, -0.0_8_1_8, -0.1_0_6_9, 0.0_6_8_8, -0.1_4_5_8, -0.4_4_4_6, -0.0_0_2_6] )
				else:
							__lowercase										=							torch.tensor(
							    [-0.2_4_2_1, 0.4_6_4_2, 0.2_5_0_7, -0.0_4_3_8, 0.0_6_8_2, 0.3_1_6_0, -0.2_0_1_8, -0.0_7_2_7, 0.2_4_8_5] )

				self.assertTrue(torch_all_close(_lowerCamelCase     ,_lowerCamelCase     ,rtol=1E-2 ) )






@slow
class 							__lowercase       (      unittest.TestCase						):







	'''simple docstring'''

	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase )							->							Optional[int]:







				'''simple docstring'''





				return f"gaussian_noise_s={seed}_shape={'_'.join([str(_lowerCamelCase ) for s in shape] )}.npy"


	def 						_UpperCAmelCase     (self )							->							Tuple:







				'''simple docstring'''





				super().tearDown()
				gc.collect()
				torch.cuda.empty_cache()


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase=0     ,_lowerCamelCase=(4, 3, 512, 512)     ,_lowerCamelCase=False )							->							str:







				'''simple docstring'''





				__lowercase										=							torch.floataa if fpaa else torch.floataa
				__lowercase										=							torch.from_numpy(load_hf_numpy(self.get_file_format(_lowerCamelCase     ,_lowerCamelCase ) ) ).to(_lowerCamelCase ).to(_lowerCamelCase )
				return image


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase="CompVis/stable-diffusion-v1-4"     ,_lowerCamelCase=False )							->							List[str]:







				'''simple docstring'''





				__lowercase										=							'''fp16''' if fpaa else None
				__lowercase										=							torch.floataa if fpaa else torch.floataa

				__lowercase										=							AutoencoderKL.from_pretrained(
				    _lowerCamelCase     ,subfolder='''vae'''     ,torch_dtype=_lowerCamelCase     ,revision=_lowerCamelCase     ,)
				model.to(_lowerCamelCase ).eval()

				return model


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase=0 )							->							Optional[Any]:







				'''simple docstring'''





				if torch_device == "mps":
							return torch.manual_seed(_lowerCamelCase )
				return torch.Generator(device=_lowerCamelCase ).manual_seed(_lowerCamelCase )


	@parameterized.expand(
	    [
	        # fmt: off
	        [33, [-0.1_6_0_3, 0.9_8_7_8, -0.0_4_9_5, -0.0_7_9_0, -0.2_7_0_9, 0.8_3_7_5, -0.2_0_6_0, -0.0_8_2_4], [-0.2_3_9_5, 0.0_0_9_8, 0.0_1_0_2, -0.0_7_0_9, -0.2_8_4_0, -0.0_2_7_4, -0.0_7_1_8, -0.1_8_2_4]],
	        [47, [-0.2_3_7_6, 0.1_1_6_8, 0.1_3_3_2, -0.4_8_4_0, -0.2_5_0_8, -0.0_7_9_1, -0.0_4_9_3, -0.4_0_8_9], [0.0_3_5_0, 0.0_8_4_7, 0.0_4_6_7, 0.0_3_4_4, -0.0_8_4_2, -0.0_5_4_7, -0.0_6_3_3, -0.1_1_3_1]],
	        # fmt: on
	    ] )
	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase     ,_lowerCamelCase )							->							List[Any]:







				'''simple docstring'''





				__lowercase										=							self.get_sd_vae_model()
				__lowercase										=							self.get_sd_image(_lowerCamelCase )
				__lowercase										=							self.get_generator(_lowerCamelCase )

				with torch.no_grad():
							__lowercase										=							model(_lowerCamelCase     ,generator=_lowerCamelCase     ,sample_posterior=_lowerCamelCase ).sample

				assert sample.shape == image.shape

				__lowercase										=							sample[-1, -2:, -2:, :2].flatten().float().cpu()
				__lowercase										=							torch.tensor(expected_slice_mps if torch_device == '''mps''' else expected_slice )

				assert torch_all_close(_lowerCamelCase     ,_lowerCamelCase     ,atol=3E-3 )


	@parameterized.expand(
	    [
	        # fmt: off
	        [33, [-0.0_5_1_3, 0.0_2_8_9, 1.3_7_9_9, 0.2_1_6_6, -0.2_5_7_3, -0.0_8_7_1, 0.5_1_0_3, -0.0_9_9_9]],
	        [47, [-0.4_1_2_8, -0.1_3_2_0, -0.3_7_0_4, 0.1_9_6_5, -0.4_1_1_6, -0.2_3_3_2, -0.3_3_4_0, 0.2_2_4_7]],
	        # fmt: on
	    ] )
	@require_torch_gpu
	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase )							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase										=							self.get_sd_vae_model(fpaa=_lowerCamelCase )
				__lowercase										=							self.get_sd_image(_lowerCamelCase     ,fpaa=_lowerCamelCase )
				__lowercase										=							self.get_generator(_lowerCamelCase )

				with torch.no_grad():
							__lowercase										=							model(_lowerCamelCase     ,generator=_lowerCamelCase     ,sample_posterior=_lowerCamelCase ).sample

				assert sample.shape == image.shape

				__lowercase										=							sample[-1, -2:, :2, -2:].flatten().float().cpu()
				__lowercase										=							torch.tensor(_lowerCamelCase )

				assert torch_all_close(_lowerCamelCase     ,_lowerCamelCase     ,atol=1E-2 )


	@parameterized.expand(
	    [
	        # fmt: off
	        [33, [-0.1_6_0_9, 0.9_8_6_6, -0.0_4_8_7, -0.0_7_7_7, -0.2_7_1_6, 0.8_3_6_8, -0.2_0_5_5, -0.0_8_1_4], [-0.2_3_9_5, 0.0_0_9_8, 0.0_1_0_2, -0.0_7_0_9, -0.2_8_4_0, -0.0_2_7_4, -0.0_7_1_8, -0.1_8_2_4]],
	        [47, [-0.2_3_7_7, 0.1_1_4_7, 0.1_3_3_3, -0.4_8_4_1, -0.2_5_0_6, -0.0_8_0_5, -0.0_4_9_1, -0.4_0_8_5], [0.0_3_5_0, 0.0_8_4_7, 0.0_4_6_7, 0.0_3_4_4, -0.0_8_4_2, -0.0_5_4_7, -0.0_6_3_3, -0.1_1_3_1]],
	        # fmt: on
	    ] )
	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase     ,_lowerCamelCase )							->							List[str]:







				'''simple docstring'''





				__lowercase										=							self.get_sd_vae_model()
				__lowercase										=							self.get_sd_image(_lowerCamelCase )

				with torch.no_grad():
							__lowercase										=							model(_lowerCamelCase ).sample

				assert sample.shape == image.shape

				__lowercase										=							sample[-1, -2:, -2:, :2].flatten().float().cpu()
				__lowercase										=							torch.tensor(expected_slice_mps if torch_device == '''mps''' else expected_slice )

				assert torch_all_close(_lowerCamelCase     ,_lowerCamelCase     ,atol=3E-3 )


	@parameterized.expand(
	    [
	        # fmt: off
	        [13, [-0.2_0_5_1, -0.1_8_0_3, -0.2_3_1_1, -0.2_1_1_4, -0.3_2_9_2, -0.3_5_7_4, -0.2_9_5_3, -0.3_3_2_3]],
	        [37, [-0.2_6_3_2, -0.2_6_2_5, -0.2_1_9_9, -0.2_7_4_1, -0.4_5_3_9, -0.4_9_9_0, -0.3_7_2_0, -0.4_9_2_5]],
	        # fmt: on
	    ] )
	@require_torch_gpu
	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase )							->							str:







				'''simple docstring'''





				__lowercase										=							self.get_sd_vae_model()
				__lowercase										=							self.get_sd_image(_lowerCamelCase     ,shape=(3, 4, 64, 64) )

				with torch.no_grad():
							__lowercase										=							model.decode(_lowerCamelCase ).sample

				assert list(sample.shape ) == [3, 3, 512, 512]

				__lowercase										=							sample[-1, -2:, :2, -2:].flatten().cpu()
				__lowercase										=							torch.tensor(_lowerCamelCase )

				assert torch_all_close(_lowerCamelCase     ,_lowerCamelCase     ,atol=1E-3 )


	@parameterized.expand(
	    [
	        # fmt: off
	        [27, [-0.0_3_6_9, 0.0_2_0_7, -0.0_7_7_6, -0.0_6_8_2, -0.1_7_4_7, -0.1_9_3_0, -0.1_4_6_5, -0.2_0_3_9]],
	        [16, [-0.1_6_2_8, -0.2_1_3_4, -0.2_7_4_7, -0.2_6_4_2, -0.3_7_7_4, -0.4_4_0_4, -0.3_6_8_7, -0.4_2_7_7]],
	        # fmt: on
	    ] )
	@require_torch_gpu
	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase )							->							str:







				'''simple docstring'''





				__lowercase										=							self.get_sd_vae_model(fpaa=_lowerCamelCase )
				__lowercase										=							self.get_sd_image(_lowerCamelCase     ,shape=(3, 4, 64, 64)     ,fpaa=_lowerCamelCase )

				with torch.no_grad():
							__lowercase										=							model.decode(_lowerCamelCase ).sample

				assert list(sample.shape ) == [3, 3, 512, 512]

				__lowercase										=							sample[-1, -2:, :2, -2:].flatten().float().cpu()
				__lowercase										=							torch.tensor(_lowerCamelCase )

				assert torch_all_close(_lowerCamelCase     ,_lowerCamelCase     ,atol=5E-3 )


	@parameterized.expand([(13,), (16,), (27,)] )
	@require_torch_gpu
	@unittest.skipIf(not is_xformers_available()     ,reason='''xformers is not required when using PyTorch 2.0.''' )
	def 						_UpperCAmelCase     (self     ,_lowerCamelCase )							->							Any:







				'''simple docstring'''





				__lowercase										=							self.get_sd_vae_model(fpaa=_lowerCamelCase )
				__lowercase										=							self.get_sd_image(_lowerCamelCase     ,shape=(3, 4, 64, 64)     ,fpaa=_lowerCamelCase )

				with torch.no_grad():
							__lowercase										=							model.decode(_lowerCamelCase ).sample

				model.enable_xformers_memory_efficient_attention()
				with torch.no_grad():
							__lowercase										=							model.decode(_lowerCamelCase ).sample

				assert list(sample.shape ) == [3, 3, 512, 512]

				assert torch_all_close(_lowerCamelCase     ,_lowerCamelCase     ,atol=1E-1 )


	@parameterized.expand([(13,), (16,), (37,)] )
	@require_torch_gpu
	@unittest.skipIf(not is_xformers_available()     ,reason='''xformers is not required when using PyTorch 2.0.''' )
	def 						_UpperCAmelCase     (self     ,_lowerCamelCase )							->							List[str]:







				'''simple docstring'''





				__lowercase										=							self.get_sd_vae_model()
				__lowercase										=							self.get_sd_image(_lowerCamelCase     ,shape=(3, 4, 64, 64) )

				with torch.no_grad():
							__lowercase										=							model.decode(_lowerCamelCase ).sample

				model.enable_xformers_memory_efficient_attention()
				with torch.no_grad():
							__lowercase										=							model.decode(_lowerCamelCase ).sample

				assert list(sample.shape ) == [3, 3, 512, 512]

				assert torch_all_close(_lowerCamelCase     ,_lowerCamelCase     ,atol=1E-2 )


	@parameterized.expand(
	    [
	        # fmt: off
	        [33, [-0.3_0_0_1, 0.0_9_1_8, -2.6_9_8_4, -3.9_7_2_0, -3.2_0_9_9, -5.0_3_5_3, 1.7_3_3_8, -0.2_0_6_5, 3.4_2_6_7]],
	        [47, [-1.5_0_3_0, -4.3_8_7_1, -6.0_3_5_5, -9.1_1_5_7, -1.6_6_6_1, -2.7_8_5_3, 2.1_6_0_7, -5.0_8_2_3, 2.5_6_3_3]],
	        # fmt: on
	    ] )
	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase )							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase										=							self.get_sd_vae_model()
				__lowercase										=							self.get_sd_image(_lowerCamelCase )
				__lowercase										=							self.get_generator(_lowerCamelCase )

				with torch.no_grad():
							__lowercase										=							model.encode(_lowerCamelCase ).latent_dist
							__lowercase										=							dist.sample(generator=_lowerCamelCase )

				assert list(sample.shape ) == [image.shape[0], 4] + [i // 8 for i in image.shape[2:]]

				__lowercase										=							sample[0, -1, -3:, -3:].flatten().cpu()
				__lowercase										=							torch.tensor(_lowerCamelCase )

				__lowercase										=							3E-3 if torch_device != '''mps''' else 1E-2
				assert torch_all_close(_lowerCamelCase     ,_lowerCamelCase     ,atol=_lowerCamelCase )












'''simple docstring'''



import argparse
from pathlib import Path

import torch

from transformers import OPTConfig, OPTModel
from transformers.utils import logging


logging.set_verbosity_info()
_SCREAMING_SNAKE_CASE					=    logging.get_logger(__name__)




def    _lowerCAmelCase						(				lowerCamelCase_			:  int				):
			__lowercase										=							torch.load(lowerCamelCase_		,       map_location='''cpu'''				)
			if "model" in sd.keys():
						__lowercase										=							torch.load(lowerCamelCase_		,       map_location='''cpu'''				)['''model''']

			# pop unnecessary weights
			__lowercase										=							[
			    '''decoder.version''',
			    '''decoder.output_projection.weight''',
			]
			for key in keys_to_delete:
						if key in sd:
									sd.pop(lowerCamelCase_				)

			__lowercase										=							{
			    '''decoder.project_in_dim.weight''': '''decoder.project_in.weight''',
			    '''decoder.project_out_dim.weight''': '''decoder.project_out.weight''',
			    '''decoder.layer_norm.weight''': '''decoder.final_layer_norm.weight''',
			    '''decoder.layer_norm.bias''': '''decoder.final_layer_norm.bias''',
			}
			for old_key, new_key in keys_to_rename.items():
						if old_key in sd:
									__lowercase										=							sd.pop(lowerCamelCase_				)

			__lowercase										=							list(sd.keys()				)
			for key in keys:
						if ".qkv_proj." in key:
									__lowercase										=							sd[key]
									# We split QKV in separate Q,K,V

									__lowercase										=							key.replace('''.qkv_proj.'''		,       '''.q_proj.'''				)
									__lowercase										=							key.replace('''.qkv_proj.'''		,       '''.k_proj.'''				)
									__lowercase										=							key.replace('''.qkv_proj.'''		,       '''.v_proj.'''				)

									__lowercase										=							value.shape[0]
									assert depth % 3 == 0
									# `SequeuceParallelTransformerBlock` has QKV weight is separated in K,V,Q despite the naming:
									# https://cs.github.com/facebookresearch/metaseq/blob/51871bd73cd04c038f239ea2a26db1d7f6b37927/metaseq/modules/sequence_parallel_transformer_layer.py#L97
									__lowercase						,		__lowercase						,		__lowercase										=							torch.split(lowerCamelCase_		,       depth // 3		,       dim=0				)

									__lowercase										=							q
									__lowercase										=							k
									__lowercase										=							v
									del sd[key]

			return sd




@torch.no_grad()
def    _lowerCAmelCase						(				lowerCamelCase_			:  Union[str, Any]		,       lowerCamelCase_			:  Tuple		,       lowerCamelCase_			:  Union[str, Any]=None				):
			__lowercase										=							load_checkpoint(lowerCamelCase_				)

			if config is not None:
						__lowercase										=							OPTConfig.from_pretrained(lowerCamelCase_				)
			else:
						__lowercase										=							OPTConfig()

			__lowercase										=							OPTModel(lowerCamelCase_				).half().eval()
			model.load_state_dict(lowerCamelCase_				)

			# Check results
			Path(lowerCamelCase_				).mkdir(exist_ok=lowerCamelCase_				)
			model.save_pretrained(lowerCamelCase_				)


if __name__ == "__main__":
	_SCREAMING_SNAKE_CASE					=    argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    '''--fairseq_path''',
	    type=str,
	    help=(
	        '''path to fairseq checkpoint in correct format. You can find all checkpoints in the correct format here:'''
	        ''' https://huggingface.co/models?other=opt_metasq'''
	    ),
	)
	parser.add_argument('''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model.''')
	parser.add_argument('''--hf_config''', default=None, type=str, help='''Define HF config.''')
	_SCREAMING_SNAKE_CASE					=    parser.parse_args()
	convert_opt_checkpoint(args.fairseq_path, args.pytorch_dump_folder_path, config=args.hf_config)





