



import argparse

import torch

from transformers import GPTaLMHeadModel, RobertaForMaskedLM


if __name__ == "__main__":
			SCREAMING_SNAKE_CASE     				=						argparse.ArgumentParser(
			    description=(
			        "Extraction some layers of the full RobertaForMaskedLM or GPT2LMHeadModel for Transfer Learned"
			        " Distillation"
			    )
			)
			parser.add_argument("--model_type", default="roberta", choices=["roberta", "gpt2"])
			parser.add_argument("--model_name", default="roberta-large", type=str)
			parser.add_argument("--dump_checkpoint", default="serialization_dir/tf_roberta_048131723.pth", type=str)
			parser.add_argument("--vocab_transform", action="store_true")
			SCREAMING_SNAKE_CASE     				=						parser.parse_args()

			if args.model_type == "roberta":
						SCREAMING_SNAKE_CASE     				=						RobertaForMaskedLM.from_pretrained(args.model_name)
						SCREAMING_SNAKE_CASE     				=						"roberta"
			elif args.model_type == "gpt2":
						SCREAMING_SNAKE_CASE     				=						GPTaLMHeadModel.from_pretrained(args.model_name)
						SCREAMING_SNAKE_CASE     				=						"transformer"

			SCREAMING_SNAKE_CASE     				=						model.state_dict()
			SCREAMING_SNAKE_CASE     				=						{}

			# Embeddings #
			if args.model_type == "gpt2":
						for param_name in ["wte.weight", "wpe.weight"]:
									SCREAMING_SNAKE_CASE     				=						state_dict[f'''{prefix}.{param_name}''']
			else:
						for w in ["word_embeddings", "position_embeddings", "token_type_embeddings"]:
									SCREAMING_SNAKE_CASE     				=						f'''{prefix}.embeddings.{w}.weight'''
									SCREAMING_SNAKE_CASE     				=						state_dict[param_name]
						for w in ["weight", "bias"]:
									SCREAMING_SNAKE_CASE     				=						f'''{prefix}.embeddings.LayerNorm.{w}'''
									SCREAMING_SNAKE_CASE     				=						state_dict[param_name]

    # Transformer Blocks #
			SCREAMING_SNAKE_CASE     				=						0
			for teacher_idx in [0, 2, 4, 7, 9, 11]:
						if args.model_type == "gpt2":
									for layer in ["ln_1", "attn.c_attn", "attn.c_proj", "ln_2", "mlp.c_fc", "mlp.c_proj"]:
												for w in ["weight", "bias"]:
															SCREAMING_SNAKE_CASE     				=						state_dict[
															    f'''{prefix}.h.{teacher_idx}.{layer}.{w}'''
															]
									SCREAMING_SNAKE_CASE     				=						state_dict[f'''{prefix}.h.{teacher_idx}.attn.bias''']
						else:
									for layer in [
									    "attention.self.query",
									    "attention.self.key",
									    "attention.self.value",
									    "attention.output.dense",
									    "attention.output.LayerNorm",
									    "intermediate.dense",
									    "output.dense",
									    "output.LayerNorm",
									]:
												for w in ["weight", "bias"]:
															SCREAMING_SNAKE_CASE     				=						state_dict[
															    f'''{prefix}.encoder.layer.{teacher_idx}.{layer}.{w}'''
															]
						std_idx += 1

			# Language Modeling Head ###s
			if args.model_type == "roberta":
						for layer in ["lm_head.decoder.weight", "lm_head.bias"]:
									SCREAMING_SNAKE_CASE     				=						state_dict[f'''{layer}''']
						if args.vocab_transform:
									for w in ["weight", "bias"]:
												SCREAMING_SNAKE_CASE     				=						state_dict[f'''lm_head.dense.{w}''']
												SCREAMING_SNAKE_CASE     				=						state_dict[f'''lm_head.layer_norm.{w}''']
			elif args.model_type == "gpt2":
						for w in ["weight", "bias"]:
									SCREAMING_SNAKE_CASE     				=						state_dict[f'''{prefix}.ln_f.{w}''']
						SCREAMING_SNAKE_CASE     				=						state_dict["lm_head.weight"]

			print(f'''N layers selected for distillation: {std_idx}''')
			print(f'''Number of params transferred for distillation: {len(compressed_sd.keys())}''')

			print(f'''Save transferred checkpoint to {args.dump_checkpoint}.''')
			torch.save(compressed_sd, args.dump_checkpoint)






import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    DPMSolverMultistepScheduler,
    TextToVideoSDPipeline,
    UNetaDConditionModel,
)
from diffusers.utils import is_xformers_available, load_numpy, skip_mps, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineTesterMixin


enable_full_determinism()



@skip_mps
class 	lowerCamelCase      (						lowercase__,     unittest.TestCase  ):







					'''simple docstring'''







					lowerCAmelCase_				:    int									=  TextToVideoSDPipeline
					lowerCAmelCase_				:    Dict									=  TEXT_TO_IMAGE_PARAMS
					lowerCAmelCase_				:    Optional[Any]									=  TEXT_TO_IMAGE_BATCH_PARAMS
					# No `output_type`.
					lowerCAmelCase_				:    Optional[Any]									=  frozenset(
					    [
					        'num_inference_steps',
					        'generator',
					        'latents',
					        'return_dict',
					        'callback',
					        'callback_steps',
					    ]  )



					def    A__		(			self     ):
							torch.manual_seed(0     )
							UpperCAmelCase_						=				UNetaDConditionModel(
							    block_out_channels=(32, 64, 64, 64)    ,	layers_per_block=2    ,	sample_size=32    ,	in_channels=4    ,	out_channels=4    ,	down_block_types=("CrossAttnDownBlock3D", "CrossAttnDownBlock3D", "CrossAttnDownBlock3D", "DownBlock3D")    ,	up_block_types=("UpBlock3D", "CrossAttnUpBlock3D", "CrossAttnUpBlock3D", "CrossAttnUpBlock3D")    ,	cross_attention_dim=32    ,	attention_head_dim=4    ,	)
							UpperCAmelCase_						=				DDIMScheduler(
							    beta_start=0.00085    ,	beta_end=0.012    ,	beta_schedule="scaled_linear"    ,	clip_sample=lowerCAmelCase    ,	set_alpha_to_one=lowerCAmelCase    ,	)
							torch.manual_seed(0     )
							UpperCAmelCase_						=				AutoencoderKL(
							    block_out_channels=[32, 64]    ,	in_channels=3    ,	out_channels=3    ,	down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"]    ,	up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"]    ,	latent_channels=4    ,	sample_size=128    ,	)
							torch.manual_seed(0     )
							UpperCAmelCase_						=				CLIPTextConfig(
							    bos_token_id=0    ,	eos_token_id=2    ,	hidden_size=32    ,	intermediate_size=37    ,	layer_norm_eps=1e-0_5    ,	num_attention_heads=4    ,	num_hidden_layers=5    ,	pad_token_id=1    ,	vocab_size=1000    ,	hidden_act="gelu"    ,	projection_dim=512    ,	)
							UpperCAmelCase_						=				CLIPTextModel(lowerCAmelCase     )
							UpperCAmelCase_						=				CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"     )

							UpperCAmelCase_						=				{
							    "unet": unet,
							    "scheduler": scheduler,
							    "vae": vae,
							    "text_encoder": text_encoder,
							    "tokenizer": tokenizer,
							}
							return components



					def    A__		(			self    ,	lowerCAmelCase    ,	lowerCAmelCase=0     ):
							if str(lowerCAmelCase     ).startswith("mps"     ):
									UpperCAmelCase_						=				torch.manual_seed(lowerCAmelCase     )
							else:
									UpperCAmelCase_						=				torch.Generator(device=lowerCAmelCase     ).manual_seed(lowerCAmelCase     )
							UpperCAmelCase_						=				{
							    "prompt": "A painting of a squirrel eating a burger",
							    "generator": generator,
							    "num_inference_steps": 2,
							    "guidance_scale": 6.0,
							    "output_type": "pt",
							}
							return inputs



					def    A__		(			self     ):
							UpperCAmelCase_						=				"cpu"  # ensure determinism for the device-dependent torch.Generator
							UpperCAmelCase_						=				self.get_dummy_components()
							UpperCAmelCase_						=				TextToVideoSDPipeline(**lowerCAmelCase     )
							UpperCAmelCase_						=				sd_pipe.to(lowerCAmelCase     )
							sd_pipe.set_progress_bar_config(disable=lowerCAmelCase     )

							UpperCAmelCase_						=				self.get_dummy_inputs(lowerCAmelCase     )
							UpperCAmelCase_						=				"np"
							UpperCAmelCase_						=				sd_pipe(**lowerCAmelCase     ).frames
							UpperCAmelCase_						=				frames[0][-3:, -3:, -1]

							assert frames[0].shape == (64, 64, 3)
							UpperCAmelCase_						=				np.array([158.0, 160.0, 153.0, 125.0, 100.0, 121.0, 111.0, 93.0, 113.0]     )

							assert np.abs(image_slice.flatten() - expected_slice     ).max() < 1e-2



					def    A__		(			self     ):
							self._test_attention_slicing_forward_pass(test_mean_pixel_difference=lowerCAmelCase    ,	expected_max_diff=3e-3     )



					@unittest.skipIf(
					    torch_device != "cuda" or not is_xformers_available()    ,	reason="XFormers attention is only available with CUDA and `xformers` installed"    ,	)
					def    A__		(			self     ):
							self._test_xformers_attention_forwardGenerator_pass(test_mean_pixel_difference=lowerCAmelCase    ,	expected_max_diff=1e-2     )



					@unittest.skip(reason="Batching needs to be properly figured out first for this pipeline."     )
					def    A__		(			self     ):
							pass



					@unittest.skip(reason="Batching needs to be properly figured out first for this pipeline."     )
					def    A__		(			self     ):
							pass



					@unittest.skip(reason="`num_images_per_prompt` argument is not supported for this pipeline."     )
					def    A__		(			self     ):
							pass



					def    A__		(			self     ):
							return super().test_progress_bar()





@slow
@skip_mps
class 	lowerCamelCase      (						unittest.TestCase  ):







					'''simple docstring'''







					def    A__		(			self     ):
							UpperCAmelCase_						=				load_numpy(
							    "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/text_to_video/video.npy"     )

							UpperCAmelCase_						=				TextToVideoSDPipeline.from_pretrained("damo-vilab/text-to-video-ms-1.7b"     )
							UpperCAmelCase_						=				DPMSolverMultistepScheduler.from_config(pipe.scheduler.config     )
							UpperCAmelCase_						=				pipe.to("cuda"     )

							UpperCAmelCase_						=				"Spiderman is surfing"
							UpperCAmelCase_						=				torch.Generator(device="cpu"     ).manual_seed(0     )

							UpperCAmelCase_						=				pipe(lowerCAmelCase    ,	generator=lowerCAmelCase    ,	num_inference_steps=25    ,	output_type="pt"     ).frames
							UpperCAmelCase_						=				video_frames.cpu().numpy()

							assert np.abs(expected_video - video     ).mean() < 5e-2



					def    A__		(			self     ):
							UpperCAmelCase_						=				load_numpy(
							    "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/text_to_video/video_2step.npy"     )

							UpperCAmelCase_						=				TextToVideoSDPipeline.from_pretrained("damo-vilab/text-to-video-ms-1.7b"     )
							UpperCAmelCase_						=				pipe.to("cuda"     )

							UpperCAmelCase_						=				"Spiderman is surfing"
							UpperCAmelCase_						=				torch.Generator(device="cpu"     ).manual_seed(0     )

							UpperCAmelCase_						=				pipe(lowerCAmelCase    ,	generator=lowerCAmelCase    ,	num_inference_steps=2    ,	output_type="pt"     ).frames
							UpperCAmelCase_						=				video_frames.cpu().numpy()

							assert np.abs(expected_video - video     ).mean() < 5e-2

