




"""simple docstring"""






import argparse
import json

import gdown
import numpy as np
import torch
from huggingface_hub import hf_hub_download

from transformers import (
    VideoMAEConfig,
    VideoMAEForPreTraining,
    VideoMAEForVideoClassification,
    VideoMAEImageProcessor,
)

def 							UpperCAmelCase		(		snake_case			:					List[str]    ):
					_lowerCAmelCase:Any									=  VideoMAEConfig()

					set_architecture_configs(__lowercase      ,       __lowercase    )

					if "finetuned" not in model_name:
										_lowerCAmelCase:int									=  False

					if "finetuned" in model_name:
										_lowerCAmelCase:Any									=  'huggingface/label-files'
										if "kinetics" in model_name:
															_lowerCAmelCase:Dict									=  400
															_lowerCAmelCase:Union[str, Any]									=  'kinetics400-id2label.json'
										elif "ssv2" in model_name:
															_lowerCAmelCase:Dict									=  174
															_lowerCAmelCase:List[Any]									=  'something-something-v2-id2label.json'
										else:
															raise ValueError('''Model name should either contain \'kinetics\' or \'ssv2\' in case it\'s fine-tuned.'''    )
										_lowerCAmelCase:Optional[int]									=  json.load(open(hf_hub_download(__lowercase      ,       __lowercase      ,       repo_type='''dataset'''    )      ,       '''r'''    )    )
										_lowerCAmelCase:Union[str, Any]									=  {int(__lowercase    ): v for k, v in idalabel.items()}
										_lowerCAmelCase:Any									=  idalabel
										_lowerCAmelCase:Dict									=  {v: k for k, v in idalabel.items()}

					return config

def 							UpperCAmelCase		(		snake_case			:					Optional[int]      ,       snake_case			:					Union[str, Any]    ):




					if "small" in model_name:
										_lowerCAmelCase:Optional[Any]									=  384
										_lowerCAmelCase:Union[str, Any]									=  1536
										_lowerCAmelCase:List[Any]									=  12
										_lowerCAmelCase:Union[str, Any]									=  16
										_lowerCAmelCase:Optional[int]									=  12
										_lowerCAmelCase:Tuple									=  3
										_lowerCAmelCase:str									=  192
										_lowerCAmelCase:Union[str, Any]									=  768
					elif "large" in model_name:
										_lowerCAmelCase:Any									=  1024
										_lowerCAmelCase:int									=  4096
										_lowerCAmelCase:int									=  24
										_lowerCAmelCase:Optional[Any]									=  16
										_lowerCAmelCase:int									=  12
										_lowerCAmelCase:Optional[int]									=  8
										_lowerCAmelCase:Union[str, Any]									=  512
										_lowerCAmelCase:List[Any]									=  2048
					elif "huge" in model_name:
										_lowerCAmelCase:Union[str, Any]									=  1280
										_lowerCAmelCase:Union[str, Any]									=  5120
										_lowerCAmelCase:List[Any]									=  32
										_lowerCAmelCase:Tuple									=  16
										_lowerCAmelCase:List[str]									=  12
										_lowerCAmelCase:List[str]									=  8
										_lowerCAmelCase:List[Any]									=  640
										_lowerCAmelCase:int									=  2560
					elif "base" not in model_name:
										raise ValueError('''Model name should include either "small", "base", "large", or "huge"'''    )

def 							UpperCAmelCase		(		snake_case			:					Any    ):




					if "encoder." in name:
										_lowerCAmelCase:int									=  name.replace('''encoder.'''      ,       ''''''    )
					if "cls_token" in name:
										_lowerCAmelCase:List[Any]									=  name.replace('''cls_token'''      ,       '''videomae.embeddings.cls_token'''    )
					if "decoder_pos_embed" in name:
										_lowerCAmelCase:List[str]									=  name.replace('''decoder_pos_embed'''      ,       '''decoder.decoder_pos_embed'''    )
					if "pos_embed" in name and "decoder" not in name:
										_lowerCAmelCase:Tuple									=  name.replace('''pos_embed'''      ,       '''videomae.embeddings.position_embeddings'''    )
					if "patch_embed.proj" in name:
										_lowerCAmelCase:Optional[Any]									=  name.replace('''patch_embed.proj'''      ,       '''videomae.embeddings.patch_embeddings.projection'''    )
					if "patch_embed.norm" in name:
										_lowerCAmelCase:int									=  name.replace('''patch_embed.norm'''      ,       '''videomae.embeddings.norm'''    )
					if "decoder.blocks" in name:
										_lowerCAmelCase:Union[str, Any]									=  name.replace('''decoder.blocks'''      ,       '''decoder.decoder_layers'''    )
					if "blocks" in name:
										_lowerCAmelCase:Dict									=  name.replace('''blocks'''      ,       '''videomae.encoder.layer'''    )
					if "attn.proj" in name:
										_lowerCAmelCase:str									=  name.replace('''attn.proj'''      ,       '''attention.output.dense'''    )
					if "attn" in name and "bias" not in name:
										_lowerCAmelCase:Optional[int]									=  name.replace('''attn'''      ,       '''attention.self'''    )
					if "attn" in name:
										_lowerCAmelCase:str									=  name.replace('''attn'''      ,       '''attention.attention'''    )
					if "norm1" in name:
										_lowerCAmelCase:int									=  name.replace('''norm1'''      ,       '''layernorm_before'''    )
					if "norm2" in name:
										_lowerCAmelCase:Tuple									=  name.replace('''norm2'''      ,       '''layernorm_after'''    )
					if "mlp.fc1" in name:
										_lowerCAmelCase:int									=  name.replace('''mlp.fc1'''      ,       '''intermediate.dense'''    )
					if "mlp.fc2" in name:
										_lowerCAmelCase:Any									=  name.replace('''mlp.fc2'''      ,       '''output.dense'''    )
					if "decoder_embed" in name:
										_lowerCAmelCase:Union[str, Any]									=  name.replace('''decoder_embed'''      ,       '''decoder.decoder_embed'''    )
					if "decoder_norm" in name:
										_lowerCAmelCase:str									=  name.replace('''decoder_norm'''      ,       '''decoder.decoder_norm'''    )
					if "decoder_pred" in name:
										_lowerCAmelCase:str									=  name.replace('''decoder_pred'''      ,       '''decoder.decoder_pred'''    )
					if "norm.weight" in name and "decoder" not in name and "fc" not in name:
										_lowerCAmelCase:Dict									=  name.replace('''norm.weight'''      ,       '''videomae.layernorm.weight'''    )
					if "norm.bias" in name and "decoder" not in name and "fc" not in name:
										_lowerCAmelCase:int									=  name.replace('''norm.bias'''      ,       '''videomae.layernorm.bias'''    )
					if "head" in name and "decoder" not in name:
										_lowerCAmelCase:Dict									=  name.replace('''head'''      ,       '''classifier'''    )

					return name

def 							UpperCAmelCase		(		snake_case			:					int      ,       snake_case			:					Any    ):




					for key in orig_state_dict.copy().keys():
										_lowerCAmelCase:Dict									=  orig_state_dict.pop(__lowercase    )

										if key.startswith('''encoder.'''    ):
															_lowerCAmelCase:Optional[int]									=  key.replace('''encoder.'''      ,       ''''''    )

										if "qkv" in key:
															_lowerCAmelCase:Any									=  key.split('''.'''    )
															if key.startswith('''decoder.blocks'''    ):
																				_lowerCAmelCase:Optional[Any]									=  config.decoder_hidden_size
																				_lowerCAmelCase:Optional[int]									=  int(key_split[2]    )
																				_lowerCAmelCase:List[Any]									=  'decoder.decoder_layers.'
																				if "weight" in key:
																									_lowerCAmelCase:Optional[Any]									=  val[:dim, :]
																									_lowerCAmelCase:Tuple									=  val[dim : dim * 2, :]
																									_lowerCAmelCase:Union[str, Any]									=  val[-dim:, :]
															else:
																				_lowerCAmelCase:str									=  config.hidden_size
																				_lowerCAmelCase:Optional[Any]									=  int(key_split[1]    )
																				_lowerCAmelCase:Optional[int]									=  'videomae.encoder.layer.'
																				if "weight" in key:
																									_lowerCAmelCase:Tuple									=  val[:dim, :]
																									_lowerCAmelCase:Optional[int]									=  val[dim : dim * 2, :]
																									_lowerCAmelCase:Dict									=  val[-dim:, :]
										else:
															_lowerCAmelCase:Optional[int]									=  val

					return orig_state_dict

def 							UpperCAmelCase		(		):
					_lowerCAmelCase:Union[str, Any]									=  hf_hub_download(
					    repo_id='''hf-internal-testing/spaghetti-video'''      ,       filename='''eating_spaghetti.npy'''      ,       repo_type='''dataset'''    )
					_lowerCAmelCase:Optional[int]									=  np.load(__lowercase    )
					return list(__lowercase    )

def 							UpperCAmelCase		(		snake_case			:					Dict      ,       snake_case			:					Any      ,       snake_case			:					Union[str, Any]      ,       snake_case			:					Any    ):
					_lowerCAmelCase:Dict									=  get_videomae_config(__lowercase    )

					if "finetuned" in model_name:
										_lowerCAmelCase:List[str]									=  VideoMAEForVideoClassification(__lowercase    )
					else:
										_lowerCAmelCase:Tuple									=  VideoMAEForPreTraining(__lowercase    )

					# download original checkpoint, hosted on Google Drive
					_lowerCAmelCase:Optional[int]									=  'pytorch_model.bin'
					gdown.cached_download(__lowercase      ,       __lowercase      ,       quiet=__lowercase    )
					_lowerCAmelCase:Union[str, Any]									=  torch.load(__lowercase      ,       map_location='''cpu'''    )
					if "model" in files:
										_lowerCAmelCase:int									=  files['model']
					else:
										_lowerCAmelCase:Union[str, Any]									=  files['module']
					_lowerCAmelCase:Tuple									=  convert_state_dict(__lowercase      ,       __lowercase    )

					model.load_state_dict(__lowercase    )
					model.eval()

					# verify model on basic input
					_lowerCAmelCase:int									=  VideoMAEImageProcessor(image_mean=[0.5, 0.5, 0.5]      ,       image_std=[0.5, 0.5, 0.5]    )
					_lowerCAmelCase:Tuple									=  prepare_video()
					_lowerCAmelCase:Union[str, Any]									=  image_processor(__lowercase      ,       return_tensors='''pt'''    )

					if "finetuned" not in model_name:
										_lowerCAmelCase:List[Any]									=  hf_hub_download(repo_id='''hf-internal-testing/bool-masked-pos'''      ,       filename='''bool_masked_pos.pt'''    )
										_lowerCAmelCase:Optional[Any]									=  torch.load(__lowercase    )

					_lowerCAmelCase:str									=  model(**__lowercase    )
					_lowerCAmelCase:Tuple									=  outputs.logits

					_lowerCAmelCase:Dict									=  [
					    'videomae-small-finetuned-kinetics',
					    'videomae-small-finetuned-ssv2',
					    # Kinetics-400 checkpoints (short = pretrained only for 800 epochs instead of 1600)
					    'videomae-base-short',
					    'videomae-base-short-finetuned-kinetics',
					    'videomae-base',
					    'videomae-base-finetuned-kinetics',
					    'videomae-large',
					    'videomae-large-finetuned-kinetics',
					    'videomae-huge-finetuned-kinetics',
					    # Something-Something-v2 checkpoints (short = pretrained only for 800 epochs instead of 2400)
					    'videomae-base-short-ssv2',
					    'videomae-base-short-finetuned-ssv2',
					    'videomae-base-ssv2',
					    'videomae-base-finetuned-ssv2',
					]

					# NOTE: logits were tested with image_mean and image_std equal to [0.5, 0.5, 0.5] and [0.5, 0.5, 0.5]
					if model_name == "videomae-small-finetuned-kinetics":
										_lowerCAmelCase:Union[str, Any]									=  torch.Size([1, 400]    )
										_lowerCAmelCase:Dict									=  torch.tensor([-0.92_91, -0.40_61, -0.93_07]    )
					elif model_name == "videomae-small-finetuned-ssv2":
										_lowerCAmelCase:Optional[int]									=  torch.Size([1, 174]    )
										_lowerCAmelCase:List[str]									=  torch.tensor([0.26_71, -0.46_89, -0.82_35]    )
					elif model_name == "videomae-base":
										_lowerCAmelCase:Any									=  torch.Size([1, 1408, 1536]    )
										_lowerCAmelCase:List[str]									=  torch.tensor([[0.77_39, 0.79_68, 0.70_89], [0.67_01, 0.74_87, 0.62_09], [0.42_87, 0.51_58, 0.47_73]]    )
					elif model_name == "videomae-base-short":
										_lowerCAmelCase:Optional[int]									=  torch.Size([1, 1408, 1536]    )
										_lowerCAmelCase:Union[str, Any]									=  torch.tensor([[0.79_94, 0.96_12, 0.85_08], [0.74_01, 0.89_58, 0.83_02], [0.58_62, 0.74_68, 0.73_25]]    )
										# we verified the loss both for normalized and unnormalized targets for this one
										_lowerCAmelCase:Dict									=  torch.tensor([0.51_42]    ) if config.norm_pix_loss else torch.tensor([0.64_69]    )
					elif model_name == "videomae-large":
										_lowerCAmelCase:Tuple									=  torch.Size([1, 1408, 1536]    )
										_lowerCAmelCase:Optional[int]									=  torch.tensor([[0.71_49, 0.79_97, 0.69_66], [0.67_68, 0.78_69, 0.69_48], [0.51_39, 0.62_21, 0.56_05]]    )
					elif model_name == "videomae-large-finetuned-kinetics":
										_lowerCAmelCase:Tuple									=  torch.Size([1, 400]    )
										_lowerCAmelCase:Union[str, Any]									=  torch.tensor([0.07_71, 0.00_11, -0.36_25]    )
					elif model_name == "videomae-huge-finetuned-kinetics":
										_lowerCAmelCase:Dict									=  torch.Size([1, 400]    )
										_lowerCAmelCase:List[Any]									=  torch.tensor([0.24_33, 0.16_32, -0.48_94]    )
					elif model_name == "videomae-base-short-finetuned-kinetics":
										_lowerCAmelCase:Any									=  torch.Size([1, 400]    )
										_lowerCAmelCase:Optional[Any]									=  torch.tensor([0.65_88, 0.09_90, -0.24_93]    )
					elif model_name == "videomae-base-finetuned-kinetics":
										_lowerCAmelCase:Any									=  torch.Size([1, 400]    )
										_lowerCAmelCase:Any									=  torch.tensor([0.36_69, -0.06_88, -0.24_21]    )
					elif model_name == "videomae-base-short-ssv2":
										_lowerCAmelCase:str									=  torch.Size([1, 1408, 1536]    )
										_lowerCAmelCase:Tuple									=  torch.tensor([[0.47_12, 0.52_96, 0.57_86], [0.22_78, 0.27_29, 0.40_26], [0.03_52, 0.07_30, 0.25_06]]    )
					elif model_name == "videomae-base-short-finetuned-ssv2":
										_lowerCAmelCase:Dict									=  torch.Size([1, 174]    )
										_lowerCAmelCase:List[str]									=  torch.tensor([-0.05_37, -0.15_39, -0.32_66]    )
					elif model_name == "videomae-base-ssv2":
										_lowerCAmelCase:Dict									=  torch.Size([1, 1408, 1536]    )
										_lowerCAmelCase:Optional[int]									=  torch.tensor([[0.81_31, 0.87_27, 0.85_46], [0.73_66, 0.93_77, 0.88_70], [0.59_35, 0.88_74, 0.85_64]]    )
					elif model_name == "videomae-base-finetuned-ssv2":
										_lowerCAmelCase:Any									=  torch.Size([1, 174]    )
										_lowerCAmelCase:Tuple									=  torch.tensor([0.19_61, -0.83_37, -0.63_89]    )
					else:
										raise ValueError(F'Model name not supported. Should be one of {model_names}'    )

					# verify logits
					assert logits.shape == expected_shape
					if "finetuned" in model_name:
										assert torch.allclose(logits[0, :3]      ,       __lowercase      ,       atol=1e-4    )
					else:
										print('''Logits:'''      ,       logits[0, :3, :3]    )
										assert torch.allclose(logits[0, :3, :3]      ,       __lowercase      ,       atol=1e-4    )
					print('''Logits ok!'''    )

					# verify loss, if applicable
					if model_name == "videomae-base-short":
										_lowerCAmelCase:Tuple									=  outputs.loss
										assert torch.allclose(__lowercase      ,       __lowercase      ,       atol=1e-4    )
										print('''Loss ok!'''    )

					if pytorch_dump_folder_path is not None:
										print(F'Saving model and image processor to {pytorch_dump_folder_path}'    )
										image_processor.save_pretrained(__lowercase    )
										model.save_pretrained(__lowercase    )

					if push_to_hub:
										print('''Pushing to the hub...'''    )
										model.push_to_hub(__lowercase      ,       organization='''nielsr'''    )


if __name__ == "__main__":
				UpperCamelCase__						=			argparse.ArgumentParser()
				# Required parameters
				parser.add_argument(
				    '''--checkpoint_url''',
				    default='''https://drive.google.com/u/1/uc?id=1tEhLyskjb755TJ65ptsrafUG2llSwQE1&amp;export=download&amp;confirm=t&amp;uuid=aa3276eb-fb7e-482a-adec-dc7171df14c4''',
				    type=str,
				    help=(
				        '''URL of the original PyTorch checkpoint (on Google Drive) you\'d like to convert. Should be a direct'''
				        ''' download link.'''
				    ),
				)
				parser.add_argument(
				    '''--pytorch_dump_folder_path''',
				    default='''/Users/nielsrogge/Documents/VideoMAE/Test''',
				    type=str,
				    help='''Path to the output PyTorch model directory.''',
				)
				parser.add_argument('''--model_name''', default='''videomae-base''', type=str, help='''Name of the model.''')
				parser.add_argument(
				    '''--push_to_hub''', action='''store_true''', help='''Whether or not to push the converted model to the 🤗 hub.'''
				)

				UpperCamelCase__						=			parser.parse_args()
				convert_videomae_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path, args.model_name, args.push_to_hub)












"""simple docstring"""






import torch

from diffusers import KDPMaDiscreteScheduler
from diffusers.utils import torch_device

from .test_schedulers import SchedulerCommonTest




class   a__			(       UpperCamelCase_  ):
						snake_case__             = (KDPMaDiscreteScheduler,)
						snake_case__             = 1_0





						def       __UpperCamelCase     (							self      :	Dict     ,**a__      :	str)						->       int:




											"""simple docstring"""







											_lowerCAmelCase:Optional[int]									=  {
											    '''num_train_timesteps''': 1100,
											    '''beta_start''': 0.0001,
											    '''beta_end''': 0.02,
											    '''beta_schedule''': '''linear''',
											}

											config.update(**a__)
											return config





						def       __UpperCamelCase     (							self      :	Optional[int])						->       Optional[int]:




											"""simple docstring"""







											for timesteps in [10, 50, 100, 1000]:
																self.check_over_configs(num_train_timesteps=a__)





						def       __UpperCamelCase     (							self      :	Union[str, Any])						->       Optional[int]:




											"""simple docstring"""







											for beta_start, beta_end in zip([0.00001, 0.0001, 0.001]     ,[0.0002, 0.002, 0.02]):
																self.check_over_configs(beta_start=a__     ,beta_end=a__)





						def       __UpperCamelCase     (							self      :	Optional[Any])						->       Union[str, Any]:




											"""simple docstring"""







											for schedule in ["linear", "scaled_linear"]:
																self.check_over_configs(beta_schedule=a__)





						def       __UpperCamelCase     (							self      :	str)						->       str:




											"""simple docstring"""







											for prediction_type in ["epsilon", "v_prediction"]:
																self.check_over_configs(prediction_type=a__)





						def       __UpperCamelCase     (							self      :	Tuple)						->       Tuple:




											"""simple docstring"""







											_lowerCAmelCase:Union[str, Any]									=  self.scheduler_classes[0]
											_lowerCAmelCase:Union[str, Any]									=  self.get_scheduler_config(prediction_type='''v_prediction''')
											_lowerCAmelCase:Tuple									=  scheduler_class(**a__)

											scheduler.set_timesteps(self.num_inference_steps)

											_lowerCAmelCase:Optional[Any]									=  self.dummy_model()
											_lowerCAmelCase:List[Any]									=  self.dummy_sample_deter * scheduler.init_noise_sigma
											_lowerCAmelCase:Optional[Any]									=  sample.to(a__)

											for i, t in enumerate(scheduler.timesteps):
																_lowerCAmelCase:Tuple									=  scheduler.scale_model_input(a__     ,a__)

																_lowerCAmelCase:Any									=  model(a__     ,a__)

																_lowerCAmelCase:Union[str, Any]									=  scheduler.step(a__     ,a__     ,a__)
																_lowerCAmelCase:Optional[int]									=  output.prev_sample

											_lowerCAmelCase:str									=  torch.sum(torch.abs(a__))
											_lowerCAmelCase:Union[str, Any]									=  torch.mean(torch.abs(a__))

											if torch_device in ["cpu", "mps"]:
																assert abs(result_sum.item() - 4.6_934E-07) < 1E-2
																assert abs(result_mean.item() - 6.1_112E-10) < 1E-3
											else:
																# CUDA
																assert abs(result_sum.item() - 4.693_428_650_170_972E-07) < 1E-2
																assert abs(result_mean.item() - 0.0002) < 1E-3





						def       __UpperCamelCase     (							self      :	Tuple)						->       Dict:




											"""simple docstring"""







											if torch_device == "mps":
																return
											_lowerCAmelCase:Tuple									=  self.scheduler_classes[0]
											_lowerCAmelCase:List[str]									=  self.get_scheduler_config()
											_lowerCAmelCase:Union[str, Any]									=  scheduler_class(**a__)

											scheduler.set_timesteps(self.num_inference_steps)

											_lowerCAmelCase:Any									=  self.dummy_model()
											_lowerCAmelCase:str									=  self.dummy_sample_deter * scheduler.init_noise_sigma
											_lowerCAmelCase:List[str]									=  sample.to(a__)

											for i, t in enumerate(scheduler.timesteps):
																_lowerCAmelCase:Tuple									=  scheduler.scale_model_input(a__     ,a__)

																_lowerCAmelCase:List[str]									=  model(a__     ,a__)

																_lowerCAmelCase:int									=  scheduler.step(a__     ,a__     ,a__)
																_lowerCAmelCase:List[Any]									=  output.prev_sample

											_lowerCAmelCase:Tuple									=  torch.sum(torch.abs(a__))
											_lowerCAmelCase:Dict									=  torch.mean(torch.abs(a__))

											if torch_device in ["cpu", "mps"]:
																assert abs(result_sum.item() - 20.4125) < 1E-2
																assert abs(result_mean.item() - 0.0266) < 1E-3
											else:
																# CUDA
																assert abs(result_sum.item() - 20.4125) < 1E-2
																assert abs(result_mean.item() - 0.0266) < 1E-3







						def       __UpperCamelCase     (							self      :	Union[str, Any])						->       List[Any]:




											"""simple docstring"""







											if torch_device == "mps":
																return
											_lowerCAmelCase:str									=  self.scheduler_classes[0]
											_lowerCAmelCase:Optional[Any]									=  self.get_scheduler_config()
											_lowerCAmelCase:Any									=  scheduler_class(**a__)

											scheduler.set_timesteps(self.num_inference_steps     ,device=a__)

											_lowerCAmelCase:Optional[int]									=  self.dummy_model()
											_lowerCAmelCase:Optional[Any]									=  self.dummy_sample_deter.to(a__) * scheduler.init_noise_sigma

											for t in scheduler.timesteps:
																_lowerCAmelCase:Union[str, Any]									=  scheduler.scale_model_input(a__     ,a__)

																_lowerCAmelCase:Tuple									=  model(a__     ,a__)

																_lowerCAmelCase:str									=  scheduler.step(a__     ,a__     ,a__)
																_lowerCAmelCase:Dict									=  output.prev_sample

											_lowerCAmelCase:Any									=  torch.sum(torch.abs(a__))
											_lowerCAmelCase:str									=  torch.mean(torch.abs(a__))

											if str(a__).startswith('''cpu'''):
																# The following sum varies between 148 and 156 on mps. Why?
																assert abs(result_sum.item() - 20.4125) < 1E-2
																assert abs(result_mean.item() - 0.0266) < 1E-3
											else:
																# CUDA
																assert abs(result_sum.item() - 20.4125) < 1E-2
																assert abs(result_mean.item() - 0.0266) < 1E-3






