





import argparse
import json

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import ViTImageProcessor, ViTMSNConfig, ViTMSNModel
from transformers.image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD


torch.set_grad_enabled(False)




def    A_(			A				,      A=False   ):
	UpperCAmelCase_								=     []
	for i in range(config.num_hidden_layers   ):
		# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
		rename_keys.append((f"""module.blocks.{i}.norm1.weight""", f"""vit.encoder.layer.{i}.layernorm_before.weight""")   )
		rename_keys.append((f"""module.blocks.{i}.norm1.bias""", f"""vit.encoder.layer.{i}.layernorm_before.bias""")   )
		rename_keys.append(
		    (f"""module.blocks.{i}.attn.proj.weight""", f"""vit.encoder.layer.{i}.attention.output.dense.weight""")   )
		rename_keys.append((f"""module.blocks.{i}.attn.proj.bias""", f"""vit.encoder.layer.{i}.attention.output.dense.bias""")   )
		rename_keys.append((f"""module.blocks.{i}.norm2.weight""", f"""vit.encoder.layer.{i}.layernorm_after.weight""")   )
		rename_keys.append((f"""module.blocks.{i}.norm2.bias""", f"""vit.encoder.layer.{i}.layernorm_after.bias""")   )
		rename_keys.append((f"""module.blocks.{i}.mlp.fc1.weight""", f"""vit.encoder.layer.{i}.intermediate.dense.weight""")   )
		rename_keys.append((f"""module.blocks.{i}.mlp.fc1.bias""", f"""vit.encoder.layer.{i}.intermediate.dense.bias""")   )
		rename_keys.append((f"""module.blocks.{i}.mlp.fc2.weight""", f"""vit.encoder.layer.{i}.output.dense.weight""")   )
		rename_keys.append((f"""module.blocks.{i}.mlp.fc2.bias""", f"""vit.encoder.layer.{i}.output.dense.bias""")   )

	# projection layer + position embeddings
	rename_keys.extend(
	    [
	        ("""module.cls_token""", """vit.embeddings.cls_token"""),
	        ("""module.patch_embed.proj.weight""", """vit.embeddings.patch_embeddings.projection.weight"""),
	        ("""module.patch_embed.proj.bias""", """vit.embeddings.patch_embeddings.projection.bias"""),
	        ("""module.pos_embed""", """vit.embeddings.position_embeddings"""),
	    ]   )

	if base_model:
		# layernorm + pooler
		rename_keys.extend(
		    [
		        ("""module.norm.weight""", """layernorm.weight"""),
		        ("""module.norm.bias""", """layernorm.bias"""),
		    ]   )

		# if just the base model, we should remove "vit" from all keys that start with "vit"
		UpperCAmelCase_								=     [(pair[0], pair[1][4:]) if pair[1].startswith("""vit"""   ) else pair for pair in rename_keys]
	else:
		# layernorm + classification head
		rename_keys.extend(
		    [
		        ("""norm.weight""", """vit.layernorm.weight"""),
		        ("""norm.bias""", """vit.layernorm.bias"""),
		        ("""head.weight""", """classifier.weight"""),
		        ("""head.bias""", """classifier.bias"""),
		    ]   )

	return rename_keys




def    A_(			A				,      A				,      A=False   ):
	for i in range(config.num_hidden_layers   ):
		if base_model:
			UpperCAmelCase_								=     """"""
		else:
			UpperCAmelCase_								=     """vit."""
		# read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
		UpperCAmelCase_								=     state_dict.pop(f"""module.blocks.{i}.attn.qkv.weight"""   )
		UpperCAmelCase_								=     state_dict.pop(f"""module.blocks.{i}.attn.qkv.bias"""   )
		# next, add query, keys and values (in that order) to the state dict
		UpperCAmelCase_								=     in_proj_weight[
		    : config.hidden_size, :
		]
		UpperCAmelCase_								=     in_proj_bias[: config.hidden_size]
		UpperCAmelCase_								=     in_proj_weight[
		    config.hidden_size : config.hidden_size * 2, :
		]
		UpperCAmelCase_								=     in_proj_bias[
		    config.hidden_size : config.hidden_size * 2
		]
		UpperCAmelCase_								=     in_proj_weight[
		    -config.hidden_size :, :
		]
		UpperCAmelCase_								=     in_proj_bias[-config.hidden_size :]




def    A_(			A   ):
	UpperCAmelCase_								=     ["""head.weight""", """head.bias"""]
	for k in ignore_keys:
		state_dict.pop(A				,      A   )




def    A_(			A   ):
	# projection head is used in the self-supervised pre-training in MSN,
	# for downstream task it's not needed.
	UpperCAmelCase_								=     [
	    """module.fc.fc1.weight""",
	    """module.fc.fc1.bias""",
	    """module.fc.bn1.weight""",
	    """module.fc.bn1.bias""",
	    """module.fc.bn1.running_mean""",
	    """module.fc.bn1.running_var""",
	    """module.fc.bn1.num_batches_tracked""",
	    """module.fc.fc2.weight""",
	    """module.fc.fc2.bias""",
	    """module.fc.bn2.weight""",
	    """module.fc.bn2.bias""",
	    """module.fc.bn2.running_mean""",
	    """module.fc.bn2.running_var""",
	    """module.fc.bn2.num_batches_tracked""",
	    """module.fc.fc3.weight""",
	    """module.fc.fc3.bias""",
	]
	for k in ignore_keys:
		state_dict.pop(A				,      A   )




def    A_(			A				,      A				,      A   ):
	UpperCAmelCase_								=     dct.pop(A   )
	UpperCAmelCase_								=     val




def    A_(			A				,      A   ):
	UpperCAmelCase_								=     ViTMSNConfig()
	UpperCAmelCase_								=     1000

	UpperCAmelCase_								=     """datasets/huggingface/label-files"""
	UpperCAmelCase_								=     """imagenet-1k-id2label.json"""
	UpperCAmelCase_								=     json.load(open(hf_hub_download(A				,      A   )				,      """r"""   )   )
	UpperCAmelCase_								=     {int(A   ): v for k, v in idalabel.items()}
	UpperCAmelCase_								=     idalabel
	UpperCAmelCase_								=     {v: k for k, v in idalabel.items()}

	if "s16" in checkpoint_url:
		UpperCAmelCase_								=     384
		UpperCAmelCase_								=     1536
		UpperCAmelCase_								=     6
	elif "l16" in checkpoint_url:
		UpperCAmelCase_								=     1024
		UpperCAmelCase_								=     4096
		UpperCAmelCase_								=     24
		UpperCAmelCase_								=     16
		UpperCAmelCase_								=     0.1
	elif "b4" in checkpoint_url:
		UpperCAmelCase_								=     4
	elif "l7" in checkpoint_url:
		UpperCAmelCase_								=     7
		UpperCAmelCase_								=     1024
		UpperCAmelCase_								=     4096
		UpperCAmelCase_								=     24
		UpperCAmelCase_								=     16
		UpperCAmelCase_								=     0.1

	UpperCAmelCase_								=     ViTMSNModel(A   )

	UpperCAmelCase_								=     torch.hub.load_state_dict_from_url(A				,      map_location="""cpu"""   )["""target_encoder"""]

	UpperCAmelCase_								=     ViTImageProcessor(size=config.image_size   )

	remove_projection_head(A   )
	UpperCAmelCase_								=     create_rename_keys(A				,      base_model=A   )

	for src, dest in rename_keys:
		rename_key(A				,      A				,      A   )
	read_in_q_k_v(A				,      A				,      base_model=A   )

	model.load_state_dict(A   )
	model.eval()

	UpperCAmelCase_								=     """http://images.cocodataset.org/val2017/000000039769.jpg"""

	UpperCAmelCase_								=     Image.open(requests.get(A				,      stream=A   ).raw   )
	UpperCAmelCase_								=     ViTImageProcessor(
	    size=config.image_size				,      image_mean=A				,      image_std=A   )
	UpperCAmelCase_								=     image_processor(images=A				,      return_tensors="""pt"""   )

	# forward pass
	torch.manual_seed(2   )
	UpperCAmelCase_								=     model(**A   )
	UpperCAmelCase_								=     outputs.last_hidden_state

	# The following Colab Notebook was used to generate these outputs:
	# https://colab.research.google.com/gist/sayakpaul/3672419a04f5997827503fd84079bdd1/scratchpad.ipynb
	if "s16" in checkpoint_url:
		UpperCAmelCase_								=     torch.tensor([[-1.0_915, -1.4_876, -1.1_809]]   )
	elif "b16" in checkpoint_url:
		UpperCAmelCase_								=     torch.tensor([[14.2_889, -18.9_045, 11.7_281]]   )
	elif "l16" in checkpoint_url:
		UpperCAmelCase_								=     torch.tensor([[41.5_028, -22.8_681, 45.6_475]]   )
	elif "b4" in checkpoint_url:
		UpperCAmelCase_								=     torch.tensor([[-4.3_868, 5.2_932, -0.4_137]]   )
	else:
		UpperCAmelCase_								=     torch.tensor([[-0.1_792, -0.6_465, 2.4_263]]   )

	# verify logits
	assert torch.allclose(last_hidden_state[:, 0, :3]				,      A				,      atol=1E-4   )

	print(f"""Saving model to {pytorch_dump_folder_path}"""   )
	model.save_pretrained(A   )

	print(f"""Saving image processor to {pytorch_dump_folder_path}"""   )
	image_processor.save_pretrained(A   )


if __name__ == "__main__":
				UpperCamelCase__   : Tuple					  = argparse.ArgumentParser()
				# Required parameters
				parser.add_argument(
				    """--checkpoint_url""",
				    default="""https://dl.fbaipublicfiles.com/msn/vits16_800ep.pth.tar""",
				    type=str,
				    help="""URL of the checkpoint you'd like to convert.""",
				)
				parser.add_argument(
				    """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model directory."""
				)

				UpperCamelCase__   : int					  = parser.parse_args()
				convert_vit_msn_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)








import json
import os
import tempfile

import datasets
from utils import generate_example_dataset, get_duration


UpperCamelCase__   : Tuple					  = 5_00_00
UpperCamelCase__   : Any					  = 50_00

UpperCamelCase__  ,			UpperCamelCase__   : Tuple					  = os.path.split(__file__)
UpperCamelCase__   : Union[str, Any]					  = os.path.join(RESULTS_BASEPATH, """results""", RESULTS_FILENAME.replace(""".py""", """.json"""))




@get_duration
def    A_(			A				,      A   ):
	for i in range(A   ):
		UpperCAmelCase_								=     dataset[i]




@get_duration
def    A_(			A				,      A				,      A   ):
	for i in range(0				,      len(A   )				,      A   ):
		UpperCAmelCase_								=     dataset[i : i + batch_size]




@get_duration
def    A_(			A				,      A				,      A   ):
	with dataset.formatted_as(type=A   ):
		for i in range(A   ):
			UpperCAmelCase_								=     dataset[i]




@get_duration
def    A_(			A				,      A				,      A				,      A   ):
	with dataset.formatted_as(type=A   ):
		for i in range(0				,      A				,      A   ):
			UpperCAmelCase_								=     dataset[i : i + batch_size]




def    A_(			):
	UpperCAmelCase_								=     {"""num examples""": SPEED_TEST_N_EXAMPLES}
	UpperCAmelCase_								=     [
	    (read, {"""length""": SMALL_TEST}),
	    (read, {"""length""": SPEED_TEST_N_EXAMPLES}),
	    (read_batch, {"""length""": SPEED_TEST_N_EXAMPLES, """batch_size""": 10}),
	    (read_batch, {"""length""": SPEED_TEST_N_EXAMPLES, """batch_size""": 100}),
	    (read_batch, {"""length""": SPEED_TEST_N_EXAMPLES, """batch_size""": 1000}),
	    (read_formatted, {"""type""": """numpy""", """length""": SMALL_TEST}),
	    (read_formatted, {"""type""": """pandas""", """length""": SMALL_TEST}),
	    (read_formatted, {"""type""": """torch""", """length""": SMALL_TEST}),
	    (read_formatted, {"""type""": """tensorflow""", """length""": SMALL_TEST}),
	    (read_formatted_batch, {"""type""": """numpy""", """length""": SMALL_TEST, """batch_size""": 10}),
	    (read_formatted_batch, {"""type""": """numpy""", """length""": SMALL_TEST, """batch_size""": 1000}),
	]

	UpperCAmelCase_								=     [
	    (read, {"""length""": SMALL_TEST}),
	    (read, {"""length""": SPEED_TEST_N_EXAMPLES}),
	    (read_batch, {"""length""": SPEED_TEST_N_EXAMPLES, """batch_size""": 10}),
	    (read_batch, {"""length""": SPEED_TEST_N_EXAMPLES, """batch_size""": 100}),
	    (read_batch, {"""length""": SPEED_TEST_N_EXAMPLES, """batch_size""": 1000}),
	    (read_formatted, {"""type""": """numpy""", """length""": SMALL_TEST}),
	    (read_formatted_batch, {"""type""": """numpy""", """length""": SMALL_TEST, """batch_size""": 10}),
	    (read_formatted_batch, {"""type""": """numpy""", """length""": SMALL_TEST, """batch_size""": 1000}),
	]
	with tempfile.TemporaryDirectory() as tmp_dir:
		print("""generating dataset"""   )
		UpperCAmelCase_								=     datasets.Features(
		    {"""list""": datasets.Sequence(datasets.Value("""float32"""   )   ), """numbers""": datasets.Value("""float32"""   )}   )
		UpperCAmelCase_								=     generate_example_dataset(
		    os.path.join(A				,      """dataset.arrow"""   )				,      A				,      num_examples=A				,      seq_shapes={"""list""": (100,)}				,      )
		print("""first set of iterations"""   )
		for func, kwargs in functions:
			print(func.__name__				,      str(A   )   )
			UpperCAmelCase_								=     func(A				,      **A   )

		print("""shuffling dataset"""   )
		UpperCAmelCase_								=     dataset.shuffle()
		print("""Second set of iterations (after shuffling"""   )
		for func, kwargs in functions_shuffled:
			print("""shuffled """				,      func.__name__				,      str(A   )   )
			UpperCAmelCase_								=     func(
			    A				,      **A   )

	with open(A				,      """wb"""   ) as f:
		f.write(json.dumps(A   ).encode("""utf-8"""   )   )


if __name__ == "__main__":  # useful to run the profiler
				benchmark_iterating()


