




"""simple docstring"""





import argparse
import re

import requests
import torch

# git clone https://github.com/salesforce/BLIP.git
from models.blip import blip_decoder
from models.blip_itm import blip_itm
from models.blip_vqa import blip_vqa
from PIL import Image
from torchvision import transforms
from torchvision.transforms.functional import InterpolationMode

from transformers import (
    BertTokenizer,
    BlipConfig,
    BlipForConditionalGeneration,
    BlipForImageTextRetrieval,
    BlipForQuestionAnswering,
)

def 					_UpperCamelCase    (			UpperCamelCase				,       UpperCamelCase							)			->			List[Any]:


			"""simple docstring"""
			__UpperCAmelCase	: Tuple		=					"https://storage.googleapis.com/sfr-vision-language-research/BLIP/demo.jpg"
			__UpperCAmelCase	: Any		=					Image.open(requests.get(UpperCamelCase				,       stream=UpperCamelCase							).raw							).convert("RGB"							)

			__UpperCAmelCase	: List[str]		=					transforms.Compose(
			    [
			        transforms.Resize((image_size, image_size)				,       interpolation=InterpolationMode.BICUBIC							),
			        transforms.ToTensor(),
			        transforms.Normalize((0.48145466, 0.4578275, 0.40821073)				,       (0.26862954, 0.26130258, 0.27577711)							),
			    ]							)
			__UpperCAmelCase	: Any		=					transform(UpperCamelCase							).unsqueeze(0							).to(UpperCamelCase							)
			return image

def 					_UpperCamelCase    (			UpperCamelCase							)			->			Optional[Any]:


			"""simple docstring"""
			if "visual_encoder" in key:
						__UpperCAmelCase	: Dict		=					re.sub("visual_encoder*"				,       "vision_model.encoder"				,       UpperCamelCase							)
			if "blocks" in key:
						__UpperCAmelCase	: Dict		=					re.sub(R"blocks"				,       "layers"				,       UpperCamelCase							)
			if "attn" in key:
						__UpperCAmelCase	: Optional[int]		=					re.sub(R"attn"				,       "self_attn"				,       UpperCamelCase							)
			if "norm1" in key:
						__UpperCAmelCase	: Any		=					re.sub(R"norm1"				,       "layer_norm1"				,       UpperCamelCase							)
			if "norm2" in key:
						__UpperCAmelCase	: Union[str, Any]		=					re.sub(R"norm2"				,       "layer_norm2"				,       UpperCamelCase							)
			if "encoder.norm" in key:
						__UpperCAmelCase	: Dict		=					re.sub(R"encoder.norm"				,       "post_layernorm"				,       UpperCamelCase							)
			if "encoder.patch_embed.proj" in key:
						__UpperCAmelCase	: Any		=					re.sub(R"encoder.patch_embed.proj"				,       "embeddings.patch_embedding"				,       UpperCamelCase							)

			if "encoder.pos_embed" in key:
						__UpperCAmelCase	: List[str]		=					re.sub(R"encoder.pos_embed"				,       "embeddings.position_embedding"				,       UpperCamelCase							)
			if "encoder.cls_token" in key:
						__UpperCAmelCase	: Optional[Any]		=					re.sub(R"encoder.cls_token"				,       "embeddings.class_embedding"				,       UpperCamelCase							)

			if "self_attn" in key:
						__UpperCAmelCase	: Optional[int]		=					re.sub(R"self_attn.proj"				,       "self_attn.projection"				,       UpperCamelCase							)

			return key

@torch.no_grad()
def 					_UpperCamelCase    (			UpperCamelCase				,       UpperCamelCase=None							)			->			Optional[Any]:


			"""simple docstring"""
			if config_path is not None:
						__UpperCAmelCase	: Tuple		=					BlipConfig.from_pretrained(UpperCamelCase							)
			else:
						__UpperCAmelCase	: List[str]		=					BlipConfig(projection_dim=512				,       text_config={}				,       vision_config={}							)

			__UpperCAmelCase	: List[Any]		=					BlipForConditionalGeneration(UpperCamelCase							).eval()

			__UpperCAmelCase	: str		=					"https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_capfilt_large.pth"

			__UpperCAmelCase	: Union[str, Any]		=					blip_decoder(pretrained=UpperCamelCase				,       image_size=384				,       vit="base"							)
			__UpperCAmelCase	: List[Any]		=					pt_model.eval()

			__UpperCAmelCase	: str		=					pt_model.state_dict()
			for key in modified_state_dict.copy():
						__UpperCAmelCase	: Optional[int]		=					modified_state_dict.pop(UpperCamelCase							)
						__UpperCAmelCase	: List[Any]		=					rename_key(UpperCamelCase							)
						__UpperCAmelCase	: Any		=					value

			hf_model.load_state_dict(UpperCamelCase							)

			__UpperCAmelCase	: Optional[Any]		=					384
			__UpperCAmelCase	: List[str]		=					load_demo_image(image_size=UpperCamelCase				,       device="cpu"							)
			__UpperCAmelCase	: Tuple		=					BertTokenizer.from_pretrained("bert-base-uncased"							)
			__UpperCAmelCase	: Any		=					tokenizer(["a picture of"]							).input_ids

			__UpperCAmelCase	: Optional[int]		=					hf_model.generate(UpperCamelCase				,       UpperCamelCase							)

			assert out[0].tolist() == [3_0522, 1037, 3861, 1997, 1037, 2450, 3564, 2006, 1996, 3509, 2007, 2014, 3899, 102]

			__UpperCAmelCase	: Any		=					hf_model.generate(UpperCamelCase							)

			assert out[0].tolist() == [3_0522, 1037, 2450, 3564, 2006, 1996, 3509, 2007, 2014, 3899, 102]

			if pytorch_dump_folder_path is not None:
						hf_model.save_pretrained(UpperCamelCase							)

			# model_url = 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_vqa.pth'
			__UpperCAmelCase	: Optional[int]		=					(
			    "https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_vqa_capfilt_large.pth"
			)

			__UpperCAmelCase	: List[str]		=					blip_vqa(pretrained=UpperCamelCase				,       image_size=UpperCamelCase				,       vit="base"							)
			vqa_model.eval()

			__UpperCAmelCase	: Tuple		=					vqa_model.state_dict()
			for key in modified_state_dict.copy():
						__UpperCAmelCase	: Union[str, Any]		=					modified_state_dict.pop(UpperCamelCase							)
						__UpperCAmelCase	: Optional[Any]		=					rename_key(UpperCamelCase							)
						__UpperCAmelCase	: Any		=					value

			__UpperCAmelCase	: Tuple		=					BlipForQuestionAnswering(UpperCamelCase							)

			hf_vqa_model.load_state_dict(UpperCamelCase							)

			__UpperCAmelCase	: int		=					["How many dogs are in this image?"]
			__UpperCAmelCase	: int		=					tokenizer(UpperCamelCase				,       return_tensors="pt"							).input_ids

			__UpperCAmelCase	: Union[str, Any]		=					hf_vqa_model.generate(UpperCamelCase				,       UpperCamelCase							)
			print(tokenizer.decode(answer[0]							)							)

			assert tokenizer.decode(answer[0]							) == "[UNK] 1 [SEP]"
			if pytorch_dump_folder_path is not None:
						hf_vqa_model.save_pretrained(pytorch_dump_folder_path + "_vqa"							)

			__UpperCAmelCase	: Tuple		=					"https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_retrieval_coco.pth"

			__UpperCAmelCase	: List[Any]		=					blip_itm(pretrained=UpperCamelCase				,       image_size=UpperCamelCase				,       vit="base"							)
			itm_model.eval()

			__UpperCAmelCase	: List[str]		=					itm_model.state_dict()
			for key in modified_state_dict.copy():
						__UpperCAmelCase	: int		=					modified_state_dict.pop(UpperCamelCase							)
						__UpperCAmelCase	: str		=					rename_key(UpperCamelCase							)
						__UpperCAmelCase	: Union[str, Any]		=					value

			__UpperCAmelCase	: List[str]		=					BlipForImageTextRetrieval(UpperCamelCase							)

			__UpperCAmelCase	: List[str]		=					["A picture of a woman with a dog sitting in a beach"]
			__UpperCAmelCase	: int		=					tokenizer(
			    UpperCamelCase				,       return_tensors="pt"				,       padding="max_length"				,       truncation=UpperCamelCase				,       max_length=35				,       ).input_ids

			hf_itm_model.load_state_dict(UpperCamelCase							)
			hf_itm_model.eval()

			__UpperCAmelCase	: Tuple		=					hf_itm_model(UpperCamelCase				,       UpperCamelCase				,       use_itm_head=UpperCamelCase							)
			__UpperCAmelCase	: Dict		=					hf_itm_model(UpperCamelCase				,       UpperCamelCase				,       use_itm_head=UpperCamelCase							)

			assert out[0].item() == 0.2110687494277954
			assert torch.nn.functional.softmax(out_itm[0]				,       dim=1							)[:, 1].item() == 0.45698845386505127

			if pytorch_dump_folder_path is not None:
						hf_itm_model.save_pretrained(pytorch_dump_folder_path + "_itm"							)


if __name__ == "__main__":
					A   =       argparse.ArgumentParser()
					parser.add_argument("""--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
					parser.add_argument("""--config_path""", default=None, type=str, help="""Path to hf config.json of model to convert""")
					A   =       parser.parse_args()

					convert_blip_checkpoint(args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path)








"""simple docstring"""





import argparse

import torch
from datasets import load_dataset
from donut import DonutModel

from transformers import (
    DonutImageProcessor,
    DonutProcessor,
    DonutSwinConfig,
    DonutSwinModel,
    MBartConfig,
    MBartForCausalLM,
    VisionEncoderDecoderModel,
    XLMRobertaTokenizerFast,
)

def 					_UpperCamelCase    (			UpperCamelCase							)			->			Tuple:


			"""simple docstring"""
			__UpperCAmelCase	: str		=					model.config

			__UpperCAmelCase	: int		=					DonutSwinConfig(
			    image_size=original_config.input_size				,       patch_size=4				,       depths=original_config.encoder_layer				,       num_heads=[4, 8, 16, 32]				,       window_size=original_config.window_size				,       embed_dim=128				,       )
			__UpperCAmelCase	: Union[str, Any]		=					MBartConfig(
			    is_decoder=UpperCamelCase				,       is_encoder_decoder=UpperCamelCase				,       add_cross_attention=UpperCamelCase				,       decoder_layers=original_config.decoder_layer				,       max_position_embeddings=original_config.max_position_embeddings				,       vocab_size=len(
			        model.decoder.tokenizer							)				,       scale_embedding=UpperCamelCase				,       add_final_layer_norm=UpperCamelCase				,       )

			return encoder_config, decoder_config

def 					_UpperCamelCase    (			UpperCamelCase							)			->			Union[str, Any]:


			"""simple docstring"""
			if "encoder.model" in name:
						__UpperCAmelCase	: Union[str, Any]		=					name.replace("encoder.model"				,       "encoder"							)
			if "decoder.model" in name:
						__UpperCAmelCase	: Dict		=					name.replace("decoder.model"				,       "decoder"							)
			if "patch_embed.proj" in name:
						__UpperCAmelCase	: Dict		=					name.replace("patch_embed.proj"				,       "embeddings.patch_embeddings.projection"							)
			if "patch_embed.norm" in name:
						__UpperCAmelCase	: str		=					name.replace("patch_embed.norm"				,       "embeddings.norm"							)
			if name.startswith("encoder"							):
						if "layers" in name:
									__UpperCAmelCase	: Dict		=					"encoder." + name
						if "attn.proj" in name:
									__UpperCAmelCase	: List[Any]		=					name.replace("attn.proj"				,       "attention.output.dense"							)
						if "attn" in name and "mask" not in name:
									__UpperCAmelCase	: int		=					name.replace("attn"				,       "attention.self"							)
						if "norm1" in name:
									__UpperCAmelCase	: Optional[Any]		=					name.replace("norm1"				,       "layernorm_before"							)
						if "norm2" in name:
									__UpperCAmelCase	: str		=					name.replace("norm2"				,       "layernorm_after"							)
						if "mlp.fc1" in name:
									__UpperCAmelCase	: str		=					name.replace("mlp.fc1"				,       "intermediate.dense"							)
						if "mlp.fc2" in name:
									__UpperCAmelCase	: Union[str, Any]		=					name.replace("mlp.fc2"				,       "output.dense"							)

						if name == "encoder.norm.weight":
									__UpperCAmelCase	: List[Any]		=					"encoder.layernorm.weight"
						if name == "encoder.norm.bias":
									__UpperCAmelCase	: Tuple		=					"encoder.layernorm.bias"

			return name

def 					_UpperCamelCase    (			UpperCamelCase				,       UpperCamelCase							)			->			List[Any]:


			"""simple docstring"""
			for key in orig_state_dict.copy().keys():
						__UpperCAmelCase	: Any		=					orig_state_dict.pop(UpperCamelCase							)

						if "qkv" in key:
									__UpperCAmelCase	: Any		=					key.split("."							)
									__UpperCAmelCase	: List[Any]		=					int(key_split[3]							)
									__UpperCAmelCase	: Any		=					int(key_split[5]							)
									__UpperCAmelCase	: str		=					model.encoder.encoder.layers[layer_num].blocks[block_num].attention.self.all_head_size

									if "weight" in key:
												__UpperCAmelCase	: List[Any]		=					val[:dim, :]
												__UpperCAmelCase	: Union[str, Any]		=					val[dim : dim * 2, :]
												__UpperCAmelCase	: Any		=					val[-dim:, :]
									else:
												__UpperCAmelCase	: List[str]		=					val[:dim]
												__UpperCAmelCase	: List[Any]		=					val[dim : dim * 2]
												__UpperCAmelCase	: Tuple		=					val[-dim:]
						elif "attn_mask" in key or key in ["encoder.model.norm.weight", "encoder.model.norm.bias"]:
									# HuggingFace implementation doesn't use attn_mask buffer
									# and model doesn't use final LayerNorms for the encoder
									pass
						else:
									__UpperCAmelCase	: Dict		=					val

			return orig_state_dict

def 					_UpperCamelCase    (			UpperCamelCase				,       UpperCamelCase=None				,       UpperCamelCase=False							)			->			List[str]:


			"""simple docstring"""
			# load original model
			__UpperCAmelCase	: Dict		=					DonutModel.from_pretrained(UpperCamelCase							).eval()

			# load HuggingFace model
			__UpperCAmelCase	,     __UpperCAmelCase	: List[str]		=					get_configs(UpperCamelCase							)
			__UpperCAmelCase	: Tuple		=					DonutSwinModel(UpperCamelCase							)
			__UpperCAmelCase	: Optional[Any]		=					MBartForCausalLM(UpperCamelCase							)
			__UpperCAmelCase	: int		=					VisionEncoderDecoderModel(encoder=UpperCamelCase				,       decoder=UpperCamelCase							)
			model.eval()

			__UpperCAmelCase	: Union[str, Any]		=					original_model.state_dict()
			__UpperCAmelCase	: List[str]		=					convert_state_dict(UpperCamelCase				,       UpperCamelCase							)
			model.load_state_dict(UpperCamelCase							)

			# verify results on scanned document
			__UpperCAmelCase	: List[Any]		=					load_dataset("hf-internal-testing/example-documents"							)
			__UpperCAmelCase	: Any		=					dataset["test"][0]["image"].convert("RGB"							)

			__UpperCAmelCase	: Union[str, Any]		=					XLMRobertaTokenizerFast.from_pretrained(UpperCamelCase				,       from_slow=UpperCamelCase							)
			__UpperCAmelCase	: str		=					DonutImageProcessor(
			    do_align_long_axis=original_model.config.align_long_axis				,       size=original_model.config.input_size[::-1]							)
			__UpperCAmelCase	: Union[str, Any]		=					DonutProcessor(UpperCamelCase				,       UpperCamelCase							)
			__UpperCAmelCase	: Union[str, Any]		=					processor(UpperCamelCase				,       return_tensors="pt"							).pixel_values

			if model_name == "naver-clova-ix/donut-base-finetuned-docvqa":
						__UpperCAmelCase	: List[Any]		=					"<s_docvqa><s_question>{user_input}</s_question><s_answer>"
						__UpperCAmelCase	: Any		=					"When is the coffee break?"
						__UpperCAmelCase	: Dict		=					task_prompt.replace("{user_input}"				,       UpperCamelCase							)
			elif model_name == "naver-clova-ix/donut-base-finetuned-rvlcdip":
						__UpperCAmelCase	: int		=					"<s_rvlcdip>"
			elif model_name in [
			    "naver-clova-ix/donut-base-finetuned-cord-v1",
			    "naver-clova-ix/donut-base-finetuned-cord-v1-2560",
			]:
						__UpperCAmelCase	: Union[str, Any]		=					"<s_cord>"
			elif model_name == "naver-clova-ix/donut-base-finetuned-cord-v2":
						__UpperCAmelCase	: Dict		=					"s_cord-v2>"
			elif model_name == "naver-clova-ix/donut-base-finetuned-zhtrainticket":
						__UpperCAmelCase	: str		=					"<s_zhtrainticket>"
			elif model_name in ["naver-clova-ix/donut-proto", "naver-clova-ix/donut-base"]:
						# use a random prompt
						__UpperCAmelCase	: Tuple		=					"hello world"
			else:
						raise ValueError("Model name not supported"							)
			__UpperCAmelCase	: Dict		=					original_model.decoder.tokenizer(UpperCamelCase				,       add_special_tokens=UpperCamelCase				,       return_tensors="pt"							)[
			    "input_ids"
			]

			__UpperCAmelCase	: int		=					original_model.encoder.model.patch_embed(UpperCamelCase							)
			__UpperCAmelCase	,     __UpperCAmelCase	: Tuple		=					model.encoder.embeddings(UpperCamelCase							)
			assert torch.allclose(UpperCamelCase				,       UpperCamelCase				,       atol=1e-3							)

			# verify encoder hidden states
			__UpperCAmelCase	: int		=					original_model.encoder(UpperCamelCase							)
			__UpperCAmelCase	: Union[str, Any]		=					model.encoder(UpperCamelCase							).last_hidden_state
			assert torch.allclose(UpperCamelCase				,       UpperCamelCase				,       atol=1e-2							)

			# verify decoder hidden states
			__UpperCAmelCase	: int		=					original_model(UpperCamelCase				,       UpperCamelCase				,       UpperCamelCase							).logits
			__UpperCAmelCase	: List[str]		=					model(UpperCamelCase				,       decoder_input_ids=UpperCamelCase							).logits
			assert torch.allclose(UpperCamelCase				,       UpperCamelCase				,       atol=1e-3							)
			print("Looks ok!"							)

			if pytorch_dump_folder_path is not None:
						print(f"Saving model and processor to {pytorch_dump_folder_path}"							)
						model.save_pretrained(UpperCamelCase							)
						processor.save_pretrained(UpperCamelCase							)

			if push_to_hub:
						model.push_to_hub("nielsr/" + model_name.split("/"							)[-1]				,       commit_message="Update model"							)
						processor.push_to_hub("nielsr/" + model_name.split("/"							)[-1]				,       commit_message="Update model"							)


if __name__ == "__main__":
					A   =       argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    """--model_name""",
					    default="""naver-clova-ix/donut-base-finetuned-docvqa""",
					    required=False,
					    type=str,
					    help="""Name of the original model you'd like to convert.""",
					)
					parser.add_argument(
					    """--pytorch_dump_folder_path""",
					    default=None,
					    required=False,
					    type=str,
					    help="""Path to the output PyTorch model directory.""",
					)
					parser.add_argument(
					    """--push_to_hub""",
					    action="""store_true""",
					    help="""Whether or not to push the converted model and processor to the 🤗 hub.""",
					)

					A   =       parser.parse_args()
					convert_donut_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)


