






"""simple docstring"""



import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis_float32 (there's also the fix_lavis branch)
# also note: to convert Vicuna checkpoints, we had to include /home/niels/python_projects/checkpoints/FastChat/vicuna-7b in lavis/configs/models/blip2/blip2_instruct_vicuna7b.yaml
# same for Vicuna-13b
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipImageProcessor,
    InstructBlipConfig,
    InstructBlipForConditionalGeneration,
    InstructBlipProcessor,
    InstructBlipQFormerConfig,
    InstructBlipVisionConfig,
    LlamaConfig,
    LlamaTokenizerFast,
    TaConfig,
    TaTokenizerFast,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD




def      _lowerCAmelCase		(							)		->Union[str, Any]:
						A__						:  Union[str, Any]		  =  """https://raw.githubusercontent.com/salesforce/LAVIS/main/docs/_static/Confusing-Pictures.jpg"""
						A__						:  str		  =  Image.open(requests.get(UpperCAmelCase__,						stream=UpperCAmelCase__      ).raw      ).convert("""RGB"""      )

						return image




def      _lowerCAmelCase		(							UpperCAmelCase__       :					Dict      )		->Union[str, Any]:
						A__						:  List[str]		  =  []
						# fmt: off

						# vision encoder
						rename_keys.append(("""visual_encoder.cls_token""", """vision_model.embeddings.class_embedding""")      )
						rename_keys.append(("""visual_encoder.pos_embed""", """vision_model.embeddings.position_embedding""")      )
						rename_keys.append(("""visual_encoder.patch_embed.proj.weight""", """vision_model.embeddings.patch_embedding.weight""")      )
						rename_keys.append(("""visual_encoder.patch_embed.proj.bias""", """vision_model.embeddings.patch_embedding.bias""")      )
						rename_keys.append(("""ln_vision.weight""", """vision_model.post_layernorm.weight""")      )
						rename_keys.append(("""ln_vision.bias""", """vision_model.post_layernorm.bias""")      )

						for i in range(config.vision_config.num_hidden_layers      ):
												rename_keys.append((f'visual_encoder.blocks.{i}.norm1.weight', f'vision_model.encoder.layers.{i}.layer_norm1.weight')      )
												rename_keys.append((f'visual_encoder.blocks.{i}.norm1.bias', f'vision_model.encoder.layers.{i}.layer_norm1.bias')      )
												rename_keys.append((f'visual_encoder.blocks.{i}.norm2.weight', f'vision_model.encoder.layers.{i}.layer_norm2.weight')      )
												rename_keys.append((f'visual_encoder.blocks.{i}.norm2.bias', f'vision_model.encoder.layers.{i}.layer_norm2.bias')      )
												rename_keys.append((f'visual_encoder.blocks.{i}.attn.qkv.weight', f'vision_model.encoder.layers.{i}.self_attn.qkv.weight')      )
												rename_keys.append((f'visual_encoder.blocks.{i}.attn.proj.weight', f'vision_model.encoder.layers.{i}.self_attn.projection.weight',)      )
												rename_keys.append((f'visual_encoder.blocks.{i}.attn.proj.bias', f'vision_model.encoder.layers.{i}.self_attn.projection.bias')      )
												rename_keys.append((f'visual_encoder.blocks.{i}.mlp.fc1.weight', f'vision_model.encoder.layers.{i}.mlp.fc1.weight')      )
												rename_keys.append((f'visual_encoder.blocks.{i}.mlp.fc1.bias', f'vision_model.encoder.layers.{i}.mlp.fc1.bias')      )
												rename_keys.append((f'visual_encoder.blocks.{i}.mlp.fc2.weight', f'vision_model.encoder.layers.{i}.mlp.fc2.weight')      )
												rename_keys.append((f'visual_encoder.blocks.{i}.mlp.fc2.bias', f'vision_model.encoder.layers.{i}.mlp.fc2.bias')      )

						# QFormer
						rename_keys.append(("""Qformer.bert.embeddings.LayerNorm.weight""", """qformer.embeddings.layernorm.weight""")      )
						rename_keys.append(("""Qformer.bert.embeddings.LayerNorm.bias""", """qformer.embeddings.layernorm.bias""")      )

						# fmt: on
						return rename_keys




def      _lowerCAmelCase		(							UpperCAmelCase__       :					List[Any],						UpperCAmelCase__       :					Union[str, Any],						UpperCAmelCase__       :					Union[str, Any]      )		->List[Any]:
						A__						:  Optional[Any]		  =  dct.pop(UpperCAmelCase__      )
						A__						:  Optional[int]		  =  val




def      _lowerCAmelCase		(							UpperCAmelCase__       :					Optional[Any],						UpperCAmelCase__       :					int      )		->Tuple:
						for i in range(config.vision_config.num_hidden_layers      ):
												# read in original q and v biases
												A__						:  Union[str, Any]		  =  state_dict.pop(f'visual_encoder.blocks.{i}.attn.q_bias'      )
												A__						:  Dict		  =  state_dict.pop(f'visual_encoder.blocks.{i}.attn.v_bias'      )

												# next, set bias in the state dict
												A__						:  Union[str, Any]		  =  torch.cat((q_bias, torch.zeros_like(UpperCAmelCase__,						requires_grad=UpperCAmelCase__      ), v_bias)      )
												A__						:  int		  =  qkv_bias




def      _lowerCAmelCase		(							UpperCAmelCase__       :					Optional[int]      )		->str:
						A__						:  Dict		  =  3_6_4 if """coco""" in model_name else 2_2_4
						A__						:  List[str]		  =  InstructBlipVisionConfig(image_size=UpperCAmelCase__      ).to_dict()

						# make sure the models have proper bos_token_id and eos_token_id set (important for generation)
						# seems like flan-T5 models don't have bos_token_id properly set?
						if "t5-xl" in model_name:
												A__						:  List[str]		  =  TaConfig.from_pretrained("""google/flan-t5-xl""",						dense_act_fn="""gelu""",						bos_token_id=1      ).to_dict()
						elif "t5-xxl" in model_name:
												A__						:  Optional[int]		  =  TaConfig.from_pretrained("""google/flan-t5-xxl""",						dense_act_fn="""gelu""",						bos_token_id=1      ).to_dict()
						elif "vicuna-7b" in model_name:
												A__						:  Optional[int]		  =  LlamaConfig.from_pretrained("""decapoda-research/llama-7b-hf""",						vocab_size=3_2_0_0_1      ).to_dict()
						elif "vicuna-13b" in model_name:
												A__						:  Optional[int]		  =  LlamaConfig.from_pretrained("""decapoda-research/llama-13b-hf""",						vocab_size=3_2_0_0_1      ).to_dict()
						else:
												raise ValueError("""Model name not supported"""      )

						# the authors add one special "[DEC]" token to the vocab of Q-Former, hence vocab size = 30522 + 1
						A__						:  Any		  =  InstructBlipQFormerConfig(vocab_size=3_0_5_2_3      ).to_dict()
						A__						:  Optional[int]		  =  InstructBlipConfig(vision_config=UpperCAmelCase__,						text_config=UpperCAmelCase__,						qformer_config=UpperCAmelCase__      )

						return config, image_size




@torch.no_grad()
def      _lowerCAmelCase		(							UpperCAmelCase__       :					int,						UpperCAmelCase__       :					int=None,						UpperCAmelCase__       :					List[Any]=False      )		->Tuple:
						A__						:  str		  =  AutoTokenizer.from_pretrained("""bert-base-uncased""",						truncation_side="""left"""      )
						qformer_tokenizer.add_special_tokens({"""bos_token""": """[DEC]"""}      )

						if "t5" in model_name:
												A__						:  List[Any]		  =  TaTokenizerFast.from_pretrained("""google/flan-t5-xl""",						truncation_side="""left"""      )
						elif "vicuna" in model_name:
												# the following was used in the original implementation:
												# tokenizer = LlamaTokenizer.from_pretrained("huggyllama/llama-7b", use_fast=False, truncation_side="left")
												# tokenizer.add_special_tokens({"pad_token": "[PAD]"})
												# tokenizer.add_special_tokens({"bos_token": "</s>"})
												# tokenizer.add_special_tokens({"eos_token": "</s>"})
												# tokenizer.add_special_tokens({"unk_token": "</s>"})
												A__						:  Optional[Any]		  =  LlamaTokenizerFast.from_pretrained(
												    """huggyllama/llama-7b""",						truncation_side="""left""",						bos_token="""</s>""",						unk_token="""</s>"""      )
												tokenizer.add_special_tokens({"""pad_token""": """[PAD]"""}      )

						A__  ,  A__						:  Optional[int]		  =  get_blipa_config(UpperCAmelCase__      )
						A__						:  Tuple		  =  InstructBlipForConditionalGeneration(UpperCAmelCase__      ).eval()

						A__						:  int		  =  {
						    """instructblip-vicuna-7b""": ("""blip2_vicuna_instruct""", """vicuna7b"""),
						    """instructblip-vicuna-13b""": ("""blip2_vicuna_instruct""", """vicuna13b"""),
						    """instructblip-flan-t5-xl""": ("""blip2_t5_instruct""", """flant5xl"""),
						    """instructblip-flan-t5-xxl""": ("""blip2_t5_instruct""", """flant5xxl"""),
						}

						A__  ,  A__						:  Dict		  =  model_name_to_original[model_name]

						# load original model
						print("""Loading original model..."""      )
						A__						:  str		  =  """cuda:1""" if torch.cuda.is_available() else """cpu"""
						A__						:  str		  =  """cuda:2""" if torch.cuda.is_available() else """cpu"""
						A__  ,  A__  ,  A__						:  Optional[Any]		  =  load_model_and_preprocess(
						    name=UpperCAmelCase__,						model_type=UpperCAmelCase__,						is_eval=UpperCAmelCase__,						device=UpperCAmelCase__      )
						original_model.eval()
						print("""Done!"""      )

						# update state dict keys
						A__						:  int		  =  original_model.state_dict()
						A__						:  List[str]		  =  create_rename_keys(UpperCAmelCase__      )
						for src, dest in rename_keys:
												rename_key(UpperCAmelCase__,						UpperCAmelCase__,						UpperCAmelCase__      )

						# some keys can be renamed efficiently
						for key, val in state_dict.copy().items():
												A__						:  List[str]		  =  state_dict.pop(UpperCAmelCase__      )
												if key.startswith("""Qformer.bert"""      ):
																		A__						:  List[Any]		  =  key.replace("""Qformer.bert""",						"""qformer"""      )
												if "attention.self" in key:
																		A__						:  int		  =  key.replace("""self""",						"""attention"""      )
												if "llm_proj" in key:
																		A__						:  List[Any]		  =  key.replace("""llm_proj""",						"""language_projection"""      )
												if "t5_proj" in key:
																		A__						:  List[str]		  =  key.replace("""t5_proj""",						"""language_projection"""      )
												if key.startswith("""llm_model"""      ):
																		A__						:  Union[str, Any]		  =  key.replace("""llm_model""",						"""language_model"""      )
												if key.startswith("""t5"""      ):
																		A__						:  Optional[Any]		  =  key.replace("""t5""",						"""language"""      )
												A__						:  Optional[int]		  =  val

						# read in qv biases
						read_in_q_v_bias(UpperCAmelCase__,						UpperCAmelCase__      )

						# note: weights get loaded in torch.float32 by default
						hf_model.load_state_dict(UpperCAmelCase__,						strict=UpperCAmelCase__      )

						A__						:  Dict		  =  load_demo_image()
						A__						:  List[str]		  =  """What is unusual about this image?"""

						# create processor
						A__						:  Union[str, Any]		  =  BlipImageProcessor(
						    size={"""height""": image_size, """width""": image_size},						image_mean=UpperCAmelCase__,						image_std=UpperCAmelCase__      )
						A__						:  Union[str, Any]		  =  InstructBlipProcessor(
						    image_processor=UpperCAmelCase__,						tokenizer=UpperCAmelCase__,						qformer_tokenizer=UpperCAmelCase__,						)
						A__						:  List[Any]		  =  processor(images=UpperCAmelCase__,						text=UpperCAmelCase__,						return_tensors="""pt"""      ).to(UpperCAmelCase__      )

						# make sure processor creates exact same pixel values
						A__						:  Union[str, Any]		  =  vis_processors["""eval"""](UpperCAmelCase__      ).unsqueeze(0      ).to(UpperCAmelCase__      )
						A__						:  int		  =  inputs.pixel_values
						assert torch.allclose(original_pixel_values.to(pixel_values.device      ),						UpperCAmelCase__      )

						original_model.to(UpperCAmelCase__      )
						hf_model.to(UpperCAmelCase__      )
						with torch.no_grad():
												if "vicuna" in model_name:
																		A__						:  str		  =  original_model({"""image""": original_pixel_values, """text_input""": [prompt]}      ).logits
																		A__						:  Optional[int]		  =  hf_model(**UpperCAmelCase__      ).logits
												else:
																		A__						:  int		  =  original_model(
																		    {"""image""": original_pixel_values, """text_input""": [prompt], """text_output""": ["""\n"""]}      ).logits
																		A__						:  Any		  =  tokenizer("""\n""",						return_tensors="""pt"""      ).input_ids.to(UpperCAmelCase__      )
																		A__						:  List[str]		  =  label_input_ids.masked_fill(label_input_ids == tokenizer.pad_token_id,						-1_0_0      )
																		A__						:  Dict		  =  hf_model(**UpperCAmelCase__,						labels=UpperCAmelCase__      ).logits

						print("""First values of original logits:""",						original_logits[0, :3, :3]      )
						print("""First values of HF logits:""",						logits[0, :3, :3]      )

						# assert values
						assert original_logits.shape == logits.shape
						A__						:  Dict		  =  1e-4 if """vicuna""" in model_name else 1e-5
						assert torch.allclose(original_logits.to(logits.device      ),						UpperCAmelCase__,						atol=UpperCAmelCase__      )
						print("""Looks ok!"""      )

						print("""Generating with original model..."""      )
						A__						:  str		  =  original_model.generate({"""image""": original_pixel_values, """prompt""": prompt},						num_beams=5      )

						# important: we need to cast the weights of the HF model to the appropriate type
						print("""Generating with HF model..."""      )
						A__						:  Union[str, Any]		  =  hf_model.generate(
						    **UpperCAmelCase__,						do_sample=UpperCAmelCase__,						num_beams=5,						max_length=2_5_6,						min_length=1,						top_p=0.9,						repetition_penalty=1.5,						length_penalty=1.0,						temperature=1,						)
						if "vicuna" in model_name:
												# convert output id 0 to 2 (eos_token_id)
												# TODO add this in the generate method?
												A__						:  Optional[Any]		  =  2
						print("""Original generation:""",						UpperCAmelCase__      )
						A__						:  Tuple		  =  processor.batch_decode(UpperCAmelCase__,						skip_special_tokens=UpperCAmelCase__      )
						A__						:  int		  =  [text.strip() for text in output_text]
						print("""HF generation:""",						UpperCAmelCase__      )

						if pytorch_dump_folder_path is not None:
												processor.save_pretrained(UpperCAmelCase__      )
												hf_model.save_pretrained(UpperCAmelCase__      )

						if push_to_hub:
												processor.push_to_hub(f'Salesforce/{model_name}'      )
												hf_model.push_to_hub(f'Salesforce/{model_name}'      )


if __name__ == "__main__":
						A_    =							argparse.ArgumentParser()
						A_    =							[
						    '''instructblip-vicuna-7b''',
						    '''instructblip-vicuna-13b''',
						    '''instructblip-flan-t5-xl''',
						    '''instructblip-flan-t5-xxl''',
						]
						parser.add_argument(
						    '''--model_name''',
						    default='''instructblip-flan-t5-xl''',
						    choices=choices,
						    type=str,
						    help='''Path to hf config.json of model to convert''',
						)
						parser.add_argument('''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model.''')
						parser.add_argument(
						    '''--push_to_hub''',
						    action='''store_true''',
						    help='''Whether to push the model and processor to the hub after converting''',
						)

						A_    =							parser.parse_args()

						convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)














"""simple docstring"""



import os

try:
						from .build_directory_md import good_file_paths
except ImportError:
						from build_directory_md import good_file_paths  # type: ignore

A_    =							list(good_file_paths())
assert filepaths, "good_file_paths() failed!"

A_    =							[file for file in filepaths if file != file.lower()]
if upper_files:
						print(F'{len(upper_files)} files contain uppercase characters:')
						print('''\n'''.join(upper_files) + '''\n''')

A_    =							[file for file in filepaths if ''' ''' in file]
if space_files:
						print(F'{len(space_files)} files contain space characters:')
						print('''\n'''.join(space_files) + '''\n''')

A_    =							[file for file in filepaths if '''-''' in file]
if hyphen_files:
						print(F'{len(hyphen_files)} files contain hyphen characters:')
						print('''\n'''.join(hyphen_files) + '''\n''')

A_    =							[file for file in filepaths if os.sep not in file]
if nodir_files:
						print(F'{len(nodir_files)} files are not in a directory:')
						print('''\n'''.join(nodir_files) + '''\n''')

A_    =							len(upper_files + space_files + hyphen_files + nodir_files)
if bad_files:
						import sys

						sys.exit(bad_files)






