Qwen

/

Qwen2.5-VL-72B-Instruct-AWQ

like 59

Follow

Qwen 41k

I keep getting this error, but only for some input images, when using the AWQ version of the model. The huge issue is that this ends up poisoning the device, and I can't do any inference on any images, even ones that worked previously, once the error happens. However, I can't seem to figure out what's going on or why it's happening, and inspecting various things about the images seem to produce no obvious differences - sizes are identical, the device has enough memory for all of the images, pixel values do range from 0 to 255, so maybe it's an overflow error?

Here is the generate function I'm using:

from fastapi import FastAPI, File, UploadFile, Form
from fastapi.responses import JSONResponse
from transformers import (
    Qwen2_5_VLForConditionalGeneration,
    AutoProcessor,
    AutoModelForCausalLM,
    AutoTokenizer
)
from qwen_vl_utils import process_vision_info
import torch
from PIL import Image
import tempfile
import uvicorn
import os

app = FastAPI()

# ----------------------------
# Model 1: Vision-Language Model (GPU1)
# ----------------------------
model_vl = Qwen2_5_VLForConditionalGeneration.from_pretrained(
    "Qwen/Qwen2.5-VL-72B-Instruct-AWQ", torch_dtype=torch.bfloat16, device_map={"": 1}
)
print(model_vl)
#processor_vl = AutoProcessor.from_pretrained("Qwen/Qwen2.5-VL-72B-Instruct-AWQ")
processor_vl = AutoProcessor.from_pretrained("Qwen/Qwen2.5-VL-72B-Instruct-AWQ")


# ----------------------------
# Vision + Language Endpoint
# ----------------------------


@app
	.post("/generate_vision")
async def generate_vision(prompt: str = Form(...), file: UploadFile = File(...)):
    try:
        filename = file.filename
        suffix = os.path.splitext(filename)[1]

        # Create a permanent storage directory if it doesn't exist
        save_dir = "saved_images"
        os.makedirs(save_dir, exist_ok=True)

        # Save image permanently
        saved_path = os.path.join(save_dir, filename)
        with open(saved_path, "wb") as f:
            f.write(await file.read())

        # Re-open the saved file for reading during processing
        img_path = saved_path

        messages = [
            {
                "role": "user",
                "content": [
                    {"type": "image", "image": img_path},
                    {"type": "text", "text": prompt},
                ],
            }
        ]

        text = processor_vl.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
        image_inputs, video_inputs = process_vision_info(messages)
        inputs = processor_vl(
            text=[text],
            images=image_inputs,
            videos=video_inputs,
            padding=True,
            return_tensors="pt",
        )
        inputs = inputs.to("cuda:1")
        for k, v in inputs.items():
            if torch.isnan(v).any():
                print(f"NaNs in {k}")
            if torch.isinf(v).any():
                print(f"Infs in {k}")
            if (v < 0).any() and k != 'attention_mask':
                print(f"Negative values in {k}")

        generated_ids = model_vl.generate(**inputs, max_new_tokens=4096)
        generated_ids_trimmed = [
            out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
        ]
        output_text = processor_vl.batch_decode(
            generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
        )

        return JSONResponse(content={"result": output_text[0], "saved_image": saved_path})
    except Exception as e:
        #If something went wrong and poisoned the device, we need to know what the problem was. 
        
        return JSONResponse(content={"result": f"processing the image failed: {str(e)}"})