Update app.py

app.py

@@ -2,6 +2,7 @@ import os
 import threading
 import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+from importlib.metadata import PackageNotFoundError
 import gradio as gr
 from fastapi import FastAPI
 from pydantic import BaseModel
@@ -22,14 +23,23 @@ MODEL_ID = "tiiuae/Falcon3-3B-Instruct"
 
 tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
 
-# Load model in 4-bit for faster CPU/GPU inference (requires bitsandbytes)
-model = AutoModelForCausalLM.from_pretrained(
-    MODEL_ID,
-    load_in_4bit=True,
-    device_map="auto",
-    torch_dtype=torch.float16,
-    trust_remote_code=True
-)
+# Attempt 4-bit quantization; fallback if bitsandbytes is not installed
+try:
+    model = AutoModelForCausalLM.from_pretrained(
+        MODEL_ID,
+        load_in_4bit=True,
+        device_map="auto",
+        torch_dtype=torch.float16,
+        trust_remote_code=True
+    )
+except PackageNotFoundError:
+    print("bitsandbytes not found; loading full model without quantization.")
+    model = AutoModelForCausalLM.from_pretrained(
+        MODEL_ID,
+        device_map="auto",
+        torch_dtype=torch.float16,
+        trust_remote_code=True
+    )
 
 # Create optimized text-generation pipeline
 pipe = pipeline(