convert_gguf.py

# /// script
# dependencies = ["unsloth[colab-new]", "torch", "transformers", "huggingface_hub", "xformers"]
# ///
"""Convert LoRA adapter to GGUF using Unsloth's built-in conversion."""

import os
import subprocess
import glob

# Pre-install cmake
print("Installing cmake...")
subprocess.run(["apt-get", "update", "-qq"], check=True)
subprocess.run(["apt-get", "install", "-y", "-qq", "cmake", "build-essential"], check=True)

from huggingface_hub import HfApi, create_repo

ADAPTER_MODEL = "victor/functiongemma-agent-finetuned"
OUTPUT_REPO = "victor/functiongemma-agent-gguf"

print("=" * 60)
print(f"Converting {ADAPTER_MODEL} to GGUF")
print("=" * 60)

from unsloth import FastLanguageModel

print("Loading model...")
model, tokenizer = FastLanguageModel.from_pretrained(
    model_name=ADAPTER_MODEL,
    max_seq_length=4096,
    load_in_4bit=False,
    load_in_16bit=True,
)

api = HfApi()
create_repo(OUTPUT_REPO, repo_type="model", exist_ok=True)

print("Converting to GGUF (q4_k_m)...")
model.save_pretrained_gguf(
    "gguf_output",
    tokenizer,
    quantization_method="q4_k_m",
)

# Find and upload ALL gguf files
print("Searching for GGUF files...")
for root, dirs, files in os.walk("."):
    for f in files:
        if f.endswith(".gguf"):
            path = os.path.join(root, f)
            print(f"Found: {path}")
            api.upload_file(
                path_or_fileobj=path,
                path_in_repo=f,
                repo_id=OUTPUT_REPO,
                repo_type="model",
            )
            print(f"Uploaded: {f}")

print("=" * 60)
print(f"DONE! https://huggingface.co/{OUTPUT_REPO}")
print("=" * 60)