fixes

README.md

@@ -1,5 +1,5 @@
 ---
-title: Saiga Nemo 12B GGUF Q4_K_M
+title: Saiga Nemo 12B GPTQ 8-bit
 emoji: 🔥
 colorFrom: pink
 colorTo: yellow
@@ -7,4 +7,4 @@ sdk: gradio
 sdk_version: 5.8.0
 app_file: app.py
 pinned: false
----
+---

app.py

@@ -1,38 +1,13 @@
-import gradio as gr
-
 import os
 
-from huggingface_hub.file_download import http_get
-from llama_cpp import Llama
-
+import gradio as gr
+from openai import OpenAI
 
 SYSTEM_PROMPT = "Ты — Сайга, русскоязычный автоматический ассистент. Ты разговариваешь с людьми и помогаешь им."
-
-
-def load_model(
-    directory: str = ".",
-    model_name: str = "saiga_nemo_12b.Q4_K_M.gguf",
-    model_url: str = "https://huggingface.co/IlyaGusev/saiga_nemo_12b_gguf/resolve/main/saiga_nemo_12b.Q4_K_M.gguf"
-):
-    final_model_path = os.path.join(directory, model_name)
-    
-    print("Downloading all files...")
-    if not os.path.exists(final_model_path):
-        with open(final_model_path, "wb") as f:
-            http_get(model_url, f)
-    os.chmod(final_model_path, 0o777)
-    print("Files downloaded!")
-    
-    model = Llama(
-        model_path=final_model_path,
-        n_ctx=8192
-    )
-    
-    print("Model loaded!")
-    return model
-
-
-MODEL = load_model()
+BASE_URL = os.getenv("BASE_URL")
+API_KEY = os.getenv("API_KEY")
+MODEL_NAME = "IlyaGusev/saiga_nemo_12b_gptq_8bit"
+CLIENT = OpenAI(base_url=BASE_URL, api_key=API_KEY)
 
 
 def user(message, history):
@@ -44,10 +19,8 @@ def bot(
     history,
     system_prompt,
     top_p,
-    top_k,
     temp
 ):
-    model = MODEL
     messages = [{"role": "system", "content": SYSTEM_PROMPT}]
 
     for user_message, bot_message in history[:-1]:
@@ -57,19 +30,21 @@ def bot(
 
     last_user_message = history[-1][0]
     messages.append({"role": "user", "content": last_user_message})
-    partial_text = ""
-    for part in model.create_chat_completion(
-        messages,
+
+    response = CLIENT.chat.completions.create(
+        model=MODEL_NAME,
+        messages=messages,
         temperature=temp,
-        top_k=top_k,
         top_p=top_p,
         stream=True,
-    ):
-        delta = part["choices"][0]["delta"]
-        if "content" in delta:
-            partial_text += delta["content"]
-            history[-1][1] = partial_text
-            yield history
+    )
+
+    partial_text = ""
+    for chunk in response:
+        content = chunk.choices[0].delta.content
+        partial_text += content
+        history[-1][1] = partial_text
+        yield history
 
 
 with gr.Blocks(
@@ -77,11 +52,11 @@ with gr.Blocks(
 ) as demo:
     favicon = '<img src="https://cdn.midjourney.com/b88e5beb-6324-4820-8504-a1a37a9ba36d/0_1.png" width="48px" style="display: inline">'
     gr.Markdown(
-        f"""<h1><center>{favicon}Saiga Nemo 12B GGUF Q4_K_M</center></h1>
+        f"""<h1><center>{favicon}Saiga Nemo 12B GPTQ 8 bit</center></h1>
 
         This is a demo of a **Russian**-speaking Mistral Nemo based model.
 
-        Это демонстрационная версия [квантованной Сайги Немо с 12 миллиардами параметров](https://huggingface.co/IlyaGusev/saiga_nemo_12b_gguf), работающая на CPU.
+        Это демонстрационная версия [квантованной Сайги Немо с 12 миллиардами параметров](https://huggingface.co/IlyaGusev/saiga_nemo_12b).
         """
     )
     with gr.Row():
@@ -98,14 +73,6 @@ with gr.Blocks(
                     interactive=True,
                     label="Top-p",
                 )
-                top_k = gr.Slider(
-                    minimum=10,
-                    maximum=100,
-                    value=30,
-                    step=5,
-                    interactive=True,
-                    label="Top-k",
-                )
                 temp = gr.Slider(
                     minimum=0.0,
                     maximum=2.0,
@@ -143,7 +110,6 @@ with gr.Blocks(
             chatbot,
             system_prompt,
             top_p,
-            top_k,
             temp
         ],
         outputs=chatbot,
@@ -162,7 +128,6 @@ with gr.Blocks(
             chatbot,
             system_prompt,
             top_p,
-            top_k,
             temp
         ],
         outputs=chatbot,

requirements.txt

@@ -1,3 +1,2 @@
-llama-cpp-python==0.3.2
-huggingface-hub==0.19.4
-gradio
+openai==1.57.0
+gradio