Update requirements.txt (#1)

- Update requirements.txt (875b34459a00833337d20f2f0ecb3e989577c51f)
- Update app.py (529ae2345094455908687d930c423cebb8abecb5)
- Create test_app.py (66b202656aa3f28fe24d857908069ff4e94f6bc7)

app.py

@@ -1,29 +1,28 @@
 import gradio as gr
-import outlines
+from outlines import Transformers
+from outlines.types import CFG
 import transformers
 import torch
-from threading import Thread
+from sentence_transformers import SentenceTransformer
 
-pipe = transformers.pipeline("text-generation", "HuggingFaceTB/SmolLM-1.7B-Instruct", torch_dtype=torch.float32)
-outlines_tokenizer = outlines.models.TransformerTokenizer(pipe.tokenizer)
+pipe = transformers.pipeline("text-generation", "HuggingFaceTB/SmolLM-360M-Instruct", dtype=torch.float32)
+# Create outlines model wrapper
+outlines_model = Transformers(pipe.model, pipe.tokenizer)
+minilm = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+write_a_list, write_a_story = minilm.encode(["write a list", "write a story"])
 
-def string_to_acrostic_grammar(s, dash_initial=True):
+def string_to_acrostic_grammar(s, bullet_initial=True):
     # this will convert a string to a CFG grammar
     chars = filter(str.isalpha, s.upper())
-    grammar_rules = [('"- " ' if dash_initial else '') + f'"{char}" /[^-\\r\\n]+/ "\\n"' for char in chars]
+    grammar_rules = [('"* " ' if bullet_initial else '') + f'"{char}" /[^-\\r\\n]+/ "\\n"' for char in chars]
     return "?start: " + " ".join(grammar_rules)
 
 def is_this_prompt_a_list(prompt):
-    # this will check if the prompt is a list
-    # ask the model if the prompt is a list, by constraining the generation to yes or no about a question whether the prompt is a list
-    question = f'You are trying to understand the desired format of output for a prompt, whether it will be a list or a story. The prompt:\n```{prompt}```\n\nIs this prompt asking for short phrases in a list, or long sentences in a story?'
-    grammar = '?start: ("list" | "story")'
-    cfg_logits_processor = outlines.processors.CFGLogitsProcessor(grammar, outlines_tokenizer)
-    output = pipe([{"role": "user", "content": question}, {"role": "assistant", "content": "The output to this prompt is a "}], logits_processor=transformers.LogitsProcessorList([cfg_logits_processor]), max_new_tokens=10,)
-    response = output[0]['generated_text'][-1]['content'].split()[-1]
-    # the last word is the answer
-    print("is this prompt a list?", response)
-    return response == "list"
+    # embed the prompt in minilm and find the euclidean distance to write_a_list and write_a_story and see which is closer
+    prompt_embedding = minilm.encode([prompt])[0]
+    distance_to_list = ((prompt_embedding - write_a_list) ** 2).sum()
+    distance_to_story = ((prompt_embedding - write_a_story) ** 2).sum()
+    return distance_to_list < distance_to_story
 
 def respond(
     message,
@@ -35,30 +34,27 @@ def respond(
     top_p,
 ):
     print({"message": message, "history": history, "system_message": system_message, "acrostic": acrostic, "max_tokens": max_tokens, "temperature": temperature, "top_p": top_p})
-    # this will generate a response to the message
-    prompt = f"<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{message}<|im_end|>\n<|im_start|>assistant\n"
-    grammar = string_to_acrostic_grammar(acrostic, dash_initial=is_this_prompt_a_list(prompt))
-    acrostic_logits_processor = outlines.processors.CFGLogitsProcessor(grammar, outlines_tokenizer)
-    streamer = transformers.TextIteratorStreamer(pipe.tokenizer, skip_prompt=True, decode_kwargs={"skip_special_tokens": True})
+
+    # Build the prompt with conversation history
     current_inputs = []
-    # take the current inputs, and for every item in the history (which is a list of [x,y], add it to the current inputs like so: {"role": "user", "content": x), {"role": "assistant", "content": y}
     for x, y in history:
         current_inputs.append({"role": "user", "content": x})
         current_inputs.append({"role": "assistant", "content": y})
-    # add the current inputs to the inputs
-    inputs = current_inputs + [{"role": "user", "content": prompt}]
+    current_inputs.append({"role": "user", "content": message})
+
+    # Apply chat template
+    prompt = pipe.tokenizer.apply_chat_template(current_inputs, tokenize=False, add_generation_prompt=True)
+
+    # Create CFG grammar for acrostic format
+    grammar_str = string_to_acrostic_grammar(acrostic, bullet_initial=is_this_prompt_a_list(message))
+    cfg_type = CFG(grammar_str)
 
-    generation_kwargs = dict(text_inputs=inputs, logits_processor=transformers.LogitsProcessorList([acrostic_logits_processor]), streamer=streamer, max_new_tokens=max_tokens, temperature=temperature, top_p=top_p, do_sample=True)
-    thread = Thread(target=pipe, kwargs=generation_kwargs)
-    thread.start()
-    # this will generate a response to the message
-    # TODO: figure out why skip special tokens doesn't skip special tokens
-    special_tokens = set([str(v) for v in pipe.tokenizer.special_tokens_map.values()])
-    response = ""
-    for new_text in streamer:
-        if new_text not in special_tokens:
-            response += new_text
-        yield response
+    # Use outlines model to generate with CFG constraints
+    # Note: streaming is not yet supported for Transformers models with CFG
+    # so we generate the full response and yield it
+    # Call the model directly (not .generate()) to use CFG
+    response = outlines_model(prompt, cfg_type, max_new_tokens=max_tokens, temperature=temperature, top_p=top_p)
+    yield response
 
 
 """

requirements.txt

@@ -2,8 +2,9 @@ huggingface_hub
 transformers
 accelerate
 optimum-quanto
-outlines==0.1.14
+outlines
+llguidance
 sentencepiece
 datasets
-# --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121
-# llama-cpp-python==0.3.4
+sentence-transformers
+gradio

test_app.py

@@ -0,0 +1,43 @@
+import pytest
+from app import is_this_prompt_a_list, respond
+
+
+def test_is_this_prompt_a_list_with_list_request():
+    """Test that 'tell me ten css class names' is correctly identified as a list"""
+    result = is_this_prompt_a_list("tell me ten css class names")
+    assert result == True, "Expected 'tell me ten css class names' to be identified as a list"
+
+
+def test_is_this_prompt_a_list_with_story_request():
+    """Test that 'tell me a tall tale' is correctly identified as NOT a list"""
+    result = is_this_prompt_a_list("tell me a tall tale")
+    assert result == False, "Expected 'tell me a tall tale' to be identified as a story (not a list)"
+
+
+def test_respond_generates_acrostic_with_dash():
+    """Test that respond generates output starting with '- G' for acrostic 'good dog'"""
+    # Get the generator and consume it to get the final response
+    response_gen = respond(
+        "give me seven great names for a dog",
+        [],
+        "You are a friendly Chatbot.",
+        "good dog",
+        512,
+        0.2,
+        0.95
+    )
+
+    # Get the final response by consuming the generator
+    final_response = None
+    for response in response_gen:
+        final_response = response
+
+    # Check that we got some output
+    assert final_response is not None, "Expected some response output"
+    assert len(final_response) > 0, "Expected non-empty response"
+
+    # Check that the last non-empty line starts with "* G" (acrostic format with dash)
+    lines = [line for line in final_response.split("\n") if line.strip()]
+    assert len(lines) > 0, "Expected at least one non-empty line"
+    last_line = lines[-1]
+    assert last_line[0:3] == "* G", f"Expected last line to start with '- G', but got: '{last_line[0:3]}'"