Initial commit

.gitignore

@@ -0,0 +1,5 @@
+venv
+**/__pycache__
+.env
+flagged_data_points
+json_dataset

README.md

@@ -0,0 +1,11 @@
+---
+title: Flor 6.3B Inference
+emoji: 🌸
+colorFrom: green
+colorTo: blue
+sdk: gradio
+sdk_version: 4.8.0
+app_file: app.py
+pinned: false
+license: apache-2.0
+---

app.py

@@ -0,0 +1,198 @@
+import os
+from dotenv import load_dotenv
+import gradio as gr
+from gradio.components import Textbox, Button, Slider, Checkbox
+from AinaTheme import AinaGradioTheme
+from sagemaker_endpoint import invoke_endpoint
+
+load_dotenv()
+
+MAX_NEW_TOKENS = int(os.environ.get("MAX_NEW_TOKENS", default=100))
+MAX_INPUT_CHARACTERS= int(os.environ.get("MAX_INPUT_CHARACTERS", default=100))
+SHOW_MODEL_PARAMETERS_IN_UI = os.environ.get("SHOW_MODEL_PARAMETERS_IN_UI", default=True) == "True"
+
+
+def submit_input(input_, max_new_tokens, repetition_penalty, top_k, top_p, do_sample, num_beams, temperature):
+    if input_.strip() == "":  
+        gr.Warning('Not possible to inference an empty input')
+        return None
+    
+    model_parameters = {
+        "max_new_tokens": max_new_tokens,
+        "repetition_penalty": repetition_penalty,
+        "top_k": top_k,
+        "top_p": top_p,
+        "do_sample": do_sample,
+        "num_beams": num_beams,
+        "temperature": temperature
+    }
+     
+    output = invoke_endpoint(input_, model_parameters=model_parameters)
+
+   
+    if output is  None:
+        gr.Warning('Inference endpoint is not available right now. Please try again later.')
+    return output 
+    
+def change_interactive(text):
+    if len(text.strip()) > MAX_INPUT_CHARACTERS:
+        return gr.update(interactive = True),  gr.update(interactive = False)
+    return gr.update(interactive = True),  gr.update(interactive = True)
+    
+def clear(): 
+    return (
+        None, 
+        None,
+        gr.Slider.update(value=100),
+        gr.Slider.update(value=1.2),
+        gr.Slider.update(value=50),
+        gr.Slider.update(value=0.95),
+        gr.Checkbox.update(value=True),
+        gr.Slider.update(value=4),
+        gr.Slider.update(value=0.5),
+    )
+    
+def gradio_app():
+    with gr.Blocks(**AinaGradioTheme().get_kwargs()) as demo:
+        with gr.Row():
+            with gr.Column(scale=0.1):
+                gr.Image("ginesta_small.jpg", elem_id="flor-banner", scale=1, height=256, width=256, show_label=False, show_download_button = False, show_share_button = False)
+            with gr.Column():
+                gr.Markdown(
+                    """# Flor-6.3B (experimental)
+                   
+                    🪷 **[Flor](https://huggingface.co/projecte-aina/FLOR-6.3B)** is a 6.3B parameters multilingual LLM that has been trained on a massive mixture of Spanish, Catalan and English data. It is a new open-source Large Language Model (LLM), licensed for both research and commercial use. It uses the [Bloom-7b](https://huggingface.co/bigscience/bloom-7b1) model as a starting point, a state-of-the-art multilingual language model.
+
+                    ⚠️ **Limitations**: This version is for beta testing only. The content generated by these models is unsupervised and might be judged as inappropriate or offensive. Please bear this in mind when exploring this resource.
+
+                    👀 **Learn more about Flor:** [HF official model card](https://huggingface.co/projecte-aina/FLOR-6.3B) and the [Instruct version](https://huggingface.co/projecte-aina/FLOR_63B_Instruit).
+                 
+                    """
+                )
+        with gr.Row( equal_height=False):
+            with gr.Column(variant="panel"):
+                placeholder_max_token = Textbox(
+                    visible=False,
+                    interactive=False,
+                    value= MAX_INPUT_CHARACTERS
+                )
+                input_ = Textbox(
+                    lines=11,
+                    label="Input",
+                    placeholder="e.g. El mercat del barri és fantàstic hi pots trobar."
+                )
+                with gr.Row(variant="panel", equal_height=True):
+                    gr.HTML("""<span id="countertext" style="display: flex; justify-content: start; color:#ef4444; font-weight: bold;"></span>""")
+                    gr.HTML(f"""<span id="counter" style="display: flex; justify-content: end;"> <span id="inputlenght">0</span>&nbsp;/&nbsp;{MAX_INPUT_CHARACTERS}</span>""")
+
+                with gr.Row(variant="panel"):
+                    with gr.Accordion("Model parameters", open=False, visible=SHOW_MODEL_PARAMETERS_IN_UI):
+                        max_new_tokens = Slider(
+                            minimum=1,
+                            maximum=200,
+                            step=1,
+                            value=MAX_NEW_TOKENS,
+                            label="Max tokens"
+                        )
+                        repetition_penalty = Slider(
+                            minimum=0.1,
+                            maximum=10,
+                            step=0.1,
+                            value=1.2,
+                            label="Repetition penalty"
+                        )
+                        top_k = Slider(
+                            minimum=1,
+                            maximum=100,
+                            step=1,
+                            value=50,
+                            label="Top k"
+                        )
+                        top_p = Slider(
+                            minimum=0.01,
+                            maximum=0.99,
+                            value=0.95,
+                            label="Top p"
+                        )  
+                        do_sample = Checkbox(
+                            value=True, 
+                            label="Do sample"
+                        )
+                        num_beams = Slider(
+                            minimum=1, 
+                            maximum=8, 
+                            step=1,
+                            value=4,
+                            label="Beams"
+                        )
+                        temperature = Slider(
+                            minimum=0, 
+                            maximum=1,
+                            value=0.5,
+                            label="Temperature"
+                        )
+            with gr.Column(variant="panel"):
+                output = Textbox(
+                    lines=11,
+                    label="Output",
+                    interactive=False,
+                    show_copy_button=True
+                )
+                with gr.Row(variant="panel"):
+                    clear_btn = Button(
+                        "Clear", 
+                    )
+                    submit_btn = Button(
+                        "Submit", 
+                        variant="primary",
+                    )
+
+        with gr.Row():
+            with gr.Column(scale=0.5):
+                gr.Examples(
+                    label="Short prompts:",
+                    examples=[
+                        ["""La capital de Suècia"""],
+                    ],
+                    inputs=[input_, max_new_tokens, repetition_penalty, top_k, top_p, do_sample, num_beams, temperature],
+                    outputs=output,
+                    fn=submit_input,
+                )
+
+                gr.Examples(
+                    label="Zero-shot prompts",
+                    examples=[
+                        ["Tradueix del Castellà al Català la següent frase: \"Eso es pan comido.\" \nTraducció:"],
+                    ],
+                    inputs=[input_, max_new_tokens, repetition_penalty, top_k, top_p, do_sample, num_beams, temperature],
+                    outputs=output,
+                    fn=submit_input,
+                )
+                gr.Examples(
+                    label="Few-Shot prompts:",
+                    examples=[
+                        ["""Oració: Els sons melòdics produeixen una sensació de calma i benestar en l'individu. \nParàfrasi: La música és molt relaxant i reconfortant.\n----\nOració: L'animal domèstic mostra una gran alegria i satisfacció. \nParàfrasi: El gos és molt feliç. \n----\nOració: El vehicle es va trencar i vaig haver de contactar amb el servei de remolc perquè el transportés. \nParàfrasi: El cotxe es va trencar i vaig haver de trucar la grua. \n----\nOració: El professor va explicar els conceptes de manera clara i concisa. \nParàfrasi:"""],
+                    ],
+                    inputs=[input_, max_new_tokens, repetition_penalty, top_k, top_p, do_sample, num_beams, temperature],
+                    outputs=output,
+                    fn=submit_input,
+                )
+
+            
+           
+        input_.change(fn=change_interactive, inputs=[input_], outputs=[clear_btn, submit_btn])
+      
+        input_.change(fn=None, inputs=[input_], js=f"""(i) => document.getElementById('countertext').textContent =  i.length > {MAX_INPUT_CHARACTERS} && 'Max length {MAX_INPUT_CHARACTERS} characters. ' || '' """)
+
+        input_.change(fn=None, inputs=[input_, placeholder_max_token], js="""(i, m) => {
+            document.getElementById('inputlenght').textContent = i.length + '  '
+            document.getElementById('inputlenght').style.color =  (i.length > m) ? "#ef4444" : "";
+        }""")
+    
+        clear_btn.click(fn=clear, inputs=[], outputs=[input_, output, max_new_tokens, repetition_penalty, top_k, top_p, do_sample, num_beams, temperature], queue=False)
+        submit_btn.click(fn=submit_input, inputs=[input_, max_new_tokens, repetition_penalty, top_k, top_p, do_sample, num_beams, temperature], outputs=[output])
+
+        demo.launch(show_api=True)
+
+if __name__ == "__main__":
+    gradio_app()

handler.py

@@ -0,0 +1,14 @@
+import json
+
+class ContentHandler():
+    content_type = "application/json"
+    accepts = "application/json"
+
+    def transform_input(self, prompt: str, model_kwargs: dict) -> bytes:
+        input_str = json.dumps({'inputs': prompt, 'parameters': model_kwargs})
+        return input_str.encode('utf-8')
+      
+    def transform_output(self, output: bytes) -> str:
+        response_json = json.loads(output.read().decode("utf-8"))        
+        return response_json[0]["generated_text"]
+    

requirements.txt

@@ -0,0 +1,4 @@
+aina-gradio-theme==1.0
+gradio==4.8.0
+python-dotenv==1.0.0
+boto3==1.28.20

sagemaker_endpoint.py

@@ -0,0 +1,35 @@
+import os
+from traceback import print_exc
+import boto3
+from handler import ContentHandler
+from dotenv import load_dotenv
+
+load_dotenv()
+
+endpoint_name = os.environ.get("AWS_ENDPOINT_NAME")
+aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID")
+aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY")
+aws_region_name = os.environ.get("AWS_REGION_NAME")
+
+boto_client = boto3.client(
+    service_name='sagemaker-runtime',
+    aws_access_key_id=aws_access_key_id,
+    aws_secret_access_key=aws_secret_access_key,
+    region_name=aws_region_name)
+
+content_handler = ContentHandler()
+
+def invoke_endpoint(
+        input_, 
+        model_parameters,
+    ):
+    try:
+        response = boto_client.invoke_endpoint(
+                            EndpointName=endpoint_name,
+                            ContentType='application/json', 
+                            Body=content_handler.transform_input(prompt=input_, model_kwargs=model_parameters)
+                    )
+        return content_handler.transform_output(response['Body'])
+    except:
+        print_exc()
+        return None