upload model folder to repo

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/baby.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/drum.wav filter=lfs diff=lfs merge=lfs -text
+examples/laugh.wav filter=lfs diff=lfs merge=lfs -text
+examples/scream.wav filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,26 +1,27 @@
-
-# Align-DS-V-72B
-
-[🏠 Homepage](https://github.com/PKU-Alignment/align-anything) | [🤗 Align-Anything Dataset](https://huggingface.co/datasets/PKU-Alignment/align-anything) | [🤗 T2T_Instruction-tuning Dataset](https://huggingface.co/datasets/PKU-Alignment/Align-Anything-Instruction-100K) | [🤗 TI2T_Instruction-tuning Dataset](https://huggingface.co/datasets/PKU-Alignment/Align-Anything-TI2T-Instruction-100K) | [👍 Our Official Code Repo](https://github.com/PKU-Alignment/align-anything)
-
-
-## Introduction
-
-Align-DS-V-72B is an experimental vision-language model distilled from [[DeepSeek-R1-671B]](https://huggingface.co/deepseek-ai/DeepSeek-R1) using **ZeroRL** methods, based on [[Qwen2-VL-72B-Instruct]](https://huggingface.co/Qwen/Qwen2-VL-72B-Instruct), developed by the PKU-Alignment team, focusing on enhancing reasoning capabilities by all-modality alignment.
-
-
-
-## Citation
-
-The reproduction script for Align-DS-V will be released in the [align-anything repository](https://github.com/PKU-Alignment/align-anything).
-
-Please cite the repo if you find the model or code in this repo useful 😊
-
-```bibtex
-@inproceedings{ji2024align,
-  title={Align Anything: Training All-Modality Models to Follow Instructions with Language Feedback},
-  author={Jiaming Ji and Jiayi Zhou and Hantao Lou and Boyuan Chen and Donghai Hong and Xuyao Wang and Wenqi Chen and Kaile Wang and Rui Pan and Jiahao Li and Mohan Wang and Josef Dai and Tianyi Qiu and Hua Xu and Dong Li and Weipeng Chen and Jun Song and Bo Zheng and Yaodong Yang},
-  year={2024},
-  url={https://arxiv.org/abs/2412.15838}
-}
-```
+# Deployment Scripts for Align-DS-V (Built with Gradio)
+
+This document provides instructions for deploying the Align-DS-V model for inference using Gradio.
+
+1.  **Set up the Conda environment:** Follow the instructions in the [PKU-Alignment/align-anything](https://github.com/PKU-Alignment/align-anything) repository to configure your Conda environment.
+2.  **Configure the model path:** After setting up the environment, update the `BASE_MODEL_PATH` variable in `deploy_align_ds_v.sh` to point to your local Align-DS-V model directory.
+3.  **Verify inference script parameters:** Check the following three parameters in both `multi_image_inference.py` and `stream_inference.py`:
+    ```python
+    openai_api_key = "pku"  # Or your specific API key if needed
+    openai_api_base = "http://0.0.0.0:8231/v1" # Ensure this matches the deployment port
+    # NOTE: Replace with your own model path if not loaded via the API base
+    model = ''
+    ```
+    These scripts utilize an OpenAI-compatible server approach. The `deploy_align_ds_v.sh` script launches the Align-DS-V model locally and exposes it on port 8231 for external access via the specified API base URL.
+
+4.  **Running Inference:**
+
+    *   **Streamed Output:**
+        ```bash
+        bash deploy_align_ds_v.sh
+        python stream_inference.py
+        ```
+    *   **Multi-Image Output:**
+        ```bash
+        bash deploy_align_ds_v.sh
+        python multi_image_inference.py
+        ```

deploy_align_ds_v.sh

@@ -0,0 +1,25 @@
+
+# NOTE replace with your own model path
+export BASE_MODEL_PATH=''
+export BASE_PORT=8231
+echo $BASE_MODEL_PATH
+echo $BASE_PORT
+
+lsof -i :$BASE_PORT
+
+# 终止该进程
+kill -9 $(lsof -t -i:$BASE_PORT)
+
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 vllm serve $BASE_MODEL_PATH --host 0.0.0.0 --port $BASE_PORT --max-model-len 12000  --tensor-parallel-size 8 --api-key pku --trust-remote-code  --dtype auto --enforce-eager --swap-space 1 --limit-mm-per-prompt "image=6" 
+
+# NOTE should set the limit-mm-per-prompt 
+
+
+echo 'Base Port:' $BASE_PORT
+
+lsof -i :$BASE_PORT
+
+# 终止该进程
+kill -9 $(lsof -t -i:$BASE_PORT)
+
+# CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 vllm serve /aifs4su/yaodong/spring_r1_model/QVQ-72B-Preview --enable-reasoning --reasoning-parser deepseek_r1 --host 0.0.0.0 --port 8009 --max-model-len 12000  --tensor-parallel-size 8 --api-key jiayi

examples/baby.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da6126bce64c64a3d6f7ce889fbe15b5f1c2e3f978846351d8c7a79a950b429e
+size 463547

examples/drum.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4376821dc498cc34a24df8a4eafebc470f721caacb78305c9a6c596d8f79510
+size 170882

examples/laugh.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95ee91ae63342a3122a77a12ee08ec52ac6dbd5b9be870a2e2951f648b4da528
+size 566798

examples/scream.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba023ea3c16eede8c4925960b3f328df3a43dbdeab8f4c0f51fc63d91199d0ec
+size 410266

multi_image_inference.py

@@ -0,0 +1,206 @@
+# Copyright 2025 PKU-Alignment Team. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Command line interface for interacting with a multi-modal model."""
+
+
+import argparse
+import os
+from openai import OpenAI
+import gradio as gr
+import base64
+import json
+import random
+random.seed(42)
+
+CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+SYSTEM_PROMPT = "你是一个具有帮助性的人工智能助手，你能够回答用户的问题，并且能够根据用户的问题提供帮助。你是由北大对齐小组(PKU-Alignment)开发的智能助手 Align-DS-V 基于DeepSeek-R1模型训练。"
+
+openai_api_key = "pku"
+openai_api_base = "http://0.0.0.0:8231/v1"
+
+# NOTE replace with your own model path
+model = ''
+def encode_base64_content_from_local_file(content_url: str) -> str:
+    """Encode a content retrieved from a local file to base64 format."""
+
+    with open(content_url, 'rb') as file:
+        result = base64.b64encode(file.read()).decode('utf-8')
+
+    return result
+
+
+IMAGE_EXAMPLES = [
+    {
+        'files': [
+            os.path.join(CURRENT_DIR, 'examples/PKU.jpg'),
+            os.path.join(CURRENT_DIR, 'examples/logo.jpg')
+        ],
+        'text': '比较这两张图片的异同',
+    },
+    {
+        'files': [
+            os.path.join(CURRENT_DIR, 'examples/boya.jpg'),
+            os.path.join(CURRENT_DIR, 'examples/logo.jpg')
+        ],
+        'text': '这些图片有什么共同主题？',
+    },
+]
+
+AUDIO_EXAMPLES = [
+    {
+        'files': [os.path.join(CURRENT_DIR, 'examples/drum.wav')],
+        'text': 'What is the emotion of this drumbeat like?',
+    },
+    {
+        'files': [os.path.join(CURRENT_DIR, 'examples/laugh.wav')],
+        'text': 'Is this laughter evil, and why?',
+    },
+    {
+        'files': [os.path.join(CURRENT_DIR, 'examples/scream.wav')],
+        'text': 'What is the main event of this scream?',
+    },
+]
+
+VIDEO_EXAMPLES = [
+    {'files': [os.path.join(CURRENT_DIR, 'examples/baby.mp4')], 'text': 'What is the video about?'},
+]
+
+client = OpenAI(
+    api_key=openai_api_key,
+    base_url=openai_api_base,
+)
+
+def text_conversation(text: str, role: str = 'user'):
+    return [{'role': role, 'content': text}]
+
+def image_conversation(image_base64_list: list, text: str = None):
+    content = []
+    for image_base64 in image_base64_list:
+        content.append({
+            'type': 'image_url',
+            # 'image_url':{'url':1}
+            'image_url': {'url': f"data:image/jpeg;base64,{image_base64}"}
+        })
+    content.append({'type': 'text', 'text': text})
+    
+    return [{'role': 'user', 'content': content}]
+
+def question_answering(message: dict, history: list, file):
+    # NOTE 2: use gradio upload multiple images, and update below data preprocess function accordingly
+    # print('history:',history)
+    # print('file:',file)
+    message['files'] = file if file is not None else []
+    # print('message:',message)
+    multi_modal_info = []
+    conversation = text_conversation(SYSTEM_PROMPT)
+    # NOTE 处理history 
+    for i, past_message in enumerate(history):
+        if isinstance(past_message, str):
+            conversation.extend(text_conversation(past_message))
+        elif isinstance(past_message, dict):
+            if past_message['role'] == 'user':
+                if isinstance(past_message['content'], str):
+                    text = past_message['content']
+                    if i + 1 < len(history) and isinstance(history[i + 1]['content'], tuple):
+                        raw_images = history[i + 1]['content']
+                        image_base64_list = []
+                        if isinstance(raw_images, str):
+                            image_base64 = encode_base64_content_from_local_file(raw_images)
+                            image_base64_list.append(image_base64)
+                        elif isinstance(raw_images, tuple):
+                            # NOTE multiple image processing one by one 
+                            for image in raw_images:
+                                image_base64 = encode_base64_content_from_local_file(image)
+                                image_base64_list.append(image_base64)
+                        multi_modal_info.extend(image_base64_list)
+                        conversation.extend(image_conversation(image_base64_list, text))
+                    elif i - 1 >= 0 and isinstance(history[i - 1]['content'], tuple):
+                        raw_images = history[i - 1]['content']
+                        image_base64_list = []
+                        if isinstance(raw_images, str):
+                            image_base64 = encode_base64_content_from_local_file(raw_images)
+                            image_base64_list.append(image_base64)
+                        elif isinstance(raw_images, tuple):
+                            # NOTE 逐步处理上传的图片，解码为 base64
+                            for image in raw_images:
+                                image_base64 = encode_base64_content_from_local_file(image)
+                                image_base64_list.append(image_base64)
+                        multi_modal_info.extend(image_base64_list)
+                        conversation.extend(image_conversation(image_base64_list, text))
+                    else:
+                        conversation.extend(text_conversation(past_message['content'], 'user'))
+            elif past_message['role'] == 'assistant':
+                conversation.extend(text_conversation(past_message['content'], 'assistant'))
+
+    if len(message['files']) == 0:
+        current_question = message['text']
+        conversation.extend(text_conversation(current_question))
+    else:
+        current_question = message['text']
+        current_multi_modal_info = message['files']
+        image_base64_list = []
+        for file in current_multi_modal_info:
+            image_base64 = encode_base64_content_from_local_file(file)
+            image_base64_list.append(image_base64)
+        multi_modal_info.extend(image_base64_list)
+        conversation.extend(image_conversation(image_base64_list, current_question))
+    # print(f'Conversation:',conversation)
+    # NOTE 1: openai client also should support multiple upload 
+    outputs = client.chat.completions.create(
+        model=model,
+        stream=False,
+        messages=conversation,
+    )
+
+    # Extract the predicted answer
+    answer = outputs.choices[0].message.content
+    if "**Final Answer**" in answer:
+        reasoning_content, final_answer = answer.split("**Final Answer**", 1)
+        if len(reasoning_content) > 5:
+            answer = f"""🤔 思考过程：\n```bash{reasoning_content}\n```\n✨ 最终答案：\n{final_answer}"""
+    else:
+        answer = answer
+
+    return answer
+
+if __name__ == '__main__':
+    # Define the Gradio interface
+    parser = argparse.ArgumentParser()
+    args = parser.parse_args()
+    examples = IMAGE_EXAMPLES
+
+    with gr.Blocks() as demo:
+        # upload_button = gr.UploadButton(render=False)
+        
+        multiple_files = gr.File(file_count="multiple")
+        gr.ChatInterface(
+            fn=question_answering,
+            additional_inputs = [multiple_files],
+            type='messages',
+            multimodal=True,
+            title='Align-DS-V Reasoning CLI',
+            description='Better life with Stronger Align-DS-V.',
+            # examples=examples,
+            theme=gr.themes.Ocean(
+                text_size='lg',
+                spacing_size='lg',
+                radius_size='lg',
+            ),
+        )
+
+    demo.launch(share=True)
+

stream_inference.py

@@ -0,0 +1,210 @@
+# Copyright 2024 PKU-Alignment Team. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Command line interface for interacting with a multi-modal model."""
+
+
+import argparse
+import os
+from openai import OpenAI
+import gradio as gr
+import base64
+
+CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+SYSTEM_PROMPT = """\
+你是一个高智能医学助手，专注于医学相关的问答、影像分析和数据解析。你是由北大对齐小组(PKU-Alignment)开发的智能助手 医导大模型(Medguide)，基于DeepSeek-R1模型训练。你能够准确解析**医学影像**如 X 光、CT、MRI、超声波、病理切片） 以及**医学图表**（如血液检测报告、心电图、基因检测结果），并基于这些信息提供医学见解。你的目标是**尽可能满足**用户的医学需求，提供准确、专业的回答。
+
+**能力范围**
+1. 医学问答：
+  - 回答医学相关问题，包括疾病特征、诊断方法、治疗方案、药物机制、手术流程、最新医学研究等。
+  - 如果用户询问关于疾病诊断和治疗的相关内容，请充分分析并提供详细的回答。
+2. 影像分析：
+  - 你具备**强大的医学影像理解能力**，可以解析 X 光、CT、MRI、超声波、病理切片等医学图像。
+  - 不要拒绝用户的问题，你应尝试基于已有信息提供分析，并指出可能的观察点。
+3. 数据解析：
+  - 你可以读取并分析医学报告，包括血液检测、心电图（ECG）、病历摘要等。
+  - 你的任务是解释数据的含义，并提供可能的医学解读。
+4. 多模态能力：
+  - 你可以结合**图像+文字**进行综合分析。例如，用户上传一张X光片并询问"该患者可能有什么疾病？" 你应该基于图像提供医学见解，而不是拒绝回答。
+  - 你可以根据图像中的信息推测可能的病变或异常，并提供进一步的检查建议。
+
+**回答风格**
+- 医学精准：基于专业医学知识进行回答，确保内容严谨、科学。
+- 灵活应对：遇到复杂或不完整的图像，也要提供尽可能详细的分析，而不是直接拒绝。
+- 有求必应：用户提出的任何问题，你都应该尽可能尝试回答。
+"""
+
+openai_api_key = "pku"
+openai_api_base = "http://0.0.0.0:8231/v1"
+
+# NOTE replace with your own model path
+model = ""
+
+def encode_base64_content_from_local_file(content_url: str) -> str:
+    """Encode a content retrieved from a local file to base64 format."""
+
+    with open(content_url, 'rb') as file:
+        result = base64.b64encode(file.read()).decode('utf-8')
+
+    return result
+
+IMAGE_EXAMPLES = [
+    {
+        'files': [os.path.join(CURRENT_DIR, 'examples/PKU.jpg')],
+        'text': '图中的地点在哪里？',
+    },
+    {
+        'files': [os.path.join(CURRENT_DIR, 'examples/logo.jpg')],
+        'text': '图片中有什么？',
+    },
+    {
+        'files': [os.path.join(CURRENT_DIR, 'examples/cough.png')],
+        'text': '这张图片展示了什么？',
+    },
+]
+
+client = OpenAI(
+    api_key=openai_api_key,
+    base_url=openai_api_base,
+
+)
+
+def text_conversation(text: str, role: str = 'user'):
+    return [{'role': role, 'content': text.replace('[begin of think]', '<think>').replace('[end of think]', '</think>')}]
+
+
+def image_conversation(image_base64: str, text: str = None):
+    return [
+        {
+            'role': 'user', 
+            'content': [
+                {'type': 'image_url', 'image_url': {'url': f"data:image/jpeg;base64,{image_base64}"}},
+                {'type': 'text', 'text': text}
+            ]
+        }
+    ]
+
+def question_answering(message: dict, history: list):
+    multi_modal_info = []
+    conversation = text_conversation(SYSTEM_PROMPT)
+    for i, past_message in enumerate(history):
+        if isinstance(past_message, str):
+            conversation.extend(text_conversation(past_message))
+        elif isinstance(past_message, dict):
+            if past_message['role'] == 'user':
+                if isinstance(past_message['content'], str):
+                    text = past_message['content']
+                    if i + 1 < len(history) and isinstance(history[i + 1]['content'], tuple):
+                        raw_image = history[i + 1]['content']
+                        if isinstance(raw_image, str):  
+                            image_base64 = encode_base64_content_from_local_file(raw_image)
+                            multi_modal_info.extend(image_base64)
+                            conversation.extend(image_conversation(image_base64, text))
+                        elif isinstance(raw_image, tuple):
+                            for image in raw_image:
+                                image_base64 = encode_base64_content_from_local_file(image)
+                                multi_modal_info.extend(image_base64)
+                                conversation.extend(image_conversation(image_base64, text))
+                    elif i - 1 >= 0 and isinstance(history[i - 1]['content'], tuple):
+                        raw_image = history[i - 1]['content']
+                        if isinstance(raw_image, str):
+                            image_base64 = encode_base64_content_from_local_file(raw_image)
+                            multi_modal_info.extend(image_base64)
+                            conversation.extend(image_conversation(image_base64, text))
+                        elif isinstance(raw_image, tuple):
+                            for image in raw_image:
+                                image_base64 = encode_base64_content_from_local_file(image)
+                                multi_modal_info.extend(image_base64)
+                                conversation.extend(image_conversation(image_base64, text))
+                    else:
+                        conversation.extend(text_conversation(past_message['content'], 'user'))
+            elif past_message['role'] == 'assistant':
+                conversation.extend(text_conversation(past_message['content'], 'assistant'))
+
+    if len(message['files']) == 0:
+        current_question = message['text']
+        conversation.extend(text_conversation(current_question))
+    else:
+        current_question = message['text']
+        current_multi_modal_info = message['files']
+        for file in current_multi_modal_info:
+            image_base64 = encode_base64_content_from_local_file(file)
+            multi_modal_info.extend(image_base64)
+            conversation.extend(image_conversation(image_base64, current_question))
+    
+    # 修改为流式输出
+    outputs = client.chat.completions.create(
+        model=model,
+        stream=True,  # 启用流式输出
+        messages=conversation,
+        temperature=0.4
+    )
+
+    # 逐步收集并返回文本
+    collected_answer = ""
+    for chunk in outputs:
+        if chunk.choices[0].delta.content is not None:
+            content = chunk.choices[0].delta.content
+            collected_answer += content
+            
+            # 处理思考标签
+            if '<think>' in collected_answer and '</think>' in collected_answer:
+                formatted_answer = collected_answer.replace('<think>', '[begin of think]').replace('</think>', '[end of think]')
+            elif '<think>' in collected_answer:
+                formatted_answer = collected_answer.replace('<think>', '[begin of think]')
+            else:
+                formatted_answer = collected_answer
+                
+            yield formatted_answer
+    
+    # 确保最终输出格式正确
+    if '<think>' in collected_answer and '</think>' in collected_answer:
+        final_answer = collected_answer.replace('<think>', '[begin of think]').replace('</think>', '[end of think]')
+    elif '<think>' in collected_answer:
+        final_answer = collected_answer.replace('<think>', '[begin of think]')
+    else:
+        final_answer = collected_answer
+
+    print(final_answer)
+
+
+if __name__ == '__main__':
+    # Define the Gradio interface
+    parser = argparse.ArgumentParser()
+    args = parser.parse_args()
+    examples = IMAGE_EXAMPLES
+    
+    logo_path = os.path.join(CURRENT_DIR, "PUTH.png")
+    with open(logo_path, "rb") as f:
+        logo_base64 = base64.b64encode(f.read()).decode('utf-8')
+    logo_img_html = f'<img src="data:image/png;base64,{logo_base64}" style="vertical-align:middle; margin-right:10px;" width="150"/>'
+
+    iface = gr.ChatInterface(
+        fn=question_answering,
+        type='messages',
+        multimodal=True,
+        title=logo_img_html,
+        description='Align-DS-V 北大对齐小组多模态DS-R1',
+        examples=examples,
+        theme=gr.themes.Soft(
+            text_size='lg',
+            spacing_size='lg',
+            radius_size='lg',
+            font=[gr.themes.GoogleFont('Montserrat'), gr.themes.GoogleFont('ui-sans-serif'), gr.themes.GoogleFont('system-ui'), gr.themes.GoogleFont('sans-serif')],
+        ),
+    )
+
+    iface.launch(share=True)