speech similarity model

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.trt filter=lfs diff=lfs merge=lfs -text
+*.pt.*.partial filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,7 @@
+__pycache__/
+.venv/
+*.pyc
+.DS_Store
+outputs/
+logs/
+eval_res/

README.md

@@ -1,3 +1,137 @@
 ---
 license: apache-2.0
 ---
+
+# Speaker Similarity Evaluator - 新格式使用说明
+
+## 概述
+
+音色相似度评估器现在支持从JSONL文件读取输入数据，并且支持两种prompt音频输入模式：
+1. **预分割模式**：直接提供S1和S2的分别音频文件
+2. **自动分割模式**：提供combined prompt音频，程序自动按说话人标签分割
+
+## 输入格式
+
+### JSONL文件格式
+
+每行是一个JSON对象，必须包含以下字段：
+
+#### 必需字段
+- `text`: 待评估的文本，包含说话人标签[S1][S2]
+- `output_audio`: 待评估的音频文件路径
+
+#### prompt音频字段（两种模式择一）
+
+**模式1：预分割模式**
+- `prompt_audio_speaker1`: S1说话人的音频文件
+- `prompt_text_speaker1`: S1说话人的文本
+- `prompt_audio_speaker2`: S2说话人的音频文件  
+- `prompt_text_speaker2`: S2说话人的文本
+
+**模式2：自动分割模式**
+- `prompt_audio`: 包含两个说话人的combined音频文件
+- `prompt_text`: 包含说话人标签的文本，如"[S1]文本1[S2]文本2"
+
+### 示例
+
+#### 预分割模式示例
+```json
+{
+  "text": "[S1]是我对不住你。[S2]没有没有！燕子幸亏咱俩没领证！",
+  "prompt_audio_speaker1": "/path/to/speaker1.wav",
+  "prompt_text_speaker1": "一共二十万我都记着呢。我一赚到钱就马上还给你。",
+  "prompt_audio_speaker2": "/path/to/speaker2.wav", 
+  "prompt_text_speaker2": "没关系，我不缺钱。",
+  "output_audio": "/path/to/output.wav"
+}
+```
+
+#### 自动分割模式示例
+```json
+{
+  "text": "[S1]今天天气真好啊。[S2]是的，阳光明媚。",
+  "prompt_audio": "/path/to/combined_prompt.wav",
+  "prompt_text": "[S1]早上好，今天怎么样？[S2]很好，谢谢你的关心。",
+  "output_audio": "/path/to/output.wav"
+}
+```
+
+#### 混合模式示例（同时提供两种模式，优先使用预分割）
+```json
+{
+  "text": "[S1]是我对不住你。[S2]没有没有！",
+  "prompt_audio": "/path/to/combined.wav",
+  "prompt_text": "[S1]一共二十万我都记着呢。[S2]没关系，我不缺钱。",
+  "prompt_audio_speaker1": "/path/to/speaker1.wav",
+  "prompt_text_speaker1": "一共二十万我都记着呢。我一赚到钱就马上还给你。",
+  "prompt_audio_speaker2": "/path/to/speaker2.wav",
+  "prompt_text_speaker2": "没关系，我不缺钱。",
+  "output_audio": "/path/to/output.wav"
+}
+```
+
+## 使用方法
+
+### 命令行运行
+
+```bash
+# 使用JSONL文件输入
+python test.py --jsonl_path /path/to/your/input.jsonl --output_dir /path/to/results
+
+# 使用默认示例数据（向后兼容）
+python test.py --output_dir /path/to/results
+```
+
+### 程序调用
+
+```python
+from test import SpeakerSimilarityEvaluator
+
+# 创建评估器
+evaluator = SpeakerSimilarityEvaluator(output_dir="/path/to/results")
+
+# 从JSONL文件处理
+evaluator.process_batch_from_jsonl("/path/to/input.jsonl")
+
+# 或者直接传入数据列表（旧接口，向后兼容）
+input_data = [
+    {
+        'prompt_audio': "/path/to/prompt.wav",
+        'prompt_text': "[S1]文本1[S2]文本2", 
+        'text': "[S1]输出文本1[S2]输出文本2",
+        'output_audio': "/path/to/output.wav"
+    }
+]
+evaluator.process_batch(input_data)
+```
+
+## 优势
+
+### 预分割模式的优势
+1. **更高精度**：避免了自动分割可能带来的误差
+2. **更快速度**：跳过音频分割步骤
+3. **更稳定**：不依赖词对齐模型的准确性
+
+### 自动分割模式的优势
+1. **便利性**：只需要提供一个combined音频文件
+2. **向后兼容**：与现有数据格式兼容
+
+## 输出文件结构
+
+```
+results_YYYYMMDD_HHMMSS/
+├── segments/           # 分割后的音频片段
+├── prompts/           # prompt音频的S1和S2片段（仅自动分割模式）
+├── temp/              # 临时文件（运行结束后清空）
+└── results/           # 评估结果
+    ├── speaker_similarity_results_YYYYMMDD_HHMMSS.jsonl
+    └── evaluation_summary_YYYYMMDD_HHMMSS.json
+```
+
+## 注意事项
+
+1. 确保所有音频文件路径正确且文件存在
+2. 文本中的说话人标签格式必须为`[S1]`和`[S2]`
+3. 如果同时提供两种模式的数据，程序优先使用预分割模式
+4. JSONL文件中的每行必须是有效的JSON格式
+5. 程序会自动验证输入数据的完整性，跳过有问题的行并继续处理 

alignment.py

@@ -0,0 +1,384 @@
+import re
+import torch
+import torchaudio.functional as F
+import torchaudio
+import uroman as ur
+import logging
+import traceback
+
+def convert_to_list_with_punctuation_mixed(text):
+    """处理中文文本（可能包含英文单词） - 中文按字符分割，英文单词保持完整"""
+    result = []
+    text = text.strip()
+    
+    if not text:
+        return result
+    
+    def is_chinese(char):
+        """检查是否是汉字"""
+        return '\u4e00' <= char <= '\u9fff'
+    
+    # 使用更精确的正则表达式来分割文本
+    # 匹配：英文单词（含数字）、单个汉字、标点符号
+    pattern = r'[a-zA-Z]+[a-zA-Z0-9]*|[\u4e00-\u9fff]|[^\w\s\u4e00-\u9fff]'
+    tokens = re.findall(pattern, text)
+    
+    for token in tokens:
+        if not token.strip():  # 跳过空字符
+            continue
+        
+        if re.match(r'^[a-zA-Z]+[a-zA-Z0-9]*$', token):  # 英文单词（可能包含数字）
+            result.append(token)
+        elif is_chinese(token):  # 单个汉字
+            result.append(token)
+        else:  # 标点符号等其他字符
+            # 标点符号加到前一个词后面
+            if result:
+                result[-1] += token
+            else:
+                # 如果是文本开头的标点，单独作为一项
+                result.append(token)
+    
+    return result
+
+def split_and_merge_punctuation(text):
+    """处理英文 - 按单词分割，保持单词完整性"""
+    # 先按空格拆分文本
+    elements = text.split()
+    
+    # 用于保存最终的结果
+    result = []
+    
+    # 遍历每个拆分后的元素
+    for ele in elements:
+        # 使用正则表达式提取连续字母、数字和标点
+        parts = re.findall(r'[a-zA-Z0-9]+|[^\w\s]+', ele)
+        
+        # 用于保存拆分后的部分
+        merged_parts = []
+        
+        for i in range(len(parts)):
+            if i % 2 == 0:  # 如果是字母或数字部分
+                # 将字母或数字部分添加到结果中
+                merged_parts.append(parts[i])
+            else:  # 如果是标点或其他符号部分
+                # 将标点部分与前面的字母或数字部分合并
+                if merged_parts:
+                    merged_parts[-1] += parts[i]
+                else:
+                    merged_parts.append(parts[i])
+        
+        # 将合并后的部分加入最终结果
+        result.extend(merged_parts)
+    
+    return result
+
+
+def get_aligned_result_text_with_punctuation(alignment_result, text, language):
+    """
+    将对齐结果转换为正确的文本tokens，英文保持单词级别，中文保持字符级别（但英文单词完整）
+    """
+    logging.info("start change text to text_tokens")
+    
+    if language == "EN":
+        text_tokens = split_and_merge_punctuation(text)  # 英文按单词分词
+    elif language == "ZH":
+        text_tokens = convert_to_list_with_punctuation_mixed(text)  # 中文按字符分割，但英文单词保持完整
+    else:
+        raise ValueError(f"Unsupported language: {language}")
+
+    logging.info(f"Text tokens count: {len(text_tokens)}, Alignment result count: {len(alignment_result)}")
+    
+    punctuations = set(',.!?;:()[]<>\'\"…·，。；：！？（）【】《》''""\、')
+
+    logging.info("start get align result text with punctuation")
+    updated_alignment_result = []
+    token_idx = 0
+    
+    for index, align_item in enumerate(alignment_result):
+        if token_idx >= len(text_tokens):
+            # 如果text_tokens用完了但还有对齐结果，跳出循环
+            logging.warning(f"Text tokens exhausted at index {token_idx}, but alignment has more items")
+            break
+            
+        start = align_item["start"]
+        end = align_item["end"]
+        text_token = text_tokens[token_idx]
+        
+        # 检查该 token 后是否有连续标点（仅对中文）
+        if language == "ZH":
+            while token_idx + 1 < len(text_tokens) and text_tokens[token_idx + 1] in punctuations:
+                assert False, "???"  # 这里理论上应该进不去？？
+                text_token += text_tokens[token_idx + 1]  # 将标点加入
+                token_idx += 1
+        else:
+            # 英文不需要特殊的标点处理，因为标点已经在split_and_merge_punctuation中处理了
+            pass
+        
+        # 更新对齐结果
+        updated_item = {
+            "start": start,
+            "end": end,
+            "transcript": text_token
+        }
+        updated_item.update({key: align_item[key] for key in align_item if key not in ["start", "end", "transcript"]})
+        
+        updated_alignment_result.append(updated_item)
+        token_idx += 1
+
+    logging.info("end get align result text with punctuation")
+    return updated_alignment_result
+
+
+class AlignmentModel:
+    def __init__(self, device, model_dir='/data-mnt/data/wy/X-Codec-2.0/checkpoints'):
+        """
+        初始化对齐模型并加载必要的资源
+        :param device: 设备类型 ("cuda" 或 "cpu")
+        :param model_dir: 模型目录路径
+        """
+        self.device = torch.device(device)
+        self.bundle = torchaudio.pipelines.MMS_FA
+        self.align_model = self.bundle.get_model(with_star=False, dl_kwargs={'model_dir': model_dir}).to(self.device)
+        self.uroman = ur.Uroman()
+        self.DICTIONARY = self.bundle.get_dict()
+
+    def align(self, emission, tokens):
+        """
+        执行强对齐
+        :param emission: 模型的输出
+        :param tokens: 目标 tokens
+        :return: 对齐的 tokens 和分数
+        """
+        alignments, scores = F.forced_align(
+            log_probs=emission,
+            targets=tokens,
+            blank=0
+        )
+        alignments, scores = alignments[0], scores[0]
+        scores = scores.exp()
+        return alignments, scores
+
+    def unflatten(self, list_, lengths):
+        """
+        将一个长列表按照长度拆分成子列表
+        :param list_: 长列表
+        :param lengths: 各子列表的长度
+        :return: 拆分后的子列表
+        """
+        assert len(list_) == sum(lengths)
+        i = 0
+        ret = []
+        for l in lengths:
+            ret.append(list_[i:i + l])
+            i += l
+        return ret
+
+    def preview_word(self, waveform, spans, num_frames, transcript, sample_rate):
+        """
+        预览每个单词的开始时间和结束时间
+        :param waveform: 音频波形
+        :param spans: 单词的跨度
+        :param num_frames: 帧数
+        :param transcript: 转录文本
+        :param sample_rate: 采样率
+        :return: 单词的对齐信息
+        """
+        end = 0
+        alignment_result = []
+        for span, trans in zip(spans, transcript):
+            ratio = waveform.size(1) / num_frames
+            x0 = int(ratio * span[0].start)
+            x1 = int(ratio * span[-1].end)
+            align_info = {
+                "transcript": trans,
+                "start": round(x0 / sample_rate, 3),
+                "end": round(x1 / sample_rate, 3)
+            }
+            align_info["pause"] = round(align_info["start"] - end, 3)
+            align_info["duration"] = round(align_info["end"] - align_info["start"], 3)
+            end = align_info["end"]
+            alignment_result.append(align_info)
+        return alignment_result
+
+    def make_wav_batch(self, wav_list):
+        """
+        将 wav_list 中的每个 wav 张量填充为相同的长度，返回填充后的张量和每个张量的原始长度。
+        :param wav_list: wav 文件列表
+        :return: 填充后的音频张量和原始长度
+        """
+        wav_lengths = torch.tensor([wav.size(0) for wav in wav_list], dtype=torch.long)
+        max_length = max(wav_lengths)
+        # 确保张量在正确的设备上
+        wavs_tensors = torch.zeros(len(wav_list), max_length, device=self.device)
+        for i, wav in enumerate(wav_list):
+            wav = wav.to(self.device)  # 确保wav在正确的设备上
+            wavs_tensors[i, :wav_lengths[i]] = wav
+        return wavs_tensors, wav_lengths.to(self.device)
+
+    def get_target(self, transcript, language):
+        """
+        获取给定转录文本的目标 tokens - 修正版本，保持英文单词完整性
+        """
+        original_transcript = transcript  # 保存原始文本用于调试
+        
+        if language == "ZH":
+            # 中文处理：保持英文单词完整，只对中文字符进行romanization
+            # 使用相同的分词逻辑
+            pattern = r'[a-zA-Z]+[a-zA-Z0-9]*|[\u4e00-\u9fff]|[^\w\s\u4e00-\u9fff]'
+            tokens = re.findall(pattern, transcript)
+            
+            # 分别处理中文字符和英文单词
+            processed_parts = []
+            for token in tokens:
+                if not token.strip():
+                    continue
+                elif re.match(r'^[a-zA-Z]+[a-zA-Z0-9]*$', token):  # 英文单词
+                    # 英文单词保持原样，不进行romanization
+                    processed_parts.append(token.lower())
+                elif '\u4e00' <= token <= '\u9fff':  # 中文字符
+                    # 只对中文字符进行romanization
+                    romanized = self.uroman.romanize_string(token)
+                    processed_parts.append(romanized)
+                else:  # 标点符号等
+                    # 标点符号直接添加，但会在后续步骤中被过滤掉
+                    processed_parts.append(token)
+            
+            # 用空格连接所有部分
+            transcript = ' '.join(processed_parts)
+        
+        elif language == "EN":
+            # 英文处理：保持单词结构，只是清理标点
+            pass
+        else:
+            assert False, f"Unsupported language: {language}"
+
+        # 清理标点符号
+        transcript = re.sub(r'[^\w\s]', r' ', transcript)
+        TRANSCRIPT = transcript.lower().split()
+        
+        # 提前获取字典中的特殊符号 token
+        star_token = self.DICTIONARY['*']
+        tokenized_transcript = []
+
+        # 统一的tokenization逻辑
+        for word in TRANSCRIPT:
+            # 对每个word中的字符进行token化
+            word_tokens = []
+            for c in word:
+                if c in self.DICTIONARY and c != '-':
+                    word_tokens.append(self.DICTIONARY[c])
+                else:
+                    word_tokens.append(star_token)
+            tokenized_transcript.extend(word_tokens)
+        
+        logging.info(f"Original transcript: {original_transcript}")
+        logging.info(f"Processed transcript: {transcript}")
+        logging.info(f"Final TRANSCRIPT: {TRANSCRIPT}")
+        
+        return torch.tensor([tokenized_transcript], dtype=torch.int32, device=self.device)
+
+    def get_alignment_result(self, emission_padded, emission_length, aligned_tokens, alignment_scores, transcript, waveform, language):
+        """
+        根据给定的 emission 和对齐信息生成对齐结果 - 修正版本
+        """
+        original_transcript = transcript  # 保存原始文本
+        
+        if language == "ZH":
+            # 使用与get_target相同的处理逻辑
+            pattern = r'[a-zA-Z]+[a-zA-Z0-9]*|[\u4e00-\u9fff]|[^\w\s\u4e00-\u9fff]'
+            tokens = re.findall(pattern, transcript)
+            
+            processed_parts = []
+            for token in tokens:
+                if not token.strip():
+                    continue
+                elif re.match(r'^[a-zA-Z]+[a-zA-Z0-9]*$', token):  # 英文单词
+                    processed_parts.append(token.lower())
+                elif '\u4e00' <= token <= '\u9fff':  # 中文字符
+                    romanized = self.uroman.romanize_string(token)
+                    processed_parts.append(romanized)
+                else:  # 标点符号等
+                    processed_parts.append(token)
+            
+            transcript = ' '.join(processed_parts)
+        elif language == "EN":
+            pass
+        else:
+            assert False, f"Unsupported language: {language}"
+        
+        transcript = re.sub(r'[^\w\s]', r' ', transcript)
+        emission = emission_padded[:emission_length, :].unsqueeze(0)
+        TRANSCRIPT = transcript.lower().split()
+        
+        token_spans = F.merge_tokens(aligned_tokens, alignment_scores)
+        
+        # 统一的分组逻辑
+        word_spans = self.unflatten(token_spans, [len(word) for word in TRANSCRIPT])
+        
+        num_frames = emission.size(1)
+        
+        logging.info(f"Original transcript for alignment: {original_transcript}")
+        logging.info(f"Processed TRANSCRIPT: {TRANSCRIPT}")
+        
+        return self.preview_word(waveform.unsqueeze(0), word_spans, num_frames, TRANSCRIPT, self.bundle.sample_rate)
+
+    def batch_alignment(self, wav_list, transcript_list, language_list):
+        """
+        批量对齐
+        :param wav_list: wav 文件列表
+        :param transcript_list: 转录文本列表
+        :param language_list: 语言类型列表
+        :return: 对齐结果列表
+        """
+        wavs_tensors, wavs_lengths_tensor = self.make_wav_batch(wav_list)
+        logging.info("start alignment model forward")
+        with torch.inference_mode():
+            emission, emission_lengths = self.align_model(wavs_tensors.to(self.device), wavs_lengths_tensor)
+            star_dim = torch.zeros((emission.shape[0], emission.size(1), 1), dtype=emission.dtype, device=self.device)
+            emission = torch.cat((emission, star_dim), dim=-1)
+        
+        logging.info("end alignment model forward")
+        
+        target_list = [self.get_target(transcript, language) for transcript, language in zip(transcript_list, language_list)]
+        
+        logging.info("align success")
+        align_results = [
+            self.align(emission_padded[:emission_length, :].unsqueeze(0), target)
+            for emission_padded, emission_length, target in zip(emission, emission_lengths, target_list)
+        ]
+        
+        logging.info("get align result")
+        batch_aligned_tokens = [align_result[0] for align_result in align_results]
+        batch_alignment_scores = [align_result[1] for align_result in align_results]
+
+        alignment_result_list = [
+            self.get_alignment_result(emission_padded, emission_length, aligned_tokens, alignment_scores, transcript, waveform, language)
+            for emission_padded, emission_length, aligned_tokens, alignment_scores, transcript, waveform, language
+            in zip(emission, emission_lengths, batch_aligned_tokens, batch_alignment_scores, transcript_list, wav_list, language_list)
+        ]
+        logging.info("get align result success")
+        return alignment_result_list
+    
+
+def batch_get_alignment_result(alignment_model, wav_list, transcript_list, language_list):
+    """
+    批量获取对齐结果的便捷函数
+    """
+    alignment_results = alignment_model.batch_alignment(
+        wav_list=wav_list, 
+        transcript_list=transcript_list, 
+        language_list=language_list
+    )
+    
+    alignments_results_with_text_and_punctuation = []
+    for alignment_result, transcript, language in zip(alignment_results, transcript_list, language_list):
+        try:
+            result = get_aligned_result_text_with_punctuation(alignment_result, transcript, language)
+            alignments_results_with_text_and_punctuation.append(result)
+        except:
+            logger = logging.getLogger("tokenize")
+            logger.error(f"Error in processing {alignment_result}")
+            traceback.print_exc()
+            alignments_results_with_text_and_punctuation.append(alignment_result)
+    return alignments_results_with_text_and_punctuation

alignment_online.py

@@ -0,0 +1,398 @@
+import base64
+import httpx
+import re
+import requests
+import torch
+import torchaudio.functional as F
+import torchaudio
+import uroman as ur
+import logging
+import traceback
+
+
+def convert_to_list_with_punctuation_mixed(text):
+    """处理中文文本（可能包含英文单词） - 中文按字符分割，英文单词保持完整"""
+    result = []
+    text = text.strip()
+    
+    if not text:
+        return result
+    
+    def is_chinese(char):
+        """检查是否是汉字"""
+        return '\u4e00' <= char <= '\u9fff'
+    
+    # 使用更精确的正则表达式来分割文本
+    # 匹配：英文单词（含数字）、单个汉字、标点符号
+    pattern = r'[a-zA-Z]+[a-zA-Z0-9]*|[\u4e00-\u9fff]|[^\w\s\u4e00-\u9fff]'
+    tokens = re.findall(pattern, text)
+    
+    for token in tokens:
+        if not token.strip():  # 跳过空字符
+            continue
+        
+        if re.match(r'^[a-zA-Z]+[a-zA-Z0-9]*$', token):  # 英文单词（可能包含数字）
+            result.append(token)
+        elif is_chinese(token):  # 单个汉字
+            result.append(token)
+        else:  # 标点符号等其他字符
+            # 标点符号加到前一个词后面
+            if result:
+                result[-1] += token
+            else:
+                # 如果是文本开头的标点，单独作为一项
+                result.append(token)
+    
+    return result
+
+def split_and_merge_punctuation(text):
+    """处理英文 - 按单词分割，保持单词完整性"""
+    # 先按空格拆分文本
+    elements = text.split()
+    
+    # 用于保存最终的结果
+    result = []
+    
+    # 遍历每个拆分后的元素
+    for ele in elements:
+        # 使用正则表达式提取连续字母、数字和标点
+        parts = re.findall(r'[a-zA-Z0-9]+|[^\w\s]+', ele)
+        
+        # 用于保存拆分后的部分
+        merged_parts = []
+        
+        for i in range(len(parts)):
+            if i % 2 == 0:  # 如果是字母或数字部分
+                # 将字母或数字部分添加到结果中
+                merged_parts.append(parts[i])
+            else:  # 如果是标点或其他符号部分
+                # 将标点部分与前面的字母或数字部分合并
+                if merged_parts:
+                    merged_parts[-1] += parts[i]
+                else:
+                    merged_parts.append(parts[i])
+        
+        # 将合并后的部分加入最终结果
+        result.extend(merged_parts)
+    
+    return result
+
+
+def get_aligned_result_text_with_punctuation(alignment_result, text, language):
+    """
+    将对齐结果转换为正确的文本tokens，英文保持单词级别，中文保持字符级别（但英文单词完整）
+    """
+    logging.info("start change text to text_tokens")
+    
+    if language == "EN":
+        text_tokens = split_and_merge_punctuation(text)  # 英文按单词分词
+    elif language == "ZH":
+        text_tokens = convert_to_list_with_punctuation_mixed(text)  # 中文按字符分割，但英文单词保持完整
+    else:
+        raise ValueError(f"Unsupported language: {language}")
+
+    logging.info(f"Text tokens count: {len(text_tokens)}, Alignment result count: {len(alignment_result)}")
+    
+    punctuations = set(',.!?;:()[]<>\'\"…·，。；：！？（）【】《》''""\、')
+
+    logging.info("start get align result text with punctuation")
+    updated_alignment_result = []
+    token_idx = 0
+    
+    for index, align_item in enumerate(alignment_result):
+        if token_idx >= len(text_tokens):
+            # 如果text_tokens用完了但还有对齐结果，跳出循环
+            logging.warning(f"Text tokens exhausted at index {token_idx}, but alignment has more items")
+            break
+            
+        start = align_item["start"]
+        end = align_item["end"]
+        text_token = text_tokens[token_idx]
+        
+        # 检查该 token 后是否有连续标点（仅对中文）
+        if language == "ZH":
+            while token_idx + 1 < len(text_tokens) and text_tokens[token_idx + 1] in punctuations:
+                assert False, "???"  # 这里理论上应该进不去？？
+                text_token += text_tokens[token_idx + 1]  # 将标点加入
+                token_idx += 1
+        else:
+            # 英文不需要特殊的标点处理，因为标点已经在split_and_merge_punctuation中处理了
+            pass
+        
+        # 更新对齐结果
+        updated_item = {
+            "start": start,
+            "end": end,
+            "transcript": text_token
+        }
+        updated_item.update({key: align_item[key] for key in align_item if key not in ["start", "end", "transcript"]})
+        
+        updated_alignment_result.append(updated_item)
+        token_idx += 1
+
+    logging.info("end get align result text with punctuation")
+    return updated_alignment_result
+
+
+class AlignmentModel:
+    def __init__(self, device, model_dir='/data-mnt/data/wy/X-Codec-2.0/checkpoints'):
+        """
+        初始化对齐模型并加载必要的资源
+        """
+        self.device = torch.device(device)
+        self.bundle = torchaudio.pipelines.MMS_FA
+        model = self.bundle.get_model(with_star=False, dl_kwargs={'model_dir': model_dir}).to(self.device)
+        
+        # --- 核心优化 ---
+        # 使用 torch.compile 对模型进行 JIT 编译
+        # mode="max-autotune" 会花费更长时间编译，但能达到最佳性能
+        print("Compiling the model... This may take a moment.")
+        self.align_model = torch.compile(model, mode="reduce-overhead", fullgraph=True) 
+        print("Model compiled successfully.")
+
+        self.uroman = ur.Uroman()
+        self.DICTIONARY = self.bundle.get_dict()
+
+    def align(self, emission, tokens):
+        """
+        执行强对齐
+        :param emission: 模型的输出
+        :param tokens: 目标 tokens
+        :return: 对齐的 tokens 和分数
+        """
+        alignments, scores = F.forced_align(
+            log_probs=emission,
+            targets=tokens,
+            blank=0
+        )
+        alignments, scores = alignments[0], scores[0]
+        scores = scores.exp()
+        return alignments, scores
+
+    def unflatten(self, list_, lengths):
+        """
+        将一个长列表按照长度拆分成子列表
+        :param list_: 长列表
+        :param lengths: 各子列表的长度
+        :return: 拆分后的子列表
+        """
+        assert len(list_) == sum(lengths)
+        i = 0
+        ret = []
+        for l in lengths:
+            ret.append(list_[i:i + l])
+            i += l
+        return ret
+
+    def preview_word(self, waveform, spans, num_frames, transcript, sample_rate):
+        """
+        预览每个单词的开始时间和结束时间
+        :param waveform: 音频波形
+        :param spans: 单词的跨度
+        :param num_frames: 帧数
+        :param transcript: 转录文本
+        :param sample_rate: 采样率
+        :return: 单词的对齐信息
+        """
+        end = 0
+        alignment_result = []
+        for span, trans in zip(spans, transcript):
+            ratio = waveform.size(1) / num_frames
+            x0 = int(ratio * span[0].start)
+            x1 = int(ratio * span[-1].end)
+            align_info = {
+                "transcript": trans,
+                "start": round(x0 / sample_rate, 3),
+                "end": round(x1 / sample_rate, 3)
+            }
+            align_info["pause"] = round(align_info["start"] - end, 3)
+            align_info["duration"] = round(align_info["end"] - align_info["start"], 3)
+            end = align_info["end"]
+            alignment_result.append(align_info)
+        return alignment_result
+
+    def make_wav_batch(self, wav_list):
+        """
+        将 wav_list 中的每个 wav 张量填充为相同的长度，返回填充后的张量和每个张量的原始长度。
+        :param wav_list: wav 文件列表
+        :return: 填充后的音频张量和原始长度
+        """
+        wav_lengths = torch.tensor([wav.size(0) for wav in wav_list], dtype=torch.long)
+        max_length = max(wav_lengths)
+        # 确保张量在正确的设备上
+        wavs_tensors = torch.zeros(len(wav_list), max_length, device=self.device)
+        for i, wav in enumerate(wav_list):
+            wav = wav.to(self.device)  # 确保wav在正确的设备上
+            wavs_tensors[i, :wav_lengths[i]] = wav
+        return wavs_tensors, wav_lengths.to(self.device)
+
+    def get_target(self, transcript, language):
+        """
+        获取给定转录文本的目标 tokens - 修正版本，保持英文单词完整性
+        """
+        original_transcript = transcript  # 保存原始文本用于调试
+        
+        if language == "ZH":
+            # 中文处理：保持英文单词完整，只对中文字符进行romanization
+            # 使用相同的分词逻辑
+            pattern = r'[a-zA-Z]+[a-zA-Z0-9]*|[\u4e00-\u9fff]|[^\w\s\u4e00-\u9fff]'
+            tokens = re.findall(pattern, transcript)
+            
+            # 分别处理中文字符和英文单词
+            processed_parts = []
+            for token in tokens:
+                if not token.strip():
+                    continue
+                elif re.match(r'^[a-zA-Z]+[a-zA-Z0-9]*$', token):  # 英文单词
+                    # 英文单词保持原样，不进行romanization
+                    processed_parts.append(token.lower())
+                elif '\u4e00' <= token <= '\u9fff':  # 中文字符
+                    # 只对中文字符进行romanization
+                    romanized = self.uroman.romanize_string(token)
+                    processed_parts.append(romanized)
+                else:  # 标点符号等
+                    # 标点符号直接添加，但会在后续步骤中被过滤掉
+                    processed_parts.append(token)
+            
+            # 用空格连接所有部分
+            transcript = ' '.join(processed_parts)
+        
+        elif language == "EN":
+            # 英文处理：保持单词结构，只是清理标点
+            pass
+        else:
+            assert False, f"Unsupported language: {language}"
+
+        # 清理标点符号
+        transcript = re.sub(r'[^\w\s]', r' ', transcript)
+        TRANSCRIPT = transcript.lower().split()
+        
+        # 提前获取字典中的特殊符号 token
+        star_token = self.DICTIONARY['*']
+        tokenized_transcript = []
+
+        # 统一的tokenization逻辑
+        for word in TRANSCRIPT:
+            # 对每个word中的字符进行token化
+            word_tokens = []
+            for c in word:
+                if c in self.DICTIONARY and c != '-':
+                    word_tokens.append(self.DICTIONARY[c])
+                else:
+                    word_tokens.append(star_token)
+            tokenized_transcript.extend(word_tokens)
+        
+        logging.info(f"Original transcript: {original_transcript}")
+        logging.info(f"Processed transcript: {transcript}")
+        logging.info(f"Final TRANSCRIPT: {TRANSCRIPT}")
+        
+        return torch.tensor([tokenized_transcript], dtype=torch.int32, device=self.device)
+
+    def get_alignment_result(self, emission_padded, emission_length, aligned_tokens, alignment_scores, transcript, waveform, language):
+        """
+        根据给定的 emission 和对齐信息生成对齐结果 - 修正版本
+        """
+        original_transcript = transcript  # 保存原始文本
+        
+        if language == "ZH":
+            # 使用与get_target相同的处理逻辑
+            pattern = r'[a-zA-Z]+[a-zA-Z0-9]*|[\u4e00-\u9fff]|[^\w\s\u4e00-\u9fff]'
+            tokens = re.findall(pattern, transcript)
+            
+            processed_parts = []
+            for token in tokens:
+                if not token.strip():
+                    continue
+                elif re.match(r'^[a-zA-Z]+[a-zA-Z0-9]*$', token):  # 英文单词
+                    processed_parts.append(token.lower())
+                elif '\u4e00' <= token <= '\u9fff':  # 中文字符
+                    romanized = self.uroman.romanize_string(token)
+                    processed_parts.append(romanized)
+                else:  # 标点符号等
+                    processed_parts.append(token)
+            
+            transcript = ' '.join(processed_parts)
+        elif language == "EN":
+            pass
+        else:
+            assert False, f"Unsupported language: {language}"
+        
+        transcript = re.sub(r'[^\w\s]', r' ', transcript)
+        emission = emission_padded[:emission_length, :].unsqueeze(0)
+        TRANSCRIPT = transcript.lower().split()
+        
+        token_spans = F.merge_tokens(aligned_tokens, alignment_scores)
+        
+        # 统一的分组逻辑
+        word_spans = self.unflatten(token_spans, [len(word) for word in TRANSCRIPT])
+        
+        num_frames = emission.size(1)
+        
+        logging.info(f"Original transcript for alignment: {original_transcript}")
+        logging.info(f"Processed TRANSCRIPT: {TRANSCRIPT}")
+        
+        return self.preview_word(waveform.unsqueeze(0), word_spans, num_frames, TRANSCRIPT, self.bundle.sample_rate)
+
+    def batch_alignment(self, wav_list, transcript_list, language_list):
+        """
+        批量对齐
+        :param wav_list: wav 文件列表
+        :param transcript_list: 转录文本列表
+        :param language_list: 语言类型列表
+        :return: 对齐结果列表
+        """
+        wavs_tensors, wavs_lengths_tensor = self.make_wav_batch(wav_list)
+        logging.info("start alignment model forward")
+        with torch.inference_mode():
+            emission, emission_lengths = self.align_model(wavs_tensors.to(self.device), wavs_lengths_tensor)
+            star_dim = torch.zeros((emission.shape[0], emission.size(1), 1), dtype=emission.dtype, device=self.device)
+            emission = torch.cat((emission, star_dim), dim=-1)
+        
+        logging.info("end alignment model forward")
+        
+        target_list = [self.get_target(transcript, language) for transcript, language in zip(transcript_list, language_list)]
+        
+        logging.info("align success")
+        align_results = [
+            self.align(emission_padded[:emission_length, :].unsqueeze(0), target)
+            for emission_padded, emission_length, target in zip(emission, emission_lengths, target_list)
+        ]
+        
+        logging.info("get align result")
+        batch_aligned_tokens = [align_result[0] for align_result in align_results]
+        batch_alignment_scores = [align_result[1] for align_result in align_results]
+
+        alignment_result_list = [
+            self.get_alignment_result(emission_padded, emission_length, aligned_tokens, alignment_scores, transcript, waveform, language)
+            for emission_padded, emission_length, aligned_tokens, alignment_scores, transcript, waveform, language
+            in zip(emission, emission_lengths, batch_aligned_tokens, batch_alignment_scores, transcript_list, wav_list, language_list)
+        ]
+        logging.info("get align result success")
+        return alignment_result_list
+
+
+async def batch_get_alignment_result_remote(alignment_url, audio_path, transcript, language):
+    """
+    通过调用远程对齐服务来批量获取对齐结果。
+    """
+    payload = {
+        "audio_path": audio_path,
+        "transcript": transcript,
+        "language": language,
+    }
+
+    try:
+        async with httpx.AsyncClient() as client:
+            response = await client.post(alignment_url, json=payload, timeout=300) # 设置较长的超时
+        response.raise_for_status()  # 如果状态码不是 2xx，则抛出异常
+        data = response.json()
+        return data['results']
+
+    except requests.exceptions.RequestException as e:
+        logging.error(f"Failed to connect to alignment service: {e}")
+        traceback.print_exc()
+        # 根据需求可以返回空列表或抛出异常
+    except Exception as e:
+        logging.error(f"An error occurred in remote alignment: {e}")
+        traceback.print_exc()
+

docker/Dockerfile

@@ -0,0 +1,40 @@
+###################################################################################################
+#
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are permitted
+# provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright notice, this list of
+#       conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright notice, this list of
+#       conditions and the following disclaimer in the documentation and/or other materials
+#       provided with the distribution.
+#     * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
+#       to endorse or promote products derived from this software without specific prior written
+#       permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+###################################################################################################
+FROM nvcr.io/nvidia/tritonserver:25.08-py3
+LABEL maintainer="NVIDIA"
+LABEL repository="tritonserver"
+
+RUN apt-get update && apt-get -y install swig && apt-get -y install python3-dev && apt-get install -y cmake && apt-get install -y libsndfile1
+RUN pip3 install kaldiio
+RUN pip3 install torch torchvision torchaudio -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+RUN pip3 install -v kaldifeat
+RUN python3 -m pip install cupy
+RUN python3 -m pip install soundfile
+RUN pip3 install --upgrade pip
+RUN pip install --extra-index-url https://pypi.nvidia.com cudf_cu12
+RUN pip install --extra-index-url https://pypi.nvidia.com cuml_cu12
+RUN pip install --extra-index-url https://pypi.nvidia.com cugraph_cu12
+WORKDIR /workspace

download_mms_model.py

@@ -0,0 +1,35 @@
+import torchaudio
+import os
+from pathlib import Path
+
+def download_mms_model(download_dir="/inspire/hdd/project/embodied-multimodality/public/yqzhang/auto_evaluation/models/mms_fa"):
+    """下载MMS-FA模型到指定目录"""
+    
+    # 创建下载目录
+    download_path = Path(download_dir)
+    download_path.mkdir(parents=True, exist_ok=True)
+    
+    print(f"开始下载MMS-FA模型到: {download_path}")
+    
+    try:
+        # 获取MMS-FA bundle
+        bundle = torchaudio.pipelines.MMS_FA
+        
+        # 下载模型
+        model = bundle.get_model(with_star=False, dl_kwargs={'model_dir': str(download_path)})
+        
+        print(f"✅ 模型下载成功！保存在: {download_path}")
+        print(f"模型文件: {list(download_path.glob('*'))}")
+        
+        return str(download_path)
+        
+    except Exception as e:
+        print(f"❌ 下载失败: {e}")
+        return None
+
+if __name__ == "__main__":
+    # 下载模型
+    model_path = download_mms_model()
+    if model_path:
+        print(f"\n使用方法:")
+        print(f"evaluator = SpeakerSimilarityEvaluator(alignment_model_dir='{model_path}')")

example_input.jsonl

@@ -0,0 +1 @@
+{"text": "[S1] Hey, do you know the AI world has been super lively lately?[S2] Oh, yeah, new news every day. It feels like, um, a lot of big companies are just pushing really hard to get ahead.[S1] Right, right, exactly. Like, big news popping up every other day. Recently, I saw something about Anthropic. Didn't they release Claude 4?[S2] Oh, Claude 4, yeah, I saw some reports. They said it's really powerful, their latest model.[S1] Mhm, they're calling it the world's best programming model,sounds super impressive.[S2] Mm.[S1] Hey, really? World's best? That title alone is pretty catchy.[S2] Yeah, that really makes you curious, actually.[S1] Right? And it claims that for long tasks requiring extreme focus and thousands of steps, it can maintain stable performance.[S2] Mm.[S1] Meaning, it doesn't crash easily.[S2] Wow, that's amazing. So, it doesn't crash easily, huh?[S1] Exactly. They said, like, the Japanese e-commerce giant Rakuten, you know them, right? They actually verified Claude Opus 4's capability. In a demanding open-source refactoring task, it ran independently for seven hours.[S2] Seven hours?[S1] And throughout that time, its performance remained completely stable.[S2] Wow, my goodness. It runs on its own for seven hours without a break? That's incredible.[S1] Yeah, for those tasks that need focused effort and thousands of steps, it can handle them steadily.[S2] Mm, that's really something.[S1] Uh, so it's especially suitable for complex coding and problem-solving scenarios.[S2] Oh, I see. So, how's its performance in programming, really? Is it actually much better than before?[S1] Yeah, they mentioned the SWE-bench evaluation, which is a benchmark test for software engineering tasks.[S2] Oh, I know that test, it's quite professional.[S1] Mm, their Claude Sonnet 4 achieved an accuracy of 72.7 percent.[S2] Mm, 72.7 percent, that's high.[S1] Right, and they also compared it to the previous Sonnet 3.7 version.[S2] Mm.[S1] The 3.7 version got 62.3 percent.[S2] Oh, that's about a ten-point difference, then.[S1] Exactly, so Sonnet 4 improved significantly.[S2] Hmm, so it seems like this upgrade is substantial, not just hype.[S1] Indeed. And they also released Claude Code, which is a dedicated programming tool.[S2] Hmm, like, for developers to use?[S1] Yes, they said Claude Code is officially launched and supported by both Claude 4 models.[S2] Oh, I see. So, not only are the models powerful, but they've also improved the tools, like a complete package.[S1] That's right. And they also said that Claude Code isn't just for programmers.[S2] Huh? If it's not for programmers, then who's it for?[S1] They said, even for people who aren't really good at programming,[S2] Mm.[S1] Like product managers, if they want to create a prototype for an idea, they can just ask Claude to do it.[S2] Wow, that's really interesting. So, you don't have to write the code yourself, you just let the AI help you realize your ideas, right?[S1] Yeah, they're saying that in the future, if you have an idea, you might not need to write a document; you can just have it help you create the prototype.[S2] Hmm, that sounds a bit like, uh, will programmers' jobs become less common in the future?[S1] Hmm, it might be more like, Scott White, who's their product lead, he said that Claude is transforming from a tool that provides answers into a truly capable collaborative partner.[S2] Oh, I understand. So, it helps you with, uh, more basic or repetitive tasks, allowing you to focus more on creative things.[S1] Yes, exactly. And the models they released this time are called Opus 4 and Sonnet 4.[S2] Mm.[S1] Opus 4, they say, is their most powerful model to date, and also the world's best programming model.[S2] Definitely the flagship model.[S1] And Sonnet 4 is a major upgrade to Sonnet 3.7.[S2] Oh, so what are the specific differences between the two?[S1] Hmm, Opus 4 is better at high-end tasks like coding, research, writing, and scientific discovery.[S2] Hmm, Opus sounds more all-around capable.[S1] Right, and Sonnet 4 is more suitable for everyday use cases; it offers cutting-edge performance for daily tasks.[S2] Oh, I see. So, one is super high-end, and the other is also super strong for everyday use.[S1] Yes, and both models use a hybrid mode design.[S2] Hybrid mode? What does that mean?[S1] It means it can provide almost instant responses, but also perform deeper reasoning and thought.[S2] Oh.[S1] Like, uh, expansive thinking.[S2] Oh, I see. So, sometimes it needs to be fast, and other times it needs to be slow and think deeply.[S1] Exactly.[S2] Hmm, so what about the pricing? Is it very expensive?[S1] The pricing is the same as the previous Opus and Sonnet models.[S2] Oh.[S1] For Opus 4, it's fifteen dollars per million input tokens and seventy-five dollars for output tokens.[S2] Wow, output is much more expensive![S1] Right. And for Sonnet 4, input is three dollars and output is fifteen dollars.[S2] Hmm, Sonnet is much more affordable then.[S1] Yes, and Sonnet 4 is also available for free users.[S2] Oh, that's good, everyone can try it out.[S1] Mm, exactly.[S2] Hey, so how does it compare to other AI giants? Where does it stand now?[S1] This release of theirs has intensified the competition with giants like OpenAI and Google in the top-tier model space.[S2] Yeah, it really feels like everyone's pushing hard lately.[S1] Right? Like, Microsoft also announced new coding agents, didn't they? And they partnered with Elon Musk's xAI.[S2] Mm.[S1] Google, meanwhile, is accelerating the integration of AI agents into their services.[S2] Right.[S1] And OpenAI is even more impressive; they just made a six-point-five-billion-dollar deal to acquire an AI hardware startup founded by the father of iPhone, former Apple design chief Jony Ive.[S2] Wow, six-point-five billion, that's a huge move. It feels like AI competition is really heating up.[S1] Exactly, so for investors, it means re-evaluating the competitive landscape in the AI sector.[S2] Hmm, makes sense. So, does this Claude 4 also bring a lot of opportunities for Anthropic?[S1] Yeah, its strong performance in coding, reasoning, and agent tasks will definitely help it capture more market share and enterprise clients.[S2] Hmm, sounds like it has huge potential indeed.[S1] Mm, it just feels like the AI competition now is all about who can push the technology to new heights.[S2] Exactly, and also who can really, uh, implement these technologies into practical applications.[S1] Right, right, exactly like that.[S2] Okay, well, this news about Claude 4 today really makes you feel like AI has taken a huge leap forward.[S1] Yeah, looking forward to it bringing more surprises in the future.", "prompt_audio_speaker1": "/inspire/hdd/project/embodied-multimodality/public/yqzhang/infer_prompt/testset/audio/moon-en1/en_spk1_moon.wav", "prompt_text_speaker1": "OK. I'm starting to see how this multi-headed approach could lead to some pretty impressive results.", "prompt_audio_speaker2": "/inspire/hdd/project/embodied-multimodality/public/yqzhang/infer_prompt/testset/audio/moon-en1/en_spk2-moon.wav", "prompt_text_speaker2": "It's not just crunching data. It's starting to develop a more sophisticated understanding of how language actually works.", "output_audio": "/inspire/hdd/project/embodied-multimodality/public/yqzhang/infer_res/from_newckpt_step40000/test_en/gpu0/output_0.wav"}

model_repo/speaker_model/1/model.trt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17a3a3b794fa886c9b8341a08ad5e22f3bb385dd994ff891ba62d591503673f5
+size 104729100

model_repo/speaker_model/config.pbtxt

@@ -0,0 +1,44 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "speaker_model"
+backend: "tensorrt"
+default_model_filename: "model.trt"
+
+max_batch_size: 16
+input [
+  {
+    name: "feats"
+    data_type: TYPE_FP32
+    dims: [ -1, 80 ] # num_mel_bins
+  }
+]
+
+output [
+  {
+    name: "embs"
+    data_type: TYPE_FP32
+    dims: [ 256 ] # [embedding_size]
+  }
+]
+dynamic_batching {
+    preferred_batch_size: [ 4, 8 ]
+    max_queue_delay_microseconds: 1000
+  }
+instance_group [
+    {
+      count: 1
+      kind: KIND_GPU
+    }
+]

models/mms_fa/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20ef12963ab4924bef49ac4fc7f58ad5da2ee43b2c11bc8c853c9b90ecdbc680
+size 1262047414

models/mms_fa/model.pt.2c7cc4fedf8e4a089a0095148cc9201b.partial

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0cf233f857de07296254c36332b4b984045cdc0964ec1fef6a0c6cc5aae00b7
+size 1056964608

models/mms_fa/model.pt.5c5fe9893a2c462e9132dcd6a3fba337.partial

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51258936b4a1a51762ef849ec0f404920f38d03c4e018550d75ea4e1e82a451a
+size 486539264

models/voxblink2_samresnet100_ft/avg_model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d92ee34668d8eb24a02df4e7869fd4bde661220a137e045f29e4a0c85eb4004
+size 201115747

models/voxblink2_samresnet100_ft/avg_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5aeee438ca23c0ca6e341bab6c6bf7f465497e1dc323bb1bc1074d6a0c778b11
+size 201318407

models/voxblink2_samresnet100_ft/config.yaml

@@ -0,0 +1,83 @@
+data_type: shard
+dataloader_args:
+  batch_size: 128
+  drop_last: true
+  num_workers: 16
+  pin_memory: false
+  prefetch_factor: 8
+dataset_args:
+  aug_prob: 0.6
+  fbank_args:
+    dither: 1.0
+    frame_length: 25
+    frame_shift: 10
+    num_mel_bins: 80
+  filter: true
+  filter_args:
+    max_num_frames: 800
+    min_num_frames: 100
+  num_frms: 200
+  resample_rate: 16000
+  sample_num_per_epoch: 0
+  shuffle: true
+  shuffle_args:
+    shuffle_size: 2500
+  spec_aug: false
+  spec_aug_args:
+    max_f: 8
+    max_t: 10
+    num_f_mask: 1
+    num_t_mask: 1
+    prob: 0.6
+  speed_perturb: true
+enable_amp: false
+exp_dir: exp/samresnet100/
+gpus:
+- 0
+- 1
+log_batch_interval: 100
+loss: CrossEntropyLoss
+loss_args: {}
+margin_scheduler: MarginScheduler
+margin_update:
+  epoch_iter: 4265
+  final_margin: 0.2
+  fix_start_epoch: 40
+  increase_start_epoch: 20
+  increase_type: exp
+  initial_margin: 0.0
+  update_margin: true
+model: SimAM_ResNet100_ASP
+model_args:
+  embed_dim: 256
+model_init: null
+noise_data: data/musan/lmdb
+num_avg: 1
+num_epochs: 150
+optimizer: SGD
+optimizer_args:
+  lr: 0.1
+  momentum: 0.9
+  nesterov: true
+  weight_decay: 0.0001
+projection_args:
+  do_lm: false
+  easy_margin: false
+  embed_dim: 256
+  num_class: 17982
+  project_type: arc_margin
+  scale: 32.0
+reverb_data: data/rirs/lmdb
+save_epoch_interval: 5
+scheduler: ExponentialDecrease
+scheduler_args:
+  epoch_iter: 4265
+  final_lr: 5.0e-05
+  initial_lr: 0.1
+  num_epochs: 150
+  scale_ratio: 4.0
+  warm_from_zero: true
+  warm_up_epoch: 6
+seed: 42
+train_data: data/vox2_dev/shard.list
+train_label: data/vox2_dev/utt2spk

models/wespeaker/chinese/config.yaml

@@ -0,0 +1,7 @@
+model: cnceleb_resnet34_LM
+task: speaker_verification
+domain: speech
+framework: onnxruntime
+dataset: cnceleb
+language: chinese
+sample_rate: 16000

models/wespeaker/chinese/model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7584940aeac8d5512d875e58ce6c09ba4ddad65d8128e1dac0d93aadd087ebb
+size 26530309

python_backend/similarity_model/1/model.py

@@ -0,0 +1,149 @@
+import io
+import math
+import numpy as np
+import torch
+import torchaudio
+import torchaudio.compliance.kaldi as kaldi
+import traceback
+from torch.utils.dlpack import from_dlpack
+import triton_python_backend_utils as pb_utils
+
+
+class TritonPythonModel:
+    def initialize(self, args):
+        self.sample_rate = 16000
+        self.feature_dim = 80
+        self.vad_enabled = True # This variable is declared but not used.
+        self.min_duration = 0.1
+        
+        # This seems correct for BLS (Business Logic Scripting)
+        self.speaker_model_name = "speaker_model"
+
+    def execute(self, requests):
+        responses = []
+        for request in requests:
+            try:
+                # 1. Get the input audio BYTES, not a file path string.
+                # The input tensor is of type TYPE_STRING, which holds bytes.
+                # .as_numpy()[0] gives you the raw bytes object.
+                audio1_bytes = pb_utils.get_input_tensor_by_name(request, "AUDIO_BYTES_1").as_numpy()[0][0]
+                audio2_bytes = pb_utils.get_input_tensor_by_name(request, "AUDIO_BYTES_2").as_numpy()[0][0]
+                
+                # 2. Preprocess audio from bytes
+                feats1 = self.preprocess(audio1_bytes)
+                feats2 = self.preprocess(audio2_bytes)
+                
+                # 3. Call the speaker_model to compute similarity
+                similarity = self.compute_similarity(feats1, feats2)
+
+                # Prepare output
+                output_tensor = pb_utils.Tensor("SIMILARITY", np.array([similarity], dtype=np.float32))
+                response = pb_utils.InferenceResponse(output_tensors=[output_tensor])
+                responses.append(response)
+
+            except pb_utils.TritonModelException as e:
+                # If a Triton-specific error occurs, create an error response
+                error_response = pb_utils.InferenceResponse(error=pb_utils.TritonError(str(e)))
+                pb_utils.Logger.log_error(error_response)
+                responses.append(error_response)
+            except Exception as e:
+                # For any other unexpected error, log it and return an error response
+                error_message = f"Unexpected error: {e}\n{traceback.format_exc()}"
+                pb_utils.Logger.log_error(error_message)
+                error_response = pb_utils.InferenceResponse(error=pb_utils.TritonError(error_message))
+                responses.append(error_response)
+
+        return responses
+
+    def preprocess(self, audio_bytes: bytes):
+        """
+        Processes audio data from an in-memory byte buffer.
+        If the audio is too short, it's padded by repetition to meet the minimum length.
+        """
+        try:
+            # Wrap the raw bytes in a file-like object for torchaudio
+            # buffer = io.BytesIO(audio_bytes)
+            buffer = audio_bytes.decode('utf-8')
+            waveform, sample_rate = torchaudio.load(buffer)
+            
+            # You might want to resample if the client's sample rate differs
+            if sample_rate != self.sample_rate:
+                # Note: This requires the 'torchaudio.transforms' module.
+                # Make sure torchaudio is fully installed in your Triton environment.
+                resampler = torchaudio.transforms.Resample(orig_freq=sample_rate, new_freq=self.sample_rate)
+                waveform = resampler(waveform)
+
+            duration = waveform.shape[1] / self.sample_rate
+            
+            if duration < self.min_duration:
+                # Audio is too short, repeat it to meet the minimum duration
+                repeat_times = math.ceil(self.min_duration / duration)
+                waveform = waveform.repeat(1, repeat_times)
+            
+            # --- THIS IS THE NEW, CRITICAL PART ---
+            # Calculate 80-dimensional Fbank features, which is what the speaker_model expects.
+            # The waveform needs to be shape [batch, time], so we squeeze it.
+            features = kaldi.fbank(
+                waveform.squeeze(0).unsqueeze(0), # Needs shape [1, T]
+                num_mel_bins=self.feature_dim,     # This is 80
+                sample_frequency=self.sample_rate,
+                frame_length=25,
+                frame_shift=10
+            )
+            # The output of fbank is [1, num_frames, num_bins], e.g., [1, 150, 80]
+            # We need [num_frames, num_bins] for the speaker model
+            return features.squeeze(0) # Returns shape [num_frames, 80]
+            
+        except Exception as e:
+            # Raise a specific exception that can be caught in execute()
+            raise pb_utils.TritonModelException(f"Failed during audio preprocessing: {e}")
+    
+    def compute_similarity(self, waveform1, waveform2):
+        # Call speaker_model to get embeddings
+        # Assuming speaker_model takes a waveform and outputs an embedding
+        e1 = torch.from_numpy(self.call_speaker_model(waveform1)).to("cuda")
+        e2 = torch.from_numpy(self.call_speaker_model(waveform2)).to("cuda")
+        
+        # Flatten the tensors
+        e1 = e1.flatten()
+        e2 = e2.flatten()
+        
+        # Calculate cosine similarity
+        dot_product = torch.dot(e1, e2)
+        norm_e1 = torch.norm(e1)
+        norm_e2 = torch.norm(e2)
+        
+        # Handle zero norms
+        if norm_e1 == 0 or norm_e2 == 0:
+            return 0.0
+        
+        similarity = (dot_product / (norm_e1 * norm_e2)).item()
+        
+        # Normalize from [-1, 1] to [0, 1]
+        return (similarity + 1) / 2
+    
+    def call_speaker_model(self, waveform):
+        """Calls the speaker_model to get an embedding vector."""
+        # Create the input tensor for the speaker_model.
+        # The name 'feats' here must match the input name in speaker_model's config.pbtxt
+        if waveform.dim() == 2:
+            waveform = waveform.unsqueeze(0)
+        input_tensor = pb_utils.Tensor("feats", waveform.cpu().numpy().astype(np.float32))
+        
+        inference_request = pb_utils.InferenceRequest(
+            model_name=self.speaker_model_name,
+            requested_output_names=["embs"], # Must match output name in speaker_model's config
+            inputs=[input_tensor]
+        )
+        
+        inference_response = inference_request.exec()
+        
+        if inference_response.has_error():
+            raise pb_utils.TritonModelException(f"Error from speaker_model: {inference_response.error().message()}")
+        
+        output_tensor = pb_utils.get_output_tensor_by_name(inference_response, "embs")
+        if output_tensor.is_cpu():
+            output_tensor = output_tensor.as_numpy()
+        else:
+            output_tensor = from_dlpack(output_tensor.to_dlpack()).detach().cpu().numpy()
+        return output_tensor

python_backend/similarity_model/1/model_old.py

@@ -0,0 +1,97 @@
+import math
+import numpy as np
+import torchaudio
+import traceback
+import triton_python_backend_utils as pb_utils
+
+
+class TritonPythonModel:
+    def initialize(self, args):
+        self.sample_rate = 16000
+        self.feature_dim = 80
+        self.vad_enabled = True
+        self.min_duration = 0.1
+        
+        # 创建与speaker_model通信的客户端
+        self.speaker_model_name = "speaker_model"
+
+    def execute(self, requests):
+        responses = []
+        for request in requests:
+            # 获取输入音频
+            audio1 = pb_utils.get_input_tensor_by_name(request, "AUDIO1").as_numpy()[0].decode('utf-8')
+            audio2 = pb_utils.get_input_tensor_by_name(request, "AUDIO2").as_numpy()[0].decode('utf-8')
+            
+            # 预处理音频
+            feats1 = self.preprocess(audio1)
+            feats2 = self.preprocess(audio2)
+            
+            # 调用speaker_model计算相似度
+            similarity = self.compute_similarity(feats1, feats2)
+            
+            # 准备输出
+            output_tensor = pb_utils.Tensor("SIMILARITY", np.array([similarity]), dtype=np.float32)
+            response = pb_utils.InferenceResponse(output_tensors=[output_tensor])
+            responses.append(response)
+            
+        return responses
+
+    def preprocess(self, audio_path):
+        """
+        处理音频文件，如果过短则复制到满足最小长度要求
+        返回处理后的音频路径和是否为临时文件的标志
+        """
+        try:
+            waveform, sample_rate = torchaudio.load(audio_path)
+            duration = waveform.shape[1] / sample_rate
+            
+            if duration >= self.min_duration:
+                # 音频长度足够，直接返回原路径
+                return waveform
+            
+            # 音频过短，需要复制
+            repeat_times = math.ceil(self.min_duration / duration)
+            
+            # 复制音频
+            return waveform.repeat(1, repeat_times)
+            
+        except Exception:
+            traceback.format_exc()
+            return None
+    
+    def compute_similarity(self, feats1, feats2):
+        # 调用speaker_model获取嵌入向量
+        e1 = self.call_speaker_model(feats1)
+        e2 = self.call_speaker_model(feats2)
+        
+        # 计算余弦相似度
+        dot_product = np.dot(e1, e2)
+        norm_e1 = np.linalg.norm(e1)
+        norm_e2 = np.linalg.norm(e2)
+        similarity = dot_product / (norm_e1 * norm_e2)
+        
+        # 归一化到[0, 1]
+        return (similarity + 1) / 2
+    
+    def call_speaker_model(self, features):
+        """调用speaker_model获取嵌入向量"""
+        # 创建输入张量
+        input_tensor = pb_utils.Tensor("feats", features.astype(np.float32))
+        
+        # 创建推理请求
+        inference_request = pb_utils.InferenceRequest(
+            model_name=self.speaker_model_name,
+            requested_output_names=["embs"],
+            inputs=[input_tensor]
+        )
+        
+        # 发送请求
+        inference_response = inference_request.exec()
+        
+        # 处理响应
+        if inference_response.has_error():
+            raise pb_utils.TritonModelException(inference_response.error().message())
+        
+        # 获取嵌入向量
+        output_tensor = pb_utils.get_output_tensor_by_name(inference_response, "embs")
+        return output_tensor.as_numpy()

python_backend/similarity_model/1/model_runnable.py

@@ -0,0 +1,149 @@
+import io
+import math
+import numpy as np
+import torch
+import torchaudio
+import torchaudio.compliance.kaldi as kaldi
+import traceback
+from torch.utils.dlpack import from_dlpack
+import triton_python_backend_utils as pb_utils
+
+
+class TritonPythonModel:
+    def initialize(self, args):
+        self.sample_rate = 16000
+        self.feature_dim = 80
+        self.vad_enabled = True # This variable is declared but not used.
+        self.min_duration = 0.1
+        
+        # This seems correct for BLS (Business Logic Scripting)
+        self.speaker_model_name = "speaker_model"
+
+    def execute(self, requests):
+        responses = []
+        for request in requests:
+            try:
+                # 1. Get the input audio BYTES, not a file path string.
+                # The input tensor is of type TYPE_STRING, which holds bytes.
+                # .as_numpy()[0] gives you the raw bytes object.
+                audio1_bytes = pb_utils.get_input_tensor_by_name(request, "AUDIO_BYTES_1").as_numpy()[0][0]
+                audio2_bytes = pb_utils.get_input_tensor_by_name(request, "AUDIO_BYTES_2").as_numpy()[0][0]
+                
+                # 2. Preprocess audio from bytes
+                feats1 = self.preprocess(audio1_bytes)
+                feats2 = self.preprocess(audio2_bytes)
+                
+                # 3. Call the speaker_model to compute similarity
+                similarity = self.compute_similarity(feats1, feats2)
+                
+                pb_utils.Logger.log_info(similarity)
+                # Prepare output
+                output_tensor = pb_utils.Tensor("SIMILARITY", np.array([similarity], dtype=np.float32))
+                response = pb_utils.InferenceResponse(output_tensors=[output_tensor])
+                responses.append(response)
+
+            except pb_utils.TritonModelException as e:
+                # If a Triton-specific error occurs, create an error response
+                error_response = pb_utils.InferenceResponse(error=pb_utils.TritonError(str(e)))
+                responses.append(error_response)
+            except Exception as e:
+                # For any other unexpected error, log it and return an error response
+                error_message = f"Unexpected error: {e}\n{traceback.format_exc()}"
+                pb_utils.Logger.log_error(error_message)
+                error_response = pb_utils.InferenceResponse(error=pb_utils.TritonError(error_message))
+                responses.append(error_response)
+
+        return responses
+
+    def preprocess(self, audio_bytes: bytes):
+        """
+        Processes audio data from an in-memory byte buffer.
+        If the audio is too short, it's padded by repetition to meet the minimum length.
+        """
+        try:
+            # Wrap the raw bytes in a file-like object for torchaudio
+            # buffer = io.BytesIO(audio_bytes)
+            buffer = audio_bytes.decode('utf-8')
+            waveform, sample_rate = torchaudio.load(buffer)
+            
+            # You might want to resample if the client's sample rate differs
+            if sample_rate != self.sample_rate:
+                # Note: This requires the 'torchaudio.transforms' module.
+                # Make sure torchaudio is fully installed in your Triton environment.
+                resampler = torchaudio.transforms.Resample(orig_freq=sample_rate, new_freq=self.sample_rate)
+                waveform = resampler(waveform)
+
+            duration = waveform.shape[1] / self.sample_rate
+            
+            if duration < self.min_duration:
+                # Audio is too short, repeat it to meet the minimum duration
+                repeat_times = math.ceil(self.min_duration / duration)
+                waveform = waveform.repeat(1, repeat_times)
+            
+            # --- THIS IS THE NEW, CRITICAL PART ---
+            # Calculate 80-dimensional Fbank features, which is what the speaker_model expects.
+            # The waveform needs to be shape [batch, time], so we squeeze it.
+            features = kaldi.fbank(
+                waveform.squeeze(0).unsqueeze(0), # Needs shape [1, T]
+                num_mel_bins=self.feature_dim,     # This is 80
+                sample_frequency=self.sample_rate,
+                frame_length=25,
+                frame_shift=10
+            )
+            # The output of fbank is [1, num_frames, num_bins], e.g., [1, 150, 80]
+            # We need [num_frames, num_bins] for the speaker model
+            return features.squeeze(0) # Returns shape [num_frames, 80]
+            
+        except Exception as e:
+            # Raise a specific exception that can be caught in execute()
+            raise pb_utils.TritonModelException(f"Failed during audio preprocessing: {e}")
+    
+    def compute_similarity(self, waveform1, waveform2):
+        # Call speaker_model to get embeddings
+        # Assuming speaker_model takes a waveform and outputs an embedding
+        e1 = torch.from_numpy(self.call_speaker_model(waveform1)).to("cuda")
+        e2 = torch.from_numpy(self.call_speaker_model(waveform2)).to("cuda")
+        
+        # Flatten the tensors
+        e1 = e1.flatten()
+        e2 = e2.flatten()
+        
+        # Calculate cosine similarity
+        dot_product = torch.dot(e1, e2)
+        norm_e1 = torch.norm(e1)
+        norm_e2 = torch.norm(e2)
+        
+        # Handle zero norms
+        if norm_e1 == 0 or norm_e2 == 0:
+            return 0.0
+        
+        similarity = (dot_product / (norm_e1 * norm_e2)).item()
+        
+        # Normalize from [-1, 1] to [0, 1]
+        return (similarity + 1) / 2
+    
+    def call_speaker_model(self, waveform):
+        """Calls the speaker_model to get an embedding vector."""
+        # Create the input tensor for the speaker_model.
+        # The name 'feats' here must match the input name in speaker_model's config.pbtxt
+        if waveform.dim() == 2:
+            waveform = waveform.unsqueeze(0)
+        input_tensor = pb_utils.Tensor("feats", waveform.cpu().numpy().astype(np.float32))
+        
+        inference_request = pb_utils.InferenceRequest(
+            model_name=self.speaker_model_name,
+            requested_output_names=["embs"], # Must match output name in speaker_model's config
+            inputs=[input_tensor]
+        )
+        
+        inference_response = inference_request.exec()
+        
+        if inference_response.has_error():
+            raise pb_utils.TritonModelException(f"Error from speaker_model: {inference_response.error().message()}")
+        
+        output_tensor = pb_utils.get_output_tensor_by_name(inference_response, "embs")
+        if output_tensor.is_cpu():
+            output_tensor = output_tensor.as_numpy()
+        else:
+            output_tensor = from_dlpack(output_tensor.to_dlpack()).detach().cpu().numpy()
+        return output_tensor

python_backend/similarity_model/config.pbtxt.back

@@ -0,0 +1,46 @@
+name: "similarity_model"
+backend: "python"
+max_batch_size: 128
+
+parameters: {
+  key: "EXECUTION_ENV_PATH",
+  value: {
+    # string_value: "/inspire/hdd/project/embodied-multimodality/public/cchang/env/audio.tar.gz"
+    # string_value: "/inspire/hdd/project/embodied-multimodality/public/cchang/env/audio_clean.tar.gz"
+    # string_value: "/inspire/hdd/project/embodied-multimodality/public/cchang/env_tar/audio_env.tar.gz"
+    string_value: "/inspire/hdd/project/embodied-multimodality/public/cchang/env/mooncast/bin/python"
+  }
+}
+
+input [
+  {
+    name: "AUDIO1"
+    data_type: TYPE_STRING
+    dims: [ 1 ]  # 音频路径
+  },
+  {
+    name: "AUDIO2"
+    data_type: TYPE_STRING
+    dims: [ 1 ]  # 音频路径
+  }
+]
+
+output [
+  {
+    name: "SIMILARITY"
+    data_type: TYPE_FP32
+    dims: [ 1 ]  # 相似度分数
+  }
+]
+
+dynamic_batching {
+  preferred_batch_size: [ 16, 32 ]
+}
+
+instance_group [
+  {
+    count: 1
+    kind: KIND_GPU
+  }
+]
+

python_backend/similarity_model/config.pbtxt.disabled

@@ -0,0 +1,26 @@
+name: "similarity_model" # Or whatever you call this model
+backend: "python"
+max_batch_size: 8
+
+# Input tensors are now raw audio bytes
+input [
+  {
+    name: "AUDIO_BYTES_1"
+    data_type: TYPE_STRING # TYPE_STRING is used for variable-length binary data
+    dims: [ 1 ]
+  },
+  {
+    name: "AUDIO_BYTES_2"
+    data_type: TYPE_STRING
+    dims: [ 1 ]
+  }
+]
+
+# Output is a single similarity score
+output [
+  {
+    name: "SIMILARITY"
+    data_type: TYPE_FP32
+    dims: [ 1 ]
+  }
+]

similarity.py

@@ -0,0 +1,412 @@
+# Copyright (c) 2023 Binbin Zhang ([email protected])
+#                    Shuai Wang ([email protected])
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+from typing import List, Tuple
+
+import numpy as np
+from silero_vad import load_silero_vad, read_audio, get_speech_timestamps
+import torch
+import torchaudio
+import torchaudio.compliance.kaldi as kaldi
+import yaml
+import kaldiio
+from tqdm import tqdm
+
+from wespeaker.cli.hub import Hub
+from wespeaker.cli.utils import get_args
+from wespeaker.models.speaker_model import get_speaker_model
+from wespeaker.utils.checkpoint import load_checkpoint
+from wespeaker.diar.umap_clusterer import cluster
+from wespeaker.diar.extract_emb import subsegment
+from wespeaker.diar.make_rttm import merge_segments
+from wespeaker.utils.utils import set_seed
+
+
+class Speaker:
+
+    def __init__(self, model_dir: str):
+        set_seed()
+
+        config_path = os.path.join(model_dir, 'config.yaml')
+        model_path = os.path.join(model_dir, 'avg_model.pt')
+        with open(config_path, 'r') as fin:
+            configs = yaml.load(fin, Loader=yaml.FullLoader)
+        self.model = get_speaker_model(
+            configs['model'])(**configs['model_args'])
+        load_checkpoint(self.model, model_path)
+        self.model.eval()
+        self.vad = load_silero_vad()
+        self.table = {}
+        self.resample_rate = 16000
+        self.apply_vad = False
+        self.device = torch.device('cpu')
+        self.wavform_norm = False
+
+        # diarization parmas
+        self.diar_min_duration = 0.255
+        self.diar_window_secs = 1.5
+        self.diar_period_secs = 0.75
+        self.diar_frame_shift = 10
+        self.diar_batch_size = 32
+        self.diar_subseg_cmn = True
+
+    def set_wavform_norm(self, wavform_norm: bool):
+        self.wavform_norm = wavform_norm
+
+    def set_resample_rate(self, resample_rate: int):
+        self.resample_rate = resample_rate
+
+    def set_vad(self, apply_vad: bool):
+        self.apply_vad = apply_vad
+
+    def set_device(self, device: str):
+        self.device = torch.device(device)
+        self.model = self.model.to(self.device)
+
+    def set_diarization_params(self,
+                               min_duration: float = 0.255,
+                               window_secs: float = 1.5,
+                               period_secs: float = 0.75,
+                               frame_shift: int = 10,
+                               batch_size: int = 32,
+                               subseg_cmn: bool = True):
+        self.diar_min_duration = min_duration
+        self.diar_window_secs = window_secs
+        self.diar_period_secs = period_secs
+        self.diar_frame_shift = frame_shift
+        self.diar_batch_size = batch_size
+        self.diar_subseg_cmn = subseg_cmn
+
+    def compute_fbank(self,
+                      wavform,
+                      sample_rate=16000,
+                      num_mel_bins=80,
+                      frame_length=25,
+                      frame_shift=10,
+                      cmn=True):
+        feat = kaldi.fbank(wavform,
+                           num_mel_bins=num_mel_bins,
+                           frame_length=frame_length,
+                           frame_shift=frame_shift,
+                           sample_frequency=sample_rate,
+                           window_type='hamming')
+        if cmn:
+            feat = feat - torch.mean(feat, 0)
+        return feat
+
+    def extract_embedding_feats(self, fbanks, batch_size, subseg_cmn):
+        fbanks_array = np.stack(fbanks)
+        if subseg_cmn:
+            fbanks_array = fbanks_array - np.mean(
+                fbanks_array, axis=1, keepdims=True)
+        embeddings = []
+        fbanks_array = torch.from_numpy(fbanks_array).to(self.device)
+        for i in tqdm(range(0, fbanks_array.shape[0], batch_size)):
+            batch_feats = fbanks_array[i:i + batch_size]
+            with torch.no_grad():
+                batch_embs = self.model(batch_feats)
+                batch_embs = batch_embs[-1] if isinstance(
+                    batch_embs, tuple) else batch_embs
+            embeddings.append(batch_embs.detach().cpu().numpy())
+        embeddings = np.vstack(embeddings)
+        return embeddings
+
+    def extract_embedding(self, audio_path: str):
+        pcm, sample_rate = torchaudio.load(audio_path,
+                                           normalize=self.wavform_norm)
+        return self.extract_embedding_from_pcm(pcm, sample_rate)
+
+    def extract_embedding_from_pcm(self, pcm: torch.Tensor, sample_rate: int):
+        if self.apply_vad:
+            # TODO(Binbin Zhang): Refine the segments logic, here we just
+            # suppose there is only silence at the start/end of the speech
+            vad_sample_rate = 16000
+            wav = pcm
+            if wav.size(0) > 1:
+                wav = wav.mean(dim=0, keepdim=True)
+
+            if sample_rate != vad_sample_rate:
+                transform = torchaudio.transforms.Resample(
+                    orig_freq=sample_rate, new_freq=vad_sample_rate)
+                wav = transform(wav)
+            segments = get_speech_timestamps(wav,
+                                             self.vad,
+                                             return_seconds=True)
+            pcmTotal = torch.Tensor()
+            if len(segments) > 0:  # remove all the silence
+                for segment in segments:
+                    start = int(segment['start'] * sample_rate)
+                    end = int(segment['end'] * sample_rate)
+                    pcmTemp = pcm[0, start:end]
+                    pcmTotal = torch.cat([pcmTotal, pcmTemp], 0)
+                pcm = pcmTotal.unsqueeze(0)
+            else:  # all silence, nospeech
+                return None
+        pcm = pcm.to(torch.float)
+        if sample_rate != self.resample_rate:
+            pcm = torchaudio.transforms.Resample(
+                orig_freq=sample_rate, new_freq=self.resample_rate)(pcm)
+        feats = self.compute_fbank(pcm,
+                                   sample_rate=self.resample_rate,
+                                   cmn=True)
+        feats = feats.unsqueeze(0)
+        feats = feats.to(self.device)
+
+        with torch.no_grad():
+            outputs = self.model(feats)
+            outputs = outputs[-1] if isinstance(outputs, tuple) else outputs
+        embedding = outputs[0].to(torch.device('cpu'))
+        return embedding
+
+    def extract_embedding_list(self, scp_path: str):
+        names = []
+        embeddings = []
+        with open(scp_path, 'r') as read_scp:
+            for line in tqdm(read_scp):
+                name, wav_path = line.strip().split()
+                names.append(name)
+                embedding = self.extract_embedding(wav_path)
+                embeddings.append(embedding.detach().numpy())
+        return names, embeddings
+
+    def compute_similarity(self, audio_path1: str, audio_path2: str) -> float:
+        e1 = self.extract_embedding(audio_path1)
+        e2 = self.extract_embedding(audio_path2)
+        if e1 is None or e2 is None:
+            return 0.0
+        else:
+            return self.cosine_similarity(e1, e2)
+
+    def compute_similarity_batch(
+            self, audio_pairs: List[Tuple[str, str]]) -> List[float]:
+        """
+        Computes cosine similarity for a batch of audio file pairs.
+        This method is optimized to extract embedding for each unique audio file
+        only once.
+
+        Args:
+            audio_pairs (List[Tuple[str, str]]): A list of tuples, where each
+                tuple contains two audio paths.
+                e.g., [('audio1.wav', 'audio2.wav'),
+                       ('audio1.wav', 'audio3.wav')]
+
+        Returns:
+            List[float]: A list of similarity scores, corresponding to the
+                         input pairs.
+        """
+        # 1. Collect all unique audio paths to avoid redundant computations
+        unique_audio_paths = set()
+        for path1, path2 in audio_pairs:
+            unique_audio_paths.add(path1)
+            unique_audio_paths.add(path2)
+
+        # 2. Extract embeddings for all unique files and store them in a cache
+        embedding_cache = {}
+        print(f"Extracting embeddings for {len(unique_audio_paths)} "
+              "unique audio files...")
+        for path in tqdm(list(unique_audio_paths)):
+            embedding_cache[path] = self.extract_embedding(path)
+
+        # 3. Compute similarity for each pair using the cached embeddings
+        scores = []
+        for path1, path2 in audio_pairs:
+            e1 = embedding_cache.get(path1)
+            e2 = embedding_cache.get(path2)
+
+            if e1 is None or e2 is None:
+                # Handle cases where embedding extraction failed (e.g., all
+                # silence)
+                scores.append(0.0)
+            else:
+                score = self.cosine_similarity(e1, e2)
+                scores.append(score)
+
+        return scores
+
+    def cosine_similarity(self, e1, e2):
+        cosine_score = torch.dot(e1, e2) / (torch.norm(e1) * torch.norm(e2))
+        cosine_score = cosine_score.item()
+        return (cosine_score + 1.0) / 2  # normalize: [-1, 1] => [0, 1]
+
+    def register(self, name: str, audio_path: str):
+        if name in self.table:
+            print('Speaker {} already registered, ignore'.format(name))
+        else:
+            self.table[name] = self.extract_embedding(audio_path)
+
+    def recognize(self, audio_path: str):
+        q = self.extract_embedding(audio_path)
+        best_score = 0.0
+        best_name = ''
+        for name, e in self.table.items():
+            score = self.cosine_similarity(q, e)
+            if best_score < score:
+                best_score = score
+                best_name = name
+        result = {}
+        result['name'] = best_name
+        result['confidence'] = best_score
+        return result
+
+    def diarize(self, audio_path: str, utt: str = "unk"):
+
+        pcm, sample_rate = torchaudio.load(audio_path, normalize=False)
+        # 1. vad
+        wav = read_audio(audio_path)
+        vad_segments = get_speech_timestamps(wav,
+                                             self.vad,
+                                             return_seconds=True)
+        if not vad_segments:
+            return []
+        # 2. extact fbanks
+        subsegs, subseg_fbanks = [], []
+        window_fs = int(self.diar_window_secs * 1000) // self.diar_frame_shift
+        period_fs = int(self.diar_period_secs * 1000) // self.diar_frame_shift
+        for item in vad_segments:
+            begin, end = item['start'], item['end']
+            if end - begin >= self.diar_min_duration:
+                begin_idx = int(begin * sample_rate)
+                end_idx = int(end * sample_rate)
+                tmp_wavform = pcm[0, begin_idx:end_idx].unsqueeze(0).to(
+                    torch.float)
+                fbank = self.compute_fbank(tmp_wavform,
+                                           sample_rate=sample_rate,
+                                           cmn=False)
+                tmp_subsegs, tmp_subseg_fbanks = subsegment(
+                    fbank=fbank,
+                    seg_id="{:08d}-{:08d}".format(int(begin * 1000),
+                                                  int(end * 1000)),
+                    window_fs=window_fs,
+                    period_fs=period_fs,
+                    frame_shift=self.diar_frame_shift)
+                subsegs.extend(tmp_subsegs)
+                subseg_fbanks.extend(tmp_subseg_fbanks)
+
+        # 3. extract embedding
+        embeddings = self.extract_embedding_feats(subseg_fbanks,
+                                                  self.diar_batch_size,
+                                                  self.diar_subseg_cmn)
+
+        # 4. cluster
+        subseg2label = []
+        labels = cluster(embeddings)
+        for (_subseg, _label) in zip(subsegs, labels):
+            # b, e = process_seg_id(_subseg, frame_shift=self.diar_frame_shift)
+            # subseg2label.append([b, e, _label])
+            begin_ms, end_ms, begin_frames, end_frames = _subseg.split('-')
+            begin = (int(begin_ms) +
+                     int(begin_frames) * self.diar_frame_shift) / 1000.0
+            end = (int(begin_ms) +
+                   int(end_frames) * self.diar_frame_shift) / 1000.0
+            subseg2label.append([begin, end, _label])
+
+        # 5. merged segments
+        # [[utt, ([begin, end, label], [])], [utt, ([], [])]]
+        merged_segment_to_labels = merge_segments({utt: subseg2label})
+
+        return merged_segment_to_labels
+
+    def diarize_list(self, scp_path: str):
+        utts = []
+        segment2labels = []
+        with open(scp_path, 'r', encoding='utf-8') as read_scp:
+            for line in tqdm(read_scp):
+                utt, wav_path = line.strip().split()
+                utts.append(utt)
+                segment2label = self.diarize(wav_path, utt)
+                segment2labels.append(segment2label)
+        return utts, segment2labels
+
+    def make_rttm(self, merged_segment_to_labels, outfile):
+        with open(outfile, 'w', encoding='utf-8') as fin:
+            for (utt, begin, end, label) in merged_segment_to_labels:
+                fin.write(
+                    "SPEAKER {} {} {:.3f} {:.3f} <NA> <NA> {} <NA> <NA>\n".
+                    format(utt, 1, float(begin),
+                           float(end) - float(begin), label))
+
+
+def load_model(language: str) -> Speaker:
+    model_path = Hub.get_model(language)
+    return Speaker(model_path)
+
+
+def load_model_local(model_dir: str) -> Speaker:
+    return Speaker(model_dir)
+
+
+def main():
+    args = get_args()
+    if args.pretrain == "":
+        if args.campplus:
+            model = load_model("campplus")
+            model.set_wavform_norm(True)
+        elif args.eres2net:
+            model = load_model("eres2net")
+            model.set_wavform_norm(True)
+        elif args.vblinkp:
+            model = load_model("vblinkp")
+        elif args.vblinkf:
+            model = load_model("vblinkf")
+        else:
+            model = load_model(args.language)
+    else:
+        model = load_model_local(args.pretrain)
+    model.set_resample_rate(args.resample_rate)
+    model.set_vad(args.vad)
+    model.set_device(args.device)
+    model.set_diarization_params(min_duration=args.diar_min_duration,
+                                 window_secs=args.diar_window_secs,
+                                 period_secs=args.diar_period_secs,
+                                 frame_shift=args.diar_frame_shift,
+                                 batch_size=args.diar_emb_bs,
+                                 subseg_cmn=args.diar_subseg_cmn)
+    if args.task == 'embedding':
+        embedding = model.extract_embedding(args.audio_file)
+        if embedding is not None:
+            np.savetxt(args.output_file, embedding.detach().numpy())
+            print('Succeed, see {}'.format(args.output_file))
+        else:
+            print('Fails to extract embedding')
+    elif args.task == 'embedding_kaldi':
+        names, embeddings = model.extract_embedding_list(args.wav_scp)
+        embed_ark = args.output_file + ".ark"
+        embed_scp = args.output_file + ".scp"
+        with kaldiio.WriteHelper('ark,scp:' + embed_ark + "," +
+                                 embed_scp) as writer:
+            for name, embedding in zip(names, embeddings):
+                writer(name, embedding)
+    elif args.task == 'similarity':
+        print(model.compute_similarity(args.audio_file, args.audio_file2))
+    elif args.task == 'diarization':
+        diar_result = model.diarize(args.audio_file)
+        if args.output_file is None:
+            for (_, start, end, spkid) in diar_result:
+                print("{:.3f}\t{:.3f}\t{:d}".format(start, end, spkid))
+        else:
+            model.make_rttm(diar_result, args.output_file)
+    elif args.task == 'diarization_list':
+        utts, segment2labels = model.diarize_list(args.wav_scp)
+        assert args.output_file is not None
+        model.make_rttm(np.vstack(segment2labels), args.output_file)
+    else:
+        print('Unsupported task {}'.format(args.task))
+        sys.exit(-1)
+
+
+if __name__ == '__main__':
+    main()

speaker_client.py

@@ -0,0 +1,149 @@
+# new_client.py
+import argparse
+import asyncio
+import numpy as np
+import torch
+import torchaudio
+import torchaudio.compliance.kaldi as kaldi
+import tritonclient.grpc.aio as grpcclient
+import sys
+import time
+import math
+
+
+class TritonSpeakerClient:
+    def __init__(self, url, model_name="speaker_model", verbose=False):
+        try:
+            self.triton_client = grpcclient.InferenceServerClient(url=url, verbose=verbose)
+        except Exception as e:
+            print(f"Channel creation failed: {e}", file=sys.stderr)
+            sys.exit(1)
+            
+        self.model_name = model_name
+        
+        # --- 从旧的 similarity_model 迁移过来的预处理参数 ---
+        self.sample_rate = 16000
+        self.feature_dim = 80
+        self.min_duration = 0.1
+        # ----------------------------------------------------
+
+    def _preprocess_audio(self, audio_path: str):
+        """
+        从音频文件路径加载并预处理音频，生成Fbank特征。
+        这段逻辑完全复制自旧的 similarity_model.py 中的 preprocess 方法。
+        """
+        try:
+            waveform, sample_rate = torchaudio.load(audio_path)
+            
+            # 如果采样率不匹配，则重采样
+            if sample_rate != self.sample_rate:
+                resampler = torchaudio.transforms.Resample(orig_freq=sample_rate, new_freq=self.sample_rate)
+                waveform = resampler(waveform)
+
+            # 如果音频太短，则重复填充以满足最小长度
+            duration = waveform.shape[1] / self.sample_rate
+            if duration < self.min_duration:
+                repeat_times = math.ceil(self.min_duration / duration)
+                waveform = waveform.repeat(1, repeat_times)
+            
+            # 计算80维Fbank特征
+            # waveform 需要是 [batch, time] 格式，所以我们移除通道维度
+            if waveform.shape[0] > 1:
+                waveform = torch.mean(waveform, dim=0, keepdim=True) # 转为单声道
+
+            features = kaldi.fbank(
+                waveform,
+                num_mel_bins=self.feature_dim,
+                sample_frequency=self.sample_rate,
+                frame_length=25,
+                frame_shift=10
+            )
+            # fbank 输出 shape [1, num_frames, num_bins], 我们需要 [num_frames, 80]
+            return features.squeeze(0).numpy().astype(np.float32)
+            
+        except Exception as e:
+            raise RuntimeError(f"Failed during audio preprocessing for {audio_path}: {e}")
+
+    def _calculate_cosine_similarity(self, emb1: np.ndarray, emb2: np.ndarray):
+        """在客户端计算余弦相似度。"""
+        e1 = torch.from_numpy(emb1).flatten()
+        e2 = torch.from_numpy(emb2).flatten()
+        
+        similarity = torch.nn.functional.cosine_similarity(e1, e2, dim=0)
+        
+        # 将相似度从 [-1, 1] 范围归一化到 [0, 1]
+        return (similarity.item() + 1) / 2
+
+    async def compute_similarity(self, audio1_path: str, audio2_path: str):
+        """
+        计算两个音频文件的相似度。
+        此函数现在包含完整的处理流程：预处理 -> 批处理 -> 推理 -> 后处理。
+        """
+        # 1. 在客户端对两个音频文件进行预处理
+        feats1 = self._preprocess_audio(audio1_path)
+        feats2 = self._preprocess_audio(audio2_path)
+        
+        # 2. 批处理：为了使用Triton的动态批处理，我们将两个特征打包成一个请求。
+        #    由于它们的长度（帧数）可能不同，我们需要将它们填充到相同的长度。
+        max_len = max(feats1.shape[0], feats2.shape[0])
+        
+        # 使用np.pad进行填充
+        padded_feats1 = np.pad(feats1, ((0, max_len - feats1.shape[0]), (0, 0)), 'constant', constant_values=0)
+        padded_feats2 = np.pad(feats2, ((0, max_len - feats2.shape[0]), (0, 0)), 'constant', constant_values=0)
+        
+        # 将填充后的特征堆叠成一个批次
+        input_batch = np.stack([padded_feats1, padded_feats2]) # Shape: [2, max_len, 80]
+
+        # 3. 创建Triton输入张量
+        #    输入名称 "feats" 必须与 speaker_model 的 config.pbtxt 中的输入名匹配
+        inputs = [
+            grpcclient.InferInput("feats", input_batch.shape, "FP32")
+        ]
+        inputs[0].set_data_from_numpy(input_batch)
+        
+        # 4. 设置请求的输出
+        #    输出名称 "embs" 必须与 speaker_model 的 config.pbtxt 中的输出名匹配
+        outputs = [grpcclient.InferRequestedOutput("embs")]
+        
+        # 5. 发送推理请求
+        response = await self.triton_client.infer(
+            model_name=self.model_name,
+            inputs=inputs,
+            outputs=outputs
+        )
+        
+        # 6. 解析结果
+        embeddings_batch = response.as_numpy("embs") # Shape: [2, embedding_dim]
+        emb1 = embeddings_batch[0]
+        emb2 = embeddings_batch[1]
+        
+        # 7. 在客户端计��相似度
+        similarity = self._calculate_cosine_similarity(emb1, emb2)
+        return similarity
+
+async def main():
+    parser = argparse.ArgumentParser(description="Triton client for speaker model (direct call).")
+    parser.add_argument('-v', '--verbose', action="store_true", default=False, help='Enable verbose output')
+    parser.add_argument('-u', '--url', type=str, default='localhost:8001', help='Inference server URL.')
+    # 注意：这里的 model_name 应该是 speaker_model
+    parser.add_argument('--model_name', default='speaker_model', help='The name of the speaker embedding model on Triton.')
+    parser.add_argument('--audio_file1', type=str, required=True, help='Path to first audio file')
+    parser.add_argument('--audio_file2', type=str, required=True, help='Path to second audio file')
+    
+    FLAGS = parser.parse_args()
+
+    client = TritonSpeakerClient(FLAGS.url, FLAGS.model_name, verbose=FLAGS.verbose)
+    
+    start_time = time.time()
+    try:
+        similarity = await client.compute_similarity(FLAGS.audio_file1, FLAGS.audio_file2)
+        elapsed = time.time() - start_time
+        print(f"Similarity: {similarity:.4f}, Time: {elapsed:.3f}s")
+    except Exception as e:
+        print(f"Error computing similarity: {e}", file=sys.stderr)
+        sys.exit(1)
+
+# 使用示例:
+# python speaker_client.py --audio_file1=/inspire/hdd/project/embodied-multimodality/public/yqzhang/infer_prompt/testset/audio/yanzi/yanzi1.wav --audio_file2=/inspire/hdd/project/embodied-multimodality/public/yqzhang/infer_prompt/testset/audio/yanzi/yanzi2.wav
+if __name__ == '__main__':
+    asyncio.run(main())

test.py

@@ -0,0 +1,1643 @@
+import json
+import re
+import os
+from typing import List, Dict, Tuple, Any
+import numpy as np
+from pathlib import Path
+import torch
+import torchaudio
+import torchaudio.functional as F
+import logging
+import wespeaker
+import shutil
+from datetime import datetime
+import multiprocessing as mp
+from functools import partial
+import math
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import random  # 添加random模块用于shuffle
+
+# 设置multiprocessing启动方式为spawn（CUDA兼容）
+mp.set_start_method('spawn', force=True)
+
+# 引用词对齐模块
+from alignment import AlignmentModel, batch_get_alignment_result
+
+class SpeakerSimilarityEvaluator:
+    """音色相似度评估器"""
+    
+    def __init__(self, device="cuda", 
+                 alignment_model_dir='/inspire/hdd/project/embodied-multimodality/public/yqzhang/auto_evaluation_new/models/mms_fa', 
+                 wespeaker_model_dir='/inspire/ssd/project/embodied-multimodality/public/zylin/speaker_embedding/wespeaker_pretrain/voxblink2_samresnet100_ft',
+                 output_dir="./evaluation_results",
+                 language="ZH",
+                 similarity_max_workers=8):
+        """初始化评估器"""
+        self.device = device
+        self.alignment_model_dir = alignment_model_dir
+        self.wespeaker_model_dir = wespeaker_model_dir
+        self.language = language.upper()  # 添加语言参数
+        self.similarity_max_workers = similarity_max_workers  # 相似度计算线程数
+        
+        # 先设置日志系统
+        logging.basicConfig(level=logging.INFO)
+        self.logger = logging.getLogger(__name__)
+        
+        # 设置输出目录结构
+        self.output_dir = Path(output_dir)
+        self.segments_dir = self.output_dir / "segments"  # 分割后的音频片段
+        self.prompts_dir = self.output_dir / "prompts"   # prompt音频的S1和S2片段
+        self.temp_dir = self.output_dir / "temp"         # 临时文件
+        self.results_dir = self.output_dir / "results"   # 评估结果
+        self.temp_results_dir = self.output_dir / "temp_results"  # 临时结果文件
+        self.alignment_dir = self.output_dir / "alignments"  # 对齐信息保存目录
+        
+        # 创建所有必要的目录
+        self._create_output_directories()
+        
+        # 在多进程环境中延迟模型初始化
+        self.alignment_model = None
+        self.similarity_model = None
+        
+        # 线程局部存储，用于线程安全的模型访问
+        self._thread_local = threading.local()
+        
+        # 记录运行信息
+        self.logger.info(f"评估结果将保存到: {self.output_dir}")
+        self.logger.info(f"对齐信息将保存到: {self.alignment_dir}")
+        self.logger.info(f"使用语言: {self.language}")
+    
+    def _create_output_directories(self):
+        """创建输出目录结构"""
+        for dir_path in [self.segments_dir, self.prompts_dir, self.temp_dir, 
+                        self.results_dir, self.temp_results_dir, self.alignment_dir]:
+            dir_path.mkdir(parents=True, exist_ok=True)
+    
+    def _get_safe_filename(self, text: str, max_length: int = 50) -> str:
+        """生成安全的文件名"""
+        # 移除特殊字符，只保留中文、英文、数字和基本符号
+        safe_text = re.sub(r'[^\u4e00-\u9fff\w\s]', '', text)
+        # 限制长度
+        if len(safe_text) > max_length:
+            safe_text = safe_text[:max_length]
+        # 替换空格为下划线
+        safe_text = safe_text.replace(' ', '_')
+        return safe_text if safe_text else "unnamed"
+    
+    def _clean_temp_files(self):
+        """清理临时文件，但保留临时目录"""
+        if self.temp_dir.exists():
+            # 只删除临时目录中的文件，不删除目录本身
+            for file_path in self.temp_dir.iterdir():
+                if file_path.is_file():
+                    try:
+                        file_path.unlink()
+                    except Exception as e:
+                        self.logger.warning(f"删除临时文件失败: {file_path}, 错误: {e}")
+        else:
+            # 如果临时目录不存在，重新创建
+            self.temp_dir.mkdir(parents=True, exist_ok=True)
+    
+    def _init_models_if_needed(self):
+        """延迟初始化模型（用于多进程环境）"""
+        # 初始化对齐模型 - 修正参数顺序
+        if self.alignment_model is None:
+            # 根据AlignmentModel的构造函数，应该是(device, model_dir)而不是(model_dir, device)
+            self.alignment_model = AlignmentModel(self.device, self.alignment_model_dir)
+        
+        # 初始化相似度模型
+        if self.similarity_model is None:
+            self._load_wespeaker_model(self.wespeaker_model_dir)
+    
+    def _is_english_text(self, text: str) -> bool:
+        """简单判断文本是否主要是英文"""
+        # 计算英文字符的比例
+        english_chars = sum(1 for c in text if c.isascii() and c.isalpha())
+        total_chars = sum(1 for c in text if c.isalpha())
+        
+        if total_chars == 0:
+            return False
+        
+        return english_chars / total_chars > 0.8  # 如果80%以上是英文字符，认为是英文
+    
+    def _detect_language_from_text(self, text: str) -> str:
+        """从文本内容检测语言"""
+        clean_text = self.remove_speaker_tags(text)
+        if self._is_english_text(clean_text):
+            return "EN"
+        else:
+            return "ZH"
+    
+    def save_alignment_info(self, alignment_data: Dict[str, Any], input_id: str, file_type: str = "output"):
+        """
+        保存对齐信息到单独的JSON文件
+        
+        Args:
+            alignment_data: 对齐信息数据
+            input_id: 输入ID
+            file_type: 文件类型 ("output", "prompt", "segment")
+        """
+        try:
+            safe_input_id = self._get_safe_filename(input_id)
+            alignment_filename = f"{safe_input_id}_{file_type}_alignment.json"
+            alignment_path = self.alignment_dir / alignment_filename
+            
+            # 添加元数据
+            alignment_info = {
+                'input_id': input_id,
+                'file_type': file_type,
+                'language': self.language,
+                'timestamp': datetime.now().isoformat(),
+                'alignment_data': alignment_data
+            }
+            
+            with open(alignment_path, 'w', encoding='utf-8') as f:
+                json.dump(alignment_info, f, ensure_ascii=False, indent=2)
+            
+            self.logger.info(f"对齐信息已保存: {alignment_path}")
+            return str(alignment_path)
+            
+        except Exception as e:
+            self.logger.error(f"保存对齐信息失败: {e}")
+            return None
+    
+    def save_detailed_alignment_info(self, alignments: List[Dict[str, Any]], 
+                                   text_segments: List[Dict[str, Any]], 
+                                   input_id: str, audio_path: str, 
+                                   original_text: str, processed_text: str):
+        """
+        保存详细的对齐信息，包括分段信息
+        
+        Args:
+            alignments: 对齐结果列表
+            text_segments: 文本分段信息
+            input_id: 输入ID
+            audio_path: 音频文件路径
+            original_text: 原始文本
+            processed_text: 处理后的文本
+        """
+        alignment_data = {
+            'original_text': original_text,
+            'processed_text': processed_text,
+            'audio_path': audio_path,
+            'language': self.language,
+            'total_alignments': len(alignments),
+            'total_segments': len(text_segments),
+            'alignments': alignments,
+            'text_segments': text_segments,
+            'segment_alignment_mapping': []
+        }
+        
+        # 建立文本段和对齐结果的映射关系
+        for segment in text_segments:
+            segment_mapping = {
+                'segment_id': segment.get('segment_id', 0),
+                'segment_text': segment.get('text', ''),
+                'speaker_label': segment.get('speaker_label', ''),
+                'start_time': segment.get('start_time', 0.0),
+                'end_time': segment.get('end_time', 0.0),
+                'corresponding_alignments': []
+            }
+            
+            # 找到对应的对齐项
+            segment_start = segment.get('start_time', 0.0)
+            segment_end = segment.get('end_time', 0.0)
+            
+            for i, align_item in enumerate(alignments):
+                align_start = align_item.get('start', 0.0)
+                align_end = align_item.get('end', 0.0)
+                
+                # 检查对齐项是否在当前段的时间范围内
+                if (align_start >= segment_start and align_end <= segment_end) or \
+                   (align_start < segment_end and align_end > segment_start):
+                    segment_mapping['corresponding_alignments'].append({
+                        'alignment_index': i,
+                        'transcript': align_item.get('transcript', ''),
+                        'start': align_start,
+                        'end': align_end,
+                        'score': align_item.get('score', 0.0) if 'score' in align_item else None
+                    })
+            
+            alignment_data['segment_alignment_mapping'].append(segment_mapping)
+        
+        return self.save_alignment_info(alignment_data, input_id, "detailed")
+    
+    def remove_speaker_tags(self, text: str) -> str:
+        """删除文本中的说话人标签[S1][S2]"""
+        return re.sub(r'\[S[12]\]', '', text).strip()
+    
+    def extract_speaker_segments(self, text: str) -> List[Dict[str, Any]]:
+        """提取文本中的说话人片段信息"""
+        segments = []
+        pattern = r'\[S([12])\]([^[]*)'
+        matches = re.findall(pattern, text)
+        
+        for speaker_id, content in matches:
+            segments.append({
+                'speaker': f'S{speaker_id}',
+                'content': content.strip()
+            })
+        return segments
+    
+    def replace_punctuation_with_comma(self, text: str, language: str = None) -> str:
+        """将所有标点符号替换为逗号，连续逗号只保留一个，根据语言选择正确的逗号类型"""
+        # 如果未指定语言，使用类的默认语言设置或自动检测
+        if language is None:
+            if hasattr(self, 'language'):
+                language = self.language
+            else:
+                language = self._detect_language_from_text(text)
+        
+        language = language.upper()
+        
+        # 根据语言选择逗号类型和处理策略
+        if language == "EN" or (language == "AUTO" and self._is_english_text(text)):
+            # 英文处理：先删除撇号，再替换其他标点符号
+            text = re.sub(r"'", '', text)  # 删除撇号（don't -> dont）
+            target_comma = ','  # 英文逗号
+            comma_pattern = r',+'  # 匹配连续英文逗号
+            # 更新正则表达式，不包含撇号
+            text = re.sub(r'[.,!?;:()\[\]<>\"…·，。；：！？（）【】《》""\\、]', target_comma, text)
+        else:
+            # 中文处理：包含撇号在替换范围内
+            target_comma = '，'  # 中文逗号
+            comma_pattern = r'，+'  # 匹配连续中文逗号
+            # 更新正则表达式以匹配更多的标点符号
+            text = re.sub(r'[.,!?;:()\[\]<>\'\"…·，。；：！？（）【】《》''""\\、]', target_comma, text)
+        
+        text = re.sub(comma_pattern, target_comma, text)
+        return text.strip(target_comma)
+    
+    def align_text_with_audio(self, text: str, audio_path: str, language=None) -> List[Dict[str, Any]]:
+        """
+        文本和音频的词对齐
+        返回每个词对应的音频时间段
+        """
+        # 确保模型已初始化
+        self._init_models_if_needed()
+        
+        # 如果未指定语言，使用类的默认语言设置或自动检测
+        if language is None:
+            if hasattr(self, 'language'):
+                language = self.language
+            else:
+                language = self._detect_language_from_text(text)
+        else:
+            language = language.upper()
+        
+        # 加载音频
+        waveform, sample_rate = torchaudio.load(audio_path)
+        
+        # 重采样到模型要求的采样率
+        if sample_rate != self.alignment_model.bundle.sample_rate:
+            waveform = F.resample(waveform, sample_rate, self.alignment_model.bundle.sample_rate)
+        
+        # 转换为单声道
+        if waveform.shape[0] > 1:
+            waveform = torch.mean(waveform, dim=0, keepdim=True)
+        
+        waveform = waveform.squeeze(0)  # 移除批次维度
+        
+        # 将音频移动到正确的设备
+        waveform = waveform.to(self.device)
+        
+        # 执行对齐
+        try:
+            alignment_results = batch_get_alignment_result(
+                self.alignment_model, 
+                [waveform], 
+                [text], 
+                [language]
+            )
+            if not alignment_results or not alignment_results[0]:
+                raise RuntimeError(f"对齐结果为空: {audio_path}")
+            return alignment_results[0]
+        except Exception as e:
+            self.logger.error(f"音频对齐失败: {audio_path}")
+            self.logger.error(f"错误详情: {e}")
+            raise RuntimeError(f"音频对齐失败，程序终止。文件: {audio_path}，错误: {e}")
+    
+    def split_audio_segment(self, audio_path: str, start_time: float, end_time: float, output_path: str):
+        """分割音频片段"""
+        waveform, sample_rate = torchaudio.load(audio_path)
+        
+        start_frame = int(start_time * sample_rate)
+        end_frame = int(end_time * sample_rate)
+        
+        segment = waveform[:, start_frame:end_frame]
+        
+        # 确保输出目录存在
+        os.makedirs(os.path.dirname(output_path), exist_ok=True)
+        
+        torchaudio.save(output_path, segment, sample_rate)
+        return output_path
+    
+    def concatenate_audio_files(self, audio_files: List[str], output_path: str):
+        """拼接多个音频文件"""
+        if not audio_files:
+            return
+        
+        waveforms = []
+        sample_rate = None
+        
+        for audio_file in audio_files:
+            if os.path.exists(audio_file):
+                waveform, sr = torchaudio.load(audio_file)
+                if sample_rate is None:
+                    sample_rate = sr
+                elif sr != sample_rate:
+                    waveform = F.resample(waveform, sr, sample_rate)
+                waveforms.append(waveform)
+        
+        if waveforms:
+            concatenated = torch.cat(waveforms, dim=1)
+            os.makedirs(os.path.dirname(output_path), exist_ok=True)
+            torchaudio.save(output_path, concatenated, sample_rate)
+    
+    def split_audio_by_speaker(self, prompt_text: str, prompt_audio: str, audio_id: str) -> Tuple[str, str]:
+        """
+        根据说话人标签分割prompt音频
+        返回S1和S2的音频片段路径
+        """
+        # 1. 提取说话人片段
+        speaker_segments = self.extract_speaker_segments(prompt_text)
+        
+        # 2. 删除标签后进行词对齐 - 如果失败则直接抛出异常
+        clean_text = self.remove_speaker_tags(prompt_text)
+        
+        # 检测语言或使用设置的语言
+        alignment_language = self.language
+        if alignment_language == "AUTO":
+            alignment_language = self._detect_language_from_text(clean_text)
+            
+        alignments = self.align_text_with_audio(clean_text, prompt_audio, alignment_language)
+        
+        # 保存prompt对齐信息
+        prompt_alignment_data = {
+            'original_text': prompt_text,
+            'clean_text': clean_text,
+            'audio_path': prompt_audio,
+            'language': alignment_language,
+            'speaker_segments': speaker_segments,
+            'alignments': alignments
+        }
+        self.save_alignment_info(prompt_alignment_data, audio_id, "prompt")
+        
+        # 3. 根据对齐结果分割音频
+        s1_segments = []
+        s2_segments = []
+        
+        # 为每个说话人片段找到对应的时间段
+        text_pos = 0
+        for seg in speaker_segments:
+            seg_text = seg['content'].strip()
+            seg_length = len(seg_text)
+            
+            # 找到这个片段在对齐结果中的起始和结束
+            start_time = None
+            end_time = None
+            
+            current_pos = 0
+            for align_item in alignments:
+                item_text = align_item['transcript']
+                item_length = len(item_text)
+                
+                if current_pos >= text_pos and current_pos < text_pos + seg_length:
+                    if start_time is None:
+                        start_time = align_item['start']
+                    end_time = align_item['end']
+                
+                current_pos += item_length
+            
+            if start_time is not None and end_time is not None:
+                if seg['speaker'] == 'S1':
+                    s1_segments.append((start_time, end_time))
+                else:
+                    s2_segments.append((start_time, end_time))
+            
+            text_pos += seg_length
+        
+        # 4. 分割并拼接音频片段
+        safe_audio_id = self._get_safe_filename(audio_id)
+        prompts1_path = str(self.prompts_dir / f"{safe_audio_id}_s1.wav")
+        prompts2_path = str(self.prompts_dir / f"{safe_audio_id}_s2.wav")
+        
+        # 分割S1的所有片段
+        if s1_segments:
+            s1_temp_segments = []
+            for i, (start, end) in enumerate(s1_segments):
+                temp_path = str(self.temp_dir / f"{safe_audio_id}_s1_temp_{i}.wav")
+                self.split_audio_segment(prompt_audio, start, end, temp_path)
+                s1_temp_segments.append(temp_path)
+            
+            # 拼接S1片段
+            self.concatenate_audio_files(s1_temp_segments, prompts1_path)
+        
+        # 分割S2的所有片段
+        if s2_segments:
+            s2_temp_segments = []
+            for i, (start, end) in enumerate(s2_segments):
+                temp_path = str(self.temp_dir / f"{safe_audio_id}_s2_temp_{i}.wav")
+                self.split_audio_segment(prompt_audio, start, end, temp_path)
+                s2_temp_segments.append(temp_path)
+            
+            # 拼接S2片段
+            self.concatenate_audio_files(s2_temp_segments, prompts2_path)
+        
+        return prompts1_path, prompts2_path
+    
+    def map_text_segments_to_speakers(self, original_text: str) -> List[Dict[str, Any]]:
+        """
+        将原始文本按说话人和标点符号同时分割，保持映射关系
+        支持英文单词级别的处理
+        """
+        segments = []
+        pattern = r'\[S([12])\]([^[]*)'
+        matches = re.findall(pattern, original_text)
+        
+        # 检测语言或使用设置的语言
+        alignment_language = self.language
+        if alignment_language == "AUTO":
+            alignment_language = self._detect_language_from_text(original_text)
+        
+        segment_id = 0
+        for speaker_id, content in matches:
+            speaker = f'S{speaker_id}'
+            clean_content = content.strip()
+            comma_content = self.replace_punctuation_with_comma(clean_content, alignment_language)
+            
+            # 根据语言选择正确的逗号分割
+            if alignment_language == "EN" or (alignment_language == "AUTO" and self._is_english_text(clean_content)):
+                # 英文：按��文逗号分割，保持单词完整性
+                parts = [part.strip() for part in comma_content.split(',') if part.strip()]
+            else:
+                # 中文：按中文逗号分割
+                parts = [part.strip() for part in comma_content.split('，') if part.strip()]
+            
+            for part in parts:
+                if part.strip():
+                    segments.append({
+                        'segment_id': segment_id,
+                        'text': part.strip(),
+                        'speaker_label': speaker,
+                        'original_speaker_content': clean_content
+                    })
+                    segment_id += 1
+        
+        return segments
+    
+    def split_output_audio_by_comma(self, text: str, output_audio: str, audio_id: str) -> List[Dict[str, Any]]:
+        """
+        根据逗号分割输出音频，返回每小段的信息 - 基于词对齐结果中的标点符号划分句子
+        """
+        # 1. 获取文本片段和对应的说话人（用于获取speaker标签）
+        text_segments = self.map_text_segments_to_speakers(text)
+        
+        # 2. 删除标签并替换标点符号
+        clean_text = self.remove_speaker_tags(text)
+        
+        # 3. 检测语言或使用设置的语言
+        alignment_language = self.language
+        if alignment_language == "AUTO":
+            alignment_language = self._detect_language_from_text(clean_text)
+        
+        # 使用检测到的语言替换标点符号
+        comma_text = self.replace_punctuation_with_comma(clean_text, alignment_language)
+        
+        # 4. 词对齐 - 如果失败则直接抛出异常
+        alignments = self.align_text_with_audio(comma_text, output_audio, alignment_language)
+        
+        # 5. 根据标点符号划分句子
+        segments = []
+        safe_audio_id = self._get_safe_filename(audio_id)
+        
+        # 确定标点符号（根据语言选择，英文不包含撇号）
+        if alignment_language == "EN" or (alignment_language == "AUTO" and self._is_english_text(clean_text)):
+            punctuation_chars = set([',', '.', '!', '?', ';', ':'])  # 不包含撇号
+        else:
+            punctuation_chars = set(['，', '。', '！', '？', '；', '：'])
+        
+        # 顺序扫描对齐结果，根据标点符号划分句子
+        sentence_start_idx = 0
+        sentence_alignments = []
+        segment_id = 0
+        
+        for i, align_item in enumerate(alignments):
+            transcript = align_item['transcript']
+            sentence_alignments.append(align_item)
+            
+            # 检查是否包含标点符号（句子结束标志）
+            has_punctuation = any(punct in transcript for punct in punctuation_chars)
+            
+            if has_punctuation or i == len(alignments) - 1:  # 遇到标点符号或最后一个词
+                # 创建句子片段
+                if sentence_alignments:
+                    # 获取句子的开始和结束时间
+                    start_time = sentence_alignments[0]['start']
+                    end_time = sentence_alignments[-1]['end']
+                    
+                    # 构建句子文本（去除标点符号）
+                    sentence_text_parts = []
+                    for align in sentence_alignments:
+                        # 根据语言选择不同的清理策略
+                        if alignment_language == "EN" or (alignment_language == "AUTO" and self._is_english_text(clean_text)):
+                            # 英文：去除标点符号，但保留撇号已被删除的单词
+                            clean_transcript = align['transcript'].rstrip(',.!?;:')
+                        else:
+                            # 中文：去除中文标点符号
+                            clean_transcript = align['transcript'].rstrip('，。！？；：')
+                        
+                        if clean_transcript.strip():
+                            sentence_text_parts.append(clean_transcript)
+                    
+                    # 根据语言选择连接方式
+                    if alignment_language == "EN" or (alignment_language == "AUTO" and self._is_english_text(clean_text)):
+                        sentence_text = ' '.join(sentence_text_parts).strip()  # 英文用空格连接
+                    else:
+                        sentence_text = ''.join(sentence_text_parts).strip()  # 中文直接连接
+                    
+                    if sentence_text:  # 只有非空句子才处理
+                        # 确定说话人标签（从原始text_segments中获取，如果可能的话）
+                        speaker_label = "S1"  # 默认
+                        if segment_id < len(text_segments):
+                            speaker_label = text_segments[segment_id]['speaker_label']
+                        elif text_segments:
+                            # 如果超出范围，使用最后一个片段的speaker
+                            speaker_label = text_segments[-1]['speaker_label']
+                        
+                        # 生成音频文件路径
+                        safe_text = self._get_safe_filename(sentence_text, 30)
+                        audio_path = str(self.segments_dir / f"{safe_audio_id}_segment_{segment_id:03d}_{safe_text}.wav")
+                        
+                        # 分割音频
+                        try:
+                            self.split_audio_segment(output_audio, start_time, end_time, audio_path)
+                        except Exception as e:
+                            self.logger.error(f"分割音频失败: {e}")
+                            # 使用默认时间间隔
+                            start_time = segment_id * 1.0
+                            end_time = (segment_id + 1) * 1.0
+                            self.split_audio_segment(output_audio, start_time, end_time, audio_path)
+                        
+                        # 创建segment
+                        segment = {
+                            'segment_id': segment_id,
+                            'text': sentence_text,
+                            'speaker_label': speaker_label,
+                            'original_speaker_content': sentence_text,  # 这里简化处理
+                            'audio_path': audio_path,
+                            'start_time': start_time,
+                            'end_time': end_time
+                        }
+                        
+                        segments.append(segment)
+                        
+                        self.logger.info(f"句子 {segment_id}: '{sentence_text}' ({speaker_label}) -> {start_time:.3f}-{end_time:.3f}s")
+                        segment_id += 1
+                
+                # 重置为下一个句子
+                sentence_alignments = []
+                sentence_start_idx = i + 1
+        
+        # 保存详细的对齐信息
+        self.save_detailed_alignment_info(
+            alignments, segments, audio_id, output_audio, text, comma_text
+        )
+        
+        self.logger.info(f"总共分割出 {len(segments)} 个句子片段")
+        return segments
+    
+    def _get_thread_local_similarity_model(self):
+        """获取线程局部的相似度模型实例（线程安全）"""
+        if not hasattr(self._thread_local, 'similarity_model'):
+            # 为当前线程创建独立的模型实例
+            self._thread_local.similarity_model = self._create_similarity_model()
+        return self._thread_local.similarity_model
+    
+    def _create_similarity_model(self):
+        """创建新的相似度模型实例"""
+        try:
+            import wespeaker
+            
+            # 使用与主模型相同的加载逻辑
+            local_model_path = '/inspire/ssd/project/embodied-multimodality/public/zylin/speaker_embedding/wespeaker_pretrain/voxblink2_samresnet100_ft'
+            
+            try:
+                model = wespeaker.load_model_local(local_model_path)
+                return model
+            except Exception as e:
+                self.logger.warning(f"加载指定本地模型失败: {e}")
+            
+            # 回退方案
+            if os.path.exists(self.wespeaker_model_dir):
+                try:
+                    model = wespeaker.load_model_local(self.wespeaker_model_dir)
+                    return model
+                except Exception as e:
+                    self.logger.warning(f"加载传入本地模型失败: {e}")
+            
+            # 最终回退到预训练模型
+            try:
+                model = wespeaker.load_model('chinese')
+                return model
+            except Exception as e:
+                model = wespeaker.load_model('english')
+                return model
+                
+        except Exception as e:
+            self.logger.error(f"创建相似度模型失败: {e}")
+            raise
+    
+    def calculate_voice_similarity_thread_safe(self, audio1_path: str, audio2_path: str) -> float:
+        """
+        线程安全的音色相似度计算
+        对于过短的音频片段，通过复制来达到最小长度要求
+        """
+        try:
+            if not os.path.exists(audio1_path) or not os.path.exists(audio2_path):
+                self.logger.warning(f"Audio file not found: {audio1_path} or {audio2_path}")
+                return None
+            
+            # 获取线程局部的模型实例
+            similarity_model = self._get_thread_local_similarity_model()
+            
+            # 检查并处理音频文件长度
+            def process_audio_for_similarity(audio_path, min_duration=0.1):
+                """
+                处理音频文件，如果过短则复制到满足最小长度要求
+                返回处理后的音频路径和是否为临时文件的标志
+                """
+                try:
+                    waveform, sample_rate = torchaudio.load(audio_path)
+                    duration = waveform.shape[1] / sample_rate
+                    
+                    if duration >= min_duration:
+                        # 音频长度足够，直接返回原路径
+                        return audio_path, False
+                    
+                    # 音频过短，需要复制
+                    repeat_times = math.ceil(min_duration / duration)
+                    thread_id = threading.get_ident()
+                    
+                    # 复制音频
+                    repeated_waveform = waveform.repeat(1, repeat_times)
+                    
+                    # 生成临时文件路径（包含线程ID避免冲突）
+                    temp_filename = f"temp_{thread_id}_{os.path.basename(audio_path)}"
+                    temp_path = str(self.temp_dir / temp_filename)
+                    
+                    # 保存复制后的音频
+                    torchaudio.save(temp_path, repeated_waveform, sample_rate)
+                    
+                    return temp_path, True
+                    
+                except Exception as e:
+                    self.logger.error(f"处理音频文件失败: {audio_path}, 错误: {e}")
+                    return audio_path, False
+            
+            # 处理两个音频文件
+            processed_audio1, is_temp1 = process_audio_for_similarity(audio1_path)
+            processed_audio2, is_temp2 = process_audio_for_similarity(audio2_path)
+            
+            # 计算相似度
+            similarity = similarity_model.compute_similarity(processed_audio1, processed_audio2)
+            
+            # 清理临时文件
+            if is_temp1 and os.path.exists(processed_audio1):
+                try:
+                    os.remove(processed_audio1)
+                except Exception as e:
+                    self.logger.warning(f"删除临时文件失败: {processed_audio1}, 错误: {e}")
+            
+            if is_temp2 and os.path.exists(processed_audio2):
+                try:
+                    os.remove(processed_audio2)
+                except Exception as e:
+                    self.logger.warning(f"删除临时文件失败: {processed_audio2}, 错误: {e}")
+            
+            return float(similarity)
+            
+        except Exception as e:
+            # 检查是否是窗口大小错误或其他计算错误
+            if "choose a window size" in str(e) or "window size" in str(e):
+                self.logger.warning(f"音频片段仍然过短，无法计算相似度: {audio1_path} vs {audio2_path}")
+                return None
+            else:
+                self.logger.error(f"Failed to compute similarity between {audio1_path} and {audio2_path}: {e}")
+                return None
+    
+    def calculate_segment_similarities_parallel(self, output_segments: List[Dict[str, Any]], 
+                                              prompts1_path: str, prompts2_path: str) -> List[Dict[str, Any]]:
+        """
+        并行计算所有segments的相似度
+        Args:
+            output_segments: 音频segments列表
+            prompts1_path: S1 prompt音频路径  
+            prompts2_path: S2 prompt音频路径
+        Returns:
+            包含相似度信息的segment列表
+        """
+        
+        def calculate_single_segment_similarity(segment):
+            """计算单个segment与两个prompts的相似度"""
+            try:
+                # 使用线程安全的相似度计算方法
+                sim1 = self.calculate_voice_similarity_thread_safe(segment['audio_path'], prompts1_path)
+                sim2 = self.calculate_voice_similarity_thread_safe(segment['audio_path'], prompts2_path)
+                
+                return {
+                    'segment': segment,
+                    'sim1': sim1,
+                    'sim2': sim2,
+                    'success': True
+                }
+            except Exception as e:
+                self.logger.error(f"计算segment {segment['segment_id']} 相似度失败: {e}")
+                return {
+                    'segment': segment,
+                    'sim1': None,
+                    'sim2': None,
+                    'success': False
+                }
+        
+        # 使用线程池并行处理所有segments
+        self.logger.info(f"开始并行计算 {len(output_segments)} 个segments的相似度，使用 {self.similarity_max_workers} 个线程")
+        
+        results = []
+        with ThreadPoolExecutor(max_workers=self.similarity_max_workers) as executor:
+            # 提交所有segment任务
+            future_to_segment = {
+                executor.submit(calculate_single_segment_similarity, segment): segment 
+                for segment in output_segments
+            }
+            
+            # 收集结果（保持原有顺序）
+            segment_to_result = {}
+            completed_count = 0
+            for future in as_completed(future_to_segment):
+                result = future.result()
+                segment_id = result['segment']['segment_id']
+                segment_to_result[segment_id] = result
+                completed_count += 1
+                
+                # 每完成10个segment报告一次进度
+                if completed_count % 10 == 0 or completed_count == len(output_segments):
+                    self.logger.info(f"相似度计算进度: {completed_count}/{len(output_segments)}")
+            
+            # 按segment_id顺序返回结果
+            for segment in output_segments:
+                segment_id = segment['segment_id']
+                if segment_id in segment_to_result:
+                    results.append(segment_to_result[segment_id])
+        
+        return results
+
+    def evaluate_single_input(self, data: Dict[str, Any], input_id: str = None) -> Dict[str, Any]:
+        """评估单个输入的音色相似度"""
+        
+        # 生成输入ID
+        if input_id is None:
+            input_id = f"input_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+        
+        self.logger.info(f"开始评估输入: {input_id}，使用语言: {self.language}")
+        
+        # 1. 获取或分割prompt音频
+        prompts1_path, prompts2_path = self.get_or_split_prompt_audio(data, f"{input_id}_prompt")
+        
+        # 2. 分割output音频（这里会保存详细对齐信息）
+        output_segments = self.split_output_audio_by_comma(data['text'], data['output_audio'], f"{input_id}_output")
+        
+        # 3. 并行计算每小段的相似度
+        similarity_results = self.calculate_segment_similarities_parallel(
+            output_segments, prompts1_path, prompts2_path
+        )
+        
+        # 4. 处理相似度结果
+        segment_results = []
+        correct_predictions = 0
+        total_segments = 0  # 只计算有效段数
+        label_similarities = []  # 每小段与其标签的相似度
+        skipped_segments = 0  # 跳过的段数
+        
+        for sim_result in similarity_results:
+            segment = sim_result['segment']
+            sim1 = sim_result['sim1']
+            sim2 = sim_result['sim2']
+            
+            # 如果任一相似度为None（音频过短或计算失败），跳过该段
+            if sim1 is None or sim2 is None:
+                skipped_segments += 1
+                self.logger.info(f"跳过段 {segment['segment_id']}: 相似度计算失败")
+                continue
+            
+            # 只有有效段才参与计算
+            total_segments += 1
+            
+            # 判断实际音色
+            predicted_speaker = 'S1' if sim1 > sim2 else 'S2'
+            actual_speaker = segment['speaker_label']
+            is_correct = predicted_speaker == actual_speaker
+            
+            if is_correct:
+                correct_predictions += 1
+            
+            # 计算与标签的相似度
+            if actual_speaker == 'S1':
+                label_similarity = sim1
+            else:
+                label_similarity = sim2
+            label_similarities.append(label_similarity)
+            
+            segment_result = {
+                'segment_id': segment['segment_id'],
+                'text': segment['text'],
+                'speaker_label': actual_speaker,
+                'predicted_speaker': predicted_speaker,
+                'sim1': sim1,
+                'sim2': sim2,
+                'label_similarity': label_similarity,
+                'is_correct': is_correct,
+                'audio_path': segment['audio_path'],
+                'start_time': segment.get('start_time', 0.0),
+                'end_time': segment.get('end_time', 1.0)
+            }
+            segment_results.append(segment_result)
+        
+        # 4. 计算整体指标（只基于有效段）
+        accuracy = correct_predictions / total_segments if total_segments > 0 else 0.0
+        average_similarity = np.mean(label_similarities) if label_similarities else 0.0
+        
+        # 5. 保存评估结果的对齐信息摘要
+        evaluation_alignment_summary = {
+            'input_id': input_id,
+            'language': self.language,
+            'prompt_alignment_files': [
+                f"{self._get_safe_filename(f'{input_id}_prompt')}_prompt_alignment.json"
+            ],
+            'output_alignment_file': f"{self._get_safe_filename(f'{input_id}_output')}_detailed_alignment.json",
+            'total_segments': total_segments,
+            'total_alignments_processed': len(output_segments),
+            'alignment_success_rate': total_segments / len(output_segments) if output_segments else 0.0
+        }
+        self.save_alignment_info(evaluation_alignment_summary, input_id, "summary")
+        
+        result = {
+            'input_id': input_id,
+            'language': self.language,
+            'input_data': data,  # 保存原始输入数据
+            'prompts1_path': prompts1_path,
+            'prompts2_path': prompts2_path,
+            'segments': segment_results,
+            'accuracy': accuracy,
+            'average_similarity': average_similarity,
+            'total_segments': total_segments,  # 有效段数
+            'correct_predictions': correct_predictions,
+            'skipped_segments': skipped_segments,  # 跳过的段数
+            'original_total_segments': len(output_segments),  # 原始总段数
+            'alignment_files': {
+                'summary': f"{self._get_safe_filename(input_id)}_summary_alignment.json",
+                'output_detailed': f"{self._get_safe_filename(f'{input_id}_output')}_detailed_alignment.json",
+                'prompt': f"{self._get_safe_filename(f'{input_id}_prompt')}_prompt_alignment.json"
+            },
+            'timestamp': datetime.now().isoformat()
+        }
+        
+        self.logger.info(f"完成评估输入: {input_id}, 语言: {self.language}, 有效段: {total_segments}/{len(output_segments)}, 跳过: {skipped_segments}, 准确率: {accuracy:.3f}, 平均相似度: {average_similarity:.3f}")
+        
+        return result
+
+    def save_results_to_jsonl(self, results: List[Dict[str, Any]], filename: str = None):
+        """保存结果到JSONL文件"""
+        if filename is None:
+            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+            filename = f"speaker_similarity_results_{self.language.lower()}_{timestamp}.jsonl"
+        
+        output_path = self.results_dir / filename
+        
+        with open(output_path, 'w', encoding='utf-8') as f:
+            for result in results:
+                f.write(json.dumps(result, ensure_ascii=False) + '\n')
+        
+        return str(output_path)
+    
+    def save_summary_report(self, results: List[Dict[str, Any]], filename: str = None):
+        """保存汇总报告"""
+        if filename is None:
+            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+            filename = f"evaluation_summary_{self.language.lower()}_{timestamp}.json"
+        
+        summary_path = self.results_dir / filename
+        
+        # 计算总体统计
+        total_accuracy = np.mean([r['accuracy'] for r in results])
+        total_avg_similarity = np.mean([r['average_similarity'] for r in results])
+        total_segments = sum([r['total_segments'] for r in results])
+        total_correct = sum([r['correct_predictions'] for r in results])
+        
+        summary = {
+            'evaluation_summary': {
+                'language': self.language,
+                'total_inputs': len(results),
+                'total_segments': total_segments,
+                'total_correct_predictions': total_correct,
+                'overall_accuracy': total_accuracy,
+                'overall_average_similarity': total_avg_similarity,
+                'evaluation_timestamp': datetime.now().isoformat(),
+                'output_directory': str(self.output_dir),
+                'alignment_directory': str(self.alignment_dir)
+            },
+            'per_input_results': [
+                {
+                    'input_id': r['input_id'],
+                    'language': r.get('language', self.language),
+                    'accuracy': r['accuracy'],
+                    'average_similarity': r['average_similarity'],
+                    'total_segments': r['total_segments'],
+                    'correct_predictions': r['correct_predictions'],
+                    'output_audio_path': r['input_data']['output_audio'],
+                    'alignment_files': r.get('alignment_files', {})
+                }
+                for r in results
+            ]
+        }
+        
+        with open(summary_path, 'w', encoding='utf-8') as f:
+            json.dump(summary, f, ensure_ascii=False, indent=2)
+        
+        return str(summary_path)
+    
+    def process_batch_from_jsonl_parallel(self, jsonl_path: str, 
+                                        processes_per_gpu: int = 16,
+                                        results_filename: str = None,
+                                        shuffle_data: bool = True):
+        """从JSONL文件并行批量处理输入数据"""
+        # 加载数据
+        input_data = self.load_data_from_jsonl(jsonl_path)
+        
+        if not input_data:
+            self.logger.error("没有有效的输入数据")
+            return []
+        
+        # 对数据进行shuffle，使分配更均匀
+        if shuffle_data:
+            random.shuffle(input_data)
+            self.logger.info(f"已对 {len(input_data)} 条数据进行随机shuffle")
+        
+        return self.process_batch_parallel(input_data, processes_per_gpu, results_filename)
+
+    def process_batch_from_jsonl(self, jsonl_path: str, results_filename: str = None):
+        """从JSONL文件批量处理输入数据（单进程版本）"""
+        # 加载数据
+        input_data = self.load_data_from_jsonl(jsonl_path)
+        
+        if not input_data:
+            self.logger.error("没有有效的输入数据")
+            return []
+        
+        return self.process_batch_from_data(input_data, results_filename)
+
+    def process_batch_from_data(self, input_data: List[Dict[str, Any]], results_filename: str = None):
+        """处理数据列表（单进程版本，用于兼容），支持增量写入"""
+        # 准备结果文件
+        if results_filename is None:
+            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+            results_filename = f"speaker_similarity_results_{self.language.lower()}_{timestamp}.jsonl"
+        
+        results_path = self.results_dir / results_filename
+        
+        # 如果文件已存在，删除它（重新开始）
+        if results_path.exists():
+            results_path.unlink()
+        
+        results = []
+        
+        self.logger.info(f"开始处理 {len(input_data)} 个输入，使用语言: {self.language}...")
+        
+        for i, data in enumerate(input_data):
+            input_id = f"input_{i+1:03d}"
+            print(f"处理第{i+1}/{len(input_data)}个输入: {input_id}，语言: {self.language}")
+            
+            try:
+                result = self.evaluate_single_input(data, input_id=input_id)
+                results.append(result)
+                
+                # 增量写入结果
+                self.append_result_to_jsonl(result, str(results_path))
+                
+            except Exception as e:
+                self.logger.error(f"处理输入{input_id}时出错: {e}")
+                continue
+        
+        if not results:
+            self.logger.error("没有成功处理的输入")
+            return []
+        
+        # 保存汇总报告
+        summary_path = self.save_summary_report(results)
+        
+        # 清理临时文件
+        self._clean_temp_files()
+        
+        # 打印总体统计
+        total_accuracy = np.mean([r['accuracy'] for r in results])
+        total_avg_similarity = np.mean([r['average_similarity'] for r in results])
+        
+        print(f"\n=== 评估完成 ===")
+        print(f"使用语言: {self.language}")
+        print(f"总体准确率: {total_accuracy:.3f}")
+        print(f"总体平均相似度: {total_avg_similarity:.3f}")
+        print(f"详细结果已保存到: {results_path}")
+        print(f"汇总报告已保存到: {summary_path}")
+        print(f"对齐信息已保存到: {self.alignment_dir}")
+        print(f"所有中间文件保存在: {self.output_dir}")
+        
+        return results
+
+    def _load_wespeaker_model(self, wespeaker_model_dir):
+        """加载wespeaker模型"""
+        try:
+            import wespeaker
+            
+            # 使用load_model_local方法加载本地模型
+            # 根据你提供的参考，使用你指定的模型路径
+            local_model_path = '/inspire/ssd/project/embodied-multimodality/public/zylin/speaker_embedding/wespeaker_pretrain/voxblink2_samresnet100_ft'
+            
+            try:
+                self.similarity_model = wespeaker.load_model_local(local_model_path)
+                self.logger.info(f"成功加载本地wespeaker模型: {local_model_path}")
+                return
+            except Exception as e:
+                self.logger.warning(f"加载指定本地模型失败: {e}")
+            
+            # 回退方案1: 尝试使用传入的模型目录
+            if os.path.exists(wespeaker_model_dir):
+                try:
+                    self.similarity_model = wespeaker.load_model_local(wespeaker_model_dir)
+                    self.logger.info(f"成功加载传入的本地wespeaker模型: {wespeaker_model_dir}")
+                    return
+                except Exception as e:
+                    self.logger.warning(f"加载传入本地模型失败: {e}")
+            
+            # 回退方案2: 使用预训练的中文模型
+            try:
+                self.similarity_model = wespeaker.load_model('chinese')
+                self.logger.info("回退到wespeaker预训练中文模型")
+                return
+            except Exception as e:
+                self.logger.warning(f"加载预训练中文模型失败: {e}")
+            
+            # 回退方案3: 使用预训练的英文模型
+            try:
+                self.similarity_model = wespeaker.load_model('english')
+                self.logger.info("回退到wespeaker预训练英文模型")
+                return
+            except Exception as e:
+                self.logger.error(f"加载英文模型也失败: {e}")
+            
+            # 如果所有方法都失败，抛出异常
+            raise Exception("无法加载任何wespeaker模型")
+            
+        except ImportError:
+            raise ImportError("请安装wespeaker: pip install git+https://github.com/wenet-e2e/wespeaker.git")
+        except Exception as e:
+            self.logger.error(f"加载wespeaker模型失败: {e}")
+            raise
+
+    def load_data_from_jsonl(self, jsonl_path: str) -> List[Dict[str, Any]]:
+        """从JSONL文件加载数据"""
+        data = []
+        try:
+            with open(jsonl_path, 'r', encoding='utf-8') as f:
+                for line_num, line in enumerate(f, 1):
+                    line = line.strip()
+                    if line:
+                        try:
+                            item = json.loads(line)
+                            # 验证必要字段
+                            required_fields = ['text', 'output_audio']
+                            for field in required_fields:
+                                if field not in item:
+                                    self.logger.error(f"第{line_num}行缺少必要字段: {field}")
+                                    continue
+                            
+                            # 验证音频路径模式：要么有prompt_audio和prompt_text，要么有分别的speaker音频文件
+                            has_combined_prompt = 'prompt_audio' in item and 'prompt_text' in item
+                            has_separate_prompts = ('prompt_audio_speaker1' in item and 
+                                                  'prompt_text_speaker1' in item and
+                                                  'prompt_audio_speaker2' in item and 
+                                                  'prompt_text_speaker2' in item)
+                            
+                            if not (has_combined_prompt or has_separate_prompts):
+                                self.logger.error(f"第{line_num}行：需要提供prompt_audio+prompt_text或者分别的speaker音频文件")
+                                continue
+                            
+                            data.append(item)
+                            
+                        except json.JSONDecodeError as e:
+                            self.logger.error(f"第{line_num}行JSON解析错误: {e}")
+                            continue
+                            
+            self.logger.info(f"从{jsonl_path}成功加载{len(data)}条数据")
+            return data
+            
+        except FileNotFoundError:
+            self.logger.error(f"JSONL文件不存在: {jsonl_path}")
+            return []
+        except Exception as e:
+            self.logger.error(f"读取JSONL文件失败: {e}")
+            return []
+
+    @staticmethod
+    def get_gpu_count():
+        """获取可用GPU数量"""
+        if torch.cuda.is_available():
+            return torch.cuda.device_count()
+        return 0
+    
+    @staticmethod
+    def split_data_by_gpu(data: List[Dict[str, Any]], num_gpus: int) -> List[List[Dict[str, Any]]]:
+        """根据GPU数量分割数据"""
+        if num_gpus == 0:
+            return [data]
+        
+        chunk_size = math.ceil(len(data) / num_gpus)
+        gpu_chunks = []
+        
+        for i in range(num_gpus):
+            start_idx = i * chunk_size
+            end_idx = min((i + 1) * chunk_size, len(data))
+            if start_idx < len(data):
+                gpu_chunks.append(data[start_idx:end_idx])
+        
+        return gpu_chunks
+    
+    @staticmethod
+    def split_data_by_processes(data: List[Dict[str, Any]], num_processes: int) -> List[List[Dict[str, Any]]]:
+        """根据进程数量分割数据"""
+        if num_processes <= 1:
+            return [data]
+        
+        chunk_size = math.ceil(len(data) / num_processes)
+        process_chunks = []
+        
+        for i in range(num_processes):
+            start_idx = i * chunk_size
+            end_idx = min((i + 1) * chunk_size, len(data))
+            if start_idx < len(data):
+                process_chunks.append(data[start_idx:end_idx])
+        
+        return process_chunks
+
+    def append_result_to_jsonl(self, result: Dict[str, Any], filepath: str):
+        """增量写入结果到JSONL文件"""
+        os.makedirs(os.path.dirname(filepath), exist_ok=True)
+        with open(filepath, 'a', encoding='utf-8') as f:
+            f.write(json.dumps(result, ensure_ascii=False) + '\n')
+            f.flush()  # 强制刷新缓冲区
+
+    def merge_temp_results(self, temp_files: List[str], final_path: str):
+        """合并临时结果文件"""
+        all_results = []
+        
+        for temp_file in temp_files:
+            if os.path.exists(temp_file):
+                try:
+                    with open(temp_file, 'r', encoding='utf-8') as f:
+                        for line in f:
+                            line = line.strip()
+                            if line:
+                                result = json.loads(line)
+                                all_results.append(result)
+                except Exception as e:
+                    self.logger.error(f"读取临时文件失败: {temp_file}, 错误: {e}")
+        
+        # 写入最终文件
+        with open(final_path, 'w', encoding='utf-8') as f:
+            for result in all_results:
+                f.write(json.dumps(result, ensure_ascii=False) + '\n')
+        
+        return all_results
+
+    def process_batch_parallel(self, input_data: List[Dict[str, Any]], 
+                             processes_per_gpu: int = 8,  # 降低进程数
+                             results_filename: str = None,
+                             shuffle_data: bool = True):
+        """并行批量处理输入数据"""
+        # 1. ���查GPU数量
+        num_gpus = self.get_gpu_count()
+        if num_gpus == 0:
+            self.logger.warning("未检测到GPU，将使用CPU单进程处理")
+            return self.process_batch_from_data(input_data, results_filename)
+        
+        # 限制每个GPU的进程数，避免CUDA内存冲突
+        max_processes_per_gpu = min(processes_per_gpu, 16)
+        self.logger.info(f"检测到 {num_gpus} 个GPU，每个GPU将使用 {max_processes_per_gpu} 个进程")
+        
+        # 2. 对数据进行shuffle（如果还没有shuffle过）
+        shuffled_data = input_data.copy()
+        if shuffle_data:
+            random.shuffle(shuffled_data)
+            self.logger.info(f"已对 {len(shuffled_data)} 条数据进行随机shuffle以平衡GPU负载")
+        
+        # 3. 按GPU分割数据
+        gpu_chunks = self.split_data_by_gpu(shuffled_data, num_gpus)
+        
+        # 打印每个GPU分配到的数据量
+        for gpu_id, gpu_data in enumerate(gpu_chunks):
+            if gpu_data:
+                self.logger.info(f"GPU {gpu_id}: 分配到 {len(gpu_data)} 条数据")
+        
+        # 4. 准备结果文件路径
+        if results_filename is None:
+            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+            results_filename = f"speaker_similarity_results_{self.language.lower()}_{timestamp}.jsonl"
+        
+        final_results_path = self.results_dir / results_filename
+        
+        # 5. 为所有GPU准备进程参数
+        all_temp_files = []
+        all_gpu_tasks = []
+        
+        for gpu_id, gpu_data in enumerate(gpu_chunks):
+            if not gpu_data:
+                continue
+                
+            self.logger.info(f"GPU {gpu_id}: 准备处理 {len(gpu_data)} 条数据")
+            
+            # 按进程数分割当前GPU的数据
+            process_chunks = self.split_data_by_processes(gpu_data, max_processes_per_gpu)
+            
+            # 为当前GPU准备所有进程参数
+            gpu_process_args = []
+            for proc_id, proc_data in enumerate(process_chunks):
+                if proc_data:
+                    temp_result_file = str(self.temp_results_dir / f"gpu{gpu_id}_proc{proc_id}_results.jsonl")
+                    all_temp_files.append(temp_result_file)
+                    
+                    # 子进程输出目录在主输出目录内部
+                    subprocess_output_dir = str(self.output_dir / f"gpu{gpu_id}_proc{proc_id}")
+                    
+                    gpu_process_args.append((
+                        proc_data, 
+                        gpu_id, 
+                        proc_id, 
+                        subprocess_output_dir,
+                        temp_result_file,
+                        self.alignment_model_dir,
+                        self.wespeaker_model_dir,
+                        self.language,  # 语言参数
+                        self.similarity_max_workers  # 添加相似度计算线程数参数
+                    ))
+            
+            if gpu_process_args:
+                all_gpu_tasks.append((gpu_id, gpu_process_args, max_processes_per_gpu))
+        
+        # 6. 使用ThreadPoolExecutor并行处理所有GPU
+        def process_gpu_tasks(gpu_task):
+            gpu_id, process_args, actual_processes = gpu_task
+            self.logger.info(f"GPU {gpu_id}: 开始并行处理 {len(process_args)} 个进程")
+            
+            # 为每个GPU使用独立的进程池，避免进程间冲突
+            with mp.Pool(processes=actual_processes) as pool:
+                pool.map(process_data_chunk_incremental, process_args)
+            
+            self.logger.info(f"GPU {gpu_id}: 所有进程处理完成")
+            return gpu_id
+
+        # 使用线程池同时处理所有GPU
+        with ThreadPoolExecutor(max_workers=num_gpus) as executor:
+            # 提交所有GPU任务
+            future_to_gpu = {executor.submit(process_gpu_tasks, gpu_task): gpu_task[0] 
+                           for gpu_task in all_gpu_tasks}
+            
+            # 等待所有GPU完成
+            completed_gpus = []
+            for future in as_completed(future_to_gpu):
+                gpu_id = future_to_gpu[future]
+                try:
+                    result_gpu_id = future.result()
+                    completed_gpus.append(result_gpu_id)
+                    self.logger.info(f"GPU {result_gpu_id} 完成处理")
+                except Exception as exc:
+                    self.logger.error(f"GPU {gpu_id} 处理时发生异常: {exc}")
+        
+        self.logger.info(f"所有GPU处理完成: {completed_gpus}")
+        
+        # 7. 合并所有临时结果文件
+        self.logger.info("合并所有临时结果文件...")
+        all_results = self.merge_temp_results(all_temp_files, str(final_results_path))
+        
+        if not all_results:
+            self.logger.error("没有成功处理的数据")
+            return []
+        
+        # 8. 生成汇总报告
+        summary_path = self.save_summary_report(all_results)
+        
+        # 9. 清理临时文件
+        for temp_file in all_temp_files:
+            if os.path.exists(temp_file):
+                os.remove(temp_file)
+        
+        # 10. 打印总体统计
+        total_accuracy = np.mean([r['accuracy'] for r in all_results])
+        total_avg_similarity = np.mean([r['average_similarity'] for r in all_results])
+        
+        print(f"\n=== 并行评估完成 ===")
+        print(f"使用语言: {self.language}")
+        print(f"使用 {num_gpus} 个GPU，每GPU {max_processes_per_gpu} 个进程")
+        print(f"总处理数据: {len(input_data)} 条")
+        print(f"成功处理: {len(all_results)} 条")
+        print(f"总体准确率: {total_accuracy:.3f}")
+        print(f"总体平均相似度: {total_avg_similarity:.3f}")
+        print(f"详细结果已保存到: {final_results_path}")
+        print(f"汇总报告已保存到: {summary_path}")
+        print(f"对齐信息已保存到: {self.alignment_dir}")
+        
+        return all_results
+
+    def get_or_split_prompt_audio(self, data: Dict[str, Any], audio_id: str) -> Tuple[str, str]:
+        """
+        获取或分割prompt音频
+        如果提供了分别的speaker音频文件则直接使用，否则从combined prompt分割
+        """
+        # 检查是否有分别的speaker音频文件
+        if ('prompt_audio_speaker1' in data and 'prompt_audio_speaker2' in data and
+            'prompt_text_speaker1' in data and 'prompt_text_speaker2' in data):
+            
+            self.logger.info(f"使用预分割的speaker音频文件")
+            
+            # 即使使用预分割的音频，也保存对齐信息
+            try:
+                # 检测语言或使用设置的语言
+                alignment_language = self.language
+                if alignment_language == "AUTO":
+                    alignment_language = self._detect_language_from_text(data['prompt_text_speaker1'])
+                
+                # 对S1音频进行对齐
+                s1_alignments = self.align_text_with_audio(
+                    data['prompt_text_speaker1'], data['prompt_audio_speaker1'], alignment_language
+                )
+                s1_alignment_data = {
+                    'speaker': 'S1',
+                    'text': data['prompt_text_speaker1'],
+                    'audio_path': data['prompt_audio_speaker1'],
+                    'language': alignment_language,
+                    'alignments': s1_alignments
+                }
+                self.save_alignment_info(s1_alignment_data, audio_id, "prompt_s1")
+                
+                # 对S2音频进行对齐
+                s2_alignments = self.align_text_with_audio(
+                    data['prompt_text_speaker2'], data['prompt_audio_speaker2'], alignment_language
+                )
+                s2_alignment_data = {
+                    'speaker': 'S2',
+                    'text': data['prompt_text_speaker2'],
+                    'audio_path': data['prompt_audio_speaker2'],
+                    'language': alignment_language,
+                    'alignments': s2_alignments
+                }
+                self.save_alignment_info(s2_alignment_data, audio_id, "prompt_s2")
+                
+            except Exception as e:
+                self.logger.warning(f"保存预分割音频对齐信息失败: {e}")
+            
+            return data['prompt_audio_speaker1'], data['prompt_audio_speaker2']
+        
+        # 否则从combined prompt分割
+        elif 'prompt_audio' in data and 'prompt_text' in data:
+            self.logger.info(f"从combined prompt音频分割speaker片段")
+            return self.split_audio_by_speaker(data['prompt_text'], data['prompt_audio'], audio_id)
+        
+        else:
+            raise ValueError("必须提供prompt_audio+prompt_text或者分别的speaker音频文件")
+
+    def calculate_voice_similarity(self, audio1_path: str, audio2_path: str) -> float:
+        """
+        计算两个音频的音色相似度（向后兼容版本）
+        对于过短的音频片段，通过复制来达到最小长度要求
+        """
+        # 如果在多线程环境中，使用线程安全版本
+        if threading.current_thread() != threading.main_thread():
+            return self.calculate_voice_similarity_thread_safe(audio1_path, audio2_path)
+        
+        # 确保模型已初始化
+        self._init_models_if_needed()
+        
+        try:
+            if not os.path.exists(audio1_path) or not os.path.exists(audio2_path):
+                self.logger.warning(f"Audio file not found: {audio1_path} or {audio2_path}")
+                return None
+            
+            # 检查并处理音频文件长度
+            def process_audio_for_similarity(audio_path, min_duration=0.1):
+                """
+                处理音频文件，如果过短则复制到满足最小长度要求
+                返回处理后的音频路径和是否为临时文件的标志
+                """
+                try:
+                    waveform, sample_rate = torchaudio.load(audio_path)
+                    duration = waveform.shape[1] / sample_rate
+                    
+                    if duration >= min_duration:
+                        # 音频长度足够，直接返回原路径
+                        return audio_path, False
+                    
+                    # 音频过短，需要复制
+                    repeat_times = math.ceil(min_duration / duration)
+                    self.logger.info(f"音频过短 ({duration:.3f}s)，复制 {repeat_times} 次达到 {min_duration}s 要求: {audio_path}")
+                    
+                    # 复制音频
+                    repeated_waveform = waveform.repeat(1, repeat_times)
+                    
+                    # 生成临时文件路径
+                    temp_filename = f"temp_{os.path.basename(audio_path)}"
+                    temp_path = str(self.temp_dir / temp_filename)
+                    
+                    # 保存复制后的音频
+                    torchaudio.save(temp_path, repeated_waveform, sample_rate)
+                    
+                    return temp_path, True
+                    
+                except Exception as e:
+                    self.logger.error(f"处理音频文件失败: {audio_path}, 错误: {e}")
+                    return audio_path, False
+            
+            # 处理两个音频文件
+            processed_audio1, is_temp1 = process_audio_for_similarity(audio1_path)
+            processed_audio2, is_temp2 = process_audio_for_similarity(audio2_path)
+            
+            # 计算相似度
+            similarity = self.similarity_model.compute_similarity(processed_audio1, processed_audio2)
+            
+            # 清理临时文件
+            if is_temp1 and os.path.exists(processed_audio1):
+                try:
+                    os.remove(processed_audio1)
+                except Exception as e:
+                    self.logger.warning(f"删除临时文件失败: {processed_audio1}, 错误: {e}")
+            
+            if is_temp2 and os.path.exists(processed_audio2):
+                try:
+                    os.remove(processed_audio2)
+                except Exception as e:
+                    self.logger.warning(f"删除临时文件失败: {processed_audio2}, 错误: {e}")
+            
+            return float(similarity)
+            
+        except Exception as e:
+            # 检查是否是窗口大小错误或其他计算错误
+            if "choose a window size" in str(e) or "window size" in str(e):
+                self.logger.warning(f"音频片段仍然过短，无法计算相似度: {audio1_path} vs {audio2_path}")
+                return None
+            else:
+                self.logger.error(f"Failed to compute similarity between {audio1_path} and {audio2_path}: {e}")
+                return None
+
+# 全局函数，用于多进程处理（支持增量写入）
+def process_data_chunk_incremental(args):
+    """处理数据块的工作函数（增量写入版本）"""
+    data_chunk, gpu_id, proc_id, output_dir, temp_result_file, alignment_model_dir, wespeaker_model_dir, language, similarity_max_workers = args
+    
+    # 设置当前进程使用的GPU
+    device = f"cuda:{gpu_id}" if torch.cuda.is_available() and gpu_id < torch.cuda.device_count() else "cpu"
+    
+    try:
+        # 清理CUDA状态，避免进程间冲突
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+            # 设置当前进程的GPU设备
+            torch.cuda.set_device(gpu_id)
+            # 添加小延迟，避免同时初始化冲突
+            time.sleep(proc_id * 0.5)
+        
+        # 创建评估器实例，传入模型路径、语言参数和相似度计算线程数
+        evaluator = SpeakerSimilarityEvaluator(
+            device=device,
+            alignment_model_dir=alignment_model_dir,
+            wespeaker_model_dir=wespeaker_model_dir,
+            output_dir=output_dir,
+            language=language,  # 传入语言参数
+            similarity_max_workers=similarity_max_workers  # 传入相似度计算线程数
+        )
+        
+        # 延迟初始化模型
+        evaluator._init_models_if_needed()
+        
+        # 清空临时结果文件（如果存在）
+        if os.path.exists(temp_result_file):
+            os.remove(temp_result_file)
+        
+        # 处理数据块
+        for i, data in enumerate(data_chunk):
+            input_id = f"gpu{gpu_id}_proc{proc_id}_input_{i+1:03d}"
+            
+            try:
+                result = evaluator.evaluate_single_input(data, input_id=input_id)
+                
+                # 立即写入结果到临时文件
+                evaluator.append_result_to_jsonl(result, temp_result_file)
+                
+                print(f"GPU{gpu_id}-进程{proc_id}: 完成 {input_id} (语言: {language}, 相似度线程: {similarity_max_workers})")
+                
+                # 每处理完一个数据项，清理CUDA缓存
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
+                    
+            except Exception as e:
+                print(f"GPU{gpu_id}-进程{proc_id}: 处理 {input_id} 失败: {e}")
+                # 出错时也清理CUDA缓存
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
+                continue
+        
+        print(f"GPU{gpu_id}-进程{proc_id}: 所有数据处理完成，结果已写入 {temp_result_file}")
+        
+    except Exception as e:
+        print(f"GPU{gpu_id}-进程{proc_id}: 初始化失败: {e}")
+        # 出错时清理CUDA缓存
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+
+def main():
+    """主函数示例"""
+    import argparse
+    
+    parser = argparse.ArgumentParser(description='Speaker Similarity Evaluator')
+    parser.add_argument('--jsonl_path', type=str, help='JSONL文件路径')
+    parser.add_argument('--output_dir', type=str, 
+                       default=f"/inspire/hdd/project/embodied-multimodality/public/yqzhang/auto_evaluation_new/eval_res/results_{datetime.now().strftime('%Y%m%d_%H%M%S')}", 
+                       help='结果保存目录')
+    parser.add_argument('--language', type=str, choices=['zh', 'en', 'auto'], default='zh',
+                       help='指定语言: zh=中文, en=英文, auto=自动检测 (默认: zh)')
+    parser.add_argument('--no_parallel', action='store_true', help='禁用并行处理（默认启用并行）')
+    parser.add_argument('--processes_per_gpu', type=int, default=4, help='每个GPU的进程数（建议不超过4）')
+    parser.add_argument('--similarity_workers', type=int, default=16, help='相似度计算的线程数（默认: 8）')
+    parser.add_argument('--no_shuffle', action='store_true', help='禁用数据shuffle（默认启用shuffle）')
+    parser.add_argument('--random_seed', type=int, default=None, help='随机种子（可选，用于结果复现）')
+    
+    args = parser.parse_args()
+    
+    # 设置随机种子（如果指定）
+    if args.random_seed is not None:
+        random.seed(args.random_seed)
+        np.random.seed(args.random_seed)
+        torch.manual_seed(args.random_seed)
+        print(f"设置随机种子: {args.random_seed}")
+    
+    # 语言参数处理
+    language = args.language.upper()
+    if language == 'AUTO':
+        language = 'AUTO'
+    elif language == 'EN':
+        language = 'EN'
+    else:
+        language = 'ZH'  # 默认中文
+    
+    # 创建评估器，指定结果保存目录、语言和相似度计算线程数
+    evaluator = SpeakerSimilarityEvaluator(
+        output_dir=args.output_dir,
+        language=language,
+        similarity_max_workers=args.similarity_workers
+    )
+    
+    # 默认使用并行处理，除非明确禁用
+    use_parallel = not args.no_parallel
+    use_shuffle = not args.no_shuffle
+    
+    print(f"使用语言设置: {language}")
+    print(f"相似度计算线程数: {args.similarity_workers}")
+    
+    if args.jsonl_path:
+        # 从JSONL文件处理数据
+        if use_parallel:
+            evaluator.process_batch_from_jsonl_parallel(
+                args.jsonl_path, 
+                processes_per_gpu=args.processes_per_gpu,
+                shuffle_data=use_shuffle
+            )
+        else:
+            evaluator.process_batch_from_jsonl(args.jsonl_path)
+    else:
+        # 使用示例数据（兼容性）
+        input_data = [
+            {
+                'prompt_audio': "/inspire/hdd/project/embodied-multimodality/public/yqzhang/infer_prompt/testset/audio/zhouxingchi/zxc_enhanced.wav",
+                'prompt_text': "[S1]你再往前半步我就把你给杀了。[S2]你应该这么做，我也应该死。",
+                'text': "[S1]至尊宝，如果有一天我不再是紫霞仙子，只是一个普通的凡人，你还会像现在这样陪着我吗？[S2]这个嘛，那我得先问问月老，看看他给不给我打折！毕竟追仙子要花好多力气的！[S1]哼！油嘴滑舌！我是认真的！[S2]紫霞，不管你是仙子还是凡人，哪怕变成一根香蕉，我都认得出你。不过……你最好别真变成香蕉，我怕我会忍不住吃掉……[S1]讨厌！谁要变成香蕉啊！那……如果有一天，我们不得不分开呢？[S2]哇！你这话比牛魔王的斧头还狠！不行不行，你得赔我精神损失费！[S1]怎么赔？[S2]很简单，让我亲一下，就当是定金！[S1]想得美！那如果有一天，你真的忘了我呢？[S2]那我就算翻遍三界，打烂阎王殿，也要把记忆找回来。紫霞，我至尊宝这辈子，赖定你了！[S1]傻瓜。",
+                'output_audio': "/inspire/hdd/project/embodied-multimodality/public/yqzhang/infer_res/from_newckpt_step145000/test_set/output_7.wav"
+            }
+        ]
+        
+        # 处理数据
+        if use_parallel:
+            evaluator.process_batch_parallel(input_data, processes_per_gpu=args.processes_per_gpu)
+        else:
+            evaluator.process_batch_from_data(input_data)
+
+
+if __name__ == "__main__":
+    main()

test.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+source /inspire/hdd/project/embodied-multimodality/public/yqzhang/miniconda3/bin/activate
+conda activate /inspire/hdd/project/embodied-multimodality/public/cchang/env/mooncast/
+
+# 设置CUDA环境变量
+export CUDA_LAUNCH_BLOCKING=1
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+
+# 创建日志目录和文件名
+LOG_DIR="/inspire/hdd/project/embodied-multimodality/public/cchang/projects/auto_evaluation_new/logs"
+mkdir -p "$LOG_DIR"
+LOG_FILE="$LOG_DIR/evaluation_$(date +%Y%m%d_%H%M%S).log"
+
+# 记录开始时间
+START_TIME=$(date +%s)
+START_TIME_READABLE=$(date '+%Y-%m-%d %H:%M:%S')
+
+echo "========================================="
+echo "音色相似度评估开始"
+echo "开始时间: $START_TIME_READABLE"
+echo "日志文件: $LOG_FILE"
+echo "========================================="
+echo "可以使用以下命令实时查看日志："
+echo "tail -f $LOG_FILE"
+echo ""
+
+# 将开始时间信息也写入日志文件
+{
+    echo "========================================="
+    echo "音色相似度评估开始"
+    echo "开始时间: $START_TIME_READABLE"
+    echo "进程配置: 每GPU 8个进程"
+    echo "语言设置: zh (中文)"
+    echo "========================================="
+    echo ""
+} | tee "$LOG_FILE"
+
+# 使用更保守的进程数
+python -u /inspire/hdd/project/embodied-multimodality/public/cchang/projects/auto_evaluation_new/test.py \
+    --jsonl_path /inspire/hdd/project/embodied-multimodality/public/yqzhang/infer_res/from_newckpt_step70000/eval_new/output.jsonl \
+    --output_dir /inspire/hdd/project/embodied-multimodality/public/cchang/projects/auto_evaluation_new/eval_res/new_test \
+    --processes_per_gpu 8 \
+    --language zh \
+    2>&1 | tee -a "$LOG_FILE"
+
+
+# 记录结束时间
+END_TIME=$(date +%s)
+END_TIME_READABLE=$(date '+%Y-%m-%d %H:%M:%S')
+
+# 计算耗时
+DURATION=$((END_TIME - START_TIME))
+HOURS=$((DURATION / 3600))
+MINUTES=$(((DURATION % 3600) / 60))
+SECONDS=$((DURATION % 60))
+
+# 输出结束信息
+{
+    echo ""
+    echo "========================================="
+    echo "音色相似度评估完成！"
+    echo "结束时间: $END_TIME_READABLE"
+    echo "总耗时: ${HOURS}小时${MINUTES}分钟${SECONDS}秒 (共${DURATION}秒)"
+    echo "日志文件: $LOG_FILE"
+    echo "========================================="
+} | tee -a "$LOG_FILE"
+
+# 显示在终端
+echo ""
+echo "评估完成！"
+echo "开始时间: $START_TIME_READABLE"
+echo "结束时间: $END_TIME_READABLE"
+echo "总耗时: ${HOURS}小时${MINUTES}分钟${SECONDS}秒"
+echo "日志已保存到: $LOG_FILE"
+
+# 如果耗时超过1小时，发送额外提醒
+if [ $DURATION -gt 3600 ]; then
+    echo ""
+    echo "⏰ 注意：本次评估耗时较长，超过1小时"
+    echo "   建议检查性能优化效果"
+fi

test_alignment.py

@@ -0,0 +1,416 @@
+import re
+import torch
+import torchaudio.functional as F
+import torchaudio
+import uroman as ur
+import logging
+from typing import List, Dict, Any, Optional
+
+def split_and_merge_punctuation(text: str) -> List[str]:
+    """
+    处理英文文本，按空格分词并将标点符号合并到前面的单词
+    
+    Args:
+        text: 输入的英文文本
+        
+    Returns:
+        处理后的单词列表，标点符号已合并到对应单词
+    """
+    # 先按空格拆分文本
+    elements = text.split()
+    
+    # 用于保存最终的结果
+    result = []
+    
+    # 遍历每个拆分后的元素
+    for ele in elements:
+        # 使用正则表达式提取连续字母、数字和标点
+        parts = re.findall(r'[a-zA-Z0-9]+|[^\w\s]+', ele)
+        
+        # 用于保存拆分后的部分
+        merged_parts = []
+        
+        for i in range(len(parts)):
+            if i % 2 == 0:  # 如果是字母或数字部分
+                # 将字母或数字部分添加到结果中
+                merged_parts.append(parts[i])
+            else:  # 如果是标点或其他符号部分
+                # 将标点部分与前面的字母或数字部分合并
+                if merged_parts:
+                    merged_parts[-1] += parts[i]
+                else:
+                    merged_parts.append(parts[i])
+        
+        # 将合并后的部分加入最终结果
+        result.extend(merged_parts)
+    
+    return result
+
+def restore_spaces_in_english_text(tokens: List[str]) -> str:
+    """
+    在英文单词之间恢复空格
+    
+    Args:
+        tokens: 单词列表
+        
+    Returns:
+        恢复空格后的文本
+    """
+    result = []
+    for i, token in enumerate(tokens):
+        # 检查是否需要在单词前添加空格
+        if i > 0 and token[0].isalnum() and not any(p in tokens[i-1] for p in ',.!?;:()[]<>\'\"…'):
+            result.append(" ")
+        result.append(token)
+    
+    return "".join(result)
+
+def get_aligned_result_with_punctuation(alignment_result: List[Dict], text: str) -> List[Dict]:
+    """
+    将对齐结果转换为包含标点符号的格式
+    
+    Args:
+        alignment_result: 原始对齐结果
+        text: 原始文本
+        
+    Returns:
+        处理后的对齐结果，标点符号已合并
+    """
+    text_tokens = split_and_merge_punctuation(text)
+    
+    updated_alignment_result = []
+    token_idx = 0
+    
+    for index, align_item in enumerate(alignment_result):
+        if token_idx >= len(text_tokens):
+            break
+            
+        start = align_item["start"]
+        end = align_item["end"]
+        text_token = text_tokens[token_idx]
+        
+        updated_item = {
+            "start": start,
+            "end": end,
+            "transcript": text_token
+        }
+        
+        # 保留原始对齐结果中的其他字段
+        updated_item.update({key: align_item[key] for key in align_item 
+                           if key not in ["start", "end", "transcript"]})
+        
+        updated_alignment_result.append(updated_item)
+        token_idx += 1
+    
+    return updated_alignment_result
+
+class EnglishAlignmentModel:
+    def __init__(self, device: str = "cuda", model_dir: Optional[str] = None):
+        """
+        初始化英文对齐模型
+        
+        Args:
+            device: 设备类型 ("cuda" 或 "cpu")
+            model_dir: 模型目录路径，如果为None则使用默认路径
+        """
+        self.device = torch.device(device)
+        self.bundle = torchaudio.pipelines.MMS_FA
+        
+        # 设置模型下载参数
+        dl_kwargs = {}
+        if model_dir:
+            dl_kwargs['model_dir'] = model_dir
+            
+        self.align_model = self.bundle.get_model(
+            with_star=False, 
+            dl_kwargs=dl_kwargs
+        ).to(self.device)
+        
+        self.uroman = ur.Uroman()
+        self.DICTIONARY = self.bundle.get_dict()
+
+    def align(self, emission: torch.Tensor, tokens: torch.Tensor):
+        """
+        执行强对齐
+        
+        Args:
+            emission: 模型的输出
+            tokens: 目标tokens
+            
+        Returns:
+            对齐的tokens和分数
+        """
+        alignments, scores = F.forced_align(
+            log_probs=emission,
+            targets=tokens,
+            blank=0
+        )
+        alignments, scores = alignments[0], scores[0]
+        scores = scores.exp()
+        return alignments, scores
+
+    def unflatten(self, list_: List, lengths: List[int]) -> List[List]:
+        """
+        将一个长列表按照长度拆分成子列表
+        
+        Args:
+            list_: 长列表
+            lengths: 各子列表的长度
+            
+        Returns:
+            拆分后的子列表
+        """
+        assert len(list_) == sum(lengths)
+        i = 0
+        ret = []
+        for l in lengths:
+            ret.append(list_[i:i + l])
+            i += l
+        return ret
+
+    def preview_word(self, waveform: torch.Tensor, spans: List, num_frames: int, 
+                    transcript: List[str], sample_rate: int) -> List[Dict]:
+        """
+        生成每个单词的时间对齐信息
+        
+        Args:
+            waveform: 音频波形
+            spans: 单词的跨度
+            num_frames: 帧数
+            transcript: 转录文本单词列表
+            sample_rate: 采样率
+            
+        Returns:
+            单词的对齐信息列表
+        """
+        end = 0
+        alignment_result = []
+        
+        for span, trans in zip(spans, transcript):
+            ratio = waveform.size(1) / num_frames
+            x0 = int(ratio * span[0].start)
+            x1 = int(ratio * span[-1].end)
+            
+            align_info = {
+                "transcript": trans,
+                "start": round(x0 / sample_rate, 3),
+                "end": round(x1 / sample_rate, 3)
+            }
+            align_info["pause"] = round(align_info["start"] - end, 3)
+            align_info["duration"] = round(align_info["end"] - align_info["start"], 3)
+            end = align_info["end"]
+            alignment_result.append(align_info)
+            
+        return alignment_result
+
+    def make_wav_batch(self, wav_list: List[torch.Tensor]):
+        """
+        将wav_list中的每个wav张量填充为相同的长度
+        
+        Args:
+            wav_list: wav文件列表
+            
+        Returns:
+            填充后的音频张量和原始长度
+        """
+        wav_lengths = torch.tensor([wav.size(0) for wav in wav_list], dtype=torch.long)
+        max_length = max(wav_lengths)
+        wavs_tensors = torch.zeros(len(wav_list), max_length, device=wav_list[0].device)
+        
+        for i, wav in enumerate(wav_list):
+            wavs_tensors[i, :wav_lengths[i]] = wav
+            
+        return wavs_tensors, wav_lengths.to(wavs_tensors.device)
+
+    def get_target(self, transcript: str) -> torch.Tensor:
+        """
+        获取给定英文转录文本的目标tokens
+        
+        Args:
+            transcript: 英文转录文本
+            
+        Returns:
+            转录文本的目标tokens
+        """
+        # 移除标点符号并转换为小写
+        transcript = re.sub(r'[^\w\s]', r' ', transcript)
+        words = transcript.lower().split()
+        
+        # 获取字典中的特殊符号token
+        star_token = self.DICTIONARY['*']
+        
+        # 将每个字符转换为对应的token
+        tokenized_transcript = []
+        for word in words:
+            tokenized_transcript.extend([
+                self.DICTIONARY[c] if c in self.DICTIONARY and c != '-' else star_token 
+                for c in word
+            ])
+        
+        return torch.tensor([tokenized_transcript], dtype=torch.int32, device=self.device)
+
+    def get_alignment_result(self, emission_padded: torch.Tensor, emission_length: int,
+                           aligned_tokens: torch.Tensor, alignment_scores: torch.Tensor,
+                           transcript: str, waveform: torch.Tensor) -> List[Dict]:
+        """
+        根据给定的emission和对齐信息生成对齐结果
+        
+        Args:
+            emission_padded: 填充后的emission
+            emission_length: emission的有效长度
+            aligned_tokens: 对齐的tokens
+            alignment_scores: 对齐的分数
+            transcript: 转录文本
+            waveform: 音频波形
+            
+        Returns:
+            对齐结果
+        """
+        # 处理文本
+        processed_transcript = re.sub(r'[^\w\s]', r' ', transcript)
+        words = processed_transcript.lower().split()
+        
+        emission = emission_padded[:emission_length, :].unsqueeze(0)
+        token_spans = F.merge_tokens(aligned_tokens, alignment_scores)
+        word_spans = self.unflatten(token_spans, [len(word) for word in words])
+        num_frames = emission.size(1)
+        
+        return self.preview_word(waveform.unsqueeze(0), word_spans, num_frames, 
+                               words, self.bundle.sample_rate)
+
+    def align_audio_text(self, waveform: torch.Tensor, transcript: str) -> List[Dict]:
+        """
+        对单个音频和文本进行对齐
+        
+        Args:
+            waveform: 音频波形张量 (1D tensor)
+            transcript: 英文转录文本
+            
+        Returns:
+            对齐结果列表，包含每个单词的时间信息
+        """
+        # 确保音频在正确的设备上
+        waveform = waveform.to(self.device)
+        
+        # 如果需要重采样
+        if hasattr(self, 'original_sample_rate'):
+            if self.original_sample_rate != self.bundle.sample_rate:
+                waveform = F.resample(waveform, self.original_sample_rate, self.bundle.sample_rate)
+        
+        # 批量处理（单个样本）
+        return self.batch_alignment([waveform], [transcript])[0]
+
+    def batch_alignment(self, wav_list: List[torch.Tensor], transcript_list: List[str]) -> List[List[Dict]]:
+        """
+        批量对齐
+        
+        Args:
+            wav_list: wav文件列表
+            transcript_list: 转录文本列表
+            
+        Returns:
+            对齐结果列表
+        """
+        wavs_tensors, wavs_lengths_tensor = self.make_wav_batch(wav_list)
+        
+        # 前向传播
+        with torch.inference_mode():
+            emission, emission_lengths = self.align_model(
+                wavs_tensors.to(self.device), 
+                wavs_lengths_tensor
+            )
+            # 添加star维度
+            star_dim = torch.zeros(
+                (emission.shape[0], emission.size(1), 1), 
+                dtype=emission.dtype, 
+                device=self.device
+            )
+            emission = torch.cat((emission, star_dim), dim=-1)
+        
+        # 获取目标tokens
+        target_list = [self.get_target(transcript) for transcript in transcript_list]
+        
+        # 执行对齐
+        align_results = [
+            self.align(emission_padded[:emission_length, :].unsqueeze(0), target)
+            for emission_padded, emission_length, target in zip(emission, emission_lengths, target_list)
+        ]
+        
+        batch_aligned_tokens = [align_result[0] for align_result in align_results]
+        batch_alignment_scores = [align_result[1] for align_result in align_results]
+
+        # 生成对齐结果
+        alignment_result_list = [
+            self.get_alignment_result(emission_padded, emission_length, aligned_tokens, 
+                                    alignment_scores, transcript, waveform)
+            for emission_padded, emission_length, aligned_tokens, alignment_scores, transcript, waveform
+            in zip(emission, emission_lengths, batch_aligned_tokens, batch_alignment_scores, 
+                  transcript_list, wav_list)
+        ]
+        
+        # 处理标点符号
+        final_results = []
+        for alignment_result, transcript in zip(alignment_result_list, transcript_list):
+            processed_result = get_aligned_result_with_punctuation(alignment_result, transcript)
+            final_results.append(processed_result)
+        
+        return final_results
+
+def align_english_audio_text(audio_path: str, transcript: str, device: str = "cuda", 
+                           model_dir: Optional[str] = None) -> List[Dict]:
+    """
+    便捷函数：对英文音频和文本进行对齐
+    
+    Args:
+        audio_path: 音频文件路径
+        transcript: 英文转录文本
+        device: 设备类型 ("cuda" 或 "cpu")
+        model_dir: 模型目录路径
+        
+    Returns:
+        对齐结果列表，包含每个单词的时间信息
+        
+    Example:
+        >>> result = align_english_audio_text("audio.wav", "Hello world!")
+        >>> print(result)
+        [
+            {"transcript": "Hello", "start": 0.0, "end": 0.5, "duration": 0.5, "pause": 0.0},
+            {"transcript": "world!", "start": 0.6, "end": 1.2, "duration": 0.6, "pause": 0.1}
+        ]
+    """
+    # 加载音频
+    waveform, sample_rate = torchaudio.load(audio_path)
+    
+    # 转换为单声道
+    if waveform.size(0) > 1:
+        waveform = torch.mean(waveform, dim=0, keepdim=True)
+    waveform = waveform.squeeze(0)  # 移除批次维度
+    
+    # 初始化模型
+    model = EnglishAlignmentModel(device=device, model_dir=model_dir)
+    model.original_sample_rate = sample_rate
+    
+    # 执行对齐
+    return model.align_audio_text(waveform, transcript)
+
+if __name__ == "__main__":
+    # 使用示例
+    audio_file = "/inspire/hdd/project/embodied-multimodality/public/yqzhang/infer_res/from_newckpt_step40000/test_en/gpu4/output_0.wav"
+    text = "[S1]Hey, did you hear about that company called MoSi AI? [S2]MoSi AI? Yeah, I think I've heard of them. Aren't they the ones doing AI stuff? What new thing have they come up with now? [S1]Yeah, that's them! They recently launched this super hot new product called, um, Asteroid. [S2]Asteroid. That's a pretty cool name. Does it mean like the space rock? [S1]Yeah, I think that's what it means. Let me tell you, this thing is incredible. They say it's currently the most realistic, human-like conversational TTS model out there. [S2]Oh, TTS technology? You mean the text-to-speech thing? Aren't there already a lot of those on the market? What makes this one so special? [S1]Well, it's completely different. They say the voice produced by Asteroid sounds almost exactly like a real person talking. And it's super smooth and natural. Not at all like, you know, that stiff robotic tone. [S2]I see. Some voice assistants do still have that mechanical feel, especially during multi-turn conversations. So how amazing is this Asteroid exactly? [S1]I heard they internally call Asteroid China's own version of NotebookLM. [S2]NotebookLM? Oh, I know that one. Isn't that the personal AI that Google made? The one that helps organize notes and answers all kinds of questions? So Asteroid has similar functions? [S1]Right. That's probably what they mean. It's not just that the voice sounds incredibly human. The intelligence level is also really high. It can have these really logical, contextual, in-depth conversations with you. It's just like chatting with a real person. [S2]Wow, that sounds amazing. If they can really achieve that... [S1]Yeah, it's basically like having a personal assistant that's both articulate and really understands you. [S2]Hmm. That does sound appealing. [S1]And some people are saying it's like the, what's it called again in the voice technology circle? Oh right, DeepSeek. [S2]DeepSeek? Isn't that the company making large language models? Their models are pretty popular now. That's high praise. So they're saying Asteroid is top-tier technology? [S1]Yeah, I think that's what they mean. It's like they've reached a whole new level in voice synthesis. Similar to the impact DeepSeek has had in natural language processing. It could be that kind of groundbreaking technology. [S2]If Asteroid is really that impressive, where could it be used? I feel like there must be huge potential there. [S1]Absolutely. Just imagine future smart customer service, audiobook reading, and those virtual livestreamers that are so popular now. The quality would improve dramatically. We might even have personal assistants using Asteroid to talk to us directly. How natural would that be? [S2]Yeah. That does sound exciting. When can we actually try it out? Are there any demos available? [S1]I haven't looked into that carefully yet. But since they've already announced it, I'm guessing it won't be long. I'm really eager to try it and see just how human-like it is. [S2]Yeah, yeah. If it can really deliver what they're promising, getting information and interacting with machines will be so much more convenient. The experience will be much better too. [S1]Exactly, exactly. We're just waiting for MoSi AI to give us this big surprise."
+    
+    # 对文本进行归一化，删除所有[S1][S2]标记
+    import re
+    normalized_text = re.sub(r'\[S[12]\]', '', text).strip()
+    
+    # 设置本地模型目录
+    alignment_model_dir = '/inspire/hdd/project/embodied-multimodality/public/yqzhang/auto_evaluation_new/models/mms_fa'
+    
+    try:
+        alignment_result = align_english_audio_text(audio_file, normalized_text, model_dir=alignment_model_dir)
+        
+        print("对齐结果:")
+        for item in alignment_result:
+            print(f"单词: '{item['transcript']}', 开始: {item['start']}s, 结束: {item['end']}s, 持续: {item['duration']}s")
+            
+    except Exception as e:
+        print(f"对齐失败: {e}")

test_online.py

@@ -0,0 +1,1550 @@
+import asyncio
+import json
+import re
+import os
+from typing import List, Dict, Tuple, Any
+import numpy as np
+from pathlib import Path
+import torch
+import torchaudio
+import torchaudio.functional as F
+import tritonclient.grpc as grpcclient
+from tritonclient.utils import *
+import logging
+import wespeaker
+import shutil
+from datetime import datetime
+import multiprocessing as mp
+from functools import partial
+import math
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import random  # 添加random模块用于shuffle
+
+# 设置multiprocessing启动方式为spawn（CUDA兼容）
+mp.set_start_method('spawn', force=True)
+
+# 引用词对齐模块
+from alignment import AlignmentModel, batch_get_alignment_result
+# from tensorrt_client import TritonSimilarityClient
+from speaker_client import TritonSpeakerClient
+
+
+class SpeakerSimilarityEvaluator:
+    """音色相似度评估器"""
+    
+    def __init__(self, device="cuda", 
+                 alignment_model_dir='./models/mms_fa', 
+                 wespeaker_model_url='localhost:8001',
+                 output_dir="./evaluation_results",
+                 language="ZH",
+                 similarity_max_workers=8):
+        """初始化评估器"""
+        self.device = device
+        self.alignment_model_dir = alignment_model_dir
+        self.wespeaker_model_url = wespeaker_model_url
+        self.language = language.upper()  # 添加语言参数
+        self.similarity_max_workers = similarity_max_workers  # 相似度计算线程数，已无效
+        
+        # 先设置日志系统
+        logging.basicConfig(level=logging.INFO)
+        self.logger = logging.getLogger(__name__)
+        
+        # 设置输出目录结构
+        self.output_dir = Path(output_dir)
+        self.segments_dir = self.output_dir / "segments"  # 分割后的音频片段
+        self.prompts_dir = self.output_dir / "prompts"   # prompt音频的S1和S2片段
+        self.temp_dir = self.output_dir / "temp"         # 临时文件
+        self.results_dir = self.output_dir / "results"   # 评估结果
+        self.temp_results_dir = self.output_dir / "temp_results"  # 临时结果文件
+        self.alignment_dir = self.output_dir / "alignments"  # 对齐信息保存目录
+        
+        # 创建所有必要的目录
+        self._create_output_directories()
+        
+        # 在多进程环境中延迟模型初始化
+        self.alignment_model = None
+        self.similarity_model = None
+        
+        # 线程局部存储，用于线程安全的模型访问
+        self._thread_local = threading.local()
+        
+        # 记录运行信息
+        self.logger.info(f"评估结果将保存到: {self.output_dir}")
+        self.logger.info(f"对齐信息将保存到: {self.alignment_dir}")
+        self.logger.info(f"使用语言: {self.language}")
+    
+    def _create_output_directories(self):
+        """创建输出目录结构"""
+        for dir_path in [self.segments_dir, self.prompts_dir, self.temp_dir, 
+                        self.results_dir, self.temp_results_dir, self.alignment_dir]:
+            dir_path.mkdir(parents=True, exist_ok=True)
+    
+    def _get_safe_filename(self, text: str, max_length: int = 50) -> str:
+        """生成安全的文件名"""
+        # 移除特殊字符，只保留中文、英文、数字和基本符号
+        safe_text = re.sub(r'[^\u4e00-\u9fff\w\s]', '', text)
+        # 限制长度
+        if len(safe_text) > max_length:
+            safe_text = safe_text[:max_length]
+        # 替换空格为下划线
+        safe_text = safe_text.replace(' ', '_')
+        return safe_text if safe_text else "unnamed"
+    
+    def _clean_temp_files(self):
+        """清理临时文件，但保留临时目录"""
+        if self.temp_dir.exists():
+            # 只删除临时目录中的文件，不删除目录本身
+            for file_path in self.temp_dir.iterdir():
+                if file_path.is_file():
+                    try:
+                        file_path.unlink()
+                    except Exception as e:
+                        self.logger.warning(f"删除临时文件失败: {file_path}, 错误: {e}")
+        else:
+            # 如果临时目录不存在，重新创建
+            self.temp_dir.mkdir(parents=True, exist_ok=True)
+    
+    def _init_models_if_needed(self):
+        """延迟初始化模型（用于多进程环境）"""
+        # 初始化对齐模型 - 修正参数顺序
+        if self.alignment_model is None:
+            # 根据AlignmentModel的构造函数，应该是(device, model_dir)而不是(model_dir, device)
+            self.alignment_model = AlignmentModel(self.device, self.alignment_model_dir)
+        
+        # 初始化相似度模型
+        if self.similarity_model is None:
+            self._load_wespeaker_model(self.wespeaker_model_url)
+    
+    def _is_english_text(self, text: str) -> bool:
+        """简单判断文本是否主要是英文"""
+        # 计算英文字符的比例
+        english_chars = sum(1 for c in text if c.isascii() and c.isalpha())
+        total_chars = sum(1 for c in text if c.isalpha())
+        
+        if total_chars == 0:
+            return False
+        
+        return english_chars / total_chars > 0.8  # 如果80%以上是英文字符，认为是英文
+    
+    def _detect_language_from_text(self, text: str) -> str:
+        """从文本内容检测语言"""
+        clean_text = self.remove_speaker_tags(text)
+        if self._is_english_text(clean_text):
+            return "EN"
+        else:
+            return "ZH"
+    
+    def save_alignment_info(self, alignment_data: Dict[str, Any], input_id: str, file_type: str = "output"):
+        """
+        保存对齐信息到单独的JSON文件
+        
+        Args:
+            alignment_data: 对齐信息数据
+            input_id: 输入ID
+            file_type: 文件类型 ("output", "prompt", "segment")
+        """
+        try:
+            safe_input_id = self._get_safe_filename(input_id)
+            alignment_filename = f"{safe_input_id}_{file_type}_alignment.json"
+            alignment_path = self.alignment_dir / alignment_filename
+            
+            # 添加元数据
+            alignment_info = {
+                'input_id': input_id,
+                'file_type': file_type,
+                'language': self.language,
+                'timestamp': datetime.now().isoformat(),
+                'alignment_data': alignment_data
+            }
+            
+            with open(alignment_path, 'w', encoding='utf-8') as f:
+                json.dump(alignment_info, f, ensure_ascii=False, indent=2)
+            
+            self.logger.info(f"对齐信息已保存: {alignment_path}")
+            return str(alignment_path)
+            
+        except Exception as e:
+            self.logger.error(f"保存对齐信息失败: {e}")
+            return None
+    
+    def save_detailed_alignment_info(self, alignments: List[Dict[str, Any]], 
+                                   text_segments: List[Dict[str, Any]], 
+                                   input_id: str, audio_path: str, 
+                                   original_text: str, processed_text: str):
+        """
+        保存详细的对齐信息，包括分段信息
+        
+        Args:
+            alignments: 对齐结果列表
+            text_segments: 文本分段信息
+            input_id: 输入ID
+            audio_path: 音频文件路径
+            original_text: 原始文本
+            processed_text: 处理后的文本
+        """
+        alignment_data = {
+            'original_text': original_text,
+            'processed_text': processed_text,
+            'audio_path': audio_path,
+            'language': self.language,
+            'total_alignments': len(alignments),
+            'total_segments': len(text_segments),
+            'alignments': alignments,
+            'text_segments': text_segments,
+            'segment_alignment_mapping': []
+        }
+        
+        # 建立文本段和对齐结果的映射关系
+        for segment in text_segments:
+            segment_mapping = {
+                'segment_id': segment.get('segment_id', 0),
+                'segment_text': segment.get('text', ''),
+                'speaker_label': segment.get('speaker_label', ''),
+                'start_time': segment.get('start_time', 0.0),
+                'end_time': segment.get('end_time', 0.0),
+                'corresponding_alignments': []
+            }
+            
+            # 找到对应的对齐项
+            segment_start = segment.get('start_time', 0.0)
+            segment_end = segment.get('end_time', 0.0)
+            
+            for i, align_item in enumerate(alignments):
+                align_start = align_item.get('start', 0.0)
+                align_end = align_item.get('end', 0.0)
+                
+                # 检查对齐项是否在当前段的时间范围内
+                if (align_start >= segment_start and align_end <= segment_end) or \
+                   (align_start < segment_end and align_end > segment_start):
+                    segment_mapping['corresponding_alignments'].append({
+                        'alignment_index': i,
+                        'transcript': align_item.get('transcript', ''),
+                        'start': align_start,
+                        'end': align_end,
+                        'score': align_item.get('score', 0.0) if 'score' in align_item else None
+                    })
+            
+            alignment_data['segment_alignment_mapping'].append(segment_mapping)
+        
+        return self.save_alignment_info(alignment_data, input_id, "detailed")
+    
+    def remove_speaker_tags(self, text: str) -> str:
+        """删除文本中的说话人标签[S1][S2]"""
+        return re.sub(r'\[S[12]\]', '', text).strip()
+    
+    def extract_speaker_segments(self, text: str) -> List[Dict[str, Any]]:
+        """提取文本中的说话人片段信息"""
+        segments = []
+        pattern = r'\[S([12])\]([^[]*)'
+        matches = re.findall(pattern, text)
+        
+        for speaker_id, content in matches:
+            segments.append({
+                'speaker': f'S{speaker_id}',
+                'content': content.strip()
+            })
+        return segments
+    
+    def replace_punctuation_with_comma(self, text: str, language: str = None) -> str:
+        """将所有标点符号替换为逗号，连续逗号只保留一个，根据语言选择正确的逗号类型"""
+        # 如果未指定语言，使用类的默认语言设置或自动检测
+        if language is None:
+            if hasattr(self, 'language'):
+                language = self.language
+            else:
+                language = self._detect_language_from_text(text)
+        
+        language = language.upper()
+        
+        # 根据语言选择逗号类型和处理策略
+        if language == "EN" or (language == "AUTO" and self._is_english_text(text)):
+            # 英文处理：先删除撇号，再替换其他标点符号
+            text = re.sub(r"'", '', text)  # 删除撇号（don't -> dont）
+            target_comma = ','  # 英文逗号
+            comma_pattern = r',+'  # 匹配连续英文逗号
+            # 更新正则表达式，不包含撇号
+            text = re.sub(r'[.,!?;:()\[\]<>\"…·，。；：！？（）【】《》""\\、]', target_comma, text)
+        else:
+            # 中文处理：包含撇号在替换范围内
+            target_comma = '，'  # 中文逗号
+            comma_pattern = r'，+'  # 匹配连续中文逗号
+            # 更新正则表达式以匹配更多的标点符号
+            text = re.sub(r'[.,!?;:()\[\]<>\'\"…·，。；：！？（）【】《》''""\\、]', target_comma, text)
+        
+        text = re.sub(comma_pattern, target_comma, text)
+        return text.strip(target_comma)
+    
+    def align_text_with_audio(self, text: str, audio_path: str, language=None) -> List[Dict[str, Any]]:
+        """
+        文本和音频的词对齐
+        返回每个词对应的音频时间段
+        """
+        # 确保模型已初始化
+        self._init_models_if_needed()
+        
+        # 如果未指定语言，使用类的默认语言设置或自动检测
+        if language is None:
+            if hasattr(self, 'language'):
+                language = self.language
+            else:
+                language = self._detect_language_from_text(text)
+        else:
+            language = language.upper()
+        
+        # 加载音频
+        waveform, sample_rate = torchaudio.load(audio_path)
+        
+        # 重采样到模型要求的采样率
+        if sample_rate != self.alignment_model.bundle.sample_rate:
+            waveform = F.resample(waveform, sample_rate, self.alignment_model.bundle.sample_rate)
+        
+        # 转换为单声道
+        if waveform.shape[0] > 1:
+            waveform = torch.mean(waveform, dim=0, keepdim=True)
+        
+        waveform = waveform.squeeze(0)  # 移除批次维度
+        
+        # 将音频移动到正确的设备
+        waveform = waveform.to(self.device)
+        
+        # 执行对齐
+        try:
+            alignment_results = batch_get_alignment_result(
+                self.alignment_model, 
+                [waveform], 
+                [text], 
+                [language]
+            )
+            if not alignment_results or not alignment_results[0]:
+                raise RuntimeError(f"对齐结果为空: {audio_path}")
+            return alignment_results[0]
+        except Exception as e:
+            self.logger.error(f"音频对齐失败: {audio_path}")
+            self.logger.error(f"错误详情: {e}")
+            raise RuntimeError(f"音频对齐失败，程序终止。文件: {audio_path}，错误: {e}")
+    
+    def split_audio_segment(self, audio_path: str, start_time: float, end_time: float, output_path: str):
+        """分割音频片段"""
+        waveform, sample_rate = torchaudio.load(audio_path)
+        
+        start_frame = int(start_time * sample_rate)
+        end_frame = int(end_time * sample_rate)
+        
+        segment = waveform[:, start_frame:end_frame]
+        
+        # 确保输出目录存在
+        os.makedirs(os.path.dirname(output_path), exist_ok=True)
+        
+        torchaudio.save(output_path, segment, sample_rate)
+        return output_path
+    
+    def concatenate_audio_files(self, audio_files: List[str], output_path: str):
+        """拼接多个音频文件"""
+        if not audio_files:
+            return
+        
+        waveforms = []
+        sample_rate = None
+        
+        for audio_file in audio_files:
+            if os.path.exists(audio_file):
+                waveform, sr = torchaudio.load(audio_file)
+                if sample_rate is None:
+                    sample_rate = sr
+                elif sr != sample_rate:
+                    waveform = F.resample(waveform, sr, sample_rate)
+                waveforms.append(waveform)
+        
+        if waveforms:
+            concatenated = torch.cat(waveforms, dim=1)
+            os.makedirs(os.path.dirname(output_path), exist_ok=True)
+            torchaudio.save(output_path, concatenated, sample_rate)
+    
+    def split_audio_by_speaker(self, prompt_text: str, prompt_audio: str, audio_id: str) -> Tuple[str, str]:
+        """
+        根据说话人标签分割prompt音频
+        返回S1和S2的音频片段路径
+        """
+        # 1. 提取说话人片段
+        speaker_segments = self.extract_speaker_segments(prompt_text)
+        
+        # 2. 删除标签后进行词对齐 - 如果失败则直接抛出异常
+        clean_text = self.remove_speaker_tags(prompt_text)
+        
+        # 检测语言或使用设置的语言
+        alignment_language = self.language
+        if alignment_language == "AUTO":
+            alignment_language = self._detect_language_from_text(clean_text)
+            
+        alignments = self.align_text_with_audio(clean_text, prompt_audio, alignment_language)
+        
+        # 保存prompt对齐信息
+        prompt_alignment_data = {
+            'original_text': prompt_text,
+            'clean_text': clean_text,
+            'audio_path': prompt_audio,
+            'language': alignment_language,
+            'speaker_segments': speaker_segments,
+            'alignments': alignments
+        }
+        self.save_alignment_info(prompt_alignment_data, audio_id, "prompt")
+        
+        # 3. 根据对齐结果分割音频
+        s1_segments = []
+        s2_segments = []
+        
+        # 为每个说话人片段找到对应的时间段
+        text_pos = 0
+        for seg in speaker_segments:
+            seg_text = seg['content'].strip()
+            seg_length = len(seg_text)
+            
+            # 找到这个片段在对齐结果中的起始和结束
+            start_time = None
+            end_time = None
+            
+            current_pos = 0
+            for align_item in alignments:
+                item_text = align_item['transcript']
+                item_length = len(item_text)
+                
+                if current_pos >= text_pos and current_pos < text_pos + seg_length:
+                    if start_time is None:
+                        start_time = align_item['start']
+                    end_time = align_item['end']
+                
+                current_pos += item_length
+            
+            if start_time is not None and end_time is not None:
+                if seg['speaker'] == 'S1':
+                    s1_segments.append((start_time, end_time))
+                else:
+                    s2_segments.append((start_time, end_time))
+            
+            text_pos += seg_length
+        
+        # 4. 分割并拼接音频片段
+        safe_audio_id = self._get_safe_filename(audio_id)
+        prompts1_path = str(self.prompts_dir / f"{safe_audio_id}_s1.wav")
+        prompts2_path = str(self.prompts_dir / f"{safe_audio_id}_s2.wav")
+        
+        # 分割S1的所有片段
+        if s1_segments:
+            s1_temp_segments = []
+            for i, (start, end) in enumerate(s1_segments):
+                temp_path = str(self.temp_dir / f"{safe_audio_id}_s1_temp_{i}.wav")
+                self.split_audio_segment(prompt_audio, start, end, temp_path)
+                s1_temp_segments.append(temp_path)
+            
+            # 拼接S1片段
+            self.concatenate_audio_files(s1_temp_segments, prompts1_path)
+        
+        # 分割S2的所有片段
+        if s2_segments:
+            s2_temp_segments = []
+            for i, (start, end) in enumerate(s2_segments):
+                temp_path = str(self.temp_dir / f"{safe_audio_id}_s2_temp_{i}.wav")
+                self.split_audio_segment(prompt_audio, start, end, temp_path)
+                s2_temp_segments.append(temp_path)
+            
+            # 拼接S2片段
+            self.concatenate_audio_files(s2_temp_segments, prompts2_path)
+        
+        return prompts1_path, prompts2_path
+    
+    def map_text_segments_to_speakers(self, original_text: str) -> List[Dict[str, Any]]:
+        """
+        将原始文本按说话人和标点符号同时分割，保持映射关系
+        支持英文单词级别的处理
+        """
+        segments = []
+        pattern = r'\[S([12])\]([^[]*)'
+        matches = re.findall(pattern, original_text)
+        
+        # 检测语言或使用设置的语言
+        alignment_language = self.language
+        if alignment_language == "AUTO":
+            alignment_language = self._detect_language_from_text(original_text)
+        
+        segment_id = 0
+        for speaker_id, content in matches:
+            speaker = f'S{speaker_id}'
+            clean_content = content.strip()
+            comma_content = self.replace_punctuation_with_comma(clean_content, alignment_language)
+            
+            # 根据语言选择正确的逗号分割
+            if alignment_language == "EN" or (alignment_language == "AUTO" and self._is_english_text(clean_content)):
+                # 英文：按英文逗号分割，保持单词完整性
+                parts = [part.strip() for part in comma_content.split(',') if part.strip()]
+            else:
+                # 中文：按中文逗号分割
+                parts = [part.strip() for part in comma_content.split('，') if part.strip()]
+            
+            for part in parts:
+                if part.strip():
+                    segments.append({
+                        'segment_id': segment_id,
+                        'text': part.strip(),
+                        'speaker_label': speaker,
+                        'original_speaker_content': clean_content
+                    })
+                    segment_id += 1
+        
+        return segments
+    
+    def split_output_audio_by_comma(self, text: str, output_audio: str, audio_id: str) -> List[Dict[str, Any]]:
+        """
+        根据逗号分割输出音频，返回每小段的信息 - 基于词对齐结果中的标点符号划分句子
+        """
+        # 1. 获取文本片段和对应的说话人（用于获取speaker标签）
+        text_segments = self.map_text_segments_to_speakers(text)
+        
+        # 2. 删除标签并替换标点符号
+        clean_text = self.remove_speaker_tags(text)
+        
+        # 3. 检测语言或使用设置的语言
+        alignment_language = self.language
+        if alignment_language == "AUTO":
+            alignment_language = self._detect_language_from_text(clean_text)
+        
+        # 使用检测到的语言替换标点符号
+        comma_text = self.replace_punctuation_with_comma(clean_text, alignment_language)
+        
+        # 4. 词对齐 - 如果失败则直接抛出异常
+        alignments = self.align_text_with_audio(comma_text, output_audio, alignment_language)
+        
+        # 5. 根据标点符号划分句子
+        segments = []
+        safe_audio_id = self._get_safe_filename(audio_id)
+        
+        # 确定标点符号（根据语言选择，英文不包含撇号）
+        if alignment_language == "EN" or (alignment_language == "AUTO" and self._is_english_text(clean_text)):
+            punctuation_chars = set([',', '.', '!', '?', ';', ':'])  # 不包含撇号
+        else:
+            punctuation_chars = set(['，', '。', '！', '？', '；', '：'])
+        
+        # 顺序扫描对齐结果，根据标点符号划分句子
+        sentence_start_idx = 0
+        sentence_alignments = []
+        segment_id = 0
+        
+        for i, align_item in enumerate(alignments):
+            transcript = align_item['transcript']
+            sentence_alignments.append(align_item)
+            
+            # 检查是否包含标点符号（句子结束标志）
+            has_punctuation = any(punct in transcript for punct in punctuation_chars)
+            
+            if has_punctuation or i == len(alignments) - 1:  # 遇到标点符号或最后一个词
+                # 创建句子片段
+                if sentence_alignments:
+                    # 获取句子的开始和结束时间
+                    start_time = sentence_alignments[0]['start']
+                    end_time = sentence_alignments[-1]['end']
+                    
+                    # 构建句子文本（去除标点符号）
+                    sentence_text_parts = []
+                    for align in sentence_alignments:
+                        # 根据语言选择不同的清理策略
+                        if alignment_language == "EN" or (alignment_language == "AUTO" and self._is_english_text(clean_text)):
+                            # 英文：去除标点符号，但保留撇号已被删除的单词
+                            clean_transcript = align['transcript'].rstrip(',.!?;:')
+                        else:
+                            # 中文：去除中文标点符号
+                            clean_transcript = align['transcript'].rstrip('，。！？；：')
+                        
+                        if clean_transcript.strip():
+                            sentence_text_parts.append(clean_transcript)
+                    
+                    # 根据语言选择连接方式
+                    if alignment_language == "EN" or (alignment_language == "AUTO" and self._is_english_text(clean_text)):
+                        sentence_text = ' '.join(sentence_text_parts).strip()  # 英文用空格连接
+                    else:
+                        sentence_text = ''.join(sentence_text_parts).strip()  # 中文直接连接
+                    
+                    if sentence_text:  # 只有非空句子才处理
+                        # 确定说话人标签（从原始text_segments中获取，如果可能的话）
+                        speaker_label = "S1"  # 默认
+                        if segment_id < len(text_segments):
+                            speaker_label = text_segments[segment_id]['speaker_label']
+                        elif text_segments:
+                            # 如果超出范围，使用最后一个片段的speaker
+                            speaker_label = text_segments[-1]['speaker_label']
+                        
+                        # 生成音频文件路径
+                        safe_text = self._get_safe_filename(sentence_text, 30)
+                        audio_path = str(self.segments_dir / f"{safe_audio_id}_segment_{segment_id:03d}_{safe_text}.wav")
+                        
+                        # 分割音频
+                        try:
+                            self.split_audio_segment(output_audio, start_time, end_time, audio_path)
+                        except Exception as e:
+                            self.logger.error(f"分割音频失败: {e}")
+                            # 使用默认时间间隔
+                            start_time = segment_id * 1.0
+                            end_time = (segment_id + 1) * 1.0
+                            self.split_audio_segment(output_audio, start_time, end_time, audio_path)
+                        
+                        # 创建segment
+                        segment = {
+                            'segment_id': segment_id,
+                            'text': sentence_text,
+                            'speaker_label': speaker_label,
+                            'original_speaker_content': sentence_text,  # 这里简化处理
+                            'audio_path': audio_path,
+                            'start_time': start_time,
+                            'end_time': end_time
+                        }
+                        
+                        segments.append(segment)
+                        
+                        self.logger.info(f"句子 {segment_id}: '{sentence_text}' ({speaker_label}) -> {start_time:.3f}-{end_time:.3f}s")
+                        segment_id += 1
+                
+                # 重置为下一个句子
+                sentence_alignments = []
+                sentence_start_idx = i + 1
+        
+        # 保存详细的对齐信息
+        self.save_detailed_alignment_info(
+            alignments, segments, audio_id, output_audio, text, comma_text
+        )
+        
+        self.logger.info(f"总共分割出 {len(segments)} 个句子片段")
+        return segments
+    
+    def _get_similarity_model_server(self):
+        """获取线程局部的相似度模型实例（线程安全）"""
+        if not hasattr(self, 'similarity_model'):
+            # 为当前线程创建独立的模型实例
+            self.similarity_model = self._create_similarity_model()
+        return self.similarity_model
+    
+    def _create_similarity_model(self):
+        """创建新的相似度模型实例"""
+        try:
+            return TritonSpeakerClient(self.wespeaker_model_url)       
+        except Exception as e:
+            self.logger.error(f"创建相似度模型失败: {e}")
+            raise
+        
+    async def compute_similarity(self, processed_audio1, processed_audio2):
+        return await self.similarity_model.compute_similarity(processed_audio1, processed_audio2)
+        
+    async def calculate_voice_similarity_thread_safe(self, audio1_path: str, audio2_path: str) -> float:
+        """
+        线程安全的音色相似度计算
+        对于过短的音频片段，通过复制来达到最小长度要求
+        """
+        try:
+            if not os.path.exists(audio1_path) or not os.path.exists(audio2_path):
+                self.logger.warning(f"Audio file not found: {audio1_path} or {audio2_path}")
+                return None
+            
+            # 获取线程局部的模型实例
+            _ = self._get_similarity_model_server()
+            
+            # 计算相似度
+            similarity = await self.compute_similarity(audio1_path, audio2_path)
+            
+            return float(similarity)
+            
+        except Exception as e:
+            # 检查是否是窗口大小错误或其他计算错误
+            if "choose a window size" in str(e) or "window size" in str(e):
+                self.logger.warning(f"音频片段仍然过短，无法计算相似度: {audio1_path} vs {audio2_path}")
+                return None
+            else:
+                self.logger.error(f"Failed to compute similarity between {audio1_path} and {audio2_path}: {e}")
+                return None
+    
+    async def calculate_segment_similarities_parallel(
+        self, output_segments: List[Dict[str, Any]], prompts1_path: str, prompts2_path: str
+    ) -> List[Dict[str, Any]]:
+        """
+        并行计算所有segments的相似度
+        Args:
+            output_segments: 音频segments列表
+            prompts1_path: S1 prompt音频路径  
+            prompts2_path: S2 prompt音频路径
+        Returns:
+            包含相似度信息的segment列表
+        """
+        
+        async def calculate_single_segment_similarity(segment):
+            """计算单个segment与两个prompts的相似度"""
+            try:
+                # 使用线程安全的相似度计算方法
+                sim1 = await self.calculate_voice_similarity_thread_safe(segment['audio_path'], prompts1_path)
+                sim2 = await self.calculate_voice_similarity_thread_safe(segment['audio_path'], prompts2_path)
+                
+                return {
+                    'segment': segment,
+                    'sim1': sim1,
+                    'sim2': sim2,
+                    'success': True
+                }
+            except Exception as e:
+                self.logger.error(f"计算segment {segment['segment_id']} 相似度失败: {e}")
+                return {
+                    'segment': segment,
+                    'sim1': None,
+                    'sim2': None,
+                    'success': False
+                }
+        
+        # 使用线程池并行处理所有segments
+        self.logger.info(f"开始异步计算 {len(output_segments)} 个segments的相似度")
+        
+        # 创建任务并保留原始segment的顺序（gather会保持顺序）
+        tasks = [
+            asyncio.create_task(calculate_single_segment_similarity(segment))
+            for segment in output_segments
+        ]
+        
+        # 正确版本：使用asyncio.as_completed实时报告进度
+        return await self._run_tasks_with_progress(tasks)
+
+    # 新增辅助方法：带进度报告的任务执行
+    async def _run_tasks_with_progress(self, tasks):
+        """执行任务集合并实时报告进度"""
+        completed_count = 0
+        total = len(tasks)
+        results = []
+        
+        # 按完成顺序处理结果
+        for future in asyncio.as_completed(tasks):
+            result = await future
+            completed_count += 1
+            
+            # 每完成10个segment报告一次进度
+            if completed_count % 10 == 0 or completed_count == total:
+                seg_id = result['segment']['segment_id']
+                self.logger.info(f"相似度计算进度: {completed_count}/{total} (最近完成: {seg_id})")
+
+            results.append(result)
+        
+        # gather返回的就是按顺序的结果，无需额外排序
+        return results
+
+    async def evaluate_single_input(self, data: Dict[str, Any], input_id: str = None) -> Dict[str, Any]:
+        """评估单个输入的音色相似度"""
+        
+        # 生成输入ID
+        if input_id is None:
+            input_id = f"input_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+        
+        self.logger.info(f"开始评估输入: {input_id}，使用语言: {self.language}")
+        
+        # 1. 获取或分割prompt音频
+        prompts1_path, prompts2_path = self.get_or_split_prompt_audio(data, f"{input_id}_prompt")
+        
+        # 2. 分割output音频（这里会保存详细对齐信息）
+        output_segments = self.split_output_audio_by_comma(data['text'], data['output_audio'], f"{input_id}_output")
+        
+        # 3. 并行计算每小段的相似度
+        similarity_results = await self.calculate_segment_similarities_parallel(
+            output_segments, prompts1_path, prompts2_path
+        )
+        
+        # 4. 处理相似度结果
+        segment_results = []
+        correct_predictions = 0
+        total_segments = 0  # 只计算有效段数
+        label_similarities = []  # 每小段与其标签的相似度
+        skipped_segments = 0  # 跳过的段数
+        
+        for sim_result in similarity_results:
+            segment = sim_result['segment']
+            sim1 = sim_result['sim1']
+            sim2 = sim_result['sim2']
+            
+            # 如果任一相似度为None（音频过短或计算失败），跳过该段
+            if sim1 is None or sim2 is None:
+                skipped_segments += 1
+                self.logger.info(f"跳过段 {segment['segment_id']}: 相似度计算失败")
+                continue
+            
+            # 只有有效段才参与计算
+            total_segments += 1
+            
+            # 判断实际音色
+            predicted_speaker = 'S1' if sim1 > sim2 else 'S2'
+            actual_speaker = segment['speaker_label']
+            is_correct = predicted_speaker == actual_speaker
+            
+            if is_correct:
+                correct_predictions += 1
+            
+            # 计算与标签的相似度
+            if actual_speaker == 'S1':
+                label_similarity = sim1
+            else:
+                label_similarity = sim2
+            label_similarities.append(label_similarity)
+            
+            segment_result = {
+                'segment_id': segment['segment_id'],
+                'text': segment['text'],
+                'speaker_label': actual_speaker,
+                'predicted_speaker': predicted_speaker,
+                'sim1': sim1,
+                'sim2': sim2,
+                'label_similarity': label_similarity,
+                'is_correct': is_correct,
+                'audio_path': segment['audio_path'],
+                'start_time': segment.get('start_time', 0.0),
+                'end_time': segment.get('end_time', 1.0)
+            }
+            segment_results.append(segment_result)
+        
+        # 4. 计算整体指标（只基于有效段）
+        accuracy = correct_predictions / total_segments if total_segments > 0 else 0.0
+        average_similarity = np.mean(label_similarities) if label_similarities else 0.0
+        
+        # 5. 保存评估结果的对齐信息摘要
+        evaluation_alignment_summary = {
+            'input_id': input_id,
+            'language': self.language,
+            'prompt_alignment_files': [
+                f"{self._get_safe_filename(f'{input_id}_prompt')}_prompt_alignment.json"
+            ],
+            'output_alignment_file': f"{self._get_safe_filename(f'{input_id}_output')}_detailed_alignment.json",
+            'total_segments': total_segments,
+            'total_alignments_processed': len(output_segments),
+            'alignment_success_rate': total_segments / len(output_segments) if output_segments else 0.0
+        }
+        self.save_alignment_info(evaluation_alignment_summary, input_id, "summary")
+        
+        result = {
+            'input_id': input_id,
+            'language': self.language,
+            'input_data': data,  # 保存原始输入数据
+            'prompts1_path': prompts1_path,
+            'prompts2_path': prompts2_path,
+            'segments': segment_results,
+            'accuracy': accuracy,
+            'average_similarity': average_similarity,
+            'total_segments': total_segments,  # 有效段数
+            'correct_predictions': correct_predictions,
+            'skipped_segments': skipped_segments,  # 跳过的段数
+            'original_total_segments': len(output_segments),  # 原始总段数
+            'alignment_files': {
+                'summary': f"{self._get_safe_filename(input_id)}_summary_alignment.json",
+                'output_detailed': f"{self._get_safe_filename(f'{input_id}_output')}_detailed_alignment.json",
+                'prompt': f"{self._get_safe_filename(f'{input_id}_prompt')}_prompt_alignment.json"
+            },
+            'timestamp': datetime.now().isoformat()
+        }
+        
+        self.logger.info(f"完成评估输入: {input_id}, 语言: {self.language}, 有效段: {total_segments}/{len(output_segments)}, 跳过: {skipped_segments}, 准确率: {accuracy:.3f}, 平均相似度: {average_similarity:.3f}")
+        
+        return result
+
+    def save_results_to_jsonl(self, results: List[Dict[str, Any]], filename: str = None):
+        """保存结果到JSONL文件"""
+        if filename is None:
+            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+            filename = f"speaker_similarity_results_{self.language.lower()}_{timestamp}.jsonl"
+        
+        output_path = self.results_dir / filename
+        
+        with open(output_path, 'w', encoding='utf-8') as f:
+            for result in results:
+                f.write(json.dumps(result, ensure_ascii=False) + '\n')
+        
+        return str(output_path)
+    
+    def save_summary_report(self, results: List[Dict[str, Any]], filename: str = None):
+        """保存汇总报告"""
+        if filename is None:
+            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+            filename = f"evaluation_summary_{self.language.lower()}_{timestamp}.json"
+        
+        summary_path = self.results_dir / filename
+        
+        # 计算总体统计
+        total_accuracy = np.mean([r['accuracy'] for r in results])
+        total_avg_similarity = np.mean([r['average_similarity'] for r in results])
+        total_segments = sum([r['total_segments'] for r in results])
+        total_correct = sum([r['correct_predictions'] for r in results])
+        
+        summary = {
+            'evaluation_summary': {
+                'language': self.language,
+                'total_inputs': len(results),
+                'total_segments': total_segments,
+                'total_correct_predictions': total_correct,
+                'overall_accuracy': total_accuracy,
+                'overall_average_similarity': total_avg_similarity,
+                'evaluation_timestamp': datetime.now().isoformat(),
+                'output_directory': str(self.output_dir),
+                'alignment_directory': str(self.alignment_dir)
+            },
+            'per_input_results': [
+                {
+                    'input_id': r['input_id'],
+                    'language': r.get('language', self.language),
+                    'accuracy': r['accuracy'],
+                    'average_similarity': r['average_similarity'],
+                    'total_segments': r['total_segments'],
+                    'correct_predictions': r['correct_predictions'],
+                    'output_audio_path': r['input_data']['output_audio'],
+                    'alignment_files': r.get('alignment_files', {})
+                }
+                for r in results
+            ]
+        }
+        
+        with open(summary_path, 'w', encoding='utf-8') as f:
+            json.dump(summary, f, ensure_ascii=False, indent=2)
+        
+        return str(summary_path)
+    
+    def process_batch_from_jsonl_parallel(self, jsonl_path: str, 
+                                        processes_per_gpu: int = 16,
+                                        results_filename: str = None,
+                                        shuffle_data: bool = True):
+        """从JSONL文件并行批量处理输入数据"""
+        # 加载数据
+        input_data = self.load_data_from_jsonl(jsonl_path)
+        
+        if not input_data:
+            self.logger.error("没有有效的输入数据")
+            return []
+        
+        # 对数据进行shuffle，使分配更均匀
+        if shuffle_data:
+            random.shuffle(input_data)
+            self.logger.info(f"已对 {len(input_data)} 条数据进行随机shuffle")
+        
+        return self.process_batch_parallel(input_data, processes_per_gpu, results_filename)
+
+    def process_batch_from_jsonl(self, jsonl_path: str, results_filename: str = None):
+        """从JSONL文件批量处理输入数据（单进程版本）"""
+        # 加载数据
+        input_data = self.load_data_from_jsonl(jsonl_path)
+        
+        if not input_data:
+            self.logger.error("没有有效的输入数据")
+            return []
+        
+        return asyncio.run(self.process_batch_from_data(input_data, results_filename))
+
+    async def process_batch_from_data(self, input_data: List[Dict[str, Any]], results_filename: str = None):
+        """处理数据列表（单进程版本，用于兼容），支持增量写入"""
+        # 准备结果文件
+        if results_filename is None:
+            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+            results_filename = f"speaker_similarity_results_{self.language.lower()}_{timestamp}.jsonl"
+        
+        results_path = self.results_dir / results_filename
+        
+        # 如果文件已存在，删除它（重新开始）
+        if results_path.exists():
+            results_path.unlink()
+        
+        results = []
+        
+        self.logger.info(f"开始处理 {len(input_data)} 个输入，使用语言: {self.language}...")
+        
+        for i, data in enumerate(input_data):
+            input_id = f"input_{i+1:03d}"
+            print(f"处理第{i+1}/{len(input_data)}个输入: {input_id}，语言: {self.language}")
+            
+            try:
+                result = await self.evaluate_single_input(data, input_id=input_id)
+                results.append(result)
+                
+                # 增量写入结果
+                self.append_result_to_jsonl(result, str(results_path))
+                
+            except Exception as e:
+                self.logger.error(f"处理输入{input_id}时出错: {e}")
+                continue
+        
+        if not results:
+            self.logger.error("没有成功处理的输入")
+            return []
+        
+        # 保存汇总报告
+        summary_path = self.save_summary_report(results)
+        
+        # 清理临时文件
+        self._clean_temp_files()
+        
+        # 打印总体统计
+        total_accuracy = np.mean([r['accuracy'] for r in results])
+        total_avg_similarity = np.mean([r['average_similarity'] for r in results])
+        
+        print(f"\n=== 评估完成 ===")
+        print(f"使用语言: {self.language}")
+        print(f"总体准确率: {total_accuracy:.3f}")
+        print(f"总体平均相似度: {total_avg_similarity:.3f}")
+        print(f"详细结果已保存到: {results_path}")
+        print(f"汇总报告已保存到: {summary_path}")
+        print(f"对齐信息已保存到: {self.alignment_dir}")
+        print(f"所有中间文件保存在: {self.output_dir}")
+        
+        return results
+
+    def _load_wespeaker_model(self, wespeaker_model_url):
+        """加载wespeaker模型"""
+        try:
+            self.similarity_model = TritonSpeakerClient(wespeaker_model_url)
+        except ImportError:
+            raise ImportError("请安装wespeaker: pip install git+https://github.com/wenet-e2e/wespeaker.git")
+        except Exception as e:
+            self.logger.error(f"加载wespeaker模型失败: {e}")
+            raise
+
+    def load_data_from_jsonl(self, jsonl_path: str) -> List[Dict[str, Any]]:
+        """从JSONL文件加载数据"""
+        data = []
+        try:
+            with open(jsonl_path, 'r', encoding='utf-8') as f:
+                for line_num, line in enumerate(f, 1):
+                    line = line.strip()
+                    if line:
+                        try:
+                            item = json.loads(line)
+                            # 验证必要字段
+                            required_fields = ['text', 'output_audio']
+                            for field in required_fields:
+                                if field not in item:
+                                    self.logger.error(f"第{line_num}行缺少必要字段: {field}")
+                                    continue
+                            
+                            # 验证音频路径模式：要么有prompt_audio和prompt_text，要么有分别的speaker音频文件
+                            has_combined_prompt = 'prompt_audio' in item and 'prompt_text' in item
+                            has_separate_prompts = ('prompt_audio_speaker1' in item and 
+                                                  'prompt_text_speaker1' in item and
+                                                  'prompt_audio_speaker2' in item and 
+                                                  'prompt_text_speaker2' in item)
+                            
+                            if not (has_combined_prompt or has_separate_prompts):
+                                self.logger.error(f"第{line_num}行：需要提供prompt_audio+prompt_text或者分别的speaker音频文件")
+                                continue
+                            
+                            data.append(item)
+                            
+                        except json.JSONDecodeError as e:
+                            self.logger.error(f"第{line_num}行JSON解析错误: {e}")
+                            continue
+                            
+            self.logger.info(f"从{jsonl_path}成功加载{len(data)}条数据")
+            return data
+            
+        except FileNotFoundError:
+            self.logger.error(f"JSONL文件不存在: {jsonl_path}")
+            return []
+        except Exception as e:
+            self.logger.error(f"读取JSONL文件失败: {e}")
+            return []
+
+    @staticmethod
+    def get_gpu_count():
+        """获取可用GPU数量"""
+        if torch.cuda.is_available():
+            return torch.cuda.device_count()
+        return 0
+    
+    @staticmethod
+    def split_data_by_gpu(data: List[Dict[str, Any]], num_gpus: int) -> List[List[Dict[str, Any]]]:
+        """根据GPU数量分割数据"""
+        if num_gpus == 0:
+            return [data]
+        
+        chunk_size = math.ceil(len(data) / num_gpus)
+        gpu_chunks = []
+        
+        for i in range(num_gpus):
+            start_idx = i * chunk_size
+            end_idx = min((i + 1) * chunk_size, len(data))
+            if start_idx < len(data):
+                gpu_chunks.append(data[start_idx:end_idx])
+        
+        return gpu_chunks
+    
+    @staticmethod
+    def split_data_by_processes(data: List[Dict[str, Any]], num_processes: int) -> List[List[Dict[str, Any]]]:
+        """根据进程数量分割数据"""
+        if num_processes <= 1:
+            return [data]
+        
+        chunk_size = math.ceil(len(data) / num_processes)
+        process_chunks = []
+        
+        for i in range(num_processes):
+            start_idx = i * chunk_size
+            end_idx = min((i + 1) * chunk_size, len(data))
+            if start_idx < len(data):
+                process_chunks.append(data[start_idx:end_idx])
+        
+        return process_chunks
+
+    def append_result_to_jsonl(self, result: Dict[str, Any], filepath: str):
+        """增量写入结果到JSONL文件"""
+        os.makedirs(os.path.dirname(filepath), exist_ok=True)
+        with open(filepath, 'a', encoding='utf-8') as f:
+            f.write(json.dumps(result, ensure_ascii=False) + '\n')
+            f.flush()  # 强制刷新缓冲区
+
+    def merge_temp_results(self, temp_files: List[str], final_path: str):
+        """合并临时结果文件"""
+        all_results = []
+        
+        for temp_file in temp_files:
+            if os.path.exists(temp_file):
+                try:
+                    with open(temp_file, 'r', encoding='utf-8') as f:
+                        for line in f:
+                            line = line.strip()
+                            if line:
+                                result = json.loads(line)
+                                all_results.append(result)
+                except Exception as e:
+                    self.logger.error(f"读取临时文件失败: {temp_file}, 错误: {e}")
+        
+        # 写入最终文件
+        with open(final_path, 'w', encoding='utf-8') as f:
+            for result in all_results:
+                f.write(json.dumps(result, ensure_ascii=False) + '\n')
+        
+        return all_results
+
+    def process_batch_parallel(self, input_data: List[Dict[str, Any]], 
+                             processes_per_gpu: int = 8,  # 降低进程数
+                             results_filename: str = None,
+                             shuffle_data: bool = True):
+        """并行批量处理输入数据"""
+        # 1. 检查GPU数量
+        num_gpus = self.get_gpu_count()
+        if num_gpus == 0:
+            self.logger.warning("未检测到GPU，将使用CPU单进程处理")
+            return asyncio.run(self.process_batch_from_data(input_data, results_filename))
+        
+        # 限制每个GPU的进程数，避免CUDA内存冲突
+        max_processes_per_gpu = min(processes_per_gpu, 16)
+        self.logger.info(f"检测到 {num_gpus} 个GPU，每个GPU将使用 {max_processes_per_gpu} 个进程")
+        
+        # 2. 对数据进行shuffle（如果还没有shuffle过）
+        shuffled_data = input_data.copy()
+        if shuffle_data:
+            random.shuffle(shuffled_data)
+            self.logger.info(f"已对 {len(shuffled_data)} 条数据进行随机shuffle以平衡GPU负载")
+        
+        # 3. 按GPU分割数据
+        gpu_chunks = self.split_data_by_gpu(shuffled_data, num_gpus)
+        
+        # 打印每个GPU分配到的数据量
+        for gpu_id, gpu_data in enumerate(gpu_chunks):
+            if gpu_data:
+                self.logger.info(f"GPU {gpu_id}: 分配到 {len(gpu_data)} 条数据")
+        
+        # 4. 准备结果文件路径
+        if results_filename is None:
+            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+            results_filename = f"speaker_similarity_results_{self.language.lower()}_{timestamp}.jsonl"
+        
+        final_results_path = self.results_dir / results_filename
+        
+        # 5. 为所有GPU准备进程参数
+        all_temp_files = []
+        all_gpu_tasks = []
+        
+        for gpu_id, gpu_data in enumerate(gpu_chunks):
+            if not gpu_data:
+                continue
+                
+            self.logger.info(f"GPU {gpu_id}: 准备处理 {len(gpu_data)} 条数据")
+            
+            # 按进程数分割当前GPU的数据
+            process_chunks = self.split_data_by_processes(gpu_data, max_processes_per_gpu)
+            
+            # 为当前GPU准备所有进程参数
+            gpu_process_args = []
+            for proc_id, proc_data in enumerate(process_chunks):
+                if proc_data:
+                    temp_result_file = str(self.temp_results_dir / f"gpu{gpu_id}_proc{proc_id}_results.jsonl")
+                    all_temp_files.append(temp_result_file)
+                    
+                    # 子进程输出目录在主输出目录内部
+                    subprocess_output_dir = str(self.output_dir / f"gpu{gpu_id}_proc{proc_id}")
+                    
+                    gpu_process_args.append((
+                        proc_data, 
+                        gpu_id, 
+                        proc_id, 
+                        subprocess_output_dir,
+                        temp_result_file,
+                        self.alignment_model_dir,
+                        self.wespeaker_model_url,
+                        self.language,  # 语言参数
+                        self.similarity_max_workers  # 添加相似度计算线程数参数
+                    ))
+            
+            if gpu_process_args:
+                all_gpu_tasks.append((gpu_id, gpu_process_args, max_processes_per_gpu))
+        
+        # 6. 使用ThreadPoolExecutor并行处理所有GPU
+        def process_gpu_tasks(gpu_task):
+            gpu_id, process_args, actual_processes = gpu_task
+            self.logger.info(f"GPU {gpu_id}: 开始并行处理 {len(process_args)} 个进程")
+            
+            # 为每个GPU使用独立的进程池，避免进程间冲突
+            with mp.Pool(processes=actual_processes) as pool:
+                # 调用同步包装器 run_async_worker，在每个子进程内部运行异步函数。
+                pool.map(run_async_worker, process_args)
+            
+            self.logger.info(f"GPU {gpu_id}: 所有进程处理完成")
+            return gpu_id
+
+        # 使用线程池同时处理所有GPU
+        with ThreadPoolExecutor(max_workers=num_gpus) as executor:
+            # 提交所有GPU任务
+            future_to_gpu = {executor.submit(process_gpu_tasks, gpu_task): gpu_task[0] 
+                           for gpu_task in all_gpu_tasks}
+            
+            # 等待所有GPU完成
+            completed_gpus = []
+            for future in as_completed(future_to_gpu):
+                gpu_id = future_to_gpu[future]
+                try:
+                    result_gpu_id = future.result()
+                    completed_gpus.append(result_gpu_id)
+                    self.logger.info(f"GPU {result_gpu_id} 完成处理")
+                except Exception as exc:
+                    self.logger.error(f"GPU {gpu_id} 处理时发生异常: {exc}")
+        
+        self.logger.info(f"所有GPU处理完成: {completed_gpus}")
+        
+        # 7. 合并所有临时结果文件
+        self.logger.info("合并所有临时结果文件...")
+        all_results = self.merge_temp_results(all_temp_files, str(final_results_path))
+        
+        if not all_results:
+            self.logger.error("没有成功处理的数据")
+            return []
+        
+        # 8. 生成汇总报告
+        summary_path = self.save_summary_report(all_results)
+        
+        # 9. 清理临时文件
+        for temp_file in all_temp_files:
+            if os.path.exists(temp_file):
+                os.remove(temp_file)
+        
+        # 10. 打印总体统计
+        total_accuracy = np.mean([r['accuracy'] for r in all_results])
+        total_avg_similarity = np.mean([r['average_similarity'] for r in all_results])
+        
+        print(f"\n=== 并行评估完成 ===")
+        print(f"使用语言: {self.language}")
+        print(f"使用 {num_gpus} 个GPU，每GPU {max_processes_per_gpu} 个进程")
+        print(f"总处理数据: {len(input_data)} 条")
+        print(f"成功处理: {len(all_results)} 条")
+        print(f"总体准确率: {total_accuracy:.3f}")
+        print(f"总体平均相似度: {total_avg_similarity:.3f}")
+        print(f"详细结果已保存到: {final_results_path}")
+        print(f"汇总报告已保存到: {summary_path}")
+        print(f"对齐信息已保存到: {self.alignment_dir}")
+        
+        return all_results
+
+    def get_or_split_prompt_audio(self, data: Dict[str, Any], audio_id: str) -> Tuple[str, str]:
+        """
+        获取或分割prompt音频
+        如果提供了分别的speaker音频文件则直接使用，否则从combined prompt分割
+        """
+        # 检查是否有分别的speaker音频文件
+        if ('prompt_audio_speaker1' in data and 'prompt_audio_speaker2' in data and
+            'prompt_text_speaker1' in data and 'prompt_text_speaker2' in data):
+            
+            self.logger.info(f"使用预分割的speaker音频文件")
+            
+            # 即使使用预分割的音频，也保存对齐信息
+            try:
+                # 检测语言或使用设置的语言
+                alignment_language = self.language
+                if alignment_language == "AUTO":
+                    alignment_language = self._detect_language_from_text(data['prompt_text_speaker1'])
+                
+                # 对S1音频进行对齐
+                s1_alignments = self.align_text_with_audio(
+                    data['prompt_text_speaker1'], data['prompt_audio_speaker1'], alignment_language
+                )
+                s1_alignment_data = {
+                    'speaker': 'S1',
+                    'text': data['prompt_text_speaker1'],
+                    'audio_path': data['prompt_audio_speaker1'],
+                    'language': alignment_language,
+                    'alignments': s1_alignments
+                }
+                self.save_alignment_info(s1_alignment_data, audio_id, "prompt_s1")
+                
+                # 对S2音频进行对齐
+                s2_alignments = self.align_text_with_audio(
+                    data['prompt_text_speaker2'], data['prompt_audio_speaker2'], alignment_language
+                )
+                s2_alignment_data = {
+                    'speaker': 'S2',
+                    'text': data['prompt_text_speaker2'],
+                    'audio_path': data['prompt_audio_speaker2'],
+                    'language': alignment_language,
+                    'alignments': s2_alignments
+                }
+                self.save_alignment_info(s2_alignment_data, audio_id, "prompt_s2")
+                
+            except Exception as e:
+                self.logger.warning(f"保存预分割音频对齐信息失败: {e}")
+            
+            return data['prompt_audio_speaker1'], data['prompt_audio_speaker2']
+        
+        # 否则从combined prompt分割
+        elif 'prompt_audio' in data and 'prompt_text' in data:
+            self.logger.info(f"从combined prompt音频分割speaker片段")
+            return self.split_audio_by_speaker(data['prompt_text'], data['prompt_audio'], audio_id)
+        
+        else:
+            raise ValueError("必须提供prompt_audio+prompt_text或者分别的speaker音频文件")
+
+    def calculate_voice_similarity(self, audio1_path: str, audio2_path: str) -> float:
+        """
+        计算两个音频的音色相似度（向后兼容版本）
+        对于过短的音频片段，通过复制来达到最小长度要求
+        """
+        # 如果在多线程环境中，使用线程安全版本
+        if threading.current_thread() != threading.main_thread():
+            return self.calculate_voice_similarity_thread_safe(audio1_path, audio2_path)
+        
+        # 确保模型已初始化
+        self._init_models_if_needed()
+        
+        try:
+            if not os.path.exists(audio1_path) or not os.path.exists(audio2_path):
+                self.logger.warning(f"Audio file not found: {audio1_path} or {audio2_path}")
+                return None
+            
+            # 检查并处理音频文件长度
+            def process_audio_for_similarity(audio_path, min_duration=0.1):
+                """
+                处理音频文件，如果过短则复制到满足最小长度要求
+                返回处理后的音频路径和是否为临时文件的标志
+                """
+                try:
+                    waveform, sample_rate = torchaudio.load(audio_path)
+                    duration = waveform.shape[1] / sample_rate
+                    
+                    if duration >= min_duration:
+                        # 音频长度足够，直接返回原路径
+                        return audio_path, False
+                    
+                    # 音频过短，需要复制
+                    repeat_times = math.ceil(min_duration / duration)
+                    self.logger.info(f"音频过短 ({duration:.3f}s)，复制 {repeat_times} 次达到 {min_duration}s 要求: {audio_path}")
+                    
+                    # 复制音频
+                    repeated_waveform = waveform.repeat(1, repeat_times)
+                    
+                    # 生成临时文件路径
+                    temp_filename = f"temp_{os.path.basename(audio_path)}"
+                    temp_path = str(self.temp_dir / temp_filename)
+                    
+                    # 保存复制后的音频
+                    torchaudio.save(temp_path, repeated_waveform, sample_rate)
+                    
+                    return temp_path, True
+                    
+                except Exception as e:
+                    self.logger.error(f"处理音频文件失败: {audio_path}, 错误: {e}")
+                    return audio_path, False
+            
+            # 处理两个音频文件
+            processed_audio1, is_temp1 = process_audio_for_similarity(audio1_path)
+            processed_audio2, is_temp2 = process_audio_for_similarity(audio2_path)
+            
+            # 计算相似度
+            similarity = self.similarity_model.compute_similarity(processed_audio1, processed_audio2)
+            
+            # 清理临时文件
+            if is_temp1 and os.path.exists(processed_audio1):
+                try:
+                    os.remove(processed_audio1)
+                except Exception as e:
+                    self.logger.warning(f"删除临时文件失败: {processed_audio1}, 错误: {e}")
+            
+            if is_temp2 and os.path.exists(processed_audio2):
+                try:
+                    os.remove(processed_audio2)
+                except Exception as e:
+                    self.logger.warning(f"删除临时文件失败: {processed_audio2}, 错误: {e}")
+            
+            return float(similarity)
+            
+        except Exception as e:
+            # 检查是否是窗口大小错误或其他计算错误
+            if "choose a window size" in str(e) or "window size" in str(e):
+                self.logger.warning(f"音频片段仍然过短，无法计算相似度: {audio1_path} vs {audio2_path}")
+                return None
+            else:
+                self.logger.error(f"Failed to compute similarity between {audio1_path} and {audio2_path}: {e}")
+                return None
+
+# 全局函数，用于多进程处理（支持增量写入）
+async def process_data_chunk_incremental(args):
+    """处理数据块的工作函数（增量写入版本）"""
+    data_chunk, gpu_id, proc_id, output_dir, temp_result_file, alignment_model_dir, wespeaker_model_url, language, similarity_max_workers = args
+    
+    # 设置当前进程使用的GPU
+    device = f"cuda:{gpu_id}" if torch.cuda.is_available() and gpu_id < torch.cuda.device_count() else "cpu"
+    
+    try:
+        # 清理CUDA状态，避免进程间冲突
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+            # 设置当前进程的GPU设备
+            torch.cuda.set_device(gpu_id)
+            # 添加小延迟，避免同时初始化冲突
+            time.sleep(proc_id * 0.5)
+        
+        # 创建评估器实例，传入模型路径、语言参数和相似度计算线程数
+        evaluator = SpeakerSimilarityEvaluator(
+            device=device,
+            alignment_model_dir=alignment_model_dir,
+            wespeaker_model_url=wespeaker_model_url,
+            output_dir=output_dir,
+            language=language,  # 传入语言参数
+            similarity_max_workers=similarity_max_workers  # 传入相似度计算线程数
+        )
+        
+        # 延迟初始化模型
+        evaluator._init_models_if_needed()
+        
+        # 清空临时结果文件（如果存在）
+        if os.path.exists(temp_result_file):
+            os.remove(temp_result_file)
+        
+        # 处理数据块
+        for i, data in enumerate(data_chunk):
+            input_id = f"gpu{gpu_id}_proc{proc_id}_input_{i+1:03d}"
+            
+            try:
+                result = await evaluator.evaluate_single_input(data, input_id=input_id)
+                
+                # 立即写入结果到临时文件
+                evaluator.append_result_to_jsonl(result, temp_result_file)
+                
+                print(f"GPU{gpu_id}-进程{proc_id}: 完成 {input_id} (语言: {language}, 相似度线程: {similarity_max_workers})")
+                
+                # 每处理完一个数据项，清理CUDA缓存
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
+                    
+            except Exception as e:
+                print(f"GPU{gpu_id}-进程{proc_id}: 处理 {input_id} 失败: {e}")
+                # 出错时也清理CUDA缓存
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
+                continue
+        
+        print(f"GPU{gpu_id}-进程{proc_id}: 所有数据处理完成，结果已写入 {temp_result_file}")
+        
+    except Exception as e:
+        print(f"GPU{gpu_id}-进程{proc_id}: 初始化失败: {e}")
+        # 出错时清理CUDA缓存
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+
+
+def run_async_worker(args):
+    """
+    一个同步包装器，为我们的异步工作函数设置并运行asyncio事件循环。
+    这是必需的，因为 multiprocessing.Pool 不能直接调用异步函数。
+    """
+    # asyncio.run() 是在每个子进程中启动和运行协程最简单、最安全的方式。
+    # 它会创建一个新的事件循环，运行协程直到完成，然后关闭事件循环。
+    return asyncio.run(process_data_chunk_incremental(args))
+
+
+def main():
+    """主函数示例"""
+    import argparse
+    
+    parser = argparse.ArgumentParser(description='Speaker Similarity Evaluator')
+    parser.add_argument('--jsonl_path', type=str, help='JSONL文件路径')
+    parser.add_argument('--output_dir', type=str, 
+                       default=f"/inspire/hdd/project/embodied-multimodality/public/yqzhang/auto_evaluation_new/eval_res/results_{datetime.now().strftime('%Y%m%d_%H%M%S')}", 
+                       help='结果保存目录')
+    parser.add_argument('--language', type=str, choices=['zh', 'en', 'auto'], default='zh',
+                       help='指定语言: zh=中文, en=英文, auto=自动检测 (默认: zh)')
+    parser.add_argument('--no_parallel', action='store_true', help='禁用并行处理（默认启用并行）')
+    parser.add_argument('--processes_per_gpu', type=int, default=4, help='每个GPU的进程数（建议不超过4）')
+    parser.add_argument('--similarity_workers', type=int, default=16, help='相似度计算的线程数（默认: 8）')
+    parser.add_argument('--no_shuffle', action='store_true', help='禁用数据shuffle（默认启用shuffle）')
+    parser.add_argument('--random_seed', type=int, default=None, help='随机种子（可选，用于结果复现）')
+    
+    args = parser.parse_args()
+    
+    # 设置随机种子（如果指定）
+    if args.random_seed is not None:
+        random.seed(args.random_seed)
+        np.random.seed(args.random_seed)
+        torch.manual_seed(args.random_seed)
+        print(f"设置随机种子: {args.random_seed}")
+    
+    # 语言参数处理
+    language = args.language.upper()
+    if language == 'AUTO':
+        language = 'AUTO'
+    elif language == 'EN':
+        language = 'EN'
+    else:
+        language = 'ZH'  # 默认中文
+    
+    # 创建评估器，指定结果保存目录、语言和相似度计算线程数
+    evaluator = SpeakerSimilarityEvaluator(
+        output_dir=args.output_dir,
+        language=language,
+        similarity_max_workers=args.similarity_workers
+    )
+    
+    # 默认使用并行处理，除非明确禁用
+    use_parallel = not args.no_parallel
+    use_shuffle = not args.no_shuffle
+    
+    print(f"使用语言设置: {language}")
+    print(f"相似度计算线程数: {args.similarity_workers}")
+    
+    if args.jsonl_path:
+        # 从JSONL文件处理数据
+        if use_parallel:
+            evaluator.process_batch_from_jsonl_parallel(
+                args.jsonl_path, 
+                processes_per_gpu=args.processes_per_gpu,
+                shuffle_data=use_shuffle
+            )
+        else:
+            asyncio.run(evaluator.process_batch_from_jsonl(args.jsonl_path))
+    else:
+        # 使用示例数据（兼容性）
+        input_data = [
+            {
+                'prompt_audio': "/inspire/hdd/project/embodied-multimodality/public/yqzhang/infer_prompt/testset/audio/zhouxingchi/zxc_enhanced.wav",
+                'prompt_text': "[S1]你再往前半步我就把你给杀了。[S2]你应该这么做，我也应该死。",
+                'text': "[S1]至尊宝，如果有一天我不再是紫霞仙子，只是一个普通的凡人，你还会像现在这样陪着我吗？[S2]这个嘛，那我得先问问月老，看看他给不给我打折！毕竟追仙子要花好多力气的！[S1]哼！油嘴滑舌！我是认真的！[S2]紫霞，不管你是仙子还是凡人，哪怕变成一根香蕉，我都认得出你。不过……你最好别真变成香蕉，我怕我会忍不住吃掉……[S1]讨厌！谁要变成香蕉啊！那……如果有一天，我们不得不分开呢？[S2]哇！你这话比牛魔王的斧头还狠！不行不行，你得赔我精神损失费！[S1]怎么赔？[S2]很简单，让我亲一下，就当是定金！[S1]想得美！那如果有一天，你真的忘了我呢？[S2]那我就算翻遍三界，打烂阎王殿，也要把记忆找回来。紫霞，我至尊宝这辈子，赖定你了！[S1]傻瓜。",
+                'output_audio': "/inspire/hdd/project/embodied-multimodality/public/yqzhang/infer_res/from_newckpt_step145000/test_set/output_7.wav"
+            }
+        ]
+        
+        # 处理数据
+        if use_parallel:
+            evaluator.process_batch_parallel(input_data, processes_per_gpu=args.processes_per_gpu)
+        else:
+            asyncio.run(evaluator.process_batch_from_data(input_data))
+
+
+if __name__ == "__main__":
+    main()

test_online.sh

@@ -0,0 +1,150 @@
+#!/bin/bash
+
+# 设置CUDA环境变量
+export CUDA_LAUNCH_BLOCKING=1
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+
+# Color variables
+RESET='\033[0m'
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[0;33m'
+BLUE='\033[0;34m'
+MAGENTA='\033[0;35m'
+CYAN='\033[0;36m'
+WHITE='\033[0;37m'
+
+# 创建日志目录和文件名
+LOG_DIR="./logs"
+mkdir -p "$LOG_DIR"
+# 记录开始时间
+START_TIME=$(date +%s)
+START_TIME_READABLE=$(date '+%Y-%m-%d %H:%M:%S')
+LOG_TIME=$(date +%Y%m%d_%H%M%S)
+LOG_FILE="$LOG_DIR/evaluation_$LOG_TIME.log"
+SERVER_FILE="$LOG_DIR/server_$LOG_TIME.log"
+
+# A function to ensure the server is killed, which we'll call on exit.
+cleanup() {
+    echo "--- Cleanup ---"
+    # Check if the server process is still running
+    if kill -0 $SERVER_PID 2>/dev/null; then
+        echo "Client has finished. Sending SIGTERM to shut down the server (PID: $SERVER_PID)..."
+        # Send the SIGTERM signal, allowing the server to shut down gracefully if it handles the signal.
+        kill $SERVER_PID
+        # Wait a moment for it to terminate
+        wait $SERVER_PID 2>/dev/null
+        echo "Server has been shut down."
+    else
+        echo "Server (PID: $SERVER_PID) was already stopped."
+    fi
+}
+
+# Use 'trap' to register the 'cleanup' function to be called when the script exits.
+# This works for normal exit, Ctrl+C (SIGINT), or termination (SIGTERM).
+trap cleanup EXIT
+
+# 1. Start the server in the background
+echo "Starting alignment models' remote_server.py in the background..."
+/opt/tritonserver/bin/tritonserver --model-repository=./model_repo 2>&1 > $SERVER_FILE &
+
+# 2. Capture the Process ID (PID) of the server
+SERVER_PID=$!
+echo "Server started with PID: $SERVER_PID"
+
+# Give the server a moment to initialize and start listening on its port.
+# This is crucial, otherwise the client might try to connect before the server is ready.
+echo "Waited 3 seconds for server to initialize."
+echo "------------------------------------------"
+sleep 3
+
+echo "${GREEN}========================================="
+echo "音色相似度评估开始"
+echo "开始时间: $START_TIME_READABLE"
+echo "日志文件: $LOG_FILE"
+echo "========================================="
+echo "可以使用以下命令实时查看日志："
+echo "tail -f $LOG_FILE${RESET}"
+echo ""
+
+# 将开始时间信息也写入日志文件
+{
+    echo "${GREEN}========================================="
+    echo "音色相似度评估开始"
+    echo "开始时间: $START_TIME_READABLE"
+    echo "进程配置: 每GPU 8个进程"
+    echo "语言设置: zh (中文)"
+    echo "=========================================${RESET}"
+    echo ""
+} | tee "$LOG_FILE"
+
+# 3. Run the client in the foreground
+echo "Starting similarity test client test.py in the foreground..."
+# The script will pause here and wait for client.py to complete.
+# We wrap this in a block to capture the exit code.
+{
+    # 使用更保守的进程数
+    python -u ./test_online.py \
+        --jsonl_path /data-mnt/data/yqzhang/testset_ttsd/test_set_zh_304/output_new.jsonl \
+        --output_dir ./eval_res/new_test_online \
+        --processes_per_gpu 8 \
+        --language zh \
+        2>&1 | tee -a "$LOG_FILE"
+    CLIENT_EXIT_CODE=$?
+}
+echo "------------------------------------------"
+echo "${YELLOW}Client.py has finished with exit code: $CLIENT_EXIT_CODE${RESET}"
+# 记录结束时间
+END_TIME=$(date +%s)
+END_TIME_READABLE=$(date '+%Y-%m-%d %H:%M:%S')
+
+# 计算耗时
+DURATION=$((END_TIME - START_TIME))
+HOURS=$((DURATION / 3600))
+MINUTES=$(((DURATION % 3600) / 60))
+SECONDS=$((DURATION % 60))
+
+# 输出结束信息
+{
+    echo "${GREEN}"
+    echo "========================================="
+    echo "音色相似度评估完成！"
+    echo "结束时间: $END_TIME_READABLE"
+    echo "总耗时: ${HOURS}小时${MINUTES}分钟${SECONDS}秒 (共${DURATION}秒)"
+    echo "日志文件: $LOG_FILE"
+    echo "========================================="
+    echo "${RESET}"
+} | tee -a "$LOG_FILE"
+
+# 显示在终端
+echo "${GREEN}"
+echo "评估完成！"
+echo "开始时间: $START_TIME_READABLE"
+echo "结束时间: $END_TIME_READABLE"
+echo "总耗时: ${HOURS}小时${MINUTES}分钟${SECONDS}秒"
+echo "日志已保存到: $LOG_FILE"
+echo "${RESET}"
+
+# 如果耗时超过1小时，发送额外提醒
+if [ $DURATION -gt 3600 ]; then
+    echo "${RED}"
+    echo "⏰ 注意：本次评估耗时较长，超过1小时"
+    echo "   建议检查性能优化效果"
+    echo "${RESET}"
+fi
+
+# The 'trap' will automatically call the 'cleanup' function now that the script is exiting.
+# The exit is triggered because the client process (the last foreground command) has finished.
+
+# You can add logic based on the client's exit code if needed.
+if [ $CLIENT_EXIT_CODE -ne 0 ]; then
+    echo "Warning: Client exited with an error."
+    exit 1 # Exit the main script with an error code as well
+fi
+
+echo "Script finished successfully."
+exit 0
+
+
+
+