Update app.py

app.py

@@ -1,22 +1,159 @@
-from nemo.collections.asr.models import ASRModel
-import torch
+import os
+import time
 import gradio as gr
-import spaces
-import gc
-import shutil
-from pathlib import Path
-from pydub import AudioSegment
 import numpy as np
-import os
+import librosa
+import soundfile as sf
+from twilio.rest import Client
+from twilio.twiml.voice_response import VoiceResponse, Dial
+import requests
+from datetime import datetime
+import tempfile
+from nemo.collections.asr.models import ASRModel
+import torch
 import gradio.themes as gr_themes
 import csv
-import datetime
-from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
-from docx import Document
+from pathlib import Path
+import shutil
+import gc
 import re
 import threading
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+from docx import Document
+
+# Optional: Diarization
+try:
+    from pyannote.audio import Pipeline
+    HAVE_PYANNOTE = True
+except ImportError:
+    HAVE_PYANNOTE = False
+
+# ========== Twilio Functions ==========
+
+def get_twilio_credentials():
+    account_sid = os.environ.get("TWILIO_ACCOUNT_SID")
+    auth_token = os.environ.get("TWILIO_AUTH_TOKEN")
+    twilio_number = os.environ.get("TWILIO_PHONE_NUMBER")
+    return account_sid, auth_token, twilio_number
+
+def make_conference_call(phone_number, conference_code, wait_time=30):
+    try:
+        account_sid, auth_token, twilio_number = get_twilio_credentials()
+        if not all([account_sid, auth_token, twilio_number]):
+            return None, "Twilio credentials not found. Please set environment variables."
+        client = Client(account_sid, auth_token)
+        response = VoiceResponse()
+        response.say("Joining conference call. This call will be recorded for diarization.")
+        response.pause(length=2)
+        if conference_code:
+            for digit in conference_code:
+                if digit.isdigit() or digit in ['*', '#']:
+                    response.play(digits=digit)
+                    response.pause(length=1)
+        response.record(timeout=0, transcribe=False, recording_status_callback="/recording-status")
+        dial = Dial()
+        dial.conference('ConferenceRoom', record='record-from-start', recording_status_callback="/recording-status")
+        response.append(dial)
+        call = client.calls.create(
+            to=phone_number,
+            from_=twilio_number,
+            twiml=str(response),
+            record=True
+        )
+        return call.sid, f"Call initiated with SID: {call.sid}. Wait for the call to complete before retrieving the recording."
+    except Exception as e:
+        return None, f"Error initiating call: {str(e)}"
+
+def check_call_status(call_sid):
+    try:
+        account_sid, auth_token, _ = get_twilio_credentials()
+        if not all([account_sid, auth_token]):
+            return None, "Twilio credentials not found. Please set environment variables."
+        client = Client(account_sid, auth_token)
+        call = client.calls(call_sid).fetch()
+        if call.status in ['in-progress', 'queued', 'ringing']:
+            return None, f"Call is still {call.status}. Please check again later."
+        recordings = client.recordings.list(call_sid=call_sid)
+        if not recordings:
+            return None, "No recordings found for this call yet. Please check again later."
+        recording = recordings[0]
+        recording_url = f"https://api.twilio.com/2010-04-01/Accounts/{account_sid}/Recordings/{recording.sid}.wav"
+        response = requests.get(recording_url, auth=(account_sid, auth_token))
+        if response.status_code != 200:
+            return None, f"Failed to download recording: {response.status_code}"
+        temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.wav')
+        temp_file.write(response.content)
+        temp_file.close()
+        return temp_file.name, f"Recording downloaded successfully: {temp_file.name}"
+    except Exception as e:
+        return None, f"Error checking call status: {str(e)}"
+
+def upsample_audio(audio_file):
+    try:
+        y, sr = librosa.load(audio_file, sr=None)
+        if sr == 16000:
+            return audio_file, f"Audio is already at 16kHz: {audio_file}"
+        y_enhanced = librosa.effects.preemphasis(y, coef=0.97)
+        y_resampled = librosa.resample(y_enhanced, orig_sr=sr, target_sr=16000)
+        output_file = f"upsampled_{datetime.now().strftime('%Y%m%d_%H%M%S')}.wav"
+        sf.write(output_file, y_resampled, 16000)
+        return output_file, f"Audio upsampled successfully: {output_file}"
+    except Exception as e:
+        return None, f"Error upsampling audio: {str(e)}"
+
+def prepare_for_diarization(recording_file):
+    try:
+        if not recording_file:
+            return None, "No recording file provided."
+        upsampled_file, message = upsample_audio(recording_file)
+        if not upsampled_file:
+            return None, message
+        y, sr = librosa.load(upsampled_file, sr=16000)
+        S = librosa.stft(y)
+        S_filtered = librosa.decompose.nn_filter(
+            np.abs(S),
+            aggregate=np.median,
+            metric='cosine',
+            width=11
+        )
+        mask = librosa.util.softmask(
+            S_filtered,
+            np.abs(S) - S_filtered,
+            power=2
+        )
+        S_enhanced = S * mask
+        y_enhanced = librosa.istft(S_enhanced)
+        output_file = f"diarization_ready_{datetime.now().strftime('%Y%m%d_%H%M%S')}.wav"
+        sf.write(output_file, y_enhanced, 16000)
+        return output_file, f"Audio processed and ready for diarization: {output_file}"
+    except Exception as e:
+        return None, f"Error preparing audio for diarization: {str(e)}"
+
+# ========== Diarization Function (pyannote) ==========
+
+def diarize_audio(audio_file):
+    if not HAVE_PYANNOTE:
+        return None, "pyannote.audio not installed."
+    try:
+        hf_token = os.environ.get("HUGGINGFACE_TOKEN", None)
+        if not hf_token:
+            return None, "Hugging Face access token not set in environment as HUGGINGFACE_TOKEN."
+        pipeline = Pipeline.from_pretrained("pyannote/speaker-diarization", use_auth_token=hf_token)
+        diarization = pipeline(audio_file)
+        diarization_file = f"diarization_{datetime.now().strftime('%Y%m%d_%H%M%S')}.rttm"
+        with open(diarization_file, "w") as f:
+            diarization.write_rttm(f)
+        # For user: return path to RTTM file and a simple string report
+        report_lines = []
+        for turn, _, speaker in diarization.itertracks(yield_label=True):
+            report_lines.append(f"{turn.start:.1f}-{turn.end:.1f}s: {speaker}")
+        return diarization_file, "\n".join(report_lines)
+    except Exception as e:
+        return None, f"Error in diarization: {str(e)}"
 
-# ========== Qwen2.5-1.5B-Instruct Setup ==========
+# ========== ASR and Meeting Minutes Setup ==========
+
+# Qwen2.5-1.5B-Instruct Setup
 QWEN_MODEL = "Qwen/Qwen2.5-1.5B-Instruct"
 qwen_tokenizer = AutoTokenizer.from_pretrained(QWEN_MODEL)
 qwen_model = AutoModelForCausalLM.from_pretrained(
@@ -34,13 +171,11 @@ qwen_pipe = pipeline(
     temperature=0.3,
 )
 
-# ========== ASR Setup ==========
+# ASR Setup
 device = "cuda" if torch.cuda.is_available() else "cpu"
-MODEL_NAME="nvidia/parakeet-tdt-0.6b-v2"
+MODEL_NAME = "nvidia/parakeet-tdt-0.6b-v2"
 model = ASRModel.from_pretrained(model_name=MODEL_NAME)
 model.eval()
-
-# ========== Thread Lock for Model State ==========
 model_lock = threading.Lock()
 
 def start_session(request: gr.Request):
@@ -58,6 +193,7 @@ def end_session(request: gr.Request):
     print(f"Session with hash {session_hash} ended.")
 
 def get_audio_segment(audio_path, start_second, end_second):
+    from pydub import AudioSegment
     if not audio_path or not Path(audio_path).exists():
         print(f"Warning: Audio path '{audio_path}' not found or invalid for clipping.")
         return None
@@ -89,6 +225,7 @@ def get_audio_segment(audio_path, start_second, end_second):
         return None
 
 def format_srt_time(seconds: float) -> str:
+    import datetime
     sanitized_total_seconds = max(0.0, seconds)
     delta = datetime.timedelta(seconds=sanitized_total_seconds)
     total_int_seconds = int(delta.total_seconds())
@@ -111,8 +248,9 @@ def generate_srt_content(segment_timestamps: list) -> str:
         srt_content.append("")
     return "\n".join(srt_content)
 
-@spaces.GPU
 def get_transcripts_and_raw_times(audio_path, session_dir):
+    from pydub import AudioSegment
+    import gradio as gr
     if not audio_path:
         gr.Error("No audio file path provided for transcription.", duration=None)
         return [], [], None, gr.DownloadButton(label="Download Transcript (CSV)", visible=False), gr.DownloadButton(label="Download Transcript (SRT)", visible=False)
@@ -262,30 +400,6 @@ def get_transcripts_and_raw_times(audio_path, session_dir):
             except Exception as e:
                 print(f"Error removing temporary audio file {processed_audio_path}: {e}")
 
-def play_segment(evt: gr.SelectData, raw_ts_list, current_audio_path):
-    if not isinstance(raw_ts_list, list):
-        print(f"Warning: raw_ts_list is not a list ({type(raw_ts_list)}). Cannot play segment.")
-        return gr.Audio(value=None, label="Selected Segment")
-    if not current_audio_path:
-        print("No audio path available to play segment from.")
-        return gr.Audio(value=None, label="Selected Segment")
-    selected_index = evt.index[0]
-    if selected_index < 0 or selected_index >= len(raw_ts_list):
-        print(f"Invalid index {selected_index} selected for list of length {len(raw_ts_list)}.")
-        return gr.Audio(value=None, label="Selected Segment")
-    if not isinstance(raw_ts_list[selected_index], (list, tuple)) or len(raw_ts_list[selected_index]) != 2:
-        print(f"Warning: Data at index {selected_index} is not in the expected format [start, end].")
-        return gr.Audio(value=None, label="Selected Segment")
-    start_time_s, end_time_s = raw_ts_list[selected_index]
-    print(f"Attempting to play segment: {current_audio_path} from {start_time_s:.2f}s to {end_time_s:.2f}s")
-    segment_data = get_audio_segment(current_audio_path, start_time_s, end_time_s)
-    if segment_data:
-        print("Segment data retrieved successfully.")
-        return gr.Audio(value=segment_data, autoplay=True, label=f"Segment: {start_time_s:.2f}s - {end_time_s:.2f}s", interactive=False)
-    else:
-        print("Failed to get audio segment data.")
-        return gr.Audio(value=None, label="Selected Segment")
-
 def strip_markdown(text):
     text = re.sub(r'(\*\*|__)(.*?)\1', r'\2', text)
     text = re.sub(r'(\*|_)(.*?)\1', r'\2', text)
@@ -318,7 +432,6 @@ def generate_meeting_minutes(session_dir):
             f"{transcript}\n"
             "Structured Meeting Minutes:"
         )
-
         print("Sending prompt to Qwen2.5-1.5B-Instruct...")
         out = qwen_pipe(prompt)
         minutes = out[0]["generated_text"][len(prompt):].strip()
@@ -334,19 +447,12 @@ def generate_meeting_minutes(session_dir):
         print("Error in generate_meeting_minutes:", e)
         return f"Error generating minutes: {e}", None, gr.update(visible=True)
 
+# ========== Gradio UI ==========
+
 nvidia_theme = gr_themes.Default(
     primary_hue=gr_themes.Color(
-        c50="#E6F1D9",
-        c100="#CEE3B3",
-        c200="#B5D58C",
-        c300="#9CC766",
-        c400="#84B940",
-        c500="#76B900",
-        c600="#68A600",
-        c700="#5A9200",
-        c800="#4C7E00",
-        c900="#3E6A00",
-        c950="#2F5600"
+        c50="#E6F1D9", c100="#CEE3B3", c200="#B5D58C", c300="#9CC766", c400="#84B940",
+        c500="#76B900", c600="#68A600", c700="#5A9200", c800="#4C7E00", c900="#3E6A00", c950="#2F5600"
     ),
     neutral_hue="gray",
     font=[gr_themes.GoogleFont("Inter"), "ui-sans-serif", "system-ui", "sans-serif"],
@@ -358,11 +464,77 @@ with gr.Blocks(theme=nvidia_theme) as demo:
     session_dir = gr.State()
     demo.load(start_session, outputs=[session_dir])
 
-    mic_input = gr.Audio(sources=["microphone"], type="filepath", label="Record Audio")
-    mic_transcribe_btn = gr.Button("Transcribe Microphone Input", variant="primary")
-    file_input = gr.Audio(sources=["upload"], type="filepath", label="Upload Audio File")
-    file_transcribe_btn = gr.Button("Transcribe Uploaded File", variant="primary")
+    # ====== Twilio Tab ======
+    with gr.Tab("Twilio Call & Recording"):
+        gr.Markdown("### 1. Make Twilio Call and Record")
+        phone_number = gr.Textbox(label="Phone Number (E.164)", placeholder="+15551234567")
+        conference_code = gr.Textbox(label="Conference Code (optional)", placeholder="123456#")
+        call_btn = gr.Button("Make Call")
+        call_sid = gr.Textbox(label="Call SID", interactive=False)
+        call_status = gr.Textbox(label="Call Status", interactive=False)
+        call_btn.click(
+            make_conference_call,
+            inputs=[phone_number, conference_code],
+            outputs=[call_sid, call_status]
+        )
+
+        gr.Markdown("### 2. Retrieve Recording")
+        sid_input = gr.Textbox(label="Call SID")
+        get_recording_btn = gr.Button("Get Recording")
+        recording_path = gr.Textbox(label="Recording File Path", interactive=False)
+        recording_status = gr.Textbox(label="Recording Status", interactive=False)
+        get_recording_btn.click(
+            check_call_status,
+            inputs=[sid_input],
+            outputs=[recording_path, recording_status]
+        )
 
+        gr.Markdown("### 3. Process for Diarization")
+        process_btn = gr.Button("Process Recording")
+        process_status = gr.Textbox(label="Processing Status", interactive=False)
+        processed_file = gr.Textbox(label="Processed Audio File", interactive=False)
+        def process_and_return_file(recording_path):
+            file_path, message = prepare_for_diarization(recording_path)
+            if file_path:
+                return message, file_path
+            else:
+                return message, None
+        process_btn.click(
+            process_and_return_file,
+            inputs=[recording_path],
+            outputs=[process_status, processed_file]
+        )
+
+        gr.Markdown("### 4. (Optional) Diarize Processed Audio")
+        diarize_btn = gr.Button("Diarize Processed Audio")
+        diarization_status = gr.Textbox(label="Diarization Status/Report", interactive=False)
+        diarization_file = gr.File(label="RTTM Diarization File")
+        def diarize_and_return_file(processed_file):
+            diar_file, report = diarize_audio(processed_file)
+            return report, diar_file
+        diarize_btn.click(
+            diarize_and_return_file,
+            inputs=[processed_file],
+            outputs=[diarization_status, diarization_file]
+        )
+
+        gr.Markdown("### 5. Transcribe and Analyze Processed Audio")
+        transcribe_btn = gr.Button("Transcribe Processed Recording")
+        vis_timestamps_df = gr.DataFrame(
+            headers=["Start (s)", "End (s)", "Segment"],
+            datatype=["number", "number", "str"],
+            wrap=True,
+            label="Transcription Segments"
+        )
+        download_btn_csv = gr.DownloadButton(label="Download Transcript (CSV)", visible=False)
+        download_btn_srt = gr.DownloadButton(label="Download Transcript (SRT)", visible=False)
+        transcribe_btn.click(
+            get_transcripts_and_raw_times,
+            inputs=[processed_file, session_dir],
+            outputs=[vis_timestamps_df, raw_timestamps_list_state, current_audio_path_state, download_btn_csv, download_btn_srt],
+        )
+
+    # ====== Your Existing UI ======
     gr.Markdown("---")
     gr.Markdown("<p><strong style='color: #FF0000; font-size: 1.2em;'>Transcription Results (Click row to play segment)</strong></p>")
 
@@ -384,28 +556,57 @@ with gr.Blocks(theme=nvidia_theme) as demo:
 
     selected_segment_player = gr.Audio(label="Selected Segment", interactive=False)
 
+    mic_input = gr.Audio(sources=["microphone"], type="filepath", label="Record Audio")
+    mic_transcribe_btn = gr.Button("Transcribe Microphone Input", variant="primary")
+    file_input = gr.Audio(sources=["upload"], type="filepath", label="Upload Audio File")
+    file_transcribe_btn = gr.Button("Transcribe Uploaded File", variant="primary")
+
     mic_transcribe_btn.click(
-        fn=get_transcripts_and_raw_times,
+        get_transcripts_and_raw_times,
         inputs=[mic_input, session_dir],
         outputs=[vis_timestamps_df, raw_timestamps_list_state, current_audio_path_state, download_btn_csv, download_btn_srt],
         api_name="transcribe_mic"
     )
 
     file_transcribe_btn.click(
-        fn=get_transcripts_and_raw_times,
+        get_transcripts_and_raw_times,
         inputs=[file_input, session_dir],
         outputs=[vis_timestamps_df, raw_timestamps_list_state, current_audio_path_state, download_btn_csv, download_btn_srt],
         api_name="transcribe_file"
     )
 
     gen_minutes_btn.click(
-        fn=generate_meeting_minutes,
+        generate_meeting_minutes,
         inputs=[session_dir],
         outputs=[minutes_output, minutes_download, minutes_download],
     )
 
+    def play_segment(evt: gr.SelectData, raw_ts_list, current_audio_path):
+        if not isinstance(raw_ts_list, list):
+            print(f"Warning: raw_ts_list is not a list ({type(raw_ts_list)}). Cannot play segment.")
+            return gr.Audio(value=None, label="Selected Segment")
+        if not current_audio_path:
+            print("No audio path available to play segment from.")
+            return gr.Audio(value=None, label="Selected Segment")
+        selected_index = evt.index[0]
+        if selected_index < 0 or selected_index >= len(raw_ts_list):
+            print(f"Invalid index {selected_index} selected for list of length {len(raw_ts_list)}.")
+            return gr.Audio(value=None, label="Selected Segment")
+        if not isinstance(raw_ts_list[selected_index], (list, tuple)) or len(raw_ts_list[selected_index]) != 2:
+            print(f"Warning: Data at index {selected_index} is not in the expected format [start, end].")
+            return gr.Audio(value=None, label="Selected Segment")
+        start_time_s, end_time_s = raw_ts_list[selected_index]
+        print(f"Attempting to play segment: {current_audio_path} from {start_time_s:.2f}s to {end_time_s:.2f}s")
+        segment_data = get_audio_segment(current_audio_path, start_time_s, end_time_s)
+        if segment_data:
+            print("Segment data retrieved successfully.")
+            return gr.Audio(value=segment_data, autoplay=True, label=f"Segment: {start_time_s:.2f}s - {end_time_s:.2f}s", interactive=False)
+        else:
+            print("Failed to get audio segment data.")
+            return gr.Audio(value=None, label="Selected Segment")
+
     vis_timestamps_df.select(
-        fn=play_segment,
+        play_segment,
         inputs=[raw_timestamps_list_state, current_audio_path_state],
         outputs=[selected_segment_player],
     )