app.py

import tempfile

import gradio as gr
import soundfile as sf
import torchaudio
from cached_path import cached_path
from omegaconf import OmegaConf

from ipa.ipa import get_ipa, parse_ipa

try:
    import spaces

    USING_SPACES = True
except ImportError:
    USING_SPACES = False

from f5_tts.infer.utils_infer import (
    infer_process,
    load_model,
    load_vocoder,
    preprocess_ref_audio_text,
    remove_silence_for_generated_wav,
    save_spectrogram,
)
from f5_tts.model import DiT


def gpu_decorator(func):
    if USING_SPACES:
        return spaces.GPU(func)
    else:
        return func


vocoder = load_vocoder()


def load_f5tts(ckpt_path, vocab_path):
    ckpt_path = str(cached_path(ckpt_path))
    F5TTS_model_cfg = dict(
        dim=1024, depth=22, heads=16, ff_mult=2, text_dim=512, conv_layers=4
    )
    vocab_path = str(cached_path(vocab_path))
    return load_model(DiT, F5TTS_model_cfg, ckpt_path, vocab_file=vocab_path)


OmegaConf.register_new_resolver("load_f5tts", load_f5tts)

models_config = OmegaConf.to_object(OmegaConf.load("configs/models.yaml"))
dialects = OmegaConf.to_object(OmegaConf.load("configs/dialects.yaml"))


DEFAULT_MODEL_ID = list(models_config.keys())[0]
DEFAULT_DIALECT = list(dialects.values())[0]


@gpu_decorator
def infer(
    ref_audio_orig,
    ref_text,
    gen_text,
    model,
    remove_silence,
    cross_fade_duration=0.15,
    nfe_step=32,
    fix_duration=1,
    show_info=gr.Info,
):
    if not ref_audio_orig:
        gr.Warning("Please provide reference audio.")
        return gr.update(), gr.update(), ref_text

    if not gen_text.strip():
        gr.Warning("Please enter text to generate.")
        return gr.update(), gr.update(), ref_text

    ref_audio, ref_text = preprocess_ref_audio_text(
        ref_audio_orig, ref_text, show_info=show_info
    )

    final_wave, final_sample_rate, combined_spectrogram = infer_process(
        ref_audio,
        ref_text,
        gen_text,
        model,
        vocoder,
        cross_fade_duration=cross_fade_duration,
        nfe_step=nfe_step,
        fix_duration=fix_duration,
        show_info=show_info,
        progress=gr.Progress(),
    )

    # Remove silence
    if remove_silence:
        with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as f:
            sf.write(f.name, final_wave, final_sample_rate)
            remove_silence_for_generated_wav(f.name)
            final_wave, _ = torchaudio.load(f.name)
        final_wave = final_wave.squeeze().cpu().numpy()

    print(f"Final wave duration: {final_wave.shape[0] / final_sample_rate:.2f}s")
    # Save the spectrogram
    with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp_spectrogram:
        spectrogram_path = tmp_spectrogram.name
        save_spectrogram(combined_spectrogram, spectrogram_path)

    return (final_sample_rate, final_wave), spectrogram_path


demo = gr.Blocks(
    title="臺灣客語語音生成系統",
    css="@import url(https://tauhu.tw/tauhu-oo.css);",
    theme=gr.themes.Default(
        font=(
            "tauhu-oo",
            gr.themes.GoogleFont("Source Sans Pro"),
            "ui-sans-serif",
            "system-ui",
            "sans-serif",
        )
    ),
)

with demo:
    gr.Markdown(
        """
        # 臺灣客語語音合成系統
        ### Taiwanese Hakka Text-to-Speech System
        ### 研發團隊
        - **[李鴻欣 Hung-Shin Lee](mailto:[email protected])（[聯和科創](https://www.104.com.tw/company/1a2x6bmu75)）**
        - **[陳力瑋 Li-Wei Chen](mailto:[email protected])（[聯和科創](https://www.104.com.tw/company/1a2x6bmu75)）**
        ### 合作單位
        - **[國立聯合大學智慧客家實驗室](https://www.gohakka.org)**
        """
    )
    with gr.Row():
        with gr.Column():
            model_drop_down = gr.Dropdown(
                models_config.keys(),
                value=DEFAULT_MODEL_ID,
                label="模型",
            )

            ref_audio_input = gr.Audio(
                type="filepath",
                waveform_options=gr.WaveformOptions(
                    sample_rate=24000,
                ),
                label="Reference Audio",
            )
            ref_text_input = gr.Textbox(
                value="",
                label="Reference Text",
            )

            ref_dialect_radio = gr.Radio(
                choices=[(k, v) for k, v in dialects.items()],
                value=DEFAULT_DIALECT,
                label="ref 腔調",
                interactive=len(dialects.keys()) > 1,
            )

            gen_text_input = gr.Textbox(
                label="Text to Generate",
                value="",
            )

            dialect_radio = gr.Radio(
                choices=[(k, v) for k, v in dialects.items()],
                value=DEFAULT_DIALECT,
                label="腔調",
                interactive=len(dialects.keys()) > 1,
            )

            generate_btn = gr.Button("Synthesize", variant="primary")

            with gr.Accordion("Advanced Settings", open=False):
                remove_silence = gr.Checkbox(
                    label="Remove Silences",
                    info="The model tends to produce silences, especially on longer audio. We can manually remove silences if needed. Note that this is an experimental feature and may produce strange results. This will also increase generation time.",
                    value=False,
                )
                speed_slider = gr.Slider(
                    label="Speed",
                    minimum=0.3,
                    maximum=2.0,
                    value=1.0,
                    step=0.1,
                    info="語速（越小越慢）",
                )
                nfe_slider = gr.Slider(
                    label="NFE Steps",
                    minimum=4,
                    maximum=64,
                    value=32,
                    step=2,
                    info="Set the number of denoising steps.",
                )
                cross_fade_duration_slider = gr.Slider(
                    label="Cross-Fade Duration (s)",
                    minimum=0.0,
                    maximum=1.0,
                    value=0.15,
                    step=0.01,
                    info="Set the duration of the cross-fade between audio clips.",
                )
        with gr.Column():
            audio_output = gr.Audio(label="Synthesized Audio")
            spectrogram_output = gr.Image(label="Spectrogram")

    @gpu_decorator
    def basic_tts(
        model_drop_down: str,
        ref_audio_input: str,
        ref_text_input: str,
        ref_dialect_radio: str,
        gen_text_input: str,
        dialect_radio: str,
        remove_silence: bool,
        cross_fade_duration_slider: float,
        nfe_slider: int,
        speed_slider: float,
    ):
        ref_audio_info = torchaudio.info(ref_audio_input)
        ref_duration = ref_audio_info.num_frames / ref_audio_info.sample_rate
        target_duration = (
            ref_duration
            * len(gen_text_input.replace(" ", ""))
            / len(ref_text_input.replace(" ", ""))
            / speed_slider
        )
        print(f"Reference duration: {ref_duration}")
        print(f"Target duration: {target_duration}")
        if len(ref_text_input) == 0:
            raise gr.Error("請勿輸入空字串。")
        words, ipa, pinyin, missing_words = get_ipa(
            ref_text_input, dialect=ref_dialect_radio
        )
        if len(missing_words) > 0:
            raise gr.Error(
                f"參考句子中的[{','.join(missing_words)}]目前無法轉成 ipa。請嘗試其他句子。"
            )
        ref_text_input = parse_ipa(ipa)

        if len(gen_text_input) == 0:
            raise gr.Error("請勿輸入空字串。")
        words, ipa, pinyin, missing_words = get_ipa(
            gen_text_input, dialect=dialect_radio
        )
        if len(missing_words) > 0:
            raise gr.Error(
                f"生成句子中的[{','.join(missing_words)}]目前無法轉成 ipa。請嘗試其他句子。"
            )
        gen_text_input = parse_ipa(ipa)

        audio_out, spectrogram_path = infer(
            ref_audio_input,
            ref_text_input,
            gen_text_input,
            models_config[model_drop_down],
            remove_silence,
            cross_fade_duration=cross_fade_duration_slider,
            nfe_step=nfe_slider,
            fix_duration=ref_duration + target_duration,
        )
        return audio_out, spectrogram_path

    generate_btn.click(
        basic_tts,
        inputs=[
            model_drop_down,
            ref_audio_input,
            ref_text_input,
            ref_dialect_radio,
            gen_text_input,
            dialect_radio,
            remove_silence,
            cross_fade_duration_slider,
            nfe_slider,
            speed_slider,
        ],
        outputs=[audio_output, spectrogram_output],
    )
    gr.Examples(
        [
            [
                "./ref_wav/0000001_0.15-0.93.wav",
                "恁早",
                "sixian",
                "食飯愛正經食，正毋會食到半出半入",
                "sixian",
            ],
            [
                "./ref_wav/0000002_0.15-2.73.wav",
                "你今晡日著到恁派頭",
                "sixian",
                "食飯愛正經食，正毋會食到半出半入",
                "sixian",
            ],
            [
                "./ref_wav/0000002_0.15-2.73.wav",
                "你今晡日著到恁派頭",
                "sixian",
                "歸條路吊等長長个花燈，祈求風調雨順，歸屋下人个心願，親像花燈下燒暖个光華",
                "sixian",
            ],
            # [
            #     "預設語者",
            #     "戴君儒",
            #     "hailu",
            #     "男女平等个時代，平平做得受教育",
            # ],
            # [
            #     "預設語者",
            #     "宋涵葳",
            #     "dapu",
            #     "客家山城乜跈緊鬧熱䟘來咧",
            # ],
            # [
            #     "預設語者",
            #     "江芮敏",
            #     "raoping",
            #     "頭擺匱人，戴个毋係菅草屋，个創商品哦",
            # ],
            # [
            #     "預設語者",
            #     "洪藝晅",
            #     "zhaoan",
            #     "歇熱个時務，阿松歸屋下轉去在客莊个老屋",
            # ],
            # [
            #     "預設語者",
            #     "江芮敏",
            #     "nansixian",
            #     "在𠊎讀小學一年生个時節，阿爸輒常用自轉車載𠊎去學校讀書",
            # ],
        ],
        label="範例",
        inputs=[
            ref_audio_input,
            ref_text_input,
            ref_dialect_radio,
            gen_text_input,
            dialect_radio,
        ],
    )

demo.launch()