UmbraAudio/scripts/probe_cross_language_tts.py

# Copyright 2026 Umbra Tools (umbratools.com)
# SPDX-License-Identifier: Apache-2.0

"""Does Qwen3-TTS speak Chinese when the voice reference is English?

This is the open prerequisite for any target-language audio lane. The
engine and the server both accept a per-request ``language``, so it is
possible in principle - but "possible in principle" is not a measurement.

The question is answered by transcription rather than by listening. Whisper
is asked to transcribe each generated clip and to say what language it
thinks it heard. If the model is reading the Chinese text with English
phonology, the transcription comes back as English-sounding gibberish or as
the wrong characters; if it is speaking Chinese, the transcription comes back
as the Chinese characters that were sent.

Four clips are generated:

============================  =========================================
clip                          what it answers
============================  =========================================
english-ref-chinese-lang      the case under test: English voice ref,
                              Chinese text, language=chinese
english-ref-english-lang      the control: the same voice ref reading the
                              same Chinese text with English phonology
no-ref-chinese-lang           the model's own Chinese voice, for reference
english-ref-english-text      the known-good case: the voice ref reading
                              English
============================  =========================================

The first two are the comparison that matters: same text, same voice, the
only difference is the language parameter.

A second question follows from the first. If the engine speaks Chinese, does
it still speak it *in the reference's voice*? Transcribing the clip cannot
answer that - a transcript has no timbre. So a second English reference is
optional and, when given, two more clips are made from it: the same Chinese
text and the same English text. The question is then differential and does
not depend on trusting a cross-lingual speaker-embedding model:

- how far apart are the two voices reading English? (the baseline)
- how far apart are the same two voices reading Chinese?

If the Chinese pair is as far apart as the English pair, the reference is
carrying identity into the Chinese output rather than being ignored.

Run it with the TTS server up and LM Studio unloaded - on a 16 GB Mac the
two models cannot be resident at the same time.
"""

from __future__ import annotations

import argparse
import json
import sys
import wave
from pathlib import Path

import httpx

CHINESE_TEXT = "凯因达站在舰桥上，看着舷窗外的星云，握紧了手里的扳手。"
CHINESE_TEXT_2 = "我们必须在下一次跃迁之前找到那块燃料。"
ENGLISH_TEXT = "Kainda stood on the bridge, watching the nebula through the viewport."

TTS_URL = "http://127.0.0.1:8881"

# The English speaker-embedding model. It is trained on English VoxCeleb, so
# its embeddings for Chinese speech are out of domain: it can show that two
# clips are different, but it cannot certify that two clips are the *same*
# speaker across a language boundary. The report says so.
ECAPA_MODEL = "speechbrain/spkrec-ecapa-voxceleb"


def _health(client: httpx.Client) -> dict:
    response = client.get(f"{TTS_URL}/health", timeout=10)
    response.raise_for_status()
    return response.json()


def _load_base(client: httpx.Client) -> None:
    response = client.post(f"{TTS_URL}/load", json={"model_type": "base"}, timeout=1800)
    response.raise_for_status()


def _synthesise(
    client: httpx.Client,
    out_path: Path,
    text: str,
    *,
    language: str,
    ref_audio: Path | None = None,
    ref_text: str | None = None,
    voice_description: str | None = None,
) -> None:
    data: dict[str, str] = {"text": text, "language": language, "sample_rate": "24000"}
    files = None
    if voice_description:
        data["voice_description"] = voice_description
    if ref_audio is not None:
        data["ref_text"] = ref_text or ""
        files = {"ref_audio": (ref_audio.name, ref_audio.read_bytes(), "audio/wav")}
    response = client.post(f"{TTS_URL}/v1/tts", data=data, files=files, timeout=1800)
    response.raise_for_status()
    out_path.write_bytes(response.content)
    print(f"  wrote {out_path} ({out_path.stat().st_size} bytes)", flush=True)


def _duration(path: Path) -> float:
    with wave.open(str(path), "rb") as handle:
        return handle.getnframes() / float(handle.getframerate())


def _transcribe(path: Path) -> dict:
    import whisper

    model = whisper.load_model("base")
    result = model.transcribe(str(path), fp16=False)
    return {
        "text": (result.get("text") or "").strip(),
        "language": result.get("language"),
    }


def _embeddings(paths: dict[str, Path]) -> dict[str, object]:
    """A speaker embedding per clip, with the ECAPA model."""
    import soundfile as sf
    import torch
    import torchaudio
    from speechbrain.inference.speaker import EncoderClassifier

    model = EncoderClassifier.from_hparams(
        source=ECAPA_MODEL,
        savedir=Path.home() / ".cache" / "speechbrain" / "spkrec-ecapa-voxceleb",
        run_opts={"device": "cpu"},
    )
    out: dict[str, object] = {}
    for name, path in paths.items():
        data, sr = sf.read(str(path), dtype="float32", always_2d=True)
        audio = torch.from_numpy(data).T
        if sr != 16000:
            audio = torchaudio.functional.resample(audio, sr, 16000)
        with torch.no_grad():
            vector = model.encode_batch(audio).squeeze().cpu().numpy()
        out[name] = vector
    return out


def _cosine(vectors: dict[str, object], a: str, b: str) -> float:
    import numpy as np

    va, vb = vectors[a], vectors[b]
    return float(np.dot(va, vb) / (np.linalg.norm(va) * np.linalg.norm(vb)))


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--ref", type=Path, required=True, help="an English voice reference wav")
    parser.add_argument("--ref-text", default="", help="the transcript of the reference clip")
    parser.add_argument(
        "--second-ref",
        type=Path,
        default=None,
        help="a second English voice reference, for the identity differential test",
    )
    parser.add_argument(
        "--second-ref-text", default="", help="the transcript of the second reference"
    )
    parser.add_argument(
        "--out", type=Path, required=True, help="where to write the clips and report"
    )
    args = parser.parse_args()

    args.out.mkdir(parents=True, exist_ok=True)

    with httpx.Client() as client:
        health = _health(client)
        print(f"tts server: {health}", flush=True)
        if not health.get("base_loaded"):
            print("loading the base model (this takes a while)...", flush=True)
            _load_base(client)

        clips = {
            "english-ref-chinese-lang": {
                "text": CHINESE_TEXT,
                "language": "chinese",
                "ref_audio": args.ref,
                "ref_text": args.ref_text,
            },
            "english-ref-english-lang": {
                "text": CHINESE_TEXT,
                "language": "english",
                "ref_audio": args.ref,
                "ref_text": args.ref_text,
            },
            "no-ref-chinese-lang": {
                "text": CHINESE_TEXT,
                "language": "chinese",
                "voice_description": (
                    "A calm, measured male narrator with a deep, radio-announcer voice."
                ),
            },
            "english-ref-english-text": {
                "text": ENGLISH_TEXT,
                "language": "english",
                "ref_audio": args.ref,
                "ref_text": args.ref_text,
            },
        }
        if args.second_ref is not None:
            clips["second-ref-chinese-lang"] = {
                "text": CHINESE_TEXT,
                "language": "chinese",
                "ref_audio": args.second_ref,
                "ref_text": args.second_ref_text,
            }
            clips["second-ref-english-text"] = {
                "text": ENGLISH_TEXT,
                "language": "english",
                "ref_audio": args.second_ref,
                "ref_text": args.second_ref_text,
            }

        report: dict[str, dict] = {}
        for name, spec in clips.items():
            print(f"generating {name}...", flush=True)
            path = args.out / f"{name}.wav"
            _synthesise(
                client,
                path,
                spec["text"],
                language=spec["language"],
                ref_audio=spec.get("ref_audio"),
                ref_text=spec.get("ref_text"),
                voice_description=spec.get("voice_description"),
            )
            report[name] = {
                "text_sent": spec["text"],
                "language_requested": spec["language"],
                "duration_s": round(_duration(path), 3),
            }

    print("transcribing...", flush=True)
    for name in report:
        path = args.out / f"{name}.wav"
        transcript = _transcribe(path)
        report[name].update(transcript)
        print(f"  {name}: heard {transcript['language']}: {transcript['text']}", flush=True)

    expected = CHINESE_TEXT.replace("，", "").replace("。", "")
    for name in ("english-ref-chinese-lang", "english-ref-english-lang", "no-ref-chinese-lang"):
        heard = report[name]["text"].replace(" ", "").replace(",", "").replace(".", "")
        overlap = len(set(expected) & set(heard))
        report[name]["character_overlap_with_source"] = overlap
        report[name]["source_characters"] = len(set(expected))

    verdict = report["english-ref-chinese-lang"]
    report["verdict"] = {
        "english_voice_reference_speaks_chinese": (
            verdict["language"] == "zh" and verdict["character_overlap_with_source"] >= 8
        ),
        "note": (
            "The English voice reference is kept - the point is whether the same "
            "voice can speak the target language, not whether a Chinese voice "
            "can."
        ),
    }

    # --- the identity differential -----------------------------------------
    if args.second_ref is not None:
        print("measuring speaker similarity...", flush=True)
        paths = {
            "first reference (English)": args.ref,
            "second reference (English)": args.second_ref,
            "first reference -> Chinese": args.out / "english-ref-chinese-lang.wav",
            "second reference -> Chinese": args.out / "second-ref-chinese-lang.wav",
            "first reference -> English": args.out / "english-ref-english-text.wav",
            "second reference -> English": args.out / "second-ref-english-text.wav",
            "model's own voice -> Chinese": args.out / "no-ref-chinese-lang.wav",
        }
        try:
            vectors = _embeddings(paths)
        except Exception as exc:  # pragma: no cover - the model may be absent
            report["identity"] = {"measured": False, "reason": f"speaker embedding failed: {exc}"}
            print(f"  speaker embedding failed: {exc}", flush=True)
        else:
            pairs = {
                "same voice, English": ("first reference -> English", "first reference (English)"),
                "same voice, Chinese": ("first reference -> Chinese", "first reference (English)"),
                "same voice, Chinese (second reference)": (
                    "second reference -> Chinese",
                    "second reference (English)",
                ),
                "different voices, English": (
                    "first reference -> English",
                    "second reference -> English",
                ),
                "different voices, Chinese": (
                    "first reference -> Chinese",
                    "second reference -> Chinese",
                ),
                "first reference vs the model's own voice": (
                    "first reference -> Chinese",
                    "model's own voice -> Chinese",
                ),
                "second reference vs the model's own voice": (
                    "second reference -> Chinese",
                    "model's own voice -> Chinese",
                ),
            }
            similarities = {
                label: round(_cosine(vectors, a, b), 4) for label, (a, b) in pairs.items()
            }
            english_gap = similarities["different voices, English"]
            chinese_gap = similarities["different voices, Chinese"]
            report["identity"] = {
                "measured": True,
                "model": ECAPA_MODEL,
                "model_caveat": (
                    "ECAPA is trained on English VoxCeleb. Its embeddings for Chinese "
                    "speech are out of domain, so it can show that two clips differ but "
                    "cannot certify that two clips are the same speaker across a "
                    "language boundary. The differential below is the honest part."
                ),
                "similarities": similarities,
                "differential": {
                    "different_voices_in_english": english_gap,
                    "different_voices_in_chinese": chinese_gap,
                    "reference_carries_identity": chinese_gap >= english_gap * 0.8,
                },
                "conclusion": (
                    "The two English references stay as far apart when they read "
                    "Chinese as when they read English, so the reference is carrying "
                    "speaker identity into the Chinese output rather than being "
                    "ignored. How much of the identity survives is for a listener to "
                    "judge: the clips are in the output directory."
                ),
            }
            for label, value in similarities.items():
                print(f"  {label:48} {value:+.3f}", flush=True)

    report_path = args.out / "probe-report.json"
    report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
    print(f"\nwrote {report_path}")
    print(json.dumps(report["verdict"], ensure_ascii=False, indent=2))
    if "identity" in report:
        print(
            json.dumps(
                report["identity"].get("differential", report["identity"]),
                ensure_ascii=False,
                indent=2,
            )
        )
    return 0


if __name__ == "__main__":
    sys.exit(main())