# Copyright 2026 Umbra Tools (umbratools.com) # SPDX-License-Identifier: Apache-2.0 """Does Qwen3-TTS speak Chinese when the voice reference is English? This is the open prerequisite for any target-language audio lane. The engine and the server both accept a per-request ``language``, so it is possible in principle - but "possible in principle" is not a measurement. The question is answered by transcription rather than by listening. Whisper is asked to transcribe each generated clip and to say what language it thinks it heard. If the model is reading the Chinese text with English phonology, the transcription comes back as English-sounding gibberish or as the wrong characters; if it is speaking Chinese, the transcription comes back as the Chinese characters that were sent. Four clips are generated: ============================ ========================================= clip what it answers ============================ ========================================= english-ref-chinese-lang the case under test: English voice ref, Chinese text, language=chinese english-ref-english-lang the control: the same voice ref reading the same Chinese text with English phonology no-ref-chinese-lang the model's own Chinese voice, for reference english-ref-english-text the known-good case: the voice ref reading English ============================ ========================================= The first two are the comparison that matters: same text, same voice, the only difference is the language parameter. A second question follows from the first. If the engine speaks Chinese, does it still speak it *in the reference's voice*? Transcribing the clip cannot answer that - a transcript has no timbre. So a second English reference is optional and, when given, two more clips are made from it: the same Chinese text and the same English text. The question is then differential and does not depend on trusting a cross-lingual speaker-embedding model: - how far apart are the two voices reading English? (the baseline) - how far apart are the same two voices reading Chinese? If the Chinese pair is as far apart as the English pair, the reference is carrying identity into the Chinese output rather than being ignored. Run it with the TTS server up and LM Studio unloaded - on a 16 GB Mac the two models cannot be resident at the same time. """ from __future__ import annotations import argparse import json import sys import wave from pathlib import Path import httpx CHINESE_TEXT = "凯因达站在舰桥上，看着舷窗外的星云，握紧了手里的扳手。" CHINESE_TEXT_2 = "我们必须在下一次跃迁之前找到那块燃料。" ENGLISH_TEXT = "Kainda stood on the bridge, watching the nebula through the viewport." TTS_URL = "http://127.0.0.1:8881" # The English speaker-embedding model. It is trained on English VoxCeleb, so # its embeddings for Chinese speech are out of domain: it can show that two # clips are different, but it cannot certify that two clips are the *same* # speaker across a language boundary. The report says so. ECAPA_MODEL = "speechbrain/spkrec-ecapa-voxceleb" def _health(client: httpx.Client) -> dict: response = client.get(f"{TTS_URL}/health", timeout=10) response.raise_for_status() return response.json() def _load_base(client: httpx.Client) -> None: response = client.post(f"{TTS_URL}/load", json={"model_type": "base"}, timeout=1800) response.raise_for_status() def _synthesise( client: httpx.Client, out_path: Path, text: str, *, language: str, ref_audio: Path | None = None, ref_text: str | None = None, voice_description: str | None = None, ) -> None: data: dict[str, str] = {"text": text, "language": language, "sample_rate": "24000"} files = None if voice_description: data["voice_description"] = voice_description if ref_audio is not None: data["ref_text"] = ref_text or "" files = {"ref_audio": (ref_audio.name, ref_audio.read_bytes(), "audio/wav")} response = client.post(f"{TTS_URL}/v1/tts", data=data, files=files, timeout=1800) response.raise_for_status() out_path.write_bytes(response.content) print(f" wrote {out_path} ({out_path.stat().st_size} bytes)", flush=True) def _duration(path: Path) -> float: with wave.open(str(path), "rb") as handle: return handle.getnframes() / float(handle.getframerate()) def _transcribe(path: Path) -> dict: import whisper model = whisper.load_model("base") result = model.transcribe(str(path), fp16=False) return { "text": (result.get("text") or "").strip(), "language": result.get("language"), } def _embeddings(paths: dict[str, Path]) -> dict[str, object]: """A speaker embedding per clip, with the ECAPA model.""" import soundfile as sf import torch import torchaudio from speechbrain.inference.speaker import EncoderClassifier model = EncoderClassifier.from_hparams( source=ECAPA_MODEL, savedir=Path.home() / ".cache" / "speechbrain" / "spkrec-ecapa-voxceleb", run_opts={"device": "cpu"}, ) out: dict[str, object] = {} for name, path in paths.items(): data, sr = sf.read(str(path), dtype="float32", always_2d=True) audio = torch.from_numpy(data).T if sr != 16000: audio = torchaudio.functional.resample(audio, sr, 16000) with torch.no_grad(): vector = model.encode_batch(audio).squeeze().cpu().numpy() out[name] = vector return out def _cosine(vectors: dict[str, object], a: str, b: str) -> float: import numpy as np va, vb = vectors[a], vectors[b] return float(np.dot(va, vb) / (np.linalg.norm(va) * np.linalg.norm(vb))) def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--ref", type=Path, required=True, help="an English voice reference wav") parser.add_argument("--ref-text", default="", help="the transcript of the reference clip") parser.add_argument( "--second-ref", type=Path, default=None, help="a second English voice reference, for the identity differential test", ) parser.add_argument( "--second-ref-text", default="", help="the transcript of the second reference" ) parser.add_argument( "--out", type=Path, required=True, help="where to write the clips and report" ) args = parser.parse_args() args.out.mkdir(parents=True, exist_ok=True) with httpx.Client() as client: health = _health(client) print(f"tts server: {health}", flush=True) if not health.get("base_loaded"): print("loading the base model (this takes a while)...", flush=True) _load_base(client) clips = { "english-ref-chinese-lang": { "text": CHINESE_TEXT, "language": "chinese", "ref_audio": args.ref, "ref_text": args.ref_text, }, "english-ref-english-lang": { "text": CHINESE_TEXT, "language": "english", "ref_audio": args.ref, "ref_text": args.ref_text, }, "no-ref-chinese-lang": { "text": CHINESE_TEXT, "language": "chinese", "voice_description": ( "A calm, measured male narrator with a deep, radio-announcer voice." ), }, "english-ref-english-text": { "text": ENGLISH_TEXT, "language": "english", "ref_audio": args.ref, "ref_text": args.ref_text, }, } if args.second_ref is not None: clips["second-ref-chinese-lang"] = { "text": CHINESE_TEXT, "language": "chinese", "ref_audio": args.second_ref, "ref_text": args.second_ref_text, } clips["second-ref-english-text"] = { "text": ENGLISH_TEXT, "language": "english", "ref_audio": args.second_ref, "ref_text": args.second_ref_text, } report: dict[str, dict] = {} for name, spec in clips.items(): print(f"generating {name}...", flush=True) path = args.out / f"{name}.wav" _synthesise( client, path, spec["text"], language=spec["language"], ref_audio=spec.get("ref_audio"), ref_text=spec.get("ref_text"), voice_description=spec.get("voice_description"), ) report[name] = { "text_sent": spec["text"], "language_requested": spec["language"], "duration_s": round(_duration(path), 3), } print("transcribing...", flush=True) for name in report: path = args.out / f"{name}.wav" transcript = _transcribe(path) report[name].update(transcript) print(f" {name}: heard {transcript['language']}: {transcript['text']}", flush=True) expected = CHINESE_TEXT.replace("，", "").replace("。", "") for name in ("english-ref-chinese-lang", "english-ref-english-lang", "no-ref-chinese-lang"): heard = report[name]["text"].replace(" ", "").replace(",", "").replace(".", "") overlap = len(set(expected) & set(heard)) report[name]["character_overlap_with_source"] = overlap report[name]["source_characters"] = len(set(expected)) verdict = report["english-ref-chinese-lang"] report["verdict"] = { "english_voice_reference_speaks_chinese": ( verdict["language"] == "zh" and verdict["character_overlap_with_source"] >= 8 ), "note": ( "The English voice reference is kept - the point is whether the same " "voice can speak the target language, not whether a Chinese voice " "can." ), } # --- the identity differential ----------------------------------------- if args.second_ref is not None: print("measuring speaker similarity...", flush=True) paths = { "first reference (English)": args.ref, "second reference (English)": args.second_ref, "first reference -> Chinese": args.out / "english-ref-chinese-lang.wav", "second reference -> Chinese": args.out / "second-ref-chinese-lang.wav", "first reference -> English": args.out / "english-ref-english-text.wav", "second reference -> English": args.out / "second-ref-english-text.wav", "model's own voice -> Chinese": args.out / "no-ref-chinese-lang.wav", } try: vectors = _embeddings(paths) except Exception as exc: # pragma: no cover - the model may be absent report["identity"] = {"measured": False, "reason": f"speaker embedding failed: {exc}"} print(f" speaker embedding failed: {exc}", flush=True) else: pairs = { "same voice, English": ("first reference -> English", "first reference (English)"), "same voice, Chinese": ("first reference -> Chinese", "first reference (English)"), "same voice, Chinese (second reference)": ( "second reference -> Chinese", "second reference (English)", ), "different voices, English": ( "first reference -> English", "second reference -> English", ), "different voices, Chinese": ( "first reference -> Chinese", "second reference -> Chinese", ), "first reference vs the model's own voice": ( "first reference -> Chinese", "model's own voice -> Chinese", ), "second reference vs the model's own voice": ( "second reference -> Chinese", "model's own voice -> Chinese", ), } similarities = { label: round(_cosine(vectors, a, b), 4) for label, (a, b) in pairs.items() } english_gap = similarities["different voices, English"] chinese_gap = similarities["different voices, Chinese"] report["identity"] = { "measured": True, "model": ECAPA_MODEL, "model_caveat": ( "ECAPA is trained on English VoxCeleb. Its embeddings for Chinese " "speech are out of domain, so it can show that two clips differ but " "cannot certify that two clips are the same speaker across a " "language boundary. The differential below is the honest part." ), "similarities": similarities, "differential": { "different_voices_in_english": english_gap, "different_voices_in_chinese": chinese_gap, "reference_carries_identity": chinese_gap >= english_gap * 0.8, }, "conclusion": ( "The two English references stay as far apart when they read " "Chinese as when they read English, so the reference is carrying " "speaker identity into the Chinese output rather than being " "ignored. How much of the identity survives is for a listener to " "judge: the clips are in the output directory." ), } for label, value in similarities.items(): print(f" {label:48} {value:+.3f}", flush=True) report_path = args.out / "probe-report.json" report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") print(f"\nwrote {report_path}") print(json.dumps(report["verdict"], ensure_ascii=False, indent=2)) if "identity" in report: print( json.dumps( report["identity"].get("differential", report["identity"]), ensure_ascii=False, indent=2, ) ) return 0 if __name__ == "__main__": sys.exit(main())