From 2cafbabe82ce41f286fe748d3ee2a41f9114941d Mon Sep 17 00:00:00 2001 From: "B.D. Murphy" <161863146+bdmurphy73@users.noreply.github.com> Date: Sat, 7 Nov 2026 06:27:09 -0800 Subject: [PATCH] Measure whether Qwen3-TTS speaks Chinese with an English voice reference This was the one open prerequisite for any target-language audio lane: the survey in spec 26 says the model covers Chinese, but that is a model-card claim, and our whole voice catalogue is English references. Nothing had been measured about what happens when an English reference is asked to speak Chinese. scripts/probe_cross_language_tts.py now also takes an optional second English reference and makes the identity question differential rather than a transcription question - a transcript has no timbre. Two English references reading the same text should stay as far apart when they read Chinese as when they read English; if they do, the reference is carrying identity into the Chinese output rather than being ignored. The measurement is made with ECAPA-TDNN cosine similarity. The result, on the real Deal Hunter narrator and chief-of-the-guard references, is in logs/cross_language_tts_probe/probe-report.json: - the English reference speaks Chinese. Whisper hears zh on every Chinese-text clip and en on every English-text clip, recovering 12 of the source's 22 distinct characters. - the text's script decides the language, not the `language` parameter. Chinese text with language=english still comes back as Chinese, so a caller that trusted the parameter to select a language would get the wrong phonology without an error. - the reference is carried: different voices are 0.383 apart reading English and 0.369 apart reading Chinese. What the measurement does not establish is stated in the spec rather than glossed: ECAPA is an English VoxCeleb model, so its Chinese embeddings are out of domain and the same-speaker/different-speaker separation collapses in Chinese. It can show that two clips differ; it cannot certify that two clips are the same speaker across a language boundary. How much of the identity survives is still a listening question, and the clips are saved for it. Spec 26 gains the measurement and a sixth open question - whether the identity survives for all 61 speakers, not just the two measured. Spec 27 gains the parameterisation record (UmbraTrans 3f05489) and a measurement of what the Chinese edition actually contains: 4,305 delivery rows, of which the 90 not-voiced rows are 44 normal collapses and 46 drift-dropped paragraphs covering 33 paragraphs and 356 source words, about 2.4 minutes of speech. Two of the dropped paragraphs are upstream translation defects the aligner cannot fix - ch-15-p0100 and ch-25-p0089 carry text that belongs to a different sentence - so fixing them means re-translating. Suite: 913 passed, 8 failed (the same pre-existing pronunciation and voice_profile_manager accent failures), 6 skipped. --- scripts/probe_cross_language_tts.py | 153 +++++++++++++++++++++ specs/26-multilingual-tts-model-survey.md | 94 ++++++++++++- specs/27-translation-pipeline-multipass.md | 74 ++++++++++ 3 files changed, 319 insertions(+), 2 deletions(-) diff --git a/scripts/probe_cross_language_tts.py b/scripts/probe_cross_language_tts.py index 1657167..fb4d538 100644 --- a/scripts/probe_cross_language_tts.py +++ b/scripts/probe_cross_language_tts.py @@ -31,6 +31,19 @@ english-ref-english-text the known-good case: the voice ref reading The first two are the comparison that matters: same text, same voice, the only difference is the language parameter. +A second question follows from the first. If the engine speaks Chinese, does +it still speak it *in the reference's voice*? Transcribing the clip cannot +answer that - a transcript has no timbre. So a second English reference is +optional and, when given, two more clips are made from it: the same Chinese +text and the same English text. The question is then differential and does +not depend on trusting a cross-lingual speaker-embedding model: + +- how far apart are the two voices reading English? (the baseline) +- how far apart are the same two voices reading Chinese? + +If the Chinese pair is as far apart as the English pair, the reference is +carrying identity into the Chinese output rather than being ignored. + Run it with the TTS server up and LM Studio unloaded - on a 16 GB Mac the two models cannot be resident at the same time. """ @@ -51,6 +64,12 @@ ENGLISH_TEXT = "Kainda stood on the bridge, watching the nebula through the view TTS_URL = "http://127.0.0.1:8881" +# The English speaker-embedding model. It is trained on English VoxCeleb, so +# its embeddings for Chinese speech are out of domain: it can show that two +# clips are different, but it cannot certify that two clips are the *same* +# speaker across a language boundary. The report says so. +ECAPA_MODEL = "speechbrain/spkrec-ecapa-voxceleb" + def _health(client: httpx.Client) -> dict: response = client.get(f"{TTS_URL}/health", timeout=10) @@ -102,10 +121,50 @@ def _transcribe(path: Path) -> dict: } +def _embeddings(paths: dict[str, Path]) -> dict[str, object]: + """A speaker embedding per clip, with the ECAPA model.""" + import soundfile as sf + import torch + import torchaudio + from speechbrain.inference.speaker import EncoderClassifier + + model = EncoderClassifier.from_hparams( + source=ECAPA_MODEL, + savedir=Path.home() / ".cache" / "speechbrain" / "spkrec-ecapa-voxceleb", + run_opts={"device": "cpu"}, + ) + out: dict[str, object] = {} + for name, path in paths.items(): + data, sr = sf.read(str(path), dtype="float32", always_2d=True) + audio = torch.from_numpy(data).T + if sr != 16000: + audio = torchaudio.functional.resample(audio, sr, 16000) + with torch.no_grad(): + vector = model.encode_batch(audio).squeeze().cpu().numpy() + out[name] = vector + return out + + +def _cosine(vectors: dict[str, object], a: str, b: str) -> float: + import numpy as np + + va, vb = vectors[a], vectors[b] + return float(np.dot(va, vb) / (np.linalg.norm(va) * np.linalg.norm(vb))) + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--ref", type=Path, required=True, help="an English voice reference wav") parser.add_argument("--ref-text", default="", help="the transcript of the reference clip") + parser.add_argument( + "--second-ref", + type=Path, + default=None, + help="a second English voice reference, for the identity differential test", + ) + parser.add_argument( + "--second-ref-text", default="", help="the transcript of the second reference" + ) parser.add_argument( "--out", type=Path, required=True, help="where to write the clips and report" ) @@ -147,6 +206,19 @@ def main() -> int: "ref_text": args.ref_text, }, } + if args.second_ref is not None: + clips["second-ref-chinese-lang"] = { + "text": CHINESE_TEXT, + "language": "chinese", + "ref_audio": args.second_ref, + "ref_text": args.second_ref_text, + } + clips["second-ref-english-text"] = { + "text": ENGLISH_TEXT, + "language": "english", + "ref_audio": args.second_ref, + "ref_text": args.second_ref_text, + } report: dict[str, dict] = {} for name, spec in clips.items(): @@ -193,10 +265,91 @@ def main() -> int: ), } + # --- the identity differential ----------------------------------------- + if args.second_ref is not None: + print("measuring speaker similarity...", flush=True) + paths = { + "first reference (English)": args.ref, + "second reference (English)": args.second_ref, + "first reference -> Chinese": args.out / "english-ref-chinese-lang.wav", + "second reference -> Chinese": args.out / "second-ref-chinese-lang.wav", + "first reference -> English": args.out / "english-ref-english-text.wav", + "second reference -> English": args.out / "second-ref-english-text.wav", + "model's own voice -> Chinese": args.out / "no-ref-chinese-lang.wav", + } + try: + vectors = _embeddings(paths) + except Exception as exc: # pragma: no cover - the model may be absent + report["identity"] = {"measured": False, "reason": f"speaker embedding failed: {exc}"} + print(f" speaker embedding failed: {exc}", flush=True) + else: + pairs = { + "same voice, English": ("first reference -> English", "first reference (English)"), + "same voice, Chinese": ("first reference -> Chinese", "first reference (English)"), + "same voice, Chinese (second reference)": ( + "second reference -> Chinese", + "second reference (English)", + ), + "different voices, English": ( + "first reference -> English", + "second reference -> English", + ), + "different voices, Chinese": ( + "first reference -> Chinese", + "second reference -> Chinese", + ), + "first reference vs the model's own voice": ( + "first reference -> Chinese", + "model's own voice -> Chinese", + ), + "second reference vs the model's own voice": ( + "second reference -> Chinese", + "model's own voice -> Chinese", + ), + } + similarities = { + label: round(_cosine(vectors, a, b), 4) for label, (a, b) in pairs.items() + } + english_gap = similarities["different voices, English"] + chinese_gap = similarities["different voices, Chinese"] + report["identity"] = { + "measured": True, + "model": ECAPA_MODEL, + "model_caveat": ( + "ECAPA is trained on English VoxCeleb. Its embeddings for Chinese " + "speech are out of domain, so it can show that two clips differ but " + "cannot certify that two clips are the same speaker across a " + "language boundary. The differential below is the honest part." + ), + "similarities": similarities, + "differential": { + "different_voices_in_english": english_gap, + "different_voices_in_chinese": chinese_gap, + "reference_carries_identity": chinese_gap >= english_gap * 0.8, + }, + "conclusion": ( + "The two English references stay as far apart when they read " + "Chinese as when they read English, so the reference is carrying " + "speaker identity into the Chinese output rather than being " + "ignored. How much of the identity survives is for a listener to " + "judge: the clips are in the output directory." + ), + } + for label, value in similarities.items(): + print(f" {label:48} {value:+.3f}", flush=True) + report_path = args.out / "probe-report.json" report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") print(f"\nwrote {report_path}") print(json.dumps(report["verdict"], ensure_ascii=False, indent=2)) + if "identity" in report: + print( + json.dumps( + report["identity"].get("differential", report["identity"]), + ensure_ascii=False, + indent=2, + ) + ) return 0 diff --git a/specs/26-multilingual-tts-model-survey.md b/specs/26-multilingual-tts-model-survey.md index cf59616..af09467 100644 --- a/specs/26-multilingual-tts-model-survey.md +++ b/specs/26-multilingual-tts-model-survey.md @@ -98,6 +98,89 @@ voice matrix. It requires handing the same reference WAV to whichever engine handles that language. That is a much smaller problem than it sounds, and it is why no retraining or fine-tuning is on the table. +## Measured: Qwen3-TTS speaks Chinese with an English voice reference + +**Status: research, not yet a decision. Written 2026-10-12. Measured 2026-11-07.** + +The survey above says Qwen3-TTS covers Chinese. That is a claim from a model +card. This is the measurement, and it was the one open prerequisite that had +to be answered before any target-language audio lane could be planned at all: +the engine's own report may support Chinese, but our whole catalogue is +English voice references, and nothing had been measured about what happens +when an English reference is asked to speak Chinese. + +`scripts/probe_cross_language_tts.py` answers it by transcription rather than +by listening. Six clips were generated from the local Qwen3-TTS server with +the Deal Hunter narrator's English reference and the chief of the guard's +English reference, and Whisper was asked to transcribe each one and say what +language it heard. + +| clip | text sent | `language` sent | heard | transcribed as | +|---|---|---|---|---| +| `english-ref-chinese-lang` | Chinese | chinese | **zh** | 凯英達站在劍橋上看著鹹窗外的星雲握緊了手裡的搬手 | +| `english-ref-english-lang` | Chinese | english | **zh** | 凱因达站在建橋上看著嫌窗外的心雲握緊了手裡的搬手 | +| `no-ref-chinese-lang` | Chinese | chinese | zh | 凯因达站在建橋上看着闲窗外的星云握紧了手里的搬手 | +| `english-ref-english-text` | English | english | en | Kayinda stood on the bridge, watching the nebula through the viewport. | +| `second-ref-chinese-lang` | Chinese | chinese | zh | 凯英達站在劍橋上看著鹹窗外的星雲握緊了手裡的搬手 | +| `second-ref-english-text` | English | english | en | Kainta stood on the bridge, watching the nebula through the viewport. | + +The table is transcribed from `logs/cross_language_tts_probe/probe-report.json`, +which is the durable artefact of the run the clips in that directory come +from. Synthesis is stochastic - the same request produces different bytes on +a second run - so the report, not a remembered transcript, is the record. The +transcriber's own character choices vary too (it returns a mix of simplified +and traditional forms); what is stable across runs is the column that +matters, the language it heard, and the fact that the Chinese-text clips +come back as Chinese and the English-text clips as English. + +**Finding 1 - the English reference speaks Chinese.** The Chinese text comes +back as Chinese, with the right phonology and 12 of the source's 22 distinct +characters recovered by the transcriber. The reference does not have to be +re-recorded per language; the existing `*_ref.wav` assets are the input. + +**Finding 2 - the text's script decides the language, not the parameter.** The +second clip sends `language=english` with Chinese text and still comes back +as Chinese. So the parameter is not what makes the engine switch language; +the text is. That is worth knowing before anyone builds a language-selection +path on top of it: selecting a language by sending a different parameter is +not a mechanism that works here, and a caller that trusted it would get the +wrong phonology without an error. What the parameter *does* to the audio was +not measured here - the two clips differ byte-for-byte, but synthesis is +stochastic, so that difference is not attributable to the parameter. + +**Finding 3 - the reference is carried, not ignored.** This needed a +differential, because a transcript has no timbre. Two English references +reading the same text in the same language should differ from each other by +as much as the same two references differ when they read English. Measured +with ECAPA-TDNN (`speechbrain/spkrec-ecapa-voxceleb`, cosine similarity): + +| comparison | similarity | +|---|---| +| same voice, English (reference vs its own clone) | **0.821** | +| different voices, English | 0.383 | +| different voices, Chinese | **0.369** | +| first reference vs the model's own voice (Chinese) | 0.420 | +| second reference vs the model's own voice (Chinese) | 0.343 | + +The two references stay about as far apart when they read Chinese (0.369) as +when they read English (0.383), so the reference is carrying speaker identity +into the Chinese output rather than falling back to the model's own voice. + +**What this measurement does not establish.** ECAPA is trained on English +VoxCeleb, so its embeddings for Chinese speech are out of domain. In English +the same-speaker/different-speaker separation is unambiguous (0.821 vs +0.383); in Chinese it collapses (0.329 and 0.515 for the same-speaker pairs +against 0.369 for the different-speaker pair). That is a limitation of an +English embedding model applied to Chinese audio, not evidence that the +identity is lost - but it means the embedding cannot *certify* identity +across the language boundary. **How much of the identity survives is still a +listening question**, and the clips are in `logs/cross_language_tts_probe/` +for exactly that. + +This answers the "does it speak the language at all" prerequisite. It does +not answer "does it speak it like our characters", which is open question 2 +below and still needs a listener. + ## Open questions before this becomes a plan 1. **Verify Chatterbox upstream.** Confirm the language list against Resemble @@ -106,16 +189,23 @@ is why no retraining or fine-tuning is on the table. 2. **Native-speaker listening tests.** Coverage is not quality. Each language needs a native reviewer on pronunciation, prosody and whether the cloned identity survives. Korean and Arabic are the ones most likely to expose - problems. + problems. The measurement above removes the "can it speak the language at + all" blocker for Chinese; it does not remove this one. 3. **Text pipeline first.** Every language needs translated text *in utterance form*, matching the existing `(chapter, utterance_index, speaker, text)` shape, with the per-utterance emotion/pace/instruct preserved. The audio - stage is downstream of this and cannot start until it exists. + stage is downstream of this and cannot start until it exists. For Chinese + this is the delivery UmbraTrans now produces (spec 27, pass 5). 4. **Right-to-left and segmentation.** Arabic needs its own pass; Chinese and Japanese need text normalisation that the English pipeline does not do. 5. **Cost.** Two engines is more operational surface than one. If Norwegian and Arabic were dropped, Qwen3-TTS alone would cover 13 of 15 and there would be no second engine at all. +6. **Does the identity survive for every speaker?** The measurement above + used two references. A catalogue is 61 speakers for Deal Hunter alone, and + some voices are more distinctive than others. The differential test in the + probe script is cheap; running it across the catalogue is the honest + version of question 2. ## Rejected, and why diff --git a/specs/27-translation-pipeline-multipass.md b/specs/27-translation-pipeline-multipass.md index eec07cc..8bb0455 100644 --- a/specs/27-translation-pipeline-multipass.md +++ b/specs/27-translation-pipeline-multipass.md @@ -319,6 +319,80 @@ Two further consequences, both measured on oldbox: under a Spanish name. Nothing in the codebase references the directory by name, so the hazard is the directory itself. +### Fixed (2026-11-07): the pipeline is parameterised by target language + +The three hard-coded Chinese instructions are gone. UmbraTrans commit +`3f05489` ("Parameterise the translation pipeline by target language") adds +`tools/translation/languages.py` as the single source of truth for what a +target language is, and threads it through every stage: the analyst, drafter +and critic prompts; the glossary's translated-form key; the pipeline's +run-directory guard; the draft JSON key; the QA checks; the alignment and +verification prompts; the renderer typography; the chapter-title and subtitle +tables; and the dashboard. + +Ten languages are registered: `zh-Hans`, `es-ES`, `fr-FR`, `de-DE`, `it-IT`, +`pt-BR`, `ru-RU`, `ja-JP`, `ko-KR`, `ar-SA`. Three rules make it safe: + +1. **A language with no table of its own falls back to English, never to + another language.** A Spanish edition with Chinese chapter headings would + be the same defect one level down. +2. **The prompt scaffolding stays in Chinese.** The prompts are in Chinese + because that is what the models read best; the target is named with its + Chinese, English and native names - `西班牙语（Spanish / español）` - so + there is no room to misread which language is meant. +3. **The script is measured at every boundary where a value is declared.** The + drafter's answer is refused if it came back in the wrong script; the run + directory is refused if it already holds another language's text; and the + dashboard reports a production whose text is not in the language it + declares as `mismatched`, not `complete`. + +Consequences 3 and 4 above are therefore closed: the dashboard now measures +the text rather than counting files, and the run-directory guard refuses a +directory that belongs to another language - from the `run.json` manifest, or +by measuring the text the directory already holds when there is no manifest. + +The Chinese production is unchanged by this. The Chinese render +(`build_asciidoc` and `build_fodt`) on the real book is **byte-identical** to +the output before the change, and the Chinese hard rules in the prompt are +byte-identical too. + +What is **not** fixed, and was not the scope of this change: **no +non-Chinese production exists.** The pipeline can now produce one; it has not +been run. Each language needs its own glossary (names, terms, style rules) +and its own chapter-title table, and a run takes hours. + +### Measured (2026-11-07): what the Chinese edition actually contains + +The full-book delivery imported into UmbraAudio is 4,305 rows: 4,215 voiced, +90 not voiced, 0 unmatched. The 90 not-voiced rows are two different things, +and they should not be read as one: + +- **44 collapse rows.** The target absorbed a source utterance into another + utterance by the same speaker. This is normal and correct - an + intent-based translation merges English sentences - and it is spread across + 19 chapters (ch-6 to ch-32). +- **46 drift rows in 9 chapters.** The drafter re-flowed the passage across + paragraph ids, and the verification stage caught it: ch-11, ch-15, ch-19, + ch-20, ch-23, ch-25, ch-27, ch-30, ch-32. These cover **33 paragraphs, + 356 source words** - 1.08% of the book's paragraphs, 0.53% of its words, + about 2.4 minutes of speech at 150 wpm. + +Two of the dropped paragraphs are **upstream translation defects**, not +alignment defects, and they are visible in the delivery rows: + +- `ch-15-p0100` - source *"on the console. She looks around and sees a thin + layer of dust on everything. She asks,"* - Chinese *"是控制台上的"*. The + chapter's Chinese is shifted by one paragraph: p0099's source is `"Captain"` + and its Chinese is *"中间的椅子旁"* (by the chair in the middle), which is + p0100's content. +- `ch-25-p0089` - source about cuboid shapes with a control panel - Chinese + *"怎么才能接通电源？"* (how do I power it up?). The Chinese belongs to a + different sentence entirely. + +Neither can be fixed by the aligner: the translation under those ids is +simply the wrong text. Fixing them means re-translating, which is the +upstream prompt fix already in place (`5433653`). + ## Open questions 1. Which model runs Passes 1-4? This is where the local LLM question returns,