From 3db6ac3b8b3c640d9e1dee0339159ca43cd36ff4 Mon Sep 17 00:00:00 2001 From: Umbra Tools Date: Fri, 6 Nov 2026 20:06:59 -0800 Subject: [PATCH] Refuse a delivery whose text is not written in the language it claims The production directory oldbox calls "deal-hunter-es" declares target_language "es-ES", its glossary says Spanish, and its 32-chapter run finished - and all 3,038 translated paragraphs are in Chinese. It is a second Chinese draft filed under a Spanish name. The cause is in the pipeline: the drafter's instructions are hard-coded to Simplified Chinese in three places (SYSTEM_DRAFTER, draft_prompt, SYSTEM_CRITIC), so the glossary's target language was ignored. No non-Chinese translation has ever been produced by that pipeline. The danger at this end is that the rows would be stored under "es", the review lane would look populated, and a reviewer would approve Chinese as Spanish. So the ingest now checks the script of the delivered text against the declared language, and refuses the delivery before writing a single row. The check is about the script, not the language: it cannot tell Spanish from Portuguese, but it can tell Chinese from Spanish, which is the mistake that actually happens. Japanese needs its kana and Korean its Hangul, and a Chinese delivery carrying kana is a Japanese translation filed in the wrong lane. An unknown language is not checked - there is no expectation to check it against. Proven on the real data: the es delivery is refused with "the text is mostly Han characters, so it is not Spanish", and the 4,300-row Chinese delivery still imports unchanged. 19 new tests. Suite: 909 passed, 8 failed (the same 8 pre-existing failures). --- specs/27-translation-pipeline-multipass.md | 44 +++++ tests/test_language_scripts.py | 205 +++++++++++++++++++++ umbraaudio/languages.py | 141 +++++++++++++- umbraaudio/translations.py | 16 ++ 4 files changed, 400 insertions(+), 6 deletions(-) create mode 100644 tests/test_language_scripts.py diff --git a/specs/27-translation-pipeline-multipass.md b/specs/27-translation-pipeline-multipass.md index d0b9b64..bf29ac3 100644 --- a/specs/27-translation-pipeline-multipass.md +++ b/specs/27-translation-pipeline-multipass.md @@ -185,6 +185,50 @@ the mechanical split, 0 partition mismatches**. The one fallback was a model answer that did not reproduce the translated text; the content check caught it and the paragraph was flagged rather than accepted. +### The pipeline is Chinese-only, and one production says otherwise + +Measured on oldbox (2026-11-06), the second production directory is +`productions/deal-hunter-es`. It declares `target_language: es-ES`, its +glossary says `"script": "Spanish"` with Spanish style rules and the Spanish +title `Cazador de Ofertas`, and it has a complete 32-chapter run (39,083 s). + +**Its text is Chinese.** All 3,038 translated paragraphs are Chinese, in the +same proportion as the Chinese production (3,034 with Han characters, 4 +Latin-only). It is a second Chinese draft filed under a Spanish name - a +different Chinese wording from the `deal-hunter-zh` run (2,792 of 3,038 +paragraphs differ), but Chinese. + +The cause is in the pipeline, not in the run: the drafter's instructions are +**hard-coded to Simplified Chinese** in three places. + +- `prompts.py` `SYSTEM_DRAFTER`: *"把英文太空冒险小说译成简体中文"* - translate the + English space adventure novel into Simplified Chinese. +- `prompts.py` `draft_prompt`: *"请把下面英文译为简体中文"* and a JSON key of + `"zh"`. +- `prompts.py` `SYSTEM_CRITIC`: *"对照英文原文和中文译文"*. + +The glossary is passed into the prompt, but the *instruction* says Chinese, so +the model followed the instruction and ignored the glossary's target. The +critic is hard-coded the same way, so it checked the Chinese text and found +nothing wrong - there was nothing wrong with it, for the language it was +actually asked about. + +Two consequences, both measured rather than inferred: + +1. **No non-Chinese translation has ever been produced by this pipeline.** The + nine other Qwen3-TTS languages exist only as the single-pass sample files + described above. +2. **The English translation output would be wrong for any other language**, + because the analyst, the drafter and the critic are all told they are + working on Chinese. + +The pipeline needs a target-language parameter before it can be run for +anything but Chinese. Until then, a production directory's name and its +`target_language` field are a *claim*, not a measurement - which is why the +delivery ingest now checks the script of the delivered text against the +declared language and refuses a delivery whose text is written in another +language (`umbraaudio/languages.py:script_mismatch`). + ## Open questions 1. Which model runs Passes 1-4? This is where the local LLM question returns, diff --git a/tests/test_language_scripts.py b/tests/test_language_scripts.py new file mode 100644 index 0000000..f317f58 --- /dev/null +++ b/tests/test_language_scripts.py @@ -0,0 +1,205 @@ +# Copyright 2026 Umbra Tools (umbratools.com) +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the script check on a translation delivery (spec 31 section 8). + +A delivery that claims one language and arrives written in another is not a +translation of that language. It is the "declared value disagrees with the +measured value" defect class, and it is checked at the boundary: once the rows +are stored, the review lane looks populated and the mistake is invisible. + +The check is about the *script*, not the language. It cannot tell Spanish +from Portuguese. It can tell Chinese from Spanish, which is the mistake that +actually happens: a pipeline configured for one language that silently +produces another. +""" + +from __future__ import annotations + +import sqlite3 + +import pytest + +from umbraaudio import languages as lang +from umbraaudio import translations as tr + +CHINESE = "欢迎收听《Deal Hunter》有声书。我是旁白，将为您介绍几位您将听到的声音。" +SPANISH = "Bienvenido al audiolibro Deal Hunter. Soy tu narrador." +RUSSIAN = "Добро пожаловать в аудиокнигу Deal Hunter. Я ваш рассказчик." +JAPANESE = "『Deal Hunter』的有声書へようこそ。私はナレーターです。" +KOREAN = "Deal Hunter 오디오북에 오신 것을 환영합니다. 저는 내레이터입니다." + + +# --- the registry's own answers -------------------------------------------- + + +def test_chinese_is_not_spanish() -> None: + reason = lang.script_mismatch("es", CHINESE) + assert reason is not None + assert "Han" in reason + assert "Spanish" in reason + + +def test_chinese_is_chinese() -> None: + assert lang.script_mismatch("zh", CHINESE) is None + + +def test_spanish_is_spanish() -> None: + assert lang.script_mismatch("es", SPANISH) is None + + +def test_russian_is_russian() -> None: + assert lang.script_mismatch("ru", RUSSIAN) is None + + +def test_latin_text_is_not_russian() -> None: + reason = lang.script_mismatch("ru", SPANISH) + assert reason is not None + assert "Latin" in reason + assert "Russian" in reason + + +def test_japanese_needs_kana() -> None: + assert lang.script_mismatch("ja", JAPANESE) is None + + +def test_chinese_text_is_not_japanese() -> None: + """Kanji are Han, so Chinese reads as Japanese without the kana.""" + reason = lang.script_mismatch("ja", CHINESE) + assert reason is not None + assert "kana" in reason + + +def test_japanese_text_is_not_chinese() -> None: + """Kana in a Chinese delivery means the target is the wrong language.""" + reason = lang.script_mismatch("zh", JAPANESE) + assert reason is not None + assert "Japanese" in reason or "kana" in reason + + +def test_korean_is_hangul() -> None: + assert lang.script_mismatch("ko", KOREAN) is None + + +def test_chinese_text_is_not_korean() -> None: + reason = lang.script_mismatch("ko", CHINESE) + assert reason is not None + + +def test_names_kept_in_english_do_not_break_the_check() -> None: + """A Chinese paragraph keeps Deal Hunter and Umbra Audio in English.""" + text = CHINESE + " Deal Hunter 是船名，Umbra Audio 是出品方。" + assert lang.script_mismatch("zh", text) is None + + +def test_an_unknown_language_is_not_checked() -> None: + """There is no script expectation for a language we do not know.""" + assert lang.script_mismatch("xx", CHINESE) is None + + +def test_empty_text_is_not_checked() -> None: + assert lang.script_mismatch("es", "") is None + assert lang.script_mismatch("es", "1234 -- ...") is None + + +# --- the gate at the ingest boundary ---------------------------------------- + + +def _seed(conn: sqlite3.Connection) -> None: + conn.executescript( + """ + CREATE TABLE project (id INTEGER PRIMARY KEY, name TEXT); + CREATE TABLE chapters (id INTEGER PRIMARY KEY, chapter_index INTEGER); + CREATE TABLE utterances ( + id INTEGER PRIMARY KEY, + chapter_id INTEGER, + utterance_index INTEGER, + speaker_name TEXT, + text TEXT, + context_before TEXT, + context_after TEXT, + language_code TEXT, + language_origin TEXT + ); + INSERT INTO project (id, name) VALUES (1, 'Test'); + INSERT INTO utterances (id, chapter_id, utterance_index, speaker_name, text) VALUES + (1, 1, 0, 'narrator', 'The ship landed.'), + (2, 1, 1, 'Kainda', 'Welcome to the ship.'); + """ + ) + conn.commit() + + +@pytest.fixture +def conn() -> sqlite3.Connection: + c = sqlite3.connect(":memory:") + c.row_factory = sqlite3.Row + _seed(c) + tr.ensure_tables(c) + yield c + c.close() + + +def test_a_mismatched_delivery_is_refused_and_writes_nothing(conn: sqlite3.Connection) -> None: + """The Chinese text filed under Spanish must not reach the database.""" + report = tr.import_translations( + conn, + 1, + { + "target_language": "es", + "utterances": [ + {"utterance_id": 1, "text": CHINESE}, + {"utterance_id": 2, "text": "我是旁白。"}, + ], + }, + ) + assert report.accepted == 0 + assert report.problems + assert "not Spanish" in report.problems[0] + assert conn.execute("SELECT COUNT(*) FROM utterance_translations").fetchone()[0] == 0 + + +def test_a_matching_delivery_is_imported(conn: sqlite3.Connection) -> None: + report = tr.import_translations( + conn, + 1, + { + "target_language": "es", + "utterances": [ + {"utterance_id": 1, "text": SPANISH}, + {"utterance_id": 2, "text": "Bienvenido a la nave."}, + ], + }, + ) + assert report.accepted == 2 + assert report.problems == [] + assert conn.execute("SELECT COUNT(*) FROM utterance_translations").fetchone()[0] == 2 + + +def test_a_locale_code_is_checked_against_its_base_language(conn: sqlite3.Connection) -> None: + """``es-ES`` is still Spanish; ``zh-Hans`` is still Chinese.""" + report = tr.import_translations( + conn, + 1, + {"target_language": "es-ES", "utterances": [{"utterance_id": 1, "text": CHINESE}]}, + ) + assert report.accepted == 0 + assert report.problems + + +def test_a_collapse_orphan_is_not_mistaken_for_a_mismatch(conn: sqlite3.Connection) -> None: + """A row with no text is a gap, not evidence of the wrong language.""" + report = tr.import_translations( + conn, + 1, + { + "target_language": "es", + "utterances": [ + {"utterance_id": 1, "text": SPANISH}, + {"utterance_id": 2, "voiced": False, "reason": "absorbed into 1"}, + ], + }, + ) + assert report.accepted == 2 + assert report.not_voiced == 1 + assert report.problems == [] diff --git a/umbraaudio/languages.py b/umbraaudio/languages.py index 5a2a94c..19a4388 100644 --- a/umbraaudio/languages.py +++ b/umbraaudio/languages.py @@ -27,6 +27,50 @@ ORIGIN_OPERATOR = "operator" ORIGINS: tuple[str, ...] = (ORIGIN_UNKNOWN, ORIGIN_UMBRATRANS, ORIGIN_OPERATOR) +# The script a language is written in. A delivery that claims one language +# and arrives written in another is not a translation of that language - it is +# the exact "declared value disagrees with the measured value" defect class +# that the release work already had to fix for audio padding. +SCRIPT_LATIN = "latin" +SCRIPT_CYRILLIC = "cyrillic" +SCRIPT_HAN = "han" +SCRIPT_KANA = "kana" +SCRIPT_HANGUL = "hangul" +SCRIPT_ARABIC = "arabic" +SCRIPT_DEVANAGARI = "devanagari" + +SCRIPT_NAMES: dict[str, str] = { + SCRIPT_LATIN: "Latin script", + SCRIPT_CYRILLIC: "Cyrillic script", + SCRIPT_HAN: "Han characters", + SCRIPT_KANA: "Japanese kana", + SCRIPT_HANGUL: "Hangul", + SCRIPT_ARABIC: "Arabic script", + SCRIPT_DEVANAGARI: "Devanagari", +} + +# The Unicode blocks that count as each script. Latin is split so that the +# accented forms are counted with it rather than falling through. +_SCRIPT_RANGES: tuple[tuple[str, int, int], ...] = ( + (SCRIPT_LATIN, 0x0041, 0x005A), + (SCRIPT_LATIN, 0x0061, 0x007A), + (SCRIPT_LATIN, 0x00C0, 0x024F), + (SCRIPT_CYRILLIC, 0x0400, 0x04FF), + (SCRIPT_HAN, 0x3400, 0x4DBF), + (SCRIPT_HAN, 0x4E00, 0x9FFF), + (SCRIPT_HAN, 0xF900, 0xFAFF), + (SCRIPT_KANA, 0x3040, 0x309F), + (SCRIPT_KANA, 0x30A0, 0x30FF), + (SCRIPT_KANA, 0x31F0, 0x31FF), + (SCRIPT_HANGUL, 0x1100, 0x11FF), + (SCRIPT_HANGUL, 0x3130, 0x318F), + (SCRIPT_HANGUL, 0xAC00, 0xD7AF), + (SCRIPT_ARABIC, 0x0600, 0x06FF), + (SCRIPT_ARABIC, 0x0750, 0x077F), + (SCRIPT_DEVANAGARI, 0x0900, 0x097F), +) + + @dataclass(frozen=True) class Language: """One language. ``tts_name`` is None when it cannot be voiced.""" @@ -34,6 +78,15 @@ class Language: code: str name: str tts_name: str | None = None + # The scripts this language is normally written in. A delivery written + # mostly in something else is a defect, not a translation. + scripts: tuple[str, ...] = (SCRIPT_LATIN,) + # A script that has to be present for the text to be this language at all: + # Japanese without kana is Chinese, and Korean without Hangul is Chinese. + required_script: str | None = None + # Scripts that would mean the delivery is a different language. Chinese + # text carrying kana is a Japanese translation filed under the wrong lane. + forbidden_scripts: tuple[str, ...] = () # The ten Qwen3-TTS languages. ``tts_name`` is copied from the model's own @@ -41,13 +94,31 @@ class Language: # reference - do not retype these by hand. QWEN_LANGUAGES: tuple[Language, ...] = ( Language("en", "English", "english"), - Language("zh", "Chinese", "chinese"), - Language("ja", "Japanese", "japanese"), - Language("ko", "Korean", "korean"), + Language( + "zh", + "Chinese", + "chinese", + scripts=(SCRIPT_HAN,), + forbidden_scripts=(SCRIPT_KANA, SCRIPT_HANGUL), + ), + Language( + "ja", + "Japanese", + "japanese", + scripts=(SCRIPT_HAN, SCRIPT_KANA), + required_script=SCRIPT_KANA, + ), + Language( + "ko", + "Korean", + "korean", + scripts=(SCRIPT_HANGUL,), + required_script=SCRIPT_HANGUL, + ), Language("de", "German", "german"), Language("es", "Spanish", "spanish"), Language("fr", "French", "french"), - Language("ru", "Russian", "russian"), + Language("ru", "Russian", "russian", scripts=(SCRIPT_CYRILLIC,)), Language("it", "Italian", "italian"), Language("pt", "Portuguese", "portuguese"), ) @@ -57,8 +128,8 @@ GAP_LANGUAGES: tuple[Language, ...] = ( Language("nl", "Dutch"), Language("no", "Norwegian"), Language("pl", "Polish"), - Language("ar", "Arabic"), - Language("hi", "Hindi"), + Language("ar", "Arabic", None, scripts=(SCRIPT_ARABIC,)), + Language("hi", "Hindi", None, scripts=(SCRIPT_DEVANAGARI,)), ) ALL_LANGUAGES: tuple[Language, ...] = QWEN_LANGUAGES + GAP_LANGUAGES @@ -109,6 +180,64 @@ def tts_name(value: str | None) -> str | None: return lang.tts_name if lang is not None else None +def script_counts(text: str) -> dict[str, int]: + """How many letters of *text* belong to each script. + + Only letters count. Punctuation, digits and whitespace say nothing about + which language a passage is written in, and a name kept in English inside + a Chinese paragraph must not pull the count towards Latin. + """ + counts: dict[str, int] = {} + for ch in text: + point = ord(ch) + for script, low, high in _SCRIPT_RANGES: + if low <= point <= high: + counts[script] = counts.get(script, 0) + 1 + break + return counts + + +def script_mismatch(code: str | None, text: str) -> str | None: + """Why *text* cannot be a translation into *code*, or None when it can. + + A delivery that claims Spanish and arrives written in Chinese is not a + Spanish translation. It is the "declared value disagrees with the measured + value" defect class, and it has to be caught at the boundary: once the + rows are in the database, the review lane looks populated and a reviewer + approves the wrong thing. + + The check is deliberately about the *script*, not the language. It cannot + tell Spanish from Portuguese, but it can tell Chinese from Spanish, which + is the mistake that actually happens - a pipeline configured for one + language that silently produces another. + """ + language = by_code(code) + if language is None: + # An unknown language has no script expectation to check against. + return None + + counts = script_counts(text) + total = sum(counts.values()) + if total == 0: + return None + + for forbidden in language.forbidden_scripts: + if counts.get(forbidden, 0) / total >= 0.10: + return f"the text is written in {SCRIPT_NAMES[forbidden]}, so it is not {language.name}" + + if language.required_script and not counts.get(language.required_script): + return ( + f"the text has no {SCRIPT_NAMES[language.required_script]}, " + f"so it is not {language.name}" + ) + + acceptable = sum(counts.get(script, 0) for script in language.scripts) + if acceptable / total < 0.5: + dominant = max(counts, key=lambda key: counts[key]) + return f"the text is mostly {SCRIPT_NAMES[dominant]}, so it is not {language.name}" + return None + + def language_name(value: str | None) -> str: """Display name for a code. Falls back to the raw value, then 'unknown'.""" lang = by_code(value) diff --git a/umbraaudio/translations.py b/umbraaudio/translations.py index d96b929..9994697 100644 --- a/umbraaudio/translations.py +++ b/umbraaudio/translations.py @@ -285,6 +285,22 @@ def import_translations( if not isinstance(rows, list): raise ValueError("'utterances' must be a list") + # A delivery that claims one language and arrives written in another is a + # defect, and it has to be caught before a single row is written. Once the + # rows are stored the lane looks populated, the review UI shows the text, + # and the mistake is invisible - which is how a Chinese translation ends up + # being approved as Spanish. + if target: + delivered_text = "\n".join( + str(row.get("text") or "") for row in rows if isinstance(row, dict) and row.get("text") + ) + mismatch = lang.script_mismatch(target, delivered_text) + if mismatch is not None: + report.problems.append( + f"delivery declares '{target}' but {mismatch}; nothing was imported" + ) + return report + for row in rows: if not isinstance(row, dict) or "utterance_id" not in row: report.problems.append(f"row without utterance_id: {json.dumps(row)[:80]}")