3f05489e25
The pipeline was hard-coded to Simplified Chinese in three places: the drafter was told to translate into Chinese, the critic was told to compare English with Chinese, and the glossary's translated forms were read from a "zh" key. A production that declared target_language: es-ES was therefore translated into Chinese and filed under a Spanish name - measured on oldbox, deal-hunter-es held 3,038 paragraphs of Chinese and was reported complete. A new registry, tools/translation/languages.py, is the single source of truth for what a target language is: its names, its script, its punctuation, its glossary key, its typography and its chapter-numbering convention. Ten languages are registered. The prompt scaffolding stays in Chinese - that is what the models read best - and the target is named with its Chinese, English and native names so there is no room to misread it. The registry now reaches every stage: - prompts.py: the analyst, drafter, critic and glossary blocks take the target. The hard rules about this book - Deal Hunter is the ship's name, the AI's name and the book's title, so it stays in English - are the language's own table for Chinese and the global rules for everything else, so a language with no table does not lose them. The Chinese rules are byte-for-byte what they were. - pipeline.py: --target-language, a run-directory guard that refuses to reuse a directory belonging to another language (from the manifest, or by measuring the text it already holds), a draft-keyed-by-target reader, and a refusal of a draft that came back in the wrong script. - book.py: per-language chapter-title and subtitle tables. A language with no table falls back to the English title, never to another language's; a language with a table and a missing chapter gets a visible marker in its own language. - qa.py: the target's quote marks, punctuation, script count and forbidden title forms. The Latin-residue check is skipped for a Latin target, where it would flag the whole book. - align.py: the aligner and verifier prompts name the target, and the sentence and clause splitting uses the target's own terminators. - render.py: the fonts, the language and country codes, the writing direction, the chapter headings and the fixed editorial strings around the translation are the target's. The Latin-run span is skipped for a Latin target, where marking the text as English would tell the renderer it is English. - project_status.py: the dashboard measures a sample of a production's text and reports a production whose text is not in the language it declares as "mismatched", not "complete". A chapter count is a claim about the files; the script check is a measurement of the text. Also fixes: the dashboard gated every language's production on the source production existing, so a Spanish production could never be reported even when it was there. The Chinese render is byte-identical to before on the real book, and the Chinese rules are unchanged. Suite: 620 passed, the same 68 pre-existing environment failures, +49 new tests.
901 lines
35 KiB
Python
901 lines
35 KiB
Python
# Copyright 2026 Umbra Tools (umbratools.com)
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
"""Tests for the utterance alignment pass.
|
|
|
|
The alignment model is an LLM, so the tests cover the deterministic parts:
|
|
the mechanical split, the repartition that guarantees the translation is
|
|
never rewritten, the validation rules that stop a line being handed to a
|
|
different speaker, and the delivery payload UmbraAudio ingests.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
sys.path.insert(0, str(HERE.parent))
|
|
|
|
from align import ( # noqa: E402
|
|
Segment,
|
|
align_chapter,
|
|
align_paragraph,
|
|
build_delivery,
|
|
is_fragment,
|
|
mechanical_split,
|
|
normalise_target_language,
|
|
parse_row_id,
|
|
parse_segments,
|
|
repartition,
|
|
source_utterances,
|
|
split_target_sentences,
|
|
system_aligner,
|
|
verify_translation,
|
|
)
|
|
from languages import resolve # noqa: E402
|
|
|
|
CHINESE = resolve("zh-Hans")
|
|
SPANISH = resolve("es-ES")
|
|
|
|
|
|
def bundle_fixture() -> dict[str, dict]:
|
|
"""Three narration utterances and one dialogue, as UmbraAudio has them."""
|
|
return {
|
|
"utt-100": {
|
|
"id": "utt-100",
|
|
"database_row_id": 100,
|
|
"utterance_index": 0,
|
|
"speaker_name_resolved": "narrator",
|
|
"utterance_type": "narration",
|
|
"text": "The ship was large.",
|
|
},
|
|
"utt-101": {
|
|
"id": "utt-101",
|
|
"database_row_id": 101,
|
|
"utterance_index": 1,
|
|
"speaker_name_resolved": "narrator",
|
|
"utterance_type": "narration",
|
|
"text": "The designer had laid it out carefully.",
|
|
},
|
|
"utt-102": {
|
|
"id": "utt-102",
|
|
"database_row_id": 102,
|
|
"utterance_index": 2,
|
|
"speaker_name_resolved": "Kainda Walters Foster Murphy",
|
|
"utterance_type": "dialogue",
|
|
"text": "I want to see the bridge.",
|
|
},
|
|
}
|
|
|
|
|
|
def paragraph_fixture() -> dict:
|
|
return {
|
|
"id": "ch-1-p0000",
|
|
"speaker": "narrator",
|
|
"type": "narration",
|
|
"text": "The ship was large. The designer had laid it out carefully.",
|
|
"source_indices": [0, 1],
|
|
"source_ids": ["utt-100", "utt-101"],
|
|
}
|
|
|
|
|
|
class LanguageTests(unittest.TestCase):
|
|
def test_locale_becomes_base_code(self) -> None:
|
|
self.assertEqual(normalise_target_language("zh-CN"), "zh")
|
|
self.assertEqual(normalise_target_language("es-ES"), "es")
|
|
|
|
def test_row_id_is_parsed_from_the_bundle_id(self) -> None:
|
|
self.assertEqual(parse_row_id("utt-83578"), 83578)
|
|
self.assertIsNone(parse_row_id("utt-abc"))
|
|
|
|
|
|
class SentenceTests(unittest.TestCase):
|
|
def test_chinese_sentences_keep_their_terminator(self) -> None:
|
|
self.assertEqual(
|
|
split_target_sentences("这艘船很大。设计师很用心。"),
|
|
["这艘船很大。", "设计师很用心。"],
|
|
)
|
|
|
|
def test_closing_quote_stays_with_its_sentence(self) -> None:
|
|
self.assertEqual(split_target_sentences("“你好。”然后她走了。"), ["“你好。”", "然后她走了。"])
|
|
|
|
def test_text_without_a_terminator_is_one_sentence(self) -> None:
|
|
self.assertEqual(split_target_sentences("没有句号"), ["没有句号"])
|
|
|
|
|
|
class MechanicalSplitTests(unittest.TestCase):
|
|
def test_exact_sentence_count(self) -> None:
|
|
text = "第一句。第二句。"
|
|
self.assertEqual(mechanical_split(text, 2), ["第一句。", "第二句。"])
|
|
|
|
def test_too_many_sentences_are_merged(self) -> None:
|
|
text = "第一句。第二句。第三句。"
|
|
parts = mechanical_split(text, 2)
|
|
self.assertIsNotNone(parts)
|
|
assert parts is not None
|
|
self.assertEqual(len(parts), 2)
|
|
self.assertEqual("".join(parts), text)
|
|
|
|
def test_too_few_sentences_are_split_at_a_comma(self) -> None:
|
|
text = "第一句很长，包含很多细节。"
|
|
parts = mechanical_split(text, 2)
|
|
self.assertIsNotNone(parts)
|
|
assert parts is not None
|
|
self.assertEqual(len(parts), 2)
|
|
self.assertEqual("".join(parts), text)
|
|
|
|
def test_nothing_is_lost(self) -> None:
|
|
text = "第一句。第二句。第三句。第四句。"
|
|
parts = mechanical_split(text, 3)
|
|
assert parts is not None
|
|
self.assertEqual("".join(parts), text)
|
|
|
|
|
|
class RepartitionTests(unittest.TestCase):
|
|
def test_gaps_go_to_the_preceding_part(self) -> None:
|
|
translation = "第一句。第二句。"
|
|
parts = repartition(translation, ["第一句。", "第二句。"])
|
|
assert parts is not None
|
|
self.assertEqual("".join(parts), translation)
|
|
|
|
def test_the_result_reproduces_the_translation(self) -> None:
|
|
translation = "这艘船很大。设计师很用心。"
|
|
parts = repartition(translation, ["这艘船很大。", "设计师很用心。"])
|
|
assert parts is not None
|
|
self.assertEqual("".join(parts), translation)
|
|
|
|
def test_a_missing_anchor_is_rejected(self) -> None:
|
|
self.assertIsNone(repartition("这艘船很大。", ["完全不同的句子。"]))
|
|
|
|
def test_whitespace_differences_are_tolerated(self) -> None:
|
|
translation = "第一句。 第二句。"
|
|
parts = repartition(translation, ["第一句。", "第二句。"])
|
|
assert parts is not None
|
|
# Each part is stripped, so the whitespace between them is dropped.
|
|
# The invariant is that nothing but whitespace is lost.
|
|
self.assertEqual("".join(parts).replace(" ", ""), translation.replace(" ", ""))
|
|
|
|
|
|
class ParseSegmentsTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.sources = source_utterances(paragraph_fixture(), bundle_fixture())
|
|
self.translation = "这艘船很大。设计师很用心。"
|
|
|
|
def test_a_valid_answer_is_accepted(self) -> None:
|
|
data = {
|
|
"segments": [
|
|
{"source_id": "utt-100", "target_text": "这艘船很大。"},
|
|
{"source_id": "utt-101", "target_text": "设计师很用心。"},
|
|
]
|
|
}
|
|
segments, problems = parse_segments(data, self.sources, self.translation)
|
|
self.assertEqual(problems, [])
|
|
self.assertEqual([s.text for s in segments], ["这艘船很大。", "设计师很用心。"])
|
|
|
|
def test_a_missing_segment_is_rejected(self) -> None:
|
|
data = {"segments": [{"source_id": "utt-100", "target_text": "这艘船很大。"}]}
|
|
_, problems = parse_segments(data, self.sources, self.translation)
|
|
self.assertTrue(problems)
|
|
|
|
def test_a_duplicated_source_id_is_rejected(self) -> None:
|
|
data = {
|
|
"segments": [
|
|
{"source_id": "utt-100", "target_text": "这艘船很大。"},
|
|
{"source_id": "utt-100", "target_text": "设计师很用心。"},
|
|
]
|
|
}
|
|
_, problems = parse_segments(data, self.sources, self.translation)
|
|
self.assertTrue(problems)
|
|
|
|
def test_text_that_is_not_in_the_translation_is_rejected(self) -> None:
|
|
data = {
|
|
"segments": [
|
|
{"source_id": "utt-100", "target_text": "这艘船很大。"},
|
|
{"source_id": "utt-101", "target_text": "凭空多出来的一句。"},
|
|
]
|
|
}
|
|
_, problems = parse_segments(data, self.sources, self.translation)
|
|
self.assertTrue(problems)
|
|
|
|
def test_a_merge_is_accepted_and_marked_not_voiced(self) -> None:
|
|
data = {
|
|
"segments": [
|
|
{"source_id": "utt-100", "target_text": "这艘船很大。设计师很用心。"},
|
|
{"source_id": "utt-101", "target_text": "", "merged_into": "utt-100"},
|
|
]
|
|
}
|
|
segments, problems = parse_segments(data, self.sources, self.translation)
|
|
self.assertEqual(problems, [])
|
|
self.assertTrue(segments[0].voiced)
|
|
self.assertFalse(segments[1].voiced)
|
|
self.assertEqual(segments[1].merged_into, "utt-100")
|
|
|
|
def test_a_line_cannot_be_handed_to_another_speaker(self) -> None:
|
|
paragraph = {
|
|
"id": "ch-1-p0001",
|
|
"speaker": "narrator",
|
|
"type": "narration",
|
|
"text": "The ship was large. I want to see the bridge.",
|
|
"source_indices": [0, 2],
|
|
"source_ids": ["utt-100", "utt-102"],
|
|
}
|
|
sources = source_utterances(paragraph, bundle_fixture())
|
|
data = {
|
|
"segments": [
|
|
{"source_id": "utt-100", "target_text": "这艘船很大。我想看看舰桥。"},
|
|
{"source_id": "utt-102", "target_text": "", "merged_into": "utt-100"},
|
|
]
|
|
}
|
|
_, problems = parse_segments(data, sources, "这艘船很大。我想看看舰桥。")
|
|
self.assertTrue(problems)
|
|
self.assertTrue(any("different speaker" in p for p in problems))
|
|
|
|
def test_a_merge_chain_is_rejected(self) -> None:
|
|
data = {
|
|
"segments": [
|
|
{"source_id": "utt-100", "target_text": "这艘船很大。"},
|
|
{"source_id": "utt-101", "target_text": "", "merged_into": "utt-101"},
|
|
]
|
|
}
|
|
_, problems = parse_segments(data, self.sources, self.translation)
|
|
self.assertTrue(problems)
|
|
|
|
|
|
class AlignParagraphTests(unittest.TestCase):
|
|
def test_a_single_source_utterance_is_copied(self) -> None:
|
|
paragraph = {
|
|
"id": "ch-1-p0002",
|
|
"speaker": "Kainda Walters Foster Murphy",
|
|
"type": "dialogue",
|
|
"text": "I want to see the bridge.",
|
|
"source_indices": [2],
|
|
"source_ids": ["utt-102"],
|
|
}
|
|
alignment = align_paragraph(paragraph, "我想看看舰桥。", bundle_fixture())
|
|
self.assertEqual(alignment.method, "one_to_one")
|
|
self.assertEqual(alignment.segments[0].text, "我想看看舰桥。")
|
|
|
|
def test_a_scene_break_has_no_translation(self) -> None:
|
|
paragraph = {
|
|
"id": "ch-1-break1",
|
|
"speaker": "",
|
|
"type": "scene_break",
|
|
"text": "* * *",
|
|
"source_indices": [3],
|
|
"source_ids": ["utt-103"],
|
|
}
|
|
alignment = align_paragraph(paragraph, "* * *", bundle_fixture())
|
|
self.assertEqual(alignment.method, "scene_break")
|
|
self.assertEqual(alignment.segments, [])
|
|
|
|
def test_without_a_model_the_split_is_mechanical_and_flagged(self) -> None:
|
|
alignment = align_paragraph(paragraph_fixture(), "这艘船很大。设计师很用心。", bundle_fixture())
|
|
self.assertEqual(alignment.method, "mechanical")
|
|
self.assertTrue(alignment.needs_review)
|
|
self.assertEqual(len(alignment.segments), 2)
|
|
|
|
def test_a_model_partition_is_used_and_verified(self) -> None:
|
|
class FakeLLM:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
self.calls += 1
|
|
return (
|
|
{
|
|
"segments": [
|
|
{"source_id": "utt-100", "target_text": "这艘船很大。"},
|
|
{"source_id": "utt-101", "target_text": "设计师很用心。"},
|
|
]
|
|
},
|
|
None,
|
|
)
|
|
|
|
llm = FakeLLM()
|
|
alignment = align_paragraph(paragraph_fixture(), "这艘船很大。设计师很用心。", bundle_fixture(), llm=llm)
|
|
self.assertEqual(alignment.method, "llm")
|
|
self.assertFalse(alignment.needs_review)
|
|
self.assertEqual("".join(s.text for s in alignment.segments), "这艘船很大。设计师很用心。")
|
|
self.assertEqual(llm.calls, 1)
|
|
|
|
|
|
class DeliveryTests(unittest.TestCase):
|
|
def test_the_payload_carries_the_database_row_id(self) -> None:
|
|
alignment = align_paragraph(paragraph_fixture(), "这艘船很大。设计师很用心。", bundle_fixture())
|
|
book = {"target_language": "zh-CN"}
|
|
delivery = build_delivery(
|
|
book,
|
|
[({"id": "ch-1", "paragraphs": [paragraph_fixture()]}, [alignment])],
|
|
bundle_fixture(),
|
|
)
|
|
self.assertEqual(delivery["target_language"], "zh")
|
|
self.assertEqual([u["utterance_id"] for u in delivery["utterances"]], [100, 101])
|
|
self.assertTrue(all(u["voiced"] for u in delivery["utterances"]))
|
|
self.assertEqual(delivery["unmatched"], [])
|
|
|
|
def test_a_collapsed_utterance_is_delivered_as_not_voiced(self) -> None:
|
|
alignment = align_paragraph(paragraph_fixture(), "这艘船很大。设计师很用心。", bundle_fixture())
|
|
alignment.segments = [
|
|
Segment(source_id="utt-100", text="这艘船很大。设计师很用心。"),
|
|
Segment(
|
|
source_id="utt-101",
|
|
text="",
|
|
voiced=False,
|
|
merged_into="utt-100",
|
|
reason="absorbed into utt-100 in the translated paragraph",
|
|
),
|
|
]
|
|
delivery = build_delivery(
|
|
{"target_language": "zh"},
|
|
[({"id": "ch-1", "paragraphs": [paragraph_fixture()]}, [alignment])],
|
|
bundle_fixture(),
|
|
)
|
|
collapsed = [u for u in delivery["utterances"] if not u["voiced"]]
|
|
self.assertEqual(len(collapsed), 1)
|
|
self.assertEqual(collapsed[0]["utterance_id"], 101)
|
|
self.assertIn("absorbed", collapsed[0]["reason"])
|
|
self.assertNotIn("text", collapsed[0])
|
|
|
|
def test_an_unverifiable_paragraph_is_delivered_as_a_visible_gap(self) -> None:
|
|
"""The row is kept and says why, rather than vanishing.
|
|
|
|
A delivery that simply omits the utterances is indistinguishable
|
|
from one that was never translated. The review lane needs to see the
|
|
gap and its reason.
|
|
"""
|
|
alignment = align_paragraph(paragraph_fixture(), "这艘船很大。设计师很用心。", bundle_fixture())
|
|
alignment.method = "mismatch"
|
|
alignment.segments = []
|
|
alignment.problems = ["translation belongs to another paragraph: it renders the next one"]
|
|
delivery = build_delivery(
|
|
{"target_language": "zh"},
|
|
[({"id": "ch-1", "paragraphs": [paragraph_fixture()]}, [alignment])],
|
|
bundle_fixture(),
|
|
)
|
|
self.assertEqual(len(delivery["utterances"]), 2)
|
|
self.assertTrue(all(not u["voiced"] for u in delivery["utterances"]))
|
|
self.assertEqual([u["utterance_id"] for u in delivery["utterances"]], [100, 101])
|
|
self.assertIn("another paragraph", delivery["utterances"][0]["reason"])
|
|
self.assertNotIn("text", delivery["utterances"][0])
|
|
|
|
|
|
class CommandTests(unittest.TestCase):
|
|
def test_the_cache_directory_is_created_before_the_first_call(self) -> None:
|
|
"""A missing cache directory would fail every model answer.
|
|
|
|
The client writes each answer as it arrives, so a cache directory that
|
|
was never created turns every model call into an error - and the
|
|
fallback would quietly turn the whole book into a mechanical split.
|
|
"""
|
|
import json
|
|
import tempfile
|
|
from unittest import mock
|
|
|
|
from align import main
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
book = root / "book.json"
|
|
book.write_text(
|
|
json.dumps(
|
|
{
|
|
"target_language": "zh-CN",
|
|
"chapters": [
|
|
{
|
|
"id": "ch-1",
|
|
"index": 1,
|
|
"title": "One",
|
|
"title_zh": "一",
|
|
"matter": "body",
|
|
"paragraphs": [],
|
|
}
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
bundle = root / "bundle.jsonl"
|
|
bundle.write_text("", encoding="utf-8")
|
|
run_dir = root / "runs"
|
|
run_dir.mkdir()
|
|
|
|
seen: dict[str, object] = {}
|
|
|
|
class StubLLM:
|
|
def __init__(self, **kwargs: object) -> None:
|
|
seen.update(kwargs)
|
|
|
|
with mock.patch.object(sys.modules["align"], "LLM", StubLLM):
|
|
exit_code = main(
|
|
[
|
|
"--book",
|
|
str(book),
|
|
"--run-dir",
|
|
str(run_dir),
|
|
"--out",
|
|
str(root / "delivery.json"),
|
|
"--bundle",
|
|
str(bundle),
|
|
]
|
|
)
|
|
self.assertEqual(exit_code, 0)
|
|
cache_dir = seen.get("cache_dir")
|
|
self.assertIsNotNone(cache_dir)
|
|
assert cache_dir is not None
|
|
self.assertTrue(Path(cache_dir).is_dir())
|
|
|
|
|
|
class DriftVerificationTests(unittest.TestCase):
|
|
"""The paragraph-boundary drift the production translation actually has."""
|
|
|
|
def test_a_fragment_is_an_english_text_without_a_sentence_end(self) -> None:
|
|
self.assertTrue(is_fragment("The chair in the middle has"))
|
|
self.assertTrue(is_fragment('"operations"'))
|
|
self.assertTrue(is_fragment("on the console. She looks around, and she asks,"))
|
|
self.assertFalse(is_fragment("The ship was large."))
|
|
self.assertFalse(is_fragment('"Yes, Captain."'))
|
|
self.assertFalse(is_fragment("“The last time was thirty-four years ago?”"))
|
|
|
|
def test_a_matching_translation_verifies(self) -> None:
|
|
class StubLLM:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
self.calls += 1
|
|
return ({"kind": "aligned", "reason": ""}, None)
|
|
|
|
llm = StubLLM()
|
|
verdict, _reason = verify_translation("The chair in the middle has", "中间的椅子旁", llm)
|
|
self.assertEqual(verdict, "aligned")
|
|
self.assertEqual(llm.calls, 1)
|
|
|
|
def test_a_drifted_translation_is_caught(self) -> None:
|
|
"""The Chinese is good - it just belongs to the next paragraph."""
|
|
|
|
class StubLLM:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
self.calls += 1
|
|
return (
|
|
{"kind": "drift", "reason": "the text is about another paragraph"},
|
|
None,
|
|
)
|
|
|
|
llm = StubLLM()
|
|
verdict, reason = verify_translation(
|
|
'"Yes, Captain."',
|
|
"“上一次有人登上这艘舰桥还是三十四年前，那时是西顿船长在任？”",
|
|
llm,
|
|
)
|
|
self.assertEqual(verdict, "drift")
|
|
self.assertIn("another paragraph", reason)
|
|
|
|
def test_an_unanswerable_verification_is_not_a_pass(self) -> None:
|
|
"""A model that cannot answer the question has verified nothing."""
|
|
|
|
class StubLLM:
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
return ({"reason": "I am not sure"}, None)
|
|
|
|
verdict, reason = verify_translation("The ship was large.", "这艘船很大。", StubLLM())
|
|
self.assertEqual(verdict, "drift")
|
|
self.assertIn("not understood", reason)
|
|
|
|
def test_the_neighbouring_paragraphs_are_shown_to_the_model(self) -> None:
|
|
"""A paragraph's translation cannot be judged without its neighbours."""
|
|
|
|
seen: dict[str, str] = {}
|
|
|
|
class StubLLM:
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
seen["user"] = user
|
|
return ({"kind": "aligned", "reason": ""}, None)
|
|
|
|
verify_translation(
|
|
"The chair in the middle has",
|
|
"中间的椅子旁",
|
|
StubLLM(),
|
|
previous_text="On the right, the console says",
|
|
next_text='"Captain"',
|
|
)
|
|
self.assertIn("On the right, the console says", seen["user"])
|
|
self.assertIn("Captain", seen["user"])
|
|
|
|
def test_an_incomplete_paragraph_keeps_its_text_and_is_flagged(self) -> None:
|
|
"""It is still the right translation in the right place."""
|
|
|
|
class StubLLM:
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
return ({"kind": "incomplete", "reason": "漏译了最后一句"}, None)
|
|
|
|
chapter = {
|
|
"id": "ch-1",
|
|
"paragraphs": [
|
|
{
|
|
"id": "ch-1-p0000",
|
|
"speaker": "narrator",
|
|
"type": "narration",
|
|
"text": "The chair in the middle has",
|
|
"source_indices": [0],
|
|
"source_ids": ["utt-100"],
|
|
}
|
|
],
|
|
}
|
|
alignments, report = align_chapter(
|
|
chapter,
|
|
{"ch-1-p0000": "中间的椅子旁"},
|
|
bundle_fixture(),
|
|
llm=StubLLM(),
|
|
verify=True,
|
|
)
|
|
self.assertNotEqual(alignments[0].method, "mismatch")
|
|
self.assertTrue(alignments[0].segments)
|
|
self.assertTrue(alignments[0].needs_review)
|
|
self.assertIn("incomplete", alignments[0].problems[0])
|
|
self.assertEqual(report["needs_review"], 1)
|
|
|
|
def test_a_drifted_paragraph_is_dropped_from_the_delivery(self) -> None:
|
|
"""A visible gap is better than a confident wrong line in the audio."""
|
|
|
|
class StubLLM:
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
return ({"kind": "drift", "reason": "belongs to another paragraph"}, None)
|
|
|
|
chapter = {
|
|
"id": "ch-1",
|
|
"paragraphs": [
|
|
{
|
|
"id": "ch-1-p0000",
|
|
"speaker": "narrator",
|
|
"type": "narration",
|
|
"text": "The chair in the middle has",
|
|
"source_indices": [0],
|
|
"source_ids": ["utt-100"],
|
|
}
|
|
],
|
|
}
|
|
translated = {"ch-1-p0000": "“Yes, Captain.”"}
|
|
alignments, report = align_chapter(
|
|
chapter,
|
|
translated,
|
|
bundle_fixture(),
|
|
llm=StubLLM(),
|
|
verify=True,
|
|
)
|
|
self.assertEqual(alignments[0].method, "mismatch")
|
|
self.assertTrue(alignments[0].needs_review)
|
|
self.assertEqual(alignments[0].segments, [])
|
|
self.assertEqual(report["needs_review"], 1)
|
|
|
|
def test_the_check_only_starts_at_a_fragment(self) -> None:
|
|
"""A complete paragraph needs no check: the drafter had no reason to re-flow it."""
|
|
|
|
class CountingLLM:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
self.calls += 1
|
|
return ({"kind": "aligned", "reason": ""}, None)
|
|
|
|
chapter = {
|
|
"id": "ch-1",
|
|
"paragraphs": [
|
|
{
|
|
"id": "ch-1-p0000",
|
|
"speaker": "narrator",
|
|
"type": "narration",
|
|
"text": "The ship was large.",
|
|
"source_indices": [0],
|
|
"source_ids": ["utt-100"],
|
|
}
|
|
],
|
|
}
|
|
llm = CountingLLM()
|
|
align_chapter(chapter, {"ch-1-p0000": "这艘船很大。"}, bundle_fixture(), llm=llm, verify=True)
|
|
self.assertEqual(llm.calls, 0)
|
|
|
|
def test_the_check_follows_the_drift_until_the_passage_resynchronises(self) -> None:
|
|
"""A drift runs on: the next paragraphs carry the wrong content too."""
|
|
|
|
class DriftingLLM:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
self.calls += 1
|
|
if self.calls <= 3:
|
|
return ({"kind": "drift", "reason": "drifted"}, None)
|
|
return ({"kind": "aligned", "reason": ""}, None)
|
|
|
|
def para(pid: str, text: str, sid: str, index: int) -> dict:
|
|
return {
|
|
"id": pid,
|
|
"speaker": "narrator",
|
|
"type": "narration",
|
|
"text": text,
|
|
"source_indices": [index],
|
|
"source_ids": [sid],
|
|
}
|
|
|
|
chapter = {
|
|
"id": "ch-1",
|
|
"paragraphs": [
|
|
para("ch-1-p0000", "The chair in the middle has", "utt-100", 0),
|
|
para("ch-1-p0001", '"Captain"', "utt-101", 1),
|
|
para("ch-1-p0002", "on the console.", "utt-102", 2),
|
|
para("ch-1-p0003", "She looked around and saw dust everywhere.", "utt-103", 3),
|
|
],
|
|
}
|
|
llm = DriftingLLM()
|
|
alignments, _report = align_chapter(
|
|
chapter,
|
|
{p["id"]: "译文" for p in chapter["paragraphs"]},
|
|
bundle_fixture(),
|
|
llm=llm,
|
|
verify=True,
|
|
)
|
|
methods = [a.method for a in alignments]
|
|
self.assertEqual(methods[0], "mismatch")
|
|
self.assertEqual(methods[1], "mismatch")
|
|
self.assertEqual(methods[2], "mismatch")
|
|
self.assertEqual(methods[3], "one_to_one")
|
|
# The drift follows the run: the third paragraph was not a fragment, but
|
|
# it was checked because the passage had not re-synchronised yet.
|
|
self.assertEqual(llm.calls, 4)
|
|
|
|
def test_an_incomplete_paragraph_inside_a_drift_run_is_part_of_the_drift(self) -> None:
|
|
"""The drafter re-flowed the passage; this paragraph is one of the pieces.
|
|
|
|
The English around a drift is often itself a sentence split across
|
|
paragraph ids, so a truncated lead-in inside a confirmed drift is the
|
|
tail of the previous paragraph, not a translation of this one.
|
|
"""
|
|
|
|
class DriftingLLM:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
self.calls += 1
|
|
if self.calls == 1:
|
|
return ({"kind": "drift", "reason": "belongs to another paragraph"}, None)
|
|
if self.calls == 2:
|
|
return ({"kind": "incomplete", "reason": "only the first phrase"}, None)
|
|
return ({"kind": "aligned", "reason": ""}, None)
|
|
|
|
def para(pid: str, text: str, sid: str, index: int) -> dict:
|
|
return {
|
|
"id": pid,
|
|
"speaker": "narrator",
|
|
"type": "narration",
|
|
"text": text,
|
|
"source_indices": [index],
|
|
"source_ids": [sid],
|
|
}
|
|
|
|
chapter = {
|
|
"id": "ch-1",
|
|
"paragraphs": [
|
|
para("ch-1-p0000", "The chair in the middle has", "utt-100", 0),
|
|
para("ch-1-p0001", "on the console.", "utt-101", 1),
|
|
para("ch-1-p0002", "She looked around and saw dust everywhere.", "utt-102", 2),
|
|
],
|
|
}
|
|
alignments, _report = align_chapter(
|
|
chapter,
|
|
{p["id"]: "译文" for p in chapter["paragraphs"]},
|
|
bundle_fixture(),
|
|
llm=DriftingLLM(),
|
|
verify=True,
|
|
)
|
|
methods = [a.method for a in alignments]
|
|
self.assertEqual(methods[0], "mismatch")
|
|
self.assertEqual(methods[1], "mismatch")
|
|
self.assertEqual(methods[2], "one_to_one")
|
|
|
|
def test_an_incomplete_paragraph_before_any_drift_keeps_its_text(self) -> None:
|
|
"""Nothing has been proven wrong yet, so the line is kept and flagged."""
|
|
|
|
class IncompleteLLM:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
self.calls += 1
|
|
return ({"kind": "incomplete", "reason": "漏译了最后一句"}, None)
|
|
|
|
chapter = {
|
|
"id": "ch-1",
|
|
"paragraphs": [
|
|
{
|
|
"id": "ch-1-p0000",
|
|
"speaker": "narrator",
|
|
"type": "narration",
|
|
"text": "The chair in the middle has",
|
|
"source_indices": [0],
|
|
"source_ids": ["utt-100"],
|
|
}
|
|
],
|
|
}
|
|
alignments, _report = align_chapter(
|
|
chapter,
|
|
{"ch-1-p0000": "中间的椅子旁"},
|
|
bundle_fixture(),
|
|
llm=IncompleteLLM(),
|
|
verify=True,
|
|
)
|
|
self.assertNotEqual(alignments[0].method, "mismatch")
|
|
self.assertTrue(alignments[0].segments)
|
|
|
|
def test_a_fragment_called_aligned_inside_a_broken_window_is_dropped(self) -> None:
|
|
"""The model calls these two "aligned" and they are wrong.
|
|
|
|
Measured on the real ch-15 window: the Chinese under the two
|
|
paragraphs before the drift belongs to the paragraph before them.
|
|
The per-paragraph verdict is not trustworthy inside a re-flowed
|
|
passage, so a fragment in a broken window is dropped even when the
|
|
model says it is where it belongs.
|
|
"""
|
|
|
|
class StubLLM:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
self.calls += 1
|
|
if self.calls == 1:
|
|
return ({"kind": "aligned", "reason": ""}, None)
|
|
if self.calls == 2:
|
|
return ({"kind": "drift", "reason": "belongs to another paragraph"}, None)
|
|
return ({"kind": "aligned", "reason": ""}, None)
|
|
|
|
def para(pid: str, text: str, sid: str, index: int) -> dict:
|
|
return {
|
|
"id": pid,
|
|
"speaker": "narrator",
|
|
"type": "narration",
|
|
"text": text,
|
|
"source_indices": [index],
|
|
"source_ids": [sid],
|
|
}
|
|
|
|
chapter = {
|
|
"id": "ch-1",
|
|
"paragraphs": [
|
|
para("ch-1-p0000", '"operations"', "utt-100", 0),
|
|
para("ch-1-p0001", "On the right, the console says", "utt-101", 1),
|
|
para("ch-1-p0002", "The chair in the middle has", "utt-102", 2),
|
|
para("ch-1-p0003", "She grabbed the dusty chair.", "utt-103", 3),
|
|
],
|
|
}
|
|
alignments, report = align_chapter(
|
|
chapter,
|
|
{p["id"]: "译文" for p in chapter["paragraphs"]},
|
|
bundle_fixture(),
|
|
llm=StubLLM(),
|
|
verify=True,
|
|
)
|
|
methods = [a.method for a in alignments]
|
|
# The first fragment was called "aligned" but sits in a broken window.
|
|
self.assertEqual(methods[0], "mismatch")
|
|
self.assertEqual(methods[1], "mismatch")
|
|
self.assertEqual(methods[2], "mismatch")
|
|
# The last paragraph is a complete sentence and re-synchronises the
|
|
# passage, so it is kept.
|
|
self.assertEqual(methods[3], "one_to_one")
|
|
self.assertEqual(report["needs_review"], 3)
|
|
|
|
def test_a_fragment_called_incomplete_at_the_head_of_a_window_keeps_its_text(self) -> None:
|
|
"""It demonstrably renders its own leading content.
|
|
|
|
Measured on the real ch-15 window: the first paragraph's Chinese is a
|
|
correct translation of its first two sentences, and only its third
|
|
sentence was re-flowed away. Dropping it would lose two good lines to
|
|
save nothing.
|
|
"""
|
|
|
|
class StubLLM:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
self.calls += 1
|
|
if self.calls == 1:
|
|
return ({"kind": "incomplete", "reason": "漏译了第三句"}, None)
|
|
if self.calls == 2:
|
|
return ({"kind": "drift", "reason": "belongs to another paragraph"}, None)
|
|
return ({"kind": "aligned", "reason": ""}, None)
|
|
|
|
def para(pid: str, text: str, sid: str, index: int) -> dict:
|
|
return {
|
|
"id": pid,
|
|
"speaker": "narrator",
|
|
"type": "narration",
|
|
"text": text,
|
|
"source_indices": [index],
|
|
"source_ids": [sid],
|
|
}
|
|
|
|
chapter = {
|
|
"id": "ch-1",
|
|
"paragraphs": [
|
|
para("ch-1-p0000", "Three chairs face forward, each equipped with a screen", "utt-100", 0),
|
|
para("ch-1-p0001", "The chair in the middle has", "utt-101", 1),
|
|
para("ch-1-p0002", "She grabbed the dusty chair.", "utt-102", 2),
|
|
],
|
|
}
|
|
alignments, report = align_chapter(
|
|
chapter,
|
|
{p["id"]: "译文" for p in chapter["paragraphs"]},
|
|
bundle_fixture(),
|
|
llm=StubLLM(),
|
|
verify=True,
|
|
)
|
|
self.assertNotEqual(alignments[0].method, "mismatch")
|
|
self.assertTrue(alignments[0].segments)
|
|
self.assertTrue(alignments[0].needs_review)
|
|
self.assertEqual(alignments[1].method, "mismatch")
|
|
self.assertEqual(alignments[2].method, "one_to_one")
|
|
self.assertEqual(report["needs_review"], 2)
|
|
|
|
|
|
class TargetLanguageTests(unittest.TestCase):
|
|
"""The alignment pass must work in the language it is aligning."""
|
|
|
|
def test_the_aligner_is_told_the_target_language(self) -> None:
|
|
self.assertIn("中文", system_aligner(CHINESE))
|
|
self.assertIn("西班牙语", system_aligner(SPANISH))
|
|
self.assertNotIn("中文", system_aligner(SPANISH))
|
|
|
|
def test_a_spanish_sentence_splits_on_the_spanish_terminators(self) -> None:
|
|
text = "La silla está sucia. ¿Who está ahí? ¡Ven!"
|
|
sentences = split_target_sentences(text, SPANISH)
|
|
self.assertEqual(len(sentences), 3)
|
|
self.assertTrue(sentences[0].endswith("sucia."))
|
|
self.assertTrue(sentences[1].endswith("ahí?"))
|
|
|
|
def test_a_chinese_sentence_splits_on_the_chinese_terminators(self) -> None:
|
|
text = "椅子很脏。谁在那里？快来！"
|
|
self.assertEqual(len(split_target_sentences(text, CHINESE)), 3)
|
|
|
|
def test_the_mechanical_split_uses_the_target_clause_marks(self) -> None:
|
|
"""A Spanish translation merges sentences with a comma, not a full stop."""
|
|
text = "La silla estaba sucia, y el suelo tenía polvo"
|
|
pieces = mechanical_split(text, 2, SPANISH)
|
|
self.assertIsNotNone(pieces)
|
|
assert pieces is not None
|
|
self.assertEqual(len(pieces), 2)
|
|
# The contract is whitespace-insensitive: the split never rewrites
|
|
# the translation, it only divides it.
|
|
self.assertEqual("".join(pieces).replace(" ", ""), text.replace(" ", ""))
|
|
|
|
def test_the_delivery_carries_the_base_language_code(self) -> None:
|
|
alignment = align_paragraph(
|
|
{
|
|
"id": "ch-1-p0000",
|
|
"type": "narration",
|
|
"text": "The ship was large.",
|
|
"source_indices": [0],
|
|
"source_ids": ["utt-100"],
|
|
},
|
|
"El barco era grande.",
|
|
bundle_fixture(),
|
|
)
|
|
delivery = build_delivery(
|
|
{"target_language": "es-ES"},
|
|
[({"id": "ch-1", "paragraphs": []}, [alignment])],
|
|
bundle_fixture(),
|
|
target=SPANISH,
|
|
)
|
|
self.assertEqual(delivery["target_language"], "es")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|