11fd8d8060
A paragraph whose translation could not be verified produced no rows at all, which is indistinguishable from one that was never translated: the review lane showed a missing utterance with no explanation. The project's own rule for a gap is "the row is kept, the gap is visible, no clip is generated", and it already applies to the collapse orphans. An unverifiable paragraph is the same kind of gap, so it now gets the same treatment: a voiced=false row carrying the reason. Measured on the real ch-15 window: the ten drifted paragraphs now produce thirteen explicit not-voiced rows instead of vanishing, and the reviewer can see that the translation was dropped because it renders another paragraph rather than because nobody translated it. 60 tests, ruff clean.
845 lines
33 KiB
Python
845 lines
33 KiB
Python
# Copyright 2026 Umbra Tools (umbratools.com)
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
"""Tests for the utterance alignment pass.
|
|
|
|
The alignment model is an LLM, so the tests cover the deterministic parts:
|
|
the mechanical split, the repartition that guarantees the translation is
|
|
never rewritten, the validation rules that stop a line being handed to a
|
|
different speaker, and the delivery payload UmbraAudio ingests.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
sys.path.insert(0, str(HERE.parent))
|
|
|
|
from align import ( # noqa: E402
|
|
Segment,
|
|
align_chapter,
|
|
align_paragraph,
|
|
build_delivery,
|
|
is_fragment,
|
|
mechanical_split,
|
|
normalise_target_language,
|
|
parse_row_id,
|
|
parse_segments,
|
|
repartition,
|
|
source_utterances,
|
|
split_target_sentences,
|
|
verify_translation,
|
|
)
|
|
|
|
|
|
def bundle_fixture() -> dict[str, dict]:
|
|
"""Three narration utterances and one dialogue, as UmbraAudio has them."""
|
|
return {
|
|
"utt-100": {
|
|
"id": "utt-100",
|
|
"database_row_id": 100,
|
|
"utterance_index": 0,
|
|
"speaker_name_resolved": "narrator",
|
|
"utterance_type": "narration",
|
|
"text": "The ship was large.",
|
|
},
|
|
"utt-101": {
|
|
"id": "utt-101",
|
|
"database_row_id": 101,
|
|
"utterance_index": 1,
|
|
"speaker_name_resolved": "narrator",
|
|
"utterance_type": "narration",
|
|
"text": "The designer had laid it out carefully.",
|
|
},
|
|
"utt-102": {
|
|
"id": "utt-102",
|
|
"database_row_id": 102,
|
|
"utterance_index": 2,
|
|
"speaker_name_resolved": "Kainda Walters Foster Murphy",
|
|
"utterance_type": "dialogue",
|
|
"text": "I want to see the bridge.",
|
|
},
|
|
}
|
|
|
|
|
|
def paragraph_fixture() -> dict:
|
|
return {
|
|
"id": "ch-1-p0000",
|
|
"speaker": "narrator",
|
|
"type": "narration",
|
|
"text": "The ship was large. The designer had laid it out carefully.",
|
|
"source_indices": [0, 1],
|
|
"source_ids": ["utt-100", "utt-101"],
|
|
}
|
|
|
|
|
|
class LanguageTests(unittest.TestCase):
|
|
def test_locale_becomes_base_code(self) -> None:
|
|
self.assertEqual(normalise_target_language("zh-CN"), "zh")
|
|
self.assertEqual(normalise_target_language("es-ES"), "es")
|
|
|
|
def test_row_id_is_parsed_from_the_bundle_id(self) -> None:
|
|
self.assertEqual(parse_row_id("utt-83578"), 83578)
|
|
self.assertIsNone(parse_row_id("utt-abc"))
|
|
|
|
|
|
class SentenceTests(unittest.TestCase):
|
|
def test_chinese_sentences_keep_their_terminator(self) -> None:
|
|
self.assertEqual(
|
|
split_target_sentences("这艘船很大。设计师很用心。"),
|
|
["这艘船很大。", "设计师很用心。"],
|
|
)
|
|
|
|
def test_closing_quote_stays_with_its_sentence(self) -> None:
|
|
self.assertEqual(split_target_sentences("“你好。”然后她走了。"), ["“你好。”", "然后她走了。"])
|
|
|
|
def test_text_without_a_terminator_is_one_sentence(self) -> None:
|
|
self.assertEqual(split_target_sentences("没有句号"), ["没有句号"])
|
|
|
|
|
|
class MechanicalSplitTests(unittest.TestCase):
|
|
def test_exact_sentence_count(self) -> None:
|
|
text = "第一句。第二句。"
|
|
self.assertEqual(mechanical_split(text, 2), ["第一句。", "第二句。"])
|
|
|
|
def test_too_many_sentences_are_merged(self) -> None:
|
|
text = "第一句。第二句。第三句。"
|
|
parts = mechanical_split(text, 2)
|
|
self.assertIsNotNone(parts)
|
|
assert parts is not None
|
|
self.assertEqual(len(parts), 2)
|
|
self.assertEqual("".join(parts), text)
|
|
|
|
def test_too_few_sentences_are_split_at_a_comma(self) -> None:
|
|
text = "第一句很长，包含很多细节。"
|
|
parts = mechanical_split(text, 2)
|
|
self.assertIsNotNone(parts)
|
|
assert parts is not None
|
|
self.assertEqual(len(parts), 2)
|
|
self.assertEqual("".join(parts), text)
|
|
|
|
def test_nothing_is_lost(self) -> None:
|
|
text = "第一句。第二句。第三句。第四句。"
|
|
parts = mechanical_split(text, 3)
|
|
assert parts is not None
|
|
self.assertEqual("".join(parts), text)
|
|
|
|
|
|
class RepartitionTests(unittest.TestCase):
|
|
def test_gaps_go_to_the_preceding_part(self) -> None:
|
|
translation = "第一句。第二句。"
|
|
parts = repartition(translation, ["第一句。", "第二句。"])
|
|
assert parts is not None
|
|
self.assertEqual("".join(parts), translation)
|
|
|
|
def test_the_result_reproduces_the_translation(self) -> None:
|
|
translation = "这艘船很大。设计师很用心。"
|
|
parts = repartition(translation, ["这艘船很大。", "设计师很用心。"])
|
|
assert parts is not None
|
|
self.assertEqual("".join(parts), translation)
|
|
|
|
def test_a_missing_anchor_is_rejected(self) -> None:
|
|
self.assertIsNone(repartition("这艘船很大。", ["完全不同的句子。"]))
|
|
|
|
def test_whitespace_differences_are_tolerated(self) -> None:
|
|
translation = "第一句。 第二句。"
|
|
parts = repartition(translation, ["第一句。", "第二句。"])
|
|
assert parts is not None
|
|
# Each part is stripped, so the whitespace between them is dropped.
|
|
# The invariant is that nothing but whitespace is lost.
|
|
self.assertEqual("".join(parts).replace(" ", ""), translation.replace(" ", ""))
|
|
|
|
|
|
class ParseSegmentsTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.sources = source_utterances(paragraph_fixture(), bundle_fixture())
|
|
self.translation = "这艘船很大。设计师很用心。"
|
|
|
|
def test_a_valid_answer_is_accepted(self) -> None:
|
|
data = {
|
|
"segments": [
|
|
{"source_id": "utt-100", "target_text": "这艘船很大。"},
|
|
{"source_id": "utt-101", "target_text": "设计师很用心。"},
|
|
]
|
|
}
|
|
segments, problems = parse_segments(data, self.sources, self.translation)
|
|
self.assertEqual(problems, [])
|
|
self.assertEqual([s.text for s in segments], ["这艘船很大。", "设计师很用心。"])
|
|
|
|
def test_a_missing_segment_is_rejected(self) -> None:
|
|
data = {"segments": [{"source_id": "utt-100", "target_text": "这艘船很大。"}]}
|
|
_, problems = parse_segments(data, self.sources, self.translation)
|
|
self.assertTrue(problems)
|
|
|
|
def test_a_duplicated_source_id_is_rejected(self) -> None:
|
|
data = {
|
|
"segments": [
|
|
{"source_id": "utt-100", "target_text": "这艘船很大。"},
|
|
{"source_id": "utt-100", "target_text": "设计师很用心。"},
|
|
]
|
|
}
|
|
_, problems = parse_segments(data, self.sources, self.translation)
|
|
self.assertTrue(problems)
|
|
|
|
def test_text_that_is_not_in_the_translation_is_rejected(self) -> None:
|
|
data = {
|
|
"segments": [
|
|
{"source_id": "utt-100", "target_text": "这艘船很大。"},
|
|
{"source_id": "utt-101", "target_text": "凭空多出来的一句。"},
|
|
]
|
|
}
|
|
_, problems = parse_segments(data, self.sources, self.translation)
|
|
self.assertTrue(problems)
|
|
|
|
def test_a_merge_is_accepted_and_marked_not_voiced(self) -> None:
|
|
data = {
|
|
"segments": [
|
|
{"source_id": "utt-100", "target_text": "这艘船很大。设计师很用心。"},
|
|
{"source_id": "utt-101", "target_text": "", "merged_into": "utt-100"},
|
|
]
|
|
}
|
|
segments, problems = parse_segments(data, self.sources, self.translation)
|
|
self.assertEqual(problems, [])
|
|
self.assertTrue(segments[0].voiced)
|
|
self.assertFalse(segments[1].voiced)
|
|
self.assertEqual(segments[1].merged_into, "utt-100")
|
|
|
|
def test_a_line_cannot_be_handed_to_another_speaker(self) -> None:
|
|
paragraph = {
|
|
"id": "ch-1-p0001",
|
|
"speaker": "narrator",
|
|
"type": "narration",
|
|
"text": "The ship was large. I want to see the bridge.",
|
|
"source_indices": [0, 2],
|
|
"source_ids": ["utt-100", "utt-102"],
|
|
}
|
|
sources = source_utterances(paragraph, bundle_fixture())
|
|
data = {
|
|
"segments": [
|
|
{"source_id": "utt-100", "target_text": "这艘船很大。我想看看舰桥。"},
|
|
{"source_id": "utt-102", "target_text": "", "merged_into": "utt-100"},
|
|
]
|
|
}
|
|
_, problems = parse_segments(data, sources, "这艘船很大。我想看看舰桥。")
|
|
self.assertTrue(problems)
|
|
self.assertTrue(any("different speaker" in p for p in problems))
|
|
|
|
def test_a_merge_chain_is_rejected(self) -> None:
|
|
data = {
|
|
"segments": [
|
|
{"source_id": "utt-100", "target_text": "这艘船很大。"},
|
|
{"source_id": "utt-101", "target_text": "", "merged_into": "utt-101"},
|
|
]
|
|
}
|
|
_, problems = parse_segments(data, self.sources, self.translation)
|
|
self.assertTrue(problems)
|
|
|
|
|
|
class AlignParagraphTests(unittest.TestCase):
|
|
def test_a_single_source_utterance_is_copied(self) -> None:
|
|
paragraph = {
|
|
"id": "ch-1-p0002",
|
|
"speaker": "Kainda Walters Foster Murphy",
|
|
"type": "dialogue",
|
|
"text": "I want to see the bridge.",
|
|
"source_indices": [2],
|
|
"source_ids": ["utt-102"],
|
|
}
|
|
alignment = align_paragraph(paragraph, "我想看看舰桥。", bundle_fixture())
|
|
self.assertEqual(alignment.method, "one_to_one")
|
|
self.assertEqual(alignment.segments[0].text, "我想看看舰桥。")
|
|
|
|
def test_a_scene_break_has_no_translation(self) -> None:
|
|
paragraph = {
|
|
"id": "ch-1-break1",
|
|
"speaker": "",
|
|
"type": "scene_break",
|
|
"text": "* * *",
|
|
"source_indices": [3],
|
|
"source_ids": ["utt-103"],
|
|
}
|
|
alignment = align_paragraph(paragraph, "* * *", bundle_fixture())
|
|
self.assertEqual(alignment.method, "scene_break")
|
|
self.assertEqual(alignment.segments, [])
|
|
|
|
def test_without_a_model_the_split_is_mechanical_and_flagged(self) -> None:
|
|
alignment = align_paragraph(paragraph_fixture(), "这艘船很大。设计师很用心。", bundle_fixture())
|
|
self.assertEqual(alignment.method, "mechanical")
|
|
self.assertTrue(alignment.needs_review)
|
|
self.assertEqual(len(alignment.segments), 2)
|
|
|
|
def test_a_model_partition_is_used_and_verified(self) -> None:
|
|
class FakeLLM:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
self.calls += 1
|
|
return (
|
|
{
|
|
"segments": [
|
|
{"source_id": "utt-100", "target_text": "这艘船很大。"},
|
|
{"source_id": "utt-101", "target_text": "设计师很用心。"},
|
|
]
|
|
},
|
|
None,
|
|
)
|
|
|
|
llm = FakeLLM()
|
|
alignment = align_paragraph(paragraph_fixture(), "这艘船很大。设计师很用心。", bundle_fixture(), llm=llm)
|
|
self.assertEqual(alignment.method, "llm")
|
|
self.assertFalse(alignment.needs_review)
|
|
self.assertEqual("".join(s.text for s in alignment.segments), "这艘船很大。设计师很用心。")
|
|
self.assertEqual(llm.calls, 1)
|
|
|
|
|
|
class DeliveryTests(unittest.TestCase):
|
|
def test_the_payload_carries_the_database_row_id(self) -> None:
|
|
alignment = align_paragraph(paragraph_fixture(), "这艘船很大。设计师很用心。", bundle_fixture())
|
|
book = {"target_language": "zh-CN"}
|
|
delivery = build_delivery(
|
|
book,
|
|
[({"id": "ch-1", "paragraphs": [paragraph_fixture()]}, [alignment])],
|
|
bundle_fixture(),
|
|
)
|
|
self.assertEqual(delivery["target_language"], "zh")
|
|
self.assertEqual([u["utterance_id"] for u in delivery["utterances"]], [100, 101])
|
|
self.assertTrue(all(u["voiced"] for u in delivery["utterances"]))
|
|
self.assertEqual(delivery["unmatched"], [])
|
|
|
|
def test_a_collapsed_utterance_is_delivered_as_not_voiced(self) -> None:
|
|
alignment = align_paragraph(paragraph_fixture(), "这艘船很大。设计师很用心。", bundle_fixture())
|
|
alignment.segments = [
|
|
Segment(source_id="utt-100", text="这艘船很大。设计师很用心。"),
|
|
Segment(
|
|
source_id="utt-101",
|
|
text="",
|
|
voiced=False,
|
|
merged_into="utt-100",
|
|
reason="absorbed into utt-100 in the translated paragraph",
|
|
),
|
|
]
|
|
delivery = build_delivery(
|
|
{"target_language": "zh"},
|
|
[({"id": "ch-1", "paragraphs": [paragraph_fixture()]}, [alignment])],
|
|
bundle_fixture(),
|
|
)
|
|
collapsed = [u for u in delivery["utterances"] if not u["voiced"]]
|
|
self.assertEqual(len(collapsed), 1)
|
|
self.assertEqual(collapsed[0]["utterance_id"], 101)
|
|
self.assertIn("absorbed", collapsed[0]["reason"])
|
|
self.assertNotIn("text", collapsed[0])
|
|
|
|
def test_an_unverifiable_paragraph_is_delivered_as_a_visible_gap(self) -> None:
|
|
"""The row is kept and says why, rather than vanishing.
|
|
|
|
A delivery that simply omits the utterances is indistinguishable
|
|
from one that was never translated. The review lane needs to see the
|
|
gap and its reason.
|
|
"""
|
|
alignment = align_paragraph(paragraph_fixture(), "这艘船很大。设计师很用心。", bundle_fixture())
|
|
alignment.method = "mismatch"
|
|
alignment.segments = []
|
|
alignment.problems = ["translation belongs to another paragraph: it renders the next one"]
|
|
delivery = build_delivery(
|
|
{"target_language": "zh"},
|
|
[({"id": "ch-1", "paragraphs": [paragraph_fixture()]}, [alignment])],
|
|
bundle_fixture(),
|
|
)
|
|
self.assertEqual(len(delivery["utterances"]), 2)
|
|
self.assertTrue(all(not u["voiced"] for u in delivery["utterances"]))
|
|
self.assertEqual([u["utterance_id"] for u in delivery["utterances"]], [100, 101])
|
|
self.assertIn("another paragraph", delivery["utterances"][0]["reason"])
|
|
self.assertNotIn("text", delivery["utterances"][0])
|
|
|
|
|
|
class CommandTests(unittest.TestCase):
|
|
def test_the_cache_directory_is_created_before_the_first_call(self) -> None:
|
|
"""A missing cache directory would fail every model answer.
|
|
|
|
The client writes each answer as it arrives, so a cache directory that
|
|
was never created turns every model call into an error - and the
|
|
fallback would quietly turn the whole book into a mechanical split.
|
|
"""
|
|
import json
|
|
import tempfile
|
|
from unittest import mock
|
|
|
|
from align import main
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
book = root / "book.json"
|
|
book.write_text(
|
|
json.dumps(
|
|
{
|
|
"target_language": "zh-CN",
|
|
"chapters": [
|
|
{
|
|
"id": "ch-1",
|
|
"index": 1,
|
|
"title": "One",
|
|
"title_zh": "一",
|
|
"matter": "body",
|
|
"paragraphs": [],
|
|
}
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
bundle = root / "bundle.jsonl"
|
|
bundle.write_text("", encoding="utf-8")
|
|
run_dir = root / "runs"
|
|
run_dir.mkdir()
|
|
|
|
seen: dict[str, object] = {}
|
|
|
|
class StubLLM:
|
|
def __init__(self, **kwargs: object) -> None:
|
|
seen.update(kwargs)
|
|
|
|
with mock.patch.object(sys.modules["align"], "LLM", StubLLM):
|
|
exit_code = main(
|
|
[
|
|
"--book",
|
|
str(book),
|
|
"--run-dir",
|
|
str(run_dir),
|
|
"--out",
|
|
str(root / "delivery.json"),
|
|
"--bundle",
|
|
str(bundle),
|
|
]
|
|
)
|
|
self.assertEqual(exit_code, 0)
|
|
cache_dir = seen.get("cache_dir")
|
|
self.assertIsNotNone(cache_dir)
|
|
assert cache_dir is not None
|
|
self.assertTrue(Path(cache_dir).is_dir())
|
|
|
|
|
|
class DriftVerificationTests(unittest.TestCase):
|
|
"""The paragraph-boundary drift the production translation actually has."""
|
|
|
|
def test_a_fragment_is_an_english_text_without_a_sentence_end(self) -> None:
|
|
self.assertTrue(is_fragment("The chair in the middle has"))
|
|
self.assertTrue(is_fragment('"operations"'))
|
|
self.assertTrue(is_fragment("on the console. She looks around, and she asks,"))
|
|
self.assertFalse(is_fragment("The ship was large."))
|
|
self.assertFalse(is_fragment('"Yes, Captain."'))
|
|
self.assertFalse(is_fragment("“The last time was thirty-four years ago?”"))
|
|
|
|
def test_a_matching_translation_verifies(self) -> None:
|
|
class StubLLM:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
self.calls += 1
|
|
return ({"kind": "aligned", "reason": ""}, None)
|
|
|
|
llm = StubLLM()
|
|
verdict, _reason = verify_translation("The chair in the middle has", "中间的椅子旁", llm)
|
|
self.assertEqual(verdict, "aligned")
|
|
self.assertEqual(llm.calls, 1)
|
|
|
|
def test_a_drifted_translation_is_caught(self) -> None:
|
|
"""The Chinese is good - it just belongs to the next paragraph."""
|
|
|
|
class StubLLM:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
self.calls += 1
|
|
return (
|
|
{"kind": "drift", "reason": "the text is about another paragraph"},
|
|
None,
|
|
)
|
|
|
|
llm = StubLLM()
|
|
verdict, reason = verify_translation(
|
|
'"Yes, Captain."',
|
|
"“上一次有人登上这艘舰桥还是三十四年前，那时是西顿船长在任？”",
|
|
llm,
|
|
)
|
|
self.assertEqual(verdict, "drift")
|
|
self.assertIn("another paragraph", reason)
|
|
|
|
def test_an_unanswerable_verification_is_not_a_pass(self) -> None:
|
|
"""A model that cannot answer the question has verified nothing."""
|
|
|
|
class StubLLM:
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
return ({"reason": "I am not sure"}, None)
|
|
|
|
verdict, reason = verify_translation("The ship was large.", "这艘船很大。", StubLLM())
|
|
self.assertEqual(verdict, "drift")
|
|
self.assertIn("not understood", reason)
|
|
|
|
def test_the_neighbouring_paragraphs_are_shown_to_the_model(self) -> None:
|
|
"""A paragraph's translation cannot be judged without its neighbours."""
|
|
|
|
seen: dict[str, str] = {}
|
|
|
|
class StubLLM:
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
seen["user"] = user
|
|
return ({"kind": "aligned", "reason": ""}, None)
|
|
|
|
verify_translation(
|
|
"The chair in the middle has",
|
|
"中间的椅子旁",
|
|
StubLLM(),
|
|
previous_text="On the right, the console says",
|
|
next_text='"Captain"',
|
|
)
|
|
self.assertIn("On the right, the console says", seen["user"])
|
|
self.assertIn("Captain", seen["user"])
|
|
|
|
def test_an_incomplete_paragraph_keeps_its_text_and_is_flagged(self) -> None:
|
|
"""It is still the right translation in the right place."""
|
|
|
|
class StubLLM:
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
return ({"kind": "incomplete", "reason": "漏译了最后一句"}, None)
|
|
|
|
chapter = {
|
|
"id": "ch-1",
|
|
"paragraphs": [
|
|
{
|
|
"id": "ch-1-p0000",
|
|
"speaker": "narrator",
|
|
"type": "narration",
|
|
"text": "The chair in the middle has",
|
|
"source_indices": [0],
|
|
"source_ids": ["utt-100"],
|
|
}
|
|
],
|
|
}
|
|
alignments, report = align_chapter(
|
|
chapter,
|
|
{"ch-1-p0000": "中间的椅子旁"},
|
|
bundle_fixture(),
|
|
llm=StubLLM(),
|
|
verify=True,
|
|
)
|
|
self.assertNotEqual(alignments[0].method, "mismatch")
|
|
self.assertTrue(alignments[0].segments)
|
|
self.assertTrue(alignments[0].needs_review)
|
|
self.assertIn("incomplete", alignments[0].problems[0])
|
|
self.assertEqual(report["needs_review"], 1)
|
|
|
|
def test_a_drifted_paragraph_is_dropped_from_the_delivery(self) -> None:
|
|
"""A visible gap is better than a confident wrong line in the audio."""
|
|
|
|
class StubLLM:
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
return ({"kind": "drift", "reason": "belongs to another paragraph"}, None)
|
|
|
|
chapter = {
|
|
"id": "ch-1",
|
|
"paragraphs": [
|
|
{
|
|
"id": "ch-1-p0000",
|
|
"speaker": "narrator",
|
|
"type": "narration",
|
|
"text": "The chair in the middle has",
|
|
"source_indices": [0],
|
|
"source_ids": ["utt-100"],
|
|
}
|
|
],
|
|
}
|
|
translated = {"ch-1-p0000": "“Yes, Captain.”"}
|
|
alignments, report = align_chapter(
|
|
chapter,
|
|
translated,
|
|
bundle_fixture(),
|
|
llm=StubLLM(),
|
|
verify=True,
|
|
)
|
|
self.assertEqual(alignments[0].method, "mismatch")
|
|
self.assertTrue(alignments[0].needs_review)
|
|
self.assertEqual(alignments[0].segments, [])
|
|
self.assertEqual(report["needs_review"], 1)
|
|
|
|
def test_the_check_only_starts_at_a_fragment(self) -> None:
|
|
"""A complete paragraph needs no check: the drafter had no reason to re-flow it."""
|
|
|
|
class CountingLLM:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
self.calls += 1
|
|
return ({"kind": "aligned", "reason": ""}, None)
|
|
|
|
chapter = {
|
|
"id": "ch-1",
|
|
"paragraphs": [
|
|
{
|
|
"id": "ch-1-p0000",
|
|
"speaker": "narrator",
|
|
"type": "narration",
|
|
"text": "The ship was large.",
|
|
"source_indices": [0],
|
|
"source_ids": ["utt-100"],
|
|
}
|
|
],
|
|
}
|
|
llm = CountingLLM()
|
|
align_chapter(chapter, {"ch-1-p0000": "这艘船很大。"}, bundle_fixture(), llm=llm, verify=True)
|
|
self.assertEqual(llm.calls, 0)
|
|
|
|
def test_the_check_follows_the_drift_until_the_passage_resynchronises(self) -> None:
|
|
"""A drift runs on: the next paragraphs carry the wrong content too."""
|
|
|
|
class DriftingLLM:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
self.calls += 1
|
|
if self.calls <= 3:
|
|
return ({"kind": "drift", "reason": "drifted"}, None)
|
|
return ({"kind": "aligned", "reason": ""}, None)
|
|
|
|
def para(pid: str, text: str, sid: str, index: int) -> dict:
|
|
return {
|
|
"id": pid,
|
|
"speaker": "narrator",
|
|
"type": "narration",
|
|
"text": text,
|
|
"source_indices": [index],
|
|
"source_ids": [sid],
|
|
}
|
|
|
|
chapter = {
|
|
"id": "ch-1",
|
|
"paragraphs": [
|
|
para("ch-1-p0000", "The chair in the middle has", "utt-100", 0),
|
|
para("ch-1-p0001", '"Captain"', "utt-101", 1),
|
|
para("ch-1-p0002", "on the console.", "utt-102", 2),
|
|
para("ch-1-p0003", "She looked around and saw dust everywhere.", "utt-103", 3),
|
|
],
|
|
}
|
|
llm = DriftingLLM()
|
|
alignments, _report = align_chapter(
|
|
chapter,
|
|
{p["id"]: "译文" for p in chapter["paragraphs"]},
|
|
bundle_fixture(),
|
|
llm=llm,
|
|
verify=True,
|
|
)
|
|
methods = [a.method for a in alignments]
|
|
self.assertEqual(methods[0], "mismatch")
|
|
self.assertEqual(methods[1], "mismatch")
|
|
self.assertEqual(methods[2], "mismatch")
|
|
self.assertEqual(methods[3], "one_to_one")
|
|
# The drift follows the run: the third paragraph was not a fragment, but
|
|
# it was checked because the passage had not re-synchronised yet.
|
|
self.assertEqual(llm.calls, 4)
|
|
|
|
def test_an_incomplete_paragraph_inside_a_drift_run_is_part_of_the_drift(self) -> None:
|
|
"""The drafter re-flowed the passage; this paragraph is one of the pieces.
|
|
|
|
The English around a drift is often itself a sentence split across
|
|
paragraph ids, so a truncated lead-in inside a confirmed drift is the
|
|
tail of the previous paragraph, not a translation of this one.
|
|
"""
|
|
|
|
class DriftingLLM:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
self.calls += 1
|
|
if self.calls == 1:
|
|
return ({"kind": "drift", "reason": "belongs to another paragraph"}, None)
|
|
if self.calls == 2:
|
|
return ({"kind": "incomplete", "reason": "only the first phrase"}, None)
|
|
return ({"kind": "aligned", "reason": ""}, None)
|
|
|
|
def para(pid: str, text: str, sid: str, index: int) -> dict:
|
|
return {
|
|
"id": pid,
|
|
"speaker": "narrator",
|
|
"type": "narration",
|
|
"text": text,
|
|
"source_indices": [index],
|
|
"source_ids": [sid],
|
|
}
|
|
|
|
chapter = {
|
|
"id": "ch-1",
|
|
"paragraphs": [
|
|
para("ch-1-p0000", "The chair in the middle has", "utt-100", 0),
|
|
para("ch-1-p0001", "on the console.", "utt-101", 1),
|
|
para("ch-1-p0002", "She looked around and saw dust everywhere.", "utt-102", 2),
|
|
],
|
|
}
|
|
alignments, _report = align_chapter(
|
|
chapter,
|
|
{p["id"]: "译文" for p in chapter["paragraphs"]},
|
|
bundle_fixture(),
|
|
llm=DriftingLLM(),
|
|
verify=True,
|
|
)
|
|
methods = [a.method for a in alignments]
|
|
self.assertEqual(methods[0], "mismatch")
|
|
self.assertEqual(methods[1], "mismatch")
|
|
self.assertEqual(methods[2], "one_to_one")
|
|
|
|
def test_an_incomplete_paragraph_before_any_drift_keeps_its_text(self) -> None:
|
|
"""Nothing has been proven wrong yet, so the line is kept and flagged."""
|
|
|
|
class IncompleteLLM:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
self.calls += 1
|
|
return ({"kind": "incomplete", "reason": "漏译了最后一句"}, None)
|
|
|
|
chapter = {
|
|
"id": "ch-1",
|
|
"paragraphs": [
|
|
{
|
|
"id": "ch-1-p0000",
|
|
"speaker": "narrator",
|
|
"type": "narration",
|
|
"text": "The chair in the middle has",
|
|
"source_indices": [0],
|
|
"source_ids": ["utt-100"],
|
|
}
|
|
],
|
|
}
|
|
alignments, _report = align_chapter(
|
|
chapter,
|
|
{"ch-1-p0000": "中间的椅子旁"},
|
|
bundle_fixture(),
|
|
llm=IncompleteLLM(),
|
|
verify=True,
|
|
)
|
|
self.assertNotEqual(alignments[0].method, "mismatch")
|
|
self.assertTrue(alignments[0].segments)
|
|
|
|
def test_a_fragment_called_aligned_inside_a_broken_window_is_dropped(self) -> None:
|
|
"""The model calls these two "aligned" and they are wrong.
|
|
|
|
Measured on the real ch-15 window: the Chinese under the two
|
|
paragraphs before the drift belongs to the paragraph before them.
|
|
The per-paragraph verdict is not trustworthy inside a re-flowed
|
|
passage, so a fragment in a broken window is dropped even when the
|
|
model says it is where it belongs.
|
|
"""
|
|
|
|
class StubLLM:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
self.calls += 1
|
|
if self.calls == 1:
|
|
return ({"kind": "aligned", "reason": ""}, None)
|
|
if self.calls == 2:
|
|
return ({"kind": "drift", "reason": "belongs to another paragraph"}, None)
|
|
return ({"kind": "aligned", "reason": ""}, None)
|
|
|
|
def para(pid: str, text: str, sid: str, index: int) -> dict:
|
|
return {
|
|
"id": pid,
|
|
"speaker": "narrator",
|
|
"type": "narration",
|
|
"text": text,
|
|
"source_indices": [index],
|
|
"source_ids": [sid],
|
|
}
|
|
|
|
chapter = {
|
|
"id": "ch-1",
|
|
"paragraphs": [
|
|
para("ch-1-p0000", '"operations"', "utt-100", 0),
|
|
para("ch-1-p0001", "On the right, the console says", "utt-101", 1),
|
|
para("ch-1-p0002", "The chair in the middle has", "utt-102", 2),
|
|
para("ch-1-p0003", "She grabbed the dusty chair.", "utt-103", 3),
|
|
],
|
|
}
|
|
alignments, report = align_chapter(
|
|
chapter,
|
|
{p["id"]: "译文" for p in chapter["paragraphs"]},
|
|
bundle_fixture(),
|
|
llm=StubLLM(),
|
|
verify=True,
|
|
)
|
|
methods = [a.method for a in alignments]
|
|
# The first fragment was called "aligned" but sits in a broken window.
|
|
self.assertEqual(methods[0], "mismatch")
|
|
self.assertEqual(methods[1], "mismatch")
|
|
self.assertEqual(methods[2], "mismatch")
|
|
# The last paragraph is a complete sentence and re-synchronises the
|
|
# passage, so it is kept.
|
|
self.assertEqual(methods[3], "one_to_one")
|
|
self.assertEqual(report["needs_review"], 3)
|
|
|
|
def test_a_fragment_called_incomplete_at_the_head_of_a_window_keeps_its_text(self) -> None:
|
|
"""It demonstrably renders its own leading content.
|
|
|
|
Measured on the real ch-15 window: the first paragraph's Chinese is a
|
|
correct translation of its first two sentences, and only its third
|
|
sentence was re-flowed away. Dropping it would lose two good lines to
|
|
save nothing.
|
|
"""
|
|
|
|
class StubLLM:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
def chat_json(self, system: str, user: str, **kwargs: object) -> tuple[dict, object]:
|
|
self.calls += 1
|
|
if self.calls == 1:
|
|
return ({"kind": "incomplete", "reason": "漏译了第三句"}, None)
|
|
if self.calls == 2:
|
|
return ({"kind": "drift", "reason": "belongs to another paragraph"}, None)
|
|
return ({"kind": "aligned", "reason": ""}, None)
|
|
|
|
def para(pid: str, text: str, sid: str, index: int) -> dict:
|
|
return {
|
|
"id": pid,
|
|
"speaker": "narrator",
|
|
"type": "narration",
|
|
"text": text,
|
|
"source_indices": [index],
|
|
"source_ids": [sid],
|
|
}
|
|
|
|
chapter = {
|
|
"id": "ch-1",
|
|
"paragraphs": [
|
|
para("ch-1-p0000", "Three chairs face forward, each equipped with a screen", "utt-100", 0),
|
|
para("ch-1-p0001", "The chair in the middle has", "utt-101", 1),
|
|
para("ch-1-p0002", "She grabbed the dusty chair.", "utt-102", 2),
|
|
],
|
|
}
|
|
alignments, report = align_chapter(
|
|
chapter,
|
|
{p["id"]: "译文" for p in chapter["paragraphs"]},
|
|
bundle_fixture(),
|
|
llm=StubLLM(),
|
|
verify=True,
|
|
)
|
|
self.assertNotEqual(alignments[0].method, "mismatch")
|
|
self.assertTrue(alignments[0].segments)
|
|
self.assertTrue(alignments[0].needs_review)
|
|
self.assertEqual(alignments[1].method, "mismatch")
|
|
self.assertEqual(alignments[2].method, "one_to_one")
|
|
self.assertEqual(report["needs_review"], 2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|