From 9bda553311c8aec416faf5722c919dbd1f72e6cb Mon Sep 17 00:00:00 2001 From: "B.D. Murphy" <161863146+bdmurphy73@users.noreply.github.com> Date: Sun, 11 Oct 2026 21:57:58 -0700 Subject: [PATCH] Add a speaker-review audit: is the attribution data fit for translation? Runs after a reviewer finishes a book and answers the two questions that gate the next stage: 1. Are there utterances with no speaker? 2. Now that speakers have been defined, renamed, merged or deleted, are there utterances that should change? Read-only - it never edits the database, and a test asserts the files are byte-identical afterwards. Output is a markdown table and a CSV, one row per finding with chapter, utterance number and a text preview, so each item is locatable rather than a count. Deal Hunter, after the completed review (56 speakers, 0 still needing review, 4319 utterances): - Dialogue with no speaker: 0. The 1803 speakerless rows are narration and scene breaks, which is correct and is reported as a number rather than silently dropped. - Utterances attributed to a deleted speaker: 0. - One speaker stored under two spellings: 6 rows across 3 speakers (narrator/Narrator, Kainda/Princess Kainda, Willamina Wake/Captain). All resolve to one voice so the audio is correct, but a per-speaker export would split them into six - caught before the translation stage, not after. - 6 speech-tag mismatches where the narration immediately before the line names a different speaker. - 10 dialogue rows voiced by the narrator: 9 are quoted labels ("Reactor.", "operations") that are correctly read by the narrator, 1 (ch28 utt46) is a real segmentation error. Classified advisory so the one real defect is not buried under nine false positives. build_alias_map moved from scripts/qa_attribution.py into attribution_qa.py so the CLI and the audit cannot disagree about what a speech tag resolves to; qa_attribution output verified byte-identical. 13 new tests. Suite: 721 passed, 8 pre-existing. --- scripts/qa_attribution.py | 24 +- scripts/speaker_review_audit.py | 374 ++++++++++++++++++++ specs/25-speaker-review-audit-dealhunter.md | 94 +++++ tests/test_speaker_review_audit.py | 178 ++++++++++ umbraaudio/analysis/attribution_qa.py | 32 ++ 5 files changed, 680 insertions(+), 22 deletions(-) create mode 100644 scripts/speaker_review_audit.py create mode 100644 specs/25-speaker-review-audit-dealhunter.md create mode 100644 tests/test_speaker_review_audit.py diff --git a/scripts/qa_attribution.py b/scripts/qa_attribution.py index 22ce62e..606806e 100644 --- a/scripts/qa_attribution.py +++ b/scripts/qa_attribution.py @@ -11,7 +11,6 @@ Usage: from __future__ import annotations import argparse -import json import sqlite3 import sys from pathlib import Path @@ -20,6 +19,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) from umbraaudio.analysis.attribution_qa import ( QAIssue, + build_alias_map, check_attribution, fix_high_confidence, verify_conversation_flow, @@ -27,26 +27,6 @@ from umbraaudio.analysis.attribution_qa import ( from umbraaudio.config import load_config -def _build_alias_map(speakers_path: Path) -> dict[str, str]: - alias_map: dict[str, str] = {} - if not speakers_path.exists(): - return alias_map - with open(speakers_path) as f: - data = json.load(f) - for s in data.get("speakers", []): - primary = s.get("primary_name", "") - if not primary: - continue - alias_map[primary.lower()] = primary - for alias in s.get("aliases", []): - if alias: - alias_map[alias.lower()] = primary - first = primary.split()[0].lower() if primary.split() else "" - if first and first not in alias_map: - alias_map[first] = primary - return alias_map - - def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -73,7 +53,7 @@ def main() -> None: print(f"Database not found: {db_path}") return - alias_map = _build_alias_map(cfg.data_dir / "speakers.json") + alias_map = build_alias_map(cfg.data_dir / "speakers.json") db = sqlite3.connect(str(db_path)) db.row_factory = sqlite3.Row diff --git a/scripts/speaker_review_audit.py b/scripts/speaker_review_audit.py new file mode 100644 index 0000000..6a5962b --- /dev/null +++ b/scripts/speaker_review_audit.py @@ -0,0 +1,374 @@ +"""Speaker-review audit: is the attribution data fit to hand to the next stage? + +Answers the two questions asked after a speaker review completes: + + 1. Are there utterances with no speaker? + 2. Now that speakers have been defined, renamed, merged or deleted, are + there utterances that should change? + +Everything here is mechanical and read-only. It never edits the database - +these are candidate lists for a human to accept or dismiss, and the report +records what was *measured*, not what is assumed. + +A null speaker on a NARRATION row is not a defect: narration is not spoken by +a character. Only dialogue without a speaker counts as missing, and a +narration row carrying a character's name is reported separately because it +usually means an inner monologue that may want the character's voice. + +Usage: + python scripts/speaker_review_audit.py --project DealHunterShip_4.0e_whole-to-audio + python scripts/speaker_review_audit.py --project Last Dwarf --json +""" +from __future__ import annotations + +import argparse +import csv +import json +import sqlite3 +import sys +from collections import defaultdict +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from umbraaudio.analysis.attribution_qa import ( # noqa: E402 + build_alias_map, + check_attribution, +) +from umbraaudio.config import load_config # noqa: E402 + +# Issues that are safe to act on mechanically: the narration immediately +# before the line names a different speaker. +HIGH_CONFIDENCE_ISSUES = {"speech_tag_mismatch"} +# Advisory only - pronoun coreference and role guessing are judgement calls. +# +# narrator_dialogue is advisory rather than an action item on purpose. In a +# text like this most of these rows are quoted labels - a console reading +# "operations", a legend saying "Reactor." - which are typed as dialogue but +# are correctly read by the narrator. Treating them as defects would bury +# the one row that is a real segmentation error under nine false positives. +ADVISORY_ISSUES = { + "role_tag_guessed", + "address_mismatch", + "narration_speaker_mismatch", + "long_monologue", +} + +PLACEHOLDER_SPEAKERS = {"", "__unassigned__", "unassigned", "unknown", "none", "null"} + + +@dataclass +class Finding: + """One reviewable row. ``chapter``/``utterance`` are what a human needs.""" + + check: str + severity: str # "action" | "advisory" | "info" + chapter: int + utterance: int + utterance_id: int + speaker: str + expected: str + detail: str + text: str + + +@dataclass +class Audit: + project: str + generated: str + findings: list[Finding] = field(default_factory=list) + stats: dict[str, Any] = field(default_factory=dict) + + def add(self, **kw: Any) -> None: + self.findings.append(Finding(**kw)) + + def by_check(self) -> dict[str, list[Finding]]: + out: dict[str, list[Finding]] = defaultdict(list) + for f in self.findings: + out[f.check].append(f) + return out + + +def _chapters(conn: sqlite3.Connection) -> list[int]: + return [ + r[0] + for r in conn.execute("SELECT DISTINCT chapter_id FROM utterances ORDER BY chapter_id") + ] + + +def _chapter_utterances(conn: sqlite3.Connection, chapter_id: int) -> list[dict[str, Any]]: + rows = conn.execute( + "SELECT id, utterance_index, speaker_name, type, text FROM utterances " + "WHERE chapter_id = ? ORDER BY utterance_index", + (chapter_id,), + ).fetchall() + return [ + { + "id": r[0], + "utterance_index": r[1], + "speaker_name": r[2], + "type": r[3], + "text": r[4] or "", + } + for r in rows + ] + + +def run_audit(project: str | None, as_json: bool) -> Audit: + cfg = load_config(project) if project else load_config() + data_dir = cfg.data_dir + conn = sqlite3.connect(f"file:{data_dir / 'project.db'}?mode=ro", uri=True) + conn.row_factory = sqlite3.Row + from datetime import UTC, datetime + + audit = Audit( + project=cfg.project_dir.name, + generated=datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%SZ"), + ) + + speakers_path = data_dir / "speakers.json" + speakers = [] + if speakers_path.exists(): + speakers = json.loads(speakers_path.read_text()).get("speakers", []) + alias_map = build_alias_map(speakers_path) + known_lower = set(alias_map.keys()) + + # ---------------------------------------------------------- 1. no speaker + missing = conn.execute( + "SELECT id, chapter_id, utterance_index, type, speaker_name, text FROM utterances " + "WHERE type = 'dialogue' AND (" + " speaker_name IS NULL OR TRIM(speaker_name) = '' " + " OR LOWER(TRIM(speaker_name)) IN (" + ",".join("?" * len(PLACEHOLDER_SPEAKERS)) + "))", + tuple(PLACEHOLDER_SPEAKERS), + ).fetchall() + for r in missing: + audit.add( + check="dialogue_missing_speaker", + severity="action", + chapter=r[1], + utterance=r[2], + utterance_id=r[0], + speaker=r[4] or "", + expected="", + detail="dialogue with no speaker", + text=(r[5] or "")[:110], + ) + + # Narration with no speaker is normal; count it so the report is honest + # about what was excluded rather than quietly dropping it. + audit.stats["narration_without_speaker"] = conn.execute( + "SELECT COUNT(*) FROM utterances WHERE type IN ('narration','scene_break') " + "AND (speaker_name IS NULL OR TRIM(speaker_name) = '')" + ).fetchone()[0] + + # ------------------------------------------------- 2. unresolvable speaker + used = { + r[0] + for r in conn.execute( + "SELECT DISTINCT speaker_name FROM utterances " + "WHERE speaker_name IS NOT NULL AND TRIM(speaker_name) <> ''" + ) + } + for name in sorted(used): + if name.casefold() in known_lower: + continue + rows = conn.execute( + "SELECT id, chapter_id, utterance_index, text FROM utterances WHERE speaker_name = ?", + (name,), + ).fetchall() + for r in rows: + audit.add( + check="speaker_not_resolvable", + severity="action", + chapter=r[1], + utterance=r[2], + utterance_id=r[0], + speaker=name, + expected="", + detail="attributed to a speaker that no longer exists", + text=(r[3] or "")[:110], + ) + + # ------------------------------------- 3. identity / case fragmentation + # The database stores whatever string was written at attribution time, so + # one speaker can appear under several spellings ("narrator" and "Narrator", + # for instance). Both resolve through the alias map so the audio is correct + # today, but anything that groups by the raw column will split them - which + # is exactly what a translation export would do. Reported here so it is a + # decision rather than a surprise downstream. + canonical: dict[str, set[str]] = defaultdict(set) + for name in used: + target = alias_map.get(name.casefold()) + if target: + canonical[target].add(name) + for target, variants in canonical.items(): + if len(variants) < 2: + continue + for variant in sorted(variants): + count = conn.execute( + "SELECT COUNT(*) FROM utterances WHERE speaker_name = ?", (variant,) + ).fetchone()[0] + if not count: + continue + audit.add( + check="speaker_name_variants", + severity="action", + chapter=0, + utterance=0, + utterance_id=0, + speaker=variant, + expected=target, + detail=( + f"{count} utterances stored under this spelling; the same " + "speaker also appears as " + f"{sorted(variants - {variant})}. They resolve to one voice, " + "but a per-speaker export would split them." + ), + text="", + ) + + # ------------------------------------- 4. new speaker vs stale attribution + # This is the "a speaker has since been defined" question: the narration + # right before the line names speaker X, and the line is not attributed + # to X. + for chapter_id in _chapters(conn): + utts = _chapter_utterances(conn, chapter_id) + result = check_attribution(utts, chapter_id, alias_map) + for issue in result.issues: + idx = issue.utterance_index + row = next((u for u in utts if u["utterance_index"] == idx), None) + if row is None: + continue + severity = ( + "action" + if issue.issue_type in HIGH_CONFIDENCE_ISSUES + else "advisory" + ) + audit.add( + check=issue.issue_type, + severity=severity, + chapter=chapter_id, + utterance=idx, + utterance_id=row["id"], + speaker=row["speaker_name"] or "", + expected=issue.expected or "", + detail=issue.evidence or "", + text=(row["text"] or "")[:110], + ) + + audit.stats.update( + utterances=conn.execute("SELECT COUNT(*) FROM utterances").fetchone()[0], + dialogue=conn.execute( + "SELECT COUNT(*) FROM utterances WHERE type='dialogue'" + ).fetchone()[0], + speakers=len(speakers), + names_and_aliases=len(known_lower), + ) + return audit + + +def write_reports(audit: Audit, out_dir: Path) -> tuple[Path, Path]: + out_dir.mkdir(parents=True, exist_ok=True) + md = out_dir / "speaker_review_audit.md" + csv_path = out_dir / "speaker_review_audit.csv" + + grouped = audit.by_check() + order = [ + ("dialogue_missing_speaker", "Dialogue with no speaker"), + ("speaker_not_resolvable", "Attributed to a speaker that no longer exists"), + ("narrator_dialogue", "Dialogue still voiced by the narrator"), + ("speech_tag_mismatch", "Narration names a different speaker"), + ("speaker_name_variants", "One speaker, several spellings in the database"), + ("role_tag_guessed", "Role tag guessed as a named character"), + ("narration_speaker_mismatch", "Narration speaker disagrees (pronoun)"), + ("address_mismatch", "Addressed to the speaker themselves"), + ("long_monologue", "Long monologue"), + ] + action = [f for f in audit.findings if f.severity == "action"] + + lines = [ + f"# Speaker review audit - {audit.project}", + "", + f"Generated {audit.generated}. Read-only: nothing was modified.", + "", + "## Summary", + "", + f"- Utterances: **{audit.stats.get('utterances')}** " + f"({audit.stats.get('dialogue')} dialogue)", + f"- Speakers: **{audit.stats.get('speakers')}** " + f"({audit.stats.get('names_and_aliases')} distinct names + aliases)", + f"- Narration rows with no speaker: " + f"**{audit.stats.get('narration_without_speaker')}** - expected, narration is " + "not spoken by a character", + f"- Findings needing a decision: **{len(action)}**", + "", + ] + + if not audit.findings: + lines += ["No findings. The attribution data is clean.", ""] + + for check, title in order: + rows = grouped.get(check) + if not rows: + continue + lines += [f"## {title} ({len(rows)})", ""] + lines += ["| Chapter | Utt | Severity | Speaker | Expected | Detail |", + "|---|---|---|---|---|---|"] + for f in sorted(rows, key=lambda r: (r.chapter, r.utterance)): + detail = (f.detail or "").replace("|", "\\|")[:70] + # Book-level findings have no single utterance; "-" beats a + # misleading chapter 0 / utterance 0. + where = f"{f.chapter} | {f.utterance}" if f.chapter else "- | -" + lines.append( + f"| {where} | {f.severity} | {f.speaker} | " + f"{f.expected} | {detail} |" + ) + lines.append("") + + md.write_text("\n".join(lines)) + + with open(csv_path, "w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=list(asdict(audit.findings[0]).keys())) + w.writeheader() + for f in audit.findings: + w.writerow(asdict(f)) + return md, csv_path + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--project") + ap.add_argument("--json", action="store_true", help="print JSON to stdout") + ap.add_argument("--out", help="output directory (default: /.umbra/reports)") + args = ap.parse_args() + + audit = run_audit(args.project, args.json) + cfg = load_config(args.project) if args.project else load_config() + out_dir = Path(args.out) if args.out else cfg.data_dir / "reports" + md, csv_path = write_reports(audit, out_dir) + + if args.json: + print(json.dumps({ + "project": audit.project, + "generated": audit.generated, + "stats": audit.stats, + "findings": [asdict(f) for f in audit.findings], + }, indent=2)) + else: + action = sum(1 for f in audit.findings if f.severity == "action") + print(f" {audit.project}: {len(audit.findings)} findings " + f"({action} need a decision)") + for f in audit.findings: + if f.severity != "action": + continue + print(f" [{f.check}] ch{f.chapter} utt{f.utterance} " + f"speaker={f.speaker!r} expected={f.expected!r}") + print(f" written: {md}") + print(f" written: {csv_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/specs/25-speaker-review-audit-dealhunter.md b/specs/25-speaker-review-audit-dealhunter.md new file mode 100644 index 0000000..146a3b9 --- /dev/null +++ b/specs/25-speaker-review-audit-dealhunter.md @@ -0,0 +1,94 @@ +# Speaker review audit - DealHunterShip_4.0e_whole-to-audio + +Generated 2026-10-12 04:57:45Z. Read-only: nothing was modified. + +## Summary + +- Utterances: **4319** (1874 dialogue) +- Speakers: **56** (116 distinct names + aliases) +- Narration rows with no speaker: **1803** - expected, narration is not spoken by a character +- Findings needing a decision: **12** + +## Dialogue still voiced by the narrator (10) + +| Chapter | Utt | Severity | Speaker | Expected | Detail | +|---|---|---|---|---|---| +| 10 | 21 | advisory | Narrator | | Dialogue line still attributed to the narrator. | +| 12 | 120 | advisory | Narrator | | Dialogue line still attributed to the narrator. | +| 15 | 122 | advisory | Narrator | | Dialogue line still attributed to the narrator. | +| 15 | 124 | advisory | Narrator | | Dialogue line still attributed to the narrator. | +| 15 | 126 | advisory | Narrator | | Dialogue line still attributed to the narrator. | +| 19 | 116 | advisory | narrator | | Dialogue line still attributed to the narrator. | +| 25 | 178 | advisory | Narrator | | Dialogue line still attributed to the narrator. | +| 25 | 187 | advisory | Narrator | | Dialogue line still attributed to the narrator. | +| 27 | 14 | advisory | Narrator | | Dialogue line still attributed to the narrator. | +| 28 | 46 | advisory | Narrator | | Dialogue line still attributed to the narrator. | + +## Narration names a different speaker (6) + +| Chapter | Utt | Severity | Speaker | Expected | Detail | +|---|---|---|---|---|---| +| 9 | 21 | action | Kainda Walters Foster Murphy | Duchess Victoria | Nearest tag: 'Vicky says assertively.' | +| 20 | 229 | action | Warbots | The Warbot | Nearest tag: 'The Warbot asks,' | +| 24 | 112 | action | Unnamed Officer | Someone | Nearest tag: 'Someone says,' | +| 25 | 132 | action | Warbots | The Warbot | Nearest tag: 'The Warbot replies in a simple, neutral voice,' | +| 26 | 196 | action | Distorted Voice | Ruler | Nearest tag: 'Ruler says,' | +| 26 | 200 | action | Distorted Voice | Ruler | Nearest tag: 'Ruler replies,' | + +## One speaker, several spellings in the database (6) + +| Chapter | Utt | Severity | Speaker | Expected | Detail | +|---|---|---|---|---|---| +| - | - | action | Captain | Willamina Wake | 3 utterances stored under this spelling; the same speaker also appears | +| - | - | action | Willamina Wake | Willamina Wake | 21 utterances stored under this spelling; the same speaker also appear | +| - | - | action | Kainda Walters Foster Murphy | Kainda Walters Foster Murphy | 793 utterances stored under this spelling; the same speaker also appea | +| - | - | action | Princess Kainda | Kainda Walters Foster Murphy | 78 utterances stored under this spelling; the same speaker also appear | +| - | - | action | Narrator | Narrator | 15 utterances stored under this spelling; the same speaker also appear | +| - | - | action | narrator | Narrator | 614 utterances stored under this spelling; the same speaker also appea | + +## Role tag guessed as a named character (3) + +| Chapter | Utt | Severity | Speaker | Expected | Detail | +|---|---|---|---|---|---| +| 6 | 158 | advisory | Kainda Walters Foster Murphy | The Voice | Nearest tag: 'The voice calls again,' | +| 6 | 212 | advisory | Nanny | The Robot | Nearest tag: 'The robot says,' | +| 26 | 152 | advisory | Warbots | The Speaker | Nearest tag: 'Every Warbots lower their weapons, and the speaker says, | + +## Narration speaker disagrees (pronoun) (13) + +| Chapter | Utt | Severity | Speaker | Expected | Detail | +|---|---|---|---|---|---| +| 10 | 116 | advisory | Brutus | It | Narration names 'It' as speaker | +| 11 | 20 | advisory | Kainda Walters Foster Murphy | She | Narration names 'She' as speaker | +| 17 | 77 | advisory | Kainda Walters Foster Murphy | She | Narration names 'She' as speaker | +| 20 | 123 | advisory | Kainda Walters Foster Murphy | She | Narration names 'She' as speaker | +| 20 | 131 | advisory | Kainda Walters Foster Murphy | She | Narration names 'She' as speaker | +| 20 | 160 | advisory | Kainda Walters Foster Murphy | She | Narration names 'She' as speaker | +| 20 | 174 | advisory | Player Four | He | Narration names 'He' as speaker | +| 25 | 153 | advisory | Kainda Walters Foster Murphy | She | Narration names 'She' as speaker | +| 30 | 84 | advisory | Kainda Walters Foster Murphy | She | Narration names 'She' as speaker | +| 30 | 150 | advisory | Kainda Walters Foster Murphy | She | Narration names 'She' as speaker | +| 30 | 162 | advisory | Kainda Walters Foster Murphy | She | Narration names 'She' as speaker | +| 31 | 12 | advisory | Interrogator | As Kainda | Narration names 'As Kainda' as speaker | +| 32 | 149 | advisory | Kainda Walters Foster Murphy | She | Narration names 'She' as speaker | + +## Addressed to the speaker themselves (3) + +| Chapter | Utt | Severity | Speaker | Expected | Detail | +|---|---|---|---|---|---| +| 11 | 166 | advisory | Service Robot | | Dialogue addresses 'Service Robot' but speaker is also 'Service Robot' | +| 31 | 109 | advisory | Willamina Wake | | Dialogue addresses 'Willamina Wake' but speaker is also 'Willamina Wak | +| 31 | 111 | advisory | Willamina Wake | | Dialogue addresses 'Willamina Wake' but speaker is also 'Willamina Wak | + +--- + +## Provenance + +Generated by `scripts/speaker_review_audit.py`, read-only. Re-run with: + +``` +python scripts/speaker_review_audit.py --project DealHunterShip_4.0e_whole-to-audio +``` + +The CSV alongside it in the project (not committed, project dirs are +gitignored) carries one row per finding including the full text preview. diff --git a/tests/test_speaker_review_audit.py b/tests/test_speaker_review_audit.py new file mode 100644 index 0000000..77c36ab --- /dev/null +++ b/tests/test_speaker_review_audit.py @@ -0,0 +1,178 @@ +"""Tests for the speaker-review audit. + +The audit is the gate between "the reviewer finished" and "the data goes to +translation". Two of its rules encode decisions that are easy to get wrong and +expensive to get wrong quietly: + +- narration with no speaker is NOT a missing speaker; only dialogue counts; +- a speaker stored under two spellings is fine for audio (both resolve via the + alias map) but splits per-speaker exports, so it must still be reported. + +The rest guards that the audit is read-only and does not invent findings. +""" +from __future__ import annotations + +import json +import sqlite3 +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from scripts.speaker_review_audit import write_reports # noqa: E402 +from umbraaudio.web.routes import SCHEMA_SQL # noqa: E402 + +SPEAKERS = [ + {"primary_name": "Narrator", "aliases": ["narrator", "the narrator"]}, + {"primary_name": "Kainda", "aliases": ["Princess Kainda"]}, +] + + +def _project(tmp_path: Path, utterances: list[tuple]) -> Path: + """Build a throwaway project. ``utterances`` = (idx, type, speaker, text).""" + d = tmp_path / "proj" / ".umbra" + d.mkdir(parents=True) + (d / "speakers.json").write_text(json.dumps({"speakers": SPEAKERS})) + conn = sqlite3.connect(str(d / "project.db")) + conn.executescript(SCHEMA_SQL) + for i, (idx, utype, speaker, text) in enumerate(utterances): + conn.execute( + "INSERT INTO utterances (id, chapter_id, utterance_index, type, " + "speaker_name, text) VALUES (?, 1, ?, ?, ?, ?)", + (i + 1, idx, utype, speaker, text), + ) + conn.commit() + conn.close() + return d + + +def _audit(tmp_path: Path, utterances, monkeypatch): + d = _project(tmp_path, utterances) + from umbraaudio.config import AppConfig + + cfg = AppConfig() + cfg.project_dir = d.parent + cfg.data_dir = d + # Patch the name where it is USED. The script does + # "from umbraaudio.config import load_config", which binds at import time, + # so patching the config module would be ignored and these tests would + # quietly audit the real project instead of the fixture. + import scripts.speaker_review_audit as audit_mod + + monkeypatch.setattr(audit_mod, "load_config", lambda _s=None: cfg) + return audit_mod.run_audit(None, False), d + + +def test_narration_without_a_speaker_is_not_a_defect(tmp_path, monkeypatch) -> None: + """Narration is not spoken by a character; flagging it would bury real bugs.""" + audit, _ = _audit( + tmp_path, + [(0, "narration", None, "The ship drifted.")], + monkeypatch, + ) + assert [f for f in audit.findings if f.check == "dialogue_missing_speaker"] == [] + assert audit.stats["narration_without_speaker"] == 1 + + +def test_dialogue_without_a_speaker_is_reported(tmp_path, monkeypatch) -> None: + audit, _ = _audit( + tmp_path, + [(0, "dialogue", None, "Who goes there?"), (1, "narration", None, "Silence.")], + monkeypatch, + ) + missing = [f for f in audit.findings if f.check == "dialogue_missing_speaker"] + assert len(missing) == 1 + assert missing[0].chapter == 1 + assert missing[0].severity == "action" + + +@pytest.mark.parametrize("placeholder", ["", " ", "__unassigned__", "unknown", "NONE"]) +def test_placeholder_speaker_names_count_as_missing( + tmp_path, monkeypatch, placeholder +) -> None: + audit, _ = _audit(tmp_path, [(0, "dialogue", placeholder, "Hi.")], monkeypatch) + assert [f for f in audit.findings if f.check == "dialogue_missing_speaker"] + + +def test_two_spellings_of_one_speaker_are_reported(tmp_path, monkeypatch) -> None: + """Both resolve to one voice, but a per-speaker export would split them.""" + audit, _ = _audit( + tmp_path, + [(0, "dialogue", "Narrator", "One."), (1, "dialogue", "narrator", "Two.")], + monkeypatch, + ) + variants = [f for f in audit.findings if f.check == "speaker_name_variants"] + assert {v.speaker for v in variants} == {"Narrator", "narrator"} + assert all(v.expected == "Narrator" for v in variants) + + +def test_a_single_spelling_is_not_reported(tmp_path, monkeypatch) -> None: + audit, _ = _audit( + tmp_path, [(0, "dialogue", "Kainda", "Hi."), (1, "dialogue", "Kainda", "Bye.")], + monkeypatch, + ) + assert [f for f in audit.findings if f.check == "speaker_name_variants"] == [] + + +def test_alias_spelling_is_reported_against_its_canonical(tmp_path, monkeypatch) -> None: + audit, _ = _audit( + tmp_path, + [(0, "dialogue", "Kainda", "A."), (1, "dialogue", "Princess Kainda", "B.")], + monkeypatch, + ) + variants = [f for f in audit.findings if f.check == "speaker_name_variants"] + assert any(v.speaker == "Princess Kainda" and v.expected == "Kainda" for v in variants) + + +def test_deleted_speaker_is_reported_with_its_rows(tmp_path, monkeypatch) -> None: + audit, _ = _audit( + tmp_path, + [(0, "dialogue", "Gone", "A."), (1, "dialogue", "Kainda", "B.")], + monkeypatch, + ) + orphans = [f for f in audit.findings if f.check == "speaker_not_resolvable"] + assert len(orphans) == 1 + assert orphans[0].speaker == "Gone" + assert orphans[0].utterance == 0 + + +def test_audit_does_not_modify_the_database(tmp_path, monkeypatch) -> None: + """It is a review aid: it must never change the data it is describing.""" + d = _project(tmp_path, [(0, "dialogue", None, "Who?"), (1, "dialogue", "Kainda", "Hi.")]) + before = d.joinpath("project.db").read_bytes() + speakers_before = d.joinpath("speakers.json").read_bytes() + + from umbraaudio.config import AppConfig + + cfg = AppConfig() + cfg.project_dir = d.parent + cfg.data_dir = d + import scripts.speaker_review_audit as audit_mod + + monkeypatch.setattr(audit_mod, "load_config", lambda _s=None: cfg) + audit_mod.run_audit(None, False) + assert d.joinpath("project.db").read_bytes() == before + assert d.joinpath("speakers.json").read_bytes() == speakers_before + + +def test_reports_are_written_with_chapter_and_utterance(tmp_path, monkeypatch) -> None: + audit, d = _audit( + tmp_path, + [ + (0, "dialogue", "Kainda", '"I have to tell you." said the captain.'), + (1, "dialogue", None, "Who is there?"), + ], + monkeypatch, + ) + out = tmp_path / "out" + md, csv_path = write_reports(audit, out) + text = md.read_text() + assert "# Speaker review audit" in text + # Every row-level finding must be locatable by a human. + missing = [f for f in audit.findings if f.check == "dialogue_missing_speaker"][0] + assert f"| {missing.chapter} | {missing.utterance} |" in text + assert csv_path.exists() + header = csv_path.read_text().splitlines()[0] + assert "chapter" in header and "utterance" in header diff --git a/umbraaudio/analysis/attribution_qa.py b/umbraaudio/analysis/attribution_qa.py index bb26ab9..361828c 100644 --- a/umbraaudio/analysis/attribution_qa.py +++ b/umbraaudio/analysis/attribution_qa.py @@ -292,6 +292,38 @@ def _find_narrated_speaker( return None +def build_alias_map(speakers_path) -> dict[str, str]: + """Lowercase name/alias -> canonical ``primary_name``, plus first-word keys. + + First-word entries let a speech tag like "Wakam" resolve to "Trader + Wakam". Lives here rather than in the CLI so every caller agrees on what a + tag can resolve to; scripts/qa_attribution.py imports this. + """ + import json + from pathlib import Path as _Path + + path = _Path(speakers_path) + if not path.exists(): + return {} + try: + data = json.loads(path.read_text()) + except (OSError, ValueError): + return {} + alias_map: dict[str, str] = {} + for s in data.get("speakers", []): + primary = s.get("primary_name", "") + if not primary: + continue + alias_map[primary.lower()] = primary + for alias in s.get("aliases", []): + if alias: + alias_map[alias.lower()] = primary + first = primary.split()[0].lower() if primary.split() else "" + if first and first not in alias_map: + alias_map[first] = primary + return alias_map + + def check_attribution( utterances: list[dict[str, Any]], chapter_id: int,