umbragit/UmbraAudio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add utterance clarity signals + review queue (spec 17 Phase 1)

Browse Source 
Persist per-utterance attribution transparency so the system can auto-accept
high-confidence lines and surface only a small review queue:

- DB: utterances gains attribution_evidence_kind, attribution_tag_name,
  attribution_uncertain, needs_review, review_reason; attribution_confidence
  default -> NULL. Idempotent migrations in database._migrate, routes
  _ensure_utterance_columns, and a new ensure in reattribute_all (raw-sqlite
  driver must add columns itself).
- llm_attributor: accumulates scene-level mechanical evidence across batches and
  stamps evidence kind + tag name on attributed utterances.
- New umbraaudio/analysis/clarity.py: deterministic evidence classification
  (name/role/pronoun/address_only/none), in-scene pronoun resolvability
  (gender-filtered candidate count), and composite needs_review rule:
  QA issues (long_monologue excluded), non-unique pronoun referent, role tag
  with no profile, none/address_only evidence < 0.75, unresolved uncertain
  flag, narrator-left dialogue, confidence < 0.7. Addressee counts only for
  known names/titles (guards 'But where...' false positives).
- Drivers persist all fields: reattribute_all (per-chapter clarity pass),
  run_llm_attribution (via update_utterance_speaker), both web attribution
  paths. _populate_exceptions keys off needs_review (was confidence<0.7 +
  keyword sniff) and fixes a latent NameError (pid not in scope).
- tests/test_clarity.py: 25 tests. Eval on Last Dwarf: 3918 dialogue lines ->
  173 needs_review (4.4%), within the 1-5% target.

Full suite 380 passed; 6 pre-existing voice_profile_manager accent failures.
ruff/mypy clean on new code; routes.py baseline unchanged (73 pre-existing).
This commit is contained in:
B.D. Murphy
2026-09-23 12:15:39 -07:00
parent b09e324431
commit 1bbbe5d92c
8 changed files with 1231 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
scripts/reattribute_all.py
View File

@@ -17,6 +17,7 @@ from umbraaudio.analysis.attribution_qa import (
check_attribution,
verify_conversation_flow,
)
from umbraaudio.analysis.clarity import assess_utterance_clarity
from umbraaudio.analysis.llm_attributor import LLMAttributor
from umbraaudio.config import load_config
from umbraaudio.voice_maintenance import (
@@ -26,6 +27,21 @@ from umbraaudio.voice_maintenance import (
from umbraaudio.voices.model_coordinator import ModelCoordinator, PipelineStage
def _ensure_utterance_columns(db: sqlite3.Connection) -> None:
"""Add clarity columns introduced after project DBs were first created."""
cols = {r[1] for r in db.execute("PRAGMA table_info(utterances)")}
for col, typedef in (
("attribution_evidence_kind", "TEXT"),
("attribution_tag_name", "TEXT"),
("attribution_uncertain", "INTEGER NOT NULL DEFAULT 0"),
("needs_review", "INTEGER NOT NULL DEFAULT 0"),
("review_reason", "TEXT"),
):
if col not in cols:
db.execute(f"ALTER TABLE utterances ADD COLUMN {col} {typedef}")
db.commit()
def _build_alias_map(speakers_path: Path) -> dict[str, str]:
if not speakers_path.exists():
return {}
@@ -151,8 +167,14 @@ def main():
db = sqlite3.connect(str(db_path))
db.row_factory = sqlite3.Row
_ensure_utterance_columns(db)
alias_map = _build_alias_map(cfg.data_dir / "speakers.json")
profiles: list[dict] = []
speakers_path = cfg.data_dir / "speakers.json"
if speakers_path.exists():
with open(speakers_path) as f:
profiles = json.load(f).get("speakers", [])
if args.chapters:
chapter_ids = [int(x.strip()) for x in args.chapters.split(",")]
@@ -221,10 +243,14 @@ def main():
verify_rounds=args.verify_rounds,
)
time.sleep(0.5)
clarity_aware = assess_utterance_clarity(
attributed, ch_id, alias_map, profiles
)
ch_elapsed = time.time() - ch_start
accepted_skipped = 0
for u in attributed:
for u in clarity_aware:
if u.get("qa_accepted"):
accepted_skipped += 1
continue
@@ -232,11 +258,18 @@ def main():
continue
db.execute(
"UPDATE utterances SET speaker_name=?, attribution_confidence=?, "
"attribution_reasoning=? WHERE id=?",
"attribution_reasoning=?, attribution_evidence_kind=?, "
"attribution_tag_name=?, attribution_uncertain=?, "
"needs_review=?, review_reason=? WHERE id=?",
(
u.get("speaker_name", "Narrator"),
u.get("attribution_confidence", u.get("confidence", 0.5)),
u.get("attribution_reasoning", "") or "",
u.get("attribution_evidence_kind") or None,
u.get("attribution_tag_name") or None,
1 if u.get("attribution_uncertain") else 0,
1 if u.get("needs_review") else 0,
u.get("review_reason") or "",
u["id"],
),
)
@@ -244,7 +277,7 @@ def main():
db.commit()
if args.dry_run:
_report_dry_run(attributed, utt_list)
_report_dry_run(clarity_aware, utt_list)
narrator_after = sum(
1

84
scripts/run_llm_attribution.py
View File

@@ -3,6 +3,7 @@ from __future__ import annotations
import logging
from typing import Any
from umbraaudio.analysis.clarity import assess_utterance_clarity
from umbraaudio.analysis.llm_attributor import LLMAttributor
from umbraaudio.config import load_config
from umbraaudio.database import Database
@@ -48,7 +49,7 @@ async def run_llm_attribution(
for ch in chapters:
chapter_id = ch["id"]
chapter_title = ch.get("title") or f"Chapter {ch['chapter_index'] + 1}"
logger.info(f"Running LLM attribution for chapter {chapter_id}: {chapter_title}")
# Get utterances from database
@@ -73,8 +74,33 @@ async def run_llm_attribution(
chapter_data=chapter_data,
)
# Deterministic clarity + review flags (evidence kind, pronoun
# uniqueness, QA issues, confidence thresholds)
alias_map: dict[str, str] = {}
try:
import json as _json
from pathlib import Path as _Path
sp = _Path(config.data_dir) / "speakers.json"
if sp.exists():
with open(sp) as f:
for s in _json.load(f).get("speakers", []):
primary = s.get("primary_name", "")
if primary:
alias_map[primary.lower()] = primary
for a in s.get("aliases", []):
if a:
alias_map[a.lower()] = primary
except Exception:
pass
clarity_aware = assess_utterance_clarity(
[dict(u) for u in updated_utterances],
chapter_id,
alias_map,
)
# Update database with new attributions
for u in updated_utterances:
for u in clarity_aware:
if u.get("type") == "dialogue" and "speaker_name" in u:
await db.update_utterance_speaker(
u["id"],
@@ -83,10 +109,15 @@ async def run_llm_attribution(
u.get("attribution_reasoning", ""),
u.get("emotion"),
u.get("emotion_confidence"),
evidence_kind=u.get("attribution_evidence_kind"),
tag_name=u.get("attribution_tag_name"),
uncertain=1 if u.get("attribution_uncertain") else 0,
needs_review=1 if u.get("needs_review") else 0,
review_reason=u.get("review_reason") or "",
)
# Populate exception reviews from confidence scores
await _populate_exceptions(db, chapter_id, updated_utterances)
# Populate exception reviews from clarity flags
await _populate_exceptions(db, pid, chapter_id, clarity_aware)
results["chapters_processed"] += 1
results["total_utterances"] += len([u for u in updated_utterances if u.get("type") == "dialogue"])
@@ -105,8 +136,10 @@ async def run_llm_attribution(
await db.close()
async def _populate_exceptions(db: Database, chapter_id: int, utterances: list[dict[str, Any]]) -> int:
"""Populate exception_reviews table from low-confidence attributions."""
async def _populate_exceptions(
db: Database, project_id: int, chapter_id: int, utterances: list[dict[str, Any]]
) -> int:
"""Populate exception_reviews table from clarity review flags."""
import json
total_added = 0
@@ -114,9 +147,7 @@ async def _populate_exceptions(db: Database, chapter_id: int, utterances: list[d
for u in utterances:
if u.get("type") != "dialogue":
continue
confidence = u.get("attribution_confidence", 1.0)
if confidence >= 0.7:
if not u.get("needs_review"):
continue
# Check if already in exception_reviews
@@ -128,21 +159,18 @@ async def _populate_exceptions(db: Database, chapter_id: int, utterances: list[d
if existing:
continue
# Determine ambiguity type
ambiguity_type = "low_confidence"
reasoning = u.get("attribution_reasoning", "")
if confidence == 0.0:
ambiguity_type = "no_confidence"
reason = u.get("review_reason") or ""
evidence_kind = u.get("attribution_evidence_kind") or ""
if "pronoun" in reason:
ambiguity_type = "pronoun"
elif "role" in reason:
ambiguity_type = "unnamed_role"
elif evidence_kind in ("none", "address_only"):
ambiguity_type = "ambiguous_tag"
elif u.get("speaker_name") == "narrator":
ambiguity_type = "narrator_dialogue"
elif "pronoun" in reasoning.lower():
ambiguity_type = "pronoun"
elif "role" in reasoning.lower() or "unnamed" in reasoning.lower():
ambiguity_type = "unnamed_role"
elif "retroactive" in reasoning.lower():
ambiguity_type = "retroactive_name"
elif "generic" in reasoning.lower():
ambiguity_type = "generic_role"
else:
ambiguity_type = "low_confidence"
await db.conn.execute("""
INSERT INTO exception_reviews (
@@ -151,14 +179,14 @@ async def _populate_exceptions(db: Database, chapter_id: int, utterances: list[d
ambiguity_type, evidence, reasoning, alternatives, text_preview, status
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending')
""", (
pid, chapter_id, u["id"], u["utterance_index"],
project_id, chapter_id, u["id"], u["utterance_index"],
u.get("speaker_name", ""), u.get("speaker_name", ""),
confidence,
u.get("attribution_confidence", 0.0),
ambiguity_type,
json.dumps([reasoning] if reasoning else ["Low confidence attribution"]),
reasoning,
json.dumps([reason] if reason else ["Needs review"]),
reason,
json.dumps([]),
u["text"][:200],
(u.get("text") or "")[:200],
))
total_added += 1
@@ -199,4 +227,4 @@ if __name__ == "__main__":
chapter_ids = [int(x) for x in sys.argv[1].split(",")]
result = asyncio.run(run_llm_attribution(chapter_ids=chapter_ids))
print(result)
print(result)

362
specs/17-speaker-confidence-review.md Normal file
View File

@@ -0,0 +1,362 @@
# Spec 17 — Speaker Identification Confidence, Orphan Aliases, and Utterance Clarity Review
Status: Phase 1 implemented (2026-09-23) — backend clarity signals live; eval measured 4.4%
queue on Last Dwarf. Phases 2-5 still planned.
Date: 2026-09-23
Consumer: translation pipeline (requires highly accurate speaker identity data); audiobook
voice assignment; future books by authors other than the user.
## 1. Goal
Design a review workflow where the system auto-accepts everything it can prove, and
surfaces a **small, high-value review queue** (~1–5% of utterances) for the human.
The reviewer should be able to fix 10 items, not re-review every utterance and change
50%. All identity/attribution data must be exportable with per-item confidence for a
downstream translation system.
Requirements driving this design:
- Identify **all** speakers and **all** aliases of each speaker.
- A "review" section on the Speakers tab for any speaker that isn't high confidence.
- Utterance clarity: "He said" with an unclear referent should be flagged for review.
- The system finds all high-confidence attributions; the human helps with the rest.
- Future books are NOT written by the user — the reviewer cannot be assumed to know the
cast. The pipeline must work when the reviewer has zero prior book knowledge.
- Data must be highly accurate for another system (translations).
## 2. Current-State Evaluation
### 2.1 What already exists (and works)
| Capability | Where | Notes |
|---|---|---|
| 4-pass LLM character discovery | `character_discovery.py` `discover_speakers` | speaking + non-speaking chars, aliases with `alias_evidence`, `unidentified` + `unidentified_reason`, `chapters_appeared` |
| Alias validation | `validate_aliases` | checks aliases appear in chapter text |
| Alias merge gating | `_merge_chapter_results` | `_has_two_proper_names` + `_alias_has_evidence` prevent garbage aliases |
| Mechanical narration evidence | `llm_attributor.py` | `narr_before`, `tag`, `tag_kind` (name/role/pronoun), `addressees` — extracted from the manuscript, not LLM guesses |
| Multi-pass attribution | `_verify_attributions` | up to 2 verify rounds, batched 20/call, invented-name gate, `_normalize_speaker` |
| Mechanical QA | `attribution_qa.py` | 6 issue types; `fix_high_confidence` writes back only resolved exact-name mismatches |
| QA accept (durable) | `utterances.qa_accepted` + `qa_accepted_sig` | signature = hash(speaker,text), so edits invalidate acceptance |
| Per-utterance confidence | `attribution_confidence` | stored, but see §2.2 weaknesses |
| Orphaned utterances page | `/orphaned-utterances` | utterance-level: `speaker_name` not in known names → reassign |
| Exception review table+page | `exception_reviews` table, `/exception-review` | filters by ambiguity type + confidence, resolve modal, export CSV, quality gate |
| Speakers tab | `/speakers`, `_speakers_list.html`, `_speaker_row.html` | flag badge, add/remove alias, sample dialogue, per-row voice gen |
### 2.2 What is weak or missing
1. **Confidence is not trustworthy as-is.** Distribution over 3,918 dialogue utterances
(Last Dwarf): **1.0 → 2480 (63%), 0.9 → 879, 0.85 → 13, 0.8 → 173, 0.7 → 58,
0.6 → 17, ≤0.5 → 4.** 94% sit at ≥0.9, and the LLM self-reports the number. There is
no calibration against actual correctness, no decomposition into *why* it is
confident (mechanical name tag vs. dialogue-content guess).
2. **The pass-1 `uncertain` flag is transient.** `attribution_uncertain` is set on
in-memory dicts during attribution (`llm_attributor.py:538`) and consumed only by the
verify pass (`:682`). It is **never persisted** — the DB has no such column, so the
review UI cannot see which lines "had no mechanical evidence to lean on".
3. **Speakers have no identification confidence.** The only flag is `_is_unknown_speaker`
(no vocal_traits + gender Unknown + empty description + Casual register) — a crude
heuristic. Discovery already produces `unidentified`, `unidentified_reason`,
`alias_evidence`, `chapters_appeared`, but none are surfaced as a confidence/certainty
signal or as a speaker-level accept/review workflow. 7 of 66 Last Dwarf speakers are
flagged `unidentified`, and many have zero aliases — no UI to resolve them.
4. **No orphan-ALIAS concept.** The orphaned-utterances page handles *utterances* whose
`speaker_name` is unknown. There is no list of *name tokens that appear in the text*
but are not attached to any speaker as primary name or alias (e.g., a name mentioned
in narration that is never attributed a line, or a role title that has no profile).
Discovery's `non_speaking_characters` capture some of this but they are not reconciled
with the alias map in the UI.
5. **No clarity classification per utterance.** The system never labels an utterance as
`name-tagged` / `role-tagged` / `pronoun-tagged` / `content-guess` at persistence time.
"He said" with a pronoun that cannot be resolved to a unique recent character is a
first-class ambiguity case the reviewer cares about, but nothing mechanically flags it.
6. **Exception review page is orphaned from navigation.** `/exception-review` exists but
is not linked in `base.html`; its populate endpoint only inserts rows for
`attribution_confidence < 0.7`, and it sniffs `attribution_reasoning` text for the
words "pronoun"/"role" to classify ambiguity. It never auto-populates during
attribution, and confidence <0.7 is 0.2% of the book — so it currently finds almost
nothing.
7. **DB default is dangerous.** `attribution_confidence REAL DEFAULT 1.0` means
un-attributed rows silently look perfect.
8. **Alias unification is a one-way street.** Discovery creates speakers + aliases; manual
alias add exists. But there is no flow to merge "this name is actually an alias of that
speaker" driven by *evidence* (the same name token to which the LLM attributed lines in
multiple chapters), and no automatic fix-up of attributed `speaker_name` when aliases
are added/merged.
## 3. Design
### 3.1 Two axes of confidence
Keep them separate — they answer different questions and flow to the translation system
differently.
**A. Speaker identification confidence** (speaker-level; stored on `speakers.json`):
how sure we are that this speaker exists and that their alias set is complete.
Derived from:
- `unidentified` / `unidentified_reason` (discovery)
- alias evidence strength: `alias_evidence` entries, chapters where the alias is attested
- chapters appeared with dialogue vs. only mentioned
- speaker-level attribution health: number of utterances, mean attribution confidence,
count of `needs_review` utterances attributed to them
- soundex/word-overlap collision warnings: two speakers whose names/aliases are near-dups
**B. Utterance attribution confidence** (utterance-level; persisted in DB): composite of
- mechanical tag strength: `name` tag > `role` tag > `pronoun` tag > `no tag`
(content guess). This is the *strongest* signal and is manuscript-derived, not LLM.
- LLM self-reported confidence (pass 1)
- verify-pass outcome: agreed / corrected / still-uncertain
- mechanical QA issues (attribution_qa)
- pronoun resolvability (see §3.3)
Composite → one of: `HIGH` (auto-accept), `MEDIUM` (review batch), `LOW` (must review).
The review queue = `MEDIUM` + `LOW` ≈ the ~1–5% the user wants to see, not 50%.
### 3.2 Speaker page changes
Layout (all on `/speakers`):
1. **Review section (top).** "Speakers needing review" — any speaker with:
- `unidentified: true`, or
- identification confidence < threshold, or
- alias-set incompleteness (name tokens in text that look like they belong to them —
fed by the orphan-alias scanner §3.4), or
- attribution-health problems (many LOW/MEDIUM utterances, tag-vs-speaker mismatches).
Each row: speaker name, **reason** (machine-readable + short human text), sample
dialogue, and actions:
- **Accept** (sets `reviewed: true`, clears the flag; the tool remembers the decision)
- **Edit** (existing speaker edit form)
- **_Create speaker_** from an orphan alias / **merge alias into this speaker** (§3.4).
Auto-accept is never silent: accepted speakers are recorded so a re-run does not re-flag.
2. **Main list (existing table, enriched).** Add per-speaker columns:
- confidence bar (identification confidence)
- utterance count + % attributed at HIGH
- alias count + "alias coverage" (of name tokens attested for them in the text)
- a small ✓/! review-status badge.
3. **Orphan aliases panel (§3.4)** — same page, below the speaker table.
### 3.3 Utterance clarity: confidence assessment + review-needed flagging
At attribution time, persist four new fields per utterance:
- `attribution_evidence_kind`: `name` | `role` | `pronoun` | `address_only` | `none`
- `attribution_tag_name`: the exact tag string from narration (e.g. `"the lead guard"`)
- `attribution_uncertain`: the pass-1 flag, now persisted (0/1)
- `needs_review`: computed composite flag + `review_reason` TEXT
Mechanical determinism (no LLM needed):
- **pronoun tag** (`He said`/`She said`): resolve against the dialogue positions just
before the tag, within the scene. If exactly one candidate of that gender is present in
the recent window → assignment is HIGH-confidence (LLM still sets the name, but the
machine has corroborated uniqueness). If **zero or multiple** candidates → flag
`needs_review` with reason `pronoun: N candidates`. This is the "He said" case the user
explicitly called out.
Role/name tags that match a profile/alias → HIGH; role tags with **no** matching profile
→ MEDIUM (review; likely a missing speaker → orphan alias / create speaker).
`none` / `address_only` (content-guess, incl. "The voice calls again") → MEDIUM only when
confidence < 0.75 (implemented `_NO_EVIDENCE_CONFIDENCE_FLOOR`); verify-confirmed
content-guess lines at 0.75-0.9 stay auto-accepted to keep the queue near 1-5%.
QA issues from `attribution_qa.py` (speech_tag_mismatch, narrator_dialogue, address_mismatch,
narration_speaker_mismatch, etc.) push to `needs_review` regardless of confidence.
`long_monologue` is deliberately excluded from the queue (weak 0.5 heuristic; intentional
4+ line speeches are common) but still reported by mechanical QA.
`attribution_uncertain` (pass-1 no-narration-evidence flag) → review only when confidence
did not recover to ≥ 0.9 after the verify pass.
A `MEDIUM`/`LOW` item still keeps its attributed speaker — the reviewer corrects only when
wrong; the queue is a triage list, not a "blank" list.
### 3.4 Orphan aliases: display and assignment
New derived concept: **unresolved name tokens** — proper-noun tokens appearing in chapter
text/narration (from the existing segmentation, `tag` fields, and discovery pass) that are
not in `_all_speaker_names_and_aliases()` and are not stopwords. Grouped by token:
```
Name token | Occurrences | Chapters | Sample line | Action
Trader Marek | 12 | 14, 15 | "Trader Marek nodded." | [Assign ▾] [Create speaker]
```
Actions per token:
- **Assign to existing speaker** — dropdown (searchable) of speakers; on save: add token
to that speaker's `aliases` **with the evidence lines**, then **re-normalize every
utterance** whose `speaker_name` equals the token → the speaker's `primary_name`
(updates `speaker_name` + bumps `attribution_reasoning`; audio markers already key on
speaker so regen is automatic via staleness).
- **Create new speaker** — pre-filled modal (primary_name = token, aliases = [token],
sample dialogue = first attributed line). Speaker enters the normal voice-design flow.
- **Dismiss** — mark as non-speaking / not-a-character (e.g., a place or a once-mentioned
person). Recorded so it does not reappear.
- **Merge two speakers** — the reverse of assign: two profiles are one person. Keep the
longer alias set, merge aliases, re-normalize utterances from the removed primary to the
kept primary.
Feeds the speaker-review section: if an unresolved token looks like a *variant* of an
existing speaker's name (word-overlap / soundex / prefix match ≥ threshold), surface it
directly as "suggested alias of X" so the reviewer can accept in one click.
Everything recorded in `speakers.json` (aliases + `alias_evidence`) and in a new
`name_tokens` table (token, status: `assigned|created|dismissed|unresolved`, target
speaker_id, evidence lines) so re-runs are stable and the translation export can prove
"every name in the text resolves to a speaker or was dismissed".
### 3.5 Review workflow & persistence
- Extend the existing `exception_reviews` table as the **review queue** (it already has
the shape: chapter, utterance, current_speaker, confidence, ambiguity_type, evidence,
alternatives, status). Re-populate it **automatically at the end of attribution** from
`needs_review` utterances (no manual populate step), and link the page in `base.html`.
- Add `speaker_name` changes through the review UI → same re-normalization + regen path as
§3.4.
- `qa_accepted_sig` already invalidates acceptance when speaker/text changes — reuse.
- **The queue should be small by construction**: only pronoun-ambiguity, role-no-profile,
no-tag, address-only, and mechanical-QA-flagged lines. That is the user's
"review 10, not 50%".
### 3.6 Translation export
New endpoint/CLI: export `identity_export.json`:
```json
{
"speaker_id": "trader-marek", // stable slug, not display name
"canonical_name": "Trader Marek",
"aliases": [{"name": "Marek", "evidence_lines": [...]}],
"identification_confidence": 0.96,
"review_status": "accepted", // accepted | needs_review | auto_high
"utterances": [
{"chapter": 14, "index": 41, "text": "...",
"effective_speaker": "trader-marek",
"confidence": 0.97, "evidence_kind": "name", "review_status": "auto_high"}
]
}
```
Rules:
- Every utterance maps to a stable `speaker_id` (never a raw display string).
- Every speaker carries identification confidence + review status.
- Every name token in the book is resolved or explicitly dismissed (the `name_tokens`
table proves it).
- A benchmark/precision metric: per `evidence_kind`, the fraction of
reviewer-confirmed items that were correct — lets the user see "name-tagged lines are
99% right, content-guesses are 60%".
### 3.7 New columns / tables
- `utterances`: `attribution_evidence_kind TEXT`, `attribution_tag_name TEXT`,
`attribution_uncertain INTEGER DEFAULT 0`, `needs_review INTEGER DEFAULT 0`,
`review_reason TEXT` (nullable), and fix `attribution_confidence` default → NULL.
- `speakers.json` per-speaker: `identification_confidence REAL`,
`review_status TEXT` (`auto_high|needs_review|accepted`), `reviewed_at TEXT`,
`alias_coverage` (derived, not stored).
- New table `name_tokens` (token, status, target_speaker, evidence_json, dismissed).
- Reuse `exception_reviews` as the review-queue table; auto-populate.
## 4. Implementation Plan
### Phase 1 — Persist clarity signals (backend) ✅ implemented
1. DB: 5 columns added (`attribution_evidence_kind`, `attribution_tag_name`,
`attribution_uncertain`, `needs_review`, `review_reason`); `attribution_confidence`
default → NULL. Migrations idempotent in `database.py._migrate`, `routes.py
_ensure_utterance_columns`, and `scripts/reattribute_all.py._ensure_utterance_columns`
(the raw-sqlite driver used for chapter re-runs must add columns itself).
2. `llm_attributor.py`: `attribute_speakers` accumulates scene evidence across batches
into `evidence_by_idx` and stamps `attribution_evidence_kind` (via
`clarity.evidence_kind_for`) + `attribution_tag_name` onto returned utterance dicts.
3. New module `umbraaudio/analysis/clarity.py`:
- `extract_mechanical_evidence` / `evidence_kind_for`: deterministic evidence kind
(`name|role|pronoun|address_only|none`); addressee only counts when it resolves to a
known name/title (guards against "But where…" false positives).
- pronoun resolvability: `_recent_candidates` walks back in-scene (gender-filtered via
`speakers.json` demographics); exactly 1 candidate → resolvable, else → review.
- composite `needs_review` — triggers: any QA issue (long_monologue excluded), pronoun
non-unique, role tag with no profile (`speaker` not in alias map), no/address-only
evidence below 0.75 confidence, `attribution_uncertain` below 0.9, narrator-left
dialogue, confidence < 0.7. `review_reason` is a "; "-joined readable summary.
4. Drivers persist everything: `reattribute_all.py` runs `assess_utterance_clarity` after
attribution and writes all 8 columns; `run_llm_attribution.py` same via
`update_utterance_speaker`; both web attribution paths in `routes.py` updated.
`_populate_exceptions` now keys off `needs_review` (was confidence<0.7 + keyword sniff)
and maps reason → `ambiguity_type`; fixed a latent `NameError` (`pid` not in scope).
5. Eval on Last Dwarf (pre-re-run confidences, 3918 dialogue): review queue = **173
(4.42%)** — top signals: none-evidence low-conf (73), pronoun ambiguity (53),
narration-speaker mismatch (18), address mismatch (5), role-no-profile (6),
narrator-left (3). Long-monologue runs (43) deliberately NOT auto-queued (weak 0.5
heuristic; 4+ line speeches are commonly intentional) — still visible in QA reports.
Tests: `tests/test_clarity.py` — 25 tests (evidence kinds, address-only false-positive
guard, pronoun 0/1/many candidates, role-with/without profile, QA coupling, uncertain
flag persistence, narrator-left, low-confidence, non-dialogue passthrough, no-mutation).
ruff + mypy clean on `clarity.py`, `llm_attributor.py` deltas, `reattribute_all.py`;
routes.py unchanged baseline (73 pre-existing). Full suite 380 passed / 6 pre-existing
`voice_profile_manager` accent failures. Migration smoke-tested against a copy of the real
project DB. To refresh real data: re-run `scripts/reattribute_all.py` (adds columns +
fills clarity fields for the reviewed chapters; `qa_accepted` rows are skipped as today).
### Phase 2 — Orphan alias scanner + assignment (backend)
1. `scripts/scan_name_tokens.py` + `umbraaudio/analysis/name_tokens.py`: extract
proper-noun tokens from chapter text/tags; diff against alias map; write `name_tokens`
table; fuzzy-suggest existing speakers (word-overlap/soundex).
2. Assignment APIs: assign-to-speaker (add alias + evidence + re-normalize utterances),
create-speaker, dismiss, merge-speakers. Reuse routes' `_all_speaker_names_and_aliases`
and the alias-modal machinery.
Tests: token extraction precision/recall on a fixture chapter; assign→re-normalize→stale
marker; merge idempotence.
### Phase 3 — UI (web)
1. Speakers tab: review section (reason + Accept/Edit/Create/Assign), enriched columns,
orphan-aliases panel with inline assign/create/dismiss.
2. Link `/exception-review` in `base.html` nav; row actions: speaker dropdown (searchable),
context, resolve (reuse existing modals), auto-regen wiring.
3. Chapter view: per-utterance clarity badge (name/role/pronoun/none) + "why flagged"
tooltip; mark `needs_review` items with the same amber styling as QA flags.
Manual verification: load a Last Dwarf chapter; confirm review queue ≈ expected small set;
confirm every orphan alias action updates speakers.json + DB + markers.
### Phase 4 — Translation-oriented metrics + export
1. `identity_export` endpoint + CLI (`scripts/export_identity.py`).
2. Precision-by-evidence-kind metric from reviewed items (requires reviewer accept counts —
starts as "n/a" until first review batch lands).
Tests: export schema golden file; every utterance ✓ effective_speaker id; every token
resolved-or-dismissed assertion.
### Phase 5 — Validation on Last Dwarf
1. Re-run attribution for ch11–49 with clarity persistence (existing workflow, markers
allow regen on output change; a full re-attribute run only writes DB fields, no audio
regen unless speaker changes).
2. Report: review queue size, per-evidence-kind distribution, unresolved-token count,
speaker identification confidences — vs. this document's targets (queue ~1–5%).
3. Commit cadence per AGENTS.md (Sunday commits; scripts/commit_check.sh).
## 5. Success Criteria
- Review queue: ≤5% of dialogue utterances, ≥80% of reviewer-corrections correspond to
items the system flagged (precision), and ≥90% of genuine errors are in the queue
(recall — measured against a gold set of the user's known books as label source).
- 100% of name tokens in the text resolve to a speaker or are explicitly dismissed.
- Translation export: deterministic, stable speaker_ids, no raw-name drift.
- Works with zero prior book knowledge: the pipeline can run end-to-end on a fixture book
with no user corrections and still produce a usable export (some review items, none
fatal).
## 6. Non-Goals / Deferred
- Automated *resolution* of pronoun ambiguity by extra LLM passes (verify rounds already
cover it; keep the human loop).
- Auto-merging speakers without a human decision (only suggestions).
- Per-utterance audio regeneration automation beyond existing staleness markers.

299
tests/test_clarity.py Normal file
View File

@@ -0,0 +1,299 @@
from __future__ import annotations
from umbraaudio.analysis.clarity import (
assess_utterance_clarity,
evidence_kind_for,
extract_mechanical_evidence,
)
ALIAS_MAP = {
"hunter": "Deal Hunter",
"deal hunter": "Deal Hunter",
"kainda": "Kainda Walters Foster Murphy",
"kainda walters foster murphy": "Kainda Walters Foster Murphy",
"princess kainda": "Kainda Walters Foster Murphy",
"kabir": "Kabir",
"captain": "Captain Weston",
"captain weston": "Captain Weston",
"chief of the guard": "Chief of the Guard",
"narrator": "Narrator",
}
PROFILES = [
{
"primary_name": "Deal Hunter",
"aliases": ["Hunter"],
"demographics": {"gender": "Non-binary"},
},
{
"primary_name": "Kainda Walters Foster Murphy",
"aliases": ["Kainda", "Princess Kainda"],
"demographics": {"gender": "Female"},
},
{
"primary_name": "Kabir",
"aliases": [],
"demographics": {"gender": "Male"},
},
{
"primary_name": "Captain Weston",
"aliases": ["Captain"],
"demographics": {"gender": "Male"},
},
{
"primary_name": "Chief of the Guard",
"aliases": [],
"demographics": {"gender": "Male"},
},
]
def _utt(idx: int, text: str, speaker: str, type_: str = "narration", **kw) -> dict:
row = {
"id": idx + 1000,
"utterance_index": idx,
"text": text,
"type": type_,
"speaker_name": speaker,
}
row.update(kw)
return row
class TestEvidenceKind:
def test_name_kind(self) -> None:
assert evidence_kind_for({"tag_kind": "name", "addressees": []}) == "name"
def test_role_kind(self) -> None:
assert evidence_kind_for({"tag_kind": "role", "addressees": []}) == "role"
def test_pronoun_kind(self) -> None:
assert evidence_kind_for({"tag_kind": "pronoun", "addressees": []}) == "pronoun"
def test_address_only_kind(self) -> None:
assert (
evidence_kind_for({"tag_kind": None, "addressees": ["Captain"]})
== "address_only"
)
def test_none_kind(self) -> None:
assert evidence_kind_for({"tag_kind": None, "addressees": []}) == "none"
class TestExtractMechanicalEvidence:
def test_name_tag(self) -> None:
utts = [
_utt(0, "Kainda says,", "narrator"),
_utt(1, "We are ready.", "Kainda Walters Foster Murphy", "dialogue"),
]
ev = extract_mechanical_evidence(utts, ALIAS_MAP)
assert ev[1]["tag_kind"] == "name"
assert ev[1]["tag"] == "Kainda Walters Foster Murphy"
def test_role_tag(self) -> None:
utts = [
_utt(0, "The lead guard said,", "narrator"),
_utt(1, "Stop there.", "Chief of the Guard", "dialogue"),
]
ev = extract_mechanical_evidence(utts, ALIAS_MAP)
assert ev[1]["tag_kind"] == "role"
def test_pronoun_tag(self) -> None:
utts = [
_utt(0, "He said,", "narrator"),
_utt(1, "I will go.", "Kabir", "dialogue"),
]
ev = extract_mechanical_evidence(utts, ALIAS_MAP)
assert ev[1]["tag_kind"] == "pronoun"
assert ev[1]["tag"] == "he"
def test_no_evidence(self) -> None:
utts = [
_utt(0, "The ship hummed quietly.", "narrator"),
_utt(1, "But where is the coat?", "Kabir", "dialogue"),
]
ev = extract_mechanical_evidence(utts, ALIAS_MAP)
assert ev[1]["tag_kind"] is None
assert ev[1]["addressees"] == []
def test_addressee_only(self) -> None:
utts = [
_utt(0, "Captain, we are taking fire.", "Kabir", "dialogue"),
]
ev = extract_mechanical_evidence(utts, ALIAS_MAP)
assert ev[0]["tag_kind"] is None
assert "Captain Weston" in ev[0]["addressees"]
class TestAssessUtteranceClarity:
def test_high_confidence_name_tag_auto_accept(self) -> None:
utts = [
_utt(0, "Kainda says,", "narrator"),
_utt(
1,
"We are ready.",
"Kainda Walters Foster Murphy",
"dialogue",
attribution_confidence=0.95,
),
]
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
dialogue = out[1]
assert dialogue["attribution_evidence_kind"] == "name"
assert dialogue["needs_review"] == 0
assert dialogue["review_reason"] == ""
def test_speech_tag_mismatch_flagged(self) -> None:
utts = [
_utt(0, "Kainda says,", "narrator"),
_utt(1, "We are ready.", "Kabir", "dialogue", attribution_confidence=0.95),
]
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
assert out[1]["needs_review"] == 1
assert "tag" in out[1]["review_reason"].lower()
def test_pronoun_unique_candidate_correct(self) -> None:
utts = [
_utt(0, "Kabir says,", "Kabir"),
_utt(1, "I am going.", "Kabir", "dialogue", attribution_confidence=0.9),
_utt(2, "He said,", "narrator"),
_utt(3, "I will return soon.", "Kabir", "dialogue", attribution_confidence=0.9),
]
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
assert out[3]["attribution_evidence_kind"] == "pronoun"
assert out[3]["needs_review"] == 0
def test_pronoun_no_candidate_flagged(self) -> None:
utts = [
_utt(0, "He said,", "narrator"),
_utt(1, "I will return soon.", "Kabir", "dialogue", attribution_confidence=0.9),
]
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
assert out[1]["needs_review"] == 1
assert "0 candidate" in out[1]["review_reason"]
def test_pronoun_ambiguous_two_candidates_flagged(self) -> None:
# Two males present (Kabir + Captain Weston) → pronoun referent ambiguous.
utts = [
_utt(0, "Kabir says,", "Kabir"),
_utt(1, "I am going.", "Kabir", "dialogue", attribution_confidence=0.9),
_utt(2, "Captain Weston says,", "Captain Weston"),
_utt(3, "Stand down.", "Captain Weston", "dialogue", attribution_confidence=0.9),
_utt(4, "He said,", "narrator"),
_utt(5, "Enough of this.", "Kabir", "dialogue", attribution_confidence=0.9),
]
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
assert out[5]["needs_review"] == 1
assert "2 candidate" in out[5]["review_reason"]
def test_role_tag_no_profile_flagged(self) -> None:
utts = [
_utt(0, "The lead guard said,", "narrator"),
# "Lead Guard" is not a profile entry → inventoried role without profile.
_utt(1, "Stop there.", "Lead Guard", "dialogue", attribution_confidence=0.9),
]
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
assert out[1]["needs_review"] == 1
assert "no profile" in out[1]["review_reason"]
def test_role_tag_with_profile_auto_accept(self) -> None:
utts = [
_utt(0, "The lead guard said,", "narrator"),
_utt(1, "Stop there.", "Chief of the Guard", "dialogue", attribution_confidence=0.95),
]
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
assert out[1]["needs_review"] == 0
def test_no_evidence_low_confidence_flagged(self) -> None:
utts = [
_utt(0, "The ship hummed quietly.", "narrator"),
_utt(1, "But where is the coat?", "Kabir", "dialogue", attribution_confidence=0.65),
]
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
assert out[1]["attribution_evidence_kind"] == "none"
assert out[1]["needs_review"] == 1
assert "no narration evidence" in out[1]["review_reason"]
def test_no_evidence_high_confidence_accepted(self) -> None:
utts = [
_utt(0, "The ship hummed quietly.", "narrator"),
_utt(1, "But where is the coat?", "Kabir", "dialogue", attribution_confidence=0.97),
]
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
assert out[1]["needs_review"] == 0
def test_narrator_dialogue_flagged(self) -> None:
utts = [
_utt(0, "Some narration.", "narrator"),
_utt(1, "I am talking.", "narrator", "dialogue", attribution_confidence=1.0),
]
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
assert out[1]["needs_review"] == 1
assert "narrator" in out[1]["review_reason"]
def test_uncertain_persisted_and_flagged_when_subhigh(self) -> None:
utts = [
_utt(0, "The ship hummed quietly.", "narrator"),
_utt(
1,
"But where is the coat?",
"Kabir",
"dialogue",
attribution_confidence=0.8,
attribution_uncertain=True,
),
]
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
assert out[1]["attribution_uncertain"] == 1
assert out[1]["needs_review"] == 1
assert "uncertain" in out[1]["review_reason"]
def test_uncertain_no_longer_flagged_after_verify_high(self) -> None:
utts = [
_utt(0, "The ship hummed quietly.", "narrator"),
_utt(
1,
"But where is the coat?",
"Kabir",
"dialogue",
attribution_confidence=0.95,
attribution_uncertain=True,
),
]
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
assert out[1]["attribution_uncertain"] == 1
assert out[1]["needs_review"] == 0
def test_low_confidence_flagged(self) -> None:
utts = [
_utt(0, "Kainda says,", "narrator"),
_utt(
1,
"We are ready.",
"Kainda Walters Foster Murphy",
"dialogue",
attribution_confidence=0.6,
),
]
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
assert out[1]["needs_review"] == 1
assert "low confidence" in out[1]["review_reason"]
def test_originals_not_mutated(self) -> None:
utts = [
_utt(0, "Kainda says,", "narrator"),
_utt(1, "We are ready.", "Kabir", "dialogue", attribution_confidence=0.95),
]
assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
assert "needs_review" not in utts[1]
assert "attribution_evidence_kind" not in utts[1]
def test_non_dialogue_pass_through(self) -> None:
utts = [
_utt(0, "Kainda says,", "narrator"),
_utt(1, "We are ready.", "Kainda Walters Foster Murphy", "dialogue"),
_utt(2, "<<<>>>", "scene_break", "scene_break"),
]
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
assert "needs_review" not in out[2]
assert out[2]["type"] == "scene_break"

327
umbraaudio/analysis/clarity.py Normal file
View File

@@ -0,0 +1,327 @@
"""Deterministic utterance-clarity classification for speaker attributions.
Runs after LLM attribution as a cheap, repeatable post-pass. It annotates every
dialogue utterance with:
- ``attribution_evidence_kind``: ``name`` | ``role`` | ``pronoun`` |
``address_only`` | ``none`` (what the narration mechanically points at)
- ``attribution_tag_name``: the exact narration tag ("Kainda", "the lead guard",
"He") or None
- ``needs_review``: 0/1 — whether a human should double-check this line
- ``review_reason``: readable summary of why the line was flagged (or "")
Nothing here calls an LLM. The rules mirror the manuscript-derived evidence the
attribution prompt already uses, so the persistence layer and the review UI agree
with what drove the attribution in the first place.
Review flagging (kept deliberately narrow so the queue is ~1-5% of dialogue):
- any mechanical QA issue (speech tag mismatch, narrator dialogue, role guessed,
address mismatch, conversation-flow issue)
- a pronoun tag ("He said") whose referent is **not unique** in the recent scene
- a role tag that resolved to a name with no profile entry
- a line with no narration evidence ("the voice calls again") at sub-0.75
confidence (verified content-guess lines at 0.75-0.9 are self-consistent)
- confidence below 0.7 from the attribution/verify passes
``long_monologue`` runs are NOT auto-queued: intentional speeches of 4+ lines
are common, and the rule is only a weak (0.5) heuristic. It still appears in
mechanical QA reports for spot checking.
"""
from __future__ import annotations
import re
from typing import Any
from umbraaudio.analysis.attribution_qa import (
SPEECH_VERBS,
QAIssue,
_detect_addressee,
_match_speech_tag,
check_attribution,
verify_conversation_flow,
)
# Matches a TAG narration line whose subject is a pronoun: "He said,", "She asks."
_PRONOUN_TAG = re.compile(
r"^\s*(H[ei]|She|They)\s+"
r"(says|said|asks|asked|replies|replied|whispers|whispered|shouts|shouted|"
r"mutters|muttered|calls|called|answers|answered)\b",
re.IGNORECASE,
)
_PRONOUN_GENDERS = {"he": "Male", "she": "Female", "his": "Male", "her": "Female"}
_DEFAULT_LOOKBACK = 15
# Confidence floor for flagging "no narration evidence" lines. Verify-confirmed
# content-guess lines at 0.75-0.9 are self-consistent; only clearly-low ones go
# to the review queue (keeps the queue near the ~1-5% target).
_NO_EVIDENCE_CONFIDENCE_FLOOR = 0.75
# Titles that count as addressing a known cast member even without a profile
# alias (mirrors the title list in attribution_qa._detect_addressee).
_TITLE_WORDS = frozenset(
{"captain", "princess", "hunter", "doctor", "sir", "ma'am", "admiral", "chief"}
)
def evidence_kind_for(entry: dict[str, Any]) -> str:
"""Classify one mechanical-evidence entry into an evidence kind string."""
tag_kind = entry.get("tag_kind")
if tag_kind in ("name", "role", "pronoun"):
return str(tag_kind)
if entry.get("addressees"):
return "address_only"
return "none"
def extract_mechanical_evidence(
utterances: list[dict[str, Any]],
alias_map: dict[str, str],
) -> dict[int, dict[str, Any]]:
"""Walk the ordered utterance list and pull narration evidence per dialogue line.
Mirrors ``LLMAttributor._build_mechanical_evidence``: up to two narration
lines before a dialogue (staying inside the scene) yield ``narr_before``,
``tag``, ``tag_kind``; the dialogue itself yields ``addressees``.
"""
evidence: dict[int, dict[str, Any]] = {}
for pos, u in enumerate(utterances):
if (u.get("type") or "").lower() != "dialogue":
continue
idx = u.get("utterance_index")
if idx is None:
idx = u.get("index")
if idx is None:
continue
entry: dict[str, Any] = {
"narr_before": "",
"tag": None,
"tag_kind": None,
"addressees": [],
}
narr_texts: list[str] = []
j = pos - 1
while j >= 0:
prev = utterances[j]
prev_text = (prev.get("text") or "").strip()
if prev_text == "<<<>>>" or (prev.get("type") or "").lower() != "narration":
break
narr_texts.append(prev_text)
if len(narr_texts) >= 2:
break
j -= 1
if narr_texts:
entry["narr_before"] = narr_texts[0][:220]
for narr_text in narr_texts:
if SPEECH_VERBS.search(narr_text):
kind, value, _resolved = _match_speech_tag(narr_text, alias_map)
if kind == "name" and value:
entry["tag"] = value
entry["tag_kind"] = "name"
break
if kind == "role" and value:
entry["tag"] = value
entry["tag_kind"] = "role"
break
pm = _PRONOUN_TAG.match(narr_text)
if pm:
entry["tag"] = pm.group(1).lower()
entry["tag_kind"] = "pronoun"
break
# Only a KNOWN name/title counts as an addressee: the shared detector
# also matches any capitalized sentence-start ("But where...") which is
# not address evidence for the speaker-clarity model.
addressee = _detect_addressee(u.get("text") or "", alias_map)
if addressee:
addr_lower = addressee.lower()
if addr_lower in alias_map or addr_lower in _TITLE_WORDS:
entry["addressees"].append(addressee)
evidence[idx] = entry
return evidence
def _pronoun_target(pronoun: str) -> str | None:
"""Return the gender the pronoun points at ('Male'/'Female'), or None = any."""
return _PRONOUN_GENDERS.get((pronoun or "").lower())
def _speaker_gender(name: str, gender_map: dict[str, str]) -> str | None:
"""Gender of a canonical speaker name, from the profile/alias gender map."""
return gender_map.get(name.strip().lower())
def _recent_candidates(
utterances: list[dict[str, Any]],
dialogue_pos: int,
gender: str | None,
gender_map: dict[str, str],
alias_map: dict[str, str],
lookback: int = _DEFAULT_LOOKBACK,
) -> list[str]:
"""Distinct plausible referents for a pronoun tag before ``dialogue_pos``.
Candidates are speakers attributed to prior dialogue lines in the scene,
plus names mechanically tagged by prior narration lines. When ``gender`` is
not None the list is filtered to that gender (unknown-gender profiles count).
"""
candidates: list[str] = []
seen: set[str] = set()
j = dialogue_pos - 1
scanned = 0
while j >= 0 and scanned < lookback:
prev = utterances[j]
prev_text = (prev.get("text") or "").strip()
if prev_text == "<<<>>>":
break
prev_type = (prev.get("type") or "").lower()
if prev_type == "dialogue":
name = (prev.get("speaker_name") or "").strip()
if name and name.lower() not in ("narrator", "the narrator"):
canonical = alias_map.get(name.lower(), name)
if canonical.lower() not in seen:
g = _speaker_gender(canonical, gender_map)
if gender is None or g is None or g.lower() == gender.lower():
seen.add(canonical.lower())
candidates.append(canonical)
elif prev_type == "narration":
if SPEECH_VERBS.search(prev_text):
kind, value, _resolved = _match_speech_tag(prev_text, alias_map)
if kind == "name" and value:
canonical = alias_map.get(value.lower(), value)
if canonical.lower() not in seen:
g = _speaker_gender(canonical, gender_map)
if gender is None or g is None or g.lower() == gender.lower():
seen.add(canonical.lower())
candidates.append(canonical)
scanned += 1
j -= 1
return candidates
def _profile_gender_map(profiles: list[dict[str, Any]] | None) -> dict[str, str]:
"""Map lowercase name+alias -> gender from discovery profile entries."""
gender_map: dict[str, str] = {}
for p in profiles or []:
primary = (p.get("primary_name") or "").strip()
gender = ((p.get("demographics") or {}).get("gender") or "").strip()
if not primary:
continue
if gender:
gender_map[primary.lower()] = gender
for alias in p.get("aliases") or []:
if alias and gender:
gender_map[alias.lower()] = gender
return gender_map
def assess_utterance_clarity(
utterances: list[dict[str, Any]],
chapter_id: int,
alias_map: dict[str, str],
profiles: list[dict[str, Any]] | None = None,
) -> list[dict[str, Any]]:
"""Annotate each dialogue utterance with evidence kind + review flags.
Returns a NEW list of dicts (originals untouched). Only dialogue utterances
gain the clarity fields; narration/scene-break rows pass through unchanged.
"""
evidence = extract_mechanical_evidence(utterances, alias_map)
gender_map = _profile_gender_map(profiles)
qa = check_attribution(utterances, chapter_id, alias_map)
flow = verify_conversation_flow(utterances, chapter_id, alias_map)
issue_by_idx: dict[int, list[QAIssue]] = {}
for issue in qa.issues + flow.issues:
# long_monologue is a weak (0.5) heuristic: 4+ same-speaker lines are
# often intentional speeches. Keep it out of the auto-review queue; it
# remains visible in mechanical QA for spot checking.
if issue.issue_type == "long_monologue":
continue
issue_by_idx.setdefault(issue.utterance_index, []).append(issue)
dialogue_positions: dict[int, int] = {}
for pos, u in enumerate(utterances):
idx = u.get("utterance_index")
if idx is not None and (u.get("type") or "").lower() == "dialogue":
dialogue_positions[idx] = pos
result: list[dict[str, Any]] = []
for u in utterances:
out = dict(u)
if (u.get("type") or "").lower() != "dialogue":
result.append(out)
continue
idx = u.get("utterance_index")
assert idx is not None
idx = int(idx)
entry = evidence.get(idx, {})
kind = evidence_kind_for(entry)
tag = entry.get("tag")
out["attribution_evidence_kind"] = kind
out["attribution_tag_name"] = tag
if u.get("attribution_uncertain"):
out["attribution_uncertain"] = 1
else:
out.setdefault("attribution_uncertain", 0)
reasons: list[str] = []
issues = issue_by_idx.get(idx, [])
for iss in issues:
reasons.append(iss.evidence or iss.issue_type)
speaker = (u.get("speaker_name") or "narrator").strip()
confidence = float(u.get("attribution_confidence") or 0.5)
# Pass-1 "no mechanical evidence" flag: still worth eyes unless the
# verify pass reached HIGH confidence.
if out["attribution_uncertain"] and confidence < 0.9:
reasons.append("uncertain: no narration evidence in first pass")
# Pronoun tags: unique referent required, else flag for review.
if kind == "pronoun" and tag:
target_gender = _pronoun_target(tag)
candidates = _recent_candidates(
utterances,
dialogue_positions.get(idx, 0),
target_gender,
gender_map,
alias_map,
)
if len(candidates) != 1:
reasons.append(
f"pronoun {tag!r}: {len(candidates)} candidate(s) in scene"
)
# Role tags resolving to a name with no profile entry → missing speaker.
if kind == "role" and tag and speaker.lower() not in alias_map:
reasons.append(f"role tag {tag!r} → no profile for {speaker!r}")
# No narration evidence at all: content-guess lines need eyes unless
# confidence cleared the verify-confidence floor.
if kind in ("none", "address_only") and confidence < _NO_EVIDENCE_CONFIDENCE_FLOOR:
reasons.append(f"no narration evidence (confidence {confidence:.2f})")
if speaker.lower() in ("narrator", "the narrator"):
reasons.append("dialogue attributed to narrator")
if confidence < 0.7:
reasons.append(f"low confidence ({confidence:.2f})")
needs_review = 1 if reasons else 0
out["needs_review"] = needs_review
out["review_reason"] = "; ".join(dict.fromkeys(reasons))
result.append(out)
return result

26
umbraaudio/analysis/llm_attributor.py
View File

@@ -10,17 +10,28 @@ from typing import Any
from openai import OpenAI
from umbraaudio.config import LLMConfig
from umbraaudio.analysis.context_analyzer import ContextAnalyzer, AttributionConfidence
from umbraaudio.analysis.tone_analyzer import ToneAnalyzer, SceneEmotionalProfile
from umbraaudio.analysis.attribution_qa import (
SPEECH_VERBS as _QA_SPEECH_VERBS,
)
from umbraaudio.analysis.attribution_qa import (
_detect_addressee as _qa_detect_addressee,
)
from umbraaudio.analysis.attribution_qa import (
_match_speech_tag as _qa_match_speech_tag,
)
from umbraaudio.analysis.attribution_qa import (
check_attribution as _qa_check_attribution,
)
from umbraaudio.analysis.attribution_qa import (
fix_high_confidence as _qa_fix_high_confidence,
)
from umbraaudio.analysis.attribution_qa import (
verify_conversation_flow as _qa_verify_flow,
)
from umbraaudio.analysis.clarity import evidence_kind_for
from umbraaudio.analysis.context_analyzer import ContextAnalyzer
from umbraaudio.analysis.tone_analyzer import ToneAnalyzer
from umbraaudio.config import LLMConfig
logger = logging.getLogger(__name__)
@@ -378,6 +389,7 @@ class LLMAttributor:
confidence_map: dict[int, float] = {}
reasoning_map: dict[int, str] = {}
uncertain_map: dict[int, bool] = {}
evidence_by_idx: dict[int, dict[str, Any]] = {}
SPEECH_VERB_SUFFIXES = (
" says", " says,", " said", " said,", " asks", " asks,",
@@ -426,6 +438,8 @@ class LLMAttributor:
evidence_map = self._build_mechanical_evidence(
scene_utterances, batch, alias_map
)
for ev_idx, ev in evidence_map.items():
evidence_by_idx[ev_idx] = ev
for u in batch:
evidence = evidence_map.get(u["utterance_index"])
if evidence is None:
@@ -514,6 +528,10 @@ class LLMAttributor:
new_u["speaker_name"] = speaker_map[idx]
new_u["attribution_confidence"] = confidence_map.get(idx, 0.5)
new_u["attribution_reasoning"] = reasoning_map.get(idx, "")
ev = evidence_by_idx.get(idx)
if ev:
new_u["attribution_evidence_kind"] = evidence_kind_for(ev)
new_u["attribution_tag_name"] = ev.get("tag")
# Add emotion from tone analysis
scene_id = self._find_scene_for_utterance(chapter_id, idx, scene_contexts)
if scene_id is not None:
@@ -556,7 +574,7 @@ class LLMAttributor:
"confidence": confidence_map.get(idx, 0.5),
"reasoning": reasoning_map.get(idx, ""),
}
for idx in speaker_map.keys()
for idx in speaker_map
]
logger.info(

43
umbraaudio/database.py
View File

@@ -92,8 +92,13 @@ CREATE TABLE IF NOT EXISTS utterances (
pause_after_ms INTEGER,
instruct TEXT,
speaker_name TEXT,
attribution_confidence REAL DEFAULT 1.0,
attribution_confidence REAL,
attribution_reasoning TEXT,
attribution_evidence_kind TEXT,
attribution_tag_name TEXT,
attribution_uncertain INTEGER NOT NULL DEFAULT 0,
needs_review INTEGER NOT NULL DEFAULT 0,
review_reason TEXT,
emotion_confidence REAL DEFAULT 1.0,
user_edited INTEGER NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'pending'
@@ -188,6 +193,17 @@ class Database:
await self._conn.execute(
"ALTER TABLE utterances ADD COLUMN user_edited INTEGER NOT NULL DEFAULT 0"
)
for col, typedef in (
("attribution_evidence_kind", "TEXT"),
("attribution_tag_name", "TEXT"),
("attribution_uncertain", "INTEGER NOT NULL DEFAULT 0"),
("needs_review", "INTEGER NOT NULL DEFAULT 0"),
("review_reason", "TEXT"),
):
if col not in utt_cols:
await self._conn.execute(
f"ALTER TABLE utterances ADD COLUMN {col} {typedef}"
)
cursor = await self._conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name='voice_review_history'"
@@ -571,6 +587,11 @@ class Database:
attribution_reasoning: str | None = None,
emotion: str | None = None,
emotion_confidence: float | None = None,
evidence_kind: str | None = None,
tag_name: str | None = None,
uncertain: int | None = None,
needs_review: int | None = None,
review_reason: str | None = None,
) -> None:
"""Update utterance with LLM attribution results."""
fields = ["speaker_name = ?"]
@@ -592,6 +613,26 @@ class Database:
fields.append("emotion_confidence = ?")
params.append(emotion_confidence)
if evidence_kind is not None:
fields.append("attribution_evidence_kind = ?")
params.append(evidence_kind)
if tag_name is not None:
fields.append("attribution_tag_name = ?")
params.append(tag_name)
if uncertain is not None:
fields.append("attribution_uncertain = ?")
params.append(uncertain)
if needs_review is not None:
fields.append("needs_review = ?")
params.append(needs_review)
if review_reason is not None:
fields.append("review_reason = ?")
params.append(review_reason)
params.append(utterance_id)
await self.conn.execute(
f"UPDATE utterances SET {', '.join(fields)} WHERE id = ?",

92
umbraaudio/web/routes.py
View File

@@ -180,8 +180,13 @@ CREATE TABLE IF NOT EXISTS utterances (
pause_after_ms INTEGER,
instruct TEXT,
speaker_name TEXT,
attribution_confidence REAL DEFAULT 1.0,
attribution_confidence REAL,
attribution_reasoning TEXT,
attribution_evidence_kind TEXT,
attribution_tag_name TEXT,
attribution_uncertain INTEGER NOT NULL DEFAULT 0,
needs_review INTEGER NOT NULL DEFAULT 0,
review_reason TEXT,
qa_accepted INTEGER NOT NULL DEFAULT 0,
qa_accepted_at TEXT,
qa_accepted_sig TEXT,
@@ -267,6 +272,16 @@ def _ensure_utterance_columns(db: sqlite3.Connection) -> None:
db.execute(f"ALTER TABLE utterances ADD COLUMN {col}")
if "user_edited" not in cols:
db.execute("ALTER TABLE utterances ADD COLUMN user_edited INTEGER NOT NULL DEFAULT 0")
for col in (
"attribution_evidence_kind TEXT",
"attribution_tag_name TEXT",
"attribution_uncertain INTEGER NOT NULL DEFAULT 0",
"needs_review INTEGER NOT NULL DEFAULT 0",
"review_reason TEXT",
):
name = col.split()[0]
if name not in cols:
db.execute(f"ALTER TABLE utterances ADD COLUMN {col}")
db.commit()
@@ -4849,10 +4864,42 @@ async def run_full_pipeline(request: Request):
ch.get("title", ""),
ch_id,
)
from umbraaudio.analysis.clarity import (
assess_utterance_clarity,
)
try:
alias_map: dict[str, str] = {}
with open(speakers_path) as f:
for s in json.load(f).get("speakers", []):
primary = s.get("primary_name", "")
if primary:
alias_map[primary.lower()] = primary
for a in s.get("aliases", []):
if a:
alias_map[a.lower()] = primary
attributed = assess_utterance_clarity(
attributed, ch_id, alias_map
)
except Exception:
pass
for u in attributed:
db.execute(
"UPDATE utterances SET speaker_name=? WHERE id=?",
(u.get("speaker_name", "Narrator"), u["id"]),
"UPDATE utterances SET speaker_name=?, "
"attribution_confidence=?, attribution_reasoning=?, "
"attribution_evidence_kind=?, attribution_tag_name=?, "
"attribution_uncertain=?, needs_review=?, review_reason=? "
"WHERE id=?",
(
u.get("speaker_name", "Narrator"),
u.get("attribution_confidence", 0.5),
u.get("attribution_reasoning", "") or "",
u.get("attribution_evidence_kind"),
u.get("attribution_tag_name"),
1 if u.get("attribution_uncertain") else 0,
1 if u.get("needs_review") else 0,
u.get("review_reason") or "",
u["id"],
),
)
db.commit()
pct = int(65 + (idx / max(total_ch, 1)) * 30)
@@ -5174,10 +5221,45 @@ async def segment_utterances(request: Request):
utt_list = [dict(u) for u in utts]
attributed = attributor.attribute_speakers(utt_list, title, ch_id)
try:
from umbraaudio.analysis.clarity import (
assess_utterance_clarity,
)
alias_map: dict[str, str] = {}
if speakers_path.exists():
with open(speakers_path) as f:
for s in json.load(f).get("speakers", []):
primary = s.get("primary_name", "")
if primary:
alias_map[primary.lower()] = primary
for a in s.get("aliases", []):
if a:
alias_map[a.lower()] = primary
attributed = assess_utterance_clarity(
attributed, ch_id, alias_map
)
except Exception:
pass
for u in attributed:
db2.execute(
"UPDATE utterances SET speaker_name=? WHERE id=?",
(u.get("speaker_name", "Narrator"), u["id"]),
"UPDATE utterances SET speaker_name=?, "
"attribution_confidence=?, attribution_reasoning=?, "
"attribution_evidence_kind=?, attribution_tag_name=?, "
"attribution_uncertain=?, needs_review=?, review_reason=? "
"WHERE id=?",
(
u.get("speaker_name", "Narrator"),
u.get("attribution_confidence", 0.5),
u.get("attribution_reasoning", "") or "",
u.get("attribution_evidence_kind"),
u.get("attribution_tag_name"),
1 if u.get("attribution_uncertain") else 0,
1 if u.get("needs_review") else 0,
u.get("review_reason") or "",
u["id"],
),
)
db2.commit()