Add orphan name-token review UI (spec 17 Phase 3)

Gives the reviewer a single place to resolve every name the manuscript uses as
a speaker that doesn't resolve to a profile, so nothing reaches the translation
export unresolved.

New (separate router module, so the UI never has to merge into the hot
routes.py):
- umbraaudio/web/name_token_routes.py: /name-tokens page, /name-tokens/partial
  (HTMX), POST /api/name-tokens/scan, and per-token assign/create/dismiss, plus
  /api/speakers/merge-by-name. Registered in create_app.
- templates/name_tokens.html + _name_tokens_list.html: token table with kind,
  use count, chapters, evidence, status, and inline Assign / New speaker /
  Dismiss actions. Shows a one-click suggestion for name-like tokens.
- base.html nav: Name Tokens + Review (/exception-review) links.
- name_tokens.py: shared scan_project() + build_alias_map() so the CLI and the
  web scan use identical logic.

Fixed a pre-existing 500 on /exception-review (now linked in the nav): it called
the old Starlette TemplateResponse(name, context) signature, so the context dict
was used as the template name. Switched to (request, name, context) and replaced
the hardcoded project name DealHunter with the active project.

Verified live: /name-tokens 200 with 8 rows; scan API persisted 9 tokens;
assign error paths return 400; dismiss round-tripped through HTTP with the
scan evidence preserved (Phase 2 fix) and was then reverted. All nav pages 200.

Full suite 424 passed; the 8 failures are the pre-existing/parallel-session
voice_profile_manager + test_pronunciation ones.

routes.py was committed with ONLY the two hunks that are mine (name_tokens
schema + the exception_review fix), via a filtered patch. The parallel session's
5 uncommitted helper_url hunks are left untouched in the working tree.
This commit is contained in:
B.D. Murphy
2026-09-25 03:20:59 -07:00
parent 4d900d77a6
commit 3bc6945bb3
8 changed files with 505 additions and 37 deletions

View File

@@ -21,10 +21,10 @@ from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent))
from umbraaudio.analysis.name_tokens import (
build_alias_map,
ensure_name_tokens_table,
merge_tokens,
persist_name_tokens,
scan_utterances,
scan_project,
)
from umbraaudio.config import load_config
@@ -34,16 +34,7 @@ def _build_alias_map(speakers_path: Path) -> dict[str, str]:
return {}
with open(speakers_path) as f:
data = json.load(f)
alias_map: dict[str, str] = {}
for s in data.get("speakers", []):
primary = s.get("primary_name", "")
if not primary:
continue
alias_map[primary.lower()] = primary
for alias in s.get("aliases", []):
if alias:
alias_map[alias.lower()] = primary
return alias_map
return build_alias_map(data.get("speakers", []))
def _project_id(db: sqlite3.Connection) -> int:
@@ -120,7 +111,6 @@ def main() -> int:
print(f"Database not found: {db_path}")
return 1
alias_map = _build_alias_map(cfg.data_dir / "speakers.json")
db = sqlite3.connect(str(db_path))
db.row_factory = sqlite3.Row
@@ -138,30 +128,11 @@ def main() -> int:
# dangling transaction is left open.
db.close()
if args.chapters:
ids = [int(x.strip()) for x in args.chapters.split(",")]
chapters = db.execute(
"SELECT id, title, chapter_index FROM chapters WHERE id IN "
f"({','.join('?' * len(ids))}) ORDER BY chapter_index",
ids,
).fetchall()
else:
chapters = db.execute(
"SELECT id, title, chapter_index FROM chapters ORDER BY chapter_index"
).fetchall()
ids = (
[int(x.strip()) for x in args.chapters.split(",")] if args.chapters else None
)
per_chapter = []
for ch in chapters:
rows = db.execute(
"SELECT id, utterance_index, text, type, speaker_name "
"FROM utterances WHERE chapter_id=? ORDER BY utterance_index",
(ch["id"],),
).fetchall()
utts = [dict(r) for r in rows]
if utts:
per_chapter.append(scan_utterances(utts, ch["id"], alias_map))
tokens = merge_tokens(per_chapter, alias_map)
tokens = scan_project(db, cfg.data_dir, ids)
if args.write:
pid_row = db.execute("SELECT id FROM project LIMIT 1").fetchone()

View File

@@ -28,6 +28,7 @@ import json
import re
import sqlite3
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from umbraaudio.analysis.attribution_qa import (
@@ -422,6 +423,63 @@ def merge_tokens(
# DB persistence (name_tokens table)
# ---------------------------------------------------------------------------
def build_alias_map(speakers: list[dict[str, Any]]) -> dict[str, str]:
"""Build a lowercase alias -> canonical primary_name map from speaker records."""
alias_map: dict[str, str] = {}
for s in speakers:
primary = (s.get("primary_name") or "").strip()
if not primary:
continue
alias_map[primary.lower()] = primary
for alias in s.get("aliases") or []:
if alias and alias.lower() not in alias_map:
alias_map[alias.lower()] = primary
return alias_map
def scan_project(
db: sqlite3.Connection,
data_dir: Path,
chapter_ids: list[int] | None = None,
) -> list[NameToken]:
"""Scan a whole project DB for orphan name tokens.
Loads speakers.json for the alias map, walks every chapter's utterances
(or only ``chapter_ids``), and returns the book-wide merged token list.
Does not write to the DB; call :func:`persist_name_tokens` for that.
"""
speakers_path = data_dir / "speakers.json"
speakers: list[dict[str, Any]] = []
if speakers_path.exists():
with open(speakers_path) as f:
speakers = json.load(f).get("speakers", [])
alias_map = build_alias_map(speakers)
if chapter_ids:
placeholders = ",".join("?" * len(chapter_ids))
chapters = db.execute(
f"SELECT id FROM chapters WHERE id IN ({placeholders}) ORDER BY chapter_index",
chapter_ids,
).fetchall()
else:
chapters = db.execute(
"SELECT id FROM chapters ORDER BY chapter_index"
).fetchall()
per_chapter: list[list[NameToken]] = []
for ch in chapters:
rows = db.execute(
"SELECT id, utterance_index, text, type, speaker_name "
"FROM utterances WHERE chapter_id=? ORDER BY utterance_index",
(ch["id"],),
).fetchall()
utts = [dict(r) for r in rows]
if utts:
per_chapter.append(scan_utterances(utts, ch["id"], alias_map))
return merge_tokens(per_chapter, alias_map)
CREATE_NAME_TOKENS_SQL = """
CREATE TABLE IF NOT EXISTS name_tokens (
id INTEGER PRIMARY KEY,

View File

@@ -28,9 +28,13 @@ def create_app(config: AppConfig) -> FastAPI:
from umbraaudio.config import list_projects
from umbraaudio.pipeline_manager import PipelineManager
from umbraaudio.voices.generation_manager import GenerationManager
from umbraaudio.web.name_token_routes import router as name_token_router
from umbraaudio.web.routes import NoActiveProjectError, router
app.include_router(router)
# Spec 17 Phase 3: orphan name-token review queue lives in its own module so
# the UI never has to be merged into the hot routes.py.
app.include_router(name_token_router)
@app.exception_handler(NoActiveProjectError)
async def _no_active_project_handler(request: Request, exc: NoActiveProjectError):

View File

@@ -0,0 +1,226 @@
"""Web routes for the orphan name-token review queue (spec 17 Phase 3).
Deliberately a SEPARATE router module so the name-token UI never has to be
merged into the large, hot ``routes.py``. ``create_app`` includes this router
alongside the main one.
Routes:
- ``GET /name-tokens`` review page (token table + actions)
- ``GET /name-tokens/partial`` HTMX partial (the token table body)
- ``POST /api/name-tokens/scan`` rescan the project and persist
- ``POST /api/name-tokens/{token}/assign`` assign token -> existing speaker
- ``POST /api/name-tokens/{token}/create`` create a speaker from the token
- ``POST /api/name-tokens/{token}/dismiss`` dismiss (not a character)
- ``POST /api/speakers/merge-by-name`` merge two speakers by primary name
The resolution actions live in ``umbraaudio.analysis.orphan_resolution`` and
are shared with ``scripts/scan_name_tokens.py``.
"""
from __future__ import annotations
import json
import logging
import sqlite3
from typing import Any
from fastapi import APIRouter, Request
from fastapi.responses import HTMLResponse, JSONResponse
from umbraaudio.analysis.name_tokens import (
NameToken,
ensure_name_tokens_table,
persist_name_tokens,
scan_project,
)
from umbraaudio.analysis.orphan_resolution import (
ResolutionError,
assign_token_to_speaker,
create_speaker_from_token,
dismiss_token,
merge_speakers,
)
from umbraaudio.web.routes import (
_get_active_config,
_get_db,
_load_speakers,
templates,
)
logger = logging.getLogger(__name__)
router = APIRouter()
def _project_id(db: sqlite3.Connection) -> int:
row = db.execute("SELECT id FROM project LIMIT 1").fetchone()
if row is None:
raise ResolutionError("No project row found in DB.")
return int(row["id"])
def _row_to_token(row: sqlite3.Row) -> dict[str, Any]:
try:
chapters = json.loads(row["chapters_json"] or "[]")
except (json.JSONDecodeError, TypeError):
chapters = []
try:
evidence = json.loads(row["evidence_json"] or "[]")
except (json.JSONDecodeError, TypeError):
evidence = []
return {
"token": row["token"],
"kind": row["kind"],
"status": row["status"],
"target_speaker": row["target_speaker"],
"occurrences": row["occurrences"],
"chapters": chapters,
"evidence": evidence,
"suggestion": row["suggestion"],
"suggestion_score": row["suggestion_score"],
}
def _load_stored_tokens(db: sqlite3.Connection, project_id: int) -> list[dict[str, Any]]:
ensure_name_tokens_table(db)
rows = db.execute(
"SELECT * FROM name_tokens WHERE project_id=? "
"ORDER BY CASE status WHEN 'unresolved' THEN 0 ELSE 1 END, "
"occurrences DESC, token COLLATE NOCASE",
(project_id,),
).fetchall()
return [_row_to_token(r) for r in rows]
@router.get("/name-tokens", response_class=HTMLResponse)
async def name_tokens_page(request: Request):
config = _get_active_config(request)
db = _get_db(request)
try:
pid = _project_id(db)
tokens = _load_stored_tokens(db, pid)
except ResolutionError:
tokens = []
finally:
db.close()
return templates.TemplateResponse(
request,
"name_tokens.html",
{
"speakers": _load_speakers(request),
"tokens": tokens,
"project_name": config.project_dir.name,
},
)
@router.get("/name-tokens/partial", response_class=HTMLResponse)
async def name_tokens_partial(request: Request):
db = _get_db(request)
try:
tokens = _load_stored_tokens(db, _project_id(db))
except ResolutionError:
tokens = []
finally:
db.close()
return templates.TemplateResponse(
request,
"_name_tokens_list.html",
{
"tokens": tokens,
# The assign dropdown needs the speaker list.
"speakers": _load_speakers(request),
},
)
@router.post("/api/name-tokens/scan")
async def rescan_name_tokens(request: Request):
config = _get_active_config(request)
db = _get_db(request)
try:
pid = _project_id(db)
ensure_name_tokens_table(db)
tokens: list[NameToken] = scan_project(db, config.data_dir)
inserted, updated = persist_name_tokens(db, pid, tokens)
except ResolutionError as e:
return JSONResponse({"error": str(e)}, status_code=400)
finally:
db.close()
return JSONResponse({"ok": True, "inserted": inserted, "updated": updated})
@router.post("/api/name-tokens/{token:path}/assign")
async def assign_name_token(request: Request, token: str):
body = await request.json()
speaker = (body.get("speaker") or "").strip()
if not speaker:
return JSONResponse({"error": "speaker required"}, status_code=400)
config = _get_active_config(request)
db = _get_db(request)
try:
result = assign_token_to_speaker(
db, config.data_dir, _project_id(db), token, speaker
)
except ResolutionError as e:
return JSONResponse({"error": str(e)}, status_code=400)
finally:
db.close()
return JSONResponse(result)
@router.post("/api/name-tokens/{token:path}/create")
async def create_from_name_token(request: Request, token: str):
try:
body = await request.json()
except Exception:
body = {}
description = (body.get("description") or "").strip()
config = _get_active_config(request)
db = _get_db(request)
try:
result = create_speaker_from_token(
db, config.data_dir, _project_id(db), token, description=description
)
except ResolutionError as e:
return JSONResponse({"error": str(e)}, status_code=400)
finally:
db.close()
return JSONResponse(result)
@router.post("/api/name-tokens/{token:path}/dismiss")
async def dismiss_name_token(request: Request, token: str):
try:
body = await request.json()
except Exception:
body = {}
reason = (body.get("reason") or "").strip()
db = _get_db(request)
try:
result = dismiss_token(db, _project_id(db), token, reason=reason)
except ResolutionError as e:
return JSONResponse({"error": str(e)}, status_code=400)
finally:
db.close()
return JSONResponse(result)
@router.post("/api/speakers/merge-by-name")
async def merge_speakers_by_name(request: Request):
body = await request.json()
source = (body.get("source") or "").strip()
target = (body.get("target") or "").strip()
if not source or not target:
return JSONResponse(
{"error": "source and target are required"}, status_code=400
)
config = _get_active_config(request)
db = _get_db(request)
try:
result = merge_speakers(db, config.data_dir, source, target)
except ResolutionError as e:
return JSONResponse({"error": str(e)}, status_code=400)
finally:
db.close()
return JSONResponse(result)

View File

@@ -250,6 +250,23 @@ CREATE TABLE IF NOT EXISTS exception_reviews (
CREATE INDEX IF NOT EXISTS idx_exception_reviews_chapter ON exception_reviews(chapter_id);
CREATE INDEX IF NOT EXISTS idx_exception_reviews_utterance ON exception_reviews(utterance_id);
CREATE INDEX IF NOT EXISTS idx_exception_reviews_status ON exception_reviews(status);
CREATE TABLE IF NOT EXISTS name_tokens (
id INTEGER PRIMARY KEY,
project_id INTEGER NOT NULL REFERENCES project(id),
token TEXT NOT NULL,
kind TEXT NOT NULL DEFAULT 'name_tag',
status TEXT NOT NULL DEFAULT 'unresolved',
target_speaker TEXT,
occurrences INTEGER NOT NULL DEFAULT 0,
chapters_json TEXT NOT NULL DEFAULT '[]',
evidence_json TEXT NOT NULL DEFAULT '[]',
suggestion TEXT,
suggestion_score REAL DEFAULT 0.0,
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
UNIQUE(project_id, token)
);
CREATE INDEX IF NOT EXISTS idx_name_tokens_status ON name_tokens(status);
CREATE INDEX IF NOT EXISTS idx_name_tokens_project ON name_tokens(project_id);
CREATE INDEX IF NOT EXISTS idx_utterances_chapter ON utterances(chapter_id);
CREATE INDEX IF NOT EXISTS idx_utterances_character ON utterances(character_id);
CREATE INDEX IF NOT EXISTS idx_generated_audio_utterance ON generated_audio(utterance_id);
@@ -5943,10 +5960,11 @@ async def exception_review_page(request: Request):
total_pages = (total + per_page - 1) // per_page
return templates.TemplateResponse(
request,
"exception_review.html",
{
"request": request,
"project": {"name": "DealHunter"},
"project": {"name": config.project_dir.name},
"exceptions": exceptions,
"chapters": chapters,
"selected_chapter": int(chapter_filter) if chapter_filter else None,

View File

@@ -0,0 +1,86 @@
{% if tokens %}
<table>
<thead>
<tr>
<th>Token</th>
<th>Kind</th>
<th>Uses</th>
<th>Chapters</th>
<th>Evidence</th>
<th>Status</th>
<th>Action</th>
</tr>
</thead>
<tbody>
{% for t in tokens %}
<tr id="nt-row-{{ loop.index }}" {% if t.status != 'unresolved' %}style="opacity:0.45"{% endif %}>
<td>
<strong style="color:#e0e0e0">{{ t.token }}</strong>
{% if t.suggestion %}
<div style="font-size:0.7rem; color:#7dd3fc; margin-top:0.15rem">
↳ likely “{{ t.suggestion }}” ({{ '%.2f'|format(t.suggestion_score) }})
</div>
{% endif %}
</td>
<td>
{% if t.kind == 'name_tag' %}
<span class="badge badge-blue">name tag</span>
{% else %}
<span class="badge badge-yellow">role speaker</span>
{% endif %}
</td>
<td style="color:#94a3b8">{{ t.occurrences }}</td>
<td style="color:#64748b; font-size:0.8rem">
{% if t.chapters %}{{ t.chapters|join(', ') }}{% else %}—{% endif %}
</td>
<td style="max-width:280px">
{% if t.evidence %}
<small style="color:#cbd5e1">“{{ t.evidence[0][:90] }}{% if t.evidence[0]|length > 90 %}…{% endif %}”</small>
{% else %}
<span style="color:#475569">—</span>
{% endif %}
</td>
<td>
{% if t.status == 'unresolved' %}
<span class="badge badge-red">unresolved</span>
{% elif t.status == 'assigned' %}
<span class="badge badge-green" title="Assigned to {{ t.target_speaker }}">assigned</span>
{% elif t.status == 'created' %}
<span class="badge badge-green">new speaker</span>
{% else %}
<span class="badge" style="background:#334155; color:#cbd5e1">dismissed</span>
{% endif %}
</td>
<td style="min-width:280px">
{% if t.status == 'unresolved' %}
<div style="display:flex; gap:0.3rem; flex-wrap:wrap; align-items:center">
<select id="nt-assign-{{ loop.index }}"
style="padding:0.25rem; background:#1a1a2e; border:1px solid #333; border-radius:3px; color:#e0e0e0; font-size:0.78rem; max-width:150px">
<option value="">Assign to…</option>
{% for sp in speakers %}
<option value="{{ sp.primary_name }}" {% if sp.primary_name == t.suggestion %}selected{% endif %}>{{ sp.primary_name }}</option>
{% endfor %}
</select>
<button class="btn btn-approve" style="font-size:0.72rem; padding:0.2rem 0.5rem"
onclick="ntAssign({{ loop.index }}, {{ t.token|tojson }})">Assign</button>
<button class="btn btn-primary" style="font-size:0.72rem; padding:0.2rem 0.5rem"
onclick="ntCreate({{ loop.index }}, {{ t.token|tojson }})">New speaker</button>
<button class="btn" style="font-size:0.72rem; padding:0.2rem 0.5rem; background:#475569; color:#e0e0e0"
onclick="ntDismiss({{ loop.index }}, {{ t.token|tojson }})">Dismiss</button>
</div>
{% else %}
<span style="color:#64748b; font-size:0.78rem">
{% if t.target_speaker %}→ {{ t.target_speaker }}{% else %}no action{% endif %}
</span>
{% endif %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<div style="padding:2rem; text-align:center; color:#64748b">
<p style="font-size:1.1rem">No name tokens recorded yet.</p>
<p style="font-size:0.85rem">Run a scan to find names in the manuscript that don’t resolve to a speaker.</p>
</div>
{% endif %}

View File

@@ -87,6 +87,8 @@
<nav class="nav">
<a href="/">Dashboard</a>
<a href="/speakers">Speakers</a>
<a href="/name-tokens">Name Tokens</a>
<a href="/exception-review">Review</a>
<a href="/proper-names">Proper Names</a>
<a href="/chapters-audio">Chapters</a>
<a href="/quality">Quality</a>

View File

@@ -0,0 +1,103 @@
{% extends "base.html" %}
{% block content %}
<div class="card">
<div style="display:flex; justify-content:space-between; align-items:center; margin-bottom:1rem">
<h3 style="margin:0">Unresolved Name Tokens</h3>
<div style="display:flex; gap:0.5rem">
<button class="btn btn-primary" id="nt-scan-btn" onclick="ntScan()">Rescan manuscript</button>
<a href="/speakers" class="btn" style="padding:0.3rem 0.8rem; background:#475569; color:#e0e0e0">← Speakers</a>
</div>
</div>
<div style="margin-bottom:1.25rem; padding:1rem; background:#1e3a5f; border-radius:8px; border-left:3px solid #38bdf8">
<h4 style="color:#38bdf8; margin:0 0 0.5rem 0">What is this?</h4>
<p style="color:#94a3b8; font-size:0.85rem; margin:0 0 0.5rem 0">
Names the manuscript uses as a speaker that don’t resolve to any speaker profile or alias —
missed aliases, misspellings, and unnamed role speakers. Every token must be
<strong style="color:#cbd5e1">assigned</strong>, turned into a <strong style="color:#cbd5e1">new speaker</strong>,
or <strong style="color:#cbd5e1">dismissed</strong>, so the translation export can prove that every name in
the text resolves to a speaker.
</p>
<p style="color:#64748b; font-size:0.78rem; margin:0">
Assigning adds the token as an alias and re-points every utterance that used it — audio marks stale
automatically and regenerates through the normal path.
</p>
</div>
<div id="nt-summary" style="margin-bottom:1rem; color:#94a3b8; font-size:0.85rem"></div>
<div id="nt-table">
{% include "_name_tokens_list.html" %}
</div>
</div>
<script>
function ntTokenize(msg) { return msg || ''; }
function ntRefresh() {
fetch('/name-tokens/partial', {headers: {'HX-Request': 'true'}})
.then(r => r.text())
.then(html => { document.getElementById('nt-table').innerHTML = html; ntSummarize(); });
}
function ntSummarize() {
var unresolved = document.querySelectorAll('#nt-table tbody tr').length;
var el = document.getElementById('nt-summary');
if (el) el.textContent = unresolved ? (unresolved + ' token(s) listed') : 'No tokens listed.';
}
function ntPost(url, payload) {
return fetch(url, {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify(payload || {})
}).then(r => r.json().then(d => ({ok: r.ok, data: d})));
}
function ntAssign(idx, token) {
var sel = document.getElementById('nt-assign-' + idx);
var speaker = sel ? sel.value : '';
if (!speaker) { alert('Pick a speaker to assign “' + token + '” to.'); return; }
ntPost('/api/name-tokens/' + encodeURIComponent(token) + '/assign', {speaker: speaker})
.then(function (res) {
if (!res.ok) { alert('Failed: ' + ntTokenize(res.data.error)); return; }
alert('Assigned “' + token + '” → ' + res.data.target_speaker + ' (' + res.data.utterances_changed + ' utterance(s) re-pointed)');
ntRefresh();
});
}
function ntCreate(idx, token) {
if (!confirm('Create a new speaker named “' + token + '”?\n\nIt will enter the normal voice-design flow on the Speakers tab.')) return;
ntPost('/api/name-tokens/' + encodeURIComponent(token) + '/create', {})
.then(function (res) {
if (!res.ok) { alert('Failed: ' + ntTokenize(res.data.error)); return; }
ntRefresh();
});
}
function ntDismiss(idx, token) {
var reason = prompt('Dismiss “' + token + '” — why is it not a character?\n(e.g. a place, a sound, a one-off mention)', '');
if (reason === null) return;
ntPost('/api/name-tokens/' + encodeURIComponent(token) + '/dismiss', {reason: reason})
.then(function (res) {
if (!res.ok) { alert('Failed: ' + ntTokenize(res.data.error)); return; }
ntRefresh();
});
}
function ntScan() {
var btn = document.getElementById('nt-scan-btn');
btn.disabled = true;
btn.textContent = 'Scanning…';
ntPost('/api/name-tokens/scan', {})
.then(function (res) {
btn.disabled = false;
btn.textContent = 'Rescan manuscript';
if (!res.ok) { alert('Scan failed: ' + ntTokenize(res.data.error)); return; }
ntRefresh();
});
}
document.addEventListener('DOMContentLoaded', ntSummarize);
</script>
{% endblock %}