2b294d7fa4
The review UI's language dropdown did nothing: every selection kept rendering the English text. Two independent causes, both fixed. 1. The translations were never being read. The pipeline's `utterance_translations` table holds one row in Deal Hunter, and that row stores the *English* text under a `zh-Hans` code. The real translations live in the on-disk sample files under `samples/translations/`. `umbraaudio/translations.py` now resolves both sources and merges them by (chapter_id, utterance_index): the table wins where it holds a real translation, the sample files fill the gaps. A candidate whose text is identical to the source is a placeholder, not a translation, so it is ignored - otherwise the page would show English under a Chinese heading and claim it was translated. 2. The query that read them was broken. `language_routes.py` selected `u.index` and `u.speaker`; the columns are `utterance_index` and `speaker_name`, so every read raised `sqlite3.OperationalError: near "index": syntax error` and `/language-mockup` returned 500. It also opened `<project>/project.db` - a 0-byte decoy - instead of `<project>/.umbra/project.db`, the file that actually holds the data. Also in this commit: * The chapter review page honours `?lang=`, remembers the choice in a cookie (set by a middleware, so it survives on every page), and shows the selected language's *name* in the "has not been translated yet" message. * The page now shows the translation, not the source, and the edit box is bound to the source text so saving cannot overwrite the English with the Chinese. * The stray language selector that sat after `</html>` in base.html is gone; the chapters-list page's dropdown had zero options because the route never passed the language list. * The language tag is wired into generation: `utterance.language_code` (NULL = project source language) reaches `engine.generate()` and the staleness marker, so a retagged line is re-rendered instead of keeping English phonology behind an unchanged text. The marker records the language; a marker written before the tag existed means English, so the ~4,300 existing clips are not invalidated. * `project.source_language` (spec 31 section 13.1) records the book's own language, with a migration for existing project DBs. * A scene-break marker deliberately records no language: it is silence, identical in every language, and tagging it would claim a phonology the audio does not have. Tests: 883 passed / 8 failed - the same 8 pre-existing failures (2 test_pronunciation, 6 voice_profile_manager accent). 26 new tests cover the resolver and the generation wiring. ruff introduces no new findings.
766 lines
27 KiB
Python
766 lines
27 KiB
Python
"""Ingest and query the source-language tag and per-language translations.
|
|
|
|
Spec 31. Two axes, deliberately separate:
|
|
|
|
* **Source** - ``utterances.language_code`` says what language the original text
|
|
is *written in*. A Spanish line in an English book is tagged ``es`` and is
|
|
**never translated**: it is story content. "Bob overhears them speaking in
|
|
German" - the German is payload the reader is meant to read.
|
|
* **Target** - ``utterance_translations`` holds one row per (utterance, target
|
|
language). The ``voiced`` flag lives here, because a line can be voiced in
|
|
English and not in German (three source sentences collapsing into two).
|
|
|
|
The functions here take a plain ``sqlite3.Connection``. Both the web layer
|
|
(raw sqlite3) and the CLI/scripts can call them, and the import logic stays
|
|
testable without standing up the async database.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import re
|
|
import sqlite3
|
|
from dataclasses import dataclass, field
|
|
from datetime import UTC, datetime
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
from umbraaudio import languages as lang
|
|
|
|
if TYPE_CHECKING:
|
|
from pathlib import Path
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Import rows are stored as 'draft' unless the caller says otherwise. A draft
|
|
# is visible and audible but explicitly not approved (spec 30 section 3.2).
|
|
STATUS_DRAFT = "draft"
|
|
STATUS_CONFIRMED = "confirmed"
|
|
|
|
|
|
def _now() -> str:
|
|
return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
|
|
|
|
def _col(row: Any, name: str, index: int) -> Any:
|
|
"""Read a column from a sqlite3.Row or a plain tuple.
|
|
|
|
Callers may not have set ``row_factory``, so both shapes are supported
|
|
rather than assuming one.
|
|
"""
|
|
if row is None:
|
|
return None
|
|
try:
|
|
return row[name]
|
|
except (TypeError, IndexError, KeyError):
|
|
return row[index]
|
|
|
|
|
|
@dataclass
|
|
class ImportReport:
|
|
"""What an import actually did. Counts only - never the text."""
|
|
|
|
accepted: int = 0
|
|
source_tagged: int = 0
|
|
skipped_confirmed: int = 0
|
|
not_voiced: int = 0
|
|
refused_voice: int = 0
|
|
foreign_untouched: int = 0
|
|
unknown_language: list[str] = field(default_factory=list)
|
|
problems: list[str] = field(default_factory=list)
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"accepted": self.accepted,
|
|
"source_tagged": self.source_tagged,
|
|
"skipped_confirmed": self.skipped_confirmed,
|
|
"not_voiced": self.not_voiced,
|
|
"refused_voice": self.refused_voice,
|
|
"foreign_untouched": self.foreign_untouched,
|
|
"unknown_language": sorted(set(self.unknown_language)),
|
|
"problems": self.problems,
|
|
}
|
|
|
|
|
|
def _translations_table_exists(conn: sqlite3.Connection) -> bool:
|
|
"""True when this connection can see the translations table.
|
|
|
|
Read-only callers exist (routes opens ``mode=ro`` in places), so the read
|
|
helpers must not create the table - they check instead, and report "nothing
|
|
translated yet" rather than raising.
|
|
"""
|
|
try:
|
|
row = conn.execute(
|
|
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='utterance_translations'"
|
|
).fetchone()
|
|
except sqlite3.OperationalError:
|
|
return False
|
|
return row is not None
|
|
|
|
|
|
def ensure_tables(conn: sqlite3.Connection) -> None:
|
|
"""Create the translations table if an old project DB lacks it.
|
|
|
|
The column and index additions live in ``database._migrate`` and
|
|
``web.routes._ensure_utterance_columns``; this is the same statement exposed
|
|
for callers that hold a bare connection (scripts, tests).
|
|
"""
|
|
conn.execute(
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS utterance_translations (
|
|
id INTEGER PRIMARY KEY,
|
|
project_id INTEGER NOT NULL REFERENCES project(id),
|
|
utterance_id INTEGER NOT NULL REFERENCES utterances(id),
|
|
language_code TEXT NOT NULL,
|
|
language_name TEXT NOT NULL,
|
|
text TEXT NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'draft',
|
|
origin TEXT NOT NULL DEFAULT 'umbratrans',
|
|
source_ref TEXT,
|
|
delivered_at TEXT,
|
|
confirmed_at TEXT,
|
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
voiced INTEGER NOT NULL DEFAULT 1,
|
|
review_reason TEXT,
|
|
UNIQUE(utterance_id, language_code)
|
|
)
|
|
"""
|
|
)
|
|
conn.execute(
|
|
"CREATE INDEX IF NOT EXISTS idx_utterance_translations_utt "
|
|
"ON utterance_translations(utterance_id)"
|
|
)
|
|
|
|
|
|
def source_language(conn: sqlite3.Connection) -> str:
|
|
"""The project's primary language, defaulting to English.
|
|
|
|
Spec 31 section 13.1 flagged this as an open question; until a column
|
|
exists, the project's source language is 'en', which is what every existing
|
|
row means anyway (NULL on the utterance = this value).
|
|
"""
|
|
try:
|
|
row = conn.execute("SELECT source_language FROM project LIMIT 1").fetchone()
|
|
except sqlite3.OperationalError:
|
|
return "en"
|
|
return lang.normalize(_col(row, "source_language", 0)) or "en"
|
|
|
|
|
|
def set_source_language(
|
|
conn: sqlite3.Connection,
|
|
utterance_id: int,
|
|
code: str | None,
|
|
origin: str = lang.ORIGIN_UMBRATRANS,
|
|
*,
|
|
force: bool = False,
|
|
) -> bool:
|
|
"""Record the language an utterance's text is written in.
|
|
|
|
Returns True when a change was written. ``code`` is normalised; an
|
|
unrecognised language is still stored (spec 31 section 4.3 - the tag is
|
|
allowed, the render is refused) but reported by the caller.
|
|
|
|
An existing ``operator`` tag is not overwritten by a machine identification
|
|
unless ``force`` is set: a human correction outranks a re-delivery of the
|
|
same machine guess.
|
|
"""
|
|
if origin not in lang.ORIGINS:
|
|
raise ValueError(f"unknown origin {origin!r}")
|
|
|
|
row = conn.execute(
|
|
"SELECT language_code, language_origin FROM utterances WHERE id=?",
|
|
(utterance_id,),
|
|
).fetchone()
|
|
if row is None:
|
|
return False
|
|
|
|
current_code = _col(row, "language_code", 0)
|
|
current_origin = _col(row, "language_origin", 1)
|
|
|
|
# An unrecognised language is STORED, not discarded (spec 31 section 4.3:
|
|
# "allow the tag, refuse the render"). Collapsing it to NULL would make the
|
|
# line look untagged, so it would fall back to the primary language and be
|
|
# spoken with the wrong phonology - the silent failure the decision exists
|
|
# to prevent. The raw value is kept so the refusal can name it.
|
|
if code:
|
|
stored: str | None = lang.normalize(code) or str(code).strip().lower()
|
|
else:
|
|
stored = None
|
|
|
|
operator_owns = current_origin == lang.ORIGIN_OPERATOR and origin != lang.ORIGIN_OPERATOR
|
|
if operator_owns and not force:
|
|
return False
|
|
if stored == current_code and origin == current_origin:
|
|
return False
|
|
|
|
conn.execute(
|
|
"UPDATE utterances SET language_code=?, language_origin=? WHERE id=?",
|
|
(stored, origin, utterance_id),
|
|
)
|
|
return True
|
|
|
|
|
|
def foreign_utterance_ids(conn: sqlite3.Connection, primary: str | None = None) -> set[int]:
|
|
"""Utterances whose text is NOT in the project's primary language.
|
|
|
|
These are excluded from translation: they are story content, not something
|
|
to render into another language.
|
|
"""
|
|
primary = lang.normalize(primary) or source_language(conn)
|
|
rows = conn.execute(
|
|
"SELECT id FROM utterances WHERE language_code IS NOT NULL AND language_code <> ?",
|
|
(primary,),
|
|
).fetchall()
|
|
return {_col(r, "id", 0) for r in rows}
|
|
|
|
|
|
def utterances_to_translate(conn: sqlite3.Connection, primary: str | None = None) -> list[int]:
|
|
"""Utterances eligible for translation: source-language ones only.
|
|
|
|
The complement of :func:`foreign_utterance_ids`. A NULL tag means "the
|
|
project's primary language" (spec 31 section 6.1), so untagged rows are
|
|
included - which is every existing utterance until UmbraTrans speaks.
|
|
"""
|
|
primary = lang.normalize(primary) or source_language(conn)
|
|
rows = conn.execute(
|
|
"SELECT id FROM utterances "
|
|
"WHERE language_code IS NULL OR language_code = ? "
|
|
"ORDER BY chapter_id, utterance_index",
|
|
(primary,),
|
|
).fetchall()
|
|
return [_col(r, "id", 0) for r in rows]
|
|
|
|
|
|
def import_translations(
|
|
conn: sqlite3.Connection,
|
|
project_id: int,
|
|
payload: dict[str, Any],
|
|
*,
|
|
origin: str = lang.ORIGIN_UMBRATRANS,
|
|
confirm: bool = False,
|
|
) -> ImportReport:
|
|
"""Store one delivery from UmbraTrans.
|
|
|
|
``payload`` shape (spec 31 section 8)::
|
|
|
|
{"target_language": "de",
|
|
"utterances": [{"utterance_id": 1, "source_language": "en",
|
|
"text": "Hallo", "voiced": true},
|
|
{"utterance_id": 2, "source_language": "es"},
|
|
{"utterance_id": 3, "voiced": false,
|
|
"reason": "absorbed into utterance 2"}]}
|
|
|
|
Three row shapes, all expected:
|
|
|
|
* ``source_language`` only - a language *identification*. This is the shape
|
|
for a foreign line, which is deliberately left untranslated. Counted, not
|
|
complained about.
|
|
* ``text`` (+ optional ``voiced``) - a translation into the target language.
|
|
* ``voiced: false`` with no text - a collapse orphan: the source sentence was
|
|
absorbed into another translated sentence. A row is still written, so the
|
|
gap is visible and reconstructable; deleting it would lose the alignment.
|
|
"""
|
|
ensure_tables(conn)
|
|
report = ImportReport()
|
|
|
|
target = lang.normalize(payload.get("target_language"))
|
|
target_name = lang.language_name(target) if target else None
|
|
accepted_status = STATUS_CONFIRMED if confirm else STATUS_DRAFT
|
|
now = _now()
|
|
|
|
rows = payload.get("utterances") or []
|
|
if not isinstance(rows, list):
|
|
raise ValueError("'utterances' must be a list")
|
|
|
|
for row in rows:
|
|
if not isinstance(row, dict) or "utterance_id" not in row:
|
|
report.problems.append(f"row without utterance_id: {json.dumps(row)[:80]}")
|
|
continue
|
|
|
|
uid = int(row["utterance_id"])
|
|
|
|
# --- source axis: what language the ORIGINAL text is in ------------
|
|
src = row.get("source_language")
|
|
if src:
|
|
if not lang.is_known(src):
|
|
report.unknown_language.append(str(src))
|
|
if set_source_language(conn, uid, src, origin):
|
|
report.source_tagged += 1
|
|
if not lang.is_voiceable(src):
|
|
# Tag honoured, render refused (spec 31 section 4.3).
|
|
report.refused_voice += 1
|
|
|
|
# --- target axis: the translated text -----------------------------
|
|
text = row.get("text")
|
|
voiced = bool(row.get("voiced", True))
|
|
|
|
if not target:
|
|
# No target language on the delivery: identification only.
|
|
if text is not None:
|
|
report.problems.append(
|
|
f"utterance {uid} carries text but the payload has no target_language"
|
|
)
|
|
continue
|
|
|
|
if text is None and voiced:
|
|
# A foreign line, or one with nothing to translate. Untouched.
|
|
report.foreign_untouched += 1
|
|
continue
|
|
|
|
reason = row.get("reason") or row.get("review_reason")
|
|
|
|
existing = conn.execute(
|
|
"SELECT status FROM utterance_translations WHERE utterance_id=? AND language_code=?",
|
|
(uid, target),
|
|
).fetchone()
|
|
if _col(existing, "status", 0) == STATUS_CONFIRMED:
|
|
report.skipped_confirmed += 1
|
|
continue
|
|
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO utterance_translations
|
|
(project_id, utterance_id, language_code, language_name, text,
|
|
status, origin, source_ref, delivered_at, created_at, updated_at,
|
|
voiced, review_reason)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(utterance_id, language_code) DO UPDATE SET
|
|
text=excluded.text,
|
|
status=excluded.status,
|
|
origin=excluded.origin,
|
|
source_ref=excluded.source_ref,
|
|
delivered_at=excluded.delivered_at,
|
|
updated_at=excluded.updated_at,
|
|
voiced=excluded.voiced,
|
|
review_reason=excluded.review_reason
|
|
""",
|
|
(
|
|
project_id,
|
|
uid,
|
|
target,
|
|
target_name or target,
|
|
str(text) if text is not None else "",
|
|
accepted_status,
|
|
origin,
|
|
row.get("source_ref"),
|
|
now,
|
|
now,
|
|
now,
|
|
1 if voiced else 0,
|
|
reason,
|
|
),
|
|
)
|
|
report.accepted += 1
|
|
if not voiced:
|
|
report.not_voiced += 1
|
|
|
|
if not lang.is_voiceable(target):
|
|
# Target language cannot be spoken: stored, but nothing renders.
|
|
report.refused_voice += 1
|
|
|
|
conn.commit()
|
|
return report
|
|
|
|
|
|
def coverage(conn: sqlite3.Connection, language_code: str) -> dict[str, int]:
|
|
"""How much of the book exists in a target language.
|
|
|
|
``voiced`` counts rows that will actually produce audio, so a collapse
|
|
orphan shows as a gap rather than quietly inflating the total.
|
|
|
|
A project with no translations table yet reports "nothing delivered" rather
|
|
than raising: that is a real state (a book that has never been sent), not an
|
|
error.
|
|
"""
|
|
code = lang.normalize(language_code) or language_code
|
|
|
|
def scalar(sql: str, args: tuple = ()) -> int:
|
|
return conn.execute(sql, args).fetchone()[0]
|
|
|
|
total = scalar("SELECT COUNT(*) FROM utterances")
|
|
if not _translations_table_exists(conn):
|
|
return {
|
|
"total": total,
|
|
"delivered": 0,
|
|
"voiced": 0,
|
|
"not_voiced": 0,
|
|
"confirmed": 0,
|
|
"missing": total,
|
|
}
|
|
|
|
delivered = scalar("SELECT COUNT(*) FROM utterance_translations WHERE language_code=?", (code,))
|
|
voiced = scalar(
|
|
"SELECT COUNT(*) FROM utterance_translations WHERE language_code=? AND voiced=1",
|
|
(code,),
|
|
)
|
|
confirmed = scalar(
|
|
"SELECT COUNT(*) FROM utterance_translations WHERE language_code=? AND status=?",
|
|
(code, STATUS_CONFIRMED),
|
|
)
|
|
return {
|
|
"total": total,
|
|
"delivered": delivered,
|
|
"voiced": voiced,
|
|
"not_voiced": delivered - voiced,
|
|
"confirmed": confirmed,
|
|
"missing": max(total - delivered, 0),
|
|
}
|
|
|
|
|
|
def not_voiced_rows(conn: sqlite3.Connection, language_code: str) -> list[dict[str, Any]]:
|
|
"""The collapse orphans for a language - what a reviewer must look at.
|
|
|
|
These are utterances where the target used fewer sentences than the source,
|
|
so a source line has no counterpart. They are *kept*, with a reason, rather
|
|
than deleted: deleting loses the alignment (spec 31 section 6.2).
|
|
"""
|
|
code = lang.normalize(language_code) or language_code
|
|
if not _translations_table_exists(conn):
|
|
return []
|
|
rows = conn.execute(
|
|
"""
|
|
SELECT t.utterance_id, t.review_reason, u.chapter_id, u.utterance_index,
|
|
u.speaker_name, u.text
|
|
FROM utterance_translations t
|
|
JOIN utterances u ON u.id = t.utterance_id
|
|
WHERE t.language_code = ? AND t.voiced = 0
|
|
ORDER BY u.chapter_id, u.utterance_index
|
|
""",
|
|
(code,),
|
|
).fetchall()
|
|
return [
|
|
{
|
|
"utterance_id": _col(r, "utterance_id", 0),
|
|
"review_reason": _col(r, "review_reason", 1),
|
|
"chapter_id": _col(r, "chapter_id", 2),
|
|
"utterance_index": _col(r, "utterance_index", 3),
|
|
"speaker_name": _col(r, "speaker_name", 4),
|
|
"text": _col(r, "text", 5),
|
|
}
|
|
for r in rows
|
|
]
|
|
|
|
|
|
def refused_render_rows(conn: sqlite3.Connection) -> list[dict[str, Any]]:
|
|
"""Utterances tagged in a language Qwen3-TTS cannot speak.
|
|
|
|
Spec 31 section 11 / decision 4.3: the tag is honoured and the render
|
|
refused. This is the list that makes the resulting hole *visible* instead of
|
|
silent. Sorted by chapter so it reads like a worklist.
|
|
"""
|
|
rows = conn.execute(
|
|
"""
|
|
SELECT id, chapter_id, utterance_index, speaker_name, text, language_code
|
|
FROM utterances
|
|
WHERE language_code IS NOT NULL
|
|
ORDER BY chapter_id, utterance_index
|
|
"""
|
|
).fetchall()
|
|
out = []
|
|
for r in rows:
|
|
code = _col(r, "language_code", 5)
|
|
if lang.is_voiceable(code):
|
|
continue
|
|
out.append(
|
|
{
|
|
"utterance_id": _col(r, "id", 0),
|
|
"chapter_id": _col(r, "chapter_id", 1),
|
|
"utterance_index": _col(r, "utterance_index", 2),
|
|
"speaker_name": _col(r, "speaker_name", 3),
|
|
"text": _col(r, "text", 4),
|
|
"language_code": code,
|
|
"reason": lang.refusal_reason(code),
|
|
}
|
|
)
|
|
return out
|
|
|
|
|
|
def effective_language(
|
|
conn: sqlite3.Connection, utterance_id: int, primary: str | None = None
|
|
) -> str:
|
|
"""The language an utterance should be spoken in.
|
|
|
|
NULL means "the project's primary language" (spec 31 section 6.1), so the
|
|
caller gets a concrete code rather than a None to branch on.
|
|
|
|
A tag that is present but unrecognised is returned **as-is**, never replaced
|
|
by the primary language. Falling back would voice a Klingon line as English,
|
|
which is exactly the silent wrongness the refuse-render decision prevents.
|
|
"""
|
|
row = conn.execute(
|
|
"SELECT language_code FROM utterances WHERE id=?", (utterance_id,)
|
|
).fetchone()
|
|
code = _col(row, "language_code", 0)
|
|
if code:
|
|
return lang.normalize(code) or str(code)
|
|
return lang.normalize(primary) or source_language(conn)
|
|
|
|
|
|
def build_export_payload(
|
|
conn: sqlite3.Connection,
|
|
primary: str | None = None,
|
|
*,
|
|
include_foreign: bool = True,
|
|
) -> dict[str, Any]:
|
|
"""The per-utterance payload for UmbraTrans (spec 31 section 7).
|
|
|
|
``language_code`` is sent when already known so a re-delivery does not ask
|
|
UmbraTrans to re-identify a line the operator corrected. It is null on the
|
|
first pass, which is normal.
|
|
|
|
``include_foreign`` defaults to True because UmbraTrans is the *authority*
|
|
on identification: it must see every line to identify it. Set it False to
|
|
send only the lines that need translating.
|
|
"""
|
|
primary = lang.normalize(primary) or source_language(conn)
|
|
foreign = foreign_utterance_ids(conn, primary)
|
|
rows = conn.execute(
|
|
"""
|
|
SELECT u.id, u.chapter_id, c.chapter_index, u.utterance_index,
|
|
u.speaker_name, u.text, u.context_before, u.context_after,
|
|
u.language_code
|
|
FROM utterances u
|
|
LEFT JOIN chapters c ON c.id = u.chapter_id
|
|
ORDER BY u.chapter_id, u.utterance_index
|
|
"""
|
|
).fetchall()
|
|
|
|
utterances = []
|
|
for r in rows:
|
|
uid = _col(r, "id", 0)
|
|
if not include_foreign and uid in foreign:
|
|
continue
|
|
utterances.append(
|
|
{
|
|
"utterance_id": uid,
|
|
"chapter_id": _col(r, "chapter_id", 1),
|
|
"chapter_index": _col(r, "chapter_index", 2),
|
|
"utterance_index": _col(r, "utterance_index", 3),
|
|
"speaker_name": _col(r, "speaker_name", 4),
|
|
"source_text": _col(r, "text", 5),
|
|
"context_before": _col(r, "context_before", 6),
|
|
"context_after": _col(r, "context_after", 7),
|
|
"language_code": _col(r, "language_code", 8),
|
|
}
|
|
)
|
|
|
|
return {
|
|
"source_system": "umbraaudio",
|
|
"book_language": primary,
|
|
"utterance_count": len(utterances),
|
|
"foreign_count": len(foreign),
|
|
"utterances": utterances,
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Resolution: where a translation actually comes from
|
|
# ---------------------------------------------------------------------------
|
|
#
|
|
# Two stores hold translations and they answer different questions:
|
|
#
|
|
# * ``utterance_translations`` - the pipeline's output. What UmbraTrans
|
|
# delivered and what the operator confirmed. Authoritative when it holds a
|
|
# real translation.
|
|
# * ``<project>/samples/translations/*.json`` - the on-disk sample files. These
|
|
# are what ``render_lang_test.py`` rendered the per-language sample audio
|
|
# from, so the text and the audio on disk agree.
|
|
#
|
|
# Today the sample files are the only place the sample chapter's translations
|
|
# exist; the table holds one placeholder row. A resolver that reads only one
|
|
# of the two shows English for a chapter that has a valid Chinese translation,
|
|
# which is the bug this layer exists to prevent.
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class TranslationEntry:
|
|
"""One resolved translation for one utterance in one target language."""
|
|
|
|
text: str
|
|
status: str = STATUS_DRAFT
|
|
origin: str = "sample"
|
|
speaker: str = ""
|
|
type: str = ""
|
|
|
|
@property
|
|
def confirmed(self) -> bool:
|
|
return self.status == STATUS_CONFIRMED
|
|
|
|
|
|
def canonical_code(value: str | None) -> str | None:
|
|
"""Normalise a language code, tolerating script subtags.
|
|
|
|
``lang.normalize`` only knows the canonical codes, so ``'zh-Hans'`` (what
|
|
UmbraTrans writes) and ``'zh_CN'`` both collapse to ``'zh'`` here.
|
|
"""
|
|
if not value:
|
|
return None
|
|
raw = str(value).strip()
|
|
if not raw:
|
|
return None
|
|
direct = lang.normalize(raw)
|
|
if direct:
|
|
return direct
|
|
base = re.split(r"[-_]", raw)[0]
|
|
return lang.normalize(base)
|
|
|
|
|
|
def sample_translations_dir(project_dir: Path) -> Path:
|
|
"""Where the on-disk sample translations live for *project_dir*.
|
|
|
|
``project_dir`` rather than ``data_dir``: ``.umbra`` is gitignored runtime
|
|
state, while the sample translations are reviewable project content that
|
|
travels with the book.
|
|
"""
|
|
return project_dir / "samples" / "translations"
|
|
|
|
|
|
def load_sample_translations(project_dir: Path) -> dict[str, dict[tuple[int, int], str]]:
|
|
"""Read every sample translation file under the project.
|
|
|
|
Returns ``{language_code: {(chapter_id, utterance_index): text}}``. A file
|
|
that cannot be read or parsed is skipped with a warning rather than taking
|
|
the page down - one bad file must not hide the other nine.
|
|
"""
|
|
out: dict[str, dict[tuple[int, int], str]] = {}
|
|
directory = sample_translations_dir(project_dir)
|
|
if not directory.is_dir():
|
|
return out
|
|
for path in sorted(directory.glob("*.json")):
|
|
try:
|
|
doc = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, ValueError):
|
|
logger.warning("translation sample unreadable: %s", path)
|
|
continue
|
|
if not isinstance(doc, dict):
|
|
logger.warning("translation sample is not an object: %s", path)
|
|
continue
|
|
code = canonical_code(doc.get("language_code"))
|
|
if code is None:
|
|
logger.warning("translation sample has no usable language_code: %s", path)
|
|
continue
|
|
chapter_id = doc.get("chapter_id")
|
|
try:
|
|
chapter_id = int(chapter_id)
|
|
except (TypeError, ValueError):
|
|
logger.warning("translation sample has no usable chapter_id: %s", path)
|
|
continue
|
|
bucket = out.setdefault(code, {})
|
|
for entry in doc.get("utterances") or []:
|
|
if not isinstance(entry, dict):
|
|
continue
|
|
index = _as_index(entry.get("index"))
|
|
text = entry.get("text")
|
|
if index is None or not isinstance(text, str) or not text.strip():
|
|
continue
|
|
bucket[(chapter_id, index)] = text
|
|
return out
|
|
|
|
|
|
def _as_index(value: Any) -> int | None:
|
|
"""The sample files store the index as a string ("0"), the DB as an int."""
|
|
try:
|
|
return int(str(value).strip())
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _db_translation_rows(
|
|
conn: sqlite3.Connection,
|
|
) -> list[dict[str, Any]]:
|
|
"""Rows from ``utterance_translations`` joined to their utterance.
|
|
|
|
The join uses ``utterances.utterance_index`` and ``utterances.speaker_name``
|
|
- the column names that actually exist. (An earlier draft of the language
|
|
UI selected ``u.index`` and ``u.speaker``, which do not, so every read
|
|
raised ``sqlite3.OperationalError: near "index": syntax error``.)
|
|
"""
|
|
if not _translations_table_exists(conn):
|
|
return []
|
|
try:
|
|
rows = conn.execute(
|
|
"""
|
|
SELECT u.chapter_id, u.utterance_index, u.text AS source_text,
|
|
t.text AS target_text, t.status, t.origin, u.speaker_name,
|
|
t.language_code
|
|
FROM utterance_translations t
|
|
JOIN utterances u ON u.id = t.utterance_id
|
|
"""
|
|
).fetchall()
|
|
except sqlite3.OperationalError:
|
|
logger.exception("utterance_translations could not be read")
|
|
return []
|
|
|
|
return [
|
|
{
|
|
"chapter_id": _col(r, "chapter_id", 0),
|
|
"utterance_index": _col(r, "utterance_index", 1),
|
|
"source_text": _col(r, "source_text", 2),
|
|
"target_text": _col(r, "target_text", 3),
|
|
"status": _col(r, "status", 4),
|
|
"origin": _col(r, "origin", 5),
|
|
"speaker_name": _col(r, "speaker_name", 6),
|
|
"language_code": _col(r, "language_code", 7),
|
|
}
|
|
for r in rows
|
|
]
|
|
|
|
|
|
def resolved_translations(
|
|
conn: sqlite3.Connection,
|
|
project_dir: Path,
|
|
code: str,
|
|
) -> dict[tuple[int, int], TranslationEntry]:
|
|
"""The translation text to show for *code*, per ``(chapter_id, index)``.
|
|
|
|
The DB wins where it holds a real translation; the on-disk sample files
|
|
fill the gaps. A candidate whose text is identical to the source text is a
|
|
placeholder rather than a translation, so it is ignored - otherwise the
|
|
page would show English under a Chinese heading and claim it was
|
|
translated.
|
|
"""
|
|
target = canonical_code(code) or str(code).strip().lower()
|
|
if not target:
|
|
return {}
|
|
|
|
resolved: dict[tuple[int, int], TranslationEntry] = {}
|
|
|
|
# --- the pipeline's output -------------------------------------------
|
|
for row in _db_translation_rows(conn):
|
|
if canonical_code(row["language_code"]) != target:
|
|
continue
|
|
source_text = row["source_text"] or ""
|
|
text = row["target_text"] or ""
|
|
if not text.strip():
|
|
continue
|
|
if source_text and text.strip() == source_text.strip():
|
|
# The English text stored under a target-language code: a
|
|
# placeholder, not a translation.
|
|
continue
|
|
key = (int(row["chapter_id"]), int(row["utterance_index"]))
|
|
resolved[key] = TranslationEntry(
|
|
text=text,
|
|
status=row["status"] or STATUS_DRAFT,
|
|
origin=row["origin"] or "umbratrans",
|
|
speaker=row["speaker_name"] or "",
|
|
)
|
|
|
|
# --- the on-disk sample files ----------------------------------------
|
|
files = load_sample_translations(project_dir)
|
|
reference = files.get("en", {})
|
|
for key, text in files.get(target, {}).items():
|
|
if key in resolved:
|
|
continue
|
|
source_text = reference.get(key)
|
|
if source_text is not None and source_text.strip() == text.strip():
|
|
# Identical to the source-language file: not translated.
|
|
continue
|
|
resolved[key] = TranslationEntry(
|
|
text=text,
|
|
status=STATUS_DRAFT,
|
|
origin="sample",
|
|
)
|
|
|
|
return resolved
|