2b294d7fa4
The review UI's language dropdown did nothing: every selection kept rendering the English text. Two independent causes, both fixed. 1. The translations were never being read. The pipeline's `utterance_translations` table holds one row in Deal Hunter, and that row stores the *English* text under a `zh-Hans` code. The real translations live in the on-disk sample files under `samples/translations/`. `umbraaudio/translations.py` now resolves both sources and merges them by (chapter_id, utterance_index): the table wins where it holds a real translation, the sample files fill the gaps. A candidate whose text is identical to the source is a placeholder, not a translation, so it is ignored - otherwise the page would show English under a Chinese heading and claim it was translated. 2. The query that read them was broken. `language_routes.py` selected `u.index` and `u.speaker`; the columns are `utterance_index` and `speaker_name`, so every read raised `sqlite3.OperationalError: near "index": syntax error` and `/language-mockup` returned 500. It also opened `<project>/project.db` - a 0-byte decoy - instead of `<project>/.umbra/project.db`, the file that actually holds the data. Also in this commit: * The chapter review page honours `?lang=`, remembers the choice in a cookie (set by a middleware, so it survives on every page), and shows the selected language's *name* in the "has not been translated yet" message. * The page now shows the translation, not the source, and the edit box is bound to the source text so saving cannot overwrite the English with the Chinese. * The stray language selector that sat after `</html>` in base.html is gone; the chapters-list page's dropdown had zero options because the route never passed the language list. * The language tag is wired into generation: `utterance.language_code` (NULL = project source language) reaches `engine.generate()` and the staleness marker, so a retagged line is re-rendered instead of keeping English phonology behind an unchanged text. The marker records the language; a marker written before the tag existed means English, so the ~4,300 existing clips are not invalidated. * `project.source_language` (spec 31 section 13.1) records the book's own language, with a migration for existing project DBs. * A scene-break marker deliberately records no language: it is silence, identical in every language, and tagging it would claim a phonology the audio does not have. Tests: 883 passed / 8 failed - the same 8 pre-existing failures (2 test_pronunciation, 6 voice_profile_manager accent). 26 new tests cover the resolver and the generation wiring. ruff introduces no new findings.
296 lines
12 KiB
Python
296 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING
|
|
|
|
from umbraaudio.generation.staleness import is_current, marker_path, write_marker
|
|
|
|
if TYPE_CHECKING:
|
|
from pathlib import Path
|
|
|
|
|
|
def test_marker_path(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0003.wav"
|
|
assert marker_path(wav) == tmp_path / "utt_0003.wav.txt"
|
|
|
|
|
|
def test_missing_wav_is_stale(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0000.wav"
|
|
assert is_current("Hello", wav) is False
|
|
|
|
|
|
def test_existing_wav_without_marker_is_stale(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0000.wav"
|
|
wav.write_bytes(b"RIFF....")
|
|
assert is_current("Hello", wav) is False
|
|
|
|
|
|
def test_matching_text_is_current(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0001.wav"
|
|
wav.write_bytes(b"RIFF....")
|
|
write_marker("Google uses Android.", wav, speaker="Narrator")
|
|
assert is_current("Google uses Android.", wav, speaker="Narrator") is True
|
|
|
|
|
|
def test_changed_speaker_is_stale(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0005.wav"
|
|
wav.write_bytes(b"RIFF....")
|
|
write_marker("Same words.", wav, speaker="Kabir")
|
|
assert is_current("Same words.", wav, speaker="Nanny") is False
|
|
|
|
|
|
def test_legacy_plain_text_marker_is_stale(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0006.wav"
|
|
wav.write_bytes(b"RIFF....")
|
|
marker_path(wav).write_text("Same words.", encoding="utf-8")
|
|
assert is_current("Same words.", wav, speaker="Kabir") is False
|
|
|
|
|
|
def test_marker_keeps_text_without_speaker_given(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0007.wav"
|
|
wav.write_bytes(b"RIFF....")
|
|
write_marker("Only text recorded.", wav, speaker="Narrator")
|
|
assert is_current("Only text recorded.", wav) is True
|
|
|
|
|
|
def test_changed_text_is_stale(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0002.wav"
|
|
wav.write_bytes(b"RIFF....")
|
|
write_marker("Old sentence from a previous segmentation.", wav, speaker="Narrator")
|
|
assert is_current("New sentence from re-segmentation.", wav, speaker="Narrator") is False
|
|
|
|
|
|
def test_write_marker_creates_sidecar(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0004.wav"
|
|
write_marker("Some text", wav, speaker="Narrator")
|
|
marker = marker_path(wav).read_text(encoding="utf-8")
|
|
assert marker == '{"speaker": "Narrator", "text": "Some text"}'
|
|
|
|
|
|
def test_voice_fingerprint_in_marker(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0010.wav"
|
|
wav.write_bytes(b"RIFF....")
|
|
ref = tmp_path / "female_ref.wav"
|
|
ref.write_bytes(b"RIFF-female-voice-data")
|
|
write_marker("Hello there.", wav, speaker="Nanny", voice_ref=ref)
|
|
marker = marker_path(wav).read_text(encoding="utf-8")
|
|
assert '"voice_fp":' in marker
|
|
|
|
|
|
def test_voice_fingerprint_mismatch_is_stale(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0011.wav"
|
|
wav.write_bytes(b"RIFF....")
|
|
ref_old = tmp_path / "old_ref.wav"
|
|
ref_old.write_bytes(b"RIFF-old-male-voice")
|
|
ref_new = tmp_path / "new_ref.wav"
|
|
ref_new.write_bytes(b"RIFF-new-female-voice")
|
|
write_marker("Same line.", wav, speaker="Nanny", voice_ref=ref_old)
|
|
assert is_current("Same line.", wav, speaker="Nanny", voice_ref=ref_new) is False
|
|
|
|
|
|
def test_voice_fingerprint_match_is_current(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0012.wav"
|
|
wav.write_bytes(b"RIFF....")
|
|
ref = tmp_path / "same_ref.wav"
|
|
ref.write_bytes(b"RIFF-same-voice")
|
|
write_marker("Same line.", wav, speaker="Nanny", voice_ref=ref)
|
|
assert is_current("Same line.", wav, speaker="Nanny", voice_ref=ref) is True
|
|
|
|
|
|
def test_voice_fingerprint_skipped_when_no_ref(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0013.wav"
|
|
wav.write_bytes(b"RIFF....")
|
|
ref = tmp_path / "the_ref.wav"
|
|
ref.write_bytes(b"RIFF-voice")
|
|
write_marker("With fp.", wav, speaker="Nanny", voice_ref=ref)
|
|
assert is_current("With fp.", wav, speaker="Nanny") is True
|
|
|
|
|
|
def test_voice_fingerprint_stale_when_ref_deleted(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0014.wav"
|
|
wav.write_bytes(b"RIFF....")
|
|
ref = tmp_path / "deleted_ref.wav"
|
|
ref.write_bytes(b"RIFF-data")
|
|
write_marker("Gone ref.", wav, speaker="Nanny", voice_ref=ref)
|
|
ref.unlink()
|
|
assert is_current("Gone ref.", wav, speaker="Nanny", voice_ref=ref) is False
|
|
|
|
|
|
def test_changed_instruct_is_stale(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0015.wav"
|
|
wav.write_bytes(b"RIFF....")
|
|
write_marker("Same words.", wav, speaker="Narrator", instruct="speak softly")
|
|
assert is_current("Same words.", wav, speaker="Narrator", instruct="speak loudly") is False
|
|
|
|
|
|
def test_matching_instruct_is_current(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0016.wav"
|
|
wav.write_bytes(b"RIFF....")
|
|
write_marker("Same words.", wav, speaker="Narrator", instruct="speak softly")
|
|
assert is_current("Same words.", wav, speaker="Narrator", instruct="speak softly") is True
|
|
|
|
|
|
def test_marker_without_instruct_matches_no_instruct_call(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0017.wav"
|
|
wav.write_bytes(b"RIFF....")
|
|
write_marker("Plain line.", wav, speaker="Narrator")
|
|
assert is_current("Plain line.", wav, speaker="Narrator") is True
|
|
|
|
|
|
def test_marker_without_instruct_stale_when_instruct_expected(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0018.wav"
|
|
wav.write_bytes(b"RIFF....")
|
|
write_marker("Plain line.", wav, speaker="Narrator")
|
|
assert is_current("Plain line.", wav, speaker="Narrator", instruct="speak softly") is False
|
|
|
|
|
|
def test_instruct_recorded_in_marker(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0019.wav"
|
|
write_marker("Some text", wav, speaker="Narrator", instruct="speak softly")
|
|
marker = marker_path(wav).read_text(encoding="utf-8")
|
|
assert marker == '{"speaker": "Narrator", "text": "Some text", "instruct": "speak softly"}'
|
|
|
|
|
|
def test_matching_fx_version_is_current(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0020.wav"
|
|
wav.write_bytes(b"RIFF....")
|
|
write_marker("Same words.", wav, speaker="Narrator", fx_version=1)
|
|
assert is_current("Same words.", wav, speaker="Narrator", fx_version=1) is True
|
|
|
|
|
|
def test_fx_version_mismatch_is_stale(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0021.wav"
|
|
wav.write_bytes(b"RIFF....")
|
|
write_marker("Same words.", wav, speaker="Narrator", fx_version=1)
|
|
assert is_current("Same words.", wav, speaker="Narrator", fx_version=2) is False
|
|
|
|
|
|
def test_marker_without_fx_stale_when_fx_expected(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0022.wav"
|
|
wav.write_bytes(b"RIFF....")
|
|
write_marker("Same words.", wav, speaker="Narrator")
|
|
assert is_current("Same words.", wav, speaker="Narrator", fx_version=1) is False
|
|
|
|
|
|
def test_fx_marked_audio_stale_for_plain_calls(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0023.wav"
|
|
wav.write_bytes(b"RIFF....")
|
|
write_marker("Same words.", wav, speaker="Narrator", fx_version=1)
|
|
assert is_current("Same words.", wav, speaker="Narrator") is False
|
|
|
|
|
|
def test_sha256_is_recorded_at_generation(tmp_path: Path) -> None:
|
|
"""Controller §5.2: every blob carries its own content hash from birth."""
|
|
import hashlib
|
|
import json
|
|
|
|
wav = tmp_path / "utt_0030.wav"
|
|
wav.write_bytes(b"RIFF-final-bytes")
|
|
write_marker("Same words.", wav, speaker="Hallr")
|
|
payload = json.loads(marker_path(wav).read_text())
|
|
assert payload["sha256"] == hashlib.sha256(b"RIFF-final-bytes").hexdigest()
|
|
|
|
|
|
def test_sha256_is_the_post_processed_bytes_not_the_raw_output(tmp_path: Path) -> None:
|
|
"""The hash must describe the delivered artifact, after effects are applied."""
|
|
import hashlib
|
|
import json
|
|
|
|
wav = tmp_path / "utt_0031.wav"
|
|
raw = b"RIFF-raw-tts"
|
|
wav.write_bytes(raw)
|
|
# Post-processing rewrites the file, exactly as apply_effects_to_file does.
|
|
wav.write_bytes(raw + b"-EFFECTED")
|
|
write_marker("Same words.", wav, speaker="Xylia", effects=["raspy"])
|
|
payload = json.loads(marker_path(wav).read_text())
|
|
assert payload["sha256"] == hashlib.sha256(raw + b"-EFFECTED").hexdigest()
|
|
assert payload["effects"] == ["raspy"]
|
|
|
|
|
|
def test_source_unit_key_is_recorded_so_audio_names_its_source_unit(tmp_path: Path) -> None:
|
|
import json
|
|
|
|
wav = tmp_path / "utt_0032.wav"
|
|
wav.write_bytes(b"RIFF")
|
|
write_marker("Same words.", wav, speaker="Hallr", source_unit_key="abc123")
|
|
payload = json.loads(marker_path(wav).read_text())
|
|
assert payload["source_unit_key"] == "abc123"
|
|
|
|
|
|
def test_sha256_verification_is_opt_in_and_detects_tampering(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0033.wav"
|
|
wav.write_bytes(b"RIFF-original")
|
|
write_marker("Same words.", wav, speaker="Narrator")
|
|
# Not verified by default: hashing every file on every validation pass is too slow.
|
|
assert is_current("Same words.", wav, speaker="Narrator") is True
|
|
assert is_current("Same words.", wav, speaker="Narrator", verify_sha256=True) is True
|
|
wav.write_bytes(b"RIFF-tampered")
|
|
assert is_current("Same words.", wav, speaker="Narrator") is True
|
|
assert is_current("Same words.", wav, speaker="Narrator", verify_sha256=True) is False
|
|
|
|
|
|
def test_language_recorded_in_marker(tmp_path: Path) -> None:
|
|
import json
|
|
|
|
wav = tmp_path / "utt_0040.wav"
|
|
wav.write_bytes(b"RIFF")
|
|
write_marker("Hola, amigo.", wav, speaker="Kabir", language="es")
|
|
payload = json.loads(marker_path(wav).read_text())
|
|
assert payload["language"] == "es"
|
|
|
|
|
|
def test_omitting_language_keeps_markers_byte_identical(tmp_path: Path) -> None:
|
|
"""Existing callers pass no language; their markers must not change shape."""
|
|
import json
|
|
|
|
wav = tmp_path / "utt_0041.wav"
|
|
write_marker("Some text", wav, speaker="Narrator")
|
|
payload = json.loads(marker_path(wav).read_text())
|
|
assert "language" not in payload
|
|
|
|
|
|
def test_changed_language_is_stale(tmp_path: Path) -> None:
|
|
"""A retagged line must re-render even when its text is untouched.
|
|
|
|
The audio is wrong (English phonology) while the text is identical, so only
|
|
the language in the marker can invalidate it (spec 31 section 10).
|
|
"""
|
|
wav = tmp_path / "utt_0042.wav"
|
|
wav.write_bytes(b"RIFF")
|
|
write_marker("Bonjour, ami.", wav, speaker="Kabir", language="en")
|
|
assert is_current("Bonjour, ami.", wav, speaker="Kabir", language="en") is True
|
|
assert is_current("Bonjour, ami.", wav, speaker="Kabir", language="fr") is False
|
|
|
|
|
|
def test_matching_language_is_current(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0043.wav"
|
|
wav.write_bytes(b"RIFF")
|
|
write_marker("Hola, amigo.", wav, speaker="Kabir", language="es")
|
|
assert is_current("Hola, amigo.", wav, speaker="Kabir", language="es") is True
|
|
|
|
|
|
def test_legacy_english_marker_stays_current(tmp_path: Path) -> None:
|
|
"""Markers written before the language tag exist stay valid.
|
|
|
|
Absence means English, so ~4,300 existing clips are not invalidated by this
|
|
feature.
|
|
"""
|
|
wav = tmp_path / "utt_0044.wav"
|
|
wav.write_bytes(b"RIFF")
|
|
write_marker("Hello there.", wav, speaker="Narrator")
|
|
assert is_current("Hello there.", wav, speaker="Narrator", language="en") is True
|
|
|
|
|
|
def test_legacy_marker_is_stale_when_a_foreign_language_is_expected(tmp_path: Path) -> None:
|
|
wav = tmp_path / "utt_0045.wav"
|
|
wav.write_bytes(b"RIFF")
|
|
write_marker("Hola, amigo.", wav, speaker="Kabir")
|
|
assert is_current("Hola, amigo.", wav, speaker="Kabir", language="es") is False
|
|
|
|
|
|
def test_language_not_compared_when_caller_omits_it(tmp_path: Path) -> None:
|
|
"""Callers not yet language-aware keep working exactly as before."""
|
|
wav = tmp_path / "utt_0046.wav"
|
|
wav.write_bytes(b"RIFF")
|
|
write_marker("Same words.", wav, speaker="Narrator", language="es")
|
|
assert is_current("Same words.", wav, speaker="Narrator") is True
|