"""Tests for the TTS facts document and the scale totals. The controller asked for a document whose every claim is checkable against the running system. These tests are the part of that claim this side can enforce: the document must not assert a value it cannot back, and a null that means "unknown" must not be rendered as a fact. They are cheap and deterministic. Nothing here generates audio, loads a model, or touches the model directory -- the generator is read-only and these tests keep it that way. """ from __future__ import annotations import importlib.util import json import sys from pathlib import Path from typing import Any import pytest REPO = Path(__file__).resolve().parent.parent def _load(name: str) -> Any: spec = importlib.util.spec_from_file_location(name, REPO / "scripts" / f"{name}.py") assert spec and spec.loader module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module facts = _load("build_tts_facts") scale = _load("build_scale_totals") @pytest.fixture(scope="module") def document() -> dict[str, Any]: return facts.build() @pytest.fixture(scope="module") def markdown(document: dict[str, Any]) -> str: return facts.render_markdown(document) # -------------------------------------------------------------------------- # the document must not invent numbers # -------------------------------------------------------------------------- def test_every_unknown_is_marked_unknown_and_has_a_reason(document: dict[str, Any]) -> None: """An unknown with no reason is a shrug; the reader cannot act on it.""" determinism = document["determinism"] assert determinism["seeded_reproducible"] == "unknown" assert determinism["seeded_reproducible_why"].strip() languages = document["languages"] assert languages["quality_weaknesses"] == "unknown" assert languages["quality_weaknesses_why"].strip() models = document["models"] assert "note" in models def test_working_set_is_not_estimated(document: dict[str, Any]) -> None: """The 48 GB node question turns on this number, so it must be absent, not guessed.""" text = json.dumps(document["models"]).lower() assert "working_set" not in text assert "vram" not in text def test_determinism_is_reported_as_non_deterministic(document: dict[str, Any]) -> None: """Sampling is on by default; claiming reproducibility would be the easy lie.""" assert document["determinism"]["identical_by_default"] is False assert "temperature" in document["determinism"]["why"] def test_authentication_is_not_claimed_when_absent(document: dict[str, Any]) -> None: """A security statement inverted by a missing word is worse than no statement. The first version of this asked ``not "Depends(" in source`` and therefore reported "required" on a server with no ``Depends`` at all. """ auth = document["runtime_surface"]["authentication"] assert auth["requires_authentication"] == bool(auth["mechanisms_found"]) assert "fastapi_dependency" in auth["mechanisms_searched"] if not auth["mechanisms_found"]: # The note must say so in words, not merely omit a field. assert "no authentication mechanism is present" in auth["note"].lower() def test_concurrency_is_stated_as_a_number(document: dict[str, Any]) -> None: concurrency = document["runtime_surface"]["concurrency"] assert concurrency["concurrent_generations"] == 1 assert concurrency["model_lock"] is True def test_models_without_weights_are_not_rendered_as_none(document: dict[str, Any]) -> None: """A missing weights file is a fact; `None` is not a rendering of it.""" for model in document["models"].get("models_present", []): if model["weights_state"] == "absent": assert model["weights_bytes"] is None assert model["weights_sha256"] is None assert facts._size(model) == "no weights on disk" def test_output_contract_is_measured_from_a_real_file(document: dict[str, Any]) -> None: out = document["output_contract"]["rendered_utterance"] assert out.get("codec"), "output contract must name a codec, not omit it" assert out.get("measured_from"), "the file it was read from must be named" assert out.get("sample_rate_hz"), "sample rate must be stated in the name/unit" def test_provenance_counts_only_our_commits(document: dict[str, Any]) -> None: provenance = document["provenance"] assert provenance["commits_ahead_of_upstream"] == len( provenance["commits_ahead_detail"]) assert provenance["upstream_base_commit"] def test_edge_voice_answer_is_not_inverted(markdown: str) -> None: """The rendered sentence must not say the Edge voice IS the render engine.""" assert "the FALLBACK ENGINE for creating the reference audio" in markdown assert "**not** the engine that renders utterances" in markdown assert "Utterances are rendered by Qwen" in markdown def test_markdown_has_no_stray_none(markdown: str) -> None: """An f-string that looked up a key in the wrong dict prints `None`. That happened once: the consistency note rendered as a bare `None` on its own line, which reads as an answer and is not one. """ for line in markdown.splitlines(): assert line.strip() != "None", f"stray None rendered as a value: {line!r}" def test_markdown_states_the_hashes_it_claims(document: dict[str, Any], markdown: str) -> None: assert document["content_hash"]["value"] in markdown for model in document["models"].get("models_present", []): if model["weights_sha256"]: assert model["weights_sha256"][:16] in markdown def test_markdown_names_every_section_the_request_asked_for(markdown: str) -> None: for heading in ("## 1. Provenance", "## 2. The model", "## 3. Input contract", "## 4. Output contract", "## 5. Determinism", "## 6. Language coverage", "## 7. Voice identity", "## 8. Failure modes", "## 9. Runtime surface"): assert heading in markdown, heading assert "What I could not measure" in markdown # -------------------------------------------------------------------------- # determinism # -------------------------------------------------------------------------- def test_document_is_reproducible_apart_from_its_timestamp() -> None: """A hash that changes on every run cannot answer 'did anything change?'.""" first = facts.build() second = facts.build() assert first["content_hash"]["value"] == second["content_hash"]["value"] for key in ("generated_at", "content_hash"): del first[key] del second[key] assert first == second def test_hash_excludes_the_timestamp() -> None: """The hash must not move when only the clock does.""" import copy import hashlib original = facts.build() shifted = copy.deepcopy(original) shifted["generated_at"] = "1999-01-01T00:00:00Z" payload = json.dumps({k: v for k, v in shifted.items() if k not in ("generated_at", "content_hash")}, sort_keys=True, separators=(",", ":")) assert hashlib.sha256(payload.encode()).hexdigest() == \ original["content_hash"]["value"] # -------------------------------------------------------------------------- # scale totals # -------------------------------------------------------------------------- def _minimal_project(root: Path) -> Path: """A project with just enough to be measurable: a populated database.""" import sqlite3 umbra = root / ".umbra" umbra.mkdir(parents=True) db = umbra / "project.db" conn = sqlite3.connect(db) conn.executescript( "CREATE TABLE chapters (id INTEGER PRIMARY KEY, word_count INTEGER);" "CREATE TABLE utterances (id INTEGER PRIMARY KEY, speaker_name TEXT);" "INSERT INTO chapters (id, word_count) VALUES (1, 10);" "INSERT INTO utterances (id, speaker_name) VALUES (1, 'narrator');" ) conn.commit() conn.close() return root def test_scale_totals_separate_audio_from_markers(tmp_path: Path) -> None: """'files' must mean one thing. A render and its marker are not the same file.""" project = _minimal_project(tmp_path / "P") umbra = project / ".umbra" / "chapter_audio" / "ch1" umbra.mkdir(parents=True) import wave as wavemod for index in range(2): path = umbra / f"utt_{index:04d}.wav" with wavemod.open(str(path), "wb") as handle: handle.setnchannels(1) handle.setsampwidth(2) handle.setframerate(24000) handle.writeframes(b"\x00\x00" * 24000) path.with_suffix(".wav.txt").write_text('{"speaker":"narrator","text":"x"}') measured = scale.measure_project(project) assert measured is not None assert measured["audio"]["audio_files"] == 2 assert measured["audio"]["marker_files_excluded_from_audio"] == 2 assert measured["audio"]["audio_duration_ms"] == 2000 def test_scale_totals_excludes_scratch_and_says_so(tmp_path: Path) -> None: project = _minimal_project(tmp_path / "P") scratch = project / ".umbra" / "chapter_audio" / "cache" / "ch1" scratch.mkdir(parents=True) (scratch / "utt_0000.wav").write_bytes(b"not audio") measured = scale.measure_project(project) assert measured["audio"]["scratch_files_excluded"] == 1 assert measured["audio"]["audio_files"] == 0 def test_scale_totals_report_a_project_with_no_database_as_absent(tmp_path: Path) -> None: (tmp_path / "Empty").mkdir() assert scale.measure_project(tmp_path / "Empty") is None def test_scale_schema_fingerprint_is_shape_not_content(tmp_path: Path) -> None: import sqlite3 db = tmp_path / "a.db" conn = sqlite3.connect(db) conn.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)") conn.execute("INSERT INTO t (name) VALUES ('one')") first = scale.schema_fingerprint(conn) conn.execute("INSERT INTO t (name) VALUES ('two')") second = scale.schema_fingerprint(conn) conn.execute("ALTER TABLE t ADD COLUMN extra TEXT") third = scale.schema_fingerprint(conn) conn.close() assert first == second, "content must not change the fingerprint" assert first != third, "a shape change must change the fingerprint"