UmbraAudio/tests/test_tts_facts.py

"""Tests for the TTS facts document and the scale totals.

The controller asked for a document whose every claim is checkable against the
running system. These tests are the part of that claim this side can enforce: the
document must not assert a value it cannot back, and a null that means "unknown"
must not be rendered as a fact.

They are cheap and deterministic. Nothing here generates audio, loads a model, or
touches the model directory -- the generator is read-only and these tests keep it
that way.
"""

from __future__ import annotations

import importlib.util
import json
import sys
from pathlib import Path
from typing import Any

import pytest

REPO = Path(__file__).resolve().parent.parent


def _load(name: str) -> Any:
    spec = importlib.util.spec_from_file_location(name, REPO / "scripts" / f"{name}.py")
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    sys.modules[spec.name] = module
    spec.loader.exec_module(module)
    return module


facts = _load("build_tts_facts")
scale = _load("build_scale_totals")


@pytest.fixture(scope="module")
def document() -> dict[str, Any]:
    return facts.build()


@pytest.fixture(scope="module")
def markdown(document: dict[str, Any]) -> str:
    return facts.render_markdown(document)


# --------------------------------------------------------------------------
# the document must not invent numbers
# --------------------------------------------------------------------------


def test_every_unknown_is_marked_unknown_and_has_a_reason(document: dict[str, Any]) -> None:
    """An unknown with no reason is a shrug; the reader cannot act on it."""

    determinism = document["determinism"]
    assert determinism["seeded_reproducible"] == "unknown"
    assert determinism["seeded_reproducible_why"].strip()

    languages = document["languages"]
    assert languages["quality_weaknesses"] == "unknown"
    assert languages["quality_weaknesses_why"].strip()

    models = document["models"]
    assert "note" in models


def test_working_set_is_not_estimated(document: dict[str, Any]) -> None:
    """The 48 GB node question turns on this number, so it must be absent, not guessed."""

    text = json.dumps(document["models"]).lower()
    assert "working_set" not in text
    assert "vram" not in text


def test_determinism_is_reported_as_non_deterministic(document: dict[str, Any]) -> None:
    """Sampling is on by default; claiming reproducibility would be the easy lie."""

    assert document["determinism"]["identical_by_default"] is False
    assert "temperature" in document["determinism"]["why"]


def test_authentication_is_not_claimed_when_absent(document: dict[str, Any]) -> None:
    """A security statement inverted by a missing word is worse than no statement.

    The first version of this asked ``not "Depends(" in source`` and therefore
    reported "required" on a server with no ``Depends`` at all.
    """

    auth = document["runtime_surface"]["authentication"]
    assert auth["requires_authentication"] == bool(auth["mechanisms_found"])
    assert "fastapi_dependency" in auth["mechanisms_searched"]
    if not auth["mechanisms_found"]:
        # The note must say so in words, not merely omit a field.
        assert "no authentication mechanism is present" in auth["note"].lower()


def test_concurrency_is_stated_as_a_number(document: dict[str, Any]) -> None:
    concurrency = document["runtime_surface"]["concurrency"]
    assert concurrency["concurrent_generations"] == 1
    assert concurrency["model_lock"] is True


def test_models_without_weights_are_not_rendered_as_none(document: dict[str, Any]) -> None:
    """A missing weights file is a fact; `None` is not a rendering of it."""

    for model in document["models"].get("models_present", []):
        if model["weights_state"] == "absent":
            assert model["weights_bytes"] is None
            assert model["weights_sha256"] is None
            assert facts._size(model) == "no weights on disk"


def test_output_contract_is_measured_from_a_real_file(document: dict[str, Any]) -> None:
    out = document["output_contract"]["rendered_utterance"]
    assert out.get("codec"), "output contract must name a codec, not omit it"
    assert out.get("measured_from"), "the file it was read from must be named"
    assert out.get("sample_rate_hz"), "sample rate must be stated in the name/unit"


def test_provenance_counts_only_our_commits(document: dict[str, Any]) -> None:
    provenance = document["provenance"]
    assert provenance["commits_ahead_of_upstream"] == len(
        provenance["commits_ahead_detail"])
    assert provenance["upstream_base_commit"]


def test_edge_voice_answer_is_not_inverted(markdown: str) -> None:
    """The rendered sentence must not say the Edge voice IS the render engine."""

    assert "the FALLBACK ENGINE for creating the reference audio" in markdown
    assert "**not** the engine that renders utterances" in markdown
    assert "Utterances are rendered by Qwen" in markdown


def test_markdown_has_no_stray_none(markdown: str) -> None:
    """An f-string that looked up a key in the wrong dict prints `None`.

    That happened once: the consistency note rendered as a bare `None` on its
    own line, which reads as an answer and is not one.
    """

    for line in markdown.splitlines():
        assert line.strip() != "None", f"stray None rendered as a value: {line!r}"


def test_markdown_states_the_hashes_it_claims(document: dict[str, Any],
                                              markdown: str) -> None:
    assert document["content_hash"]["value"] in markdown
    for model in document["models"].get("models_present", []):
        if model["weights_sha256"]:
            assert model["weights_sha256"][:16] in markdown


def test_markdown_names_every_section_the_request_asked_for(markdown: str) -> None:
    for heading in ("## 1. Provenance", "## 2. The model", "## 3. Input contract",
                    "## 4. Output contract", "## 5. Determinism",
                    "## 6. Language coverage", "## 7. Voice identity",
                    "## 8. Failure modes", "## 9. Runtime surface"):
        assert heading in markdown, heading
    assert "What I could not measure" in markdown


# --------------------------------------------------------------------------
# determinism
# --------------------------------------------------------------------------


def test_document_is_reproducible_apart_from_its_timestamp() -> None:
    """A hash that changes on every run cannot answer 'did anything change?'."""

    first = facts.build()
    second = facts.build()
    assert first["content_hash"]["value"] == second["content_hash"]["value"]

    for key in ("generated_at", "content_hash"):
        del first[key]
        del second[key]
    assert first == second


def test_hash_excludes_the_timestamp() -> None:
    """The hash must not move when only the clock does."""

    import copy
    import hashlib

    original = facts.build()
    shifted = copy.deepcopy(original)
    shifted["generated_at"] = "1999-01-01T00:00:00Z"
    payload = json.dumps({k: v for k, v in shifted.items()
                          if k not in ("generated_at", "content_hash")},
                         sort_keys=True, separators=(",", ":"))
    assert hashlib.sha256(payload.encode()).hexdigest() == \
        original["content_hash"]["value"]


# --------------------------------------------------------------------------
# scale totals
# --------------------------------------------------------------------------


def _minimal_project(root: Path) -> Path:
    """A project with just enough to be measurable: a populated database."""

    import sqlite3
    umbra = root / ".umbra"
    umbra.mkdir(parents=True)
    db = umbra / "project.db"
    conn = sqlite3.connect(db)
    conn.executescript(
        "CREATE TABLE chapters (id INTEGER PRIMARY KEY, word_count INTEGER);"
        "CREATE TABLE utterances (id INTEGER PRIMARY KEY, speaker_name TEXT);"
        "INSERT INTO chapters (id, word_count) VALUES (1, 10);"
        "INSERT INTO utterances (id, speaker_name) VALUES (1, 'narrator');"
    )
    conn.commit()
    conn.close()
    return root


def test_scale_totals_separate_audio_from_markers(tmp_path: Path) -> None:
    """'files' must mean one thing. A render and its marker are not the same file."""

    project = _minimal_project(tmp_path / "P")
    umbra = project / ".umbra" / "chapter_audio" / "ch1"
    umbra.mkdir(parents=True)
    import wave as wavemod
    for index in range(2):
        path = umbra / f"utt_{index:04d}.wav"
        with wavemod.open(str(path), "wb") as handle:
            handle.setnchannels(1)
            handle.setsampwidth(2)
            handle.setframerate(24000)
            handle.writeframes(b"\x00\x00" * 24000)
        path.with_suffix(".wav.txt").write_text('{"speaker":"narrator","text":"x"}')

    measured = scale.measure_project(project)
    assert measured is not None
    assert measured["audio"]["audio_files"] == 2
    assert measured["audio"]["marker_files_excluded_from_audio"] == 2
    assert measured["audio"]["audio_duration_ms"] == 2000


def test_scale_totals_excludes_scratch_and_says_so(tmp_path: Path) -> None:
    project = _minimal_project(tmp_path / "P")
    scratch = project / ".umbra" / "chapter_audio" / "cache" / "ch1"
    scratch.mkdir(parents=True)
    (scratch / "utt_0000.wav").write_bytes(b"not audio")
    measured = scale.measure_project(project)
    assert measured["audio"]["scratch_files_excluded"] == 1
    assert measured["audio"]["audio_files"] == 0


def test_scale_totals_report_a_project_with_no_database_as_absent(tmp_path: Path) -> None:
    (tmp_path / "Empty").mkdir()
    assert scale.measure_project(tmp_path / "Empty") is None


def test_scale_schema_fingerprint_is_shape_not_content(tmp_path: Path) -> None:
    import sqlite3
    db = tmp_path / "a.db"
    conn = sqlite3.connect(db)
    conn.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)")
    conn.execute("INSERT INTO t (name) VALUES ('one')")
    first = scale.schema_fingerprint(conn)
    conn.execute("INSERT INTO t (name) VALUES ('two')")
    second = scale.schema_fingerprint(conn)
    conn.execute("ALTER TABLE t ADD COLUMN extra TEXT")
    third = scale.schema_fingerprint(conn)
    conn.close()
    assert first == second, "content must not change the fingerprint"
    assert first != third, "a shape change must change the fingerprint"