UmbraAudio/scripts/umbra_node_report.py

#!/usr/bin/env python3
"""Read-only Umbra node report: the facts another machine needs to align with.

Every Umbra machine is a *node* in the Umbra system (spec 21). Before any data
model can be aligned, or any task assigned, the controller needs a factual
description of what a node actually holds: its git state, its projects, the
per-project SQLite shape (row counts plus a schema fingerprint), the JSON state
files, and the audio artifact counts.

This script produces exactly that, and nothing else. It is strictly read-only:
it never opens a database for writing, never mutates a project, and never reads
secret values (only the *names* of ``.env`` keys).

Usage:
    python scripts/umbra_node_report.py                  # human-readable
    python scripts/umbra_node_report.py --json           # machine-readable
    python scripts/umbra_node_report.py --check          # profile vs reality
    python scripts/umbra_node_report.py --project-dir "Last Dwarf"

``--check`` compares ``infra/umbra-node.json`` (the node's declared profile)
against measured reality and exits non-zero on drift. That is the "confirmation"
half of spec 21 §5: a node confirms the profile it advertises. The check is
deliberately limited to *stable* facts -- contract version, project set, and the
per-project schema fingerprints. Volatile facts (git HEAD, which ports happen to
be listening right now) are reported but never treated as drift.
"""
from __future__ import annotations

import argparse
import hashlib
import json
import platform
import socket
import sqlite3
import subprocess
import sys
from datetime import UTC, datetime
from pathlib import Path
from typing import Any

sys.path.insert(0, str(Path(__file__).parent.parent))

from umbraaudio.config import list_projects, slugify_project_name
from umbraaudio.definitions import DEFINITION_SET_VERSION
from umbraaudio.generation.staleness import file_sha256
from umbraaudio.source_units import key_report

REPO_ROOT = Path(__file__).resolve().parent.parent
NODE_PROFILE = REPO_ROOT / "infra" / "umbra-node.json"

# Version of the node-report contract itself. Bump when the shape below changes;
# `infra/umbra-node.json` must declare the same value for `--check` to confirm.
NODE_CONTRACT_VERSION = 1

# Ports that carry meaning for this system. Reported only; never contacted.
MEANINGFUL_PORTS = {
    8420: "umbraaudio web UI (FastAPI/HTMX)",
    8881: "Qwen3-TTS local API",
    18881: "qwen-relay listener (loopback -> LAN GPU box)",
    1234: "LM Studio OpenAI-compatible server",
    5432: "PostgreSQL (control plane, not hosted here)",
}

# Tables whose row counts matter for alignment. Seven of eleven tables are empty
# in every project today (manifest.md §1.2) - that emptiness is itself a fact
# the shared schema must absorb.
TRACKED_TABLES = (
    "project",
    "chapters",
    "utterances",
    "name_tokens",
    "characters",
    "generated_audio",
    "pronunciation_guide",
    "quality_reports",
    "review_decisions",
    "exception_reviews",
    "voice_review_history",
)

# JSON state files, classified per manifest.md §0.
STATE_FILES = {
    "speakers.json": "AUTHORITATIVE",
    "voice_profiles.json": "AUTHORITATIVE",
    "voice_status.json": "AUTHORITATIVE",
    "voice_engines.json": "AUTHORITATIVE",
    "voice_assignments.json": "AUTHORITATIVE (legacy)",
    "phonetic_names.json": "AUTHORITATIVE",
    "m4b_meta.json": "AUTHORITATIVE",
    "analysis_status.json": "CACHE",
    "pipeline_status.json": "CACHE",
    "generation_status.json": "CACHE",
}


def _run(*cmd: str) -> str | None:
    """Return stdout of a command, or None if it is unavailable/fails."""
    try:
        out = subprocess.run(
            cmd, cwd=REPO_ROOT, capture_output=True, text=True, timeout=20, check=False
        )
    except (OSError, subprocess.SubprocessError):
        return None
    return out.stdout.strip() or None


def _git_state() -> dict[str, Any]:
    head = _run("git", "rev-parse", "--short", "HEAD")
    branch = _run("git", "rev-parse", "--abbrev-ref", "HEAD")
    dirty = _run("git", "status", "--porcelain")
    changed = [line[3:] for line in (dirty or "").splitlines() if len(line) > 3]
    return {
        "head": head,
        "branch": branch,
        "clean": dirty == "",
        "uncommitted_files": changed,
    }


def _listening_ports() -> dict[str, Any]:
    """Report which meaningful ports this host listens on (best effort)."""
    listening: list[int] = []
    raw = _run("lsof", "-nP", "-iTCP", "-sTCP:LISTEN")
    if raw:
        for line in raw.splitlines()[1:]:
            for token in line.split():
                if token.count(":") == 1:
                    port_text = token.rsplit(":", 1)[1]
                    if port_text.isdigit():
                        listening.append(int(port_text))
    return {
        "listening": sorted(set(listening)),
        "meaningful": {
            str(port): role
            for port, role in MEANINGFUL_PORTS.items()
            if port in listening
        },
    }


def _env_key_names() -> list[str]:
    """Return ``.env`` KEY NAMES only. Values are never read or reported."""
    env_path = REPO_ROOT / ".env"
    if not env_path.exists():
        return []
    names = set()
    for line in env_path.read_text(encoding="utf-8", errors="replace").splitlines():
        line = line.strip()
        if not line or line.startswith("#") or "=" not in line:
            continue
        names.add(line.split("=", 1)[0].strip())
    return sorted(names)


def _schema_fingerprint(db: sqlite3.Connection) -> tuple[str, int]:
    """Hash the logical schema (tables + columns + declared types).

    ``ALTER TABLE`` reorders columns, so the hash is computed over *sorted*
    tuples: it detects added/dropped/renamed/retyped columns without reporting
    spurious drift from ordering alone.
    """
    rows = db.execute(
        "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"
    ).fetchall()
    parts: list[str] = []
    for (table,) in rows:
        for col in db.execute(f'PRAGMA table_info("{table}")'):
            parts.append(f"{table}.{col[1]}:{col[2]}")
    digest = hashlib.sha256("\n".join(sorted(parts)).encode()).hexdigest()
    return digest[:16], len(parts)


def _project_fingerprint(project_dir: Path) -> dict[str, Any]:
    data_dir = project_dir / ".umbra"
    db_path = data_dir / "project.db"
    report: dict[str, Any] = {
        "project_dir": str(project_dir.relative_to(REPO_ROOT)),
        "db_exists": db_path.exists(),
        "db_size_bytes": db_path.stat().st_size if db_path.exists() else 0,
        "state_files": {},
        "voice_refs": 0,
        "vmatrix_pt": 0,
        "voice_embeddings": 0,
        "utterance_wavs": 0,
        "markers": 0,
        "full_stitches": 0,
        "voice_assets": [],
        "source_unit_keys": {"column_present": False},
        "exports": [],
    }

    for name, classification in STATE_FILES.items():
        path = data_dir / name
        if path.exists():
            report["state_files"][name] = {
                "classification": classification,
                "size_bytes": path.stat().st_size,
            }

    voices = data_dir / "voices"
    if voices.is_dir():
        report["voice_refs"] = sum(1 for _ in voices.rglob("*_ref.wav"))
    prompts = data_dir / "voice_prompts"
    if prompts.is_dir():
        report["vmatrix_pt"] = sum(1 for _ in prompts.rglob("*.pt"))
    embeddings = data_dir / "voice_embeddings"
    if embeddings.is_dir():
        report["voice_embeddings"] = sum(1 for _ in embeddings.rglob("*.npy"))

    audio = data_dir / "chapter_audio"
    if audio.is_dir():
        report["utterance_wavs"] = sum(1 for _ in audio.rglob("utt_*.wav"))
        report["markers"] = sum(1 for _ in audio.rglob("utt_*.wav.txt"))
        report["full_stitches"] = sum(1 for _ in audio.rglob("ch*_full.*"))

    # Voice assets carry their own content hash (controller §5.2: every blob is hashed
    # at creation). Ref WAVs are the voice asset; this is the identity the shared
    # model needs to key a voice on bytes instead of on a name-derived slug.
    voices_dir = data_dir / "voices"
    voice_assets: list[dict[str, Any]] = []
    if voices_dir.is_dir():
        for ref in sorted(voices_dir.rglob("*_ref.wav")):
            voice_assets.append(
                {
                    "slug": ref.stem.removesuffix("_ref"),
                    "object_key": str(ref.relative_to(data_dir)),
                    "sha256": file_sha256(ref),
                    "bytes": ref.stat().st_size,
                }
            )
    report["voice_assets"] = voice_assets

    export_dir = data_dir / "export"
    if export_dir.is_dir():
        report["exports"] = sorted(
            p.name for p in export_dir.iterdir() if p.is_file() and p.suffix in {".m4b", ".mp3"}
        )

    if not db_path.exists():
        report["schema_fingerprint"] = None
        report["tables"] = {}
        return report

    # Read-only URI: never creates, migrates, or writes. The UmbraTrans spec 18
    # §4.2.1 forbids helpers whose side effects touch the source project.
    con = sqlite3.connect(f"file:{db_path.resolve()}?mode=ro", uri=True)
    try:
        fingerprint, column_count = _schema_fingerprint(con)
        report["schema_fingerprint"] = fingerprint
        report["schema_columns"] = column_count
        present = {
            row[0]
            for row in con.execute(
                "SELECT name FROM sqlite_master WHERE type='table'"
            )
        }
        counts: dict[str, int] = {}
        for table in TRACKED_TABLES:
            if table not in present:
                counts[table] = -1  # table absent = a different schema shape
                continue
            try:
                counts[table] = con.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0]
            except sqlite3.Error:
                counts[table] = -2
        report["tables"] = counts
        report["source_unit_keys"] = key_report(con)
    finally:
        con.close()
    return report


def build_report(project_dir: str | None = None) -> dict[str, Any]:
    """Collect the full node report. Read-only."""
    projects = list_projects()
    if project_dir:
        projects = [p for p in projects if p["slug"] == project_dir] or [
            {"slug": slugify_project_name(project_dir), "name": project_dir}
        ]

    fingerprints = [
        _project_fingerprint(REPO_ROOT / p["slug"])
        for p in projects
        if (REPO_ROOT / p["slug"]).is_dir()
    ]

    report: dict[str, Any] = {
        "report_version": NODE_CONTRACT_VERSION,
        "node_contract_version": NODE_CONTRACT_VERSION,
        "generated_at": datetime.now(UTC).isoformat(timespec="seconds"),
        "definition_set_version": DEFINITION_SET_VERSION,
        "node_profile_path": str(NODE_PROFILE.relative_to(REPO_ROOT)),
        "host": {
            "hostname": socket.gethostname(),
            "platform": platform.platform(),
            "python": platform.python_version(),
        },
        "git": _git_state(),
        "listening": _listening_ports(),
        "env_key_names": _env_key_names(),
        "project_count": len(projects),
        "projects": fingerprints,
        "schema_fingerprint_set": sorted(
            {p["schema_fingerprint"] for p in fingerprints if p["schema_fingerprint"]}
        ),
    }

    # Facts the controller must know that are *not* derivable from the repo.
    report["facts"] = {
        "project_base_hard_coded": True,
        "project_base_value": "/Volumes/1TB_APL_Storage/UmbraAudio",
        "authoritative_sql_tables": ["project", "chapters", "utterances", "name_tokens"],
        "empty_sql_tables": [
            "characters",
            "generated_audio",
            "pronunciation_guide",
            "quality_reports",
            "review_decisions",
            "voice_review_history",
            "exception_reviews",
        ],
        "speaker_identity_lives_in": "speakers.json + voice_profiles.json + ref WAVs",
        "audio_record_lives_in": "filesystem + *.wav.txt markers",
        "authentication": "none; active_project is an unsigned browser cookie",
        "shared_database_present_here": False,
        "controller_endpoint": "http://192.168.123.14:8442 (Host: umbratrans.bdmurphy.me)",
        "controller_stage_reached": "0 (reachability verified 2026-10-03)",
        "secret_values_reported": False,
    }
    return report


def check_profile(report: dict[str, Any]) -> list[str]:
    """Compare the declared node profile with measured reality.

    Only stable facts are checked (contract version, project set, schema
    fingerprints). Returns a list of drift messages; empty means confirmed.
    """
    if not NODE_PROFILE.exists():
        return [f"missing node profile: {NODE_PROFILE.relative_to(REPO_ROOT)}"]

    declared = json.loads(NODE_PROFILE.read_text(encoding="utf-8"))
    drift: list[str] = []

    declared_version = declared.get("node_contract_version")
    if declared_version != NODE_CONTRACT_VERSION:
        drift.append(
            f"node_contract_version declared {declared_version} != supported "
            f"{NODE_CONTRACT_VERSION}"
        )

    declared_projects = set(declared.get("projects", []))
    measured_projects = {p["project_dir"] for p in report["projects"]}
    if declared_projects != measured_projects:
        drift.append(
            f"projects declared {sorted(declared_projects)} != measured {sorted(measured_projects)}"
        )

    if declared.get("schema_fingerprints"):
        declared_set = sorted(declared["schema_fingerprints"])
        if declared_set != report["schema_fingerprint_set"]:
            drift.append(
                f"schema fingerprints declared {declared_set} != measured "
                f"{report['schema_fingerprint_set']}"
            )
    return drift


def _print_human(report: dict[str, Any], drift: list[str], checked: bool) -> None:
    host = report["host"]
    git = report["git"]
    print("UMBRA NODE REPORT (read-only)")
    print(f"  generated_at      {report['generated_at']}")
    print(f"  host              {host['hostname']}  {host['platform']}")
    dirty_note = "clean" if git["clean"] else f"{len(git['uncommitted_files'])} uncommitted"
    print(f"  python            {host['python']}")
    print(f"  git               {git['branch']} @ {git['head']} ({dirty_note})")
    print(f"  listening         {report['listening']['meaningful'] or 'no known ports'}")
    print(f"  env keys          {len(report['env_key_names'])} (names only, no values)")
    print(f"  schema fingerprints {report['schema_fingerprint_set']}")

    print("\n  projects:")
    for proj in report["projects"]:
        populated = {t: n for t, n in proj["tables"].items() if n > 0}
        empty = [t for t, n in proj["tables"].items() if n == 0]
        absent = [t for t, n in proj["tables"].items() if n == -1]
        print(f"    - {proj['project_dir']}  db={proj['db_size_bytes']}B "
              f"schema={proj['schema_fingerprint']}")
        print(f"      populated: {populated}")
        if empty:
            print(f"      empty:     {empty}")
        if absent:
            print(f"      absent:    {absent}")
        print(f"      audio: refs={proj['voice_refs']} vmatrix={proj['vmatrix_pt']} "
              f"wavs={proj['utterance_wavs']} markers={proj['markers']} "
              f"stitches={proj['full_stitches']} exports={proj['exports']}")

    print("\n  system facts:")
    for key, value in report["facts"].items():
        print(f"    {key}: {value}")

    if drift:
        print("\n  PROFILE DRIFT (declared profile does not match measured reality):")
        for d in drift:
            print(f"    ! {d}")
    elif checked:
        print("\n  profile check: CONFIRMED (declared profile matches measured reality)")


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--json", action="store_true", help="machine-readable output")
    parser.add_argument("--check", action="store_true",
                        help="compare infra/umbra-node.json to reality; exit 2 on drift")
    parser.add_argument("--project-dir", default=None, help="limit to one project directory")
    args = parser.parse_args()

    report = build_report(project_dir=args.project_dir)
    drift = check_profile(report) if args.check else []

    if args.json:
        report["profile_drift"] = drift
        report["profile_checked"] = args.check
        print(json.dumps(report, indent=2, sort_keys=True))
    else:
        _print_human(report, drift, args.check)
    return 2 if drift else 0


if __name__ == "__main__":
    raise SystemExit(main())