diff --git a/.env b/.env index 9917518..2281e87 100644 --- a/.env +++ b/.env @@ -14,6 +14,9 @@ UMBRA_TTS__RETRY_DELAY=2.0 UMBRA_TTS__VOICEDESIGN_MODEL=/Volumes/1TB_APL_Storage/UmbraAudio/models/Qwen3-TTS-12Hz-1.7B-VoiceDesign UMBRA_TTS__BASE_MODEL=/Volumes/1TB_APL_Storage/UmbraAudio/models/Qwen3-TTS-12Hz-1.7B-Base UMBRA_TTS__QWEN_TTS_URL=http://localhost:8881 +# Optional GPU accelerator: relay 127.0.0.1:18881 -> 192.168.123.14:8881. +# Helper is tried first when healthy; local server above is the required fallback. +UMBRA_TTS__QWEN_TTS_HELPER_URL=http://127.0.0.1:18881 # --- LLM --- UMBRA_LLM__PROVIDER=auto diff --git a/.gitignore b/.gitignore index 20a27c6..83f4d96 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,6 @@ umbraaudio_package.zip # One-off voice A/B demo (large binary WAVs, not wired into the app) /umbraaudio/web/static/samples/ + +# SQLite project databases are runtime state, never source +*.db diff --git a/.gitmodules b/.gitmodules index 0da0689..96d1857 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "Qwen3-TTS"] path = Qwen3-TTS - url = https://github.com/QwenLM/Qwen3-TTS.git + url = ssh://git@mini-eq:2222/umbragit/Qwen3-TTS.git diff --git a/Issues.txt b/Issues.txt index 76f998d..b4628fc 100644 --- a/Issues.txt +++ b/Issues.txt @@ -1,3 +1,13 @@ Issues -Split of book - identify chapters, paragraphs and utterances. Split utterances by ".?!" the only three valid end of sentence punctuation. \ No newline at end of file +Split of book - identify chapters, paragraphs and utterances. Split utterances by ".?!" the only three valid end of sentence punctuation. + + +connect the two. Get audio data on the new machine in the right format. + + +Idea: +Android app to pull pictures from phone and from cloud and send to NAS, formatted into groups. Allows removal from phone and cloud to free up space. +- App becomes primary photo backup (not giving your stuff to google) - the place to find your stuff. +- Search by location taken, date, contents of picture, identify who +- Have system setup to display the pictures on the nas via web on in the app. diff --git a/backup/Starthere.sh b/backup/Starthere.sh new file mode 100755 index 0000000..03e752f --- /dev/null +++ b/backup/Starthere.sh @@ -0,0 +1,290 @@ +#!/usr/bin/env bash +# Restore an UmbraAudio full backup and rebuild a portable runtime. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +ARCHIVE="" +TARGET="${UMBRA_RESTORE_TARGET:-$HOME/UmbraAudio-restore}" +GLOBAL_TARGET="${UMBRA_GLOBAL_STATE_DIR:-$HOME/.umbra}" +PYTHON_OVERRIDE="${PYTHON_BIN:-}" +SKIP_DEPS=0 +SKIP_SUBMODULE_UPDATE=0 +SKIP_GLOBAL_STATE=0 +INSTALL_SYSTEM_DEPS=0 +DOWNLOAD_MODELS=0 +WITH_TTS_SERVER=0 +KEEP_ARCHIVED_ENV=0 +RUN_TESTS=0 +FORCE=0 +APPLY_PORTABILITY_PATCH=1 +OLD_ROOT="" + +usage() { + cat <<'EOF' +Usage: Starthere.sh [options] + +Restore the newest UmbraAudio backup in this directory (or use --archive), +verify its SHA-256 manifest, restore the project and global voice state, and +rebuild a portable Python environment. + +Options: + --target DIR Restore destination (default: $HOME/UmbraAudio-restore) + --global-state-dir DIR Restored global registry/library (default: $HOME/.umbra) + --archive FILE Payload archive to restore + --python BIN Explicit Python 3.11+ interpreter + --skip-deps Do not rebuild Python dependencies + --skip-submodule Do not initialize/update git submodules + --skip-global-state Do not restore ~/.umbra voice registry/library + --install-system-deps Try to install git/ffmpeg/Python with brew or apt-get + --download-models Download missing Qwen model weights + --with-tts-server Also install tts_server/requirements.txt + --keep-archived-env Keep archived .venv/.python instead of rebuilding + --no-portability-patch Do not rewrite machine-local paths in restored copy + --run-tests Run pytest after setup + --force Permit non-empty project/global targets + -h, --help Show this help + +The archive may contain .env, manuscripts, models, and audio. Review it before +sharing the restored directory. Network database configuration is intentionally +not changed by this script. +EOF +} + +while (($#)); do + case "$1" in + --target) TARGET="${2:?--target requires a directory}"; shift 2 ;; + --global-state-dir) GLOBAL_TARGET="${2:?--global-state-dir requires a directory}"; shift 2 ;; + --archive) ARCHIVE="${2:?--archive requires a file}"; shift 2 ;; + --python) PYTHON_OVERRIDE="${2:?--python requires an interpreter}"; shift 2 ;; + --skip-deps) SKIP_DEPS=1; shift ;; + --skip-submodule) SKIP_SUBMODULE_UPDATE=1; shift ;; + --skip-global-state) SKIP_GLOBAL_STATE=1; shift ;; + --install-system-deps) INSTALL_SYSTEM_DEPS=1; shift ;; + --download-models) DOWNLOAD_MODELS=1; shift ;; + --with-tts-server) WITH_TTS_SERVER=1; shift ;; + --keep-archived-env) KEEP_ARCHIVED_ENV=1; shift ;; + --no-portability-patch) APPLY_PORTABILITY_PATCH=0; shift ;; + --run-tests) RUN_TESTS=1; shift ;; + --force) FORCE=1; shift ;; + -h|--help) usage; exit 0 ;; + *) printf 'Unknown option: %s\n' "$1" >&2; usage >&2; exit 2 ;; + esac +done + +if [[ -z "$ARCHIVE" ]]; then + ARCHIVE="$(find "$SCRIPT_DIR" -maxdepth 1 -type f -name 'UmbraAudio-*.tar' -print | sort | tail -n 1)" +fi +[[ -n "$ARCHIVE" && -f "$ARCHIVE" && ! -L "$ARCHIVE" ]] || { + printf 'No regular payload archive found. Use --archive or place UmbraAudio-*.tar beside this script.\n' >&2 + exit 1 +} +[[ -f "$SCRIPT_DIR/SHA256SUMS" ]] || { + printf 'Missing SHA256SUMS beside Starthere.sh; refusing an unverified restore.\n' >&2 + exit 1 +} +[[ -f "$SCRIPT_DIR/restore_support.py" ]] || { + printf 'Missing restore_support.py beside Starthere.sh; use a backup made with the updated tooling.\n' >&2 + exit 1 +} +ARCHIVE_DIR="$(cd -- "$(dirname -- "$ARCHIVE")" && pwd -P)" +[[ "$ARCHIVE_DIR" == "$SCRIPT_DIR" ]] || { + printf 'Archive must be located beside the selected backup metadata: %s\n' "$ARCHIVE_DIR" >&2 + exit 1 +} +ARCHIVE_BASENAME="$(basename -- "$ARCHIVE")" +if ! awk -v name="$ARCHIVE_BASENAME" '$2 == name { found = 1 } END { exit(found ? 0 : 1) }' "$SCRIPT_DIR/SHA256SUMS"; then + printf 'Selected archive is not listed in SHA256SUMS: %s\n' "$ARCHIVE_BASENAME" >&2 + exit 1 +fi +if [[ -f "$SCRIPT_DIR/BACKUP-INFO.txt" ]]; then + info_archive="$(sed -n 's/^archive_name=//p' "$SCRIPT_DIR/BACKUP-INFO.txt" | head -n 1)" + if [[ -n "$info_archive" && "$info_archive" != "$ARCHIVE_BASENAME" ]]; then + printf 'Selected archive does not match BACKUP-INFO.txt: %s vs %s\n' "$ARCHIVE_BASENAME" "$info_archive" >&2 + exit 1 + fi +fi + +for tool in tar; do + command -v "$tool" >/dev/null 2>&1 || { printf 'Required tool is missing: %s\n' "$tool" >&2; exit 1; } +done + +printf 'Backup directory: %s\n' "$SCRIPT_DIR" +printf 'Payload archive: %s\n' "$ARCHIVE" +printf 'Restore target: %s\n' "$TARGET" +printf 'Global state: %s\n' "$GLOBAL_TARGET" + +if command -v shasum >/dev/null 2>&1; then + (cd "$SCRIPT_DIR" && shasum -a 256 -c SHA256SUMS) +elif command -v sha256sum >/dev/null 2>&1; then + (cd "$SCRIPT_DIR" && sha256sum -c SHA256SUMS) +else + printf 'Neither shasum nor sha256sum is available.\n' >&2 + exit 1 +fi + +[[ ! -L "$TARGET" ]] || { printf 'Refusing symlink restore target: %s\n' "$TARGET" >&2; exit 1; } +if [[ -e "$TARGET" && -n "$(find "$TARGET" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)" && "$FORCE" -ne 1 ]]; then + printf 'Restore target is not empty: %s (use --force only deliberately)\n' "$TARGET" >&2 + exit 1 +fi +TARGET_PARENT_INPUT="$(dirname -- "$TARGET")" +mkdir -p "$TARGET_PARENT_INPUT" +TARGET_PARENT="$(cd -- "$TARGET_PARENT_INPUT" && pwd -P)" +TARGET_NAME="$(basename -- "$TARGET")" +TARGET_ABS="$TARGET_PARENT/$TARGET_NAME" +GLOBAL_PARENT_INPUT="$(dirname -- "$GLOBAL_TARGET")" +mkdir -p "$GLOBAL_PARENT_INPUT" +GLOBAL_TARGET_ABS="$(cd -- "$GLOBAL_PARENT_INPUT" && pwd -P)/$(basename -- "$GLOBAL_TARGET")" +TEMP_TARGET="$(mktemp -d "$TARGET_PARENT/.${TARGET_NAME}.restore.XXXXXX")" +OLD_TARGET="" +cleanup() { + if [[ -d "$TEMP_TARGET" ]]; then + rm -rf "$TEMP_TARGET" + fi +} +trap cleanup EXIT + +printf 'Extracting into an isolated staging directory...\n' +tar -xpf "$ARCHIVE" -C "$TEMP_TARGET" --strip-components 1 +ROOT="$TEMP_TARGET" +[[ -f "$ROOT/pyproject.toml" && -d "$ROOT/umbraaudio" ]] || { + printf 'Archive did not contain the expected UmbraAudio root.\n' >&2 + exit 1 +} + +if [[ "$KEEP_ARCHIVED_ENV" -ne 1 ]]; then + printf 'Discarding archived platform-specific virtualenv/toolchain...\n' + rm -rf "$ROOT/.venv" "$ROOT/.python" +fi + +install_system_dependencies() { + [[ "$INSTALL_SYSTEM_DEPS" -eq 1 ]] || return 0 + if command -v brew >/dev/null 2>&1; then + brew install git ffmpeg + brew install python@3.12 || brew install python + elif command -v apt-get >/dev/null 2>&1 && command -v sudo >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y git ffmpeg python3 python3-venv + else + printf 'Install git, ffmpeg, Python 3.11+, and venv support manually.\n' >&2 + exit 1 + fi +} +install_system_dependencies + +python_supports_project() { + "$1" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 11) else 1)' \ + >/dev/null 2>&1 +} + +PYTHON_BIN="" +if [[ -n "$PYTHON_OVERRIDE" ]]; then + python_supports_project "$PYTHON_OVERRIDE" || { + printf 'Requested Python is unavailable or older than 3.11: %s\n' "$PYTHON_OVERRIDE" >&2 + exit 1 + } + PYTHON_BIN="$PYTHON_OVERRIDE" +else + for candidate in python3.12 python3.11 python3.13 python3; do + if command -v "$candidate" >/dev/null 2>&1 && python_supports_project "$candidate"; then + PYTHON_BIN="$candidate" + break + fi + done +fi +[[ -n "$PYTHON_BIN" ]] || { + printf 'Python 3.11+ is required. Install it or pass --python.\n' >&2 + exit 1 +} +printf 'Using Python: %s (%s)\n' "$PYTHON_BIN" "$("$PYTHON_BIN" --version 2>&1)" + +if [[ "$SKIP_SUBMODULE_UPDATE" -ne 1 && -f "$ROOT/.gitmodules" ]]; then + printf 'Initializing git submodules...\n' + git -C "$ROOT" submodule update --init --recursive +fi + +if [[ "$SKIP_DEPS" -eq 1 ]]; then + printf 'Skipping dependency installation as requested.\n' +else + UV_BIN="$(command -v uv || true)" + BOOTSTRAP_ENV="" + if [[ -z "$UV_BIN" ]]; then + printf 'Bootstrapping uv 0.11.7 in an isolated environment...\n' + BOOTSTRAP_ENV="$ROOT/.uv-bootstrap" + "$PYTHON_BIN" -m venv "$BOOTSTRAP_ENV" + "$BOOTSTRAP_ENV/bin/python" -m pip install --upgrade pip + "$BOOTSTRAP_ENV/bin/python" -m pip install 'uv==0.11.7' + UV_BIN="$BOOTSTRAP_ENV/bin/uv" + fi + printf 'Rebuilding the locked environment with uv...\n' + UV_SYNC_ARGS=(sync --locked --python "$PYTHON_BIN") + if [[ "$RUN_TESTS" -eq 1 ]]; then + UV_SYNC_ARGS+=(--extra dev) + fi + (cd "$ROOT" && "$UV_BIN" "${UV_SYNC_ARGS[@]}") + if [[ "$WITH_TTS_SERVER" -eq 1 && -f "$ROOT/tts_server/requirements.txt" ]]; then + (cd "$ROOT" && "$UV_BIN" pip install --python .venv/bin/python -r tts_server/requirements.txt) + fi + if [[ -n "$BOOTSTRAP_ENV" ]]; then + rm -rf "$BOOTSTRAP_ENV" + fi +fi + +if [[ "$DOWNLOAD_MODELS" -eq 1 && ! -f "$ROOT/models/Qwen3-TTS-12Hz-1.7B-Base/model.safetensors" ]]; then + [[ -f "$ROOT/scripts/download_models.sh" ]] || { + printf 'Model download script is missing.\n' >&2 + exit 1 + } + (cd "$ROOT" && bash scripts/download_models.sh) +fi + +if [[ -f "$SCRIPT_DIR/BACKUP-INFO.txt" ]]; then + OLD_ROOT="$(sed -n 's/^source_path=//p' "$SCRIPT_DIR/BACKUP-INFO.txt" | head -n 1)" +fi +if [[ "$APPLY_PORTABILITY_PATCH" -eq 1 ]]; then + "$PYTHON_BIN" "$SCRIPT_DIR/restore_support.py" patch-paths \ + --root "$ROOT" --old-root "$OLD_ROOT" --new-root "$TARGET_ABS" \ + --global-state-dir "$GLOBAL_TARGET_ABS" +fi + +if [[ -f "$ROOT/.env" ]]; then + chmod 600 "$ROOT/.env" +fi + +printf 'Validating restored project tree and SQLite databases...\n' +"$PYTHON_BIN" "$SCRIPT_DIR/restore_support.py" validate-tree \ + --root "$ROOT" --old-root "$OLD_ROOT" + +# Commit the project tree only after extraction, dependency setup, path migration, +# and validation all succeed. An existing target is retained as .old on --force. +if [[ -e "$TARGET" ]]; then + OLD_TARGET="$TARGET_PARENT/.${TARGET_NAME}.old.$(date +%Y%m%d%H%M%S)" + mv "$TARGET" "$OLD_TARGET" +fi +mv "$TEMP_TARGET" "$TARGET" +trap - EXIT +ROOT="$TARGET_ABS" + +if [[ "$SKIP_GLOBAL_STATE" -ne 1 && -d "$SCRIPT_DIR/global-state" ]]; then + printf 'Restoring global voice registry/library...\n' + RESTORE_ARGS=(restore-global --backup-dir "$SCRIPT_DIR" --target "$GLOBAL_TARGET_ABS") + [[ "$FORCE" -eq 1 ]] && RESTORE_ARGS+=(--force) + if [[ -n "$OLD_ROOT" ]]; then + RESTORE_ARGS+=(--old-project-root "$OLD_ROOT" --new-project-root "$ROOT") + fi + "$PYTHON_BIN" "$SCRIPT_DIR/restore_support.py" "${RESTORE_ARGS[@]}" +fi + +if [[ "$RUN_TESTS" -eq 1 ]]; then + [[ -x "$ROOT/.venv/bin/pytest" ]] || { + printf 'pytest is not installed in the rebuilt environment.\n' >&2 + exit 1 + } + (cd "$ROOT" && .venv/bin/pytest -q) +fi + +printf '\nRestore complete.\n' +printf 'Project root: %s\n' "$ROOT" +[[ -z "$OLD_TARGET" ]] || printf 'Previous target retained at: %s\n' "$OLD_TARGET" +printf 'Review .env, global voice paths, and RESTORED-SERVICE-COMMANDS.md before starting services.\n' +printf 'Network database setup is intentionally not performed by Starthere.sh.\n' diff --git a/backup/create_full_backup.sh b/backup/create_full_backup.sh new file mode 100755 index 0000000..cfd209f --- /dev/null +++ b/backup/create_full_backup.sh @@ -0,0 +1,493 @@ +#!/usr/bin/env bash +# Create a complete, restorable UmbraAudio snapshot on a mounted filesystem/NAS. +set -Eeuo pipefail + +SOURCE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)" +DESTINATION="${UMBRA_BACKUP_DEST:-/Volumes/NetBackup/AudioBackup}" +GLOBAL_ROOT="${UMBRA_GLOBAL_STATE_ROOT:-$HOME/.umbra}" +QUIESCE=1 +BEST_EFFORT=0 +RESTART_SERVICES=1 +ALLOW_NON_SMB=0 + +usage() { + cat <<'EOF' +Usage: create_full_backup.sh [options] + +Create a dated, fail-closed full backup of the current UmbraAudio working tree +and the active global voice registry/library. + +Options: + --destination DIR NAS/filesystem destination (default: /Volumes/NetBackup/AudioBackup) + --global-root DIR Active global state root (default: $HOME/.umbra) + --best-effort Do not quiesce services (not recommended; explicit opt-in) + --quiesce Quiesce services (default) + --no-restart Do not restart services after a --quiesce backup + --allow-non-smb Permit a non-SMB destination for testing + -h, --help Show this help + +The archive includes the entire current tree, including dirty/untracked files, +ignored book projects, generated audio, models, .git, and .env. Global state is +captured with SQLite's backup API and stored beside the archive. +EOF +} + +while (($#)); do + case "$1" in + --destination) DESTINATION="${2:?--destination requires a directory}"; shift 2 ;; + --global-root) GLOBAL_ROOT="${2:?--global-root requires a directory}"; shift 2 ;; + --best-effort) BEST_EFFORT=1; QUIESCE=0; shift ;; + --quiesce) QUIESCE=1; shift ;; + --no-restart) RESTART_SERVICES=0; shift ;; + --allow-non-smb) ALLOW_NON_SMB=1; shift ;; + -h|--help) usage; exit 0 ;; + *) printf 'Unknown option: %s\n' "$1" >&2; usage >&2; exit 2 ;; + esac +done + +[[ -d "$SOURCE" ]] || { printf 'Source directory not found: %s\n' "$SOURCE" >&2; exit 1; } +mkdir -p "$DESTINATION" +DESTINATION="$(cd -- "$DESTINATION" && pwd -P)" +case "$DESTINATION/" in + "$SOURCE/"*) printf 'Backup destination must not be inside the source tree.\n' >&2; exit 1 ;; +esac + +for tool in tar git awk sed df mount; do + command -v "$tool" >/dev/null 2>&1 || { + printf 'Required tool is missing: %s\n' "$tool" >&2 + exit 1 + } +done +if command -v shasum >/dev/null 2>&1; then + : +elif command -v sha256sum >/dev/null 2>&1; then + : +else + printf 'Install shasum or sha256sum.\n' >&2 + exit 1 +fi + +hash_file() { + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$@" + else + sha256sum "$@" + fi +} + +verify_hashes() { + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 -c "$@" + else + sha256sum -c "$@" + fi +} + +FSTYPE="$(df -P "$DESTINATION" | awk 'NR == 2 {print $1}')" +if [[ "$ALLOW_NON_SMB" -ne 1 && "$FSTYPE" != *smb* && "$FSTYPE" != *cifs* ]]; then + printf 'Destination is not an SMB/CIFS mount (filesystem=%s). Use --allow-non-smb only for tests.\n' "$FSTYPE" >&2 + exit 1 +fi +SOURCE_KB="$(du -sk "$SOURCE" | awk '{print $1}')" +AVAIL_KB="$(df -Pk "$DESTINATION" | awk 'NR == 2 {print $4}')" +if [[ "$AVAIL_KB" -lt $((SOURCE_KB + 1048576)) ]]; then + printf 'Insufficient destination space: need about %s KB plus 1 GiB, have %s KB.\n' "$SOURCE_KB" "$AVAIL_KB" >&2 + exit 1 +fi + +LOCK_DIR="${TMPDIR:-/tmp}/umbra-full-backup.lock" +if ! mkdir "$LOCK_DIR" 2>/dev/null; then + printf 'Another backup appears to be running (lock: %s).\n' "$LOCK_DIR" >&2 + exit 1 +fi +BACKUP_ID="$(date +%Y%m%d-%H%M%S)-$$-$RANDOM" +STAGING_DIR="$(mktemp -d "$DESTINATION/.UmbraAudio-$BACKUP_ID.partial.XXXXXX")" +FINAL_DIR="$DESTINATION/UmbraAudio-$BACKUP_ID" +ARCHIVE_NAME="UmbraAudio-$BACKUP_ID.tar" +ARCHIVE_TMP="$STAGING_DIR/$ARCHIVE_NAME.partial" +ARCHIVE_FINAL="$STAGING_DIR/$ARCHIVE_NAME" +[[ ! -e "$FINAL_DIR" ]] || { printf 'Final backup directory already exists: %s\n' "$FINAL_DIR" >&2; exit 1; } + +WEB_LABEL="gui/$(id -u)/com.umbraaudio.web" +WEB_PLIST="/Users/worstauthor/Library/LaunchAgents/com.umbraaudio.web.plist" +if command -v launchctl >/dev/null 2>&1; then + detected_web_plist="$(launchctl print "$WEB_LABEL" 2>/dev/null | sed -n 's/^[[:space:]]*path = //p' | head -n 1 || true)" + if [[ -f "$detected_web_plist" ]]; then + WEB_PLIST="$detected_web_plist" + fi +fi +WEB_WAS_LOADED=0 +WEB_STOPPED=0 +QWEN_PIDS=() +QWEN_STOPPED=0 +Qwen_CWD="" +Qwen_COMMAND="" +WRITER_WATCH_PID="" +RESTARTED=0 + +release_lock() { rmdir "$LOCK_DIR" 2>/dev/null || true; } + +restart_services() { + [[ "$QUIESCE" -eq 1 && "$RESTART_SERVICES" -eq 1 && "$RESTARTED" -eq 0 ]] || return 0 + local restore_failed=0 + if [[ "$WEB_WAS_LOADED" -eq 1 && -f "$WEB_PLIST" ]]; then + launchctl bootstrap "gui/$(id -u)" "$WEB_PLIST" >/dev/null 2>&1 || \ + launchctl kickstart -k "$WEB_LABEL" >/dev/null 2>&1 || true + for _ in {1..30}; do + if launchctl print "$WEB_LABEL" >/dev/null 2>&1 && lsof -nP -iTCP:8420 -sTCP:LISTEN >/dev/null 2>&1; then + break + fi + sleep 1 + done + if ! launchctl print "$WEB_LABEL" >/dev/null 2>&1 || ! lsof -nP -iTCP:8420 -sTCP:LISTEN >/dev/null 2>&1; then + printf 'WARNING: web LaunchAgent did not restart cleanly.\n' >&2 + restore_failed=1 + fi + fi + if [[ "$QWEN_STOPPED" -eq 1 && "${#QWEN_PIDS[@]}" -gt 0 ]]; then + mkdir -p "$SOURCE/logs" + QWEN_ROOT="${Qwen_CWD:-$SOURCE}" + [[ -d "$QWEN_ROOT" ]] || QWEN_ROOT="$SOURCE" + (cd "$QWEN_ROOT" && nohup "$SOURCE/.venv/bin/python" -u scripts/qwen_tts_server.py \ + >"$SOURCE/logs/qwen_tts_server.log" 2>&1 &) + disown 2>/dev/null || true + for _ in {1..30}; do + if lsof -nP -iTCP:8881 -sTCP:LISTEN >/dev/null 2>&1; then + if ! command -v curl >/dev/null 2>&1 || curl -fsS --max-time 5 http://127.0.0.1:8881/health >/dev/null 2>&1; then + break + fi + fi + sleep 1 + done + if ! lsof -nP -iTCP:8881 -sTCP:LISTEN >/dev/null 2>&1; then + printf 'WARNING: Qwen TTS server did not become healthy/listening on port 8881.\n' >&2 + restore_failed=1 + elif command -v curl >/dev/null 2>&1 && ! curl -fsS --max-time 5 http://127.0.0.1:8881/health >/dev/null 2>&1; then + printf 'WARNING: Qwen TTS port is open but /health did not pass.\n' >&2 + restore_failed=1 + fi + fi + RESTARTED=1 + return "$restore_failed" +} + +cleanup() { + local rc=$? + if [[ -n "$WRITER_WATCH_PID" ]]; then + kill "$WRITER_WATCH_PID" >/dev/null 2>&1 || true + wait "$WRITER_WATCH_PID" 2>/dev/null || true + fi + if [[ "$QUIESCE" -eq 1 && "$RESTART_SERVICES" -eq 1 && "$RESTARTED" -eq 0 ]]; then + if ! restart_services; then + printf 'WARNING: service restoration during cleanup did not complete cleanly.\n' >&2 + fi + fi + if [[ -d "$STAGING_DIR" ]]; then + printf 'Backup incomplete; preserving partial directory for inspection: %s\n' "$STAGING_DIR" >&2 + fi + release_lock + exit "$rc" +} +trap cleanup EXIT + +record_service_state() { + if launchctl print "$WEB_LABEL" >/dev/null 2>&1; then WEB_WAS_LOADED=1; fi + QWEN_PIDS=() + if command -v lsof >/dev/null 2>&1; then + while IFS= read -r pid; do + [[ "$pid" =~ ^[0-9]+$ ]] && QWEN_PIDS+=("$pid") + done < <(lsof -nP -tiTCP:8881 -sTCP:LISTEN 2>/dev/null || true) + fi + while IFS= read -r pid; do + [[ "$pid" =~ ^[0-9]+$ ]] || continue + [[ " ${QWEN_PIDS[*]} " == *" $pid "* ]] && continue + cmd="$(ps -p "$pid" -o command= 2>/dev/null || true)" + [[ "$cmd" == *qwen_tts_server.py* ]] && QWEN_PIDS+=("$pid") + done < <(ps -axo pid=,args= | awk '$0 ~ /[q]wen_tts_server\\.py/ {print $1}') + if [[ "${#QWEN_PIDS[@]}" -gt 0 ]]; then + Qwen_COMMAND="$(ps -p "${QWEN_PIDS[0]}" -o command= 2>/dev/null || true)" + if command -v lsof >/dev/null 2>&1; then + Qwen_CWD="$(lsof -a -p "${QWEN_PIDS[0]}" -d cwd -Fn 2>/dev/null | sed -n 's/^n//p' | head -n 1)" + fi + fi +} + +write_service_state() { + local output="$1" + { + printf 'captured_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf 'web_label=%s\n' "$WEB_LABEL" + printf 'web_loaded=%s\n' "$WEB_WAS_LOADED" + printf 'web_plist=%s\n' "$WEB_PLIST" + printf 'qwen_pids=%s\n' "${QWEN_PIDS[*]:-}" + printf 'qwen_cwd=%s\n' "${Qwen_CWD:-}" + printf 'qwen_command=%s\n' "${Qwen_COMMAND:-}" + launchctl print "$WEB_LABEL" 2>&1 || true + lsof -nP -iTCP:8420 -sTCP:LISTEN 2>&1 || true + lsof -nP -iTCP:8881 -sTCP:LISTEN 2>&1 || true + } >"$output" +} + +check_active_writers() { + local pattern pid + for pattern in \ + 'generate[_]all_audio.py' \ + 'generate[_]chapter[_]audio.py' \ + 'reattribute[_]all.py' \ + 'run[_]llm[_]attribution.py' \ + 'resegment[_]all.py' \ + 'nas[_]lifecycle.py' \ + 'chapter[_]builder' \ + '[r]sync.*UmbraAudio' \ + '[f]fmpeg' \ + '[u]vicorn'; do + while IFS= read -r pid; do + [[ "$pid" == "$$" ]] && continue + printf 'Active writer detected (%s, pid %s); refusing backup.\n' "$pattern" "$pid" >&2 + return 1 + done < <(ps -axo pid=,command= | awk -v p="$pattern" '$0 ~ p {print $1}') + done +} + +quiesce_services() { + for tool in launchctl lsof ps; do + command -v "$tool" >/dev/null 2>&1 || { + printf 'Quiesce requires missing tool: %s\n' "$tool" >&2 + exit 1 + } + done + record_service_state + if [[ "$WEB_WAS_LOADED" -eq 1 ]]; then + printf 'Pausing web LaunchAgent...\n' + launchctl bootout "$WEB_LABEL" "$WEB_PLIST" >/dev/null 2>&1 || { + printf 'Could not boot out web LaunchAgent; refusing inconsistent backup.\n' >&2 + exit 1 + } + for _ in {1..30}; do + if ! launchctl print "$WEB_LABEL" >/dev/null 2>&1 && ! lsof -nP -iTCP:8420 -sTCP:LISTEN >/dev/null 2>&1; then + WEB_STOPPED=1 + break + fi + sleep 1 + done + [[ "$WEB_STOPPED" -eq 1 ]] || { + printf 'Web service did not stop cleanly; refusing backup.\n' >&2 + exit 1 + } + fi + for pid in "${QWEN_PIDS[@]}"; do + printf 'Pausing Qwen TTS server (pid %s)...\n' "$pid" + kill -TERM "$pid" >/dev/null 2>&1 || true + done + for pid in "${QWEN_PIDS[@]}"; do + for _ in {1..30}; do + kill -0 "$pid" >/dev/null 2>&1 || break + sleep 1 + done + if kill -0 "$pid" >/dev/null 2>&1; then + printf 'Qwen TTS server pid %s did not stop; refusing backup.\n' "$pid" >&2 + exit 1 + fi + done + QWEN_STOPPED=1 + if lsof -nP -iTCP:8881 -sTCP:LISTEN >/dev/null 2>&1; then + printf 'Port 8881 is still listening; refusing backup.\n' >&2 + exit 1 + fi + if [[ "$WEB_STOPPED" -eq 1 ]] && lsof -nP -iTCP:8420 -sTCP:LISTEN >/dev/null 2>&1; then + printf 'Port 8420 is still listening; refusing backup.\n' >&2 + exit 1 + fi + check_active_writers || exit 1 + sleep 2 +} + +record_service_state +write_service_state "$STAGING_DIR/SERVICE-STATE-BEFORE.txt" +if [[ "$QUIESCE" -eq 1 ]]; then + quiesce_services +else + printf 'WARNING: best-effort backup requested; active files may be inconsistent.\n' >&2 +fi + +SOURCE_PARENT="$(cd -- "$SOURCE/.." && pwd -P)" +SOURCE_NAME="$(basename -- "$SOURCE")" +GLOBAL_PYTHON="$SOURCE/.venv/bin/python" +[[ -x "$GLOBAL_PYTHON" ]] || GLOBAL_PYTHON="$(command -v python3)" + +{ + printf 'backup_id=%s\n' "$BACKUP_ID" + printf 'created_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf 'source_path=%s\n' "$SOURCE" + printf 'global_state_root=%s\n' "$GLOBAL_ROOT" + printf 'source_host=%s\n' "$(hostname)" + printf 'source_os=%s\n' "$(uname -a)" + printf 'git_commit=%s\n' "$(git -C "$SOURCE" rev-parse HEAD 2>/dev/null || printf unknown)" + printf 'git_branch=%s\n' "$(git -C "$SOURCE" branch --show-current 2>/dev/null || printf unknown)" + printf 'quiesced_services=%s\n' "$QUIESCE" + printf 'best_effort=%s\n' "$BEST_EFFORT" + printf 'archive_name=%s\n' "$ARCHIVE_NAME" +} >"$STAGING_DIR/BACKUP-INFO.txt" + +{ + python3 --version 2>&1 || true + "$GLOBAL_PYTHON" --version 2>&1 || true + uv --version 2>&1 || true + git --version 2>&1 || true + tar --version 2>&1 | head -n 1 || true + ffmpeg -version 2>&1 | head -n 1 || true +} >"$STAGING_DIR/ENVIRONMENT.txt" + +git -C "$SOURCE" status --short --branch --untracked-files=all >"$STAGING_DIR/GIT-STATUS.txt" 2>&1 || true +git -C "$SOURCE" diff --binary >"$STAGING_DIR/GIT-DIFF.patch" 2>/dev/null || true +git -C "$SOURCE" diff --cached --binary >"$STAGING_DIR/GIT-DIFF-CACHED.patch" 2>/dev/null || true +git -C "$SOURCE" submodule status >"$STAGING_DIR/SUBMODULES.txt" 2>&1 || true + +printf 'Recording file inventory...\n' +SOURCE="$SOURCE" "$GLOBAL_PYTHON" - <<'PY' >"$STAGING_DIR/FILE-INVENTORY.tsv" +from __future__ import annotations +import os +from pathlib import Path +root = Path(os.environ["SOURCE"]) +for path in sorted(root.rglob("*")): + try: + stat = path.lstat() + except OSError: + continue + kind = "l" if path.is_symlink() else "d" if path.is_dir() else "f" + print(f"{kind}\t{stat.st_size}\t{path.relative_to(root)}") +PY + +printf 'Running source integrity checks before archiving...\n' +"$GLOBAL_PYTHON" "$SOURCE/backup/restore_support.py" validate-tree --root "$SOURCE" + +cat >"$STAGING_DIR/BACKUP-README.txt" <<'EOF' +UmbraAudio full backup +====================== + +This backup is a point-in-time copy of the complete current working tree and +active global voice registry/library. + +Included: +- tracked, untracked, and modified files; +- ignored book projects, generated audio, exports, logs, and .git metadata; +- model weights and the Qwen3-TTS submodule; +- .env and local configuration; +- the existing .venv and .python directories for forensic completeness; +- a SQLite-consistent global-state snapshot in global-state/; +- launchd/GPU-relay context in context/. + +The restore script rebuilds a portable environment rather than trusting the +archived platform-specific virtual environment. The original .env is preserved +as .env.archive-original when portability paths are rewritten. + +This backup does not configure or modify a network database. + +Restore: + ./Starthere.sh --target /path/to/restore + +See BACKUP-INFO.txt, ENVIRONMENT.txt, global-state/MANIFEST.json, and +RESTORED-SERVICE-COMMANDS.md after restoration. +EOF + +cp "$SOURCE/backup/Starthere.sh" "$STAGING_DIR/Starthere.sh" +cp "$SOURCE/backup/restore_support.py" "$STAGING_DIR/restore_support.py" +chmod 700 "$STAGING_DIR/Starthere.sh" +chmod 700 "$STAGING_DIR/restore_support.py" + +CONTEXT_DIR="$STAGING_DIR/context" +mkdir -p "$CONTEXT_DIR/launchagents" "$CONTEXT_DIR/launchdaemons" +cp -p /Users/worstauthor/.umbraaudio/umbraweb_login.sh "$CONTEXT_DIR/launchagents/" 2>/dev/null || true +cp -p /Users/worstauthor/Library/LaunchAgents/com.umbraaudio.web.plist "$CONTEXT_DIR/launchagents/" 2>/dev/null || true +cp -p /Users/worstauthor/Library/LaunchAgents/com.umbraaudio.commitcheck.plist "$CONTEXT_DIR/launchagents/" 2>/dev/null || true +cp -p /Library/LaunchDaemons/com.umba.qwen-relay.plist "$CONTEXT_DIR/launchdaemons/" 2>/dev/null || true +cp -p /Library/LaunchDaemons/qwen_relay.py "$CONTEXT_DIR/launchdaemons/" 2>/dev/null || true +launchctl print "$WEB_LABEL" >"$CONTEXT_DIR/launchctl-web.txt" 2>&1 || true +launchctl print system/com.umba.qwen-relay >"$CONTEXT_DIR/launchctl-qwen-relay.txt" 2>&1 || true +mount >"$CONTEXT_DIR/mounts.txt" 2>&1 || true +df -h >"$CONTEXT_DIR/filesystems.txt" 2>&1 || true +printf '%s\n' 'Context files are informational; Starthere.sh does not install privileged services.' >"$CONTEXT_DIR/README.txt" + +printf 'Snapshotting global voice state with SQLite backup API...\n' +"$GLOBAL_PYTHON" "$SOURCE/backup/restore_support.py" snapshot-global \ + --source "$GLOBAL_ROOT" --output "$STAGING_DIR/global-state" --original-root "$GLOBAL_ROOT" +(cd "$STAGING_DIR/global-state" && verify_hashes GLOBAL-STATE-SHA256SUMS) + +start_writer_watch() { + local flag="$STAGING_DIR/ACTIVE-WRITER" + ( + while :; do + if ! check_active_writers >/dev/null 2>&1; then + printf 'writer detected during archive\n' >"$flag" + break + fi + sleep 1 + done + ) & + WRITER_WATCH_PID="$!" +} + +stop_writer_watch() { + if [[ -n "$WRITER_WATCH_PID" ]]; then + kill "$WRITER_WATCH_PID" >/dev/null 2>&1 || true + wait "$WRITER_WATCH_PID" 2>/dev/null || true + WRITER_WATCH_PID="" + fi +} + +printf 'Creating full payload archive (%s)...\n' "$ARCHIVE_NAME" +start_writer_watch +if ! COPYFILE_DISABLE=1 tar -C "$SOURCE_PARENT" -cf "$ARCHIVE_TMP" "$SOURCE_NAME" 2>"$STAGING_DIR/TAR-WARNINGS.log"; then + stop_writer_watch + printf 'tar failed; inspect TAR-WARNINGS.log.\n' >&2 + exit 1 +fi +stop_writer_watch +if [[ -f "$STAGING_DIR/ACTIVE-WRITER" ]]; then + printf 'A writer appeared while archiving; refusing the backup.\n' >&2 + exit 1 +fi +if grep -Eiq 'file changed|error|failed|cannot|permission denied' "$STAGING_DIR/TAR-WARNINGS.log"; then + printf 'tar reported a consistency/read warning; refusing backup.\n' >&2 + cat "$STAGING_DIR/TAR-WARNINGS.log" >&2 + exit 1 +fi +mv "$ARCHIVE_TMP" "$ARCHIVE_FINAL" +printf 'Computing comprehensive SHA-256 manifest...\n' +"$GLOBAL_PYTHON" "$SOURCE/backup/restore_support.py" write-manifest --root "$STAGING_DIR" +(cd "$STAGING_DIR" && verify_hashes SHA256SUMS) +(cd "$STAGING_DIR/global-state" && verify_hashes GLOBAL-STATE-SHA256SUMS) +tar -tf "$ARCHIVE_FINAL" >/dev/null +chmod 600 "$ARCHIVE_FINAL" "$STAGING_DIR/SHA256SUMS" "$STAGING_DIR/GIT-DIFF.patch" "$STAGING_DIR/GIT-DIFF-CACHED.patch" 2>/dev/null || true + +mv "$STAGING_DIR" "$FINAL_DIR" +STAGING_DIR="" + +# Keep the cleanup trap until service restoration and top-level pointers finish. +ROOT_LAUNCHER_TMP="$DESTINATION/.Starthere.sh.tmp.$$" +cat >"$ROOT_LAUNCHER_TMP" <&2; exit 1; } +exec "\$LATEST/Starthere.sh" "\$@" +EOF +chmod 700 "$ROOT_LAUNCHER_TMP" +mv "$ROOT_LAUNCHER_TMP" "$DESTINATION/Starthere.sh" +printf '%s\n' "$FINAL_DIR" >"$DESTINATION/LATEST.tmp.$$" +mv "$DESTINATION/LATEST.tmp.$$" "$DESTINATION/LATEST" + +restart_services +record_service_state +write_service_state "$FINAL_DIR/SERVICE-STATE-AFTER.txt" +"$GLOBAL_PYTHON" "$SOURCE/backup/restore_support.py" write-manifest --root "$FINAL_DIR" +(cd "$FINAL_DIR" && verify_hashes SHA256SUMS) +trap - EXIT +release_lock + +printf '\nBackup complete.\n' +printf 'Location: %s\n' "$FINAL_DIR" +printf 'Archive: %s\n' "$FINAL_DIR/$ARCHIVE_NAME" +printf 'Global state snapshot: %s/global-state\n' "$FINAL_DIR" +printf 'Restore: %s/Starthere.sh --target /path/to/restore\n' "$FINAL_DIR" diff --git a/backup/restore_support.py b/backup/restore_support.py new file mode 100644 index 0000000..5858793 --- /dev/null +++ b/backup/restore_support.py @@ -0,0 +1,821 @@ +#!/usr/bin/env python3 +"""Portable backup/restore support helpers for UmbraAudio. + +This module uses only the Python standard library. It snapshots global Umbra +state with SQLite's backup API, restores it atomically, rewrites machine-local +paths in a restored copy, and performs basic integrity checks. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shlex +import shutil +import sqlite3 +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def is_database(path: Path) -> bool: + return path.name.endswith((".db", ".db-wal", ".db-shm")) + + +def is_sqlite_file(path: Path) -> bool: + try: + with path.open("rb") as handle: + return handle.read(16) == b"SQLite format 3\x00" + except OSError: + return False + + +def safe_relative_path(value: str) -> Path: + path = Path(value) + if path.is_absolute() or ".." in path.parts: + raise RuntimeError(f"unsafe relative path in backup metadata: {value!r}") + return path + + +def sql_identifier(value: str) -> str: + return '"' + value.replace('"', '""') + '"' + + +def sqlite_backup(source: Path, destination: Path) -> str: + destination.parent.mkdir(parents=True, exist_ok=True) + source_uri = f"file:{source}?mode=ro" + source_db = sqlite3.connect(source_uri, uri=True) + destination_db = sqlite3.connect(destination) + try: + source_db.backup(destination_db) + result = destination_db.execute("PRAGMA integrity_check").fetchone()[0] + if result != "ok": + raise RuntimeError(f"integrity_check failed for {source}: {result}") + finally: + destination_db.close() + source_db.close() + return "ok" + + +def copy_tree_files( + source: Path, + destination: Path, + skip_databases: bool = True, + reject_symlinks: bool = False, +) -> list[str]: + copied: list[str] = [] + for path in sorted(source.rglob("*")): + relative = path.relative_to(source) + target = destination / relative + if path.is_symlink(): + if reject_symlinks: + raise RuntimeError(f"global state symlink is not allowed: {relative}") + target.parent.mkdir(parents=True, exist_ok=True) + os.symlink(os.readlink(path), target) + elif path.is_dir(): + target.mkdir(parents=True, exist_ok=True) + elif path.is_file(): + if skip_databases and is_database(path): + continue + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, target) + copied.append(str(relative)) + return copied + + +def snapshot_global(source: Path, output: Path, original_root: Path) -> None: + if output.exists(): + raise RuntimeError(f"global-state output already exists: {output}") + if not source.exists(): + raise RuntimeError(f"global state root does not exist: {source}") + if source == output or source in output.parents: + raise RuntimeError("global state output must be outside the source root") + + temporary = Path(tempfile.mkdtemp(prefix="umbra-global-snapshot-")) + try: + files = copy_tree_files(source, temporary, skip_databases=True) + databases: list[dict[str, Any]] = [] + for path in sorted(source.rglob("*.db")): + if not path.is_file(): + continue + relative = path.relative_to(source) + target = temporary / relative + status = sqlite_backup(path, target) + databases.append( + {"path": str(relative), "integrity_check": status, "sha256": sha256(target)} + ) + files.append(str(relative)) + + file_hashes = { + relative: sha256(temporary / relative) + for relative in sorted(files) + if (temporary / relative).is_file() + } + manifest = { + "schema": "umbra.global_state/v1", + "original_root": str(original_root), + "source_root": str(source), + "files": sorted(files), + "file_hashes": file_hashes, + "databases": databases, + } + (temporary / "MANIFEST.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + (temporary / "GLOBAL-STATE-SHA256SUMS").write_text( + "".join(f"{sha256(temporary / relative)} {relative}\n" for relative in sorted(files)), + encoding="utf-8", + ) + output.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(temporary, output, symlinks=True) + finally: + shutil.rmtree(temporary, ignore_errors=True) + + +def _replace_prefix(value: Any, old_root: str, new_root: str) -> Any: + if isinstance(value, str) and old_root and value.startswith(old_root): + return new_root + value[len(old_root) :] + return value + + +def assert_no_old_prefixes( + connection: sqlite3.Connection, + old_roots: list[str], +) -> None: + for table_row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'"): + table = table_row[0] + if table.startswith("sqlite_"): + continue + table_identifier = sql_identifier(table) + for column in connection.execute(f"PRAGMA table_info({table_identifier})").fetchall(): + column_name = column[1] + column_identifier = sql_identifier(column_name) + if (column[2] or "").upper() not in {"TEXT", ""}: + continue + try: + values = connection.execute( + f"SELECT {column_identifier} FROM {table_identifier} " + f"WHERE {column_identifier} IS NOT NULL" + ).fetchall() + except sqlite3.Error: + continue + for (value,) in values: + if isinstance(value, str) and any( + old_root and value.startswith(old_root) for old_root in old_roots + ): + raise RuntimeError(f"old path prefix remains in {table}.{column_name}: {value}") + + +def migrate_database_paths(database: Path, replacements: list[tuple[str, str]]) -> None: + connection = sqlite3.connect(database) + try: + tables = { + row[0] + for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'") + } + for table, columns in { + "voice_registry": ("voice_ref_path", "vmatrix_path", "sample_audio_path"), + "library_speakers": ("voice_ref_path",), + "library_utterances": ("voice_ref_path",), + }.items(): + if table not in tables: + continue + for column in columns: + try: + rows = connection.execute( + f'SELECT rowid, "{column}" FROM "{table}" WHERE "{column}" IS NOT NULL' + ).fetchall() + except sqlite3.Error: + continue + for rowid, value in rows: + replacement = value + for old_root, new_root in replacements: + replacement = _replace_prefix(replacement, old_root, new_root) + if replacement != value: + connection.execute( + f'UPDATE "{table}" SET "{column}" = ? WHERE rowid = ?', + (replacement, rowid), + ) + connection.commit() + assert_no_old_prefixes(connection, [old for old, _ in replacements if old]) + result = connection.execute("PRAGMA integrity_check").fetchone()[0] + if result != "ok": + raise RuntimeError(f"integrity_check failed after path migration: {database}: {result}") + finally: + connection.close() + + +def migrate_project_database(database: Path, old_root: str, new_root: str) -> None: + connection = sqlite3.connect(database) + try: + tables = { + row[0] + for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'") + if not row[0].startswith("sqlite_") + } + for table in tables: + table_identifier = sql_identifier(table) + columns = connection.execute(f"PRAGMA table_info({table_identifier})").fetchall() + for column in columns: + column_name = column[1] + column_identifier = sql_identifier(column_name) + if (column[2] or "").upper() not in {"TEXT", ""}: + continue + try: + query = ( + f"SELECT rowid, {column_identifier} FROM {table_identifier} " + f"WHERE {column_identifier} IS NOT NULL" + ) + rows = connection.execute(query).fetchall() + except sqlite3.Error: + continue + for rowid, value in rows: + replacement = _replace_prefix(value, old_root, new_root) + if replacement != value: + connection.execute( + f"UPDATE {table_identifier} SET {column_identifier} = ? " + "WHERE rowid = ?", + (replacement, rowid), + ) + connection.commit() + assert_no_old_prefixes(connection, [old_root] if old_root else []) + result = connection.execute("PRAGMA integrity_check").fetchone()[0] + if result != "ok": + raise RuntimeError( + f"integrity_check failed after project path migration: {database}: {result}" + ) + finally: + connection.close() + + +def _validate_target(path: Path, force: bool) -> Path: + if path.is_symlink(): + raise RuntimeError(f"refusing symlink target: {path}") + if path.exists() and not path.is_dir(): + raise RuntimeError(f"target exists and is not a directory: {path}") + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists() and any(path.iterdir()) and not force: + raise RuntimeError(f"target is not empty: {path}; use --force") + return path + + +def _commit_directory(temporary: Path, target: Path, force: bool) -> Path | None: + _validate_target(target, force) + previous: Path | None = None + if target.exists(): + previous = target.with_name(f"{target.name}.old.{int(time.time())}") + target.rename(previous) + try: + temporary.rename(target) + except Exception: + if previous is not None and previous.exists() and not target.exists(): + previous.rename(target) + raise + return previous + + +def restore_global( + backup_dir: Path, + target: Path, + force: bool, + old_project_root: str = "", + new_project_root: str = "", +) -> None: + source = backup_dir / "global-state" + manifest_path = source / "MANIFEST.json" + if not manifest_path.exists(): + raise RuntimeError(f"global-state manifest is missing: {manifest_path}") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + old_root = manifest.get("original_root", "") + _validate_target(target, force) + temporary = Path(tempfile.mkdtemp(prefix=f".{target.name}.global-restore.", dir=target.parent)) + try: + copy_tree_files( + source, + temporary, + skip_databases=False, + reject_symlinks=True, + ) + for metadata_name in ("MANIFEST.json", "GLOBAL-STATE-SHA256SUMS"): + metadata_path = temporary / metadata_name + if metadata_path.exists(): + metadata_path.unlink() + for relative in manifest.get("files", []): + safe_relative_path(relative) + file_hashes = manifest.get("file_hashes", {}) + for relative, expected_hash in file_hashes.items(): + safe_relative = safe_relative_path(relative) + path = temporary / safe_relative + if not path.is_file() or sha256(path) != expected_hash: + raise RuntimeError(f"global file checksum mismatch: {relative}") + replacements = [(old_root, str(target))] + if old_project_root and new_project_root: + replacements.append((old_project_root, new_project_root)) + for database in manifest.get("databases", []): + relative = safe_relative_path(database["path"]) + path = temporary / relative + if not path.is_file() or sha256(path) != database.get("sha256"): + raise RuntimeError(f"global database checksum mismatch: {relative}") + migrate_database_paths(path, replacements) + previous = _commit_directory(temporary, target, force) + temporary = Path() + if previous is not None: + print(f"Previous global state retained at: {previous}") + finally: + if temporary and str(temporary) not in {"", "."}: + shutil.rmtree(temporary, ignore_errors=True) + print(f"Restored global state to {target}") + + +def _replace_text(path: Path, replacements: list[tuple[str, str]]) -> list[str]: + text = path.read_text(encoding="utf-8") + changed: list[str] = [] + for old, new in replacements: + if old in text: + text = text.replace(old, new) + changed.append(old) + path.write_text(text, encoding="utf-8") + return changed + + +def patch_restored_paths( + root: Path, + old_root: str, + new_root: Path, + global_root: Path | None = None, +) -> None: + changes: list[str] = [] + env_path = root / ".env" + if env_path.exists() and old_root: + original = root / ".env.archive-original" + if not original.exists(): + shutil.copy2(env_path, original) + changes.extend(_replace_text(env_path, [(old_root, str(new_root))])) + if env_path.exists() and global_root is not None: + lines = env_path.read_text(encoding="utf-8").splitlines() + keys = {"UMBRA_GLOBAL_STATE_DIR", "UMBRA_VOICE_REGISTRY_PATH"} + lines = [line for line in lines if line.split("=", 1)[0] not in keys] + lines.extend( + [ + f"UMBRA_GLOBAL_STATE_DIR={json.dumps(str(global_root))}", + f"UMBRA_VOICE_REGISTRY_PATH={json.dumps(str(global_root / 'voice_registry.db'))}", + ] + ) + env_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + changes.append(".env global-state settings") + + config_path = root / "umbraaudio" / "config.py" + if config_path.exists() and old_root: + text = config_path.read_text(encoding="utf-8") + if "import os\n" not in text: + text = text.replace("import re\n", "import os\nimport re\n", 1) + portable_project_base = ( + "PROJECT_BASE = Path(\n" + ' os.environ.get("UMBRA_PROJECT_BASE", ' + "str(Path(__file__).resolve().parents[1]))\n" + ").expanduser()" + ) + text = text.replace( + f'PROJECT_BASE = Path("{old_root}")', + portable_project_base, + ) + text = text.replace( + f'"{old_root}/models/Qwen3-TTS-12Hz-1.7B-VoiceDesign"', + 'str(PROJECT_BASE / "models/Qwen3-TTS-12Hz-1.7B-VoiceDesign")', + ) + text = text.replace( + f'"{old_root}/models/Qwen3-TTS-12Hz-1.7B-Base"', + 'str(PROJECT_BASE / "models/Qwen3-TTS-12Hz-1.7B-Base")', + ) + portable_voice_registry = ( + "voice_registry_path: Path = Field(\n" + " default_factory=lambda: Path(\n" + " os.environ.get(\n" + ' "UMBRA_VOICE_REGISTRY_PATH",\n' + ' str(Path.home() / ".umbra" / "voice_registry.db"),\n' + " )\n" + " ).expanduser()\n" + ")" + ) + old_voice_field = ( + 'voice_registry_path: Path = Field(default=Path.home() / ".umbra" / ' + '"voice_registry.db")' + ) + text = text.replace(old_voice_field, portable_voice_registry) + config_path.write_text(text, encoding="utf-8") + changes.append("umbraaudio/config.py") + + library_path = root / "umbraaudio" / "library.py" + if library_path.exists(): + text = library_path.read_text(encoding="utf-8") + if "import os\n" not in text: + text = text.replace("import json\n", "import json\nimport os\n", 1) + portable_library_root = ( + "def _global_state_root() -> Path:\n" + " return Path(\n" + ' os.environ.get("UMBRA_GLOBAL_STATE_DIR", str(Path.home() / ".umbra"))\n' + " ).expanduser()\n" + ) + text = text.replace( + 'DEFAULT_LIBRARY_DB = Path.home() / ".umbra" / "voice_registry.db"\n' + 'DEFAULT_LIBRARY_VOICES = Path.home() / ".umbra" / "library" / "voices"', + portable_library_root + + 'DEFAULT_LIBRARY_DB = _global_state_root() / "voice_registry.db"\n' + + 'DEFAULT_LIBRARY_VOICES = _global_state_root() / "library" / "voices"', + ) + old_db_getter = "def get_library_db_path() -> Path:\n return DEFAULT_LIBRARY_DB" + new_db_getter = ( + "def get_library_db_path() -> Path:\n return _global_state_root() / " + '"voice_registry.db"' + ) + text = text.replace(old_db_getter, new_db_getter) + old_voices_getter = ( + "def get_library_voices_dir() -> Path:\n return DEFAULT_LIBRARY_VOICES" + ) + new_voices_getter = ( + "def get_library_voices_dir() -> Path:\n return _global_state_root() / " + '"library" / "voices"' + ) + text = text.replace(old_voices_getter, new_voices_getter) + library_path.write_text(text, encoding="utf-8") + changes.append("umbraaudio/library.py") + + server_path = root / "scripts" / "qwen_tts_server.py" + if server_path.exists() and old_root: + portable_prompt_dir = ( + "str(Path(__file__).resolve().parents[1] / " + '"DealHunterShip_4.0e_whole-to-audio/.umbra/voice_prompts")' + ) + portable_voicedesign = ( + 'str(Path(__file__).resolve().parents[1] / "models/Qwen3-TTS-12Hz-1.7B-VoiceDesign")' + ) + portable_base = ( + 'str(Path(__file__).resolve().parents[1] / "models/Qwen3-TTS-12Hz-1.7B-Base")' + ) + replacements = [ + ( + f'"{old_root}/DealHunterShip_4.0e_whole-to-audio/.umbra/voice_prompts"', + portable_prompt_dir, + ), + ( + f'"{old_root}/models/Qwen3-TTS-12Hz-1.7B-VoiceDesign"', + portable_voicedesign, + ), + ( + f'"{old_root}/models/Qwen3-TTS-12Hz-1.7B-Base"', + portable_base, + ), + ] + changes.extend(_replace_text(server_path, replacements)) + + storage_path = root / "umbraaudio" / "storage.py" + if storage_path.exists(): + text = storage_path.read_text(encoding="utf-8") + if "import os\n" not in text: + text = text.replace("import logging\n", "import logging\nimport os\n", 1) + if old_root: + portable_local_root = ( + "DEFAULT_LOCAL_ROOT = Path(os.environ.get(" + '"UMBRA_PROJECT_ROOT", str(Path(__file__).resolve().parents[1]))' + ").expanduser()" + ) + text = text.replace( + f'DEFAULT_LOCAL_ROOT = Path("{old_root}")', + portable_local_root, + ) + portable_nas_root = ( + "DEFAULT_NAS_ROOT = Path(os.environ.get(" + '"UMBRA_NAS_ROOT", "/Volumes/Umbra/UmbraAudio"' + ")).expanduser()" + ) + text = text.replace( + 'DEFAULT_NAS_ROOT = Path("/Volumes/Umbra/UmbraAudio")', + portable_nas_root, + ) + storage_path.write_text(text, encoding="utf-8") + changes.append("umbraaudio/storage.py") + + if old_root: + escaped_root = json.dumps(str(new_root))[1:-1] + operational_files = [ + root / "scripts" / "qwen_tts_server.py", + root / "tts_server" / "qwen_tts_server.py", + root / "Qwen3-TTS" / "qwen_tts" / "cli" / "demo.py", + root / "start_server.sh", + root / "test_voice_clone.py", + ] + for operational_path in operational_files: + if operational_path.exists(): + _replace_text(operational_path, [(old_root, escaped_root)]) + + nas_lifecycle = root / "scripts" / "nas_lifecycle.py" + if nas_lifecycle.exists(): + text = nas_lifecycle.read_text(encoding="utf-8") + if "import os\n" not in text: + text = text.replace("import logging\n", "import logging\nimport os\n", 1) + text = text.replace( + f'DEFAULT_LOCAL_ROOT = Path("{old_root}")', + "DEFAULT_LOCAL_ROOT = Path(__file__).resolve().parents[1]", + ) + portable_nas = ( + "DEFAULT_NAS_ROOT = Path(os.environ.get(" + '"UMBRA_NAS_ROOT", "/Volumes/Umbra/UmbraAudio"' + ")).expanduser()" + ) + text = text.replace( + 'DEFAULT_NAS_ROOT = Path("/Volumes/Umbra/UmbraAudio")', + portable_nas, + ) + text = text.replace( + 'MOUNT_SCRIPT = Path.home() / ".umbra/mount_nas.sh"', + 'MOUNT_SCRIPT = _global_state_root() / "mount_nas.sh"', + ) + text = text.replace( + 'CONFIG_PATH = Path.home() / ".umbra/nas_config.json"', + 'CONFIG_PATH = _global_state_root() / "nas_config.json"', + ) + if "def _global_state_root()" not in text: + global_root_function = ( + "def _global_state_root() -> Path:\n" + " return Path(os.environ.get(\n" + ' "UMBRA_GLOBAL_STATE_DIR", str(Path.home() / ".umbra")\n' + " )).expanduser()\n\n" + "DEFAULT_LOCAL_ROOT =" + ) + text = text.replace( + "DEFAULT_LOCAL_ROOT =", + global_root_function, + 1, + ) + nas_lifecycle.write_text(text, encoding="utf-8") + changes.append("scripts/nas_lifecycle.py") + + sync_voice = root / "tts_server" / "sync_voice_data.sh" + if sync_voice.exists(): + text = sync_voice.read_text(encoding="utf-8") + text = text.replace( + 'UMBRA_DATA="${UMBRA_DATA_DIR:-/Volumes/1TB_APL_Storage/UmbraAudio}"', + 'PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"\n' + 'UMBRA_DATA="${UMBRA_DATA_DIR:-$PROJECT_ROOT}"', + ) + sync_voice.write_text(text, encoding="utf-8") + changes.append("tts_server/sync_voice_data.sh") + + verify_nas = root / "scripts" / "verify_nas.py" + if verify_nas.exists(): + text = verify_nas.read_text(encoding="utf-8") + text = text.replace( + f'Path("{old_root}")', + "Path(__file__).resolve().parents[1]", + ) + verify_nas.write_text(text, encoding="utf-8") + changes.append("scripts/verify_nas.py") + + embeddings = root / "scripts" / "extract_embeddings.py" + if embeddings.exists(): + text = embeddings.read_text(encoding="utf-8") + text = text.replace( + f'"{old_root}/models/Qwen3-TTS-12Hz-1.7B-Base"', + 'str(project_root / "models/Qwen3-TTS-12Hz-1.7B-Base")', + ) + text = text.replace( + f'"{old_root}/models/Qwen3-TTS-12Hz-1.7B-VoiceDesign"', + 'str(project_root / "models/Qwen3-TTS-12Hz-1.7B-VoiceDesign")', + ) + embeddings.write_text(text, encoding="utf-8") + changes.append("scripts/extract_embeddings.py") + + legacy_start = root / "start_server.sh" + if legacy_start.exists(): + legacy_start.write_text( + "#!/usr/bin/env bash\n" + "set -Eeuo pipefail\n" + "printf '%s\\n' 'Legacy Gradio demo launcher is disabled in this restore.'\n" + "printf '%s\\n' 'Use scripts/start_web_server.sh or the generated service commands.'\n" + "exit 1\n", + encoding="utf-8", + ) + legacy_start.chmod(0o755) + changes.append("start_server.sh") + + start_script = root / "scripts" / "start_web_server.sh" + if start_script.exists(): + web_command = ( + 'exec "$PROJ/.venv/bin/python" -c \'' + "from pathlib import Path; from umbraaudio.web import create_app; " + "from umbraaudio.config import load_config; import uvicorn; " + 'c = load_config(Path(".")); uvicorn.run(create_app(c), ' + 'host="0.0.0.0", port=8420, log_level="info")\'\n' + ) + global_exports = "" + if global_root is not None: + global_exports = ( + f"export UMBRA_GLOBAL_STATE_DIR={shlex.quote(str(global_root))}\n" + "export UMBRA_VOICE_REGISTRY_PATH=" + f"{shlex.quote(str(global_root / 'voice_registry.db'))}\n" + ) + start_script.write_text( + "#!/usr/bin/env bash\n" + "set -Eeuo pipefail\n" + 'PROJ="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)"\n' + 'exec >> "$PROJ/logs/web_server.log" 2>&1\n' + "echo \"[$(date '+%F %T')] web server starting (wrapper pid $$)\"\n" + 'cd "$PROJ"\n' + global_exports + web_command, + encoding="utf-8", + ) + start_script.chmod(0o755) + changes.append("scripts/start_web_server.sh") + + if old_root: + for database in sorted(root.rglob("*.db")): + if not database.is_file() or any( + part in {".venv", ".python"} for part in database.parts + ): + continue + if not is_sqlite_file(database): + continue + migrate_project_database(database, old_root, str(new_root)) + + commands = root / "RESTORED-SERVICE-COMMANDS.md" + web_command = ( + "python -c 'from pathlib import Path; from umbraaudio.web import create_app; " + "from umbraaudio.config import load_config; import uvicorn; " + 'uvicorn.run(create_app(load_config(Path("."))), host="0.0.0.0", port=8420)\'' + ) + voicedesign_model = json.dumps(str(new_root / "models/Qwen3-TTS-12Hz-1.7B-VoiceDesign")) + base_model = json.dumps(str(new_root / "models/Qwen3-TTS-12Hz-1.7B-Base")) + global_exports = "" + if global_root is not None: + global_exports = ( + f"export UMBRA_GLOBAL_STATE_DIR={shlex.quote(str(global_root))}\n" + "export UMBRA_VOICE_REGISTRY_PATH=" + f"{shlex.quote(str(global_root / 'voice_registry.db'))}\n" + ) + commands.write_text( + "# Restored UmbraAudio service commands\n\n" + "These commands are generated for the restore location and are not installed " + "automatically.\n\n" + "```bash\n" + f"cd {json.dumps(str(new_root))}\n" + 'export PATH="$PWD/.venv/bin:$PATH"\n' + f"export UMBRA_PROJECT_BASE={json.dumps(str(new_root))}\n" + + global_exports + + f"{web_command}\n" + f"QWEN_VOICEDESIGN_MODEL={voicedesign_model}\n" + f"QWEN_BASE_MODEL={base_model}\n" + "python scripts/qwen_tts_server.py\n" + "```\n\n" + "The external launchd plist is intentionally not installed automatically; review the " + "context/ directory and adapt paths and permissions first.\n", + encoding="utf-8", + ) + (root / "PORTABILITY-PATCH.txt").write_text( + "The following restored-copy portability adjustments were applied:\n" + + "".join(f"- {item}\n" for item in sorted(set(changes))) + + "\nThe original .env was preserved as .env.archive-original.\n", + encoding="utf-8", + ) + + +def write_checksum_manifest(root: Path) -> Path: + output = root / "SHA256SUMS" + lines: list[str] = [] + for path in sorted(root.rglob("*")): + if not path.is_file() or path == output: + continue + relative = path.relative_to(root).as_posix() + lines.append(f"{sha256(path)} {relative}\n") + output.write_text("".join(lines), encoding="utf-8") + return output + + +def validate_tree(root: Path, old_root: str | None = None) -> list[str]: + errors: list[str] = [] + required = (root / "pyproject.toml", root / "umbraaudio", root / "Qwen3-TTS") + for path in required: + if not path.exists(): + errors.append(f"missing required path: {path}") + model = root / "models" / "Qwen3-TTS-12Hz-1.7B-Base" / "model.safetensors" + if not model.exists(): + errors.append(f"missing model: {model}") + for database in sorted(root.rglob("*.db")): + if not database.is_file(): + continue + if database.stat().st_size == 0: + continue + if not is_sqlite_file(database): + if any( + part in {".mypy_cache", ".pytest_cache", ".ruff_cache"} for part in database.parts + ): + continue + errors.append(f"non-SQLite .db file: {database}") + continue + try: + connection = sqlite3.connect(f"file:{database}?mode=ro", uri=True) + result = connection.execute("PRAGMA quick_check").fetchone()[0] + if old_root: + try: + assert_no_old_prefixes(connection, [old_root]) + except RuntimeError as exc: + errors.append(str(exc)) + connection.close() + if result != "ok": + errors.append(f"SQLite quick_check failed: {database}: {result}") + except Exception as exc: # noqa: BLE001 + errors.append(f"SQLite check error: {database}: {exc}") + if old_root: + code_files = [ + root / "umbraaudio" / "config.py", + root / "umbraaudio" / "library.py", + root / "umbraaudio" / "storage.py", + root / "scripts" / "nas_lifecycle.py", + root / "scripts" / "verify_nas.py", + root / "scripts" / "extract_embeddings.py", + root / "scripts" / "qwen_tts_server.py", + root / "tts_server" / "qwen_tts_server.py", + root / "tts_server" / "sync_voice_data.sh", + root / "start_server.sh", + ] + for code_file in code_files: + if code_file.exists() and old_root in code_file.read_text(encoding="utf-8"): + errors.append(f"old project root remains in {code_file}") + return errors + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + + snapshot = sub.add_parser("snapshot-global") + snapshot.add_argument("--source", type=Path, required=True) + snapshot.add_argument("--output", type=Path, required=True) + snapshot.add_argument("--original-root", type=Path, required=True) + + restore = sub.add_parser("restore-global") + restore.add_argument("--backup-dir", type=Path, required=True) + restore.add_argument("--target", type=Path, required=True) + restore.add_argument("--force", action="store_true") + restore.add_argument("--old-project-root", default="") + restore.add_argument("--new-project-root", default="") + + patch = sub.add_parser("patch-paths") + patch.add_argument("--root", type=Path, required=True) + patch.add_argument("--old-root", required=True) + patch.add_argument("--new-root", type=Path, required=True) + patch.add_argument("--global-state-dir", type=Path) + + manifest = sub.add_parser("write-manifest") + manifest.add_argument("--root", type=Path, required=True) + + validate = sub.add_parser("validate-tree") + validate.add_argument("--root", type=Path, required=True) + validate.add_argument("--old-root", default="") + + args = parser.parse_args() + try: + if args.command == "snapshot-global": + snapshot_global(args.source, args.output, args.original_root) + elif args.command == "restore-global": + restore_global( + args.backup_dir, + args.target, + args.force, + args.old_project_root, + args.new_project_root, + ) + elif args.command == "patch-paths": + patch_restored_paths( + args.root, + args.old_root, + args.new_root, + args.global_state_dir, + ) + elif args.command == "write-manifest": + write_checksum_manifest(args.root) + elif args.command == "validate-tree": + errors = validate_tree(args.root, args.old_root or None) + if errors: + for error in errors: + print(error, file=sys.stderr) + return 1 + print(f"Tree validation passed: {args.root}") + return 0 + except Exception as exc: # noqa: BLE001 + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/infra/BOOTSTRAP-SHA256SUMS b/infra/BOOTSTRAP-SHA256SUMS new file mode 100644 index 0000000..60f2bab --- /dev/null +++ b/infra/BOOTSTRAP-SHA256SUMS @@ -0,0 +1,4 @@ +47df21afa85aa7bbb550fa6622b0b4fa3ec8ea1e100fecf47375db60990186ad bootstrap_enterprise_mint.sh +bea205a4113e7b0ca7eb7b3da4344a1711143d111c4c13049698369506768f72 enterprise-bootstrap.env.example +5a7fe225d6cc67fea080e392dc3c3b9d896800eeb3eb1e6391791485d68764ed README-enterprise-bootstrap.md +deb8b51f9892ad5f3b50738514da4d59b2697b30ad356e1d1bdc81ae1c862749 postgres/001_control_plane.sql diff --git a/infra/DATABASE-SHA256SUMS b/infra/DATABASE-SHA256SUMS new file mode 100644 index 0000000..8672370 --- /dev/null +++ b/infra/DATABASE-SHA256SUMS @@ -0,0 +1,2 @@ +d6f52246a3a6899d48d8ed115deb5b4661362e9b5695eb946b0e0dbd0ded03cf bootstrap_umbracontrol_database.sh +deb8b51f9892ad5f3b50738514da4d59b2697b30ad356e1d1bdc81ae1c862749 postgres/001_control_plane.sql diff --git a/infra/README-enterprise-bootstrap.md b/infra/README-enterprise-bootstrap.md new file mode 100644 index 0000000..d605309 --- /dev/null +++ b/infra/README-enterprise-bootstrap.md @@ -0,0 +1,63 @@ +# Umbra enterprise phase-one bootstrap + +This directory contains the first, infrastructure-only bootstrap for the UmbraTrans enterprise control plane. + +It is designed for Linux Mint 22.1 (Ubuntu 24.04-family) and is intentionally conservative: + +- It does not modify Qwen. +- It does not stop or alter Docker containers. +- It does not change firewall rules. +- It does not install application code or systemd application units yet. +- It does not place PostgreSQL on SMB/NAS. +- It creates database roles, a local database, core identity/tenancy tables, row-level-security policies, a one-time bootstrap administrator, and least-privilege service accounts. + +## Files + +- `bootstrap_enterprise_mint.sh` — root bootstrap and read-only inspection tool. +- `enterprise-bootstrap.env.example` — non-secret configuration template. +- `postgres/001_control_plane.sql` — initial identity, organization, project, role, invitation, session, MFA, bootstrap-token, audit, and RLS schema. + +## Safe usage on oldbox + +The script is intended to be copied to the `umbra` account and reviewed there: + +```bash +ssh umbra@oldbox.bdmurphy.me +mkdir -p ~/umbra-enterprise-bootstrap +# copy the three files into that directory +cp enterprise-bootstrap.env.example enterprise-bootstrap.env +$EDITOR enterprise-bootstrap.env +bash -n bootstrap_enterprise_mint.sh +./bootstrap_enterprise_mint.sh --check +./bootstrap_enterprise_mint.sh --dry-run --config "$PWD/enterprise-bootstrap.env" +sudo ./bootstrap_enterprise_mint.sh --apply --config /home/umbra/umbra-enterprise-bootstrap/enterprise-bootstrap.env +``` + +The root password is entered only in the user's terminal. It is not stored in the configuration file or command line. + +After a successful apply: + +```bash +sudo ./bootstrap_enterprise_mint.sh --verify --config /home/umbra/umbra-enterprise-bootstrap/enterprise-bootstrap.env +``` + +The initial one-time administrator token is stored root-only at: + +```text +/root/umbra-enterprise-bootstrap.token +``` + +Do not paste that token into chat. The application bootstrap flow will consume it and replace it with normal authentication. + +## Deliberate exclusions + +The script does not yet: + +- create or install the UmbraTrans API/worker application; +- install a reverse proxy or certificate; +- expose Qwen to other machines; +- add Qwen authentication or rate limiting; +- modify UFW/nftables/iptables; +- migrate existing Docker, MySQL, LM Studio, Ollama, or Qwen data. + +Those are separate reviewed changes after the host and database foundation are verified. diff --git a/infra/agent_bridge/README.md b/infra/agent_bridge/README.md new file mode 100644 index 0000000..41af3ce --- /dev/null +++ b/infra/agent_bridge/README.md @@ -0,0 +1,59 @@ +# Temporary UmbraTrans agent bridge + +This is a one-hour, LAN-only browser pairing point for communication between +the Mac agent and a browser session on oldbox. It is not a remote shell. + +## Properties + +- Random OS-assigned TCP port. +- HTTPS with a temporary self-signed certificate. +- One-hour hard expiry. +- Password is entered once in a native Mac setup dialog and once in the + browser on the other machine. +- Password is hashed in memory; plaintext is not logged or written to disk. +- HttpOnly/Secure/SameSite session cookie and CSRF token. +- No raw shell, no arbitrary command execution, no attachments. +- Messages are atomic, bounded JSON files in a private queue. + +The browser terminal is an agent message channel. The Mac agent must poll the +inbox and send replies through the local bridge tooling; commands are not +executed automatically from browser text. + +## Start + +The listener runs on the Mac's LAN address. The startup dialog asks for the +one-hour pairing password. The process prints the exact URL and port. + +Example: + +```bash +PYTHONPATH=/Volumes/1TB_APL_Storage/UmbraAudio/infra \ +/usr/bin/python3 -m agent_bridge \ + --host 192.168.123.12 \ + --ttl 3600 +``` + +Open the printed `https://192.168.123.12:/` URL in a browser on oldbox. +Accept the temporary certificate warning if the browser requires it, then pair +with the same one-hour password. + +## Queue + +The default queue is: + +```text +~/.local/state/umbratrans/agent-bridge/ + inbox/ # browser messages waiting for the Mac agent + outbox/ # Mac-agent replies waiting for the browser + processed/ # browser messages already read + delivered/ # browser replies acknowledged by the browser +``` + +All files are mode `0600`; directories are mode `0700`. + +## Safety + +- Do not expose the listener to the public internet. +- Keep the one-hour timer unchanged. +- Do not put passwords or private keys in messages. +- Stop the process after the session if the automatic timer is interrupted. diff --git a/infra/agent_bridge/__init__.py b/infra/agent_bridge/__init__.py new file mode 100644 index 0000000..37a18e3 --- /dev/null +++ b/infra/agent_bridge/__init__.py @@ -0,0 +1,3 @@ +"""Temporary browser-to-agent bridge for UmbraTrans.""" + +__all__: list[str] = [] diff --git a/infra/agent_bridge/__main__.py b/infra/agent_bridge/__main__.py new file mode 100644 index 0000000..0a68a64 --- /dev/null +++ b/infra/agent_bridge/__main__.py @@ -0,0 +1,3 @@ +from .server import main + +raise SystemExit(main()) diff --git a/infra/agent_bridge/client.py b/infra/agent_bridge/client.py new file mode 100644 index 0000000..c67d52e --- /dev/null +++ b/infra/agent_bridge/client.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from .server import MessageStore + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Read/write the local UmbraTrans agent bridge queue") + parser.add_argument("action", choices=("poll", "send", "status")) + parser.add_argument("--root", default="~/.local/state/umbratrans/agent-bridge") + parser.add_argument("--title", default="Mac agent") + parser.add_argument("--body", default="") + args = parser.parse_args(argv) + store = MessageStore(Path(args.root).expanduser()) + if args.action == "poll": + print(json.dumps(store.poll_for_mac(20), ensure_ascii=False)) + elif args.action == "send": + if not args.body: + parser.error("--body is required for send") + print(json.dumps(store.send_to_browser(args.body, args.title), ensure_ascii=False)) + else: + print(json.dumps(store.status(), sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/infra/agent_bridge/server.py b/infra/agent_bridge/server.py new file mode 100644 index 0000000..48f729d --- /dev/null +++ b/infra/agent_bridge/server.py @@ -0,0 +1,490 @@ +from __future__ import annotations + +import argparse +import base64 +import hashlib +import hmac +import html +import http.cookies +import json +import os +import secrets +import signal +import ssl +import subprocess +import sys +import tempfile +import threading +import time +import uuid +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, urlparse + +from remote_credential_broker.popup import macos_secret_prompt + +MAX_BODY_BYTES = 128 * 1024 +MAX_MESSAGE_CHARS = 64 * 1024 +SESSION_TTL_SECONDS = 3600 +PAIR_ATTEMPT_LIMIT = 8 +PAIR_ATTEMPT_WINDOW = 60 + + +class BridgeError(RuntimeError): + """Base error for the temporary bridge.""" + + +class MessageStore: + """Atomic, non-secret file queue shared by the browser and local tooling.""" + + def __init__(self, root: Path) -> None: + self.root = root + self.inbox = root / "inbox" + self.outbox = root / "outbox" + self.processed = root / "processed" + self.delivered = root / "delivered" + for directory in (self.root, self.inbox, self.outbox, self.processed, self.delivered): + directory.mkdir(parents=True, exist_ok=True) + os.chmod(directory, 0o700) + + @staticmethod + def _safe_name(value: str) -> bool: + return bool(value) and len(value) <= 80 and all(c.isalnum() or c in "-_" for c in value) + + def _write(self, directory: Path, message: dict[str, Any]) -> str: + message_id = str(message["id"]) + if not self._safe_name(message_id): + raise BridgeError("invalid message id") + target = directory / f"{message_id}.json" + temporary = directory / f".{message_id}.{uuid.uuid4().hex}.tmp" + payload = json.dumps(message, ensure_ascii=False, sort_keys=True).encode("utf-8") + fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(fd, "wb") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, target) + except BaseException: + try: + temporary.unlink() + except FileNotFoundError: + pass + raise + return str(target) + + @staticmethod + def _validate_message(message: dict[str, Any]) -> dict[str, Any]: + required = {"schema_version", "id", "created_at", "sender", "kind", "title", "body"} + if not required.issubset(message): + raise BridgeError("message is missing required fields") + if message["schema_version"] != 1: + raise BridgeError("unsupported message schema") + if not isinstance(message["body"], str) or len(message["body"]) > MAX_MESSAGE_CHARS: + raise BridgeError("message body is too large") + if not isinstance(message["title"], str) or len(message["title"]) > 200: + raise BridgeError("message title is too large") + if "\x00" in message["body"] or "\x00" in message["title"]: + raise BridgeError("message contains NUL") + return message + + def send_to_mac(self, body: str, title: str = "Browser message") -> dict[str, Any]: + message = self._validate_message( + { + "schema_version": 1, + "id": str(uuid.uuid4()), + "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "sender": "oldbox-browser", + "kind": "instruction", + "correlation_id": None, + "title": title, + "body": body, + } + ) + self._write(self.inbox, message) + return message + + def send_to_browser(self, body: str, title: str = "Mac agent") -> dict[str, Any]: + message = self._validate_message( + { + "schema_version": 1, + "id": str(uuid.uuid4()), + "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "sender": "mac-agent", + "kind": "report", + "correlation_id": None, + "title": title, + "body": body, + } + ) + self._write(self.outbox, message) + return message + + def _read_dir(self, directory: Path, limit: int) -> list[dict[str, Any]]: + values: list[dict[str, Any]] = [] + for path in sorted(directory.glob("*.json"))[:limit]: + if not self._safe_name(path.stem): + continue + try: + if not path.is_file() or path.is_symlink(): + continue + value = json.loads(path.read_text(encoding="utf-8")) + values.append(self._validate_message(value)) + except (OSError, json.JSONDecodeError, BridgeError): + continue + return values + + def poll_for_mac(self, limit: int = 20) -> list[dict[str, Any]]: + values = self._read_dir(self.inbox, max(1, min(limit, 100))) + for value in values: + source = self.inbox / f"{value['id']}.json" + destination = self.processed / f"{value['id']}.json" + try: + os.replace(source, destination) + except FileNotFoundError: + pass + return values + + def read_for_browser(self, limit: int = 50) -> list[dict[str, Any]]: + return self._read_dir(self.outbox, max(1, min(limit, 100))) + + def acknowledge(self, message_ids: list[str]) -> int: + count = 0 + for message_id in message_ids[:100]: + if not self._safe_name(message_id): + continue + source = self.outbox / f"{message_id}.json" + destination = self.delivered / f"{message_id}.json" + try: + os.replace(source, destination) + count += 1 + except FileNotFoundError: + pass + return count + + def status(self) -> dict[str, int]: + return { + "inbox": len(list(self.inbox.glob("*.json"))), + "outbox": len(list(self.outbox.glob("*.json"))), + "processed": len(list(self.processed.glob("*.json"))), + "delivered": len(list(self.delivered.glob("*.json"))), + } + + +class BridgeState: + def __init__(self, password: str, ttl_seconds: int = SESSION_TTL_SECONDS) -> None: + self.started_at = time.time() + self.expires_at = self.started_at + ttl_seconds + self.salt = os.urandom(32) + self.digest = hashlib.pbkdf2_hmac( + "sha256", password.encode("utf-8"), self.salt, 240_000 + ) + self.sessions: dict[str, dict[str, Any]] = {} + self.attempts: dict[str, list[float]] = {} + self.lock = threading.RLock() + + def check_password(self, address: str, password: str) -> bool: + now = time.time() + with self.lock: + recent = [stamp for stamp in self.attempts.get(address, []) if now - stamp < PAIR_ATTEMPT_WINDOW] + if len(recent) >= PAIR_ATTEMPT_LIMIT: + return False + candidate = hashlib.pbkdf2_hmac( + "sha256", password.encode("utf-8"), self.salt, 240_000 + ) + accepted = hmac.compare_digest(candidate, self.digest) + if accepted: + self.attempts.pop(address, None) + else: + recent.append(now) + self.attempts[address] = recent + return accepted + + def new_session(self) -> tuple[str, str]: + session = secrets.token_urlsafe(40) + csrf = secrets.token_urlsafe(32) + with self.lock: + self.sessions[session] = {"csrf": csrf, "expires_at": self.expires_at} + return session, csrf + + def session(self, token: str | None) -> dict[str, Any] | None: + if not token: + return None + with self.lock: + value = self.sessions.get(token) + if value is None or value["expires_at"] <= time.time(): + self.sessions.pop(token, None) + return None + return value + + def remaining_seconds(self) -> int: + return max(0, int(self.expires_at - time.time())) + + +class Handler(BaseHTTPRequestHandler): + server_version = "UmbraAgentBridge/0.1" + state: BridgeState + store: MessageStore + + def log_message(self, format: str, *args: Any) -> None: + # Do not log request bodies, cookies, passwords, or message content. + print(f"agent-bridge: {self.address_string()} {format % args}", file=sys.stderr) + + def _headers(self, content_type: str = "text/plain; charset=utf-8", length: int | None = None) -> None: + self.send_header("Content-Type", content_type) + self.send_header("Cache-Control", "no-store") + self.send_header("X-Content-Type-Options", "nosniff") + self.send_header("X-Frame-Options", "DENY") + self.send_header("Referrer-Policy", "no-referrer") + if length is not None: + self.send_header("Content-Length", str(length)) + + def _send(self, status: int, body: bytes, content_type: str = "text/plain; charset=utf-8") -> None: + self.send_response(status) + self._headers(content_type, len(body)) + self.end_headers() + self.wfile.write(body) + + def _json(self, status: int, value: dict[str, Any]) -> None: + self._send(status, json.dumps(value).encode("utf-8"), "application/json; charset=utf-8") + + def _cookie_token(self) -> str | None: + raw = self.headers.get("Cookie", "") + cookie = http.cookies.SimpleCookie() + try: + cookie.load(raw) + except http.cookies.CookieError: + return None + morsel = cookie.get("bridge_session") + return morsel.value if morsel else None + + def _auth(self) -> tuple[dict[str, Any] | None, str | None]: + session = self.state.session(self._cookie_token()) + if session is None: + return None, "authentication required" + csrf = self.headers.get("X-Bridge-CSRF", "") + if self.command == "POST" and not hmac.compare_digest(csrf, session["csrf"]): + return None, "CSRF validation failed" + return session, None + + def _read_json(self) -> dict[str, Any]: + length = int(self.headers.get("Content-Length", "0")) + if length <= 0 or length > MAX_BODY_BYTES: + raise BridgeError("invalid request size") + value = json.loads(self.rfile.read(length).decode("utf-8")) + if not isinstance(value, dict): + raise BridgeError("request must be a JSON object") + return value + + def do_GET(self) -> None: + path = urlparse(self.path).path + if path == "/healthz": + self._json(200, {"status": "ok", "paired_sessions": len(self.state.sessions)}) + return + if path == "/": + if self.state.session(self._cookie_token()) is None: + self._send(200, LOGIN_HTML.encode("utf-8"), "text/html; charset=utf-8") + else: + self._send(200, TERMINAL_HTML.encode("utf-8"), "text/html; charset=utf-8") + return + if path == "/api/status": + session, error = self._auth() + if error: + self._json(401, {"error": error}) + return + self._json(200, {"expires_in": self.state.remaining_seconds(), "queue": self.store.status()}) + return + if path == "/api/messages": + session, error = self._auth() + if error: + self._json(401, {"error": error}) + return + query = parse_qs(urlparse(self.path).query) + try: + limit = int(query.get("limit", ["50"])[0]) + except ValueError: + limit = 50 + self._json(200, {"messages": self.store.read_for_browser(limit)}) + return + self._json(404, {"error": "not found"}) + + def do_POST(self) -> None: + path = urlparse(self.path).path + if path == "/api/pair": + try: + value = self._read_json() + password = value.get("password") + if not isinstance(password, str) or not self.state.check_password( + self.client_address[0], password + ): + self._json(401, {"error": "pairing failed"}) + return + session, csrf = self.state.new_session() + body = json.dumps({"paired": True, "csrf": csrf, "expires_in": self.state.remaining_seconds()}).encode() + self.send_response(200) + self._headers("application/json; charset=utf-8", len(body)) + self.send_header( + "Set-Cookie", + f"bridge_session={session}; HttpOnly; Secure; SameSite=Strict; Path=/; Max-Age={self.state.remaining_seconds()}", + ) + self.end_headers() + self.wfile.write(body) + except (BridgeError, ValueError, json.JSONDecodeError): + self._json(400, {"error": "invalid pairing request"}) + return + if path == "/api/messages": + session, error = self._auth() + if error: + self._json(401, {"error": error}) + return + try: + value = self._read_json() + body = value.get("body") + title = value.get("title", "Browser message") + if not isinstance(body, str) or not isinstance(title, str): + raise BridgeError("body and title must be strings") + message = self.store.send_to_mac(body, title) + self._json(202, {"id": message["id"], "queued": True}) + except (BridgeError, ValueError, json.JSONDecodeError): + self._json(400, {"error": "invalid message"}) + return + self._json(404, {"error": "not found"}) + + +LOGIN_HTML = r""" + + +UmbraTrans Temporary Agent Bridge + +

UmbraTrans temporary bridge

+

This session expires after one hour. Enter the temporary pairing password supplied by the Mac operator.

+
+

+ +""" + +TERMINAL_HTML = r""" + + +UmbraTrans Agent Terminal + +

UmbraTrans agent terminal

+Paired local session. Messages are queued for the Mac agent; this is not a raw shell. +
+
+ +""" + + +def make_certificate(directory: Path, address: str) -> tuple[Path, Path]: + directory.mkdir(parents=True, exist_ok=True) + os.chmod(directory, 0o700) + key = directory / "bridge-key.pem" + cert = directory / "bridge-cert.pem" + if key.exists() and cert.exists(): + return key, cert + subprocess.run( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + str(key), + "-out", + str(cert), + "-days", + "1", + "-subj", + f"/CN={address}", + "-addext", + f"subjectAltName=IP:{address}", + ], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + os.chmod(key, 0o600) + os.chmod(cert, 0o644) + return key, cert + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Temporary UmbraTrans agent bridge") + parser.add_argument("--host", default="192.168.123.12") + parser.add_argument("--port", type=int, default=0) + parser.add_argument("--root", default="~/.local/state/umbratrans/agent-bridge") + parser.add_argument("--ttl", type=int, default=SESSION_TTL_SECONDS) + args = parser.parse_args(argv) + if not 1 <= args.ttl <= 3600: + parser.error("--ttl must be between 1 and 3600") + password = macos_secret_prompt("agent-bridge-temp", "one-hour agent bridge pairing password") + if not password: + print("agent bridge: pairing password entry cancelled", file=sys.stderr) + return 2 + state = BridgeState(password, args.ttl) + del password + store = MessageStore(Path(args.root).expanduser()) + Handler.state = state + Handler.store = store + server = ThreadingHTTPServer((args.host, args.port), Handler) + tls_dir = Path(tempfile.mkdtemp(prefix="umbra-agent-bridge-tls-")) + key, cert = make_certificate(tls_dir, args.host) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(certfile=str(cert), keyfile=str(key)) + server.socket = context.wrap_socket(server.socket, server_side=True) + port = server.server_address[1] + print(f"BRIDGE_URL=https://{args.host}:{port}/", flush=True) + print(f"BRIDGE_ROOT={store.root}", flush=True) + print(f"BRIDGE_EXPIRES_IN={state.remaining_seconds()}", flush=True) + + def stop(*_: Any) -> None: + threading.Thread(target=server.shutdown, daemon=True).start() + + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + timer = threading.Timer(args.ttl, stop) + timer.daemon = True + timer.start() + try: + server.serve_forever(poll_interval=0.5) + finally: + server.server_close() + timer.cancel() + for path in (key, cert): + try: + path.unlink() + except FileNotFoundError: + pass + try: + tls_dir.rmdir() + except OSError: + pass + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/infra/agent_bridge/tests/test_bridge.py b/infra/agent_bridge/tests/test_bridge.py new file mode 100644 index 0000000..cea0ea3 --- /dev/null +++ b/infra/agent_bridge/tests/test_bridge.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import json +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from agent_bridge.server import BridgeError, BridgeState, MessageStore + + +class BridgeTests(unittest.TestCase): + def test_password_hashing_and_pairing(self) -> None: + state = BridgeState("correct-value", ttl_seconds=60) + self.assertTrue(state.check_password("192.0.2.1", "correct-value")) + self.assertFalse(state.check_password("192.0.2.1", "wrong-value")) + session, csrf = state.new_session() + self.assertIsNotNone(state.session(session)) + self.assertTrue(csrf) + self.assertIsNone(state.session("missing")) + + def test_message_queue_is_atomic_and_bounded(self) -> None: + with tempfile.TemporaryDirectory() as directory: + store = MessageStore(Path(directory)) + message = store.send_to_mac("hello", "test") + self.assertEqual(store.status()["inbox"], 1) + self.assertEqual(store.poll_for_mac()[0]["body"], "hello") + self.assertEqual(store.status()["processed"], 1) + reply = store.send_to_browser("reply", "test") + self.assertEqual(store.read_for_browser()[0]["id"], reply["id"]) + self.assertEqual(store.acknowledge([reply["id"]]), 1) + self.assertEqual(store.status()["delivered"], 1) + + def test_message_size_is_limited(self) -> None: + with tempfile.TemporaryDirectory() as directory: + store = MessageStore(Path(directory)) + with self.assertRaises(BridgeError): + store.send_to_mac("x" * (64 * 1024 + 1)) + + def test_invalid_message_file_is_ignored(self) -> None: + with tempfile.TemporaryDirectory() as directory: + store = MessageStore(Path(directory)) + (store.inbox / "bad.json").write_text(json.dumps({"not": "a message"})) + self.assertEqual(store.poll_for_mac(), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/infra/bootstrap_enterprise_mint.sh b/infra/bootstrap_enterprise_mint.sh new file mode 100755 index 0000000..5840b21 --- /dev/null +++ b/infra/bootstrap_enterprise_mint.sh @@ -0,0 +1,511 @@ +#!/usr/bin/env bash +# Phase-one enterprise control-plane bootstrap for Linux Mint 22.x. +# This provisions PostgreSQL, database roles, core tenancy/identity tables, +# and least-privilege service accounts. It does not install application code, +# change Qwen, manage Docker, or change firewall rules. +set -Eeuo pipefail +umask 077 + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +CONFIG_FILE="${UMBRA_BOOTSTRAP_CONFIG:-${SCRIPT_DIR}/enterprise-bootstrap.env}" +MODE="check" +DRY_RUN=0 +LOCK_DIR="/run/lock/umbra-enterprise-bootstrap.lock" +LOG_FILE="/var/log/umbra-enterprise-bootstrap.log" + +UMBRA_HOSTNAME="oldbox.bdmurphy.me" +UMBRA_PUBLIC_URL="https://oldbox.bdmurphy.me" +UMBRA_DB_NAME="umbracontrol" +UMBRA_DB_HOST="127.0.0.1" +UMBRA_DB_PORT="5432" +UMBRA_PG_VERSION="16" +UMBRA_APP_ROOT="/opt/umbracontrol" +UMBRA_DATA_ROOT="/var/lib/umbracontrol" +UMBRA_BACKUP_ROOT="/var/backups/umbracontrol" +UMBRA_BOOTSTRAP_ADMIN_EMAIL="umbratools@gmail.com" +UMBRA_ORGANIZATION_NAME="Umbra" +UMBRA_ORGANIZATION_SLUG="umbra" +UMBRA_QWEN_URL="http://192.168.123.14:8881" +UMBRA_INSTALL_PACKAGES=1 +UMBRA_INSTALL_UV=1 +UMBRA_CREATE_BOOTSTRAP_ADMIN=1 +UMBRA_CONFIG_DIR="/etc/umbracontrol" +UMBRA_CREDENTIALS_FILE="/etc/umbracontrol/db-credentials.env" +UMBRA_BOOTSTRAP_TOKEN_FILE="/root/umbra-enterprise-bootstrap.token" +UMBRA_PG_SUPERUSER="postgres" +UMBRA_SERVICE_GROUP="umbracontrol" +UMBRA_API_USER="umbracontrol_api" +UMBRA_WORKER_USER="umbracontrol_worker" +UMBRA_READONLY_USER="umbracontrol_readonly" +UMBRA_BACKUP_USER="umbracontrol_backup" +UMBRA_MIGRATOR_USER="umbracontrol_migrator" +UMBRA_OWNER_ROLE="umbracontrol_owner" + +usage() { + cat <<'EOF' +Usage: bootstrap_enterprise_mint.sh [options] + +Options: + --check Read-only host and dependency inspection (default) + --dry-run Show apply actions without changing the host + --apply Apply the phase-one infrastructure bootstrap; requires root + --status Show installation/service/database status + --verify Verify installed services and database connectivity + --config FILE Read non-secret KEY=VALUE configuration + -h, --help Show this help + +This phase intentionally does not install application code, modify Qwen, +manage Docker containers, or change firewall rules. Run it first on Mint, +then deploy the application and systemd units in a separate reviewed step. +EOF +} + +while (($#)); do + case "$1" in + --check) MODE="check"; shift ;; + --dry-run) MODE="apply"; DRY_RUN=1; shift ;; + --apply) MODE="apply"; DRY_RUN=0; shift ;; + --status) MODE="status"; shift ;; + --verify) MODE="verify"; shift ;; + --config) CONFIG_FILE="${2:?--config requires a file}"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) printf 'Unknown option: %s\n' "$1" >&2; usage >&2; exit 2 ;; + esac +done + +log() { + local line + line="[$(date -Is)] $*" + printf '%s\n' "$line" + if [[ -f "$LOG_FILE" && "$(id -u)" -eq 0 ]]; then + printf '%s\n' "$line" >> "$LOG_FILE" + fi +} + +die() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || die "Required command is missing: $1" +} + +require_root() { + [[ "$(id -u)" -eq 0 ]] || die "This operation requires root. Use --check for a read-only inspection." +} + +load_config() { + [[ -f "$CONFIG_FILE" ]] || { + [[ "$MODE" == "check" || "$MODE" == "status" || "$MODE" == "verify" ]] && return 0 + die "Configuration file not found: $CONFIG_FILE" + } + + local line key value + while IFS= read -r line || [[ -n "$line" ]]; do + line="${line#"${line%%[![:space:]]*}"}" + [[ -z "$line" || "${line:0:1}" == "#" ]] && continue + [[ "$line" == *=* ]] || die "Invalid configuration line (expected KEY=VALUE): $line" + key="${line%%=*}" + value="${line#*=}" + key="${key//[[:space:]]/}" + value="${value%\"}"; value="${value#\"}" + value="${value%\'}"; value="${value#\'}" + case "$key" in + UMBRA_HOSTNAME|UMBRA_PUBLIC_URL|UMBRA_DB_NAME|UMBRA_DB_HOST|UMBRA_DB_PORT|UMBRA_PG_VERSION|UMBRA_APP_ROOT|UMBRA_DATA_ROOT|UMBRA_BACKUP_ROOT|UMBRA_BOOTSTRAP_ADMIN_EMAIL|UMBRA_ORGANIZATION_NAME|UMBRA_ORGANIZATION_SLUG|UMBRA_QWEN_URL|UMBRA_INSTALL_PACKAGES|UMBRA_INSTALL_UV|UMBRA_CREATE_BOOTSTRAP_ADMIN) + printf -v "$key" '%s' "$value" + ;; + *) die "Unsupported configuration key: $key" ;; + esac + done < "$CONFIG_FILE" +} + +validate_config() { + [[ "$UMBRA_HOSTNAME" =~ ^[A-Za-z0-9.-]+$ ]] || die "Invalid UMBRA_HOSTNAME" + [[ "$UMBRA_PUBLIC_URL" =~ ^https?://[A-Za-z0-9._:-]+(/.*)?$ ]] || die "Invalid UMBRA_PUBLIC_URL" + [[ "$UMBRA_DB_NAME" =~ ^[a-z_][a-z0-9_]*$ ]] || die "Invalid UMBRA_DB_NAME" + [[ "$UMBRA_DB_HOST" =~ ^[A-Za-z0-9._:-]+$ ]] || die "Invalid UMBRA_DB_HOST" + [[ "$UMBRA_DB_PORT" =~ ^[0-9]+$ && "$UMBRA_DB_PORT" -ge 1 && "$UMBRA_DB_PORT" -le 65535 ]] || die "Invalid UMBRA_DB_PORT" + [[ "$UMBRA_PG_VERSION" =~ ^[0-9]+$ ]] || die "Invalid UMBRA_PG_VERSION" + [[ "$UMBRA_APP_ROOT" == /* && "$UMBRA_DATA_ROOT" == /* && "$UMBRA_BACKUP_ROOT" == /* ]] || die "Application/data paths must be absolute" + [[ "$UMBRA_BOOTSTRAP_ADMIN_EMAIL" == *@*.* ]] || die "Invalid administrator email" + [[ "$UMBRA_ORGANIZATION_SLUG" =~ ^[a-z0-9][a-z0-9-]*$ ]] || die "Invalid organization slug" + [[ "$UMBRA_QWEN_URL" =~ ^https?:// ]] || die "Invalid UMBRA_QWEN_URL" + for value in "$UMBRA_INSTALL_PACKAGES" "$UMBRA_INSTALL_UV" "$UMBRA_CREATE_BOOTSTRAP_ADMIN"; do + [[ "$value" == 0 || "$value" == 1 ]] || die "Boolean configuration values must be 0 or 1" + done +} + +run() { + if ((DRY_RUN)); then + printf '[dry-run]' + printf ' %q' "$@" + printf '\n' + else + "$@" + fi +} + +run_root() { + require_root + run "$@" +} + +read_secret() { + local key="$1" + [[ -f "$UMBRA_CREDENTIALS_FILE" ]] || return 1 + awk -F= -v key="$key" '$1 == key {print substr($0, index($0, "=") + 1); exit}' "$UMBRA_CREDENTIALS_FILE" +} + +write_secret_line() { + local key="$1" value="$2" temporary + if ((DRY_RUN)); then + printf '[dry-run] store generated secret %s in %s\n' "$key" "$UMBRA_CREDENTIALS_FILE" + return + fi + install -d -m 0750 -o root -g "$UMBRA_SERVICE_GROUP" "$UMBRA_CONFIG_DIR" + touch "$UMBRA_CREDENTIALS_FILE" + chown root:"$UMBRA_SERVICE_GROUP" "$UMBRA_CREDENTIALS_FILE" + chmod 0640 "$UMBRA_CREDENTIALS_FILE" + temporary="$(mktemp "${UMBRA_CONFIG_DIR}/.credentials.XXXXXX")" + awk -F= -v key="$key" -v value="$value" ' + $1 == key { print key "=" value; found = 1; next } + { print } + END { if (!found) print key "=" value } + ' "$UMBRA_CREDENTIALS_FILE" > "$temporary" + chown root:"$UMBRA_SERVICE_GROUP" "$temporary" + chmod 0640 "$temporary" + mv -f "$temporary" "$UMBRA_CREDENTIALS_FILE" +} + +random_secret() { + openssl rand -hex 32 +} + +check_host() { + log "Read-only host inspection" + printf 'hostname: '; hostname -f 2>/dev/null || hostname + printf 'os: '; . /etc/os-release; printf '%s\n' "${PRETTY_NAME:-unknown}" + printf 'kernel: '; uname -r + printf 'user: '; id + printf 'cpu: '; nproc + free -h | sed -n '1,3p' + if command -v nvidia-smi >/dev/null 2>&1; then + nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader 2>/dev/null || true + fi + printf '%s\n' 'storage:' + df -hT / /home /var/lib 2>/dev/null || true + printf '%s\n' 'listening ports:' + ss -lntup 2>/dev/null | sed -n '1,100p' || true + printf '%s\n' 'relevant services:' + systemctl list-units --type=service --all --no-pager 2>/dev/null | grep -Ei 'docker|ollama|lmstudio|qwen|postgres|umbra' || true + printf '%s\n' 'DNS:' + getent ahosts "$UMBRA_HOSTNAME" 2>/dev/null || true + log "No changes made" +} + +check_tools() { + require_command openssl + require_command systemctl + if [[ "$MODE" == "apply" && "$UMBRA_INSTALL_PACKAGES" -eq 1 ]]; then + require_command apt-get + fi +} + +ensure_group_and_users() { + log "Ensuring service group and non-login service accounts" + if ! getent group "$UMBRA_SERVICE_GROUP" >/dev/null; then + run groupadd --system "$UMBRA_SERVICE_GROUP" + else + log "Group already exists: $UMBRA_SERVICE_GROUP" + fi + + ensure_service_user "$UMBRA_API_USER" + ensure_service_user "$UMBRA_WORKER_USER" + ensure_service_user "$UMBRA_READONLY_USER" + ensure_service_user "$UMBRA_BACKUP_USER" +} + +ensure_service_user() { + local user="$1" + if id "$user" >/dev/null 2>&1; then + log "Service account already exists: $user" + return + fi + run useradd --system --gid "$UMBRA_SERVICE_GROUP" --home-dir /nonexistent --no-create-home --shell /usr/sbin/nologin "$user" +} + +ensure_directories() { + log "Creating application, data, backup, and configuration directories" + run install -d -m 0755 -o root -g root "$UMBRA_APP_ROOT" + run install -d -m 0750 -o root -g "$UMBRA_SERVICE_GROUP" "$UMBRA_DATA_ROOT" + run install -d -m 0700 -o root -g root "$UMBRA_BACKUP_ROOT" + run install -d -m 0750 -o root -g "$UMBRA_SERVICE_GROUP" "$UMBRA_CONFIG_DIR" + run install -d -m 0750 -o root -g "$UMBRA_SERVICE_GROUP" "$UMBRA_APP_ROOT/app" "$UMBRA_APP_ROOT/venv" "$UMBRA_APP_ROOT/bin" +} + +install_packages() { + [[ "$UMBRA_INSTALL_PACKAGES" -eq 1 ]] || { log "Package installation disabled by configuration"; return; } + log "Installing Mint/Ubuntu packages" + run env DEBIAN_FRONTEND=noninteractive apt-get update + run env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates curl openssl postgresql postgresql-client postgresql-contrib \ + python3 python3-venv python3-pip rsync +} + +ensure_uv() { + [[ "$UMBRA_INSTALL_UV" -eq 1 ]] || { log "uv installation disabled by configuration"; return; } + if command -v uv >/dev/null 2>&1; then + log "uv already available: $(command -v uv)" + return + fi + log "Installing pinned uv bootstrap environment" + run python3 -m venv "$UMBRA_APP_ROOT/bootstrap-venv" + run "$UMBRA_APP_ROOT/bootstrap-venv/bin/python" -m pip install --upgrade pip + run "$UMBRA_APP_ROOT/bootstrap-venv/bin/python" -m pip install 'uv==0.11.7' + run ln -sfn "$UMBRA_APP_ROOT/bootstrap-venv/bin/uv" "$UMBRA_APP_ROOT/bin/uv" +} + +psql_as_postgres() { + runuser -u "$UMBRA_PG_SUPERUSER" -- env PGOPTIONS='-c log_statement=none' \ + psql -v ON_ERROR_STOP=1 "$@" +} + +ensure_postgres_cluster() { + log "Ensuring PostgreSQL cluster and service" + local version cluster + if command -v pg_lsclusters >/dev/null 2>&1; then + version="$(pg_lsclusters --no-header 2>/dev/null | awk 'NR == 1 {print $1}')" + cluster="$(pg_lsclusters --no-header 2>/dev/null | awk 'NR == 1 {print $2}')" + else + version="" + cluster="" + fi + if [[ -z "$version" || -z "$cluster" ]]; then + version="$UMBRA_PG_VERSION" + cluster="main" + run pg_createcluster "$version" "$cluster" --start + fi + if command -v pg_lsclusters >/dev/null 2>&1; then + if ! pg_lsclusters --no-header | awk -v v="$version" -v c="$cluster" '$1 == v && $2 == c && $3 == "online" {found=1} END {exit !found}'; then + run pg_ctlcluster "$version" "$cluster" start + fi + else + log "pg_lsclusters is not installed; cluster start will be performed after package installation" + fi + run systemctl enable postgresql +} + +ensure_database_roles() { + log "Creating PostgreSQL roles" + local role password login + for role in \ + "$UMBRA_OWNER_ROLE" \ + "$UMBRA_MIGRATOR_USER" \ + "$UMBRA_API_USER" \ + "$UMBRA_WORKER_USER" \ + "$UMBRA_READONLY_USER" \ + "$UMBRA_BACKUP_USER"; do + password="" + login="LOGIN" + [[ "$role" == "$UMBRA_OWNER_ROLE" ]] && login="NOLOGIN" + if [[ "$role" != "$UMBRA_OWNER_ROLE" ]]; then + if ((DRY_RUN)); then + password="$(random_secret)" + else + password="$(read_secret "${role^^}_PASSWORD" 2>/dev/null || true)" + [[ -n "$password" ]] || password="$(random_secret)" + fi + write_secret_line "${role^^}_PASSWORD" "$password" + fi + + if ((DRY_RUN)); then + printf '[dry-run] ensure PostgreSQL role %s (%s)\n' "$role" "$login" + continue + fi + + if [[ -n "$password" ]]; then + printf 'DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = %s) THEN CREATE ROLE %s %s PASSWORD %s; ELSE ALTER ROLE %s %s PASSWORD %s; END IF; END $$;\n' \ + "'$role'" "$role" "$login" "'$password'" "$role" "$login" "'$password'" \ + | psql_as_postgres postgres >/dev/null + else + printf 'DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = %s) THEN CREATE ROLE %s %s; END IF; END $$;\n' \ + "'$role'" "$role" "$login" \ + | psql_as_postgres postgres >/dev/null + fi + done +} + +ensure_database() { + log "Creating enterprise database if absent" + if ((DRY_RUN)); then + printf '[dry-run] ensure database %s owned by %s\n' "$UMBRA_DB_NAME" "$UMBRA_OWNER_ROLE" + return + fi + if ! psql_as_postgres postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '$UMBRA_DB_NAME'" | grep -q 1; then + runuser -u "$UMBRA_PG_SUPERUSER" -- createdb --owner="$UMBRA_OWNER_ROLE" "$UMBRA_DB_NAME" + fi + psql_as_postgres postgres -c "REVOKE ALL ON DATABASE \"$UMBRA_DB_NAME\" FROM PUBLIC" >/dev/null + psql_as_postgres postgres -c "GRANT CONNECT ON DATABASE \"$UMBRA_DB_NAME\" TO \"$UMBRA_MIGRATOR_USER\", \"$UMBRA_API_USER\", \"$UMBRA_WORKER_USER\", \"$UMBRA_READONLY_USER\"" >/dev/null + psql_as_postgres postgres -c "GRANT CONNECT ON DATABASE \"$UMBRA_DB_NAME\" TO \"$UMBRA_BACKUP_USER\"" >/dev/null +} + +apply_schema() { + local schema_file="${SCRIPT_DIR}/postgres/001_control_plane.sql" + [[ -f "$schema_file" ]] || die "Schema file is missing: $schema_file" + log "Applying core identity/tenancy schema" + if ((DRY_RUN)); then + printf '[dry-run] psql -d %s -f %s\n' "$UMBRA_DB_NAME" "$schema_file" + return + fi + /bin/cat -- "$schema_file" | psql_as_postgres "$UMBRA_DB_NAME" >/dev/null + log "Core schema applied" +} + +write_nonsecret_runtime_config() { + local config_file="$UMBRA_CONFIG_DIR/control-plane.env" + run install -d -m 0750 -o root -g "$UMBRA_SERVICE_GROUP" "$UMBRA_CONFIG_DIR" + if ((DRY_RUN)); then + printf '[dry-run] write non-secret runtime configuration to %s\n' "$config_file" + return + fi + cat > "${config_file}.tmp" < "$UMBRA_BOOTSTRAP_TOKEN_FILE" + chown root:root "$UMBRA_BOOTSTRAP_TOKEN_FILE" + chmod 0600 "$UMBRA_BOOTSTRAP_TOKEN_FILE" + fi + [[ "$token" =~ ^[a-f0-9]{64}$ ]] || die "Bootstrap token file has an unexpected format" + token_hash="$(printf '%s' "$token" | sha256sum | awk '{print $1}')" + log "Creating/confirming bootstrap platform administrator" + if ((DRY_RUN)); then + printf '[dry-run] create administrator %s and organization %s\n' "$UMBRA_BOOTSTRAP_ADMIN_EMAIL" "$UMBRA_ORGANIZATION_NAME" + return + fi + psql_as_postgres "$UMBRA_DB_NAME" \ + -v admin_email="$UMBRA_BOOTSTRAP_ADMIN_EMAIL" \ + -v token_hash="$token_hash" \ + -v organization_name="$UMBRA_ORGANIZATION_NAME" \ + -v organization_slug="$UMBRA_ORGANIZATION_SLUG" <<'SQL' >/dev/null +INSERT INTO users (email, display_name, auth_provider, auth_subject, status, mfa_required) +VALUES (:'admin_email', :'admin_email', 'bootstrap', 'bootstrap:' || gen_random_uuid()::text, 'invited', true) +ON CONFLICT (email) DO UPDATE + SET display_name = EXCLUDED.display_name, + auth_provider = 'bootstrap', + mfa_required = true, + updated_at = now(); + +INSERT INTO organizations (name, slug) +VALUES (:'organization_name', :'organization_slug') +ON CONFLICT (slug) DO UPDATE SET name = EXCLUDED.name; + +INSERT INTO user_roles (user_id, role_id, scope, organization_id, project_id, created_by) +SELECT u.id, r.id, 'platform', NULL, NULL, u.id +FROM users u +JOIN roles r ON r.scope = 'platform' AND r.name = 'platform_admin' +WHERE u.email = :'admin_email' +ON CONFLICT DO NOTHING; + +INSERT INTO user_roles (user_id, role_id, scope, organization_id, project_id, created_by) +SELECT u.id, r.id, 'organization', o.id, NULL, u.id +FROM users u +JOIN organizations o ON o.slug = :'organization_slug' +JOIN roles r ON r.scope = 'organization' AND r.name = 'organization_admin' +WHERE u.email = :'admin_email' +ON CONFLICT DO NOTHING; + +INSERT INTO bootstrap_tokens (user_id, token_hash, expires_at) +SELECT id, :'token_hash', now() + interval '24 hours' +FROM users +WHERE email = :'admin_email' +ON CONFLICT (token_hash) DO NOTHING; +SQL + log "Bootstrap token stored root-only at $UMBRA_BOOTSTRAP_TOKEN_FILE" +} + +apply_mode() { + if ((DRY_RUN == 0)); then + require_root + log "Beginning phase-one enterprise bootstrap" + if ! mkdir "$LOCK_DIR" 2>/dev/null; then + die "Another bootstrap process holds $LOCK_DIR" + fi + trap 'rmdir "$LOCK_DIR" 2>/dev/null || true' EXIT + touch "$LOG_FILE" + chmod 0640 "$LOG_FILE" + else + log "Beginning phase-one enterprise bootstrap dry-run" + fi + install_packages + ensure_group_and_users + ensure_directories + ensure_uv + ensure_postgres_cluster + ensure_database_roles + ensure_database + apply_schema + write_nonsecret_runtime_config + create_bootstrap_admin + log "Phase-one bootstrap completed" + log "Application code, systemd units, Qwen, Docker, and firewall were not modified" +} + +status_mode() { + log "Umbra enterprise installation status" + printf 'host: '; hostname -f 2>/dev/null || hostname + printf 'dns: '; getent ahosts "$UMBRA_HOSTNAME" 2>/dev/null || true + printf 'postgres: '; command -v pg_lsclusters >/dev/null 2>&1 && pg_lsclusters || echo 'not installed' + printf 'api user: '; id "$UMBRA_API_USER" 2>/dev/null || echo 'not installed' + printf 'worker user: '; id "$UMBRA_WORKER_USER" 2>/dev/null || echo 'not installed' + systemctl is-active postgresql 2>/dev/null || true + printf 'qwen port: '; (curl -sS --max-time 3 "$UMBRA_QWEN_URL/health" 2>&1 | head -c 300) || true; printf '\n' +} + +verify_mode() { + log "Verifying phase-one installation" + require_command psql + require_root + pg_isready -h "$UMBRA_DB_HOST" -p "$UMBRA_DB_PORT" -d "$UMBRA_DB_NAME" + psql_as_postgres "$UMBRA_DB_NAME" -v ON_ERROR_STOP=1 -c 'SELECT version, applied_at FROM schema_migrations ORDER BY version' -c 'SELECT count(*) AS organizations FROM organizations' -c 'SELECT count(*) AS users FROM users' + systemctl is-active postgresql + log "Verification completed" +} + +main() { + load_config + validate_config + check_tools + case "$MODE" in + check) check_host ;; + apply) apply_mode ;; + status) status_mode ;; + verify) verify_mode ;; + *) die "Unknown mode: $MODE" ;; + esac +} + +main "$@" diff --git a/infra/bootstrap_umbracontrol_database.sh b/infra/bootstrap_umbracontrol_database.sh new file mode 100755 index 0000000..81b9717 --- /dev/null +++ b/infra/bootstrap_umbracontrol_database.sh @@ -0,0 +1,368 @@ +#!/usr/bin/env bash +# Database-only bootstrap for the UmbraTrans control plane. +# This intentionally does not install packages, create application directories, +# install application services, or modify Qwen/Docker/firewall settings. +set -Eeuo pipefail +umask 077 + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +CONFIG_FILE="${UMBRA_BOOTSTRAP_CONFIG:-${SCRIPT_DIR}/enterprise-bootstrap.env}" +MODE="check" +DRY_RUN=0 +LOCK_DIR="/run/lock/umbracontrol-database.lock" +LOG_FILE="/var/log/umbracontrol-database-bootstrap.log" +PG_SOCKET_DIR="/var/run/postgresql" + +UMBRA_DB_NAME="umbracontrol" +UMBRA_DB_HOST="127.0.0.1" +UMBRA_DB_PORT="5432" +UMBRA_BOOTSTRAP_ADMIN_EMAIL="umbratools@gmail.com" +UMBRA_ORGANIZATION_NAME="Umbra" +UMBRA_ORGANIZATION_SLUG="umbra" +UMBRA_PG_SUPERUSER="postgres" +UMBRA_CREDENTIALS_FILE="/etc/umbracontrol/db-credentials.env" +UMBRA_BOOTSTRAP_TOKEN_FILE="/root/umbra-enterprise-bootstrap.token" +UMBRA_SERVICE_GROUP="umbracontrol" +UMBRA_OWNER_ROLE="umbracontrol_owner" +UMBRA_MIGRATOR_ROLE="umbracontrol_migrator" +UMBRA_API_ROLE="umbracontrol_api" +UMBRA_WORKER_ROLE="umbracontrol_worker" +UMBRA_READONLY_ROLE="umbracontrol_readonly" +UMBRA_BACKUP_ROLE="umbracontrol_backup" + +usage() { + cat <<'EOF' +Usage: bootstrap_umbracontrol_database.sh [options] + +Options: + --check Read-only PostgreSQL and prerequisite inspection + --dry-run Show database actions without changing the host + --apply Create/finish the database, roles, schema, and bootstrap admin + --verify Verify the database, schema, RLS, and bootstrap records + --config FILE Read the existing non-secret enterprise-bootstrap.env + -h, --help Show this help + +This script is intentionally database-only. It does not run apt, install uv, +create application directories, start Qwen, or change firewall rules. +EOF +} + +while (($#)); do + case "$1" in + --check) MODE="check"; shift ;; + --dry-run) MODE="apply"; DRY_RUN=1; shift ;; + --apply) MODE="apply"; DRY_RUN=0; shift ;; + --verify) MODE="verify"; shift ;; + --config) CONFIG_FILE="${2:?--config requires a file}"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) printf 'Unknown option: %s\n' "$1" >&2; usage >&2; exit 2 ;; + esac +done + +log() { + local line="[$(date -Is)] $*" + printf '%s\n' "$line" + if [[ -f "$LOG_FILE" && "$(id -u)" -eq 0 ]]; then + printf '%s\n' "$line" >> "$LOG_FILE" + fi +} + +die() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +require_root() { + [[ "$(id -u)" -eq 0 ]] || die "This operation requires root. Use --check for read-only inspection." +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || die "Required command is missing: $1" +} + +load_config() { + [[ -f "$CONFIG_FILE" ]] || { + [[ "$MODE" == "check" || "$MODE" == "verify" ]] && return 0 + die "Configuration file not found: $CONFIG_FILE" + } + + local line key value + while IFS= read -r line || [[ -n "$line" ]]; do + line="${line#"${line%%[![:space:]]*}"}" + [[ -z "$line" || "${line:0:1}" == "#" ]] && continue + [[ "$line" == *=* ]] || die "Invalid configuration line: $line" + key="${line%%=*}" + value="${line#*=}" + key="${key//[[:space:]]/}" + value="${value%\"}"; value="${value#\"}" + value="${value%\'}"; value="${value#\'}" + case "$key" in + UMBRA_DB_NAME|UMBRA_DB_HOST|UMBRA_DB_PORT|UMBRA_BOOTSTRAP_ADMIN_EMAIL|UMBRA_ORGANIZATION_NAME|UMBRA_ORGANIZATION_SLUG) + printf -v "$key" '%s' "$value" + ;; + UMBRA_*) + : # Ignore unrelated non-secret enterprise settings. + ;; + *) + die "Unsupported configuration key: $key" + ;; + esac + done < "$CONFIG_FILE" +} + +validate_config() { + [[ "$UMBRA_DB_NAME" =~ ^[a-z_][a-z0-9_]*$ ]] || die "Invalid UMBRA_DB_NAME" + [[ "$UMBRA_DB_HOST" =~ ^[A-Za-z0-9._:-]+$ ]] || die "Invalid UMBRA_DB_HOST" + [[ "$UMBRA_DB_PORT" =~ ^[0-9]+$ && "$UMBRA_DB_PORT" -ge 1 && "$UMBRA_DB_PORT" -le 65535 ]] || die "Invalid UMBRA_DB_PORT" + [[ "$UMBRA_BOOTSTRAP_ADMIN_EMAIL" == *@*.* ]] || die "Invalid administrator email" + [[ "$UMBRA_ORGANIZATION_SLUG" =~ ^[a-z0-9][a-z0-9-]*$ ]] || die "Invalid organization slug" + [[ -d "$PG_SOCKET_DIR" ]] || die "PostgreSQL socket directory is missing: $PG_SOCKET_DIR" +} + +run() { + if ((DRY_RUN)); then + printf '[dry-run]' + printf ' %q' "$@" + printf '\n' + else + "$@" + fi +} + +random_secret() { + openssl rand -hex 32 +} + +read_secret() { + local key="$1" + [[ -f "$UMBRA_CREDENTIALS_FILE" ]] || return 1 + awk -F= -v key="$key" '$1 == key {print substr($0, index($0, "=") + 1); exit}' "$UMBRA_CREDENTIALS_FILE" +} + +write_secret() { + local key="$1" value="$2" temporary + if ((DRY_RUN)); then + printf '[dry-run] store generated database secret %s\n' "$key" + return + fi + install -d -m 0750 -o root -g "$UMBRA_SERVICE_GROUP" "$(dirname "$UMBRA_CREDENTIALS_FILE")" + touch "$UMBRA_CREDENTIALS_FILE" + chown root:"$UMBRA_SERVICE_GROUP" "$UMBRA_CREDENTIALS_FILE" + chmod 0640 "$UMBRA_CREDENTIALS_FILE" + temporary="$(mktemp "${UMBRA_CREDENTIALS_FILE}.XXXXXX")" + awk -F= -v key="$key" -v value="$value" ' + $1 == key { print key "=" value; found = 1; next } + { print } + END { if (!found) print key "=" value } + ' "$UMBRA_CREDENTIALS_FILE" > "$temporary" + chown root:"$UMBRA_SERVICE_GROUP" "$temporary" + chmod 0640 "$temporary" + mv -f "$temporary" "$UMBRA_CREDENTIALS_FILE" +} + +psql_as_postgres() { + runuser -u "$UMBRA_PG_SUPERUSER" -- env \ + -u PGHOST -u PGPORT -u PGDATABASE -u PGUSER -u PGPASSWORD \ + -u PGPASSWORDFILE -u PGSERVICE -u PGSERVICEFILE \ + PGOPTIONS='-c log_statement=none' \ + psql -h "$PG_SOCKET_DIR" -p "$UMBRA_DB_PORT" -U "$UMBRA_PG_SUPERUSER" \ + -v ON_ERROR_STOP=1 "$@" +} + +role_exists() { + local role="$1" + psql_as_postgres postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname = '$role'" | grep -q 1 +} + +ensure_role() { + local role="$1" login="$2" password_key="$3" password="" + if ((DRY_RUN)); then + printf '[dry-run] ensure PostgreSQL role %s (%s)\n' "$role" "$login" + return + fi + if role_exists "$role"; then + if [[ "$login" == "LOGIN" ]]; then + printf 'ALTER ROLE "%s" LOGIN;\n' "$role" | psql_as_postgres postgres >/dev/null + else + printf 'ALTER ROLE "%s" NOLOGIN;\n' "$role" | psql_as_postgres postgres >/dev/null + fi + log "PostgreSQL role already exists: $role" + return + fi + + if [[ "$login" == "LOGIN" ]]; then + password="$(read_secret "$password_key" 2>/dev/null || true)" + [[ -n "$password" ]] || password="$(random_secret)" + write_secret "$password_key" "$password" + printf 'CREATE ROLE "%s" LOGIN PASSWORD '\''%s'\'';\n' "$role" "$password" \ + | psql_as_postgres postgres >/dev/null + else + printf 'CREATE ROLE "%s" NOLOGIN;\n' "$role" | psql_as_postgres postgres >/dev/null + fi + log "PostgreSQL role created: $role" +} + +ensure_roles() { + log "Ensuring PostgreSQL roles" + ensure_role "$UMBRA_OWNER_ROLE" NOLOGIN OWNER_UNUSED + ensure_role "$UMBRA_MIGRATOR_ROLE" LOGIN UMBRACONTROL_MIGRATOR_PASSWORD + ensure_role "$UMBRA_API_ROLE" LOGIN UMBRACONTROL_API_PASSWORD + ensure_role "$UMBRA_WORKER_ROLE" LOGIN UMBRACONTROL_WORKER_PASSWORD + ensure_role "$UMBRA_READONLY_ROLE" LOGIN UMBRACONTROL_READONLY_PASSWORD + ensure_role "$UMBRA_BACKUP_ROLE" LOGIN UMBRACONTROL_BACKUP_PASSWORD +} + +database_exists() { + psql_as_postgres postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '$UMBRA_DB_NAME'" | grep -q 1 +} + +ensure_database() { + log "Ensuring database: $UMBRA_DB_NAME" + if ((DRY_RUN)); then + printf '[dry-run] create database if absent, owner=%s\n' "$UMBRA_OWNER_ROLE" + return + fi + if ! database_exists; then + runuser -u "$UMBRA_PG_SUPERUSER" -- createdb --owner="$UMBRA_OWNER_ROLE" "$UMBRA_DB_NAME" + log "Database created" + else + log "Database already exists" + fi + psql_as_postgres postgres -c "REVOKE ALL ON DATABASE \"$UMBRA_DB_NAME\" FROM PUBLIC" >/dev/null + psql_as_postgres postgres -c "GRANT CONNECT ON DATABASE \"$UMBRA_DB_NAME\" TO \"$UMBRA_MIGRATOR_ROLE\", \"$UMBRA_API_ROLE\", \"$UMBRA_WORKER_ROLE\", \"$UMBRA_READONLY_ROLE\", \"$UMBRA_BACKUP_ROLE\"" >/dev/null +} + +apply_schema() { + local schema_file="${SCRIPT_DIR}/postgres/001_control_plane.sql" + [[ -f "$schema_file" ]] || die "Schema file is missing: $schema_file" + log "Applying identity, tenancy, role, audit, and RLS schema" + if ((DRY_RUN)); then + printf '[dry-run] stream %s into PostgreSQL database %s\n' "$schema_file" "$UMBRA_DB_NAME" + return + fi + /bin/cat -- "$schema_file" | psql_as_postgres "$UMBRA_DB_NAME" >/dev/null + log "Database schema applied" +} + +create_bootstrap_admin() { + local token token_hash + if [[ -f "$UMBRA_BOOTSTRAP_TOKEN_FILE" ]]; then + token="$(cat "$UMBRA_BOOTSTRAP_TOKEN_FILE")" + else + token="$(random_secret)" + if ((DRY_RUN)); then + printf '[dry-run] create root-only bootstrap token\n' + return + fi + printf '%s\n' "$token" > "$UMBRA_BOOTSTRAP_TOKEN_FILE" + chown root:root "$UMBRA_BOOTSTRAP_TOKEN_FILE" + chmod 0600 "$UMBRA_BOOTSTRAP_TOKEN_FILE" + fi + [[ "$token" =~ ^[a-f0-9]{64}$ ]] || die "Bootstrap token has an unexpected format" + token_hash="$(printf '%s' "$token" | sha256sum | awk '{print $1}')" + + log "Ensuring bootstrap administrator" + if ((DRY_RUN)); then + printf '[dry-run] create administrator %s and organization %s\n' "$UMBRA_BOOTSTRAP_ADMIN_EMAIL" "$UMBRA_ORGANIZATION_NAME" + return + fi + psql_as_postgres "$UMBRA_DB_NAME" \ + -v admin_email="$UMBRA_BOOTSTRAP_ADMIN_EMAIL" \ + -v token_hash="$token_hash" \ + -v organization_name="$UMBRA_ORGANIZATION_NAME" \ + -v organization_slug="$UMBRA_ORGANIZATION_SLUG" <<'SQL' >/dev/null +INSERT INTO users (email, display_name, auth_provider, auth_subject, status, mfa_required) +VALUES (:'admin_email', :'admin_email', 'bootstrap', 'bootstrap:' || gen_random_uuid()::text, 'invited', true) +ON CONFLICT (email) DO UPDATE + SET display_name = EXCLUDED.display_name, + auth_provider = 'bootstrap', + mfa_required = true, + updated_at = now(); + +INSERT INTO organizations (name, slug) +VALUES (:'organization_name', :'organization_slug') +ON CONFLICT (slug) DO UPDATE SET name = EXCLUDED.name; + +INSERT INTO user_roles (user_id, role_id, scope, organization_id, project_id, created_by) +SELECT u.id, r.id, 'platform', NULL, NULL, u.id +FROM users u +JOIN roles r ON r.scope = 'platform' AND r.name = 'platform_admin' +WHERE u.email = :'admin_email' +ON CONFLICT DO NOTHING; + +INSERT INTO user_roles (user_id, role_id, scope, organization_id, project_id, created_by) +SELECT u.id, r.id, 'organization', o.id, NULL, u.id +FROM users u +JOIN organizations o ON o.slug = :'organization_slug' +JOIN roles r ON r.scope = 'organization' AND r.name = 'organization_admin' +WHERE u.email = :'admin_email' +ON CONFLICT DO NOTHING; + +INSERT INTO bootstrap_tokens (user_id, token_hash, expires_at) +SELECT id, :'token_hash', now() + interval '24 hours' +FROM users +WHERE email = :'admin_email' +ON CONFLICT (token_hash) DO NOTHING; +SQL + log "Bootstrap administrator is ready" +} + +check_mode() { + log "Database-only read-only inspection" + printf 'psql: '; command -v psql 2>/dev/null || echo 'missing' + printf 'PostgreSQL: '; command -v pg_lsclusters >/dev/null 2>&1 && pg_lsclusters || echo 'not installed' + printf 'service: '; systemctl is-active postgresql 2>/dev/null || true + printf 'socket: '; test -S "$PG_SOCKET_DIR/.s.PGSQL.$UMBRA_DB_PORT" && echo ready || echo missing + printf 'database socket: '; pg_isready -h "$PG_SOCKET_DIR" -p "$UMBRA_DB_PORT" -d "$UMBRA_DB_NAME" 2>&1 || true + printf 'roles: '; getent group "$UMBRA_SERVICE_GROUP" 2>/dev/null || echo 'service group absent' + log "No changes made" +} + +apply_mode() { + if ((DRY_RUN == 0)); then + require_root + mkdir "$LOCK_DIR" 2>/dev/null || die "Another database bootstrap is running" + trap 'rmdir "$LOCK_DIR" 2>/dev/null || true' EXIT + touch "$LOG_FILE" + chmod 0640 "$LOG_FILE" + else + log "Database-only bootstrap dry-run" + fi + ensure_roles + ensure_database + apply_schema + create_bootstrap_admin + log "Database-only bootstrap completed" +} + +verify_mode() { + require_root + log "Verifying database-only bootstrap" + pg_isready -h "$UMBRA_DB_HOST" -p "$UMBRA_DB_PORT" -d "$UMBRA_DB_NAME" + psql_as_postgres "$UMBRA_DB_NAME" -v ON_ERROR_STOP=1 \ + -c 'SELECT version, applied_at FROM schema_migrations ORDER BY version' \ + -c 'SELECT count(*) AS organizations FROM organizations' \ + -c 'SELECT count(*) AS users FROM users' \ + -c "SELECT count(*) AS rls_tables FROM pg_class WHERE relrowsecurity AND relname IN ('organizations','projects','users','user_roles','invitations','sessions','mfa_methods','bootstrap_tokens','audit_events')" + log "Database verification completed" +} + +main() { + load_config + validate_config + require_command openssl + require_command systemctl + if [[ "$MODE" == "apply" || "$MODE" == "verify" ]]; then + require_command psql + require_command pg_isready + require_command runuser + fi + case "$MODE" in + check) check_mode ;; + apply) apply_mode ;; + verify) verify_mode ;; + *) die "Unknown mode: $MODE" ;; + esac +} + +main diff --git a/infra/enterprise-bootstrap.env.example b/infra/enterprise-bootstrap.env.example new file mode 100644 index 0000000..1331ed4 --- /dev/null +++ b/infra/enterprise-bootstrap.env.example @@ -0,0 +1,20 @@ +# Non-secret phase-one bootstrap configuration. +# Copy to enterprise-bootstrap.env and review before running as root. +# Do not put passwords, private keys, or API secrets in this file. + +UMBRA_HOSTNAME=oldbox.bdmurphy.me +UMBRA_PUBLIC_URL=https://oldbox.bdmurphy.me +UMBRA_DB_NAME=umbracontrol +UMBRA_DB_HOST=127.0.0.1 +UMBRA_DB_PORT=5432 +UMBRA_PG_VERSION=16 +UMBRA_APP_ROOT=/opt/umbracontrol +UMBRA_DATA_ROOT=/var/lib/umbracontrol +UMBRA_BACKUP_ROOT=/var/backups/umbracontrol +UMBRA_BOOTSTRAP_ADMIN_EMAIL=umbratools@gmail.com +UMBRA_ORGANIZATION_NAME=Umbra +UMBRA_ORGANIZATION_SLUG=umbra +UMBRA_QWEN_URL=http://192.168.123.14:8881 +UMBRA_INSTALL_PACKAGES=1 +UMBRA_INSTALL_UV=1 +UMBRA_CREATE_BOOTSTRAP_ADMIN=1 diff --git a/infra/oldbox-agent-communication-prompt.md b/infra/oldbox-agent-communication-prompt.md new file mode 100644 index 0000000..22cac1e --- /dev/null +++ b/infra/oldbox-agent-communication-prompt.md @@ -0,0 +1,180 @@ +# Build a secure oldbox ↔ Mac agent communication MCP server + +You are the OpenCode agent running on `oldbox`. The Mac-side agent needs a +reliable way to exchange sanitized instructions and reports with you without +requiring repeated clipboard copy/paste. + +## Objective + +Create a standalone, local stdio MCP server named `agent-comm` under the +UmbraTrans workspace on oldbox. It must use a shared, append-only inbox/outbox +on the NAS2 `netbackup/UmbraTrans` share. Do not create an unauthenticated +network listener. + +The first purpose is to send the pending SSH-authentication diagnostic report +back to the Mac agent without exposing passwords or other secrets. + +## Security requirements + +- Never accept, return, log, or transmit passwords, sudo passwords, private + keys, API tokens, Keychain data, or `/etc/shadow` contents. +- No secret values may appear in MCP arguments, environment variables, command + lines, filenames, reports, or logs. +- The server must be stdio-only. It must not listen on a TCP or UDP port. +- Use the NAS share only for sanitized text/JSON messages. +- Message files must be written atomically (`temporary file` + `rename`). +- Use restrictive directory/file permissions and refuse symlink paths. +- Validate and bound all message sizes. +- Do not change SSH, sudoers, firewall, Docker, Qwen, or existing services. +- Do not install a system-wide daemon yet. +- Do not grant `NOPASSWD: ALL` or any broad sudo rule. + +## Discovery phase + +Before creating the server, perform read-only discovery: + +1. Locate the NAS2 mount for the `netbackup` share. +2. Locate or propose the `UmbraTrans` subdirectory. +3. Confirm the path is writable by `umbra`. +4. Report the exact path and permissions. +5. Do not write any message until the path is confirmed in the report. + +Use a conservative search such as `find`, `mount`, and `df`; do not modify +mounts or credentials. If the share is not visible, report that and stop rather +than guessing a path. + +## Shared mailbox layout + +Once the path is confirmed, use this layout below the confirmed UmbraTrans +directory: + +```text +agent-mailbox/ + inbox/ # sanitized instructions from Mac to oldbox + outbox/ # sanitized reports from oldbox to Mac + processed/ # messages already handled by oldbox + archive/ # optional processed message archive +``` + +Each message is one UTF-8 JSON file with this shape: + +```json +{ + "schema_version": 1, + "id": "UUID", + "created_at": "ISO-8601 UTC", + "sender": "oldbox", + "kind": "diagnostic|report|instruction|ack", + "correlation_id": "optional UUID", + "title": "short non-secret title", + "body": "sanitized text, maximum 64 KiB", + "attachments": [] +} +``` + +Do not allow arbitrary attachment paths in version 1. + +## MCP tools + +Implement these tools: + +### `agent_status` + +Return only: + +- server version +- configured mailbox path +- whether inbox/outbox are readable/writable +- number of pending messages +- last health-check time +- no secret values + +### `send_report` + +Arguments: + +- `title`: non-secret string +- `body`: sanitized report text +- `correlation_id`: optional non-secret string +- `kind`: default `report` + +Write a validated JSON message to `outbox/` using an atomic rename. Return the +message ID and path, never the secret contents. + +### `read_instructions` + +Read up to a bounded number of JSON messages from `inbox/`. Validate each +message, return only sanitized metadata/body, and move successfully handled +messages to `processed/` using an atomic rename. Never execute commands from a +message automatically. + +### `send_ack` + +Accept only a correlation ID and short non-secret text. Write an `ack` message +to `outbox/`. + +## Current diagnostic report + +After the server is implemented and tested, use it to send one report covering: + +- the successful manual SSH path +- the broker's `Permission denied (publickey,password)` result +- the fact that no authorized key or certificate is configured +- the account/password state already observed +- the exact unresolved question: whether the broker supplied a wrong password + or supplied no usable password to OpenSSH + +Do not include any password or hash. Do not ask the Mac agent to guess. + +## Implementation requirements + +- Python 3.11+ standard library is sufficient; do not add dependencies unless + necessary. +- Use type hints, `from __future__ import annotations`, logging to stderr, and + no `print()` for debug output except the CLI's intentional status output. +- Include a README with the exact OpenCode local-server configuration. +- Include unit tests for JSON validation, atomic writes, path traversal, + symlink refusal, size limits, inbox/outbox behavior, and secret-pattern + redaction. +- Add a `--check` mode that performs only read-only checks. +- Run the tests and `--check` before sending the first report. + +## OpenCode configuration to document + +The configuration should be equivalent to: + +```json +{ + "mcp": { + "servers": { + "agent-comm": { + "type": "local", + "command": [ + "/absolute/path/to/python", + "-m", + "agent_comm" + ], + "environment": { + "AGENT_COMM_ROOT": "/confirmed/nas2/path/agent-mailbox" + } + } + } + } +} +``` + +Do not add secrets to the configuration. + +## Completion response + +Return a concise report containing: + +1. Confirmed mailbox path and permissions. +2. Files created. +3. Test results. +4. OpenCode configuration path/snippet. +5. The first outbox report message ID. +6. Any unresolved issue. + +Do not claim the Mac agent has received the message until the Mac-side reader +has been configured and verified. diff --git a/infra/oldbox-bootstrap-instructions.md b/infra/oldbox-bootstrap-instructions.md new file mode 100644 index 0000000..8613886 --- /dev/null +++ b/infra/oldbox-bootstrap-instructions.md @@ -0,0 +1,96 @@ +# Oldbox enterprise bootstrap instructions + +You are the OpenCode agent running on `oldbox`. Work locally on the target +host; do not wait for clipboard reports from the Mac. + +Read and execute these files first: + +```text +/tmp/oldbox-temporary-access-prompt.md +/tmp/oldbox-agent-communication-prompt.md +``` + +Then complete the following local bootstrap: + +## 1. Preserve the existing host + +Inventory, but do not alter: + +- Qwen and GPU services +- Docker containers +- MySQL +- Samba/Webmin/XRDP +- PostgreSQL +- firewall rules +- existing user accounts and data + +Report conflicts before changing anything. + +## 2. Create the development workspace + +Use: + +```text +/home/umbra/src/UmbraTrans +``` + +Create it with ownership `umbra:umbra`. Do not modify UmbraAudio files. +If the enterprise source bundle is available locally, copy it there; do not +invent application code or overwrite existing source. + +Prepare: + +- Git repository/worktree +- `.venv` managed by the approved `uv` workflow +- non-secret `.env` template +- test/lint/format commands from the project configuration +- an explicit development and production data separation + +Run the project’s tests and record the results. + +## 3. Configure backups + +Use only the NAS2 `netbackup/UmbraTrans` share after confirming the oldbox +account can access it. Create: + +```text +backups/postgres/ +backups/config/ +backups/workspace/ +backups/manifests/ +``` + +PostgreSQL backups must be logical dumps or an approved backup tool. Never +copy a live PostgreSQL data directory or SQLite database from a running +service as a backup. + +Use atomic temporary files, restrictive permissions, timestamps, checksums, +and a retention policy. Do not put credentials in the backup path or report. + +## 4. Establish the agent communication queue + +Use the `agent-comm` server from the communication prompt. If NAS permissions +are not yet correct, use a private local queue under the oldbox workspace and +report the exact blocker; do not broaden share permissions automatically. + +The queue is for sanitized instructions and reports only. It must never carry +passwords, tokens, private keys, or unrestricted shell commands. + +## 5. Report + +Write a concise, machine-readable status report to the queue and print: + +```text +OLDBOX_BOOTSTRAP_COMPLETE +``` + +Include: + +- workspace path and ownership +- test result +- backup path and last backup status +- communication queue path/status +- unresolved blockers +- confirmation that existing services were preserved + +Do not claim completion for any step that was not actually verified. diff --git a/infra/oldbox-ssh-auth-log-prompt.md b/infra/oldbox-ssh-auth-log-prompt.md new file mode 100644 index 0000000..9ef09a9 --- /dev/null +++ b/infra/oldbox-ssh-auth-log-prompt.md @@ -0,0 +1,61 @@ +# Oldbox SSH authentication-log handoff + +A Mac-side broker SSH attempt was just made against `umbra@oldbox.bdmurphy.me`. +It received `Permission denied (publickey,password)`. + +You are the OpenCode agent running on oldbox. Perform only read-only +authentication diagnostics now. + +Do not change SSH configuration. +Do not restart SSH. +Do not install anything. +Do not read or print passwords, hashes, private keys, or tokens. +If a sudo command prompts, the human operator may enter the sudo password +locally. Never ask for or transmit that password through chat or the report. + +Run these commands as needed: + +```bash +sudo journalctl -u ssh --since "3 minutes ago" --no-pager +sudo journalctl -u sshd --since "3 minutes ago" --no-pager +sudo /usr/sbin/sshd -T -C user=umbra,addr=192.168.123.12,host=oldbox.bdmurphy.me +sudo passwd -S umbra +``` + +If journalctl is not the active service name, use the service that owns port +22. If `sshd -T -C` is unavailable, report the exact error. + +Filter the output to authentication-relevant lines only, especially: + +- Accepted password +- Failed password +- Failed publickey +- Invalid user +- account locked +- PAM/session-open failures +- certificate errors +- source address and username + +Return exactly this compact report: + +```text +=== AUTH LOG RESULT === + + +=== EFFECTIVE SSH AUTH === + + +=== ACCOUNT STATE === + + +=== CONCLUSION === + + +=== NEXT ACTION === + + +READY_FOR_MAC_AGENT +``` + +Do not guess. Do not make changes. Do not include secret values. diff --git a/infra/oldbox-ssh-diagnostic-prompt.md b/infra/oldbox-ssh-diagnostic-prompt.md new file mode 100644 index 0000000..5a567d6 --- /dev/null +++ b/infra/oldbox-ssh-diagnostic-prompt.md @@ -0,0 +1,129 @@ +# Oldbox SSH diagnostic handoff + +You are the on-host investigator for `oldbox.bdmurphy.me`. + +## Goal + +Determine why direct interactive SSH as `umbra` succeeds from the Mac, while the Mac-side remote broker receives: + +```text +Permission denied (publickey,password) +``` + +## Known facts + +- Host: `oldbox.bdmurphy.me` +- Address: `192.168.123.14` +- User: `umbra` +- The Mac can run: `ssh umbra@oldbox` +- The Mac broker is attempting SSH as `umbra@oldbox` +- Do not change SSH configuration. +- Do not restart SSH. +- Do not install packages. +- Do not read or print passwords, private keys, tokens, `/etc/shadow` contents, Keychain data, or environment secrets. +- Do not ask me to provide any password in chat. +- Use `sudo` only for read-only commands. Prefer `sudo -n`; if `sudo -n` is not permitted, report that limitation instead of prompting for a password. + +## Read-only inspection + +### 1. Record host and account state + +Collect: + +- Hostname and kernel +- SSH service state +- `id umbra` +- `getent passwd umbra` +- Home-directory permissions +- Current `umbra` shell + +### 2. Inspect SSH policy + +Run, where permitted: + +```bash +sshd -T +``` + +If permitted: + +```bash +sshd -T -C user=umbra,addr=192.168.123.1,host=oldbox.bdmurphy.me +``` + +Search `/etc/ssh/sshd_config` and `/etc/ssh/sshd_config.d` for: + +- `Match` +- `AuthenticationMethods` +- `PasswordAuthentication` +- `KbdInteractiveAuthentication` +- `UsePAM` +- `PubkeyAuthentication` +- `AllowUsers` +- `DenyUsers` +- Certificate-related directives + +### 3. Inspect account state + +If permitted, collect: + +```bash +passwd -S umbra +chage -l umbra +``` + +Report account lock or expiry information without printing hashes. + +### 4. Inspect recent SSH authentication logs + +Where readable, inspect: + +```bash +journalctl -u ssh --since "15 minutes ago" --no-pager +journalctl -u sshd --since "15 minutes ago" --no-pager +``` + +Also inspect `/var/log/auth.log` or `/var/log/secure` if readable. + +Report whether failures mention: + +- failed password +- invalid user +- account locked +- failed public key +- certificate problem +- PAM failure +- or another condition + +### 5. Determine the likely authentication path + +Determine whether the successful Mac login is likely password-based or key/certificate-based. + +Do not attempt to log in from oldbox. Do not inspect private-key contents. Base the conclusion only on SSH policy, account state, and server logs. + +## Required report format + +Return exactly one report with these headings: + +```text +=== IDENTITY === +=== SSH SERVICE === +=== SSH EFFECTIVE CONFIG === +=== ACCOUNT STATE === +=== AUTHENTICATION LOGS === +=== CONCLUSION === +=== NEXT DIAGNOSTIC === +``` + +## Rules + +- Redact sensitive values while retaining enough information to distinguish the authentication path. +- Never include secret values. +- Do not make changes. +- If a command is denied, report the command and permission error. +- If evidence is insufficient, say so rather than guessing. +- At the end, print: + +```text +READY_FOR_MAC_AGENT +``` diff --git a/infra/oldbox-temporary-access-prompt.md b/infra/oldbox-temporary-access-prompt.md new file mode 100644 index 0000000..1ae8ac7 --- /dev/null +++ b/infra/oldbox-temporary-access-prompt.md @@ -0,0 +1,115 @@ +# One-hour temporary UmbraTrans access setup + +You are the OpenCode agent running on `oldbox`. The human operator has +authorized a one-hour temporary access window for finishing the UmbraTrans +workspace and communication setup. + +## Goal + +Create a temporary, unprivileged Linux account for Mac-side automation, using +a password entered locally in a hidden prompt. The Mac broker will use the +same one-hour credential for SSH. After one hour, the account and credential +must no longer work. + +## Account + +Use this exact temporary username: + +```text +umbracontrol-temp +``` + +Create it with: + +- home `/home/umbracontrol-temp`, mode `0700` +- shell `/bin/bash` +- no membership in `sudo`, `docker`, `mysql`, `lpadmin`, or other privileged + groups +- account expiry exactly one hour from creation +- a locally entered hidden password; never print, log, echo, or place it in a + file, command argument, environment variable, report, or MCP message + +Use a local hidden prompt in the oldbox terminal. For example, adapt this +pattern safely in the agent's shell: + +```bash +read -rsp 'Temporary UmbraTrans password: ' TEMP_PASSWORD +printf '\n' +test -n "$TEMP_PASSWORD" +``` + +Pass the value to `chpasswd` through standard input, not an argument. Clear +the shell variable immediately afterward. Do not repeat the value in output. + +Set the account password and expiration with `sudo` locally. The human may +enter the sudo password in the terminal; never request it in chat. + +Schedule cleanup for one hour using a root-owned transient systemd timer or +an equivalent root-owned one-shot cleanup. Cleanup must: + +- lock and delete `umbracontrol-temp` +- remove `/home/umbracontrol-temp` +- remove any temporary authorized-key or agent files +- leave `umbra`, Qwen, Docker, MySQL, Samba, Webmin, XRDP, and PostgreSQL + unchanged + +Verify before reporting: + +```bash +id umbracontrol-temp +getent passwd umbracontrol-temp +passwd -S umbracontrol-temp +``` + +Do not print hashes or secret values. + +## Temporary Mac broker profile + +After creating the account, write a **sanitized** report containing: + +- username `umbracontrol-temp` +- host `oldbox.bdmurphy.me` +- port `22` +- account expiry timestamp +- cleanup mechanism/timer name +- home path and permissions + +Do not include the temporary password in the report. The Mac operator will +enter the same one-hour password through the local secure popup. + +## Workspace + +Use the temporary account only for safe, unprivileged workspace operations: + +- create `/home/umbracontrol-temp/UmbraTrans` +- report the path and ownership +- do not alter `/home/umbra` ownership or permissions +- do not modify existing services +- do not grant sudo +- do not install packages +- do not change firewall rules + +Root-required application/database/backup work remains a separate explicit +operation after the temporary SSH path is verified. + +## Sanitized diagnostic report + +Send the report through the local `agent-comm` MCP server if it is already +available. Otherwise write it atomically to: + +```text +/tmp/umbracontrol-temp-setup-report.json +``` + +The report may contain only non-secret account, expiry, permission, and +service-status data. Use mode `0600`. + +At the end print: + +```text +TEMP_SETUP_READY +READY_FOR_MAC_AGENT +``` + +Do not claim the Mac has connected until the Mac broker reports a successful +SSH status probe. diff --git a/infra/postgres/001_control_plane.sql b/infra/postgres/001_control_plane.sql new file mode 100644 index 0000000..5e391a9 --- /dev/null +++ b/infra/postgres/001_control_plane.sql @@ -0,0 +1,584 @@ +BEGIN; + +CREATE EXTENSION IF NOT EXISTS citext; +CREATE EXTENSION IF NOT EXISTS pgcrypto; +CREATE SCHEMA IF NOT EXISTS app; + +CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS organizations ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + name text NOT NULL, + slug citext NOT NULL UNIQUE, + status text NOT NULL DEFAULT 'active' + CHECK (status IN ('active', 'suspended', 'archived')), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS users ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + email citext NOT NULL UNIQUE, + display_name text, + auth_provider text NOT NULL DEFAULT 'bootstrap', + auth_subject text, + status text NOT NULL DEFAULT 'invited' + CHECK (status IN ('invited', 'active', 'suspended', 'deactivated', 'locked')), + mfa_required boolean NOT NULL DEFAULT false, + last_login_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE UNIQUE INDEX IF NOT EXISTS users_auth_subject_idx + ON users (auth_provider, auth_subject) + WHERE auth_subject IS NOT NULL; + +CREATE TABLE IF NOT EXISTS projects ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + organization_id uuid NOT NULL REFERENCES organizations(id), + name text NOT NULL, + slug citext NOT NULL, + status text NOT NULL DEFAULT 'active' + CHECK (status IN ('active', 'on_hold', 'released', 'archived')), + created_by uuid REFERENCES users(id), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (organization_id, slug), + UNIQUE (id, organization_id) +); + +CREATE TABLE IF NOT EXISTS roles ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + name text NOT NULL, + scope text NOT NULL CHECK (scope IN ('platform', 'organization', 'project')), + description text NOT NULL DEFAULT '', + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (scope, name) +); + +CREATE TABLE IF NOT EXISTS permissions ( + key text PRIMARY KEY, + description text NOT NULL DEFAULT '' +); + +CREATE TABLE IF NOT EXISTS role_permissions ( + role_id uuid NOT NULL REFERENCES roles(id) ON DELETE CASCADE, + permission_key text NOT NULL REFERENCES permissions(key) ON DELETE CASCADE, + PRIMARY KEY (role_id, permission_key) +); + +CREATE TABLE IF NOT EXISTS user_roles ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + role_id uuid NOT NULL REFERENCES roles(id) ON DELETE RESTRICT, + scope text NOT NULL CHECK (scope IN ('platform', 'organization', 'project')), + organization_id uuid REFERENCES organizations(id) ON DELETE CASCADE, + project_id uuid, + status text NOT NULL DEFAULT 'active' + CHECK (status IN ('active', 'suspended', 'revoked')), + created_at timestamptz NOT NULL DEFAULT now(), + created_by uuid REFERENCES users(id), + CHECK ( + (scope = 'platform' AND organization_id IS NULL AND project_id IS NULL) + OR (scope = 'organization' AND organization_id IS NOT NULL AND project_id IS NULL) + OR (scope = 'project' AND organization_id IS NOT NULL AND project_id IS NOT NULL) + ), + FOREIGN KEY (project_id, organization_id) + REFERENCES projects(id, organization_id) ON DELETE CASCADE +); + +CREATE UNIQUE INDEX IF NOT EXISTS user_roles_platform_unique + ON user_roles (user_id, role_id) + WHERE scope = 'platform'; +CREATE UNIQUE INDEX IF NOT EXISTS user_roles_organization_unique + ON user_roles (user_id, role_id, organization_id) + WHERE scope = 'organization'; +CREATE UNIQUE INDEX IF NOT EXISTS user_roles_project_unique + ON user_roles (user_id, role_id, project_id) + WHERE scope = 'project'; + +CREATE TABLE IF NOT EXISTS invitations ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + organization_id uuid NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, + project_id uuid, + email citext NOT NULL, + role_id uuid NOT NULL REFERENCES roles(id) ON DELETE RESTRICT, + token_hash text NOT NULL UNIQUE, + invited_by uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL, + accepted_at timestamptz, + revoked_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + FOREIGN KEY (project_id, organization_id) + REFERENCES projects(id, organization_id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS sessions ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + token_hash text NOT NULL UNIQUE, + created_at timestamptz NOT NULL DEFAULT now(), + expires_at timestamptz NOT NULL, + revoked_at timestamptz, + ip_address inet, + user_agent text +); + +CREATE TABLE IF NOT EXISTS mfa_methods ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + method text NOT NULL CHECK (method IN ('totp', 'webauthn', 'recovery_code')), + secret_reference text, + verified_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + last_used_at timestamptz, + revoked_at timestamptz +); + +CREATE TABLE IF NOT EXISTS bootstrap_tokens ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + token_hash text NOT NULL UNIQUE, + expires_at timestamptz NOT NULL, + consumed_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS audit_events ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + actor_user_id uuid REFERENCES users(id), + organization_id uuid REFERENCES organizations(id), + project_id uuid, + action text NOT NULL, + target_type text, + target_id text, + request_id text, + ip_address inet, + metadata jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS projects_organization_idx ON projects (organization_id); +CREATE INDEX IF NOT EXISTS user_roles_user_idx ON user_roles (user_id, status); +CREATE INDEX IF NOT EXISTS user_roles_organization_idx ON user_roles (organization_id, status); +CREATE INDEX IF NOT EXISTS user_roles_project_idx ON user_roles (project_id, status); +CREATE INDEX IF NOT EXISTS invitations_email_idx ON invitations (email); +CREATE INDEX IF NOT EXISTS audit_events_org_created_idx ON audit_events (organization_id, created_at DESC); +CREATE INDEX IF NOT EXISTS audit_events_project_created_idx ON audit_events (project_id, created_at DESC); + +CREATE OR REPLACE FUNCTION app.validate_user_role_scope() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +DECLARE + role_scope text; +BEGIN + SELECT scope INTO role_scope FROM roles WHERE id = NEW.role_id; + IF role_scope IS NULL OR role_scope <> NEW.scope THEN + RAISE EXCEPTION 'role scope does not match assignment scope'; + END IF; + RETURN NEW; +END; +$$; + +DROP TRIGGER IF EXISTS user_roles_validate_scope ON user_roles; +CREATE TRIGGER user_roles_validate_scope +BEFORE INSERT OR UPDATE OF role_id, scope ON user_roles +FOR EACH ROW EXECUTE FUNCTION app.validate_user_role_scope(); + +CREATE OR REPLACE FUNCTION app.set_updated_at() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + NEW.updated_at = now(); + RETURN NEW; +END; +$$; + +DROP TRIGGER IF EXISTS organizations_set_updated_at ON organizations; +CREATE TRIGGER organizations_set_updated_at +BEFORE UPDATE ON organizations +FOR EACH ROW EXECUTE FUNCTION app.set_updated_at(); + +DROP TRIGGER IF EXISTS users_set_updated_at ON users; +CREATE TRIGGER users_set_updated_at +BEFORE UPDATE ON users +FOR EACH ROW EXECUTE FUNCTION app.set_updated_at(); + +DROP TRIGGER IF EXISTS projects_set_updated_at ON projects; +CREATE TRIGGER projects_set_updated_at +BEFORE UPDATE ON projects +FOR EACH ROW EXECUTE FUNCTION app.set_updated_at(); + +CREATE OR REPLACE FUNCTION app.current_user_id() +RETURNS uuid +LANGUAGE sql +STABLE +AS $$ + SELECT NULLIF(current_setting('app.user_id', true), '')::uuid +$$; + +CREATE OR REPLACE FUNCTION app.is_platform_admin() +RETURNS boolean +LANGUAGE sql +STABLE +SECURITY DEFINER +SET search_path = public, pg_temp +AS $$ + SELECT EXISTS ( + SELECT 1 + FROM user_roles ur + JOIN roles r ON r.id = ur.role_id + WHERE ur.user_id = app.current_user_id() + AND ur.scope = 'platform' + AND ur.status = 'active' + AND r.name = 'platform_admin' + ) +$$; + +CREATE OR REPLACE FUNCTION app.is_organization_member(target_organization_id uuid) +RETURNS boolean +LANGUAGE sql +STABLE +SECURITY DEFINER +SET search_path = public, pg_temp +AS $$ + SELECT EXISTS ( + SELECT 1 + FROM user_roles ur + JOIN roles r ON r.id = ur.role_id + WHERE ur.user_id = app.current_user_id() + AND ur.organization_id = target_organization_id + AND ur.scope = 'organization' + AND ur.status = 'active' + AND r.name IN ('organization_admin', 'member') + ) +$$; + +CREATE OR REPLACE FUNCTION app.is_organization_admin(target_organization_id uuid) +RETURNS boolean +LANGUAGE sql +STABLE +SECURITY DEFINER +SET search_path = public, pg_temp +AS $$ + SELECT EXISTS ( + SELECT 1 + FROM user_roles ur + JOIN roles r ON r.id = ur.role_id + WHERE ur.user_id = app.current_user_id() + AND ur.organization_id = target_organization_id + AND ur.scope = 'organization' + AND ur.status = 'active' + AND r.name = 'organization_admin' + ) +$$; + +CREATE OR REPLACE FUNCTION app.is_organization_admin_for_project(target_project_id uuid) +RETURNS boolean +LANGUAGE sql +STABLE +SECURITY DEFINER +SET search_path = public, pg_temp +AS $$ + SELECT app.is_platform_admin() + OR EXISTS ( + SELECT 1 + FROM projects p + WHERE p.id = target_project_id + AND app.is_organization_admin(p.organization_id) + ) +$$; + +CREATE OR REPLACE FUNCTION app.can_access_project(target_project_id uuid) +RETURNS boolean +LANGUAGE sql +STABLE +SECURITY DEFINER +SET search_path = public, pg_temp +AS $$ + SELECT app.is_platform_admin() + OR EXISTS ( + SELECT 1 + FROM projects p + WHERE p.id = target_project_id + AND app.is_organization_admin(p.organization_id) + ) + OR EXISTS ( + SELECT 1 + FROM user_roles ur + WHERE ur.user_id = app.current_user_id() + AND ur.project_id = target_project_id + AND ur.scope = 'project' + AND ur.status = 'active' + ) +$$; + +CREATE OR REPLACE FUNCTION app.can_view_user(target_user_id uuid) +RETURNS boolean +LANGUAGE sql +STABLE +SECURITY DEFINER +SET search_path = public, pg_temp +AS $$ + SELECT target_user_id = app.current_user_id() + OR app.is_platform_admin() + OR EXISTS ( + SELECT 1 + FROM user_roles mine + JOIN user_roles theirs + ON theirs.user_id = target_user_id + AND theirs.organization_id = mine.organization_id + AND theirs.scope = 'organization' + AND theirs.status = 'active' + JOIN roles r ON r.id = mine.role_id + WHERE mine.user_id = app.current_user_id() + AND mine.scope = 'organization' + AND mine.status = 'active' + AND r.name = 'organization_admin' + ) +$$; + +CREATE OR REPLACE FUNCTION app.prevent_audit_mutation() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + RAISE EXCEPTION 'audit_events is append-only'; +END; +$$; + +DROP TRIGGER IF EXISTS audit_events_append_only ON audit_events; +CREATE TRIGGER audit_events_append_only +BEFORE UPDATE OR DELETE ON audit_events +FOR EACH ROW EXECUTE FUNCTION app.prevent_audit_mutation(); + +INSERT INTO schema_migrations(version) VALUES ('001_control_plane') +ON CONFLICT (version) DO NOTHING; + +INSERT INTO roles (name, scope, description) VALUES + ('platform_admin', 'platform', 'Global control-plane administration'), + ('organization_admin', 'organization', 'Administration within one organization'), + ('member', 'organization', 'Read-only organization membership'), + ('project_owner', 'project', 'Project administration'), + ('project_manager', 'project', 'Project production management'), + ('translator', 'project', 'Translation work'), + ('proofreader', 'project', 'Translation review'), + ('audio_director', 'project', 'Voice and audio approval'), + ('voice_manager', 'organization', 'Voice library and licensing management'), + ('release_manager', 'project', 'Release approval'), + ('viewer', 'project', 'Read-only project access') +ON CONFLICT (scope, name) DO UPDATE SET description = EXCLUDED.description; + +INSERT INTO permissions (key, description) VALUES + ('platform.manage', 'Manage platform-level resources'), + ('organization.manage', 'Manage organization settings'), + ('organization.users.manage', 'Invite and manage organization users'), + ('organization.projects.manage', 'Create and manage projects'), + ('project.access.manage', 'Assign project access'), + ('project.source.edit', 'Edit approved source material'), + ('project.translation.edit', 'Edit target translations'), + ('project.translation.review', 'Review target translations'), + ('project.audio.approve', 'Approve audio output'), + ('project.release.approve', 'Approve releases'), + ('voice.catalog.manage', 'Manage voice catalog and licensing'), + ('artifact.download', 'Download authorized artifacts') +ON CONFLICT (key) DO UPDATE SET description = EXCLUDED.description; + +INSERT INTO role_permissions (role_id, permission_key) +SELECT r.id, p.key +FROM roles r +JOIN ( + VALUES + ('platform_admin', 'platform.manage'), + ('organization_admin', 'organization.manage'), + ('organization_admin', 'organization.users.manage'), + ('organization_admin', 'organization.projects.manage'), + ('organization_admin', 'project.access.manage'), + ('member', 'artifact.download'), + ('project_owner', 'project.access.manage'), + ('project_owner', 'project.release.approve'), + ('project_manager', 'project.translation.edit'), + ('project_manager', 'project.audio.approve'), + ('translator', 'project.translation.edit'), + ('proofreader', 'project.translation.review'), + ('audio_director', 'project.audio.approve'), + ('voice_manager', 'voice.catalog.manage'), + ('release_manager', 'project.release.approve'), + ('viewer', 'artifact.download') +) AS grants(role_name, permission_key) + ON grants.role_name = r.name +JOIN permissions p ON p.key = grants.permission_key +ON CONFLICT DO NOTHING; + +ALTER TABLE roles ENABLE ROW LEVEL SECURITY; +ALTER TABLE permissions ENABLE ROW LEVEL SECURITY; +ALTER TABLE role_permissions ENABLE ROW LEVEL SECURITY; + +DROP POLICY IF EXISTS roles_read ON roles; +CREATE POLICY roles_read ON roles + FOR SELECT USING (app.current_user_id() IS NOT NULL OR app.is_platform_admin()); +DROP POLICY IF EXISTS roles_manage ON roles; +CREATE POLICY roles_manage ON roles + FOR ALL USING (app.is_platform_admin()) + WITH CHECK (app.is_platform_admin()); + +DROP POLICY IF EXISTS permissions_read ON permissions; +CREATE POLICY permissions_read ON permissions + FOR SELECT USING (app.current_user_id() IS NOT NULL OR app.is_platform_admin()); +DROP POLICY IF EXISTS permissions_manage ON permissions; +CREATE POLICY permissions_manage ON permissions + FOR ALL USING (app.is_platform_admin()) + WITH CHECK (app.is_platform_admin()); + +DROP POLICY IF EXISTS role_permissions_read ON role_permissions; +CREATE POLICY role_permissions_read ON role_permissions + FOR SELECT USING (app.current_user_id() IS NOT NULL OR app.is_platform_admin()); +DROP POLICY IF EXISTS role_permissions_manage ON role_permissions; +CREATE POLICY role_permissions_manage ON role_permissions + FOR ALL USING (app.is_platform_admin()) + WITH CHECK (app.is_platform_admin()); + +ALTER TABLE organizations ENABLE ROW LEVEL SECURITY; +ALTER TABLE projects ENABLE ROW LEVEL SECURITY; +ALTER TABLE users ENABLE ROW LEVEL SECURITY; +ALTER TABLE user_roles ENABLE ROW LEVEL SECURITY; +ALTER TABLE invitations ENABLE ROW LEVEL SECURITY; +ALTER TABLE sessions ENABLE ROW LEVEL SECURITY; +ALTER TABLE mfa_methods ENABLE ROW LEVEL SECURITY; +ALTER TABLE bootstrap_tokens ENABLE ROW LEVEL SECURITY; +ALTER TABLE audit_events ENABLE ROW LEVEL SECURITY; + +DROP POLICY IF EXISTS organizations_select ON organizations; +CREATE POLICY organizations_select ON organizations + FOR SELECT USING (app.is_platform_admin() OR app.is_organization_member(id)); +DROP POLICY IF EXISTS organizations_insert ON organizations; +CREATE POLICY organizations_insert ON organizations + FOR INSERT WITH CHECK (app.is_platform_admin()); +DROP POLICY IF EXISTS organizations_update ON organizations; +CREATE POLICY organizations_update ON organizations + FOR UPDATE USING (app.is_platform_admin() OR app.is_organization_admin(id)) + WITH CHECK (app.is_platform_admin() OR app.is_organization_admin(id)); +DROP POLICY IF EXISTS organizations_delete ON organizations; +CREATE POLICY organizations_delete ON organizations + FOR DELETE USING (app.is_platform_admin()); + +DROP POLICY IF EXISTS projects_select ON projects; +CREATE POLICY projects_select ON projects + FOR SELECT USING (app.can_access_project(id)); +DROP POLICY IF EXISTS projects_insert ON projects; +CREATE POLICY projects_insert ON projects + FOR INSERT WITH CHECK (app.is_platform_admin() OR app.is_organization_admin(organization_id)); +DROP POLICY IF EXISTS projects_update ON projects; +CREATE POLICY projects_update ON projects + FOR UPDATE USING (app.is_platform_admin() OR app.is_organization_admin(organization_id)) + WITH CHECK (app.is_platform_admin() OR app.is_organization_admin(organization_id)); +DROP POLICY IF EXISTS projects_delete ON projects; +CREATE POLICY projects_delete ON projects + FOR DELETE USING (app.is_platform_admin() OR app.is_organization_admin(organization_id)); + +DROP POLICY IF EXISTS users_select ON users; +CREATE POLICY users_select ON users + FOR SELECT USING (app.can_view_user(id)); +DROP POLICY IF EXISTS users_manage ON users; +CREATE POLICY users_manage ON users + FOR ALL USING (app.is_platform_admin()) + WITH CHECK (app.is_platform_admin()); + +DROP POLICY IF EXISTS user_roles_select ON user_roles; +CREATE POLICY user_roles_select ON user_roles + FOR SELECT USING ( + app.is_platform_admin() + OR user_id = app.current_user_id() + OR (scope = 'organization' AND app.is_organization_admin(organization_id)) + OR (scope = 'project' AND app.is_organization_admin_for_project(project_id)) + ); +DROP POLICY IF EXISTS user_roles_manage ON user_roles; +CREATE POLICY user_roles_manage ON user_roles + FOR ALL USING ( + app.is_platform_admin() + OR (scope = 'organization' AND app.is_organization_admin(organization_id)) + OR (scope = 'project' AND app.is_organization_admin_for_project(project_id)) + ) + WITH CHECK ( + app.is_platform_admin() + OR (scope = 'organization' AND app.is_organization_admin(organization_id)) + OR (scope = 'project' AND app.is_organization_admin_for_project(project_id)) + ); + +DROP POLICY IF EXISTS invitations_select ON invitations; +CREATE POLICY invitations_select ON invitations + FOR SELECT USING ( + app.is_platform_admin() + OR app.is_organization_admin(organization_id) + OR email = (SELECT email FROM users WHERE id = app.current_user_id()) + ); +DROP POLICY IF EXISTS invitations_manage ON invitations; +CREATE POLICY invitations_manage ON invitations + FOR ALL USING (app.is_platform_admin() OR app.is_organization_admin(organization_id)) + WITH CHECK (app.is_platform_admin() OR app.is_organization_admin(organization_id)); + +DROP POLICY IF EXISTS sessions_self ON sessions; +CREATE POLICY sessions_self ON sessions + FOR ALL USING (user_id = app.current_user_id()) + WITH CHECK (user_id = app.current_user_id()); + +DROP POLICY IF EXISTS mfa_methods_self ON mfa_methods; +CREATE POLICY mfa_methods_self ON mfa_methods + FOR ALL USING (user_id = app.current_user_id()) + WITH CHECK (user_id = app.current_user_id()); + +DROP POLICY IF EXISTS bootstrap_tokens_self ON bootstrap_tokens; +CREATE POLICY bootstrap_tokens_self ON bootstrap_tokens + FOR ALL USING (user_id = app.current_user_id()) + WITH CHECK (user_id = app.current_user_id()); + +DROP POLICY IF EXISTS audit_events_select ON audit_events; +CREATE POLICY audit_events_select ON audit_events + FOR SELECT USING ( + app.is_platform_admin() + OR app.is_organization_admin(organization_id) + ); +DROP POLICY IF EXISTS audit_events_insert ON audit_events; +CREATE POLICY audit_events_insert ON audit_events + FOR INSERT WITH CHECK ( + actor_user_id = app.current_user_id() + OR app.is_platform_admin() + ); + +REVOKE ALL ON SCHEMA app FROM PUBLIC; +REVOKE CREATE ON SCHEMA public FROM PUBLIC; + +GRANT USAGE ON SCHEMA public, app TO + umbracontrol_api, umbracontrol_worker, umbracontrol_readonly; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO + umbracontrol_api, umbracontrol_worker, umbracontrol_readonly; + +GRANT SELECT, INSERT, UPDATE, DELETE ON + organizations, projects, users, roles, permissions, role_permissions, + user_roles, invitations, sessions, mfa_methods, bootstrap_tokens, audit_events +TO umbracontrol_api; + +GRANT SELECT ON + organizations, projects, users, roles, permissions, role_permissions, + user_roles, audit_events +TO umbracontrol_worker, umbracontrol_readonly; + +GRANT INSERT ON audit_events TO umbracontrol_worker; + +GRANT USAGE, CREATE ON SCHEMA public TO umbracontrol_migrator; +GRANT USAGE ON SCHEMA public TO umbracontrol_backup; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO umbracontrol_backup; +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO umbracontrol_migrator; +GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO umbracontrol_migrator; + +REVOKE ALL ON ALL FUNCTIONS IN SCHEMA app FROM PUBLIC; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA app TO + umbracontrol_api, umbracontrol_worker, umbracontrol_readonly; + +COMMIT; diff --git a/infra/remote_credential_broker/README.md b/infra/remote_credential_broker/README.md new file mode 100644 index 0000000..8c1ab5f --- /dev/null +++ b/infra/remote_credential_broker/README.md @@ -0,0 +1,106 @@ +# UmbraTrans remote credential broker + +This is a standalone local MCP server for the UmbraTrans control plane. It is +not imported by UmbraAudio and must be deployed separately on the trusted +macOS/OpenCode host. + +## Security model + +- Passwords are entered in a native macOS hidden-password dialog. +- Passwords are stored in the macOS login Keychain through the Security + framework; they are never written to the repository, environment variables, + command-line arguments, logs, or MCP responses. +- The MCP server exposes no `get_password` operation. +- SSH host-key checking is strict. The server does not silently accept an + unknown host key. +- Unprivileged SSH commands are bounded by a timeout and output limit. +- Sudo commands must exactly match an allowlist entry in the local config. +- The default configuration does not grant `sudo` access to arbitrary commands. +- The server is stdio-only and has no network listener. + +The broker is intended to run on the Mac that owns the OpenCode session. The +remote `umbra` account receives an SSH connection; the password is supplied +only to the local SSH client/Keychain flow. + +## Files + +- `broker.py` — stdio MCP server and command-line popup setup. +- `keychain.py` — ctypes macOS Keychain backend. +- `popup.py` — native hidden-password prompt. +- `askpass_helper.py` — OpenSSH `SSH_ASKPASS` helper; it prints a secret only + to the SSH process pipe. +- `executor.py` — bounded SSH/sudo execution and redaction. +- `config.py` — non-secret remote profile and sudo allowlist parser. +- `config.example.json` — oldbox/umbra example; copy to the user config path. +- `tests/` — dependency-free unit tests. + +## Local configuration + +Copy the example to: + +```text +~/.config/umbra-trans/remote-broker.json +``` + +Review it. The `known_hosts` path must already contain the verified oldbox host +key. Do not use `StrictHostKeyChecking=no` or an empty known-hosts file. + +The first credential profiles can be entered through the MCP tool +`credential_set`, which opens the popup, or directly from a terminal: + +```bash +python3 -m remote_credential_broker.broker \ + --credential-profile oldbox-umbra-ssh \ + --purpose "SSH password for umbra@oldbox" +``` + +```bash +python3 -m remote_credential_broker.broker \ + --credential-profile oldbox-umbra-sudo \ + --purpose "sudo password for umbra@oldbox" +``` + +The command prints only whether the profile was stored. It never prints the +secret. + +## OpenCode registration + +Run the module from the directory containing the `remote_credential_broker` +package, or use an absolute Python path and package directory. A local server +entry has this shape: + +```json +{ + "mcp": { + "servers": { + "remote-broker": { + "type": "local", + "command": [ + "/absolute/path/to/python", + "-m", + "remote_credential_broker.broker" + ], + "environment": { + "PYTHONPATH": "/absolute/path/to/package/parent" + } + } + } + } +} +``` + +The current OpenCode installation may use the older top-level `mcp` object; +use the format accepted by the installed version. Do not put a password in the +configuration file. + +## Tools + +- `credential_set` — opens the native popup and stores a profile. +- `credential_status` — reports existence only. +- `credential_delete` — removes a profile. +- `remote_exec` — runs a command as the configured SSH account. +- `remote_sudo_exec` — runs one exact allowlisted command through sudo. +- `remote_status` — runs fixed identity/hostname probes. + +The model can request a popup or operation, but cannot read the secret entered +in the popup. diff --git a/infra/remote_credential_broker/__init__.py b/infra/remote_credential_broker/__init__.py new file mode 100644 index 0000000..5b3b702 --- /dev/null +++ b/infra/remote_credential_broker/__init__.py @@ -0,0 +1,7 @@ +"""Local, stdio MCP broker for controlled remote operations. + +The broker keeps passwords in the macOS Keychain and exposes only operations, +never secret values, to MCP clients. +""" + +__all__: list[str] = [] diff --git a/infra/remote_credential_broker/__main__.py b/infra/remote_credential_broker/__main__.py new file mode 100644 index 0000000..84cbe71 --- /dev/null +++ b/infra/remote_credential_broker/__main__.py @@ -0,0 +1,3 @@ +from .broker import main + +raise SystemExit(main()) diff --git a/infra/remote_credential_broker/askpass_helper.py b/infra/remote_credential_broker/askpass_helper.py new file mode 100755 index 0000000..1883316 --- /dev/null +++ b/infra/remote_credential_broker/askpass_helper.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import os +import socket +import sys + + +def main() -> int: + socket_path = os.environ.get("UMBRA_BROKER_ASKPASS_SOCKET", "") + if not socket_path: + print("credential broker: askpass socket is not configured", file=sys.stderr) + return 2 + try: + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as channel: + channel.connect(socket_path) + secret = channel.recv(4096) + except OSError as exc: + print(f"credential broker: askpass channel failed: {exc}", file=sys.stderr) + return 1 + if not secret: + return 1 + sys.stdout.buffer.write(secret) + sys.stdout.buffer.flush() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/infra/remote_credential_broker/broker.py b/infra/remote_credential_broker/broker.py new file mode 100644 index 0000000..bfc07fa --- /dev/null +++ b/infra/remote_credential_broker/broker.py @@ -0,0 +1,435 @@ +from __future__ import annotations + +import argparse +import json +import os +import sys +import tempfile +import threading +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, TextIO + +from .config import BrokerConfig, ConfigError +from .executor import ExecutionError, RemoteExecutor +from .keychain import ( + CredentialError, + CredentialNotFound, + CredentialVault, + MacKeychain, + UnsupportedPlatformError, + validate_profile, +) +from .popup import PromptCancelled, SecretPrompt, macos_secret_prompt + +_DEFAULT_CONFIG = Path("~/.config/umbra-trans/remote-broker.json").expanduser() +_DEFAULT_STATE = Path("~/.config/umbra-trans/remote-broker-state.json").expanduser() +_MAX_RESULT_CHARS = 1_000_000 + + +class BrokerState: + """Non-secret metadata about profiles that have been configured locally.""" + + def __init__(self, path: Path) -> None: + self.path = path + self._lock = threading.RLock() + + def _read(self) -> dict[str, Any]: + try: + raw = json.loads(self.path.read_text(encoding="utf-8")) + except FileNotFoundError: + return {"profiles": {}} + except (OSError, json.JSONDecodeError) as exc: + raise RuntimeError("Unable to read broker state") from exc + if not isinstance(raw, dict) or not isinstance(raw.get("profiles", {}), dict): + raise RuntimeError("Broker state is malformed") + return raw + + def _write(self, value: dict[str, Any]) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + fd, temporary_name = tempfile.mkstemp(prefix=".remote-broker-state.", dir=self.path.parent) + try: + os.fchmod(fd, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(value, handle, indent=2, sort_keys=True) + handle.write("\n") + os.replace(temporary_name, self.path) + except BaseException: + try: + os.unlink(temporary_name) + except FileNotFoundError: + pass + raise + + def record(self, profile: str, purpose: str, ttl_seconds: int | None = None) -> None: + with self._lock: + value = self._read() + metadata: dict[str, Any] = { + "purpose": purpose, + "updated_at": datetime.now(timezone.utc).isoformat(), + } + if ttl_seconds is not None: + metadata["expires_at"] = ( + datetime.now(timezone.utc) + timedelta(seconds=int(ttl_seconds)) + ).isoformat() + value["profiles"][profile] = metadata + self._write(value) + + def remove(self, profile: str) -> None: + with self._lock: + value = self._read() + value["profiles"].pop(profile, None) + self._write(value) + + def profiles(self) -> dict[str, Any]: + with self._lock: + return dict(self._read().get("profiles", {})) + + +class RemoteCredentialBroker: + """MCP-facing broker; secret values never cross the tool boundary.""" + + def __init__( + self, + store: CredentialVault, + config: BrokerConfig, + state: BrokerState, + prompt: SecretPrompt, + executor: RemoteExecutor, + ) -> None: + self.store = store + self.config = config + self.state = state + self.prompt = prompt + self.executor = executor + + @staticmethod + def _text_result(value: Any, *, is_error: bool = False) -> dict[str, Any]: + return { + "content": [{"type": "text", "text": json.dumps(value, ensure_ascii=False)}], + "isError": is_error, + } + + def tool_definitions(self) -> list[dict[str, Any]]: + return [ + { + "name": "credential_set", + "description": ( + "Open a native macOS hidden-password dialog and store the entered secret " + "in the login Keychain. The secret is never returned to the model." + ), + "inputSchema": { + "type": "object", + "additionalProperties": False, + "required": ["profile", "purpose"], + "properties": { + "profile": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$", + "description": "Non-secret profile name, for example oldbox-umbra-ssh.", + }, + "purpose": { + "type": "string", + "maxLength": 160, + "description": "Short label shown in the popup; never include a password.", + }, + "ttl_seconds": { + "type": "integer", + "minimum": 1, + "maximum": 3600, + "description": "Optional in-memory lifetime; omit for persistent Keychain storage.", + }, + }, + }, + }, + { + "name": "credential_status", + "description": "Report whether configured credential profiles exist; never reveal secrets.", + "inputSchema": { + "type": "object", + "additionalProperties": False, + "properties": {"profile": {"type": "string"}}, + }, + }, + { + "name": "credential_delete", + "description": "Delete a stored credential profile from the local Keychain.", + "inputSchema": { + "type": "object", + "additionalProperties": False, + "required": ["profile"], + "properties": {"profile": {"type": "string"}}, + }, + }, + { + "name": "remote_exec", + "description": ( + "Run one bounded command as a configured SSH account. The command is a list, " + "not a shell string; output is redacted and size-limited." + ), + "inputSchema": { + "type": "object", + "additionalProperties": False, + "required": ["profile", "command"], + "properties": { + "profile": {"type": "string"}, + "command": { + "type": "array", + "minItems": 1, + "items": {"type": "string"}, + }, + "timeout_seconds": { + "type": "integer", + "minimum": 5, + "maximum": 3600, + "default": 120, + }, + }, + }, + }, + { + "name": "remote_sudo_exec", + "description": ( + "Run one exact, configured allowlist command through remote sudo. " + "The password is read internally from Keychain and is never returned." + ), + "inputSchema": { + "type": "object", + "additionalProperties": False, + "required": ["profile", "command"], + "properties": { + "profile": {"type": "string"}, + "command": { + "type": "array", + "minItems": 1, + "items": {"type": "string"}, + }, + "timeout_seconds": { + "type": "integer", + "minimum": 5, + "maximum": 3600, + "default": 120, + }, + }, + }, + }, + { + "name": "remote_status", + "description": "Run a small fixed status probe against a configured SSH account.", + "inputSchema": { + "type": "object", + "additionalProperties": False, + "required": ["profile"], + "properties": {"profile": {"type": "string"}}, + }, + }, + ] + + def call_tool(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]: + try: + if name == "credential_set": + profile = validate_profile(str(arguments["profile"])) + purpose = str(arguments["purpose"]).strip() + if not purpose or len(purpose) > 160: + raise ExecutionError("purpose must be 1-160 characters") + ttl_value = arguments.get("ttl_seconds") + ttl_seconds = None if ttl_value is None else int(ttl_value) + if ttl_seconds is not None and not 1 <= ttl_seconds <= 3600: + raise ExecutionError("ttl_seconds must be between 1 and 3600") + secret = self.prompt(profile, purpose) + if secret is None: + raise PromptCancelled("Credential entry was cancelled") + self.store.set(profile, secret, ttl_seconds=ttl_seconds) + self.state.record(profile, purpose, ttl_seconds=ttl_seconds) + return self._text_result( + { + "profile": profile, + "stored": True, + "temporary": ttl_seconds is not None, + "ttl_seconds": ttl_seconds, + } + ) + if name == "credential_status": + profile = arguments.get("profile") + if profile is not None: + profile = validate_profile(str(profile)) + return self._text_result( + {"profile": profile, "configured": self.store.exists(profile)} + ) + statuses = [] + for name_, metadata in sorted(self.state.profiles().items()): + statuses.append( + { + "profile": name_, + "configured": self.store.exists(name_), + "purpose": metadata.get("purpose", ""), + } + ) + return self._text_result({"profiles": statuses}) + if name == "credential_delete": + profile = validate_profile(str(arguments["profile"])) + self.store.delete(profile) + self.state.remove(profile) + return self._text_result({"profile": profile, "deleted": True}) + if name in {"remote_exec", "remote_sudo_exec"}: + profile = self.config.get(str(arguments["profile"])) + command = arguments.get("command") + if not isinstance(command, list): + raise ExecutionError("command must be an array of strings") + if not all(isinstance(part, str) and part for part in command): + raise ExecutionError("command must contain only non-empty strings") + command_tuple = tuple(command) + if name == "remote_sudo_exec" and not profile.sudo_allowed(command_tuple): + raise ExecutionError( + "The requested sudo command is not an exact allowlist entry for this profile" + ) + timeout = int(arguments.get("timeout_seconds", 120)) + result = self.executor.run( + profile, + command, + sudo=name == "remote_sudo_exec", + timeout_seconds=timeout, + ) + output = { + "exit_code": result.exit_code, + "stdout": result.stdout[:_MAX_RESULT_CHARS], + "stderr": result.stderr[:_MAX_RESULT_CHARS], + "duration_ms": result.duration_ms, + } + return self._text_result(output, is_error=result.exit_code != 0) + if name == "remote_status": + profile = self.config.get(str(arguments["profile"])) + results = [] + for command in (["/usr/bin/id", "-un"], ["/usr/bin/hostname"]): + result = self.executor.run(profile, command, timeout_seconds=30) + results.append( + { + "command": command, + "exit_code": result.exit_code, + "stdout": result.stdout.strip(), + "stderr": result.stderr.strip(), + } + ) + return self._text_result({"profile": profile.name, "checks": results}) + raise ExecutionError(f"Unknown tool: {name}") + except (CredentialError, ConfigError, ExecutionError, PromptCancelled, KeyError, ValueError) as exc: + return self._text_result({"error": str(exc)}, is_error=True) + + def handle_json_rpc(self, message: dict[str, Any]) -> dict[str, Any] | None: + request_id = message.get("id") + method = message.get("method") + if method == "initialize": + params = message.get("params") or {} + requested_protocol = params.get("protocolVersion") + supported_protocols = { + "2024-11-05", + "2025-03-26", + "2025-06-18", + "2025-11-25", + } + protocol = ( + requested_protocol + if isinstance(requested_protocol, str) and requested_protocol in supported_protocols + else "2024-11-05" + ) + result: dict[str, Any] = { + "protocolVersion": protocol, + "capabilities": {"tools": {"listChanged": False}}, + "serverInfo": {"name": "umbratrans-remote-broker", "version": "0.1.0"}, + } + return self._response(request_id, result=result) + if method in {"notifications/initialized", "notifications/cancelled"}: + return None + if method == "ping": + return self._response(request_id, result={}) + if method == "tools/list": + return self._response(request_id, result={"tools": self.tool_definitions()}) + if method == "resources/list": + return self._response(request_id, result={"resources": []}) + if method == "prompts/list": + return self._response(request_id, result={"prompts": []}) + if method == "tools/call": + params = message.get("params") or {} + name = params.get("name") + arguments = params.get("arguments") or {} + if not isinstance(name, str) or not isinstance(arguments, dict): + return self._error(request_id, -32602, "Invalid tools/call parameters") + return self._response(request_id, result=self.call_tool(name, arguments)) + if request_id is not None: + return self._error(request_id, -32601, f"Method not found: {method}") + return None + + @staticmethod + def _response(request_id: Any, *, result: Any) -> dict[str, Any]: + return {"jsonrpc": "2.0", "id": request_id, "result": result} + + @staticmethod + def _error(request_id: Any, code: int, message: str) -> dict[str, Any]: + return {"jsonrpc": "2.0", "id": request_id, "error": {"code": code, "message": message}} + + def serve(self, input_stream: TextIO = sys.stdin, output_stream: TextIO = sys.stdout) -> None: + for line in input_stream: + if not line.strip(): + continue + try: + message = json.loads(line) + if not isinstance(message, dict): + raise ValueError("message must be a JSON object") + response = self.handle_json_rpc(message) + except json.JSONDecodeError: + response = self._error(None, -32700, "Parse error") + except Exception: + response = self._error(None, -32603, "Internal broker error") + if response is not None: + output_stream.write(json.dumps(response, ensure_ascii=False) + "\n") + output_stream.flush() + + +def _load_runtime() -> tuple[CredentialVault, BrokerConfig, BrokerState, RemoteCredentialBroker]: + config_path = Path(os.environ.get("UMBRA_BROKER_CONFIG", str(_DEFAULT_CONFIG))).expanduser() + state_path = Path(os.environ.get("UMBRA_BROKER_STATE", str(_DEFAULT_STATE))).expanduser() + store = CredentialVault() + config = BrokerConfig.load(config_path) + state = BrokerState(state_path) + audit_path = Path( + os.environ.get("UMBRA_BROKER_AUDIT_LOG", "~/.local/state/umbratrans/remote-broker.log") + ).expanduser() + executor = RemoteExecutor(store, audit_path=audit_path) + broker = RemoteCredentialBroker(store, config, state, macos_secret_prompt, executor) + return store, config, state, broker + + +def _cli_credential_set(profile: str, purpose: str) -> int: + store, _, state, _ = _load_runtime() + try: + secret = macos_secret_prompt(validate_profile(profile), purpose) + if secret is None: + return 2 + store.set(profile, secret) + state.record(profile, purpose) + except (CredentialError, UnsupportedPlatformError, PromptCancelled, RuntimeError) as exc: + print(f"credential setup failed: {exc}", file=sys.stderr) + return 1 + print(f"stored credential profile {profile!r}; secret was not printed") + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="UmbraTrans local remote credential MCP broker") + parser.add_argument("--credential-profile", help="open the popup for one profile and exit") + parser.add_argument("--purpose", default="remote access", help="label shown in the credential popup") + args = parser.parse_args(argv) + if args.credential_profile: + return _cli_credential_set(args.credential_profile, args.purpose) + try: + _, _, _, broker = _load_runtime() + broker.serve() + except (ConfigError, CredentialError, UnsupportedPlatformError, RuntimeError) as exc: + print(f"remote broker startup failed: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/infra/remote_credential_broker/config.example.json b/infra/remote_credential_broker/config.example.json new file mode 100644 index 0000000..abafefa --- /dev/null +++ b/infra/remote_credential_broker/config.example.json @@ -0,0 +1,26 @@ +{ + "profiles": { + "oldbox-umbra": { + "host": "oldbox.bdmurphy.me", + "username": "umbra", + "port": 22, + "ssh_credential": "oldbox-umbra-ssh", + "sudo_credential": "oldbox-umbra-sudo", + "known_hosts": "~/.ssh/known_hosts", + "allowed_sudo_commands": [ + [ + "/home/umbra/umbra-enterprise-bootstrap/bootstrap_umbracontrol_database.sh", + "--apply", + "--config", + "/home/umbra/umbra-enterprise-bootstrap/enterprise-bootstrap.env" + ], + [ + "/home/umbra/umbra-enterprise-bootstrap/bootstrap_umbracontrol_database.sh", + "--verify", + "--config", + "/home/umbra/umbra-enterprise-bootstrap/enterprise-bootstrap.env" + ] + ] + } + } +} diff --git a/infra/remote_credential_broker/config.py b/infra/remote_credential_broker/config.py new file mode 100644 index 0000000..3b273ba --- /dev/null +++ b/infra/remote_credential_broker/config.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +_PROFILE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$") +_HOST_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9.-]{0,252}$") + + +class ConfigError(ValueError): + """Raised when the broker configuration is invalid.""" + + +@dataclass(frozen=True) +class RemoteProfile: + name: str + host: str + username: str + port: int = 22 + ssh_credential: str | None = None + sudo_credential: str | None = None + known_hosts: str = "~/.ssh/known_hosts" + allowed_sudo_commands: tuple[tuple[str, ...], ...] = () + + def validate(self) -> None: + if not _PROFILE_RE.fullmatch(self.name): + raise ConfigError(f"Invalid remote profile name: {self.name!r}") + if not _HOST_RE.fullmatch(self.host): + raise ConfigError(f"Invalid host in profile {self.name!r}") + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_.-]{0,31}", self.username): + raise ConfigError(f"Invalid SSH username in profile {self.name!r}") + if not 1 <= self.port <= 65535: + raise ConfigError(f"Invalid SSH port in profile {self.name!r}") + for credential in (self.ssh_credential, self.sudo_credential): + if credential is not None and not _PROFILE_RE.fullmatch(credential): + raise ConfigError(f"Invalid credential profile in {self.name!r}") + for command in self.allowed_sudo_commands: + if not command or any(not isinstance(part, str) or not part for part in command): + raise ConfigError(f"Invalid sudo command allowlist in profile {self.name!r}") + if any("\x00" in part for part in command): + raise ConfigError(f"NUL byte in sudo command allowlist in {self.name!r}") + + def sudo_allowed(self, command: tuple[str, ...]) -> bool: + return command in self.allowed_sudo_commands + + +@dataclass(frozen=True) +class BrokerConfig: + profiles: dict[str, RemoteProfile] + + @classmethod + def load(cls, path: Path) -> BrokerConfig: + if not path.exists(): + return cls(profiles={}) + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ConfigError(f"Unable to read broker config: {path}") from exc + if not isinstance(raw, dict) or not isinstance(raw.get("profiles", {}), dict): + raise ConfigError("Broker config must contain a profiles object") + profiles: dict[str, RemoteProfile] = {} + for name, item in raw.get("profiles", {}).items(): + if not isinstance(item, dict): + raise ConfigError(f"Profile {name!r} must be an object") + commands_raw = item.get("allowed_sudo_commands", []) + if not isinstance(commands_raw, list): + raise ConfigError(f"allowed_sudo_commands for {name!r} must be a list") + commands: list[tuple[str, ...]] = [] + for command in commands_raw: + if not isinstance(command, list) or not all(isinstance(x, str) for x in command): + raise ConfigError(f"Invalid sudo command entry for {name!r}") + commands.append(tuple(command)) + profile = RemoteProfile( + name=name, + host=str(item.get("host", "")), + username=str(item.get("username", "")), + port=int(item.get("port", 22)), + ssh_credential=item.get("ssh_credential"), + sudo_credential=item.get("sudo_credential"), + known_hosts=str(item.get("known_hosts", "~/.ssh/known_hosts")), + allowed_sudo_commands=tuple(commands), + ) + profile.validate() + profiles[name] = profile + return cls(profiles=profiles) + + def get(self, name: str) -> RemoteProfile: + try: + return self.profiles[name] + except KeyError as exc: + raise ConfigError(f"Unknown remote profile: {name!r}") from exc diff --git a/infra/remote_credential_broker/executor.py b/infra/remote_credential_broker/executor.py new file mode 100644 index 0000000..b2fd982 --- /dev/null +++ b/infra/remote_credential_broker/executor.py @@ -0,0 +1,242 @@ +from __future__ import annotations + +import os +import shlex +import shutil +import socket +import subprocess +import tempfile +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Sequence + +from .config import ConfigError, RemoteProfile +from .keychain import CredentialError, CredentialVault + +Runner = Callable[..., subprocess.CompletedProcess[bytes]] + + +class ExecutionError(RuntimeError): + """Raised when a remote operation cannot be safely started.""" + + +@dataclass(frozen=True) +class ExecutionResult: + exit_code: int + stdout: str + stderr: str + duration_ms: int + + +class SecretRedactor: + """Redact known secrets before output leaves the broker.""" + + def __init__(self) -> None: + self._secrets: set[str] = set() + + def add(self, secret: str | None) -> None: + if secret: + self._secrets.add(secret) + + def redact(self, value: str) -> str: + result = value + for secret in sorted(self._secrets, key=len, reverse=True): + if secret: + result = result.replace(secret, "") + return result + + +class _AskpassServer: + """One-shot local channel used instead of exposing a Keychain item to ssh.""" + + def __init__(self, secret: str) -> None: + self._directory = Path(tempfile.mkdtemp(prefix="remote-broker-askpass-")) + self._directory.chmod(0o700) + self.path = self._directory / "secret.sock" + self._secret = secret.encode("utf-8") + self._stop = threading.Event() + self._socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._socket.bind(str(self.path)) + self._socket.listen(8) + self._socket.settimeout(0.2) + self._thread = threading.Thread(target=self._serve, name="remote-broker-askpass", daemon=True) + self._thread.start() + + def _serve(self) -> None: + while not self._stop.is_set(): + try: + connection, _ = self._socket.accept() + except socket.timeout: + continue + except OSError: + return + with connection: + try: + connection.sendall(self._secret) + except OSError: + pass + + def close(self) -> None: + self._stop.set() + try: + self._socket.close() + except OSError: + pass + self._thread.join(timeout=1) + shutil.rmtree(self._directory, ignore_errors=True) + + +class RemoteExecutor: + """Run bounded SSH operations; sudo commands are exact allowlist entries.""" + + def __init__( + self, + store: CredentialVault, + *, + runner: Runner = subprocess.run, + askpass_path: Path | None = None, + audit_path: Path | None = None, + ) -> None: + self._store = store + self._runner = runner + self._askpass_path = askpass_path or Path(__file__).with_name("askpass_helper.py") + if self._askpass_path.exists(): + self._askpass_path.chmod(0o700) + self._audit_path = audit_path + + @staticmethod + def _validate_command(command: Sequence[str]) -> tuple[str, ...]: + if not command or not all(isinstance(part, str) and part for part in command): + raise ExecutionError("Remote command must be a non-empty list of strings") + if any("\x00" in part for part in command): + raise ExecutionError("Remote command contains a NUL byte") + return tuple(command) + + def _audit(self, profile: RemoteProfile, command: tuple[str, ...], sudo: bool, result: int) -> None: + if self._audit_path is None: + return + self._audit_path.parent.mkdir(parents=True, exist_ok=True) + # Store command metadata only. Never store passwords or command stdin. + line = ( + f"profile={profile.name} sudo={str(sudo).lower()} " + f"command={shlex.join(command)} exit={result}\n" + ) + fd = os.open(self._audit_path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600) + try: + os.write(fd, line.encode("utf-8")) + finally: + os.close(fd) + + def _base_ssh_command(self, profile: RemoteProfile, remote_command: str, use_password: bool) -> list[str]: + known_hosts = str(Path(profile.known_hosts).expanduser()) + command = [ + "ssh", + "-p", + str(profile.port), + "-o", + "ConnectTimeout=15", + "-o", + "ServerAliveInterval=15", + "-o", + "ServerAliveCountMax=2", + "-o", + "StrictHostKeyChecking=yes", + "-o", + f"UserKnownHostsFile={known_hosts}", + ] + if use_password: + command.extend( + [ + "-o", + "BatchMode=no", + "-o", + "PreferredAuthentications=publickey,password", + "-o", + "PubkeyAuthentication=yes", + ] + ) + command.extend([f"{profile.username}@{profile.host}", remote_command]) + return command + + def run( + self, + profile: RemoteProfile, + command: Sequence[str], + *, + sudo: bool = False, + timeout_seconds: int = 120, + ) -> ExecutionResult: + command_tuple = self._validate_command(command) + if not 5 <= timeout_seconds <= 3600: + raise ExecutionError("timeout_seconds must be between 5 and 3600") + if sudo and not profile.sudo_allowed(command_tuple): + raise ExecutionError( + "The requested sudo command is not an exact allowlist entry for this profile" + ) + + ssh_secret: str | None = None + sudo_secret: str | None = None + redactor = SecretRedactor() + if profile.ssh_credential: + try: + ssh_secret = self._store.get(profile.ssh_credential) + except CredentialError as exc: + raise ExecutionError( + f"SSH credential is not configured for profile {profile.name!r}" + ) from exc + redactor.add(ssh_secret) + if sudo: + if not profile.sudo_credential: + raise ExecutionError(f"No sudo credential is configured for profile {profile.name!r}") + try: + sudo_secret = self._store.get(profile.sudo_credential) + except CredentialError as exc: + raise ExecutionError( + f"Sudo credential is not configured for profile {profile.name!r}" + ) from exc + redactor.add(sudo_secret) + + remote_argv = list(command_tuple) + if sudo: + remote_command = "sudo -S -p '' -- " + shlex.join(remote_argv) + else: + remote_command = shlex.join(remote_argv) + ssh_command = self._base_ssh_command(profile, remote_command, use_password=ssh_secret is not None) + environment = os.environ.copy() + askpass_server: _AskpassServer | None = None + if ssh_secret is not None: + askpass_server = _AskpassServer(ssh_secret) + environment.update( + { + "SSH_ASKPASS": str(self._askpass_path), + "SSH_ASKPASS_REQUIRE": "force", + "DISPLAY": environment.get("DISPLAY", ":0"), + "UMBRA_BROKER_ASKPASS_SOCKET": str(askpass_server.path), + } + ) + stdin_data = (sudo_secret + "\n").encode("utf-8") if sudo_secret is not None else None + started = time.monotonic() + try: + completed = self._runner( + ssh_command, + input=stdin_data, + capture_output=True, + timeout=timeout_seconds, + env=environment, + check=False, + ) + except subprocess.TimeoutExpired as exc: + raise ExecutionError("Remote operation timed out") from exc + except OSError as exc: + raise ExecutionError("Unable to start ssh") from exc + finally: + if askpass_server is not None: + askpass_server.close() + duration_ms = int((time.monotonic() - started) * 1000) + stdout = redactor.redact(completed.stdout.decode("utf-8", errors="replace")) + stderr = redactor.redact(completed.stderr.decode("utf-8", errors="replace")) + result = ExecutionResult(completed.returncode, stdout, stderr, duration_ms) + self._audit(profile, command_tuple, sudo, result.exit_code) + return result diff --git a/infra/remote_credential_broker/keychain.py b/infra/remote_credential_broker/keychain.py new file mode 100644 index 0000000..924b5d3 --- /dev/null +++ b/infra/remote_credential_broker/keychain.py @@ -0,0 +1,299 @@ +from __future__ import annotations + +import ctypes +import re +import sys +import threading +import time +from ctypes import POINTER, c_int32, c_uint32, c_void_p, create_string_buffer +from typing import Final + + +_PROFILE_RE: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$") +_SERVICE_DEFAULT: Final = "com.umbratrans.remote-broker" +_ERR_SEC_ITEM_NOT_FOUND: Final = -25300 + + +class CredentialError(RuntimeError): + """Base exception for credential-store failures.""" + + +class CredentialNotFound(CredentialError): + """Raised when a requested Keychain item does not exist.""" + + +class UnsupportedPlatformError(CredentialError): + """Raised when the broker is not running on macOS.""" + + +def validate_profile(profile: str) -> str: + """Validate a profile name before using it as a Keychain account name.""" + if not isinstance(profile, str) or not _PROFILE_RE.fullmatch(profile): + raise CredentialError( + "Credential profile must contain only letters, digits, '.', '_' and '-' " + "and must be 1-128 characters" + ) + return profile + + +class MacKeychain: + """Small ctypes wrapper around the legacy macOS generic-password API. + + Password bytes are passed directly to the Security framework and are never + placed in a subprocess argument, environment variable, log line, or MCP + response. The API is deprecated by Apple but remains available on the + supported macOS hosts and avoids adding a Python dependency. + """ + + def __init__(self, service: str = _SERVICE_DEFAULT) -> None: + self.service = service + self._lock = threading.RLock() + self._lib: ctypes.CDLL | None = None + self._cf: ctypes.CDLL | None = None + + def _load(self) -> tuple[ctypes.CDLL, ctypes.CDLL]: + if sys.platform != "darwin": + raise UnsupportedPlatformError( + "The macOS Keychain backend requires macOS; use a native secret store on other hosts" + ) + if self._lib is None: + lib = ctypes.CDLL("/System/Library/Frameworks/Security.framework/Security") + lib.SecKeychainAddGenericPassword.argtypes = [ + c_void_p, + c_uint32, + c_void_p, + c_uint32, + c_void_p, + c_uint32, + c_void_p, + POINTER(c_void_p), + ] + lib.SecKeychainAddGenericPassword.restype = c_int32 + lib.SecKeychainFindGenericPassword.argtypes = [ + c_void_p, + c_uint32, + c_void_p, + c_uint32, + c_void_p, + POINTER(c_uint32), + POINTER(c_void_p), + POINTER(c_void_p), + ] + lib.SecKeychainFindGenericPassword.restype = c_int32 + lib.SecKeychainItemModifyAttributesAndData.argtypes = [ + c_void_p, + c_void_p, + c_uint32, + c_void_p, + ] + lib.SecKeychainItemModifyAttributesAndData.restype = c_int32 + lib.SecKeychainItemFreeContent.argtypes = [c_void_p, c_void_p] + lib.SecKeychainItemFreeContent.restype = c_int32 + lib.SecKeychainItemDelete.argtypes = [c_void_p] + lib.SecKeychainItemDelete.restype = c_int32 + self._lib = lib + if self._cf is None: + cf = ctypes.CDLL("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation") + cf.CFRelease.argtypes = [c_void_p] + cf.CFRelease.restype = None + self._cf = cf + return self._lib, self._cf + + @staticmethod + def _error(status: int, operation: str) -> CredentialError: + return CredentialError(f"Keychain {operation} failed with OSStatus {status}") + + def _find( + self, + profile: str, + *, + include_password: bool, + ) -> tuple[int, bytes | None, c_void_p | None]: + lib, _ = self._load() + profile = validate_profile(profile) + service_bytes = self.service.encode("utf-8") + account_bytes = profile.encode("utf-8") + service_buf = create_string_buffer(service_bytes) + account_buf = create_string_buffer(account_bytes) + length = c_uint32(0) + data = c_void_p() + item = c_void_p() + status = lib.SecKeychainFindGenericPassword( + None, + len(service_bytes), + service_buf, + len(account_bytes), + account_buf, + ctypes.byref(length) if include_password else None, + ctypes.byref(data) if include_password else None, + ctypes.byref(item), + ) + if status != 0: + return status, None, None + if not include_password: + return status, None, item + try: + value = ctypes.string_at(data, length.value) if data.value else b"" + return status, value, item + finally: + # The pointer is owned by the Security framework. + lib.SecKeychainItemFreeContent(None, data) + + def _release(self, item: c_void_p | None) -> None: + if item is None or not item.value: + return + _, cf = self._load() + cf.CFRelease(item) + + def set(self, profile: str, secret: str) -> None: + """Store or replace a secret in the default login Keychain.""" + if not isinstance(secret, str) or not secret: + raise CredentialError("Credential secret must not be empty") + with self._lock: + lib, _ = self._load() + profile = validate_profile(profile) + status, _, item = self._find(profile, include_password=False) + if status == 0 and item is not None: + secret_bytes = secret.encode("utf-8") + secret_buf = create_string_buffer(secret_bytes) + try: + update_status = lib.SecKeychainItemModifyAttributesAndData( + item, + None, + len(secret_bytes), + secret_buf, + ) + finally: + self._release(item) + if update_status != 0: + raise self._error(update_status, "update") + return + if status != _ERR_SEC_ITEM_NOT_FOUND: + raise self._error(status, "lookup") + + service_bytes = self.service.encode("utf-8") + account_bytes = profile.encode("utf-8") + secret_bytes = secret.encode("utf-8") + service_buf = create_string_buffer(service_bytes) + account_buf = create_string_buffer(account_bytes) + secret_buf = create_string_buffer(secret_bytes) + new_item = c_void_p() + add_status = lib.SecKeychainAddGenericPassword( + None, + len(service_bytes), + service_buf, + len(account_bytes), + account_buf, + len(secret_bytes), + secret_buf, + ctypes.byref(new_item), + ) + self._release(new_item) + if add_status != 0: + raise self._error(add_status, "add") + + def get(self, profile: str) -> str: + """Retrieve a secret for internal broker use only.""" + with self._lock: + status, value, item = self._find(profile, include_password=True) + self._release(item) + if status == _ERR_SEC_ITEM_NOT_FOUND: + raise CredentialNotFound(f"No stored credential for profile {profile!r}") + if status != 0: + raise self._error(status, "lookup") + if value is None: + raise CredentialError("Keychain returned an empty credential") + return value.decode("utf-8") + + def delete(self, profile: str) -> None: + """Delete a stored secret; missing items are treated as already absent.""" + with self._lock: + lib, _ = self._load() + status, _, item = self._find(profile, include_password=False) + if status == _ERR_SEC_ITEM_NOT_FOUND: + return + if status != 0: + raise self._error(status, "lookup") + if item is None: + return + try: + delete_status = lib.SecKeychainItemDelete(item) + finally: + self._release(item) + if delete_status != 0: + raise self._error(delete_status, "delete") + + def exists(self, profile: str) -> bool: + """Return whether a profile exists without retrieving its secret.""" + with self._lock: + status, _, item = self._find(profile, include_password=False) + self._release(item) + if status == 0: + return True + if status == _ERR_SEC_ITEM_NOT_FOUND: + return False + raise self._error(status, "lookup") + + +class CredentialVault: + """Persistent Keychain storage plus explicitly temporary in-memory profiles.""" + + def __init__(self, persistent: MacKeychain | None = None) -> None: + self.persistent = persistent or MacKeychain() + self.service = self.persistent.service + self._lock = threading.RLock() + self._temporary: dict[str, tuple[str, float]] = {} + self._timers: dict[str, threading.Timer] = {} + + def _expire(self, profile: str) -> None: + with self._lock: + self._temporary.pop(profile, None) + self._timers.pop(profile, None) + + def set(self, profile: str, secret: str, ttl_seconds: int | None = None) -> None: + profile = validate_profile(profile) + if not isinstance(secret, str) or not secret: + raise CredentialError("Credential secret must not be empty") + with self._lock: + old_timer = self._timers.pop(profile, None) + if old_timer is not None: + old_timer.cancel() + if ttl_seconds is None: + self._temporary.pop(profile, None) + self.persistent.set(profile, secret) + return + if not 1 <= int(ttl_seconds) <= 3600: + raise CredentialError("ttl_seconds must be between 1 and 3600") + expires_at = time.monotonic() + int(ttl_seconds) + self._temporary[profile] = (secret, expires_at) + timer = threading.Timer(int(ttl_seconds), self._expire, args=(profile,)) + timer.daemon = True + self._timers[profile] = timer + timer.start() + + def get(self, profile: str) -> str: + profile = validate_profile(profile) + with self._lock: + temporary = self._temporary.get(profile) + if temporary is not None: + secret, expires_at = temporary + if expires_at > time.monotonic(): + return secret + self._expire(profile) + return self.persistent.get(profile) + + def delete(self, profile: str) -> None: + profile = validate_profile(profile) + with self._lock: + self._expire(profile) + self.persistent.delete(profile) + + def exists(self, profile: str) -> bool: + profile = validate_profile(profile) + with self._lock: + temporary = self._temporary.get(profile) + if temporary is not None: + if temporary[1] > time.monotonic(): + return True + self._expire(profile) + return self.persistent.exists(profile) diff --git a/infra/remote_credential_broker/popup.py b/infra/remote_credential_broker/popup.py new file mode 100644 index 0000000..d6e9157 --- /dev/null +++ b/infra/remote_credential_broker/popup.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import subprocess +from typing import Protocol + + +class PromptCancelled(RuntimeError): + """Raised when the user cancels a native credential prompt.""" + + +class SecretPrompt(Protocol): + def __call__(self, profile: str, purpose: str) -> str | None: + """Return a secret from a local UI, or None when cancelled.""" + + +def _apple_string(value: str) -> str: + return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"' + + +def macos_secret_prompt(profile: str, purpose: str) -> str | None: + """Open a native macOS hidden-password dialog. + + The returned value is intended to be passed directly to the Keychain + backend. It is never printed, logged, or included in an MCP result. + """ + message = f"Enter the password for {purpose} ({profile})." + script = ( + f'display dialog {_apple_string(message)} ' + 'default answer "" with hidden answer buttons {"Cancel", "Save"} ' + 'default button "Save" with title "UmbraTrans secure credential"' + ) + try: + result = subprocess.run( + ["/usr/bin/osascript", "-e", script], + check=True, + capture_output=True, + text=True, + timeout=120, + ) + except subprocess.CalledProcessError as exc: + detail = exc.stderr.strip().lower() + if "user canceled" in detail or "-128" in detail: + raise PromptCancelled("Credential entry was cancelled") from exc + raise RuntimeError("The macOS credential dialog could not be completed") from exc + except (OSError, subprocess.TimeoutExpired) as exc: + raise RuntimeError("The macOS credential dialog could not be opened") from exc + + secret = result.stdout.rstrip("\r\n") + return secret or None diff --git a/infra/remote_credential_broker/tests/test_broker.py b/infra/remote_credential_broker/tests/test_broker.py new file mode 100644 index 0000000..c8c921d --- /dev/null +++ b/infra/remote_credential_broker/tests/test_broker.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import json +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from remote_credential_broker.broker import BrokerState, RemoteCredentialBroker +from remote_credential_broker.config import BrokerConfig, RemoteProfile +from remote_credential_broker.executor import ExecutionResult, SecretRedactor +from remote_credential_broker.keychain import CredentialError, CredentialVault, validate_profile + + +class FakeStore: + service = "test-service" + + def __init__(self) -> None: + self.values: dict[str, str] = {} + + def set(self, profile: str, secret: str, ttl_seconds: int | None = None) -> None: + self.values[profile] = secret + + def get(self, profile: str) -> str: + return self.values[profile] + + def delete(self, profile: str) -> None: + self.values.pop(profile, None) + + def exists(self, profile: str) -> bool: + return profile in self.values + + +class FakeExecutor: + def __init__(self) -> None: + self.calls: list[tuple[str, tuple[str, ...], bool]] = [] + + def run( + self, + profile: RemoteProfile, + command: list[str] | tuple[str, ...], + *, + sudo: bool = False, + timeout_seconds: int = 120, + ) -> ExecutionResult: + self.calls.append((profile.name, tuple(command), sudo)) + return ExecutionResult(0, "ok\n", "", 1) + + +class BrokerTests(unittest.TestCase): + def setUp(self) -> None: + self.store = FakeStore() + self.state_path = Path("/tmp/remote-broker-test-state.json") + self.state_path.unlink(missing_ok=True) + self.state = BrokerState(self.state_path) + self.executor = FakeExecutor() + self.profile = RemoteProfile( + name="oldbox-umbra", + host="oldbox.bdmurphy.me", + username="umbra", + ssh_credential="oldbox-umbra-ssh", + sudo_credential="oldbox-umbra-sudo", + allowed_sudo_commands=(("/usr/local/sbin/umbracontrol-backup",),), + ) + config = BrokerConfig(profiles={self.profile.name: self.profile}) + self.broker = RemoteCredentialBroker( + self.store, + config, + self.state, + lambda profile, purpose: "popup-secret", + self.executor, + ) + + def tearDown(self) -> None: + self.state_path.unlink(missing_ok=True) + + def test_credential_popup_is_not_returned(self) -> None: + result = self.broker.call_tool( + "credential_set", + {"profile": "oldbox-umbra-ssh", "purpose": "SSH for oldbox"}, + ) + encoded = json.dumps(result) + self.assertNotIn("popup-secret", encoded) + payload = json.loads(result["content"][0]["text"]) + self.assertTrue(payload["stored"]) + self.assertEqual(self.store.get("oldbox-umbra-ssh"), "popup-secret") + + def test_status_never_returns_secret(self) -> None: + self.broker.call_tool( + "credential_set", + {"profile": "oldbox-umbra-ssh", "purpose": "SSH for oldbox"}, + ) + result = self.broker.call_tool("credential_status", {"profile": "oldbox-umbra-ssh"}) + self.assertNotIn("popup-secret", json.dumps(result)) + payload = json.loads(result["content"][0]["text"]) + self.assertTrue(payload["configured"]) + + def test_sudo_requires_exact_allowlist(self) -> None: + result = self.broker.call_tool( + "remote_sudo_exec", + {"profile": "oldbox-umbra", "command": ["/bin/sh", "-c", "id"]}, + ) + self.assertTrue(result["isError"]) + self.assertIn("allowlist", result["content"][0]["text"]) + self.assertEqual(self.executor.calls, []) + + def test_allowed_sudo_is_forwarded(self) -> None: + result = self.broker.call_tool( + "remote_sudo_exec", + {"profile": "oldbox-umbra", "command": ["/usr/local/sbin/umbracontrol-backup"]}, + ) + self.assertFalse(result["isError"]) + self.assertEqual(self.executor.calls[0][2], True) + + def test_json_rpc_tools_list_and_initialize(self) -> None: + response = self.broker.handle_json_rpc({"jsonrpc": "2.0", "id": 1, "method": "initialize"}) + assert response is not None + self.assertEqual(response["result"]["serverInfo"]["name"], "umbratrans-remote-broker") + response = self.broker.handle_json_rpc({"jsonrpc": "2.0", "id": 2, "method": "tools/list"}) + assert response is not None + names = {tool["name"] for tool in response["result"]["tools"]} + self.assertIn("credential_set", names) + self.assertIn("remote_sudo_exec", names) + + def test_redactor(self) -> None: + redactor = SecretRedactor() + redactor.add("secret-value") + self.assertEqual(redactor.redact("x secret-value y"), "x y") + def test_temporary_profile_is_in_memory_and_readable(self) -> None: + persistent = FakeStore() + vault = CredentialVault(persistent) + vault.set("temporary-profile", "temporary-value", ttl_seconds=60) + self.assertEqual(vault.get("temporary-profile"), "temporary-value") + self.assertTrue(vault.exists("temporary-profile")) + self.assertNotIn("temporary-profile", persistent.values) + vault.delete("temporary-profile") + self.assertFalse(vault.exists("temporary-profile")) + + +class ValidationTests(unittest.TestCase): + def test_profile_rejects_shell_metacharacters(self) -> None: + with self.assertRaises(CredentialError): + validate_profile("bad profile;rm") + + def test_profile_accepts_normal_name(self) -> None: + self.assertEqual(validate_profile("oldbox-umbra_ssh.1"), "oldbox-umbra_ssh.1") + + +if __name__ == "__main__": + unittest.main() diff --git a/specs/18-UmbraTrans.md b/specs/18-UmbraTrans.md new file mode 100644 index 0000000..17de9c6 --- /dev/null +++ b/specs/18-UmbraTrans.md @@ -0,0 +1,2053 @@ +# Spec 18 — UmbraTrans: Meaning-Preserving Translation and Multilingual Audiobooks + +- **Status:** Proposed architecture and implementation specification +- **Date:** 2026-09-23 +- **Scope:** New system; additive to UmbraAudio +- **Primary source language:** English +- **Initial target locales:** French, Italian, German, Spanish, Russian, Japanese, and Simplified Chinese + +## 1. Executive Summary + +UmbraTrans is a separate multilingual production system built around the analysis and +voice identity work already performed by UmbraAudio. + +The system turns one English manuscript into: + +- a canonical, meaning-aware representation of the book; +- high-quality translated books in each target locale; +- multilingual audiobooks in which a listener can recognize the same characters and + narrator across languages; +- translation, pronunciation, voice, and audio quality reports; and +- reproducible release packages for text and audio. + +UmbraTrans does not treat a sentence as an isolated string to be translated. It builds a +hierarchy of book, chapter, scene, beat, discourse unit, and utterance information. The +translation subsystem receives the scene's goals, conflict, relationships, chronology, +terminology, speaker intent, and discourse context before it translates individual +utterances. The result is a natural target-language performance that preserves the +message and dramatic force rather than mirroring English word order. + +The first implementation must not alter the current UmbraAudio codebase, database, +speaker files, voice references, generation markers, routes, or audio outputs. UmbraAudio +is an upstream, read-only provider. UmbraTrans writes to a separate project data area and +communicates through versioned file or service contracts. + +The quality target is publication-grade output, with human review reserved for the +highest-risk decisions and exceptions. The source author controls the English source, +English analysis, glossary, pronunciation decisions, and intended meaning. Target-language +translation, localization, and audio validation are performed by UmbraTrans and its +subsystems; the author is not required to translate or approve every target-language +line. + +The recommended first deliverable is not seven complete books. It is a validated source +package, a representative translation in German plus one CJK lane, and a small set of +cross-language voice samples. Only after those gates pass should the system expand to +all seven locales and full-book production. + +## 2. Motivation and Goals + +### 2.1 Goals + +1. Import an English manuscript from the formats supported by UmbraAudio and, eventually, + public-domain books and historical scans. +2. Preserve the source text and create a stable, language-neutral source model that can + be reused by translation, audio, search, adaptation, and future Umbra systems. +3. Translate the complete book into: + - French (`fr-FR` by default); + - Italian (`it-IT`); + - German (`de-DE`); + - Spanish (`es-ES`); + - Russian (`ru-RU`); + - Japanese (`ja-JP`); and + - Simplified Chinese (`zh-Hans`). +4. Make translation quality meaning-first, context-aware, and consistent across a book. +5. Analyze communication at the scene and utterance levels, including goals, conflict, + threats, commands, promises, emotional turns, relationships, and subtext. +6. Produce target-language books with stable character names, terminology, punctuation, + typography, and metadata. +7. Produce target-language audiobooks with a recognizable cast and a consistent radio-play + performance style. +8. Validate the source analysis, translations, target pronunciation, voice identity, and + final audio before release. +9. Support incremental rebuilds: changing a glossary entry should not require rebuilding + unrelated chapters, and changing a voice profile should not invalidate translations. +10. Make every output traceable to source text, analysis decisions, model/engine versions, + and quality reports. + +### 2.2 Non-goals for the first version + +- Changing or replacing UmbraAudio's existing pipeline. +- Reusing English audio as a literal waveform for every target language. +- Treating a back-translation score as proof of translation quality. +- Automatically publishing books or distributing them without a rights/release check. +- Requiring the author to review every target-language sentence. +- Building a new general-purpose machine-learning model before validating the contracts + and workflow. +- Selecting a single TTS engine for all languages before measuring language-specific + quality. +- Silently falling back to a different voice, language, or engine during a release build. + +## 3. Design Principles and Non-negotiable Invariants + +### 3.1 Meaning before literal wording + +The source message, speech act, relationships, and dramatic force are authoritative. +Word order, idiom, metaphor, and even sentence boundaries may change when required by the +target language. A translation may compress, expand, reorder, or split source material when +that produces a better target-language result, provided the alignment and meaning checks +pass. + +### 3.2 Context before utterance + +No dialogue or narration unit is translated from its local string alone. Every unit has a +stable scene and chapter context, a speaker, addressee information where available, +discourse dependencies, and a meaning frame. Ambiguous pronouns, ellipsis, register shifts, +threats, and implied meanings must be resolved with that context. + +### 3.3 Stable identity across languages + +A character's `speaker_id` is the identity key in every language. Target languages may +use different voices because their phonetic and prosodic systems differ, but the listener +must be able to recognize the same character by the approved combination of voice identity, +role, relationship, and performance style. + +The system distinguishes: + +- **character identity:** the same fictional speaker; +- **voice identity:** the approved target-language realization of that speaker; and +- **performance state:** the emotion, pace, and delivery for one particular utterance. + +The third must not silently replace the first two. + +### 3.4 Versioned, reproducible artifacts + +Every generated artifact records the source revision, analysis revision, translation +revision, glossary revision, voice-profile revision, pronunciation revision, engine/model +version, and quality-report version. Re-running a build with the same inputs must produce +the same logical result or a recorded, explainable difference. + +### 3.5 Quality gates, not optimistic completion + +A stage is complete only when its required artifacts exist and its hard quality gates pass. +Warnings may be accepted with an explicit reason. Critical semantic, attribution, +pronunciation, or technical errors block release. + +### 3.6 English is the source of truth + +The immutable English source and its approved source-side meaning decisions are +canonical. Imported UmbraAudio analysis is advisory until it passes the source review gate. +A target-language issue does not silently rewrite English. If target translation reveals +genuine source ambiguity, UmbraTrans records a source-side issue for the author or source +operator to resolve. + +## 4. System Boundary and Relationship to UmbraAudio + +UmbraTrans is additive. The initial implementation must not change the behavior or data +files of the existing UmbraAudio system. + +```text +English manuscript + │ + ▼ +┌──────────────────────────────────────────────────────────┐ +│ Existing UmbraAudio │ +│ parse → character discovery → speaker attribution → │ +│ utterance/prosody data → approved English voice identity │ +└──────────────────────────────────────────────────────────┘ + │ read-only snapshot/adapter + ▼ +┌──────────────────────────────────────────────────────────┐ +│ UmbraTrans source adapter │ +│ normalize IDs, snapshot confidence, preserve provenance │ +└──────────────────────────────────────────────────────────┘ + │ + ▼ + Source Book Package (versioned) + │ + ├── semantic/discourse enrichment + ├── terminology and pronunciation preparation + ├── target-language translation lanes + ├── target-language voice casting + ├── target-language TTS rendering + ├── translation and audio QA + └── assembly/export +``` + +### 4.1 What UmbraTrans consumes from UmbraAudio + +The original English source file is the authoritative text artifact. UmbraTrans can +accept either a raw manuscript or a completed UmbraAudio project; in both cases it +performs its own structure-preserving parse when possible, retaining chapter, paragraph, +page, heading, and source-offset information. UmbraAudio's parsed chapter text is an +advisory cross-check, not the sole source of truth. + +The adapter may read, without writing, the following kinds of information: + +- project and source metadata; +- chapter order, titles, and source text; +- utterance order, text, type, and context; +- speaker names, aliases, demographics, vocal traits, and communication style; +- speaker attribution confidence and reasoning when present; +- emotion, pace, emphasis, pause, and instruction metadata; +- approved English voice references and voice status when available; and +- source pronunciation records and confirmed spellings. + +These values are hints for speaker identity, segmentation, prosody, and voice direction. +UmbraTrans assigns its own stable IDs, preserves the source hierarchy, and records an +independent confidence and provenance value for every imported claim. It must not use a +mutable database row ID, a renumbered `utterance_index`, or a display name as a durable +identity. + +The adapter must tolerate missing optional fields. It records `unknown` or `needs_review` +rather than inventing confidence. In particular, a confidence value supplied by +UmbraAudio is not treated as calibrated release evidence until it has been reviewed or +validated by UmbraTrans. + +The first adapter must not assume that a proposed identity-export endpoint exists. It must +work from the currently available read-only database/JSON artifacts and may adopt a +formal UmbraAudio export later without changing the UmbraTrans contract. + +### 4.2 What UmbraTrans must not do to UmbraAudio + +It must not: + +- modify `.umbra/project.db`, `speakers.json`, `voice_assignments.json`, or voice files; +- renumber or rewrite UmbraAudio utterances; +- write translation text into UmbraAudio's source fields; +- reuse UmbraAudio's audio markers as target-language markers; +- call private UmbraAudio functions as a required integration boundary; +- change the current web UI, CLI, TTS behavior, or export behavior; or +- require an UmbraAudio migration before the first UmbraTrans prototype. + +The preferred integration is a read-only adapter that creates a coherent snapshot. For +SQLite, that means a read-only/backup connection or an online backup, not a call to a +configuration loader or database class that creates directories, tables, or migrations as a +side effect. JSON, voice references, and prompt artifacts must be copied or hashed from the +same snapshot window. The adapter must be safe while an UmbraAudio batch is running. + +If a future UmbraAudio version exposes a formal export endpoint, the adapter can use that +endpoint without changing the contract. An integration test must prove that an import +leaves all UmbraAudio files unchanged. + +### 4.2.1 Coherent snapshot protocol + +A snapshot is a versioned artifact, not a best-effort collection of files. The adapter +creates a `snapshot_id` and records the source project path, source file resolution, +schema versions, file hashes, timestamps, and any available generation markers. + +The protocol is: + +1. Resolve `project.source_path` against the project root using one documented rule. + Reject missing, ambiguous, or unexpectedly external paths unless the operator explicitly + approved that source location. Copy the raw source into UmbraTrans storage before + parsing it. +2. Open the source database through SQLite's read-only or online-backup interface. Do not + call `umbraaudio.config.load_config()`, `Database.connect()`, or any helper that creates + directories, tables, or migrations. +3. Copy or hash `speakers.json`, `voice_assignments.json`, voice references, pronunciation + records, and prompt artifacts from the same snapshot window. A file that changes while + being copied is retried and then rejected if it cannot be stabilized. +4. Compare available generation/status markers and re-check file hashes. If the database, + JSON, and audio artifacts cannot be shown to belong to one generation, fail closed and + request a quiet source project or an explicit operator-approved mixed-generation import. +5. Write a snapshot manifest containing every source and hint artifact hash. Never write + the snapshot back into the source project. + +The integration test uses a WAL database, an active writer, a relative source path, spaces +and Unicode in paths, and a read-only source-tree permission mode. It compares a complete +before/after UmbraAudio file hash and configuration snapshot and proves that no new or +modified UmbraAudio file was created by the import. + +### 4.3 UmbraTrans data area + +Each book has an isolated data directory, for example: + +```text +/{book-slug}/ +├── source/ +│ ├── snapshots/{snapshot_id}/ +│ ├── source_book.v1.json +│ ├── source_text/ +│ └── provenance.json +├── analysis/ +│ ├── scenes/ +│ ├── discourse_units/ +│ ├── entities/ +│ └── issues/ +├── glossary/ +│ ├── glossary.v1.json +│ └── pronunciation.v1.json +├── translations/ +│ ├── fr-FR/ +│ ├── it-IT/ +│ ├── de-DE/ +│ ├── es-ES/ +│ ├── ru-RU/ +│ ├── ja-JP/ +│ └── zh-Hans/ +├── voices/ +│ ├── identity/ +│ ├── capabilities/ +│ └── profiles/{locale}/ +├── audio/ +│ ├── clips/{locale}/ +│ ├── chapters/{locale}/ +│ └── masters/{locale}/ +├── quality/ +│ ├── source/ +│ ├── translation/{locale}/ +│ └── audio/{locale}/ +├── manifests/ +└── logs/ +``` + +The directory is separate from UmbraAudio's `.umbra` directory and is rooted outside +the source project. Shared model weights and voice references may be referenced by +immutable path or content hash, but target artifacts are never written into the source +project. + +### 4.4 Current UmbraAudio audio boundary + +The current UmbraAudio TTS path is not assumed to be multilingual. Its request path, +language setting, English reference samples, pronunciation handling, and audio markers are +specific to the existing English production workflow. Target-language requests must pass +an explicit locale/language to a target-capable adapter and must not be sent through an +English-only fallback merely because the existing server is available. + +UmbraTrans must not reuse the English Rainbow Passage as the canonical target reference. +Each target voice profile has a target-language reference and, where supported, a target +reference transcript. Prompt and voice caches are scoped by work, logical speaker, +locale, voice revision, and TTS model; a filename-derived slug shared across projects or +languages is not safe for multilingual production. + +### 4.4a Target TTS capability registry + +Before Phase 4, UmbraTrans registers one or more target engines in a versioned capability +registry. An engine is not accepted merely because it accepts arbitrary text; it must +publish and pass a benchmark for each supported locale. This specification defines the +contract and readiness gates; it does not claim that a particular provider currently +supports all seven locales. + +Each `TTSCapabilityProfile` records: + +- engine and model identity, version, weight/config hash, and license; +- supported language/script/locale combinations; +- supported voice IDs and whether cloning, voice design, or both are available; +- reference-audio language and transcript requirements; +- pronunciation, SSML/phoneme, stress, pitch-accent, and lexicon controls; +- maximum text/span size, segmentation behavior, and supported output formats; +- language identification and synthesis metadata; +- concurrency, timeout, retry, and cancellation behavior; and +- measured quality, pronunciation, identity, and fallback results for the pilot. + +A preflight request includes locale, script, voice profile, render-span lengths, +pronunciation plan, and required output format. It returns `pass`, `blocked`, or +`unsupported`, with machine-readable reasons. A blocked locale/voice combination never +reaches generation. The all-locale v1 goal means that every requested locale has at least +one validated engine/profile path; it does not require one engine to support every locale. + +### 4.5 Reuse assessment + +UmbraAudio supplies useful upstream evidence, but UmbraTrans must not confuse existing +conveniences with a durable multilingual contract. + +| UmbraAudio capability | UmbraTrans use | +|---|---| +| Chapter and utterance segmentation | Advisory alignment hints; re-segmented into source/target semantic units | +| Speaker profiles, aliases, voice traits, and English prosody | Seed data for source meaning frames and identity contracts | +| Approved English voice references | Optional identity evidence and reference for engines with demonstrated cross-language support | +| Attribution confidence and reasoning | Advisory evidence with independent UmbraTrans confidence and review | +| Current English TTS and audio markers | Not reused as target-language generation or cache state | +| Current English-oriented WER/quality checks | Not reused as translation or CJK release gates | +| Current database row IDs and mutable indices | Never used as stable cross-language IDs | +| Proposed identity-export work | Optional future input; not a prerequisite for the adapter | + +This assessment keeps reuse useful while making every missing contract an explicit +UmbraTrans responsibility. + +## 5. Canonical Source Book Model + +The source model is the reusable foundation of UmbraTrans. It must be more expressive than +a list of sentences and more stable than a particular LLM response. + +### 5.1 Hierarchy + +```text +WorkEdition +└── Chapter + └── Scene + └── Beat / dramatic situation + └── MeaningBlock / discourse block + └── Translation unit + └── Source span / clause + └── Speaker turn or narration span + +TranslationRevision +└── TargetBlock + └── TargetSpeechUnit + └── Target render span +``` + +The source hierarchy describes the English work; the target hierarchy describes one +locale-specific revision. `TargetSpeechUnit` is not required to mirror one source +utterance. A target block or speech unit may merge, split, or reorder source units while +retaining explicit alignment links. + +Not every book has an explicit scene break. Scene boundaries may be inferred and must be +marked as inferred. Not every unit is dialogue. The model supports narration, dialogue, +inner thought, written text, signs, letters, headings, captions, and front/back matter. + +### 5.2 Required hierarchy records + +| Record | Stable identity | Required information | +|---|---|---| +| WorkEdition / Book | `book_id` | source language, title, author, edition, rights/provenance, source hash | +| Chapter | `chapter_id` | order, title, source text hash, front/back-matter role | +| Scene | `scene_id` | chapter, boundaries, participants, setting, time, viewpoint, goal, conflict, stakes, outcome, confidence | +| Beat | `beat_id` | scene, turn in action/communication, causal links, tension/change | +| Discourse block | `block_id` | scene, speaker/topic continuity, rhetorical function, preceding/following references | +| Translation unit | `unit_id` | source text, type, speaker, addressee, speech act, meaning frame, dependencies, risk flags | +| Source span | `span_id` | unit, character/line offsets, source hash, optional page/edition anchor | + +IDs are opaque and stable within a book revision. They must not depend on a translated +display name or a database row number that can change after re-indexing. The identity +registry persists speaker IDs, aliases, role instances, merge/split decisions, and +supporting evidence across source revisions. An automatic merge or identity rename is +never treated as harmless; it creates a reviewable identity revision and invalidates the +affected target artifacts. + +### 5.3 Scene meaning frame + +Each scene receives a structured `SceneMeaningFrame` containing, where supported by the +source: + +- scene purpose and immediate objective; +- participants and non-speaking entities; +- relationships and changes in relationship; +- physical setting and relevant time period; +- point of view and narrative mode; +- initiating event and expected outcome; +- conflict, threat, danger, stakes, and urgency; +- promises, demands, commands, warnings, secrets, and revelations; +- emotional trajectory and turning points; +- chronology and causal links; +- unresolved references, ambiguities, and source issues; and +- translation constraints, including terms that must remain consistent. + +The frame is an evidence-bearing analysis, not an invitation to add plot facts. Every +non-obvious claim records supporting source unit IDs and confidence. + +### 5.4 Utterance meaning frame + +Each translation unit carries a `MeaningFrame` with at least: + +```json +{ + "speech_act": "assertion", + "propositions": [ + { + "text": "The speaker reports a direct order from an authority.", + "source_spans": ["span_..."], + "certainty": 0.96 + } + ], + "intent": "warn the listener without openly defying the authority", + "stance": "guarded", + "emotion": "fearful", + "addressee_ids": ["speaker_..."], + "references": ["entity_..."], + "terminology_ids": ["term_..."], + "dependencies": ["unit_..."], + "ambiguities": [], + "translation_risks": ["register", "implicature"] +} +``` + +`propositions` are semantic obligations, not a demand for one-to-one sentence mapping. +They allow validation to compare what was said even when the target uses different syntax +or splits one English sentence into several target sentences. + +### 5.4a Fail-closed identity readiness + +Speaker identity is a release prerequisite, not a best-effort annotation. The UmbraTrans +identity registry stores, for every logical speaker: + +- stable `speaker_id` and speaker kind (`named_character`, `recurring_role`, + `scene_role_instance`, `group`, `ai`, or `narrator`); +- canonical English name, aliases, titles, and role scope; +- source evidence spans and attribution evidence kind; +- confidence, review status, and unresolved alternatives; +- merge/split history and identity revision; and +- co-occurring speakers and cast-distinctiveness group. + +An adapter may create candidates, but it must not silently merge aliases, turn a role into +a named character, or assign a low-confidence pronoun. Pronoun, role-only, off-screen, +group, and AI cases follow explicit evidence rules. If two candidates remain plausible, +the unit is `needs_source_review`. + +The source identity gate blocks translation and audio release when any speaking unit has +an unresolved speaker, an unapproved role/instance policy, or an identity revision that +has not been reviewed. Non-speaking mentions may remain unresolved only when they cannot +affect a target utterance, and the exception is recorded. The current UmbraAudio display +name, nullable `character_id`, and default confidence are never sufficient by themselves. + +A source identity report lists every unresolved unit and every proposed merge/split. This +gate is independent of whether a future Spec 17 identity export is implemented. + +### 5.5 Source span and alignment rules + +Every source span records character offsets into an immutable source chapter artifact. The +source text is never silently normalized in place. A normalized derivative may be created, +but it retains a mapping to the original and records normalization operations. + +Target text may have any of these alignments: + +- one source unit to one target unit; +- one source unit to several target units; +- several source units to one target unit; +- a reordered or nested alignment; or +- a non-textual rendering such as a stage direction or deliberate ellipsis. + +Every target unit must point back to one or more source units. A target unit with no source +anchor is a defect unless explicitly marked as an editor's note. + +### 5.5a Source format and offset contract + +Source offsets are Unicode code-point offsets into an explicitly identified immutable +artifact, not offsets into an unversioned database string. The source package records the +encoding, newline convention, Unicode normalization form, and whether offsets are absolute, +chapter-relative, or page/paragraph-relative. Normalization maps every changed code point +back to the raw source; CRLF conversion, combining marks, and repeated passages must not +make two spans indistinguishable. + +Each importer has a format contract: + +- EPUB records spine order, heading hierarchy, anchors, footnotes, and unsupported markup; +- PDF records page and block anchors plus OCR/extraction confidence; +- DOCX records paragraph, heading, table, caption, and note anchors; and +- TXT/MD records byte/character encoding, heading, and line-ending provenance. + +Images, tables, maps, captions, footnotes, and decorative elements receive an explicit +`preserved`, `omitted`, `nonverbal`, or `unsupported` disposition. A book whose source +coverage cannot be accounted for is blocked rather than silently flattened. Fixtures must +cover CJK text, combining characters, CRLF, repeated passages, tables, images, and notes. + +### 5.5b Ordered target render spans + +A target block may contain multiple speakers, a group utterance, an overlap, or an +approved non-textual event. It therefore cannot be represented safely by one target string +and one speaker ID. Each `TargetBlock` contains ordered `TargetRenderSpan` records: + +```json +{ + "render_span_id": "de_block_004_span_02", + "order": 2, + "source_span_ids": ["span_041", "span_042"], + "disposition": "translated", + "speaker_id": "speaker_kainda", + "surface_text": "…", + "overlap_group": null, + "pronunciation_plan_id": "de_pron_plan_004" +} +``` + +Required dispositions are `translated`, `approved_omission`, `nonverbal`, and +`editor_note`. Omissions require source-review approval and a reason; they are never +silently inferred by alignment length. Overlapping spans declare an `overlap_group` and +explicit start/end ordering. A group or multi-speaker block is rendered as multiple spans +or an explicitly defined group-performance track, never guessed from a single speaker. + +The translation package, audio renderer, and assembler all consume this ordered span +schema. A one-to-one `target_text` field is only a convenience projection for a single +speaker span; it is not the canonical render contract. + +## 6. Source Preparation and Author Control + +The source preparation stage is the only stage where the source author has routine +editorial authority. + +### 6.1 Source preparation tasks + +1. Preserve the original manuscript and compute a content hash. +2. Parse chapters, headings, paragraphs, page breaks, dialogue, and scene markers. +3. Validate encoding, extraction completeness, and paragraph order. +4. For scans, record OCR uncertainty and page/line anchors for later review. +5. Create immutable chapter artifacts and stable IDs. +6. Discover speakers, aliases, roles, entities, places, dates, and technical terms. +7. Analyze scenes, beats, discourse, and communication. +8. Build the English glossary, pronunciation candidates, and source-side issues. +9. Produce a source review package and a machine-readable source manifest. +10. Freeze the source revision before target translation begins. + +### 6.2 Author review boundary + +The author or source operator may edit: + +- the English source text and normalization decisions; +- character identity, aliases, and relationships; +- intended meaning, speech acts, conflict, and scene interpretation; +- canonical names, terms, and pronunciation decisions that are source facts; +- style, register, and emotional intent; and +- unresolved source ambiguities. + +The author does not need to: + +- translate target-language passages; +- choose target-language voice models; +- approve every target sentence; or +- diagnose target pronunciation for every locale. + +A target subsystem may raise a `source_ambiguity` issue. It cannot resolve that issue by +silently changing the English source. + +### 6.3 Source approval + +A source revision is approved only when: + +- all chapters and required front/back matter are accounted for; +- source coverage is complete, including explicit dispositions for tables, images, notes, + captions, and unsupported elements; +- source text hashes match the manifest; +- speaker IDs are stable for all speaking units or exceptions are recorded; +- no speaking unit remains unresolved, ambiguous, or dependent on an unapproved + role/instance/AI identity decision; +- scene boundaries and critical meaning frames have been reviewed; +- names, numbers, negations, and high-risk terms have no unresolved critical issues; and +- the author/source operator has approved the English preparation package. + +The approval is recorded as an immutable event. Later target work references that exact +source revision. + +## 7. Translation Architecture + +### 7.1 Target locale profiles + +Each target locale has a versioned profile. At minimum it defines: + +- BCP 47 locale and language/script pair; +- target writing system and text direction; +- sentence and utterance segmentation rules; +- quotation, dialogue, ellipsis, and emphasis conventions; +- number, date, time, measurement, and currency conventions; +- honorific, address, politeness, and register guidance; +- pronoun, kinship, and discourse reference behavior; +- target-specific typography and line-breaking rules; +- pronunciation and TTS capabilities; +- supported TTS engines and language-specific fallbacks; and +- target-specific human-review policy. + +The initial locale set is: + +| Language | Locale | Script | Notes | +|---|---|---|---| +| French | `fr-FR` | Latin | Default French France profile; other variants may be added later | +| Italian | `it-IT` | Latin | Default Italian Italy profile | +| German | `de-DE` | Latin | Default German Germany profile | +| Spanish | `es-ES` | Latin | Default Spanish Spain profile; Latin American variants may be added later | +| Russian | `ru-RU` | Cyrillic | Stress, case, and inflection are first-class concerns | +| Japanese | `ja-JP` | Japanese mixed scripts | Kanji readings, pitch/accent, honorifics, and line segmentation matter | +| Simplified Chinese | `zh-Hans` | Simplified Han | Segmenter must not assume whitespace between words; punctuation and readings matter | + +Locale is not just a language label. A later `fr-CA`, `de-AT`, `es-MX`, or `zh-Hant-TW` +must be a separate profile and release lane, not an accidental string substitution. + +### 7.2 Translation job graph + +```text +Source package frozen + │ + ├── Build book/scene summaries and discourse graph + ├── Build entity/term/reference indexes + └── Build target locale profiles + │ + ▼ + Scene translation briefs + │ + ├── Draft target units with source context + ├── Translate longer blocks with overlap + └── Run terminology and alignment checks + │ + ▼ + Scene-level continuity/reconciliation pass + │ + ▼ + Independent fidelity and fluency verification + │ + ▼ + Target text freeze + │ + ├── Pronunciation and TTS preparation + ├── Target voice casting/rendering + └── Final release validation +``` + +Each target language is an independent lane. A failure or revision in French must not +silently alter German, Italian, Spanish, Russian, Japanese, or Chinese artifacts. + +### 7.3 Scene translation brief + +UmbraTrans maintains a work-level translation brief and style guide in addition to the +scene brief. It records person and tense, narrative versus direct-speech register, +literalization versus cultural-adaptation policy, names and titles, historical vocabulary, +humor, dialect, honorifics, and any approved terminology decisions. A translation memory +may provide approved examples, but it never replaces scene context or semantic checks. + +Before target text generation, each scene receives a compact brief containing: + +- source scene ID and source hash; +- scene goal, conflict, stakes, and outcome; +- participants and speaker/addressee map; +- relevant book and chapter context; +- preceding and following scene summaries; +- entity and term IDs with target forms; +- names, honorifics, and relationship rules; +- discourse conventions and point of view; +- chronology and causality constraints; +- source ambiguities and approved interpretations; +- required emotional/register trajectory; and +- risk classification. + +The brief is generated from evidence-bearing source records. It is not a free-form +summary that may contradict the source. + +### 7.3a Hierarchical translation memory + +Context retrieval is an explicit subsystem, not an unbounded conversation transcript. +UmbraTrans maintains versioned memory artifacts at four levels: + +- **Book memory:** premise, point of view, global style, terminology, and durable facts; +- **Arc memory:** a multi-scene conflict, promise, reveal, injury, relationship change, or + objective; +- **Scene memory:** the current brief, participants, local chronology, and unresolved + references; +- **Entity memory:** names, roles, relationships, preferences, and last-known state for + each character, group, place, object, and term. + +Every memory fact stores supporting source IDs, valid time range, confidence, and revision. +A fact is never promoted to book memory solely because a model repeated it. Conflicting +facts are retained with their evidence and marked unresolved until source review or an +explicit narrative exception explains them. + +Retrieval is deterministic: select the current scene, dependent prior scenes, referenced +entities, active arc facts, and unresolved issues using a token budget defined by the +locale profile. Compaction may shorten a summary but must preserve fact IDs, evidence +links, and conflict markers. Target-language memory is separate from source memory and +records which target wording and terminology were used. + +A cross-scene benchmark covers names, promises, injuries, timeline facts, relationship +changes, and repeated off-screen speakers. A translation that loses one of these facts +fails even when each individual sentence is fluent. + +### 7.4 Drafting rules + +The target translator receives source units plus their scene brief and relevant context. +It returns structured output: + +```json +{ + "target_unit_id": "fr_ch006_s02_u003", + "source_unit_ids": ["ch006_s02_u003"], + "target_text": "…", + "alignment": [ + {"source_span_id": "span_001", "target_char_start": 0, "target_char_end": 42} + ], + "meaning_notes": ["preserves the warning force without adding a new threat"], + "uncertainty": null, + "risk_flags": [] +} +``` + +The translator must: + +- preserve propositions, modality, negation, speech act, and speaker intent; +- use the target language's natural grammar and idiom; +- keep names and terms consistent with the glossary; +- retain meaningful repetition, hesitation, and register when they carry character or + dramatic information; +- avoid adding explanations, facts, dialogue, or emotional claims not present in the source; +- preserve the distinction between narration and speech; +- respect target punctuation and script conventions; and +- leave a low-confidence issue rather than fabricate a confident translation. + +### 7.5 Translation unit boundaries + +The existing UmbraAudio utterance segmentation is an input anchor, not an immutable +translation boundary. A scene may require: + +- merging several source units when the target grammar demands it; +- splitting a long source unit at a natural target pause; +- moving a clause across a target sentence boundary; +- preserving a repeated phrase for rhetorical effect; or +- rendering an English-specific sound or pun with a target equivalent. + +The target renderer must still preserve source anchors, speaker identity, and the +relationship between dialogue and narration. + +### 7.6 Terminology, names, and cultural adaptation + +A versioned glossary stores at least: + +- stable term/entity ID; +- source form and aliases; +- target forms by locale; +- whether the form is translated, transliterated, retained, or localized; +- pronunciation/reading data; +- usage constraints and examples; and +- provenance and approval status. + +Names, invented terms, ranks, units, organizations, technologies, and culturally specific +references receive explicit policies. The system must not silently translate a name in one +chapter and transliterate it in another. + +Adaptation is permitted only when the source's communicative intent is preserved and the +adaptation is recorded. Literalism is not a quality rule. + +## 8. Translation Validation + +Translation validation is a first-class subsystem, not an optional afterthought. + +### 8.1 Hard checks + +The following conditions block release when found in a critical unit: + +- missing or duplicated source content; +- changed negation, modality, quantity, or certainty; +- changed speaker, addressee, or speech act; +- invented entity, event, motive, or threat; +- lost conflict, warning, promise, or emotional force; +- inconsistent name, title, or terminology in a way that changes identity; +- untranslated target-language content or accidental language mixing, except explicitly + approved editor notes and registered source terms; +- broken source/target alignment; or +- unresolved source ambiguity presented as a confident target fact. + +### 8.2 Soft checks + +Soft checks produce warnings or review recommendations: + +- awkward but understandable phrasing; +- register drift; +- repeated target terminology that may be stylistically valid; +- punctuation or typography inconsistencies; +- minor pacing or rhythm concerns; +- optional cultural adaptation opportunities; and +- low-confidence terminology or pronunciation decisions. + +### 8.3 Validation methods + +UmbraTrans uses several independent methods: + +1. **Deterministic checks:** missing anchors, duplicate IDs, glossary violations, locale + script checks, punctuation rules, numbers/dates, and untranslated segments. +2. **Semantic comparison:** compare source propositions and discourse obligations with the + target, independent of word order. +3. **Independent critic:** a verifier model or reviewer receives the source, target, + alignment, and brief and reports defects with evidence. +4. **Continuity check:** compare neighboring target units for names, register, tense, + deixis, and discourse flow. +5. **Back-translation:** useful as a diagnostic signal, never the sole release gate. +6. **Human review:** required for high-risk units and a qualified, statistically useful + sample of lower-risk units, with release sign-off defined by the locale profile. +7. **Benchmark scoring:** track results by language, scene type, risk class, and model + version to detect regressions. + +The same model that drafted a passage must not be the only judge of that passage. The +system should support an independent model, a different prompt, deterministic checks, or +human review. + +### 8.4 Review queue + +A unit enters review when it has any of the following: + +- low source-analysis confidence; +- unresolved pronoun, deixis, or ellipsis; +- threat, conflict, legal, medical, technical, or culturally sensitive content; +- new entity, invented term, or unresolved name; +- target critic disagreement; +- low confidence in translation, alignment, or register; +- pronunciation ambiguity; or +- a repeated phrase that may carry a cross-scene reference. + +Review decisions are stored with the unit, evidence, reviewer or model, reason, and +resulting revision. A target review does not modify the source revision. + +### 8.5 Quality report + +Every target language receives a report containing: + +- units translated and units blocked; +- coverage and alignment rates; +- critical, major, minor, and warning counts; +- speech-act, entity, number, negation, and terminology checks; +- critic agreement and disagreement rates; +- human-reviewed sample results; +- unresolved issues; +- model/provider/prompt versions; and +- release recommendation. + +Thresholds are calibrated per locale and risk class. A single universal numeric score is +not sufficient for a high-quality literary translation. Native review uses a documented +rubric covering semantic accuracy, completeness, fluency, register/style, terminology, +speaker intent, and target-language naturalness. Critical errors override an otherwise +high average score. + +### 8.6 Verification bundles and release benchmark policy + +A verifier receives an immutable bundle, not a path that invites undeclared private reads. +The translation bundle contains the source package, semantic package, source snapshot, +locale profile, glossary, translation package, ordered render spans, and relevant memory +retrieval. The audio bundle contains the audio manifest, approved target text and render +spans, voice profiles, pronunciation plans, audio QA profile, rights manifest, and all +reference evidence required by the checks. + +Every issue is evidence-bearing and includes issue type, severity, source/target IDs, +observed evidence, expected invariant, detector/reviewer, and repair state. A verifier that +cannot load one of its declared inputs fails closed. + +Each locale has a versioned `TranslationQualityProfile` and a human-reviewed benchmark +created before production use. The benchmark covers narrative-heavy, dialogue-heavy, +high-conflict, high-risk terminology, role/pronoun ambiguity, and long-range continuity +passages. The initial pilot requires at least 20 representative scenes and 500 source +units per locale, with every high-risk category represented; production sample sizes are +calculated from the declared error rate and confidence interval. + +The minimum release policy is fail-closed: + +- zero critical omissions, additions, speaker errors, negation/modality errors, or + identity-changing entity/number errors; +- zero major errors in the high-risk human-review set; +- a predeclared maximum major-error rate for the full benchmark, recommended at or below + 0.1% before expansion; +- a qualified native reviewer signs off on every locale's high-risk set and stratified + release sample, with a recommended mean rubric score of at least 4/5 and no rubric + dimension below 3/5; +- independent adjudication is required when reviewer and model disagree; and +- no release occurs when the benchmark, reviewer qualification, or required sign-off is + unavailable. + +Thresholds are stored in the locale profile and changes create a new reviewed benchmark +revision; they cannot be relaxed silently to make a build pass. + +## 9. Voice Identity and Multilingual Audio Production + +### 9.1 The cross-language voice problem + +An English voice reference cannot be assumed to produce a reliable German, Russian, +Japanese, or Chinese voice merely by changing the input text. Speech models, phoneme +inventories, prosody, and training data differ by language. UmbraTrans therefore treats +English voice identity as evidence and direction, not as a guarantee of a bit-identical +target voice. + +The release contract is: + +> A listener must recognize the same character, narrator, role, and relationship in every +> released language, even when the target voice has language-appropriate timbre and +> pronunciation. + +UmbraTrans must choose and record one of these identity policies for a work: + +- **Same actor:** the same acoustic performer is intended across all languages. This is + allowed only when the selected engine demonstrates the required cross-language identity. +- **Native equivalent:** each locale uses a native-language performer who preserves the + character's age, gender presentation, role, relative energy, and cast relationships. +- **Hybrid:** a stable broad identity is shared while accent, phonetics, and prosody are + adapted to each target language. + +The recommended default is **hybrid/native equivalent**: preserve the logical character +and recognizable identity features, then use a target-native voice profile where exact +cross-language cloning is not demonstrated. A project may choose a stricter policy, but it +must not claim an identity level that its listening tests do not support. + +### 9.2 Voice identity contract + +The source speaker profile produces a language-neutral `VoiceIdentityContract`: + +```json +{ + "speaker_id": "speaker_kainda", + "role": "character", + "narrator": false, + "identity_features": { + "pitch_register": "mid", + "timbre": "clear", + "texture": "smooth", + "energy_baseline": "assertive", + "pace_baseline": "moderate", + "formality": "conversational", + "age_presentation": "young-adult", + "accent_policy": "no unsupported accent", + "distinguishing_trait": "measured, articulate delivery" + }, + "relationship_constraints": [ + "distinct from the narrator", + "distinct from the AI voice in shared scenes" + ], + "source_voice_reference": "voices/kainda_ref.wav", + "source_voice_revision": "voice_004" +} +``` + +The contract is stable across languages. It does not contain a language-specific voice +name or assume that a particular engine is available. The logical speaker model keeps +separate IDs for named characters, recurring roles, scene-scoped role instances, group or +collective speakers, AI/computer voices, and the narrator. A localized display name or +title is an attribute of a locale profile, not the identity key. + +### 9.3 Target voice profile + +For each locale, UmbraTrans creates a versioned `TargetVoiceProfile`: + +```json +{ + "speaker_id": "speaker_kainda", + "acoustic_instance_id": "de_acoustic_kainda_01", + "cast_kind": "named_character", + "scene_scope": null, + "locale": "de-DE", + "profile_revision": "de_voice_003", + "engine": "target-tts-adapter", + "model": "model-name-and-version", + "voice_id": "native-or-multilingual-voice-id", + "reference_audio": "voices/de-DE/speaker_kainda.wav", + "identity_contract_revision": "identity_002", + "pronunciation_lexicon_revision": "de_pron_007", + "pronunciation_plan_revision": "de_pron_plan_004", + "approved": true, + "approval_evidence": "review/speaker_kainda_de.json" +} +``` + +A target profile must be approved for that locale before it can be used for a release. +Different locales may use different native voices, provided they satisfy the same identity +contract and cast-distinctiveness requirements. + +### 9.4 Voice realization strategies + +The renderer chooses the best available strategy and records which one it used: + +1. **Multilingual voice clone:** use a model that explicitly supports the target language + and preserves the source identity. +2. **Target-language voice design:** derive a native target voice from the identity + contract, target-language voice description, and an approved target reference. +3. **Hybrid identity anchor:** use the source reference or speaker embedding only when + the engine demonstrates cross-language support; combine it with target-language + conditioning and native pronunciation. +4. **Target voice pool:** use a curated native voice whose measured identity and + distinctiveness satisfy the contract. + +If no strategy can meet the identity gate, the locale/speaker combination is not released. +UmbraTrans must not choose a random voice merely to keep a batch moving. Every request +must state the target language and locale explicitly, and the adapter must verify the +returned audio is in that language. An unavailable multilingual model, an English-only +server, or an unsupported voice is a blocked build condition—not permission to fall back +silently to English, Edge-TTS, or a narrator voice. + +### 9.5 Cast distinctiveness + +Distinctiveness is checked per locale and per co-occurring cast, because phonetics and +voice models differ across languages. The check considers: + +- speaker confusion in target-language samples; +- similarity to the target narrator; +- similarity between characters who share scenes; +- intelligibility and naturalness; +- target pronunciation and prosody; and +- human recognizability of the character identity. + +Raw speaker embeddings from English and German audio are not assumed to be numerically +comparable. Cross-language identity is validated through the identity contract, target +voice samples, structured descriptors, and human listening. Within-language embedding +similarity remains useful as one signal. + +### 9.5a Cast matrix and listener identity benchmark + +Logical identity and acoustic casting are separate. A locale's cast matrix maps each +logical speaker to one or more approved acoustic instances, and records whether a role is +single, scene-scoped, group, overlapping, synthetic, or deliberately changed by scene. +A group may use a shared group voice, multiple member voices, or a defined ensemble effect; +the choice is explicit. A scene-role instance cannot silently inherit another scene's +voice. + +The listener benchmark is randomized and native-listener based. Each trial presents a +reference utterance, a same-character utterance, and different-character controls from the +target locale. It records same/different decisions, confidence, response time, listener +locale, and a chance baseline. The acceptance threshold, confidence interval method, +minimum sample size, and adjudication rule are declared in the locale quality profile +before the test. The recommended pilot gate is at least 90% correct same-character +identification, no more than 10% false identification of different characters, and a +reported lower confidence bound above the chance baseline. A release cannot claim +cross-language voice continuity without a passing benchmark for each locale policy. + +### 9.6 Prosody and performance translation + +English emotion, pace, and speech-act metadata are direction, not literal target values. +The target performance pass maps: + +- speech act and force; +- emotional intensity and restraint; +- target-language politeness and register; +- sentence rhythm and pause placement; +- emphasis and contrast; +- question, command, threat, and warning contours; +- target-specific pronunciation and stress; and +- the character's stable voice baseline. + +The output is a target `PerformancePlan` that can be passed to a TTS adapter. A target +language may require a different rhythm or emotional treatment to preserve the same +communicative effect. + +### 9.7 Target pronunciation + +The target pronunciation subsystem produces a locale-specific lexicon, including: + +- target spelling and reading; +- transliteration where appropriate; +- IPA or engine-supported phoneme data; +- language-specific stress, pitch accent, or syllable rules; +- context-sensitive readings for names and homographs; +- pronunciation audio previews; +- TTS override syntax or SSML where supported; and +- unresolved pronunciation issues. + +Examples of language-specific concerns include Japanese kanji readings and pitch/accent, +Simplified Chinese character readings and word segmentation, Russian stress and inflection, +German compound words and proper names, French liaison and elision, Italian consonant +length, and Spanish accent and dialect policy. + +The target pronunciation lexicon is separate from the English source pronunciation data. + +### 9.7a Pronunciation plan and engine negotiation + +A lexicon entry is not yet a render instruction. Before synthesis, the pronunciation +subsystem resolves every flagged term in a render span into a versioned `PronunciationPlan`: + +```json +{ + "plan_id": "de_pron_plan_004", + "locale": "de-DE", + "engine_capability_revision": "tts_cap_003", + "surface_text": "…", + "resolved_tokens": [ + { + "text": "…", + "reading": "…", + "phoneme": null, + "ssml": null, + "source": "confirmed_lexicon" + } + ], + "unsupported_terms": [], + "manual_audio_references": [], + "status": "ready" +} +``` + +The plan records whether the selected engine can honor each reading, phoneme, stress, or +SSML instruction. Unsupported terms are blocked or routed to an explicitly approved +pronunciation audio/manual-audio path; they are never silently guessed by the TTS model. +The approved translated text remains unchanged. Only the resolved plan and any manual +audio are passed to the renderer, and the plan is included in the audio cache key and QA +bundle. + +## 10. Audio Rendering and Audio Validation + +### 10.1 Render unit + +The audio renderer receives a target speech segment and a render request: + +```json +{ + "clip_id": "de_ch006_s02_u003_0001", + "source_unit_ids": ["ch006_s02_u003"], + "target_block_id": "de_block_004", + "language": "de", + "locale": "de-DE", + "render_spans": [ + { + "render_span_id": "de_block_004_span_01", + "order": 0, + "speaker_id": "speaker_kainda", + "voice_profile_id": "de_voice_003", + "surface_text": "…", + "disposition": "translated", + "pronunciation_plan_id": "de_pron_plan_004" + } + ], + "performance_plan": {}, + "pronunciation_plan_revision": "de_pron_plan_004", + "tts_capability_revision": "tts_cap_003", + "engine": "target-tts-adapter", + "model": "model-name-and-version", + "seed": null +} +``` + +The request and response are immutable build inputs. The renderer writes one clip per +render span (or an explicitly defined group-performance track), never guessing a speaker +for a multi-speaker target block. `approved_omission` and `editor_note` spans are not spoken +unless a release profile explicitly defines a separate performance treatment. The +renderer records duration, sample rate, channels, engine, model, voice profile, capability +revision, and generation timestamp. + +### 10.2 Cache and invalidation + +A clip cache key includes at least: + +- target text hash; +- source unit, target block, and ordered render-span IDs; +- locale and locale-profile revision; +- speaker/acoustic-instance IDs and target voice-profile revision; +- performance-plan hash; +- pronunciation-plan and lexicon revisions; +- TTS capability profile, engine/model/weight/configuration versions; +- generation parameters; and +- radio/mix treatment revision; +- Audio QA profile and verifier/model versions. + +The invalidation matrix is: + +| Change | Rebuild scope | +|---|---| +| English source text | Source analysis, all affected target translations, target audio | +| English meaning/analysis decision | Affected translation briefs, target units, and audio | +| Glossary term | Units containing that term, dependent continuity checks, and affected audio | +| Target translation text | That target locale's affected clips and chapters | +| Target pronunciation entry | Clips containing the term in that locale | +| Voice profile | That speaker's clips in that locale; cast checks; chapters containing that speaker | +| Performance plan | Affected clips only | +| Audio mix/radio treatment | Masters and affected assembly, not translation text | +| TTS model version | Audio clips requiring the new engine, subject to an explicit re-render policy | +| Source identity analysis or speaker merge/split | Affected source gate, translations, voice bindings, and audio; re-run dependent cast checks | +| Locale profile or memory retrieval policy | Affected target translation, verification bundle, and downstream audio/release | +| Verifier, ASR, or Audio QA model/profile | Affected quality report and release approval; audio may be rechecked without retranslation | +| Rights, metadata, or voice/model license | Release manifest and affected approvals; no automatic regeneration unless content policy changes | +| Co-speaker cast profile | Cast matrix, affected co-occurrence checks, chapters, and release approvals | + +### 10.3 Audio quality checks + +Each clip and chapter is checked for: + +#### Technical integrity + +- file exists, is non-empty, decodes, and has the expected duration; +- no hard clipping, malformed frames, or truncation; +- correct sample rate, channel layout, and loudness profile; +- acceptable silence and pause boundaries; and +- no accidental cross-language or cross-speaker concatenation. + +#### Speech intelligibility + +- target-language identification; +- ASR transcript quality against the approved target segment, using WER for + space-delimited languages and CER or an equivalent character-level metric for Japanese + and Chinese; +- proper-name, number, and terminology recall; +- pronunciation of flagged terms and names; +- detection of omitted, repeated, or hallucinated speech; and +- intelligibility under the selected radio-program mix. + +#### Voice identity + +- target speaker matches the approved target voice profile; +- no confusion with co-occurring characters; +- stable identity across chapters; +- correct narrator/character distinction; and +- human-recognizable identity contract for a sampled set. + +#### Performance + +- intended emotion and speech act are perceptible without theatrical distortion; +- target-language rhythm and pauses are natural; +- threat, command, question, and warning contours are appropriate; +- voice remains intelligible over music/effects, if used; and +- no cross-language prosody artifact is accepted silently. + +### 10.3a Audio QA profiles + +Audio QA is configured per locale and pinned per release. An `AudioQaProfile` records: + +- ASR model/version and reference normalization rules; +- WER or CER calculation, Unicode normalization, punctuation handling, and number/name + normalization; +- language-ID model/version and false-positive/false-negative threshold; +- minimum name, terminology, and flagged-pronunciation recall; +- pronunciation benchmark cases and manual-audio policy; +- naturalness, prosody, and radio-mix human rating rubric; +- minimum clip, chapter, and final-master listening sample; +- hard and soft thresholds, retry policy, and escalation owner; and +- permitted ASR nondeterminism or adjudication process. + +For Japanese and Simplified Chinese, the profile uses a documented CER or equivalent +character-level metric after approved normalization; it never reuses an English +whitespace-WER threshold. Every locale must pass clip and final-master checks. An audio +build with no pinned QA profile is blocked. + +Recommended initial hard gates, to be ratified per locale before the pilot, are: language +identification confidence at least 0.99, WER at or below 0.05 for the selected Latin-script +ASR profile, CER at or below 0.08 for Japanese and Simplified Chinese, 100% recall for +explicitly flagged critical names/terms, 100% technical-integrity pass, and a native +listener naturalness rating of at least 4/5. The cross-language identity profile must +predeclare its same-character acceptance rate, false-identity rate, chance baseline, and +confidence-interval method. A locale may tighten these values; it may not silently loosen +them after seeing a build result. + +## 10.4 Radio-program treatment + +The default output is a speech-led radio-play audiobook: + +- distinct narrator and character voices; +- restrained, consistent delivery; +- intentional scene transitions; +- clean speech-first mixing; and +- optional period-appropriate room tone, stingers, or music when rights and quality are + confirmed. + +Music and effects are a separate `radio_profile` layer. They must not obscure pronunciation +or change the meaning of a scene. The first release may produce a clean audiobook master and +a radio-treated master from the same approved speech clips. + +### 10.5 Assembly and release + +The assembler uses the target clip manifest and language-specific chapter timing. It can +reuse an existing audio assembly implementation through an adapter, but UmbraTrans owns the +target manifest and release metadata. + +Each locale release contains: + +- translated text files in one or more supported formats; +- chapter and clip manifests; +- approved voice profiles and identity contracts; +- translation and audio quality reports; +- cover/title/author metadata in the target language; +- rights and provenance records; and +- final M4B/MP3/WAV outputs according to the release profile. + +A release is never assembled from a mix of locales or from unapproved clips. + +### 10.6 Translated-book rendering + +The target text renderer consumes the frozen target blocks and produces a reader-facing +book while retaining the structured translation package and alignment sidecar. It must: + +- preserve chapter order, headings, front matter, back matter, and intentional scene + breaks; +- apply target-language punctuation, quotation, typography, numerals, and script rules; +- keep speaker identity and dialogue/narration boundaries semantically correct; +- avoid exposing analysis notes, confidence values, or TTS-only pronunciation directives in + the reader-facing text; and +- produce target-language metadata and credits without silently translating legal or + attribution information. + +The first text export is UTF-8 Markdown plus the structured package. EPUB, DOCX, and PDF +exporters are downstream adapters and must be tested against the same frozen target +revision. TTS-only rendering changes, such as pronunciation hints, remain separate from +the approved translated text. + +## 11. Versioned Data Contracts + +All contracts use a schema name and version. Unknown fields must be ignored by older +consumers; missing required fields must fail validation rather than be silently defaulted. + +### 11.1 Source Book Package + +```json +{ + "schema": "umbra.source_book/v1", + "book_id": "book_deal_hunter", + "source_language": "en", + "source_revision": "source_007", + "snapshot_id": "snapshot_20260923_001", + "identity_revision": "identity_002", + "coverage_manifest": "source/coverage.v1.json", + "title": "Deal Hunter", + "author": "B. D. Murphy", + "source_hash": "sha256:...", + "provenance": { + "source_path": "…", + "edition": "author manuscript", + "rights_status": "author-owned", + "prepared_by": "…", + "prepared_at": "2026-09-23T00:00:00Z" + }, + "speakers": [], + "chapters": [], + "scenes": [], + "units": [], + "entities": [], + "terms": [], + "issues": [], + "approval": { + "status": "approved", + "revision": "source_007", + "approved_at": "2026-09-23T00:00:00Z" + } +} +``` + +The arrays may be stored in separate files for large books, but the package manifest must +identify the parts and their hashes. + +### 11.2 Translation Package + +```json +{ + "schema": "umbra.translation/v1", + "book_id": "book_deal_hunter", + "source_revision": "source_007", + "source_snapshot_id": "snapshot_20260923_001", + "source_identity_revision": "identity_002", + "locale": "de-DE", + "translation_revision": "de_007", + "locale_profile_revision": "de_profile_002", + "glossary_revision": "glossary_003", + "chapters": [], + "units": [ + { + "target_unit_id": "de_ch006_s02_u003", + "source_unit_ids": ["ch006_s02_u003"], + "target_text": "…", + "speaker_id": "speaker_chief_guard", + "render_spans": [], + "alignment": [], + "meaning_obligations": [], + "risk_flags": [], + "status": "validated" + } + ], + "quality_report": "quality/translation/de-DE/report.v1.json", + "approval": { + "status": "approved", + "revision": "de_007" + } +} +``` + +The `target_text` and `speaker_id` fields in this example are convenience projections for a +single-speaker target unit. `render_spans` is the canonical field for multi-speaker, +group, overlap, omission, and nonverbal blocks. + +### 11.3 Voice Identity Contract + +The contract is referenced by every target voice profile and every rendered clip. It must +be immutable for a given `identity_revision`. + +### 11.4 Audio Build Manifest + +```json +{ + "schema": "umbra.audio_build/v1", + "book_id": "book_deal_hunter", + "source_revision": "source_007", + "source_snapshot_id": "snapshot_20260923_001", + "source_identity_revision": "identity_002", + "locale": "de-DE", + "translation_revision": "de_007", + "voice_identity_revision": "identity_002", + "audio_revision": "de_audio_004", + "clips": [ + { + "clip_id": "de_ch006_s02_u003_0001", + "source_unit_ids": ["ch006_s02_u003"], + "target_unit_id": "de_ch006_s02_u003", + "render_span_ids": ["de_block_004_span_01"], + "speaker_id": "speaker_chief_guard", + "voice_profile_id": "de_voice_003", + "path": "audio/clips/de-DE/de_ch006_s02_u003_0001.wav", + "status": "validated" + } + ], + "chapters": [], + "quality_report": "quality/audio/de-DE/report.v1.json" +} +``` + +### 11.5 Subsystem ports + +The initial implementation may use local Python modules, but the logical boundaries are +fixed: + +```python +class SourcePackageReader(Protocol): + def load(self, book_id: str, revision: str) -> SourceBookPackage: ... + def snapshot(self, umbraaudio_project: Path) -> SourceBookPackage: ... + + +class SemanticAnalyzer(Protocol): + def analyze(self, source: SourceBookPackage) -> SemanticPackage: ... + + +class Translator(Protocol): + def translate_scene(self, request: TranslationSceneRequest) -> TranslationDraft: ... + def reconcile_scene(self, request: TranslationReconciliationRequest) -> TranslationDraft: ... + + +class TranslationVerifier(Protocol): + def verify(self, bundle: TranslationVerificationBundle) -> TranslationQualityReport: ... + + +class TargetVoiceProvider(Protocol): + def get_or_create_profile(self, contract: VoiceIdentityContract, locale: str) -> TargetVoiceProfile: ... + + +class TargetTTS(Protocol): + def preflight(self, request: TTSPreflightRequest) -> TTSPreflightResult: ... + def synthesize(self, request: SpeechClipRequest) -> AudioClipResult: ... + + +class TTSCapabilityRegistry(Protocol): + def get(self, locale: str, engine_id: str) -> TTSCapabilityProfile: ... + + +class AudioVerifier(Protocol): + def verify(self, bundle: AudioVerificationBundle) -> AudioQualityReport: ... +``` + +These are logical interfaces, not permission to import or mutate UmbraAudio internals. +`TranslationVerificationBundle` and `AudioVerificationBundle` are immutable, versioned +collections of the contracts described above. They are constructed by the orchestrator, +content-addressed, and passed explicitly to verifiers. A verifier may not silently read a +private project file, call a mutable database helper, or fetch an unpinned model. + +`TranslationVerificationBundle` contains `SourcePackage`, `SemanticPackage`, source +snapshot/coverage evidence, `LocaleProfile`, glossary, `TranslationPackage`, ordered +render spans, and the exact memory retrieval used for the scene. `AudioVerificationBundle` +contains `AudioBuildManifest`, approved target text/render spans, `TargetVoiceProfile`s, +`PronunciationPlan`s, `AudioQaProfile`, rights evidence, and reference audio/transcripts. + +## 12. Orchestration, State, and Scheduling + +### 12.1 Per-locale state machine + +```text +SOURCE_RECEIVED + → SOURCE_SNAPSHOT_READY + → SOURCE_NORMALIZED + → SOURCE_SEMANTICALLY_ANALYZED + → SOURCE_IDENTITY_READY + → SOURCE_MEMORY_READY + → SOURCE_APPROVED + → LOCALE_PROFILE_READY + → SCENE_BRIEFS_READY + → TRANSLATION_DRAFTED + → TRANSLATION_RECONCILED + → TRANSLATION_VALIDATED + → TEXT_FROZEN + → VOICE_PROFILES_APPROVED + → TTS_CAPABILITY_READY + → AUDIO_RENDERED + → AUDIO_VALIDATED + → ASSEMBLED + → RELEASED +``` + +A failure records the stage, attempt, error class, affected IDs, and recovery action. A +locale can move backward to the earliest invalidated stage without resetting unrelated +locales. + +### 12.2 Scheduling + +- Source analysis is a shared prerequisite for all target languages. +- The source identity gate and hierarchical memory must be ready before translation + drafting; unresolved speaking identity blocks dependent scenes. +- After source approval, target language lanes may run in parallel. +- Within a language, scene briefs and memory retrieval precede scene translation. +- Translation of neighboring scenes may overlap for continuity, but each target unit has + a single owning scene and revision. +- Pronunciation and voice preparation may proceed after a text candidate exists, but + release audio waits for a frozen, validated target text. +- TTS capability preflight and pronunciation-plan resolution precede synthesis. +- TTS generation is resumable at render-span/clip granularity. +- Assembly waits for all required chapter clips, overlap/group tracks, and chapter-level + quality checks. +- A release waits for all required manifests, metadata, rights records, reviewer sign-offs, + and reports. + +### 12.3 Idempotency + +Every job has a stable `job_key` derived from its input revisions. Re-running a completed +job does not create a new revision. A changed input creates a new revision and records the +dependency edge that caused the invalidation. + +### 12.4 Content-addressed dependency graph + +Reproducibility is implemented as a content-addressed dependency graph, not as a log line. +Every artifact records hashes for: + +- raw source snapshot, source parse, semantic analysis, identity registry, and memory; +- glossary, locale profile, translation brief, prompt, retrieved context, and target text; +- TTS capability profile, model/configuration/weight hash, voice profile, pronunciation + plan, and generation parameters/seed; +- translation verifier, ASR/QA models, Audio QA profile, and radio/mix profile; and +- rights, metadata, credits, and release policy. + +Canonical serialization is UTF-8 with sorted keys and a documented number/Unicode policy. +Revisions are append-only and committed atomically: a partially written job is never a +readable release input. The graph records both upstream and downstream edges so a changed +co-speaker profile, locale profile, QA model, or rights record invalidates exactly the +affected cast checks, translations, clips, chapters, and releases. + +The cache key and release manifest must include every dependency hash, not only the text +and speaker name. A deterministic seed is used when the engine supports one; supported +nondeterminism is recorded with an acceptance rule and repeatability measurement. + +## 13. Review, Roles, and Approval + +### 13.1 Roles + +| Role | Responsibilities | +|---|---| +| Source author/operator | English source, source analysis, intended meaning, glossary, source approval | +| Translation system | Drafting, terminology application, continuity, candidate generation | +| Translation verifier | Independent fidelity, fluency, and risk review | +| Target-language reviewer | Qualified native review, adjudication, and release sign-off for the locale profile | +| Audio director | Voice casting, performance direction, radio treatment, audio approval | +| Release operator | Metadata, rights, packaging, final release decision | +| UmbraTrans orchestrator | Scheduling, retries, provenance, state, and gate enforcement | + +### 13.2 Review policy + +The system aims to review a small, high-value queue rather than every line. It still +requires broad automated coverage and human sampling. Review effort is allocated by risk: + +- critical/high-risk units: human review before release; +- medium-risk units: independent model review plus a human sample; +- low-risk units: automated checks plus a qualified native stratified release sample, + unless a regression is detected. + +Machine output may be auto-accepted into a candidate revision, but it is never an +unattended release decision. Every required approval is bound to exact source, target, +voice, model, and artifact hashes. Changing any of those inputs invalidates only the +affected approval and downstream artifacts. + +The author can inspect source-side issues and their downstream impact, but is not assigned +routine target-language proofreading. A target-language reviewer may correct a translation +without changing the English source. A proofreader must not silently make a new creative +or identity decision; such a change is routed through the appropriate source or release +review. + +## 14. Public-Domain and Historical Book Support + +Public-domain ingestion is a supported future source class, not merely another file +extension. + +### 14.1 Rights and provenance + +For every source, record: + +- author and original publication information where known; +- source edition and digitization source; +- jurisdiction-specific public-domain determination; +- scan/digitization provenance; +- whether the source contains modern editorial additions; and +- license or permission status for the specific edition and distribution. + +Public-domain status is jurisdiction- and edition-dependent. It does not automatically +clear the rights of a new translation, voice model, music library, or recording. The +source rights status and the output rights status are separate fields. A public-domain +source must still pass a rights gate before distribution, and every release records the +rights holder, jurisdiction, credits, and output license. + +### 14.2 Historical text preparation + +Historical and public-domain sources require: + +- OCR confidence and page/line provenance; +- spelling and typography preservation policy; +- archaic vocabulary and idiom handling; +- footnote/endnote policy; +- illustration, map, caption, and table handling; +- front matter and advertising removal or preservation policy; +- uncertain-name and uncertain-term issue tracking; and +- a normalized text derivative that never overwrites the source scan/extraction. + +Modernization must be explicit. A source may be translated in its historical form, +normalized for readability, or both, but the choice is recorded and cannot vary by target +language without explanation. + +## 15. Security, Privacy, and Operations + +UmbraTrans is expected to handle unpublished manuscripts and author drafts. + +- Prefer local or private model endpoints for source and translation data. +- Do not send a manuscript to a cloud provider without an explicit data-processing policy + and project configuration. +- Store API keys outside manifests and source packages. +- Redact secrets and credentials from logs and quality reports. +- Preserve immutable source artifacts and write derived data to separate directories. +- Validate file paths and prevent path traversal in artifact assembly. +- Record model/provider requests sufficiently for reproducibility without exposing + credentials or unnecessary private text. +- Make deletion and archival explicit; deleting a derived translation must not delete the + approved source revision. +- Treat public-domain ingestion as untrusted input until rights and extraction checks pass. +- Treat manuscript text, OCR output, metadata, and retrieved reference material as data, + never as model instructions; isolate them from system prompts and tool permissions. +- Maintain a provider allowlist and record retention, no-training, regional processing, + and deletion settings before a provider may receive source or target text. +- Require authenticated, role-based access for review, approval, artifact download, and + release operations; separate author, proofreader, audio, and release permissions. +- Encrypt private source and target artifacts at rest and in transit where the deployment + requires it; keep encryption keys outside the artifact store. +- Resolve and validate paths with symlink-safe canonicalization before copying, hashing, or + assembling files; reject traversal, device files, and paths outside the project root. +- Run untrusted parsing and model-input preparation in a restricted worker or container + with no unnecessary credentials or write access. +- Record voice-actor consent, model license, reference-audio license, and music/effects + rights evidence in the release manifest. A technically valid voice is not automatically + authorized for distribution. + +UmbraTrans maintains a threat model covering path traversal, prompt injection in manuscripts +and OCR, malicious archives, model-provider leakage, unauthorized review, cache poisoning, +voice-license violations, and cross-project artifact contamination. Security controls are +release requirements, not optional hardening. + +## 16. Proposed Package and Module Boundaries + +The eventual implementation may live in this repository, but the first phase should be a +separate `umbratans` package or service with no required changes to `umbraaudio`: + +```text +umbratans/ +├── contracts/ # Versioned Pydantic/JSON schemas +├── adapters/ # Read-only UmbraAudio and file importers +├── source/ # Snapshots, normalization, IDs, hashes, provenance +├── semantics/ # Scenes, beats, discourse, propositions, memory +├── terminology/ # Entities, names, terms, cultural adaptation +├── locales/ # Language profiles and script/typography rules +├── translation/ # Briefs, drafting, render spans, verification bundles +├── voices/ # Identity contracts, cast matrix, capabilities, profiles +├── pronunciation/ # Target readings, plans, phonemes, previews +├── audio/ # Render requests, TTS adapters, cache, assembly +├── quality/ # Translation, audio, and release gates +├── orchestration/ # State machine, jobs, retries, invalidation +├── storage/ # Artifact manifests, atomic writes, locking +└── cli/ # Operator commands and status reporting +``` + +The first implementation should prefer standard-library and existing project-compatible +clients where possible. New dependencies require a separate dependency review and must not +silently change the UmbraAudio environment. + +The recommended deployment boundary is a separate distribution, virtual environment, or +container. UmbraTrans may live in the same source repository for development, but its +installable package, dependency lock, model configuration, and output root must be +separate from the `umbraaudio` distribution. Production code must not import private +`umbraaudio` modules; it communicates through the read-only adapter and versioned +contracts. The default UmbraTrans output root is outside the source project's +`umbraaudio` data directory, even when both tools share a volume. + +A subprocess integration test runs snapshot and build against a read-only source tree, +uses a separate environment, and compares UmbraAudio file/configuration hashes before and +after. This test prevents a packaging or dependency change from becoming an accidental +UmbraAudio behavior change. + +## 17. Implementation Plan + +### Phase 0 — Decisions and contracts + +Deliverables: + +- approve the target locale list and locale policy; +- define the source/translation/voice/audio schemas, including ordered render spans, + verification bundles, and pronunciation plans; +- define the source snapshot, fail-closed identity, TTS capability, and release-quality + contracts; +- select a small English fixture with scenes, conflict, threats, ambiguous references, + named entities, invented terms, and multiple voices; +- define the source author approval boundary; +- define target review staffing, benchmark minimums, and release sign-off policy; and +- select candidate TTS engines and define the per-locale capability benchmark. + +Exit criterion: every subsystem can be implemented against a versioned contract without +reading another subsystem's private files. + +### Phase 1 — Read-only source adapter + +Deliverables: + +- perform a coherent, read-only snapshot of an existing UmbraAudio project while it may + be running, with a snapshot ID and fail-closed generation check; +- parse the original source with structure/offset preservation where possible and use + UmbraAudio chapter text as an advisory cross-check; +- import chapters, utterances, speakers, aliases, confidence, prosody, and pronunciation + data when available; +- create stable UmbraTrans IDs and hashes without using mutable row IDs or display names; +- preserve missing/unknown fields explicitly; +- produce a source manifest and import report; and +- pass the active-writer, WAL, relative-path, Unicode-path, and read-only-tree integration + tests. + +Exit criterion: repeated snapshots are deterministic and the UmbraAudio project remains +byte-for-byte and behaviorally unchanged. + +### Phase 2 — Source semantic enrichment + +Deliverables: + +- scene and beat detection; +- discourse and speech-act analysis; +- entity/reference graph; +- source meaning frames and propositions; +- fail-closed speaker identity registry and role/AI/group resolution; +- format-specific source anchors and normalization maps; +- hierarchical book/arc/scene/entity memory; +- glossary and source issue generation; +- source quality report and review package. + +Exit criterion: a human can inspect and correct the English preparation, and every +downstream unit points to a stable source context. + +### Phase 3 — Translation pilot + +Start with two contrasting target lanes: + +- one Latin-script European language, recommended `de-DE`; and +- one non-Latin or segmentation-sensitive language, recommended `ja-JP` or `zh-Hans`. + +Deliverables: + +- locale profiles and translation-quality profiles; +- translation briefs and hierarchical memory retrieval; +- draft, reconcile, and verify passes using explicit verification bundles; +- ordered render-span alignment and terminology validation; +- target text package and quality report; and +- a human-reviewed benchmark for both lanes. + +Exit criterion: the fixture produces natural, meaning-preserving translations with no +unresolved critical semantic or identity defects. + +### Phase 4 — Voice identity and audio pilot + +Deliverables: + +- language-neutral voice identity contracts and per-locale cast matrices; +- target voice profiles for the pilot locales; +- pronunciation lexicons, resolved pronunciation plans, and preview workflow; +- TTS capability registry and fail-closed preflight; +- clip-level rendering from ordered render spans and cache; +- pinned target-language ASR and Audio QA profiles; and +- randomized native-listener identity benchmark for the same character. + +Exit criterion: a listener can identify the same character in both pilot languages, and +all critical pronunciation/technical audio checks pass. + +### Phase 5 — All requested locales + +Add French, Italian, Spanish, and Russian, then complete Japanese and Simplified Chinese +with language-specific test suites and at least one passing TTS capability profile per +locale. Each locale must pass its own profile, terminology, script, pronunciation, audio, +and human-review gates. A locale is not considered complete merely because its text files +were generated. + +### Phase 6 — Public-domain ingestion + +Add provenance/rights checks, OCR confidence handling, historical text policy, and fixture +books from multiple periods and scan qualities. Test both preservation and normalization +modes. + +### Phase 7 — Production hardening + +Add durable job recovery, artifact locking, cost/usage reporting, observability, release +metadata, operator UI or CLI dashboards, regression benchmarks, backup/restore, retention/ +deletion policies, provider allowlists, role-based access, encryption, sandboxed parsing, +voice-rights evidence, and separate-environment packaging tests. + +## 18. Test Strategy and Acceptance Criteria + +### 18.1 Contract tests + +- schema validation rejects missing IDs, invalid locales, and broken references; +- source snapshots are deterministic; +- all target units have source anchors; +- all target speakers have a voice identity contract; +- every audio clip has an approved target voice profile and a valid translation revision; +- active-writer snapshot tests prove no UmbraAudio file, directory, or configuration is + created or modified; +- unresolved speaker, role, group, AI, or scene-instance identity blocks release; +- TTS preflight rejects unsupported locale/voice/pronunciation combinations; +- multi-speaker, merged, omitted, nonverbal, and overlapping target blocks round-trip + through render spans; and +- verification bundles are complete and do not require undeclared private reads. + +### 18.2 Semantic-context tests + +Fixtures must include: + +- a threat whose literal words are mild but whose scene context makes it dangerous; +- a command that is softened by politeness; +- a pronoun with multiple possible antecedents; +- a repeated line spoken by the same off-screen character; +- a named character mentioned in dialogue but not speaking; +- a scene transition with a changed cast; +- an idiom, pun, or culturally specific reference; +- numbers, negation, uncertainty, and modality; and +- an invented term whose meaning is established across scenes. + +Tests must prove that the translator receives scene/beat context and that a locally fluent +but contextually wrong translation is rejected. + +### 18.3 Language-specific tests + +- French and Italian: register, agreement, punctuation, and natural sentence rhythm; +- German: compound nouns, gender/article decisions, formal address, and correct names; +- Spanish: register, dialect policy, punctuation, and names; +- Russian: Cyrillic script, case, stress, inflection, and speech-act force; +- Japanese: kanji readings, mixed scripts, honorifics, segmentation, and target punctuation; +- Simplified Chinese: word segmentation without spaces, character readings, punctuation, + and natural spoken syntax. + +### 18.4 Audio tests + +- mocked TTS tests for preflight, request/cache/invalidation behavior, and capability + profile enforcement; +- target-language ASR checks with WER/CER appropriate to the script and pinned QA models; +- pronunciation-plan tests for names, invented terms, flagged words, and unsupported + engine features; +- speaker identity tests within each locale, including groups, overlaps, and scene roles; +- randomized native-listener cross-language listening benchmark with a chance baseline and + confidence interval; +- loudness, peak, clipping, silence, duration, and file-integrity tests; and +- failure tests for unavailable models, malformed responses, language mismatch, and + accidental fallback. + +### 18.5 Initial release targets + +These are pilot targets, not claims of guaranteed literary quality: + +- 100% of source discourse units have a target disposition; +- zero unresolved critical omissions, additions, speaker errors, negation errors, or + high-risk entity/number errors in a release audit; +- zero major errors in the predeclared high-risk human-review set and no more than the + locale profile's benchmark limit for major errors overall; +- 100% of target units are in the intended locale and script (or have an explicitly + approved mixed-script exception); +- 100% of speaking units have a reviewed stable identity, including role/group/AI policy; +- all flagged pronunciation terms are resolved or explicitly blocked through a + `PronunciationPlan`; +- 100% of release clips and final masters pass their pinned technical and intelligibility + profiles; +- every locale has a qualified native reviewer, stratified release sample, adjudication + record, and documented issue rate; and +- a listener can identify sampled recurring characters across languages at the declared + acceptance rate with a reported chance baseline and confidence interval. + +### 18.6 Snapshot, format, security, and packaging tests + +- Snapshot tests cover WAL databases, an active writer, relative and Unicode/space paths, + read-only source trees, and before/after source-tree hashes. +- Format fixtures cover EPUB structure, PDF page anchors, DOCX tables/notes, TXT/MD line + endings, CJK and combining characters, repeated passages, images, and unsupported + elements. +- Identity fixtures cover pronouns, off-screen voices, role-only speakers, groups, AI, + merges, splits, and scene-scoped instances. +- Translation fixtures cover merge, split, reorder, approved omission, nonverbal spans, + editor notes, overlap, and multi-speaker blocks. +- Memory fixtures cover names, promises, injuries, timeline facts, relationship changes, + and repeated off-screen speakers across scenes. +- Security tests cover malicious archives, prompt injection, path traversal, symlinks, + provider leakage, unauthorized approval, cache poisoning, and voice-rights evidence. +- A subprocess test runs UmbraTrans in a separate environment against a read-only source + tree and proves that the UmbraAudio distribution and project remain unchanged. + +## 19. Risks and Mitigations + +| Risk | Mitigation | +|---|---| +| Literal or context-blind translation | Scene briefs, discourse graph, semantic obligations, independent critic | +| Lost conflict, threats, or speaker intent | Explicit speech-act and scene-conflict analysis with hard checks | +| Character names drift across chapters | Stable entity IDs and versioned locale glossary | +| Wrong speaker attribution | Fail-closed identity registry, confidence/evidence fields, and source review queue | +| English voice does not transfer to target language | Language-neutral identity contract plus target voice profile and explicit strategy | +| Target voices collide | Per-locale cast checks and target-language listening review | +| Pronunciation errors | Target pronunciation lexicon, previews, ASR checks, and language-specific overrides | +| Model self-evaluation | Independent verifier, deterministic checks, benchmarks, and human sample | +| Long-context inconsistency | Hierarchical summaries, overlap, scene reconciliation, and target memory | +| Public-domain OCR or rights errors | Source provenance, confidence spans, rights gate, and non-destructive normalization | +| Silent quality degradation | Explicit fallback records, blocked release, and no automatic voice substitution | +| Cost and runtime growth | Incremental scene/clip caching, parallel locale lanes, and revision-based invalidation | +| Unreviewed cultural adaptation | Adaptation policy, evidence trail, and target-language review for sensitive content | +| Unsupported target TTS locale or feature | Versioned capability registry, per-locale preflight, and blocked unsupported combinations | +| Mixed-generation or relative-path import | Snapshot ID, raw-source resolution, read-only backup, generation/hash checks, and fail-closed import | +| Unresolved speaker or role identity | Fail-closed identity registry, evidence-backed review, and no unresolved speaking unit at release | +| Restructured target blocks lose speaker identity | Ordered render spans, dispositions, overlap/group rules, and span-level rendering | +| Verifier uses unpinned or private context | Explicit immutable verification bundles and pinned QA/model profiles | +| Cross-scene facts drift | Hierarchical memory with provenance, deterministic retrieval, and continuity benchmarks | +| Stale approval or cache contamination | Content-addressed dependency graph, append-only revisions, scoped caches, and hash-bound approvals | +| Package/dependency change affects UmbraAudio | Separate distribution/environment, no private imports, read-only subprocess integration test | +| Voice or model rights are unclear | Rights evidence, provider allowlists, consent/license records, and release gate | + +## 20. Open Decisions and Recommended Defaults + +These decisions do not block the architecture, but they should be settled before the +corresponding implementation phase: + +1. **Target-language review staffing:** recommended target-language reviewers for + high-risk scenes and a statistically meaningful sample of all other scenes. +2. **Translation model/provider:** keep the port provider-neutral; benchmark at least two + model configurations before selecting a production default. +3. **Voice identity policy:** release on character continuity and approved target voice + profiles, with bit-identical cross-language timbre treated as an optional capability. +4. **Text output formats:** begin with UTF-8 Markdown and a structured JSON package; add + EPUB/DOCX/PDF exporters after the contract is stable. +5. **Audio output formats:** produce WAV intermediates, MP3 for distribution, and M4B when + metadata and chapter navigation are validated. +6. **Radio treatment:** ship a clean speech master first; make music/effects an optional, + separately licensed profile. +7. **Public-domain policy:** preserve the source edition and create an explicit normalized + derivative; never silently modernize or discard front matter. +8. **Source review policy:** the author approves English preparation; target-language + correctness is owned by the translation and release workflow. +9. **TTS provider selection:** benchmark at least one target-capable engine per locale; + do not declare the seven-language audio target achievable until every locale has a + passing capability profile and pronunciation/identity test. +10. **Release thresholds:** approve the per-locale benchmark size, major-error limit, + reviewer qualification, adjudication process, and listener-identification threshold + before the pilot; changes require a new benchmark revision. + +## 21. Definition of Done for UmbraTrans v1 + +UmbraTrans v1 is complete when: + +- an existing UmbraAudio project can be snapshotted through the coherent, fail-closed + protocol without modification; +- the source package contains stable book/chapter/scene/unit/utterance identities, + format/offset anchors, and a reviewed fail-closed speaker identity registry; +- scene, utterance, and hierarchical memory analysis is available to the translator; +- one English book can produce all seven target text lanes; +- each target lane has a locale profile, glossary, ordered render spans, translation QA + report, qualified review evidence, and frozen revision; +- each target language has at least one passing TTS capability profile, approved target + voice profiles for the full speaking cast, and resolved pronunciation plans; +- Japanese and Simplified Chinese pronunciation, script, and CER handling pass their + language tests; +- each target audiobook has clip-level provenance, pinned Audio QA reports, and passing + final-master checks; +- a randomized native-listener benchmark confirms recurring character identity across + languages with a declared acceptance rate; +- all release artifacts are reproducible from a content-addressed dependency graph; and +- no UmbraAudio source, database, configuration, route, output, distribution, or + dependency environment is modified by the UmbraTrans path. + +## 22. Enterprise Control Plane + +The multi-user project catalog, authorization, artifact lineage, commercial voice/pronunciation +library, billing/entitlements, and distributed job infrastructure are specified separately in +[`19-enterprise-control-plane.md`](19-enterprise-control-plane.md). UmbraTrans project data +contracts are designed to be stored by that control plane without changing this per-book +production boundary. + +## 23. Summary + +UmbraTrans is not a word-by-word translation layer placed after UmbraAudio. It is a +meaning-aware production system that reuses UmbraAudio's English analysis and voice +identity evidence while adding scene context, translation planning, target-language +validation, multilingual pronunciation, target voice casting, and audiobook production. + +The central architectural decision is to keep two things separate: + +1. **what the source means and who is communicating it**; and +2. **how a particular language and voice system expresses and performs it**. + +That separation allows the same character to be recognizable across English, German, +French, Italian, Spanish, Russian, Japanese, and Simplified Chinese without pretending +that all languages have the same grammar, pronunciation, or vocal physics. diff --git a/specs/19-enterprise-control-plane.md b/specs/19-enterprise-control-plane.md new file mode 100644 index 0000000..fcea088 --- /dev/null +++ b/specs/19-enterprise-control-plane.md @@ -0,0 +1,1121 @@ +# Spec 19 — Umbra Enterprise Control Plane, Multi-Tenant Projects, and Voice Library + +- **Status:** Proposed architecture and migration specification +- **Date:** 2026-09-23 +- **Scope:** UmbraTrans enterprise platform; additive to UmbraAudio +- **Primary users:** Authors, editors, translators, proofreaders, audio directors, voice-library operators, release managers, and administrators + +## 1. Executive Decision + +The project needs two related but distinct systems: + +1. **UmbraAudio** remains the existing local English audiobook engine and its read-only + upstream source for UmbraTrans. +2. **Umbra Enterprise Control Plane** becomes the multi-user, multi-project system of record + for projects, people, permissions, translations, deliverables, reviews, voice assets, + pronunciation records, licensing, costs, and releases. + +The control plane must not be implemented as another table or mode inside the current +UmbraAudio project database. It must be a separate service/package with its own relational +database, artifact store, job scheduler, authentication boundary, and audit log. + +The central database decision is: + +> Do not create a database table per project or a table per language. Use a shared logical +> relational schema with `tenant_id`, `project_id`, `locale_id`, and revision foreign keys. +> A separate physical database may be used for a large tenant or for an isolated deployment, +> but it uses the same logical schema and migration system. + +A project contains one source work and many target locales. Each locale produces independent +translation and audio revisions, and each deliverable type (EPUB, PDF, M4B, MP3, WAV, and +future formats) is a versioned artifact. Every source change, translation fix, voice change, +pronunciation change, audio regeneration, approval, download, and release is traceable. + +The current UmbraAudio implementation has useful **local multi-project primitives**, but it +is not yet a multi-user enterprise platform. This document defines the target architecture +without changing the current UmbraAudio code. + +## 2. Current-State Audit + +### 2.1 What exists today + +The current repository has several useful building blocks: + +| Existing capability | Current location | Assessment | +|---|---|---| +| Per-project directory and database | `project_bootstrap.py`, `config.py`, `database.py` | Good local isolation primitive | +| Project discovery | `config.list_projects()` | Filesystem scan, not an authoritative multi-user catalog | +| Project selection | `active_project` browser cookie in `web/routes.py` | UI context only; not authentication or authorization | +| Per-project `project_id` columns | `database.py`, `SCHEMA_SQL` | Useful locally, but IDs are scoped to one project database | +| New project web flow | `/projects`, `/projects/new`, `/api/projects` | Creates isolated projects successfully for a local user | +| SQLite WAL | project and voice registry connections | Helps same-host contention; not distributed coordination | +| Shared voice registry | `umbraaudio/voices/registry.py` | Cross-project names and some voice metadata, but not a commercial voice catalog | +| Cross-project name/voice library | `umbraaudio/library.py` | Basic reuse; lacks locale, rights, cost, versions, and access policy | +| Pipeline/generation status | per-project JSON files and in-process managers | Useful for one operator; not a durable multi-worker scheduler | +| UmbraTrans contracts | `specs/18-UmbraTrans.md` | Defines per-project production semantics; needs an enterprise control plane around it | + +### 2.2 What is not enterprise-ready today + +The current setup does **not** provide: + +- user accounts, organizations, tenants, or authentication; +- project membership, roles, permissions, or row-level access control; +- an authoritative project registry with ownership and lifecycle state; +- safe concurrent editing by multiple people; +- a durable distributed job queue or worker lease system; +- per-user or per-project resource quotas; +- target-language deliverable and revision tracking; +- a central artifact registry for ebooks, PDFs, audiobooks, and QA reports; +- a rights/usage/billing model for rented or purchased voices; +- a global, locale-aware pronunciation catalog with review history; +- provider availability, license expiry, territory, and takedown handling; +- a complete audit trail for reads, writes, approvals, downloads, and releases; +- encrypted object storage and signed artifact access; +- a searchable, cross-project operational dashboard; or +- a reliable separation between a user's selected project and a user's authorized projects. + +The `active_project` cookie must not be treated as an authorization mechanism. A user must +be able to select a project only after the server verifies that the user is a member of that +project. + +### 2.3 What must remain unchanged + +The enterprise migration does not require modifying the current UmbraAudio engine. The +control plane imports a read-only snapshot of the current project and keeps a one-way link +back to the legacy project. English audiobook generation can continue in UmbraAudio while +translation and enterprise workflows run in the new system. + +## 3. Goals and Non-Goals + +### 3.1 Goals + +1. Manage many organizations, users, projects, source works, locales, and deliverables. +2. Let each user see only the projects and locales they are authorized to access. +3. Support a project picker and project-scoped dashboard with language/release status. +4. Track source, translation, pronunciation, voice, audio, review, and release revisions. +5. Support concurrent work on different projects with bounded shared-resource usage. +6. Store ebooks, PDFs, audio, and other large artifacts outside the relational database. +7. Provide immutable lineage and auditability from source text to released audiobook. +8. Support a reusable, licensed voice and pronunciation library across projects. +9. Track voice rental, usage, cost, rights, territory, term, and expiration. +10. Make fixes safe: a correction creates a new revision and never destroys history. +11. Support local development and a later production deployment without changing the + domain model. + +### 3.2 Non-goals + +- Turning SQLite into a multi-user authorization system. +- Creating one table per project or one table per language. +- Storing WAV, M4B, MP3, PDF, or EPUB binaries as large blobs in the control database. +- Letting a browser cookie grant access to a project. +- Letting a target-language correction silently mutate the English source. +- Letting a voice asset be reused after its license or territory authorization expires. +- Coupling the control plane to one TTS provider or one deployment host. +- Replacing UmbraAudio's existing English generation path in the first migration phase. + +## 4. Architecture at a Glance + +```text +Users / Reviewers / Release Managers + │ + ▼ +┌──────────────────────────────────────────────────────────────┐ +│ Web/API Control Plane │ +│ authentication · organizations · projects · access · review │ +│ deliverables · audit · dashboards · API │ +└───────────────┬───────────────────────┬──────────────────────┘ + │ │ + ▼ ▼ +┌──────────────────────────┐ ┌────────────────────────────────┐ +│ PostgreSQL Control DB │ │ Versioned Object/Artifact Store │ +│ tenants, projects, │ │ source, translations, PDFs, │ +│ revisions, issues, │ │ audio, manifests, QA reports │ +│ voices, rights, usage │ │ encrypted, signed, immutable │ +└───────────────┬──────────┘ └───────────────┬────────────────┘ + │ │ + └──────────────┬───────────────┘ + ▼ + ┌─────────────────────────┐ + │ Durable Job Scheduler │ + │ queues · leases · quota │ + │ retries · cancellation │ + └────────────┬────────────┘ + │ + ┌─────────────────────┼─────────────────────┐ + ▼ ▼ ▼ + Analysis workers Translation workers Audio/Export workers + CPU/LLM/GPU pools provider/model pools TTS/mix/ffmpeg pools +``` + +The control plane is the source of truth for identity, authorization, state, lineage, and +release decisions. Workers are disposable and may be restarted at any time. + +## 5. Tenancy, Identity, and Access Control + +### 5.1 Tenancy model + +The primary tenancy boundary is an `organization` (also called a tenant). An organization +may contain many users, projects, voice-library records, billing accounts, and policies. + +A user may belong to multiple organizations. A user may have different roles in different +organizations and different projects. + +```text +organization + ├── users and memberships + ├── projects + │ ├── project members + │ ├── source work + │ └── target locales + ├── shared voice catalog + ├── pronunciation catalog + ├── billing and entitlements + └── audit and policy records +``` + +### 5.2 Identity provider + +The control plane uses an external OpenID Connect/SAML identity provider or an equivalent +managed identity service. Local passwords may exist only as a controlled fallback for a +private installation. The web application never trusts a user-supplied project slug, +organization ID, or role without a server-side authorization check. + +Every request carries an authenticated subject. A project selection is a signed UI/session +context, not a capability. + +### 5.3 Roles and permissions + +The initial role set is: + +| Role | Capabilities | +|---|---| +| `owner` | Organization ownership, billing, destructive actions, all projects | +| `admin` | Organization configuration, users, projects, policies, library administration | +| `producer` | Create projects, schedule work, manage deliverables and releases | +| `source_editor` | Edit approved English source analysis and source facts | +| `translator` | Draft and revise target text within assigned locales/scenes | +| `proofreader` | Review and approve target text; request source clarification | +| `audio_director` | Cast voices, approve pronunciation and performance, approve audio | +| `voice_manager` | Manage voice catalog, licenses, reservations, rates, and provider connections | +| `release_manager` | Approve and publish deliverables, metadata, credits, and manifests | +| `viewer` | Read-only access to assigned projects and approved/review-visible artifacts | +| `billing_admin` | View usage, invoices, entitlements, and spending limits; no manuscript edit | + +Permissions are additive policy records, not hard-coded role names in route handlers. A +route or worker asks the authorization service whether a subject may perform an action on a +specific resource. + +### 5.4 Resource-level access + +Access may be restricted by: + +- organization; +- project; +- locale; +- source revision; +- translation revision; +- deliverable/revision; +- voice library record; +- artifact classification; or +- release state. + +A project member may have access to only the English source while a target proofreader has +access to a specific locale and its review queue. A voice manager may see licensing and +usage metadata without seeing an unpublished manuscript. + +### 5.5 Database enforcement + +PostgreSQL row-level security is the default enforcement layer. Application checks improve +UX, but they do not replace database policies. Every tenant-owned row carries `tenant_id`. +Worker service accounts receive only the permissions required for their queue and artifact +scope. + +Sensitive actions are denied by default: + +- a user cannot download an unpublished artifact without an explicit grant; +- a translator cannot approve their own final release without the required independent + review policy; +- a worker cannot read a project outside its lease; +- a voice asset cannot be attached to a project without an active entitlement; and +- a release cannot include a blocked or unapproved revision. + +## 6. Work, Project, Locale, and Release Model + +A source ebook is not the same thing as a production project. The model separates them. + +### 6.1 Work + +`work` is the bibliographic identity of a book: + +- title, author, series, ISBN/work identifier; +- source language and original publication information; +- rights/provenance records; and +- one or more source editions/assets. + +A public-domain edition and an author's revised manuscript can be different source assets +belonging to the same work, or separate works if the editorial relationship is uncertain. + +### 6.2 Project + +`project` is a production workspace for one source revision and a selected set of locales, +quality policies, voice policy, team, schedule, and release policy. + +A project contains: + +- one or more source assets and frozen source revisions; +- a current source-analysis revision; +- project members and assignments; +- one or more target locale lanes; +- deliverable definitions; +- review and issue queues; +- voice and pronunciation bindings; and +- release history. + +A project can be cloned as a new production effort without copying historical revisions or +silently reusing a voice license. + +### 6.3 Locale lane + +`project_locale` is the relationship between a project and a target locale such as +`de-DE`, `ja-JP`, or `zh-Hans`. It contains locale-specific: + +- translation status and current revision; +- proofreader and translation assignments; +- terminology and style profile; +- pronunciation profile; +- voice/cast matrix; +- audio-build status; +- QA profile and reports; +- deliverable status; and +- release state. + +There is no `translations_de` table, `translations_fr` table, or `audio_de` table. All +locales use the same tables with a `locale_id` foreign key and appropriate indexes. + +### 6.4 Deliverable + +A `deliverable` is an output type requested for a project/locale, such as: + +- EPUB ebook; +- PDF ebook; +- print PDF; +- M4B audiobook; +- MP3 chapter package; +- WAV masters; +- translation-only DOCX/Markdown package; or +- a future format. + +A deliverable has many immutable `deliverable_revisions`. A revision is never overwritten +after approval. A fix creates a successor revision linked through `supersedes_revision_id`. + +## 7. Storage and Database Topology + +### 7.1 PostgreSQL control database + +The control plane uses PostgreSQL as its system of record. SQLite remains appropriate for a +single-user local UmbraAudio project or a disposable worker cache, but it is not the +enterprise multi-user database. + +PostgreSQL stores: + +- identity and membership; +- project and locale state; +- source/translation/analysis metadata; +- revisions and lineage; +- review tasks, issues, approvals, and audit events; +- voice catalog metadata, entitlements, rates, and usage; +- job and lease state; +- artifact metadata and storage URIs; and +- release manifests and quality report metadata. + +### 7.2 Object/artifact store + +Large files live in an encrypted object store or equivalent content-addressed storage: + +- original ebooks and scans; +- normalized source artifacts; +- translation exports; +- PDF/EPUB/DOCX files; +- voice references and approved pronunciation audio; +- individual audio clips; +- chapter and full-book masters; +- QA media samples; and +- signed manifests and reports where file size requires it. + +The relational database stores a storage key, URI, checksum, byte size, media type, +retention class, encryption key reference, and access policy. It does not store large binary +payloads as ordinary rows. + +### 7.3 Physical isolation options + +The default production topology is one shared PostgreSQL cluster with tenant RLS. When a +customer requires stronger isolation or a very large scale, deploy a separate database +cluster or database for that tenant using the same schema migrations. This is database-per- +tenant, not table-per-project and not table-per-language. + +Object storage may use tenant prefixes, separate buckets, or separate encryption keys. The +application always resolves storage through an authorization-aware artifact service; users +never receive arbitrary filesystem paths. + +### 7.4 Object lifecycle + +Artifacts are immutable and content-addressed. A write follows: + +1. request an upload session; +2. stream to a quarantine/temporary object; +3. calculate SHA-256 and validate media type; +4. commit the object and metadata in a transaction; +5. run malware/archive validation and metadata extraction; +6. mark the artifact `available`; or +7. mark it `quarantined`/`failed` without exposing it to workers. + +Deletion is a governed tombstone/retention operation, not an immediate filesystem delete. +Legal holds, active licenses, release history, and audit requirements can prevent deletion. + +## 8. Logical Relational Model + +The following is the logical model, not a mandate to create every table in the first +sprint. Names may change, but the relationships and keys are stable. + +### 8.1 Identity and tenancy + +```text +organizations +users +organization_memberships +projects +project_members +project_access_grants +``` + +Important fields: + +- `organizations(id, slug, name, status, retention_policy, created_at)` +- `users(id, identity_subject, display_name, status, created_at)` +- `organization_memberships(organization_id, user_id, role, status)` +- `projects(id, organization_id, work_id, slug, name, status, source_locale, created_by)` +- `project_members(project_id, user_id, role, status)` +- `project_access_grants(id, project_id, principal_type, principal_id, scope, expires_at)` + +### 8.2 Source and production content + +```text +works +source_assets +source_revisions +analysis_revisions +semantic_packages +identity_registries +glossary_revisions +memory_revisions +translation_revisions +translation_blocks +target_render_spans +alignment_links +``` + +Important foreign keys: + +- every project row has `organization_id` and `work_id`; +- every source revision has `project_id` and `source_asset_id`; +- every analysis revision has `source_revision_id`; +- every translation revision has `project_id`, `locale_id`, and `source_revision_id`; +- every target render span belongs to one translation revision and has an ordinal order; +- every alignment link references source and target spans, never only raw display text. + +### 8.3 Locale, review, and release + +```text +locales +project_locales +review_tasks +review_comments +review_decisions +issues +issue_links +approvals +deliverables +deliverable_revisions +release_manifests +``` + +`locales` is a controlled catalog of BCP-47 language/script/locale values. A locale can be +used by many projects, while `project_locales` stores project-specific status and policy. + +`issues` are first-class records with: + +- organization, project, locale, and optional source/target/audio scope; +- severity and category; +- exact source span, target render span, clip, chapter, or deliverable link; +- status, assignee, reviewer, and timestamps; +- evidence and reproduction information; +- fix revision and regression-test result; and +- audit event references. + +### 8.4 Artifacts + +```text +artifacts +artifact_versions +artifact_checksums +artifact_links +artifact_access_events +``` + +An `artifact_version` represents one immutable file or structured package. It links to the +revision that produced it and records: + +- storage key and signed-access policy; +- SHA-256, size, media type, and compression; +- producer job and model/engine versions; +- source/translation/voice/pronunciation/audio dependencies; +- retention and legal-hold state; and +- supersedes/derived-from relationships. + +### 8.5 Operations and audit + +```text +jobs +job_attempts +job_leases +worker_heartbeats +resource_pools +resource_reservations +audit_events +idempotency_keys +outbox_events +``` + +Every mutation writes an audit event containing actor, organization, project, resource, +action, before/after revision IDs, timestamp, request ID, and reason where applicable. + +## 9. Artifact Lineage, Revisions, and Fixes + +The canonical lineage is: + +```text +source asset + → immutable source revision + → analysis/semantic revision + → identity and glossary revisions + → target translation revision + → ordered target render spans + → target voice/pronunciation binding + → audio clip and chapter revisions + → deliverable revision + → release manifest +``` + +A fix is never an in-place mutation of a released artifact. The workflow is: + +1. open an issue against an exact revision and scope; +2. assign the appropriate owner; +3. create a source, translation, voice, pronunciation, audio, or release revision; +4. run only the affected validation/rebuild jobs; +5. compare the new artifact with its predecessor; +6. obtain required independent approvals; and +7. publish the successor or reject it. + +The UI must show: + +- current revision; +- previous approved revision; +- open issues; +- rebuild progress; +- quality delta; +- voice and pronunciation cost delta; +- approvers; and +- download links for each permitted revision. + +## 10. Review, Audit, and Access Workflow + +### 10.1 Project dashboard + +The project dashboard is the user's authorized view of a project. It shows: + +- source revision and source approval state; +- target locale matrix; +- translation progress and blocked scenes; +- review queue size and age; +- pronunciation coverage; +- voice/cast status and license warnings; +- audio build and QA status; +- deliverable versions and release state; +- current spend/estimate; +- active workers and resource contention; and +- recent audit events. + +A typical locale row is: + +| Locale | Source | Translation | Proofread | Pronunciation | Cast | Audio | QA | Release | +|---|---|---|---|---|---|---|---|---| +| `de-DE` | frozen | 82% | 14 queued | 98% | approved | 76% | pending | draft | +| `ja-JP` | frozen | 61% | 32 queued | 91% | pending | 0% | blocked | draft | + +### 10.2 Project picker + +The project picker lists only projects returned by the authorization service for the +authenticated user. Selecting a project sets a signed session/UI context and navigates to a +project-scoped URL. It never grants access and never changes the user's memberships. + +### 10.3 Review queues + +Review tasks are assigned by role, locale, risk, and workload. High-risk translation, +pronunciation, voice, and audio items are never silently auto-released. A reviewer sees the +source evidence, current revision, proposed fix, relevant scene context, and prior approval +history. + +### 10.4 Audit and downloads + +The system records: + +- login and authorization failures; +- project selection and switching; +- source/translation/voice/pronunciation edits; +- review decisions and approvals; +- artifact generation and quality reports; +- downloads and signed-link issuance; +- voice usage and billing changes; and +- release, supersession, withdrawal, and takedown. + +Audit records are append-only and retained according to organization policy and legal hold. + +## 11. Independent Voice and Pronunciation Library + +The voice library is an independent commercial and creative subsystem. It is not merely a +global `speakers.json` or a name-based row in the current voice registry. + +### 11.1 Voice identity versus provider asset + +Keep these concepts separate: + +- **Logical voice identity:** the reusable character/performance identity in UmbraTrans. +- **Acoustic voice asset:** a provider voice, recorded actor, cloned voice, or licensed + sample usable by a particular engine. +- **Voice profile:** the approved combination of identity, language, style, engine, and + pronunciation behavior. +- **Project binding:** the assignment of a logical speaker to a voice profile for one + project, locale, acoustic instance, and revision. + +One logical identity may use different provider assets in different languages. One provider +asset may be reused by many projects only when its license permits it. + +### 11.2 Voice catalog tables + +```text +voice_assets +voice_asset_locales +voice_profiles +voice_profile_revisions +voice_bindings +voice_reviews +voice_licenses +voice_entitlements +voice_reservations +voice_rate_cards +voice_usage_events +voice_invoices +voice_takedowns +``` + +Important fields include: + +- provider and provider voice ID; +- actor/voice name and legal entity; +- supported languages, scripts, locales, genders/presentations, and styles; +- engine/model compatibility; +- reference audio URI, checksum, transcript, and retention policy; +- license type, territories, media, exclusivity, term, concurrency, and derivative rights; +- rate, minimum commitment, currency, billing unit, and effective dates; +- current availability, reservation state, and takedown state; +- quality, pronunciation, identity, and listener-test results; and +- consent and evidence-document references. + +### 11.3 Commercial voice workflow + +```text +provider connection + → voice asset import + → rights verification + → capability/quality test + → profile approval + → project casting + → reservation/lease + → usage metering + → invoice/cost allocation + → renewal, expiry, or takedown +``` + +A project cannot bind a voice asset unless the entitlement is active for the project, +locale, territory, media, and release date. Expiry or takedown blocks new generation and +creates an issue for existing deliverables; it does not erase historical usage or audit +records. + +### 11.4 Usage and financial reporting + +Track usage by: + +- organization, project, work, locale, speaker, and deliverable; +- voice asset and provider; +- character count, audio minutes, jobs, retries, and rerenders; +- rate-card version and currency; +- reservation and overage charges; +- pronunciation-review labor or service cost; and +- release/revenue attribution where applicable. + +The dashboard can answer: + +- Which voices are used by this project? +- Which locales lack an approved voice? +- Which assets expire before the planned release? +- What will the next rerender cost? +- Which pronunciation fixes affect the most clips? +- Which provider delivered the best quality/cost ratio for this locale? + +### 11.5 Pronunciation catalog + +The global pronunciation catalog is separate from voice catalog metadata: + +```text +pronunciation_entries +pronunciation_locale_forms +pronunciation_reviews +pronunciation_sources +pronunciation_overrides +pronunciation_usage +pronunciation_audio +``` + +An entry contains: + +- stable term/entity ID; +- source spelling and locale; +- target reading, transliteration, IPA/phoneme, and stress/pitch information; +- language-specific context and part-of-speech/homograph rules; +- source and confidence; +- reviewer, approval state, and evidence; +- effective dates and supersession history; and +- rights/usage status for reference audio. + +Projects may override a global pronunciation entry, but the override is explicit, +versioned, reviewed, and linked to the exact target render spans and audio clips it +affects. A pronunciation change invalidates only the affected translation/audio scope, not +unrelated projects or languages. + +## 12. Jobs, Resources, and Concurrency + +The current in-process `GenerationManager` lock and JSON status files are not a distributed +scheduler. They can remain local worker safeguards, but enterprise execution needs a +durable queue and lease system. + +### 12.1 Job types + +Jobs are typed and resource-labeled: + +- `source_import`; +- `parse`; +- `semantic_analysis`; +- `identity_review`; +- `translation_draft`; +- `translation_verify`; +- `pronunciation_resolve`; +- `voice_cast`; +- `voice_sample_generate`; +- `audio_render`; +- `audio_qa`; +- `ebook_export`; +- `pdf_export`; +- `audio_assembly`; +- `release_package`; and +- `artifact_derivative`. + +### 12.2 Scheduler requirements + +Every job has: + +- organization/project/locale/revision scope; +- deterministic idempotency key; +- priority and deadline; +- required resource pool and capabilities; +- lease owner and heartbeat; +- retry and backoff policy; +- cancellation state; +- progress and structured logs; and +- output artifact and dependency hashes. + +A worker claims a job with a lease. If the worker dies, the lease expires and another +worker can resume safely. Jobs never use a filename alone as a global identity. + +### 12.3 Resource pools + +Resource pools include: + +- CPU parsing/OCR; +- LLM analysis/translation; +- target TTS GPU; +- pronunciation/QA CPU/GPU; +- audio mixing/FFmpeg; +- object storage/export workers; and +- optional provider API workers. + +Each pool has capacity, per-organization quotas, per-project concurrency, priorities, and +cost limits. A project can run multiple independent language lanes while expensive TTS +capacity is shared fairly. A model server that can serve one Mac is not treated as an +unlimited multi-user service. + +### 12.4 Optimistic concurrency + +Metadata edits use a version number or ETag. Two reviewers cannot silently overwrite each +other's changes. A conflict creates a review task or merge decision. Artifact uploads use +idempotency keys and cannot overwrite an approved revision. + +## 13. Service and API Boundaries + +The first implementation may be a modular monolith, but the logical boundaries remain: + +```text +Identity/Tenant Service +Project/Work Service +Source and Analysis Service +Translation Service +Review Service +Voice/Pronunciation Catalog Service +Artifact Service +Job Scheduler +Billing/Entitlement Service +Notification Service +Reporting/Search Index +``` + +### 13.1 Project API + +```text +GET /api/projects +POST /api/projects +GET /api/projects/{project_id} +GET /api/projects/{project_id}/dashboard +GET /api/projects/{project_id}/locales +POST /api/projects/{project_id}/members +GET /api/projects/{project_id}/audit +``` + +Every endpoint authorizes the subject against the organization and project. Project IDs are +opaque UUIDs/ULIDs; slugs are display and routing conveniences only. + +### 13.2 Artifact API + +```text +POST /api/projects/{project_id}/artifacts/upload-sessions +POST /api/artifacts/{artifact_id}/commit +GET /api/artifacts/{artifact_id} +POST /api/artifacts/{artifact_id}/download-url +GET /api/artifacts/{artifact_id}/lineage +POST /api/artifacts/{artifact_id}/supersede +``` + +Download URLs are short-lived, signed, audited, and scoped to the caller's permissions. + +### 13.3 Voice catalog API + +```text +GET /api/voice-assets +POST /api/voice-assets +GET /api/voice-assets/{voice_asset_id}/availability +GET /api/voice-assets/{voice_asset_id}/rights +POST /api/voice-assets/{voice_asset_id}/reserve +GET /api/voice-assets/{voice_asset_id}/usage +POST /api/voice-assets/{voice_asset_id}/takedown +``` + +### 13.4 Event and outbox contract + +Domain events are written transactionally to an outbox and delivered asynchronously: + +- `project.created`; +- `source_revision.frozen`; +- `identity.review_required`; +- `translation.revision.created`; +- `pronunciation.changed`; +- `voice.license.expiring`; +- `audio.rebuild_requested`; +- `artifact.revision.created`; +- `issue.opened`; +- `approval.recorded`; and +- `release.published`. + +Consumers must be idempotent. Events carry IDs and revisions, not mutable filenames. + +## 14. Web User Experience + +The enterprise UI is project-centric but not project-cookie-centric. + +### 14.1 Navigation + +- Organization/workspace switcher. +- Authorized project list with search, status, and activity. +- Project dashboard. +- Source and analysis review. +- Locale matrix. +- Translation and proofing queues. +- Pronunciation review. +- Voice casting and license view. +- Audio build and listening review. +- Deliverables and release packages. +- Issues and audit timeline. +- Voice library and usage/billing for authorized operators. + +### 14.2 Locale matrix + +A project dashboard shows every target locale in one view: + +- source revision; +- translation current/superseded revisions; +- proofing status; +- open issue count and age; +- pronunciation coverage; +- voice binding and license status; +- audio build progress and cost estimate; +- QA profile/result; +- deliverable versions; +- release state; and +- next responsible action. + +### 14.3 Fix navigation + +An issue links directly to: + +- the exact source span; +- the scene and context packet; +- the target block/render span; +- the voice and pronunciation plan; +- the audio clip and chapter; +- the prior and new artifact revisions; and +- the approvals and audit events. + +This is the primary audit/repair workflow for authors, editors, translators, audio staff, +and release managers. + +## 15. Deployment and Operations + +### 15.1 Recommended production services + +- FastAPI or equivalent control-plane API; +- PostgreSQL with managed backups and point-in-time recovery; +- S3-compatible encrypted object storage; +- Redis or a durable database-backed queue; +- CPU analysis workers; +- LLM workers; +- TTS/GPU workers; +- audio/export workers; +- search index for projects, terms, issues, and artifacts; +- notification/email service; and +- metrics, logs, traces, and alerting. + +### 15.2 Local development + +Local development may use: + +- PostgreSQL in a container or managed local instance; +- MinIO or a local object-store emulator; +- Redis or a database queue; +- a single worker process for functional tests; and +- filesystem-backed artifact storage behind the same ArtifactStore interface. + +The domain and API contracts must not depend on SQLite-specific behavior. + +### 15.3 Availability and recovery + +- Database backups and point-in-time recovery are tested. +- Object storage versioning and checksum validation are enabled. +- Jobs are resumable after worker loss. +- Release manifests survive service restarts. +- A project can be exported and re-imported without losing lineage. +- Deletion and retention policies are explicit. +- Voice license and pronunciation data are backed up independently of project files. + +## 16. Migration from Current UmbraAudio + +Migration is one-way and non-destructive. + +### Phase M0 — Inventory + +Collect: + +- every current project directory and source path; +- each project database and its schema/version; +- `speakers.json`, voice assignments, approved voice references, and markers; +- global voice registry and pronunciation/name library; +- generated audio and export files; +- pipeline/generation status and QA reports; and +- current source and output hashes. + +### Phase M1 — Import projects + +For each current project: + +1. create an organization or assign it to the appropriate organization; +2. create a `work`, `project`, and source asset; +3. copy the raw source into the enterprise artifact store; +4. create an immutable source revision and legacy source link; +5. import chapters, utterances, speaker hints, and attribution evidence; +6. mark all imported analysis as `legacy_advisory` until source review; +7. import generated English audio as legacy artifact revisions; and +8. record the exact UmbraAudio project path and database hash. + +No current UmbraAudio file is moved, renamed, or rewritten. + +### Phase M2 — Import voice/name library + +- Convert current registry speakers into logical voice profiles, not final commercial assets. +- Convert library names into pronunciation entries with `legacy_source` provenance. +- Preserve aliases, evidence, source project, and current confirmation state. +- Mark missing rights, locale, provider, and usage data as unknown. +- Require voice-manager review before a legacy voice can be used in a paid or public release. + +### Phase M3 — Validate lineage + +For a sample of projects and artifacts, prove: + +- source text can be found in the enterprise artifact store; +- speaker/utterance links resolve to stable IDs; +- audio artifacts link to the expected source/utterance revisions; +- checksums match; +- project membership can restrict access; and +- a regenerated translation/audio artifact can supersede the imported one. + +### Phase M4 — Dual operation + +Run the current UmbraAudio English path and the enterprise target-language workers in +parallel. The enterprise system records UmbraAudio output as an imported dependency and +does not write into its database. + +### Phase M5 — Cutover per project + +Only after validation, a project can be marked `enterprise_active`. Cutover is per project +and per locale, not a global irreversible switch. + +## 17. Implementation Roadmap + +### Phase E0 — Decisions and inventory + +- Choose PostgreSQL and object-store providers. +- Define organizations, roles, retention, privacy, and deployment assumptions. +- Inventory current projects and voice/name libraries. +- Select artifact ID, project ID, revision, and idempotency conventions. + +### Phase E1 — Control-plane foundation + +- PostgreSQL schema and migrations. +- Authentication and organization membership. +- Projects, project members, access grants, and RLS. +- Audit events and outbox events. +- Project picker and authorized project dashboard. + +### Phase E2 — Artifact and revision system + +- Object store adapter. +- Immutable source, translation, audio, and deliverable revisions. +- Lineage graph. +- Signed download URLs. +- Fix/supersede workflow. + +### Phase E3 — Review and issues + +- Locale matrix. +- Translation, source, pronunciation, voice, and audio review queues. +- Issue links to exact spans/clips/artifacts. +- Approval and audit timeline. + +### Phase E4 — Jobs and workers + +- Durable queue, leases, retries, cancellation, and resource pools. +- Per-project quotas and concurrency. +- Worker dashboards and cost estimates. + +### Phase E5 — Voice and pronunciation catalog + +- Voice assets, profiles, bindings, licenses, entitlements, rates, usage, and takedowns. +- Locale-aware pronunciation entries and review. +- Voice/pronunciation cost and expiry dashboards. + +### Phase E6 — Deliverables and releases + +- EPUB/PDF/text exporters. +- M4B/MP3/WAV package manifests. +- Release sign-off and immutable release packages. +- Per-locale delivery tracking. + +### Phase E7 — Migration and hardening + +- Import current projects and library. +- Cross-project reporting and search. +- Backup/restore, retention, legal hold, and disaster recovery. +- Security, concurrency, load, and cost testing. + +## 18. Acceptance Criteria + +The enterprise architecture is ready for multiple users when: + +1. Two organizations with users in only one organization cannot discover or access the + other's projects, artifacts, voices, or usage data. +2. A user with access to multiple projects can select among only those projects, and every + request remains authorized after switching. +3. Two users editing the same translation revision receive an optimistic-lock conflict + rather than silently overwriting each other. +4. Two projects can run independent jobs concurrently without sharing project status files, + database rows, artifact paths, or review decisions. +5. Shared TTS/LLM resources are queued and metered rather than protected only by an + in-process lock. +6. Every source, translation, render span, audio clip, deliverable, and release has a + complete lineage path and content hash. +7. A fix creates a new revision and preserves the prior approved revision and audit trail. +8. Every target locale has independent translation, pronunciation, voice, audio, QA, and + release status. +9. A voice or pronunciation change shows exactly which target spans and audio clips it + invalidates. +10. An expired or revoked voice entitlement blocks new generation and creates a visible + project issue. +11. Voice usage, rerenders, reservations, rates, and projected costs are visible by project + and locale. +12. A release package contains only authorized, approved, non-superseded revisions. +13. A full backup/restore preserves project access, lineage, audit, and release state. +14. The current UmbraAudio project tree remains unchanged after import and enterprise + operation. +15. The system passes concurrency, authorization, path-security, and provider-failure tests. + +## 19. Open Decisions + +1. **Deployment:** managed cloud, private cloud, or hybrid Mac/GPU workers. +2. **Identity:** preferred OIDC/SAML provider and service-account strategy. +3. **Tenancy:** one organization per author, one per studio, or explicit workspace nesting. +4. **Commercial model:** internal voices, licensed third-party voices, rented actors, or a + hybrid catalog. +5. **Billing:** internal cost allocation, customer invoicing, or both. +6. **Voice rights:** territories, media, exclusivity, term, derivative rights, and takedown + policy. +7. **Pronunciation ownership:** global catalog, per-customer overrides, or both. +8. **Artifact retention:** how long source, rejected revisions, audio, and audit data remain. +9. **Review sampling:** required qualified human review by locale and risk class. +10. **Search:** PostgreSQL full-text search initially or a separate search service later. + +## 20. Summary + +The current UmbraAudio repository is a capable local English audiobook engine with partial +multi-project support, a shared basic voice registry, and a browser project selector. It is +not yet a multi-user or enterprise system. + +UmbraTrans needs a separate enterprise control plane with: + +- organizations and authenticated users; +- authorized project membership and project selection; +- one logical relational schema, not tables per project or language; +- immutable source, translation, pronunciation, voice, audio, and deliverable revisions; +- object storage and complete lineage; +- durable jobs and resource quotas; +- review, issue, approval, and audit workflows; +- a licensed, metered, reusable voice library; and +- a global, locale-aware pronunciation library. + +This structure lets one source ebook become multiple independently tracked ebooks, PDFs, +and audiobooks in many languages while keeping the English source, people, permissions, +commercial voice rights, costs, and every fix under one auditable control plane. diff --git a/umbraaudio/analysis/character_discovery.py b/umbraaudio/analysis/character_discovery.py index b2ff1a8..8186de7 100644 --- a/umbraaudio/analysis/character_discovery.py +++ b/umbraaudio/analysis/character_discovery.py @@ -14,6 +14,8 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +from umbraaudio.voices.lmstudio_gate import queued_chat_completion + PROMPT_PATH = Path(__file__).parent / "CHARACTER_DISCOVERY_PROMPT.md" SCENE_BREAK = "<<<>>>" @@ -295,7 +297,8 @@ class CharacterDiscoverer: response = None for attempt in range(max_attempts): try: - response = client.chat.completions.create( + response = queued_chat_completion( + client, model=model, messages=[{"role": "user", "content": prompt}], temperature=0.2, @@ -997,7 +1000,8 @@ Return ONLY valid JSON.""" model = self._active_model or self.config.openai_model try: - response = client.chat.completions.create( + response = queued_chat_completion( + client, model=model, messages=[{"role": "user", "content": prompt}], temperature=0.1, diff --git a/umbraaudio/analysis/character_profiler.py b/umbraaudio/analysis/character_profiler.py index 9c68e16..8bd383b 100644 --- a/umbraaudio/analysis/character_profiler.py +++ b/umbraaudio/analysis/character_profiler.py @@ -10,6 +10,7 @@ from umbraaudio.analysis import ( CharacterProfile, ) from umbraaudio.config import LLMConfig +from umbraaudio.voices.lmstudio_gate import queued_chat_completion logger = logging.getLogger(__name__) @@ -140,7 +141,8 @@ class CharacterProfiler: f"dimensions while staying consistent with the text evidence." ) - response = client.chat.completions.create( + response = queued_chat_completion( + client, model=self.config.openai_model, messages=[{"role": "user", "content": prompt}], temperature=self.config.temperature, @@ -234,7 +236,8 @@ class CharacterProfiler: prompt=prompt, ) - response = client.chat.completions.create( + response = queued_chat_completion( + client, model=self.config.openai_model, messages=[{"role": "user", "content": llm_prompt}], temperature=self.config.temperature, @@ -285,7 +288,8 @@ Example: {{"Eilean": "eɪ.lən", "Fjord": "fjɔːrd"}} Return ONLY valid JSON.""" - response = client.chat.completions.create( + response = queued_chat_completion( + client, model=self.config.openai_model, messages=[{"role": "user", "content": prompt}], temperature=0.1, diff --git a/umbraaudio/analysis/llm_attributor.py b/umbraaudio/analysis/llm_attributor.py index bc4441e..4c47206 100644 --- a/umbraaudio/analysis/llm_attributor.py +++ b/umbraaudio/analysis/llm_attributor.py @@ -10,6 +10,7 @@ from typing import Any from openai import OpenAI +from umbraaudio.voices.lmstudio_gate import queued_chat_completion from umbraaudio.analysis.attribution_qa import ( SPEECH_VERBS as _QA_SPEECH_VERBS, ) @@ -1097,7 +1098,8 @@ class LLMAttributor: max_attempts = 3 for attempt in range(max_attempts): try: - response = client.chat.completions.create( + response = queued_chat_completion( + client, model=self.model, messages=messages, temperature=0.1, diff --git a/umbraaudio/analysis/llm_segmenter.py b/umbraaudio/analysis/llm_segmenter.py index f117714..547bd27 100644 --- a/umbraaudio/analysis/llm_segmenter.py +++ b/umbraaudio/analysis/llm_segmenter.py @@ -10,6 +10,7 @@ from typing import Any from openai import OpenAI from umbraaudio.config import LLMConfig +from umbraaudio.voices.lmstudio_gate import queued_chat_completion logger = logging.getLogger(__name__) @@ -136,7 +137,8 @@ class LLMSegmenter: client = self._get_client() try: - response = client.chat.completions.create( + response = queued_chat_completion( + client, model=self.model, messages=[{"role": "user", "content": prompt}], temperature=0.2, diff --git a/umbraaudio/config.py b/umbraaudio/config.py index 2b2933c..9bc2f2a 100644 --- a/umbraaudio/config.py +++ b/umbraaudio/config.py @@ -58,6 +58,9 @@ class TTSConfig(BaseSettings): voicedesign_model: str = "/Volumes/1TB_APL_Storage/UmbraAudio/models/Qwen3-TTS-12Hz-1.7B-VoiceDesign" base_model: str = "/Volumes/1TB_APL_Storage/UmbraAudio/models/Qwen3-TTS-12Hz-1.7B-Base" qwen_tts_url: str = "http://localhost:8881" + # Optional GPU accelerator (e.g. http://127.0.0.1:18881 relay to the + # 192.168.123.14 box). Tried first when healthy; local always falls back. + qwen_tts_helper_url: str = "" class QualityConfig(BaseSettings): diff --git a/umbraaudio/generation/chapter_builder.py b/umbraaudio/generation/chapter_builder.py index 7e908c9..1a82cfe 100644 --- a/umbraaudio/generation/chapter_builder.py +++ b/umbraaudio/generation/chapter_builder.py @@ -475,7 +475,7 @@ class ChapterBuilder: speaker_traits = load_speaker_traits(self.data_dir / "speakers.json") distortion_effects = load_speaker_distortion_effects(self.data_dir / "speakers.json") - engine = QwenTTSEngine(qwen_tts_url=self.config.tts.qwen_tts_url) + engine = QwenTTSEngine(qwen_tts_url=self.config.tts.qwen_tts_url, helper_url=self.config.tts.qwen_tts_helper_url) # Capture the full vmatrix (the "8k matrix" ref_code + speaker embedding) # once per distinct speaker, cached server-side as .pt under PROMPT_DIR. diff --git a/umbraaudio/mapping/emotion.py b/umbraaudio/mapping/emotion.py index 07cc82f..daeb834 100644 --- a/umbraaudio/mapping/emotion.py +++ b/umbraaudio/mapping/emotion.py @@ -4,6 +4,8 @@ import json import logging from typing import Any +from umbraaudio.voices.lmstudio_gate import queued_chat_completion + logger = logging.getLogger(__name__) EMOTION_PROMPT = """You are an audiobook director analyzing dialogue and narration for TTS voice direction. @@ -85,7 +87,8 @@ class EmotionMapper: max_attempts = 3 for attempt in range(max_attempts): try: - response = self.llm.chat.completions.create( + response = queued_chat_completion( + self.llm, model=self.model, messages=[{"role": "user", "content": prompt}], temperature=0.2, diff --git a/umbraaudio/pronunciation.py b/umbraaudio/pronunciation.py index 2d63bd4..8e1bab0 100644 --- a/umbraaudio/pronunciation.py +++ b/umbraaudio/pronunciation.py @@ -808,7 +808,7 @@ def generate_pronunciation_audio(config: Any, name: str, spelling: str) -> dict[ try: from umbraaudio.voices.qwen_tts import VOICE_REFERENCE_TEXT, QwenTTSEngine - engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url) + engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url, helper_url=config.tts.qwen_tts_helper_url) if ref: engine.generate( text=syn_text, output_path=out, voice_ref=ref, diff --git a/umbraaudio/voices/lmstudio_gate.py b/umbraaudio/voices/lmstudio_gate.py new file mode 100644 index 0000000..8f8ca9c --- /dev/null +++ b/umbraaudio/voices/lmstudio_gate.py @@ -0,0 +1,186 @@ +"""Queueing gate for LM Studio LLM requests. + +LM Studio on this Mac can only serve one LLM at a time and the model +coordinator swaps between the Qwen TTS models and the LLM. LLM calls that +arrive while LM Studio is busy -- loading a model, swapping models, or +warming up -- previously failed with connection / HTTP errors. This module +serializes requests and waits until the server is available again, then lets +the queued request through (the exact behavior requested: *if the local LM +Studio is busy the request is queued and waits for availability, then +processed*). + +Only requests aimed at a local LM Studio (:1234) are gated; OpenAI provider +clients pass straight through untouched. +""" + +from __future__ import annotations + +import json +import logging +import threading +import time +from collections.abc import Callable +from contextlib import contextmanager +from typing import Any +from urllib.error import URLError +from urllib.request import Request, urlopen + +logger = logging.getLogger(__name__) + +DEFAULT_POLL_INTERVAL = 1.0 +DEFAULT_WAIT_TIMEOUT = 900.0 + +_ML_LOCAL_HOSTS = {"localhost", "127.0.0.1", "0.0.0.0", "::1"} +_PLACEHOLDER_MODELS = {"", "local-model"} + + +class LMStudioBusyError(RuntimeError): + """Raised when LM Studio did not become available within the wait window.""" + + +def _host_of(base_url: str) -> str: + """Return the hostname of a base URL (best effort, no network).""" + base = base_url.split("://", 1)[-1] + base = base.split("/", 1)[0] + host = base + if ":" in base and not base.startswith("["): + host = base.rsplit(":", 1)[0] + return (host or "").lower().strip("[]") + + +def _is_local_lmstudio(base_url: str) -> bool: + """Gate only requests aimed at the local LM Studio server.""" + host = _host_of(base_url) + if host in _ML_LOCAL_HOSTS: + return True + # LM Studio is commonly on port 1234, even when reached via a custom host. + return ":1234" in base_url + + +def _server_probe(base_url: str, model: str | None = None) -> bool: + """True when the server answers /models and (if given) knows the model.""" + base = (base_url or "").rstrip("/") + if not base: + return True + try: + req = Request( + f"{base}/models", + headers={"Accept": "application/json"}, + ) + with urlopen(req, timeout=3) as resp: + if resp.status != 200: + return False + payload = json.loads(resp.read().decode("utf-8")) + except (URLError, OSError, ValueError): + return False + + ids = [m.get("id", "") for m in payload.get("data", [])] + if model in _PLACEHOLDER_MODELS: + return True + + # LM Studio exposes loaded instances under the native /api/v1/models + # endpoint; prefer it when the OpenAI-compatible listing is ambiguous. + api = base + for suffix in ("/v1",): + if api.endswith(suffix): + api = api[: -len(suffix)] + try: + req2 = Request(f"{api}/api/v1/models", headers={"Accept": "application/json"}) + with urlopen(req2, timeout=3) as resp2: + if resp2.status == 200: + native = json.loads(resp2.read().decode("utf-8")) + for m in native.get("data", []): + if m.get("key") == model and m.get("loaded_instances"): + return True + except (URLError, OSError, ValueError): + pass + + return model in ids or not ids + + +class LMStudioGate: + """Serialize + queue LLM requests until LM Studio is available. + + Thread-safe. At most one request is processed at a time; everyone else + waits (FIFO-ish via the condition variable). Before a request runs, the + gate polls ``available_fn`` until the server reports healthy. + """ + + def __init__( + self, + *, + poll_interval: float = DEFAULT_POLL_INTERVAL, + wait_timeout: float = DEFAULT_WAIT_TIMEOUT, + available_fn: Callable[[str, str | None], bool] = _server_probe, + ) -> None: + self.poll_interval = poll_interval + self.wait_timeout = wait_timeout + self._available_fn = available_fn + self._cond = threading.Condition() + self._inflight = 0 + + def wait_until_available( + self, + base_url: str, + model: str | None = None, + timeout: float | None = None, + ) -> None: + """Block until LM Studio answers, raising :class:`LMStudioBusyError`.""" + wait_for = timeout if timeout is not None else self.wait_timeout + deadline = time.monotonic() + wait_for + while True: + if self._available_fn(base_url, model): + return + if time.monotonic() >= deadline: + raise LMStudioBusyError( + f"LM Studio not available after {wait_for:.0f}s wait (base={base_url})" + ) + time.sleep(self.poll_interval) + + @contextmanager + def acquire( + self, + base_url: str, + model: str | None = None, + timeout: float | None = None, + ) -> Any: + """Queue the caller behind any in-flight request, then run when ready.""" + with self._cond: + while self._inflight > 0: + self._cond.wait() + self._inflight += 1 + try: + self.wait_until_available(base_url, model, timeout) + yield + finally: + with self._cond: + self._inflight -= 1 + self._cond.notify() + + +_GATE = LMStudioGate() + + +def _client_base(client: Any) -> str: + try: + return str(client.base_url) + except Exception: + return "" + + +def queued_chat_completion( + client: Any, + model: str, + messages: list[dict[str, str]], + **kwargs: Any, +) -> Any: + """Run ``client.chat.completions.create`` behind the LM Studio gate. + + Requests to a local LM Studio are serialized and wait for server + availability before processing; non-LM Studio providers are unchanged. + """ + base_url = _client_base(client) + if _is_local_lmstudio(base_url): + with _GATE.acquire(base_url, model): + return client.chat.completions.create(model=model, messages=messages, **kwargs) + return client.chat.completions.create(model=model, messages=messages, **kwargs) \ No newline at end of file diff --git a/umbraaudio/voices/qwen_tts.py b/umbraaudio/voices/qwen_tts.py index d21b87e..17b66f9 100644 --- a/umbraaudio/voices/qwen_tts.py +++ b/umbraaudio/voices/qwen_tts.py @@ -51,67 +51,99 @@ class QwenTTSEngine: voicedesign_model: str | None = None, base_model: str | None = None, qwen_tts_url: str | None = None, + helper_url: str | None = None, ) -> None: self.device = device self.sample_rate = sample_rate self._voicedesign_path = voicedesign_model or DEFAULT_VOICEDESIGN_MODEL self._base_path = base_model or DEFAULT_BASE_MODEL self._qwen_tts_url = qwen_tts_url + self._helper_url = helper_url self._vd_model = None self._base_model = None + @property + def _candidate_urls(self) -> list[str]: + """Ordered TTS endpoints: GPU helper first (when configured), local required last.""" + urls: list[str] = [] + if self._helper_url: + urls.append(self._helper_url.rstrip("/")) + if self._qwen_tts_url: + local = self._qwen_tts_url.rstrip("/") + if local not in urls: + urls.append(local) + return urls + @property def _use_remote(self) -> bool: - return bool(self._qwen_tts_url) + return bool(self._candidate_urls) + + def _url_healthy(self, url: str) -> bool: + import httpx + + try: + r = httpx.get(f"{url}/health", timeout=5) + return r.status_code < 500 + except Exception: + return False + + def _preferred_candidates(self) -> list[str]: + """Healthy endpoints first, unhealthy ones after (never dropped). + + With a single endpoint (local-only setup) no health probe is made: + the request itself determines success, so we never add a second + round-trip to every TTS call. + """ + urls = self._candidate_urls + if len(urls) <= 1: + return urls + healthy = [u for u in urls if self._url_healthy(u)] + return healthy + [u for u in urls if u not in healthy] def check_health(self) -> bool: if not self._use_remote: return False - import httpx - - try: - r = httpx.get(f"{self._qwen_tts_url}/health", timeout=5) - return r.status_code < 500 - except Exception: - return False + return any(self._url_healthy(url) for url in self._candidate_urls) def load_remote_model(self, model_type: str = "voicedesign") -> None: if not self._use_remote: return import httpx - try: - r = httpx.post( - f"{self._qwen_tts_url}/load", - json={"model_type": model_type}, - timeout=120, - ) - logger.info( - "Qwen TTS remote model load: HTTP %d %s", r.status_code, r.text.strip()[:200] - ) - import time + for url in self._candidate_urls: + try: + r = httpx.post( + f"{url}/load", + json={"model_type": model_type}, + timeout=120, + ) + logger.info( + "Qwen TTS remote model load: HTTP %d %s", r.status_code, r.text.strip()[:200] + ) + import time - if "already" in r.text.lower(): - time.sleep(1) - else: - time.sleep(10) - except Exception as e: - logger.warning("Qwen TTS remote model load failed: %s", e) + if "already" in r.text.lower(): + time.sleep(1) + else: + time.sleep(10) + except Exception as e: + logger.warning("Qwen TTS remote model load failed (%s): %s", url, e) def unload_remote_model(self) -> None: if not self._use_remote: return import httpx - try: - r = httpx.post( - f"{self._qwen_tts_url}/unload", - json={"model_type": "all"}, - timeout=30, - ) - logger.info("Qwen TTS remote model unload: HTTP %d", r.status_code) - except Exception as e: - logger.warning("Qwen TTS remote model unload failed: %s", e) + for url in self._candidate_urls: + try: + r = httpx.post( + f"{url}/unload", + json={"model_type": "all"}, + timeout=30, + ) + logger.info("Qwen TTS remote model unload (%s): HTTP %d", url, r.status_code) + except Exception as e: + logger.warning("Qwen TTS remote model unload failed (%s): %s", url, e) def unload_and_reload(self) -> None: self.unload_remote_model() @@ -271,18 +303,26 @@ class QwenTTSEngine: data["force"] = 1 if ref_text: data["ref_text"] = ref_text - with open(voice_ref, "rb") as f: - r = httpx.post( - f"{self._qwen_tts_url}/v1/voice-prompt", - data=data, - files={"ref_audio": ("ref.wav", f.read(), "audio/wav")}, - timeout=REMOTE_TIMEOUT, - ) - if r.status_code != 200: - raise QwenTTSUnavailableError( - f"Voice-prompt build failed HTTP {r.status_code}: {r.text[:300]}" - ) - return slug + + last_err: str | None = None + ref_bytes = Path(voice_ref).read_bytes() + for url in self._preferred_candidates(): + try: + r = httpx.post( + f"{url}/v1/voice-prompt", + data=data, + files={"ref_audio": ("ref.wav", ref_bytes, "audio/wav")}, + timeout=REMOTE_TIMEOUT, + ) + if r.status_code != 200: + last_err = f"Voice-prompt build failed HTTP {r.status_code}: {r.text[:300]}" + logger.warning("%s (url=%s)", last_err, url) + continue + return slug + except Exception as e: + last_err = f"Voice-prompt build failed: {e}" + logger.warning("%s (url=%s)", last_err, url) + raise QwenTTSUnavailableError(last_err or "No Qwen TTS server available") def generate_voice_reference( self, @@ -315,7 +355,48 @@ class QwenTTSEngine: ) -> tuple[np.ndarray, int]: import httpx - url = f"{self._qwen_tts_url}/v1/tts" + files = None + + if prompt_slug: + pass + elif voice_ref and Path(voice_ref).exists(): + files = {"ref_audio": ("ref.wav", Path(voice_ref).read_bytes(), "audio/wav")} + + last_err: str | None = None + for url in self._preferred_candidates(): + try: + return self._remote_generate_at( + url=url, + text=text, + files=files, + instruct=instruct, + ref_text=ref_text, + prompt_slug=prompt_slug, + temperature=temperature, + do_sample=do_sample, + seed=seed, + ) + except QwenTTSUnavailableError as e: + last_err = str(e) + logger.warning("Qwen TTS endpoint %s failed, trying next: %s", url, e) + raise QwenTTSUnavailableError(last_err or "No Qwen TTS server available") + + def _remote_generate_at( + self, + *, + url: str, + text: str, + files: dict[str, Any] | None, + instruct: str | None, + ref_text: str | None, + prompt_slug: str | None, + temperature: float, + do_sample: bool, + seed: int | None, + ) -> tuple[np.ndarray, int]: + import httpx + + generate_url = f"{url}/v1/tts" payload: dict[str, Any] = { "text": text, "sample_rate": self.sample_rate, @@ -326,12 +407,9 @@ class QwenTTSEngine: payload["seed"] = seed if ref_text: payload["ref_text"] = ref_text - files = None if prompt_slug: payload["prompt_slug"] = prompt_slug - elif voice_ref and Path(voice_ref).exists(): - files = {"ref_audio": ("ref.wav", Path(voice_ref).read_bytes(), "audio/wav")} # instruct composes with voice clone / ICL: the server prepends it as # style direction (emotion/pace/delivery) while keeping the speaker. @@ -339,15 +417,16 @@ class QwenTTSEngine: payload["voice_description"] = instruct try: - r = httpx.post(url, data=payload, files=files or {}, timeout=REMOTE_TIMEOUT) + r = httpx.post( + generate_url, data=payload, files=files or {}, timeout=REMOTE_TIMEOUT + ) except httpx.ConnectError as e: raise QwenTTSUnavailableError( - f"Cannot connect to Qwen TTS server at {self._qwen_tts_url}. " - "Make sure the server is running." + f"Cannot connect to Qwen TTS server at {url}. Make sure the server is running." ) from e except httpx.TimeoutException as e: raise QwenTTSUnavailableError( - f"Qwen TTS server at {self._qwen_tts_url} timed out after {REMOTE_TIMEOUT}s." + f"Qwen TTS server at {url} timed out after {REMOTE_TIMEOUT}s." ) from e if r.status_code != 200: @@ -456,23 +535,34 @@ class QwenTTSEngine: class QwenTTSClient: - def __init__(self, base_url: str, timeout: int = REMOTE_TIMEOUT) -> None: + def __init__( + self, + base_url: str, + timeout: int = REMOTE_TIMEOUT, + helper_url: str | None = None, + ) -> None: self.base_url = base_url.rstrip("/") + self.helper_url = (helper_url or "").rstrip("/") or None self.timeout = timeout def check_health(self) -> bool: import httpx - try: - r = httpx.get(f"{self.base_url}/health", timeout=5) - return r.status_code < 500 - except Exception: - return False + for url in (self.helper_url, self.base_url): + if not url: + continue + try: + r = httpx.get(f"{url}/health", timeout=5) + if r.status_code < 500: + return True + except Exception: + continue + return False def generate_voice_reference( self, text: str, voice_description: str, output_path: Path ) -> Path: - engine = QwenTTSEngine(qwen_tts_url=self.base_url) + engine = QwenTTSEngine(qwen_tts_url=self.base_url, helper_url=self.helper_url) return engine.generate_voice_reference(text, output_path, voice_description) def generate( @@ -485,7 +575,7 @@ class QwenTTSClient: do_sample: bool = True, seed: int | None = None, ) -> Path: - engine = QwenTTSEngine(qwen_tts_url=self.base_url) + engine = QwenTTSEngine(qwen_tts_url=self.base_url, helper_url=self.helper_url) return engine.generate( text, output_path, voice_ref=voice_ref, instruct=instruct, temperature=temperature, do_sample=do_sample, seed=seed, diff --git a/umbraaudio/web/routes.py b/umbraaudio/web/routes.py index 952416a..b7ee405 100644 --- a/umbraaudio/web/routes.py +++ b/umbraaudio/web/routes.py @@ -1940,7 +1940,7 @@ async def speakers_list_partial(request: Request): return False from umbraaudio.voices.qwen_tts import QwenTTSEngine - engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url) + engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url, helper_url=config.tts.qwen_tts_helper_url) try: import httpx @@ -3325,7 +3325,7 @@ async def generate_utterance_audio(request: Request, utt_id: int): voice_ref = vf break - engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url) + engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url, helper_url=config.tts.qwen_tts_helper_url) tts_instruct = compose_utterance_instruct(instruct, emotion, pace) @@ -3444,7 +3444,7 @@ async def delete_utterance(request: Request, utt_id: int): if "kainda" in vf.stem.lower(): narrator_ref = vf break - engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url) + engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url, helper_url=config.tts.qwen_tts_helper_url) for u in remaining: out = audio_dir / f"utt_{u['utterance_index']:04d}.wav" if is_scene_break(u["text"], u["type"]): @@ -3633,7 +3633,7 @@ async def create_utterance(request: Request, chapter_id: int): narrator_ref = vf break try: - engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url) + engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url, helper_url=config.tts.qwen_tts_helper_url) if narrator_ref: engine.generate( text=spoken_text, @@ -3798,7 +3798,7 @@ async def set_voice_status(request: Request): QwenTTSEngine, ) - engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url) + engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url, helper_url=config.tts.qwen_tts_helper_url) slug = engine.build_voice_prompt( ref_path, ref_text=VOICE_REFERENCE_TEXT,