Add enterprise/infra tooling and repoint Qwen3-TTS submodule to internal fork
Two groups of changes, committed together as the follow-up to the initial push: 1. .gitmodules — submodule Qwen3-TTS now points at the internal Gitea fork (ssh://git@mini-eq:2222/umbragit/Qwen3-TTS.git) instead of upstream GitHub, so a clone actually gets the UmbraAudio branding/integration commit 012e452 rather than vanilla upstream. 2. New infra/ tooling for the cross-machine work in spec 21: agent_bridge, remote_credential_broker (Keychain-backed, secrets read at runtime and generated via read_secret/random_secret — no hardcoded credentials), the enterprise bootstrap scripts, and the oldbox SSH/auth prompts. Plus backup/ restore scripts, two specs (18 UmbraTrans, 19 enterprise control plane), umbraaudio/voices/lmstudio_gate.py, and in-flight edits to the discovery/attribution/profiling modules, config, chapter_builder, emotion, pronunciation, qwen_tts and routes. Also adds *.db to .gitignore: SQLite project databases are runtime state and no .db file was tracked before. Verified before staging: .env's only change is UMBRA_TTS__QWEN_TTS_HELPER_URL and the committed copy has zero secret-looking values; the broker ships only config.example.json.
This commit is contained in:
B.D. Murphy
3
.env
3
.env
@@ -14,6 +14,9 @@ UMBRA_TTS__RETRY_DELAY=2.0
|
||||
UMBRA_TTS__VOICEDESIGN_MODEL=/Volumes/1TB_APL_Storage/UmbraAudio/models/Qwen3-TTS-12Hz-1.7B-VoiceDesign
|
||||
UMBRA_TTS__BASE_MODEL=/Volumes/1TB_APL_Storage/UmbraAudio/models/Qwen3-TTS-12Hz-1.7B-Base
|
||||
UMBRA_TTS__QWEN_TTS_URL=http://localhost:8881
|
||||
# Optional GPU accelerator: relay 127.0.0.1:18881 -> 192.168.123.14:8881.
|
||||
# Helper is tried first when healthy; local server above is the required fallback.
|
||||
UMBRA_TTS__QWEN_TTS_HELPER_URL=http://127.0.0.1:18881
|
||||
|
||||
# --- LLM ---
|
||||
UMBRA_LLM__PROVIDER=auto
|
||||
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
@@ -47,3 +47,6 @@ umbraaudio_package.zip
|
||||
|
||||
# One-off voice A/B demo (large binary WAVs, not wired into the app)
|
||||
/umbraaudio/web/static/samples/
|
||||
|
||||
# SQLite project databases are runtime state, never source
|
||||
*.db
|
||||
|
||||
2
.gitmodules
vendored
2
.gitmodules
vendored
@@ -1,3 +1,3 @@
|
||||
[submodule "Qwen3-TTS"]
|
||||
path = Qwen3-TTS
|
||||
url = https://github.com/QwenLM/Qwen3-TTS.git
|
||||
url = ssh://git@mini-eq:2222/umbragit/Qwen3-TTS.git
|
||||
|
||||
10
Issues.txt
10
Issues.txt
@@ -1,3 +1,13 @@
|
||||
Issues
|
||||
|
||||
Split of book - identify chapters, paragraphs and utterances. Split utterances by ".?!" the only three valid end of sentence punctuation.
|
||||
|
||||
|
||||
connect the two. Get audio data on the new machine in the right format.
|
||||
|
||||
|
||||
Idea:
|
||||
Android app to pull pictures from phone and from cloud and send to NAS, formatted into groups. Allows removal from phone and cloud to free up space.
|
||||
- App becomes primary photo backup (not giving your stuff to google) - the place to find your stuff.
|
||||
- Search by location taken, date, contents of picture, identify who
|
||||
- Have system setup to display the pictures on the nas via web on in the app.
|
||||
|
||||
290
backup/Starthere.sh
Executable file
290
backup/Starthere.sh
Executable file
@@ -0,0 +1,290 @@
|
||||
#!/usr/bin/env bash
|
||||
# Restore an UmbraAudio full backup and rebuild a portable runtime.
|
||||
set -Eeuo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
ARCHIVE=""
|
||||
TARGET="${UMBRA_RESTORE_TARGET:-$HOME/UmbraAudio-restore}"
|
||||
GLOBAL_TARGET="${UMBRA_GLOBAL_STATE_DIR:-$HOME/.umbra}"
|
||||
PYTHON_OVERRIDE="${PYTHON_BIN:-}"
|
||||
SKIP_DEPS=0
|
||||
SKIP_SUBMODULE_UPDATE=0
|
||||
SKIP_GLOBAL_STATE=0
|
||||
INSTALL_SYSTEM_DEPS=0
|
||||
DOWNLOAD_MODELS=0
|
||||
WITH_TTS_SERVER=0
|
||||
KEEP_ARCHIVED_ENV=0
|
||||
RUN_TESTS=0
|
||||
FORCE=0
|
||||
APPLY_PORTABILITY_PATCH=1
|
||||
OLD_ROOT=""
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: Starthere.sh [options]
|
||||
|
||||
Restore the newest UmbraAudio backup in this directory (or use --archive),
|
||||
verify its SHA-256 manifest, restore the project and global voice state, and
|
||||
rebuild a portable Python environment.
|
||||
|
||||
Options:
|
||||
--target DIR Restore destination (default: $HOME/UmbraAudio-restore)
|
||||
--global-state-dir DIR Restored global registry/library (default: $HOME/.umbra)
|
||||
--archive FILE Payload archive to restore
|
||||
--python BIN Explicit Python 3.11+ interpreter
|
||||
--skip-deps Do not rebuild Python dependencies
|
||||
--skip-submodule Do not initialize/update git submodules
|
||||
--skip-global-state Do not restore ~/.umbra voice registry/library
|
||||
--install-system-deps Try to install git/ffmpeg/Python with brew or apt-get
|
||||
--download-models Download missing Qwen model weights
|
||||
--with-tts-server Also install tts_server/requirements.txt
|
||||
--keep-archived-env Keep archived .venv/.python instead of rebuilding
|
||||
--no-portability-patch Do not rewrite machine-local paths in restored copy
|
||||
--run-tests Run pytest after setup
|
||||
--force Permit non-empty project/global targets
|
||||
-h, --help Show this help
|
||||
|
||||
The archive may contain .env, manuscripts, models, and audio. Review it before
|
||||
sharing the restored directory. Network database configuration is intentionally
|
||||
not changed by this script.
|
||||
EOF
|
||||
}
|
||||
|
||||
while (($#)); do
|
||||
case "$1" in
|
||||
--target) TARGET="${2:?--target requires a directory}"; shift 2 ;;
|
||||
--global-state-dir) GLOBAL_TARGET="${2:?--global-state-dir requires a directory}"; shift 2 ;;
|
||||
--archive) ARCHIVE="${2:?--archive requires a file}"; shift 2 ;;
|
||||
--python) PYTHON_OVERRIDE="${2:?--python requires an interpreter}"; shift 2 ;;
|
||||
--skip-deps) SKIP_DEPS=1; shift ;;
|
||||
--skip-submodule) SKIP_SUBMODULE_UPDATE=1; shift ;;
|
||||
--skip-global-state) SKIP_GLOBAL_STATE=1; shift ;;
|
||||
--install-system-deps) INSTALL_SYSTEM_DEPS=1; shift ;;
|
||||
--download-models) DOWNLOAD_MODELS=1; shift ;;
|
||||
--with-tts-server) WITH_TTS_SERVER=1; shift ;;
|
||||
--keep-archived-env) KEEP_ARCHIVED_ENV=1; shift ;;
|
||||
--no-portability-patch) APPLY_PORTABILITY_PATCH=0; shift ;;
|
||||
--run-tests) RUN_TESTS=1; shift ;;
|
||||
--force) FORCE=1; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) printf 'Unknown option: %s\n' "$1" >&2; usage >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "$ARCHIVE" ]]; then
|
||||
ARCHIVE="$(find "$SCRIPT_DIR" -maxdepth 1 -type f -name 'UmbraAudio-*.tar' -print | sort | tail -n 1)"
|
||||
fi
|
||||
[[ -n "$ARCHIVE" && -f "$ARCHIVE" && ! -L "$ARCHIVE" ]] || {
|
||||
printf 'No regular payload archive found. Use --archive or place UmbraAudio-*.tar beside this script.\n' >&2
|
||||
exit 1
|
||||
}
|
||||
[[ -f "$SCRIPT_DIR/SHA256SUMS" ]] || {
|
||||
printf 'Missing SHA256SUMS beside Starthere.sh; refusing an unverified restore.\n' >&2
|
||||
exit 1
|
||||
}
|
||||
[[ -f "$SCRIPT_DIR/restore_support.py" ]] || {
|
||||
printf 'Missing restore_support.py beside Starthere.sh; use a backup made with the updated tooling.\n' >&2
|
||||
exit 1
|
||||
}
|
||||
ARCHIVE_DIR="$(cd -- "$(dirname -- "$ARCHIVE")" && pwd -P)"
|
||||
[[ "$ARCHIVE_DIR" == "$SCRIPT_DIR" ]] || {
|
||||
printf 'Archive must be located beside the selected backup metadata: %s\n' "$ARCHIVE_DIR" >&2
|
||||
exit 1
|
||||
}
|
||||
ARCHIVE_BASENAME="$(basename -- "$ARCHIVE")"
|
||||
if ! awk -v name="$ARCHIVE_BASENAME" '$2 == name { found = 1 } END { exit(found ? 0 : 1) }' "$SCRIPT_DIR/SHA256SUMS"; then
|
||||
printf 'Selected archive is not listed in SHA256SUMS: %s\n' "$ARCHIVE_BASENAME" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -f "$SCRIPT_DIR/BACKUP-INFO.txt" ]]; then
|
||||
info_archive="$(sed -n 's/^archive_name=//p' "$SCRIPT_DIR/BACKUP-INFO.txt" | head -n 1)"
|
||||
if [[ -n "$info_archive" && "$info_archive" != "$ARCHIVE_BASENAME" ]]; then
|
||||
printf 'Selected archive does not match BACKUP-INFO.txt: %s vs %s\n' "$ARCHIVE_BASENAME" "$info_archive" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
for tool in tar; do
|
||||
command -v "$tool" >/dev/null 2>&1 || { printf 'Required tool is missing: %s\n' "$tool" >&2; exit 1; }
|
||||
done
|
||||
|
||||
printf 'Backup directory: %s\n' "$SCRIPT_DIR"
|
||||
printf 'Payload archive: %s\n' "$ARCHIVE"
|
||||
printf 'Restore target: %s\n' "$TARGET"
|
||||
printf 'Global state: %s\n' "$GLOBAL_TARGET"
|
||||
|
||||
if command -v shasum >/dev/null 2>&1; then
|
||||
(cd "$SCRIPT_DIR" && shasum -a 256 -c SHA256SUMS)
|
||||
elif command -v sha256sum >/dev/null 2>&1; then
|
||||
(cd "$SCRIPT_DIR" && sha256sum -c SHA256SUMS)
|
||||
else
|
||||
printf 'Neither shasum nor sha256sum is available.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
[[ ! -L "$TARGET" ]] || { printf 'Refusing symlink restore target: %s\n' "$TARGET" >&2; exit 1; }
|
||||
if [[ -e "$TARGET" && -n "$(find "$TARGET" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)" && "$FORCE" -ne 1 ]]; then
|
||||
printf 'Restore target is not empty: %s (use --force only deliberately)\n' "$TARGET" >&2
|
||||
exit 1
|
||||
fi
|
||||
TARGET_PARENT_INPUT="$(dirname -- "$TARGET")"
|
||||
mkdir -p "$TARGET_PARENT_INPUT"
|
||||
TARGET_PARENT="$(cd -- "$TARGET_PARENT_INPUT" && pwd -P)"
|
||||
TARGET_NAME="$(basename -- "$TARGET")"
|
||||
TARGET_ABS="$TARGET_PARENT/$TARGET_NAME"
|
||||
GLOBAL_PARENT_INPUT="$(dirname -- "$GLOBAL_TARGET")"
|
||||
mkdir -p "$GLOBAL_PARENT_INPUT"
|
||||
GLOBAL_TARGET_ABS="$(cd -- "$GLOBAL_PARENT_INPUT" && pwd -P)/$(basename -- "$GLOBAL_TARGET")"
|
||||
TEMP_TARGET="$(mktemp -d "$TARGET_PARENT/.${TARGET_NAME}.restore.XXXXXX")"
|
||||
OLD_TARGET=""
|
||||
cleanup() {
|
||||
if [[ -d "$TEMP_TARGET" ]]; then
|
||||
rm -rf "$TEMP_TARGET"
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
printf 'Extracting into an isolated staging directory...\n'
|
||||
tar -xpf "$ARCHIVE" -C "$TEMP_TARGET" --strip-components 1
|
||||
ROOT="$TEMP_TARGET"
|
||||
[[ -f "$ROOT/pyproject.toml" && -d "$ROOT/umbraaudio" ]] || {
|
||||
printf 'Archive did not contain the expected UmbraAudio root.\n' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [[ "$KEEP_ARCHIVED_ENV" -ne 1 ]]; then
|
||||
printf 'Discarding archived platform-specific virtualenv/toolchain...\n'
|
||||
rm -rf "$ROOT/.venv" "$ROOT/.python"
|
||||
fi
|
||||
|
||||
install_system_dependencies() {
|
||||
[[ "$INSTALL_SYSTEM_DEPS" -eq 1 ]] || return 0
|
||||
if command -v brew >/dev/null 2>&1; then
|
||||
brew install git ffmpeg
|
||||
brew install python@3.12 || brew install python
|
||||
elif command -v apt-get >/dev/null 2>&1 && command -v sudo >/dev/null 2>&1; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y git ffmpeg python3 python3-venv
|
||||
else
|
||||
printf 'Install git, ffmpeg, Python 3.11+, and venv support manually.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
install_system_dependencies
|
||||
|
||||
python_supports_project() {
|
||||
"$1" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 11) else 1)' \
|
||||
>/dev/null 2>&1
|
||||
}
|
||||
|
||||
PYTHON_BIN=""
|
||||
if [[ -n "$PYTHON_OVERRIDE" ]]; then
|
||||
python_supports_project "$PYTHON_OVERRIDE" || {
|
||||
printf 'Requested Python is unavailable or older than 3.11: %s\n' "$PYTHON_OVERRIDE" >&2
|
||||
exit 1
|
||||
}
|
||||
PYTHON_BIN="$PYTHON_OVERRIDE"
|
||||
else
|
||||
for candidate in python3.12 python3.11 python3.13 python3; do
|
||||
if command -v "$candidate" >/dev/null 2>&1 && python_supports_project "$candidate"; then
|
||||
PYTHON_BIN="$candidate"
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
[[ -n "$PYTHON_BIN" ]] || {
|
||||
printf 'Python 3.11+ is required. Install it or pass --python.\n' >&2
|
||||
exit 1
|
||||
}
|
||||
printf 'Using Python: %s (%s)\n' "$PYTHON_BIN" "$("$PYTHON_BIN" --version 2>&1)"
|
||||
|
||||
if [[ "$SKIP_SUBMODULE_UPDATE" -ne 1 && -f "$ROOT/.gitmodules" ]]; then
|
||||
printf 'Initializing git submodules...\n'
|
||||
git -C "$ROOT" submodule update --init --recursive
|
||||
fi
|
||||
|
||||
if [[ "$SKIP_DEPS" -eq 1 ]]; then
|
||||
printf 'Skipping dependency installation as requested.\n'
|
||||
else
|
||||
UV_BIN="$(command -v uv || true)"
|
||||
BOOTSTRAP_ENV=""
|
||||
if [[ -z "$UV_BIN" ]]; then
|
||||
printf 'Bootstrapping uv 0.11.7 in an isolated environment...\n'
|
||||
BOOTSTRAP_ENV="$ROOT/.uv-bootstrap"
|
||||
"$PYTHON_BIN" -m venv "$BOOTSTRAP_ENV"
|
||||
"$BOOTSTRAP_ENV/bin/python" -m pip install --upgrade pip
|
||||
"$BOOTSTRAP_ENV/bin/python" -m pip install 'uv==0.11.7'
|
||||
UV_BIN="$BOOTSTRAP_ENV/bin/uv"
|
||||
fi
|
||||
printf 'Rebuilding the locked environment with uv...\n'
|
||||
UV_SYNC_ARGS=(sync --locked --python "$PYTHON_BIN")
|
||||
if [[ "$RUN_TESTS" -eq 1 ]]; then
|
||||
UV_SYNC_ARGS+=(--extra dev)
|
||||
fi
|
||||
(cd "$ROOT" && "$UV_BIN" "${UV_SYNC_ARGS[@]}")
|
||||
if [[ "$WITH_TTS_SERVER" -eq 1 && -f "$ROOT/tts_server/requirements.txt" ]]; then
|
||||
(cd "$ROOT" && "$UV_BIN" pip install --python .venv/bin/python -r tts_server/requirements.txt)
|
||||
fi
|
||||
if [[ -n "$BOOTSTRAP_ENV" ]]; then
|
||||
rm -rf "$BOOTSTRAP_ENV"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$DOWNLOAD_MODELS" -eq 1 && ! -f "$ROOT/models/Qwen3-TTS-12Hz-1.7B-Base/model.safetensors" ]]; then
|
||||
[[ -f "$ROOT/scripts/download_models.sh" ]] || {
|
||||
printf 'Model download script is missing.\n' >&2
|
||||
exit 1
|
||||
}
|
||||
(cd "$ROOT" && bash scripts/download_models.sh)
|
||||
fi
|
||||
|
||||
if [[ -f "$SCRIPT_DIR/BACKUP-INFO.txt" ]]; then
|
||||
OLD_ROOT="$(sed -n 's/^source_path=//p' "$SCRIPT_DIR/BACKUP-INFO.txt" | head -n 1)"
|
||||
fi
|
||||
if [[ "$APPLY_PORTABILITY_PATCH" -eq 1 ]]; then
|
||||
"$PYTHON_BIN" "$SCRIPT_DIR/restore_support.py" patch-paths \
|
||||
--root "$ROOT" --old-root "$OLD_ROOT" --new-root "$TARGET_ABS" \
|
||||
--global-state-dir "$GLOBAL_TARGET_ABS"
|
||||
fi
|
||||
|
||||
if [[ -f "$ROOT/.env" ]]; then
|
||||
chmod 600 "$ROOT/.env"
|
||||
fi
|
||||
|
||||
printf 'Validating restored project tree and SQLite databases...\n'
|
||||
"$PYTHON_BIN" "$SCRIPT_DIR/restore_support.py" validate-tree \
|
||||
--root "$ROOT" --old-root "$OLD_ROOT"
|
||||
|
||||
# Commit the project tree only after extraction, dependency setup, path migration,
|
||||
# and validation all succeed. An existing target is retained as .old on --force.
|
||||
if [[ -e "$TARGET" ]]; then
|
||||
OLD_TARGET="$TARGET_PARENT/.${TARGET_NAME}.old.$(date +%Y%m%d%H%M%S)"
|
||||
mv "$TARGET" "$OLD_TARGET"
|
||||
fi
|
||||
mv "$TEMP_TARGET" "$TARGET"
|
||||
trap - EXIT
|
||||
ROOT="$TARGET_ABS"
|
||||
|
||||
if [[ "$SKIP_GLOBAL_STATE" -ne 1 && -d "$SCRIPT_DIR/global-state" ]]; then
|
||||
printf 'Restoring global voice registry/library...\n'
|
||||
RESTORE_ARGS=(restore-global --backup-dir "$SCRIPT_DIR" --target "$GLOBAL_TARGET_ABS")
|
||||
[[ "$FORCE" -eq 1 ]] && RESTORE_ARGS+=(--force)
|
||||
if [[ -n "$OLD_ROOT" ]]; then
|
||||
RESTORE_ARGS+=(--old-project-root "$OLD_ROOT" --new-project-root "$ROOT")
|
||||
fi
|
||||
"$PYTHON_BIN" "$SCRIPT_DIR/restore_support.py" "${RESTORE_ARGS[@]}"
|
||||
fi
|
||||
|
||||
if [[ "$RUN_TESTS" -eq 1 ]]; then
|
||||
[[ -x "$ROOT/.venv/bin/pytest" ]] || {
|
||||
printf 'pytest is not installed in the rebuilt environment.\n' >&2
|
||||
exit 1
|
||||
}
|
||||
(cd "$ROOT" && .venv/bin/pytest -q)
|
||||
fi
|
||||
|
||||
printf '\nRestore complete.\n'
|
||||
printf 'Project root: %s\n' "$ROOT"
|
||||
[[ -z "$OLD_TARGET" ]] || printf 'Previous target retained at: %s\n' "$OLD_TARGET"
|
||||
printf 'Review .env, global voice paths, and RESTORED-SERVICE-COMMANDS.md before starting services.\n'
|
||||
printf 'Network database setup is intentionally not performed by Starthere.sh.\n'
|
||||
493
backup/create_full_backup.sh
Executable file
493
backup/create_full_backup.sh
Executable file
@@ -0,0 +1,493 @@
|
||||
#!/usr/bin/env bash
|
||||
# Create a complete, restorable UmbraAudio snapshot on a mounted filesystem/NAS.
|
||||
set -Eeuo pipefail
|
||||
|
||||
SOURCE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
||||
DESTINATION="${UMBRA_BACKUP_DEST:-/Volumes/NetBackup/AudioBackup}"
|
||||
GLOBAL_ROOT="${UMBRA_GLOBAL_STATE_ROOT:-$HOME/.umbra}"
|
||||
QUIESCE=1
|
||||
BEST_EFFORT=0
|
||||
RESTART_SERVICES=1
|
||||
ALLOW_NON_SMB=0
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: create_full_backup.sh [options]
|
||||
|
||||
Create a dated, fail-closed full backup of the current UmbraAudio working tree
|
||||
and the active global voice registry/library.
|
||||
|
||||
Options:
|
||||
--destination DIR NAS/filesystem destination (default: /Volumes/NetBackup/AudioBackup)
|
||||
--global-root DIR Active global state root (default: $HOME/.umbra)
|
||||
--best-effort Do not quiesce services (not recommended; explicit opt-in)
|
||||
--quiesce Quiesce services (default)
|
||||
--no-restart Do not restart services after a --quiesce backup
|
||||
--allow-non-smb Permit a non-SMB destination for testing
|
||||
-h, --help Show this help
|
||||
|
||||
The archive includes the entire current tree, including dirty/untracked files,
|
||||
ignored book projects, generated audio, models, .git, and .env. Global state is
|
||||
captured with SQLite's backup API and stored beside the archive.
|
||||
EOF
|
||||
}
|
||||
|
||||
while (($#)); do
|
||||
case "$1" in
|
||||
--destination) DESTINATION="${2:?--destination requires a directory}"; shift 2 ;;
|
||||
--global-root) GLOBAL_ROOT="${2:?--global-root requires a directory}"; shift 2 ;;
|
||||
--best-effort) BEST_EFFORT=1; QUIESCE=0; shift ;;
|
||||
--quiesce) QUIESCE=1; shift ;;
|
||||
--no-restart) RESTART_SERVICES=0; shift ;;
|
||||
--allow-non-smb) ALLOW_NON_SMB=1; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) printf 'Unknown option: %s\n' "$1" >&2; usage >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -d "$SOURCE" ]] || { printf 'Source directory not found: %s\n' "$SOURCE" >&2; exit 1; }
|
||||
mkdir -p "$DESTINATION"
|
||||
DESTINATION="$(cd -- "$DESTINATION" && pwd -P)"
|
||||
case "$DESTINATION/" in
|
||||
"$SOURCE/"*) printf 'Backup destination must not be inside the source tree.\n' >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
for tool in tar git awk sed df mount; do
|
||||
command -v "$tool" >/dev/null 2>&1 || {
|
||||
printf 'Required tool is missing: %s\n' "$tool" >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
if command -v shasum >/dev/null 2>&1; then
|
||||
:
|
||||
elif command -v sha256sum >/dev/null 2>&1; then
|
||||
:
|
||||
else
|
||||
printf 'Install shasum or sha256sum.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
hash_file() {
|
||||
if command -v shasum >/dev/null 2>&1; then
|
||||
shasum -a 256 "$@"
|
||||
else
|
||||
sha256sum "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
verify_hashes() {
|
||||
if command -v shasum >/dev/null 2>&1; then
|
||||
shasum -a 256 -c "$@"
|
||||
else
|
||||
sha256sum -c "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
FSTYPE="$(df -P "$DESTINATION" | awk 'NR == 2 {print $1}')"
|
||||
if [[ "$ALLOW_NON_SMB" -ne 1 && "$FSTYPE" != *smb* && "$FSTYPE" != *cifs* ]]; then
|
||||
printf 'Destination is not an SMB/CIFS mount (filesystem=%s). Use --allow-non-smb only for tests.\n' "$FSTYPE" >&2
|
||||
exit 1
|
||||
fi
|
||||
SOURCE_KB="$(du -sk "$SOURCE" | awk '{print $1}')"
|
||||
AVAIL_KB="$(df -Pk "$DESTINATION" | awk 'NR == 2 {print $4}')"
|
||||
if [[ "$AVAIL_KB" -lt $((SOURCE_KB + 1048576)) ]]; then
|
||||
printf 'Insufficient destination space: need about %s KB plus 1 GiB, have %s KB.\n' "$SOURCE_KB" "$AVAIL_KB" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
LOCK_DIR="${TMPDIR:-/tmp}/umbra-full-backup.lock"
|
||||
if ! mkdir "$LOCK_DIR" 2>/dev/null; then
|
||||
printf 'Another backup appears to be running (lock: %s).\n' "$LOCK_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
BACKUP_ID="$(date +%Y%m%d-%H%M%S)-$$-$RANDOM"
|
||||
STAGING_DIR="$(mktemp -d "$DESTINATION/.UmbraAudio-$BACKUP_ID.partial.XXXXXX")"
|
||||
FINAL_DIR="$DESTINATION/UmbraAudio-$BACKUP_ID"
|
||||
ARCHIVE_NAME="UmbraAudio-$BACKUP_ID.tar"
|
||||
ARCHIVE_TMP="$STAGING_DIR/$ARCHIVE_NAME.partial"
|
||||
ARCHIVE_FINAL="$STAGING_DIR/$ARCHIVE_NAME"
|
||||
[[ ! -e "$FINAL_DIR" ]] || { printf 'Final backup directory already exists: %s\n' "$FINAL_DIR" >&2; exit 1; }
|
||||
|
||||
WEB_LABEL="gui/$(id -u)/com.umbraaudio.web"
|
||||
WEB_PLIST="/Users/worstauthor/Library/LaunchAgents/com.umbraaudio.web.plist"
|
||||
if command -v launchctl >/dev/null 2>&1; then
|
||||
detected_web_plist="$(launchctl print "$WEB_LABEL" 2>/dev/null | sed -n 's/^[[:space:]]*path = //p' | head -n 1 || true)"
|
||||
if [[ -f "$detected_web_plist" ]]; then
|
||||
WEB_PLIST="$detected_web_plist"
|
||||
fi
|
||||
fi
|
||||
WEB_WAS_LOADED=0
|
||||
WEB_STOPPED=0
|
||||
QWEN_PIDS=()
|
||||
QWEN_STOPPED=0
|
||||
Qwen_CWD=""
|
||||
Qwen_COMMAND=""
|
||||
WRITER_WATCH_PID=""
|
||||
RESTARTED=0
|
||||
|
||||
release_lock() { rmdir "$LOCK_DIR" 2>/dev/null || true; }
|
||||
|
||||
restart_services() {
|
||||
[[ "$QUIESCE" -eq 1 && "$RESTART_SERVICES" -eq 1 && "$RESTARTED" -eq 0 ]] || return 0
|
||||
local restore_failed=0
|
||||
if [[ "$WEB_WAS_LOADED" -eq 1 && -f "$WEB_PLIST" ]]; then
|
||||
launchctl bootstrap "gui/$(id -u)" "$WEB_PLIST" >/dev/null 2>&1 || \
|
||||
launchctl kickstart -k "$WEB_LABEL" >/dev/null 2>&1 || true
|
||||
for _ in {1..30}; do
|
||||
if launchctl print "$WEB_LABEL" >/dev/null 2>&1 && lsof -nP -iTCP:8420 -sTCP:LISTEN >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if ! launchctl print "$WEB_LABEL" >/dev/null 2>&1 || ! lsof -nP -iTCP:8420 -sTCP:LISTEN >/dev/null 2>&1; then
|
||||
printf 'WARNING: web LaunchAgent did not restart cleanly.\n' >&2
|
||||
restore_failed=1
|
||||
fi
|
||||
fi
|
||||
if [[ "$QWEN_STOPPED" -eq 1 && "${#QWEN_PIDS[@]}" -gt 0 ]]; then
|
||||
mkdir -p "$SOURCE/logs"
|
||||
QWEN_ROOT="${Qwen_CWD:-$SOURCE}"
|
||||
[[ -d "$QWEN_ROOT" ]] || QWEN_ROOT="$SOURCE"
|
||||
(cd "$QWEN_ROOT" && nohup "$SOURCE/.venv/bin/python" -u scripts/qwen_tts_server.py \
|
||||
</dev/null >>"$SOURCE/logs/qwen_tts_server.log" 2>&1 &)
|
||||
disown 2>/dev/null || true
|
||||
for _ in {1..30}; do
|
||||
if lsof -nP -iTCP:8881 -sTCP:LISTEN >/dev/null 2>&1; then
|
||||
if ! command -v curl >/dev/null 2>&1 || curl -fsS --max-time 5 http://127.0.0.1:8881/health >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if ! lsof -nP -iTCP:8881 -sTCP:LISTEN >/dev/null 2>&1; then
|
||||
printf 'WARNING: Qwen TTS server did not become healthy/listening on port 8881.\n' >&2
|
||||
restore_failed=1
|
||||
elif command -v curl >/dev/null 2>&1 && ! curl -fsS --max-time 5 http://127.0.0.1:8881/health >/dev/null 2>&1; then
|
||||
printf 'WARNING: Qwen TTS port is open but /health did not pass.\n' >&2
|
||||
restore_failed=1
|
||||
fi
|
||||
fi
|
||||
RESTARTED=1
|
||||
return "$restore_failed"
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
local rc=$?
|
||||
if [[ -n "$WRITER_WATCH_PID" ]]; then
|
||||
kill "$WRITER_WATCH_PID" >/dev/null 2>&1 || true
|
||||
wait "$WRITER_WATCH_PID" 2>/dev/null || true
|
||||
fi
|
||||
if [[ "$QUIESCE" -eq 1 && "$RESTART_SERVICES" -eq 1 && "$RESTARTED" -eq 0 ]]; then
|
||||
if ! restart_services; then
|
||||
printf 'WARNING: service restoration during cleanup did not complete cleanly.\n' >&2
|
||||
fi
|
||||
fi
|
||||
if [[ -d "$STAGING_DIR" ]]; then
|
||||
printf 'Backup incomplete; preserving partial directory for inspection: %s\n' "$STAGING_DIR" >&2
|
||||
fi
|
||||
release_lock
|
||||
exit "$rc"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
record_service_state() {
|
||||
if launchctl print "$WEB_LABEL" >/dev/null 2>&1; then WEB_WAS_LOADED=1; fi
|
||||
QWEN_PIDS=()
|
||||
if command -v lsof >/dev/null 2>&1; then
|
||||
while IFS= read -r pid; do
|
||||
[[ "$pid" =~ ^[0-9]+$ ]] && QWEN_PIDS+=("$pid")
|
||||
done < <(lsof -nP -tiTCP:8881 -sTCP:LISTEN 2>/dev/null || true)
|
||||
fi
|
||||
while IFS= read -r pid; do
|
||||
[[ "$pid" =~ ^[0-9]+$ ]] || continue
|
||||
[[ " ${QWEN_PIDS[*]} " == *" $pid "* ]] && continue
|
||||
cmd="$(ps -p "$pid" -o command= 2>/dev/null || true)"
|
||||
[[ "$cmd" == *qwen_tts_server.py* ]] && QWEN_PIDS+=("$pid")
|
||||
done < <(ps -axo pid=,args= | awk '$0 ~ /[q]wen_tts_server\\.py/ {print $1}')
|
||||
if [[ "${#QWEN_PIDS[@]}" -gt 0 ]]; then
|
||||
Qwen_COMMAND="$(ps -p "${QWEN_PIDS[0]}" -o command= 2>/dev/null || true)"
|
||||
if command -v lsof >/dev/null 2>&1; then
|
||||
Qwen_CWD="$(lsof -a -p "${QWEN_PIDS[0]}" -d cwd -Fn 2>/dev/null | sed -n 's/^n//p' | head -n 1)"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
write_service_state() {
|
||||
local output="$1"
|
||||
{
|
||||
printf 'captured_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
printf 'web_label=%s\n' "$WEB_LABEL"
|
||||
printf 'web_loaded=%s\n' "$WEB_WAS_LOADED"
|
||||
printf 'web_plist=%s\n' "$WEB_PLIST"
|
||||
printf 'qwen_pids=%s\n' "${QWEN_PIDS[*]:-}"
|
||||
printf 'qwen_cwd=%s\n' "${Qwen_CWD:-}"
|
||||
printf 'qwen_command=%s\n' "${Qwen_COMMAND:-}"
|
||||
launchctl print "$WEB_LABEL" 2>&1 || true
|
||||
lsof -nP -iTCP:8420 -sTCP:LISTEN 2>&1 || true
|
||||
lsof -nP -iTCP:8881 -sTCP:LISTEN 2>&1 || true
|
||||
} >"$output"
|
||||
}
|
||||
|
||||
check_active_writers() {
|
||||
local pattern pid
|
||||
for pattern in \
|
||||
'generate[_]all_audio.py' \
|
||||
'generate[_]chapter[_]audio.py' \
|
||||
'reattribute[_]all.py' \
|
||||
'run[_]llm[_]attribution.py' \
|
||||
'resegment[_]all.py' \
|
||||
'nas[_]lifecycle.py' \
|
||||
'chapter[_]builder' \
|
||||
'[r]sync.*UmbraAudio' \
|
||||
'[f]fmpeg' \
|
||||
'[u]vicorn'; do
|
||||
while IFS= read -r pid; do
|
||||
[[ "$pid" == "$$" ]] && continue
|
||||
printf 'Active writer detected (%s, pid %s); refusing backup.\n' "$pattern" "$pid" >&2
|
||||
return 1
|
||||
done < <(ps -axo pid=,command= | awk -v p="$pattern" '$0 ~ p {print $1}')
|
||||
done
|
||||
}
|
||||
|
||||
quiesce_services() {
|
||||
for tool in launchctl lsof ps; do
|
||||
command -v "$tool" >/dev/null 2>&1 || {
|
||||
printf 'Quiesce requires missing tool: %s\n' "$tool" >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
record_service_state
|
||||
if [[ "$WEB_WAS_LOADED" -eq 1 ]]; then
|
||||
printf 'Pausing web LaunchAgent...\n'
|
||||
launchctl bootout "$WEB_LABEL" "$WEB_PLIST" >/dev/null 2>&1 || {
|
||||
printf 'Could not boot out web LaunchAgent; refusing inconsistent backup.\n' >&2
|
||||
exit 1
|
||||
}
|
||||
for _ in {1..30}; do
|
||||
if ! launchctl print "$WEB_LABEL" >/dev/null 2>&1 && ! lsof -nP -iTCP:8420 -sTCP:LISTEN >/dev/null 2>&1; then
|
||||
WEB_STOPPED=1
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
[[ "$WEB_STOPPED" -eq 1 ]] || {
|
||||
printf 'Web service did not stop cleanly; refusing backup.\n' >&2
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
for pid in "${QWEN_PIDS[@]}"; do
|
||||
printf 'Pausing Qwen TTS server (pid %s)...\n' "$pid"
|
||||
kill -TERM "$pid" >/dev/null 2>&1 || true
|
||||
done
|
||||
for pid in "${QWEN_PIDS[@]}"; do
|
||||
for _ in {1..30}; do
|
||||
kill -0 "$pid" >/dev/null 2>&1 || break
|
||||
sleep 1
|
||||
done
|
||||
if kill -0 "$pid" >/dev/null 2>&1; then
|
||||
printf 'Qwen TTS server pid %s did not stop; refusing backup.\n' "$pid" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
QWEN_STOPPED=1
|
||||
if lsof -nP -iTCP:8881 -sTCP:LISTEN >/dev/null 2>&1; then
|
||||
printf 'Port 8881 is still listening; refusing backup.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$WEB_STOPPED" -eq 1 ]] && lsof -nP -iTCP:8420 -sTCP:LISTEN >/dev/null 2>&1; then
|
||||
printf 'Port 8420 is still listening; refusing backup.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
check_active_writers || exit 1
|
||||
sleep 2
|
||||
}
|
||||
|
||||
record_service_state
|
||||
write_service_state "$STAGING_DIR/SERVICE-STATE-BEFORE.txt"
|
||||
if [[ "$QUIESCE" -eq 1 ]]; then
|
||||
quiesce_services
|
||||
else
|
||||
printf 'WARNING: best-effort backup requested; active files may be inconsistent.\n' >&2
|
||||
fi
|
||||
|
||||
SOURCE_PARENT="$(cd -- "$SOURCE/.." && pwd -P)"
|
||||
SOURCE_NAME="$(basename -- "$SOURCE")"
|
||||
GLOBAL_PYTHON="$SOURCE/.venv/bin/python"
|
||||
[[ -x "$GLOBAL_PYTHON" ]] || GLOBAL_PYTHON="$(command -v python3)"
|
||||
|
||||
{
|
||||
printf 'backup_id=%s\n' "$BACKUP_ID"
|
||||
printf 'created_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
printf 'source_path=%s\n' "$SOURCE"
|
||||
printf 'global_state_root=%s\n' "$GLOBAL_ROOT"
|
||||
printf 'source_host=%s\n' "$(hostname)"
|
||||
printf 'source_os=%s\n' "$(uname -a)"
|
||||
printf 'git_commit=%s\n' "$(git -C "$SOURCE" rev-parse HEAD 2>/dev/null || printf unknown)"
|
||||
printf 'git_branch=%s\n' "$(git -C "$SOURCE" branch --show-current 2>/dev/null || printf unknown)"
|
||||
printf 'quiesced_services=%s\n' "$QUIESCE"
|
||||
printf 'best_effort=%s\n' "$BEST_EFFORT"
|
||||
printf 'archive_name=%s\n' "$ARCHIVE_NAME"
|
||||
} >"$STAGING_DIR/BACKUP-INFO.txt"
|
||||
|
||||
{
|
||||
python3 --version 2>&1 || true
|
||||
"$GLOBAL_PYTHON" --version 2>&1 || true
|
||||
uv --version 2>&1 || true
|
||||
git --version 2>&1 || true
|
||||
tar --version 2>&1 | head -n 1 || true
|
||||
ffmpeg -version 2>&1 | head -n 1 || true
|
||||
} >"$STAGING_DIR/ENVIRONMENT.txt"
|
||||
|
||||
git -C "$SOURCE" status --short --branch --untracked-files=all >"$STAGING_DIR/GIT-STATUS.txt" 2>&1 || true
|
||||
git -C "$SOURCE" diff --binary >"$STAGING_DIR/GIT-DIFF.patch" 2>/dev/null || true
|
||||
git -C "$SOURCE" diff --cached --binary >"$STAGING_DIR/GIT-DIFF-CACHED.patch" 2>/dev/null || true
|
||||
git -C "$SOURCE" submodule status >"$STAGING_DIR/SUBMODULES.txt" 2>&1 || true
|
||||
|
||||
printf 'Recording file inventory...\n'
|
||||
SOURCE="$SOURCE" "$GLOBAL_PYTHON" - <<'PY' >"$STAGING_DIR/FILE-INVENTORY.tsv"
|
||||
from __future__ import annotations
|
||||
import os
|
||||
from pathlib import Path
|
||||
root = Path(os.environ["SOURCE"])
|
||||
for path in sorted(root.rglob("*")):
|
||||
try:
|
||||
stat = path.lstat()
|
||||
except OSError:
|
||||
continue
|
||||
kind = "l" if path.is_symlink() else "d" if path.is_dir() else "f"
|
||||
print(f"{kind}\t{stat.st_size}\t{path.relative_to(root)}")
|
||||
PY
|
||||
|
||||
printf 'Running source integrity checks before archiving...\n'
|
||||
"$GLOBAL_PYTHON" "$SOURCE/backup/restore_support.py" validate-tree --root "$SOURCE"
|
||||
|
||||
cat >"$STAGING_DIR/BACKUP-README.txt" <<'EOF'
|
||||
UmbraAudio full backup
|
||||
======================
|
||||
|
||||
This backup is a point-in-time copy of the complete current working tree and
|
||||
active global voice registry/library.
|
||||
|
||||
Included:
|
||||
- tracked, untracked, and modified files;
|
||||
- ignored book projects, generated audio, exports, logs, and .git metadata;
|
||||
- model weights and the Qwen3-TTS submodule;
|
||||
- .env and local configuration;
|
||||
- the existing .venv and .python directories for forensic completeness;
|
||||
- a SQLite-consistent global-state snapshot in global-state/;
|
||||
- launchd/GPU-relay context in context/.
|
||||
|
||||
The restore script rebuilds a portable environment rather than trusting the
|
||||
archived platform-specific virtual environment. The original .env is preserved
|
||||
as .env.archive-original when portability paths are rewritten.
|
||||
|
||||
This backup does not configure or modify a network database.
|
||||
|
||||
Restore:
|
||||
./Starthere.sh --target /path/to/restore
|
||||
|
||||
See BACKUP-INFO.txt, ENVIRONMENT.txt, global-state/MANIFEST.json, and
|
||||
RESTORED-SERVICE-COMMANDS.md after restoration.
|
||||
EOF
|
||||
|
||||
cp "$SOURCE/backup/Starthere.sh" "$STAGING_DIR/Starthere.sh"
|
||||
cp "$SOURCE/backup/restore_support.py" "$STAGING_DIR/restore_support.py"
|
||||
chmod 700 "$STAGING_DIR/Starthere.sh"
|
||||
chmod 700 "$STAGING_DIR/restore_support.py"
|
||||
|
||||
CONTEXT_DIR="$STAGING_DIR/context"
|
||||
mkdir -p "$CONTEXT_DIR/launchagents" "$CONTEXT_DIR/launchdaemons"
|
||||
cp -p /Users/worstauthor/.umbraaudio/umbraweb_login.sh "$CONTEXT_DIR/launchagents/" 2>/dev/null || true
|
||||
cp -p /Users/worstauthor/Library/LaunchAgents/com.umbraaudio.web.plist "$CONTEXT_DIR/launchagents/" 2>/dev/null || true
|
||||
cp -p /Users/worstauthor/Library/LaunchAgents/com.umbraaudio.commitcheck.plist "$CONTEXT_DIR/launchagents/" 2>/dev/null || true
|
||||
cp -p /Library/LaunchDaemons/com.umba.qwen-relay.plist "$CONTEXT_DIR/launchdaemons/" 2>/dev/null || true
|
||||
cp -p /Library/LaunchDaemons/qwen_relay.py "$CONTEXT_DIR/launchdaemons/" 2>/dev/null || true
|
||||
launchctl print "$WEB_LABEL" >"$CONTEXT_DIR/launchctl-web.txt" 2>&1 || true
|
||||
launchctl print system/com.umba.qwen-relay >"$CONTEXT_DIR/launchctl-qwen-relay.txt" 2>&1 || true
|
||||
mount >"$CONTEXT_DIR/mounts.txt" 2>&1 || true
|
||||
df -h >"$CONTEXT_DIR/filesystems.txt" 2>&1 || true
|
||||
printf '%s\n' 'Context files are informational; Starthere.sh does not install privileged services.' >"$CONTEXT_DIR/README.txt"
|
||||
|
||||
printf 'Snapshotting global voice state with SQLite backup API...\n'
|
||||
"$GLOBAL_PYTHON" "$SOURCE/backup/restore_support.py" snapshot-global \
|
||||
--source "$GLOBAL_ROOT" --output "$STAGING_DIR/global-state" --original-root "$GLOBAL_ROOT"
|
||||
(cd "$STAGING_DIR/global-state" && verify_hashes GLOBAL-STATE-SHA256SUMS)
|
||||
|
||||
start_writer_watch() {
|
||||
local flag="$STAGING_DIR/ACTIVE-WRITER"
|
||||
(
|
||||
while :; do
|
||||
if ! check_active_writers >/dev/null 2>&1; then
|
||||
printf 'writer detected during archive\n' >"$flag"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
) &
|
||||
WRITER_WATCH_PID="$!"
|
||||
}
|
||||
|
||||
stop_writer_watch() {
|
||||
if [[ -n "$WRITER_WATCH_PID" ]]; then
|
||||
kill "$WRITER_WATCH_PID" >/dev/null 2>&1 || true
|
||||
wait "$WRITER_WATCH_PID" 2>/dev/null || true
|
||||
WRITER_WATCH_PID=""
|
||||
fi
|
||||
}
|
||||
|
||||
printf 'Creating full payload archive (%s)...\n' "$ARCHIVE_NAME"
|
||||
start_writer_watch
|
||||
if ! COPYFILE_DISABLE=1 tar -C "$SOURCE_PARENT" -cf "$ARCHIVE_TMP" "$SOURCE_NAME" 2>"$STAGING_DIR/TAR-WARNINGS.log"; then
|
||||
stop_writer_watch
|
||||
printf 'tar failed; inspect TAR-WARNINGS.log.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
stop_writer_watch
|
||||
if [[ -f "$STAGING_DIR/ACTIVE-WRITER" ]]; then
|
||||
printf 'A writer appeared while archiving; refusing the backup.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
if grep -Eiq 'file changed|error|failed|cannot|permission denied' "$STAGING_DIR/TAR-WARNINGS.log"; then
|
||||
printf 'tar reported a consistency/read warning; refusing backup.\n' >&2
|
||||
cat "$STAGING_DIR/TAR-WARNINGS.log" >&2
|
||||
exit 1
|
||||
fi
|
||||
mv "$ARCHIVE_TMP" "$ARCHIVE_FINAL"
|
||||
printf 'Computing comprehensive SHA-256 manifest...\n'
|
||||
"$GLOBAL_PYTHON" "$SOURCE/backup/restore_support.py" write-manifest --root "$STAGING_DIR"
|
||||
(cd "$STAGING_DIR" && verify_hashes SHA256SUMS)
|
||||
(cd "$STAGING_DIR/global-state" && verify_hashes GLOBAL-STATE-SHA256SUMS)
|
||||
tar -tf "$ARCHIVE_FINAL" >/dev/null
|
||||
chmod 600 "$ARCHIVE_FINAL" "$STAGING_DIR/SHA256SUMS" "$STAGING_DIR/GIT-DIFF.patch" "$STAGING_DIR/GIT-DIFF-CACHED.patch" 2>/dev/null || true
|
||||
|
||||
mv "$STAGING_DIR" "$FINAL_DIR"
|
||||
STAGING_DIR=""
|
||||
|
||||
# Keep the cleanup trap until service restoration and top-level pointers finish.
|
||||
ROOT_LAUNCHER_TMP="$DESTINATION/.Starthere.sh.tmp.$$"
|
||||
cat >"$ROOT_LAUNCHER_TMP" <<EOF
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
ROOT="\$(cd -- "\$(dirname -- "\${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
LATEST="\$(find "\$ROOT" -maxdepth 1 -type d -name 'UmbraAudio-*' ! -name '.*' -print | sort | tail -n 1)"
|
||||
[[ -n "\$LATEST" && -x "\$LATEST/Starthere.sh" ]] || { printf 'No completed backup found under %s\n' "\$ROOT" >&2; exit 1; }
|
||||
exec "\$LATEST/Starthere.sh" "\$@"
|
||||
EOF
|
||||
chmod 700 "$ROOT_LAUNCHER_TMP"
|
||||
mv "$ROOT_LAUNCHER_TMP" "$DESTINATION/Starthere.sh"
|
||||
printf '%s\n' "$FINAL_DIR" >"$DESTINATION/LATEST.tmp.$$"
|
||||
mv "$DESTINATION/LATEST.tmp.$$" "$DESTINATION/LATEST"
|
||||
|
||||
restart_services
|
||||
record_service_state
|
||||
write_service_state "$FINAL_DIR/SERVICE-STATE-AFTER.txt"
|
||||
"$GLOBAL_PYTHON" "$SOURCE/backup/restore_support.py" write-manifest --root "$FINAL_DIR"
|
||||
(cd "$FINAL_DIR" && verify_hashes SHA256SUMS)
|
||||
trap - EXIT
|
||||
release_lock
|
||||
|
||||
printf '\nBackup complete.\n'
|
||||
printf 'Location: %s\n' "$FINAL_DIR"
|
||||
printf 'Archive: %s\n' "$FINAL_DIR/$ARCHIVE_NAME"
|
||||
printf 'Global state snapshot: %s/global-state\n' "$FINAL_DIR"
|
||||
printf 'Restore: %s/Starthere.sh --target /path/to/restore\n' "$FINAL_DIR"
|
||||
821
backup/restore_support.py
Normal file
821
backup/restore_support.py
Normal file
@@ -0,0 +1,821 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Portable backup/restore support helpers for UmbraAudio.
|
||||
|
||||
This module uses only the Python standard library. It snapshots global Umbra
|
||||
state with SQLite's backup API, restores it atomically, rewrites machine-local
|
||||
paths in a restored copy, and performs basic integrity checks.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def is_database(path: Path) -> bool:
|
||||
return path.name.endswith((".db", ".db-wal", ".db-shm"))
|
||||
|
||||
|
||||
def is_sqlite_file(path: Path) -> bool:
|
||||
try:
|
||||
with path.open("rb") as handle:
|
||||
return handle.read(16) == b"SQLite format 3\x00"
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def safe_relative_path(value: str) -> Path:
|
||||
path = Path(value)
|
||||
if path.is_absolute() or ".." in path.parts:
|
||||
raise RuntimeError(f"unsafe relative path in backup metadata: {value!r}")
|
||||
return path
|
||||
|
||||
|
||||
def sql_identifier(value: str) -> str:
|
||||
return '"' + value.replace('"', '""') + '"'
|
||||
|
||||
|
||||
def sqlite_backup(source: Path, destination: Path) -> str:
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
source_uri = f"file:{source}?mode=ro"
|
||||
source_db = sqlite3.connect(source_uri, uri=True)
|
||||
destination_db = sqlite3.connect(destination)
|
||||
try:
|
||||
source_db.backup(destination_db)
|
||||
result = destination_db.execute("PRAGMA integrity_check").fetchone()[0]
|
||||
if result != "ok":
|
||||
raise RuntimeError(f"integrity_check failed for {source}: {result}")
|
||||
finally:
|
||||
destination_db.close()
|
||||
source_db.close()
|
||||
return "ok"
|
||||
|
||||
|
||||
def copy_tree_files(
|
||||
source: Path,
|
||||
destination: Path,
|
||||
skip_databases: bool = True,
|
||||
reject_symlinks: bool = False,
|
||||
) -> list[str]:
|
||||
copied: list[str] = []
|
||||
for path in sorted(source.rglob("*")):
|
||||
relative = path.relative_to(source)
|
||||
target = destination / relative
|
||||
if path.is_symlink():
|
||||
if reject_symlinks:
|
||||
raise RuntimeError(f"global state symlink is not allowed: {relative}")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
os.symlink(os.readlink(path), target)
|
||||
elif path.is_dir():
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
elif path.is_file():
|
||||
if skip_databases and is_database(path):
|
||||
continue
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(path, target)
|
||||
copied.append(str(relative))
|
||||
return copied
|
||||
|
||||
|
||||
def snapshot_global(source: Path, output: Path, original_root: Path) -> None:
|
||||
if output.exists():
|
||||
raise RuntimeError(f"global-state output already exists: {output}")
|
||||
if not source.exists():
|
||||
raise RuntimeError(f"global state root does not exist: {source}")
|
||||
if source == output or source in output.parents:
|
||||
raise RuntimeError("global state output must be outside the source root")
|
||||
|
||||
temporary = Path(tempfile.mkdtemp(prefix="umbra-global-snapshot-"))
|
||||
try:
|
||||
files = copy_tree_files(source, temporary, skip_databases=True)
|
||||
databases: list[dict[str, Any]] = []
|
||||
for path in sorted(source.rglob("*.db")):
|
||||
if not path.is_file():
|
||||
continue
|
||||
relative = path.relative_to(source)
|
||||
target = temporary / relative
|
||||
status = sqlite_backup(path, target)
|
||||
databases.append(
|
||||
{"path": str(relative), "integrity_check": status, "sha256": sha256(target)}
|
||||
)
|
||||
files.append(str(relative))
|
||||
|
||||
file_hashes = {
|
||||
relative: sha256(temporary / relative)
|
||||
for relative in sorted(files)
|
||||
if (temporary / relative).is_file()
|
||||
}
|
||||
manifest = {
|
||||
"schema": "umbra.global_state/v1",
|
||||
"original_root": str(original_root),
|
||||
"source_root": str(source),
|
||||
"files": sorted(files),
|
||||
"file_hashes": file_hashes,
|
||||
"databases": databases,
|
||||
}
|
||||
(temporary / "MANIFEST.json").write_text(
|
||||
json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
(temporary / "GLOBAL-STATE-SHA256SUMS").write_text(
|
||||
"".join(f"{sha256(temporary / relative)} {relative}\n" for relative in sorted(files)),
|
||||
encoding="utf-8",
|
||||
)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copytree(temporary, output, symlinks=True)
|
||||
finally:
|
||||
shutil.rmtree(temporary, ignore_errors=True)
|
||||
|
||||
|
||||
def _replace_prefix(value: Any, old_root: str, new_root: str) -> Any:
|
||||
if isinstance(value, str) and old_root and value.startswith(old_root):
|
||||
return new_root + value[len(old_root) :]
|
||||
return value
|
||||
|
||||
|
||||
def assert_no_old_prefixes(
|
||||
connection: sqlite3.Connection,
|
||||
old_roots: list[str],
|
||||
) -> None:
|
||||
for table_row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'"):
|
||||
table = table_row[0]
|
||||
if table.startswith("sqlite_"):
|
||||
continue
|
||||
table_identifier = sql_identifier(table)
|
||||
for column in connection.execute(f"PRAGMA table_info({table_identifier})").fetchall():
|
||||
column_name = column[1]
|
||||
column_identifier = sql_identifier(column_name)
|
||||
if (column[2] or "").upper() not in {"TEXT", ""}:
|
||||
continue
|
||||
try:
|
||||
values = connection.execute(
|
||||
f"SELECT {column_identifier} FROM {table_identifier} "
|
||||
f"WHERE {column_identifier} IS NOT NULL"
|
||||
).fetchall()
|
||||
except sqlite3.Error:
|
||||
continue
|
||||
for (value,) in values:
|
||||
if isinstance(value, str) and any(
|
||||
old_root and value.startswith(old_root) for old_root in old_roots
|
||||
):
|
||||
raise RuntimeError(f"old path prefix remains in {table}.{column_name}: {value}")
|
||||
|
||||
|
||||
def migrate_database_paths(database: Path, replacements: list[tuple[str, str]]) -> None:
|
||||
connection = sqlite3.connect(database)
|
||||
try:
|
||||
tables = {
|
||||
row[0]
|
||||
for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")
|
||||
}
|
||||
for table, columns in {
|
||||
"voice_registry": ("voice_ref_path", "vmatrix_path", "sample_audio_path"),
|
||||
"library_speakers": ("voice_ref_path",),
|
||||
"library_utterances": ("voice_ref_path",),
|
||||
}.items():
|
||||
if table not in tables:
|
||||
continue
|
||||
for column in columns:
|
||||
try:
|
||||
rows = connection.execute(
|
||||
f'SELECT rowid, "{column}" FROM "{table}" WHERE "{column}" IS NOT NULL'
|
||||
).fetchall()
|
||||
except sqlite3.Error:
|
||||
continue
|
||||
for rowid, value in rows:
|
||||
replacement = value
|
||||
for old_root, new_root in replacements:
|
||||
replacement = _replace_prefix(replacement, old_root, new_root)
|
||||
if replacement != value:
|
||||
connection.execute(
|
||||
f'UPDATE "{table}" SET "{column}" = ? WHERE rowid = ?',
|
||||
(replacement, rowid),
|
||||
)
|
||||
connection.commit()
|
||||
assert_no_old_prefixes(connection, [old for old, _ in replacements if old])
|
||||
result = connection.execute("PRAGMA integrity_check").fetchone()[0]
|
||||
if result != "ok":
|
||||
raise RuntimeError(f"integrity_check failed after path migration: {database}: {result}")
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def migrate_project_database(database: Path, old_root: str, new_root: str) -> None:
|
||||
connection = sqlite3.connect(database)
|
||||
try:
|
||||
tables = {
|
||||
row[0]
|
||||
for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")
|
||||
if not row[0].startswith("sqlite_")
|
||||
}
|
||||
for table in tables:
|
||||
table_identifier = sql_identifier(table)
|
||||
columns = connection.execute(f"PRAGMA table_info({table_identifier})").fetchall()
|
||||
for column in columns:
|
||||
column_name = column[1]
|
||||
column_identifier = sql_identifier(column_name)
|
||||
if (column[2] or "").upper() not in {"TEXT", ""}:
|
||||
continue
|
||||
try:
|
||||
query = (
|
||||
f"SELECT rowid, {column_identifier} FROM {table_identifier} "
|
||||
f"WHERE {column_identifier} IS NOT NULL"
|
||||
)
|
||||
rows = connection.execute(query).fetchall()
|
||||
except sqlite3.Error:
|
||||
continue
|
||||
for rowid, value in rows:
|
||||
replacement = _replace_prefix(value, old_root, new_root)
|
||||
if replacement != value:
|
||||
connection.execute(
|
||||
f"UPDATE {table_identifier} SET {column_identifier} = ? "
|
||||
"WHERE rowid = ?",
|
||||
(replacement, rowid),
|
||||
)
|
||||
connection.commit()
|
||||
assert_no_old_prefixes(connection, [old_root] if old_root else [])
|
||||
result = connection.execute("PRAGMA integrity_check").fetchone()[0]
|
||||
if result != "ok":
|
||||
raise RuntimeError(
|
||||
f"integrity_check failed after project path migration: {database}: {result}"
|
||||
)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def _validate_target(path: Path, force: bool) -> Path:
|
||||
if path.is_symlink():
|
||||
raise RuntimeError(f"refusing symlink target: {path}")
|
||||
if path.exists() and not path.is_dir():
|
||||
raise RuntimeError(f"target exists and is not a directory: {path}")
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if path.exists() and any(path.iterdir()) and not force:
|
||||
raise RuntimeError(f"target is not empty: {path}; use --force")
|
||||
return path
|
||||
|
||||
|
||||
def _commit_directory(temporary: Path, target: Path, force: bool) -> Path | None:
|
||||
_validate_target(target, force)
|
||||
previous: Path | None = None
|
||||
if target.exists():
|
||||
previous = target.with_name(f"{target.name}.old.{int(time.time())}")
|
||||
target.rename(previous)
|
||||
try:
|
||||
temporary.rename(target)
|
||||
except Exception:
|
||||
if previous is not None and previous.exists() and not target.exists():
|
||||
previous.rename(target)
|
||||
raise
|
||||
return previous
|
||||
|
||||
|
||||
def restore_global(
|
||||
backup_dir: Path,
|
||||
target: Path,
|
||||
force: bool,
|
||||
old_project_root: str = "",
|
||||
new_project_root: str = "",
|
||||
) -> None:
|
||||
source = backup_dir / "global-state"
|
||||
manifest_path = source / "MANIFEST.json"
|
||||
if not manifest_path.exists():
|
||||
raise RuntimeError(f"global-state manifest is missing: {manifest_path}")
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
old_root = manifest.get("original_root", "")
|
||||
_validate_target(target, force)
|
||||
temporary = Path(tempfile.mkdtemp(prefix=f".{target.name}.global-restore.", dir=target.parent))
|
||||
try:
|
||||
copy_tree_files(
|
||||
source,
|
||||
temporary,
|
||||
skip_databases=False,
|
||||
reject_symlinks=True,
|
||||
)
|
||||
for metadata_name in ("MANIFEST.json", "GLOBAL-STATE-SHA256SUMS"):
|
||||
metadata_path = temporary / metadata_name
|
||||
if metadata_path.exists():
|
||||
metadata_path.unlink()
|
||||
for relative in manifest.get("files", []):
|
||||
safe_relative_path(relative)
|
||||
file_hashes = manifest.get("file_hashes", {})
|
||||
for relative, expected_hash in file_hashes.items():
|
||||
safe_relative = safe_relative_path(relative)
|
||||
path = temporary / safe_relative
|
||||
if not path.is_file() or sha256(path) != expected_hash:
|
||||
raise RuntimeError(f"global file checksum mismatch: {relative}")
|
||||
replacements = [(old_root, str(target))]
|
||||
if old_project_root and new_project_root:
|
||||
replacements.append((old_project_root, new_project_root))
|
||||
for database in manifest.get("databases", []):
|
||||
relative = safe_relative_path(database["path"])
|
||||
path = temporary / relative
|
||||
if not path.is_file() or sha256(path) != database.get("sha256"):
|
||||
raise RuntimeError(f"global database checksum mismatch: {relative}")
|
||||
migrate_database_paths(path, replacements)
|
||||
previous = _commit_directory(temporary, target, force)
|
||||
temporary = Path()
|
||||
if previous is not None:
|
||||
print(f"Previous global state retained at: {previous}")
|
||||
finally:
|
||||
if temporary and str(temporary) not in {"", "."}:
|
||||
shutil.rmtree(temporary, ignore_errors=True)
|
||||
print(f"Restored global state to {target}")
|
||||
|
||||
|
||||
def _replace_text(path: Path, replacements: list[tuple[str, str]]) -> list[str]:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
changed: list[str] = []
|
||||
for old, new in replacements:
|
||||
if old in text:
|
||||
text = text.replace(old, new)
|
||||
changed.append(old)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
return changed
|
||||
|
||||
|
||||
def patch_restored_paths(
|
||||
root: Path,
|
||||
old_root: str,
|
||||
new_root: Path,
|
||||
global_root: Path | None = None,
|
||||
) -> None:
|
||||
changes: list[str] = []
|
||||
env_path = root / ".env"
|
||||
if env_path.exists() and old_root:
|
||||
original = root / ".env.archive-original"
|
||||
if not original.exists():
|
||||
shutil.copy2(env_path, original)
|
||||
changes.extend(_replace_text(env_path, [(old_root, str(new_root))]))
|
||||
if env_path.exists() and global_root is not None:
|
||||
lines = env_path.read_text(encoding="utf-8").splitlines()
|
||||
keys = {"UMBRA_GLOBAL_STATE_DIR", "UMBRA_VOICE_REGISTRY_PATH"}
|
||||
lines = [line for line in lines if line.split("=", 1)[0] not in keys]
|
||||
lines.extend(
|
||||
[
|
||||
f"UMBRA_GLOBAL_STATE_DIR={json.dumps(str(global_root))}",
|
||||
f"UMBRA_VOICE_REGISTRY_PATH={json.dumps(str(global_root / 'voice_registry.db'))}",
|
||||
]
|
||||
)
|
||||
env_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
changes.append(".env global-state settings")
|
||||
|
||||
config_path = root / "umbraaudio" / "config.py"
|
||||
if config_path.exists() and old_root:
|
||||
text = config_path.read_text(encoding="utf-8")
|
||||
if "import os\n" not in text:
|
||||
text = text.replace("import re\n", "import os\nimport re\n", 1)
|
||||
portable_project_base = (
|
||||
"PROJECT_BASE = Path(\n"
|
||||
' os.environ.get("UMBRA_PROJECT_BASE", '
|
||||
"str(Path(__file__).resolve().parents[1]))\n"
|
||||
").expanduser()"
|
||||
)
|
||||
text = text.replace(
|
||||
f'PROJECT_BASE = Path("{old_root}")',
|
||||
portable_project_base,
|
||||
)
|
||||
text = text.replace(
|
||||
f'"{old_root}/models/Qwen3-TTS-12Hz-1.7B-VoiceDesign"',
|
||||
'str(PROJECT_BASE / "models/Qwen3-TTS-12Hz-1.7B-VoiceDesign")',
|
||||
)
|
||||
text = text.replace(
|
||||
f'"{old_root}/models/Qwen3-TTS-12Hz-1.7B-Base"',
|
||||
'str(PROJECT_BASE / "models/Qwen3-TTS-12Hz-1.7B-Base")',
|
||||
)
|
||||
portable_voice_registry = (
|
||||
"voice_registry_path: Path = Field(\n"
|
||||
" default_factory=lambda: Path(\n"
|
||||
" os.environ.get(\n"
|
||||
' "UMBRA_VOICE_REGISTRY_PATH",\n'
|
||||
' str(Path.home() / ".umbra" / "voice_registry.db"),\n'
|
||||
" )\n"
|
||||
" ).expanduser()\n"
|
||||
")"
|
||||
)
|
||||
old_voice_field = (
|
||||
'voice_registry_path: Path = Field(default=Path.home() / ".umbra" / '
|
||||
'"voice_registry.db")'
|
||||
)
|
||||
text = text.replace(old_voice_field, portable_voice_registry)
|
||||
config_path.write_text(text, encoding="utf-8")
|
||||
changes.append("umbraaudio/config.py")
|
||||
|
||||
library_path = root / "umbraaudio" / "library.py"
|
||||
if library_path.exists():
|
||||
text = library_path.read_text(encoding="utf-8")
|
||||
if "import os\n" not in text:
|
||||
text = text.replace("import json\n", "import json\nimport os\n", 1)
|
||||
portable_library_root = (
|
||||
"def _global_state_root() -> Path:\n"
|
||||
" return Path(\n"
|
||||
' os.environ.get("UMBRA_GLOBAL_STATE_DIR", str(Path.home() / ".umbra"))\n'
|
||||
" ).expanduser()\n"
|
||||
)
|
||||
text = text.replace(
|
||||
'DEFAULT_LIBRARY_DB = Path.home() / ".umbra" / "voice_registry.db"\n'
|
||||
'DEFAULT_LIBRARY_VOICES = Path.home() / ".umbra" / "library" / "voices"',
|
||||
portable_library_root
|
||||
+ 'DEFAULT_LIBRARY_DB = _global_state_root() / "voice_registry.db"\n'
|
||||
+ 'DEFAULT_LIBRARY_VOICES = _global_state_root() / "library" / "voices"',
|
||||
)
|
||||
old_db_getter = "def get_library_db_path() -> Path:\n return DEFAULT_LIBRARY_DB"
|
||||
new_db_getter = (
|
||||
"def get_library_db_path() -> Path:\n return _global_state_root() / "
|
||||
'"voice_registry.db"'
|
||||
)
|
||||
text = text.replace(old_db_getter, new_db_getter)
|
||||
old_voices_getter = (
|
||||
"def get_library_voices_dir() -> Path:\n return DEFAULT_LIBRARY_VOICES"
|
||||
)
|
||||
new_voices_getter = (
|
||||
"def get_library_voices_dir() -> Path:\n return _global_state_root() / "
|
||||
'"library" / "voices"'
|
||||
)
|
||||
text = text.replace(old_voices_getter, new_voices_getter)
|
||||
library_path.write_text(text, encoding="utf-8")
|
||||
changes.append("umbraaudio/library.py")
|
||||
|
||||
server_path = root / "scripts" / "qwen_tts_server.py"
|
||||
if server_path.exists() and old_root:
|
||||
portable_prompt_dir = (
|
||||
"str(Path(__file__).resolve().parents[1] / "
|
||||
'"DealHunterShip_4.0e_whole-to-audio/.umbra/voice_prompts")'
|
||||
)
|
||||
portable_voicedesign = (
|
||||
'str(Path(__file__).resolve().parents[1] / "models/Qwen3-TTS-12Hz-1.7B-VoiceDesign")'
|
||||
)
|
||||
portable_base = (
|
||||
'str(Path(__file__).resolve().parents[1] / "models/Qwen3-TTS-12Hz-1.7B-Base")'
|
||||
)
|
||||
replacements = [
|
||||
(
|
||||
f'"{old_root}/DealHunterShip_4.0e_whole-to-audio/.umbra/voice_prompts"',
|
||||
portable_prompt_dir,
|
||||
),
|
||||
(
|
||||
f'"{old_root}/models/Qwen3-TTS-12Hz-1.7B-VoiceDesign"',
|
||||
portable_voicedesign,
|
||||
),
|
||||
(
|
||||
f'"{old_root}/models/Qwen3-TTS-12Hz-1.7B-Base"',
|
||||
portable_base,
|
||||
),
|
||||
]
|
||||
changes.extend(_replace_text(server_path, replacements))
|
||||
|
||||
storage_path = root / "umbraaudio" / "storage.py"
|
||||
if storage_path.exists():
|
||||
text = storage_path.read_text(encoding="utf-8")
|
||||
if "import os\n" not in text:
|
||||
text = text.replace("import logging\n", "import logging\nimport os\n", 1)
|
||||
if old_root:
|
||||
portable_local_root = (
|
||||
"DEFAULT_LOCAL_ROOT = Path(os.environ.get("
|
||||
'"UMBRA_PROJECT_ROOT", str(Path(__file__).resolve().parents[1]))'
|
||||
").expanduser()"
|
||||
)
|
||||
text = text.replace(
|
||||
f'DEFAULT_LOCAL_ROOT = Path("{old_root}")',
|
||||
portable_local_root,
|
||||
)
|
||||
portable_nas_root = (
|
||||
"DEFAULT_NAS_ROOT = Path(os.environ.get("
|
||||
'"UMBRA_NAS_ROOT", "/Volumes/Umbra/UmbraAudio"'
|
||||
")).expanduser()"
|
||||
)
|
||||
text = text.replace(
|
||||
'DEFAULT_NAS_ROOT = Path("/Volumes/Umbra/UmbraAudio")',
|
||||
portable_nas_root,
|
||||
)
|
||||
storage_path.write_text(text, encoding="utf-8")
|
||||
changes.append("umbraaudio/storage.py")
|
||||
|
||||
if old_root:
|
||||
escaped_root = json.dumps(str(new_root))[1:-1]
|
||||
operational_files = [
|
||||
root / "scripts" / "qwen_tts_server.py",
|
||||
root / "tts_server" / "qwen_tts_server.py",
|
||||
root / "Qwen3-TTS" / "qwen_tts" / "cli" / "demo.py",
|
||||
root / "start_server.sh",
|
||||
root / "test_voice_clone.py",
|
||||
]
|
||||
for operational_path in operational_files:
|
||||
if operational_path.exists():
|
||||
_replace_text(operational_path, [(old_root, escaped_root)])
|
||||
|
||||
nas_lifecycle = root / "scripts" / "nas_lifecycle.py"
|
||||
if nas_lifecycle.exists():
|
||||
text = nas_lifecycle.read_text(encoding="utf-8")
|
||||
if "import os\n" not in text:
|
||||
text = text.replace("import logging\n", "import logging\nimport os\n", 1)
|
||||
text = text.replace(
|
||||
f'DEFAULT_LOCAL_ROOT = Path("{old_root}")',
|
||||
"DEFAULT_LOCAL_ROOT = Path(__file__).resolve().parents[1]",
|
||||
)
|
||||
portable_nas = (
|
||||
"DEFAULT_NAS_ROOT = Path(os.environ.get("
|
||||
'"UMBRA_NAS_ROOT", "/Volumes/Umbra/UmbraAudio"'
|
||||
")).expanduser()"
|
||||
)
|
||||
text = text.replace(
|
||||
'DEFAULT_NAS_ROOT = Path("/Volumes/Umbra/UmbraAudio")',
|
||||
portable_nas,
|
||||
)
|
||||
text = text.replace(
|
||||
'MOUNT_SCRIPT = Path.home() / ".umbra/mount_nas.sh"',
|
||||
'MOUNT_SCRIPT = _global_state_root() / "mount_nas.sh"',
|
||||
)
|
||||
text = text.replace(
|
||||
'CONFIG_PATH = Path.home() / ".umbra/nas_config.json"',
|
||||
'CONFIG_PATH = _global_state_root() / "nas_config.json"',
|
||||
)
|
||||
if "def _global_state_root()" not in text:
|
||||
global_root_function = (
|
||||
"def _global_state_root() -> Path:\n"
|
||||
" return Path(os.environ.get(\n"
|
||||
' "UMBRA_GLOBAL_STATE_DIR", str(Path.home() / ".umbra")\n'
|
||||
" )).expanduser()\n\n"
|
||||
"DEFAULT_LOCAL_ROOT ="
|
||||
)
|
||||
text = text.replace(
|
||||
"DEFAULT_LOCAL_ROOT =",
|
||||
global_root_function,
|
||||
1,
|
||||
)
|
||||
nas_lifecycle.write_text(text, encoding="utf-8")
|
||||
changes.append("scripts/nas_lifecycle.py")
|
||||
|
||||
sync_voice = root / "tts_server" / "sync_voice_data.sh"
|
||||
if sync_voice.exists():
|
||||
text = sync_voice.read_text(encoding="utf-8")
|
||||
text = text.replace(
|
||||
'UMBRA_DATA="${UMBRA_DATA_DIR:-/Volumes/1TB_APL_Storage/UmbraAudio}"',
|
||||
'PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"\n'
|
||||
'UMBRA_DATA="${UMBRA_DATA_DIR:-$PROJECT_ROOT}"',
|
||||
)
|
||||
sync_voice.write_text(text, encoding="utf-8")
|
||||
changes.append("tts_server/sync_voice_data.sh")
|
||||
|
||||
verify_nas = root / "scripts" / "verify_nas.py"
|
||||
if verify_nas.exists():
|
||||
text = verify_nas.read_text(encoding="utf-8")
|
||||
text = text.replace(
|
||||
f'Path("{old_root}")',
|
||||
"Path(__file__).resolve().parents[1]",
|
||||
)
|
||||
verify_nas.write_text(text, encoding="utf-8")
|
||||
changes.append("scripts/verify_nas.py")
|
||||
|
||||
embeddings = root / "scripts" / "extract_embeddings.py"
|
||||
if embeddings.exists():
|
||||
text = embeddings.read_text(encoding="utf-8")
|
||||
text = text.replace(
|
||||
f'"{old_root}/models/Qwen3-TTS-12Hz-1.7B-Base"',
|
||||
'str(project_root / "models/Qwen3-TTS-12Hz-1.7B-Base")',
|
||||
)
|
||||
text = text.replace(
|
||||
f'"{old_root}/models/Qwen3-TTS-12Hz-1.7B-VoiceDesign"',
|
||||
'str(project_root / "models/Qwen3-TTS-12Hz-1.7B-VoiceDesign")',
|
||||
)
|
||||
embeddings.write_text(text, encoding="utf-8")
|
||||
changes.append("scripts/extract_embeddings.py")
|
||||
|
||||
legacy_start = root / "start_server.sh"
|
||||
if legacy_start.exists():
|
||||
legacy_start.write_text(
|
||||
"#!/usr/bin/env bash\n"
|
||||
"set -Eeuo pipefail\n"
|
||||
"printf '%s\\n' 'Legacy Gradio demo launcher is disabled in this restore.'\n"
|
||||
"printf '%s\\n' 'Use scripts/start_web_server.sh or the generated service commands.'\n"
|
||||
"exit 1\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
legacy_start.chmod(0o755)
|
||||
changes.append("start_server.sh")
|
||||
|
||||
start_script = root / "scripts" / "start_web_server.sh"
|
||||
if start_script.exists():
|
||||
web_command = (
|
||||
'exec "$PROJ/.venv/bin/python" -c \''
|
||||
"from pathlib import Path; from umbraaudio.web import create_app; "
|
||||
"from umbraaudio.config import load_config; import uvicorn; "
|
||||
'c = load_config(Path(".")); uvicorn.run(create_app(c), '
|
||||
'host="0.0.0.0", port=8420, log_level="info")\'\n'
|
||||
)
|
||||
global_exports = ""
|
||||
if global_root is not None:
|
||||
global_exports = (
|
||||
f"export UMBRA_GLOBAL_STATE_DIR={shlex.quote(str(global_root))}\n"
|
||||
"export UMBRA_VOICE_REGISTRY_PATH="
|
||||
f"{shlex.quote(str(global_root / 'voice_registry.db'))}\n"
|
||||
)
|
||||
start_script.write_text(
|
||||
"#!/usr/bin/env bash\n"
|
||||
"set -Eeuo pipefail\n"
|
||||
'PROJ="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)"\n'
|
||||
'exec >> "$PROJ/logs/web_server.log" 2>&1\n'
|
||||
"echo \"[$(date '+%F %T')] web server starting (wrapper pid $$)\"\n"
|
||||
'cd "$PROJ"\n' + global_exports + web_command,
|
||||
encoding="utf-8",
|
||||
)
|
||||
start_script.chmod(0o755)
|
||||
changes.append("scripts/start_web_server.sh")
|
||||
|
||||
if old_root:
|
||||
for database in sorted(root.rglob("*.db")):
|
||||
if not database.is_file() or any(
|
||||
part in {".venv", ".python"} for part in database.parts
|
||||
):
|
||||
continue
|
||||
if not is_sqlite_file(database):
|
||||
continue
|
||||
migrate_project_database(database, old_root, str(new_root))
|
||||
|
||||
commands = root / "RESTORED-SERVICE-COMMANDS.md"
|
||||
web_command = (
|
||||
"python -c 'from pathlib import Path; from umbraaudio.web import create_app; "
|
||||
"from umbraaudio.config import load_config; import uvicorn; "
|
||||
'uvicorn.run(create_app(load_config(Path("."))), host="0.0.0.0", port=8420)\''
|
||||
)
|
||||
voicedesign_model = json.dumps(str(new_root / "models/Qwen3-TTS-12Hz-1.7B-VoiceDesign"))
|
||||
base_model = json.dumps(str(new_root / "models/Qwen3-TTS-12Hz-1.7B-Base"))
|
||||
global_exports = ""
|
||||
if global_root is not None:
|
||||
global_exports = (
|
||||
f"export UMBRA_GLOBAL_STATE_DIR={shlex.quote(str(global_root))}\n"
|
||||
"export UMBRA_VOICE_REGISTRY_PATH="
|
||||
f"{shlex.quote(str(global_root / 'voice_registry.db'))}\n"
|
||||
)
|
||||
commands.write_text(
|
||||
"# Restored UmbraAudio service commands\n\n"
|
||||
"These commands are generated for the restore location and are not installed "
|
||||
"automatically.\n\n"
|
||||
"```bash\n"
|
||||
f"cd {json.dumps(str(new_root))}\n"
|
||||
'export PATH="$PWD/.venv/bin:$PATH"\n'
|
||||
f"export UMBRA_PROJECT_BASE={json.dumps(str(new_root))}\n"
|
||||
+ global_exports
|
||||
+ f"{web_command}\n"
|
||||
f"QWEN_VOICEDESIGN_MODEL={voicedesign_model}\n"
|
||||
f"QWEN_BASE_MODEL={base_model}\n"
|
||||
"python scripts/qwen_tts_server.py\n"
|
||||
"```\n\n"
|
||||
"The external launchd plist is intentionally not installed automatically; review the "
|
||||
"context/ directory and adapt paths and permissions first.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "PORTABILITY-PATCH.txt").write_text(
|
||||
"The following restored-copy portability adjustments were applied:\n"
|
||||
+ "".join(f"- {item}\n" for item in sorted(set(changes)))
|
||||
+ "\nThe original .env was preserved as .env.archive-original.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def write_checksum_manifest(root: Path) -> Path:
|
||||
output = root / "SHA256SUMS"
|
||||
lines: list[str] = []
|
||||
for path in sorted(root.rglob("*")):
|
||||
if not path.is_file() or path == output:
|
||||
continue
|
||||
relative = path.relative_to(root).as_posix()
|
||||
lines.append(f"{sha256(path)} {relative}\n")
|
||||
output.write_text("".join(lines), encoding="utf-8")
|
||||
return output
|
||||
|
||||
|
||||
def validate_tree(root: Path, old_root: str | None = None) -> list[str]:
|
||||
errors: list[str] = []
|
||||
required = (root / "pyproject.toml", root / "umbraaudio", root / "Qwen3-TTS")
|
||||
for path in required:
|
||||
if not path.exists():
|
||||
errors.append(f"missing required path: {path}")
|
||||
model = root / "models" / "Qwen3-TTS-12Hz-1.7B-Base" / "model.safetensors"
|
||||
if not model.exists():
|
||||
errors.append(f"missing model: {model}")
|
||||
for database in sorted(root.rglob("*.db")):
|
||||
if not database.is_file():
|
||||
continue
|
||||
if database.stat().st_size == 0:
|
||||
continue
|
||||
if not is_sqlite_file(database):
|
||||
if any(
|
||||
part in {".mypy_cache", ".pytest_cache", ".ruff_cache"} for part in database.parts
|
||||
):
|
||||
continue
|
||||
errors.append(f"non-SQLite .db file: {database}")
|
||||
continue
|
||||
try:
|
||||
connection = sqlite3.connect(f"file:{database}?mode=ro", uri=True)
|
||||
result = connection.execute("PRAGMA quick_check").fetchone()[0]
|
||||
if old_root:
|
||||
try:
|
||||
assert_no_old_prefixes(connection, [old_root])
|
||||
except RuntimeError as exc:
|
||||
errors.append(str(exc))
|
||||
connection.close()
|
||||
if result != "ok":
|
||||
errors.append(f"SQLite quick_check failed: {database}: {result}")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
errors.append(f"SQLite check error: {database}: {exc}")
|
||||
if old_root:
|
||||
code_files = [
|
||||
root / "umbraaudio" / "config.py",
|
||||
root / "umbraaudio" / "library.py",
|
||||
root / "umbraaudio" / "storage.py",
|
||||
root / "scripts" / "nas_lifecycle.py",
|
||||
root / "scripts" / "verify_nas.py",
|
||||
root / "scripts" / "extract_embeddings.py",
|
||||
root / "scripts" / "qwen_tts_server.py",
|
||||
root / "tts_server" / "qwen_tts_server.py",
|
||||
root / "tts_server" / "sync_voice_data.sh",
|
||||
root / "start_server.sh",
|
||||
]
|
||||
for code_file in code_files:
|
||||
if code_file.exists() and old_root in code_file.read_text(encoding="utf-8"):
|
||||
errors.append(f"old project root remains in {code_file}")
|
||||
return errors
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
snapshot = sub.add_parser("snapshot-global")
|
||||
snapshot.add_argument("--source", type=Path, required=True)
|
||||
snapshot.add_argument("--output", type=Path, required=True)
|
||||
snapshot.add_argument("--original-root", type=Path, required=True)
|
||||
|
||||
restore = sub.add_parser("restore-global")
|
||||
restore.add_argument("--backup-dir", type=Path, required=True)
|
||||
restore.add_argument("--target", type=Path, required=True)
|
||||
restore.add_argument("--force", action="store_true")
|
||||
restore.add_argument("--old-project-root", default="")
|
||||
restore.add_argument("--new-project-root", default="")
|
||||
|
||||
patch = sub.add_parser("patch-paths")
|
||||
patch.add_argument("--root", type=Path, required=True)
|
||||
patch.add_argument("--old-root", required=True)
|
||||
patch.add_argument("--new-root", type=Path, required=True)
|
||||
patch.add_argument("--global-state-dir", type=Path)
|
||||
|
||||
manifest = sub.add_parser("write-manifest")
|
||||
manifest.add_argument("--root", type=Path, required=True)
|
||||
|
||||
validate = sub.add_parser("validate-tree")
|
||||
validate.add_argument("--root", type=Path, required=True)
|
||||
validate.add_argument("--old-root", default="")
|
||||
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
if args.command == "snapshot-global":
|
||||
snapshot_global(args.source, args.output, args.original_root)
|
||||
elif args.command == "restore-global":
|
||||
restore_global(
|
||||
args.backup_dir,
|
||||
args.target,
|
||||
args.force,
|
||||
args.old_project_root,
|
||||
args.new_project_root,
|
||||
)
|
||||
elif args.command == "patch-paths":
|
||||
patch_restored_paths(
|
||||
args.root,
|
||||
args.old_root,
|
||||
args.new_root,
|
||||
args.global_state_dir,
|
||||
)
|
||||
elif args.command == "write-manifest":
|
||||
write_checksum_manifest(args.root)
|
||||
elif args.command == "validate-tree":
|
||||
errors = validate_tree(args.root, args.old_root or None)
|
||||
if errors:
|
||||
for error in errors:
|
||||
print(error, file=sys.stderr)
|
||||
return 1
|
||||
print(f"Tree validation passed: {args.root}")
|
||||
return 0
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
4
infra/BOOTSTRAP-SHA256SUMS
Normal file
4
infra/BOOTSTRAP-SHA256SUMS
Normal file
@@ -0,0 +1,4 @@
|
||||
47df21afa85aa7bbb550fa6622b0b4fa3ec8ea1e100fecf47375db60990186ad bootstrap_enterprise_mint.sh
|
||||
bea205a4113e7b0ca7eb7b3da4344a1711143d111c4c13049698369506768f72 enterprise-bootstrap.env.example
|
||||
5a7fe225d6cc67fea080e392dc3c3b9d896800eeb3eb1e6391791485d68764ed README-enterprise-bootstrap.md
|
||||
deb8b51f9892ad5f3b50738514da4d59b2697b30ad356e1d1bdc81ae1c862749 postgres/001_control_plane.sql
|
||||
2
infra/DATABASE-SHA256SUMS
Normal file
2
infra/DATABASE-SHA256SUMS
Normal file
@@ -0,0 +1,2 @@
|
||||
d6f52246a3a6899d48d8ed115deb5b4661362e9b5695eb946b0e0dbd0ded03cf bootstrap_umbracontrol_database.sh
|
||||
deb8b51f9892ad5f3b50738514da4d59b2697b30ad356e1d1bdc81ae1c862749 postgres/001_control_plane.sql
|
||||
63
infra/README-enterprise-bootstrap.md
Normal file
63
infra/README-enterprise-bootstrap.md
Normal file
@@ -0,0 +1,63 @@
|
||||
# Umbra enterprise phase-one bootstrap
|
||||
|
||||
This directory contains the first, infrastructure-only bootstrap for the UmbraTrans enterprise control plane.
|
||||
|
||||
It is designed for Linux Mint 22.1 (Ubuntu 24.04-family) and is intentionally conservative:
|
||||
|
||||
- It does not modify Qwen.
|
||||
- It does not stop or alter Docker containers.
|
||||
- It does not change firewall rules.
|
||||
- It does not install application code or systemd application units yet.
|
||||
- It does not place PostgreSQL on SMB/NAS.
|
||||
- It creates database roles, a local database, core identity/tenancy tables, row-level-security policies, a one-time bootstrap administrator, and least-privilege service accounts.
|
||||
|
||||
## Files
|
||||
|
||||
- `bootstrap_enterprise_mint.sh` — root bootstrap and read-only inspection tool.
|
||||
- `enterprise-bootstrap.env.example` — non-secret configuration template.
|
||||
- `postgres/001_control_plane.sql` — initial identity, organization, project, role, invitation, session, MFA, bootstrap-token, audit, and RLS schema.
|
||||
|
||||
## Safe usage on oldbox
|
||||
|
||||
The script is intended to be copied to the `umbra` account and reviewed there:
|
||||
|
||||
```bash
|
||||
ssh umbra@oldbox.bdmurphy.me
|
||||
mkdir -p ~/umbra-enterprise-bootstrap
|
||||
# copy the three files into that directory
|
||||
cp enterprise-bootstrap.env.example enterprise-bootstrap.env
|
||||
$EDITOR enterprise-bootstrap.env
|
||||
bash -n bootstrap_enterprise_mint.sh
|
||||
./bootstrap_enterprise_mint.sh --check
|
||||
./bootstrap_enterprise_mint.sh --dry-run --config "$PWD/enterprise-bootstrap.env"
|
||||
sudo ./bootstrap_enterprise_mint.sh --apply --config /home/umbra/umbra-enterprise-bootstrap/enterprise-bootstrap.env
|
||||
```
|
||||
|
||||
The root password is entered only in the user's terminal. It is not stored in the configuration file or command line.
|
||||
|
||||
After a successful apply:
|
||||
|
||||
```bash
|
||||
sudo ./bootstrap_enterprise_mint.sh --verify --config /home/umbra/umbra-enterprise-bootstrap/enterprise-bootstrap.env
|
||||
```
|
||||
|
||||
The initial one-time administrator token is stored root-only at:
|
||||
|
||||
```text
|
||||
/root/umbra-enterprise-bootstrap.token
|
||||
```
|
||||
|
||||
Do not paste that token into chat. The application bootstrap flow will consume it and replace it with normal authentication.
|
||||
|
||||
## Deliberate exclusions
|
||||
|
||||
The script does not yet:
|
||||
|
||||
- create or install the UmbraTrans API/worker application;
|
||||
- install a reverse proxy or certificate;
|
||||
- expose Qwen to other machines;
|
||||
- add Qwen authentication or rate limiting;
|
||||
- modify UFW/nftables/iptables;
|
||||
- migrate existing Docker, MySQL, LM Studio, Ollama, or Qwen data.
|
||||
|
||||
Those are separate reviewed changes after the host and database foundation are verified.
|
||||
59
infra/agent_bridge/README.md
Normal file
59
infra/agent_bridge/README.md
Normal file
@@ -0,0 +1,59 @@
|
||||
# Temporary UmbraTrans agent bridge
|
||||
|
||||
This is a one-hour, LAN-only browser pairing point for communication between
|
||||
the Mac agent and a browser session on oldbox. It is not a remote shell.
|
||||
|
||||
## Properties
|
||||
|
||||
- Random OS-assigned TCP port.
|
||||
- HTTPS with a temporary self-signed certificate.
|
||||
- One-hour hard expiry.
|
||||
- Password is entered once in a native Mac setup dialog and once in the
|
||||
browser on the other machine.
|
||||
- Password is hashed in memory; plaintext is not logged or written to disk.
|
||||
- HttpOnly/Secure/SameSite session cookie and CSRF token.
|
||||
- No raw shell, no arbitrary command execution, no attachments.
|
||||
- Messages are atomic, bounded JSON files in a private queue.
|
||||
|
||||
The browser terminal is an agent message channel. The Mac agent must poll the
|
||||
inbox and send replies through the local bridge tooling; commands are not
|
||||
executed automatically from browser text.
|
||||
|
||||
## Start
|
||||
|
||||
The listener runs on the Mac's LAN address. The startup dialog asks for the
|
||||
one-hour pairing password. The process prints the exact URL and port.
|
||||
|
||||
Example:
|
||||
|
||||
```bash
|
||||
PYTHONPATH=/Volumes/1TB_APL_Storage/UmbraAudio/infra \
|
||||
/usr/bin/python3 -m agent_bridge \
|
||||
--host 192.168.123.12 \
|
||||
--ttl 3600
|
||||
```
|
||||
|
||||
Open the printed `https://192.168.123.12:<port>/` URL in a browser on oldbox.
|
||||
Accept the temporary certificate warning if the browser requires it, then pair
|
||||
with the same one-hour password.
|
||||
|
||||
## Queue
|
||||
|
||||
The default queue is:
|
||||
|
||||
```text
|
||||
~/.local/state/umbratrans/agent-bridge/
|
||||
inbox/ # browser messages waiting for the Mac agent
|
||||
outbox/ # Mac-agent replies waiting for the browser
|
||||
processed/ # browser messages already read
|
||||
delivered/ # browser replies acknowledged by the browser
|
||||
```
|
||||
|
||||
All files are mode `0600`; directories are mode `0700`.
|
||||
|
||||
## Safety
|
||||
|
||||
- Do not expose the listener to the public internet.
|
||||
- Keep the one-hour timer unchanged.
|
||||
- Do not put passwords or private keys in messages.
|
||||
- Stop the process after the session if the automatic timer is interrupted.
|
||||
3
infra/agent_bridge/__init__.py
Normal file
3
infra/agent_bridge/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
"""Temporary browser-to-agent bridge for UmbraTrans."""
|
||||
|
||||
__all__: list[str] = []
|
||||
3
infra/agent_bridge/__main__.py
Normal file
3
infra/agent_bridge/__main__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
from .server import main
|
||||
|
||||
raise SystemExit(main())
|
||||
31
infra/agent_bridge/client.py
Normal file
31
infra/agent_bridge/client.py
Normal file
@@ -0,0 +1,31 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from .server import MessageStore
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Read/write the local UmbraTrans agent bridge queue")
|
||||
parser.add_argument("action", choices=("poll", "send", "status"))
|
||||
parser.add_argument("--root", default="~/.local/state/umbratrans/agent-bridge")
|
||||
parser.add_argument("--title", default="Mac agent")
|
||||
parser.add_argument("--body", default="")
|
||||
args = parser.parse_args(argv)
|
||||
store = MessageStore(Path(args.root).expanduser())
|
||||
if args.action == "poll":
|
||||
print(json.dumps(store.poll_for_mac(20), ensure_ascii=False))
|
||||
elif args.action == "send":
|
||||
if not args.body:
|
||||
parser.error("--body is required for send")
|
||||
print(json.dumps(store.send_to_browser(args.body, args.title), ensure_ascii=False))
|
||||
else:
|
||||
print(json.dumps(store.status(), sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
490
infra/agent_bridge/server.py
Normal file
490
infra/agent_bridge/server.py
Normal file
@@ -0,0 +1,490 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import html
|
||||
import http.cookies
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import signal
|
||||
import ssl
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from http import HTTPStatus
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from remote_credential_broker.popup import macos_secret_prompt
|
||||
|
||||
MAX_BODY_BYTES = 128 * 1024
|
||||
MAX_MESSAGE_CHARS = 64 * 1024
|
||||
SESSION_TTL_SECONDS = 3600
|
||||
PAIR_ATTEMPT_LIMIT = 8
|
||||
PAIR_ATTEMPT_WINDOW = 60
|
||||
|
||||
|
||||
class BridgeError(RuntimeError):
|
||||
"""Base error for the temporary bridge."""
|
||||
|
||||
|
||||
class MessageStore:
|
||||
"""Atomic, non-secret file queue shared by the browser and local tooling."""
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = root
|
||||
self.inbox = root / "inbox"
|
||||
self.outbox = root / "outbox"
|
||||
self.processed = root / "processed"
|
||||
self.delivered = root / "delivered"
|
||||
for directory in (self.root, self.inbox, self.outbox, self.processed, self.delivered):
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
os.chmod(directory, 0o700)
|
||||
|
||||
@staticmethod
|
||||
def _safe_name(value: str) -> bool:
|
||||
return bool(value) and len(value) <= 80 and all(c.isalnum() or c in "-_" for c in value)
|
||||
|
||||
def _write(self, directory: Path, message: dict[str, Any]) -> str:
|
||||
message_id = str(message["id"])
|
||||
if not self._safe_name(message_id):
|
||||
raise BridgeError("invalid message id")
|
||||
target = directory / f"{message_id}.json"
|
||||
temporary = directory / f".{message_id}.{uuid.uuid4().hex}.tmp"
|
||||
payload = json.dumps(message, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
try:
|
||||
with os.fdopen(fd, "wb") as handle:
|
||||
handle.write(payload)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary, target)
|
||||
except BaseException:
|
||||
try:
|
||||
temporary.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
raise
|
||||
return str(target)
|
||||
|
||||
@staticmethod
|
||||
def _validate_message(message: dict[str, Any]) -> dict[str, Any]:
|
||||
required = {"schema_version", "id", "created_at", "sender", "kind", "title", "body"}
|
||||
if not required.issubset(message):
|
||||
raise BridgeError("message is missing required fields")
|
||||
if message["schema_version"] != 1:
|
||||
raise BridgeError("unsupported message schema")
|
||||
if not isinstance(message["body"], str) or len(message["body"]) > MAX_MESSAGE_CHARS:
|
||||
raise BridgeError("message body is too large")
|
||||
if not isinstance(message["title"], str) or len(message["title"]) > 200:
|
||||
raise BridgeError("message title is too large")
|
||||
if "\x00" in message["body"] or "\x00" in message["title"]:
|
||||
raise BridgeError("message contains NUL")
|
||||
return message
|
||||
|
||||
def send_to_mac(self, body: str, title: str = "Browser message") -> dict[str, Any]:
|
||||
message = self._validate_message(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"id": str(uuid.uuid4()),
|
||||
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"sender": "oldbox-browser",
|
||||
"kind": "instruction",
|
||||
"correlation_id": None,
|
||||
"title": title,
|
||||
"body": body,
|
||||
}
|
||||
)
|
||||
self._write(self.inbox, message)
|
||||
return message
|
||||
|
||||
def send_to_browser(self, body: str, title: str = "Mac agent") -> dict[str, Any]:
|
||||
message = self._validate_message(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"id": str(uuid.uuid4()),
|
||||
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"sender": "mac-agent",
|
||||
"kind": "report",
|
||||
"correlation_id": None,
|
||||
"title": title,
|
||||
"body": body,
|
||||
}
|
||||
)
|
||||
self._write(self.outbox, message)
|
||||
return message
|
||||
|
||||
def _read_dir(self, directory: Path, limit: int) -> list[dict[str, Any]]:
|
||||
values: list[dict[str, Any]] = []
|
||||
for path in sorted(directory.glob("*.json"))[:limit]:
|
||||
if not self._safe_name(path.stem):
|
||||
continue
|
||||
try:
|
||||
if not path.is_file() or path.is_symlink():
|
||||
continue
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
values.append(self._validate_message(value))
|
||||
except (OSError, json.JSONDecodeError, BridgeError):
|
||||
continue
|
||||
return values
|
||||
|
||||
def poll_for_mac(self, limit: int = 20) -> list[dict[str, Any]]:
|
||||
values = self._read_dir(self.inbox, max(1, min(limit, 100)))
|
||||
for value in values:
|
||||
source = self.inbox / f"{value['id']}.json"
|
||||
destination = self.processed / f"{value['id']}.json"
|
||||
try:
|
||||
os.replace(source, destination)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return values
|
||||
|
||||
def read_for_browser(self, limit: int = 50) -> list[dict[str, Any]]:
|
||||
return self._read_dir(self.outbox, max(1, min(limit, 100)))
|
||||
|
||||
def acknowledge(self, message_ids: list[str]) -> int:
|
||||
count = 0
|
||||
for message_id in message_ids[:100]:
|
||||
if not self._safe_name(message_id):
|
||||
continue
|
||||
source = self.outbox / f"{message_id}.json"
|
||||
destination = self.delivered / f"{message_id}.json"
|
||||
try:
|
||||
os.replace(source, destination)
|
||||
count += 1
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return count
|
||||
|
||||
def status(self) -> dict[str, int]:
|
||||
return {
|
||||
"inbox": len(list(self.inbox.glob("*.json"))),
|
||||
"outbox": len(list(self.outbox.glob("*.json"))),
|
||||
"processed": len(list(self.processed.glob("*.json"))),
|
||||
"delivered": len(list(self.delivered.glob("*.json"))),
|
||||
}
|
||||
|
||||
|
||||
class BridgeState:
|
||||
def __init__(self, password: str, ttl_seconds: int = SESSION_TTL_SECONDS) -> None:
|
||||
self.started_at = time.time()
|
||||
self.expires_at = self.started_at + ttl_seconds
|
||||
self.salt = os.urandom(32)
|
||||
self.digest = hashlib.pbkdf2_hmac(
|
||||
"sha256", password.encode("utf-8"), self.salt, 240_000
|
||||
)
|
||||
self.sessions: dict[str, dict[str, Any]] = {}
|
||||
self.attempts: dict[str, list[float]] = {}
|
||||
self.lock = threading.RLock()
|
||||
|
||||
def check_password(self, address: str, password: str) -> bool:
|
||||
now = time.time()
|
||||
with self.lock:
|
||||
recent = [stamp for stamp in self.attempts.get(address, []) if now - stamp < PAIR_ATTEMPT_WINDOW]
|
||||
if len(recent) >= PAIR_ATTEMPT_LIMIT:
|
||||
return False
|
||||
candidate = hashlib.pbkdf2_hmac(
|
||||
"sha256", password.encode("utf-8"), self.salt, 240_000
|
||||
)
|
||||
accepted = hmac.compare_digest(candidate, self.digest)
|
||||
if accepted:
|
||||
self.attempts.pop(address, None)
|
||||
else:
|
||||
recent.append(now)
|
||||
self.attempts[address] = recent
|
||||
return accepted
|
||||
|
||||
def new_session(self) -> tuple[str, str]:
|
||||
session = secrets.token_urlsafe(40)
|
||||
csrf = secrets.token_urlsafe(32)
|
||||
with self.lock:
|
||||
self.sessions[session] = {"csrf": csrf, "expires_at": self.expires_at}
|
||||
return session, csrf
|
||||
|
||||
def session(self, token: str | None) -> dict[str, Any] | None:
|
||||
if not token:
|
||||
return None
|
||||
with self.lock:
|
||||
value = self.sessions.get(token)
|
||||
if value is None or value["expires_at"] <= time.time():
|
||||
self.sessions.pop(token, None)
|
||||
return None
|
||||
return value
|
||||
|
||||
def remaining_seconds(self) -> int:
|
||||
return max(0, int(self.expires_at - time.time()))
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server_version = "UmbraAgentBridge/0.1"
|
||||
state: BridgeState
|
||||
store: MessageStore
|
||||
|
||||
def log_message(self, format: str, *args: Any) -> None:
|
||||
# Do not log request bodies, cookies, passwords, or message content.
|
||||
print(f"agent-bridge: {self.address_string()} {format % args}", file=sys.stderr)
|
||||
|
||||
def _headers(self, content_type: str = "text/plain; charset=utf-8", length: int | None = None) -> None:
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
self.send_header("X-Frame-Options", "DENY")
|
||||
self.send_header("Referrer-Policy", "no-referrer")
|
||||
if length is not None:
|
||||
self.send_header("Content-Length", str(length))
|
||||
|
||||
def _send(self, status: int, body: bytes, content_type: str = "text/plain; charset=utf-8") -> None:
|
||||
self.send_response(status)
|
||||
self._headers(content_type, len(body))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _json(self, status: int, value: dict[str, Any]) -> None:
|
||||
self._send(status, json.dumps(value).encode("utf-8"), "application/json; charset=utf-8")
|
||||
|
||||
def _cookie_token(self) -> str | None:
|
||||
raw = self.headers.get("Cookie", "")
|
||||
cookie = http.cookies.SimpleCookie()
|
||||
try:
|
||||
cookie.load(raw)
|
||||
except http.cookies.CookieError:
|
||||
return None
|
||||
morsel = cookie.get("bridge_session")
|
||||
return morsel.value if morsel else None
|
||||
|
||||
def _auth(self) -> tuple[dict[str, Any] | None, str | None]:
|
||||
session = self.state.session(self._cookie_token())
|
||||
if session is None:
|
||||
return None, "authentication required"
|
||||
csrf = self.headers.get("X-Bridge-CSRF", "")
|
||||
if self.command == "POST" and not hmac.compare_digest(csrf, session["csrf"]):
|
||||
return None, "CSRF validation failed"
|
||||
return session, None
|
||||
|
||||
def _read_json(self) -> dict[str, Any]:
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
if length <= 0 or length > MAX_BODY_BYTES:
|
||||
raise BridgeError("invalid request size")
|
||||
value = json.loads(self.rfile.read(length).decode("utf-8"))
|
||||
if not isinstance(value, dict):
|
||||
raise BridgeError("request must be a JSON object")
|
||||
return value
|
||||
|
||||
def do_GET(self) -> None:
|
||||
path = urlparse(self.path).path
|
||||
if path == "/healthz":
|
||||
self._json(200, {"status": "ok", "paired_sessions": len(self.state.sessions)})
|
||||
return
|
||||
if path == "/":
|
||||
if self.state.session(self._cookie_token()) is None:
|
||||
self._send(200, LOGIN_HTML.encode("utf-8"), "text/html; charset=utf-8")
|
||||
else:
|
||||
self._send(200, TERMINAL_HTML.encode("utf-8"), "text/html; charset=utf-8")
|
||||
return
|
||||
if path == "/api/status":
|
||||
session, error = self._auth()
|
||||
if error:
|
||||
self._json(401, {"error": error})
|
||||
return
|
||||
self._json(200, {"expires_in": self.state.remaining_seconds(), "queue": self.store.status()})
|
||||
return
|
||||
if path == "/api/messages":
|
||||
session, error = self._auth()
|
||||
if error:
|
||||
self._json(401, {"error": error})
|
||||
return
|
||||
query = parse_qs(urlparse(self.path).query)
|
||||
try:
|
||||
limit = int(query.get("limit", ["50"])[0])
|
||||
except ValueError:
|
||||
limit = 50
|
||||
self._json(200, {"messages": self.store.read_for_browser(limit)})
|
||||
return
|
||||
self._json(404, {"error": "not found"})
|
||||
|
||||
def do_POST(self) -> None:
|
||||
path = urlparse(self.path).path
|
||||
if path == "/api/pair":
|
||||
try:
|
||||
value = self._read_json()
|
||||
password = value.get("password")
|
||||
if not isinstance(password, str) or not self.state.check_password(
|
||||
self.client_address[0], password
|
||||
):
|
||||
self._json(401, {"error": "pairing failed"})
|
||||
return
|
||||
session, csrf = self.state.new_session()
|
||||
body = json.dumps({"paired": True, "csrf": csrf, "expires_in": self.state.remaining_seconds()}).encode()
|
||||
self.send_response(200)
|
||||
self._headers("application/json; charset=utf-8", len(body))
|
||||
self.send_header(
|
||||
"Set-Cookie",
|
||||
f"bridge_session={session}; HttpOnly; Secure; SameSite=Strict; Path=/; Max-Age={self.state.remaining_seconds()}",
|
||||
)
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
except (BridgeError, ValueError, json.JSONDecodeError):
|
||||
self._json(400, {"error": "invalid pairing request"})
|
||||
return
|
||||
if path == "/api/messages":
|
||||
session, error = self._auth()
|
||||
if error:
|
||||
self._json(401, {"error": error})
|
||||
return
|
||||
try:
|
||||
value = self._read_json()
|
||||
body = value.get("body")
|
||||
title = value.get("title", "Browser message")
|
||||
if not isinstance(body, str) or not isinstance(title, str):
|
||||
raise BridgeError("body and title must be strings")
|
||||
message = self.store.send_to_mac(body, title)
|
||||
self._json(202, {"id": message["id"], "queued": True})
|
||||
except (BridgeError, ValueError, json.JSONDecodeError):
|
||||
self._json(400, {"error": "invalid message"})
|
||||
return
|
||||
self._json(404, {"error": "not found"})
|
||||
|
||||
|
||||
LOGIN_HTML = r"""<!doctype html>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>UmbraTrans Temporary Agent Bridge</title>
|
||||
<style>
|
||||
body{font:16px -apple-system,BlinkMacSystemFont,sans-serif;background:#111;color:#eee;max-width:720px;margin:10vh auto;padding:24px}
|
||||
input,button{font:inherit;padding:12px;border-radius:6px;border:1px solid #555;background:#222;color:#eee}
|
||||
input{width:min(90%,420px)}button{background:#2866c7;color:white;cursor:pointer;margin-left:8px}
|
||||
small{color:#aaa}p{line-height:1.4}
|
||||
</style>
|
||||
<h1>UmbraTrans temporary bridge</h1>
|
||||
<p>This session expires after one hour. Enter the temporary pairing password supplied by the Mac operator.</p>
|
||||
<form id="pair"><input id="password" type="password" autocomplete="off" required><button>Pair</button></form>
|
||||
<p id="status"></p>
|
||||
<script>
|
||||
const form=document.querySelector('#pair'), status=document.querySelector('#status');
|
||||
form.addEventListener('submit',async e=>{e.preventDefault();status.textContent='Pairing…';const r=await fetch('/api/pair',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({password:document.querySelector('#password').value})});if(r.ok){sessionStorage.setItem('bridgeCsrf',(await r.json()).csrf);location.reload()}else{status.textContent='Pairing failed';form.reset();}});
|
||||
</script>
|
||||
"""
|
||||
|
||||
TERMINAL_HTML = r"""<!doctype html>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>UmbraTrans Agent Terminal</title>
|
||||
<style>
|
||||
body{font:15px ui-monospace,SFMono-Regular,Menlo,monospace;background:#0b0d10;color:#d7e0ea;max-width:1100px;margin:4vh auto;padding:20px}
|
||||
#log{height:65vh;overflow:auto;background:#11151a;border:1px solid #303943;padding:14px;white-space:pre-wrap}
|
||||
form{display:flex;gap:8px;margin-top:12px}input,button{font:inherit;padding:10px;border:1px solid #3c4652;border-radius:5px;background:#171c22;color:#eee}input{flex:1}button{background:#2866c7;color:white;cursor:pointer}
|
||||
small{color:#9ba8b5}
|
||||
</style>
|
||||
<h1>UmbraTrans agent terminal</h1>
|
||||
<small>Paired local session. Messages are queued for the Mac agent; this is not a raw shell.</small>
|
||||
<div id="log"></div>
|
||||
<form id="send"><input id="body" autocomplete="off" placeholder="Describe the next task or question…" required><button>Send</button></form>
|
||||
<script>
|
||||
const csrf=sessionStorage.getItem('bridgeCsrf'), log=document.querySelector('#log');
|
||||
function show(m){const who=m.sender==='mac-agent'?'Mac agent':'Oldbox browser';log.textContent+=`[${m.created_at}] ${who}: ${m.body}\n`;log.scrollTop=log.scrollHeight;}
|
||||
async function poll(){const r=await fetch('/api/messages',{credentials:'include'});if(r.ok){const d=await r.json();for(const m of d.messages)show(m);if(d.messages.length){await fetch('/api/messages',{method:'POST',credentials:'include',headers:{'Content-Type':'application/json','X-Bridge-CSRF':csrf},body:JSON.stringify({title:'ack',body:'acknowledged'})});}}}
|
||||
setInterval(poll,2000);poll();
|
||||
document.querySelector('#send').addEventListener('submit',async e=>{e.preventDefault();const input=document.querySelector('#body');const r=await fetch('/api/messages',{method:'POST',credentials:'include',headers:{'Content-Type':'application/json','X-Bridge-CSRF':csrf},body:JSON.stringify({title:'Browser task',body:input.value})});if(r.ok){log.textContent+=`[sent] ${input.value}\n`;input.value=''}else{log.textContent+='[error] message not queued\n'}});
|
||||
</script>
|
||||
"""
|
||||
|
||||
|
||||
def make_certificate(directory: Path, address: str) -> tuple[Path, Path]:
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
os.chmod(directory, 0o700)
|
||||
key = directory / "bridge-key.pem"
|
||||
cert = directory / "bridge-cert.pem"
|
||||
if key.exists() and cert.exists():
|
||||
return key, cert
|
||||
subprocess.run(
|
||||
[
|
||||
"openssl",
|
||||
"req",
|
||||
"-x509",
|
||||
"-newkey",
|
||||
"rsa:2048",
|
||||
"-nodes",
|
||||
"-keyout",
|
||||
str(key),
|
||||
"-out",
|
||||
str(cert),
|
||||
"-days",
|
||||
"1",
|
||||
"-subj",
|
||||
f"/CN={address}",
|
||||
"-addext",
|
||||
f"subjectAltName=IP:{address}",
|
||||
],
|
||||
check=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
os.chmod(key, 0o600)
|
||||
os.chmod(cert, 0o644)
|
||||
return key, cert
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Temporary UmbraTrans agent bridge")
|
||||
parser.add_argument("--host", default="192.168.123.12")
|
||||
parser.add_argument("--port", type=int, default=0)
|
||||
parser.add_argument("--root", default="~/.local/state/umbratrans/agent-bridge")
|
||||
parser.add_argument("--ttl", type=int, default=SESSION_TTL_SECONDS)
|
||||
args = parser.parse_args(argv)
|
||||
if not 1 <= args.ttl <= 3600:
|
||||
parser.error("--ttl must be between 1 and 3600")
|
||||
password = macos_secret_prompt("agent-bridge-temp", "one-hour agent bridge pairing password")
|
||||
if not password:
|
||||
print("agent bridge: pairing password entry cancelled", file=sys.stderr)
|
||||
return 2
|
||||
state = BridgeState(password, args.ttl)
|
||||
del password
|
||||
store = MessageStore(Path(args.root).expanduser())
|
||||
Handler.state = state
|
||||
Handler.store = store
|
||||
server = ThreadingHTTPServer((args.host, args.port), Handler)
|
||||
tls_dir = Path(tempfile.mkdtemp(prefix="umbra-agent-bridge-tls-"))
|
||||
key, cert = make_certificate(tls_dir, args.host)
|
||||
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
context.load_cert_chain(certfile=str(cert), keyfile=str(key))
|
||||
server.socket = context.wrap_socket(server.socket, server_side=True)
|
||||
port = server.server_address[1]
|
||||
print(f"BRIDGE_URL=https://{args.host}:{port}/", flush=True)
|
||||
print(f"BRIDGE_ROOT={store.root}", flush=True)
|
||||
print(f"BRIDGE_EXPIRES_IN={state.remaining_seconds()}", flush=True)
|
||||
|
||||
def stop(*_: Any) -> None:
|
||||
threading.Thread(target=server.shutdown, daemon=True).start()
|
||||
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
timer = threading.Timer(args.ttl, stop)
|
||||
timer.daemon = True
|
||||
timer.start()
|
||||
try:
|
||||
server.serve_forever(poll_interval=0.5)
|
||||
finally:
|
||||
server.server_close()
|
||||
timer.cancel()
|
||||
for path in (key, cert):
|
||||
try:
|
||||
path.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
try:
|
||||
tls_dir.rmdir()
|
||||
except OSError:
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
50
infra/agent_bridge/tests/test_bridge.py
Normal file
50
infra/agent_bridge/tests/test_bridge.py
Normal file
@@ -0,0 +1,50 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
from agent_bridge.server import BridgeError, BridgeState, MessageStore
|
||||
|
||||
|
||||
class BridgeTests(unittest.TestCase):
|
||||
def test_password_hashing_and_pairing(self) -> None:
|
||||
state = BridgeState("correct-value", ttl_seconds=60)
|
||||
self.assertTrue(state.check_password("192.0.2.1", "correct-value"))
|
||||
self.assertFalse(state.check_password("192.0.2.1", "wrong-value"))
|
||||
session, csrf = state.new_session()
|
||||
self.assertIsNotNone(state.session(session))
|
||||
self.assertTrue(csrf)
|
||||
self.assertIsNone(state.session("missing"))
|
||||
|
||||
def test_message_queue_is_atomic_and_bounded(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = MessageStore(Path(directory))
|
||||
message = store.send_to_mac("hello", "test")
|
||||
self.assertEqual(store.status()["inbox"], 1)
|
||||
self.assertEqual(store.poll_for_mac()[0]["body"], "hello")
|
||||
self.assertEqual(store.status()["processed"], 1)
|
||||
reply = store.send_to_browser("reply", "test")
|
||||
self.assertEqual(store.read_for_browser()[0]["id"], reply["id"])
|
||||
self.assertEqual(store.acknowledge([reply["id"]]), 1)
|
||||
self.assertEqual(store.status()["delivered"], 1)
|
||||
|
||||
def test_message_size_is_limited(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = MessageStore(Path(directory))
|
||||
with self.assertRaises(BridgeError):
|
||||
store.send_to_mac("x" * (64 * 1024 + 1))
|
||||
|
||||
def test_invalid_message_file_is_ignored(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = MessageStore(Path(directory))
|
||||
(store.inbox / "bad.json").write_text(json.dumps({"not": "a message"}))
|
||||
self.assertEqual(store.poll_for_mac(), [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
511
infra/bootstrap_enterprise_mint.sh
Executable file
511
infra/bootstrap_enterprise_mint.sh
Executable file
@@ -0,0 +1,511 @@
|
||||
#!/usr/bin/env bash
|
||||
# Phase-one enterprise control-plane bootstrap for Linux Mint 22.x.
|
||||
# This provisions PostgreSQL, database roles, core tenancy/identity tables,
|
||||
# and least-privilege service accounts. It does not install application code,
|
||||
# change Qwen, manage Docker, or change firewall rules.
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
CONFIG_FILE="${UMBRA_BOOTSTRAP_CONFIG:-${SCRIPT_DIR}/enterprise-bootstrap.env}"
|
||||
MODE="check"
|
||||
DRY_RUN=0
|
||||
LOCK_DIR="/run/lock/umbra-enterprise-bootstrap.lock"
|
||||
LOG_FILE="/var/log/umbra-enterprise-bootstrap.log"
|
||||
|
||||
UMBRA_HOSTNAME="oldbox.bdmurphy.me"
|
||||
UMBRA_PUBLIC_URL="https://oldbox.bdmurphy.me"
|
||||
UMBRA_DB_NAME="umbracontrol"
|
||||
UMBRA_DB_HOST="127.0.0.1"
|
||||
UMBRA_DB_PORT="5432"
|
||||
UMBRA_PG_VERSION="16"
|
||||
UMBRA_APP_ROOT="/opt/umbracontrol"
|
||||
UMBRA_DATA_ROOT="/var/lib/umbracontrol"
|
||||
UMBRA_BACKUP_ROOT="/var/backups/umbracontrol"
|
||||
UMBRA_BOOTSTRAP_ADMIN_EMAIL="umbratools@gmail.com"
|
||||
UMBRA_ORGANIZATION_NAME="Umbra"
|
||||
UMBRA_ORGANIZATION_SLUG="umbra"
|
||||
UMBRA_QWEN_URL="http://192.168.123.14:8881"
|
||||
UMBRA_INSTALL_PACKAGES=1
|
||||
UMBRA_INSTALL_UV=1
|
||||
UMBRA_CREATE_BOOTSTRAP_ADMIN=1
|
||||
UMBRA_CONFIG_DIR="/etc/umbracontrol"
|
||||
UMBRA_CREDENTIALS_FILE="/etc/umbracontrol/db-credentials.env"
|
||||
UMBRA_BOOTSTRAP_TOKEN_FILE="/root/umbra-enterprise-bootstrap.token"
|
||||
UMBRA_PG_SUPERUSER="postgres"
|
||||
UMBRA_SERVICE_GROUP="umbracontrol"
|
||||
UMBRA_API_USER="umbracontrol_api"
|
||||
UMBRA_WORKER_USER="umbracontrol_worker"
|
||||
UMBRA_READONLY_USER="umbracontrol_readonly"
|
||||
UMBRA_BACKUP_USER="umbracontrol_backup"
|
||||
UMBRA_MIGRATOR_USER="umbracontrol_migrator"
|
||||
UMBRA_OWNER_ROLE="umbracontrol_owner"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: bootstrap_enterprise_mint.sh [options]
|
||||
|
||||
Options:
|
||||
--check Read-only host and dependency inspection (default)
|
||||
--dry-run Show apply actions without changing the host
|
||||
--apply Apply the phase-one infrastructure bootstrap; requires root
|
||||
--status Show installation/service/database status
|
||||
--verify Verify installed services and database connectivity
|
||||
--config FILE Read non-secret KEY=VALUE configuration
|
||||
-h, --help Show this help
|
||||
|
||||
This phase intentionally does not install application code, modify Qwen,
|
||||
manage Docker containers, or change firewall rules. Run it first on Mint,
|
||||
then deploy the application and systemd units in a separate reviewed step.
|
||||
EOF
|
||||
}
|
||||
|
||||
while (($#)); do
|
||||
case "$1" in
|
||||
--check) MODE="check"; shift ;;
|
||||
--dry-run) MODE="apply"; DRY_RUN=1; shift ;;
|
||||
--apply) MODE="apply"; DRY_RUN=0; shift ;;
|
||||
--status) MODE="status"; shift ;;
|
||||
--verify) MODE="verify"; shift ;;
|
||||
--config) CONFIG_FILE="${2:?--config requires a file}"; shift 2 ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) printf 'Unknown option: %s\n' "$1" >&2; usage >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
log() {
|
||||
local line
|
||||
line="[$(date -Is)] $*"
|
||||
printf '%s\n' "$line"
|
||||
if [[ -f "$LOG_FILE" && "$(id -u)" -eq 0 ]]; then
|
||||
printf '%s\n' "$line" >> "$LOG_FILE"
|
||||
fi
|
||||
}
|
||||
|
||||
die() {
|
||||
printf 'ERROR: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_command() {
|
||||
command -v "$1" >/dev/null 2>&1 || die "Required command is missing: $1"
|
||||
}
|
||||
|
||||
require_root() {
|
||||
[[ "$(id -u)" -eq 0 ]] || die "This operation requires root. Use --check for a read-only inspection."
|
||||
}
|
||||
|
||||
load_config() {
|
||||
[[ -f "$CONFIG_FILE" ]] || {
|
||||
[[ "$MODE" == "check" || "$MODE" == "status" || "$MODE" == "verify" ]] && return 0
|
||||
die "Configuration file not found: $CONFIG_FILE"
|
||||
}
|
||||
|
||||
local line key value
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
line="${line#"${line%%[![:space:]]*}"}"
|
||||
[[ -z "$line" || "${line:0:1}" == "#" ]] && continue
|
||||
[[ "$line" == *=* ]] || die "Invalid configuration line (expected KEY=VALUE): $line"
|
||||
key="${line%%=*}"
|
||||
value="${line#*=}"
|
||||
key="${key//[[:space:]]/}"
|
||||
value="${value%\"}"; value="${value#\"}"
|
||||
value="${value%\'}"; value="${value#\'}"
|
||||
case "$key" in
|
||||
UMBRA_HOSTNAME|UMBRA_PUBLIC_URL|UMBRA_DB_NAME|UMBRA_DB_HOST|UMBRA_DB_PORT|UMBRA_PG_VERSION|UMBRA_APP_ROOT|UMBRA_DATA_ROOT|UMBRA_BACKUP_ROOT|UMBRA_BOOTSTRAP_ADMIN_EMAIL|UMBRA_ORGANIZATION_NAME|UMBRA_ORGANIZATION_SLUG|UMBRA_QWEN_URL|UMBRA_INSTALL_PACKAGES|UMBRA_INSTALL_UV|UMBRA_CREATE_BOOTSTRAP_ADMIN)
|
||||
printf -v "$key" '%s' "$value"
|
||||
;;
|
||||
*) die "Unsupported configuration key: $key" ;;
|
||||
esac
|
||||
done < "$CONFIG_FILE"
|
||||
}
|
||||
|
||||
validate_config() {
|
||||
[[ "$UMBRA_HOSTNAME" =~ ^[A-Za-z0-9.-]+$ ]] || die "Invalid UMBRA_HOSTNAME"
|
||||
[[ "$UMBRA_PUBLIC_URL" =~ ^https?://[A-Za-z0-9._:-]+(/.*)?$ ]] || die "Invalid UMBRA_PUBLIC_URL"
|
||||
[[ "$UMBRA_DB_NAME" =~ ^[a-z_][a-z0-9_]*$ ]] || die "Invalid UMBRA_DB_NAME"
|
||||
[[ "$UMBRA_DB_HOST" =~ ^[A-Za-z0-9._:-]+$ ]] || die "Invalid UMBRA_DB_HOST"
|
||||
[[ "$UMBRA_DB_PORT" =~ ^[0-9]+$ && "$UMBRA_DB_PORT" -ge 1 && "$UMBRA_DB_PORT" -le 65535 ]] || die "Invalid UMBRA_DB_PORT"
|
||||
[[ "$UMBRA_PG_VERSION" =~ ^[0-9]+$ ]] || die "Invalid UMBRA_PG_VERSION"
|
||||
[[ "$UMBRA_APP_ROOT" == /* && "$UMBRA_DATA_ROOT" == /* && "$UMBRA_BACKUP_ROOT" == /* ]] || die "Application/data paths must be absolute"
|
||||
[[ "$UMBRA_BOOTSTRAP_ADMIN_EMAIL" == *@*.* ]] || die "Invalid administrator email"
|
||||
[[ "$UMBRA_ORGANIZATION_SLUG" =~ ^[a-z0-9][a-z0-9-]*$ ]] || die "Invalid organization slug"
|
||||
[[ "$UMBRA_QWEN_URL" =~ ^https?:// ]] || die "Invalid UMBRA_QWEN_URL"
|
||||
for value in "$UMBRA_INSTALL_PACKAGES" "$UMBRA_INSTALL_UV" "$UMBRA_CREATE_BOOTSTRAP_ADMIN"; do
|
||||
[[ "$value" == 0 || "$value" == 1 ]] || die "Boolean configuration values must be 0 or 1"
|
||||
done
|
||||
}
|
||||
|
||||
run() {
|
||||
if ((DRY_RUN)); then
|
||||
printf '[dry-run]'
|
||||
printf ' %q' "$@"
|
||||
printf '\n'
|
||||
else
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
run_root() {
|
||||
require_root
|
||||
run "$@"
|
||||
}
|
||||
|
||||
read_secret() {
|
||||
local key="$1"
|
||||
[[ -f "$UMBRA_CREDENTIALS_FILE" ]] || return 1
|
||||
awk -F= -v key="$key" '$1 == key {print substr($0, index($0, "=") + 1); exit}' "$UMBRA_CREDENTIALS_FILE"
|
||||
}
|
||||
|
||||
write_secret_line() {
|
||||
local key="$1" value="$2" temporary
|
||||
if ((DRY_RUN)); then
|
||||
printf '[dry-run] store generated secret %s in %s\n' "$key" "$UMBRA_CREDENTIALS_FILE"
|
||||
return
|
||||
fi
|
||||
install -d -m 0750 -o root -g "$UMBRA_SERVICE_GROUP" "$UMBRA_CONFIG_DIR"
|
||||
touch "$UMBRA_CREDENTIALS_FILE"
|
||||
chown root:"$UMBRA_SERVICE_GROUP" "$UMBRA_CREDENTIALS_FILE"
|
||||
chmod 0640 "$UMBRA_CREDENTIALS_FILE"
|
||||
temporary="$(mktemp "${UMBRA_CONFIG_DIR}/.credentials.XXXXXX")"
|
||||
awk -F= -v key="$key" -v value="$value" '
|
||||
$1 == key { print key "=" value; found = 1; next }
|
||||
{ print }
|
||||
END { if (!found) print key "=" value }
|
||||
' "$UMBRA_CREDENTIALS_FILE" > "$temporary"
|
||||
chown root:"$UMBRA_SERVICE_GROUP" "$temporary"
|
||||
chmod 0640 "$temporary"
|
||||
mv -f "$temporary" "$UMBRA_CREDENTIALS_FILE"
|
||||
}
|
||||
|
||||
random_secret() {
|
||||
openssl rand -hex 32
|
||||
}
|
||||
|
||||
check_host() {
|
||||
log "Read-only host inspection"
|
||||
printf 'hostname: '; hostname -f 2>/dev/null || hostname
|
||||
printf 'os: '; . /etc/os-release; printf '%s\n' "${PRETTY_NAME:-unknown}"
|
||||
printf 'kernel: '; uname -r
|
||||
printf 'user: '; id
|
||||
printf 'cpu: '; nproc
|
||||
free -h | sed -n '1,3p'
|
||||
if command -v nvidia-smi >/dev/null 2>&1; then
|
||||
nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader 2>/dev/null || true
|
||||
fi
|
||||
printf '%s\n' 'storage:'
|
||||
df -hT / /home /var/lib 2>/dev/null || true
|
||||
printf '%s\n' 'listening ports:'
|
||||
ss -lntup 2>/dev/null | sed -n '1,100p' || true
|
||||
printf '%s\n' 'relevant services:'
|
||||
systemctl list-units --type=service --all --no-pager 2>/dev/null | grep -Ei 'docker|ollama|lmstudio|qwen|postgres|umbra' || true
|
||||
printf '%s\n' 'DNS:'
|
||||
getent ahosts "$UMBRA_HOSTNAME" 2>/dev/null || true
|
||||
log "No changes made"
|
||||
}
|
||||
|
||||
check_tools() {
|
||||
require_command openssl
|
||||
require_command systemctl
|
||||
if [[ "$MODE" == "apply" && "$UMBRA_INSTALL_PACKAGES" -eq 1 ]]; then
|
||||
require_command apt-get
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_group_and_users() {
|
||||
log "Ensuring service group and non-login service accounts"
|
||||
if ! getent group "$UMBRA_SERVICE_GROUP" >/dev/null; then
|
||||
run groupadd --system "$UMBRA_SERVICE_GROUP"
|
||||
else
|
||||
log "Group already exists: $UMBRA_SERVICE_GROUP"
|
||||
fi
|
||||
|
||||
ensure_service_user "$UMBRA_API_USER"
|
||||
ensure_service_user "$UMBRA_WORKER_USER"
|
||||
ensure_service_user "$UMBRA_READONLY_USER"
|
||||
ensure_service_user "$UMBRA_BACKUP_USER"
|
||||
}
|
||||
|
||||
ensure_service_user() {
|
||||
local user="$1"
|
||||
if id "$user" >/dev/null 2>&1; then
|
||||
log "Service account already exists: $user"
|
||||
return
|
||||
fi
|
||||
run useradd --system --gid "$UMBRA_SERVICE_GROUP" --home-dir /nonexistent --no-create-home --shell /usr/sbin/nologin "$user"
|
||||
}
|
||||
|
||||
ensure_directories() {
|
||||
log "Creating application, data, backup, and configuration directories"
|
||||
run install -d -m 0755 -o root -g root "$UMBRA_APP_ROOT"
|
||||
run install -d -m 0750 -o root -g "$UMBRA_SERVICE_GROUP" "$UMBRA_DATA_ROOT"
|
||||
run install -d -m 0700 -o root -g root "$UMBRA_BACKUP_ROOT"
|
||||
run install -d -m 0750 -o root -g "$UMBRA_SERVICE_GROUP" "$UMBRA_CONFIG_DIR"
|
||||
run install -d -m 0750 -o root -g "$UMBRA_SERVICE_GROUP" "$UMBRA_APP_ROOT/app" "$UMBRA_APP_ROOT/venv" "$UMBRA_APP_ROOT/bin"
|
||||
}
|
||||
|
||||
install_packages() {
|
||||
[[ "$UMBRA_INSTALL_PACKAGES" -eq 1 ]] || { log "Package installation disabled by configuration"; return; }
|
||||
log "Installing Mint/Ubuntu packages"
|
||||
run env DEBIAN_FRONTEND=noninteractive apt-get update
|
||||
run env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl openssl postgresql postgresql-client postgresql-contrib \
|
||||
python3 python3-venv python3-pip rsync
|
||||
}
|
||||
|
||||
ensure_uv() {
|
||||
[[ "$UMBRA_INSTALL_UV" -eq 1 ]] || { log "uv installation disabled by configuration"; return; }
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
log "uv already available: $(command -v uv)"
|
||||
return
|
||||
fi
|
||||
log "Installing pinned uv bootstrap environment"
|
||||
run python3 -m venv "$UMBRA_APP_ROOT/bootstrap-venv"
|
||||
run "$UMBRA_APP_ROOT/bootstrap-venv/bin/python" -m pip install --upgrade pip
|
||||
run "$UMBRA_APP_ROOT/bootstrap-venv/bin/python" -m pip install 'uv==0.11.7'
|
||||
run ln -sfn "$UMBRA_APP_ROOT/bootstrap-venv/bin/uv" "$UMBRA_APP_ROOT/bin/uv"
|
||||
}
|
||||
|
||||
psql_as_postgres() {
|
||||
runuser -u "$UMBRA_PG_SUPERUSER" -- env PGOPTIONS='-c log_statement=none' \
|
||||
psql -v ON_ERROR_STOP=1 "$@"
|
||||
}
|
||||
|
||||
ensure_postgres_cluster() {
|
||||
log "Ensuring PostgreSQL cluster and service"
|
||||
local version cluster
|
||||
if command -v pg_lsclusters >/dev/null 2>&1; then
|
||||
version="$(pg_lsclusters --no-header 2>/dev/null | awk 'NR == 1 {print $1}')"
|
||||
cluster="$(pg_lsclusters --no-header 2>/dev/null | awk 'NR == 1 {print $2}')"
|
||||
else
|
||||
version=""
|
||||
cluster=""
|
||||
fi
|
||||
if [[ -z "$version" || -z "$cluster" ]]; then
|
||||
version="$UMBRA_PG_VERSION"
|
||||
cluster="main"
|
||||
run pg_createcluster "$version" "$cluster" --start
|
||||
fi
|
||||
if command -v pg_lsclusters >/dev/null 2>&1; then
|
||||
if ! pg_lsclusters --no-header | awk -v v="$version" -v c="$cluster" '$1 == v && $2 == c && $3 == "online" {found=1} END {exit !found}'; then
|
||||
run pg_ctlcluster "$version" "$cluster" start
|
||||
fi
|
||||
else
|
||||
log "pg_lsclusters is not installed; cluster start will be performed after package installation"
|
||||
fi
|
||||
run systemctl enable postgresql
|
||||
}
|
||||
|
||||
ensure_database_roles() {
|
||||
log "Creating PostgreSQL roles"
|
||||
local role password login
|
||||
for role in \
|
||||
"$UMBRA_OWNER_ROLE" \
|
||||
"$UMBRA_MIGRATOR_USER" \
|
||||
"$UMBRA_API_USER" \
|
||||
"$UMBRA_WORKER_USER" \
|
||||
"$UMBRA_READONLY_USER" \
|
||||
"$UMBRA_BACKUP_USER"; do
|
||||
password=""
|
||||
login="LOGIN"
|
||||
[[ "$role" == "$UMBRA_OWNER_ROLE" ]] && login="NOLOGIN"
|
||||
if [[ "$role" != "$UMBRA_OWNER_ROLE" ]]; then
|
||||
if ((DRY_RUN)); then
|
||||
password="$(random_secret)"
|
||||
else
|
||||
password="$(read_secret "${role^^}_PASSWORD" 2>/dev/null || true)"
|
||||
[[ -n "$password" ]] || password="$(random_secret)"
|
||||
fi
|
||||
write_secret_line "${role^^}_PASSWORD" "$password"
|
||||
fi
|
||||
|
||||
if ((DRY_RUN)); then
|
||||
printf '[dry-run] ensure PostgreSQL role %s (%s)\n' "$role" "$login"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ -n "$password" ]]; then
|
||||
printf 'DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = %s) THEN CREATE ROLE %s %s PASSWORD %s; ELSE ALTER ROLE %s %s PASSWORD %s; END IF; END $$;\n' \
|
||||
"'$role'" "$role" "$login" "'$password'" "$role" "$login" "'$password'" \
|
||||
| psql_as_postgres postgres >/dev/null
|
||||
else
|
||||
printf 'DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = %s) THEN CREATE ROLE %s %s; END IF; END $$;\n' \
|
||||
"'$role'" "$role" "$login" \
|
||||
| psql_as_postgres postgres >/dev/null
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
ensure_database() {
|
||||
log "Creating enterprise database if absent"
|
||||
if ((DRY_RUN)); then
|
||||
printf '[dry-run] ensure database %s owned by %s\n' "$UMBRA_DB_NAME" "$UMBRA_OWNER_ROLE"
|
||||
return
|
||||
fi
|
||||
if ! psql_as_postgres postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '$UMBRA_DB_NAME'" | grep -q 1; then
|
||||
runuser -u "$UMBRA_PG_SUPERUSER" -- createdb --owner="$UMBRA_OWNER_ROLE" "$UMBRA_DB_NAME"
|
||||
fi
|
||||
psql_as_postgres postgres -c "REVOKE ALL ON DATABASE \"$UMBRA_DB_NAME\" FROM PUBLIC" >/dev/null
|
||||
psql_as_postgres postgres -c "GRANT CONNECT ON DATABASE \"$UMBRA_DB_NAME\" TO \"$UMBRA_MIGRATOR_USER\", \"$UMBRA_API_USER\", \"$UMBRA_WORKER_USER\", \"$UMBRA_READONLY_USER\"" >/dev/null
|
||||
psql_as_postgres postgres -c "GRANT CONNECT ON DATABASE \"$UMBRA_DB_NAME\" TO \"$UMBRA_BACKUP_USER\"" >/dev/null
|
||||
}
|
||||
|
||||
apply_schema() {
|
||||
local schema_file="${SCRIPT_DIR}/postgres/001_control_plane.sql"
|
||||
[[ -f "$schema_file" ]] || die "Schema file is missing: $schema_file"
|
||||
log "Applying core identity/tenancy schema"
|
||||
if ((DRY_RUN)); then
|
||||
printf '[dry-run] psql -d %s -f %s\n' "$UMBRA_DB_NAME" "$schema_file"
|
||||
return
|
||||
fi
|
||||
/bin/cat -- "$schema_file" | psql_as_postgres "$UMBRA_DB_NAME" >/dev/null
|
||||
log "Core schema applied"
|
||||
}
|
||||
|
||||
write_nonsecret_runtime_config() {
|
||||
local config_file="$UMBRA_CONFIG_DIR/control-plane.env"
|
||||
run install -d -m 0750 -o root -g "$UMBRA_SERVICE_GROUP" "$UMBRA_CONFIG_DIR"
|
||||
if ((DRY_RUN)); then
|
||||
printf '[dry-run] write non-secret runtime configuration to %s\n' "$config_file"
|
||||
return
|
||||
fi
|
||||
cat > "${config_file}.tmp" <<EOF
|
||||
UMBRA_PUBLIC_URL=$UMBRA_PUBLIC_URL
|
||||
UMBRA_HOSTNAME=$UMBRA_HOSTNAME
|
||||
UMBRA_DB_NAME=$UMBRA_DB_NAME
|
||||
UMBRA_DB_HOST=$UMBRA_DB_HOST
|
||||
UMBRA_DB_PORT=$UMBRA_DB_PORT
|
||||
UMBRA_QWEN_URL=$UMBRA_QWEN_URL
|
||||
UMBRA_APP_ROOT=$UMBRA_APP_ROOT
|
||||
UMBRA_DATA_ROOT=$UMBRA_DATA_ROOT
|
||||
EOF
|
||||
chown root:"$UMBRA_SERVICE_GROUP" "${config_file}.tmp"
|
||||
chmod 0640 "${config_file}.tmp"
|
||||
mv -f "${config_file}.tmp" "$config_file"
|
||||
}
|
||||
|
||||
create_bootstrap_admin() {
|
||||
[[ "$UMBRA_CREATE_BOOTSTRAP_ADMIN" -eq 1 ]] || { log "Bootstrap admin creation disabled"; return; }
|
||||
local token token_hash
|
||||
if [[ -f "$UMBRA_BOOTSTRAP_TOKEN_FILE" ]]; then
|
||||
token="$(cat "$UMBRA_BOOTSTRAP_TOKEN_FILE")"
|
||||
else
|
||||
token="$(random_secret)"
|
||||
if ((DRY_RUN)); then
|
||||
printf '[dry-run] create one-time admin token at %s\n' "$UMBRA_BOOTSTRAP_TOKEN_FILE"
|
||||
return
|
||||
fi
|
||||
printf '%s\n' "$token" > "$UMBRA_BOOTSTRAP_TOKEN_FILE"
|
||||
chown root:root "$UMBRA_BOOTSTRAP_TOKEN_FILE"
|
||||
chmod 0600 "$UMBRA_BOOTSTRAP_TOKEN_FILE"
|
||||
fi
|
||||
[[ "$token" =~ ^[a-f0-9]{64}$ ]] || die "Bootstrap token file has an unexpected format"
|
||||
token_hash="$(printf '%s' "$token" | sha256sum | awk '{print $1}')"
|
||||
log "Creating/confirming bootstrap platform administrator"
|
||||
if ((DRY_RUN)); then
|
||||
printf '[dry-run] create administrator %s and organization %s\n' "$UMBRA_BOOTSTRAP_ADMIN_EMAIL" "$UMBRA_ORGANIZATION_NAME"
|
||||
return
|
||||
fi
|
||||
psql_as_postgres "$UMBRA_DB_NAME" \
|
||||
-v admin_email="$UMBRA_BOOTSTRAP_ADMIN_EMAIL" \
|
||||
-v token_hash="$token_hash" \
|
||||
-v organization_name="$UMBRA_ORGANIZATION_NAME" \
|
||||
-v organization_slug="$UMBRA_ORGANIZATION_SLUG" <<'SQL' >/dev/null
|
||||
INSERT INTO users (email, display_name, auth_provider, auth_subject, status, mfa_required)
|
||||
VALUES (:'admin_email', :'admin_email', 'bootstrap', 'bootstrap:' || gen_random_uuid()::text, 'invited', true)
|
||||
ON CONFLICT (email) DO UPDATE
|
||||
SET display_name = EXCLUDED.display_name,
|
||||
auth_provider = 'bootstrap',
|
||||
mfa_required = true,
|
||||
updated_at = now();
|
||||
|
||||
INSERT INTO organizations (name, slug)
|
||||
VALUES (:'organization_name', :'organization_slug')
|
||||
ON CONFLICT (slug) DO UPDATE SET name = EXCLUDED.name;
|
||||
|
||||
INSERT INTO user_roles (user_id, role_id, scope, organization_id, project_id, created_by)
|
||||
SELECT u.id, r.id, 'platform', NULL, NULL, u.id
|
||||
FROM users u
|
||||
JOIN roles r ON r.scope = 'platform' AND r.name = 'platform_admin'
|
||||
WHERE u.email = :'admin_email'
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO user_roles (user_id, role_id, scope, organization_id, project_id, created_by)
|
||||
SELECT u.id, r.id, 'organization', o.id, NULL, u.id
|
||||
FROM users u
|
||||
JOIN organizations o ON o.slug = :'organization_slug'
|
||||
JOIN roles r ON r.scope = 'organization' AND r.name = 'organization_admin'
|
||||
WHERE u.email = :'admin_email'
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO bootstrap_tokens (user_id, token_hash, expires_at)
|
||||
SELECT id, :'token_hash', now() + interval '24 hours'
|
||||
FROM users
|
||||
WHERE email = :'admin_email'
|
||||
ON CONFLICT (token_hash) DO NOTHING;
|
||||
SQL
|
||||
log "Bootstrap token stored root-only at $UMBRA_BOOTSTRAP_TOKEN_FILE"
|
||||
}
|
||||
|
||||
apply_mode() {
|
||||
if ((DRY_RUN == 0)); then
|
||||
require_root
|
||||
log "Beginning phase-one enterprise bootstrap"
|
||||
if ! mkdir "$LOCK_DIR" 2>/dev/null; then
|
||||
die "Another bootstrap process holds $LOCK_DIR"
|
||||
fi
|
||||
trap 'rmdir "$LOCK_DIR" 2>/dev/null || true' EXIT
|
||||
touch "$LOG_FILE"
|
||||
chmod 0640 "$LOG_FILE"
|
||||
else
|
||||
log "Beginning phase-one enterprise bootstrap dry-run"
|
||||
fi
|
||||
install_packages
|
||||
ensure_group_and_users
|
||||
ensure_directories
|
||||
ensure_uv
|
||||
ensure_postgres_cluster
|
||||
ensure_database_roles
|
||||
ensure_database
|
||||
apply_schema
|
||||
write_nonsecret_runtime_config
|
||||
create_bootstrap_admin
|
||||
log "Phase-one bootstrap completed"
|
||||
log "Application code, systemd units, Qwen, Docker, and firewall were not modified"
|
||||
}
|
||||
|
||||
status_mode() {
|
||||
log "Umbra enterprise installation status"
|
||||
printf 'host: '; hostname -f 2>/dev/null || hostname
|
||||
printf 'dns: '; getent ahosts "$UMBRA_HOSTNAME" 2>/dev/null || true
|
||||
printf 'postgres: '; command -v pg_lsclusters >/dev/null 2>&1 && pg_lsclusters || echo 'not installed'
|
||||
printf 'api user: '; id "$UMBRA_API_USER" 2>/dev/null || echo 'not installed'
|
||||
printf 'worker user: '; id "$UMBRA_WORKER_USER" 2>/dev/null || echo 'not installed'
|
||||
systemctl is-active postgresql 2>/dev/null || true
|
||||
printf 'qwen port: '; (curl -sS --max-time 3 "$UMBRA_QWEN_URL/health" 2>&1 | head -c 300) || true; printf '\n'
|
||||
}
|
||||
|
||||
verify_mode() {
|
||||
log "Verifying phase-one installation"
|
||||
require_command psql
|
||||
require_root
|
||||
pg_isready -h "$UMBRA_DB_HOST" -p "$UMBRA_DB_PORT" -d "$UMBRA_DB_NAME"
|
||||
psql_as_postgres "$UMBRA_DB_NAME" -v ON_ERROR_STOP=1 -c 'SELECT version, applied_at FROM schema_migrations ORDER BY version' -c 'SELECT count(*) AS organizations FROM organizations' -c 'SELECT count(*) AS users FROM users'
|
||||
systemctl is-active postgresql
|
||||
log "Verification completed"
|
||||
}
|
||||
|
||||
main() {
|
||||
load_config
|
||||
validate_config
|
||||
check_tools
|
||||
case "$MODE" in
|
||||
check) check_host ;;
|
||||
apply) apply_mode ;;
|
||||
status) status_mode ;;
|
||||
verify) verify_mode ;;
|
||||
*) die "Unknown mode: $MODE" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
368
infra/bootstrap_umbracontrol_database.sh
Executable file
368
infra/bootstrap_umbracontrol_database.sh
Executable file
@@ -0,0 +1,368 @@
|
||||
#!/usr/bin/env bash
|
||||
# Database-only bootstrap for the UmbraTrans control plane.
|
||||
# This intentionally does not install packages, create application directories,
|
||||
# install application services, or modify Qwen/Docker/firewall settings.
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
CONFIG_FILE="${UMBRA_BOOTSTRAP_CONFIG:-${SCRIPT_DIR}/enterprise-bootstrap.env}"
|
||||
MODE="check"
|
||||
DRY_RUN=0
|
||||
LOCK_DIR="/run/lock/umbracontrol-database.lock"
|
||||
LOG_FILE="/var/log/umbracontrol-database-bootstrap.log"
|
||||
PG_SOCKET_DIR="/var/run/postgresql"
|
||||
|
||||
UMBRA_DB_NAME="umbracontrol"
|
||||
UMBRA_DB_HOST="127.0.0.1"
|
||||
UMBRA_DB_PORT="5432"
|
||||
UMBRA_BOOTSTRAP_ADMIN_EMAIL="umbratools@gmail.com"
|
||||
UMBRA_ORGANIZATION_NAME="Umbra"
|
||||
UMBRA_ORGANIZATION_SLUG="umbra"
|
||||
UMBRA_PG_SUPERUSER="postgres"
|
||||
UMBRA_CREDENTIALS_FILE="/etc/umbracontrol/db-credentials.env"
|
||||
UMBRA_BOOTSTRAP_TOKEN_FILE="/root/umbra-enterprise-bootstrap.token"
|
||||
UMBRA_SERVICE_GROUP="umbracontrol"
|
||||
UMBRA_OWNER_ROLE="umbracontrol_owner"
|
||||
UMBRA_MIGRATOR_ROLE="umbracontrol_migrator"
|
||||
UMBRA_API_ROLE="umbracontrol_api"
|
||||
UMBRA_WORKER_ROLE="umbracontrol_worker"
|
||||
UMBRA_READONLY_ROLE="umbracontrol_readonly"
|
||||
UMBRA_BACKUP_ROLE="umbracontrol_backup"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: bootstrap_umbracontrol_database.sh [options]
|
||||
|
||||
Options:
|
||||
--check Read-only PostgreSQL and prerequisite inspection
|
||||
--dry-run Show database actions without changing the host
|
||||
--apply Create/finish the database, roles, schema, and bootstrap admin
|
||||
--verify Verify the database, schema, RLS, and bootstrap records
|
||||
--config FILE Read the existing non-secret enterprise-bootstrap.env
|
||||
-h, --help Show this help
|
||||
|
||||
This script is intentionally database-only. It does not run apt, install uv,
|
||||
create application directories, start Qwen, or change firewall rules.
|
||||
EOF
|
||||
}
|
||||
|
||||
while (($#)); do
|
||||
case "$1" in
|
||||
--check) MODE="check"; shift ;;
|
||||
--dry-run) MODE="apply"; DRY_RUN=1; shift ;;
|
||||
--apply) MODE="apply"; DRY_RUN=0; shift ;;
|
||||
--verify) MODE="verify"; shift ;;
|
||||
--config) CONFIG_FILE="${2:?--config requires a file}"; shift 2 ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) printf 'Unknown option: %s\n' "$1" >&2; usage >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
log() {
|
||||
local line="[$(date -Is)] $*"
|
||||
printf '%s\n' "$line"
|
||||
if [[ -f "$LOG_FILE" && "$(id -u)" -eq 0 ]]; then
|
||||
printf '%s\n' "$line" >> "$LOG_FILE"
|
||||
fi
|
||||
}
|
||||
|
||||
die() {
|
||||
printf 'ERROR: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_root() {
|
||||
[[ "$(id -u)" -eq 0 ]] || die "This operation requires root. Use --check for read-only inspection."
|
||||
}
|
||||
|
||||
require_command() {
|
||||
command -v "$1" >/dev/null 2>&1 || die "Required command is missing: $1"
|
||||
}
|
||||
|
||||
load_config() {
|
||||
[[ -f "$CONFIG_FILE" ]] || {
|
||||
[[ "$MODE" == "check" || "$MODE" == "verify" ]] && return 0
|
||||
die "Configuration file not found: $CONFIG_FILE"
|
||||
}
|
||||
|
||||
local line key value
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
line="${line#"${line%%[![:space:]]*}"}"
|
||||
[[ -z "$line" || "${line:0:1}" == "#" ]] && continue
|
||||
[[ "$line" == *=* ]] || die "Invalid configuration line: $line"
|
||||
key="${line%%=*}"
|
||||
value="${line#*=}"
|
||||
key="${key//[[:space:]]/}"
|
||||
value="${value%\"}"; value="${value#\"}"
|
||||
value="${value%\'}"; value="${value#\'}"
|
||||
case "$key" in
|
||||
UMBRA_DB_NAME|UMBRA_DB_HOST|UMBRA_DB_PORT|UMBRA_BOOTSTRAP_ADMIN_EMAIL|UMBRA_ORGANIZATION_NAME|UMBRA_ORGANIZATION_SLUG)
|
||||
printf -v "$key" '%s' "$value"
|
||||
;;
|
||||
UMBRA_*)
|
||||
: # Ignore unrelated non-secret enterprise settings.
|
||||
;;
|
||||
*)
|
||||
die "Unsupported configuration key: $key"
|
||||
;;
|
||||
esac
|
||||
done < "$CONFIG_FILE"
|
||||
}
|
||||
|
||||
validate_config() {
|
||||
[[ "$UMBRA_DB_NAME" =~ ^[a-z_][a-z0-9_]*$ ]] || die "Invalid UMBRA_DB_NAME"
|
||||
[[ "$UMBRA_DB_HOST" =~ ^[A-Za-z0-9._:-]+$ ]] || die "Invalid UMBRA_DB_HOST"
|
||||
[[ "$UMBRA_DB_PORT" =~ ^[0-9]+$ && "$UMBRA_DB_PORT" -ge 1 && "$UMBRA_DB_PORT" -le 65535 ]] || die "Invalid UMBRA_DB_PORT"
|
||||
[[ "$UMBRA_BOOTSTRAP_ADMIN_EMAIL" == *@*.* ]] || die "Invalid administrator email"
|
||||
[[ "$UMBRA_ORGANIZATION_SLUG" =~ ^[a-z0-9][a-z0-9-]*$ ]] || die "Invalid organization slug"
|
||||
[[ -d "$PG_SOCKET_DIR" ]] || die "PostgreSQL socket directory is missing: $PG_SOCKET_DIR"
|
||||
}
|
||||
|
||||
run() {
|
||||
if ((DRY_RUN)); then
|
||||
printf '[dry-run]'
|
||||
printf ' %q' "$@"
|
||||
printf '\n'
|
||||
else
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
random_secret() {
|
||||
openssl rand -hex 32
|
||||
}
|
||||
|
||||
read_secret() {
|
||||
local key="$1"
|
||||
[[ -f "$UMBRA_CREDENTIALS_FILE" ]] || return 1
|
||||
awk -F= -v key="$key" '$1 == key {print substr($0, index($0, "=") + 1); exit}' "$UMBRA_CREDENTIALS_FILE"
|
||||
}
|
||||
|
||||
write_secret() {
|
||||
local key="$1" value="$2" temporary
|
||||
if ((DRY_RUN)); then
|
||||
printf '[dry-run] store generated database secret %s\n' "$key"
|
||||
return
|
||||
fi
|
||||
install -d -m 0750 -o root -g "$UMBRA_SERVICE_GROUP" "$(dirname "$UMBRA_CREDENTIALS_FILE")"
|
||||
touch "$UMBRA_CREDENTIALS_FILE"
|
||||
chown root:"$UMBRA_SERVICE_GROUP" "$UMBRA_CREDENTIALS_FILE"
|
||||
chmod 0640 "$UMBRA_CREDENTIALS_FILE"
|
||||
temporary="$(mktemp "${UMBRA_CREDENTIALS_FILE}.XXXXXX")"
|
||||
awk -F= -v key="$key" -v value="$value" '
|
||||
$1 == key { print key "=" value; found = 1; next }
|
||||
{ print }
|
||||
END { if (!found) print key "=" value }
|
||||
' "$UMBRA_CREDENTIALS_FILE" > "$temporary"
|
||||
chown root:"$UMBRA_SERVICE_GROUP" "$temporary"
|
||||
chmod 0640 "$temporary"
|
||||
mv -f "$temporary" "$UMBRA_CREDENTIALS_FILE"
|
||||
}
|
||||
|
||||
psql_as_postgres() {
|
||||
runuser -u "$UMBRA_PG_SUPERUSER" -- env \
|
||||
-u PGHOST -u PGPORT -u PGDATABASE -u PGUSER -u PGPASSWORD \
|
||||
-u PGPASSWORDFILE -u PGSERVICE -u PGSERVICEFILE \
|
||||
PGOPTIONS='-c log_statement=none' \
|
||||
psql -h "$PG_SOCKET_DIR" -p "$UMBRA_DB_PORT" -U "$UMBRA_PG_SUPERUSER" \
|
||||
-v ON_ERROR_STOP=1 "$@"
|
||||
}
|
||||
|
||||
role_exists() {
|
||||
local role="$1"
|
||||
psql_as_postgres postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname = '$role'" | grep -q 1
|
||||
}
|
||||
|
||||
ensure_role() {
|
||||
local role="$1" login="$2" password_key="$3" password=""
|
||||
if ((DRY_RUN)); then
|
||||
printf '[dry-run] ensure PostgreSQL role %s (%s)\n' "$role" "$login"
|
||||
return
|
||||
fi
|
||||
if role_exists "$role"; then
|
||||
if [[ "$login" == "LOGIN" ]]; then
|
||||
printf 'ALTER ROLE "%s" LOGIN;\n' "$role" | psql_as_postgres postgres >/dev/null
|
||||
else
|
||||
printf 'ALTER ROLE "%s" NOLOGIN;\n' "$role" | psql_as_postgres postgres >/dev/null
|
||||
fi
|
||||
log "PostgreSQL role already exists: $role"
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ "$login" == "LOGIN" ]]; then
|
||||
password="$(read_secret "$password_key" 2>/dev/null || true)"
|
||||
[[ -n "$password" ]] || password="$(random_secret)"
|
||||
write_secret "$password_key" "$password"
|
||||
printf 'CREATE ROLE "%s" LOGIN PASSWORD '\''%s'\'';\n' "$role" "$password" \
|
||||
| psql_as_postgres postgres >/dev/null
|
||||
else
|
||||
printf 'CREATE ROLE "%s" NOLOGIN;\n' "$role" | psql_as_postgres postgres >/dev/null
|
||||
fi
|
||||
log "PostgreSQL role created: $role"
|
||||
}
|
||||
|
||||
ensure_roles() {
|
||||
log "Ensuring PostgreSQL roles"
|
||||
ensure_role "$UMBRA_OWNER_ROLE" NOLOGIN OWNER_UNUSED
|
||||
ensure_role "$UMBRA_MIGRATOR_ROLE" LOGIN UMBRACONTROL_MIGRATOR_PASSWORD
|
||||
ensure_role "$UMBRA_API_ROLE" LOGIN UMBRACONTROL_API_PASSWORD
|
||||
ensure_role "$UMBRA_WORKER_ROLE" LOGIN UMBRACONTROL_WORKER_PASSWORD
|
||||
ensure_role "$UMBRA_READONLY_ROLE" LOGIN UMBRACONTROL_READONLY_PASSWORD
|
||||
ensure_role "$UMBRA_BACKUP_ROLE" LOGIN UMBRACONTROL_BACKUP_PASSWORD
|
||||
}
|
||||
|
||||
database_exists() {
|
||||
psql_as_postgres postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '$UMBRA_DB_NAME'" | grep -q 1
|
||||
}
|
||||
|
||||
ensure_database() {
|
||||
log "Ensuring database: $UMBRA_DB_NAME"
|
||||
if ((DRY_RUN)); then
|
||||
printf '[dry-run] create database if absent, owner=%s\n' "$UMBRA_OWNER_ROLE"
|
||||
return
|
||||
fi
|
||||
if ! database_exists; then
|
||||
runuser -u "$UMBRA_PG_SUPERUSER" -- createdb --owner="$UMBRA_OWNER_ROLE" "$UMBRA_DB_NAME"
|
||||
log "Database created"
|
||||
else
|
||||
log "Database already exists"
|
||||
fi
|
||||
psql_as_postgres postgres -c "REVOKE ALL ON DATABASE \"$UMBRA_DB_NAME\" FROM PUBLIC" >/dev/null
|
||||
psql_as_postgres postgres -c "GRANT CONNECT ON DATABASE \"$UMBRA_DB_NAME\" TO \"$UMBRA_MIGRATOR_ROLE\", \"$UMBRA_API_ROLE\", \"$UMBRA_WORKER_ROLE\", \"$UMBRA_READONLY_ROLE\", \"$UMBRA_BACKUP_ROLE\"" >/dev/null
|
||||
}
|
||||
|
||||
apply_schema() {
|
||||
local schema_file="${SCRIPT_DIR}/postgres/001_control_plane.sql"
|
||||
[[ -f "$schema_file" ]] || die "Schema file is missing: $schema_file"
|
||||
log "Applying identity, tenancy, role, audit, and RLS schema"
|
||||
if ((DRY_RUN)); then
|
||||
printf '[dry-run] stream %s into PostgreSQL database %s\n' "$schema_file" "$UMBRA_DB_NAME"
|
||||
return
|
||||
fi
|
||||
/bin/cat -- "$schema_file" | psql_as_postgres "$UMBRA_DB_NAME" >/dev/null
|
||||
log "Database schema applied"
|
||||
}
|
||||
|
||||
create_bootstrap_admin() {
|
||||
local token token_hash
|
||||
if [[ -f "$UMBRA_BOOTSTRAP_TOKEN_FILE" ]]; then
|
||||
token="$(cat "$UMBRA_BOOTSTRAP_TOKEN_FILE")"
|
||||
else
|
||||
token="$(random_secret)"
|
||||
if ((DRY_RUN)); then
|
||||
printf '[dry-run] create root-only bootstrap token\n'
|
||||
return
|
||||
fi
|
||||
printf '%s\n' "$token" > "$UMBRA_BOOTSTRAP_TOKEN_FILE"
|
||||
chown root:root "$UMBRA_BOOTSTRAP_TOKEN_FILE"
|
||||
chmod 0600 "$UMBRA_BOOTSTRAP_TOKEN_FILE"
|
||||
fi
|
||||
[[ "$token" =~ ^[a-f0-9]{64}$ ]] || die "Bootstrap token has an unexpected format"
|
||||
token_hash="$(printf '%s' "$token" | sha256sum | awk '{print $1}')"
|
||||
|
||||
log "Ensuring bootstrap administrator"
|
||||
if ((DRY_RUN)); then
|
||||
printf '[dry-run] create administrator %s and organization %s\n' "$UMBRA_BOOTSTRAP_ADMIN_EMAIL" "$UMBRA_ORGANIZATION_NAME"
|
||||
return
|
||||
fi
|
||||
psql_as_postgres "$UMBRA_DB_NAME" \
|
||||
-v admin_email="$UMBRA_BOOTSTRAP_ADMIN_EMAIL" \
|
||||
-v token_hash="$token_hash" \
|
||||
-v organization_name="$UMBRA_ORGANIZATION_NAME" \
|
||||
-v organization_slug="$UMBRA_ORGANIZATION_SLUG" <<'SQL' >/dev/null
|
||||
INSERT INTO users (email, display_name, auth_provider, auth_subject, status, mfa_required)
|
||||
VALUES (:'admin_email', :'admin_email', 'bootstrap', 'bootstrap:' || gen_random_uuid()::text, 'invited', true)
|
||||
ON CONFLICT (email) DO UPDATE
|
||||
SET display_name = EXCLUDED.display_name,
|
||||
auth_provider = 'bootstrap',
|
||||
mfa_required = true,
|
||||
updated_at = now();
|
||||
|
||||
INSERT INTO organizations (name, slug)
|
||||
VALUES (:'organization_name', :'organization_slug')
|
||||
ON CONFLICT (slug) DO UPDATE SET name = EXCLUDED.name;
|
||||
|
||||
INSERT INTO user_roles (user_id, role_id, scope, organization_id, project_id, created_by)
|
||||
SELECT u.id, r.id, 'platform', NULL, NULL, u.id
|
||||
FROM users u
|
||||
JOIN roles r ON r.scope = 'platform' AND r.name = 'platform_admin'
|
||||
WHERE u.email = :'admin_email'
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO user_roles (user_id, role_id, scope, organization_id, project_id, created_by)
|
||||
SELECT u.id, r.id, 'organization', o.id, NULL, u.id
|
||||
FROM users u
|
||||
JOIN organizations o ON o.slug = :'organization_slug'
|
||||
JOIN roles r ON r.scope = 'organization' AND r.name = 'organization_admin'
|
||||
WHERE u.email = :'admin_email'
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO bootstrap_tokens (user_id, token_hash, expires_at)
|
||||
SELECT id, :'token_hash', now() + interval '24 hours'
|
||||
FROM users
|
||||
WHERE email = :'admin_email'
|
||||
ON CONFLICT (token_hash) DO NOTHING;
|
||||
SQL
|
||||
log "Bootstrap administrator is ready"
|
||||
}
|
||||
|
||||
check_mode() {
|
||||
log "Database-only read-only inspection"
|
||||
printf 'psql: '; command -v psql 2>/dev/null || echo 'missing'
|
||||
printf 'PostgreSQL: '; command -v pg_lsclusters >/dev/null 2>&1 && pg_lsclusters || echo 'not installed'
|
||||
printf 'service: '; systemctl is-active postgresql 2>/dev/null || true
|
||||
printf 'socket: '; test -S "$PG_SOCKET_DIR/.s.PGSQL.$UMBRA_DB_PORT" && echo ready || echo missing
|
||||
printf 'database socket: '; pg_isready -h "$PG_SOCKET_DIR" -p "$UMBRA_DB_PORT" -d "$UMBRA_DB_NAME" 2>&1 || true
|
||||
printf 'roles: '; getent group "$UMBRA_SERVICE_GROUP" 2>/dev/null || echo 'service group absent'
|
||||
log "No changes made"
|
||||
}
|
||||
|
||||
apply_mode() {
|
||||
if ((DRY_RUN == 0)); then
|
||||
require_root
|
||||
mkdir "$LOCK_DIR" 2>/dev/null || die "Another database bootstrap is running"
|
||||
trap 'rmdir "$LOCK_DIR" 2>/dev/null || true' EXIT
|
||||
touch "$LOG_FILE"
|
||||
chmod 0640 "$LOG_FILE"
|
||||
else
|
||||
log "Database-only bootstrap dry-run"
|
||||
fi
|
||||
ensure_roles
|
||||
ensure_database
|
||||
apply_schema
|
||||
create_bootstrap_admin
|
||||
log "Database-only bootstrap completed"
|
||||
}
|
||||
|
||||
verify_mode() {
|
||||
require_root
|
||||
log "Verifying database-only bootstrap"
|
||||
pg_isready -h "$UMBRA_DB_HOST" -p "$UMBRA_DB_PORT" -d "$UMBRA_DB_NAME"
|
||||
psql_as_postgres "$UMBRA_DB_NAME" -v ON_ERROR_STOP=1 \
|
||||
-c 'SELECT version, applied_at FROM schema_migrations ORDER BY version' \
|
||||
-c 'SELECT count(*) AS organizations FROM organizations' \
|
||||
-c 'SELECT count(*) AS users FROM users' \
|
||||
-c "SELECT count(*) AS rls_tables FROM pg_class WHERE relrowsecurity AND relname IN ('organizations','projects','users','user_roles','invitations','sessions','mfa_methods','bootstrap_tokens','audit_events')"
|
||||
log "Database verification completed"
|
||||
}
|
||||
|
||||
main() {
|
||||
load_config
|
||||
validate_config
|
||||
require_command openssl
|
||||
require_command systemctl
|
||||
if [[ "$MODE" == "apply" || "$MODE" == "verify" ]]; then
|
||||
require_command psql
|
||||
require_command pg_isready
|
||||
require_command runuser
|
||||
fi
|
||||
case "$MODE" in
|
||||
check) check_mode ;;
|
||||
apply) apply_mode ;;
|
||||
verify) verify_mode ;;
|
||||
*) die "Unknown mode: $MODE" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
main
|
||||
20
infra/enterprise-bootstrap.env.example
Normal file
20
infra/enterprise-bootstrap.env.example
Normal file
@@ -0,0 +1,20 @@
|
||||
# Non-secret phase-one bootstrap configuration.
|
||||
# Copy to enterprise-bootstrap.env and review before running as root.
|
||||
# Do not put passwords, private keys, or API secrets in this file.
|
||||
|
||||
UMBRA_HOSTNAME=oldbox.bdmurphy.me
|
||||
UMBRA_PUBLIC_URL=https://oldbox.bdmurphy.me
|
||||
UMBRA_DB_NAME=umbracontrol
|
||||
UMBRA_DB_HOST=127.0.0.1
|
||||
UMBRA_DB_PORT=5432
|
||||
UMBRA_PG_VERSION=16
|
||||
UMBRA_APP_ROOT=/opt/umbracontrol
|
||||
UMBRA_DATA_ROOT=/var/lib/umbracontrol
|
||||
UMBRA_BACKUP_ROOT=/var/backups/umbracontrol
|
||||
UMBRA_BOOTSTRAP_ADMIN_EMAIL=umbratools@gmail.com
|
||||
UMBRA_ORGANIZATION_NAME=Umbra
|
||||
UMBRA_ORGANIZATION_SLUG=umbra
|
||||
UMBRA_QWEN_URL=http://192.168.123.14:8881
|
||||
UMBRA_INSTALL_PACKAGES=1
|
||||
UMBRA_INSTALL_UV=1
|
||||
UMBRA_CREATE_BOOTSTRAP_ADMIN=1
|
||||
180
infra/oldbox-agent-communication-prompt.md
Normal file
180
infra/oldbox-agent-communication-prompt.md
Normal file
@@ -0,0 +1,180 @@
|
||||
# Build a secure oldbox ↔ Mac agent communication MCP server
|
||||
|
||||
You are the OpenCode agent running on `oldbox`. The Mac-side agent needs a
|
||||
reliable way to exchange sanitized instructions and reports with you without
|
||||
requiring repeated clipboard copy/paste.
|
||||
|
||||
## Objective
|
||||
|
||||
Create a standalone, local stdio MCP server named `agent-comm` under the
|
||||
UmbraTrans workspace on oldbox. It must use a shared, append-only inbox/outbox
|
||||
on the NAS2 `netbackup/UmbraTrans` share. Do not create an unauthenticated
|
||||
network listener.
|
||||
|
||||
The first purpose is to send the pending SSH-authentication diagnostic report
|
||||
back to the Mac agent without exposing passwords or other secrets.
|
||||
|
||||
## Security requirements
|
||||
|
||||
- Never accept, return, log, or transmit passwords, sudo passwords, private
|
||||
keys, API tokens, Keychain data, or `/etc/shadow` contents.
|
||||
- No secret values may appear in MCP arguments, environment variables, command
|
||||
lines, filenames, reports, or logs.
|
||||
- The server must be stdio-only. It must not listen on a TCP or UDP port.
|
||||
- Use the NAS share only for sanitized text/JSON messages.
|
||||
- Message files must be written atomically (`temporary file` + `rename`).
|
||||
- Use restrictive directory/file permissions and refuse symlink paths.
|
||||
- Validate and bound all message sizes.
|
||||
- Do not change SSH, sudoers, firewall, Docker, Qwen, or existing services.
|
||||
- Do not install a system-wide daemon yet.
|
||||
- Do not grant `NOPASSWD: ALL` or any broad sudo rule.
|
||||
|
||||
## Discovery phase
|
||||
|
||||
Before creating the server, perform read-only discovery:
|
||||
|
||||
1. Locate the NAS2 mount for the `netbackup` share.
|
||||
2. Locate or propose the `UmbraTrans` subdirectory.
|
||||
3. Confirm the path is writable by `umbra`.
|
||||
4. Report the exact path and permissions.
|
||||
5. Do not write any message until the path is confirmed in the report.
|
||||
|
||||
Use a conservative search such as `find`, `mount`, and `df`; do not modify
|
||||
mounts or credentials. If the share is not visible, report that and stop rather
|
||||
than guessing a path.
|
||||
|
||||
## Shared mailbox layout
|
||||
|
||||
Once the path is confirmed, use this layout below the confirmed UmbraTrans
|
||||
directory:
|
||||
|
||||
```text
|
||||
agent-mailbox/
|
||||
inbox/ # sanitized instructions from Mac to oldbox
|
||||
outbox/ # sanitized reports from oldbox to Mac
|
||||
processed/ # messages already handled by oldbox
|
||||
archive/ # optional processed message archive
|
||||
```
|
||||
|
||||
Each message is one UTF-8 JSON file with this shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": 1,
|
||||
"id": "UUID",
|
||||
"created_at": "ISO-8601 UTC",
|
||||
"sender": "oldbox",
|
||||
"kind": "diagnostic|report|instruction|ack",
|
||||
"correlation_id": "optional UUID",
|
||||
"title": "short non-secret title",
|
||||
"body": "sanitized text, maximum 64 KiB",
|
||||
"attachments": []
|
||||
}
|
||||
```
|
||||
|
||||
Do not allow arbitrary attachment paths in version 1.
|
||||
|
||||
## MCP tools
|
||||
|
||||
Implement these tools:
|
||||
|
||||
### `agent_status`
|
||||
|
||||
Return only:
|
||||
|
||||
- server version
|
||||
- configured mailbox path
|
||||
- whether inbox/outbox are readable/writable
|
||||
- number of pending messages
|
||||
- last health-check time
|
||||
- no secret values
|
||||
|
||||
### `send_report`
|
||||
|
||||
Arguments:
|
||||
|
||||
- `title`: non-secret string
|
||||
- `body`: sanitized report text
|
||||
- `correlation_id`: optional non-secret string
|
||||
- `kind`: default `report`
|
||||
|
||||
Write a validated JSON message to `outbox/` using an atomic rename. Return the
|
||||
message ID and path, never the secret contents.
|
||||
|
||||
### `read_instructions`
|
||||
|
||||
Read up to a bounded number of JSON messages from `inbox/`. Validate each
|
||||
message, return only sanitized metadata/body, and move successfully handled
|
||||
messages to `processed/` using an atomic rename. Never execute commands from a
|
||||
message automatically.
|
||||
|
||||
### `send_ack`
|
||||
|
||||
Accept only a correlation ID and short non-secret text. Write an `ack` message
|
||||
to `outbox/`.
|
||||
|
||||
## Current diagnostic report
|
||||
|
||||
After the server is implemented and tested, use it to send one report covering:
|
||||
|
||||
- the successful manual SSH path
|
||||
- the broker's `Permission denied (publickey,password)` result
|
||||
- the fact that no authorized key or certificate is configured
|
||||
- the account/password state already observed
|
||||
- the exact unresolved question: whether the broker supplied a wrong password
|
||||
or supplied no usable password to OpenSSH
|
||||
|
||||
Do not include any password or hash. Do not ask the Mac agent to guess.
|
||||
|
||||
## Implementation requirements
|
||||
|
||||
- Python 3.11+ standard library is sufficient; do not add dependencies unless
|
||||
necessary.
|
||||
- Use type hints, `from __future__ import annotations`, logging to stderr, and
|
||||
no `print()` for debug output except the CLI's intentional status output.
|
||||
- Include a README with the exact OpenCode local-server configuration.
|
||||
- Include unit tests for JSON validation, atomic writes, path traversal,
|
||||
symlink refusal, size limits, inbox/outbox behavior, and secret-pattern
|
||||
redaction.
|
||||
- Add a `--check` mode that performs only read-only checks.
|
||||
- Run the tests and `--check` before sending the first report.
|
||||
|
||||
## OpenCode configuration to document
|
||||
|
||||
The configuration should be equivalent to:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcp": {
|
||||
"servers": {
|
||||
"agent-comm": {
|
||||
"type": "local",
|
||||
"command": [
|
||||
"/absolute/path/to/python",
|
||||
"-m",
|
||||
"agent_comm"
|
||||
],
|
||||
"environment": {
|
||||
"AGENT_COMM_ROOT": "/confirmed/nas2/path/agent-mailbox"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Do not add secrets to the configuration.
|
||||
|
||||
## Completion response
|
||||
|
||||
Return a concise report containing:
|
||||
|
||||
1. Confirmed mailbox path and permissions.
|
||||
2. Files created.
|
||||
3. Test results.
|
||||
4. OpenCode configuration path/snippet.
|
||||
5. The first outbox report message ID.
|
||||
6. Any unresolved issue.
|
||||
|
||||
Do not claim the Mac agent has received the message until the Mac-side reader
|
||||
has been configured and verified.
|
||||
96
infra/oldbox-bootstrap-instructions.md
Normal file
96
infra/oldbox-bootstrap-instructions.md
Normal file
@@ -0,0 +1,96 @@
|
||||
# Oldbox enterprise bootstrap instructions
|
||||
|
||||
You are the OpenCode agent running on `oldbox`. Work locally on the target
|
||||
host; do not wait for clipboard reports from the Mac.
|
||||
|
||||
Read and execute these files first:
|
||||
|
||||
```text
|
||||
/tmp/oldbox-temporary-access-prompt.md
|
||||
/tmp/oldbox-agent-communication-prompt.md
|
||||
```
|
||||
|
||||
Then complete the following local bootstrap:
|
||||
|
||||
## 1. Preserve the existing host
|
||||
|
||||
Inventory, but do not alter:
|
||||
|
||||
- Qwen and GPU services
|
||||
- Docker containers
|
||||
- MySQL
|
||||
- Samba/Webmin/XRDP
|
||||
- PostgreSQL
|
||||
- firewall rules
|
||||
- existing user accounts and data
|
||||
|
||||
Report conflicts before changing anything.
|
||||
|
||||
## 2. Create the development workspace
|
||||
|
||||
Use:
|
||||
|
||||
```text
|
||||
/home/umbra/src/UmbraTrans
|
||||
```
|
||||
|
||||
Create it with ownership `umbra:umbra`. Do not modify UmbraAudio files.
|
||||
If the enterprise source bundle is available locally, copy it there; do not
|
||||
invent application code or overwrite existing source.
|
||||
|
||||
Prepare:
|
||||
|
||||
- Git repository/worktree
|
||||
- `.venv` managed by the approved `uv` workflow
|
||||
- non-secret `.env` template
|
||||
- test/lint/format commands from the project configuration
|
||||
- an explicit development and production data separation
|
||||
|
||||
Run the project’s tests and record the results.
|
||||
|
||||
## 3. Configure backups
|
||||
|
||||
Use only the NAS2 `netbackup/UmbraTrans` share after confirming the oldbox
|
||||
account can access it. Create:
|
||||
|
||||
```text
|
||||
backups/postgres/
|
||||
backups/config/
|
||||
backups/workspace/
|
||||
backups/manifests/
|
||||
```
|
||||
|
||||
PostgreSQL backups must be logical dumps or an approved backup tool. Never
|
||||
copy a live PostgreSQL data directory or SQLite database from a running
|
||||
service as a backup.
|
||||
|
||||
Use atomic temporary files, restrictive permissions, timestamps, checksums,
|
||||
and a retention policy. Do not put credentials in the backup path or report.
|
||||
|
||||
## 4. Establish the agent communication queue
|
||||
|
||||
Use the `agent-comm` server from the communication prompt. If NAS permissions
|
||||
are not yet correct, use a private local queue under the oldbox workspace and
|
||||
report the exact blocker; do not broaden share permissions automatically.
|
||||
|
||||
The queue is for sanitized instructions and reports only. It must never carry
|
||||
passwords, tokens, private keys, or unrestricted shell commands.
|
||||
|
||||
## 5. Report
|
||||
|
||||
Write a concise, machine-readable status report to the queue and print:
|
||||
|
||||
```text
|
||||
OLDBOX_BOOTSTRAP_COMPLETE
|
||||
```
|
||||
|
||||
Include:
|
||||
|
||||
- workspace path and ownership
|
||||
- test result
|
||||
- backup path and last backup status
|
||||
- communication queue path/status
|
||||
- unresolved blockers
|
||||
- confirmation that existing services were preserved
|
||||
|
||||
Do not claim completion for any step that was not actually verified.
|
||||
61
infra/oldbox-ssh-auth-log-prompt.md
Normal file
61
infra/oldbox-ssh-auth-log-prompt.md
Normal file
@@ -0,0 +1,61 @@
|
||||
# Oldbox SSH authentication-log handoff
|
||||
|
||||
A Mac-side broker SSH attempt was just made against `umbra@oldbox.bdmurphy.me`.
|
||||
It received `Permission denied (publickey,password)`.
|
||||
|
||||
You are the OpenCode agent running on oldbox. Perform only read-only
|
||||
authentication diagnostics now.
|
||||
|
||||
Do not change SSH configuration.
|
||||
Do not restart SSH.
|
||||
Do not install anything.
|
||||
Do not read or print passwords, hashes, private keys, or tokens.
|
||||
If a sudo command prompts, the human operator may enter the sudo password
|
||||
locally. Never ask for or transmit that password through chat or the report.
|
||||
|
||||
Run these commands as needed:
|
||||
|
||||
```bash
|
||||
sudo journalctl -u ssh --since "3 minutes ago" --no-pager
|
||||
sudo journalctl -u sshd --since "3 minutes ago" --no-pager
|
||||
sudo /usr/sbin/sshd -T -C user=umbra,addr=192.168.123.12,host=oldbox.bdmurphy.me
|
||||
sudo passwd -S umbra
|
||||
```
|
||||
|
||||
If journalctl is not the active service name, use the service that owns port
|
||||
22. If `sshd -T -C` is unavailable, report the exact error.
|
||||
|
||||
Filter the output to authentication-relevant lines only, especially:
|
||||
|
||||
- Accepted password
|
||||
- Failed password
|
||||
- Failed publickey
|
||||
- Invalid user
|
||||
- account locked
|
||||
- PAM/session-open failures
|
||||
- certificate errors
|
||||
- source address and username
|
||||
|
||||
Return exactly this compact report:
|
||||
|
||||
```text
|
||||
=== AUTH LOG RESULT ===
|
||||
<paste only relevant sanitized lines, or the exact permission/error>
|
||||
|
||||
=== EFFECTIVE SSH AUTH ===
|
||||
<paste only relevant key/value lines, or the exact error>
|
||||
|
||||
=== ACCOUNT STATE ===
|
||||
<passwd -S result; no hash>
|
||||
|
||||
=== CONCLUSION ===
|
||||
<one short conclusion: wrong password, key/certificate path, PAM issue,
|
||||
or insufficient evidence>
|
||||
|
||||
=== NEXT ACTION ===
|
||||
<one specific next step>
|
||||
|
||||
READY_FOR_MAC_AGENT
|
||||
```
|
||||
|
||||
Do not guess. Do not make changes. Do not include secret values.
|
||||
129
infra/oldbox-ssh-diagnostic-prompt.md
Normal file
129
infra/oldbox-ssh-diagnostic-prompt.md
Normal file
@@ -0,0 +1,129 @@
|
||||
# Oldbox SSH diagnostic handoff
|
||||
|
||||
You are the on-host investigator for `oldbox.bdmurphy.me`.
|
||||
|
||||
## Goal
|
||||
|
||||
Determine why direct interactive SSH as `umbra` succeeds from the Mac, while the Mac-side remote broker receives:
|
||||
|
||||
```text
|
||||
Permission denied (publickey,password)
|
||||
```
|
||||
|
||||
## Known facts
|
||||
|
||||
- Host: `oldbox.bdmurphy.me`
|
||||
- Address: `192.168.123.14`
|
||||
- User: `umbra`
|
||||
- The Mac can run: `ssh umbra@oldbox`
|
||||
- The Mac broker is attempting SSH as `umbra@oldbox`
|
||||
- Do not change SSH configuration.
|
||||
- Do not restart SSH.
|
||||
- Do not install packages.
|
||||
- Do not read or print passwords, private keys, tokens, `/etc/shadow` contents, Keychain data, or environment secrets.
|
||||
- Do not ask me to provide any password in chat.
|
||||
- Use `sudo` only for read-only commands. Prefer `sudo -n`; if `sudo -n` is not permitted, report that limitation instead of prompting for a password.
|
||||
|
||||
## Read-only inspection
|
||||
|
||||
### 1. Record host and account state
|
||||
|
||||
Collect:
|
||||
|
||||
- Hostname and kernel
|
||||
- SSH service state
|
||||
- `id umbra`
|
||||
- `getent passwd umbra`
|
||||
- Home-directory permissions
|
||||
- Current `umbra` shell
|
||||
|
||||
### 2. Inspect SSH policy
|
||||
|
||||
Run, where permitted:
|
||||
|
||||
```bash
|
||||
sshd -T
|
||||
```
|
||||
|
||||
If permitted:
|
||||
|
||||
```bash
|
||||
sshd -T -C user=umbra,addr=192.168.123.1,host=oldbox.bdmurphy.me
|
||||
```
|
||||
|
||||
Search `/etc/ssh/sshd_config` and `/etc/ssh/sshd_config.d` for:
|
||||
|
||||
- `Match`
|
||||
- `AuthenticationMethods`
|
||||
- `PasswordAuthentication`
|
||||
- `KbdInteractiveAuthentication`
|
||||
- `UsePAM`
|
||||
- `PubkeyAuthentication`
|
||||
- `AllowUsers`
|
||||
- `DenyUsers`
|
||||
- Certificate-related directives
|
||||
|
||||
### 3. Inspect account state
|
||||
|
||||
If permitted, collect:
|
||||
|
||||
```bash
|
||||
passwd -S umbra
|
||||
chage -l umbra
|
||||
```
|
||||
|
||||
Report account lock or expiry information without printing hashes.
|
||||
|
||||
### 4. Inspect recent SSH authentication logs
|
||||
|
||||
Where readable, inspect:
|
||||
|
||||
```bash
|
||||
journalctl -u ssh --since "15 minutes ago" --no-pager
|
||||
journalctl -u sshd --since "15 minutes ago" --no-pager
|
||||
```
|
||||
|
||||
Also inspect `/var/log/auth.log` or `/var/log/secure` if readable.
|
||||
|
||||
Report whether failures mention:
|
||||
|
||||
- failed password
|
||||
- invalid user
|
||||
- account locked
|
||||
- failed public key
|
||||
- certificate problem
|
||||
- PAM failure
|
||||
- or another condition
|
||||
|
||||
### 5. Determine the likely authentication path
|
||||
|
||||
Determine whether the successful Mac login is likely password-based or key/certificate-based.
|
||||
|
||||
Do not attempt to log in from oldbox. Do not inspect private-key contents. Base the conclusion only on SSH policy, account state, and server logs.
|
||||
|
||||
## Required report format
|
||||
|
||||
Return exactly one report with these headings:
|
||||
|
||||
```text
|
||||
=== IDENTITY ===
|
||||
=== SSH SERVICE ===
|
||||
=== SSH EFFECTIVE CONFIG ===
|
||||
=== ACCOUNT STATE ===
|
||||
=== AUTHENTICATION LOGS ===
|
||||
=== CONCLUSION ===
|
||||
=== NEXT DIAGNOSTIC ===
|
||||
```
|
||||
|
||||
## Rules
|
||||
|
||||
- Redact sensitive values while retaining enough information to distinguish the authentication path.
|
||||
- Never include secret values.
|
||||
- Do not make changes.
|
||||
- If a command is denied, report the command and permission error.
|
||||
- If evidence is insufficient, say so rather than guessing.
|
||||
- At the end, print:
|
||||
|
||||
```text
|
||||
READY_FOR_MAC_AGENT
|
||||
```
|
||||
115
infra/oldbox-temporary-access-prompt.md
Normal file
115
infra/oldbox-temporary-access-prompt.md
Normal file
@@ -0,0 +1,115 @@
|
||||
# One-hour temporary UmbraTrans access setup
|
||||
|
||||
You are the OpenCode agent running on `oldbox`. The human operator has
|
||||
authorized a one-hour temporary access window for finishing the UmbraTrans
|
||||
workspace and communication setup.
|
||||
|
||||
## Goal
|
||||
|
||||
Create a temporary, unprivileged Linux account for Mac-side automation, using
|
||||
a password entered locally in a hidden prompt. The Mac broker will use the
|
||||
same one-hour credential for SSH. After one hour, the account and credential
|
||||
must no longer work.
|
||||
|
||||
## Account
|
||||
|
||||
Use this exact temporary username:
|
||||
|
||||
```text
|
||||
umbracontrol-temp
|
||||
```
|
||||
|
||||
Create it with:
|
||||
|
||||
- home `/home/umbracontrol-temp`, mode `0700`
|
||||
- shell `/bin/bash`
|
||||
- no membership in `sudo`, `docker`, `mysql`, `lpadmin`, or other privileged
|
||||
groups
|
||||
- account expiry exactly one hour from creation
|
||||
- a locally entered hidden password; never print, log, echo, or place it in a
|
||||
file, command argument, environment variable, report, or MCP message
|
||||
|
||||
Use a local hidden prompt in the oldbox terminal. For example, adapt this
|
||||
pattern safely in the agent's shell:
|
||||
|
||||
```bash
|
||||
read -rsp 'Temporary UmbraTrans password: ' TEMP_PASSWORD
|
||||
printf '\n'
|
||||
test -n "$TEMP_PASSWORD"
|
||||
```
|
||||
|
||||
Pass the value to `chpasswd` through standard input, not an argument. Clear
|
||||
the shell variable immediately afterward. Do not repeat the value in output.
|
||||
|
||||
Set the account password and expiration with `sudo` locally. The human may
|
||||
enter the sudo password in the terminal; never request it in chat.
|
||||
|
||||
Schedule cleanup for one hour using a root-owned transient systemd timer or
|
||||
an equivalent root-owned one-shot cleanup. Cleanup must:
|
||||
|
||||
- lock and delete `umbracontrol-temp`
|
||||
- remove `/home/umbracontrol-temp`
|
||||
- remove any temporary authorized-key or agent files
|
||||
- leave `umbra`, Qwen, Docker, MySQL, Samba, Webmin, XRDP, and PostgreSQL
|
||||
unchanged
|
||||
|
||||
Verify before reporting:
|
||||
|
||||
```bash
|
||||
id umbracontrol-temp
|
||||
getent passwd umbracontrol-temp
|
||||
passwd -S umbracontrol-temp
|
||||
```
|
||||
|
||||
Do not print hashes or secret values.
|
||||
|
||||
## Temporary Mac broker profile
|
||||
|
||||
After creating the account, write a **sanitized** report containing:
|
||||
|
||||
- username `umbracontrol-temp`
|
||||
- host `oldbox.bdmurphy.me`
|
||||
- port `22`
|
||||
- account expiry timestamp
|
||||
- cleanup mechanism/timer name
|
||||
- home path and permissions
|
||||
|
||||
Do not include the temporary password in the report. The Mac operator will
|
||||
enter the same one-hour password through the local secure popup.
|
||||
|
||||
## Workspace
|
||||
|
||||
Use the temporary account only for safe, unprivileged workspace operations:
|
||||
|
||||
- create `/home/umbracontrol-temp/UmbraTrans`
|
||||
- report the path and ownership
|
||||
- do not alter `/home/umbra` ownership or permissions
|
||||
- do not modify existing services
|
||||
- do not grant sudo
|
||||
- do not install packages
|
||||
- do not change firewall rules
|
||||
|
||||
Root-required application/database/backup work remains a separate explicit
|
||||
operation after the temporary SSH path is verified.
|
||||
|
||||
## Sanitized diagnostic report
|
||||
|
||||
Send the report through the local `agent-comm` MCP server if it is already
|
||||
available. Otherwise write it atomically to:
|
||||
|
||||
```text
|
||||
/tmp/umbracontrol-temp-setup-report.json
|
||||
```
|
||||
|
||||
The report may contain only non-secret account, expiry, permission, and
|
||||
service-status data. Use mode `0600`.
|
||||
|
||||
At the end print:
|
||||
|
||||
```text
|
||||
TEMP_SETUP_READY
|
||||
READY_FOR_MAC_AGENT
|
||||
```
|
||||
|
||||
Do not claim the Mac has connected until the Mac broker reports a successful
|
||||
SSH status probe.
|
||||
584
infra/postgres/001_control_plane.sql
Normal file
584
infra/postgres/001_control_plane.sql
Normal file
@@ -0,0 +1,584 @@
|
||||
BEGIN;
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS citext;
|
||||
CREATE EXTENSION IF NOT EXISTS pgcrypto;
|
||||
CREATE SCHEMA IF NOT EXISTS app;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version text PRIMARY KEY,
|
||||
applied_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS organizations (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
name text NOT NULL,
|
||||
slug citext NOT NULL UNIQUE,
|
||||
status text NOT NULL DEFAULT 'active'
|
||||
CHECK (status IN ('active', 'suspended', 'archived')),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
email citext NOT NULL UNIQUE,
|
||||
display_name text,
|
||||
auth_provider text NOT NULL DEFAULT 'bootstrap',
|
||||
auth_subject text,
|
||||
status text NOT NULL DEFAULT 'invited'
|
||||
CHECK (status IN ('invited', 'active', 'suspended', 'deactivated', 'locked')),
|
||||
mfa_required boolean NOT NULL DEFAULT false,
|
||||
last_login_at timestamptz,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS users_auth_subject_idx
|
||||
ON users (auth_provider, auth_subject)
|
||||
WHERE auth_subject IS NOT NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS projects (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
organization_id uuid NOT NULL REFERENCES organizations(id),
|
||||
name text NOT NULL,
|
||||
slug citext NOT NULL,
|
||||
status text NOT NULL DEFAULT 'active'
|
||||
CHECK (status IN ('active', 'on_hold', 'released', 'archived')),
|
||||
created_by uuid REFERENCES users(id),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (organization_id, slug),
|
||||
UNIQUE (id, organization_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS roles (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
name text NOT NULL,
|
||||
scope text NOT NULL CHECK (scope IN ('platform', 'organization', 'project')),
|
||||
description text NOT NULL DEFAULT '',
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (scope, name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS permissions (
|
||||
key text PRIMARY KEY,
|
||||
description text NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS role_permissions (
|
||||
role_id uuid NOT NULL REFERENCES roles(id) ON DELETE CASCADE,
|
||||
permission_key text NOT NULL REFERENCES permissions(key) ON DELETE CASCADE,
|
||||
PRIMARY KEY (role_id, permission_key)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS user_roles (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
role_id uuid NOT NULL REFERENCES roles(id) ON DELETE RESTRICT,
|
||||
scope text NOT NULL CHECK (scope IN ('platform', 'organization', 'project')),
|
||||
organization_id uuid REFERENCES organizations(id) ON DELETE CASCADE,
|
||||
project_id uuid,
|
||||
status text NOT NULL DEFAULT 'active'
|
||||
CHECK (status IN ('active', 'suspended', 'revoked')),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
created_by uuid REFERENCES users(id),
|
||||
CHECK (
|
||||
(scope = 'platform' AND organization_id IS NULL AND project_id IS NULL)
|
||||
OR (scope = 'organization' AND organization_id IS NOT NULL AND project_id IS NULL)
|
||||
OR (scope = 'project' AND organization_id IS NOT NULL AND project_id IS NOT NULL)
|
||||
),
|
||||
FOREIGN KEY (project_id, organization_id)
|
||||
REFERENCES projects(id, organization_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS user_roles_platform_unique
|
||||
ON user_roles (user_id, role_id)
|
||||
WHERE scope = 'platform';
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS user_roles_organization_unique
|
||||
ON user_roles (user_id, role_id, organization_id)
|
||||
WHERE scope = 'organization';
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS user_roles_project_unique
|
||||
ON user_roles (user_id, role_id, project_id)
|
||||
WHERE scope = 'project';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS invitations (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
organization_id uuid NOT NULL REFERENCES organizations(id) ON DELETE CASCADE,
|
||||
project_id uuid,
|
||||
email citext NOT NULL,
|
||||
role_id uuid NOT NULL REFERENCES roles(id) ON DELETE RESTRICT,
|
||||
token_hash text NOT NULL UNIQUE,
|
||||
invited_by uuid NOT NULL REFERENCES users(id),
|
||||
expires_at timestamptz NOT NULL,
|
||||
accepted_at timestamptz,
|
||||
revoked_at timestamptz,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
FOREIGN KEY (project_id, organization_id)
|
||||
REFERENCES projects(id, organization_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
token_hash text NOT NULL UNIQUE,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
revoked_at timestamptz,
|
||||
ip_address inet,
|
||||
user_agent text
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS mfa_methods (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
method text NOT NULL CHECK (method IN ('totp', 'webauthn', 'recovery_code')),
|
||||
secret_reference text,
|
||||
verified_at timestamptz,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
last_used_at timestamptz,
|
||||
revoked_at timestamptz
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS bootstrap_tokens (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
token_hash text NOT NULL UNIQUE,
|
||||
expires_at timestamptz NOT NULL,
|
||||
consumed_at timestamptz,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS audit_events (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
actor_user_id uuid REFERENCES users(id),
|
||||
organization_id uuid REFERENCES organizations(id),
|
||||
project_id uuid,
|
||||
action text NOT NULL,
|
||||
target_type text,
|
||||
target_id text,
|
||||
request_id text,
|
||||
ip_address inet,
|
||||
metadata jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS projects_organization_idx ON projects (organization_id);
|
||||
CREATE INDEX IF NOT EXISTS user_roles_user_idx ON user_roles (user_id, status);
|
||||
CREATE INDEX IF NOT EXISTS user_roles_organization_idx ON user_roles (organization_id, status);
|
||||
CREATE INDEX IF NOT EXISTS user_roles_project_idx ON user_roles (project_id, status);
|
||||
CREATE INDEX IF NOT EXISTS invitations_email_idx ON invitations (email);
|
||||
CREATE INDEX IF NOT EXISTS audit_events_org_created_idx ON audit_events (organization_id, created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS audit_events_project_created_idx ON audit_events (project_id, created_at DESC);
|
||||
|
||||
CREATE OR REPLACE FUNCTION app.validate_user_role_scope()
|
||||
RETURNS trigger
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
role_scope text;
|
||||
BEGIN
|
||||
SELECT scope INTO role_scope FROM roles WHERE id = NEW.role_id;
|
||||
IF role_scope IS NULL OR role_scope <> NEW.scope THEN
|
||||
RAISE EXCEPTION 'role scope does not match assignment scope';
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
|
||||
DROP TRIGGER IF EXISTS user_roles_validate_scope ON user_roles;
|
||||
CREATE TRIGGER user_roles_validate_scope
|
||||
BEFORE INSERT OR UPDATE OF role_id, scope ON user_roles
|
||||
FOR EACH ROW EXECUTE FUNCTION app.validate_user_role_scope();
|
||||
|
||||
CREATE OR REPLACE FUNCTION app.set_updated_at()
|
||||
RETURNS trigger
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = now();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
|
||||
DROP TRIGGER IF EXISTS organizations_set_updated_at ON organizations;
|
||||
CREATE TRIGGER organizations_set_updated_at
|
||||
BEFORE UPDATE ON organizations
|
||||
FOR EACH ROW EXECUTE FUNCTION app.set_updated_at();
|
||||
|
||||
DROP TRIGGER IF EXISTS users_set_updated_at ON users;
|
||||
CREATE TRIGGER users_set_updated_at
|
||||
BEFORE UPDATE ON users
|
||||
FOR EACH ROW EXECUTE FUNCTION app.set_updated_at();
|
||||
|
||||
DROP TRIGGER IF EXISTS projects_set_updated_at ON projects;
|
||||
CREATE TRIGGER projects_set_updated_at
|
||||
BEFORE UPDATE ON projects
|
||||
FOR EACH ROW EXECUTE FUNCTION app.set_updated_at();
|
||||
|
||||
CREATE OR REPLACE FUNCTION app.current_user_id()
|
||||
RETURNS uuid
|
||||
LANGUAGE sql
|
||||
STABLE
|
||||
AS $$
|
||||
SELECT NULLIF(current_setting('app.user_id', true), '')::uuid
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION app.is_platform_admin()
|
||||
RETURNS boolean
|
||||
LANGUAGE sql
|
||||
STABLE
|
||||
SECURITY DEFINER
|
||||
SET search_path = public, pg_temp
|
||||
AS $$
|
||||
SELECT EXISTS (
|
||||
SELECT 1
|
||||
FROM user_roles ur
|
||||
JOIN roles r ON r.id = ur.role_id
|
||||
WHERE ur.user_id = app.current_user_id()
|
||||
AND ur.scope = 'platform'
|
||||
AND ur.status = 'active'
|
||||
AND r.name = 'platform_admin'
|
||||
)
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION app.is_organization_member(target_organization_id uuid)
|
||||
RETURNS boolean
|
||||
LANGUAGE sql
|
||||
STABLE
|
||||
SECURITY DEFINER
|
||||
SET search_path = public, pg_temp
|
||||
AS $$
|
||||
SELECT EXISTS (
|
||||
SELECT 1
|
||||
FROM user_roles ur
|
||||
JOIN roles r ON r.id = ur.role_id
|
||||
WHERE ur.user_id = app.current_user_id()
|
||||
AND ur.organization_id = target_organization_id
|
||||
AND ur.scope = 'organization'
|
||||
AND ur.status = 'active'
|
||||
AND r.name IN ('organization_admin', 'member')
|
||||
)
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION app.is_organization_admin(target_organization_id uuid)
|
||||
RETURNS boolean
|
||||
LANGUAGE sql
|
||||
STABLE
|
||||
SECURITY DEFINER
|
||||
SET search_path = public, pg_temp
|
||||
AS $$
|
||||
SELECT EXISTS (
|
||||
SELECT 1
|
||||
FROM user_roles ur
|
||||
JOIN roles r ON r.id = ur.role_id
|
||||
WHERE ur.user_id = app.current_user_id()
|
||||
AND ur.organization_id = target_organization_id
|
||||
AND ur.scope = 'organization'
|
||||
AND ur.status = 'active'
|
||||
AND r.name = 'organization_admin'
|
||||
)
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION app.is_organization_admin_for_project(target_project_id uuid)
|
||||
RETURNS boolean
|
||||
LANGUAGE sql
|
||||
STABLE
|
||||
SECURITY DEFINER
|
||||
SET search_path = public, pg_temp
|
||||
AS $$
|
||||
SELECT app.is_platform_admin()
|
||||
OR EXISTS (
|
||||
SELECT 1
|
||||
FROM projects p
|
||||
WHERE p.id = target_project_id
|
||||
AND app.is_organization_admin(p.organization_id)
|
||||
)
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION app.can_access_project(target_project_id uuid)
|
||||
RETURNS boolean
|
||||
LANGUAGE sql
|
||||
STABLE
|
||||
SECURITY DEFINER
|
||||
SET search_path = public, pg_temp
|
||||
AS $$
|
||||
SELECT app.is_platform_admin()
|
||||
OR EXISTS (
|
||||
SELECT 1
|
||||
FROM projects p
|
||||
WHERE p.id = target_project_id
|
||||
AND app.is_organization_admin(p.organization_id)
|
||||
)
|
||||
OR EXISTS (
|
||||
SELECT 1
|
||||
FROM user_roles ur
|
||||
WHERE ur.user_id = app.current_user_id()
|
||||
AND ur.project_id = target_project_id
|
||||
AND ur.scope = 'project'
|
||||
AND ur.status = 'active'
|
||||
)
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION app.can_view_user(target_user_id uuid)
|
||||
RETURNS boolean
|
||||
LANGUAGE sql
|
||||
STABLE
|
||||
SECURITY DEFINER
|
||||
SET search_path = public, pg_temp
|
||||
AS $$
|
||||
SELECT target_user_id = app.current_user_id()
|
||||
OR app.is_platform_admin()
|
||||
OR EXISTS (
|
||||
SELECT 1
|
||||
FROM user_roles mine
|
||||
JOIN user_roles theirs
|
||||
ON theirs.user_id = target_user_id
|
||||
AND theirs.organization_id = mine.organization_id
|
||||
AND theirs.scope = 'organization'
|
||||
AND theirs.status = 'active'
|
||||
JOIN roles r ON r.id = mine.role_id
|
||||
WHERE mine.user_id = app.current_user_id()
|
||||
AND mine.scope = 'organization'
|
||||
AND mine.status = 'active'
|
||||
AND r.name = 'organization_admin'
|
||||
)
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION app.prevent_audit_mutation()
|
||||
RETURNS trigger
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
BEGIN
|
||||
RAISE EXCEPTION 'audit_events is append-only';
|
||||
END;
|
||||
$$;
|
||||
|
||||
DROP TRIGGER IF EXISTS audit_events_append_only ON audit_events;
|
||||
CREATE TRIGGER audit_events_append_only
|
||||
BEFORE UPDATE OR DELETE ON audit_events
|
||||
FOR EACH ROW EXECUTE FUNCTION app.prevent_audit_mutation();
|
||||
|
||||
INSERT INTO schema_migrations(version) VALUES ('001_control_plane')
|
||||
ON CONFLICT (version) DO NOTHING;
|
||||
|
||||
INSERT INTO roles (name, scope, description) VALUES
|
||||
('platform_admin', 'platform', 'Global control-plane administration'),
|
||||
('organization_admin', 'organization', 'Administration within one organization'),
|
||||
('member', 'organization', 'Read-only organization membership'),
|
||||
('project_owner', 'project', 'Project administration'),
|
||||
('project_manager', 'project', 'Project production management'),
|
||||
('translator', 'project', 'Translation work'),
|
||||
('proofreader', 'project', 'Translation review'),
|
||||
('audio_director', 'project', 'Voice and audio approval'),
|
||||
('voice_manager', 'organization', 'Voice library and licensing management'),
|
||||
('release_manager', 'project', 'Release approval'),
|
||||
('viewer', 'project', 'Read-only project access')
|
||||
ON CONFLICT (scope, name) DO UPDATE SET description = EXCLUDED.description;
|
||||
|
||||
INSERT INTO permissions (key, description) VALUES
|
||||
('platform.manage', 'Manage platform-level resources'),
|
||||
('organization.manage', 'Manage organization settings'),
|
||||
('organization.users.manage', 'Invite and manage organization users'),
|
||||
('organization.projects.manage', 'Create and manage projects'),
|
||||
('project.access.manage', 'Assign project access'),
|
||||
('project.source.edit', 'Edit approved source material'),
|
||||
('project.translation.edit', 'Edit target translations'),
|
||||
('project.translation.review', 'Review target translations'),
|
||||
('project.audio.approve', 'Approve audio output'),
|
||||
('project.release.approve', 'Approve releases'),
|
||||
('voice.catalog.manage', 'Manage voice catalog and licensing'),
|
||||
('artifact.download', 'Download authorized artifacts')
|
||||
ON CONFLICT (key) DO UPDATE SET description = EXCLUDED.description;
|
||||
|
||||
INSERT INTO role_permissions (role_id, permission_key)
|
||||
SELECT r.id, p.key
|
||||
FROM roles r
|
||||
JOIN (
|
||||
VALUES
|
||||
('platform_admin', 'platform.manage'),
|
||||
('organization_admin', 'organization.manage'),
|
||||
('organization_admin', 'organization.users.manage'),
|
||||
('organization_admin', 'organization.projects.manage'),
|
||||
('organization_admin', 'project.access.manage'),
|
||||
('member', 'artifact.download'),
|
||||
('project_owner', 'project.access.manage'),
|
||||
('project_owner', 'project.release.approve'),
|
||||
('project_manager', 'project.translation.edit'),
|
||||
('project_manager', 'project.audio.approve'),
|
||||
('translator', 'project.translation.edit'),
|
||||
('proofreader', 'project.translation.review'),
|
||||
('audio_director', 'project.audio.approve'),
|
||||
('voice_manager', 'voice.catalog.manage'),
|
||||
('release_manager', 'project.release.approve'),
|
||||
('viewer', 'artifact.download')
|
||||
) AS grants(role_name, permission_key)
|
||||
ON grants.role_name = r.name
|
||||
JOIN permissions p ON p.key = grants.permission_key
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
ALTER TABLE roles ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE permissions ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE role_permissions ENABLE ROW LEVEL SECURITY;
|
||||
|
||||
DROP POLICY IF EXISTS roles_read ON roles;
|
||||
CREATE POLICY roles_read ON roles
|
||||
FOR SELECT USING (app.current_user_id() IS NOT NULL OR app.is_platform_admin());
|
||||
DROP POLICY IF EXISTS roles_manage ON roles;
|
||||
CREATE POLICY roles_manage ON roles
|
||||
FOR ALL USING (app.is_platform_admin())
|
||||
WITH CHECK (app.is_platform_admin());
|
||||
|
||||
DROP POLICY IF EXISTS permissions_read ON permissions;
|
||||
CREATE POLICY permissions_read ON permissions
|
||||
FOR SELECT USING (app.current_user_id() IS NOT NULL OR app.is_platform_admin());
|
||||
DROP POLICY IF EXISTS permissions_manage ON permissions;
|
||||
CREATE POLICY permissions_manage ON permissions
|
||||
FOR ALL USING (app.is_platform_admin())
|
||||
WITH CHECK (app.is_platform_admin());
|
||||
|
||||
DROP POLICY IF EXISTS role_permissions_read ON role_permissions;
|
||||
CREATE POLICY role_permissions_read ON role_permissions
|
||||
FOR SELECT USING (app.current_user_id() IS NOT NULL OR app.is_platform_admin());
|
||||
DROP POLICY IF EXISTS role_permissions_manage ON role_permissions;
|
||||
CREATE POLICY role_permissions_manage ON role_permissions
|
||||
FOR ALL USING (app.is_platform_admin())
|
||||
WITH CHECK (app.is_platform_admin());
|
||||
|
||||
ALTER TABLE organizations ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE projects ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE users ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE user_roles ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE invitations ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE sessions ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE mfa_methods ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE bootstrap_tokens ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE audit_events ENABLE ROW LEVEL SECURITY;
|
||||
|
||||
DROP POLICY IF EXISTS organizations_select ON organizations;
|
||||
CREATE POLICY organizations_select ON organizations
|
||||
FOR SELECT USING (app.is_platform_admin() OR app.is_organization_member(id));
|
||||
DROP POLICY IF EXISTS organizations_insert ON organizations;
|
||||
CREATE POLICY organizations_insert ON organizations
|
||||
FOR INSERT WITH CHECK (app.is_platform_admin());
|
||||
DROP POLICY IF EXISTS organizations_update ON organizations;
|
||||
CREATE POLICY organizations_update ON organizations
|
||||
FOR UPDATE USING (app.is_platform_admin() OR app.is_organization_admin(id))
|
||||
WITH CHECK (app.is_platform_admin() OR app.is_organization_admin(id));
|
||||
DROP POLICY IF EXISTS organizations_delete ON organizations;
|
||||
CREATE POLICY organizations_delete ON organizations
|
||||
FOR DELETE USING (app.is_platform_admin());
|
||||
|
||||
DROP POLICY IF EXISTS projects_select ON projects;
|
||||
CREATE POLICY projects_select ON projects
|
||||
FOR SELECT USING (app.can_access_project(id));
|
||||
DROP POLICY IF EXISTS projects_insert ON projects;
|
||||
CREATE POLICY projects_insert ON projects
|
||||
FOR INSERT WITH CHECK (app.is_platform_admin() OR app.is_organization_admin(organization_id));
|
||||
DROP POLICY IF EXISTS projects_update ON projects;
|
||||
CREATE POLICY projects_update ON projects
|
||||
FOR UPDATE USING (app.is_platform_admin() OR app.is_organization_admin(organization_id))
|
||||
WITH CHECK (app.is_platform_admin() OR app.is_organization_admin(organization_id));
|
||||
DROP POLICY IF EXISTS projects_delete ON projects;
|
||||
CREATE POLICY projects_delete ON projects
|
||||
FOR DELETE USING (app.is_platform_admin() OR app.is_organization_admin(organization_id));
|
||||
|
||||
DROP POLICY IF EXISTS users_select ON users;
|
||||
CREATE POLICY users_select ON users
|
||||
FOR SELECT USING (app.can_view_user(id));
|
||||
DROP POLICY IF EXISTS users_manage ON users;
|
||||
CREATE POLICY users_manage ON users
|
||||
FOR ALL USING (app.is_platform_admin())
|
||||
WITH CHECK (app.is_platform_admin());
|
||||
|
||||
DROP POLICY IF EXISTS user_roles_select ON user_roles;
|
||||
CREATE POLICY user_roles_select ON user_roles
|
||||
FOR SELECT USING (
|
||||
app.is_platform_admin()
|
||||
OR user_id = app.current_user_id()
|
||||
OR (scope = 'organization' AND app.is_organization_admin(organization_id))
|
||||
OR (scope = 'project' AND app.is_organization_admin_for_project(project_id))
|
||||
);
|
||||
DROP POLICY IF EXISTS user_roles_manage ON user_roles;
|
||||
CREATE POLICY user_roles_manage ON user_roles
|
||||
FOR ALL USING (
|
||||
app.is_platform_admin()
|
||||
OR (scope = 'organization' AND app.is_organization_admin(organization_id))
|
||||
OR (scope = 'project' AND app.is_organization_admin_for_project(project_id))
|
||||
)
|
||||
WITH CHECK (
|
||||
app.is_platform_admin()
|
||||
OR (scope = 'organization' AND app.is_organization_admin(organization_id))
|
||||
OR (scope = 'project' AND app.is_organization_admin_for_project(project_id))
|
||||
);
|
||||
|
||||
DROP POLICY IF EXISTS invitations_select ON invitations;
|
||||
CREATE POLICY invitations_select ON invitations
|
||||
FOR SELECT USING (
|
||||
app.is_platform_admin()
|
||||
OR app.is_organization_admin(organization_id)
|
||||
OR email = (SELECT email FROM users WHERE id = app.current_user_id())
|
||||
);
|
||||
DROP POLICY IF EXISTS invitations_manage ON invitations;
|
||||
CREATE POLICY invitations_manage ON invitations
|
||||
FOR ALL USING (app.is_platform_admin() OR app.is_organization_admin(organization_id))
|
||||
WITH CHECK (app.is_platform_admin() OR app.is_organization_admin(organization_id));
|
||||
|
||||
DROP POLICY IF EXISTS sessions_self ON sessions;
|
||||
CREATE POLICY sessions_self ON sessions
|
||||
FOR ALL USING (user_id = app.current_user_id())
|
||||
WITH CHECK (user_id = app.current_user_id());
|
||||
|
||||
DROP POLICY IF EXISTS mfa_methods_self ON mfa_methods;
|
||||
CREATE POLICY mfa_methods_self ON mfa_methods
|
||||
FOR ALL USING (user_id = app.current_user_id())
|
||||
WITH CHECK (user_id = app.current_user_id());
|
||||
|
||||
DROP POLICY IF EXISTS bootstrap_tokens_self ON bootstrap_tokens;
|
||||
CREATE POLICY bootstrap_tokens_self ON bootstrap_tokens
|
||||
FOR ALL USING (user_id = app.current_user_id())
|
||||
WITH CHECK (user_id = app.current_user_id());
|
||||
|
||||
DROP POLICY IF EXISTS audit_events_select ON audit_events;
|
||||
CREATE POLICY audit_events_select ON audit_events
|
||||
FOR SELECT USING (
|
||||
app.is_platform_admin()
|
||||
OR app.is_organization_admin(organization_id)
|
||||
);
|
||||
DROP POLICY IF EXISTS audit_events_insert ON audit_events;
|
||||
CREATE POLICY audit_events_insert ON audit_events
|
||||
FOR INSERT WITH CHECK (
|
||||
actor_user_id = app.current_user_id()
|
||||
OR app.is_platform_admin()
|
||||
);
|
||||
|
||||
REVOKE ALL ON SCHEMA app FROM PUBLIC;
|
||||
REVOKE CREATE ON SCHEMA public FROM PUBLIC;
|
||||
|
||||
GRANT USAGE ON SCHEMA public, app TO
|
||||
umbracontrol_api, umbracontrol_worker, umbracontrol_readonly;
|
||||
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO
|
||||
umbracontrol_api, umbracontrol_worker, umbracontrol_readonly;
|
||||
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON
|
||||
organizations, projects, users, roles, permissions, role_permissions,
|
||||
user_roles, invitations, sessions, mfa_methods, bootstrap_tokens, audit_events
|
||||
TO umbracontrol_api;
|
||||
|
||||
GRANT SELECT ON
|
||||
organizations, projects, users, roles, permissions, role_permissions,
|
||||
user_roles, audit_events
|
||||
TO umbracontrol_worker, umbracontrol_readonly;
|
||||
|
||||
GRANT INSERT ON audit_events TO umbracontrol_worker;
|
||||
|
||||
GRANT USAGE, CREATE ON SCHEMA public TO umbracontrol_migrator;
|
||||
GRANT USAGE ON SCHEMA public TO umbracontrol_backup;
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA public TO umbracontrol_backup;
|
||||
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO umbracontrol_migrator;
|
||||
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO umbracontrol_migrator;
|
||||
|
||||
REVOKE ALL ON ALL FUNCTIONS IN SCHEMA app FROM PUBLIC;
|
||||
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA app TO
|
||||
umbracontrol_api, umbracontrol_worker, umbracontrol_readonly;
|
||||
|
||||
COMMIT;
|
||||
106
infra/remote_credential_broker/README.md
Normal file
106
infra/remote_credential_broker/README.md
Normal file
@@ -0,0 +1,106 @@
|
||||
# UmbraTrans remote credential broker
|
||||
|
||||
This is a standalone local MCP server for the UmbraTrans control plane. It is
|
||||
not imported by UmbraAudio and must be deployed separately on the trusted
|
||||
macOS/OpenCode host.
|
||||
|
||||
## Security model
|
||||
|
||||
- Passwords are entered in a native macOS hidden-password dialog.
|
||||
- Passwords are stored in the macOS login Keychain through the Security
|
||||
framework; they are never written to the repository, environment variables,
|
||||
command-line arguments, logs, or MCP responses.
|
||||
- The MCP server exposes no `get_password` operation.
|
||||
- SSH host-key checking is strict. The server does not silently accept an
|
||||
unknown host key.
|
||||
- Unprivileged SSH commands are bounded by a timeout and output limit.
|
||||
- Sudo commands must exactly match an allowlist entry in the local config.
|
||||
- The default configuration does not grant `sudo` access to arbitrary commands.
|
||||
- The server is stdio-only and has no network listener.
|
||||
|
||||
The broker is intended to run on the Mac that owns the OpenCode session. The
|
||||
remote `umbra` account receives an SSH connection; the password is supplied
|
||||
only to the local SSH client/Keychain flow.
|
||||
|
||||
## Files
|
||||
|
||||
- `broker.py` — stdio MCP server and command-line popup setup.
|
||||
- `keychain.py` — ctypes macOS Keychain backend.
|
||||
- `popup.py` — native hidden-password prompt.
|
||||
- `askpass_helper.py` — OpenSSH `SSH_ASKPASS` helper; it prints a secret only
|
||||
to the SSH process pipe.
|
||||
- `executor.py` — bounded SSH/sudo execution and redaction.
|
||||
- `config.py` — non-secret remote profile and sudo allowlist parser.
|
||||
- `config.example.json` — oldbox/umbra example; copy to the user config path.
|
||||
- `tests/` — dependency-free unit tests.
|
||||
|
||||
## Local configuration
|
||||
|
||||
Copy the example to:
|
||||
|
||||
```text
|
||||
~/.config/umbra-trans/remote-broker.json
|
||||
```
|
||||
|
||||
Review it. The `known_hosts` path must already contain the verified oldbox host
|
||||
key. Do not use `StrictHostKeyChecking=no` or an empty known-hosts file.
|
||||
|
||||
The first credential profiles can be entered through the MCP tool
|
||||
`credential_set`, which opens the popup, or directly from a terminal:
|
||||
|
||||
```bash
|
||||
python3 -m remote_credential_broker.broker \
|
||||
--credential-profile oldbox-umbra-ssh \
|
||||
--purpose "SSH password for umbra@oldbox"
|
||||
```
|
||||
|
||||
```bash
|
||||
python3 -m remote_credential_broker.broker \
|
||||
--credential-profile oldbox-umbra-sudo \
|
||||
--purpose "sudo password for umbra@oldbox"
|
||||
```
|
||||
|
||||
The command prints only whether the profile was stored. It never prints the
|
||||
secret.
|
||||
|
||||
## OpenCode registration
|
||||
|
||||
Run the module from the directory containing the `remote_credential_broker`
|
||||
package, or use an absolute Python path and package directory. A local server
|
||||
entry has this shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcp": {
|
||||
"servers": {
|
||||
"remote-broker": {
|
||||
"type": "local",
|
||||
"command": [
|
||||
"/absolute/path/to/python",
|
||||
"-m",
|
||||
"remote_credential_broker.broker"
|
||||
],
|
||||
"environment": {
|
||||
"PYTHONPATH": "/absolute/path/to/package/parent"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The current OpenCode installation may use the older top-level `mcp` object;
|
||||
use the format accepted by the installed version. Do not put a password in the
|
||||
configuration file.
|
||||
|
||||
## Tools
|
||||
|
||||
- `credential_set` — opens the native popup and stores a profile.
|
||||
- `credential_status` — reports existence only.
|
||||
- `credential_delete` — removes a profile.
|
||||
- `remote_exec` — runs a command as the configured SSH account.
|
||||
- `remote_sudo_exec` — runs one exact allowlisted command through sudo.
|
||||
- `remote_status` — runs fixed identity/hostname probes.
|
||||
|
||||
The model can request a popup or operation, but cannot read the secret entered
|
||||
in the popup.
|
||||
7
infra/remote_credential_broker/__init__.py
Normal file
7
infra/remote_credential_broker/__init__.py
Normal file
@@ -0,0 +1,7 @@
|
||||
"""Local, stdio MCP broker for controlled remote operations.
|
||||
|
||||
The broker keeps passwords in the macOS Keychain and exposes only operations,
|
||||
never secret values, to MCP clients.
|
||||
"""
|
||||
|
||||
__all__: list[str] = []
|
||||
3
infra/remote_credential_broker/__main__.py
Normal file
3
infra/remote_credential_broker/__main__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
from .broker import main
|
||||
|
||||
raise SystemExit(main())
|
||||
29
infra/remote_credential_broker/askpass_helper.py
Executable file
29
infra/remote_credential_broker/askpass_helper.py
Executable file
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
|
||||
|
||||
def main() -> int:
|
||||
socket_path = os.environ.get("UMBRA_BROKER_ASKPASS_SOCKET", "")
|
||||
if not socket_path:
|
||||
print("credential broker: askpass socket is not configured", file=sys.stderr)
|
||||
return 2
|
||||
try:
|
||||
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as channel:
|
||||
channel.connect(socket_path)
|
||||
secret = channel.recv(4096)
|
||||
except OSError as exc:
|
||||
print(f"credential broker: askpass channel failed: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
if not secret:
|
||||
return 1
|
||||
sys.stdout.buffer.write(secret)
|
||||
sys.stdout.buffer.flush()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
435
infra/remote_credential_broker/broker.py
Normal file
435
infra/remote_credential_broker/broker.py
Normal file
@@ -0,0 +1,435 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, TextIO
|
||||
|
||||
from .config import BrokerConfig, ConfigError
|
||||
from .executor import ExecutionError, RemoteExecutor
|
||||
from .keychain import (
|
||||
CredentialError,
|
||||
CredentialNotFound,
|
||||
CredentialVault,
|
||||
MacKeychain,
|
||||
UnsupportedPlatformError,
|
||||
validate_profile,
|
||||
)
|
||||
from .popup import PromptCancelled, SecretPrompt, macos_secret_prompt
|
||||
|
||||
_DEFAULT_CONFIG = Path("~/.config/umbra-trans/remote-broker.json").expanduser()
|
||||
_DEFAULT_STATE = Path("~/.config/umbra-trans/remote-broker-state.json").expanduser()
|
||||
_MAX_RESULT_CHARS = 1_000_000
|
||||
|
||||
|
||||
class BrokerState:
|
||||
"""Non-secret metadata about profiles that have been configured locally."""
|
||||
|
||||
def __init__(self, path: Path) -> None:
|
||||
self.path = path
|
||||
self._lock = threading.RLock()
|
||||
|
||||
def _read(self) -> dict[str, Any]:
|
||||
try:
|
||||
raw = json.loads(self.path.read_text(encoding="utf-8"))
|
||||
except FileNotFoundError:
|
||||
return {"profiles": {}}
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError("Unable to read broker state") from exc
|
||||
if not isinstance(raw, dict) or not isinstance(raw.get("profiles", {}), dict):
|
||||
raise RuntimeError("Broker state is malformed")
|
||||
return raw
|
||||
|
||||
def _write(self, value: dict[str, Any]) -> None:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, temporary_name = tempfile.mkstemp(prefix=".remote-broker-state.", dir=self.path.parent)
|
||||
try:
|
||||
os.fchmod(fd, 0o600)
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
json.dump(value, handle, indent=2, sort_keys=True)
|
||||
handle.write("\n")
|
||||
os.replace(temporary_name, self.path)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(temporary_name)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
raise
|
||||
|
||||
def record(self, profile: str, purpose: str, ttl_seconds: int | None = None) -> None:
|
||||
with self._lock:
|
||||
value = self._read()
|
||||
metadata: dict[str, Any] = {
|
||||
"purpose": purpose,
|
||||
"updated_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
if ttl_seconds is not None:
|
||||
metadata["expires_at"] = (
|
||||
datetime.now(timezone.utc) + timedelta(seconds=int(ttl_seconds))
|
||||
).isoformat()
|
||||
value["profiles"][profile] = metadata
|
||||
self._write(value)
|
||||
|
||||
def remove(self, profile: str) -> None:
|
||||
with self._lock:
|
||||
value = self._read()
|
||||
value["profiles"].pop(profile, None)
|
||||
self._write(value)
|
||||
|
||||
def profiles(self) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
return dict(self._read().get("profiles", {}))
|
||||
|
||||
|
||||
class RemoteCredentialBroker:
|
||||
"""MCP-facing broker; secret values never cross the tool boundary."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store: CredentialVault,
|
||||
config: BrokerConfig,
|
||||
state: BrokerState,
|
||||
prompt: SecretPrompt,
|
||||
executor: RemoteExecutor,
|
||||
) -> None:
|
||||
self.store = store
|
||||
self.config = config
|
||||
self.state = state
|
||||
self.prompt = prompt
|
||||
self.executor = executor
|
||||
|
||||
@staticmethod
|
||||
def _text_result(value: Any, *, is_error: bool = False) -> dict[str, Any]:
|
||||
return {
|
||||
"content": [{"type": "text", "text": json.dumps(value, ensure_ascii=False)}],
|
||||
"isError": is_error,
|
||||
}
|
||||
|
||||
def tool_definitions(self) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"name": "credential_set",
|
||||
"description": (
|
||||
"Open a native macOS hidden-password dialog and store the entered secret "
|
||||
"in the login Keychain. The secret is never returned to the model."
|
||||
),
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"required": ["profile", "purpose"],
|
||||
"properties": {
|
||||
"profile": {
|
||||
"type": "string",
|
||||
"pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$",
|
||||
"description": "Non-secret profile name, for example oldbox-umbra-ssh.",
|
||||
},
|
||||
"purpose": {
|
||||
"type": "string",
|
||||
"maxLength": 160,
|
||||
"description": "Short label shown in the popup; never include a password.",
|
||||
},
|
||||
"ttl_seconds": {
|
||||
"type": "integer",
|
||||
"minimum": 1,
|
||||
"maximum": 3600,
|
||||
"description": "Optional in-memory lifetime; omit for persistent Keychain storage.",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "credential_status",
|
||||
"description": "Report whether configured credential profiles exist; never reveal secrets.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"properties": {"profile": {"type": "string"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "credential_delete",
|
||||
"description": "Delete a stored credential profile from the local Keychain.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"required": ["profile"],
|
||||
"properties": {"profile": {"type": "string"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "remote_exec",
|
||||
"description": (
|
||||
"Run one bounded command as a configured SSH account. The command is a list, "
|
||||
"not a shell string; output is redacted and size-limited."
|
||||
),
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"required": ["profile", "command"],
|
||||
"properties": {
|
||||
"profile": {"type": "string"},
|
||||
"command": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": {"type": "string"},
|
||||
},
|
||||
"timeout_seconds": {
|
||||
"type": "integer",
|
||||
"minimum": 5,
|
||||
"maximum": 3600,
|
||||
"default": 120,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "remote_sudo_exec",
|
||||
"description": (
|
||||
"Run one exact, configured allowlist command through remote sudo. "
|
||||
"The password is read internally from Keychain and is never returned."
|
||||
),
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"required": ["profile", "command"],
|
||||
"properties": {
|
||||
"profile": {"type": "string"},
|
||||
"command": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": {"type": "string"},
|
||||
},
|
||||
"timeout_seconds": {
|
||||
"type": "integer",
|
||||
"minimum": 5,
|
||||
"maximum": 3600,
|
||||
"default": 120,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "remote_status",
|
||||
"description": "Run a small fixed status probe against a configured SSH account.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"required": ["profile"],
|
||||
"properties": {"profile": {"type": "string"}},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
def call_tool(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]:
|
||||
try:
|
||||
if name == "credential_set":
|
||||
profile = validate_profile(str(arguments["profile"]))
|
||||
purpose = str(arguments["purpose"]).strip()
|
||||
if not purpose or len(purpose) > 160:
|
||||
raise ExecutionError("purpose must be 1-160 characters")
|
||||
ttl_value = arguments.get("ttl_seconds")
|
||||
ttl_seconds = None if ttl_value is None else int(ttl_value)
|
||||
if ttl_seconds is not None and not 1 <= ttl_seconds <= 3600:
|
||||
raise ExecutionError("ttl_seconds must be between 1 and 3600")
|
||||
secret = self.prompt(profile, purpose)
|
||||
if secret is None:
|
||||
raise PromptCancelled("Credential entry was cancelled")
|
||||
self.store.set(profile, secret, ttl_seconds=ttl_seconds)
|
||||
self.state.record(profile, purpose, ttl_seconds=ttl_seconds)
|
||||
return self._text_result(
|
||||
{
|
||||
"profile": profile,
|
||||
"stored": True,
|
||||
"temporary": ttl_seconds is not None,
|
||||
"ttl_seconds": ttl_seconds,
|
||||
}
|
||||
)
|
||||
if name == "credential_status":
|
||||
profile = arguments.get("profile")
|
||||
if profile is not None:
|
||||
profile = validate_profile(str(profile))
|
||||
return self._text_result(
|
||||
{"profile": profile, "configured": self.store.exists(profile)}
|
||||
)
|
||||
statuses = []
|
||||
for name_, metadata in sorted(self.state.profiles().items()):
|
||||
statuses.append(
|
||||
{
|
||||
"profile": name_,
|
||||
"configured": self.store.exists(name_),
|
||||
"purpose": metadata.get("purpose", ""),
|
||||
}
|
||||
)
|
||||
return self._text_result({"profiles": statuses})
|
||||
if name == "credential_delete":
|
||||
profile = validate_profile(str(arguments["profile"]))
|
||||
self.store.delete(profile)
|
||||
self.state.remove(profile)
|
||||
return self._text_result({"profile": profile, "deleted": True})
|
||||
if name in {"remote_exec", "remote_sudo_exec"}:
|
||||
profile = self.config.get(str(arguments["profile"]))
|
||||
command = arguments.get("command")
|
||||
if not isinstance(command, list):
|
||||
raise ExecutionError("command must be an array of strings")
|
||||
if not all(isinstance(part, str) and part for part in command):
|
||||
raise ExecutionError("command must contain only non-empty strings")
|
||||
command_tuple = tuple(command)
|
||||
if name == "remote_sudo_exec" and not profile.sudo_allowed(command_tuple):
|
||||
raise ExecutionError(
|
||||
"The requested sudo command is not an exact allowlist entry for this profile"
|
||||
)
|
||||
timeout = int(arguments.get("timeout_seconds", 120))
|
||||
result = self.executor.run(
|
||||
profile,
|
||||
command,
|
||||
sudo=name == "remote_sudo_exec",
|
||||
timeout_seconds=timeout,
|
||||
)
|
||||
output = {
|
||||
"exit_code": result.exit_code,
|
||||
"stdout": result.stdout[:_MAX_RESULT_CHARS],
|
||||
"stderr": result.stderr[:_MAX_RESULT_CHARS],
|
||||
"duration_ms": result.duration_ms,
|
||||
}
|
||||
return self._text_result(output, is_error=result.exit_code != 0)
|
||||
if name == "remote_status":
|
||||
profile = self.config.get(str(arguments["profile"]))
|
||||
results = []
|
||||
for command in (["/usr/bin/id", "-un"], ["/usr/bin/hostname"]):
|
||||
result = self.executor.run(profile, command, timeout_seconds=30)
|
||||
results.append(
|
||||
{
|
||||
"command": command,
|
||||
"exit_code": result.exit_code,
|
||||
"stdout": result.stdout.strip(),
|
||||
"stderr": result.stderr.strip(),
|
||||
}
|
||||
)
|
||||
return self._text_result({"profile": profile.name, "checks": results})
|
||||
raise ExecutionError(f"Unknown tool: {name}")
|
||||
except (CredentialError, ConfigError, ExecutionError, PromptCancelled, KeyError, ValueError) as exc:
|
||||
return self._text_result({"error": str(exc)}, is_error=True)
|
||||
|
||||
def handle_json_rpc(self, message: dict[str, Any]) -> dict[str, Any] | None:
|
||||
request_id = message.get("id")
|
||||
method = message.get("method")
|
||||
if method == "initialize":
|
||||
params = message.get("params") or {}
|
||||
requested_protocol = params.get("protocolVersion")
|
||||
supported_protocols = {
|
||||
"2024-11-05",
|
||||
"2025-03-26",
|
||||
"2025-06-18",
|
||||
"2025-11-25",
|
||||
}
|
||||
protocol = (
|
||||
requested_protocol
|
||||
if isinstance(requested_protocol, str) and requested_protocol in supported_protocols
|
||||
else "2024-11-05"
|
||||
)
|
||||
result: dict[str, Any] = {
|
||||
"protocolVersion": protocol,
|
||||
"capabilities": {"tools": {"listChanged": False}},
|
||||
"serverInfo": {"name": "umbratrans-remote-broker", "version": "0.1.0"},
|
||||
}
|
||||
return self._response(request_id, result=result)
|
||||
if method in {"notifications/initialized", "notifications/cancelled"}:
|
||||
return None
|
||||
if method == "ping":
|
||||
return self._response(request_id, result={})
|
||||
if method == "tools/list":
|
||||
return self._response(request_id, result={"tools": self.tool_definitions()})
|
||||
if method == "resources/list":
|
||||
return self._response(request_id, result={"resources": []})
|
||||
if method == "prompts/list":
|
||||
return self._response(request_id, result={"prompts": []})
|
||||
if method == "tools/call":
|
||||
params = message.get("params") or {}
|
||||
name = params.get("name")
|
||||
arguments = params.get("arguments") or {}
|
||||
if not isinstance(name, str) or not isinstance(arguments, dict):
|
||||
return self._error(request_id, -32602, "Invalid tools/call parameters")
|
||||
return self._response(request_id, result=self.call_tool(name, arguments))
|
||||
if request_id is not None:
|
||||
return self._error(request_id, -32601, f"Method not found: {method}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _response(request_id: Any, *, result: Any) -> dict[str, Any]:
|
||||
return {"jsonrpc": "2.0", "id": request_id, "result": result}
|
||||
|
||||
@staticmethod
|
||||
def _error(request_id: Any, code: int, message: str) -> dict[str, Any]:
|
||||
return {"jsonrpc": "2.0", "id": request_id, "error": {"code": code, "message": message}}
|
||||
|
||||
def serve(self, input_stream: TextIO = sys.stdin, output_stream: TextIO = sys.stdout) -> None:
|
||||
for line in input_stream:
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
message = json.loads(line)
|
||||
if not isinstance(message, dict):
|
||||
raise ValueError("message must be a JSON object")
|
||||
response = self.handle_json_rpc(message)
|
||||
except json.JSONDecodeError:
|
||||
response = self._error(None, -32700, "Parse error")
|
||||
except Exception:
|
||||
response = self._error(None, -32603, "Internal broker error")
|
||||
if response is not None:
|
||||
output_stream.write(json.dumps(response, ensure_ascii=False) + "\n")
|
||||
output_stream.flush()
|
||||
|
||||
|
||||
def _load_runtime() -> tuple[CredentialVault, BrokerConfig, BrokerState, RemoteCredentialBroker]:
|
||||
config_path = Path(os.environ.get("UMBRA_BROKER_CONFIG", str(_DEFAULT_CONFIG))).expanduser()
|
||||
state_path = Path(os.environ.get("UMBRA_BROKER_STATE", str(_DEFAULT_STATE))).expanduser()
|
||||
store = CredentialVault()
|
||||
config = BrokerConfig.load(config_path)
|
||||
state = BrokerState(state_path)
|
||||
audit_path = Path(
|
||||
os.environ.get("UMBRA_BROKER_AUDIT_LOG", "~/.local/state/umbratrans/remote-broker.log")
|
||||
).expanduser()
|
||||
executor = RemoteExecutor(store, audit_path=audit_path)
|
||||
broker = RemoteCredentialBroker(store, config, state, macos_secret_prompt, executor)
|
||||
return store, config, state, broker
|
||||
|
||||
|
||||
def _cli_credential_set(profile: str, purpose: str) -> int:
|
||||
store, _, state, _ = _load_runtime()
|
||||
try:
|
||||
secret = macos_secret_prompt(validate_profile(profile), purpose)
|
||||
if secret is None:
|
||||
return 2
|
||||
store.set(profile, secret)
|
||||
state.record(profile, purpose)
|
||||
except (CredentialError, UnsupportedPlatformError, PromptCancelled, RuntimeError) as exc:
|
||||
print(f"credential setup failed: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
print(f"stored credential profile {profile!r}; secret was not printed")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="UmbraTrans local remote credential MCP broker")
|
||||
parser.add_argument("--credential-profile", help="open the popup for one profile and exit")
|
||||
parser.add_argument("--purpose", default="remote access", help="label shown in the credential popup")
|
||||
args = parser.parse_args(argv)
|
||||
if args.credential_profile:
|
||||
return _cli_credential_set(args.credential_profile, args.purpose)
|
||||
try:
|
||||
_, _, _, broker = _load_runtime()
|
||||
broker.serve()
|
||||
except (ConfigError, CredentialError, UnsupportedPlatformError, RuntimeError) as exc:
|
||||
print(f"remote broker startup failed: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
26
infra/remote_credential_broker/config.example.json
Normal file
26
infra/remote_credential_broker/config.example.json
Normal file
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"profiles": {
|
||||
"oldbox-umbra": {
|
||||
"host": "oldbox.bdmurphy.me",
|
||||
"username": "umbra",
|
||||
"port": 22,
|
||||
"ssh_credential": "oldbox-umbra-ssh",
|
||||
"sudo_credential": "oldbox-umbra-sudo",
|
||||
"known_hosts": "~/.ssh/known_hosts",
|
||||
"allowed_sudo_commands": [
|
||||
[
|
||||
"/home/umbra/umbra-enterprise-bootstrap/bootstrap_umbracontrol_database.sh",
|
||||
"--apply",
|
||||
"--config",
|
||||
"/home/umbra/umbra-enterprise-bootstrap/enterprise-bootstrap.env"
|
||||
],
|
||||
[
|
||||
"/home/umbra/umbra-enterprise-bootstrap/bootstrap_umbracontrol_database.sh",
|
||||
"--verify",
|
||||
"--config",
|
||||
"/home/umbra/umbra-enterprise-bootstrap/enterprise-bootstrap.env"
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
95
infra/remote_credential_broker/config.py
Normal file
95
infra/remote_credential_broker/config.py
Normal file
@@ -0,0 +1,95 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
_PROFILE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
|
||||
_HOST_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9.-]{0,252}$")
|
||||
|
||||
|
||||
class ConfigError(ValueError):
|
||||
"""Raised when the broker configuration is invalid."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RemoteProfile:
|
||||
name: str
|
||||
host: str
|
||||
username: str
|
||||
port: int = 22
|
||||
ssh_credential: str | None = None
|
||||
sudo_credential: str | None = None
|
||||
known_hosts: str = "~/.ssh/known_hosts"
|
||||
allowed_sudo_commands: tuple[tuple[str, ...], ...] = ()
|
||||
|
||||
def validate(self) -> None:
|
||||
if not _PROFILE_RE.fullmatch(self.name):
|
||||
raise ConfigError(f"Invalid remote profile name: {self.name!r}")
|
||||
if not _HOST_RE.fullmatch(self.host):
|
||||
raise ConfigError(f"Invalid host in profile {self.name!r}")
|
||||
if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_.-]{0,31}", self.username):
|
||||
raise ConfigError(f"Invalid SSH username in profile {self.name!r}")
|
||||
if not 1 <= self.port <= 65535:
|
||||
raise ConfigError(f"Invalid SSH port in profile {self.name!r}")
|
||||
for credential in (self.ssh_credential, self.sudo_credential):
|
||||
if credential is not None and not _PROFILE_RE.fullmatch(credential):
|
||||
raise ConfigError(f"Invalid credential profile in {self.name!r}")
|
||||
for command in self.allowed_sudo_commands:
|
||||
if not command or any(not isinstance(part, str) or not part for part in command):
|
||||
raise ConfigError(f"Invalid sudo command allowlist in profile {self.name!r}")
|
||||
if any("\x00" in part for part in command):
|
||||
raise ConfigError(f"NUL byte in sudo command allowlist in {self.name!r}")
|
||||
|
||||
def sudo_allowed(self, command: tuple[str, ...]) -> bool:
|
||||
return command in self.allowed_sudo_commands
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BrokerConfig:
|
||||
profiles: dict[str, RemoteProfile]
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: Path) -> BrokerConfig:
|
||||
if not path.exists():
|
||||
return cls(profiles={})
|
||||
try:
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise ConfigError(f"Unable to read broker config: {path}") from exc
|
||||
if not isinstance(raw, dict) or not isinstance(raw.get("profiles", {}), dict):
|
||||
raise ConfigError("Broker config must contain a profiles object")
|
||||
profiles: dict[str, RemoteProfile] = {}
|
||||
for name, item in raw.get("profiles", {}).items():
|
||||
if not isinstance(item, dict):
|
||||
raise ConfigError(f"Profile {name!r} must be an object")
|
||||
commands_raw = item.get("allowed_sudo_commands", [])
|
||||
if not isinstance(commands_raw, list):
|
||||
raise ConfigError(f"allowed_sudo_commands for {name!r} must be a list")
|
||||
commands: list[tuple[str, ...]] = []
|
||||
for command in commands_raw:
|
||||
if not isinstance(command, list) or not all(isinstance(x, str) for x in command):
|
||||
raise ConfigError(f"Invalid sudo command entry for {name!r}")
|
||||
commands.append(tuple(command))
|
||||
profile = RemoteProfile(
|
||||
name=name,
|
||||
host=str(item.get("host", "")),
|
||||
username=str(item.get("username", "")),
|
||||
port=int(item.get("port", 22)),
|
||||
ssh_credential=item.get("ssh_credential"),
|
||||
sudo_credential=item.get("sudo_credential"),
|
||||
known_hosts=str(item.get("known_hosts", "~/.ssh/known_hosts")),
|
||||
allowed_sudo_commands=tuple(commands),
|
||||
)
|
||||
profile.validate()
|
||||
profiles[name] = profile
|
||||
return cls(profiles=profiles)
|
||||
|
||||
def get(self, name: str) -> RemoteProfile:
|
||||
try:
|
||||
return self.profiles[name]
|
||||
except KeyError as exc:
|
||||
raise ConfigError(f"Unknown remote profile: {name!r}") from exc
|
||||
242
infra/remote_credential_broker/executor.py
Normal file
242
infra/remote_credential_broker/executor.py
Normal file
@@ -0,0 +1,242 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Callable, Sequence
|
||||
|
||||
from .config import ConfigError, RemoteProfile
|
||||
from .keychain import CredentialError, CredentialVault
|
||||
|
||||
Runner = Callable[..., subprocess.CompletedProcess[bytes]]
|
||||
|
||||
|
||||
class ExecutionError(RuntimeError):
|
||||
"""Raised when a remote operation cannot be safely started."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ExecutionResult:
|
||||
exit_code: int
|
||||
stdout: str
|
||||
stderr: str
|
||||
duration_ms: int
|
||||
|
||||
|
||||
class SecretRedactor:
|
||||
"""Redact known secrets before output leaves the broker."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._secrets: set[str] = set()
|
||||
|
||||
def add(self, secret: str | None) -> None:
|
||||
if secret:
|
||||
self._secrets.add(secret)
|
||||
|
||||
def redact(self, value: str) -> str:
|
||||
result = value
|
||||
for secret in sorted(self._secrets, key=len, reverse=True):
|
||||
if secret:
|
||||
result = result.replace(secret, "<redacted>")
|
||||
return result
|
||||
|
||||
|
||||
class _AskpassServer:
|
||||
"""One-shot local channel used instead of exposing a Keychain item to ssh."""
|
||||
|
||||
def __init__(self, secret: str) -> None:
|
||||
self._directory = Path(tempfile.mkdtemp(prefix="remote-broker-askpass-"))
|
||||
self._directory.chmod(0o700)
|
||||
self.path = self._directory / "secret.sock"
|
||||
self._secret = secret.encode("utf-8")
|
||||
self._stop = threading.Event()
|
||||
self._socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
self._socket.bind(str(self.path))
|
||||
self._socket.listen(8)
|
||||
self._socket.settimeout(0.2)
|
||||
self._thread = threading.Thread(target=self._serve, name="remote-broker-askpass", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def _serve(self) -> None:
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
connection, _ = self._socket.accept()
|
||||
except socket.timeout:
|
||||
continue
|
||||
except OSError:
|
||||
return
|
||||
with connection:
|
||||
try:
|
||||
connection.sendall(self._secret)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def close(self) -> None:
|
||||
self._stop.set()
|
||||
try:
|
||||
self._socket.close()
|
||||
except OSError:
|
||||
pass
|
||||
self._thread.join(timeout=1)
|
||||
shutil.rmtree(self._directory, ignore_errors=True)
|
||||
|
||||
|
||||
class RemoteExecutor:
|
||||
"""Run bounded SSH operations; sudo commands are exact allowlist entries."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store: CredentialVault,
|
||||
*,
|
||||
runner: Runner = subprocess.run,
|
||||
askpass_path: Path | None = None,
|
||||
audit_path: Path | None = None,
|
||||
) -> None:
|
||||
self._store = store
|
||||
self._runner = runner
|
||||
self._askpass_path = askpass_path or Path(__file__).with_name("askpass_helper.py")
|
||||
if self._askpass_path.exists():
|
||||
self._askpass_path.chmod(0o700)
|
||||
self._audit_path = audit_path
|
||||
|
||||
@staticmethod
|
||||
def _validate_command(command: Sequence[str]) -> tuple[str, ...]:
|
||||
if not command or not all(isinstance(part, str) and part for part in command):
|
||||
raise ExecutionError("Remote command must be a non-empty list of strings")
|
||||
if any("\x00" in part for part in command):
|
||||
raise ExecutionError("Remote command contains a NUL byte")
|
||||
return tuple(command)
|
||||
|
||||
def _audit(self, profile: RemoteProfile, command: tuple[str, ...], sudo: bool, result: int) -> None:
|
||||
if self._audit_path is None:
|
||||
return
|
||||
self._audit_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Store command metadata only. Never store passwords or command stdin.
|
||||
line = (
|
||||
f"profile={profile.name} sudo={str(sudo).lower()} "
|
||||
f"command={shlex.join(command)} exit={result}\n"
|
||||
)
|
||||
fd = os.open(self._audit_path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
|
||||
try:
|
||||
os.write(fd, line.encode("utf-8"))
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
def _base_ssh_command(self, profile: RemoteProfile, remote_command: str, use_password: bool) -> list[str]:
|
||||
known_hosts = str(Path(profile.known_hosts).expanduser())
|
||||
command = [
|
||||
"ssh",
|
||||
"-p",
|
||||
str(profile.port),
|
||||
"-o",
|
||||
"ConnectTimeout=15",
|
||||
"-o",
|
||||
"ServerAliveInterval=15",
|
||||
"-o",
|
||||
"ServerAliveCountMax=2",
|
||||
"-o",
|
||||
"StrictHostKeyChecking=yes",
|
||||
"-o",
|
||||
f"UserKnownHostsFile={known_hosts}",
|
||||
]
|
||||
if use_password:
|
||||
command.extend(
|
||||
[
|
||||
"-o",
|
||||
"BatchMode=no",
|
||||
"-o",
|
||||
"PreferredAuthentications=publickey,password",
|
||||
"-o",
|
||||
"PubkeyAuthentication=yes",
|
||||
]
|
||||
)
|
||||
command.extend([f"{profile.username}@{profile.host}", remote_command])
|
||||
return command
|
||||
|
||||
def run(
|
||||
self,
|
||||
profile: RemoteProfile,
|
||||
command: Sequence[str],
|
||||
*,
|
||||
sudo: bool = False,
|
||||
timeout_seconds: int = 120,
|
||||
) -> ExecutionResult:
|
||||
command_tuple = self._validate_command(command)
|
||||
if not 5 <= timeout_seconds <= 3600:
|
||||
raise ExecutionError("timeout_seconds must be between 5 and 3600")
|
||||
if sudo and not profile.sudo_allowed(command_tuple):
|
||||
raise ExecutionError(
|
||||
"The requested sudo command is not an exact allowlist entry for this profile"
|
||||
)
|
||||
|
||||
ssh_secret: str | None = None
|
||||
sudo_secret: str | None = None
|
||||
redactor = SecretRedactor()
|
||||
if profile.ssh_credential:
|
||||
try:
|
||||
ssh_secret = self._store.get(profile.ssh_credential)
|
||||
except CredentialError as exc:
|
||||
raise ExecutionError(
|
||||
f"SSH credential is not configured for profile {profile.name!r}"
|
||||
) from exc
|
||||
redactor.add(ssh_secret)
|
||||
if sudo:
|
||||
if not profile.sudo_credential:
|
||||
raise ExecutionError(f"No sudo credential is configured for profile {profile.name!r}")
|
||||
try:
|
||||
sudo_secret = self._store.get(profile.sudo_credential)
|
||||
except CredentialError as exc:
|
||||
raise ExecutionError(
|
||||
f"Sudo credential is not configured for profile {profile.name!r}"
|
||||
) from exc
|
||||
redactor.add(sudo_secret)
|
||||
|
||||
remote_argv = list(command_tuple)
|
||||
if sudo:
|
||||
remote_command = "sudo -S -p '' -- " + shlex.join(remote_argv)
|
||||
else:
|
||||
remote_command = shlex.join(remote_argv)
|
||||
ssh_command = self._base_ssh_command(profile, remote_command, use_password=ssh_secret is not None)
|
||||
environment = os.environ.copy()
|
||||
askpass_server: _AskpassServer | None = None
|
||||
if ssh_secret is not None:
|
||||
askpass_server = _AskpassServer(ssh_secret)
|
||||
environment.update(
|
||||
{
|
||||
"SSH_ASKPASS": str(self._askpass_path),
|
||||
"SSH_ASKPASS_REQUIRE": "force",
|
||||
"DISPLAY": environment.get("DISPLAY", ":0"),
|
||||
"UMBRA_BROKER_ASKPASS_SOCKET": str(askpass_server.path),
|
||||
}
|
||||
)
|
||||
stdin_data = (sudo_secret + "\n").encode("utf-8") if sudo_secret is not None else None
|
||||
started = time.monotonic()
|
||||
try:
|
||||
completed = self._runner(
|
||||
ssh_command,
|
||||
input=stdin_data,
|
||||
capture_output=True,
|
||||
timeout=timeout_seconds,
|
||||
env=environment,
|
||||
check=False,
|
||||
)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
raise ExecutionError("Remote operation timed out") from exc
|
||||
except OSError as exc:
|
||||
raise ExecutionError("Unable to start ssh") from exc
|
||||
finally:
|
||||
if askpass_server is not None:
|
||||
askpass_server.close()
|
||||
duration_ms = int((time.monotonic() - started) * 1000)
|
||||
stdout = redactor.redact(completed.stdout.decode("utf-8", errors="replace"))
|
||||
stderr = redactor.redact(completed.stderr.decode("utf-8", errors="replace"))
|
||||
result = ExecutionResult(completed.returncode, stdout, stderr, duration_ms)
|
||||
self._audit(profile, command_tuple, sudo, result.exit_code)
|
||||
return result
|
||||
299
infra/remote_credential_broker/keychain.py
Normal file
299
infra/remote_credential_broker/keychain.py
Normal file
@@ -0,0 +1,299 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from ctypes import POINTER, c_int32, c_uint32, c_void_p, create_string_buffer
|
||||
from typing import Final
|
||||
|
||||
|
||||
_PROFILE_RE: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
|
||||
_SERVICE_DEFAULT: Final = "com.umbratrans.remote-broker"
|
||||
_ERR_SEC_ITEM_NOT_FOUND: Final = -25300
|
||||
|
||||
|
||||
class CredentialError(RuntimeError):
|
||||
"""Base exception for credential-store failures."""
|
||||
|
||||
|
||||
class CredentialNotFound(CredentialError):
|
||||
"""Raised when a requested Keychain item does not exist."""
|
||||
|
||||
|
||||
class UnsupportedPlatformError(CredentialError):
|
||||
"""Raised when the broker is not running on macOS."""
|
||||
|
||||
|
||||
def validate_profile(profile: str) -> str:
|
||||
"""Validate a profile name before using it as a Keychain account name."""
|
||||
if not isinstance(profile, str) or not _PROFILE_RE.fullmatch(profile):
|
||||
raise CredentialError(
|
||||
"Credential profile must contain only letters, digits, '.', '_' and '-' "
|
||||
"and must be 1-128 characters"
|
||||
)
|
||||
return profile
|
||||
|
||||
|
||||
class MacKeychain:
|
||||
"""Small ctypes wrapper around the legacy macOS generic-password API.
|
||||
|
||||
Password bytes are passed directly to the Security framework and are never
|
||||
placed in a subprocess argument, environment variable, log line, or MCP
|
||||
response. The API is deprecated by Apple but remains available on the
|
||||
supported macOS hosts and avoids adding a Python dependency.
|
||||
"""
|
||||
|
||||
def __init__(self, service: str = _SERVICE_DEFAULT) -> None:
|
||||
self.service = service
|
||||
self._lock = threading.RLock()
|
||||
self._lib: ctypes.CDLL | None = None
|
||||
self._cf: ctypes.CDLL | None = None
|
||||
|
||||
def _load(self) -> tuple[ctypes.CDLL, ctypes.CDLL]:
|
||||
if sys.platform != "darwin":
|
||||
raise UnsupportedPlatformError(
|
||||
"The macOS Keychain backend requires macOS; use a native secret store on other hosts"
|
||||
)
|
||||
if self._lib is None:
|
||||
lib = ctypes.CDLL("/System/Library/Frameworks/Security.framework/Security")
|
||||
lib.SecKeychainAddGenericPassword.argtypes = [
|
||||
c_void_p,
|
||||
c_uint32,
|
||||
c_void_p,
|
||||
c_uint32,
|
||||
c_void_p,
|
||||
c_uint32,
|
||||
c_void_p,
|
||||
POINTER(c_void_p),
|
||||
]
|
||||
lib.SecKeychainAddGenericPassword.restype = c_int32
|
||||
lib.SecKeychainFindGenericPassword.argtypes = [
|
||||
c_void_p,
|
||||
c_uint32,
|
||||
c_void_p,
|
||||
c_uint32,
|
||||
c_void_p,
|
||||
POINTER(c_uint32),
|
||||
POINTER(c_void_p),
|
||||
POINTER(c_void_p),
|
||||
]
|
||||
lib.SecKeychainFindGenericPassword.restype = c_int32
|
||||
lib.SecKeychainItemModifyAttributesAndData.argtypes = [
|
||||
c_void_p,
|
||||
c_void_p,
|
||||
c_uint32,
|
||||
c_void_p,
|
||||
]
|
||||
lib.SecKeychainItemModifyAttributesAndData.restype = c_int32
|
||||
lib.SecKeychainItemFreeContent.argtypes = [c_void_p, c_void_p]
|
||||
lib.SecKeychainItemFreeContent.restype = c_int32
|
||||
lib.SecKeychainItemDelete.argtypes = [c_void_p]
|
||||
lib.SecKeychainItemDelete.restype = c_int32
|
||||
self._lib = lib
|
||||
if self._cf is None:
|
||||
cf = ctypes.CDLL("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation")
|
||||
cf.CFRelease.argtypes = [c_void_p]
|
||||
cf.CFRelease.restype = None
|
||||
self._cf = cf
|
||||
return self._lib, self._cf
|
||||
|
||||
@staticmethod
|
||||
def _error(status: int, operation: str) -> CredentialError:
|
||||
return CredentialError(f"Keychain {operation} failed with OSStatus {status}")
|
||||
|
||||
def _find(
|
||||
self,
|
||||
profile: str,
|
||||
*,
|
||||
include_password: bool,
|
||||
) -> tuple[int, bytes | None, c_void_p | None]:
|
||||
lib, _ = self._load()
|
||||
profile = validate_profile(profile)
|
||||
service_bytes = self.service.encode("utf-8")
|
||||
account_bytes = profile.encode("utf-8")
|
||||
service_buf = create_string_buffer(service_bytes)
|
||||
account_buf = create_string_buffer(account_bytes)
|
||||
length = c_uint32(0)
|
||||
data = c_void_p()
|
||||
item = c_void_p()
|
||||
status = lib.SecKeychainFindGenericPassword(
|
||||
None,
|
||||
len(service_bytes),
|
||||
service_buf,
|
||||
len(account_bytes),
|
||||
account_buf,
|
||||
ctypes.byref(length) if include_password else None,
|
||||
ctypes.byref(data) if include_password else None,
|
||||
ctypes.byref(item),
|
||||
)
|
||||
if status != 0:
|
||||
return status, None, None
|
||||
if not include_password:
|
||||
return status, None, item
|
||||
try:
|
||||
value = ctypes.string_at(data, length.value) if data.value else b""
|
||||
return status, value, item
|
||||
finally:
|
||||
# The pointer is owned by the Security framework.
|
||||
lib.SecKeychainItemFreeContent(None, data)
|
||||
|
||||
def _release(self, item: c_void_p | None) -> None:
|
||||
if item is None or not item.value:
|
||||
return
|
||||
_, cf = self._load()
|
||||
cf.CFRelease(item)
|
||||
|
||||
def set(self, profile: str, secret: str) -> None:
|
||||
"""Store or replace a secret in the default login Keychain."""
|
||||
if not isinstance(secret, str) or not secret:
|
||||
raise CredentialError("Credential secret must not be empty")
|
||||
with self._lock:
|
||||
lib, _ = self._load()
|
||||
profile = validate_profile(profile)
|
||||
status, _, item = self._find(profile, include_password=False)
|
||||
if status == 0 and item is not None:
|
||||
secret_bytes = secret.encode("utf-8")
|
||||
secret_buf = create_string_buffer(secret_bytes)
|
||||
try:
|
||||
update_status = lib.SecKeychainItemModifyAttributesAndData(
|
||||
item,
|
||||
None,
|
||||
len(secret_bytes),
|
||||
secret_buf,
|
||||
)
|
||||
finally:
|
||||
self._release(item)
|
||||
if update_status != 0:
|
||||
raise self._error(update_status, "update")
|
||||
return
|
||||
if status != _ERR_SEC_ITEM_NOT_FOUND:
|
||||
raise self._error(status, "lookup")
|
||||
|
||||
service_bytes = self.service.encode("utf-8")
|
||||
account_bytes = profile.encode("utf-8")
|
||||
secret_bytes = secret.encode("utf-8")
|
||||
service_buf = create_string_buffer(service_bytes)
|
||||
account_buf = create_string_buffer(account_bytes)
|
||||
secret_buf = create_string_buffer(secret_bytes)
|
||||
new_item = c_void_p()
|
||||
add_status = lib.SecKeychainAddGenericPassword(
|
||||
None,
|
||||
len(service_bytes),
|
||||
service_buf,
|
||||
len(account_bytes),
|
||||
account_buf,
|
||||
len(secret_bytes),
|
||||
secret_buf,
|
||||
ctypes.byref(new_item),
|
||||
)
|
||||
self._release(new_item)
|
||||
if add_status != 0:
|
||||
raise self._error(add_status, "add")
|
||||
|
||||
def get(self, profile: str) -> str:
|
||||
"""Retrieve a secret for internal broker use only."""
|
||||
with self._lock:
|
||||
status, value, item = self._find(profile, include_password=True)
|
||||
self._release(item)
|
||||
if status == _ERR_SEC_ITEM_NOT_FOUND:
|
||||
raise CredentialNotFound(f"No stored credential for profile {profile!r}")
|
||||
if status != 0:
|
||||
raise self._error(status, "lookup")
|
||||
if value is None:
|
||||
raise CredentialError("Keychain returned an empty credential")
|
||||
return value.decode("utf-8")
|
||||
|
||||
def delete(self, profile: str) -> None:
|
||||
"""Delete a stored secret; missing items are treated as already absent."""
|
||||
with self._lock:
|
||||
lib, _ = self._load()
|
||||
status, _, item = self._find(profile, include_password=False)
|
||||
if status == _ERR_SEC_ITEM_NOT_FOUND:
|
||||
return
|
||||
if status != 0:
|
||||
raise self._error(status, "lookup")
|
||||
if item is None:
|
||||
return
|
||||
try:
|
||||
delete_status = lib.SecKeychainItemDelete(item)
|
||||
finally:
|
||||
self._release(item)
|
||||
if delete_status != 0:
|
||||
raise self._error(delete_status, "delete")
|
||||
|
||||
def exists(self, profile: str) -> bool:
|
||||
"""Return whether a profile exists without retrieving its secret."""
|
||||
with self._lock:
|
||||
status, _, item = self._find(profile, include_password=False)
|
||||
self._release(item)
|
||||
if status == 0:
|
||||
return True
|
||||
if status == _ERR_SEC_ITEM_NOT_FOUND:
|
||||
return False
|
||||
raise self._error(status, "lookup")
|
||||
|
||||
|
||||
class CredentialVault:
|
||||
"""Persistent Keychain storage plus explicitly temporary in-memory profiles."""
|
||||
|
||||
def __init__(self, persistent: MacKeychain | None = None) -> None:
|
||||
self.persistent = persistent or MacKeychain()
|
||||
self.service = self.persistent.service
|
||||
self._lock = threading.RLock()
|
||||
self._temporary: dict[str, tuple[str, float]] = {}
|
||||
self._timers: dict[str, threading.Timer] = {}
|
||||
|
||||
def _expire(self, profile: str) -> None:
|
||||
with self._lock:
|
||||
self._temporary.pop(profile, None)
|
||||
self._timers.pop(profile, None)
|
||||
|
||||
def set(self, profile: str, secret: str, ttl_seconds: int | None = None) -> None:
|
||||
profile = validate_profile(profile)
|
||||
if not isinstance(secret, str) or not secret:
|
||||
raise CredentialError("Credential secret must not be empty")
|
||||
with self._lock:
|
||||
old_timer = self._timers.pop(profile, None)
|
||||
if old_timer is not None:
|
||||
old_timer.cancel()
|
||||
if ttl_seconds is None:
|
||||
self._temporary.pop(profile, None)
|
||||
self.persistent.set(profile, secret)
|
||||
return
|
||||
if not 1 <= int(ttl_seconds) <= 3600:
|
||||
raise CredentialError("ttl_seconds must be between 1 and 3600")
|
||||
expires_at = time.monotonic() + int(ttl_seconds)
|
||||
self._temporary[profile] = (secret, expires_at)
|
||||
timer = threading.Timer(int(ttl_seconds), self._expire, args=(profile,))
|
||||
timer.daemon = True
|
||||
self._timers[profile] = timer
|
||||
timer.start()
|
||||
|
||||
def get(self, profile: str) -> str:
|
||||
profile = validate_profile(profile)
|
||||
with self._lock:
|
||||
temporary = self._temporary.get(profile)
|
||||
if temporary is not None:
|
||||
secret, expires_at = temporary
|
||||
if expires_at > time.monotonic():
|
||||
return secret
|
||||
self._expire(profile)
|
||||
return self.persistent.get(profile)
|
||||
|
||||
def delete(self, profile: str) -> None:
|
||||
profile = validate_profile(profile)
|
||||
with self._lock:
|
||||
self._expire(profile)
|
||||
self.persistent.delete(profile)
|
||||
|
||||
def exists(self, profile: str) -> bool:
|
||||
profile = validate_profile(profile)
|
||||
with self._lock:
|
||||
temporary = self._temporary.get(profile)
|
||||
if temporary is not None:
|
||||
if temporary[1] > time.monotonic():
|
||||
return True
|
||||
self._expire(profile)
|
||||
return self.persistent.exists(profile)
|
||||
49
infra/remote_credential_broker/popup.py
Normal file
49
infra/remote_credential_broker/popup.py
Normal file
@@ -0,0 +1,49 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from typing import Protocol
|
||||
|
||||
|
||||
class PromptCancelled(RuntimeError):
|
||||
"""Raised when the user cancels a native credential prompt."""
|
||||
|
||||
|
||||
class SecretPrompt(Protocol):
|
||||
def __call__(self, profile: str, purpose: str) -> str | None:
|
||||
"""Return a secret from a local UI, or None when cancelled."""
|
||||
|
||||
|
||||
def _apple_string(value: str) -> str:
|
||||
return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
||||
|
||||
|
||||
def macos_secret_prompt(profile: str, purpose: str) -> str | None:
|
||||
"""Open a native macOS hidden-password dialog.
|
||||
|
||||
The returned value is intended to be passed directly to the Keychain
|
||||
backend. It is never printed, logged, or included in an MCP result.
|
||||
"""
|
||||
message = f"Enter the password for {purpose} ({profile})."
|
||||
script = (
|
||||
f'display dialog {_apple_string(message)} '
|
||||
'default answer "" with hidden answer buttons {"Cancel", "Save"} '
|
||||
'default button "Save" with title "UmbraTrans secure credential"'
|
||||
)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["/usr/bin/osascript", "-e", script],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
except subprocess.CalledProcessError as exc:
|
||||
detail = exc.stderr.strip().lower()
|
||||
if "user canceled" in detail or "-128" in detail:
|
||||
raise PromptCancelled("Credential entry was cancelled") from exc
|
||||
raise RuntimeError("The macOS credential dialog could not be completed") from exc
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
raise RuntimeError("The macOS credential dialog could not be opened") from exc
|
||||
|
||||
secret = result.stdout.rstrip("\r\n")
|
||||
return secret or None
|
||||
151
infra/remote_credential_broker/tests/test_broker.py
Normal file
151
infra/remote_credential_broker/tests/test_broker.py
Normal file
@@ -0,0 +1,151 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
from remote_credential_broker.broker import BrokerState, RemoteCredentialBroker
|
||||
from remote_credential_broker.config import BrokerConfig, RemoteProfile
|
||||
from remote_credential_broker.executor import ExecutionResult, SecretRedactor
|
||||
from remote_credential_broker.keychain import CredentialError, CredentialVault, validate_profile
|
||||
|
||||
|
||||
class FakeStore:
|
||||
service = "test-service"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.values: dict[str, str] = {}
|
||||
|
||||
def set(self, profile: str, secret: str, ttl_seconds: int | None = None) -> None:
|
||||
self.values[profile] = secret
|
||||
|
||||
def get(self, profile: str) -> str:
|
||||
return self.values[profile]
|
||||
|
||||
def delete(self, profile: str) -> None:
|
||||
self.values.pop(profile, None)
|
||||
|
||||
def exists(self, profile: str) -> bool:
|
||||
return profile in self.values
|
||||
|
||||
|
||||
class FakeExecutor:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[str, tuple[str, ...], bool]] = []
|
||||
|
||||
def run(
|
||||
self,
|
||||
profile: RemoteProfile,
|
||||
command: list[str] | tuple[str, ...],
|
||||
*,
|
||||
sudo: bool = False,
|
||||
timeout_seconds: int = 120,
|
||||
) -> ExecutionResult:
|
||||
self.calls.append((profile.name, tuple(command), sudo))
|
||||
return ExecutionResult(0, "ok\n", "", 1)
|
||||
|
||||
|
||||
class BrokerTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.store = FakeStore()
|
||||
self.state_path = Path("/tmp/remote-broker-test-state.json")
|
||||
self.state_path.unlink(missing_ok=True)
|
||||
self.state = BrokerState(self.state_path)
|
||||
self.executor = FakeExecutor()
|
||||
self.profile = RemoteProfile(
|
||||
name="oldbox-umbra",
|
||||
host="oldbox.bdmurphy.me",
|
||||
username="umbra",
|
||||
ssh_credential="oldbox-umbra-ssh",
|
||||
sudo_credential="oldbox-umbra-sudo",
|
||||
allowed_sudo_commands=(("/usr/local/sbin/umbracontrol-backup",),),
|
||||
)
|
||||
config = BrokerConfig(profiles={self.profile.name: self.profile})
|
||||
self.broker = RemoteCredentialBroker(
|
||||
self.store,
|
||||
config,
|
||||
self.state,
|
||||
lambda profile, purpose: "popup-secret",
|
||||
self.executor,
|
||||
)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.state_path.unlink(missing_ok=True)
|
||||
|
||||
def test_credential_popup_is_not_returned(self) -> None:
|
||||
result = self.broker.call_tool(
|
||||
"credential_set",
|
||||
{"profile": "oldbox-umbra-ssh", "purpose": "SSH for oldbox"},
|
||||
)
|
||||
encoded = json.dumps(result)
|
||||
self.assertNotIn("popup-secret", encoded)
|
||||
payload = json.loads(result["content"][0]["text"])
|
||||
self.assertTrue(payload["stored"])
|
||||
self.assertEqual(self.store.get("oldbox-umbra-ssh"), "popup-secret")
|
||||
|
||||
def test_status_never_returns_secret(self) -> None:
|
||||
self.broker.call_tool(
|
||||
"credential_set",
|
||||
{"profile": "oldbox-umbra-ssh", "purpose": "SSH for oldbox"},
|
||||
)
|
||||
result = self.broker.call_tool("credential_status", {"profile": "oldbox-umbra-ssh"})
|
||||
self.assertNotIn("popup-secret", json.dumps(result))
|
||||
payload = json.loads(result["content"][0]["text"])
|
||||
self.assertTrue(payload["configured"])
|
||||
|
||||
def test_sudo_requires_exact_allowlist(self) -> None:
|
||||
result = self.broker.call_tool(
|
||||
"remote_sudo_exec",
|
||||
{"profile": "oldbox-umbra", "command": ["/bin/sh", "-c", "id"]},
|
||||
)
|
||||
self.assertTrue(result["isError"])
|
||||
self.assertIn("allowlist", result["content"][0]["text"])
|
||||
self.assertEqual(self.executor.calls, [])
|
||||
|
||||
def test_allowed_sudo_is_forwarded(self) -> None:
|
||||
result = self.broker.call_tool(
|
||||
"remote_sudo_exec",
|
||||
{"profile": "oldbox-umbra", "command": ["/usr/local/sbin/umbracontrol-backup"]},
|
||||
)
|
||||
self.assertFalse(result["isError"])
|
||||
self.assertEqual(self.executor.calls[0][2], True)
|
||||
|
||||
def test_json_rpc_tools_list_and_initialize(self) -> None:
|
||||
response = self.broker.handle_json_rpc({"jsonrpc": "2.0", "id": 1, "method": "initialize"})
|
||||
assert response is not None
|
||||
self.assertEqual(response["result"]["serverInfo"]["name"], "umbratrans-remote-broker")
|
||||
response = self.broker.handle_json_rpc({"jsonrpc": "2.0", "id": 2, "method": "tools/list"})
|
||||
assert response is not None
|
||||
names = {tool["name"] for tool in response["result"]["tools"]}
|
||||
self.assertIn("credential_set", names)
|
||||
self.assertIn("remote_sudo_exec", names)
|
||||
|
||||
def test_redactor(self) -> None:
|
||||
redactor = SecretRedactor()
|
||||
redactor.add("secret-value")
|
||||
self.assertEqual(redactor.redact("x secret-value y"), "x <redacted> y")
|
||||
def test_temporary_profile_is_in_memory_and_readable(self) -> None:
|
||||
persistent = FakeStore()
|
||||
vault = CredentialVault(persistent)
|
||||
vault.set("temporary-profile", "temporary-value", ttl_seconds=60)
|
||||
self.assertEqual(vault.get("temporary-profile"), "temporary-value")
|
||||
self.assertTrue(vault.exists("temporary-profile"))
|
||||
self.assertNotIn("temporary-profile", persistent.values)
|
||||
vault.delete("temporary-profile")
|
||||
self.assertFalse(vault.exists("temporary-profile"))
|
||||
|
||||
|
||||
class ValidationTests(unittest.TestCase):
|
||||
def test_profile_rejects_shell_metacharacters(self) -> None:
|
||||
with self.assertRaises(CredentialError):
|
||||
validate_profile("bad profile;rm")
|
||||
|
||||
def test_profile_accepts_normal_name(self) -> None:
|
||||
self.assertEqual(validate_profile("oldbox-umbra_ssh.1"), "oldbox-umbra_ssh.1")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
2053
specs/18-UmbraTrans.md
Normal file
2053
specs/18-UmbraTrans.md
Normal file
File diff suppressed because it is too large
Load Diff
1121
specs/19-enterprise-control-plane.md
Normal file
1121
specs/19-enterprise-control-plane.md
Normal file
File diff suppressed because it is too large
Load Diff
@@ -14,6 +14,8 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from umbraaudio.voices.lmstudio_gate import queued_chat_completion
|
||||
|
||||
PROMPT_PATH = Path(__file__).parent / "CHARACTER_DISCOVERY_PROMPT.md"
|
||||
SCENE_BREAK = "<<<>>>"
|
||||
|
||||
@@ -295,7 +297,8 @@ class CharacterDiscoverer:
|
||||
response = None
|
||||
for attempt in range(max_attempts):
|
||||
try:
|
||||
response = client.chat.completions.create(
|
||||
response = queued_chat_completion(
|
||||
client,
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.2,
|
||||
@@ -997,7 +1000,8 @@ Return ONLY valid JSON."""
|
||||
model = self._active_model or self.config.openai_model
|
||||
|
||||
try:
|
||||
response = client.chat.completions.create(
|
||||
response = queued_chat_completion(
|
||||
client,
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.1,
|
||||
|
||||
@@ -10,6 +10,7 @@ from umbraaudio.analysis import (
|
||||
CharacterProfile,
|
||||
)
|
||||
from umbraaudio.config import LLMConfig
|
||||
from umbraaudio.voices.lmstudio_gate import queued_chat_completion
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -140,7 +141,8 @@ class CharacterProfiler:
|
||||
f"dimensions while staying consistent with the text evidence."
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
response = queued_chat_completion(
|
||||
client,
|
||||
model=self.config.openai_model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=self.config.temperature,
|
||||
@@ -234,7 +236,8 @@ class CharacterProfiler:
|
||||
prompt=prompt,
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
response = queued_chat_completion(
|
||||
client,
|
||||
model=self.config.openai_model,
|
||||
messages=[{"role": "user", "content": llm_prompt}],
|
||||
temperature=self.config.temperature,
|
||||
@@ -285,7 +288,8 @@ Example: {{"Eilean": "eɪ.lən", "Fjord": "fjɔːrd"}}
|
||||
|
||||
Return ONLY valid JSON."""
|
||||
|
||||
response = client.chat.completions.create(
|
||||
response = queued_chat_completion(
|
||||
client,
|
||||
model=self.config.openai_model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.1,
|
||||
|
||||
@@ -10,6 +10,7 @@ from typing import Any
|
||||
|
||||
from openai import OpenAI
|
||||
|
||||
from umbraaudio.voices.lmstudio_gate import queued_chat_completion
|
||||
from umbraaudio.analysis.attribution_qa import (
|
||||
SPEECH_VERBS as _QA_SPEECH_VERBS,
|
||||
)
|
||||
@@ -1097,7 +1098,8 @@ class LLMAttributor:
|
||||
max_attempts = 3
|
||||
for attempt in range(max_attempts):
|
||||
try:
|
||||
response = client.chat.completions.create(
|
||||
response = queued_chat_completion(
|
||||
client,
|
||||
model=self.model,
|
||||
messages=messages,
|
||||
temperature=0.1,
|
||||
|
||||
@@ -10,6 +10,7 @@ from typing import Any
|
||||
from openai import OpenAI
|
||||
|
||||
from umbraaudio.config import LLMConfig
|
||||
from umbraaudio.voices.lmstudio_gate import queued_chat_completion
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -136,7 +137,8 @@ class LLMSegmenter:
|
||||
client = self._get_client()
|
||||
|
||||
try:
|
||||
response = client.chat.completions.create(
|
||||
response = queued_chat_completion(
|
||||
client,
|
||||
model=self.model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.2,
|
||||
|
||||
@@ -58,6 +58,9 @@ class TTSConfig(BaseSettings):
|
||||
voicedesign_model: str = "/Volumes/1TB_APL_Storage/UmbraAudio/models/Qwen3-TTS-12Hz-1.7B-VoiceDesign"
|
||||
base_model: str = "/Volumes/1TB_APL_Storage/UmbraAudio/models/Qwen3-TTS-12Hz-1.7B-Base"
|
||||
qwen_tts_url: str = "http://localhost:8881"
|
||||
# Optional GPU accelerator (e.g. http://127.0.0.1:18881 relay to the
|
||||
# 192.168.123.14 box). Tried first when healthy; local always falls back.
|
||||
qwen_tts_helper_url: str = ""
|
||||
|
||||
|
||||
class QualityConfig(BaseSettings):
|
||||
|
||||
@@ -475,7 +475,7 @@ class ChapterBuilder:
|
||||
speaker_traits = load_speaker_traits(self.data_dir / "speakers.json")
|
||||
distortion_effects = load_speaker_distortion_effects(self.data_dir / "speakers.json")
|
||||
|
||||
engine = QwenTTSEngine(qwen_tts_url=self.config.tts.qwen_tts_url)
|
||||
engine = QwenTTSEngine(qwen_tts_url=self.config.tts.qwen_tts_url, helper_url=self.config.tts.qwen_tts_helper_url)
|
||||
|
||||
# Capture the full vmatrix (the "8k matrix" ref_code + speaker embedding)
|
||||
# once per distinct speaker, cached server-side as .pt under PROMPT_DIR.
|
||||
|
||||
@@ -4,6 +4,8 @@ import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from umbraaudio.voices.lmstudio_gate import queued_chat_completion
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
EMOTION_PROMPT = """You are an audiobook director analyzing dialogue and narration for TTS voice direction.
|
||||
@@ -85,7 +87,8 @@ class EmotionMapper:
|
||||
max_attempts = 3
|
||||
for attempt in range(max_attempts):
|
||||
try:
|
||||
response = self.llm.chat.completions.create(
|
||||
response = queued_chat_completion(
|
||||
self.llm,
|
||||
model=self.model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.2,
|
||||
|
||||
@@ -808,7 +808,7 @@ def generate_pronunciation_audio(config: Any, name: str, spelling: str) -> dict[
|
||||
try:
|
||||
from umbraaudio.voices.qwen_tts import VOICE_REFERENCE_TEXT, QwenTTSEngine
|
||||
|
||||
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url)
|
||||
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url, helper_url=config.tts.qwen_tts_helper_url)
|
||||
if ref:
|
||||
engine.generate(
|
||||
text=syn_text, output_path=out, voice_ref=ref,
|
||||
|
||||
186
umbraaudio/voices/lmstudio_gate.py
Normal file
186
umbraaudio/voices/lmstudio_gate.py
Normal file
@@ -0,0 +1,186 @@
|
||||
"""Queueing gate for LM Studio LLM requests.
|
||||
|
||||
LM Studio on this Mac can only serve one LLM at a time and the model
|
||||
coordinator swaps between the Qwen TTS models and the LLM. LLM calls that
|
||||
arrive while LM Studio is busy -- loading a model, swapping models, or
|
||||
warming up -- previously failed with connection / HTTP errors. This module
|
||||
serializes requests and waits until the server is available again, then lets
|
||||
the queued request through (the exact behavior requested: *if the local LM
|
||||
Studio is busy the request is queued and waits for availability, then
|
||||
processed*).
|
||||
|
||||
Only requests aimed at a local LM Studio (:1234) are gated; OpenAI provider
|
||||
clients pass straight through untouched.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
from urllib.error import URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_POLL_INTERVAL = 1.0
|
||||
DEFAULT_WAIT_TIMEOUT = 900.0
|
||||
|
||||
_ML_LOCAL_HOSTS = {"localhost", "127.0.0.1", "0.0.0.0", "::1"}
|
||||
_PLACEHOLDER_MODELS = {"", "local-model"}
|
||||
|
||||
|
||||
class LMStudioBusyError(RuntimeError):
|
||||
"""Raised when LM Studio did not become available within the wait window."""
|
||||
|
||||
|
||||
def _host_of(base_url: str) -> str:
|
||||
"""Return the hostname of a base URL (best effort, no network)."""
|
||||
base = base_url.split("://", 1)[-1]
|
||||
base = base.split("/", 1)[0]
|
||||
host = base
|
||||
if ":" in base and not base.startswith("["):
|
||||
host = base.rsplit(":", 1)[0]
|
||||
return (host or "").lower().strip("[]")
|
||||
|
||||
|
||||
def _is_local_lmstudio(base_url: str) -> bool:
|
||||
"""Gate only requests aimed at the local LM Studio server."""
|
||||
host = _host_of(base_url)
|
||||
if host in _ML_LOCAL_HOSTS:
|
||||
return True
|
||||
# LM Studio is commonly on port 1234, even when reached via a custom host.
|
||||
return ":1234" in base_url
|
||||
|
||||
|
||||
def _server_probe(base_url: str, model: str | None = None) -> bool:
|
||||
"""True when the server answers /models and (if given) knows the model."""
|
||||
base = (base_url or "").rstrip("/")
|
||||
if not base:
|
||||
return True
|
||||
try:
|
||||
req = Request(
|
||||
f"{base}/models",
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
with urlopen(req, timeout=3) as resp:
|
||||
if resp.status != 200:
|
||||
return False
|
||||
payload = json.loads(resp.read().decode("utf-8"))
|
||||
except (URLError, OSError, ValueError):
|
||||
return False
|
||||
|
||||
ids = [m.get("id", "") for m in payload.get("data", [])]
|
||||
if model in _PLACEHOLDER_MODELS:
|
||||
return True
|
||||
|
||||
# LM Studio exposes loaded instances under the native /api/v1/models
|
||||
# endpoint; prefer it when the OpenAI-compatible listing is ambiguous.
|
||||
api = base
|
||||
for suffix in ("/v1",):
|
||||
if api.endswith(suffix):
|
||||
api = api[: -len(suffix)]
|
||||
try:
|
||||
req2 = Request(f"{api}/api/v1/models", headers={"Accept": "application/json"})
|
||||
with urlopen(req2, timeout=3) as resp2:
|
||||
if resp2.status == 200:
|
||||
native = json.loads(resp2.read().decode("utf-8"))
|
||||
for m in native.get("data", []):
|
||||
if m.get("key") == model and m.get("loaded_instances"):
|
||||
return True
|
||||
except (URLError, OSError, ValueError):
|
||||
pass
|
||||
|
||||
return model in ids or not ids
|
||||
|
||||
|
||||
class LMStudioGate:
|
||||
"""Serialize + queue LLM requests until LM Studio is available.
|
||||
|
||||
Thread-safe. At most one request is processed at a time; everyone else
|
||||
waits (FIFO-ish via the condition variable). Before a request runs, the
|
||||
gate polls ``available_fn`` until the server reports healthy.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
poll_interval: float = DEFAULT_POLL_INTERVAL,
|
||||
wait_timeout: float = DEFAULT_WAIT_TIMEOUT,
|
||||
available_fn: Callable[[str, str | None], bool] = _server_probe,
|
||||
) -> None:
|
||||
self.poll_interval = poll_interval
|
||||
self.wait_timeout = wait_timeout
|
||||
self._available_fn = available_fn
|
||||
self._cond = threading.Condition()
|
||||
self._inflight = 0
|
||||
|
||||
def wait_until_available(
|
||||
self,
|
||||
base_url: str,
|
||||
model: str | None = None,
|
||||
timeout: float | None = None,
|
||||
) -> None:
|
||||
"""Block until LM Studio answers, raising :class:`LMStudioBusyError`."""
|
||||
wait_for = timeout if timeout is not None else self.wait_timeout
|
||||
deadline = time.monotonic() + wait_for
|
||||
while True:
|
||||
if self._available_fn(base_url, model):
|
||||
return
|
||||
if time.monotonic() >= deadline:
|
||||
raise LMStudioBusyError(
|
||||
f"LM Studio not available after {wait_for:.0f}s wait (base={base_url})"
|
||||
)
|
||||
time.sleep(self.poll_interval)
|
||||
|
||||
@contextmanager
|
||||
def acquire(
|
||||
self,
|
||||
base_url: str,
|
||||
model: str | None = None,
|
||||
timeout: float | None = None,
|
||||
) -> Any:
|
||||
"""Queue the caller behind any in-flight request, then run when ready."""
|
||||
with self._cond:
|
||||
while self._inflight > 0:
|
||||
self._cond.wait()
|
||||
self._inflight += 1
|
||||
try:
|
||||
self.wait_until_available(base_url, model, timeout)
|
||||
yield
|
||||
finally:
|
||||
with self._cond:
|
||||
self._inflight -= 1
|
||||
self._cond.notify()
|
||||
|
||||
|
||||
_GATE = LMStudioGate()
|
||||
|
||||
|
||||
def _client_base(client: Any) -> str:
|
||||
try:
|
||||
return str(client.base_url)
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def queued_chat_completion(
|
||||
client: Any,
|
||||
model: str,
|
||||
messages: list[dict[str, str]],
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""Run ``client.chat.completions.create`` behind the LM Studio gate.
|
||||
|
||||
Requests to a local LM Studio are serialized and wait for server
|
||||
availability before processing; non-LM Studio providers are unchanged.
|
||||
"""
|
||||
base_url = _client_base(client)
|
||||
if _is_local_lmstudio(base_url):
|
||||
with _GATE.acquire(base_url, model):
|
||||
return client.chat.completions.create(model=model, messages=messages, **kwargs)
|
||||
return client.chat.completions.create(model=model, messages=messages, **kwargs)
|
||||
@@ -51,67 +51,99 @@ class QwenTTSEngine:
|
||||
voicedesign_model: str | None = None,
|
||||
base_model: str | None = None,
|
||||
qwen_tts_url: str | None = None,
|
||||
helper_url: str | None = None,
|
||||
) -> None:
|
||||
self.device = device
|
||||
self.sample_rate = sample_rate
|
||||
self._voicedesign_path = voicedesign_model or DEFAULT_VOICEDESIGN_MODEL
|
||||
self._base_path = base_model or DEFAULT_BASE_MODEL
|
||||
self._qwen_tts_url = qwen_tts_url
|
||||
self._helper_url = helper_url
|
||||
self._vd_model = None
|
||||
self._base_model = None
|
||||
|
||||
@property
|
||||
def _candidate_urls(self) -> list[str]:
|
||||
"""Ordered TTS endpoints: GPU helper first (when configured), local required last."""
|
||||
urls: list[str] = []
|
||||
if self._helper_url:
|
||||
urls.append(self._helper_url.rstrip("/"))
|
||||
if self._qwen_tts_url:
|
||||
local = self._qwen_tts_url.rstrip("/")
|
||||
if local not in urls:
|
||||
urls.append(local)
|
||||
return urls
|
||||
|
||||
@property
|
||||
def _use_remote(self) -> bool:
|
||||
return bool(self._qwen_tts_url)
|
||||
return bool(self._candidate_urls)
|
||||
|
||||
def _url_healthy(self, url: str) -> bool:
|
||||
import httpx
|
||||
|
||||
try:
|
||||
r = httpx.get(f"{url}/health", timeout=5)
|
||||
return r.status_code < 500
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _preferred_candidates(self) -> list[str]:
|
||||
"""Healthy endpoints first, unhealthy ones after (never dropped).
|
||||
|
||||
With a single endpoint (local-only setup) no health probe is made:
|
||||
the request itself determines success, so we never add a second
|
||||
round-trip to every TTS call.
|
||||
"""
|
||||
urls = self._candidate_urls
|
||||
if len(urls) <= 1:
|
||||
return urls
|
||||
healthy = [u for u in urls if self._url_healthy(u)]
|
||||
return healthy + [u for u in urls if u not in healthy]
|
||||
|
||||
def check_health(self) -> bool:
|
||||
if not self._use_remote:
|
||||
return False
|
||||
import httpx
|
||||
|
||||
try:
|
||||
r = httpx.get(f"{self._qwen_tts_url}/health", timeout=5)
|
||||
return r.status_code < 500
|
||||
except Exception:
|
||||
return False
|
||||
return any(self._url_healthy(url) for url in self._candidate_urls)
|
||||
|
||||
def load_remote_model(self, model_type: str = "voicedesign") -> None:
|
||||
if not self._use_remote:
|
||||
return
|
||||
import httpx
|
||||
|
||||
try:
|
||||
r = httpx.post(
|
||||
f"{self._qwen_tts_url}/load",
|
||||
json={"model_type": model_type},
|
||||
timeout=120,
|
||||
)
|
||||
logger.info(
|
||||
"Qwen TTS remote model load: HTTP %d %s", r.status_code, r.text.strip()[:200]
|
||||
)
|
||||
import time
|
||||
for url in self._candidate_urls:
|
||||
try:
|
||||
r = httpx.post(
|
||||
f"{url}/load",
|
||||
json={"model_type": model_type},
|
||||
timeout=120,
|
||||
)
|
||||
logger.info(
|
||||
"Qwen TTS remote model load: HTTP %d %s", r.status_code, r.text.strip()[:200]
|
||||
)
|
||||
import time
|
||||
|
||||
if "already" in r.text.lower():
|
||||
time.sleep(1)
|
||||
else:
|
||||
time.sleep(10)
|
||||
except Exception as e:
|
||||
logger.warning("Qwen TTS remote model load failed: %s", e)
|
||||
if "already" in r.text.lower():
|
||||
time.sleep(1)
|
||||
else:
|
||||
time.sleep(10)
|
||||
except Exception as e:
|
||||
logger.warning("Qwen TTS remote model load failed (%s): %s", url, e)
|
||||
|
||||
def unload_remote_model(self) -> None:
|
||||
if not self._use_remote:
|
||||
return
|
||||
import httpx
|
||||
|
||||
try:
|
||||
r = httpx.post(
|
||||
f"{self._qwen_tts_url}/unload",
|
||||
json={"model_type": "all"},
|
||||
timeout=30,
|
||||
)
|
||||
logger.info("Qwen TTS remote model unload: HTTP %d", r.status_code)
|
||||
except Exception as e:
|
||||
logger.warning("Qwen TTS remote model unload failed: %s", e)
|
||||
for url in self._candidate_urls:
|
||||
try:
|
||||
r = httpx.post(
|
||||
f"{url}/unload",
|
||||
json={"model_type": "all"},
|
||||
timeout=30,
|
||||
)
|
||||
logger.info("Qwen TTS remote model unload (%s): HTTP %d", url, r.status_code)
|
||||
except Exception as e:
|
||||
logger.warning("Qwen TTS remote model unload failed (%s): %s", url, e)
|
||||
|
||||
def unload_and_reload(self) -> None:
|
||||
self.unload_remote_model()
|
||||
@@ -271,18 +303,26 @@ class QwenTTSEngine:
|
||||
data["force"] = 1
|
||||
if ref_text:
|
||||
data["ref_text"] = ref_text
|
||||
with open(voice_ref, "rb") as f:
|
||||
r = httpx.post(
|
||||
f"{self._qwen_tts_url}/v1/voice-prompt",
|
||||
data=data,
|
||||
files={"ref_audio": ("ref.wav", f.read(), "audio/wav")},
|
||||
timeout=REMOTE_TIMEOUT,
|
||||
)
|
||||
if r.status_code != 200:
|
||||
raise QwenTTSUnavailableError(
|
||||
f"Voice-prompt build failed HTTP {r.status_code}: {r.text[:300]}"
|
||||
)
|
||||
return slug
|
||||
|
||||
last_err: str | None = None
|
||||
ref_bytes = Path(voice_ref).read_bytes()
|
||||
for url in self._preferred_candidates():
|
||||
try:
|
||||
r = httpx.post(
|
||||
f"{url}/v1/voice-prompt",
|
||||
data=data,
|
||||
files={"ref_audio": ("ref.wav", ref_bytes, "audio/wav")},
|
||||
timeout=REMOTE_TIMEOUT,
|
||||
)
|
||||
if r.status_code != 200:
|
||||
last_err = f"Voice-prompt build failed HTTP {r.status_code}: {r.text[:300]}"
|
||||
logger.warning("%s (url=%s)", last_err, url)
|
||||
continue
|
||||
return slug
|
||||
except Exception as e:
|
||||
last_err = f"Voice-prompt build failed: {e}"
|
||||
logger.warning("%s (url=%s)", last_err, url)
|
||||
raise QwenTTSUnavailableError(last_err or "No Qwen TTS server available")
|
||||
|
||||
def generate_voice_reference(
|
||||
self,
|
||||
@@ -315,7 +355,48 @@ class QwenTTSEngine:
|
||||
) -> tuple[np.ndarray, int]:
|
||||
import httpx
|
||||
|
||||
url = f"{self._qwen_tts_url}/v1/tts"
|
||||
files = None
|
||||
|
||||
if prompt_slug:
|
||||
pass
|
||||
elif voice_ref and Path(voice_ref).exists():
|
||||
files = {"ref_audio": ("ref.wav", Path(voice_ref).read_bytes(), "audio/wav")}
|
||||
|
||||
last_err: str | None = None
|
||||
for url in self._preferred_candidates():
|
||||
try:
|
||||
return self._remote_generate_at(
|
||||
url=url,
|
||||
text=text,
|
||||
files=files,
|
||||
instruct=instruct,
|
||||
ref_text=ref_text,
|
||||
prompt_slug=prompt_slug,
|
||||
temperature=temperature,
|
||||
do_sample=do_sample,
|
||||
seed=seed,
|
||||
)
|
||||
except QwenTTSUnavailableError as e:
|
||||
last_err = str(e)
|
||||
logger.warning("Qwen TTS endpoint %s failed, trying next: %s", url, e)
|
||||
raise QwenTTSUnavailableError(last_err or "No Qwen TTS server available")
|
||||
|
||||
def _remote_generate_at(
|
||||
self,
|
||||
*,
|
||||
url: str,
|
||||
text: str,
|
||||
files: dict[str, Any] | None,
|
||||
instruct: str | None,
|
||||
ref_text: str | None,
|
||||
prompt_slug: str | None,
|
||||
temperature: float,
|
||||
do_sample: bool,
|
||||
seed: int | None,
|
||||
) -> tuple[np.ndarray, int]:
|
||||
import httpx
|
||||
|
||||
generate_url = f"{url}/v1/tts"
|
||||
payload: dict[str, Any] = {
|
||||
"text": text,
|
||||
"sample_rate": self.sample_rate,
|
||||
@@ -326,12 +407,9 @@ class QwenTTSEngine:
|
||||
payload["seed"] = seed
|
||||
if ref_text:
|
||||
payload["ref_text"] = ref_text
|
||||
files = None
|
||||
|
||||
if prompt_slug:
|
||||
payload["prompt_slug"] = prompt_slug
|
||||
elif voice_ref and Path(voice_ref).exists():
|
||||
files = {"ref_audio": ("ref.wav", Path(voice_ref).read_bytes(), "audio/wav")}
|
||||
|
||||
# instruct composes with voice clone / ICL: the server prepends it as
|
||||
# style direction (emotion/pace/delivery) while keeping the speaker.
|
||||
@@ -339,15 +417,16 @@ class QwenTTSEngine:
|
||||
payload["voice_description"] = instruct
|
||||
|
||||
try:
|
||||
r = httpx.post(url, data=payload, files=files or {}, timeout=REMOTE_TIMEOUT)
|
||||
r = httpx.post(
|
||||
generate_url, data=payload, files=files or {}, timeout=REMOTE_TIMEOUT
|
||||
)
|
||||
except httpx.ConnectError as e:
|
||||
raise QwenTTSUnavailableError(
|
||||
f"Cannot connect to Qwen TTS server at {self._qwen_tts_url}. "
|
||||
"Make sure the server is running."
|
||||
f"Cannot connect to Qwen TTS server at {url}. Make sure the server is running."
|
||||
) from e
|
||||
except httpx.TimeoutException as e:
|
||||
raise QwenTTSUnavailableError(
|
||||
f"Qwen TTS server at {self._qwen_tts_url} timed out after {REMOTE_TIMEOUT}s."
|
||||
f"Qwen TTS server at {url} timed out after {REMOTE_TIMEOUT}s."
|
||||
) from e
|
||||
|
||||
if r.status_code != 200:
|
||||
@@ -456,23 +535,34 @@ class QwenTTSEngine:
|
||||
|
||||
|
||||
class QwenTTSClient:
|
||||
def __init__(self, base_url: str, timeout: int = REMOTE_TIMEOUT) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str,
|
||||
timeout: int = REMOTE_TIMEOUT,
|
||||
helper_url: str | None = None,
|
||||
) -> None:
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.helper_url = (helper_url or "").rstrip("/") or None
|
||||
self.timeout = timeout
|
||||
|
||||
def check_health(self) -> bool:
|
||||
import httpx
|
||||
|
||||
try:
|
||||
r = httpx.get(f"{self.base_url}/health", timeout=5)
|
||||
return r.status_code < 500
|
||||
except Exception:
|
||||
return False
|
||||
for url in (self.helper_url, self.base_url):
|
||||
if not url:
|
||||
continue
|
||||
try:
|
||||
r = httpx.get(f"{url}/health", timeout=5)
|
||||
if r.status_code < 500:
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
return False
|
||||
|
||||
def generate_voice_reference(
|
||||
self, text: str, voice_description: str, output_path: Path
|
||||
) -> Path:
|
||||
engine = QwenTTSEngine(qwen_tts_url=self.base_url)
|
||||
engine = QwenTTSEngine(qwen_tts_url=self.base_url, helper_url=self.helper_url)
|
||||
return engine.generate_voice_reference(text, output_path, voice_description)
|
||||
|
||||
def generate(
|
||||
@@ -485,7 +575,7 @@ class QwenTTSClient:
|
||||
do_sample: bool = True,
|
||||
seed: int | None = None,
|
||||
) -> Path:
|
||||
engine = QwenTTSEngine(qwen_tts_url=self.base_url)
|
||||
engine = QwenTTSEngine(qwen_tts_url=self.base_url, helper_url=self.helper_url)
|
||||
return engine.generate(
|
||||
text, output_path, voice_ref=voice_ref, instruct=instruct,
|
||||
temperature=temperature, do_sample=do_sample, seed=seed,
|
||||
|
||||
@@ -1940,7 +1940,7 @@ async def speakers_list_partial(request: Request):
|
||||
return False
|
||||
from umbraaudio.voices.qwen_tts import QwenTTSEngine
|
||||
|
||||
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url)
|
||||
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url, helper_url=config.tts.qwen_tts_helper_url)
|
||||
try:
|
||||
import httpx
|
||||
|
||||
@@ -3325,7 +3325,7 @@ async def generate_utterance_audio(request: Request, utt_id: int):
|
||||
voice_ref = vf
|
||||
break
|
||||
|
||||
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url)
|
||||
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url, helper_url=config.tts.qwen_tts_helper_url)
|
||||
|
||||
tts_instruct = compose_utterance_instruct(instruct, emotion, pace)
|
||||
|
||||
@@ -3444,7 +3444,7 @@ async def delete_utterance(request: Request, utt_id: int):
|
||||
if "kainda" in vf.stem.lower():
|
||||
narrator_ref = vf
|
||||
break
|
||||
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url)
|
||||
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url, helper_url=config.tts.qwen_tts_helper_url)
|
||||
for u in remaining:
|
||||
out = audio_dir / f"utt_{u['utterance_index']:04d}.wav"
|
||||
if is_scene_break(u["text"], u["type"]):
|
||||
@@ -3633,7 +3633,7 @@ async def create_utterance(request: Request, chapter_id: int):
|
||||
narrator_ref = vf
|
||||
break
|
||||
try:
|
||||
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url)
|
||||
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url, helper_url=config.tts.qwen_tts_helper_url)
|
||||
if narrator_ref:
|
||||
engine.generate(
|
||||
text=spoken_text,
|
||||
@@ -3798,7 +3798,7 @@ async def set_voice_status(request: Request):
|
||||
QwenTTSEngine,
|
||||
)
|
||||
|
||||
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url)
|
||||
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url, helper_url=config.tts.qwen_tts_helper_url)
|
||||
slug = engine.build_voice_prompt(
|
||||
ref_path,
|
||||
ref_text=VOICE_REFERENCE_TEXT,
|
||||
|
||||
Reference in New Issue
Block a user