Add enterprise/infra tooling and repoint Qwen3-TTS submodule to internal fork
Two groups of changes, committed together as the follow-up to the initial push: 1. .gitmodules — submodule Qwen3-TTS now points at the internal Gitea fork (ssh://git@mini-eq:2222/umbragit/Qwen3-TTS.git) instead of upstream GitHub, so a clone actually gets the UmbraAudio branding/integration commit 012e452 rather than vanilla upstream. 2. New infra/ tooling for the cross-machine work in spec 21: agent_bridge, remote_credential_broker (Keychain-backed, secrets read at runtime and generated via read_secret/random_secret — no hardcoded credentials), the enterprise bootstrap scripts, and the oldbox SSH/auth prompts. Plus backup/ restore scripts, two specs (18 UmbraTrans, 19 enterprise control plane), umbraaudio/voices/lmstudio_gate.py, and in-flight edits to the discovery/attribution/profiling modules, config, chapter_builder, emotion, pronunciation, qwen_tts and routes. Also adds *.db to .gitignore: SQLite project databases are runtime state and no .db file was tracked before. Verified before staging: .env's only change is UMBRA_TTS__QWEN_TTS_HELPER_URL and the committed copy has zero secret-looking values; the broker ships only config.example.json.
This commit is contained in:
3
.env
3
.env
@@ -14,6 +14,9 @@ UMBRA_TTS__RETRY_DELAY=2.0
|
|||||||
UMBRA_TTS__VOICEDESIGN_MODEL=/Volumes/1TB_APL_Storage/UmbraAudio/models/Qwen3-TTS-12Hz-1.7B-VoiceDesign
|
UMBRA_TTS__VOICEDESIGN_MODEL=/Volumes/1TB_APL_Storage/UmbraAudio/models/Qwen3-TTS-12Hz-1.7B-VoiceDesign
|
||||||
UMBRA_TTS__BASE_MODEL=/Volumes/1TB_APL_Storage/UmbraAudio/models/Qwen3-TTS-12Hz-1.7B-Base
|
UMBRA_TTS__BASE_MODEL=/Volumes/1TB_APL_Storage/UmbraAudio/models/Qwen3-TTS-12Hz-1.7B-Base
|
||||||
UMBRA_TTS__QWEN_TTS_URL=http://localhost:8881
|
UMBRA_TTS__QWEN_TTS_URL=http://localhost:8881
|
||||||
|
# Optional GPU accelerator: relay 127.0.0.1:18881 -> 192.168.123.14:8881.
|
||||||
|
# Helper is tried first when healthy; local server above is the required fallback.
|
||||||
|
UMBRA_TTS__QWEN_TTS_HELPER_URL=http://127.0.0.1:18881
|
||||||
|
|
||||||
# --- LLM ---
|
# --- LLM ---
|
||||||
UMBRA_LLM__PROVIDER=auto
|
UMBRA_LLM__PROVIDER=auto
|
||||||
|
|||||||
3
.gitignore
vendored
3
.gitignore
vendored
@@ -47,3 +47,6 @@ umbraaudio_package.zip
|
|||||||
|
|
||||||
# One-off voice A/B demo (large binary WAVs, not wired into the app)
|
# One-off voice A/B demo (large binary WAVs, not wired into the app)
|
||||||
/umbraaudio/web/static/samples/
|
/umbraaudio/web/static/samples/
|
||||||
|
|
||||||
|
# SQLite project databases are runtime state, never source
|
||||||
|
*.db
|
||||||
|
|||||||
2
.gitmodules
vendored
2
.gitmodules
vendored
@@ -1,3 +1,3 @@
|
|||||||
[submodule "Qwen3-TTS"]
|
[submodule "Qwen3-TTS"]
|
||||||
path = Qwen3-TTS
|
path = Qwen3-TTS
|
||||||
url = https://github.com/QwenLM/Qwen3-TTS.git
|
url = ssh://git@mini-eq:2222/umbragit/Qwen3-TTS.git
|
||||||
|
|||||||
12
Issues.txt
12
Issues.txt
@@ -1,3 +1,13 @@
|
|||||||
Issues
|
Issues
|
||||||
|
|
||||||
Split of book - identify chapters, paragraphs and utterances. Split utterances by ".?!" the only three valid end of sentence punctuation.
|
Split of book - identify chapters, paragraphs and utterances. Split utterances by ".?!" the only three valid end of sentence punctuation.
|
||||||
|
|
||||||
|
|
||||||
|
connect the two. Get audio data on the new machine in the right format.
|
||||||
|
|
||||||
|
|
||||||
|
Idea:
|
||||||
|
Android app to pull pictures from phone and from cloud and send to NAS, formatted into groups. Allows removal from phone and cloud to free up space.
|
||||||
|
- App becomes primary photo backup (not giving your stuff to google) - the place to find your stuff.
|
||||||
|
- Search by location taken, date, contents of picture, identify who
|
||||||
|
- Have system setup to display the pictures on the nas via web on in the app.
|
||||||
|
|||||||
290
backup/Starthere.sh
Executable file
290
backup/Starthere.sh
Executable file
@@ -0,0 +1,290 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Restore an UmbraAudio full backup and rebuild a portable runtime.
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||||
|
ARCHIVE=""
|
||||||
|
TARGET="${UMBRA_RESTORE_TARGET:-$HOME/UmbraAudio-restore}"
|
||||||
|
GLOBAL_TARGET="${UMBRA_GLOBAL_STATE_DIR:-$HOME/.umbra}"
|
||||||
|
PYTHON_OVERRIDE="${PYTHON_BIN:-}"
|
||||||
|
SKIP_DEPS=0
|
||||||
|
SKIP_SUBMODULE_UPDATE=0
|
||||||
|
SKIP_GLOBAL_STATE=0
|
||||||
|
INSTALL_SYSTEM_DEPS=0
|
||||||
|
DOWNLOAD_MODELS=0
|
||||||
|
WITH_TTS_SERVER=0
|
||||||
|
KEEP_ARCHIVED_ENV=0
|
||||||
|
RUN_TESTS=0
|
||||||
|
FORCE=0
|
||||||
|
APPLY_PORTABILITY_PATCH=1
|
||||||
|
OLD_ROOT=""
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'EOF'
|
||||||
|
Usage: Starthere.sh [options]
|
||||||
|
|
||||||
|
Restore the newest UmbraAudio backup in this directory (or use --archive),
|
||||||
|
verify its SHA-256 manifest, restore the project and global voice state, and
|
||||||
|
rebuild a portable Python environment.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--target DIR Restore destination (default: $HOME/UmbraAudio-restore)
|
||||||
|
--global-state-dir DIR Restored global registry/library (default: $HOME/.umbra)
|
||||||
|
--archive FILE Payload archive to restore
|
||||||
|
--python BIN Explicit Python 3.11+ interpreter
|
||||||
|
--skip-deps Do not rebuild Python dependencies
|
||||||
|
--skip-submodule Do not initialize/update git submodules
|
||||||
|
--skip-global-state Do not restore ~/.umbra voice registry/library
|
||||||
|
--install-system-deps Try to install git/ffmpeg/Python with brew or apt-get
|
||||||
|
--download-models Download missing Qwen model weights
|
||||||
|
--with-tts-server Also install tts_server/requirements.txt
|
||||||
|
--keep-archived-env Keep archived .venv/.python instead of rebuilding
|
||||||
|
--no-portability-patch Do not rewrite machine-local paths in restored copy
|
||||||
|
--run-tests Run pytest after setup
|
||||||
|
--force Permit non-empty project/global targets
|
||||||
|
-h, --help Show this help
|
||||||
|
|
||||||
|
The archive may contain .env, manuscripts, models, and audio. Review it before
|
||||||
|
sharing the restored directory. Network database configuration is intentionally
|
||||||
|
not changed by this script.
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
while (($#)); do
|
||||||
|
case "$1" in
|
||||||
|
--target) TARGET="${2:?--target requires a directory}"; shift 2 ;;
|
||||||
|
--global-state-dir) GLOBAL_TARGET="${2:?--global-state-dir requires a directory}"; shift 2 ;;
|
||||||
|
--archive) ARCHIVE="${2:?--archive requires a file}"; shift 2 ;;
|
||||||
|
--python) PYTHON_OVERRIDE="${2:?--python requires an interpreter}"; shift 2 ;;
|
||||||
|
--skip-deps) SKIP_DEPS=1; shift ;;
|
||||||
|
--skip-submodule) SKIP_SUBMODULE_UPDATE=1; shift ;;
|
||||||
|
--skip-global-state) SKIP_GLOBAL_STATE=1; shift ;;
|
||||||
|
--install-system-deps) INSTALL_SYSTEM_DEPS=1; shift ;;
|
||||||
|
--download-models) DOWNLOAD_MODELS=1; shift ;;
|
||||||
|
--with-tts-server) WITH_TTS_SERVER=1; shift ;;
|
||||||
|
--keep-archived-env) KEEP_ARCHIVED_ENV=1; shift ;;
|
||||||
|
--no-portability-patch) APPLY_PORTABILITY_PATCH=0; shift ;;
|
||||||
|
--run-tests) RUN_TESTS=1; shift ;;
|
||||||
|
--force) FORCE=1; shift ;;
|
||||||
|
-h|--help) usage; exit 0 ;;
|
||||||
|
*) printf 'Unknown option: %s\n' "$1" >&2; usage >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ -z "$ARCHIVE" ]]; then
|
||||||
|
ARCHIVE="$(find "$SCRIPT_DIR" -maxdepth 1 -type f -name 'UmbraAudio-*.tar' -print | sort | tail -n 1)"
|
||||||
|
fi
|
||||||
|
[[ -n "$ARCHIVE" && -f "$ARCHIVE" && ! -L "$ARCHIVE" ]] || {
|
||||||
|
printf 'No regular payload archive found. Use --archive or place UmbraAudio-*.tar beside this script.\n' >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
[[ -f "$SCRIPT_DIR/SHA256SUMS" ]] || {
|
||||||
|
printf 'Missing SHA256SUMS beside Starthere.sh; refusing an unverified restore.\n' >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
[[ -f "$SCRIPT_DIR/restore_support.py" ]] || {
|
||||||
|
printf 'Missing restore_support.py beside Starthere.sh; use a backup made with the updated tooling.\n' >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
ARCHIVE_DIR="$(cd -- "$(dirname -- "$ARCHIVE")" && pwd -P)"
|
||||||
|
[[ "$ARCHIVE_DIR" == "$SCRIPT_DIR" ]] || {
|
||||||
|
printf 'Archive must be located beside the selected backup metadata: %s\n' "$ARCHIVE_DIR" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
ARCHIVE_BASENAME="$(basename -- "$ARCHIVE")"
|
||||||
|
if ! awk -v name="$ARCHIVE_BASENAME" '$2 == name { found = 1 } END { exit(found ? 0 : 1) }' "$SCRIPT_DIR/SHA256SUMS"; then
|
||||||
|
printf 'Selected archive is not listed in SHA256SUMS: %s\n' "$ARCHIVE_BASENAME" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [[ -f "$SCRIPT_DIR/BACKUP-INFO.txt" ]]; then
|
||||||
|
info_archive="$(sed -n 's/^archive_name=//p' "$SCRIPT_DIR/BACKUP-INFO.txt" | head -n 1)"
|
||||||
|
if [[ -n "$info_archive" && "$info_archive" != "$ARCHIVE_BASENAME" ]]; then
|
||||||
|
printf 'Selected archive does not match BACKUP-INFO.txt: %s vs %s\n' "$ARCHIVE_BASENAME" "$info_archive" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
for tool in tar; do
|
||||||
|
command -v "$tool" >/dev/null 2>&1 || { printf 'Required tool is missing: %s\n' "$tool" >&2; exit 1; }
|
||||||
|
done
|
||||||
|
|
||||||
|
printf 'Backup directory: %s\n' "$SCRIPT_DIR"
|
||||||
|
printf 'Payload archive: %s\n' "$ARCHIVE"
|
||||||
|
printf 'Restore target: %s\n' "$TARGET"
|
||||||
|
printf 'Global state: %s\n' "$GLOBAL_TARGET"
|
||||||
|
|
||||||
|
if command -v shasum >/dev/null 2>&1; then
|
||||||
|
(cd "$SCRIPT_DIR" && shasum -a 256 -c SHA256SUMS)
|
||||||
|
elif command -v sha256sum >/dev/null 2>&1; then
|
||||||
|
(cd "$SCRIPT_DIR" && sha256sum -c SHA256SUMS)
|
||||||
|
else
|
||||||
|
printf 'Neither shasum nor sha256sum is available.\n' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
[[ ! -L "$TARGET" ]] || { printf 'Refusing symlink restore target: %s\n' "$TARGET" >&2; exit 1; }
|
||||||
|
if [[ -e "$TARGET" && -n "$(find "$TARGET" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)" && "$FORCE" -ne 1 ]]; then
|
||||||
|
printf 'Restore target is not empty: %s (use --force only deliberately)\n' "$TARGET" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
TARGET_PARENT_INPUT="$(dirname -- "$TARGET")"
|
||||||
|
mkdir -p "$TARGET_PARENT_INPUT"
|
||||||
|
TARGET_PARENT="$(cd -- "$TARGET_PARENT_INPUT" && pwd -P)"
|
||||||
|
TARGET_NAME="$(basename -- "$TARGET")"
|
||||||
|
TARGET_ABS="$TARGET_PARENT/$TARGET_NAME"
|
||||||
|
GLOBAL_PARENT_INPUT="$(dirname -- "$GLOBAL_TARGET")"
|
||||||
|
mkdir -p "$GLOBAL_PARENT_INPUT"
|
||||||
|
GLOBAL_TARGET_ABS="$(cd -- "$GLOBAL_PARENT_INPUT" && pwd -P)/$(basename -- "$GLOBAL_TARGET")"
|
||||||
|
TEMP_TARGET="$(mktemp -d "$TARGET_PARENT/.${TARGET_NAME}.restore.XXXXXX")"
|
||||||
|
OLD_TARGET=""
|
||||||
|
cleanup() {
|
||||||
|
if [[ -d "$TEMP_TARGET" ]]; then
|
||||||
|
rm -rf "$TEMP_TARGET"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
printf 'Extracting into an isolated staging directory...\n'
|
||||||
|
tar -xpf "$ARCHIVE" -C "$TEMP_TARGET" --strip-components 1
|
||||||
|
ROOT="$TEMP_TARGET"
|
||||||
|
[[ -f "$ROOT/pyproject.toml" && -d "$ROOT/umbraaudio" ]] || {
|
||||||
|
printf 'Archive did not contain the expected UmbraAudio root.\n' >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if [[ "$KEEP_ARCHIVED_ENV" -ne 1 ]]; then
|
||||||
|
printf 'Discarding archived platform-specific virtualenv/toolchain...\n'
|
||||||
|
rm -rf "$ROOT/.venv" "$ROOT/.python"
|
||||||
|
fi
|
||||||
|
|
||||||
|
install_system_dependencies() {
|
||||||
|
[[ "$INSTALL_SYSTEM_DEPS" -eq 1 ]] || return 0
|
||||||
|
if command -v brew >/dev/null 2>&1; then
|
||||||
|
brew install git ffmpeg
|
||||||
|
brew install python@3.12 || brew install python
|
||||||
|
elif command -v apt-get >/dev/null 2>&1 && command -v sudo >/dev/null 2>&1; then
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get install -y git ffmpeg python3 python3-venv
|
||||||
|
else
|
||||||
|
printf 'Install git, ffmpeg, Python 3.11+, and venv support manually.\n' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
install_system_dependencies
|
||||||
|
|
||||||
|
python_supports_project() {
|
||||||
|
"$1" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 11) else 1)' \
|
||||||
|
>/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
PYTHON_BIN=""
|
||||||
|
if [[ -n "$PYTHON_OVERRIDE" ]]; then
|
||||||
|
python_supports_project "$PYTHON_OVERRIDE" || {
|
||||||
|
printf 'Requested Python is unavailable or older than 3.11: %s\n' "$PYTHON_OVERRIDE" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
PYTHON_BIN="$PYTHON_OVERRIDE"
|
||||||
|
else
|
||||||
|
for candidate in python3.12 python3.11 python3.13 python3; do
|
||||||
|
if command -v "$candidate" >/dev/null 2>&1 && python_supports_project "$candidate"; then
|
||||||
|
PYTHON_BIN="$candidate"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
[[ -n "$PYTHON_BIN" ]] || {
|
||||||
|
printf 'Python 3.11+ is required. Install it or pass --python.\n' >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
printf 'Using Python: %s (%s)\n' "$PYTHON_BIN" "$("$PYTHON_BIN" --version 2>&1)"
|
||||||
|
|
||||||
|
if [[ "$SKIP_SUBMODULE_UPDATE" -ne 1 && -f "$ROOT/.gitmodules" ]]; then
|
||||||
|
printf 'Initializing git submodules...\n'
|
||||||
|
git -C "$ROOT" submodule update --init --recursive
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$SKIP_DEPS" -eq 1 ]]; then
|
||||||
|
printf 'Skipping dependency installation as requested.\n'
|
||||||
|
else
|
||||||
|
UV_BIN="$(command -v uv || true)"
|
||||||
|
BOOTSTRAP_ENV=""
|
||||||
|
if [[ -z "$UV_BIN" ]]; then
|
||||||
|
printf 'Bootstrapping uv 0.11.7 in an isolated environment...\n'
|
||||||
|
BOOTSTRAP_ENV="$ROOT/.uv-bootstrap"
|
||||||
|
"$PYTHON_BIN" -m venv "$BOOTSTRAP_ENV"
|
||||||
|
"$BOOTSTRAP_ENV/bin/python" -m pip install --upgrade pip
|
||||||
|
"$BOOTSTRAP_ENV/bin/python" -m pip install 'uv==0.11.7'
|
||||||
|
UV_BIN="$BOOTSTRAP_ENV/bin/uv"
|
||||||
|
fi
|
||||||
|
printf 'Rebuilding the locked environment with uv...\n'
|
||||||
|
UV_SYNC_ARGS=(sync --locked --python "$PYTHON_BIN")
|
||||||
|
if [[ "$RUN_TESTS" -eq 1 ]]; then
|
||||||
|
UV_SYNC_ARGS+=(--extra dev)
|
||||||
|
fi
|
||||||
|
(cd "$ROOT" && "$UV_BIN" "${UV_SYNC_ARGS[@]}")
|
||||||
|
if [[ "$WITH_TTS_SERVER" -eq 1 && -f "$ROOT/tts_server/requirements.txt" ]]; then
|
||||||
|
(cd "$ROOT" && "$UV_BIN" pip install --python .venv/bin/python -r tts_server/requirements.txt)
|
||||||
|
fi
|
||||||
|
if [[ -n "$BOOTSTRAP_ENV" ]]; then
|
||||||
|
rm -rf "$BOOTSTRAP_ENV"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$DOWNLOAD_MODELS" -eq 1 && ! -f "$ROOT/models/Qwen3-TTS-12Hz-1.7B-Base/model.safetensors" ]]; then
|
||||||
|
[[ -f "$ROOT/scripts/download_models.sh" ]] || {
|
||||||
|
printf 'Model download script is missing.\n' >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
(cd "$ROOT" && bash scripts/download_models.sh)
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -f "$SCRIPT_DIR/BACKUP-INFO.txt" ]]; then
|
||||||
|
OLD_ROOT="$(sed -n 's/^source_path=//p' "$SCRIPT_DIR/BACKUP-INFO.txt" | head -n 1)"
|
||||||
|
fi
|
||||||
|
if [[ "$APPLY_PORTABILITY_PATCH" -eq 1 ]]; then
|
||||||
|
"$PYTHON_BIN" "$SCRIPT_DIR/restore_support.py" patch-paths \
|
||||||
|
--root "$ROOT" --old-root "$OLD_ROOT" --new-root "$TARGET_ABS" \
|
||||||
|
--global-state-dir "$GLOBAL_TARGET_ABS"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -f "$ROOT/.env" ]]; then
|
||||||
|
chmod 600 "$ROOT/.env"
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf 'Validating restored project tree and SQLite databases...\n'
|
||||||
|
"$PYTHON_BIN" "$SCRIPT_DIR/restore_support.py" validate-tree \
|
||||||
|
--root "$ROOT" --old-root "$OLD_ROOT"
|
||||||
|
|
||||||
|
# Commit the project tree only after extraction, dependency setup, path migration,
|
||||||
|
# and validation all succeed. An existing target is retained as .old on --force.
|
||||||
|
if [[ -e "$TARGET" ]]; then
|
||||||
|
OLD_TARGET="$TARGET_PARENT/.${TARGET_NAME}.old.$(date +%Y%m%d%H%M%S)"
|
||||||
|
mv "$TARGET" "$OLD_TARGET"
|
||||||
|
fi
|
||||||
|
mv "$TEMP_TARGET" "$TARGET"
|
||||||
|
trap - EXIT
|
||||||
|
ROOT="$TARGET_ABS"
|
||||||
|
|
||||||
|
if [[ "$SKIP_GLOBAL_STATE" -ne 1 && -d "$SCRIPT_DIR/global-state" ]]; then
|
||||||
|
printf 'Restoring global voice registry/library...\n'
|
||||||
|
RESTORE_ARGS=(restore-global --backup-dir "$SCRIPT_DIR" --target "$GLOBAL_TARGET_ABS")
|
||||||
|
[[ "$FORCE" -eq 1 ]] && RESTORE_ARGS+=(--force)
|
||||||
|
if [[ -n "$OLD_ROOT" ]]; then
|
||||||
|
RESTORE_ARGS+=(--old-project-root "$OLD_ROOT" --new-project-root "$ROOT")
|
||||||
|
fi
|
||||||
|
"$PYTHON_BIN" "$SCRIPT_DIR/restore_support.py" "${RESTORE_ARGS[@]}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$RUN_TESTS" -eq 1 ]]; then
|
||||||
|
[[ -x "$ROOT/.venv/bin/pytest" ]] || {
|
||||||
|
printf 'pytest is not installed in the rebuilt environment.\n' >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
(cd "$ROOT" && .venv/bin/pytest -q)
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf '\nRestore complete.\n'
|
||||||
|
printf 'Project root: %s\n' "$ROOT"
|
||||||
|
[[ -z "$OLD_TARGET" ]] || printf 'Previous target retained at: %s\n' "$OLD_TARGET"
|
||||||
|
printf 'Review .env, global voice paths, and RESTORED-SERVICE-COMMANDS.md before starting services.\n'
|
||||||
|
printf 'Network database setup is intentionally not performed by Starthere.sh.\n'
|
||||||
493
backup/create_full_backup.sh
Executable file
493
backup/create_full_backup.sh
Executable file
@@ -0,0 +1,493 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Create a complete, restorable UmbraAudio snapshot on a mounted filesystem/NAS.
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
SOURCE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
||||||
|
DESTINATION="${UMBRA_BACKUP_DEST:-/Volumes/NetBackup/AudioBackup}"
|
||||||
|
GLOBAL_ROOT="${UMBRA_GLOBAL_STATE_ROOT:-$HOME/.umbra}"
|
||||||
|
QUIESCE=1
|
||||||
|
BEST_EFFORT=0
|
||||||
|
RESTART_SERVICES=1
|
||||||
|
ALLOW_NON_SMB=0
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'EOF'
|
||||||
|
Usage: create_full_backup.sh [options]
|
||||||
|
|
||||||
|
Create a dated, fail-closed full backup of the current UmbraAudio working tree
|
||||||
|
and the active global voice registry/library.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--destination DIR NAS/filesystem destination (default: /Volumes/NetBackup/AudioBackup)
|
||||||
|
--global-root DIR Active global state root (default: $HOME/.umbra)
|
||||||
|
--best-effort Do not quiesce services (not recommended; explicit opt-in)
|
||||||
|
--quiesce Quiesce services (default)
|
||||||
|
--no-restart Do not restart services after a --quiesce backup
|
||||||
|
--allow-non-smb Permit a non-SMB destination for testing
|
||||||
|
-h, --help Show this help
|
||||||
|
|
||||||
|
The archive includes the entire current tree, including dirty/untracked files,
|
||||||
|
ignored book projects, generated audio, models, .git, and .env. Global state is
|
||||||
|
captured with SQLite's backup API and stored beside the archive.
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
while (($#)); do
|
||||||
|
case "$1" in
|
||||||
|
--destination) DESTINATION="${2:?--destination requires a directory}"; shift 2 ;;
|
||||||
|
--global-root) GLOBAL_ROOT="${2:?--global-root requires a directory}"; shift 2 ;;
|
||||||
|
--best-effort) BEST_EFFORT=1; QUIESCE=0; shift ;;
|
||||||
|
--quiesce) QUIESCE=1; shift ;;
|
||||||
|
--no-restart) RESTART_SERVICES=0; shift ;;
|
||||||
|
--allow-non-smb) ALLOW_NON_SMB=1; shift ;;
|
||||||
|
-h|--help) usage; exit 0 ;;
|
||||||
|
*) printf 'Unknown option: %s\n' "$1" >&2; usage >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
[[ -d "$SOURCE" ]] || { printf 'Source directory not found: %s\n' "$SOURCE" >&2; exit 1; }
|
||||||
|
mkdir -p "$DESTINATION"
|
||||||
|
DESTINATION="$(cd -- "$DESTINATION" && pwd -P)"
|
||||||
|
case "$DESTINATION/" in
|
||||||
|
"$SOURCE/"*) printf 'Backup destination must not be inside the source tree.\n' >&2; exit 1 ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
for tool in tar git awk sed df mount; do
|
||||||
|
command -v "$tool" >/dev/null 2>&1 || {
|
||||||
|
printf 'Required tool is missing: %s\n' "$tool" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
done
|
||||||
|
if command -v shasum >/dev/null 2>&1; then
|
||||||
|
:
|
||||||
|
elif command -v sha256sum >/dev/null 2>&1; then
|
||||||
|
:
|
||||||
|
else
|
||||||
|
printf 'Install shasum or sha256sum.\n' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
hash_file() {
|
||||||
|
if command -v shasum >/dev/null 2>&1; then
|
||||||
|
shasum -a 256 "$@"
|
||||||
|
else
|
||||||
|
sha256sum "$@"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
verify_hashes() {
|
||||||
|
if command -v shasum >/dev/null 2>&1; then
|
||||||
|
shasum -a 256 -c "$@"
|
||||||
|
else
|
||||||
|
sha256sum -c "$@"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
FSTYPE="$(df -P "$DESTINATION" | awk 'NR == 2 {print $1}')"
|
||||||
|
if [[ "$ALLOW_NON_SMB" -ne 1 && "$FSTYPE" != *smb* && "$FSTYPE" != *cifs* ]]; then
|
||||||
|
printf 'Destination is not an SMB/CIFS mount (filesystem=%s). Use --allow-non-smb only for tests.\n' "$FSTYPE" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
SOURCE_KB="$(du -sk "$SOURCE" | awk '{print $1}')"
|
||||||
|
AVAIL_KB="$(df -Pk "$DESTINATION" | awk 'NR == 2 {print $4}')"
|
||||||
|
if [[ "$AVAIL_KB" -lt $((SOURCE_KB + 1048576)) ]]; then
|
||||||
|
printf 'Insufficient destination space: need about %s KB plus 1 GiB, have %s KB.\n' "$SOURCE_KB" "$AVAIL_KB" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
LOCK_DIR="${TMPDIR:-/tmp}/umbra-full-backup.lock"
|
||||||
|
if ! mkdir "$LOCK_DIR" 2>/dev/null; then
|
||||||
|
printf 'Another backup appears to be running (lock: %s).\n' "$LOCK_DIR" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
BACKUP_ID="$(date +%Y%m%d-%H%M%S)-$$-$RANDOM"
|
||||||
|
STAGING_DIR="$(mktemp -d "$DESTINATION/.UmbraAudio-$BACKUP_ID.partial.XXXXXX")"
|
||||||
|
FINAL_DIR="$DESTINATION/UmbraAudio-$BACKUP_ID"
|
||||||
|
ARCHIVE_NAME="UmbraAudio-$BACKUP_ID.tar"
|
||||||
|
ARCHIVE_TMP="$STAGING_DIR/$ARCHIVE_NAME.partial"
|
||||||
|
ARCHIVE_FINAL="$STAGING_DIR/$ARCHIVE_NAME"
|
||||||
|
[[ ! -e "$FINAL_DIR" ]] || { printf 'Final backup directory already exists: %s\n' "$FINAL_DIR" >&2; exit 1; }
|
||||||
|
|
||||||
|
WEB_LABEL="gui/$(id -u)/com.umbraaudio.web"
|
||||||
|
WEB_PLIST="/Users/worstauthor/Library/LaunchAgents/com.umbraaudio.web.plist"
|
||||||
|
if command -v launchctl >/dev/null 2>&1; then
|
||||||
|
detected_web_plist="$(launchctl print "$WEB_LABEL" 2>/dev/null | sed -n 's/^[[:space:]]*path = //p' | head -n 1 || true)"
|
||||||
|
if [[ -f "$detected_web_plist" ]]; then
|
||||||
|
WEB_PLIST="$detected_web_plist"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
WEB_WAS_LOADED=0
|
||||||
|
WEB_STOPPED=0
|
||||||
|
QWEN_PIDS=()
|
||||||
|
QWEN_STOPPED=0
|
||||||
|
Qwen_CWD=""
|
||||||
|
Qwen_COMMAND=""
|
||||||
|
WRITER_WATCH_PID=""
|
||||||
|
RESTARTED=0
|
||||||
|
|
||||||
|
release_lock() { rmdir "$LOCK_DIR" 2>/dev/null || true; }
|
||||||
|
|
||||||
|
restart_services() {
|
||||||
|
[[ "$QUIESCE" -eq 1 && "$RESTART_SERVICES" -eq 1 && "$RESTARTED" -eq 0 ]] || return 0
|
||||||
|
local restore_failed=0
|
||||||
|
if [[ "$WEB_WAS_LOADED" -eq 1 && -f "$WEB_PLIST" ]]; then
|
||||||
|
launchctl bootstrap "gui/$(id -u)" "$WEB_PLIST" >/dev/null 2>&1 || \
|
||||||
|
launchctl kickstart -k "$WEB_LABEL" >/dev/null 2>&1 || true
|
||||||
|
for _ in {1..30}; do
|
||||||
|
if launchctl print "$WEB_LABEL" >/dev/null 2>&1 && lsof -nP -iTCP:8420 -sTCP:LISTEN >/dev/null 2>&1; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
if ! launchctl print "$WEB_LABEL" >/dev/null 2>&1 || ! lsof -nP -iTCP:8420 -sTCP:LISTEN >/dev/null 2>&1; then
|
||||||
|
printf 'WARNING: web LaunchAgent did not restart cleanly.\n' >&2
|
||||||
|
restore_failed=1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
if [[ "$QWEN_STOPPED" -eq 1 && "${#QWEN_PIDS[@]}" -gt 0 ]]; then
|
||||||
|
mkdir -p "$SOURCE/logs"
|
||||||
|
QWEN_ROOT="${Qwen_CWD:-$SOURCE}"
|
||||||
|
[[ -d "$QWEN_ROOT" ]] || QWEN_ROOT="$SOURCE"
|
||||||
|
(cd "$QWEN_ROOT" && nohup "$SOURCE/.venv/bin/python" -u scripts/qwen_tts_server.py \
|
||||||
|
</dev/null >>"$SOURCE/logs/qwen_tts_server.log" 2>&1 &)
|
||||||
|
disown 2>/dev/null || true
|
||||||
|
for _ in {1..30}; do
|
||||||
|
if lsof -nP -iTCP:8881 -sTCP:LISTEN >/dev/null 2>&1; then
|
||||||
|
if ! command -v curl >/dev/null 2>&1 || curl -fsS --max-time 5 http://127.0.0.1:8881/health >/dev/null 2>&1; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
if ! lsof -nP -iTCP:8881 -sTCP:LISTEN >/dev/null 2>&1; then
|
||||||
|
printf 'WARNING: Qwen TTS server did not become healthy/listening on port 8881.\n' >&2
|
||||||
|
restore_failed=1
|
||||||
|
elif command -v curl >/dev/null 2>&1 && ! curl -fsS --max-time 5 http://127.0.0.1:8881/health >/dev/null 2>&1; then
|
||||||
|
printf 'WARNING: Qwen TTS port is open but /health did not pass.\n' >&2
|
||||||
|
restore_failed=1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
RESTARTED=1
|
||||||
|
return "$restore_failed"
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
local rc=$?
|
||||||
|
if [[ -n "$WRITER_WATCH_PID" ]]; then
|
||||||
|
kill "$WRITER_WATCH_PID" >/dev/null 2>&1 || true
|
||||||
|
wait "$WRITER_WATCH_PID" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
if [[ "$QUIESCE" -eq 1 && "$RESTART_SERVICES" -eq 1 && "$RESTARTED" -eq 0 ]]; then
|
||||||
|
if ! restart_services; then
|
||||||
|
printf 'WARNING: service restoration during cleanup did not complete cleanly.\n' >&2
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
if [[ -d "$STAGING_DIR" ]]; then
|
||||||
|
printf 'Backup incomplete; preserving partial directory for inspection: %s\n' "$STAGING_DIR" >&2
|
||||||
|
fi
|
||||||
|
release_lock
|
||||||
|
exit "$rc"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
record_service_state() {
|
||||||
|
if launchctl print "$WEB_LABEL" >/dev/null 2>&1; then WEB_WAS_LOADED=1; fi
|
||||||
|
QWEN_PIDS=()
|
||||||
|
if command -v lsof >/dev/null 2>&1; then
|
||||||
|
while IFS= read -r pid; do
|
||||||
|
[[ "$pid" =~ ^[0-9]+$ ]] && QWEN_PIDS+=("$pid")
|
||||||
|
done < <(lsof -nP -tiTCP:8881 -sTCP:LISTEN 2>/dev/null || true)
|
||||||
|
fi
|
||||||
|
while IFS= read -r pid; do
|
||||||
|
[[ "$pid" =~ ^[0-9]+$ ]] || continue
|
||||||
|
[[ " ${QWEN_PIDS[*]} " == *" $pid "* ]] && continue
|
||||||
|
cmd="$(ps -p "$pid" -o command= 2>/dev/null || true)"
|
||||||
|
[[ "$cmd" == *qwen_tts_server.py* ]] && QWEN_PIDS+=("$pid")
|
||||||
|
done < <(ps -axo pid=,args= | awk '$0 ~ /[q]wen_tts_server\\.py/ {print $1}')
|
||||||
|
if [[ "${#QWEN_PIDS[@]}" -gt 0 ]]; then
|
||||||
|
Qwen_COMMAND="$(ps -p "${QWEN_PIDS[0]}" -o command= 2>/dev/null || true)"
|
||||||
|
if command -v lsof >/dev/null 2>&1; then
|
||||||
|
Qwen_CWD="$(lsof -a -p "${QWEN_PIDS[0]}" -d cwd -Fn 2>/dev/null | sed -n 's/^n//p' | head -n 1)"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
write_service_state() {
|
||||||
|
local output="$1"
|
||||||
|
{
|
||||||
|
printf 'captured_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||||
|
printf 'web_label=%s\n' "$WEB_LABEL"
|
||||||
|
printf 'web_loaded=%s\n' "$WEB_WAS_LOADED"
|
||||||
|
printf 'web_plist=%s\n' "$WEB_PLIST"
|
||||||
|
printf 'qwen_pids=%s\n' "${QWEN_PIDS[*]:-}"
|
||||||
|
printf 'qwen_cwd=%s\n' "${Qwen_CWD:-}"
|
||||||
|
printf 'qwen_command=%s\n' "${Qwen_COMMAND:-}"
|
||||||
|
launchctl print "$WEB_LABEL" 2>&1 || true
|
||||||
|
lsof -nP -iTCP:8420 -sTCP:LISTEN 2>&1 || true
|
||||||
|
lsof -nP -iTCP:8881 -sTCP:LISTEN 2>&1 || true
|
||||||
|
} >"$output"
|
||||||
|
}
|
||||||
|
|
||||||
|
check_active_writers() {
|
||||||
|
local pattern pid
|
||||||
|
for pattern in \
|
||||||
|
'generate[_]all_audio.py' \
|
||||||
|
'generate[_]chapter[_]audio.py' \
|
||||||
|
'reattribute[_]all.py' \
|
||||||
|
'run[_]llm[_]attribution.py' \
|
||||||
|
'resegment[_]all.py' \
|
||||||
|
'nas[_]lifecycle.py' \
|
||||||
|
'chapter[_]builder' \
|
||||||
|
'[r]sync.*UmbraAudio' \
|
||||||
|
'[f]fmpeg' \
|
||||||
|
'[u]vicorn'; do
|
||||||
|
while IFS= read -r pid; do
|
||||||
|
[[ "$pid" == "$$" ]] && continue
|
||||||
|
printf 'Active writer detected (%s, pid %s); refusing backup.\n' "$pattern" "$pid" >&2
|
||||||
|
return 1
|
||||||
|
done < <(ps -axo pid=,command= | awk -v p="$pattern" '$0 ~ p {print $1}')
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
quiesce_services() {
|
||||||
|
for tool in launchctl lsof ps; do
|
||||||
|
command -v "$tool" >/dev/null 2>&1 || {
|
||||||
|
printf 'Quiesce requires missing tool: %s\n' "$tool" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
done
|
||||||
|
record_service_state
|
||||||
|
if [[ "$WEB_WAS_LOADED" -eq 1 ]]; then
|
||||||
|
printf 'Pausing web LaunchAgent...\n'
|
||||||
|
launchctl bootout "$WEB_LABEL" "$WEB_PLIST" >/dev/null 2>&1 || {
|
||||||
|
printf 'Could not boot out web LaunchAgent; refusing inconsistent backup.\n' >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
for _ in {1..30}; do
|
||||||
|
if ! launchctl print "$WEB_LABEL" >/dev/null 2>&1 && ! lsof -nP -iTCP:8420 -sTCP:LISTEN >/dev/null 2>&1; then
|
||||||
|
WEB_STOPPED=1
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
[[ "$WEB_STOPPED" -eq 1 ]] || {
|
||||||
|
printf 'Web service did not stop cleanly; refusing backup.\n' >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
fi
|
||||||
|
for pid in "${QWEN_PIDS[@]}"; do
|
||||||
|
printf 'Pausing Qwen TTS server (pid %s)...\n' "$pid"
|
||||||
|
kill -TERM "$pid" >/dev/null 2>&1 || true
|
||||||
|
done
|
||||||
|
for pid in "${QWEN_PIDS[@]}"; do
|
||||||
|
for _ in {1..30}; do
|
||||||
|
kill -0 "$pid" >/dev/null 2>&1 || break
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
if kill -0 "$pid" >/dev/null 2>&1; then
|
||||||
|
printf 'Qwen TTS server pid %s did not stop; refusing backup.\n' "$pid" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
QWEN_STOPPED=1
|
||||||
|
if lsof -nP -iTCP:8881 -sTCP:LISTEN >/dev/null 2>&1; then
|
||||||
|
printf 'Port 8881 is still listening; refusing backup.\n' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [[ "$WEB_STOPPED" -eq 1 ]] && lsof -nP -iTCP:8420 -sTCP:LISTEN >/dev/null 2>&1; then
|
||||||
|
printf 'Port 8420 is still listening; refusing backup.\n' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
check_active_writers || exit 1
|
||||||
|
sleep 2
|
||||||
|
}
|
||||||
|
|
||||||
|
record_service_state
|
||||||
|
write_service_state "$STAGING_DIR/SERVICE-STATE-BEFORE.txt"
|
||||||
|
if [[ "$QUIESCE" -eq 1 ]]; then
|
||||||
|
quiesce_services
|
||||||
|
else
|
||||||
|
printf 'WARNING: best-effort backup requested; active files may be inconsistent.\n' >&2
|
||||||
|
fi
|
||||||
|
|
||||||
|
SOURCE_PARENT="$(cd -- "$SOURCE/.." && pwd -P)"
|
||||||
|
SOURCE_NAME="$(basename -- "$SOURCE")"
|
||||||
|
GLOBAL_PYTHON="$SOURCE/.venv/bin/python"
|
||||||
|
[[ -x "$GLOBAL_PYTHON" ]] || GLOBAL_PYTHON="$(command -v python3)"
|
||||||
|
|
||||||
|
{
|
||||||
|
printf 'backup_id=%s\n' "$BACKUP_ID"
|
||||||
|
printf 'created_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||||
|
printf 'source_path=%s\n' "$SOURCE"
|
||||||
|
printf 'global_state_root=%s\n' "$GLOBAL_ROOT"
|
||||||
|
printf 'source_host=%s\n' "$(hostname)"
|
||||||
|
printf 'source_os=%s\n' "$(uname -a)"
|
||||||
|
printf 'git_commit=%s\n' "$(git -C "$SOURCE" rev-parse HEAD 2>/dev/null || printf unknown)"
|
||||||
|
printf 'git_branch=%s\n' "$(git -C "$SOURCE" branch --show-current 2>/dev/null || printf unknown)"
|
||||||
|
printf 'quiesced_services=%s\n' "$QUIESCE"
|
||||||
|
printf 'best_effort=%s\n' "$BEST_EFFORT"
|
||||||
|
printf 'archive_name=%s\n' "$ARCHIVE_NAME"
|
||||||
|
} >"$STAGING_DIR/BACKUP-INFO.txt"
|
||||||
|
|
||||||
|
{
|
||||||
|
python3 --version 2>&1 || true
|
||||||
|
"$GLOBAL_PYTHON" --version 2>&1 || true
|
||||||
|
uv --version 2>&1 || true
|
||||||
|
git --version 2>&1 || true
|
||||||
|
tar --version 2>&1 | head -n 1 || true
|
||||||
|
ffmpeg -version 2>&1 | head -n 1 || true
|
||||||
|
} >"$STAGING_DIR/ENVIRONMENT.txt"
|
||||||
|
|
||||||
|
git -C "$SOURCE" status --short --branch --untracked-files=all >"$STAGING_DIR/GIT-STATUS.txt" 2>&1 || true
|
||||||
|
git -C "$SOURCE" diff --binary >"$STAGING_DIR/GIT-DIFF.patch" 2>/dev/null || true
|
||||||
|
git -C "$SOURCE" diff --cached --binary >"$STAGING_DIR/GIT-DIFF-CACHED.patch" 2>/dev/null || true
|
||||||
|
git -C "$SOURCE" submodule status >"$STAGING_DIR/SUBMODULES.txt" 2>&1 || true
|
||||||
|
|
||||||
|
printf 'Recording file inventory...\n'
|
||||||
|
SOURCE="$SOURCE" "$GLOBAL_PYTHON" - <<'PY' >"$STAGING_DIR/FILE-INVENTORY.tsv"
|
||||||
|
from __future__ import annotations
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
root = Path(os.environ["SOURCE"])
|
||||||
|
for path in sorted(root.rglob("*")):
|
||||||
|
try:
|
||||||
|
stat = path.lstat()
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
kind = "l" if path.is_symlink() else "d" if path.is_dir() else "f"
|
||||||
|
print(f"{kind}\t{stat.st_size}\t{path.relative_to(root)}")
|
||||||
|
PY
|
||||||
|
|
||||||
|
printf 'Running source integrity checks before archiving...\n'
|
||||||
|
"$GLOBAL_PYTHON" "$SOURCE/backup/restore_support.py" validate-tree --root "$SOURCE"
|
||||||
|
|
||||||
|
cat >"$STAGING_DIR/BACKUP-README.txt" <<'EOF'
|
||||||
|
UmbraAudio full backup
|
||||||
|
======================
|
||||||
|
|
||||||
|
This backup is a point-in-time copy of the complete current working tree and
|
||||||
|
active global voice registry/library.
|
||||||
|
|
||||||
|
Included:
|
||||||
|
- tracked, untracked, and modified files;
|
||||||
|
- ignored book projects, generated audio, exports, logs, and .git metadata;
|
||||||
|
- model weights and the Qwen3-TTS submodule;
|
||||||
|
- .env and local configuration;
|
||||||
|
- the existing .venv and .python directories for forensic completeness;
|
||||||
|
- a SQLite-consistent global-state snapshot in global-state/;
|
||||||
|
- launchd/GPU-relay context in context/.
|
||||||
|
|
||||||
|
The restore script rebuilds a portable environment rather than trusting the
|
||||||
|
archived platform-specific virtual environment. The original .env is preserved
|
||||||
|
as .env.archive-original when portability paths are rewritten.
|
||||||
|
|
||||||
|
This backup does not configure or modify a network database.
|
||||||
|
|
||||||
|
Restore:
|
||||||
|
./Starthere.sh --target /path/to/restore
|
||||||
|
|
||||||
|
See BACKUP-INFO.txt, ENVIRONMENT.txt, global-state/MANIFEST.json, and
|
||||||
|
RESTORED-SERVICE-COMMANDS.md after restoration.
|
||||||
|
EOF
|
||||||
|
|
||||||
|
cp "$SOURCE/backup/Starthere.sh" "$STAGING_DIR/Starthere.sh"
|
||||||
|
cp "$SOURCE/backup/restore_support.py" "$STAGING_DIR/restore_support.py"
|
||||||
|
chmod 700 "$STAGING_DIR/Starthere.sh"
|
||||||
|
chmod 700 "$STAGING_DIR/restore_support.py"
|
||||||
|
|
||||||
|
CONTEXT_DIR="$STAGING_DIR/context"
|
||||||
|
mkdir -p "$CONTEXT_DIR/launchagents" "$CONTEXT_DIR/launchdaemons"
|
||||||
|
cp -p /Users/worstauthor/.umbraaudio/umbraweb_login.sh "$CONTEXT_DIR/launchagents/" 2>/dev/null || true
|
||||||
|
cp -p /Users/worstauthor/Library/LaunchAgents/com.umbraaudio.web.plist "$CONTEXT_DIR/launchagents/" 2>/dev/null || true
|
||||||
|
cp -p /Users/worstauthor/Library/LaunchAgents/com.umbraaudio.commitcheck.plist "$CONTEXT_DIR/launchagents/" 2>/dev/null || true
|
||||||
|
cp -p /Library/LaunchDaemons/com.umba.qwen-relay.plist "$CONTEXT_DIR/launchdaemons/" 2>/dev/null || true
|
||||||
|
cp -p /Library/LaunchDaemons/qwen_relay.py "$CONTEXT_DIR/launchdaemons/" 2>/dev/null || true
|
||||||
|
launchctl print "$WEB_LABEL" >"$CONTEXT_DIR/launchctl-web.txt" 2>&1 || true
|
||||||
|
launchctl print system/com.umba.qwen-relay >"$CONTEXT_DIR/launchctl-qwen-relay.txt" 2>&1 || true
|
||||||
|
mount >"$CONTEXT_DIR/mounts.txt" 2>&1 || true
|
||||||
|
df -h >"$CONTEXT_DIR/filesystems.txt" 2>&1 || true
|
||||||
|
printf '%s\n' 'Context files are informational; Starthere.sh does not install privileged services.' >"$CONTEXT_DIR/README.txt"
|
||||||
|
|
||||||
|
printf 'Snapshotting global voice state with SQLite backup API...\n'
|
||||||
|
"$GLOBAL_PYTHON" "$SOURCE/backup/restore_support.py" snapshot-global \
|
||||||
|
--source "$GLOBAL_ROOT" --output "$STAGING_DIR/global-state" --original-root "$GLOBAL_ROOT"
|
||||||
|
(cd "$STAGING_DIR/global-state" && verify_hashes GLOBAL-STATE-SHA256SUMS)
|
||||||
|
|
||||||
|
start_writer_watch() {
|
||||||
|
local flag="$STAGING_DIR/ACTIVE-WRITER"
|
||||||
|
(
|
||||||
|
while :; do
|
||||||
|
if ! check_active_writers >/dev/null 2>&1; then
|
||||||
|
printf 'writer detected during archive\n' >"$flag"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
) &
|
||||||
|
WRITER_WATCH_PID="$!"
|
||||||
|
}
|
||||||
|
|
||||||
|
stop_writer_watch() {
|
||||||
|
if [[ -n "$WRITER_WATCH_PID" ]]; then
|
||||||
|
kill "$WRITER_WATCH_PID" >/dev/null 2>&1 || true
|
||||||
|
wait "$WRITER_WATCH_PID" 2>/dev/null || true
|
||||||
|
WRITER_WATCH_PID=""
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
printf 'Creating full payload archive (%s)...\n' "$ARCHIVE_NAME"
|
||||||
|
start_writer_watch
|
||||||
|
if ! COPYFILE_DISABLE=1 tar -C "$SOURCE_PARENT" -cf "$ARCHIVE_TMP" "$SOURCE_NAME" 2>"$STAGING_DIR/TAR-WARNINGS.log"; then
|
||||||
|
stop_writer_watch
|
||||||
|
printf 'tar failed; inspect TAR-WARNINGS.log.\n' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
stop_writer_watch
|
||||||
|
if [[ -f "$STAGING_DIR/ACTIVE-WRITER" ]]; then
|
||||||
|
printf 'A writer appeared while archiving; refusing the backup.\n' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if grep -Eiq 'file changed|error|failed|cannot|permission denied' "$STAGING_DIR/TAR-WARNINGS.log"; then
|
||||||
|
printf 'tar reported a consistency/read warning; refusing backup.\n' >&2
|
||||||
|
cat "$STAGING_DIR/TAR-WARNINGS.log" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
mv "$ARCHIVE_TMP" "$ARCHIVE_FINAL"
|
||||||
|
printf 'Computing comprehensive SHA-256 manifest...\n'
|
||||||
|
"$GLOBAL_PYTHON" "$SOURCE/backup/restore_support.py" write-manifest --root "$STAGING_DIR"
|
||||||
|
(cd "$STAGING_DIR" && verify_hashes SHA256SUMS)
|
||||||
|
(cd "$STAGING_DIR/global-state" && verify_hashes GLOBAL-STATE-SHA256SUMS)
|
||||||
|
tar -tf "$ARCHIVE_FINAL" >/dev/null
|
||||||
|
chmod 600 "$ARCHIVE_FINAL" "$STAGING_DIR/SHA256SUMS" "$STAGING_DIR/GIT-DIFF.patch" "$STAGING_DIR/GIT-DIFF-CACHED.patch" 2>/dev/null || true
|
||||||
|
|
||||||
|
mv "$STAGING_DIR" "$FINAL_DIR"
|
||||||
|
STAGING_DIR=""
|
||||||
|
|
||||||
|
# Keep the cleanup trap until service restoration and top-level pointers finish.
|
||||||
|
ROOT_LAUNCHER_TMP="$DESTINATION/.Starthere.sh.tmp.$$"
|
||||||
|
cat >"$ROOT_LAUNCHER_TMP" <<EOF
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -Eeuo pipefail
|
||||||
|
ROOT="\$(cd -- "\$(dirname -- "\${BASH_SOURCE[0]}")" && pwd -P)"
|
||||||
|
LATEST="\$(find "\$ROOT" -maxdepth 1 -type d -name 'UmbraAudio-*' ! -name '.*' -print | sort | tail -n 1)"
|
||||||
|
[[ -n "\$LATEST" && -x "\$LATEST/Starthere.sh" ]] || { printf 'No completed backup found under %s\n' "\$ROOT" >&2; exit 1; }
|
||||||
|
exec "\$LATEST/Starthere.sh" "\$@"
|
||||||
|
EOF
|
||||||
|
chmod 700 "$ROOT_LAUNCHER_TMP"
|
||||||
|
mv "$ROOT_LAUNCHER_TMP" "$DESTINATION/Starthere.sh"
|
||||||
|
printf '%s\n' "$FINAL_DIR" >"$DESTINATION/LATEST.tmp.$$"
|
||||||
|
mv "$DESTINATION/LATEST.tmp.$$" "$DESTINATION/LATEST"
|
||||||
|
|
||||||
|
restart_services
|
||||||
|
record_service_state
|
||||||
|
write_service_state "$FINAL_DIR/SERVICE-STATE-AFTER.txt"
|
||||||
|
"$GLOBAL_PYTHON" "$SOURCE/backup/restore_support.py" write-manifest --root "$FINAL_DIR"
|
||||||
|
(cd "$FINAL_DIR" && verify_hashes SHA256SUMS)
|
||||||
|
trap - EXIT
|
||||||
|
release_lock
|
||||||
|
|
||||||
|
printf '\nBackup complete.\n'
|
||||||
|
printf 'Location: %s\n' "$FINAL_DIR"
|
||||||
|
printf 'Archive: %s\n' "$FINAL_DIR/$ARCHIVE_NAME"
|
||||||
|
printf 'Global state snapshot: %s/global-state\n' "$FINAL_DIR"
|
||||||
|
printf 'Restore: %s/Starthere.sh --target /path/to/restore\n' "$FINAL_DIR"
|
||||||
821
backup/restore_support.py
Normal file
821
backup/restore_support.py
Normal file
@@ -0,0 +1,821 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Portable backup/restore support helpers for UmbraAudio.
|
||||||
|
|
||||||
|
This module uses only the Python standard library. It snapshots global Umbra
|
||||||
|
state with SQLite's backup API, restores it atomically, rewrites machine-local
|
||||||
|
paths in a restored copy, and performs basic integrity checks.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shlex
|
||||||
|
import shutil
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
def sha256(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as handle:
|
||||||
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def is_database(path: Path) -> bool:
|
||||||
|
return path.name.endswith((".db", ".db-wal", ".db-shm"))
|
||||||
|
|
||||||
|
|
||||||
|
def is_sqlite_file(path: Path) -> bool:
|
||||||
|
try:
|
||||||
|
with path.open("rb") as handle:
|
||||||
|
return handle.read(16) == b"SQLite format 3\x00"
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def safe_relative_path(value: str) -> Path:
|
||||||
|
path = Path(value)
|
||||||
|
if path.is_absolute() or ".." in path.parts:
|
||||||
|
raise RuntimeError(f"unsafe relative path in backup metadata: {value!r}")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def sql_identifier(value: str) -> str:
|
||||||
|
return '"' + value.replace('"', '""') + '"'
|
||||||
|
|
||||||
|
|
||||||
|
def sqlite_backup(source: Path, destination: Path) -> str:
|
||||||
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
source_uri = f"file:{source}?mode=ro"
|
||||||
|
source_db = sqlite3.connect(source_uri, uri=True)
|
||||||
|
destination_db = sqlite3.connect(destination)
|
||||||
|
try:
|
||||||
|
source_db.backup(destination_db)
|
||||||
|
result = destination_db.execute("PRAGMA integrity_check").fetchone()[0]
|
||||||
|
if result != "ok":
|
||||||
|
raise RuntimeError(f"integrity_check failed for {source}: {result}")
|
||||||
|
finally:
|
||||||
|
destination_db.close()
|
||||||
|
source_db.close()
|
||||||
|
return "ok"
|
||||||
|
|
||||||
|
|
||||||
|
def copy_tree_files(
|
||||||
|
source: Path,
|
||||||
|
destination: Path,
|
||||||
|
skip_databases: bool = True,
|
||||||
|
reject_symlinks: bool = False,
|
||||||
|
) -> list[str]:
|
||||||
|
copied: list[str] = []
|
||||||
|
for path in sorted(source.rglob("*")):
|
||||||
|
relative = path.relative_to(source)
|
||||||
|
target = destination / relative
|
||||||
|
if path.is_symlink():
|
||||||
|
if reject_symlinks:
|
||||||
|
raise RuntimeError(f"global state symlink is not allowed: {relative}")
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
os.symlink(os.readlink(path), target)
|
||||||
|
elif path.is_dir():
|
||||||
|
target.mkdir(parents=True, exist_ok=True)
|
||||||
|
elif path.is_file():
|
||||||
|
if skip_databases and is_database(path):
|
||||||
|
continue
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.copy2(path, target)
|
||||||
|
copied.append(str(relative))
|
||||||
|
return copied
|
||||||
|
|
||||||
|
|
||||||
|
def snapshot_global(source: Path, output: Path, original_root: Path) -> None:
|
||||||
|
if output.exists():
|
||||||
|
raise RuntimeError(f"global-state output already exists: {output}")
|
||||||
|
if not source.exists():
|
||||||
|
raise RuntimeError(f"global state root does not exist: {source}")
|
||||||
|
if source == output or source in output.parents:
|
||||||
|
raise RuntimeError("global state output must be outside the source root")
|
||||||
|
|
||||||
|
temporary = Path(tempfile.mkdtemp(prefix="umbra-global-snapshot-"))
|
||||||
|
try:
|
||||||
|
files = copy_tree_files(source, temporary, skip_databases=True)
|
||||||
|
databases: list[dict[str, Any]] = []
|
||||||
|
for path in sorted(source.rglob("*.db")):
|
||||||
|
if not path.is_file():
|
||||||
|
continue
|
||||||
|
relative = path.relative_to(source)
|
||||||
|
target = temporary / relative
|
||||||
|
status = sqlite_backup(path, target)
|
||||||
|
databases.append(
|
||||||
|
{"path": str(relative), "integrity_check": status, "sha256": sha256(target)}
|
||||||
|
)
|
||||||
|
files.append(str(relative))
|
||||||
|
|
||||||
|
file_hashes = {
|
||||||
|
relative: sha256(temporary / relative)
|
||||||
|
for relative in sorted(files)
|
||||||
|
if (temporary / relative).is_file()
|
||||||
|
}
|
||||||
|
manifest = {
|
||||||
|
"schema": "umbra.global_state/v1",
|
||||||
|
"original_root": str(original_root),
|
||||||
|
"source_root": str(source),
|
||||||
|
"files": sorted(files),
|
||||||
|
"file_hashes": file_hashes,
|
||||||
|
"databases": databases,
|
||||||
|
}
|
||||||
|
(temporary / "MANIFEST.json").write_text(
|
||||||
|
json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||||
|
)
|
||||||
|
(temporary / "GLOBAL-STATE-SHA256SUMS").write_text(
|
||||||
|
"".join(f"{sha256(temporary / relative)} {relative}\n" for relative in sorted(files)),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.copytree(temporary, output, symlinks=True)
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(temporary, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _replace_prefix(value: Any, old_root: str, new_root: str) -> Any:
|
||||||
|
if isinstance(value, str) and old_root and value.startswith(old_root):
|
||||||
|
return new_root + value[len(old_root) :]
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def assert_no_old_prefixes(
|
||||||
|
connection: sqlite3.Connection,
|
||||||
|
old_roots: list[str],
|
||||||
|
) -> None:
|
||||||
|
for table_row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'"):
|
||||||
|
table = table_row[0]
|
||||||
|
if table.startswith("sqlite_"):
|
||||||
|
continue
|
||||||
|
table_identifier = sql_identifier(table)
|
||||||
|
for column in connection.execute(f"PRAGMA table_info({table_identifier})").fetchall():
|
||||||
|
column_name = column[1]
|
||||||
|
column_identifier = sql_identifier(column_name)
|
||||||
|
if (column[2] or "").upper() not in {"TEXT", ""}:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
values = connection.execute(
|
||||||
|
f"SELECT {column_identifier} FROM {table_identifier} "
|
||||||
|
f"WHERE {column_identifier} IS NOT NULL"
|
||||||
|
).fetchall()
|
||||||
|
except sqlite3.Error:
|
||||||
|
continue
|
||||||
|
for (value,) in values:
|
||||||
|
if isinstance(value, str) and any(
|
||||||
|
old_root and value.startswith(old_root) for old_root in old_roots
|
||||||
|
):
|
||||||
|
raise RuntimeError(f"old path prefix remains in {table}.{column_name}: {value}")
|
||||||
|
|
||||||
|
|
||||||
|
def migrate_database_paths(database: Path, replacements: list[tuple[str, str]]) -> None:
|
||||||
|
connection = sqlite3.connect(database)
|
||||||
|
try:
|
||||||
|
tables = {
|
||||||
|
row[0]
|
||||||
|
for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")
|
||||||
|
}
|
||||||
|
for table, columns in {
|
||||||
|
"voice_registry": ("voice_ref_path", "vmatrix_path", "sample_audio_path"),
|
||||||
|
"library_speakers": ("voice_ref_path",),
|
||||||
|
"library_utterances": ("voice_ref_path",),
|
||||||
|
}.items():
|
||||||
|
if table not in tables:
|
||||||
|
continue
|
||||||
|
for column in columns:
|
||||||
|
try:
|
||||||
|
rows = connection.execute(
|
||||||
|
f'SELECT rowid, "{column}" FROM "{table}" WHERE "{column}" IS NOT NULL'
|
||||||
|
).fetchall()
|
||||||
|
except sqlite3.Error:
|
||||||
|
continue
|
||||||
|
for rowid, value in rows:
|
||||||
|
replacement = value
|
||||||
|
for old_root, new_root in replacements:
|
||||||
|
replacement = _replace_prefix(replacement, old_root, new_root)
|
||||||
|
if replacement != value:
|
||||||
|
connection.execute(
|
||||||
|
f'UPDATE "{table}" SET "{column}" = ? WHERE rowid = ?',
|
||||||
|
(replacement, rowid),
|
||||||
|
)
|
||||||
|
connection.commit()
|
||||||
|
assert_no_old_prefixes(connection, [old for old, _ in replacements if old])
|
||||||
|
result = connection.execute("PRAGMA integrity_check").fetchone()[0]
|
||||||
|
if result != "ok":
|
||||||
|
raise RuntimeError(f"integrity_check failed after path migration: {database}: {result}")
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
|
def migrate_project_database(database: Path, old_root: str, new_root: str) -> None:
|
||||||
|
connection = sqlite3.connect(database)
|
||||||
|
try:
|
||||||
|
tables = {
|
||||||
|
row[0]
|
||||||
|
for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")
|
||||||
|
if not row[0].startswith("sqlite_")
|
||||||
|
}
|
||||||
|
for table in tables:
|
||||||
|
table_identifier = sql_identifier(table)
|
||||||
|
columns = connection.execute(f"PRAGMA table_info({table_identifier})").fetchall()
|
||||||
|
for column in columns:
|
||||||
|
column_name = column[1]
|
||||||
|
column_identifier = sql_identifier(column_name)
|
||||||
|
if (column[2] or "").upper() not in {"TEXT", ""}:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
query = (
|
||||||
|
f"SELECT rowid, {column_identifier} FROM {table_identifier} "
|
||||||
|
f"WHERE {column_identifier} IS NOT NULL"
|
||||||
|
)
|
||||||
|
rows = connection.execute(query).fetchall()
|
||||||
|
except sqlite3.Error:
|
||||||
|
continue
|
||||||
|
for rowid, value in rows:
|
||||||
|
replacement = _replace_prefix(value, old_root, new_root)
|
||||||
|
if replacement != value:
|
||||||
|
connection.execute(
|
||||||
|
f"UPDATE {table_identifier} SET {column_identifier} = ? "
|
||||||
|
"WHERE rowid = ?",
|
||||||
|
(replacement, rowid),
|
||||||
|
)
|
||||||
|
connection.commit()
|
||||||
|
assert_no_old_prefixes(connection, [old_root] if old_root else [])
|
||||||
|
result = connection.execute("PRAGMA integrity_check").fetchone()[0]
|
||||||
|
if result != "ok":
|
||||||
|
raise RuntimeError(
|
||||||
|
f"integrity_check failed after project path migration: {database}: {result}"
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_target(path: Path, force: bool) -> Path:
|
||||||
|
if path.is_symlink():
|
||||||
|
raise RuntimeError(f"refusing symlink target: {path}")
|
||||||
|
if path.exists() and not path.is_dir():
|
||||||
|
raise RuntimeError(f"target exists and is not a directory: {path}")
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
if path.exists() and any(path.iterdir()) and not force:
|
||||||
|
raise RuntimeError(f"target is not empty: {path}; use --force")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _commit_directory(temporary: Path, target: Path, force: bool) -> Path | None:
|
||||||
|
_validate_target(target, force)
|
||||||
|
previous: Path | None = None
|
||||||
|
if target.exists():
|
||||||
|
previous = target.with_name(f"{target.name}.old.{int(time.time())}")
|
||||||
|
target.rename(previous)
|
||||||
|
try:
|
||||||
|
temporary.rename(target)
|
||||||
|
except Exception:
|
||||||
|
if previous is not None and previous.exists() and not target.exists():
|
||||||
|
previous.rename(target)
|
||||||
|
raise
|
||||||
|
return previous
|
||||||
|
|
||||||
|
|
||||||
|
def restore_global(
|
||||||
|
backup_dir: Path,
|
||||||
|
target: Path,
|
||||||
|
force: bool,
|
||||||
|
old_project_root: str = "",
|
||||||
|
new_project_root: str = "",
|
||||||
|
) -> None:
|
||||||
|
source = backup_dir / "global-state"
|
||||||
|
manifest_path = source / "MANIFEST.json"
|
||||||
|
if not manifest_path.exists():
|
||||||
|
raise RuntimeError(f"global-state manifest is missing: {manifest_path}")
|
||||||
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
old_root = manifest.get("original_root", "")
|
||||||
|
_validate_target(target, force)
|
||||||
|
temporary = Path(tempfile.mkdtemp(prefix=f".{target.name}.global-restore.", dir=target.parent))
|
||||||
|
try:
|
||||||
|
copy_tree_files(
|
||||||
|
source,
|
||||||
|
temporary,
|
||||||
|
skip_databases=False,
|
||||||
|
reject_symlinks=True,
|
||||||
|
)
|
||||||
|
for metadata_name in ("MANIFEST.json", "GLOBAL-STATE-SHA256SUMS"):
|
||||||
|
metadata_path = temporary / metadata_name
|
||||||
|
if metadata_path.exists():
|
||||||
|
metadata_path.unlink()
|
||||||
|
for relative in manifest.get("files", []):
|
||||||
|
safe_relative_path(relative)
|
||||||
|
file_hashes = manifest.get("file_hashes", {})
|
||||||
|
for relative, expected_hash in file_hashes.items():
|
||||||
|
safe_relative = safe_relative_path(relative)
|
||||||
|
path = temporary / safe_relative
|
||||||
|
if not path.is_file() or sha256(path) != expected_hash:
|
||||||
|
raise RuntimeError(f"global file checksum mismatch: {relative}")
|
||||||
|
replacements = [(old_root, str(target))]
|
||||||
|
if old_project_root and new_project_root:
|
||||||
|
replacements.append((old_project_root, new_project_root))
|
||||||
|
for database in manifest.get("databases", []):
|
||||||
|
relative = safe_relative_path(database["path"])
|
||||||
|
path = temporary / relative
|
||||||
|
if not path.is_file() or sha256(path) != database.get("sha256"):
|
||||||
|
raise RuntimeError(f"global database checksum mismatch: {relative}")
|
||||||
|
migrate_database_paths(path, replacements)
|
||||||
|
previous = _commit_directory(temporary, target, force)
|
||||||
|
temporary = Path()
|
||||||
|
if previous is not None:
|
||||||
|
print(f"Previous global state retained at: {previous}")
|
||||||
|
finally:
|
||||||
|
if temporary and str(temporary) not in {"", "."}:
|
||||||
|
shutil.rmtree(temporary, ignore_errors=True)
|
||||||
|
print(f"Restored global state to {target}")
|
||||||
|
|
||||||
|
|
||||||
|
def _replace_text(path: Path, replacements: list[tuple[str, str]]) -> list[str]:
|
||||||
|
text = path.read_text(encoding="utf-8")
|
||||||
|
changed: list[str] = []
|
||||||
|
for old, new in replacements:
|
||||||
|
if old in text:
|
||||||
|
text = text.replace(old, new)
|
||||||
|
changed.append(old)
|
||||||
|
path.write_text(text, encoding="utf-8")
|
||||||
|
return changed
|
||||||
|
|
||||||
|
|
||||||
|
def patch_restored_paths(
|
||||||
|
root: Path,
|
||||||
|
old_root: str,
|
||||||
|
new_root: Path,
|
||||||
|
global_root: Path | None = None,
|
||||||
|
) -> None:
|
||||||
|
changes: list[str] = []
|
||||||
|
env_path = root / ".env"
|
||||||
|
if env_path.exists() and old_root:
|
||||||
|
original = root / ".env.archive-original"
|
||||||
|
if not original.exists():
|
||||||
|
shutil.copy2(env_path, original)
|
||||||
|
changes.extend(_replace_text(env_path, [(old_root, str(new_root))]))
|
||||||
|
if env_path.exists() and global_root is not None:
|
||||||
|
lines = env_path.read_text(encoding="utf-8").splitlines()
|
||||||
|
keys = {"UMBRA_GLOBAL_STATE_DIR", "UMBRA_VOICE_REGISTRY_PATH"}
|
||||||
|
lines = [line for line in lines if line.split("=", 1)[0] not in keys]
|
||||||
|
lines.extend(
|
||||||
|
[
|
||||||
|
f"UMBRA_GLOBAL_STATE_DIR={json.dumps(str(global_root))}",
|
||||||
|
f"UMBRA_VOICE_REGISTRY_PATH={json.dumps(str(global_root / 'voice_registry.db'))}",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
env_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||||
|
changes.append(".env global-state settings")
|
||||||
|
|
||||||
|
config_path = root / "umbraaudio" / "config.py"
|
||||||
|
if config_path.exists() and old_root:
|
||||||
|
text = config_path.read_text(encoding="utf-8")
|
||||||
|
if "import os\n" not in text:
|
||||||
|
text = text.replace("import re\n", "import os\nimport re\n", 1)
|
||||||
|
portable_project_base = (
|
||||||
|
"PROJECT_BASE = Path(\n"
|
||||||
|
' os.environ.get("UMBRA_PROJECT_BASE", '
|
||||||
|
"str(Path(__file__).resolve().parents[1]))\n"
|
||||||
|
").expanduser()"
|
||||||
|
)
|
||||||
|
text = text.replace(
|
||||||
|
f'PROJECT_BASE = Path("{old_root}")',
|
||||||
|
portable_project_base,
|
||||||
|
)
|
||||||
|
text = text.replace(
|
||||||
|
f'"{old_root}/models/Qwen3-TTS-12Hz-1.7B-VoiceDesign"',
|
||||||
|
'str(PROJECT_BASE / "models/Qwen3-TTS-12Hz-1.7B-VoiceDesign")',
|
||||||
|
)
|
||||||
|
text = text.replace(
|
||||||
|
f'"{old_root}/models/Qwen3-TTS-12Hz-1.7B-Base"',
|
||||||
|
'str(PROJECT_BASE / "models/Qwen3-TTS-12Hz-1.7B-Base")',
|
||||||
|
)
|
||||||
|
portable_voice_registry = (
|
||||||
|
"voice_registry_path: Path = Field(\n"
|
||||||
|
" default_factory=lambda: Path(\n"
|
||||||
|
" os.environ.get(\n"
|
||||||
|
' "UMBRA_VOICE_REGISTRY_PATH",\n'
|
||||||
|
' str(Path.home() / ".umbra" / "voice_registry.db"),\n'
|
||||||
|
" )\n"
|
||||||
|
" ).expanduser()\n"
|
||||||
|
")"
|
||||||
|
)
|
||||||
|
old_voice_field = (
|
||||||
|
'voice_registry_path: Path = Field(default=Path.home() / ".umbra" / '
|
||||||
|
'"voice_registry.db")'
|
||||||
|
)
|
||||||
|
text = text.replace(old_voice_field, portable_voice_registry)
|
||||||
|
config_path.write_text(text, encoding="utf-8")
|
||||||
|
changes.append("umbraaudio/config.py")
|
||||||
|
|
||||||
|
library_path = root / "umbraaudio" / "library.py"
|
||||||
|
if library_path.exists():
|
||||||
|
text = library_path.read_text(encoding="utf-8")
|
||||||
|
if "import os\n" not in text:
|
||||||
|
text = text.replace("import json\n", "import json\nimport os\n", 1)
|
||||||
|
portable_library_root = (
|
||||||
|
"def _global_state_root() -> Path:\n"
|
||||||
|
" return Path(\n"
|
||||||
|
' os.environ.get("UMBRA_GLOBAL_STATE_DIR", str(Path.home() / ".umbra"))\n'
|
||||||
|
" ).expanduser()\n"
|
||||||
|
)
|
||||||
|
text = text.replace(
|
||||||
|
'DEFAULT_LIBRARY_DB = Path.home() / ".umbra" / "voice_registry.db"\n'
|
||||||
|
'DEFAULT_LIBRARY_VOICES = Path.home() / ".umbra" / "library" / "voices"',
|
||||||
|
portable_library_root
|
||||||
|
+ 'DEFAULT_LIBRARY_DB = _global_state_root() / "voice_registry.db"\n'
|
||||||
|
+ 'DEFAULT_LIBRARY_VOICES = _global_state_root() / "library" / "voices"',
|
||||||
|
)
|
||||||
|
old_db_getter = "def get_library_db_path() -> Path:\n return DEFAULT_LIBRARY_DB"
|
||||||
|
new_db_getter = (
|
||||||
|
"def get_library_db_path() -> Path:\n return _global_state_root() / "
|
||||||
|
'"voice_registry.db"'
|
||||||
|
)
|
||||||
|
text = text.replace(old_db_getter, new_db_getter)
|
||||||
|
old_voices_getter = (
|
||||||
|
"def get_library_voices_dir() -> Path:\n return DEFAULT_LIBRARY_VOICES"
|
||||||
|
)
|
||||||
|
new_voices_getter = (
|
||||||
|
"def get_library_voices_dir() -> Path:\n return _global_state_root() / "
|
||||||
|
'"library" / "voices"'
|
||||||
|
)
|
||||||
|
text = text.replace(old_voices_getter, new_voices_getter)
|
||||||
|
library_path.write_text(text, encoding="utf-8")
|
||||||
|
changes.append("umbraaudio/library.py")
|
||||||
|
|
||||||
|
server_path = root / "scripts" / "qwen_tts_server.py"
|
||||||
|
if server_path.exists() and old_root:
|
||||||
|
portable_prompt_dir = (
|
||||||
|
"str(Path(__file__).resolve().parents[1] / "
|
||||||
|
'"DealHunterShip_4.0e_whole-to-audio/.umbra/voice_prompts")'
|
||||||
|
)
|
||||||
|
portable_voicedesign = (
|
||||||
|
'str(Path(__file__).resolve().parents[1] / "models/Qwen3-TTS-12Hz-1.7B-VoiceDesign")'
|
||||||
|
)
|
||||||
|
portable_base = (
|
||||||
|
'str(Path(__file__).resolve().parents[1] / "models/Qwen3-TTS-12Hz-1.7B-Base")'
|
||||||
|
)
|
||||||
|
replacements = [
|
||||||
|
(
|
||||||
|
f'"{old_root}/DealHunterShip_4.0e_whole-to-audio/.umbra/voice_prompts"',
|
||||||
|
portable_prompt_dir,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
f'"{old_root}/models/Qwen3-TTS-12Hz-1.7B-VoiceDesign"',
|
||||||
|
portable_voicedesign,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
f'"{old_root}/models/Qwen3-TTS-12Hz-1.7B-Base"',
|
||||||
|
portable_base,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
changes.extend(_replace_text(server_path, replacements))
|
||||||
|
|
||||||
|
storage_path = root / "umbraaudio" / "storage.py"
|
||||||
|
if storage_path.exists():
|
||||||
|
text = storage_path.read_text(encoding="utf-8")
|
||||||
|
if "import os\n" not in text:
|
||||||
|
text = text.replace("import logging\n", "import logging\nimport os\n", 1)
|
||||||
|
if old_root:
|
||||||
|
portable_local_root = (
|
||||||
|
"DEFAULT_LOCAL_ROOT = Path(os.environ.get("
|
||||||
|
'"UMBRA_PROJECT_ROOT", str(Path(__file__).resolve().parents[1]))'
|
||||||
|
").expanduser()"
|
||||||
|
)
|
||||||
|
text = text.replace(
|
||||||
|
f'DEFAULT_LOCAL_ROOT = Path("{old_root}")',
|
||||||
|
portable_local_root,
|
||||||
|
)
|
||||||
|
portable_nas_root = (
|
||||||
|
"DEFAULT_NAS_ROOT = Path(os.environ.get("
|
||||||
|
'"UMBRA_NAS_ROOT", "/Volumes/Umbra/UmbraAudio"'
|
||||||
|
")).expanduser()"
|
||||||
|
)
|
||||||
|
text = text.replace(
|
||||||
|
'DEFAULT_NAS_ROOT = Path("/Volumes/Umbra/UmbraAudio")',
|
||||||
|
portable_nas_root,
|
||||||
|
)
|
||||||
|
storage_path.write_text(text, encoding="utf-8")
|
||||||
|
changes.append("umbraaudio/storage.py")
|
||||||
|
|
||||||
|
if old_root:
|
||||||
|
escaped_root = json.dumps(str(new_root))[1:-1]
|
||||||
|
operational_files = [
|
||||||
|
root / "scripts" / "qwen_tts_server.py",
|
||||||
|
root / "tts_server" / "qwen_tts_server.py",
|
||||||
|
root / "Qwen3-TTS" / "qwen_tts" / "cli" / "demo.py",
|
||||||
|
root / "start_server.sh",
|
||||||
|
root / "test_voice_clone.py",
|
||||||
|
]
|
||||||
|
for operational_path in operational_files:
|
||||||
|
if operational_path.exists():
|
||||||
|
_replace_text(operational_path, [(old_root, escaped_root)])
|
||||||
|
|
||||||
|
nas_lifecycle = root / "scripts" / "nas_lifecycle.py"
|
||||||
|
if nas_lifecycle.exists():
|
||||||
|
text = nas_lifecycle.read_text(encoding="utf-8")
|
||||||
|
if "import os\n" not in text:
|
||||||
|
text = text.replace("import logging\n", "import logging\nimport os\n", 1)
|
||||||
|
text = text.replace(
|
||||||
|
f'DEFAULT_LOCAL_ROOT = Path("{old_root}")',
|
||||||
|
"DEFAULT_LOCAL_ROOT = Path(__file__).resolve().parents[1]",
|
||||||
|
)
|
||||||
|
portable_nas = (
|
||||||
|
"DEFAULT_NAS_ROOT = Path(os.environ.get("
|
||||||
|
'"UMBRA_NAS_ROOT", "/Volumes/Umbra/UmbraAudio"'
|
||||||
|
")).expanduser()"
|
||||||
|
)
|
||||||
|
text = text.replace(
|
||||||
|
'DEFAULT_NAS_ROOT = Path("/Volumes/Umbra/UmbraAudio")',
|
||||||
|
portable_nas,
|
||||||
|
)
|
||||||
|
text = text.replace(
|
||||||
|
'MOUNT_SCRIPT = Path.home() / ".umbra/mount_nas.sh"',
|
||||||
|
'MOUNT_SCRIPT = _global_state_root() / "mount_nas.sh"',
|
||||||
|
)
|
||||||
|
text = text.replace(
|
||||||
|
'CONFIG_PATH = Path.home() / ".umbra/nas_config.json"',
|
||||||
|
'CONFIG_PATH = _global_state_root() / "nas_config.json"',
|
||||||
|
)
|
||||||
|
if "def _global_state_root()" not in text:
|
||||||
|
global_root_function = (
|
||||||
|
"def _global_state_root() -> Path:\n"
|
||||||
|
" return Path(os.environ.get(\n"
|
||||||
|
' "UMBRA_GLOBAL_STATE_DIR", str(Path.home() / ".umbra")\n'
|
||||||
|
" )).expanduser()\n\n"
|
||||||
|
"DEFAULT_LOCAL_ROOT ="
|
||||||
|
)
|
||||||
|
text = text.replace(
|
||||||
|
"DEFAULT_LOCAL_ROOT =",
|
||||||
|
global_root_function,
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
nas_lifecycle.write_text(text, encoding="utf-8")
|
||||||
|
changes.append("scripts/nas_lifecycle.py")
|
||||||
|
|
||||||
|
sync_voice = root / "tts_server" / "sync_voice_data.sh"
|
||||||
|
if sync_voice.exists():
|
||||||
|
text = sync_voice.read_text(encoding="utf-8")
|
||||||
|
text = text.replace(
|
||||||
|
'UMBRA_DATA="${UMBRA_DATA_DIR:-/Volumes/1TB_APL_Storage/UmbraAudio}"',
|
||||||
|
'PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"\n'
|
||||||
|
'UMBRA_DATA="${UMBRA_DATA_DIR:-$PROJECT_ROOT}"',
|
||||||
|
)
|
||||||
|
sync_voice.write_text(text, encoding="utf-8")
|
||||||
|
changes.append("tts_server/sync_voice_data.sh")
|
||||||
|
|
||||||
|
verify_nas = root / "scripts" / "verify_nas.py"
|
||||||
|
if verify_nas.exists():
|
||||||
|
text = verify_nas.read_text(encoding="utf-8")
|
||||||
|
text = text.replace(
|
||||||
|
f'Path("{old_root}")',
|
||||||
|
"Path(__file__).resolve().parents[1]",
|
||||||
|
)
|
||||||
|
verify_nas.write_text(text, encoding="utf-8")
|
||||||
|
changes.append("scripts/verify_nas.py")
|
||||||
|
|
||||||
|
embeddings = root / "scripts" / "extract_embeddings.py"
|
||||||
|
if embeddings.exists():
|
||||||
|
text = embeddings.read_text(encoding="utf-8")
|
||||||
|
text = text.replace(
|
||||||
|
f'"{old_root}/models/Qwen3-TTS-12Hz-1.7B-Base"',
|
||||||
|
'str(project_root / "models/Qwen3-TTS-12Hz-1.7B-Base")',
|
||||||
|
)
|
||||||
|
text = text.replace(
|
||||||
|
f'"{old_root}/models/Qwen3-TTS-12Hz-1.7B-VoiceDesign"',
|
||||||
|
'str(project_root / "models/Qwen3-TTS-12Hz-1.7B-VoiceDesign")',
|
||||||
|
)
|
||||||
|
embeddings.write_text(text, encoding="utf-8")
|
||||||
|
changes.append("scripts/extract_embeddings.py")
|
||||||
|
|
||||||
|
legacy_start = root / "start_server.sh"
|
||||||
|
if legacy_start.exists():
|
||||||
|
legacy_start.write_text(
|
||||||
|
"#!/usr/bin/env bash\n"
|
||||||
|
"set -Eeuo pipefail\n"
|
||||||
|
"printf '%s\\n' 'Legacy Gradio demo launcher is disabled in this restore.'\n"
|
||||||
|
"printf '%s\\n' 'Use scripts/start_web_server.sh or the generated service commands.'\n"
|
||||||
|
"exit 1\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
legacy_start.chmod(0o755)
|
||||||
|
changes.append("start_server.sh")
|
||||||
|
|
||||||
|
start_script = root / "scripts" / "start_web_server.sh"
|
||||||
|
if start_script.exists():
|
||||||
|
web_command = (
|
||||||
|
'exec "$PROJ/.venv/bin/python" -c \''
|
||||||
|
"from pathlib import Path; from umbraaudio.web import create_app; "
|
||||||
|
"from umbraaudio.config import load_config; import uvicorn; "
|
||||||
|
'c = load_config(Path(".")); uvicorn.run(create_app(c), '
|
||||||
|
'host="0.0.0.0", port=8420, log_level="info")\'\n'
|
||||||
|
)
|
||||||
|
global_exports = ""
|
||||||
|
if global_root is not None:
|
||||||
|
global_exports = (
|
||||||
|
f"export UMBRA_GLOBAL_STATE_DIR={shlex.quote(str(global_root))}\n"
|
||||||
|
"export UMBRA_VOICE_REGISTRY_PATH="
|
||||||
|
f"{shlex.quote(str(global_root / 'voice_registry.db'))}\n"
|
||||||
|
)
|
||||||
|
start_script.write_text(
|
||||||
|
"#!/usr/bin/env bash\n"
|
||||||
|
"set -Eeuo pipefail\n"
|
||||||
|
'PROJ="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)"\n'
|
||||||
|
'exec >> "$PROJ/logs/web_server.log" 2>&1\n'
|
||||||
|
"echo \"[$(date '+%F %T')] web server starting (wrapper pid $$)\"\n"
|
||||||
|
'cd "$PROJ"\n' + global_exports + web_command,
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
start_script.chmod(0o755)
|
||||||
|
changes.append("scripts/start_web_server.sh")
|
||||||
|
|
||||||
|
if old_root:
|
||||||
|
for database in sorted(root.rglob("*.db")):
|
||||||
|
if not database.is_file() or any(
|
||||||
|
part in {".venv", ".python"} for part in database.parts
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
if not is_sqlite_file(database):
|
||||||
|
continue
|
||||||
|
migrate_project_database(database, old_root, str(new_root))
|
||||||
|
|
||||||
|
commands = root / "RESTORED-SERVICE-COMMANDS.md"
|
||||||
|
web_command = (
|
||||||
|
"python -c 'from pathlib import Path; from umbraaudio.web import create_app; "
|
||||||
|
"from umbraaudio.config import load_config; import uvicorn; "
|
||||||
|
'uvicorn.run(create_app(load_config(Path("."))), host="0.0.0.0", port=8420)\''
|
||||||
|
)
|
||||||
|
voicedesign_model = json.dumps(str(new_root / "models/Qwen3-TTS-12Hz-1.7B-VoiceDesign"))
|
||||||
|
base_model = json.dumps(str(new_root / "models/Qwen3-TTS-12Hz-1.7B-Base"))
|
||||||
|
global_exports = ""
|
||||||
|
if global_root is not None:
|
||||||
|
global_exports = (
|
||||||
|
f"export UMBRA_GLOBAL_STATE_DIR={shlex.quote(str(global_root))}\n"
|
||||||
|
"export UMBRA_VOICE_REGISTRY_PATH="
|
||||||
|
f"{shlex.quote(str(global_root / 'voice_registry.db'))}\n"
|
||||||
|
)
|
||||||
|
commands.write_text(
|
||||||
|
"# Restored UmbraAudio service commands\n\n"
|
||||||
|
"These commands are generated for the restore location and are not installed "
|
||||||
|
"automatically.\n\n"
|
||||||
|
"```bash\n"
|
||||||
|
f"cd {json.dumps(str(new_root))}\n"
|
||||||
|
'export PATH="$PWD/.venv/bin:$PATH"\n'
|
||||||
|
f"export UMBRA_PROJECT_BASE={json.dumps(str(new_root))}\n"
|
||||||
|
+ global_exports
|
||||||
|
+ f"{web_command}\n"
|
||||||
|
f"QWEN_VOICEDESIGN_MODEL={voicedesign_model}\n"
|
||||||
|
f"QWEN_BASE_MODEL={base_model}\n"
|
||||||
|
"python scripts/qwen_tts_server.py\n"
|
||||||
|
"```\n\n"
|
||||||
|
"The external launchd plist is intentionally not installed automatically; review the "
|
||||||
|
"context/ directory and adapt paths and permissions first.\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
(root / "PORTABILITY-PATCH.txt").write_text(
|
||||||
|
"The following restored-copy portability adjustments were applied:\n"
|
||||||
|
+ "".join(f"- {item}\n" for item in sorted(set(changes)))
|
||||||
|
+ "\nThe original .env was preserved as .env.archive-original.\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def write_checksum_manifest(root: Path) -> Path:
|
||||||
|
output = root / "SHA256SUMS"
|
||||||
|
lines: list[str] = []
|
||||||
|
for path in sorted(root.rglob("*")):
|
||||||
|
if not path.is_file() or path == output:
|
||||||
|
continue
|
||||||
|
relative = path.relative_to(root).as_posix()
|
||||||
|
lines.append(f"{sha256(path)} {relative}\n")
|
||||||
|
output.write_text("".join(lines), encoding="utf-8")
|
||||||
|
return output
|
||||||
|
|
||||||
|
|
||||||
|
def validate_tree(root: Path, old_root: str | None = None) -> list[str]:
|
||||||
|
errors: list[str] = []
|
||||||
|
required = (root / "pyproject.toml", root / "umbraaudio", root / "Qwen3-TTS")
|
||||||
|
for path in required:
|
||||||
|
if not path.exists():
|
||||||
|
errors.append(f"missing required path: {path}")
|
||||||
|
model = root / "models" / "Qwen3-TTS-12Hz-1.7B-Base" / "model.safetensors"
|
||||||
|
if not model.exists():
|
||||||
|
errors.append(f"missing model: {model}")
|
||||||
|
for database in sorted(root.rglob("*.db")):
|
||||||
|
if not database.is_file():
|
||||||
|
continue
|
||||||
|
if database.stat().st_size == 0:
|
||||||
|
continue
|
||||||
|
if not is_sqlite_file(database):
|
||||||
|
if any(
|
||||||
|
part in {".mypy_cache", ".pytest_cache", ".ruff_cache"} for part in database.parts
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
errors.append(f"non-SQLite .db file: {database}")
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
connection = sqlite3.connect(f"file:{database}?mode=ro", uri=True)
|
||||||
|
result = connection.execute("PRAGMA quick_check").fetchone()[0]
|
||||||
|
if old_root:
|
||||||
|
try:
|
||||||
|
assert_no_old_prefixes(connection, [old_root])
|
||||||
|
except RuntimeError as exc:
|
||||||
|
errors.append(str(exc))
|
||||||
|
connection.close()
|
||||||
|
if result != "ok":
|
||||||
|
errors.append(f"SQLite quick_check failed: {database}: {result}")
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
errors.append(f"SQLite check error: {database}: {exc}")
|
||||||
|
if old_root:
|
||||||
|
code_files = [
|
||||||
|
root / "umbraaudio" / "config.py",
|
||||||
|
root / "umbraaudio" / "library.py",
|
||||||
|
root / "umbraaudio" / "storage.py",
|
||||||
|
root / "scripts" / "nas_lifecycle.py",
|
||||||
|
root / "scripts" / "verify_nas.py",
|
||||||
|
root / "scripts" / "extract_embeddings.py",
|
||||||
|
root / "scripts" / "qwen_tts_server.py",
|
||||||
|
root / "tts_server" / "qwen_tts_server.py",
|
||||||
|
root / "tts_server" / "sync_voice_data.sh",
|
||||||
|
root / "start_server.sh",
|
||||||
|
]
|
||||||
|
for code_file in code_files:
|
||||||
|
if code_file.exists() and old_root in code_file.read_text(encoding="utf-8"):
|
||||||
|
errors.append(f"old project root remains in {code_file}")
|
||||||
|
return errors
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
sub = parser.add_subparsers(dest="command", required=True)
|
||||||
|
|
||||||
|
snapshot = sub.add_parser("snapshot-global")
|
||||||
|
snapshot.add_argument("--source", type=Path, required=True)
|
||||||
|
snapshot.add_argument("--output", type=Path, required=True)
|
||||||
|
snapshot.add_argument("--original-root", type=Path, required=True)
|
||||||
|
|
||||||
|
restore = sub.add_parser("restore-global")
|
||||||
|
restore.add_argument("--backup-dir", type=Path, required=True)
|
||||||
|
restore.add_argument("--target", type=Path, required=True)
|
||||||
|
restore.add_argument("--force", action="store_true")
|
||||||
|
restore.add_argument("--old-project-root", default="")
|
||||||
|
restore.add_argument("--new-project-root", default="")
|
||||||
|
|
||||||
|
patch = sub.add_parser("patch-paths")
|
||||||
|
patch.add_argument("--root", type=Path, required=True)
|
||||||
|
patch.add_argument("--old-root", required=True)
|
||||||
|
patch.add_argument("--new-root", type=Path, required=True)
|
||||||
|
patch.add_argument("--global-state-dir", type=Path)
|
||||||
|
|
||||||
|
manifest = sub.add_parser("write-manifest")
|
||||||
|
manifest.add_argument("--root", type=Path, required=True)
|
||||||
|
|
||||||
|
validate = sub.add_parser("validate-tree")
|
||||||
|
validate.add_argument("--root", type=Path, required=True)
|
||||||
|
validate.add_argument("--old-root", default="")
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
try:
|
||||||
|
if args.command == "snapshot-global":
|
||||||
|
snapshot_global(args.source, args.output, args.original_root)
|
||||||
|
elif args.command == "restore-global":
|
||||||
|
restore_global(
|
||||||
|
args.backup_dir,
|
||||||
|
args.target,
|
||||||
|
args.force,
|
||||||
|
args.old_project_root,
|
||||||
|
args.new_project_root,
|
||||||
|
)
|
||||||
|
elif args.command == "patch-paths":
|
||||||
|
patch_restored_paths(
|
||||||
|
args.root,
|
||||||
|
args.old_root,
|
||||||
|
args.new_root,
|
||||||
|
args.global_state_dir,
|
||||||
|
)
|
||||||
|
elif args.command == "write-manifest":
|
||||||
|
write_checksum_manifest(args.root)
|
||||||
|
elif args.command == "validate-tree":
|
||||||
|
errors = validate_tree(args.root, args.old_root or None)
|
||||||
|
if errors:
|
||||||
|
for error in errors:
|
||||||
|
print(error, file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
print(f"Tree validation passed: {args.root}")
|
||||||
|
return 0
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
print(f"ERROR: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
4
infra/BOOTSTRAP-SHA256SUMS
Normal file
4
infra/BOOTSTRAP-SHA256SUMS
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
47df21afa85aa7bbb550fa6622b0b4fa3ec8ea1e100fecf47375db60990186ad bootstrap_enterprise_mint.sh
|
||||||
|
bea205a4113e7b0ca7eb7b3da4344a1711143d111c4c13049698369506768f72 enterprise-bootstrap.env.example
|
||||||
|
5a7fe225d6cc67fea080e392dc3c3b9d896800eeb3eb1e6391791485d68764ed README-enterprise-bootstrap.md
|
||||||
|
deb8b51f9892ad5f3b50738514da4d59b2697b30ad356e1d1bdc81ae1c862749 postgres/001_control_plane.sql
|
||||||
2
infra/DATABASE-SHA256SUMS
Normal file
2
infra/DATABASE-SHA256SUMS
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
d6f52246a3a6899d48d8ed115deb5b4661362e9b5695eb946b0e0dbd0ded03cf bootstrap_umbracontrol_database.sh
|
||||||
|
deb8b51f9892ad5f3b50738514da4d59b2697b30ad356e1d1bdc81ae1c862749 postgres/001_control_plane.sql
|
||||||
63
infra/README-enterprise-bootstrap.md
Normal file
63
infra/README-enterprise-bootstrap.md
Normal file
@@ -0,0 +1,63 @@
|
|||||||
|
# Umbra enterprise phase-one bootstrap
|
||||||
|
|
||||||
|
This directory contains the first, infrastructure-only bootstrap for the UmbraTrans enterprise control plane.
|
||||||
|
|
||||||
|
It is designed for Linux Mint 22.1 (Ubuntu 24.04-family) and is intentionally conservative:
|
||||||
|
|
||||||
|
- It does not modify Qwen.
|
||||||
|
- It does not stop or alter Docker containers.
|
||||||
|
- It does not change firewall rules.
|
||||||
|
- It does not install application code or systemd application units yet.
|
||||||
|
- It does not place PostgreSQL on SMB/NAS.
|
||||||
|
- It creates database roles, a local database, core identity/tenancy tables, row-level-security policies, a one-time bootstrap administrator, and least-privilege service accounts.
|
||||||
|
|
||||||
|
## Files
|
||||||
|
|
||||||
|
- `bootstrap_enterprise_mint.sh` — root bootstrap and read-only inspection tool.
|
||||||
|
- `enterprise-bootstrap.env.example` — non-secret configuration template.
|
||||||
|
- `postgres/001_control_plane.sql` — initial identity, organization, project, role, invitation, session, MFA, bootstrap-token, audit, and RLS schema.
|
||||||
|
|
||||||
|
## Safe usage on oldbox
|
||||||
|
|
||||||
|
The script is intended to be copied to the `umbra` account and reviewed there:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ssh umbra@oldbox.bdmurphy.me
|
||||||
|
mkdir -p ~/umbra-enterprise-bootstrap
|
||||||
|
# copy the three files into that directory
|
||||||
|
cp enterprise-bootstrap.env.example enterprise-bootstrap.env
|
||||||
|
$EDITOR enterprise-bootstrap.env
|
||||||
|
bash -n bootstrap_enterprise_mint.sh
|
||||||
|
./bootstrap_enterprise_mint.sh --check
|
||||||
|
./bootstrap_enterprise_mint.sh --dry-run --config "$PWD/enterprise-bootstrap.env"
|
||||||
|
sudo ./bootstrap_enterprise_mint.sh --apply --config /home/umbra/umbra-enterprise-bootstrap/enterprise-bootstrap.env
|
||||||
|
```
|
||||||
|
|
||||||
|
The root password is entered only in the user's terminal. It is not stored in the configuration file or command line.
|
||||||
|
|
||||||
|
After a successful apply:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo ./bootstrap_enterprise_mint.sh --verify --config /home/umbra/umbra-enterprise-bootstrap/enterprise-bootstrap.env
|
||||||
|
```
|
||||||
|
|
||||||
|
The initial one-time administrator token is stored root-only at:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/root/umbra-enterprise-bootstrap.token
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not paste that token into chat. The application bootstrap flow will consume it and replace it with normal authentication.
|
||||||
|
|
||||||
|
## Deliberate exclusions
|
||||||
|
|
||||||
|
The script does not yet:
|
||||||
|
|
||||||
|
- create or install the UmbraTrans API/worker application;
|
||||||
|
- install a reverse proxy or certificate;
|
||||||
|
- expose Qwen to other machines;
|
||||||
|
- add Qwen authentication or rate limiting;
|
||||||
|
- modify UFW/nftables/iptables;
|
||||||
|
- migrate existing Docker, MySQL, LM Studio, Ollama, or Qwen data.
|
||||||
|
|
||||||
|
Those are separate reviewed changes after the host and database foundation are verified.
|
||||||
59
infra/agent_bridge/README.md
Normal file
59
infra/agent_bridge/README.md
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
# Temporary UmbraTrans agent bridge
|
||||||
|
|
||||||
|
This is a one-hour, LAN-only browser pairing point for communication between
|
||||||
|
the Mac agent and a browser session on oldbox. It is not a remote shell.
|
||||||
|
|
||||||
|
## Properties
|
||||||
|
|
||||||
|
- Random OS-assigned TCP port.
|
||||||
|
- HTTPS with a temporary self-signed certificate.
|
||||||
|
- One-hour hard expiry.
|
||||||
|
- Password is entered once in a native Mac setup dialog and once in the
|
||||||
|
browser on the other machine.
|
||||||
|
- Password is hashed in memory; plaintext is not logged or written to disk.
|
||||||
|
- HttpOnly/Secure/SameSite session cookie and CSRF token.
|
||||||
|
- No raw shell, no arbitrary command execution, no attachments.
|
||||||
|
- Messages are atomic, bounded JSON files in a private queue.
|
||||||
|
|
||||||
|
The browser terminal is an agent message channel. The Mac agent must poll the
|
||||||
|
inbox and send replies through the local bridge tooling; commands are not
|
||||||
|
executed automatically from browser text.
|
||||||
|
|
||||||
|
## Start
|
||||||
|
|
||||||
|
The listener runs on the Mac's LAN address. The startup dialog asks for the
|
||||||
|
one-hour pairing password. The process prints the exact URL and port.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
PYTHONPATH=/Volumes/1TB_APL_Storage/UmbraAudio/infra \
|
||||||
|
/usr/bin/python3 -m agent_bridge \
|
||||||
|
--host 192.168.123.12 \
|
||||||
|
--ttl 3600
|
||||||
|
```
|
||||||
|
|
||||||
|
Open the printed `https://192.168.123.12:<port>/` URL in a browser on oldbox.
|
||||||
|
Accept the temporary certificate warning if the browser requires it, then pair
|
||||||
|
with the same one-hour password.
|
||||||
|
|
||||||
|
## Queue
|
||||||
|
|
||||||
|
The default queue is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
~/.local/state/umbratrans/agent-bridge/
|
||||||
|
inbox/ # browser messages waiting for the Mac agent
|
||||||
|
outbox/ # Mac-agent replies waiting for the browser
|
||||||
|
processed/ # browser messages already read
|
||||||
|
delivered/ # browser replies acknowledged by the browser
|
||||||
|
```
|
||||||
|
|
||||||
|
All files are mode `0600`; directories are mode `0700`.
|
||||||
|
|
||||||
|
## Safety
|
||||||
|
|
||||||
|
- Do not expose the listener to the public internet.
|
||||||
|
- Keep the one-hour timer unchanged.
|
||||||
|
- Do not put passwords or private keys in messages.
|
||||||
|
- Stop the process after the session if the automatic timer is interrupted.
|
||||||
3
infra/agent_bridge/__init__.py
Normal file
3
infra/agent_bridge/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
"""Temporary browser-to-agent bridge for UmbraTrans."""
|
||||||
|
|
||||||
|
__all__: list[str] = []
|
||||||
3
infra/agent_bridge/__main__.py
Normal file
3
infra/agent_bridge/__main__.py
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
from .server import main
|
||||||
|
|
||||||
|
raise SystemExit(main())
|
||||||
31
infra/agent_bridge/client.py
Normal file
31
infra/agent_bridge/client.py
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .server import MessageStore
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
parser = argparse.ArgumentParser(description="Read/write the local UmbraTrans agent bridge queue")
|
||||||
|
parser.add_argument("action", choices=("poll", "send", "status"))
|
||||||
|
parser.add_argument("--root", default="~/.local/state/umbratrans/agent-bridge")
|
||||||
|
parser.add_argument("--title", default="Mac agent")
|
||||||
|
parser.add_argument("--body", default="")
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
store = MessageStore(Path(args.root).expanduser())
|
||||||
|
if args.action == "poll":
|
||||||
|
print(json.dumps(store.poll_for_mac(20), ensure_ascii=False))
|
||||||
|
elif args.action == "send":
|
||||||
|
if not args.body:
|
||||||
|
parser.error("--body is required for send")
|
||||||
|
print(json.dumps(store.send_to_browser(args.body, args.title), ensure_ascii=False))
|
||||||
|
else:
|
||||||
|
print(json.dumps(store.status(), sort_keys=True))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
490
infra/agent_bridge/server.py
Normal file
490
infra/agent_bridge/server.py
Normal file
@@ -0,0 +1,490 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import base64
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
import html
|
||||||
|
import http.cookies
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import secrets
|
||||||
|
import signal
|
||||||
|
import ssl
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from http import HTTPStatus
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
from urllib.parse import parse_qs, urlparse
|
||||||
|
|
||||||
|
from remote_credential_broker.popup import macos_secret_prompt
|
||||||
|
|
||||||
|
MAX_BODY_BYTES = 128 * 1024
|
||||||
|
MAX_MESSAGE_CHARS = 64 * 1024
|
||||||
|
SESSION_TTL_SECONDS = 3600
|
||||||
|
PAIR_ATTEMPT_LIMIT = 8
|
||||||
|
PAIR_ATTEMPT_WINDOW = 60
|
||||||
|
|
||||||
|
|
||||||
|
class BridgeError(RuntimeError):
|
||||||
|
"""Base error for the temporary bridge."""
|
||||||
|
|
||||||
|
|
||||||
|
class MessageStore:
|
||||||
|
"""Atomic, non-secret file queue shared by the browser and local tooling."""
|
||||||
|
|
||||||
|
def __init__(self, root: Path) -> None:
|
||||||
|
self.root = root
|
||||||
|
self.inbox = root / "inbox"
|
||||||
|
self.outbox = root / "outbox"
|
||||||
|
self.processed = root / "processed"
|
||||||
|
self.delivered = root / "delivered"
|
||||||
|
for directory in (self.root, self.inbox, self.outbox, self.processed, self.delivered):
|
||||||
|
directory.mkdir(parents=True, exist_ok=True)
|
||||||
|
os.chmod(directory, 0o700)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _safe_name(value: str) -> bool:
|
||||||
|
return bool(value) and len(value) <= 80 and all(c.isalnum() or c in "-_" for c in value)
|
||||||
|
|
||||||
|
def _write(self, directory: Path, message: dict[str, Any]) -> str:
|
||||||
|
message_id = str(message["id"])
|
||||||
|
if not self._safe_name(message_id):
|
||||||
|
raise BridgeError("invalid message id")
|
||||||
|
target = directory / f"{message_id}.json"
|
||||||
|
temporary = directory / f".{message_id}.{uuid.uuid4().hex}.tmp"
|
||||||
|
payload = json.dumps(message, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||||
|
fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||||
|
try:
|
||||||
|
with os.fdopen(fd, "wb") as handle:
|
||||||
|
handle.write(payload)
|
||||||
|
handle.flush()
|
||||||
|
os.fsync(handle.fileno())
|
||||||
|
os.replace(temporary, target)
|
||||||
|
except BaseException:
|
||||||
|
try:
|
||||||
|
temporary.unlink()
|
||||||
|
except FileNotFoundError:
|
||||||
|
pass
|
||||||
|
raise
|
||||||
|
return str(target)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _validate_message(message: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
required = {"schema_version", "id", "created_at", "sender", "kind", "title", "body"}
|
||||||
|
if not required.issubset(message):
|
||||||
|
raise BridgeError("message is missing required fields")
|
||||||
|
if message["schema_version"] != 1:
|
||||||
|
raise BridgeError("unsupported message schema")
|
||||||
|
if not isinstance(message["body"], str) or len(message["body"]) > MAX_MESSAGE_CHARS:
|
||||||
|
raise BridgeError("message body is too large")
|
||||||
|
if not isinstance(message["title"], str) or len(message["title"]) > 200:
|
||||||
|
raise BridgeError("message title is too large")
|
||||||
|
if "\x00" in message["body"] or "\x00" in message["title"]:
|
||||||
|
raise BridgeError("message contains NUL")
|
||||||
|
return message
|
||||||
|
|
||||||
|
def send_to_mac(self, body: str, title: str = "Browser message") -> dict[str, Any]:
|
||||||
|
message = self._validate_message(
|
||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"id": str(uuid.uuid4()),
|
||||||
|
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||||
|
"sender": "oldbox-browser",
|
||||||
|
"kind": "instruction",
|
||||||
|
"correlation_id": None,
|
||||||
|
"title": title,
|
||||||
|
"body": body,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self._write(self.inbox, message)
|
||||||
|
return message
|
||||||
|
|
||||||
|
def send_to_browser(self, body: str, title: str = "Mac agent") -> dict[str, Any]:
|
||||||
|
message = self._validate_message(
|
||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"id": str(uuid.uuid4()),
|
||||||
|
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||||
|
"sender": "mac-agent",
|
||||||
|
"kind": "report",
|
||||||
|
"correlation_id": None,
|
||||||
|
"title": title,
|
||||||
|
"body": body,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self._write(self.outbox, message)
|
||||||
|
return message
|
||||||
|
|
||||||
|
def _read_dir(self, directory: Path, limit: int) -> list[dict[str, Any]]:
|
||||||
|
values: list[dict[str, Any]] = []
|
||||||
|
for path in sorted(directory.glob("*.json"))[:limit]:
|
||||||
|
if not self._safe_name(path.stem):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
if not path.is_file() or path.is_symlink():
|
||||||
|
continue
|
||||||
|
value = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
values.append(self._validate_message(value))
|
||||||
|
except (OSError, json.JSONDecodeError, BridgeError):
|
||||||
|
continue
|
||||||
|
return values
|
||||||
|
|
||||||
|
def poll_for_mac(self, limit: int = 20) -> list[dict[str, Any]]:
|
||||||
|
values = self._read_dir(self.inbox, max(1, min(limit, 100)))
|
||||||
|
for value in values:
|
||||||
|
source = self.inbox / f"{value['id']}.json"
|
||||||
|
destination = self.processed / f"{value['id']}.json"
|
||||||
|
try:
|
||||||
|
os.replace(source, destination)
|
||||||
|
except FileNotFoundError:
|
||||||
|
pass
|
||||||
|
return values
|
||||||
|
|
||||||
|
def read_for_browser(self, limit: int = 50) -> list[dict[str, Any]]:
|
||||||
|
return self._read_dir(self.outbox, max(1, min(limit, 100)))
|
||||||
|
|
||||||
|
def acknowledge(self, message_ids: list[str]) -> int:
|
||||||
|
count = 0
|
||||||
|
for message_id in message_ids[:100]:
|
||||||
|
if not self._safe_name(message_id):
|
||||||
|
continue
|
||||||
|
source = self.outbox / f"{message_id}.json"
|
||||||
|
destination = self.delivered / f"{message_id}.json"
|
||||||
|
try:
|
||||||
|
os.replace(source, destination)
|
||||||
|
count += 1
|
||||||
|
except FileNotFoundError:
|
||||||
|
pass
|
||||||
|
return count
|
||||||
|
|
||||||
|
def status(self) -> dict[str, int]:
|
||||||
|
return {
|
||||||
|
"inbox": len(list(self.inbox.glob("*.json"))),
|
||||||
|
"outbox": len(list(self.outbox.glob("*.json"))),
|
||||||
|
"processed": len(list(self.processed.glob("*.json"))),
|
||||||
|
"delivered": len(list(self.delivered.glob("*.json"))),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class BridgeState:
|
||||||
|
def __init__(self, password: str, ttl_seconds: int = SESSION_TTL_SECONDS) -> None:
|
||||||
|
self.started_at = time.time()
|
||||||
|
self.expires_at = self.started_at + ttl_seconds
|
||||||
|
self.salt = os.urandom(32)
|
||||||
|
self.digest = hashlib.pbkdf2_hmac(
|
||||||
|
"sha256", password.encode("utf-8"), self.salt, 240_000
|
||||||
|
)
|
||||||
|
self.sessions: dict[str, dict[str, Any]] = {}
|
||||||
|
self.attempts: dict[str, list[float]] = {}
|
||||||
|
self.lock = threading.RLock()
|
||||||
|
|
||||||
|
def check_password(self, address: str, password: str) -> bool:
|
||||||
|
now = time.time()
|
||||||
|
with self.lock:
|
||||||
|
recent = [stamp for stamp in self.attempts.get(address, []) if now - stamp < PAIR_ATTEMPT_WINDOW]
|
||||||
|
if len(recent) >= PAIR_ATTEMPT_LIMIT:
|
||||||
|
return False
|
||||||
|
candidate = hashlib.pbkdf2_hmac(
|
||||||
|
"sha256", password.encode("utf-8"), self.salt, 240_000
|
||||||
|
)
|
||||||
|
accepted = hmac.compare_digest(candidate, self.digest)
|
||||||
|
if accepted:
|
||||||
|
self.attempts.pop(address, None)
|
||||||
|
else:
|
||||||
|
recent.append(now)
|
||||||
|
self.attempts[address] = recent
|
||||||
|
return accepted
|
||||||
|
|
||||||
|
def new_session(self) -> tuple[str, str]:
|
||||||
|
session = secrets.token_urlsafe(40)
|
||||||
|
csrf = secrets.token_urlsafe(32)
|
||||||
|
with self.lock:
|
||||||
|
self.sessions[session] = {"csrf": csrf, "expires_at": self.expires_at}
|
||||||
|
return session, csrf
|
||||||
|
|
||||||
|
def session(self, token: str | None) -> dict[str, Any] | None:
|
||||||
|
if not token:
|
||||||
|
return None
|
||||||
|
with self.lock:
|
||||||
|
value = self.sessions.get(token)
|
||||||
|
if value is None or value["expires_at"] <= time.time():
|
||||||
|
self.sessions.pop(token, None)
|
||||||
|
return None
|
||||||
|
return value
|
||||||
|
|
||||||
|
def remaining_seconds(self) -> int:
|
||||||
|
return max(0, int(self.expires_at - time.time()))
|
||||||
|
|
||||||
|
|
||||||
|
class Handler(BaseHTTPRequestHandler):
|
||||||
|
server_version = "UmbraAgentBridge/0.1"
|
||||||
|
state: BridgeState
|
||||||
|
store: MessageStore
|
||||||
|
|
||||||
|
def log_message(self, format: str, *args: Any) -> None:
|
||||||
|
# Do not log request bodies, cookies, passwords, or message content.
|
||||||
|
print(f"agent-bridge: {self.address_string()} {format % args}", file=sys.stderr)
|
||||||
|
|
||||||
|
def _headers(self, content_type: str = "text/plain; charset=utf-8", length: int | None = None) -> None:
|
||||||
|
self.send_header("Content-Type", content_type)
|
||||||
|
self.send_header("Cache-Control", "no-store")
|
||||||
|
self.send_header("X-Content-Type-Options", "nosniff")
|
||||||
|
self.send_header("X-Frame-Options", "DENY")
|
||||||
|
self.send_header("Referrer-Policy", "no-referrer")
|
||||||
|
if length is not None:
|
||||||
|
self.send_header("Content-Length", str(length))
|
||||||
|
|
||||||
|
def _send(self, status: int, body: bytes, content_type: str = "text/plain; charset=utf-8") -> None:
|
||||||
|
self.send_response(status)
|
||||||
|
self._headers(content_type, len(body))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(body)
|
||||||
|
|
||||||
|
def _json(self, status: int, value: dict[str, Any]) -> None:
|
||||||
|
self._send(status, json.dumps(value).encode("utf-8"), "application/json; charset=utf-8")
|
||||||
|
|
||||||
|
def _cookie_token(self) -> str | None:
|
||||||
|
raw = self.headers.get("Cookie", "")
|
||||||
|
cookie = http.cookies.SimpleCookie()
|
||||||
|
try:
|
||||||
|
cookie.load(raw)
|
||||||
|
except http.cookies.CookieError:
|
||||||
|
return None
|
||||||
|
morsel = cookie.get("bridge_session")
|
||||||
|
return morsel.value if morsel else None
|
||||||
|
|
||||||
|
def _auth(self) -> tuple[dict[str, Any] | None, str | None]:
|
||||||
|
session = self.state.session(self._cookie_token())
|
||||||
|
if session is None:
|
||||||
|
return None, "authentication required"
|
||||||
|
csrf = self.headers.get("X-Bridge-CSRF", "")
|
||||||
|
if self.command == "POST" and not hmac.compare_digest(csrf, session["csrf"]):
|
||||||
|
return None, "CSRF validation failed"
|
||||||
|
return session, None
|
||||||
|
|
||||||
|
def _read_json(self) -> dict[str, Any]:
|
||||||
|
length = int(self.headers.get("Content-Length", "0"))
|
||||||
|
if length <= 0 or length > MAX_BODY_BYTES:
|
||||||
|
raise BridgeError("invalid request size")
|
||||||
|
value = json.loads(self.rfile.read(length).decode("utf-8"))
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
raise BridgeError("request must be a JSON object")
|
||||||
|
return value
|
||||||
|
|
||||||
|
def do_GET(self) -> None:
|
||||||
|
path = urlparse(self.path).path
|
||||||
|
if path == "/healthz":
|
||||||
|
self._json(200, {"status": "ok", "paired_sessions": len(self.state.sessions)})
|
||||||
|
return
|
||||||
|
if path == "/":
|
||||||
|
if self.state.session(self._cookie_token()) is None:
|
||||||
|
self._send(200, LOGIN_HTML.encode("utf-8"), "text/html; charset=utf-8")
|
||||||
|
else:
|
||||||
|
self._send(200, TERMINAL_HTML.encode("utf-8"), "text/html; charset=utf-8")
|
||||||
|
return
|
||||||
|
if path == "/api/status":
|
||||||
|
session, error = self._auth()
|
||||||
|
if error:
|
||||||
|
self._json(401, {"error": error})
|
||||||
|
return
|
||||||
|
self._json(200, {"expires_in": self.state.remaining_seconds(), "queue": self.store.status()})
|
||||||
|
return
|
||||||
|
if path == "/api/messages":
|
||||||
|
session, error = self._auth()
|
||||||
|
if error:
|
||||||
|
self._json(401, {"error": error})
|
||||||
|
return
|
||||||
|
query = parse_qs(urlparse(self.path).query)
|
||||||
|
try:
|
||||||
|
limit = int(query.get("limit", ["50"])[0])
|
||||||
|
except ValueError:
|
||||||
|
limit = 50
|
||||||
|
self._json(200, {"messages": self.store.read_for_browser(limit)})
|
||||||
|
return
|
||||||
|
self._json(404, {"error": "not found"})
|
||||||
|
|
||||||
|
def do_POST(self) -> None:
|
||||||
|
path = urlparse(self.path).path
|
||||||
|
if path == "/api/pair":
|
||||||
|
try:
|
||||||
|
value = self._read_json()
|
||||||
|
password = value.get("password")
|
||||||
|
if not isinstance(password, str) or not self.state.check_password(
|
||||||
|
self.client_address[0], password
|
||||||
|
):
|
||||||
|
self._json(401, {"error": "pairing failed"})
|
||||||
|
return
|
||||||
|
session, csrf = self.state.new_session()
|
||||||
|
body = json.dumps({"paired": True, "csrf": csrf, "expires_in": self.state.remaining_seconds()}).encode()
|
||||||
|
self.send_response(200)
|
||||||
|
self._headers("application/json; charset=utf-8", len(body))
|
||||||
|
self.send_header(
|
||||||
|
"Set-Cookie",
|
||||||
|
f"bridge_session={session}; HttpOnly; Secure; SameSite=Strict; Path=/; Max-Age={self.state.remaining_seconds()}",
|
||||||
|
)
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(body)
|
||||||
|
except (BridgeError, ValueError, json.JSONDecodeError):
|
||||||
|
self._json(400, {"error": "invalid pairing request"})
|
||||||
|
return
|
||||||
|
if path == "/api/messages":
|
||||||
|
session, error = self._auth()
|
||||||
|
if error:
|
||||||
|
self._json(401, {"error": error})
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
value = self._read_json()
|
||||||
|
body = value.get("body")
|
||||||
|
title = value.get("title", "Browser message")
|
||||||
|
if not isinstance(body, str) or not isinstance(title, str):
|
||||||
|
raise BridgeError("body and title must be strings")
|
||||||
|
message = self.store.send_to_mac(body, title)
|
||||||
|
self._json(202, {"id": message["id"], "queued": True})
|
||||||
|
except (BridgeError, ValueError, json.JSONDecodeError):
|
||||||
|
self._json(400, {"error": "invalid message"})
|
||||||
|
return
|
||||||
|
self._json(404, {"error": "not found"})
|
||||||
|
|
||||||
|
|
||||||
|
LOGIN_HTML = r"""<!doctype html>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||||
|
<title>UmbraTrans Temporary Agent Bridge</title>
|
||||||
|
<style>
|
||||||
|
body{font:16px -apple-system,BlinkMacSystemFont,sans-serif;background:#111;color:#eee;max-width:720px;margin:10vh auto;padding:24px}
|
||||||
|
input,button{font:inherit;padding:12px;border-radius:6px;border:1px solid #555;background:#222;color:#eee}
|
||||||
|
input{width:min(90%,420px)}button{background:#2866c7;color:white;cursor:pointer;margin-left:8px}
|
||||||
|
small{color:#aaa}p{line-height:1.4}
|
||||||
|
</style>
|
||||||
|
<h1>UmbraTrans temporary bridge</h1>
|
||||||
|
<p>This session expires after one hour. Enter the temporary pairing password supplied by the Mac operator.</p>
|
||||||
|
<form id="pair"><input id="password" type="password" autocomplete="off" required><button>Pair</button></form>
|
||||||
|
<p id="status"></p>
|
||||||
|
<script>
|
||||||
|
const form=document.querySelector('#pair'), status=document.querySelector('#status');
|
||||||
|
form.addEventListener('submit',async e=>{e.preventDefault();status.textContent='Pairing…';const r=await fetch('/api/pair',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({password:document.querySelector('#password').value})});if(r.ok){sessionStorage.setItem('bridgeCsrf',(await r.json()).csrf);location.reload()}else{status.textContent='Pairing failed';form.reset();}});
|
||||||
|
</script>
|
||||||
|
"""
|
||||||
|
|
||||||
|
TERMINAL_HTML = r"""<!doctype html>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||||
|
<title>UmbraTrans Agent Terminal</title>
|
||||||
|
<style>
|
||||||
|
body{font:15px ui-monospace,SFMono-Regular,Menlo,monospace;background:#0b0d10;color:#d7e0ea;max-width:1100px;margin:4vh auto;padding:20px}
|
||||||
|
#log{height:65vh;overflow:auto;background:#11151a;border:1px solid #303943;padding:14px;white-space:pre-wrap}
|
||||||
|
form{display:flex;gap:8px;margin-top:12px}input,button{font:inherit;padding:10px;border:1px solid #3c4652;border-radius:5px;background:#171c22;color:#eee}input{flex:1}button{background:#2866c7;color:white;cursor:pointer}
|
||||||
|
small{color:#9ba8b5}
|
||||||
|
</style>
|
||||||
|
<h1>UmbraTrans agent terminal</h1>
|
||||||
|
<small>Paired local session. Messages are queued for the Mac agent; this is not a raw shell.</small>
|
||||||
|
<div id="log"></div>
|
||||||
|
<form id="send"><input id="body" autocomplete="off" placeholder="Describe the next task or question…" required><button>Send</button></form>
|
||||||
|
<script>
|
||||||
|
const csrf=sessionStorage.getItem('bridgeCsrf'), log=document.querySelector('#log');
|
||||||
|
function show(m){const who=m.sender==='mac-agent'?'Mac agent':'Oldbox browser';log.textContent+=`[${m.created_at}] ${who}: ${m.body}\n`;log.scrollTop=log.scrollHeight;}
|
||||||
|
async function poll(){const r=await fetch('/api/messages',{credentials:'include'});if(r.ok){const d=await r.json();for(const m of d.messages)show(m);if(d.messages.length){await fetch('/api/messages',{method:'POST',credentials:'include',headers:{'Content-Type':'application/json','X-Bridge-CSRF':csrf},body:JSON.stringify({title:'ack',body:'acknowledged'})});}}}
|
||||||
|
setInterval(poll,2000);poll();
|
||||||
|
document.querySelector('#send').addEventListener('submit',async e=>{e.preventDefault();const input=document.querySelector('#body');const r=await fetch('/api/messages',{method:'POST',credentials:'include',headers:{'Content-Type':'application/json','X-Bridge-CSRF':csrf},body:JSON.stringify({title:'Browser task',body:input.value})});if(r.ok){log.textContent+=`[sent] ${input.value}\n`;input.value=''}else{log.textContent+='[error] message not queued\n'}});
|
||||||
|
</script>
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def make_certificate(directory: Path, address: str) -> tuple[Path, Path]:
|
||||||
|
directory.mkdir(parents=True, exist_ok=True)
|
||||||
|
os.chmod(directory, 0o700)
|
||||||
|
key = directory / "bridge-key.pem"
|
||||||
|
cert = directory / "bridge-cert.pem"
|
||||||
|
if key.exists() and cert.exists():
|
||||||
|
return key, cert
|
||||||
|
subprocess.run(
|
||||||
|
[
|
||||||
|
"openssl",
|
||||||
|
"req",
|
||||||
|
"-x509",
|
||||||
|
"-newkey",
|
||||||
|
"rsa:2048",
|
||||||
|
"-nodes",
|
||||||
|
"-keyout",
|
||||||
|
str(key),
|
||||||
|
"-out",
|
||||||
|
str(cert),
|
||||||
|
"-days",
|
||||||
|
"1",
|
||||||
|
"-subj",
|
||||||
|
f"/CN={address}",
|
||||||
|
"-addext",
|
||||||
|
f"subjectAltName=IP:{address}",
|
||||||
|
],
|
||||||
|
check=True,
|
||||||
|
stdout=subprocess.DEVNULL,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
)
|
||||||
|
os.chmod(key, 0o600)
|
||||||
|
os.chmod(cert, 0o644)
|
||||||
|
return key, cert
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
parser = argparse.ArgumentParser(description="Temporary UmbraTrans agent bridge")
|
||||||
|
parser.add_argument("--host", default="192.168.123.12")
|
||||||
|
parser.add_argument("--port", type=int, default=0)
|
||||||
|
parser.add_argument("--root", default="~/.local/state/umbratrans/agent-bridge")
|
||||||
|
parser.add_argument("--ttl", type=int, default=SESSION_TTL_SECONDS)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
if not 1 <= args.ttl <= 3600:
|
||||||
|
parser.error("--ttl must be between 1 and 3600")
|
||||||
|
password = macos_secret_prompt("agent-bridge-temp", "one-hour agent bridge pairing password")
|
||||||
|
if not password:
|
||||||
|
print("agent bridge: pairing password entry cancelled", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
state = BridgeState(password, args.ttl)
|
||||||
|
del password
|
||||||
|
store = MessageStore(Path(args.root).expanduser())
|
||||||
|
Handler.state = state
|
||||||
|
Handler.store = store
|
||||||
|
server = ThreadingHTTPServer((args.host, args.port), Handler)
|
||||||
|
tls_dir = Path(tempfile.mkdtemp(prefix="umbra-agent-bridge-tls-"))
|
||||||
|
key, cert = make_certificate(tls_dir, args.host)
|
||||||
|
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||||
|
context.load_cert_chain(certfile=str(cert), keyfile=str(key))
|
||||||
|
server.socket = context.wrap_socket(server.socket, server_side=True)
|
||||||
|
port = server.server_address[1]
|
||||||
|
print(f"BRIDGE_URL=https://{args.host}:{port}/", flush=True)
|
||||||
|
print(f"BRIDGE_ROOT={store.root}", flush=True)
|
||||||
|
print(f"BRIDGE_EXPIRES_IN={state.remaining_seconds()}", flush=True)
|
||||||
|
|
||||||
|
def stop(*_: Any) -> None:
|
||||||
|
threading.Thread(target=server.shutdown, daemon=True).start()
|
||||||
|
|
||||||
|
signal.signal(signal.SIGTERM, stop)
|
||||||
|
signal.signal(signal.SIGINT, stop)
|
||||||
|
timer = threading.Timer(args.ttl, stop)
|
||||||
|
timer.daemon = True
|
||||||
|
timer.start()
|
||||||
|
try:
|
||||||
|
server.serve_forever(poll_interval=0.5)
|
||||||
|
finally:
|
||||||
|
server.server_close()
|
||||||
|
timer.cancel()
|
||||||
|
for path in (key, cert):
|
||||||
|
try:
|
||||||
|
path.unlink()
|
||||||
|
except FileNotFoundError:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
tls_dir.rmdir()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
50
infra/agent_bridge/tests/test_bridge.py
Normal file
50
infra/agent_bridge/tests/test_bridge.py
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||||
|
|
||||||
|
from agent_bridge.server import BridgeError, BridgeState, MessageStore
|
||||||
|
|
||||||
|
|
||||||
|
class BridgeTests(unittest.TestCase):
|
||||||
|
def test_password_hashing_and_pairing(self) -> None:
|
||||||
|
state = BridgeState("correct-value", ttl_seconds=60)
|
||||||
|
self.assertTrue(state.check_password("192.0.2.1", "correct-value"))
|
||||||
|
self.assertFalse(state.check_password("192.0.2.1", "wrong-value"))
|
||||||
|
session, csrf = state.new_session()
|
||||||
|
self.assertIsNotNone(state.session(session))
|
||||||
|
self.assertTrue(csrf)
|
||||||
|
self.assertIsNone(state.session("missing"))
|
||||||
|
|
||||||
|
def test_message_queue_is_atomic_and_bounded(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
store = MessageStore(Path(directory))
|
||||||
|
message = store.send_to_mac("hello", "test")
|
||||||
|
self.assertEqual(store.status()["inbox"], 1)
|
||||||
|
self.assertEqual(store.poll_for_mac()[0]["body"], "hello")
|
||||||
|
self.assertEqual(store.status()["processed"], 1)
|
||||||
|
reply = store.send_to_browser("reply", "test")
|
||||||
|
self.assertEqual(store.read_for_browser()[0]["id"], reply["id"])
|
||||||
|
self.assertEqual(store.acknowledge([reply["id"]]), 1)
|
||||||
|
self.assertEqual(store.status()["delivered"], 1)
|
||||||
|
|
||||||
|
def test_message_size_is_limited(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
store = MessageStore(Path(directory))
|
||||||
|
with self.assertRaises(BridgeError):
|
||||||
|
store.send_to_mac("x" * (64 * 1024 + 1))
|
||||||
|
|
||||||
|
def test_invalid_message_file_is_ignored(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
store = MessageStore(Path(directory))
|
||||||
|
(store.inbox / "bad.json").write_text(json.dumps({"not": "a message"}))
|
||||||
|
self.assertEqual(store.poll_for_mac(), [])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
511
infra/bootstrap_enterprise_mint.sh
Executable file
511
infra/bootstrap_enterprise_mint.sh
Executable file
@@ -0,0 +1,511 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Phase-one enterprise control-plane bootstrap for Linux Mint 22.x.
|
||||||
|
# This provisions PostgreSQL, database roles, core tenancy/identity tables,
|
||||||
|
# and least-privilege service accounts. It does not install application code,
|
||||||
|
# change Qwen, manage Docker, or change firewall rules.
|
||||||
|
set -Eeuo pipefail
|
||||||
|
umask 077
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||||
|
CONFIG_FILE="${UMBRA_BOOTSTRAP_CONFIG:-${SCRIPT_DIR}/enterprise-bootstrap.env}"
|
||||||
|
MODE="check"
|
||||||
|
DRY_RUN=0
|
||||||
|
LOCK_DIR="/run/lock/umbra-enterprise-bootstrap.lock"
|
||||||
|
LOG_FILE="/var/log/umbra-enterprise-bootstrap.log"
|
||||||
|
|
||||||
|
UMBRA_HOSTNAME="oldbox.bdmurphy.me"
|
||||||
|
UMBRA_PUBLIC_URL="https://oldbox.bdmurphy.me"
|
||||||
|
UMBRA_DB_NAME="umbracontrol"
|
||||||
|
UMBRA_DB_HOST="127.0.0.1"
|
||||||
|
UMBRA_DB_PORT="5432"
|
||||||
|
UMBRA_PG_VERSION="16"
|
||||||
|
UMBRA_APP_ROOT="/opt/umbracontrol"
|
||||||
|
UMBRA_DATA_ROOT="/var/lib/umbracontrol"
|
||||||
|
UMBRA_BACKUP_ROOT="/var/backups/umbracontrol"
|
||||||
|
UMBRA_BOOTSTRAP_ADMIN_EMAIL="umbratools@gmail.com"
|
||||||
|
UMBRA_ORGANIZATION_NAME="Umbra"
|
||||||
|
UMBRA_ORGANIZATION_SLUG="umbra"
|
||||||
|
UMBRA_QWEN_URL="http://192.168.123.14:8881"
|
||||||
|
UMBRA_INSTALL_PACKAGES=1
|
||||||
|
UMBRA_INSTALL_UV=1
|
||||||
|
UMBRA_CREATE_BOOTSTRAP_ADMIN=1
|
||||||
|
UMBRA_CONFIG_DIR="/etc/umbracontrol"
|
||||||
|
UMBRA_CREDENTIALS_FILE="/etc/umbracontrol/db-credentials.env"
|
||||||
|
UMBRA_BOOTSTRAP_TOKEN_FILE="/root/umbra-enterprise-bootstrap.token"
|
||||||
|
UMBRA_PG_SUPERUSER="postgres"
|
||||||
|
UMBRA_SERVICE_GROUP="umbracontrol"
|
||||||
|
UMBRA_API_USER="umbracontrol_api"
|
||||||
|
UMBRA_WORKER_USER="umbracontrol_worker"
|
||||||
|
UMBRA_READONLY_USER="umbracontrol_readonly"
|
||||||
|
UMBRA_BACKUP_USER="umbracontrol_backup"
|
||||||
|
UMBRA_MIGRATOR_USER="umbracontrol_migrator"
|
||||||
|
UMBRA_OWNER_ROLE="umbracontrol_owner"
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'EOF'
|
||||||
|
Usage: bootstrap_enterprise_mint.sh [options]
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--check Read-only host and dependency inspection (default)
|
||||||
|
--dry-run Show apply actions without changing the host
|
||||||
|
--apply Apply the phase-one infrastructure bootstrap; requires root
|
||||||
|
--status Show installation/service/database status
|
||||||
|
--verify Verify installed services and database connectivity
|
||||||
|
--config FILE Read non-secret KEY=VALUE configuration
|
||||||
|
-h, --help Show this help
|
||||||
|
|
||||||
|
This phase intentionally does not install application code, modify Qwen,
|
||||||
|
manage Docker containers, or change firewall rules. Run it first on Mint,
|
||||||
|
then deploy the application and systemd units in a separate reviewed step.
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
while (($#)); do
|
||||||
|
case "$1" in
|
||||||
|
--check) MODE="check"; shift ;;
|
||||||
|
--dry-run) MODE="apply"; DRY_RUN=1; shift ;;
|
||||||
|
--apply) MODE="apply"; DRY_RUN=0; shift ;;
|
||||||
|
--status) MODE="status"; shift ;;
|
||||||
|
--verify) MODE="verify"; shift ;;
|
||||||
|
--config) CONFIG_FILE="${2:?--config requires a file}"; shift 2 ;;
|
||||||
|
-h|--help) usage; exit 0 ;;
|
||||||
|
*) printf 'Unknown option: %s\n' "$1" >&2; usage >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
log() {
|
||||||
|
local line
|
||||||
|
line="[$(date -Is)] $*"
|
||||||
|
printf '%s\n' "$line"
|
||||||
|
if [[ -f "$LOG_FILE" && "$(id -u)" -eq 0 ]]; then
|
||||||
|
printf '%s\n' "$line" >> "$LOG_FILE"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
die() {
|
||||||
|
printf 'ERROR: %s\n' "$*" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
require_command() {
|
||||||
|
command -v "$1" >/dev/null 2>&1 || die "Required command is missing: $1"
|
||||||
|
}
|
||||||
|
|
||||||
|
require_root() {
|
||||||
|
[[ "$(id -u)" -eq 0 ]] || die "This operation requires root. Use --check for a read-only inspection."
|
||||||
|
}
|
||||||
|
|
||||||
|
load_config() {
|
||||||
|
[[ -f "$CONFIG_FILE" ]] || {
|
||||||
|
[[ "$MODE" == "check" || "$MODE" == "status" || "$MODE" == "verify" ]] && return 0
|
||||||
|
die "Configuration file not found: $CONFIG_FILE"
|
||||||
|
}
|
||||||
|
|
||||||
|
local line key value
|
||||||
|
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||||
|
line="${line#"${line%%[![:space:]]*}"}"
|
||||||
|
[[ -z "$line" || "${line:0:1}" == "#" ]] && continue
|
||||||
|
[[ "$line" == *=* ]] || die "Invalid configuration line (expected KEY=VALUE): $line"
|
||||||
|
key="${line%%=*}"
|
||||||
|
value="${line#*=}"
|
||||||
|
key="${key//[[:space:]]/}"
|
||||||
|
value="${value%\"}"; value="${value#\"}"
|
||||||
|
value="${value%\'}"; value="${value#\'}"
|
||||||
|
case "$key" in
|
||||||
|
UMBRA_HOSTNAME|UMBRA_PUBLIC_URL|UMBRA_DB_NAME|UMBRA_DB_HOST|UMBRA_DB_PORT|UMBRA_PG_VERSION|UMBRA_APP_ROOT|UMBRA_DATA_ROOT|UMBRA_BACKUP_ROOT|UMBRA_BOOTSTRAP_ADMIN_EMAIL|UMBRA_ORGANIZATION_NAME|UMBRA_ORGANIZATION_SLUG|UMBRA_QWEN_URL|UMBRA_INSTALL_PACKAGES|UMBRA_INSTALL_UV|UMBRA_CREATE_BOOTSTRAP_ADMIN)
|
||||||
|
printf -v "$key" '%s' "$value"
|
||||||
|
;;
|
||||||
|
*) die "Unsupported configuration key: $key" ;;
|
||||||
|
esac
|
||||||
|
done < "$CONFIG_FILE"
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_config() {
|
||||||
|
[[ "$UMBRA_HOSTNAME" =~ ^[A-Za-z0-9.-]+$ ]] || die "Invalid UMBRA_HOSTNAME"
|
||||||
|
[[ "$UMBRA_PUBLIC_URL" =~ ^https?://[A-Za-z0-9._:-]+(/.*)?$ ]] || die "Invalid UMBRA_PUBLIC_URL"
|
||||||
|
[[ "$UMBRA_DB_NAME" =~ ^[a-z_][a-z0-9_]*$ ]] || die "Invalid UMBRA_DB_NAME"
|
||||||
|
[[ "$UMBRA_DB_HOST" =~ ^[A-Za-z0-9._:-]+$ ]] || die "Invalid UMBRA_DB_HOST"
|
||||||
|
[[ "$UMBRA_DB_PORT" =~ ^[0-9]+$ && "$UMBRA_DB_PORT" -ge 1 && "$UMBRA_DB_PORT" -le 65535 ]] || die "Invalid UMBRA_DB_PORT"
|
||||||
|
[[ "$UMBRA_PG_VERSION" =~ ^[0-9]+$ ]] || die "Invalid UMBRA_PG_VERSION"
|
||||||
|
[[ "$UMBRA_APP_ROOT" == /* && "$UMBRA_DATA_ROOT" == /* && "$UMBRA_BACKUP_ROOT" == /* ]] || die "Application/data paths must be absolute"
|
||||||
|
[[ "$UMBRA_BOOTSTRAP_ADMIN_EMAIL" == *@*.* ]] || die "Invalid administrator email"
|
||||||
|
[[ "$UMBRA_ORGANIZATION_SLUG" =~ ^[a-z0-9][a-z0-9-]*$ ]] || die "Invalid organization slug"
|
||||||
|
[[ "$UMBRA_QWEN_URL" =~ ^https?:// ]] || die "Invalid UMBRA_QWEN_URL"
|
||||||
|
for value in "$UMBRA_INSTALL_PACKAGES" "$UMBRA_INSTALL_UV" "$UMBRA_CREATE_BOOTSTRAP_ADMIN"; do
|
||||||
|
[[ "$value" == 0 || "$value" == 1 ]] || die "Boolean configuration values must be 0 or 1"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
run() {
|
||||||
|
if ((DRY_RUN)); then
|
||||||
|
printf '[dry-run]'
|
||||||
|
printf ' %q' "$@"
|
||||||
|
printf '\n'
|
||||||
|
else
|
||||||
|
"$@"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
run_root() {
|
||||||
|
require_root
|
||||||
|
run "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
read_secret() {
|
||||||
|
local key="$1"
|
||||||
|
[[ -f "$UMBRA_CREDENTIALS_FILE" ]] || return 1
|
||||||
|
awk -F= -v key="$key" '$1 == key {print substr($0, index($0, "=") + 1); exit}' "$UMBRA_CREDENTIALS_FILE"
|
||||||
|
}
|
||||||
|
|
||||||
|
write_secret_line() {
|
||||||
|
local key="$1" value="$2" temporary
|
||||||
|
if ((DRY_RUN)); then
|
||||||
|
printf '[dry-run] store generated secret %s in %s\n' "$key" "$UMBRA_CREDENTIALS_FILE"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
install -d -m 0750 -o root -g "$UMBRA_SERVICE_GROUP" "$UMBRA_CONFIG_DIR"
|
||||||
|
touch "$UMBRA_CREDENTIALS_FILE"
|
||||||
|
chown root:"$UMBRA_SERVICE_GROUP" "$UMBRA_CREDENTIALS_FILE"
|
||||||
|
chmod 0640 "$UMBRA_CREDENTIALS_FILE"
|
||||||
|
temporary="$(mktemp "${UMBRA_CONFIG_DIR}/.credentials.XXXXXX")"
|
||||||
|
awk -F= -v key="$key" -v value="$value" '
|
||||||
|
$1 == key { print key "=" value; found = 1; next }
|
||||||
|
{ print }
|
||||||
|
END { if (!found) print key "=" value }
|
||||||
|
' "$UMBRA_CREDENTIALS_FILE" > "$temporary"
|
||||||
|
chown root:"$UMBRA_SERVICE_GROUP" "$temporary"
|
||||||
|
chmod 0640 "$temporary"
|
||||||
|
mv -f "$temporary" "$UMBRA_CREDENTIALS_FILE"
|
||||||
|
}
|
||||||
|
|
||||||
|
random_secret() {
|
||||||
|
openssl rand -hex 32
|
||||||
|
}
|
||||||
|
|
||||||
|
check_host() {
|
||||||
|
log "Read-only host inspection"
|
||||||
|
printf 'hostname: '; hostname -f 2>/dev/null || hostname
|
||||||
|
printf 'os: '; . /etc/os-release; printf '%s\n' "${PRETTY_NAME:-unknown}"
|
||||||
|
printf 'kernel: '; uname -r
|
||||||
|
printf 'user: '; id
|
||||||
|
printf 'cpu: '; nproc
|
||||||
|
free -h | sed -n '1,3p'
|
||||||
|
if command -v nvidia-smi >/dev/null 2>&1; then
|
||||||
|
nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
printf '%s\n' 'storage:'
|
||||||
|
df -hT / /home /var/lib 2>/dev/null || true
|
||||||
|
printf '%s\n' 'listening ports:'
|
||||||
|
ss -lntup 2>/dev/null | sed -n '1,100p' || true
|
||||||
|
printf '%s\n' 'relevant services:'
|
||||||
|
systemctl list-units --type=service --all --no-pager 2>/dev/null | grep -Ei 'docker|ollama|lmstudio|qwen|postgres|umbra' || true
|
||||||
|
printf '%s\n' 'DNS:'
|
||||||
|
getent ahosts "$UMBRA_HOSTNAME" 2>/dev/null || true
|
||||||
|
log "No changes made"
|
||||||
|
}
|
||||||
|
|
||||||
|
check_tools() {
|
||||||
|
require_command openssl
|
||||||
|
require_command systemctl
|
||||||
|
if [[ "$MODE" == "apply" && "$UMBRA_INSTALL_PACKAGES" -eq 1 ]]; then
|
||||||
|
require_command apt-get
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
ensure_group_and_users() {
|
||||||
|
log "Ensuring service group and non-login service accounts"
|
||||||
|
if ! getent group "$UMBRA_SERVICE_GROUP" >/dev/null; then
|
||||||
|
run groupadd --system "$UMBRA_SERVICE_GROUP"
|
||||||
|
else
|
||||||
|
log "Group already exists: $UMBRA_SERVICE_GROUP"
|
||||||
|
fi
|
||||||
|
|
||||||
|
ensure_service_user "$UMBRA_API_USER"
|
||||||
|
ensure_service_user "$UMBRA_WORKER_USER"
|
||||||
|
ensure_service_user "$UMBRA_READONLY_USER"
|
||||||
|
ensure_service_user "$UMBRA_BACKUP_USER"
|
||||||
|
}
|
||||||
|
|
||||||
|
ensure_service_user() {
|
||||||
|
local user="$1"
|
||||||
|
if id "$user" >/dev/null 2>&1; then
|
||||||
|
log "Service account already exists: $user"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
run useradd --system --gid "$UMBRA_SERVICE_GROUP" --home-dir /nonexistent --no-create-home --shell /usr/sbin/nologin "$user"
|
||||||
|
}
|
||||||
|
|
||||||
|
ensure_directories() {
|
||||||
|
log "Creating application, data, backup, and configuration directories"
|
||||||
|
run install -d -m 0755 -o root -g root "$UMBRA_APP_ROOT"
|
||||||
|
run install -d -m 0750 -o root -g "$UMBRA_SERVICE_GROUP" "$UMBRA_DATA_ROOT"
|
||||||
|
run install -d -m 0700 -o root -g root "$UMBRA_BACKUP_ROOT"
|
||||||
|
run install -d -m 0750 -o root -g "$UMBRA_SERVICE_GROUP" "$UMBRA_CONFIG_DIR"
|
||||||
|
run install -d -m 0750 -o root -g "$UMBRA_SERVICE_GROUP" "$UMBRA_APP_ROOT/app" "$UMBRA_APP_ROOT/venv" "$UMBRA_APP_ROOT/bin"
|
||||||
|
}
|
||||||
|
|
||||||
|
install_packages() {
|
||||||
|
[[ "$UMBRA_INSTALL_PACKAGES" -eq 1 ]] || { log "Package installation disabled by configuration"; return; }
|
||||||
|
log "Installing Mint/Ubuntu packages"
|
||||||
|
run env DEBIAN_FRONTEND=noninteractive apt-get update
|
||||||
|
run env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||||
|
ca-certificates curl openssl postgresql postgresql-client postgresql-contrib \
|
||||||
|
python3 python3-venv python3-pip rsync
|
||||||
|
}
|
||||||
|
|
||||||
|
ensure_uv() {
|
||||||
|
[[ "$UMBRA_INSTALL_UV" -eq 1 ]] || { log "uv installation disabled by configuration"; return; }
|
||||||
|
if command -v uv >/dev/null 2>&1; then
|
||||||
|
log "uv already available: $(command -v uv)"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
log "Installing pinned uv bootstrap environment"
|
||||||
|
run python3 -m venv "$UMBRA_APP_ROOT/bootstrap-venv"
|
||||||
|
run "$UMBRA_APP_ROOT/bootstrap-venv/bin/python" -m pip install --upgrade pip
|
||||||
|
run "$UMBRA_APP_ROOT/bootstrap-venv/bin/python" -m pip install 'uv==0.11.7'
|
||||||
|
run ln -sfn "$UMBRA_APP_ROOT/bootstrap-venv/bin/uv" "$UMBRA_APP_ROOT/bin/uv"
|
||||||
|
}
|
||||||
|
|
||||||
|
psql_as_postgres() {
|
||||||
|
runuser -u "$UMBRA_PG_SUPERUSER" -- env PGOPTIONS='-c log_statement=none' \
|
||||||
|
psql -v ON_ERROR_STOP=1 "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
ensure_postgres_cluster() {
|
||||||
|
log "Ensuring PostgreSQL cluster and service"
|
||||||
|
local version cluster
|
||||||
|
if command -v pg_lsclusters >/dev/null 2>&1; then
|
||||||
|
version="$(pg_lsclusters --no-header 2>/dev/null | awk 'NR == 1 {print $1}')"
|
||||||
|
cluster="$(pg_lsclusters --no-header 2>/dev/null | awk 'NR == 1 {print $2}')"
|
||||||
|
else
|
||||||
|
version=""
|
||||||
|
cluster=""
|
||||||
|
fi
|
||||||
|
if [[ -z "$version" || -z "$cluster" ]]; then
|
||||||
|
version="$UMBRA_PG_VERSION"
|
||||||
|
cluster="main"
|
||||||
|
run pg_createcluster "$version" "$cluster" --start
|
||||||
|
fi
|
||||||
|
if command -v pg_lsclusters >/dev/null 2>&1; then
|
||||||
|
if ! pg_lsclusters --no-header | awk -v v="$version" -v c="$cluster" '$1 == v && $2 == c && $3 == "online" {found=1} END {exit !found}'; then
|
||||||
|
run pg_ctlcluster "$version" "$cluster" start
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
log "pg_lsclusters is not installed; cluster start will be performed after package installation"
|
||||||
|
fi
|
||||||
|
run systemctl enable postgresql
|
||||||
|
}
|
||||||
|
|
||||||
|
ensure_database_roles() {
|
||||||
|
log "Creating PostgreSQL roles"
|
||||||
|
local role password login
|
||||||
|
for role in \
|
||||||
|
"$UMBRA_OWNER_ROLE" \
|
||||||
|
"$UMBRA_MIGRATOR_USER" \
|
||||||
|
"$UMBRA_API_USER" \
|
||||||
|
"$UMBRA_WORKER_USER" \
|
||||||
|
"$UMBRA_READONLY_USER" \
|
||||||
|
"$UMBRA_BACKUP_USER"; do
|
||||||
|
password=""
|
||||||
|
login="LOGIN"
|
||||||
|
[[ "$role" == "$UMBRA_OWNER_ROLE" ]] && login="NOLOGIN"
|
||||||
|
if [[ "$role" != "$UMBRA_OWNER_ROLE" ]]; then
|
||||||
|
if ((DRY_RUN)); then
|
||||||
|
password="$(random_secret)"
|
||||||
|
else
|
||||||
|
password="$(read_secret "${role^^}_PASSWORD" 2>/dev/null || true)"
|
||||||
|
[[ -n "$password" ]] || password="$(random_secret)"
|
||||||
|
fi
|
||||||
|
write_secret_line "${role^^}_PASSWORD" "$password"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ((DRY_RUN)); then
|
||||||
|
printf '[dry-run] ensure PostgreSQL role %s (%s)\n' "$role" "$login"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -n "$password" ]]; then
|
||||||
|
printf 'DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = %s) THEN CREATE ROLE %s %s PASSWORD %s; ELSE ALTER ROLE %s %s PASSWORD %s; END IF; END $$;\n' \
|
||||||
|
"'$role'" "$role" "$login" "'$password'" "$role" "$login" "'$password'" \
|
||||||
|
| psql_as_postgres postgres >/dev/null
|
||||||
|
else
|
||||||
|
printf 'DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = %s) THEN CREATE ROLE %s %s; END IF; END $$;\n' \
|
||||||
|
"'$role'" "$role" "$login" \
|
||||||
|
| psql_as_postgres postgres >/dev/null
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
ensure_database() {
|
||||||
|
log "Creating enterprise database if absent"
|
||||||
|
if ((DRY_RUN)); then
|
||||||
|
printf '[dry-run] ensure database %s owned by %s\n' "$UMBRA_DB_NAME" "$UMBRA_OWNER_ROLE"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
if ! psql_as_postgres postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '$UMBRA_DB_NAME'" | grep -q 1; then
|
||||||
|
runuser -u "$UMBRA_PG_SUPERUSER" -- createdb --owner="$UMBRA_OWNER_ROLE" "$UMBRA_DB_NAME"
|
||||||
|
fi
|
||||||
|
psql_as_postgres postgres -c "REVOKE ALL ON DATABASE \"$UMBRA_DB_NAME\" FROM PUBLIC" >/dev/null
|
||||||
|
psql_as_postgres postgres -c "GRANT CONNECT ON DATABASE \"$UMBRA_DB_NAME\" TO \"$UMBRA_MIGRATOR_USER\", \"$UMBRA_API_USER\", \"$UMBRA_WORKER_USER\", \"$UMBRA_READONLY_USER\"" >/dev/null
|
||||||
|
psql_as_postgres postgres -c "GRANT CONNECT ON DATABASE \"$UMBRA_DB_NAME\" TO \"$UMBRA_BACKUP_USER\"" >/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
apply_schema() {
|
||||||
|
local schema_file="${SCRIPT_DIR}/postgres/001_control_plane.sql"
|
||||||
|
[[ -f "$schema_file" ]] || die "Schema file is missing: $schema_file"
|
||||||
|
log "Applying core identity/tenancy schema"
|
||||||
|
if ((DRY_RUN)); then
|
||||||
|
printf '[dry-run] psql -d %s -f %s\n' "$UMBRA_DB_NAME" "$schema_file"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
/bin/cat -- "$schema_file" | psql_as_postgres "$UMBRA_DB_NAME" >/dev/null
|
||||||
|
log "Core schema applied"
|
||||||
|
}
|
||||||
|
|
||||||
|
write_nonsecret_runtime_config() {
|
||||||
|
local config_file="$UMBRA_CONFIG_DIR/control-plane.env"
|
||||||
|
run install -d -m 0750 -o root -g "$UMBRA_SERVICE_GROUP" "$UMBRA_CONFIG_DIR"
|
||||||
|
if ((DRY_RUN)); then
|
||||||
|
printf '[dry-run] write non-secret runtime configuration to %s\n' "$config_file"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
cat > "${config_file}.tmp" <<EOF
|
||||||
|
UMBRA_PUBLIC_URL=$UMBRA_PUBLIC_URL
|
||||||
|
UMBRA_HOSTNAME=$UMBRA_HOSTNAME
|
||||||
|
UMBRA_DB_NAME=$UMBRA_DB_NAME
|
||||||
|
UMBRA_DB_HOST=$UMBRA_DB_HOST
|
||||||
|
UMBRA_DB_PORT=$UMBRA_DB_PORT
|
||||||
|
UMBRA_QWEN_URL=$UMBRA_QWEN_URL
|
||||||
|
UMBRA_APP_ROOT=$UMBRA_APP_ROOT
|
||||||
|
UMBRA_DATA_ROOT=$UMBRA_DATA_ROOT
|
||||||
|
EOF
|
||||||
|
chown root:"$UMBRA_SERVICE_GROUP" "${config_file}.tmp"
|
||||||
|
chmod 0640 "${config_file}.tmp"
|
||||||
|
mv -f "${config_file}.tmp" "$config_file"
|
||||||
|
}
|
||||||
|
|
||||||
|
create_bootstrap_admin() {
|
||||||
|
[[ "$UMBRA_CREATE_BOOTSTRAP_ADMIN" -eq 1 ]] || { log "Bootstrap admin creation disabled"; return; }
|
||||||
|
local token token_hash
|
||||||
|
if [[ -f "$UMBRA_BOOTSTRAP_TOKEN_FILE" ]]; then
|
||||||
|
token="$(cat "$UMBRA_BOOTSTRAP_TOKEN_FILE")"
|
||||||
|
else
|
||||||
|
token="$(random_secret)"
|
||||||
|
if ((DRY_RUN)); then
|
||||||
|
printf '[dry-run] create one-time admin token at %s\n' "$UMBRA_BOOTSTRAP_TOKEN_FILE"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
printf '%s\n' "$token" > "$UMBRA_BOOTSTRAP_TOKEN_FILE"
|
||||||
|
chown root:root "$UMBRA_BOOTSTRAP_TOKEN_FILE"
|
||||||
|
chmod 0600 "$UMBRA_BOOTSTRAP_TOKEN_FILE"
|
||||||
|
fi
|
||||||
|
[[ "$token" =~ ^[a-f0-9]{64}$ ]] || die "Bootstrap token file has an unexpected format"
|
||||||
|
token_hash="$(printf '%s' "$token" | sha256sum | awk '{print $1}')"
|
||||||
|
log "Creating/confirming bootstrap platform administrator"
|
||||||
|
if ((DRY_RUN)); then
|
||||||
|
printf '[dry-run] create administrator %s and organization %s\n' "$UMBRA_BOOTSTRAP_ADMIN_EMAIL" "$UMBRA_ORGANIZATION_NAME"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
psql_as_postgres "$UMBRA_DB_NAME" \
|
||||||
|
-v admin_email="$UMBRA_BOOTSTRAP_ADMIN_EMAIL" \
|
||||||
|
-v token_hash="$token_hash" \
|
||||||
|
-v organization_name="$UMBRA_ORGANIZATION_NAME" \
|
||||||
|
-v organization_slug="$UMBRA_ORGANIZATION_SLUG" <<'SQL' >/dev/null
|
||||||
|
INSERT INTO users (email, display_name, auth_provider, auth_subject, status, mfa_required)
|
||||||
|
VALUES (:'admin_email', :'admin_email', 'bootstrap', 'bootstrap:' || gen_random_uuid()::text, 'invited', true)
|
||||||
|
ON CONFLICT (email) DO UPDATE
|
||||||
|
SET display_name = EXCLUDED.display_name,
|
||||||
|
auth_provider = 'bootstrap',
|
||||||
|
mfa_required = true,
|
||||||
|
updated_at = now();
|
||||||
|
|
||||||
|
INSERT INTO organizations (name, slug)
|
||||||
|
VALUES (:'organization_name', :'organization_slug')
|
||||||
|
ON CONFLICT (slug) DO UPDATE SET name = EXCLUDED.name;
|
||||||
|
|
||||||
|
INSERT INTO user_roles (user_id, role_id, scope, organization_id, project_id, created_by)
|
||||||
|
SELECT u.id, r.id, 'platform', NULL, NULL, u.id
|
||||||
|
FROM users u
|
||||||
|
JOIN roles r ON r.scope = 'platform' AND r.name = 'platform_admin'
|
||||||
|
WHERE u.email = :'admin_email'
|
||||||
|
ON CONFLICT DO NOTHING;
|
||||||
|
|
||||||
|
INSERT INTO user_roles (user_id, role_id, scope, organization_id, project_id, created_by)
|
||||||
|
SELECT u.id, r.id, 'organization', o.id, NULL, u.id
|
||||||
|
FROM users u
|
||||||
|
JOIN organizations o ON o.slug = :'organization_slug'
|
||||||
|
JOIN roles r ON r.scope = 'organization' AND r.name = 'organization_admin'
|
||||||
|
WHERE u.email = :'admin_email'
|
||||||
|
ON CONFLICT DO NOTHING;
|
||||||
|
|
||||||
|
INSERT INTO bootstrap_tokens (user_id, token_hash, expires_at)
|
||||||
|
SELECT id, :'token_hash', now() + interval '24 hours'
|
||||||
|
FROM users
|
||||||
|
WHERE email = :'admin_email'
|
||||||
|
ON CONFLICT (token_hash) DO NOTHING;
|
||||||
|
SQL
|
||||||
|
log "Bootstrap token stored root-only at $UMBRA_BOOTSTRAP_TOKEN_FILE"
|
||||||
|
}
|
||||||
|
|
||||||
|
apply_mode() {
|
||||||
|
if ((DRY_RUN == 0)); then
|
||||||
|
require_root
|
||||||
|
log "Beginning phase-one enterprise bootstrap"
|
||||||
|
if ! mkdir "$LOCK_DIR" 2>/dev/null; then
|
||||||
|
die "Another bootstrap process holds $LOCK_DIR"
|
||||||
|
fi
|
||||||
|
trap 'rmdir "$LOCK_DIR" 2>/dev/null || true' EXIT
|
||||||
|
touch "$LOG_FILE"
|
||||||
|
chmod 0640 "$LOG_FILE"
|
||||||
|
else
|
||||||
|
log "Beginning phase-one enterprise bootstrap dry-run"
|
||||||
|
fi
|
||||||
|
install_packages
|
||||||
|
ensure_group_and_users
|
||||||
|
ensure_directories
|
||||||
|
ensure_uv
|
||||||
|
ensure_postgres_cluster
|
||||||
|
ensure_database_roles
|
||||||
|
ensure_database
|
||||||
|
apply_schema
|
||||||
|
write_nonsecret_runtime_config
|
||||||
|
create_bootstrap_admin
|
||||||
|
log "Phase-one bootstrap completed"
|
||||||
|
log "Application code, systemd units, Qwen, Docker, and firewall were not modified"
|
||||||
|
}
|
||||||
|
|
||||||
|
status_mode() {
|
||||||
|
log "Umbra enterprise installation status"
|
||||||
|
printf 'host: '; hostname -f 2>/dev/null || hostname
|
||||||
|
printf 'dns: '; getent ahosts "$UMBRA_HOSTNAME" 2>/dev/null || true
|
||||||
|
printf 'postgres: '; command -v pg_lsclusters >/dev/null 2>&1 && pg_lsclusters || echo 'not installed'
|
||||||
|
printf 'api user: '; id "$UMBRA_API_USER" 2>/dev/null || echo 'not installed'
|
||||||
|
printf 'worker user: '; id "$UMBRA_WORKER_USER" 2>/dev/null || echo 'not installed'
|
||||||
|
systemctl is-active postgresql 2>/dev/null || true
|
||||||
|
printf 'qwen port: '; (curl -sS --max-time 3 "$UMBRA_QWEN_URL/health" 2>&1 | head -c 300) || true; printf '\n'
|
||||||
|
}
|
||||||
|
|
||||||
|
verify_mode() {
|
||||||
|
log "Verifying phase-one installation"
|
||||||
|
require_command psql
|
||||||
|
require_root
|
||||||
|
pg_isready -h "$UMBRA_DB_HOST" -p "$UMBRA_DB_PORT" -d "$UMBRA_DB_NAME"
|
||||||
|
psql_as_postgres "$UMBRA_DB_NAME" -v ON_ERROR_STOP=1 -c 'SELECT version, applied_at FROM schema_migrations ORDER BY version' -c 'SELECT count(*) AS organizations FROM organizations' -c 'SELECT count(*) AS users FROM users'
|
||||||
|
systemctl is-active postgresql
|
||||||
|
log "Verification completed"
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
load_config
|
||||||
|
validate_config
|
||||||
|
check_tools
|
||||||
|
case "$MODE" in
|
||||||
|
check) check_host ;;
|
||||||
|
apply) apply_mode ;;
|
||||||
|
status) status_mode ;;
|
||||||
|
verify) verify_mode ;;
|
||||||
|
*) die "Unknown mode: $MODE" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
368
infra/bootstrap_umbracontrol_database.sh
Executable file
368
infra/bootstrap_umbracontrol_database.sh
Executable file
@@ -0,0 +1,368 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Database-only bootstrap for the UmbraTrans control plane.
|
||||||
|
# This intentionally does not install packages, create application directories,
|
||||||
|
# install application services, or modify Qwen/Docker/firewall settings.
|
||||||
|
set -Eeuo pipefail
|
||||||
|
umask 077
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||||
|
CONFIG_FILE="${UMBRA_BOOTSTRAP_CONFIG:-${SCRIPT_DIR}/enterprise-bootstrap.env}"
|
||||||
|
MODE="check"
|
||||||
|
DRY_RUN=0
|
||||||
|
LOCK_DIR="/run/lock/umbracontrol-database.lock"
|
||||||
|
LOG_FILE="/var/log/umbracontrol-database-bootstrap.log"
|
||||||
|
PG_SOCKET_DIR="/var/run/postgresql"
|
||||||
|
|
||||||
|
UMBRA_DB_NAME="umbracontrol"
|
||||||
|
UMBRA_DB_HOST="127.0.0.1"
|
||||||
|
UMBRA_DB_PORT="5432"
|
||||||
|
UMBRA_BOOTSTRAP_ADMIN_EMAIL="umbratools@gmail.com"
|
||||||
|
UMBRA_ORGANIZATION_NAME="Umbra"
|
||||||
|
UMBRA_ORGANIZATION_SLUG="umbra"
|
||||||
|
UMBRA_PG_SUPERUSER="postgres"
|
||||||
|
UMBRA_CREDENTIALS_FILE="/etc/umbracontrol/db-credentials.env"
|
||||||
|
UMBRA_BOOTSTRAP_TOKEN_FILE="/root/umbra-enterprise-bootstrap.token"
|
||||||
|
UMBRA_SERVICE_GROUP="umbracontrol"
|
||||||
|
UMBRA_OWNER_ROLE="umbracontrol_owner"
|
||||||
|
UMBRA_MIGRATOR_ROLE="umbracontrol_migrator"
|
||||||
|
UMBRA_API_ROLE="umbracontrol_api"
|
||||||
|
UMBRA_WORKER_ROLE="umbracontrol_worker"
|
||||||
|
UMBRA_READONLY_ROLE="umbracontrol_readonly"
|
||||||
|
UMBRA_BACKUP_ROLE="umbracontrol_backup"
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'EOF'
|
||||||
|
Usage: bootstrap_umbracontrol_database.sh [options]
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--check Read-only PostgreSQL and prerequisite inspection
|
||||||
|
--dry-run Show database actions without changing the host
|
||||||
|
--apply Create/finish the database, roles, schema, and bootstrap admin
|
||||||
|
--verify Verify the database, schema, RLS, and bootstrap records
|
||||||
|
--config FILE Read the existing non-secret enterprise-bootstrap.env
|
||||||
|
-h, --help Show this help
|
||||||
|
|
||||||
|
This script is intentionally database-only. It does not run apt, install uv,
|
||||||
|
create application directories, start Qwen, or change firewall rules.
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
while (($#)); do
|
||||||
|
case "$1" in
|
||||||
|
--check) MODE="check"; shift ;;
|
||||||
|
--dry-run) MODE="apply"; DRY_RUN=1; shift ;;
|
||||||
|
--apply) MODE="apply"; DRY_RUN=0; shift ;;
|
||||||
|
--verify) MODE="verify"; shift ;;
|
||||||
|
--config) CONFIG_FILE="${2:?--config requires a file}"; shift 2 ;;
|
||||||
|
-h|--help) usage; exit 0 ;;
|
||||||
|
*) printf 'Unknown option: %s\n' "$1" >&2; usage >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
log() {
|
||||||
|
local line="[$(date -Is)] $*"
|
||||||
|
printf '%s\n' "$line"
|
||||||
|
if [[ -f "$LOG_FILE" && "$(id -u)" -eq 0 ]]; then
|
||||||
|
printf '%s\n' "$line" >> "$LOG_FILE"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
die() {
|
||||||
|
printf 'ERROR: %s\n' "$*" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
require_root() {
|
||||||
|
[[ "$(id -u)" -eq 0 ]] || die "This operation requires root. Use --check for read-only inspection."
|
||||||
|
}
|
||||||
|
|
||||||
|
require_command() {
|
||||||
|
command -v "$1" >/dev/null 2>&1 || die "Required command is missing: $1"
|
||||||
|
}
|
||||||
|
|
||||||
|
load_config() {
|
||||||
|
[[ -f "$CONFIG_FILE" ]] || {
|
||||||
|
[[ "$MODE" == "check" || "$MODE" == "verify" ]] && return 0
|
||||||
|
die "Configuration file not found: $CONFIG_FILE"
|
||||||
|
}
|
||||||
|
|
||||||
|
local line key value
|
||||||
|
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||||
|
line="${line#"${line%%[![:space:]]*}"}"
|
||||||
|
[[ -z "$line" || "${line:0:1}" == "#" ]] && continue
|
||||||
|
[[ "$line" == *=* ]] || die "Invalid configuration line: $line"
|
||||||
|
key="${line%%=*}"
|
||||||
|
value="${line#*=}"
|
||||||
|
key="${key//[[:space:]]/}"
|
||||||
|
value="${value%\"}"; value="${value#\"}"
|
||||||
|
value="${value%\'}"; value="${value#\'}"
|
||||||
|
case "$key" in
|
||||||
|
UMBRA_DB_NAME|UMBRA_DB_HOST|UMBRA_DB_PORT|UMBRA_BOOTSTRAP_ADMIN_EMAIL|UMBRA_ORGANIZATION_NAME|UMBRA_ORGANIZATION_SLUG)
|
||||||
|
printf -v "$key" '%s' "$value"
|
||||||
|
;;
|
||||||
|
UMBRA_*)
|
||||||
|
: # Ignore unrelated non-secret enterprise settings.
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
die "Unsupported configuration key: $key"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done < "$CONFIG_FILE"
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_config() {
|
||||||
|
[[ "$UMBRA_DB_NAME" =~ ^[a-z_][a-z0-9_]*$ ]] || die "Invalid UMBRA_DB_NAME"
|
||||||
|
[[ "$UMBRA_DB_HOST" =~ ^[A-Za-z0-9._:-]+$ ]] || die "Invalid UMBRA_DB_HOST"
|
||||||
|
[[ "$UMBRA_DB_PORT" =~ ^[0-9]+$ && "$UMBRA_DB_PORT" -ge 1 && "$UMBRA_DB_PORT" -le 65535 ]] || die "Invalid UMBRA_DB_PORT"
|
||||||
|
[[ "$UMBRA_BOOTSTRAP_ADMIN_EMAIL" == *@*.* ]] || die "Invalid administrator email"
|
||||||
|
[[ "$UMBRA_ORGANIZATION_SLUG" =~ ^[a-z0-9][a-z0-9-]*$ ]] || die "Invalid organization slug"
|
||||||
|
[[ -d "$PG_SOCKET_DIR" ]] || die "PostgreSQL socket directory is missing: $PG_SOCKET_DIR"
|
||||||
|
}
|
||||||
|
|
||||||
|
run() {
|
||||||
|
if ((DRY_RUN)); then
|
||||||
|
printf '[dry-run]'
|
||||||
|
printf ' %q' "$@"
|
||||||
|
printf '\n'
|
||||||
|
else
|
||||||
|
"$@"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
random_secret() {
|
||||||
|
openssl rand -hex 32
|
||||||
|
}
|
||||||
|
|
||||||
|
read_secret() {
|
||||||
|
local key="$1"
|
||||||
|
[[ -f "$UMBRA_CREDENTIALS_FILE" ]] || return 1
|
||||||
|
awk -F= -v key="$key" '$1 == key {print substr($0, index($0, "=") + 1); exit}' "$UMBRA_CREDENTIALS_FILE"
|
||||||
|
}
|
||||||
|
|
||||||
|
write_secret() {
|
||||||
|
local key="$1" value="$2" temporary
|
||||||
|
if ((DRY_RUN)); then
|
||||||
|
printf '[dry-run] store generated database secret %s\n' "$key"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
install -d -m 0750 -o root -g "$UMBRA_SERVICE_GROUP" "$(dirname "$UMBRA_CREDENTIALS_FILE")"
|
||||||
|
touch "$UMBRA_CREDENTIALS_FILE"
|
||||||
|
chown root:"$UMBRA_SERVICE_GROUP" "$UMBRA_CREDENTIALS_FILE"
|
||||||
|
chmod 0640 "$UMBRA_CREDENTIALS_FILE"
|
||||||
|
temporary="$(mktemp "${UMBRA_CREDENTIALS_FILE}.XXXXXX")"
|
||||||
|
awk -F= -v key="$key" -v value="$value" '
|
||||||
|
$1 == key { print key "=" value; found = 1; next }
|
||||||
|
{ print }
|
||||||
|
END { if (!found) print key "=" value }
|
||||||
|
' "$UMBRA_CREDENTIALS_FILE" > "$temporary"
|
||||||
|
chown root:"$UMBRA_SERVICE_GROUP" "$temporary"
|
||||||
|
chmod 0640 "$temporary"
|
||||||
|
mv -f "$temporary" "$UMBRA_CREDENTIALS_FILE"
|
||||||
|
}
|
||||||
|
|
||||||
|
psql_as_postgres() {
|
||||||
|
runuser -u "$UMBRA_PG_SUPERUSER" -- env \
|
||||||
|
-u PGHOST -u PGPORT -u PGDATABASE -u PGUSER -u PGPASSWORD \
|
||||||
|
-u PGPASSWORDFILE -u PGSERVICE -u PGSERVICEFILE \
|
||||||
|
PGOPTIONS='-c log_statement=none' \
|
||||||
|
psql -h "$PG_SOCKET_DIR" -p "$UMBRA_DB_PORT" -U "$UMBRA_PG_SUPERUSER" \
|
||||||
|
-v ON_ERROR_STOP=1 "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
role_exists() {
|
||||||
|
local role="$1"
|
||||||
|
psql_as_postgres postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname = '$role'" | grep -q 1
|
||||||
|
}
|
||||||
|
|
||||||
|
ensure_role() {
|
||||||
|
local role="$1" login="$2" password_key="$3" password=""
|
||||||
|
if ((DRY_RUN)); then
|
||||||
|
printf '[dry-run] ensure PostgreSQL role %s (%s)\n' "$role" "$login"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
if role_exists "$role"; then
|
||||||
|
if [[ "$login" == "LOGIN" ]]; then
|
||||||
|
printf 'ALTER ROLE "%s" LOGIN;\n' "$role" | psql_as_postgres postgres >/dev/null
|
||||||
|
else
|
||||||
|
printf 'ALTER ROLE "%s" NOLOGIN;\n' "$role" | psql_as_postgres postgres >/dev/null
|
||||||
|
fi
|
||||||
|
log "PostgreSQL role already exists: $role"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$login" == "LOGIN" ]]; then
|
||||||
|
password="$(read_secret "$password_key" 2>/dev/null || true)"
|
||||||
|
[[ -n "$password" ]] || password="$(random_secret)"
|
||||||
|
write_secret "$password_key" "$password"
|
||||||
|
printf 'CREATE ROLE "%s" LOGIN PASSWORD '\''%s'\'';\n' "$role" "$password" \
|
||||||
|
| psql_as_postgres postgres >/dev/null
|
||||||
|
else
|
||||||
|
printf 'CREATE ROLE "%s" NOLOGIN;\n' "$role" | psql_as_postgres postgres >/dev/null
|
||||||
|
fi
|
||||||
|
log "PostgreSQL role created: $role"
|
||||||
|
}
|
||||||
|
|
||||||
|
ensure_roles() {
|
||||||
|
log "Ensuring PostgreSQL roles"
|
||||||
|
ensure_role "$UMBRA_OWNER_ROLE" NOLOGIN OWNER_UNUSED
|
||||||
|
ensure_role "$UMBRA_MIGRATOR_ROLE" LOGIN UMBRACONTROL_MIGRATOR_PASSWORD
|
||||||
|
ensure_role "$UMBRA_API_ROLE" LOGIN UMBRACONTROL_API_PASSWORD
|
||||||
|
ensure_role "$UMBRA_WORKER_ROLE" LOGIN UMBRACONTROL_WORKER_PASSWORD
|
||||||
|
ensure_role "$UMBRA_READONLY_ROLE" LOGIN UMBRACONTROL_READONLY_PASSWORD
|
||||||
|
ensure_role "$UMBRA_BACKUP_ROLE" LOGIN UMBRACONTROL_BACKUP_PASSWORD
|
||||||
|
}
|
||||||
|
|
||||||
|
database_exists() {
|
||||||
|
psql_as_postgres postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '$UMBRA_DB_NAME'" | grep -q 1
|
||||||
|
}
|
||||||
|
|
||||||
|
ensure_database() {
|
||||||
|
log "Ensuring database: $UMBRA_DB_NAME"
|
||||||
|
if ((DRY_RUN)); then
|
||||||
|
printf '[dry-run] create database if absent, owner=%s\n' "$UMBRA_OWNER_ROLE"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
if ! database_exists; then
|
||||||
|
runuser -u "$UMBRA_PG_SUPERUSER" -- createdb --owner="$UMBRA_OWNER_ROLE" "$UMBRA_DB_NAME"
|
||||||
|
log "Database created"
|
||||||
|
else
|
||||||
|
log "Database already exists"
|
||||||
|
fi
|
||||||
|
psql_as_postgres postgres -c "REVOKE ALL ON DATABASE \"$UMBRA_DB_NAME\" FROM PUBLIC" >/dev/null
|
||||||
|
psql_as_postgres postgres -c "GRANT CONNECT ON DATABASE \"$UMBRA_DB_NAME\" TO \"$UMBRA_MIGRATOR_ROLE\", \"$UMBRA_API_ROLE\", \"$UMBRA_WORKER_ROLE\", \"$UMBRA_READONLY_ROLE\", \"$UMBRA_BACKUP_ROLE\"" >/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
apply_schema() {
|
||||||
|
local schema_file="${SCRIPT_DIR}/postgres/001_control_plane.sql"
|
||||||
|
[[ -f "$schema_file" ]] || die "Schema file is missing: $schema_file"
|
||||||
|
log "Applying identity, tenancy, role, audit, and RLS schema"
|
||||||
|
if ((DRY_RUN)); then
|
||||||
|
printf '[dry-run] stream %s into PostgreSQL database %s\n' "$schema_file" "$UMBRA_DB_NAME"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
/bin/cat -- "$schema_file" | psql_as_postgres "$UMBRA_DB_NAME" >/dev/null
|
||||||
|
log "Database schema applied"
|
||||||
|
}
|
||||||
|
|
||||||
|
create_bootstrap_admin() {
|
||||||
|
local token token_hash
|
||||||
|
if [[ -f "$UMBRA_BOOTSTRAP_TOKEN_FILE" ]]; then
|
||||||
|
token="$(cat "$UMBRA_BOOTSTRAP_TOKEN_FILE")"
|
||||||
|
else
|
||||||
|
token="$(random_secret)"
|
||||||
|
if ((DRY_RUN)); then
|
||||||
|
printf '[dry-run] create root-only bootstrap token\n'
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
printf '%s\n' "$token" > "$UMBRA_BOOTSTRAP_TOKEN_FILE"
|
||||||
|
chown root:root "$UMBRA_BOOTSTRAP_TOKEN_FILE"
|
||||||
|
chmod 0600 "$UMBRA_BOOTSTRAP_TOKEN_FILE"
|
||||||
|
fi
|
||||||
|
[[ "$token" =~ ^[a-f0-9]{64}$ ]] || die "Bootstrap token has an unexpected format"
|
||||||
|
token_hash="$(printf '%s' "$token" | sha256sum | awk '{print $1}')"
|
||||||
|
|
||||||
|
log "Ensuring bootstrap administrator"
|
||||||
|
if ((DRY_RUN)); then
|
||||||
|
printf '[dry-run] create administrator %s and organization %s\n' "$UMBRA_BOOTSTRAP_ADMIN_EMAIL" "$UMBRA_ORGANIZATION_NAME"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
psql_as_postgres "$UMBRA_DB_NAME" \
|
||||||
|
-v admin_email="$UMBRA_BOOTSTRAP_ADMIN_EMAIL" \
|
||||||
|
-v token_hash="$token_hash" \
|
||||||
|
-v organization_name="$UMBRA_ORGANIZATION_NAME" \
|
||||||
|
-v organization_slug="$UMBRA_ORGANIZATION_SLUG" <<'SQL' >/dev/null
|
||||||
|
INSERT INTO users (email, display_name, auth_provider, auth_subject, status, mfa_required)
|
||||||
|
VALUES (:'admin_email', :'admin_email', 'bootstrap', 'bootstrap:' || gen_random_uuid()::text, 'invited', true)
|
||||||
|
ON CONFLICT (email) DO UPDATE
|
||||||
|
SET display_name = EXCLUDED.display_name,
|
||||||
|
auth_provider = 'bootstrap',
|
||||||
|
mfa_required = true,
|
||||||
|
updated_at = now();
|
||||||
|
|
||||||
|
INSERT INTO organizations (name, slug)
|
||||||
|
VALUES (:'organization_name', :'organization_slug')
|
||||||
|
ON CONFLICT (slug) DO UPDATE SET name = EXCLUDED.name;
|
||||||
|
|
||||||
|
INSERT INTO user_roles (user_id, role_id, scope, organization_id, project_id, created_by)
|
||||||
|
SELECT u.id, r.id, 'platform', NULL, NULL, u.id
|
||||||
|
FROM users u
|
||||||
|
JOIN roles r ON r.scope = 'platform' AND r.name = 'platform_admin'
|
||||||
|
WHERE u.email = :'admin_email'
|
||||||
|
ON CONFLICT DO NOTHING;
|
||||||
|
|
||||||
|
INSERT INTO user_roles (user_id, role_id, scope, organization_id, project_id, created_by)
|
||||||
|
SELECT u.id, r.id, 'organization', o.id, NULL, u.id
|
||||||
|
FROM users u
|
||||||
|
JOIN organizations o ON o.slug = :'organization_slug'
|
||||||
|
JOIN roles r ON r.scope = 'organization' AND r.name = 'organization_admin'
|
||||||
|
WHERE u.email = :'admin_email'
|
||||||
|
ON CONFLICT DO NOTHING;
|
||||||
|
|
||||||
|
INSERT INTO bootstrap_tokens (user_id, token_hash, expires_at)
|
||||||
|
SELECT id, :'token_hash', now() + interval '24 hours'
|
||||||
|
FROM users
|
||||||
|
WHERE email = :'admin_email'
|
||||||
|
ON CONFLICT (token_hash) DO NOTHING;
|
||||||
|
SQL
|
||||||
|
log "Bootstrap administrator is ready"
|
||||||
|
}
|
||||||
|
|
||||||
|
check_mode() {
|
||||||
|
log "Database-only read-only inspection"
|
||||||
|
printf 'psql: '; command -v psql 2>/dev/null || echo 'missing'
|
||||||
|
printf 'PostgreSQL: '; command -v pg_lsclusters >/dev/null 2>&1 && pg_lsclusters || echo 'not installed'
|
||||||
|
printf 'service: '; systemctl is-active postgresql 2>/dev/null || true
|
||||||
|
printf 'socket: '; test -S "$PG_SOCKET_DIR/.s.PGSQL.$UMBRA_DB_PORT" && echo ready || echo missing
|
||||||
|
printf 'database socket: '; pg_isready -h "$PG_SOCKET_DIR" -p "$UMBRA_DB_PORT" -d "$UMBRA_DB_NAME" 2>&1 || true
|
||||||
|
printf 'roles: '; getent group "$UMBRA_SERVICE_GROUP" 2>/dev/null || echo 'service group absent'
|
||||||
|
log "No changes made"
|
||||||
|
}
|
||||||
|
|
||||||
|
apply_mode() {
|
||||||
|
if ((DRY_RUN == 0)); then
|
||||||
|
require_root
|
||||||
|
mkdir "$LOCK_DIR" 2>/dev/null || die "Another database bootstrap is running"
|
||||||
|
trap 'rmdir "$LOCK_DIR" 2>/dev/null || true' EXIT
|
||||||
|
touch "$LOG_FILE"
|
||||||
|
chmod 0640 "$LOG_FILE"
|
||||||
|
else
|
||||||
|
log "Database-only bootstrap dry-run"
|
||||||
|
fi
|
||||||
|
ensure_roles
|
||||||
|
ensure_database
|
||||||
|
apply_schema
|
||||||
|
create_bootstrap_admin
|
||||||
|
log "Database-only bootstrap completed"
|
||||||
|
}
|
||||||
|
|
||||||
|
verify_mode() {
|
||||||
|
require_root
|
||||||
|
log "Verifying database-only bootstrap"
|
||||||
|
pg_isready -h "$UMBRA_DB_HOST" -p "$UMBRA_DB_PORT" -d "$UMBRA_DB_NAME"
|
||||||
|
psql_as_postgres "$UMBRA_DB_NAME" -v ON_ERROR_STOP=1 \
|
||||||
|
-c 'SELECT version, applied_at FROM schema_migrations ORDER BY version' \
|
||||||
|
-c 'SELECT count(*) AS organizations FROM organizations' \
|
||||||
|
-c 'SELECT count(*) AS users FROM users' \
|
||||||
|
-c "SELECT count(*) AS rls_tables FROM pg_class WHERE relrowsecurity AND relname IN ('organizations','projects','users','user_roles','invitations','sessions','mfa_methods','bootstrap_tokens','audit_events')"
|
||||||
|
log "Database verification completed"
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
load_config
|
||||||
|
validate_config
|
||||||
|
require_command openssl
|
||||||
|
require_command systemctl
|
||||||
|
if [[ "$MODE" == "apply" || "$MODE" == "verify" ]]; then
|
||||||
|
require_command psql
|
||||||
|
require_command pg_isready
|
||||||
|
require_command runuser
|
||||||
|
fi
|
||||||
|
case "$MODE" in
|
||||||
|
check) check_mode ;;
|
||||||
|
apply) apply_mode ;;
|
||||||
|
verify) verify_mode ;;
|
||||||
|
*) die "Unknown mode: $MODE" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
main
|
||||||
20
infra/enterprise-bootstrap.env.example
Normal file
20
infra/enterprise-bootstrap.env.example
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
# Non-secret phase-one bootstrap configuration.
|
||||||
|
# Copy to enterprise-bootstrap.env and review before running as root.
|
||||||
|
# Do not put passwords, private keys, or API secrets in this file.
|
||||||
|
|
||||||
|
UMBRA_HOSTNAME=oldbox.bdmurphy.me
|
||||||
|
UMBRA_PUBLIC_URL=https://oldbox.bdmurphy.me
|
||||||
|
UMBRA_DB_NAME=umbracontrol
|
||||||
|
UMBRA_DB_HOST=127.0.0.1
|
||||||
|
UMBRA_DB_PORT=5432
|
||||||
|
UMBRA_PG_VERSION=16
|
||||||
|
UMBRA_APP_ROOT=/opt/umbracontrol
|
||||||
|
UMBRA_DATA_ROOT=/var/lib/umbracontrol
|
||||||
|
UMBRA_BACKUP_ROOT=/var/backups/umbracontrol
|
||||||
|
UMBRA_BOOTSTRAP_ADMIN_EMAIL=umbratools@gmail.com
|
||||||
|
UMBRA_ORGANIZATION_NAME=Umbra
|
||||||
|
UMBRA_ORGANIZATION_SLUG=umbra
|
||||||
|
UMBRA_QWEN_URL=http://192.168.123.14:8881
|
||||||
|
UMBRA_INSTALL_PACKAGES=1
|
||||||
|
UMBRA_INSTALL_UV=1
|
||||||
|
UMBRA_CREATE_BOOTSTRAP_ADMIN=1
|
||||||
180
infra/oldbox-agent-communication-prompt.md
Normal file
180
infra/oldbox-agent-communication-prompt.md
Normal file
@@ -0,0 +1,180 @@
|
|||||||
|
# Build a secure oldbox ↔ Mac agent communication MCP server
|
||||||
|
|
||||||
|
You are the OpenCode agent running on `oldbox`. The Mac-side agent needs a
|
||||||
|
reliable way to exchange sanitized instructions and reports with you without
|
||||||
|
requiring repeated clipboard copy/paste.
|
||||||
|
|
||||||
|
## Objective
|
||||||
|
|
||||||
|
Create a standalone, local stdio MCP server named `agent-comm` under the
|
||||||
|
UmbraTrans workspace on oldbox. It must use a shared, append-only inbox/outbox
|
||||||
|
on the NAS2 `netbackup/UmbraTrans` share. Do not create an unauthenticated
|
||||||
|
network listener.
|
||||||
|
|
||||||
|
The first purpose is to send the pending SSH-authentication diagnostic report
|
||||||
|
back to the Mac agent without exposing passwords or other secrets.
|
||||||
|
|
||||||
|
## Security requirements
|
||||||
|
|
||||||
|
- Never accept, return, log, or transmit passwords, sudo passwords, private
|
||||||
|
keys, API tokens, Keychain data, or `/etc/shadow` contents.
|
||||||
|
- No secret values may appear in MCP arguments, environment variables, command
|
||||||
|
lines, filenames, reports, or logs.
|
||||||
|
- The server must be stdio-only. It must not listen on a TCP or UDP port.
|
||||||
|
- Use the NAS share only for sanitized text/JSON messages.
|
||||||
|
- Message files must be written atomically (`temporary file` + `rename`).
|
||||||
|
- Use restrictive directory/file permissions and refuse symlink paths.
|
||||||
|
- Validate and bound all message sizes.
|
||||||
|
- Do not change SSH, sudoers, firewall, Docker, Qwen, or existing services.
|
||||||
|
- Do not install a system-wide daemon yet.
|
||||||
|
- Do not grant `NOPASSWD: ALL` or any broad sudo rule.
|
||||||
|
|
||||||
|
## Discovery phase
|
||||||
|
|
||||||
|
Before creating the server, perform read-only discovery:
|
||||||
|
|
||||||
|
1. Locate the NAS2 mount for the `netbackup` share.
|
||||||
|
2. Locate or propose the `UmbraTrans` subdirectory.
|
||||||
|
3. Confirm the path is writable by `umbra`.
|
||||||
|
4. Report the exact path and permissions.
|
||||||
|
5. Do not write any message until the path is confirmed in the report.
|
||||||
|
|
||||||
|
Use a conservative search such as `find`, `mount`, and `df`; do not modify
|
||||||
|
mounts or credentials. If the share is not visible, report that and stop rather
|
||||||
|
than guessing a path.
|
||||||
|
|
||||||
|
## Shared mailbox layout
|
||||||
|
|
||||||
|
Once the path is confirmed, use this layout below the confirmed UmbraTrans
|
||||||
|
directory:
|
||||||
|
|
||||||
|
```text
|
||||||
|
agent-mailbox/
|
||||||
|
inbox/ # sanitized instructions from Mac to oldbox
|
||||||
|
outbox/ # sanitized reports from oldbox to Mac
|
||||||
|
processed/ # messages already handled by oldbox
|
||||||
|
archive/ # optional processed message archive
|
||||||
|
```
|
||||||
|
|
||||||
|
Each message is one UTF-8 JSON file with this shape:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"id": "UUID",
|
||||||
|
"created_at": "ISO-8601 UTC",
|
||||||
|
"sender": "oldbox",
|
||||||
|
"kind": "diagnostic|report|instruction|ack",
|
||||||
|
"correlation_id": "optional UUID",
|
||||||
|
"title": "short non-secret title",
|
||||||
|
"body": "sanitized text, maximum 64 KiB",
|
||||||
|
"attachments": []
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not allow arbitrary attachment paths in version 1.
|
||||||
|
|
||||||
|
## MCP tools
|
||||||
|
|
||||||
|
Implement these tools:
|
||||||
|
|
||||||
|
### `agent_status`
|
||||||
|
|
||||||
|
Return only:
|
||||||
|
|
||||||
|
- server version
|
||||||
|
- configured mailbox path
|
||||||
|
- whether inbox/outbox are readable/writable
|
||||||
|
- number of pending messages
|
||||||
|
- last health-check time
|
||||||
|
- no secret values
|
||||||
|
|
||||||
|
### `send_report`
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
|
||||||
|
- `title`: non-secret string
|
||||||
|
- `body`: sanitized report text
|
||||||
|
- `correlation_id`: optional non-secret string
|
||||||
|
- `kind`: default `report`
|
||||||
|
|
||||||
|
Write a validated JSON message to `outbox/` using an atomic rename. Return the
|
||||||
|
message ID and path, never the secret contents.
|
||||||
|
|
||||||
|
### `read_instructions`
|
||||||
|
|
||||||
|
Read up to a bounded number of JSON messages from `inbox/`. Validate each
|
||||||
|
message, return only sanitized metadata/body, and move successfully handled
|
||||||
|
messages to `processed/` using an atomic rename. Never execute commands from a
|
||||||
|
message automatically.
|
||||||
|
|
||||||
|
### `send_ack`
|
||||||
|
|
||||||
|
Accept only a correlation ID and short non-secret text. Write an `ack` message
|
||||||
|
to `outbox/`.
|
||||||
|
|
||||||
|
## Current diagnostic report
|
||||||
|
|
||||||
|
After the server is implemented and tested, use it to send one report covering:
|
||||||
|
|
||||||
|
- the successful manual SSH path
|
||||||
|
- the broker's `Permission denied (publickey,password)` result
|
||||||
|
- the fact that no authorized key or certificate is configured
|
||||||
|
- the account/password state already observed
|
||||||
|
- the exact unresolved question: whether the broker supplied a wrong password
|
||||||
|
or supplied no usable password to OpenSSH
|
||||||
|
|
||||||
|
Do not include any password or hash. Do not ask the Mac agent to guess.
|
||||||
|
|
||||||
|
## Implementation requirements
|
||||||
|
|
||||||
|
- Python 3.11+ standard library is sufficient; do not add dependencies unless
|
||||||
|
necessary.
|
||||||
|
- Use type hints, `from __future__ import annotations`, logging to stderr, and
|
||||||
|
no `print()` for debug output except the CLI's intentional status output.
|
||||||
|
- Include a README with the exact OpenCode local-server configuration.
|
||||||
|
- Include unit tests for JSON validation, atomic writes, path traversal,
|
||||||
|
symlink refusal, size limits, inbox/outbox behavior, and secret-pattern
|
||||||
|
redaction.
|
||||||
|
- Add a `--check` mode that performs only read-only checks.
|
||||||
|
- Run the tests and `--check` before sending the first report.
|
||||||
|
|
||||||
|
## OpenCode configuration to document
|
||||||
|
|
||||||
|
The configuration should be equivalent to:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"mcp": {
|
||||||
|
"servers": {
|
||||||
|
"agent-comm": {
|
||||||
|
"type": "local",
|
||||||
|
"command": [
|
||||||
|
"/absolute/path/to/python",
|
||||||
|
"-m",
|
||||||
|
"agent_comm"
|
||||||
|
],
|
||||||
|
"environment": {
|
||||||
|
"AGENT_COMM_ROOT": "/confirmed/nas2/path/agent-mailbox"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not add secrets to the configuration.
|
||||||
|
|
||||||
|
## Completion response
|
||||||
|
|
||||||
|
Return a concise report containing:
|
||||||
|
|
||||||
|
1. Confirmed mailbox path and permissions.
|
||||||
|
2. Files created.
|
||||||
|
3. Test results.
|
||||||
|
4. OpenCode configuration path/snippet.
|
||||||
|
5. The first outbox report message ID.
|
||||||
|
6. Any unresolved issue.
|
||||||
|
|
||||||
|
Do not claim the Mac agent has received the message until the Mac-side reader
|
||||||
|
has been configured and verified.
|
||||||
96
infra/oldbox-bootstrap-instructions.md
Normal file
96
infra/oldbox-bootstrap-instructions.md
Normal file
@@ -0,0 +1,96 @@
|
|||||||
|
# Oldbox enterprise bootstrap instructions
|
||||||
|
|
||||||
|
You are the OpenCode agent running on `oldbox`. Work locally on the target
|
||||||
|
host; do not wait for clipboard reports from the Mac.
|
||||||
|
|
||||||
|
Read and execute these files first:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/tmp/oldbox-temporary-access-prompt.md
|
||||||
|
/tmp/oldbox-agent-communication-prompt.md
|
||||||
|
```
|
||||||
|
|
||||||
|
Then complete the following local bootstrap:
|
||||||
|
|
||||||
|
## 1. Preserve the existing host
|
||||||
|
|
||||||
|
Inventory, but do not alter:
|
||||||
|
|
||||||
|
- Qwen and GPU services
|
||||||
|
- Docker containers
|
||||||
|
- MySQL
|
||||||
|
- Samba/Webmin/XRDP
|
||||||
|
- PostgreSQL
|
||||||
|
- firewall rules
|
||||||
|
- existing user accounts and data
|
||||||
|
|
||||||
|
Report conflicts before changing anything.
|
||||||
|
|
||||||
|
## 2. Create the development workspace
|
||||||
|
|
||||||
|
Use:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/home/umbra/src/UmbraTrans
|
||||||
|
```
|
||||||
|
|
||||||
|
Create it with ownership `umbra:umbra`. Do not modify UmbraAudio files.
|
||||||
|
If the enterprise source bundle is available locally, copy it there; do not
|
||||||
|
invent application code or overwrite existing source.
|
||||||
|
|
||||||
|
Prepare:
|
||||||
|
|
||||||
|
- Git repository/worktree
|
||||||
|
- `.venv` managed by the approved `uv` workflow
|
||||||
|
- non-secret `.env` template
|
||||||
|
- test/lint/format commands from the project configuration
|
||||||
|
- an explicit development and production data separation
|
||||||
|
|
||||||
|
Run the project’s tests and record the results.
|
||||||
|
|
||||||
|
## 3. Configure backups
|
||||||
|
|
||||||
|
Use only the NAS2 `netbackup/UmbraTrans` share after confirming the oldbox
|
||||||
|
account can access it. Create:
|
||||||
|
|
||||||
|
```text
|
||||||
|
backups/postgres/
|
||||||
|
backups/config/
|
||||||
|
backups/workspace/
|
||||||
|
backups/manifests/
|
||||||
|
```
|
||||||
|
|
||||||
|
PostgreSQL backups must be logical dumps or an approved backup tool. Never
|
||||||
|
copy a live PostgreSQL data directory or SQLite database from a running
|
||||||
|
service as a backup.
|
||||||
|
|
||||||
|
Use atomic temporary files, restrictive permissions, timestamps, checksums,
|
||||||
|
and a retention policy. Do not put credentials in the backup path or report.
|
||||||
|
|
||||||
|
## 4. Establish the agent communication queue
|
||||||
|
|
||||||
|
Use the `agent-comm` server from the communication prompt. If NAS permissions
|
||||||
|
are not yet correct, use a private local queue under the oldbox workspace and
|
||||||
|
report the exact blocker; do not broaden share permissions automatically.
|
||||||
|
|
||||||
|
The queue is for sanitized instructions and reports only. It must never carry
|
||||||
|
passwords, tokens, private keys, or unrestricted shell commands.
|
||||||
|
|
||||||
|
## 5. Report
|
||||||
|
|
||||||
|
Write a concise, machine-readable status report to the queue and print:
|
||||||
|
|
||||||
|
```text
|
||||||
|
OLDBOX_BOOTSTRAP_COMPLETE
|
||||||
|
```
|
||||||
|
|
||||||
|
Include:
|
||||||
|
|
||||||
|
- workspace path and ownership
|
||||||
|
- test result
|
||||||
|
- backup path and last backup status
|
||||||
|
- communication queue path/status
|
||||||
|
- unresolved blockers
|
||||||
|
- confirmation that existing services were preserved
|
||||||
|
|
||||||
|
Do not claim completion for any step that was not actually verified.
|
||||||
61
infra/oldbox-ssh-auth-log-prompt.md
Normal file
61
infra/oldbox-ssh-auth-log-prompt.md
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
# Oldbox SSH authentication-log handoff
|
||||||
|
|
||||||
|
A Mac-side broker SSH attempt was just made against `umbra@oldbox.bdmurphy.me`.
|
||||||
|
It received `Permission denied (publickey,password)`.
|
||||||
|
|
||||||
|
You are the OpenCode agent running on oldbox. Perform only read-only
|
||||||
|
authentication diagnostics now.
|
||||||
|
|
||||||
|
Do not change SSH configuration.
|
||||||
|
Do not restart SSH.
|
||||||
|
Do not install anything.
|
||||||
|
Do not read or print passwords, hashes, private keys, or tokens.
|
||||||
|
If a sudo command prompts, the human operator may enter the sudo password
|
||||||
|
locally. Never ask for or transmit that password through chat or the report.
|
||||||
|
|
||||||
|
Run these commands as needed:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo journalctl -u ssh --since "3 minutes ago" --no-pager
|
||||||
|
sudo journalctl -u sshd --since "3 minutes ago" --no-pager
|
||||||
|
sudo /usr/sbin/sshd -T -C user=umbra,addr=192.168.123.12,host=oldbox.bdmurphy.me
|
||||||
|
sudo passwd -S umbra
|
||||||
|
```
|
||||||
|
|
||||||
|
If journalctl is not the active service name, use the service that owns port
|
||||||
|
22. If `sshd -T -C` is unavailable, report the exact error.
|
||||||
|
|
||||||
|
Filter the output to authentication-relevant lines only, especially:
|
||||||
|
|
||||||
|
- Accepted password
|
||||||
|
- Failed password
|
||||||
|
- Failed publickey
|
||||||
|
- Invalid user
|
||||||
|
- account locked
|
||||||
|
- PAM/session-open failures
|
||||||
|
- certificate errors
|
||||||
|
- source address and username
|
||||||
|
|
||||||
|
Return exactly this compact report:
|
||||||
|
|
||||||
|
```text
|
||||||
|
=== AUTH LOG RESULT ===
|
||||||
|
<paste only relevant sanitized lines, or the exact permission/error>
|
||||||
|
|
||||||
|
=== EFFECTIVE SSH AUTH ===
|
||||||
|
<paste only relevant key/value lines, or the exact error>
|
||||||
|
|
||||||
|
=== ACCOUNT STATE ===
|
||||||
|
<passwd -S result; no hash>
|
||||||
|
|
||||||
|
=== CONCLUSION ===
|
||||||
|
<one short conclusion: wrong password, key/certificate path, PAM issue,
|
||||||
|
or insufficient evidence>
|
||||||
|
|
||||||
|
=== NEXT ACTION ===
|
||||||
|
<one specific next step>
|
||||||
|
|
||||||
|
READY_FOR_MAC_AGENT
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not guess. Do not make changes. Do not include secret values.
|
||||||
129
infra/oldbox-ssh-diagnostic-prompt.md
Normal file
129
infra/oldbox-ssh-diagnostic-prompt.md
Normal file
@@ -0,0 +1,129 @@
|
|||||||
|
# Oldbox SSH diagnostic handoff
|
||||||
|
|
||||||
|
You are the on-host investigator for `oldbox.bdmurphy.me`.
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Determine why direct interactive SSH as `umbra` succeeds from the Mac, while the Mac-side remote broker receives:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Permission denied (publickey,password)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Known facts
|
||||||
|
|
||||||
|
- Host: `oldbox.bdmurphy.me`
|
||||||
|
- Address: `192.168.123.14`
|
||||||
|
- User: `umbra`
|
||||||
|
- The Mac can run: `ssh umbra@oldbox`
|
||||||
|
- The Mac broker is attempting SSH as `umbra@oldbox`
|
||||||
|
- Do not change SSH configuration.
|
||||||
|
- Do not restart SSH.
|
||||||
|
- Do not install packages.
|
||||||
|
- Do not read or print passwords, private keys, tokens, `/etc/shadow` contents, Keychain data, or environment secrets.
|
||||||
|
- Do not ask me to provide any password in chat.
|
||||||
|
- Use `sudo` only for read-only commands. Prefer `sudo -n`; if `sudo -n` is not permitted, report that limitation instead of prompting for a password.
|
||||||
|
|
||||||
|
## Read-only inspection
|
||||||
|
|
||||||
|
### 1. Record host and account state
|
||||||
|
|
||||||
|
Collect:
|
||||||
|
|
||||||
|
- Hostname and kernel
|
||||||
|
- SSH service state
|
||||||
|
- `id umbra`
|
||||||
|
- `getent passwd umbra`
|
||||||
|
- Home-directory permissions
|
||||||
|
- Current `umbra` shell
|
||||||
|
|
||||||
|
### 2. Inspect SSH policy
|
||||||
|
|
||||||
|
Run, where permitted:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sshd -T
|
||||||
|
```
|
||||||
|
|
||||||
|
If permitted:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sshd -T -C user=umbra,addr=192.168.123.1,host=oldbox.bdmurphy.me
|
||||||
|
```
|
||||||
|
|
||||||
|
Search `/etc/ssh/sshd_config` and `/etc/ssh/sshd_config.d` for:
|
||||||
|
|
||||||
|
- `Match`
|
||||||
|
- `AuthenticationMethods`
|
||||||
|
- `PasswordAuthentication`
|
||||||
|
- `KbdInteractiveAuthentication`
|
||||||
|
- `UsePAM`
|
||||||
|
- `PubkeyAuthentication`
|
||||||
|
- `AllowUsers`
|
||||||
|
- `DenyUsers`
|
||||||
|
- Certificate-related directives
|
||||||
|
|
||||||
|
### 3. Inspect account state
|
||||||
|
|
||||||
|
If permitted, collect:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
passwd -S umbra
|
||||||
|
chage -l umbra
|
||||||
|
```
|
||||||
|
|
||||||
|
Report account lock or expiry information without printing hashes.
|
||||||
|
|
||||||
|
### 4. Inspect recent SSH authentication logs
|
||||||
|
|
||||||
|
Where readable, inspect:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
journalctl -u ssh --since "15 minutes ago" --no-pager
|
||||||
|
journalctl -u sshd --since "15 minutes ago" --no-pager
|
||||||
|
```
|
||||||
|
|
||||||
|
Also inspect `/var/log/auth.log` or `/var/log/secure` if readable.
|
||||||
|
|
||||||
|
Report whether failures mention:
|
||||||
|
|
||||||
|
- failed password
|
||||||
|
- invalid user
|
||||||
|
- account locked
|
||||||
|
- failed public key
|
||||||
|
- certificate problem
|
||||||
|
- PAM failure
|
||||||
|
- or another condition
|
||||||
|
|
||||||
|
### 5. Determine the likely authentication path
|
||||||
|
|
||||||
|
Determine whether the successful Mac login is likely password-based or key/certificate-based.
|
||||||
|
|
||||||
|
Do not attempt to log in from oldbox. Do not inspect private-key contents. Base the conclusion only on SSH policy, account state, and server logs.
|
||||||
|
|
||||||
|
## Required report format
|
||||||
|
|
||||||
|
Return exactly one report with these headings:
|
||||||
|
|
||||||
|
```text
|
||||||
|
=== IDENTITY ===
|
||||||
|
=== SSH SERVICE ===
|
||||||
|
=== SSH EFFECTIVE CONFIG ===
|
||||||
|
=== ACCOUNT STATE ===
|
||||||
|
=== AUTHENTICATION LOGS ===
|
||||||
|
=== CONCLUSION ===
|
||||||
|
=== NEXT DIAGNOSTIC ===
|
||||||
|
```
|
||||||
|
|
||||||
|
## Rules
|
||||||
|
|
||||||
|
- Redact sensitive values while retaining enough information to distinguish the authentication path.
|
||||||
|
- Never include secret values.
|
||||||
|
- Do not make changes.
|
||||||
|
- If a command is denied, report the command and permission error.
|
||||||
|
- If evidence is insufficient, say so rather than guessing.
|
||||||
|
- At the end, print:
|
||||||
|
|
||||||
|
```text
|
||||||
|
READY_FOR_MAC_AGENT
|
||||||
|
```
|
||||||
115
infra/oldbox-temporary-access-prompt.md
Normal file
115
infra/oldbox-temporary-access-prompt.md
Normal file
@@ -0,0 +1,115 @@
|
|||||||
|
# One-hour temporary UmbraTrans access setup
|
||||||
|
|
||||||
|
You are the OpenCode agent running on `oldbox`. The human operator has
|
||||||
|
authorized a one-hour temporary access window for finishing the UmbraTrans
|
||||||
|
workspace and communication setup.
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Create a temporary, unprivileged Linux account for Mac-side automation, using
|
||||||
|
a password entered locally in a hidden prompt. The Mac broker will use the
|
||||||
|
same one-hour credential for SSH. After one hour, the account and credential
|
||||||
|
must no longer work.
|
||||||
|
|
||||||
|
## Account
|
||||||
|
|
||||||
|
Use this exact temporary username:
|
||||||
|
|
||||||
|
```text
|
||||||
|
umbracontrol-temp
|
||||||
|
```
|
||||||
|
|
||||||
|
Create it with:
|
||||||
|
|
||||||
|
- home `/home/umbracontrol-temp`, mode `0700`
|
||||||
|
- shell `/bin/bash`
|
||||||
|
- no membership in `sudo`, `docker`, `mysql`, `lpadmin`, or other privileged
|
||||||
|
groups
|
||||||
|
- account expiry exactly one hour from creation
|
||||||
|
- a locally entered hidden password; never print, log, echo, or place it in a
|
||||||
|
file, command argument, environment variable, report, or MCP message
|
||||||
|
|
||||||
|
Use a local hidden prompt in the oldbox terminal. For example, adapt this
|
||||||
|
pattern safely in the agent's shell:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
read -rsp 'Temporary UmbraTrans password: ' TEMP_PASSWORD
|
||||||
|
printf '\n'
|
||||||
|
test -n "$TEMP_PASSWORD"
|
||||||
|
```
|
||||||
|
|
||||||
|
Pass the value to `chpasswd` through standard input, not an argument. Clear
|
||||||
|
the shell variable immediately afterward. Do not repeat the value in output.
|
||||||
|
|
||||||
|
Set the account password and expiration with `sudo` locally. The human may
|
||||||
|
enter the sudo password in the terminal; never request it in chat.
|
||||||
|
|
||||||
|
Schedule cleanup for one hour using a root-owned transient systemd timer or
|
||||||
|
an equivalent root-owned one-shot cleanup. Cleanup must:
|
||||||
|
|
||||||
|
- lock and delete `umbracontrol-temp`
|
||||||
|
- remove `/home/umbracontrol-temp`
|
||||||
|
- remove any temporary authorized-key or agent files
|
||||||
|
- leave `umbra`, Qwen, Docker, MySQL, Samba, Webmin, XRDP, and PostgreSQL
|
||||||
|
unchanged
|
||||||
|
|
||||||
|
Verify before reporting:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
id umbracontrol-temp
|
||||||
|
getent passwd umbracontrol-temp
|
||||||
|
passwd -S umbracontrol-temp
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not print hashes or secret values.
|
||||||
|
|
||||||
|
## Temporary Mac broker profile
|
||||||
|
|
||||||
|
After creating the account, write a **sanitized** report containing:
|
||||||
|
|
||||||
|
- username `umbracontrol-temp`
|
||||||
|
- host `oldbox.bdmurphy.me`
|
||||||
|
- port `22`
|
||||||
|
- account expiry timestamp
|
||||||
|
- cleanup mechanism/timer name
|
||||||
|
- home path and permissions
|
||||||
|
|
||||||
|
Do not include the temporary password in the report. The Mac operator will
|
||||||
|
enter the same one-hour password through the local secure popup.
|
||||||
|
|
||||||
|
## Workspace
|
||||||
|
|
||||||
|
Use the temporary account only for safe, unprivileged workspace operations:
|
||||||
|
|
||||||
|
- create `/home/umbracontrol-temp/UmbraTrans`
|
||||||
|
- report the path and ownership
|
||||||
|
- do not alter `/home/umbra` ownership or permissions
|
||||||
|
- do not modify existing services
|
||||||
|
- do not grant sudo
|
||||||
|
- do not install packages
|
||||||
|
- do not change firewall rules
|
||||||
|
|
||||||
|
Root-required application/database/backup work remains a separate explicit
|
||||||
|
operation after the temporary SSH path is verified.
|
||||||
|
|
||||||
|
## Sanitized diagnostic report
|
||||||
|
|
||||||
|
Send the report through the local `agent-comm` MCP server if it is already
|
||||||
|
available. Otherwise write it atomically to:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/tmp/umbracontrol-temp-setup-report.json
|
||||||
|
```
|
||||||
|
|
||||||
|
The report may contain only non-secret account, expiry, permission, and
|
||||||
|
service-status data. Use mode `0600`.
|
||||||
|
|
||||||
|
At the end print:
|
||||||
|
|
||||||
|
```text
|
||||||
|
TEMP_SETUP_READY
|
||||||
|
READY_FOR_MAC_AGENT
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not claim the Mac has connected until the Mac broker reports a successful
|
||||||
|
SSH status probe.
|
||||||
584
infra/postgres/001_control_plane.sql
Normal file
584
infra/postgres/001_control_plane.sql
Normal file
@@ -0,0 +1,584 @@
|
|||||||
|
BEGIN;
|
||||||
|
|
||||||
|
CREATE EXTENSION IF NOT EXISTS citext;
|
||||||
|
CREATE EXTENSION IF NOT EXISTS pgcrypto;
|
||||||
|
CREATE SCHEMA IF NOT EXISTS app;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||||
|
version text PRIMARY KEY,
|
||||||
|
applied_at timestamptz NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS organizations (
|
||||||
|
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
name text NOT NULL,
|
||||||
|
slug citext NOT NULL UNIQUE,
|
||||||
|
status text NOT NULL DEFAULT 'active'
|
||||||
|
CHECK (status IN ('active', 'suspended', 'archived')),
|
||||||
|
created_at timestamptz NOT NULL DEFAULT now(),
|
||||||
|
updated_at timestamptz NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS users (
|
||||||
|
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
email citext NOT NULL UNIQUE,
|
||||||
|
display_name text,
|
||||||
|
auth_provider text NOT NULL DEFAULT 'bootstrap',
|
||||||
|
auth_subject text,
|
||||||
|
status text NOT NULL DEFAULT 'invited'
|
||||||
|
CHECK (status IN ('invited', 'active', 'suspended', 'deactivated', 'locked')),
|
||||||
|
mfa_required boolean NOT NULL DEFAULT false,
|
||||||
|
last_login_at timestamptz,
|
||||||
|
created_at timestamptz NOT NULL DEFAULT now(),
|
||||||
|
updated_at timestamptz NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS users_auth_subject_idx
|
||||||
|
ON users (auth_provider, auth_subject)
|
||||||
|
WHERE auth_subject IS NOT NULL;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS projects (
|
||||||
|
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
organization_id uuid NOT NULL REFERENCES organizations(id),
|
||||||
|
name text NOT NULL,
|
||||||
|
slug citext NOT NULL,
|
||||||
|
status text NOT NULL DEFAULT 'active'
|
||||||
|
CHECK (status IN ('active', 'on_hold', 'released', 'archived')),
|
||||||
|
created_by uuid REFERENCES users(id),
|
||||||
|
created_at timestamptz NOT NULL DEFAULT now(),
|
||||||
|
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||||
|
UNIQUE (organization_id, slug),
|
||||||
|
UNIQUE (id, organization_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS roles (
|
||||||
|
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
name text NOT NULL,
|
||||||
|
scope text NOT NULL CHECK (scope IN ('platform', 'organization', 'project')),
|
||||||
|
description text NOT NULL DEFAULT '',
|
||||||
|
created_at timestamptz NOT NULL DEFAULT now(),
|
||||||
|
UNIQUE (scope, name)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS permissions (
|
||||||
|
key text PRIMARY KEY,
|
||||||
|
description text NOT NULL DEFAULT ''
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS role_permissions (
|
||||||
|
role_id uuid NOT NULL REFERENCES roles(id) ON DELETE CASCADE,
|
||||||
|
permission_key text NOT NULL REFERENCES permissions(key) ON DELETE CASCADE,
|
||||||
|
PRIMARY KEY (role_id, permission_key)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS user_roles (
|
||||||
|
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
role_id uuid NOT NULL REFERENCES roles(id) ON DELETE RESTRICT,
|
||||||
|
scope text NOT NULL CHECK (scope IN ('platform', 'organization', 'project')),
|
||||||
|
organization_id uuid REFERENCES organizations(id) ON DELETE CASCADE,
|
||||||
|
project_id uuid,
|
||||||
|
status text NOT NULL DEFAULT 'active'
|
||||||
|
CHECK (status IN ('active', 'suspended', 'revoked')),
|
||||||
|
created_at timestamptz NOT NULL DEFAULT now(),
|
||||||
|
created_by uuid REFERENCES users(id),
|
||||||
|
CHECK (
|
||||||
|
(scope = 'platform' AND organization_id IS NULL AND project_id IS NULL)
|
||||||
|
OR (scope = 'organization' AND organization_id IS NOT NULL AND project_id IS NULL)
|
||||||
|
OR (scope = 'project' AND organization_id IS NOT NULL AND project_id IS NOT NULL)
|
||||||
|
),
|
||||||
|
FOREIGN KEY (project_id, organization_id)
|
||||||
|
REFERENCES projects(id, organization_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS user_roles_platform_unique
|
||||||
|
ON user_roles (user_id, role_id)
|
||||||
|
WHERE scope = 'platform';
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS user_roles_organization_unique
|
||||||
|
ON user_roles (user_id, role_id, organization_id)
|
||||||
|
WHERE scope = 'organization';
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS user_roles_project_unique
|
||||||
|
ON user_roles (user_id, role_id, project_id)
|
||||||
|
WHERE scope = 'project';
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS invitations (
|
||||||
|
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
organization_id uuid NOT NULL REFERENCES organizations(id) ON DELETE CASCADE,
|
||||||
|
project_id uuid,
|
||||||
|
email citext NOT NULL,
|
||||||
|
role_id uuid NOT NULL REFERENCES roles(id) ON DELETE RESTRICT,
|
||||||
|
token_hash text NOT NULL UNIQUE,
|
||||||
|
invited_by uuid NOT NULL REFERENCES users(id),
|
||||||
|
expires_at timestamptz NOT NULL,
|
||||||
|
accepted_at timestamptz,
|
||||||
|
revoked_at timestamptz,
|
||||||
|
created_at timestamptz NOT NULL DEFAULT now(),
|
||||||
|
FOREIGN KEY (project_id, organization_id)
|
||||||
|
REFERENCES projects(id, organization_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS sessions (
|
||||||
|
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
token_hash text NOT NULL UNIQUE,
|
||||||
|
created_at timestamptz NOT NULL DEFAULT now(),
|
||||||
|
expires_at timestamptz NOT NULL,
|
||||||
|
revoked_at timestamptz,
|
||||||
|
ip_address inet,
|
||||||
|
user_agent text
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS mfa_methods (
|
||||||
|
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
method text NOT NULL CHECK (method IN ('totp', 'webauthn', 'recovery_code')),
|
||||||
|
secret_reference text,
|
||||||
|
verified_at timestamptz,
|
||||||
|
created_at timestamptz NOT NULL DEFAULT now(),
|
||||||
|
last_used_at timestamptz,
|
||||||
|
revoked_at timestamptz
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS bootstrap_tokens (
|
||||||
|
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
token_hash text NOT NULL UNIQUE,
|
||||||
|
expires_at timestamptz NOT NULL,
|
||||||
|
consumed_at timestamptz,
|
||||||
|
created_at timestamptz NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS audit_events (
|
||||||
|
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||||
|
actor_user_id uuid REFERENCES users(id),
|
||||||
|
organization_id uuid REFERENCES organizations(id),
|
||||||
|
project_id uuid,
|
||||||
|
action text NOT NULL,
|
||||||
|
target_type text,
|
||||||
|
target_id text,
|
||||||
|
request_id text,
|
||||||
|
ip_address inet,
|
||||||
|
metadata jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||||
|
created_at timestamptz NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS projects_organization_idx ON projects (organization_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS user_roles_user_idx ON user_roles (user_id, status);
|
||||||
|
CREATE INDEX IF NOT EXISTS user_roles_organization_idx ON user_roles (organization_id, status);
|
||||||
|
CREATE INDEX IF NOT EXISTS user_roles_project_idx ON user_roles (project_id, status);
|
||||||
|
CREATE INDEX IF NOT EXISTS invitations_email_idx ON invitations (email);
|
||||||
|
CREATE INDEX IF NOT EXISTS audit_events_org_created_idx ON audit_events (organization_id, created_at DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS audit_events_project_created_idx ON audit_events (project_id, created_at DESC);
|
||||||
|
|
||||||
|
CREATE OR REPLACE FUNCTION app.validate_user_role_scope()
|
||||||
|
RETURNS trigger
|
||||||
|
LANGUAGE plpgsql
|
||||||
|
AS $$
|
||||||
|
DECLARE
|
||||||
|
role_scope text;
|
||||||
|
BEGIN
|
||||||
|
SELECT scope INTO role_scope FROM roles WHERE id = NEW.role_id;
|
||||||
|
IF role_scope IS NULL OR role_scope <> NEW.scope THEN
|
||||||
|
RAISE EXCEPTION 'role scope does not match assignment scope';
|
||||||
|
END IF;
|
||||||
|
RETURN NEW;
|
||||||
|
END;
|
||||||
|
$$;
|
||||||
|
|
||||||
|
DROP TRIGGER IF EXISTS user_roles_validate_scope ON user_roles;
|
||||||
|
CREATE TRIGGER user_roles_validate_scope
|
||||||
|
BEFORE INSERT OR UPDATE OF role_id, scope ON user_roles
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION app.validate_user_role_scope();
|
||||||
|
|
||||||
|
CREATE OR REPLACE FUNCTION app.set_updated_at()
|
||||||
|
RETURNS trigger
|
||||||
|
LANGUAGE plpgsql
|
||||||
|
AS $$
|
||||||
|
BEGIN
|
||||||
|
NEW.updated_at = now();
|
||||||
|
RETURN NEW;
|
||||||
|
END;
|
||||||
|
$$;
|
||||||
|
|
||||||
|
DROP TRIGGER IF EXISTS organizations_set_updated_at ON organizations;
|
||||||
|
CREATE TRIGGER organizations_set_updated_at
|
||||||
|
BEFORE UPDATE ON organizations
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION app.set_updated_at();
|
||||||
|
|
||||||
|
DROP TRIGGER IF EXISTS users_set_updated_at ON users;
|
||||||
|
CREATE TRIGGER users_set_updated_at
|
||||||
|
BEFORE UPDATE ON users
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION app.set_updated_at();
|
||||||
|
|
||||||
|
DROP TRIGGER IF EXISTS projects_set_updated_at ON projects;
|
||||||
|
CREATE TRIGGER projects_set_updated_at
|
||||||
|
BEFORE UPDATE ON projects
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION app.set_updated_at();
|
||||||
|
|
||||||
|
CREATE OR REPLACE FUNCTION app.current_user_id()
|
||||||
|
RETURNS uuid
|
||||||
|
LANGUAGE sql
|
||||||
|
STABLE
|
||||||
|
AS $$
|
||||||
|
SELECT NULLIF(current_setting('app.user_id', true), '')::uuid
|
||||||
|
$$;
|
||||||
|
|
||||||
|
CREATE OR REPLACE FUNCTION app.is_platform_admin()
|
||||||
|
RETURNS boolean
|
||||||
|
LANGUAGE sql
|
||||||
|
STABLE
|
||||||
|
SECURITY DEFINER
|
||||||
|
SET search_path = public, pg_temp
|
||||||
|
AS $$
|
||||||
|
SELECT EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM user_roles ur
|
||||||
|
JOIN roles r ON r.id = ur.role_id
|
||||||
|
WHERE ur.user_id = app.current_user_id()
|
||||||
|
AND ur.scope = 'platform'
|
||||||
|
AND ur.status = 'active'
|
||||||
|
AND r.name = 'platform_admin'
|
||||||
|
)
|
||||||
|
$$;
|
||||||
|
|
||||||
|
CREATE OR REPLACE FUNCTION app.is_organization_member(target_organization_id uuid)
|
||||||
|
RETURNS boolean
|
||||||
|
LANGUAGE sql
|
||||||
|
STABLE
|
||||||
|
SECURITY DEFINER
|
||||||
|
SET search_path = public, pg_temp
|
||||||
|
AS $$
|
||||||
|
SELECT EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM user_roles ur
|
||||||
|
JOIN roles r ON r.id = ur.role_id
|
||||||
|
WHERE ur.user_id = app.current_user_id()
|
||||||
|
AND ur.organization_id = target_organization_id
|
||||||
|
AND ur.scope = 'organization'
|
||||||
|
AND ur.status = 'active'
|
||||||
|
AND r.name IN ('organization_admin', 'member')
|
||||||
|
)
|
||||||
|
$$;
|
||||||
|
|
||||||
|
CREATE OR REPLACE FUNCTION app.is_organization_admin(target_organization_id uuid)
|
||||||
|
RETURNS boolean
|
||||||
|
LANGUAGE sql
|
||||||
|
STABLE
|
||||||
|
SECURITY DEFINER
|
||||||
|
SET search_path = public, pg_temp
|
||||||
|
AS $$
|
||||||
|
SELECT EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM user_roles ur
|
||||||
|
JOIN roles r ON r.id = ur.role_id
|
||||||
|
WHERE ur.user_id = app.current_user_id()
|
||||||
|
AND ur.organization_id = target_organization_id
|
||||||
|
AND ur.scope = 'organization'
|
||||||
|
AND ur.status = 'active'
|
||||||
|
AND r.name = 'organization_admin'
|
||||||
|
)
|
||||||
|
$$;
|
||||||
|
|
||||||
|
CREATE OR REPLACE FUNCTION app.is_organization_admin_for_project(target_project_id uuid)
|
||||||
|
RETURNS boolean
|
||||||
|
LANGUAGE sql
|
||||||
|
STABLE
|
||||||
|
SECURITY DEFINER
|
||||||
|
SET search_path = public, pg_temp
|
||||||
|
AS $$
|
||||||
|
SELECT app.is_platform_admin()
|
||||||
|
OR EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM projects p
|
||||||
|
WHERE p.id = target_project_id
|
||||||
|
AND app.is_organization_admin(p.organization_id)
|
||||||
|
)
|
||||||
|
$$;
|
||||||
|
|
||||||
|
CREATE OR REPLACE FUNCTION app.can_access_project(target_project_id uuid)
|
||||||
|
RETURNS boolean
|
||||||
|
LANGUAGE sql
|
||||||
|
STABLE
|
||||||
|
SECURITY DEFINER
|
||||||
|
SET search_path = public, pg_temp
|
||||||
|
AS $$
|
||||||
|
SELECT app.is_platform_admin()
|
||||||
|
OR EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM projects p
|
||||||
|
WHERE p.id = target_project_id
|
||||||
|
AND app.is_organization_admin(p.organization_id)
|
||||||
|
)
|
||||||
|
OR EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM user_roles ur
|
||||||
|
WHERE ur.user_id = app.current_user_id()
|
||||||
|
AND ur.project_id = target_project_id
|
||||||
|
AND ur.scope = 'project'
|
||||||
|
AND ur.status = 'active'
|
||||||
|
)
|
||||||
|
$$;
|
||||||
|
|
||||||
|
CREATE OR REPLACE FUNCTION app.can_view_user(target_user_id uuid)
|
||||||
|
RETURNS boolean
|
||||||
|
LANGUAGE sql
|
||||||
|
STABLE
|
||||||
|
SECURITY DEFINER
|
||||||
|
SET search_path = public, pg_temp
|
||||||
|
AS $$
|
||||||
|
SELECT target_user_id = app.current_user_id()
|
||||||
|
OR app.is_platform_admin()
|
||||||
|
OR EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM user_roles mine
|
||||||
|
JOIN user_roles theirs
|
||||||
|
ON theirs.user_id = target_user_id
|
||||||
|
AND theirs.organization_id = mine.organization_id
|
||||||
|
AND theirs.scope = 'organization'
|
||||||
|
AND theirs.status = 'active'
|
||||||
|
JOIN roles r ON r.id = mine.role_id
|
||||||
|
WHERE mine.user_id = app.current_user_id()
|
||||||
|
AND mine.scope = 'organization'
|
||||||
|
AND mine.status = 'active'
|
||||||
|
AND r.name = 'organization_admin'
|
||||||
|
)
|
||||||
|
$$;
|
||||||
|
|
||||||
|
CREATE OR REPLACE FUNCTION app.prevent_audit_mutation()
|
||||||
|
RETURNS trigger
|
||||||
|
LANGUAGE plpgsql
|
||||||
|
AS $$
|
||||||
|
BEGIN
|
||||||
|
RAISE EXCEPTION 'audit_events is append-only';
|
||||||
|
END;
|
||||||
|
$$;
|
||||||
|
|
||||||
|
DROP TRIGGER IF EXISTS audit_events_append_only ON audit_events;
|
||||||
|
CREATE TRIGGER audit_events_append_only
|
||||||
|
BEFORE UPDATE OR DELETE ON audit_events
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION app.prevent_audit_mutation();
|
||||||
|
|
||||||
|
INSERT INTO schema_migrations(version) VALUES ('001_control_plane')
|
||||||
|
ON CONFLICT (version) DO NOTHING;
|
||||||
|
|
||||||
|
INSERT INTO roles (name, scope, description) VALUES
|
||||||
|
('platform_admin', 'platform', 'Global control-plane administration'),
|
||||||
|
('organization_admin', 'organization', 'Administration within one organization'),
|
||||||
|
('member', 'organization', 'Read-only organization membership'),
|
||||||
|
('project_owner', 'project', 'Project administration'),
|
||||||
|
('project_manager', 'project', 'Project production management'),
|
||||||
|
('translator', 'project', 'Translation work'),
|
||||||
|
('proofreader', 'project', 'Translation review'),
|
||||||
|
('audio_director', 'project', 'Voice and audio approval'),
|
||||||
|
('voice_manager', 'organization', 'Voice library and licensing management'),
|
||||||
|
('release_manager', 'project', 'Release approval'),
|
||||||
|
('viewer', 'project', 'Read-only project access')
|
||||||
|
ON CONFLICT (scope, name) DO UPDATE SET description = EXCLUDED.description;
|
||||||
|
|
||||||
|
INSERT INTO permissions (key, description) VALUES
|
||||||
|
('platform.manage', 'Manage platform-level resources'),
|
||||||
|
('organization.manage', 'Manage organization settings'),
|
||||||
|
('organization.users.manage', 'Invite and manage organization users'),
|
||||||
|
('organization.projects.manage', 'Create and manage projects'),
|
||||||
|
('project.access.manage', 'Assign project access'),
|
||||||
|
('project.source.edit', 'Edit approved source material'),
|
||||||
|
('project.translation.edit', 'Edit target translations'),
|
||||||
|
('project.translation.review', 'Review target translations'),
|
||||||
|
('project.audio.approve', 'Approve audio output'),
|
||||||
|
('project.release.approve', 'Approve releases'),
|
||||||
|
('voice.catalog.manage', 'Manage voice catalog and licensing'),
|
||||||
|
('artifact.download', 'Download authorized artifacts')
|
||||||
|
ON CONFLICT (key) DO UPDATE SET description = EXCLUDED.description;
|
||||||
|
|
||||||
|
INSERT INTO role_permissions (role_id, permission_key)
|
||||||
|
SELECT r.id, p.key
|
||||||
|
FROM roles r
|
||||||
|
JOIN (
|
||||||
|
VALUES
|
||||||
|
('platform_admin', 'platform.manage'),
|
||||||
|
('organization_admin', 'organization.manage'),
|
||||||
|
('organization_admin', 'organization.users.manage'),
|
||||||
|
('organization_admin', 'organization.projects.manage'),
|
||||||
|
('organization_admin', 'project.access.manage'),
|
||||||
|
('member', 'artifact.download'),
|
||||||
|
('project_owner', 'project.access.manage'),
|
||||||
|
('project_owner', 'project.release.approve'),
|
||||||
|
('project_manager', 'project.translation.edit'),
|
||||||
|
('project_manager', 'project.audio.approve'),
|
||||||
|
('translator', 'project.translation.edit'),
|
||||||
|
('proofreader', 'project.translation.review'),
|
||||||
|
('audio_director', 'project.audio.approve'),
|
||||||
|
('voice_manager', 'voice.catalog.manage'),
|
||||||
|
('release_manager', 'project.release.approve'),
|
||||||
|
('viewer', 'artifact.download')
|
||||||
|
) AS grants(role_name, permission_key)
|
||||||
|
ON grants.role_name = r.name
|
||||||
|
JOIN permissions p ON p.key = grants.permission_key
|
||||||
|
ON CONFLICT DO NOTHING;
|
||||||
|
|
||||||
|
ALTER TABLE roles ENABLE ROW LEVEL SECURITY;
|
||||||
|
ALTER TABLE permissions ENABLE ROW LEVEL SECURITY;
|
||||||
|
ALTER TABLE role_permissions ENABLE ROW LEVEL SECURITY;
|
||||||
|
|
||||||
|
DROP POLICY IF EXISTS roles_read ON roles;
|
||||||
|
CREATE POLICY roles_read ON roles
|
||||||
|
FOR SELECT USING (app.current_user_id() IS NOT NULL OR app.is_platform_admin());
|
||||||
|
DROP POLICY IF EXISTS roles_manage ON roles;
|
||||||
|
CREATE POLICY roles_manage ON roles
|
||||||
|
FOR ALL USING (app.is_platform_admin())
|
||||||
|
WITH CHECK (app.is_platform_admin());
|
||||||
|
|
||||||
|
DROP POLICY IF EXISTS permissions_read ON permissions;
|
||||||
|
CREATE POLICY permissions_read ON permissions
|
||||||
|
FOR SELECT USING (app.current_user_id() IS NOT NULL OR app.is_platform_admin());
|
||||||
|
DROP POLICY IF EXISTS permissions_manage ON permissions;
|
||||||
|
CREATE POLICY permissions_manage ON permissions
|
||||||
|
FOR ALL USING (app.is_platform_admin())
|
||||||
|
WITH CHECK (app.is_platform_admin());
|
||||||
|
|
||||||
|
DROP POLICY IF EXISTS role_permissions_read ON role_permissions;
|
||||||
|
CREATE POLICY role_permissions_read ON role_permissions
|
||||||
|
FOR SELECT USING (app.current_user_id() IS NOT NULL OR app.is_platform_admin());
|
||||||
|
DROP POLICY IF EXISTS role_permissions_manage ON role_permissions;
|
||||||
|
CREATE POLICY role_permissions_manage ON role_permissions
|
||||||
|
FOR ALL USING (app.is_platform_admin())
|
||||||
|
WITH CHECK (app.is_platform_admin());
|
||||||
|
|
||||||
|
ALTER TABLE organizations ENABLE ROW LEVEL SECURITY;
|
||||||
|
ALTER TABLE projects ENABLE ROW LEVEL SECURITY;
|
||||||
|
ALTER TABLE users ENABLE ROW LEVEL SECURITY;
|
||||||
|
ALTER TABLE user_roles ENABLE ROW LEVEL SECURITY;
|
||||||
|
ALTER TABLE invitations ENABLE ROW LEVEL SECURITY;
|
||||||
|
ALTER TABLE sessions ENABLE ROW LEVEL SECURITY;
|
||||||
|
ALTER TABLE mfa_methods ENABLE ROW LEVEL SECURITY;
|
||||||
|
ALTER TABLE bootstrap_tokens ENABLE ROW LEVEL SECURITY;
|
||||||
|
ALTER TABLE audit_events ENABLE ROW LEVEL SECURITY;
|
||||||
|
|
||||||
|
DROP POLICY IF EXISTS organizations_select ON organizations;
|
||||||
|
CREATE POLICY organizations_select ON organizations
|
||||||
|
FOR SELECT USING (app.is_platform_admin() OR app.is_organization_member(id));
|
||||||
|
DROP POLICY IF EXISTS organizations_insert ON organizations;
|
||||||
|
CREATE POLICY organizations_insert ON organizations
|
||||||
|
FOR INSERT WITH CHECK (app.is_platform_admin());
|
||||||
|
DROP POLICY IF EXISTS organizations_update ON organizations;
|
||||||
|
CREATE POLICY organizations_update ON organizations
|
||||||
|
FOR UPDATE USING (app.is_platform_admin() OR app.is_organization_admin(id))
|
||||||
|
WITH CHECK (app.is_platform_admin() OR app.is_organization_admin(id));
|
||||||
|
DROP POLICY IF EXISTS organizations_delete ON organizations;
|
||||||
|
CREATE POLICY organizations_delete ON organizations
|
||||||
|
FOR DELETE USING (app.is_platform_admin());
|
||||||
|
|
||||||
|
DROP POLICY IF EXISTS projects_select ON projects;
|
||||||
|
CREATE POLICY projects_select ON projects
|
||||||
|
FOR SELECT USING (app.can_access_project(id));
|
||||||
|
DROP POLICY IF EXISTS projects_insert ON projects;
|
||||||
|
CREATE POLICY projects_insert ON projects
|
||||||
|
FOR INSERT WITH CHECK (app.is_platform_admin() OR app.is_organization_admin(organization_id));
|
||||||
|
DROP POLICY IF EXISTS projects_update ON projects;
|
||||||
|
CREATE POLICY projects_update ON projects
|
||||||
|
FOR UPDATE USING (app.is_platform_admin() OR app.is_organization_admin(organization_id))
|
||||||
|
WITH CHECK (app.is_platform_admin() OR app.is_organization_admin(organization_id));
|
||||||
|
DROP POLICY IF EXISTS projects_delete ON projects;
|
||||||
|
CREATE POLICY projects_delete ON projects
|
||||||
|
FOR DELETE USING (app.is_platform_admin() OR app.is_organization_admin(organization_id));
|
||||||
|
|
||||||
|
DROP POLICY IF EXISTS users_select ON users;
|
||||||
|
CREATE POLICY users_select ON users
|
||||||
|
FOR SELECT USING (app.can_view_user(id));
|
||||||
|
DROP POLICY IF EXISTS users_manage ON users;
|
||||||
|
CREATE POLICY users_manage ON users
|
||||||
|
FOR ALL USING (app.is_platform_admin())
|
||||||
|
WITH CHECK (app.is_platform_admin());
|
||||||
|
|
||||||
|
DROP POLICY IF EXISTS user_roles_select ON user_roles;
|
||||||
|
CREATE POLICY user_roles_select ON user_roles
|
||||||
|
FOR SELECT USING (
|
||||||
|
app.is_platform_admin()
|
||||||
|
OR user_id = app.current_user_id()
|
||||||
|
OR (scope = 'organization' AND app.is_organization_admin(organization_id))
|
||||||
|
OR (scope = 'project' AND app.is_organization_admin_for_project(project_id))
|
||||||
|
);
|
||||||
|
DROP POLICY IF EXISTS user_roles_manage ON user_roles;
|
||||||
|
CREATE POLICY user_roles_manage ON user_roles
|
||||||
|
FOR ALL USING (
|
||||||
|
app.is_platform_admin()
|
||||||
|
OR (scope = 'organization' AND app.is_organization_admin(organization_id))
|
||||||
|
OR (scope = 'project' AND app.is_organization_admin_for_project(project_id))
|
||||||
|
)
|
||||||
|
WITH CHECK (
|
||||||
|
app.is_platform_admin()
|
||||||
|
OR (scope = 'organization' AND app.is_organization_admin(organization_id))
|
||||||
|
OR (scope = 'project' AND app.is_organization_admin_for_project(project_id))
|
||||||
|
);
|
||||||
|
|
||||||
|
DROP POLICY IF EXISTS invitations_select ON invitations;
|
||||||
|
CREATE POLICY invitations_select ON invitations
|
||||||
|
FOR SELECT USING (
|
||||||
|
app.is_platform_admin()
|
||||||
|
OR app.is_organization_admin(organization_id)
|
||||||
|
OR email = (SELECT email FROM users WHERE id = app.current_user_id())
|
||||||
|
);
|
||||||
|
DROP POLICY IF EXISTS invitations_manage ON invitations;
|
||||||
|
CREATE POLICY invitations_manage ON invitations
|
||||||
|
FOR ALL USING (app.is_platform_admin() OR app.is_organization_admin(organization_id))
|
||||||
|
WITH CHECK (app.is_platform_admin() OR app.is_organization_admin(organization_id));
|
||||||
|
|
||||||
|
DROP POLICY IF EXISTS sessions_self ON sessions;
|
||||||
|
CREATE POLICY sessions_self ON sessions
|
||||||
|
FOR ALL USING (user_id = app.current_user_id())
|
||||||
|
WITH CHECK (user_id = app.current_user_id());
|
||||||
|
|
||||||
|
DROP POLICY IF EXISTS mfa_methods_self ON mfa_methods;
|
||||||
|
CREATE POLICY mfa_methods_self ON mfa_methods
|
||||||
|
FOR ALL USING (user_id = app.current_user_id())
|
||||||
|
WITH CHECK (user_id = app.current_user_id());
|
||||||
|
|
||||||
|
DROP POLICY IF EXISTS bootstrap_tokens_self ON bootstrap_tokens;
|
||||||
|
CREATE POLICY bootstrap_tokens_self ON bootstrap_tokens
|
||||||
|
FOR ALL USING (user_id = app.current_user_id())
|
||||||
|
WITH CHECK (user_id = app.current_user_id());
|
||||||
|
|
||||||
|
DROP POLICY IF EXISTS audit_events_select ON audit_events;
|
||||||
|
CREATE POLICY audit_events_select ON audit_events
|
||||||
|
FOR SELECT USING (
|
||||||
|
app.is_platform_admin()
|
||||||
|
OR app.is_organization_admin(organization_id)
|
||||||
|
);
|
||||||
|
DROP POLICY IF EXISTS audit_events_insert ON audit_events;
|
||||||
|
CREATE POLICY audit_events_insert ON audit_events
|
||||||
|
FOR INSERT WITH CHECK (
|
||||||
|
actor_user_id = app.current_user_id()
|
||||||
|
OR app.is_platform_admin()
|
||||||
|
);
|
||||||
|
|
||||||
|
REVOKE ALL ON SCHEMA app FROM PUBLIC;
|
||||||
|
REVOKE CREATE ON SCHEMA public FROM PUBLIC;
|
||||||
|
|
||||||
|
GRANT USAGE ON SCHEMA public, app TO
|
||||||
|
umbracontrol_api, umbracontrol_worker, umbracontrol_readonly;
|
||||||
|
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO
|
||||||
|
umbracontrol_api, umbracontrol_worker, umbracontrol_readonly;
|
||||||
|
|
||||||
|
GRANT SELECT, INSERT, UPDATE, DELETE ON
|
||||||
|
organizations, projects, users, roles, permissions, role_permissions,
|
||||||
|
user_roles, invitations, sessions, mfa_methods, bootstrap_tokens, audit_events
|
||||||
|
TO umbracontrol_api;
|
||||||
|
|
||||||
|
GRANT SELECT ON
|
||||||
|
organizations, projects, users, roles, permissions, role_permissions,
|
||||||
|
user_roles, audit_events
|
||||||
|
TO umbracontrol_worker, umbracontrol_readonly;
|
||||||
|
|
||||||
|
GRANT INSERT ON audit_events TO umbracontrol_worker;
|
||||||
|
|
||||||
|
GRANT USAGE, CREATE ON SCHEMA public TO umbracontrol_migrator;
|
||||||
|
GRANT USAGE ON SCHEMA public TO umbracontrol_backup;
|
||||||
|
GRANT SELECT ON ALL TABLES IN SCHEMA public TO umbracontrol_backup;
|
||||||
|
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO umbracontrol_migrator;
|
||||||
|
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO umbracontrol_migrator;
|
||||||
|
|
||||||
|
REVOKE ALL ON ALL FUNCTIONS IN SCHEMA app FROM PUBLIC;
|
||||||
|
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA app TO
|
||||||
|
umbracontrol_api, umbracontrol_worker, umbracontrol_readonly;
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
106
infra/remote_credential_broker/README.md
Normal file
106
infra/remote_credential_broker/README.md
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
# UmbraTrans remote credential broker
|
||||||
|
|
||||||
|
This is a standalone local MCP server for the UmbraTrans control plane. It is
|
||||||
|
not imported by UmbraAudio and must be deployed separately on the trusted
|
||||||
|
macOS/OpenCode host.
|
||||||
|
|
||||||
|
## Security model
|
||||||
|
|
||||||
|
- Passwords are entered in a native macOS hidden-password dialog.
|
||||||
|
- Passwords are stored in the macOS login Keychain through the Security
|
||||||
|
framework; they are never written to the repository, environment variables,
|
||||||
|
command-line arguments, logs, or MCP responses.
|
||||||
|
- The MCP server exposes no `get_password` operation.
|
||||||
|
- SSH host-key checking is strict. The server does not silently accept an
|
||||||
|
unknown host key.
|
||||||
|
- Unprivileged SSH commands are bounded by a timeout and output limit.
|
||||||
|
- Sudo commands must exactly match an allowlist entry in the local config.
|
||||||
|
- The default configuration does not grant `sudo` access to arbitrary commands.
|
||||||
|
- The server is stdio-only and has no network listener.
|
||||||
|
|
||||||
|
The broker is intended to run on the Mac that owns the OpenCode session. The
|
||||||
|
remote `umbra` account receives an SSH connection; the password is supplied
|
||||||
|
only to the local SSH client/Keychain flow.
|
||||||
|
|
||||||
|
## Files
|
||||||
|
|
||||||
|
- `broker.py` — stdio MCP server and command-line popup setup.
|
||||||
|
- `keychain.py` — ctypes macOS Keychain backend.
|
||||||
|
- `popup.py` — native hidden-password prompt.
|
||||||
|
- `askpass_helper.py` — OpenSSH `SSH_ASKPASS` helper; it prints a secret only
|
||||||
|
to the SSH process pipe.
|
||||||
|
- `executor.py` — bounded SSH/sudo execution and redaction.
|
||||||
|
- `config.py` — non-secret remote profile and sudo allowlist parser.
|
||||||
|
- `config.example.json` — oldbox/umbra example; copy to the user config path.
|
||||||
|
- `tests/` — dependency-free unit tests.
|
||||||
|
|
||||||
|
## Local configuration
|
||||||
|
|
||||||
|
Copy the example to:
|
||||||
|
|
||||||
|
```text
|
||||||
|
~/.config/umbra-trans/remote-broker.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Review it. The `known_hosts` path must already contain the verified oldbox host
|
||||||
|
key. Do not use `StrictHostKeyChecking=no` or an empty known-hosts file.
|
||||||
|
|
||||||
|
The first credential profiles can be entered through the MCP tool
|
||||||
|
`credential_set`, which opens the popup, or directly from a terminal:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 -m remote_credential_broker.broker \
|
||||||
|
--credential-profile oldbox-umbra-ssh \
|
||||||
|
--purpose "SSH password for umbra@oldbox"
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 -m remote_credential_broker.broker \
|
||||||
|
--credential-profile oldbox-umbra-sudo \
|
||||||
|
--purpose "sudo password for umbra@oldbox"
|
||||||
|
```
|
||||||
|
|
||||||
|
The command prints only whether the profile was stored. It never prints the
|
||||||
|
secret.
|
||||||
|
|
||||||
|
## OpenCode registration
|
||||||
|
|
||||||
|
Run the module from the directory containing the `remote_credential_broker`
|
||||||
|
package, or use an absolute Python path and package directory. A local server
|
||||||
|
entry has this shape:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"mcp": {
|
||||||
|
"servers": {
|
||||||
|
"remote-broker": {
|
||||||
|
"type": "local",
|
||||||
|
"command": [
|
||||||
|
"/absolute/path/to/python",
|
||||||
|
"-m",
|
||||||
|
"remote_credential_broker.broker"
|
||||||
|
],
|
||||||
|
"environment": {
|
||||||
|
"PYTHONPATH": "/absolute/path/to/package/parent"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The current OpenCode installation may use the older top-level `mcp` object;
|
||||||
|
use the format accepted by the installed version. Do not put a password in the
|
||||||
|
configuration file.
|
||||||
|
|
||||||
|
## Tools
|
||||||
|
|
||||||
|
- `credential_set` — opens the native popup and stores a profile.
|
||||||
|
- `credential_status` — reports existence only.
|
||||||
|
- `credential_delete` — removes a profile.
|
||||||
|
- `remote_exec` — runs a command as the configured SSH account.
|
||||||
|
- `remote_sudo_exec` — runs one exact allowlisted command through sudo.
|
||||||
|
- `remote_status` — runs fixed identity/hostname probes.
|
||||||
|
|
||||||
|
The model can request a popup or operation, but cannot read the secret entered
|
||||||
|
in the popup.
|
||||||
7
infra/remote_credential_broker/__init__.py
Normal file
7
infra/remote_credential_broker/__init__.py
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
"""Local, stdio MCP broker for controlled remote operations.
|
||||||
|
|
||||||
|
The broker keeps passwords in the macOS Keychain and exposes only operations,
|
||||||
|
never secret values, to MCP clients.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__all__: list[str] = []
|
||||||
3
infra/remote_credential_broker/__main__.py
Normal file
3
infra/remote_credential_broker/__main__.py
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
from .broker import main
|
||||||
|
|
||||||
|
raise SystemExit(main())
|
||||||
29
infra/remote_credential_broker/askpass_helper.py
Executable file
29
infra/remote_credential_broker/askpass_helper.py
Executable file
@@ -0,0 +1,29 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
socket_path = os.environ.get("UMBRA_BROKER_ASKPASS_SOCKET", "")
|
||||||
|
if not socket_path:
|
||||||
|
print("credential broker: askpass socket is not configured", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
try:
|
||||||
|
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as channel:
|
||||||
|
channel.connect(socket_path)
|
||||||
|
secret = channel.recv(4096)
|
||||||
|
except OSError as exc:
|
||||||
|
print(f"credential broker: askpass channel failed: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
if not secret:
|
||||||
|
return 1
|
||||||
|
sys.stdout.buffer.write(secret)
|
||||||
|
sys.stdout.buffer.flush()
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
435
infra/remote_credential_broker/broker.py
Normal file
435
infra/remote_credential_broker/broker.py
Normal file
@@ -0,0 +1,435 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, TextIO
|
||||||
|
|
||||||
|
from .config import BrokerConfig, ConfigError
|
||||||
|
from .executor import ExecutionError, RemoteExecutor
|
||||||
|
from .keychain import (
|
||||||
|
CredentialError,
|
||||||
|
CredentialNotFound,
|
||||||
|
CredentialVault,
|
||||||
|
MacKeychain,
|
||||||
|
UnsupportedPlatformError,
|
||||||
|
validate_profile,
|
||||||
|
)
|
||||||
|
from .popup import PromptCancelled, SecretPrompt, macos_secret_prompt
|
||||||
|
|
||||||
|
_DEFAULT_CONFIG = Path("~/.config/umbra-trans/remote-broker.json").expanduser()
|
||||||
|
_DEFAULT_STATE = Path("~/.config/umbra-trans/remote-broker-state.json").expanduser()
|
||||||
|
_MAX_RESULT_CHARS = 1_000_000
|
||||||
|
|
||||||
|
|
||||||
|
class BrokerState:
|
||||||
|
"""Non-secret metadata about profiles that have been configured locally."""
|
||||||
|
|
||||||
|
def __init__(self, path: Path) -> None:
|
||||||
|
self.path = path
|
||||||
|
self._lock = threading.RLock()
|
||||||
|
|
||||||
|
def _read(self) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
raw = json.loads(self.path.read_text(encoding="utf-8"))
|
||||||
|
except FileNotFoundError:
|
||||||
|
return {"profiles": {}}
|
||||||
|
except (OSError, json.JSONDecodeError) as exc:
|
||||||
|
raise RuntimeError("Unable to read broker state") from exc
|
||||||
|
if not isinstance(raw, dict) or not isinstance(raw.get("profiles", {}), dict):
|
||||||
|
raise RuntimeError("Broker state is malformed")
|
||||||
|
return raw
|
||||||
|
|
||||||
|
def _write(self, value: dict[str, Any]) -> None:
|
||||||
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
fd, temporary_name = tempfile.mkstemp(prefix=".remote-broker-state.", dir=self.path.parent)
|
||||||
|
try:
|
||||||
|
os.fchmod(fd, 0o600)
|
||||||
|
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||||
|
json.dump(value, handle, indent=2, sort_keys=True)
|
||||||
|
handle.write("\n")
|
||||||
|
os.replace(temporary_name, self.path)
|
||||||
|
except BaseException:
|
||||||
|
try:
|
||||||
|
os.unlink(temporary_name)
|
||||||
|
except FileNotFoundError:
|
||||||
|
pass
|
||||||
|
raise
|
||||||
|
|
||||||
|
def record(self, profile: str, purpose: str, ttl_seconds: int | None = None) -> None:
|
||||||
|
with self._lock:
|
||||||
|
value = self._read()
|
||||||
|
metadata: dict[str, Any] = {
|
||||||
|
"purpose": purpose,
|
||||||
|
"updated_at": datetime.now(timezone.utc).isoformat(),
|
||||||
|
}
|
||||||
|
if ttl_seconds is not None:
|
||||||
|
metadata["expires_at"] = (
|
||||||
|
datetime.now(timezone.utc) + timedelta(seconds=int(ttl_seconds))
|
||||||
|
).isoformat()
|
||||||
|
value["profiles"][profile] = metadata
|
||||||
|
self._write(value)
|
||||||
|
|
||||||
|
def remove(self, profile: str) -> None:
|
||||||
|
with self._lock:
|
||||||
|
value = self._read()
|
||||||
|
value["profiles"].pop(profile, None)
|
||||||
|
self._write(value)
|
||||||
|
|
||||||
|
def profiles(self) -> dict[str, Any]:
|
||||||
|
with self._lock:
|
||||||
|
return dict(self._read().get("profiles", {}))
|
||||||
|
|
||||||
|
|
||||||
|
class RemoteCredentialBroker:
|
||||||
|
"""MCP-facing broker; secret values never cross the tool boundary."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
store: CredentialVault,
|
||||||
|
config: BrokerConfig,
|
||||||
|
state: BrokerState,
|
||||||
|
prompt: SecretPrompt,
|
||||||
|
executor: RemoteExecutor,
|
||||||
|
) -> None:
|
||||||
|
self.store = store
|
||||||
|
self.config = config
|
||||||
|
self.state = state
|
||||||
|
self.prompt = prompt
|
||||||
|
self.executor = executor
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _text_result(value: Any, *, is_error: bool = False) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"content": [{"type": "text", "text": json.dumps(value, ensure_ascii=False)}],
|
||||||
|
"isError": is_error,
|
||||||
|
}
|
||||||
|
|
||||||
|
def tool_definitions(self) -> list[dict[str, Any]]:
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"name": "credential_set",
|
||||||
|
"description": (
|
||||||
|
"Open a native macOS hidden-password dialog and store the entered secret "
|
||||||
|
"in the login Keychain. The secret is never returned to the model."
|
||||||
|
),
|
||||||
|
"inputSchema": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": False,
|
||||||
|
"required": ["profile", "purpose"],
|
||||||
|
"properties": {
|
||||||
|
"profile": {
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$",
|
||||||
|
"description": "Non-secret profile name, for example oldbox-umbra-ssh.",
|
||||||
|
},
|
||||||
|
"purpose": {
|
||||||
|
"type": "string",
|
||||||
|
"maxLength": 160,
|
||||||
|
"description": "Short label shown in the popup; never include a password.",
|
||||||
|
},
|
||||||
|
"ttl_seconds": {
|
||||||
|
"type": "integer",
|
||||||
|
"minimum": 1,
|
||||||
|
"maximum": 3600,
|
||||||
|
"description": "Optional in-memory lifetime; omit for persistent Keychain storage.",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "credential_status",
|
||||||
|
"description": "Report whether configured credential profiles exist; never reveal secrets.",
|
||||||
|
"inputSchema": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": False,
|
||||||
|
"properties": {"profile": {"type": "string"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "credential_delete",
|
||||||
|
"description": "Delete a stored credential profile from the local Keychain.",
|
||||||
|
"inputSchema": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": False,
|
||||||
|
"required": ["profile"],
|
||||||
|
"properties": {"profile": {"type": "string"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "remote_exec",
|
||||||
|
"description": (
|
||||||
|
"Run one bounded command as a configured SSH account. The command is a list, "
|
||||||
|
"not a shell string; output is redacted and size-limited."
|
||||||
|
),
|
||||||
|
"inputSchema": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": False,
|
||||||
|
"required": ["profile", "command"],
|
||||||
|
"properties": {
|
||||||
|
"profile": {"type": "string"},
|
||||||
|
"command": {
|
||||||
|
"type": "array",
|
||||||
|
"minItems": 1,
|
||||||
|
"items": {"type": "string"},
|
||||||
|
},
|
||||||
|
"timeout_seconds": {
|
||||||
|
"type": "integer",
|
||||||
|
"minimum": 5,
|
||||||
|
"maximum": 3600,
|
||||||
|
"default": 120,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "remote_sudo_exec",
|
||||||
|
"description": (
|
||||||
|
"Run one exact, configured allowlist command through remote sudo. "
|
||||||
|
"The password is read internally from Keychain and is never returned."
|
||||||
|
),
|
||||||
|
"inputSchema": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": False,
|
||||||
|
"required": ["profile", "command"],
|
||||||
|
"properties": {
|
||||||
|
"profile": {"type": "string"},
|
||||||
|
"command": {
|
||||||
|
"type": "array",
|
||||||
|
"minItems": 1,
|
||||||
|
"items": {"type": "string"},
|
||||||
|
},
|
||||||
|
"timeout_seconds": {
|
||||||
|
"type": "integer",
|
||||||
|
"minimum": 5,
|
||||||
|
"maximum": 3600,
|
||||||
|
"default": 120,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "remote_status",
|
||||||
|
"description": "Run a small fixed status probe against a configured SSH account.",
|
||||||
|
"inputSchema": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": False,
|
||||||
|
"required": ["profile"],
|
||||||
|
"properties": {"profile": {"type": "string"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
def call_tool(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
if name == "credential_set":
|
||||||
|
profile = validate_profile(str(arguments["profile"]))
|
||||||
|
purpose = str(arguments["purpose"]).strip()
|
||||||
|
if not purpose or len(purpose) > 160:
|
||||||
|
raise ExecutionError("purpose must be 1-160 characters")
|
||||||
|
ttl_value = arguments.get("ttl_seconds")
|
||||||
|
ttl_seconds = None if ttl_value is None else int(ttl_value)
|
||||||
|
if ttl_seconds is not None and not 1 <= ttl_seconds <= 3600:
|
||||||
|
raise ExecutionError("ttl_seconds must be between 1 and 3600")
|
||||||
|
secret = self.prompt(profile, purpose)
|
||||||
|
if secret is None:
|
||||||
|
raise PromptCancelled("Credential entry was cancelled")
|
||||||
|
self.store.set(profile, secret, ttl_seconds=ttl_seconds)
|
||||||
|
self.state.record(profile, purpose, ttl_seconds=ttl_seconds)
|
||||||
|
return self._text_result(
|
||||||
|
{
|
||||||
|
"profile": profile,
|
||||||
|
"stored": True,
|
||||||
|
"temporary": ttl_seconds is not None,
|
||||||
|
"ttl_seconds": ttl_seconds,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if name == "credential_status":
|
||||||
|
profile = arguments.get("profile")
|
||||||
|
if profile is not None:
|
||||||
|
profile = validate_profile(str(profile))
|
||||||
|
return self._text_result(
|
||||||
|
{"profile": profile, "configured": self.store.exists(profile)}
|
||||||
|
)
|
||||||
|
statuses = []
|
||||||
|
for name_, metadata in sorted(self.state.profiles().items()):
|
||||||
|
statuses.append(
|
||||||
|
{
|
||||||
|
"profile": name_,
|
||||||
|
"configured": self.store.exists(name_),
|
||||||
|
"purpose": metadata.get("purpose", ""),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return self._text_result({"profiles": statuses})
|
||||||
|
if name == "credential_delete":
|
||||||
|
profile = validate_profile(str(arguments["profile"]))
|
||||||
|
self.store.delete(profile)
|
||||||
|
self.state.remove(profile)
|
||||||
|
return self._text_result({"profile": profile, "deleted": True})
|
||||||
|
if name in {"remote_exec", "remote_sudo_exec"}:
|
||||||
|
profile = self.config.get(str(arguments["profile"]))
|
||||||
|
command = arguments.get("command")
|
||||||
|
if not isinstance(command, list):
|
||||||
|
raise ExecutionError("command must be an array of strings")
|
||||||
|
if not all(isinstance(part, str) and part for part in command):
|
||||||
|
raise ExecutionError("command must contain only non-empty strings")
|
||||||
|
command_tuple = tuple(command)
|
||||||
|
if name == "remote_sudo_exec" and not profile.sudo_allowed(command_tuple):
|
||||||
|
raise ExecutionError(
|
||||||
|
"The requested sudo command is not an exact allowlist entry for this profile"
|
||||||
|
)
|
||||||
|
timeout = int(arguments.get("timeout_seconds", 120))
|
||||||
|
result = self.executor.run(
|
||||||
|
profile,
|
||||||
|
command,
|
||||||
|
sudo=name == "remote_sudo_exec",
|
||||||
|
timeout_seconds=timeout,
|
||||||
|
)
|
||||||
|
output = {
|
||||||
|
"exit_code": result.exit_code,
|
||||||
|
"stdout": result.stdout[:_MAX_RESULT_CHARS],
|
||||||
|
"stderr": result.stderr[:_MAX_RESULT_CHARS],
|
||||||
|
"duration_ms": result.duration_ms,
|
||||||
|
}
|
||||||
|
return self._text_result(output, is_error=result.exit_code != 0)
|
||||||
|
if name == "remote_status":
|
||||||
|
profile = self.config.get(str(arguments["profile"]))
|
||||||
|
results = []
|
||||||
|
for command in (["/usr/bin/id", "-un"], ["/usr/bin/hostname"]):
|
||||||
|
result = self.executor.run(profile, command, timeout_seconds=30)
|
||||||
|
results.append(
|
||||||
|
{
|
||||||
|
"command": command,
|
||||||
|
"exit_code": result.exit_code,
|
||||||
|
"stdout": result.stdout.strip(),
|
||||||
|
"stderr": result.stderr.strip(),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return self._text_result({"profile": profile.name, "checks": results})
|
||||||
|
raise ExecutionError(f"Unknown tool: {name}")
|
||||||
|
except (CredentialError, ConfigError, ExecutionError, PromptCancelled, KeyError, ValueError) as exc:
|
||||||
|
return self._text_result({"error": str(exc)}, is_error=True)
|
||||||
|
|
||||||
|
def handle_json_rpc(self, message: dict[str, Any]) -> dict[str, Any] | None:
|
||||||
|
request_id = message.get("id")
|
||||||
|
method = message.get("method")
|
||||||
|
if method == "initialize":
|
||||||
|
params = message.get("params") or {}
|
||||||
|
requested_protocol = params.get("protocolVersion")
|
||||||
|
supported_protocols = {
|
||||||
|
"2024-11-05",
|
||||||
|
"2025-03-26",
|
||||||
|
"2025-06-18",
|
||||||
|
"2025-11-25",
|
||||||
|
}
|
||||||
|
protocol = (
|
||||||
|
requested_protocol
|
||||||
|
if isinstance(requested_protocol, str) and requested_protocol in supported_protocols
|
||||||
|
else "2024-11-05"
|
||||||
|
)
|
||||||
|
result: dict[str, Any] = {
|
||||||
|
"protocolVersion": protocol,
|
||||||
|
"capabilities": {"tools": {"listChanged": False}},
|
||||||
|
"serverInfo": {"name": "umbratrans-remote-broker", "version": "0.1.0"},
|
||||||
|
}
|
||||||
|
return self._response(request_id, result=result)
|
||||||
|
if method in {"notifications/initialized", "notifications/cancelled"}:
|
||||||
|
return None
|
||||||
|
if method == "ping":
|
||||||
|
return self._response(request_id, result={})
|
||||||
|
if method == "tools/list":
|
||||||
|
return self._response(request_id, result={"tools": self.tool_definitions()})
|
||||||
|
if method == "resources/list":
|
||||||
|
return self._response(request_id, result={"resources": []})
|
||||||
|
if method == "prompts/list":
|
||||||
|
return self._response(request_id, result={"prompts": []})
|
||||||
|
if method == "tools/call":
|
||||||
|
params = message.get("params") or {}
|
||||||
|
name = params.get("name")
|
||||||
|
arguments = params.get("arguments") or {}
|
||||||
|
if not isinstance(name, str) or not isinstance(arguments, dict):
|
||||||
|
return self._error(request_id, -32602, "Invalid tools/call parameters")
|
||||||
|
return self._response(request_id, result=self.call_tool(name, arguments))
|
||||||
|
if request_id is not None:
|
||||||
|
return self._error(request_id, -32601, f"Method not found: {method}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _response(request_id: Any, *, result: Any) -> dict[str, Any]:
|
||||||
|
return {"jsonrpc": "2.0", "id": request_id, "result": result}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _error(request_id: Any, code: int, message: str) -> dict[str, Any]:
|
||||||
|
return {"jsonrpc": "2.0", "id": request_id, "error": {"code": code, "message": message}}
|
||||||
|
|
||||||
|
def serve(self, input_stream: TextIO = sys.stdin, output_stream: TextIO = sys.stdout) -> None:
|
||||||
|
for line in input_stream:
|
||||||
|
if not line.strip():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
message = json.loads(line)
|
||||||
|
if not isinstance(message, dict):
|
||||||
|
raise ValueError("message must be a JSON object")
|
||||||
|
response = self.handle_json_rpc(message)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
response = self._error(None, -32700, "Parse error")
|
||||||
|
except Exception:
|
||||||
|
response = self._error(None, -32603, "Internal broker error")
|
||||||
|
if response is not None:
|
||||||
|
output_stream.write(json.dumps(response, ensure_ascii=False) + "\n")
|
||||||
|
output_stream.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def _load_runtime() -> tuple[CredentialVault, BrokerConfig, BrokerState, RemoteCredentialBroker]:
|
||||||
|
config_path = Path(os.environ.get("UMBRA_BROKER_CONFIG", str(_DEFAULT_CONFIG))).expanduser()
|
||||||
|
state_path = Path(os.environ.get("UMBRA_BROKER_STATE", str(_DEFAULT_STATE))).expanduser()
|
||||||
|
store = CredentialVault()
|
||||||
|
config = BrokerConfig.load(config_path)
|
||||||
|
state = BrokerState(state_path)
|
||||||
|
audit_path = Path(
|
||||||
|
os.environ.get("UMBRA_BROKER_AUDIT_LOG", "~/.local/state/umbratrans/remote-broker.log")
|
||||||
|
).expanduser()
|
||||||
|
executor = RemoteExecutor(store, audit_path=audit_path)
|
||||||
|
broker = RemoteCredentialBroker(store, config, state, macos_secret_prompt, executor)
|
||||||
|
return store, config, state, broker
|
||||||
|
|
||||||
|
|
||||||
|
def _cli_credential_set(profile: str, purpose: str) -> int:
|
||||||
|
store, _, state, _ = _load_runtime()
|
||||||
|
try:
|
||||||
|
secret = macos_secret_prompt(validate_profile(profile), purpose)
|
||||||
|
if secret is None:
|
||||||
|
return 2
|
||||||
|
store.set(profile, secret)
|
||||||
|
state.record(profile, purpose)
|
||||||
|
except (CredentialError, UnsupportedPlatformError, PromptCancelled, RuntimeError) as exc:
|
||||||
|
print(f"credential setup failed: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
print(f"stored credential profile {profile!r}; secret was not printed")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
parser = argparse.ArgumentParser(description="UmbraTrans local remote credential MCP broker")
|
||||||
|
parser.add_argument("--credential-profile", help="open the popup for one profile and exit")
|
||||||
|
parser.add_argument("--purpose", default="remote access", help="label shown in the credential popup")
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
if args.credential_profile:
|
||||||
|
return _cli_credential_set(args.credential_profile, args.purpose)
|
||||||
|
try:
|
||||||
|
_, _, _, broker = _load_runtime()
|
||||||
|
broker.serve()
|
||||||
|
except (ConfigError, CredentialError, UnsupportedPlatformError, RuntimeError) as exc:
|
||||||
|
print(f"remote broker startup failed: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
26
infra/remote_credential_broker/config.example.json
Normal file
26
infra/remote_credential_broker/config.example.json
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
{
|
||||||
|
"profiles": {
|
||||||
|
"oldbox-umbra": {
|
||||||
|
"host": "oldbox.bdmurphy.me",
|
||||||
|
"username": "umbra",
|
||||||
|
"port": 22,
|
||||||
|
"ssh_credential": "oldbox-umbra-ssh",
|
||||||
|
"sudo_credential": "oldbox-umbra-sudo",
|
||||||
|
"known_hosts": "~/.ssh/known_hosts",
|
||||||
|
"allowed_sudo_commands": [
|
||||||
|
[
|
||||||
|
"/home/umbra/umbra-enterprise-bootstrap/bootstrap_umbracontrol_database.sh",
|
||||||
|
"--apply",
|
||||||
|
"--config",
|
||||||
|
"/home/umbra/umbra-enterprise-bootstrap/enterprise-bootstrap.env"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"/home/umbra/umbra-enterprise-bootstrap/bootstrap_umbracontrol_database.sh",
|
||||||
|
"--verify",
|
||||||
|
"--config",
|
||||||
|
"/home/umbra/umbra-enterprise-bootstrap/enterprise-bootstrap.env"
|
||||||
|
]
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
95
infra/remote_credential_broker/config.py
Normal file
95
infra/remote_credential_broker/config.py
Normal file
@@ -0,0 +1,95 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
_PROFILE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
|
||||||
|
_HOST_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9.-]{0,252}$")
|
||||||
|
|
||||||
|
|
||||||
|
class ConfigError(ValueError):
|
||||||
|
"""Raised when the broker configuration is invalid."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RemoteProfile:
|
||||||
|
name: str
|
||||||
|
host: str
|
||||||
|
username: str
|
||||||
|
port: int = 22
|
||||||
|
ssh_credential: str | None = None
|
||||||
|
sudo_credential: str | None = None
|
||||||
|
known_hosts: str = "~/.ssh/known_hosts"
|
||||||
|
allowed_sudo_commands: tuple[tuple[str, ...], ...] = ()
|
||||||
|
|
||||||
|
def validate(self) -> None:
|
||||||
|
if not _PROFILE_RE.fullmatch(self.name):
|
||||||
|
raise ConfigError(f"Invalid remote profile name: {self.name!r}")
|
||||||
|
if not _HOST_RE.fullmatch(self.host):
|
||||||
|
raise ConfigError(f"Invalid host in profile {self.name!r}")
|
||||||
|
if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_.-]{0,31}", self.username):
|
||||||
|
raise ConfigError(f"Invalid SSH username in profile {self.name!r}")
|
||||||
|
if not 1 <= self.port <= 65535:
|
||||||
|
raise ConfigError(f"Invalid SSH port in profile {self.name!r}")
|
||||||
|
for credential in (self.ssh_credential, self.sudo_credential):
|
||||||
|
if credential is not None and not _PROFILE_RE.fullmatch(credential):
|
||||||
|
raise ConfigError(f"Invalid credential profile in {self.name!r}")
|
||||||
|
for command in self.allowed_sudo_commands:
|
||||||
|
if not command or any(not isinstance(part, str) or not part for part in command):
|
||||||
|
raise ConfigError(f"Invalid sudo command allowlist in profile {self.name!r}")
|
||||||
|
if any("\x00" in part for part in command):
|
||||||
|
raise ConfigError(f"NUL byte in sudo command allowlist in {self.name!r}")
|
||||||
|
|
||||||
|
def sudo_allowed(self, command: tuple[str, ...]) -> bool:
|
||||||
|
return command in self.allowed_sudo_commands
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class BrokerConfig:
|
||||||
|
profiles: dict[str, RemoteProfile]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def load(cls, path: Path) -> BrokerConfig:
|
||||||
|
if not path.exists():
|
||||||
|
return cls(profiles={})
|
||||||
|
try:
|
||||||
|
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, json.JSONDecodeError) as exc:
|
||||||
|
raise ConfigError(f"Unable to read broker config: {path}") from exc
|
||||||
|
if not isinstance(raw, dict) or not isinstance(raw.get("profiles", {}), dict):
|
||||||
|
raise ConfigError("Broker config must contain a profiles object")
|
||||||
|
profiles: dict[str, RemoteProfile] = {}
|
||||||
|
for name, item in raw.get("profiles", {}).items():
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
raise ConfigError(f"Profile {name!r} must be an object")
|
||||||
|
commands_raw = item.get("allowed_sudo_commands", [])
|
||||||
|
if not isinstance(commands_raw, list):
|
||||||
|
raise ConfigError(f"allowed_sudo_commands for {name!r} must be a list")
|
||||||
|
commands: list[tuple[str, ...]] = []
|
||||||
|
for command in commands_raw:
|
||||||
|
if not isinstance(command, list) or not all(isinstance(x, str) for x in command):
|
||||||
|
raise ConfigError(f"Invalid sudo command entry for {name!r}")
|
||||||
|
commands.append(tuple(command))
|
||||||
|
profile = RemoteProfile(
|
||||||
|
name=name,
|
||||||
|
host=str(item.get("host", "")),
|
||||||
|
username=str(item.get("username", "")),
|
||||||
|
port=int(item.get("port", 22)),
|
||||||
|
ssh_credential=item.get("ssh_credential"),
|
||||||
|
sudo_credential=item.get("sudo_credential"),
|
||||||
|
known_hosts=str(item.get("known_hosts", "~/.ssh/known_hosts")),
|
||||||
|
allowed_sudo_commands=tuple(commands),
|
||||||
|
)
|
||||||
|
profile.validate()
|
||||||
|
profiles[name] = profile
|
||||||
|
return cls(profiles=profiles)
|
||||||
|
|
||||||
|
def get(self, name: str) -> RemoteProfile:
|
||||||
|
try:
|
||||||
|
return self.profiles[name]
|
||||||
|
except KeyError as exc:
|
||||||
|
raise ConfigError(f"Unknown remote profile: {name!r}") from exc
|
||||||
242
infra/remote_credential_broker/executor.py
Normal file
242
infra/remote_credential_broker/executor.py
Normal file
@@ -0,0 +1,242 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import shlex
|
||||||
|
import shutil
|
||||||
|
import socket
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Callable, Sequence
|
||||||
|
|
||||||
|
from .config import ConfigError, RemoteProfile
|
||||||
|
from .keychain import CredentialError, CredentialVault
|
||||||
|
|
||||||
|
Runner = Callable[..., subprocess.CompletedProcess[bytes]]
|
||||||
|
|
||||||
|
|
||||||
|
class ExecutionError(RuntimeError):
|
||||||
|
"""Raised when a remote operation cannot be safely started."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ExecutionResult:
|
||||||
|
exit_code: int
|
||||||
|
stdout: str
|
||||||
|
stderr: str
|
||||||
|
duration_ms: int
|
||||||
|
|
||||||
|
|
||||||
|
class SecretRedactor:
|
||||||
|
"""Redact known secrets before output leaves the broker."""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self._secrets: set[str] = set()
|
||||||
|
|
||||||
|
def add(self, secret: str | None) -> None:
|
||||||
|
if secret:
|
||||||
|
self._secrets.add(secret)
|
||||||
|
|
||||||
|
def redact(self, value: str) -> str:
|
||||||
|
result = value
|
||||||
|
for secret in sorted(self._secrets, key=len, reverse=True):
|
||||||
|
if secret:
|
||||||
|
result = result.replace(secret, "<redacted>")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
class _AskpassServer:
|
||||||
|
"""One-shot local channel used instead of exposing a Keychain item to ssh."""
|
||||||
|
|
||||||
|
def __init__(self, secret: str) -> None:
|
||||||
|
self._directory = Path(tempfile.mkdtemp(prefix="remote-broker-askpass-"))
|
||||||
|
self._directory.chmod(0o700)
|
||||||
|
self.path = self._directory / "secret.sock"
|
||||||
|
self._secret = secret.encode("utf-8")
|
||||||
|
self._stop = threading.Event()
|
||||||
|
self._socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||||
|
self._socket.bind(str(self.path))
|
||||||
|
self._socket.listen(8)
|
||||||
|
self._socket.settimeout(0.2)
|
||||||
|
self._thread = threading.Thread(target=self._serve, name="remote-broker-askpass", daemon=True)
|
||||||
|
self._thread.start()
|
||||||
|
|
||||||
|
def _serve(self) -> None:
|
||||||
|
while not self._stop.is_set():
|
||||||
|
try:
|
||||||
|
connection, _ = self._socket.accept()
|
||||||
|
except socket.timeout:
|
||||||
|
continue
|
||||||
|
except OSError:
|
||||||
|
return
|
||||||
|
with connection:
|
||||||
|
try:
|
||||||
|
connection.sendall(self._secret)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
self._stop.set()
|
||||||
|
try:
|
||||||
|
self._socket.close()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
self._thread.join(timeout=1)
|
||||||
|
shutil.rmtree(self._directory, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
class RemoteExecutor:
|
||||||
|
"""Run bounded SSH operations; sudo commands are exact allowlist entries."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
store: CredentialVault,
|
||||||
|
*,
|
||||||
|
runner: Runner = subprocess.run,
|
||||||
|
askpass_path: Path | None = None,
|
||||||
|
audit_path: Path | None = None,
|
||||||
|
) -> None:
|
||||||
|
self._store = store
|
||||||
|
self._runner = runner
|
||||||
|
self._askpass_path = askpass_path or Path(__file__).with_name("askpass_helper.py")
|
||||||
|
if self._askpass_path.exists():
|
||||||
|
self._askpass_path.chmod(0o700)
|
||||||
|
self._audit_path = audit_path
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _validate_command(command: Sequence[str]) -> tuple[str, ...]:
|
||||||
|
if not command or not all(isinstance(part, str) and part for part in command):
|
||||||
|
raise ExecutionError("Remote command must be a non-empty list of strings")
|
||||||
|
if any("\x00" in part for part in command):
|
||||||
|
raise ExecutionError("Remote command contains a NUL byte")
|
||||||
|
return tuple(command)
|
||||||
|
|
||||||
|
def _audit(self, profile: RemoteProfile, command: tuple[str, ...], sudo: bool, result: int) -> None:
|
||||||
|
if self._audit_path is None:
|
||||||
|
return
|
||||||
|
self._audit_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
# Store command metadata only. Never store passwords or command stdin.
|
||||||
|
line = (
|
||||||
|
f"profile={profile.name} sudo={str(sudo).lower()} "
|
||||||
|
f"command={shlex.join(command)} exit={result}\n"
|
||||||
|
)
|
||||||
|
fd = os.open(self._audit_path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
|
||||||
|
try:
|
||||||
|
os.write(fd, line.encode("utf-8"))
|
||||||
|
finally:
|
||||||
|
os.close(fd)
|
||||||
|
|
||||||
|
def _base_ssh_command(self, profile: RemoteProfile, remote_command: str, use_password: bool) -> list[str]:
|
||||||
|
known_hosts = str(Path(profile.known_hosts).expanduser())
|
||||||
|
command = [
|
||||||
|
"ssh",
|
||||||
|
"-p",
|
||||||
|
str(profile.port),
|
||||||
|
"-o",
|
||||||
|
"ConnectTimeout=15",
|
||||||
|
"-o",
|
||||||
|
"ServerAliveInterval=15",
|
||||||
|
"-o",
|
||||||
|
"ServerAliveCountMax=2",
|
||||||
|
"-o",
|
||||||
|
"StrictHostKeyChecking=yes",
|
||||||
|
"-o",
|
||||||
|
f"UserKnownHostsFile={known_hosts}",
|
||||||
|
]
|
||||||
|
if use_password:
|
||||||
|
command.extend(
|
||||||
|
[
|
||||||
|
"-o",
|
||||||
|
"BatchMode=no",
|
||||||
|
"-o",
|
||||||
|
"PreferredAuthentications=publickey,password",
|
||||||
|
"-o",
|
||||||
|
"PubkeyAuthentication=yes",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
command.extend([f"{profile.username}@{profile.host}", remote_command])
|
||||||
|
return command
|
||||||
|
|
||||||
|
def run(
|
||||||
|
self,
|
||||||
|
profile: RemoteProfile,
|
||||||
|
command: Sequence[str],
|
||||||
|
*,
|
||||||
|
sudo: bool = False,
|
||||||
|
timeout_seconds: int = 120,
|
||||||
|
) -> ExecutionResult:
|
||||||
|
command_tuple = self._validate_command(command)
|
||||||
|
if not 5 <= timeout_seconds <= 3600:
|
||||||
|
raise ExecutionError("timeout_seconds must be between 5 and 3600")
|
||||||
|
if sudo and not profile.sudo_allowed(command_tuple):
|
||||||
|
raise ExecutionError(
|
||||||
|
"The requested sudo command is not an exact allowlist entry for this profile"
|
||||||
|
)
|
||||||
|
|
||||||
|
ssh_secret: str | None = None
|
||||||
|
sudo_secret: str | None = None
|
||||||
|
redactor = SecretRedactor()
|
||||||
|
if profile.ssh_credential:
|
||||||
|
try:
|
||||||
|
ssh_secret = self._store.get(profile.ssh_credential)
|
||||||
|
except CredentialError as exc:
|
||||||
|
raise ExecutionError(
|
||||||
|
f"SSH credential is not configured for profile {profile.name!r}"
|
||||||
|
) from exc
|
||||||
|
redactor.add(ssh_secret)
|
||||||
|
if sudo:
|
||||||
|
if not profile.sudo_credential:
|
||||||
|
raise ExecutionError(f"No sudo credential is configured for profile {profile.name!r}")
|
||||||
|
try:
|
||||||
|
sudo_secret = self._store.get(profile.sudo_credential)
|
||||||
|
except CredentialError as exc:
|
||||||
|
raise ExecutionError(
|
||||||
|
f"Sudo credential is not configured for profile {profile.name!r}"
|
||||||
|
) from exc
|
||||||
|
redactor.add(sudo_secret)
|
||||||
|
|
||||||
|
remote_argv = list(command_tuple)
|
||||||
|
if sudo:
|
||||||
|
remote_command = "sudo -S -p '' -- " + shlex.join(remote_argv)
|
||||||
|
else:
|
||||||
|
remote_command = shlex.join(remote_argv)
|
||||||
|
ssh_command = self._base_ssh_command(profile, remote_command, use_password=ssh_secret is not None)
|
||||||
|
environment = os.environ.copy()
|
||||||
|
askpass_server: _AskpassServer | None = None
|
||||||
|
if ssh_secret is not None:
|
||||||
|
askpass_server = _AskpassServer(ssh_secret)
|
||||||
|
environment.update(
|
||||||
|
{
|
||||||
|
"SSH_ASKPASS": str(self._askpass_path),
|
||||||
|
"SSH_ASKPASS_REQUIRE": "force",
|
||||||
|
"DISPLAY": environment.get("DISPLAY", ":0"),
|
||||||
|
"UMBRA_BROKER_ASKPASS_SOCKET": str(askpass_server.path),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
stdin_data = (sudo_secret + "\n").encode("utf-8") if sudo_secret is not None else None
|
||||||
|
started = time.monotonic()
|
||||||
|
try:
|
||||||
|
completed = self._runner(
|
||||||
|
ssh_command,
|
||||||
|
input=stdin_data,
|
||||||
|
capture_output=True,
|
||||||
|
timeout=timeout_seconds,
|
||||||
|
env=environment,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
except subprocess.TimeoutExpired as exc:
|
||||||
|
raise ExecutionError("Remote operation timed out") from exc
|
||||||
|
except OSError as exc:
|
||||||
|
raise ExecutionError("Unable to start ssh") from exc
|
||||||
|
finally:
|
||||||
|
if askpass_server is not None:
|
||||||
|
askpass_server.close()
|
||||||
|
duration_ms = int((time.monotonic() - started) * 1000)
|
||||||
|
stdout = redactor.redact(completed.stdout.decode("utf-8", errors="replace"))
|
||||||
|
stderr = redactor.redact(completed.stderr.decode("utf-8", errors="replace"))
|
||||||
|
result = ExecutionResult(completed.returncode, stdout, stderr, duration_ms)
|
||||||
|
self._audit(profile, command_tuple, sudo, result.exit_code)
|
||||||
|
return result
|
||||||
299
infra/remote_credential_broker/keychain.py
Normal file
299
infra/remote_credential_broker/keychain.py
Normal file
@@ -0,0 +1,299 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import ctypes
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from ctypes import POINTER, c_int32, c_uint32, c_void_p, create_string_buffer
|
||||||
|
from typing import Final
|
||||||
|
|
||||||
|
|
||||||
|
_PROFILE_RE: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
|
||||||
|
_SERVICE_DEFAULT: Final = "com.umbratrans.remote-broker"
|
||||||
|
_ERR_SEC_ITEM_NOT_FOUND: Final = -25300
|
||||||
|
|
||||||
|
|
||||||
|
class CredentialError(RuntimeError):
|
||||||
|
"""Base exception for credential-store failures."""
|
||||||
|
|
||||||
|
|
||||||
|
class CredentialNotFound(CredentialError):
|
||||||
|
"""Raised when a requested Keychain item does not exist."""
|
||||||
|
|
||||||
|
|
||||||
|
class UnsupportedPlatformError(CredentialError):
|
||||||
|
"""Raised when the broker is not running on macOS."""
|
||||||
|
|
||||||
|
|
||||||
|
def validate_profile(profile: str) -> str:
|
||||||
|
"""Validate a profile name before using it as a Keychain account name."""
|
||||||
|
if not isinstance(profile, str) or not _PROFILE_RE.fullmatch(profile):
|
||||||
|
raise CredentialError(
|
||||||
|
"Credential profile must contain only letters, digits, '.', '_' and '-' "
|
||||||
|
"and must be 1-128 characters"
|
||||||
|
)
|
||||||
|
return profile
|
||||||
|
|
||||||
|
|
||||||
|
class MacKeychain:
|
||||||
|
"""Small ctypes wrapper around the legacy macOS generic-password API.
|
||||||
|
|
||||||
|
Password bytes are passed directly to the Security framework and are never
|
||||||
|
placed in a subprocess argument, environment variable, log line, or MCP
|
||||||
|
response. The API is deprecated by Apple but remains available on the
|
||||||
|
supported macOS hosts and avoids adding a Python dependency.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, service: str = _SERVICE_DEFAULT) -> None:
|
||||||
|
self.service = service
|
||||||
|
self._lock = threading.RLock()
|
||||||
|
self._lib: ctypes.CDLL | None = None
|
||||||
|
self._cf: ctypes.CDLL | None = None
|
||||||
|
|
||||||
|
def _load(self) -> tuple[ctypes.CDLL, ctypes.CDLL]:
|
||||||
|
if sys.platform != "darwin":
|
||||||
|
raise UnsupportedPlatformError(
|
||||||
|
"The macOS Keychain backend requires macOS; use a native secret store on other hosts"
|
||||||
|
)
|
||||||
|
if self._lib is None:
|
||||||
|
lib = ctypes.CDLL("/System/Library/Frameworks/Security.framework/Security")
|
||||||
|
lib.SecKeychainAddGenericPassword.argtypes = [
|
||||||
|
c_void_p,
|
||||||
|
c_uint32,
|
||||||
|
c_void_p,
|
||||||
|
c_uint32,
|
||||||
|
c_void_p,
|
||||||
|
c_uint32,
|
||||||
|
c_void_p,
|
||||||
|
POINTER(c_void_p),
|
||||||
|
]
|
||||||
|
lib.SecKeychainAddGenericPassword.restype = c_int32
|
||||||
|
lib.SecKeychainFindGenericPassword.argtypes = [
|
||||||
|
c_void_p,
|
||||||
|
c_uint32,
|
||||||
|
c_void_p,
|
||||||
|
c_uint32,
|
||||||
|
c_void_p,
|
||||||
|
POINTER(c_uint32),
|
||||||
|
POINTER(c_void_p),
|
||||||
|
POINTER(c_void_p),
|
||||||
|
]
|
||||||
|
lib.SecKeychainFindGenericPassword.restype = c_int32
|
||||||
|
lib.SecKeychainItemModifyAttributesAndData.argtypes = [
|
||||||
|
c_void_p,
|
||||||
|
c_void_p,
|
||||||
|
c_uint32,
|
||||||
|
c_void_p,
|
||||||
|
]
|
||||||
|
lib.SecKeychainItemModifyAttributesAndData.restype = c_int32
|
||||||
|
lib.SecKeychainItemFreeContent.argtypes = [c_void_p, c_void_p]
|
||||||
|
lib.SecKeychainItemFreeContent.restype = c_int32
|
||||||
|
lib.SecKeychainItemDelete.argtypes = [c_void_p]
|
||||||
|
lib.SecKeychainItemDelete.restype = c_int32
|
||||||
|
self._lib = lib
|
||||||
|
if self._cf is None:
|
||||||
|
cf = ctypes.CDLL("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation")
|
||||||
|
cf.CFRelease.argtypes = [c_void_p]
|
||||||
|
cf.CFRelease.restype = None
|
||||||
|
self._cf = cf
|
||||||
|
return self._lib, self._cf
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _error(status: int, operation: str) -> CredentialError:
|
||||||
|
return CredentialError(f"Keychain {operation} failed with OSStatus {status}")
|
||||||
|
|
||||||
|
def _find(
|
||||||
|
self,
|
||||||
|
profile: str,
|
||||||
|
*,
|
||||||
|
include_password: bool,
|
||||||
|
) -> tuple[int, bytes | None, c_void_p | None]:
|
||||||
|
lib, _ = self._load()
|
||||||
|
profile = validate_profile(profile)
|
||||||
|
service_bytes = self.service.encode("utf-8")
|
||||||
|
account_bytes = profile.encode("utf-8")
|
||||||
|
service_buf = create_string_buffer(service_bytes)
|
||||||
|
account_buf = create_string_buffer(account_bytes)
|
||||||
|
length = c_uint32(0)
|
||||||
|
data = c_void_p()
|
||||||
|
item = c_void_p()
|
||||||
|
status = lib.SecKeychainFindGenericPassword(
|
||||||
|
None,
|
||||||
|
len(service_bytes),
|
||||||
|
service_buf,
|
||||||
|
len(account_bytes),
|
||||||
|
account_buf,
|
||||||
|
ctypes.byref(length) if include_password else None,
|
||||||
|
ctypes.byref(data) if include_password else None,
|
||||||
|
ctypes.byref(item),
|
||||||
|
)
|
||||||
|
if status != 0:
|
||||||
|
return status, None, None
|
||||||
|
if not include_password:
|
||||||
|
return status, None, item
|
||||||
|
try:
|
||||||
|
value = ctypes.string_at(data, length.value) if data.value else b""
|
||||||
|
return status, value, item
|
||||||
|
finally:
|
||||||
|
# The pointer is owned by the Security framework.
|
||||||
|
lib.SecKeychainItemFreeContent(None, data)
|
||||||
|
|
||||||
|
def _release(self, item: c_void_p | None) -> None:
|
||||||
|
if item is None or not item.value:
|
||||||
|
return
|
||||||
|
_, cf = self._load()
|
||||||
|
cf.CFRelease(item)
|
||||||
|
|
||||||
|
def set(self, profile: str, secret: str) -> None:
|
||||||
|
"""Store or replace a secret in the default login Keychain."""
|
||||||
|
if not isinstance(secret, str) or not secret:
|
||||||
|
raise CredentialError("Credential secret must not be empty")
|
||||||
|
with self._lock:
|
||||||
|
lib, _ = self._load()
|
||||||
|
profile = validate_profile(profile)
|
||||||
|
status, _, item = self._find(profile, include_password=False)
|
||||||
|
if status == 0 and item is not None:
|
||||||
|
secret_bytes = secret.encode("utf-8")
|
||||||
|
secret_buf = create_string_buffer(secret_bytes)
|
||||||
|
try:
|
||||||
|
update_status = lib.SecKeychainItemModifyAttributesAndData(
|
||||||
|
item,
|
||||||
|
None,
|
||||||
|
len(secret_bytes),
|
||||||
|
secret_buf,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
self._release(item)
|
||||||
|
if update_status != 0:
|
||||||
|
raise self._error(update_status, "update")
|
||||||
|
return
|
||||||
|
if status != _ERR_SEC_ITEM_NOT_FOUND:
|
||||||
|
raise self._error(status, "lookup")
|
||||||
|
|
||||||
|
service_bytes = self.service.encode("utf-8")
|
||||||
|
account_bytes = profile.encode("utf-8")
|
||||||
|
secret_bytes = secret.encode("utf-8")
|
||||||
|
service_buf = create_string_buffer(service_bytes)
|
||||||
|
account_buf = create_string_buffer(account_bytes)
|
||||||
|
secret_buf = create_string_buffer(secret_bytes)
|
||||||
|
new_item = c_void_p()
|
||||||
|
add_status = lib.SecKeychainAddGenericPassword(
|
||||||
|
None,
|
||||||
|
len(service_bytes),
|
||||||
|
service_buf,
|
||||||
|
len(account_bytes),
|
||||||
|
account_buf,
|
||||||
|
len(secret_bytes),
|
||||||
|
secret_buf,
|
||||||
|
ctypes.byref(new_item),
|
||||||
|
)
|
||||||
|
self._release(new_item)
|
||||||
|
if add_status != 0:
|
||||||
|
raise self._error(add_status, "add")
|
||||||
|
|
||||||
|
def get(self, profile: str) -> str:
|
||||||
|
"""Retrieve a secret for internal broker use only."""
|
||||||
|
with self._lock:
|
||||||
|
status, value, item = self._find(profile, include_password=True)
|
||||||
|
self._release(item)
|
||||||
|
if status == _ERR_SEC_ITEM_NOT_FOUND:
|
||||||
|
raise CredentialNotFound(f"No stored credential for profile {profile!r}")
|
||||||
|
if status != 0:
|
||||||
|
raise self._error(status, "lookup")
|
||||||
|
if value is None:
|
||||||
|
raise CredentialError("Keychain returned an empty credential")
|
||||||
|
return value.decode("utf-8")
|
||||||
|
|
||||||
|
def delete(self, profile: str) -> None:
|
||||||
|
"""Delete a stored secret; missing items are treated as already absent."""
|
||||||
|
with self._lock:
|
||||||
|
lib, _ = self._load()
|
||||||
|
status, _, item = self._find(profile, include_password=False)
|
||||||
|
if status == _ERR_SEC_ITEM_NOT_FOUND:
|
||||||
|
return
|
||||||
|
if status != 0:
|
||||||
|
raise self._error(status, "lookup")
|
||||||
|
if item is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
delete_status = lib.SecKeychainItemDelete(item)
|
||||||
|
finally:
|
||||||
|
self._release(item)
|
||||||
|
if delete_status != 0:
|
||||||
|
raise self._error(delete_status, "delete")
|
||||||
|
|
||||||
|
def exists(self, profile: str) -> bool:
|
||||||
|
"""Return whether a profile exists without retrieving its secret."""
|
||||||
|
with self._lock:
|
||||||
|
status, _, item = self._find(profile, include_password=False)
|
||||||
|
self._release(item)
|
||||||
|
if status == 0:
|
||||||
|
return True
|
||||||
|
if status == _ERR_SEC_ITEM_NOT_FOUND:
|
||||||
|
return False
|
||||||
|
raise self._error(status, "lookup")
|
||||||
|
|
||||||
|
|
||||||
|
class CredentialVault:
|
||||||
|
"""Persistent Keychain storage plus explicitly temporary in-memory profiles."""
|
||||||
|
|
||||||
|
def __init__(self, persistent: MacKeychain | None = None) -> None:
|
||||||
|
self.persistent = persistent or MacKeychain()
|
||||||
|
self.service = self.persistent.service
|
||||||
|
self._lock = threading.RLock()
|
||||||
|
self._temporary: dict[str, tuple[str, float]] = {}
|
||||||
|
self._timers: dict[str, threading.Timer] = {}
|
||||||
|
|
||||||
|
def _expire(self, profile: str) -> None:
|
||||||
|
with self._lock:
|
||||||
|
self._temporary.pop(profile, None)
|
||||||
|
self._timers.pop(profile, None)
|
||||||
|
|
||||||
|
def set(self, profile: str, secret: str, ttl_seconds: int | None = None) -> None:
|
||||||
|
profile = validate_profile(profile)
|
||||||
|
if not isinstance(secret, str) or not secret:
|
||||||
|
raise CredentialError("Credential secret must not be empty")
|
||||||
|
with self._lock:
|
||||||
|
old_timer = self._timers.pop(profile, None)
|
||||||
|
if old_timer is not None:
|
||||||
|
old_timer.cancel()
|
||||||
|
if ttl_seconds is None:
|
||||||
|
self._temporary.pop(profile, None)
|
||||||
|
self.persistent.set(profile, secret)
|
||||||
|
return
|
||||||
|
if not 1 <= int(ttl_seconds) <= 3600:
|
||||||
|
raise CredentialError("ttl_seconds must be between 1 and 3600")
|
||||||
|
expires_at = time.monotonic() + int(ttl_seconds)
|
||||||
|
self._temporary[profile] = (secret, expires_at)
|
||||||
|
timer = threading.Timer(int(ttl_seconds), self._expire, args=(profile,))
|
||||||
|
timer.daemon = True
|
||||||
|
self._timers[profile] = timer
|
||||||
|
timer.start()
|
||||||
|
|
||||||
|
def get(self, profile: str) -> str:
|
||||||
|
profile = validate_profile(profile)
|
||||||
|
with self._lock:
|
||||||
|
temporary = self._temporary.get(profile)
|
||||||
|
if temporary is not None:
|
||||||
|
secret, expires_at = temporary
|
||||||
|
if expires_at > time.monotonic():
|
||||||
|
return secret
|
||||||
|
self._expire(profile)
|
||||||
|
return self.persistent.get(profile)
|
||||||
|
|
||||||
|
def delete(self, profile: str) -> None:
|
||||||
|
profile = validate_profile(profile)
|
||||||
|
with self._lock:
|
||||||
|
self._expire(profile)
|
||||||
|
self.persistent.delete(profile)
|
||||||
|
|
||||||
|
def exists(self, profile: str) -> bool:
|
||||||
|
profile = validate_profile(profile)
|
||||||
|
with self._lock:
|
||||||
|
temporary = self._temporary.get(profile)
|
||||||
|
if temporary is not None:
|
||||||
|
if temporary[1] > time.monotonic():
|
||||||
|
return True
|
||||||
|
self._expire(profile)
|
||||||
|
return self.persistent.exists(profile)
|
||||||
49
infra/remote_credential_broker/popup.py
Normal file
49
infra/remote_credential_broker/popup.py
Normal file
@@ -0,0 +1,49 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import subprocess
|
||||||
|
from typing import Protocol
|
||||||
|
|
||||||
|
|
||||||
|
class PromptCancelled(RuntimeError):
|
||||||
|
"""Raised when the user cancels a native credential prompt."""
|
||||||
|
|
||||||
|
|
||||||
|
class SecretPrompt(Protocol):
|
||||||
|
def __call__(self, profile: str, purpose: str) -> str | None:
|
||||||
|
"""Return a secret from a local UI, or None when cancelled."""
|
||||||
|
|
||||||
|
|
||||||
|
def _apple_string(value: str) -> str:
|
||||||
|
return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
||||||
|
|
||||||
|
|
||||||
|
def macos_secret_prompt(profile: str, purpose: str) -> str | None:
|
||||||
|
"""Open a native macOS hidden-password dialog.
|
||||||
|
|
||||||
|
The returned value is intended to be passed directly to the Keychain
|
||||||
|
backend. It is never printed, logged, or included in an MCP result.
|
||||||
|
"""
|
||||||
|
message = f"Enter the password for {purpose} ({profile})."
|
||||||
|
script = (
|
||||||
|
f'display dialog {_apple_string(message)} '
|
||||||
|
'default answer "" with hidden answer buttons {"Cancel", "Save"} '
|
||||||
|
'default button "Save" with title "UmbraTrans secure credential"'
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
["/usr/bin/osascript", "-e", script],
|
||||||
|
check=True,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=120,
|
||||||
|
)
|
||||||
|
except subprocess.CalledProcessError as exc:
|
||||||
|
detail = exc.stderr.strip().lower()
|
||||||
|
if "user canceled" in detail or "-128" in detail:
|
||||||
|
raise PromptCancelled("Credential entry was cancelled") from exc
|
||||||
|
raise RuntimeError("The macOS credential dialog could not be completed") from exc
|
||||||
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||||
|
raise RuntimeError("The macOS credential dialog could not be opened") from exc
|
||||||
|
|
||||||
|
secret = result.stdout.rstrip("\r\n")
|
||||||
|
return secret or None
|
||||||
151
infra/remote_credential_broker/tests/test_broker.py
Normal file
151
infra/remote_credential_broker/tests/test_broker.py
Normal file
@@ -0,0 +1,151 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||||
|
|
||||||
|
from remote_credential_broker.broker import BrokerState, RemoteCredentialBroker
|
||||||
|
from remote_credential_broker.config import BrokerConfig, RemoteProfile
|
||||||
|
from remote_credential_broker.executor import ExecutionResult, SecretRedactor
|
||||||
|
from remote_credential_broker.keychain import CredentialError, CredentialVault, validate_profile
|
||||||
|
|
||||||
|
|
||||||
|
class FakeStore:
|
||||||
|
service = "test-service"
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.values: dict[str, str] = {}
|
||||||
|
|
||||||
|
def set(self, profile: str, secret: str, ttl_seconds: int | None = None) -> None:
|
||||||
|
self.values[profile] = secret
|
||||||
|
|
||||||
|
def get(self, profile: str) -> str:
|
||||||
|
return self.values[profile]
|
||||||
|
|
||||||
|
def delete(self, profile: str) -> None:
|
||||||
|
self.values.pop(profile, None)
|
||||||
|
|
||||||
|
def exists(self, profile: str) -> bool:
|
||||||
|
return profile in self.values
|
||||||
|
|
||||||
|
|
||||||
|
class FakeExecutor:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.calls: list[tuple[str, tuple[str, ...], bool]] = []
|
||||||
|
|
||||||
|
def run(
|
||||||
|
self,
|
||||||
|
profile: RemoteProfile,
|
||||||
|
command: list[str] | tuple[str, ...],
|
||||||
|
*,
|
||||||
|
sudo: bool = False,
|
||||||
|
timeout_seconds: int = 120,
|
||||||
|
) -> ExecutionResult:
|
||||||
|
self.calls.append((profile.name, tuple(command), sudo))
|
||||||
|
return ExecutionResult(0, "ok\n", "", 1)
|
||||||
|
|
||||||
|
|
||||||
|
class BrokerTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.store = FakeStore()
|
||||||
|
self.state_path = Path("/tmp/remote-broker-test-state.json")
|
||||||
|
self.state_path.unlink(missing_ok=True)
|
||||||
|
self.state = BrokerState(self.state_path)
|
||||||
|
self.executor = FakeExecutor()
|
||||||
|
self.profile = RemoteProfile(
|
||||||
|
name="oldbox-umbra",
|
||||||
|
host="oldbox.bdmurphy.me",
|
||||||
|
username="umbra",
|
||||||
|
ssh_credential="oldbox-umbra-ssh",
|
||||||
|
sudo_credential="oldbox-umbra-sudo",
|
||||||
|
allowed_sudo_commands=(("/usr/local/sbin/umbracontrol-backup",),),
|
||||||
|
)
|
||||||
|
config = BrokerConfig(profiles={self.profile.name: self.profile})
|
||||||
|
self.broker = RemoteCredentialBroker(
|
||||||
|
self.store,
|
||||||
|
config,
|
||||||
|
self.state,
|
||||||
|
lambda profile, purpose: "popup-secret",
|
||||||
|
self.executor,
|
||||||
|
)
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.state_path.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
def test_credential_popup_is_not_returned(self) -> None:
|
||||||
|
result = self.broker.call_tool(
|
||||||
|
"credential_set",
|
||||||
|
{"profile": "oldbox-umbra-ssh", "purpose": "SSH for oldbox"},
|
||||||
|
)
|
||||||
|
encoded = json.dumps(result)
|
||||||
|
self.assertNotIn("popup-secret", encoded)
|
||||||
|
payload = json.loads(result["content"][0]["text"])
|
||||||
|
self.assertTrue(payload["stored"])
|
||||||
|
self.assertEqual(self.store.get("oldbox-umbra-ssh"), "popup-secret")
|
||||||
|
|
||||||
|
def test_status_never_returns_secret(self) -> None:
|
||||||
|
self.broker.call_tool(
|
||||||
|
"credential_set",
|
||||||
|
{"profile": "oldbox-umbra-ssh", "purpose": "SSH for oldbox"},
|
||||||
|
)
|
||||||
|
result = self.broker.call_tool("credential_status", {"profile": "oldbox-umbra-ssh"})
|
||||||
|
self.assertNotIn("popup-secret", json.dumps(result))
|
||||||
|
payload = json.loads(result["content"][0]["text"])
|
||||||
|
self.assertTrue(payload["configured"])
|
||||||
|
|
||||||
|
def test_sudo_requires_exact_allowlist(self) -> None:
|
||||||
|
result = self.broker.call_tool(
|
||||||
|
"remote_sudo_exec",
|
||||||
|
{"profile": "oldbox-umbra", "command": ["/bin/sh", "-c", "id"]},
|
||||||
|
)
|
||||||
|
self.assertTrue(result["isError"])
|
||||||
|
self.assertIn("allowlist", result["content"][0]["text"])
|
||||||
|
self.assertEqual(self.executor.calls, [])
|
||||||
|
|
||||||
|
def test_allowed_sudo_is_forwarded(self) -> None:
|
||||||
|
result = self.broker.call_tool(
|
||||||
|
"remote_sudo_exec",
|
||||||
|
{"profile": "oldbox-umbra", "command": ["/usr/local/sbin/umbracontrol-backup"]},
|
||||||
|
)
|
||||||
|
self.assertFalse(result["isError"])
|
||||||
|
self.assertEqual(self.executor.calls[0][2], True)
|
||||||
|
|
||||||
|
def test_json_rpc_tools_list_and_initialize(self) -> None:
|
||||||
|
response = self.broker.handle_json_rpc({"jsonrpc": "2.0", "id": 1, "method": "initialize"})
|
||||||
|
assert response is not None
|
||||||
|
self.assertEqual(response["result"]["serverInfo"]["name"], "umbratrans-remote-broker")
|
||||||
|
response = self.broker.handle_json_rpc({"jsonrpc": "2.0", "id": 2, "method": "tools/list"})
|
||||||
|
assert response is not None
|
||||||
|
names = {tool["name"] for tool in response["result"]["tools"]}
|
||||||
|
self.assertIn("credential_set", names)
|
||||||
|
self.assertIn("remote_sudo_exec", names)
|
||||||
|
|
||||||
|
def test_redactor(self) -> None:
|
||||||
|
redactor = SecretRedactor()
|
||||||
|
redactor.add("secret-value")
|
||||||
|
self.assertEqual(redactor.redact("x secret-value y"), "x <redacted> y")
|
||||||
|
def test_temporary_profile_is_in_memory_and_readable(self) -> None:
|
||||||
|
persistent = FakeStore()
|
||||||
|
vault = CredentialVault(persistent)
|
||||||
|
vault.set("temporary-profile", "temporary-value", ttl_seconds=60)
|
||||||
|
self.assertEqual(vault.get("temporary-profile"), "temporary-value")
|
||||||
|
self.assertTrue(vault.exists("temporary-profile"))
|
||||||
|
self.assertNotIn("temporary-profile", persistent.values)
|
||||||
|
vault.delete("temporary-profile")
|
||||||
|
self.assertFalse(vault.exists("temporary-profile"))
|
||||||
|
|
||||||
|
|
||||||
|
class ValidationTests(unittest.TestCase):
|
||||||
|
def test_profile_rejects_shell_metacharacters(self) -> None:
|
||||||
|
with self.assertRaises(CredentialError):
|
||||||
|
validate_profile("bad profile;rm")
|
||||||
|
|
||||||
|
def test_profile_accepts_normal_name(self) -> None:
|
||||||
|
self.assertEqual(validate_profile("oldbox-umbra_ssh.1"), "oldbox-umbra_ssh.1")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
2053
specs/18-UmbraTrans.md
Normal file
2053
specs/18-UmbraTrans.md
Normal file
File diff suppressed because it is too large
Load Diff
1121
specs/19-enterprise-control-plane.md
Normal file
1121
specs/19-enterprise-control-plane.md
Normal file
File diff suppressed because it is too large
Load Diff
@@ -14,6 +14,8 @@ if TYPE_CHECKING:
|
|||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
from umbraaudio.voices.lmstudio_gate import queued_chat_completion
|
||||||
|
|
||||||
PROMPT_PATH = Path(__file__).parent / "CHARACTER_DISCOVERY_PROMPT.md"
|
PROMPT_PATH = Path(__file__).parent / "CHARACTER_DISCOVERY_PROMPT.md"
|
||||||
SCENE_BREAK = "<<<>>>"
|
SCENE_BREAK = "<<<>>>"
|
||||||
|
|
||||||
@@ -295,7 +297,8 @@ class CharacterDiscoverer:
|
|||||||
response = None
|
response = None
|
||||||
for attempt in range(max_attempts):
|
for attempt in range(max_attempts):
|
||||||
try:
|
try:
|
||||||
response = client.chat.completions.create(
|
response = queued_chat_completion(
|
||||||
|
client,
|
||||||
model=model,
|
model=model,
|
||||||
messages=[{"role": "user", "content": prompt}],
|
messages=[{"role": "user", "content": prompt}],
|
||||||
temperature=0.2,
|
temperature=0.2,
|
||||||
@@ -997,7 +1000,8 @@ Return ONLY valid JSON."""
|
|||||||
model = self._active_model or self.config.openai_model
|
model = self._active_model or self.config.openai_model
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = client.chat.completions.create(
|
response = queued_chat_completion(
|
||||||
|
client,
|
||||||
model=model,
|
model=model,
|
||||||
messages=[{"role": "user", "content": prompt}],
|
messages=[{"role": "user", "content": prompt}],
|
||||||
temperature=0.1,
|
temperature=0.1,
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ from umbraaudio.analysis import (
|
|||||||
CharacterProfile,
|
CharacterProfile,
|
||||||
)
|
)
|
||||||
from umbraaudio.config import LLMConfig
|
from umbraaudio.config import LLMConfig
|
||||||
|
from umbraaudio.voices.lmstudio_gate import queued_chat_completion
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -140,7 +141,8 @@ class CharacterProfiler:
|
|||||||
f"dimensions while staying consistent with the text evidence."
|
f"dimensions while staying consistent with the text evidence."
|
||||||
)
|
)
|
||||||
|
|
||||||
response = client.chat.completions.create(
|
response = queued_chat_completion(
|
||||||
|
client,
|
||||||
model=self.config.openai_model,
|
model=self.config.openai_model,
|
||||||
messages=[{"role": "user", "content": prompt}],
|
messages=[{"role": "user", "content": prompt}],
|
||||||
temperature=self.config.temperature,
|
temperature=self.config.temperature,
|
||||||
@@ -234,7 +236,8 @@ class CharacterProfiler:
|
|||||||
prompt=prompt,
|
prompt=prompt,
|
||||||
)
|
)
|
||||||
|
|
||||||
response = client.chat.completions.create(
|
response = queued_chat_completion(
|
||||||
|
client,
|
||||||
model=self.config.openai_model,
|
model=self.config.openai_model,
|
||||||
messages=[{"role": "user", "content": llm_prompt}],
|
messages=[{"role": "user", "content": llm_prompt}],
|
||||||
temperature=self.config.temperature,
|
temperature=self.config.temperature,
|
||||||
@@ -285,7 +288,8 @@ Example: {{"Eilean": "eɪ.lən", "Fjord": "fjɔːrd"}}
|
|||||||
|
|
||||||
Return ONLY valid JSON."""
|
Return ONLY valid JSON."""
|
||||||
|
|
||||||
response = client.chat.completions.create(
|
response = queued_chat_completion(
|
||||||
|
client,
|
||||||
model=self.config.openai_model,
|
model=self.config.openai_model,
|
||||||
messages=[{"role": "user", "content": prompt}],
|
messages=[{"role": "user", "content": prompt}],
|
||||||
temperature=0.1,
|
temperature=0.1,
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ from typing import Any
|
|||||||
|
|
||||||
from openai import OpenAI
|
from openai import OpenAI
|
||||||
|
|
||||||
|
from umbraaudio.voices.lmstudio_gate import queued_chat_completion
|
||||||
from umbraaudio.analysis.attribution_qa import (
|
from umbraaudio.analysis.attribution_qa import (
|
||||||
SPEECH_VERBS as _QA_SPEECH_VERBS,
|
SPEECH_VERBS as _QA_SPEECH_VERBS,
|
||||||
)
|
)
|
||||||
@@ -1097,7 +1098,8 @@ class LLMAttributor:
|
|||||||
max_attempts = 3
|
max_attempts = 3
|
||||||
for attempt in range(max_attempts):
|
for attempt in range(max_attempts):
|
||||||
try:
|
try:
|
||||||
response = client.chat.completions.create(
|
response = queued_chat_completion(
|
||||||
|
client,
|
||||||
model=self.model,
|
model=self.model,
|
||||||
messages=messages,
|
messages=messages,
|
||||||
temperature=0.1,
|
temperature=0.1,
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ from typing import Any
|
|||||||
from openai import OpenAI
|
from openai import OpenAI
|
||||||
|
|
||||||
from umbraaudio.config import LLMConfig
|
from umbraaudio.config import LLMConfig
|
||||||
|
from umbraaudio.voices.lmstudio_gate import queued_chat_completion
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -136,7 +137,8 @@ class LLMSegmenter:
|
|||||||
client = self._get_client()
|
client = self._get_client()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = client.chat.completions.create(
|
response = queued_chat_completion(
|
||||||
|
client,
|
||||||
model=self.model,
|
model=self.model,
|
||||||
messages=[{"role": "user", "content": prompt}],
|
messages=[{"role": "user", "content": prompt}],
|
||||||
temperature=0.2,
|
temperature=0.2,
|
||||||
|
|||||||
@@ -58,6 +58,9 @@ class TTSConfig(BaseSettings):
|
|||||||
voicedesign_model: str = "/Volumes/1TB_APL_Storage/UmbraAudio/models/Qwen3-TTS-12Hz-1.7B-VoiceDesign"
|
voicedesign_model: str = "/Volumes/1TB_APL_Storage/UmbraAudio/models/Qwen3-TTS-12Hz-1.7B-VoiceDesign"
|
||||||
base_model: str = "/Volumes/1TB_APL_Storage/UmbraAudio/models/Qwen3-TTS-12Hz-1.7B-Base"
|
base_model: str = "/Volumes/1TB_APL_Storage/UmbraAudio/models/Qwen3-TTS-12Hz-1.7B-Base"
|
||||||
qwen_tts_url: str = "http://localhost:8881"
|
qwen_tts_url: str = "http://localhost:8881"
|
||||||
|
# Optional GPU accelerator (e.g. http://127.0.0.1:18881 relay to the
|
||||||
|
# 192.168.123.14 box). Tried first when healthy; local always falls back.
|
||||||
|
qwen_tts_helper_url: str = ""
|
||||||
|
|
||||||
|
|
||||||
class QualityConfig(BaseSettings):
|
class QualityConfig(BaseSettings):
|
||||||
|
|||||||
@@ -475,7 +475,7 @@ class ChapterBuilder:
|
|||||||
speaker_traits = load_speaker_traits(self.data_dir / "speakers.json")
|
speaker_traits = load_speaker_traits(self.data_dir / "speakers.json")
|
||||||
distortion_effects = load_speaker_distortion_effects(self.data_dir / "speakers.json")
|
distortion_effects = load_speaker_distortion_effects(self.data_dir / "speakers.json")
|
||||||
|
|
||||||
engine = QwenTTSEngine(qwen_tts_url=self.config.tts.qwen_tts_url)
|
engine = QwenTTSEngine(qwen_tts_url=self.config.tts.qwen_tts_url, helper_url=self.config.tts.qwen_tts_helper_url)
|
||||||
|
|
||||||
# Capture the full vmatrix (the "8k matrix" ref_code + speaker embedding)
|
# Capture the full vmatrix (the "8k matrix" ref_code + speaker embedding)
|
||||||
# once per distinct speaker, cached server-side as .pt under PROMPT_DIR.
|
# once per distinct speaker, cached server-side as .pt under PROMPT_DIR.
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ import json
|
|||||||
import logging
|
import logging
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
from umbraaudio.voices.lmstudio_gate import queued_chat_completion
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
EMOTION_PROMPT = """You are an audiobook director analyzing dialogue and narration for TTS voice direction.
|
EMOTION_PROMPT = """You are an audiobook director analyzing dialogue and narration for TTS voice direction.
|
||||||
@@ -85,7 +87,8 @@ class EmotionMapper:
|
|||||||
max_attempts = 3
|
max_attempts = 3
|
||||||
for attempt in range(max_attempts):
|
for attempt in range(max_attempts):
|
||||||
try:
|
try:
|
||||||
response = self.llm.chat.completions.create(
|
response = queued_chat_completion(
|
||||||
|
self.llm,
|
||||||
model=self.model,
|
model=self.model,
|
||||||
messages=[{"role": "user", "content": prompt}],
|
messages=[{"role": "user", "content": prompt}],
|
||||||
temperature=0.2,
|
temperature=0.2,
|
||||||
|
|||||||
@@ -808,7 +808,7 @@ def generate_pronunciation_audio(config: Any, name: str, spelling: str) -> dict[
|
|||||||
try:
|
try:
|
||||||
from umbraaudio.voices.qwen_tts import VOICE_REFERENCE_TEXT, QwenTTSEngine
|
from umbraaudio.voices.qwen_tts import VOICE_REFERENCE_TEXT, QwenTTSEngine
|
||||||
|
|
||||||
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url)
|
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url, helper_url=config.tts.qwen_tts_helper_url)
|
||||||
if ref:
|
if ref:
|
||||||
engine.generate(
|
engine.generate(
|
||||||
text=syn_text, output_path=out, voice_ref=ref,
|
text=syn_text, output_path=out, voice_ref=ref,
|
||||||
|
|||||||
186
umbraaudio/voices/lmstudio_gate.py
Normal file
186
umbraaudio/voices/lmstudio_gate.py
Normal file
@@ -0,0 +1,186 @@
|
|||||||
|
"""Queueing gate for LM Studio LLM requests.
|
||||||
|
|
||||||
|
LM Studio on this Mac can only serve one LLM at a time and the model
|
||||||
|
coordinator swaps between the Qwen TTS models and the LLM. LLM calls that
|
||||||
|
arrive while LM Studio is busy -- loading a model, swapping models, or
|
||||||
|
warming up -- previously failed with connection / HTTP errors. This module
|
||||||
|
serializes requests and waits until the server is available again, then lets
|
||||||
|
the queued request through (the exact behavior requested: *if the local LM
|
||||||
|
Studio is busy the request is queued and waits for availability, then
|
||||||
|
processed*).
|
||||||
|
|
||||||
|
Only requests aimed at a local LM Studio (:1234) are gated; OpenAI provider
|
||||||
|
clients pass straight through untouched.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from collections.abc import Callable
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from typing import Any
|
||||||
|
from urllib.error import URLError
|
||||||
|
from urllib.request import Request, urlopen
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
DEFAULT_POLL_INTERVAL = 1.0
|
||||||
|
DEFAULT_WAIT_TIMEOUT = 900.0
|
||||||
|
|
||||||
|
_ML_LOCAL_HOSTS = {"localhost", "127.0.0.1", "0.0.0.0", "::1"}
|
||||||
|
_PLACEHOLDER_MODELS = {"", "local-model"}
|
||||||
|
|
||||||
|
|
||||||
|
class LMStudioBusyError(RuntimeError):
|
||||||
|
"""Raised when LM Studio did not become available within the wait window."""
|
||||||
|
|
||||||
|
|
||||||
|
def _host_of(base_url: str) -> str:
|
||||||
|
"""Return the hostname of a base URL (best effort, no network)."""
|
||||||
|
base = base_url.split("://", 1)[-1]
|
||||||
|
base = base.split("/", 1)[0]
|
||||||
|
host = base
|
||||||
|
if ":" in base and not base.startswith("["):
|
||||||
|
host = base.rsplit(":", 1)[0]
|
||||||
|
return (host or "").lower().strip("[]")
|
||||||
|
|
||||||
|
|
||||||
|
def _is_local_lmstudio(base_url: str) -> bool:
|
||||||
|
"""Gate only requests aimed at the local LM Studio server."""
|
||||||
|
host = _host_of(base_url)
|
||||||
|
if host in _ML_LOCAL_HOSTS:
|
||||||
|
return True
|
||||||
|
# LM Studio is commonly on port 1234, even when reached via a custom host.
|
||||||
|
return ":1234" in base_url
|
||||||
|
|
||||||
|
|
||||||
|
def _server_probe(base_url: str, model: str | None = None) -> bool:
|
||||||
|
"""True when the server answers /models and (if given) knows the model."""
|
||||||
|
base = (base_url or "").rstrip("/")
|
||||||
|
if not base:
|
||||||
|
return True
|
||||||
|
try:
|
||||||
|
req = Request(
|
||||||
|
f"{base}/models",
|
||||||
|
headers={"Accept": "application/json"},
|
||||||
|
)
|
||||||
|
with urlopen(req, timeout=3) as resp:
|
||||||
|
if resp.status != 200:
|
||||||
|
return False
|
||||||
|
payload = json.loads(resp.read().decode("utf-8"))
|
||||||
|
except (URLError, OSError, ValueError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
ids = [m.get("id", "") for m in payload.get("data", [])]
|
||||||
|
if model in _PLACEHOLDER_MODELS:
|
||||||
|
return True
|
||||||
|
|
||||||
|
# LM Studio exposes loaded instances under the native /api/v1/models
|
||||||
|
# endpoint; prefer it when the OpenAI-compatible listing is ambiguous.
|
||||||
|
api = base
|
||||||
|
for suffix in ("/v1",):
|
||||||
|
if api.endswith(suffix):
|
||||||
|
api = api[: -len(suffix)]
|
||||||
|
try:
|
||||||
|
req2 = Request(f"{api}/api/v1/models", headers={"Accept": "application/json"})
|
||||||
|
with urlopen(req2, timeout=3) as resp2:
|
||||||
|
if resp2.status == 200:
|
||||||
|
native = json.loads(resp2.read().decode("utf-8"))
|
||||||
|
for m in native.get("data", []):
|
||||||
|
if m.get("key") == model and m.get("loaded_instances"):
|
||||||
|
return True
|
||||||
|
except (URLError, OSError, ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
return model in ids or not ids
|
||||||
|
|
||||||
|
|
||||||
|
class LMStudioGate:
|
||||||
|
"""Serialize + queue LLM requests until LM Studio is available.
|
||||||
|
|
||||||
|
Thread-safe. At most one request is processed at a time; everyone else
|
||||||
|
waits (FIFO-ish via the condition variable). Before a request runs, the
|
||||||
|
gate polls ``available_fn`` until the server reports healthy.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
poll_interval: float = DEFAULT_POLL_INTERVAL,
|
||||||
|
wait_timeout: float = DEFAULT_WAIT_TIMEOUT,
|
||||||
|
available_fn: Callable[[str, str | None], bool] = _server_probe,
|
||||||
|
) -> None:
|
||||||
|
self.poll_interval = poll_interval
|
||||||
|
self.wait_timeout = wait_timeout
|
||||||
|
self._available_fn = available_fn
|
||||||
|
self._cond = threading.Condition()
|
||||||
|
self._inflight = 0
|
||||||
|
|
||||||
|
def wait_until_available(
|
||||||
|
self,
|
||||||
|
base_url: str,
|
||||||
|
model: str | None = None,
|
||||||
|
timeout: float | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Block until LM Studio answers, raising :class:`LMStudioBusyError`."""
|
||||||
|
wait_for = timeout if timeout is not None else self.wait_timeout
|
||||||
|
deadline = time.monotonic() + wait_for
|
||||||
|
while True:
|
||||||
|
if self._available_fn(base_url, model):
|
||||||
|
return
|
||||||
|
if time.monotonic() >= deadline:
|
||||||
|
raise LMStudioBusyError(
|
||||||
|
f"LM Studio not available after {wait_for:.0f}s wait (base={base_url})"
|
||||||
|
)
|
||||||
|
time.sleep(self.poll_interval)
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def acquire(
|
||||||
|
self,
|
||||||
|
base_url: str,
|
||||||
|
model: str | None = None,
|
||||||
|
timeout: float | None = None,
|
||||||
|
) -> Any:
|
||||||
|
"""Queue the caller behind any in-flight request, then run when ready."""
|
||||||
|
with self._cond:
|
||||||
|
while self._inflight > 0:
|
||||||
|
self._cond.wait()
|
||||||
|
self._inflight += 1
|
||||||
|
try:
|
||||||
|
self.wait_until_available(base_url, model, timeout)
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
with self._cond:
|
||||||
|
self._inflight -= 1
|
||||||
|
self._cond.notify()
|
||||||
|
|
||||||
|
|
||||||
|
_GATE = LMStudioGate()
|
||||||
|
|
||||||
|
|
||||||
|
def _client_base(client: Any) -> str:
|
||||||
|
try:
|
||||||
|
return str(client.base_url)
|
||||||
|
except Exception:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def queued_chat_completion(
|
||||||
|
client: Any,
|
||||||
|
model: str,
|
||||||
|
messages: list[dict[str, str]],
|
||||||
|
**kwargs: Any,
|
||||||
|
) -> Any:
|
||||||
|
"""Run ``client.chat.completions.create`` behind the LM Studio gate.
|
||||||
|
|
||||||
|
Requests to a local LM Studio are serialized and wait for server
|
||||||
|
availability before processing; non-LM Studio providers are unchanged.
|
||||||
|
"""
|
||||||
|
base_url = _client_base(client)
|
||||||
|
if _is_local_lmstudio(base_url):
|
||||||
|
with _GATE.acquire(base_url, model):
|
||||||
|
return client.chat.completions.create(model=model, messages=messages, **kwargs)
|
||||||
|
return client.chat.completions.create(model=model, messages=messages, **kwargs)
|
||||||
@@ -51,67 +51,99 @@ class QwenTTSEngine:
|
|||||||
voicedesign_model: str | None = None,
|
voicedesign_model: str | None = None,
|
||||||
base_model: str | None = None,
|
base_model: str | None = None,
|
||||||
qwen_tts_url: str | None = None,
|
qwen_tts_url: str | None = None,
|
||||||
|
helper_url: str | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.device = device
|
self.device = device
|
||||||
self.sample_rate = sample_rate
|
self.sample_rate = sample_rate
|
||||||
self._voicedesign_path = voicedesign_model or DEFAULT_VOICEDESIGN_MODEL
|
self._voicedesign_path = voicedesign_model or DEFAULT_VOICEDESIGN_MODEL
|
||||||
self._base_path = base_model or DEFAULT_BASE_MODEL
|
self._base_path = base_model or DEFAULT_BASE_MODEL
|
||||||
self._qwen_tts_url = qwen_tts_url
|
self._qwen_tts_url = qwen_tts_url
|
||||||
|
self._helper_url = helper_url
|
||||||
self._vd_model = None
|
self._vd_model = None
|
||||||
self._base_model = None
|
self._base_model = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def _candidate_urls(self) -> list[str]:
|
||||||
|
"""Ordered TTS endpoints: GPU helper first (when configured), local required last."""
|
||||||
|
urls: list[str] = []
|
||||||
|
if self._helper_url:
|
||||||
|
urls.append(self._helper_url.rstrip("/"))
|
||||||
|
if self._qwen_tts_url:
|
||||||
|
local = self._qwen_tts_url.rstrip("/")
|
||||||
|
if local not in urls:
|
||||||
|
urls.append(local)
|
||||||
|
return urls
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def _use_remote(self) -> bool:
|
def _use_remote(self) -> bool:
|
||||||
return bool(self._qwen_tts_url)
|
return bool(self._candidate_urls)
|
||||||
|
|
||||||
|
def _url_healthy(self, url: str) -> bool:
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
try:
|
||||||
|
r = httpx.get(f"{url}/health", timeout=5)
|
||||||
|
return r.status_code < 500
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _preferred_candidates(self) -> list[str]:
|
||||||
|
"""Healthy endpoints first, unhealthy ones after (never dropped).
|
||||||
|
|
||||||
|
With a single endpoint (local-only setup) no health probe is made:
|
||||||
|
the request itself determines success, so we never add a second
|
||||||
|
round-trip to every TTS call.
|
||||||
|
"""
|
||||||
|
urls = self._candidate_urls
|
||||||
|
if len(urls) <= 1:
|
||||||
|
return urls
|
||||||
|
healthy = [u for u in urls if self._url_healthy(u)]
|
||||||
|
return healthy + [u for u in urls if u not in healthy]
|
||||||
|
|
||||||
def check_health(self) -> bool:
|
def check_health(self) -> bool:
|
||||||
if not self._use_remote:
|
if not self._use_remote:
|
||||||
return False
|
return False
|
||||||
import httpx
|
return any(self._url_healthy(url) for url in self._candidate_urls)
|
||||||
|
|
||||||
try:
|
|
||||||
r = httpx.get(f"{self._qwen_tts_url}/health", timeout=5)
|
|
||||||
return r.status_code < 500
|
|
||||||
except Exception:
|
|
||||||
return False
|
|
||||||
|
|
||||||
def load_remote_model(self, model_type: str = "voicedesign") -> None:
|
def load_remote_model(self, model_type: str = "voicedesign") -> None:
|
||||||
if not self._use_remote:
|
if not self._use_remote:
|
||||||
return
|
return
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
try:
|
for url in self._candidate_urls:
|
||||||
r = httpx.post(
|
try:
|
||||||
f"{self._qwen_tts_url}/load",
|
r = httpx.post(
|
||||||
json={"model_type": model_type},
|
f"{url}/load",
|
||||||
timeout=120,
|
json={"model_type": model_type},
|
||||||
)
|
timeout=120,
|
||||||
logger.info(
|
)
|
||||||
"Qwen TTS remote model load: HTTP %d %s", r.status_code, r.text.strip()[:200]
|
logger.info(
|
||||||
)
|
"Qwen TTS remote model load: HTTP %d %s", r.status_code, r.text.strip()[:200]
|
||||||
import time
|
)
|
||||||
|
import time
|
||||||
|
|
||||||
if "already" in r.text.lower():
|
if "already" in r.text.lower():
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
else:
|
else:
|
||||||
time.sleep(10)
|
time.sleep(10)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("Qwen TTS remote model load failed: %s", e)
|
logger.warning("Qwen TTS remote model load failed (%s): %s", url, e)
|
||||||
|
|
||||||
def unload_remote_model(self) -> None:
|
def unload_remote_model(self) -> None:
|
||||||
if not self._use_remote:
|
if not self._use_remote:
|
||||||
return
|
return
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
try:
|
for url in self._candidate_urls:
|
||||||
r = httpx.post(
|
try:
|
||||||
f"{self._qwen_tts_url}/unload",
|
r = httpx.post(
|
||||||
json={"model_type": "all"},
|
f"{url}/unload",
|
||||||
timeout=30,
|
json={"model_type": "all"},
|
||||||
)
|
timeout=30,
|
||||||
logger.info("Qwen TTS remote model unload: HTTP %d", r.status_code)
|
)
|
||||||
except Exception as e:
|
logger.info("Qwen TTS remote model unload (%s): HTTP %d", url, r.status_code)
|
||||||
logger.warning("Qwen TTS remote model unload failed: %s", e)
|
except Exception as e:
|
||||||
|
logger.warning("Qwen TTS remote model unload failed (%s): %s", url, e)
|
||||||
|
|
||||||
def unload_and_reload(self) -> None:
|
def unload_and_reload(self) -> None:
|
||||||
self.unload_remote_model()
|
self.unload_remote_model()
|
||||||
@@ -271,18 +303,26 @@ class QwenTTSEngine:
|
|||||||
data["force"] = 1
|
data["force"] = 1
|
||||||
if ref_text:
|
if ref_text:
|
||||||
data["ref_text"] = ref_text
|
data["ref_text"] = ref_text
|
||||||
with open(voice_ref, "rb") as f:
|
|
||||||
r = httpx.post(
|
last_err: str | None = None
|
||||||
f"{self._qwen_tts_url}/v1/voice-prompt",
|
ref_bytes = Path(voice_ref).read_bytes()
|
||||||
data=data,
|
for url in self._preferred_candidates():
|
||||||
files={"ref_audio": ("ref.wav", f.read(), "audio/wav")},
|
try:
|
||||||
timeout=REMOTE_TIMEOUT,
|
r = httpx.post(
|
||||||
)
|
f"{url}/v1/voice-prompt",
|
||||||
if r.status_code != 200:
|
data=data,
|
||||||
raise QwenTTSUnavailableError(
|
files={"ref_audio": ("ref.wav", ref_bytes, "audio/wav")},
|
||||||
f"Voice-prompt build failed HTTP {r.status_code}: {r.text[:300]}"
|
timeout=REMOTE_TIMEOUT,
|
||||||
)
|
)
|
||||||
return slug
|
if r.status_code != 200:
|
||||||
|
last_err = f"Voice-prompt build failed HTTP {r.status_code}: {r.text[:300]}"
|
||||||
|
logger.warning("%s (url=%s)", last_err, url)
|
||||||
|
continue
|
||||||
|
return slug
|
||||||
|
except Exception as e:
|
||||||
|
last_err = f"Voice-prompt build failed: {e}"
|
||||||
|
logger.warning("%s (url=%s)", last_err, url)
|
||||||
|
raise QwenTTSUnavailableError(last_err or "No Qwen TTS server available")
|
||||||
|
|
||||||
def generate_voice_reference(
|
def generate_voice_reference(
|
||||||
self,
|
self,
|
||||||
@@ -315,7 +355,48 @@ class QwenTTSEngine:
|
|||||||
) -> tuple[np.ndarray, int]:
|
) -> tuple[np.ndarray, int]:
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
url = f"{self._qwen_tts_url}/v1/tts"
|
files = None
|
||||||
|
|
||||||
|
if prompt_slug:
|
||||||
|
pass
|
||||||
|
elif voice_ref and Path(voice_ref).exists():
|
||||||
|
files = {"ref_audio": ("ref.wav", Path(voice_ref).read_bytes(), "audio/wav")}
|
||||||
|
|
||||||
|
last_err: str | None = None
|
||||||
|
for url in self._preferred_candidates():
|
||||||
|
try:
|
||||||
|
return self._remote_generate_at(
|
||||||
|
url=url,
|
||||||
|
text=text,
|
||||||
|
files=files,
|
||||||
|
instruct=instruct,
|
||||||
|
ref_text=ref_text,
|
||||||
|
prompt_slug=prompt_slug,
|
||||||
|
temperature=temperature,
|
||||||
|
do_sample=do_sample,
|
||||||
|
seed=seed,
|
||||||
|
)
|
||||||
|
except QwenTTSUnavailableError as e:
|
||||||
|
last_err = str(e)
|
||||||
|
logger.warning("Qwen TTS endpoint %s failed, trying next: %s", url, e)
|
||||||
|
raise QwenTTSUnavailableError(last_err or "No Qwen TTS server available")
|
||||||
|
|
||||||
|
def _remote_generate_at(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
url: str,
|
||||||
|
text: str,
|
||||||
|
files: dict[str, Any] | None,
|
||||||
|
instruct: str | None,
|
||||||
|
ref_text: str | None,
|
||||||
|
prompt_slug: str | None,
|
||||||
|
temperature: float,
|
||||||
|
do_sample: bool,
|
||||||
|
seed: int | None,
|
||||||
|
) -> tuple[np.ndarray, int]:
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
generate_url = f"{url}/v1/tts"
|
||||||
payload: dict[str, Any] = {
|
payload: dict[str, Any] = {
|
||||||
"text": text,
|
"text": text,
|
||||||
"sample_rate": self.sample_rate,
|
"sample_rate": self.sample_rate,
|
||||||
@@ -326,12 +407,9 @@ class QwenTTSEngine:
|
|||||||
payload["seed"] = seed
|
payload["seed"] = seed
|
||||||
if ref_text:
|
if ref_text:
|
||||||
payload["ref_text"] = ref_text
|
payload["ref_text"] = ref_text
|
||||||
files = None
|
|
||||||
|
|
||||||
if prompt_slug:
|
if prompt_slug:
|
||||||
payload["prompt_slug"] = prompt_slug
|
payload["prompt_slug"] = prompt_slug
|
||||||
elif voice_ref and Path(voice_ref).exists():
|
|
||||||
files = {"ref_audio": ("ref.wav", Path(voice_ref).read_bytes(), "audio/wav")}
|
|
||||||
|
|
||||||
# instruct composes with voice clone / ICL: the server prepends it as
|
# instruct composes with voice clone / ICL: the server prepends it as
|
||||||
# style direction (emotion/pace/delivery) while keeping the speaker.
|
# style direction (emotion/pace/delivery) while keeping the speaker.
|
||||||
@@ -339,15 +417,16 @@ class QwenTTSEngine:
|
|||||||
payload["voice_description"] = instruct
|
payload["voice_description"] = instruct
|
||||||
|
|
||||||
try:
|
try:
|
||||||
r = httpx.post(url, data=payload, files=files or {}, timeout=REMOTE_TIMEOUT)
|
r = httpx.post(
|
||||||
|
generate_url, data=payload, files=files or {}, timeout=REMOTE_TIMEOUT
|
||||||
|
)
|
||||||
except httpx.ConnectError as e:
|
except httpx.ConnectError as e:
|
||||||
raise QwenTTSUnavailableError(
|
raise QwenTTSUnavailableError(
|
||||||
f"Cannot connect to Qwen TTS server at {self._qwen_tts_url}. "
|
f"Cannot connect to Qwen TTS server at {url}. Make sure the server is running."
|
||||||
"Make sure the server is running."
|
|
||||||
) from e
|
) from e
|
||||||
except httpx.TimeoutException as e:
|
except httpx.TimeoutException as e:
|
||||||
raise QwenTTSUnavailableError(
|
raise QwenTTSUnavailableError(
|
||||||
f"Qwen TTS server at {self._qwen_tts_url} timed out after {REMOTE_TIMEOUT}s."
|
f"Qwen TTS server at {url} timed out after {REMOTE_TIMEOUT}s."
|
||||||
) from e
|
) from e
|
||||||
|
|
||||||
if r.status_code != 200:
|
if r.status_code != 200:
|
||||||
@@ -456,23 +535,34 @@ class QwenTTSEngine:
|
|||||||
|
|
||||||
|
|
||||||
class QwenTTSClient:
|
class QwenTTSClient:
|
||||||
def __init__(self, base_url: str, timeout: int = REMOTE_TIMEOUT) -> None:
|
def __init__(
|
||||||
|
self,
|
||||||
|
base_url: str,
|
||||||
|
timeout: int = REMOTE_TIMEOUT,
|
||||||
|
helper_url: str | None = None,
|
||||||
|
) -> None:
|
||||||
self.base_url = base_url.rstrip("/")
|
self.base_url = base_url.rstrip("/")
|
||||||
|
self.helper_url = (helper_url or "").rstrip("/") or None
|
||||||
self.timeout = timeout
|
self.timeout = timeout
|
||||||
|
|
||||||
def check_health(self) -> bool:
|
def check_health(self) -> bool:
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
try:
|
for url in (self.helper_url, self.base_url):
|
||||||
r = httpx.get(f"{self.base_url}/health", timeout=5)
|
if not url:
|
||||||
return r.status_code < 500
|
continue
|
||||||
except Exception:
|
try:
|
||||||
return False
|
r = httpx.get(f"{url}/health", timeout=5)
|
||||||
|
if r.status_code < 500:
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
return False
|
||||||
|
|
||||||
def generate_voice_reference(
|
def generate_voice_reference(
|
||||||
self, text: str, voice_description: str, output_path: Path
|
self, text: str, voice_description: str, output_path: Path
|
||||||
) -> Path:
|
) -> Path:
|
||||||
engine = QwenTTSEngine(qwen_tts_url=self.base_url)
|
engine = QwenTTSEngine(qwen_tts_url=self.base_url, helper_url=self.helper_url)
|
||||||
return engine.generate_voice_reference(text, output_path, voice_description)
|
return engine.generate_voice_reference(text, output_path, voice_description)
|
||||||
|
|
||||||
def generate(
|
def generate(
|
||||||
@@ -485,7 +575,7 @@ class QwenTTSClient:
|
|||||||
do_sample: bool = True,
|
do_sample: bool = True,
|
||||||
seed: int | None = None,
|
seed: int | None = None,
|
||||||
) -> Path:
|
) -> Path:
|
||||||
engine = QwenTTSEngine(qwen_tts_url=self.base_url)
|
engine = QwenTTSEngine(qwen_tts_url=self.base_url, helper_url=self.helper_url)
|
||||||
return engine.generate(
|
return engine.generate(
|
||||||
text, output_path, voice_ref=voice_ref, instruct=instruct,
|
text, output_path, voice_ref=voice_ref, instruct=instruct,
|
||||||
temperature=temperature, do_sample=do_sample, seed=seed,
|
temperature=temperature, do_sample=do_sample, seed=seed,
|
||||||
|
|||||||
@@ -1940,7 +1940,7 @@ async def speakers_list_partial(request: Request):
|
|||||||
return False
|
return False
|
||||||
from umbraaudio.voices.qwen_tts import QwenTTSEngine
|
from umbraaudio.voices.qwen_tts import QwenTTSEngine
|
||||||
|
|
||||||
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url)
|
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url, helper_url=config.tts.qwen_tts_helper_url)
|
||||||
try:
|
try:
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
@@ -3325,7 +3325,7 @@ async def generate_utterance_audio(request: Request, utt_id: int):
|
|||||||
voice_ref = vf
|
voice_ref = vf
|
||||||
break
|
break
|
||||||
|
|
||||||
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url)
|
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url, helper_url=config.tts.qwen_tts_helper_url)
|
||||||
|
|
||||||
tts_instruct = compose_utterance_instruct(instruct, emotion, pace)
|
tts_instruct = compose_utterance_instruct(instruct, emotion, pace)
|
||||||
|
|
||||||
@@ -3444,7 +3444,7 @@ async def delete_utterance(request: Request, utt_id: int):
|
|||||||
if "kainda" in vf.stem.lower():
|
if "kainda" in vf.stem.lower():
|
||||||
narrator_ref = vf
|
narrator_ref = vf
|
||||||
break
|
break
|
||||||
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url)
|
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url, helper_url=config.tts.qwen_tts_helper_url)
|
||||||
for u in remaining:
|
for u in remaining:
|
||||||
out = audio_dir / f"utt_{u['utterance_index']:04d}.wav"
|
out = audio_dir / f"utt_{u['utterance_index']:04d}.wav"
|
||||||
if is_scene_break(u["text"], u["type"]):
|
if is_scene_break(u["text"], u["type"]):
|
||||||
@@ -3633,7 +3633,7 @@ async def create_utterance(request: Request, chapter_id: int):
|
|||||||
narrator_ref = vf
|
narrator_ref = vf
|
||||||
break
|
break
|
||||||
try:
|
try:
|
||||||
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url)
|
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url, helper_url=config.tts.qwen_tts_helper_url)
|
||||||
if narrator_ref:
|
if narrator_ref:
|
||||||
engine.generate(
|
engine.generate(
|
||||||
text=spoken_text,
|
text=spoken_text,
|
||||||
@@ -3798,7 +3798,7 @@ async def set_voice_status(request: Request):
|
|||||||
QwenTTSEngine,
|
QwenTTSEngine,
|
||||||
)
|
)
|
||||||
|
|
||||||
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url)
|
engine = QwenTTSEngine(qwen_tts_url=config.tts.qwen_tts_url, helper_url=config.tts.qwen_tts_helper_url)
|
||||||
slug = engine.build_voice_prompt(
|
slug = engine.build_voice_prompt(
|
||||||
ref_path,
|
ref_path,
|
||||||
ref_text=VOICE_REFERENCE_TEXT,
|
ref_text=VOICE_REFERENCE_TEXT,
|
||||||
|
|||||||
Reference in New Issue
Block a user