497 lines
19 KiB
Bash
Executable File
497 lines
19 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Copyright 2026 Umbra Tools (umbratools.com)
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
# Create a complete, restorable UmbraAudio snapshot on a mounted filesystem/NAS.
|
|
set -Eeuo pipefail
|
|
|
|
SOURCE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
|
DESTINATION="${UMBRA_BACKUP_DEST:-/Volumes/NetBackup/AudioBackup}"
|
|
GLOBAL_ROOT="${UMBRA_GLOBAL_STATE_ROOT:-$HOME/.umbra}"
|
|
QUIESCE=1
|
|
BEST_EFFORT=0
|
|
RESTART_SERVICES=1
|
|
ALLOW_NON_SMB=0
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage: create_full_backup.sh [options]
|
|
|
|
Create a dated, fail-closed full backup of the current UmbraAudio working tree
|
|
and the active global voice registry/library.
|
|
|
|
Options:
|
|
--destination DIR NAS/filesystem destination (default: /Volumes/NetBackup/AudioBackup)
|
|
--global-root DIR Active global state root (default: $HOME/.umbra)
|
|
--best-effort Do not quiesce services (not recommended; explicit opt-in)
|
|
--quiesce Quiesce services (default)
|
|
--no-restart Do not restart services after a --quiesce backup
|
|
--allow-non-smb Permit a non-SMB destination for testing
|
|
-h, --help Show this help
|
|
|
|
The archive includes the entire current tree, including dirty/untracked files,
|
|
ignored book projects, generated audio, models, .git, and .env. Global state is
|
|
captured with SQLite's backup API and stored beside the archive.
|
|
EOF
|
|
}
|
|
|
|
while (($#)); do
|
|
case "$1" in
|
|
--destination) DESTINATION="${2:?--destination requires a directory}"; shift 2 ;;
|
|
--global-root) GLOBAL_ROOT="${2:?--global-root requires a directory}"; shift 2 ;;
|
|
--best-effort) BEST_EFFORT=1; QUIESCE=0; shift ;;
|
|
--quiesce) QUIESCE=1; shift ;;
|
|
--no-restart) RESTART_SERVICES=0; shift ;;
|
|
--allow-non-smb) ALLOW_NON_SMB=1; shift ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) printf 'Unknown option: %s\n' "$1" >&2; usage >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
[[ -d "$SOURCE" ]] || { printf 'Source directory not found: %s\n' "$SOURCE" >&2; exit 1; }
|
|
mkdir -p "$DESTINATION"
|
|
DESTINATION="$(cd -- "$DESTINATION" && pwd -P)"
|
|
case "$DESTINATION/" in
|
|
"$SOURCE/"*) printf 'Backup destination must not be inside the source tree.\n' >&2; exit 1 ;;
|
|
esac
|
|
|
|
for tool in tar git awk sed df mount; do
|
|
command -v "$tool" >/dev/null 2>&1 || {
|
|
printf 'Required tool is missing: %s\n' "$tool" >&2
|
|
exit 1
|
|
}
|
|
done
|
|
if command -v shasum >/dev/null 2>&1; then
|
|
:
|
|
elif command -v sha256sum >/dev/null 2>&1; then
|
|
:
|
|
else
|
|
printf 'Install shasum or sha256sum.\n' >&2
|
|
exit 1
|
|
fi
|
|
|
|
hash_file() {
|
|
if command -v shasum >/dev/null 2>&1; then
|
|
shasum -a 256 "$@"
|
|
else
|
|
sha256sum "$@"
|
|
fi
|
|
}
|
|
|
|
verify_hashes() {
|
|
if command -v shasum >/dev/null 2>&1; then
|
|
shasum -a 256 -c "$@"
|
|
else
|
|
sha256sum -c "$@"
|
|
fi
|
|
}
|
|
|
|
FSTYPE="$(df -P "$DESTINATION" | awk 'NR == 2 {print $1}')"
|
|
if [[ "$ALLOW_NON_SMB" -ne 1 && "$FSTYPE" != *smb* && "$FSTYPE" != *cifs* ]]; then
|
|
printf 'Destination is not an SMB/CIFS mount (filesystem=%s). Use --allow-non-smb only for tests.\n' "$FSTYPE" >&2
|
|
exit 1
|
|
fi
|
|
SOURCE_KB="$(du -sk "$SOURCE" | awk '{print $1}')"
|
|
AVAIL_KB="$(df -Pk "$DESTINATION" | awk 'NR == 2 {print $4}')"
|
|
if [[ "$AVAIL_KB" -lt $((SOURCE_KB + 1048576)) ]]; then
|
|
printf 'Insufficient destination space: need about %s KB plus 1 GiB, have %s KB.\n' "$SOURCE_KB" "$AVAIL_KB" >&2
|
|
exit 1
|
|
fi
|
|
|
|
LOCK_DIR="${TMPDIR:-/tmp}/umbra-full-backup.lock"
|
|
if ! mkdir "$LOCK_DIR" 2>/dev/null; then
|
|
printf 'Another backup appears to be running (lock: %s).\n' "$LOCK_DIR" >&2
|
|
exit 1
|
|
fi
|
|
BACKUP_ID="$(date +%Y%m%d-%H%M%S)-$$-$RANDOM"
|
|
STAGING_DIR="$(mktemp -d "$DESTINATION/.UmbraAudio-$BACKUP_ID.partial.XXXXXX")"
|
|
FINAL_DIR="$DESTINATION/UmbraAudio-$BACKUP_ID"
|
|
ARCHIVE_NAME="UmbraAudio-$BACKUP_ID.tar"
|
|
ARCHIVE_TMP="$STAGING_DIR/$ARCHIVE_NAME.partial"
|
|
ARCHIVE_FINAL="$STAGING_DIR/$ARCHIVE_NAME"
|
|
[[ ! -e "$FINAL_DIR" ]] || { printf 'Final backup directory already exists: %s\n' "$FINAL_DIR" >&2; exit 1; }
|
|
|
|
WEB_LABEL="gui/$(id -u)/com.umbraaudio.web"
|
|
WEB_PLIST="/Users/worstauthor/Library/LaunchAgents/com.umbraaudio.web.plist"
|
|
if command -v launchctl >/dev/null 2>&1; then
|
|
detected_web_plist="$(launchctl print "$WEB_LABEL" 2>/dev/null | sed -n 's/^[[:space:]]*path = //p' | head -n 1 || true)"
|
|
if [[ -f "$detected_web_plist" ]]; then
|
|
WEB_PLIST="$detected_web_plist"
|
|
fi
|
|
fi
|
|
WEB_WAS_LOADED=0
|
|
WEB_STOPPED=0
|
|
QWEN_PIDS=()
|
|
QWEN_STOPPED=0
|
|
Qwen_CWD=""
|
|
Qwen_COMMAND=""
|
|
WRITER_WATCH_PID=""
|
|
RESTARTED=0
|
|
|
|
release_lock() { rmdir "$LOCK_DIR" 2>/dev/null || true; }
|
|
|
|
restart_services() {
|
|
[[ "$QUIESCE" -eq 1 && "$RESTART_SERVICES" -eq 1 && "$RESTARTED" -eq 0 ]] || return 0
|
|
local restore_failed=0
|
|
if [[ "$WEB_WAS_LOADED" -eq 1 && -f "$WEB_PLIST" ]]; then
|
|
launchctl bootstrap "gui/$(id -u)" "$WEB_PLIST" >/dev/null 2>&1 || \
|
|
launchctl kickstart -k "$WEB_LABEL" >/dev/null 2>&1 || true
|
|
for _ in {1..30}; do
|
|
if launchctl print "$WEB_LABEL" >/dev/null 2>&1 && lsof -nP -iTCP:8420 -sTCP:LISTEN >/dev/null 2>&1; then
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
if ! launchctl print "$WEB_LABEL" >/dev/null 2>&1 || ! lsof -nP -iTCP:8420 -sTCP:LISTEN >/dev/null 2>&1; then
|
|
printf 'WARNING: web LaunchAgent did not restart cleanly.\n' >&2
|
|
restore_failed=1
|
|
fi
|
|
fi
|
|
if [[ "$QWEN_STOPPED" -eq 1 && "${#QWEN_PIDS[@]}" -gt 0 ]]; then
|
|
mkdir -p "$SOURCE/logs"
|
|
QWEN_ROOT="${Qwen_CWD:-$SOURCE}"
|
|
[[ -d "$QWEN_ROOT" ]] || QWEN_ROOT="$SOURCE"
|
|
(cd "$QWEN_ROOT" && nohup "$SOURCE/.venv/bin/python" -u scripts/qwen_tts_server.py \
|
|
</dev/null >>"$SOURCE/logs/qwen_tts_server.log" 2>&1 &)
|
|
disown 2>/dev/null || true
|
|
for _ in {1..30}; do
|
|
if lsof -nP -iTCP:8881 -sTCP:LISTEN >/dev/null 2>&1; then
|
|
if ! command -v curl >/dev/null 2>&1 || curl -fsS --max-time 5 http://127.0.0.1:8881/health >/dev/null 2>&1; then
|
|
break
|
|
fi
|
|
fi
|
|
sleep 1
|
|
done
|
|
if ! lsof -nP -iTCP:8881 -sTCP:LISTEN >/dev/null 2>&1; then
|
|
printf 'WARNING: Qwen TTS server did not become healthy/listening on port 8881.\n' >&2
|
|
restore_failed=1
|
|
elif command -v curl >/dev/null 2>&1 && ! curl -fsS --max-time 5 http://127.0.0.1:8881/health >/dev/null 2>&1; then
|
|
printf 'WARNING: Qwen TTS port is open but /health did not pass.\n' >&2
|
|
restore_failed=1
|
|
fi
|
|
fi
|
|
RESTARTED=1
|
|
return "$restore_failed"
|
|
}
|
|
|
|
cleanup() {
|
|
local rc=$?
|
|
if [[ -n "$WRITER_WATCH_PID" ]]; then
|
|
kill "$WRITER_WATCH_PID" >/dev/null 2>&1 || true
|
|
wait "$WRITER_WATCH_PID" 2>/dev/null || true
|
|
fi
|
|
if [[ "$QUIESCE" -eq 1 && "$RESTART_SERVICES" -eq 1 && "$RESTARTED" -eq 0 ]]; then
|
|
if ! restart_services; then
|
|
printf 'WARNING: service restoration during cleanup did not complete cleanly.\n' >&2
|
|
fi
|
|
fi
|
|
if [[ -d "$STAGING_DIR" ]]; then
|
|
printf 'Backup incomplete; preserving partial directory for inspection: %s\n' "$STAGING_DIR" >&2
|
|
fi
|
|
release_lock
|
|
exit "$rc"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
record_service_state() {
|
|
if launchctl print "$WEB_LABEL" >/dev/null 2>&1; then WEB_WAS_LOADED=1; fi
|
|
QWEN_PIDS=()
|
|
if command -v lsof >/dev/null 2>&1; then
|
|
while IFS= read -r pid; do
|
|
[[ "$pid" =~ ^[0-9]+$ ]] && QWEN_PIDS+=("$pid")
|
|
done < <(lsof -nP -tiTCP:8881 -sTCP:LISTEN 2>/dev/null || true)
|
|
fi
|
|
while IFS= read -r pid; do
|
|
[[ "$pid" =~ ^[0-9]+$ ]] || continue
|
|
[[ " ${QWEN_PIDS[*]} " == *" $pid "* ]] && continue
|
|
cmd="$(ps -p "$pid" -o command= 2>/dev/null || true)"
|
|
[[ "$cmd" == *qwen_tts_server.py* ]] && QWEN_PIDS+=("$pid")
|
|
done < <(ps -axo pid=,args= | awk '$0 ~ /[q]wen_tts_server\\.py/ {print $1}')
|
|
if [[ "${#QWEN_PIDS[@]}" -gt 0 ]]; then
|
|
Qwen_COMMAND="$(ps -p "${QWEN_PIDS[0]}" -o command= 2>/dev/null || true)"
|
|
if command -v lsof >/dev/null 2>&1; then
|
|
Qwen_CWD="$(lsof -a -p "${QWEN_PIDS[0]}" -d cwd -Fn 2>/dev/null | sed -n 's/^n//p' | head -n 1)"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
write_service_state() {
|
|
local output="$1"
|
|
{
|
|
printf 'captured_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
printf 'web_label=%s\n' "$WEB_LABEL"
|
|
printf 'web_loaded=%s\n' "$WEB_WAS_LOADED"
|
|
printf 'web_plist=%s\n' "$WEB_PLIST"
|
|
printf 'qwen_pids=%s\n' "${QWEN_PIDS[*]:-}"
|
|
printf 'qwen_cwd=%s\n' "${Qwen_CWD:-}"
|
|
printf 'qwen_command=%s\n' "${Qwen_COMMAND:-}"
|
|
launchctl print "$WEB_LABEL" 2>&1 || true
|
|
lsof -nP -iTCP:8420 -sTCP:LISTEN 2>&1 || true
|
|
lsof -nP -iTCP:8881 -sTCP:LISTEN 2>&1 || true
|
|
} >"$output"
|
|
}
|
|
|
|
check_active_writers() {
|
|
local pattern pid
|
|
for pattern in \
|
|
'generate[_]all_audio.py' \
|
|
'generate[_]chapter[_]audio.py' \
|
|
'reattribute[_]all.py' \
|
|
'run[_]llm[_]attribution.py' \
|
|
'resegment[_]all.py' \
|
|
'nas[_]lifecycle.py' \
|
|
'chapter[_]builder' \
|
|
'[r]sync.*UmbraAudio' \
|
|
'[f]fmpeg' \
|
|
'[u]vicorn'; do
|
|
while IFS= read -r pid; do
|
|
[[ "$pid" == "$$" ]] && continue
|
|
printf 'Active writer detected (%s, pid %s); refusing backup.\n' "$pattern" "$pid" >&2
|
|
return 1
|
|
done < <(ps -axo pid=,command= | awk -v p="$pattern" '$0 ~ p {print $1}')
|
|
done
|
|
}
|
|
|
|
quiesce_services() {
|
|
for tool in launchctl lsof ps; do
|
|
command -v "$tool" >/dev/null 2>&1 || {
|
|
printf 'Quiesce requires missing tool: %s\n' "$tool" >&2
|
|
exit 1
|
|
}
|
|
done
|
|
record_service_state
|
|
if [[ "$WEB_WAS_LOADED" -eq 1 ]]; then
|
|
printf 'Pausing web LaunchAgent...\n'
|
|
launchctl bootout "$WEB_LABEL" "$WEB_PLIST" >/dev/null 2>&1 || {
|
|
printf 'Could not boot out web LaunchAgent; refusing inconsistent backup.\n' >&2
|
|
exit 1
|
|
}
|
|
for _ in {1..30}; do
|
|
if ! launchctl print "$WEB_LABEL" >/dev/null 2>&1 && ! lsof -nP -iTCP:8420 -sTCP:LISTEN >/dev/null 2>&1; then
|
|
WEB_STOPPED=1
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
[[ "$WEB_STOPPED" -eq 1 ]] || {
|
|
printf 'Web service did not stop cleanly; refusing backup.\n' >&2
|
|
exit 1
|
|
}
|
|
fi
|
|
for pid in "${QWEN_PIDS[@]}"; do
|
|
printf 'Pausing Qwen TTS server (pid %s)...\n' "$pid"
|
|
kill -TERM "$pid" >/dev/null 2>&1 || true
|
|
done
|
|
for pid in "${QWEN_PIDS[@]}"; do
|
|
for _ in {1..30}; do
|
|
kill -0 "$pid" >/dev/null 2>&1 || break
|
|
sleep 1
|
|
done
|
|
if kill -0 "$pid" >/dev/null 2>&1; then
|
|
printf 'Qwen TTS server pid %s did not stop; refusing backup.\n' "$pid" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
QWEN_STOPPED=1
|
|
if lsof -nP -iTCP:8881 -sTCP:LISTEN >/dev/null 2>&1; then
|
|
printf 'Port 8881 is still listening; refusing backup.\n' >&2
|
|
exit 1
|
|
fi
|
|
if [[ "$WEB_STOPPED" -eq 1 ]] && lsof -nP -iTCP:8420 -sTCP:LISTEN >/dev/null 2>&1; then
|
|
printf 'Port 8420 is still listening; refusing backup.\n' >&2
|
|
exit 1
|
|
fi
|
|
check_active_writers || exit 1
|
|
sleep 2
|
|
}
|
|
|
|
record_service_state
|
|
write_service_state "$STAGING_DIR/SERVICE-STATE-BEFORE.txt"
|
|
if [[ "$QUIESCE" -eq 1 ]]; then
|
|
quiesce_services
|
|
else
|
|
printf 'WARNING: best-effort backup requested; active files may be inconsistent.\n' >&2
|
|
fi
|
|
|
|
SOURCE_PARENT="$(cd -- "$SOURCE/.." && pwd -P)"
|
|
SOURCE_NAME="$(basename -- "$SOURCE")"
|
|
GLOBAL_PYTHON="$SOURCE/.venv/bin/python"
|
|
[[ -x "$GLOBAL_PYTHON" ]] || GLOBAL_PYTHON="$(command -v python3)"
|
|
|
|
{
|
|
printf 'backup_id=%s\n' "$BACKUP_ID"
|
|
printf 'created_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
printf 'source_path=%s\n' "$SOURCE"
|
|
printf 'global_state_root=%s\n' "$GLOBAL_ROOT"
|
|
printf 'source_host=%s\n' "$(hostname)"
|
|
printf 'source_os=%s\n' "$(uname -a)"
|
|
printf 'git_commit=%s\n' "$(git -C "$SOURCE" rev-parse HEAD 2>/dev/null || printf unknown)"
|
|
printf 'git_branch=%s\n' "$(git -C "$SOURCE" branch --show-current 2>/dev/null || printf unknown)"
|
|
printf 'quiesced_services=%s\n' "$QUIESCE"
|
|
printf 'best_effort=%s\n' "$BEST_EFFORT"
|
|
printf 'archive_name=%s\n' "$ARCHIVE_NAME"
|
|
} >"$STAGING_DIR/BACKUP-INFO.txt"
|
|
|
|
{
|
|
python3 --version 2>&1 || true
|
|
"$GLOBAL_PYTHON" --version 2>&1 || true
|
|
uv --version 2>&1 || true
|
|
git --version 2>&1 || true
|
|
tar --version 2>&1 | head -n 1 || true
|
|
ffmpeg -version 2>&1 | head -n 1 || true
|
|
} >"$STAGING_DIR/ENVIRONMENT.txt"
|
|
|
|
git -C "$SOURCE" status --short --branch --untracked-files=all >"$STAGING_DIR/GIT-STATUS.txt" 2>&1 || true
|
|
git -C "$SOURCE" diff --binary >"$STAGING_DIR/GIT-DIFF.patch" 2>/dev/null || true
|
|
git -C "$SOURCE" diff --cached --binary >"$STAGING_DIR/GIT-DIFF-CACHED.patch" 2>/dev/null || true
|
|
git -C "$SOURCE" submodule status >"$STAGING_DIR/SUBMODULES.txt" 2>&1 || true
|
|
|
|
printf 'Recording file inventory...\n'
|
|
SOURCE="$SOURCE" "$GLOBAL_PYTHON" - <<'PY' >"$STAGING_DIR/FILE-INVENTORY.tsv"
|
|
from __future__ import annotations
|
|
import os
|
|
from pathlib import Path
|
|
root = Path(os.environ["SOURCE"])
|
|
for path in sorted(root.rglob("*")):
|
|
try:
|
|
stat = path.lstat()
|
|
except OSError:
|
|
continue
|
|
kind = "l" if path.is_symlink() else "d" if path.is_dir() else "f"
|
|
print(f"{kind}\t{stat.st_size}\t{path.relative_to(root)}")
|
|
PY
|
|
|
|
printf 'Running source integrity checks before archiving...\n'
|
|
"$GLOBAL_PYTHON" "$SOURCE/backup/restore_support.py" validate-tree --root "$SOURCE"
|
|
|
|
cat >"$STAGING_DIR/BACKUP-README.txt" <<'EOF'
|
|
UmbraAudio full backup
|
|
======================
|
|
|
|
This backup is a point-in-time copy of the complete current working tree and
|
|
active global voice registry/library.
|
|
|
|
Included:
|
|
- tracked, untracked, and modified files;
|
|
- ignored book projects, generated audio, exports, logs, and .git metadata;
|
|
- model weights and the Qwen3-TTS submodule;
|
|
- .env and local configuration;
|
|
- the existing .venv and .python directories for forensic completeness;
|
|
- a SQLite-consistent global-state snapshot in global-state/;
|
|
- launchd/GPU-relay context in context/.
|
|
|
|
The restore script rebuilds a portable environment rather than trusting the
|
|
archived platform-specific virtual environment. The original .env is preserved
|
|
as .env.archive-original when portability paths are rewritten.
|
|
|
|
This backup does not configure or modify a network database.
|
|
|
|
Restore:
|
|
./Starthere.sh --target /path/to/restore
|
|
|
|
See BACKUP-INFO.txt, ENVIRONMENT.txt, global-state/MANIFEST.json, and
|
|
RESTORED-SERVICE-COMMANDS.md after restoration.
|
|
EOF
|
|
|
|
cp "$SOURCE/backup/Starthere.sh" "$STAGING_DIR/Starthere.sh"
|
|
cp "$SOURCE/backup/restore_support.py" "$STAGING_DIR/restore_support.py"
|
|
chmod 700 "$STAGING_DIR/Starthere.sh"
|
|
chmod 700 "$STAGING_DIR/restore_support.py"
|
|
|
|
CONTEXT_DIR="$STAGING_DIR/context"
|
|
mkdir -p "$CONTEXT_DIR/launchagents" "$CONTEXT_DIR/launchdaemons"
|
|
cp -p /Users/worstauthor/.umbraaudio/umbraweb_login.sh "$CONTEXT_DIR/launchagents/" 2>/dev/null || true
|
|
cp -p /Users/worstauthor/Library/LaunchAgents/com.umbraaudio.web.plist "$CONTEXT_DIR/launchagents/" 2>/dev/null || true
|
|
cp -p /Users/worstauthor/Library/LaunchAgents/com.umbraaudio.commitcheck.plist "$CONTEXT_DIR/launchagents/" 2>/dev/null || true
|
|
cp -p /Library/LaunchDaemons/com.umba.qwen-relay.plist "$CONTEXT_DIR/launchdaemons/" 2>/dev/null || true
|
|
cp -p /Library/LaunchDaemons/qwen_relay.py "$CONTEXT_DIR/launchdaemons/" 2>/dev/null || true
|
|
launchctl print "$WEB_LABEL" >"$CONTEXT_DIR/launchctl-web.txt" 2>&1 || true
|
|
launchctl print system/com.umba.qwen-relay >"$CONTEXT_DIR/launchctl-qwen-relay.txt" 2>&1 || true
|
|
mount >"$CONTEXT_DIR/mounts.txt" 2>&1 || true
|
|
df -h >"$CONTEXT_DIR/filesystems.txt" 2>&1 || true
|
|
printf '%s\n' 'Context files are informational; Starthere.sh does not install privileged services.' >"$CONTEXT_DIR/README.txt"
|
|
|
|
printf 'Snapshotting global voice state with SQLite backup API...\n'
|
|
"$GLOBAL_PYTHON" "$SOURCE/backup/restore_support.py" snapshot-global \
|
|
--source "$GLOBAL_ROOT" --output "$STAGING_DIR/global-state" --original-root "$GLOBAL_ROOT"
|
|
(cd "$STAGING_DIR/global-state" && verify_hashes GLOBAL-STATE-SHA256SUMS)
|
|
|
|
start_writer_watch() {
|
|
local flag="$STAGING_DIR/ACTIVE-WRITER"
|
|
(
|
|
while :; do
|
|
if ! check_active_writers >/dev/null 2>&1; then
|
|
printf 'writer detected during archive\n' >"$flag"
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
) &
|
|
WRITER_WATCH_PID="$!"
|
|
}
|
|
|
|
stop_writer_watch() {
|
|
if [[ -n "$WRITER_WATCH_PID" ]]; then
|
|
kill "$WRITER_WATCH_PID" >/dev/null 2>&1 || true
|
|
wait "$WRITER_WATCH_PID" 2>/dev/null || true
|
|
WRITER_WATCH_PID=""
|
|
fi
|
|
}
|
|
|
|
printf 'Creating full payload archive (%s)...\n' "$ARCHIVE_NAME"
|
|
start_writer_watch
|
|
if ! COPYFILE_DISABLE=1 tar -C "$SOURCE_PARENT" -cf "$ARCHIVE_TMP" "$SOURCE_NAME" 2>"$STAGING_DIR/TAR-WARNINGS.log"; then
|
|
stop_writer_watch
|
|
printf 'tar failed; inspect TAR-WARNINGS.log.\n' >&2
|
|
exit 1
|
|
fi
|
|
stop_writer_watch
|
|
if [[ -f "$STAGING_DIR/ACTIVE-WRITER" ]]; then
|
|
printf 'A writer appeared while archiving; refusing the backup.\n' >&2
|
|
exit 1
|
|
fi
|
|
if grep -Eiq 'file changed|error|failed|cannot|permission denied' "$STAGING_DIR/TAR-WARNINGS.log"; then
|
|
printf 'tar reported a consistency/read warning; refusing backup.\n' >&2
|
|
cat "$STAGING_DIR/TAR-WARNINGS.log" >&2
|
|
exit 1
|
|
fi
|
|
mv "$ARCHIVE_TMP" "$ARCHIVE_FINAL"
|
|
printf 'Computing comprehensive SHA-256 manifest...\n'
|
|
"$GLOBAL_PYTHON" "$SOURCE/backup/restore_support.py" write-manifest --root "$STAGING_DIR"
|
|
(cd "$STAGING_DIR" && verify_hashes SHA256SUMS)
|
|
(cd "$STAGING_DIR/global-state" && verify_hashes GLOBAL-STATE-SHA256SUMS)
|
|
tar -tf "$ARCHIVE_FINAL" >/dev/null
|
|
chmod 600 "$ARCHIVE_FINAL" "$STAGING_DIR/SHA256SUMS" "$STAGING_DIR/GIT-DIFF.patch" "$STAGING_DIR/GIT-DIFF-CACHED.patch" 2>/dev/null || true
|
|
|
|
mv "$STAGING_DIR" "$FINAL_DIR"
|
|
STAGING_DIR=""
|
|
|
|
# Keep the cleanup trap until service restoration and top-level pointers finish.
|
|
ROOT_LAUNCHER_TMP="$DESTINATION/.Starthere.sh.tmp.$$"
|
|
cat >"$ROOT_LAUNCHER_TMP" <<EOF
|
|
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
ROOT="\$(cd -- "\$(dirname -- "\${BASH_SOURCE[0]}")" && pwd -P)"
|
|
LATEST="\$(find "\$ROOT" -maxdepth 1 -type d -name 'UmbraAudio-*' ! -name '.*' -print | sort | tail -n 1)"
|
|
[[ -n "\$LATEST" && -x "\$LATEST/Starthere.sh" ]] || { printf 'No completed backup found under %s\n' "\$ROOT" >&2; exit 1; }
|
|
exec "\$LATEST/Starthere.sh" "\$@"
|
|
EOF
|
|
chmod 700 "$ROOT_LAUNCHER_TMP"
|
|
mv "$ROOT_LAUNCHER_TMP" "$DESTINATION/Starthere.sh"
|
|
printf '%s\n' "$FINAL_DIR" >"$DESTINATION/LATEST.tmp.$$"
|
|
mv "$DESTINATION/LATEST.tmp.$$" "$DESTINATION/LATEST"
|
|
|
|
restart_services
|
|
record_service_state
|
|
write_service_state "$FINAL_DIR/SERVICE-STATE-AFTER.txt"
|
|
"$GLOBAL_PYTHON" "$SOURCE/backup/restore_support.py" write-manifest --root "$FINAL_DIR"
|
|
(cd "$FINAL_DIR" && verify_hashes SHA256SUMS)
|
|
trap - EXIT
|
|
release_lock
|
|
|
|
printf '\nBackup complete.\n'
|
|
printf 'Location: %s\n' "$FINAL_DIR"
|
|
printf 'Archive: %s\n' "$FINAL_DIR/$ARCHIVE_NAME"
|
|
printf 'Global state snapshot: %s/global-state\n' "$FINAL_DIR"
|
|
printf 'Restore: %s/Starthere.sh --target /path/to/restore\n' "$FINAL_DIR"
|