680 lines
28 KiB
Bash
Executable File
680 lines
28 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Copyright 2026 Umbra Tools (umbratools.com)
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
#
|
|
# Set up and verify the node-agent service account for the compute proof of concept.
|
|
#
|
|
# sudo poc/poc_node_user.sh setup create umbra-poc, install the agent (needs root)
|
|
# sudo poc/poc_node_user.sh update install a new agent version, with a way back (needs root)
|
|
# poc/poc_node_user.sh verify assert the boundary holds (needs sudo -u)
|
|
# poc/poc_node_user.sh run verify, then run the whole PoC
|
|
# poc/poc_node_user.sh status what exists right now
|
|
# poc/poc_node_user.sh wait [url] [n] wait for a controller, then stop
|
|
#
|
|
# Why an account at all, when the agent is one file and no dependencies: because
|
|
# the boundary worth proving is that a worker on a node has *no* database
|
|
# credential and no access to the controller's files. Running the agent as
|
|
# `umbra` would make that true by assertion; running it as somebody else makes it
|
|
# true by construction. The account is the test.
|
|
#
|
|
# The agent is stdlib-only, so it needs no virtualenv and no access to
|
|
# /home/umbra. That is the whole reason the setup is this short: no group
|
|
# memberships, no path grants, nothing to widen later.
|
|
#
|
|
# Two things `run` needs from the host, both about privilege and neither obvious:
|
|
#
|
|
# 1. `sudo -n -u umbra-poc` must work without a password. It is run from inside
|
|
# a loop, so a prompt would hang rather than fail.
|
|
# 2. sudo must carry the node key through to the agent. sudo resets the
|
|
# environment by policy, so `UMBRA_NODE_KEY` does not arrive unless the
|
|
# sudoers policy allows it. The demonstration checks this before it starts
|
|
# and, if the key will not survive, sends it down a pipe instead - so the
|
|
# run succeeds either way, and says which way it used. `verify` reports it.
|
|
#
|
|
# Nothing here touches the deployment database or the deployment service.
|
|
|
|
set -euo pipefail
|
|
|
|
# Overridable so the checks can be exercised against a throwaway account while
|
|
# developing the script, and so a real deployment that already has a service user
|
|
# can point at it rather than creating a second one.
|
|
AGENT_USER="${UMBRA_AGENT_USER:-umbra-poc}"
|
|
AGENT_HOME="${UMBRA_AGENT_HOME:-/var/lib/umbra-poc}"
|
|
AGENT_DIR="${UMBRA_AGENT_DIR:-/opt/umbra/node-agent}"
|
|
AGENT_SRC="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/umbratrans_web/node_agent.py"
|
|
ENV_FILE="/home/umbra/.config/umbratrans/umbratrans.env"
|
|
REPO_DIR="/home/umbra/src/UmbraTrans"
|
|
CONFIG="$AGENT_HOME/node.json"
|
|
|
|
# Facts about this host, discovered rather than assumed. Overridable so the
|
|
# script is not welded to one machine.
|
|
#
|
|
# Two interpreters, and they are not interchangeable:
|
|
#
|
|
# PYTHON_BIN the *agent's* interpreter. /usr/bin/python3, outside any
|
|
# home directory, because umbra-poc cannot reach umbra's.
|
|
# CONTROLLER_PY the *controller's*. Inside umbra's home and inside the
|
|
# project venv, which is fine: the controller runs as umbra.
|
|
# Resolved explicitly rather than shelling out to `uv`, which
|
|
# is not on PATH in a non-login shell and produced
|
|
# "exec: uv: not found" the first time this ran unattended.
|
|
CONTROLLER_URL="${UMBRA_CONTROLLER_URL:-http://127.0.0.1:8543}"
|
|
PYTHON_BIN="${UMBRA_PYTHON_BIN:-/usr/bin/python3}"
|
|
|
|
pass_count=0
|
|
fail_count=0
|
|
|
|
# The controller's interpreter: the project venv if it is there, otherwise
|
|
# whatever python3 is on PATH. The controller runs as umbra, so reading the venv
|
|
# is expected; it is only the *agent* that must not need it.
|
|
resolve_controller_python() {
|
|
local repo venv
|
|
repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
venv="$repo/.venv/bin/python"
|
|
if [[ -x "$venv" ]]; then
|
|
printf '%s' "$venv"
|
|
elif command -v python3 >/dev/null 2>&1; then
|
|
command -v python3
|
|
else
|
|
printf ''
|
|
fi
|
|
}
|
|
CONTROLLER_PY="$(resolve_controller_python)"
|
|
|
|
bold() { printf '\033[1m%s\033[0m\n' "$*"; }
|
|
ok() { printf ' \033[32mok\033[0m %s\n' "$*"; pass_count=$((pass_count + 1)); }
|
|
bad() { printf ' \033[31mFAIL\033[0m %s\n' "$*"; fail_count=$((fail_count + 1)); }
|
|
note() { printf ' %s\n' "$*"; }
|
|
head_() { printf '\n\033[1m%s\033[0m\n' "$*"; }
|
|
|
|
# -n: fail fast rather than prompting. A prompt inside a loop of checks turns
|
|
# into a hang, and worse, into a *false pass* - see require_agent_runner.
|
|
as_agent() { sudo -n -u "$AGENT_USER" -- "$@"; }
|
|
|
|
# A check that passes because its mechanism failed is worse than a check that
|
|
# fails. When `sudo -u` cannot run, the positive checks ("can read the agent")
|
|
# correctly fail, but every negative check ("cannot read the DSN") passes for the
|
|
# wrong reason - the test never ran, and `test -r` on an unreadable file returns
|
|
# false, which is exactly what "correctly denied" looks like. So the runner is
|
|
# confirmed once, up front, and a failure there aborts rather than reporting.
|
|
# "no" and "could not check" are different answers and a status display that
|
|
# conflates them will report a boundary as holding when nothing was tested.
|
|
agent_can_read() {
|
|
if ! as_agent true 2>/dev/null; then
|
|
printf 'unknown (no privilege drop)'
|
|
return
|
|
fi
|
|
if as_agent test -r "$1" 2>/dev/null; then printf 'yes'; else printf 'no'; fi
|
|
}
|
|
|
|
require_agent_runner() {
|
|
local actual
|
|
if ! actual="$(as_agent id -un 2>/dev/null)"; then
|
|
bad "cannot run commands as $AGENT_USER"
|
|
note "sudo -n -u $AGENT_USER failed. Every \"must not reach\" check below"
|
|
note "would pass for the wrong reason, so this run is abandoned rather than"
|
|
note "than reported. Either run this with a valid sudo credential, or add"
|
|
note "a NOPASSWD rule for 'sudo -n -u $AGENT_USER'."
|
|
return 1
|
|
fi
|
|
if [[ "$actual" != "$AGENT_USER" ]]; then
|
|
bad "sudo -u $AGENT_USER actually ran as '$actual'"
|
|
return 1
|
|
fi
|
|
ok "can run commands as $AGENT_USER (id -un = $actual)"
|
|
return 0
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# wait for the controller
|
|
# ---------------------------------------------------------------------------
|
|
#
|
|
# Bounded retry with visible progress, because "no controller" has two very
|
|
# different causes and only one of them is fixed by waiting: the process is
|
|
# still starting, or it is not there and nothing is going to bring it. An
|
|
# immediate failure is unhelpful for the first; an unbounded wait is harmful for
|
|
# the second. So: poll, log every attempt, back off to a cap, and stop.
|
|
#
|
|
# Each attempt records *why* it failed, because the three reasons need different
|
|
# responses and a bare "unreachable" does not distinguish them:
|
|
#
|
|
# connection refused the process is not listening; usually still starting
|
|
# name resolution a typo in the URL, or no DNS for that host
|
|
# HTTP 500 the controller is up and broken; waiting may not help,
|
|
# and the log is where the answer is
|
|
|
|
WAIT_ATTEMPTS="${UMBRA_WAIT_ATTEMPTS:-12}"
|
|
WAIT_INTERVAL="${UMBRA_WAIT_INTERVAL:-1}"
|
|
WAIT_MAX_INTERVAL="${UMBRA_WAIT_MAX_INTERVAL:-10}"
|
|
WAIT_LOG_FILE="${UMBRA_WAIT_LOG:-}"
|
|
|
|
# Every line of the wait goes to stderr, so a caller running this in a command
|
|
# substitution does not capture the progress as if it were a result. When a log
|
|
# file is set it gets a copy. Written with a function rather than a duplicated
|
|
# file descriptor, because juggling fd 3 across a loop and two early returns is
|
|
# how a diagnostic ends up printing "Bad file descriptor" when it is the one thing
|
|
# you most need to read.
|
|
wait_say() {
|
|
if [[ -n "$WAIT_LOG_FILE" ]]; then
|
|
printf '%s\n' "$*" | tee -a "$WAIT_LOG_FILE" >&2
|
|
else
|
|
printf '%s\n' "$*" >&2
|
|
fi
|
|
}
|
|
|
|
# The host part of a URL, without the port. `getent hosts 127.0.0.1:8543` fails
|
|
# on the port, and a literal address reported as "does not resolve" sends whoever
|
|
# is reading it to look at DNS when the actual problem is nothing listening.
|
|
url_host() {
|
|
local rest="${1#*://}"
|
|
rest="${rest%%/*}"
|
|
printf '%s' "${rest%%:*}"
|
|
}
|
|
|
|
# Accept either a base URL or a health URL. Callers reasonably pass
|
|
# http://127.0.0.1:8543 and reasonably expect that to work, and probing it
|
|
# verbatim 404s and reports a healthy controller as broken.
|
|
health_url() {
|
|
case "$1" in
|
|
*/api/health) printf '%s' "$1" ;;
|
|
*/) printf '%s' "${1}api/health" ;;
|
|
*/*) printf '%s/api/health' "$1" ;;
|
|
*) printf '%s/api/health' "$1" ;;
|
|
esac
|
|
}
|
|
|
|
# Ask the controller's own health endpoint and classify the answer.
|
|
probe_controller() {
|
|
local url host code
|
|
url="$(health_url "$1")"
|
|
host="$(url_host "$url")"
|
|
host="$(url_host "$url")"
|
|
if curl -fsS --max-time 4 "$url" >/dev/null 2>&1; then
|
|
printf 'up'
|
|
return 0
|
|
fi
|
|
code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 4 "$url" 2>/dev/null || true)"
|
|
if [[ "$code" == "404" ]]; then
|
|
# Something is listening and it is not the health endpoint. Usually the
|
|
# wrong path, and "connection refused" would send the reader to the wrong
|
|
# place entirely.
|
|
printf 'answering 404 (wrong path? %s)' "$url"
|
|
return 1
|
|
fi
|
|
if [[ "$code" != "000" ]]; then
|
|
# Up and unhappy. The controller's own log is where the reason is, and
|
|
# waiting may not be the fix.
|
|
printf 'answering HTTP %s' "$code"
|
|
return 1
|
|
fi
|
|
if [[ "$host" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || [[ "$host" == "localhost" ]]; then
|
|
# A literal address or localhost: nothing is listening, or a firewall is
|
|
# dropping it. "Refused" is the far more common of the two.
|
|
printf 'connection refused'
|
|
elif getent hosts "$host" >/dev/null 2>&1; then
|
|
printf 'connection refused'
|
|
else
|
|
printf 'host does not resolve'
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
do_wait() {
|
|
local url="${1:-$CONTROLLER_URL}"
|
|
local attempts="${2:-$WAIT_ATTEMPTS}"
|
|
|
|
wait_say ""
|
|
wait_say "Waiting for the controller at $(health_url "$url")"
|
|
local started elapsed interval attempt reason
|
|
started=$(date +%s)
|
|
interval="$WAIT_INTERVAL"
|
|
for (( attempt = 1; attempt <= attempts; attempt++ )); do
|
|
elapsed=$(( $(date +%s) - started ))
|
|
reason="$(probe_controller "$url" || true)"
|
|
if [[ "$reason" == "up" ]]; then
|
|
wait_say " ok controller is up after ${attempt} attempt(s), ${elapsed}s"
|
|
return 0
|
|
fi
|
|
if (( attempt < attempts )); then
|
|
wait_say " .. attempt ${attempt}/${attempts} at +${elapsed}s: ${reason}; retrying in ${interval}s"
|
|
else
|
|
wait_say " .. attempt ${attempt}/${attempts} at +${elapsed}s: ${reason}"
|
|
fi
|
|
sleep "$interval"
|
|
if (( interval < WAIT_MAX_INTERVAL )); then
|
|
interval=$(( interval * 2 ))
|
|
(( interval > WAIT_MAX_INTERVAL )) && interval="$WAIT_MAX_INTERVAL"
|
|
fi
|
|
done
|
|
elapsed=$(( $(date +%s) - started ))
|
|
|
|
wait_say ""
|
|
wait_say " FAIL no controller after ${attempts} attempt(s) over ${elapsed}s"
|
|
# A likely cause: the controller is up somewhere else. The PoC runs on 8543
|
|
# and the deployment on 8442, and asking for the wrong one is the single most
|
|
# likely reason for being here.
|
|
local other
|
|
for other in ${UMBRA_ALT_CONTROLLERS:-http://127.0.0.1:8543 http://127.0.0.1:8442}; do
|
|
if [[ "$(health_url "$other")" != "$(health_url "$url")" ]] && [[ "$(probe_controller "$other" || true)" == "up" ]]; then
|
|
wait_say " note: a controller IS answering at ${other}"
|
|
wait_say " you asked for ${url}. Set UMBRA_CONTROLLER_URL to use it."
|
|
fi
|
|
done
|
|
wait_say ""
|
|
wait_say " start one with:"
|
|
wait_say " UMBRATRANS_DATABASE_DSN=postgresql:///umbratrans_scratch \\"
|
|
wait_say " UMBRA_ENV=test uv run python -m umbratrans_web serve \\"
|
|
wait_say " --host 127.0.0.1 --port ${url##*:} --repository postgres"
|
|
wait_say ""
|
|
wait_say " or wait longer: UMBRA_WAIT_ATTEMPTS=40 $0 wait"
|
|
return 1
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# setup
|
|
# ---------------------------------------------------------------------------
|
|
|
|
do_setup() {
|
|
if [[ $EUID -ne 0 ]]; then
|
|
echo "setup must run as root: sudo $0 setup" >&2
|
|
return 2
|
|
fi
|
|
if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then
|
|
echo "no python at $PYTHON_BIN; set UMBRA_PYTHON_BIN" >&2
|
|
return 2
|
|
fi
|
|
"$PYTHON_BIN" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 10) else 1)' || {
|
|
echo "$PYTHON_BIN is older than 3.10 and the agent needs it" >&2
|
|
return 2
|
|
}
|
|
|
|
bold "Creating $AGENT_USER"
|
|
|
|
if id "$AGENT_USER" >/dev/null 2>&1; then
|
|
note "$AGENT_USER already exists; leaving the account alone"
|
|
else
|
|
# --system : no aging, no mail, a service account rather than a person
|
|
# --groups "" : no supplementary groups. This is the load-bearing part -
|
|
# group `umbra` would grant read on the repo, group `users`
|
|
# would grant other people's world-readable files, and
|
|
# neither is needed for an agent that is one stdlib file.
|
|
#
|
|
# The exception is `video`, added by
|
|
# poc/nvidia-device-access.sh rather than here, because it
|
|
# grants a *device* and not a filesystem. The agent runs
|
|
# `nvidia-smi` to measure the card, and nvidia-smi opens
|
|
# /dev/nvidia0, /dev/nvidiactl and /dev/nvidia-uvm - so
|
|
# without it the agent starts, heartbeats, and reports a
|
|
# machine with no GPU. It fails quietly, as a scheduling
|
|
# problem rather than a permissions one.
|
|
#
|
|
# --nologin : there is no reason to sign in as a worker
|
|
useradd \
|
|
--system \
|
|
--home-dir "$AGENT_HOME" \
|
|
--no-create-home \
|
|
--shell /usr/sbin/nologin \
|
|
--groups "" \
|
|
"$AGENT_USER"
|
|
ok "created $AGENT_USER with no supplementary groups and no login shell"
|
|
fi
|
|
|
|
install -d -o "$AGENT_USER" -g "$AGENT_USER" -m 0750 "$AGENT_HOME"
|
|
ok "$AGENT_HOME owned by $AGENT_USER, mode 0750"
|
|
|
|
install -d -o root -g root -m 0755 "$AGENT_DIR"
|
|
install -o root -g root -m 0755 "$AGENT_SRC" "$AGENT_DIR/node_agent.py"
|
|
ok "agent installed at $AGENT_DIR/node_agent.py (root-owned, world-readable)"
|
|
|
|
# A config with no key in it. The key travels in the environment, so this file
|
|
# holds nothing worth protecting and 0640 is belt-and-braces rather than the
|
|
# thing keeping the secret out of a backup.
|
|
if [[ ! -f "$CONFIG" ]]; then
|
|
cat > "$CONFIG" <<JSON
|
|
{
|
|
"controller_url": "$CONTROLLER_URL",
|
|
"node_name": "poc-agent-node",
|
|
"kinds": ["sleep"],
|
|
"lease_seconds": 30,
|
|
"poll_interval_seconds": 0.4
|
|
}
|
|
JSON
|
|
chown "$AGENT_USER:$AGENT_USER" "$CONFIG"
|
|
chmod 0640 "$CONFIG"
|
|
ok "config written to $CONFIG (no key in it)"
|
|
else
|
|
note "$CONFIG exists; leaving it alone"
|
|
fi
|
|
|
|
printf '\n'
|
|
note "controller: $CONTROLLER_URL"
|
|
note "next: $0 verify"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# update
|
|
# ---------------------------------------------------------------------------
|
|
#
|
|
# `setup` installs the agent too, so it has always doubled as an upgrade. It is
|
|
# idempotent and it leaves the account alone, which is most of what is needed -
|
|
# but it installs unconditionally, which is the one thing an upgrade path should
|
|
# not do. A broken file replaces a working agent on every node at once and there
|
|
# is no way back to the one that worked.
|
|
#
|
|
# So an update: names what it is replacing, keeps the old file, checks the new
|
|
# one can actually run *as the agent* before declaring success, and puts the old
|
|
# one back if it cannot. A version bump is not proof of anything; the agent
|
|
# starting is.
|
|
|
|
#: What version an installed agent reports. Read from the file rather than
|
|
#: tracked separately, because a number stored beside the thing it describes
|
|
#: drifts from it, and then the version is wrong in the one place you would look
|
|
#: to find out.
|
|
installed_agent_version() {
|
|
local file="${1:-$AGENT_DIR/node_agent.py}"
|
|
[[ -r "$file" ]] || { printf 'none'; return; }
|
|
grep -m1 -oE '^AGENT_VERSION *= *"[^"]+"' "$file" 2>/dev/null \
|
|
| sed -E 's/.*"(.*)"/\1/' || printf 'unknown'
|
|
}
|
|
|
|
do_update() {
|
|
[[ "${EUID:-$(id -u)}" -eq 0 ]] || {
|
|
bad "update must run as root: sudo $0 update"
|
|
return 2
|
|
}
|
|
local source="${1:-$AGENT_SRC}"
|
|
if [[ ! -r "$source" ]]; then
|
|
bad "no agent source at $source"
|
|
return 2
|
|
fi
|
|
if ! grep -q '^AGENT_VERSION' "$source"; then
|
|
bad "$source has no AGENT_VERSION; that is not the agent file"
|
|
return 2
|
|
fi
|
|
|
|
local from to
|
|
from="$(installed_agent_version)"
|
|
to="$(installed_agent_version "$source")"
|
|
|
|
bold "Updating the node agent"
|
|
note "installed : $from $AGENT_DIR/node_agent.py"
|
|
note "source : $to $source"
|
|
if [[ "$from" == "$to" ]]; then
|
|
note "same version; checking it still matches byte for byte"
|
|
if cmp -s "$source" "$AGENT_DIR/node_agent.py"; then
|
|
ok "already up to date ($to)"
|
|
return 0
|
|
fi
|
|
note "the files differ despite the same version number; installing anyway"
|
|
fi
|
|
|
|
# Keep the old one. Not as a backup that accumulates, but as the one way out:
|
|
# if the new agent will not start, this is what gets restored.
|
|
if [[ -r "$AGENT_DIR/node_agent.py" ]]; then
|
|
install -o root -g root -m 0755 "$AGENT_DIR/node_agent.py" "$AGENT_DIR/node_agent.py.previous"
|
|
note "previous version kept at $AGENT_DIR/node_agent.py.previous"
|
|
fi
|
|
|
|
install -o root -g root -m 0755 "$source" "$AGENT_DIR/node_agent.py"
|
|
ok "installed $to"
|
|
|
|
# The check that matters: not "the file parses" - the file the agent actually
|
|
# runs, as the account that runs it, on this host, with the interpreter that
|
|
# will run it. A file that imports cleanly under umbra's virtualenv may not
|
|
# even be readable by the account the agent runs as.
|
|
if as_agent "$PYTHON_BIN" "$AGENT_DIR/node_agent.py" --help >/dev/null 2>&1; then
|
|
ok "the new agent runs as $AGENT_USER and reports its usage"
|
|
else
|
|
bad "the new agent will not start as $AGENT_USER; restoring $AGENT_DIR/node_agent.py.previous"
|
|
if [[ -r "$AGENT_DIR/node_agent.py.previous" ]]; then
|
|
install -o root -g root -m 0755 "$AGENT_DIR/node_agent.py.previous" "$AGENT_DIR/node_agent.py"
|
|
note "restored $from"
|
|
else
|
|
note "there was no previous version to restore - the node has no working agent"
|
|
fi
|
|
return 1
|
|
fi
|
|
|
|
printf '\n'
|
|
if ! "$0" verify >/dev/null 2>&1; then
|
|
bad "the boundary checks did not pass with the new agent in place"
|
|
note "run: $0 verify to see which one failed"
|
|
else
|
|
ok "the boundary checks still pass"
|
|
fi
|
|
note "nodes pick this up when their agent restarts; a running agent keeps the old file"
|
|
note "open. Restart it with whatever supervises it, or by hand."
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# verify
|
|
# ---------------------------------------------------------------------------
|
|
|
|
do_verify() {
|
|
bold "Node-agent service account: $AGENT_USER"
|
|
|
|
if ! id "$AGENT_USER" >/dev/null 2>&1; then
|
|
bad "$AGENT_USER does not exist yet"
|
|
note "run: sudo $0 setup"
|
|
return 1
|
|
fi
|
|
|
|
if ! require_agent_runner; then
|
|
printf '\n\033[31mCannot verify without a working privilege drop; 0 checks attempted.\033[0m\n'
|
|
return 1
|
|
fi
|
|
|
|
head_ "The account itself"
|
|
local shell groups
|
|
shell="$(getent passwd "$AGENT_USER" | cut -d: -f7)"
|
|
groups="$(id -nG "$AGENT_USER" 2>/dev/null | tr ' ' ',')"
|
|
note "shell : $shell"
|
|
note "groups : ${groups:-<none>}"
|
|
if [[ "$shell" == */nologin || "$shell" == "false" ]]; then
|
|
ok "cannot sign in as it"
|
|
else
|
|
bad "shell is '$shell'; expected nologin"
|
|
fi
|
|
# One supplementary group is now expected: `video`, and only that one. It
|
|
# grants a *device* and not a filesystem, which is what makes it different in
|
|
# kind from the filesystem groups this check exists to keep out - group
|
|
# `umbra` would grant read on the repository, group `users` would grant other
|
|
# people's world-readable files.
|
|
#
|
|
# It is needed because the agent measures the card with `nvidia-smi`, and
|
|
# traced, nvidia-smi opens /dev/nvidia0, /dev/nvidiactl and /dev/nvidia-uvm.
|
|
# Without it the agent still starts, still heartbeats, and reports a machine
|
|
# with no GPU - so this check and poc/nvidia-device-access.sh agree: the group
|
|
# is added there, and refused anywhere else.
|
|
local extra=""
|
|
# $groups is comma-separated for display, so split on that and not on
|
|
# whitespace - iterating the comma form makes every group look like a group
|
|
# that is not its own name, which reports even the allowed ones as extra.
|
|
local normalised="${groups//,/ }"
|
|
for group in $normalised; do
|
|
[[ "$group" == "$AGENT_USER" || "$group" == "video" ]] || extra="$extra $group"
|
|
done
|
|
if [[ -z "$extra" ]]; then
|
|
ok "no supplementary groups but $groups (video is the GPU, not a filesystem)"
|
|
else
|
|
bad "extra groups:$extra"
|
|
fi
|
|
if id -nG "$AGENT_USER" 2>/dev/null | grep -qw sudo; then
|
|
bad "$AGENT_USER is in the sudo group"
|
|
else
|
|
ok "not in the sudo group"
|
|
fi
|
|
|
|
head_ "What it can reach"
|
|
if as_agent test -r "$AGENT_DIR/node_agent.py"; then
|
|
ok "its own code is readable"
|
|
else
|
|
bad "cannot read $AGENT_DIR/node_agent.py"
|
|
fi
|
|
if as_agent test -r "$CONFIG"; then
|
|
ok "its own config is readable"
|
|
else
|
|
bad "cannot read $CONFIG"
|
|
fi
|
|
if as_agent "$PYTHON_BIN" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 10) else 1)'; then
|
|
ok "the system python runs for it"
|
|
else
|
|
bad "$PYTHON_BIN is unusable as $AGENT_USER"
|
|
fi
|
|
if as_agent "$PYTHON_BIN" "$AGENT_DIR/node_agent.py" --help >/dev/null 2>&1; then
|
|
ok "the agent starts and prints its usage"
|
|
else
|
|
bad "the agent does not start as $AGENT_USER"
|
|
fi
|
|
|
|
head_ "What it must not reach (these are the point)"
|
|
if as_agent test -r "$ENV_FILE"; then
|
|
bad "CAN READ $ENV_FILE <- the database DSN and SMTP credentials"
|
|
else
|
|
ok "cannot read the controller's env file (DSN, SMTP credentials)"
|
|
fi
|
|
if as_agent test -r "$REPO_DIR"; then
|
|
bad "CAN READ $REPO_DIR"
|
|
else
|
|
ok "cannot read the repository"
|
|
fi
|
|
if as_agent test -x /home/umbra; then
|
|
bad "CAN ENTER /home/umbra"
|
|
else
|
|
ok "cannot enter /home/umbra"
|
|
fi
|
|
|
|
head_ "Database access, denied twice over"
|
|
if as_agent psql -d umbratrans -c 'select 1' >/dev/null 2>&1; then
|
|
bad "CAN CONNECT to the database as $AGENT_USER"
|
|
else
|
|
ok "peer authentication refuses the connection"
|
|
fi
|
|
# Even with the DSN it could not connect, because peer auth maps the unix user
|
|
# to a database role and there is no role for it. Checked directly so the
|
|
# refusal is not mistaken for the file permission doing the work.
|
|
if [[ -n "$(psql -d umbratrans -tAc "select rolname from pg_roles where rolname = '$AGENT_USER'" 2>/dev/null)" ]]; then
|
|
bad "a PostgreSQL role named $AGENT_USER exists; peer auth would accept it"
|
|
else
|
|
ok "no PostgreSQL role named $AGENT_USER, so the DSN would not help either"
|
|
fi
|
|
|
|
head_ "The GPU, and what it means"
|
|
local perms
|
|
perms="$(stat -c '%A %U:%G' /dev/nvidiactl 2>/dev/null || echo 'absent')"
|
|
note "/dev/nvidiactl is $perms"
|
|
if [[ "$perms" == crw-rw-rw-* ]]; then
|
|
note "World-accessible. The OS does NOT stop one node from using another's"
|
|
note "share of VRAM - placement does, and nothing below the scheduler does."
|
|
note "A udev rule would make the kernel agree: poc/nvidia-device-access.sh"
|
|
note "apply. Note that it will not separate two agents running as this same"
|
|
note "account; it stops every *other* account on the box, above all the"
|
|
note "controller, which can open the card today."
|
|
else
|
|
note "Restricted to a group. See poc/nvidia-device-access.sh show."
|
|
if ! as_agent nvidia-smi -L >/dev/null 2>&1; then
|
|
bad "$AGENT_USER cannot measure the GPU"
|
|
note "nvidia-smi opens /dev/nvidia0, /dev/nvidiactl and /dev/nvidia-uvm."
|
|
note "Without access the agent still starts and still heartbeats; it"
|
|
note "reports a machine with no GPU. Add it: usermod -aG video $AGENT_USER"
|
|
else
|
|
ok "$AGENT_USER can measure the GPU"
|
|
fi
|
|
fi
|
|
|
|
printf '\n'
|
|
if (( fail_count == 0 )); then
|
|
printf '\033[32m%d check(s) passed, 0 failed.\033[0m\n' "$pass_count"
|
|
return 0
|
|
fi
|
|
printf '\033[31m%d check(s) passed, %d FAILED.\033[0m\n' "$pass_count" "$fail_count"
|
|
return 1
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# run
|
|
# ---------------------------------------------------------------------------
|
|
|
|
do_run() {
|
|
do_verify || true
|
|
head_ "Running the proof of concept with the agent as $AGENT_USER"
|
|
# Wait rather than failing on the first refusal: the controller may be a few
|
|
# seconds behind a restart. Bounded, and every attempt logged, so a genuine
|
|
# absence says so instead of hanging.
|
|
if ! do_wait "$CONTROLLER_URL" "$WAIT_ATTEMPTS"; then
|
|
return 2
|
|
fi
|
|
|
|
if [[ $fail_count -gt 0 ]]; then
|
|
printf '\n\033[33mRunning anyway; the boundary problems above are the interesting part.\033[0m\n'
|
|
fi
|
|
|
|
if [[ -z "$CONTROLLER_PY" ]]; then
|
|
bad "no python for the controller; set one in the venv or on PATH"
|
|
return 2
|
|
fi
|
|
local repo_dir; repo_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
( cd "$repo_dir" && exec "$CONTROLLER_PY" poc/run_poc.py \
|
|
--controller "$CONTROLLER_URL" \
|
|
--agent-user "$AGENT_USER" \
|
|
"$@" )
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# status
|
|
# ---------------------------------------------------------------------------
|
|
|
|
do_status() {
|
|
bold "$AGENT_USER"
|
|
if id "$AGENT_USER" >/dev/null 2>&1; then
|
|
id "$AGENT_USER"
|
|
note "home : $(getent passwd "$AGENT_USER" | cut -d: -f6)"
|
|
note "shell: $(getent passwd "$AGENT_USER" | cut -d: -f7)"
|
|
for path in "$AGENT_DIR/node_agent.py" "$CONFIG" "$ENV_FILE" "$REPO_DIR"; do
|
|
if [[ -r "$path" ]]; then
|
|
note "$(stat -c '%A %U:%G %n' "$path")"
|
|
elif [[ -e "$path" ]]; then
|
|
# Exists, but this user cannot stat it. Reporting "missing" here
|
|
# would be a confident answer to a question nothing was consulted
|
|
# for, which is the failure this script has now hit twice.
|
|
note "exists, not readable by you $path"
|
|
else
|
|
note "missing $path"
|
|
fi
|
|
done
|
|
# The agent's own view of its home, which is the only way to learn what is
|
|
# in a directory this user cannot enter.
|
|
if as_agent true 2>/dev/null; then
|
|
for entry in "$AGENT_HOME" "$CONFIG"; do
|
|
note "as $AGENT_USER: $(as_agent sh -c "test -e '$entry' && echo present || echo absent" 2>/dev/null) $entry"
|
|
done
|
|
else
|
|
note "as $AGENT_USER: unknown (no privilege drop)"
|
|
fi
|
|
printf '\n'
|
|
note "can read agent : $(agent_can_read "$AGENT_DIR/node_agent.py")"
|
|
note "can read env file: $(agent_can_read "$ENV_FILE")"
|
|
else
|
|
note "not created. run: sudo $0 setup"
|
|
fi
|
|
printf '\n'
|
|
note "controller: $(curl -fsS --max-time 3 "$CONTROLLER_URL/api/health" 2>/dev/null | head -c 80 || echo unreachable)"
|
|
}
|
|
|
|
case "${1:-}" in
|
|
setup) shift; do_setup "$@" ;;
|
|
update) shift; do_update "$@" ;;
|
|
verify) shift; do_verify "$@" ;;
|
|
run) shift; do_run "$@" ;;
|
|
wait) shift
|
|
controller="${1:-$CONTROLLER_URL}"
|
|
attempts="${2:-$WAIT_ATTEMPTS}"
|
|
do_wait "$controller" "$attempts" "${UMBRA_WAIT_LOG:-}" ;;
|
|
status) shift; do_status "$@" ;;
|
|
*)
|
|
sed -n '3,20p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
|
|
exit 2
|
|
;;
|
|
esac
|