UmbraDatastore/migrations/0006_node_removal_and_assignments.sql

-- 0006: a node can be removed from the fleet for good.
--
-- Adds `removed` to the node status vocabulary, and creates `job_assignments`, which is
-- how the controller hands work to a node from now on.
--
-- Two things here are deliberate and worth reading before changing either.

-- ---------------------------------------------------------------------------
-- 1. `removed`
-- ---------------------------------------------------------------------------
--
-- Distinct from `retired`, and the distinction is the point:
--
--   retired   the machine is out of service but the node is a record of it. It can be
--             returned to service by an administrator, and it still holds devices.
--   removed   the machine is gone. Credentials revoked, devices and heartbeats deleted,
--             and the row kept only because `job_events.node_id` refers to it.
--
-- The row cannot be deleted. `job_events.node_id` and `node_pairing_codes.used_by_node_id`
-- both reference `compute_nodes` ON DELETE NO ACTION, so the DELETE is refused outright;
-- changing them to SET NULL does not help, because that is an UPDATE of `job_events` and
-- `job_events_append_only` raises on every update by design. Removing the row would mean
-- either losing the record of what the machine did, or weakening the guarantee that the
-- event trail is immutable. The trail is worth more than a tidy DELETE.
--
-- `removed` is terminal. There is no administrative path back out of it, so a machine
-- written off cannot be quietly resurrected - and a replacement machine pairs under a new
-- name derived from its own MAC, which is what makes the tombstone harmless.

ALTER TABLE public.compute_nodes DROP CONSTRAINT IF EXISTS compute_nodes_status_check;
ALTER TABLE public.compute_nodes ADD CONSTRAINT compute_nodes_status_check
    CHECK (status = ANY (ARRAY[
        'provisioning', 'online', 'draining', 'offline', 'retired', 'removed'
    ]));

-- A removed node must never be resurrected by a heartbeat. `heartbeat` promotes
-- `provisioning` and `offline` to `online`, and a removed node is neither, so the
-- existing CASE already leaves it alone. Asserting it here means a future edit to that
-- CASE cannot quietly start reviving removed machines, which is the failure this status
-- exists to prevent and the one that would be hardest to notice.

CREATE OR REPLACE FUNCTION app.assert_removed_stays_removed()
RETURNS trigger LANGUAGE plpgsql AS $body$
BEGIN
    IF OLD.status = 'removed' AND NEW.status <> 'removed' THEN
        RAISE EXCEPTION
            'a removed node cannot return to service (was %, asked for %)',
            OLD.status, NEW.status;
    END IF;
    RETURN NEW;
END;
$body$;

DROP TRIGGER IF EXISTS compute_nodes_removed_is_terminal ON public.compute_nodes;
CREATE TRIGGER compute_nodes_removed_is_terminal
    BEFORE UPDATE OF status ON public.compute_nodes
    FOR EACH ROW EXECUTE FUNCTION app.assert_removed_stays_removed();

-- ---------------------------------------------------------------------------
-- 2. job_assignments
-- ---------------------------------------------------------------------------
--
-- The controller assigns work; a node no longer claims it. `jobs.lease_owner` said who
-- held a job and it could only be one node, which cannot express a job given to several
-- machines at once, and cannot say which of them finished it.
--
-- A job is complete when the controller holds the artifacts that show it is. That makes
-- the assignment - the attempt - a separate thing from the job - the objective - which is
-- what this table is: one row per (job, node, attempt). Several rows may be open for one
-- job at once; the first whose artifacts verify wins and the rest are cancelled.
--
-- `state` is deliberately not the job's status. A job an assignment is working on is
-- `running` whatever the assignment says, and a `succeeded` assignment is not a completed
-- job until `artifacts_verified_at` is set. Collapsing the two is how a node's word
-- becomes the definition of done.

CREATE TABLE IF NOT EXISTS public.job_assignments (
    id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
    job_id uuid NOT NULL REFERENCES public.jobs (id) ON DELETE CASCADE,
    node_id uuid NOT NULL REFERENCES public.compute_nodes (id) ON DELETE RESTRICT,
    state text NOT NULL DEFAULT 'assigned',
    attempt integer NOT NULL DEFAULT 1,
    assigned_at timestamptz NOT NULL DEFAULT now(),
    started_at timestamptz,
    finished_at timestamptz,
    lease_expires_at timestamptz,
    artifacts_verified_at timestamptz,
    result jsonb NOT NULL DEFAULT '{}'::jsonb,
    error text,
    CHECK (state = ANY (ARRAY[
        'assigned',    -- handed to the node, not yet picked up
        'running',     -- the node says it started
        'succeeded',   -- the node says it finished; NOT yet verified
        'failed',      -- the node gave up, or its lease ran out
        'cancelled'    -- superseded: another node won, or an administrator stopped it
    ])),
    CHECK (finished_at IS NULL OR finished_at >= assigned_at)
);

-- One open assignment per (job, node): a node cannot be handed the same job twice at the
-- same time, but may be handed it again after a failure, which is what `attempt` counts.
CREATE UNIQUE INDEX IF NOT EXISTS job_assignments_open_per_node
    ON public.job_assignments (job_id, node_id)
    WHERE state IN ('assigned', 'running', 'succeeded');

CREATE INDEX IF NOT EXISTS job_assignments_by_node
    ON public.job_assignments (node_id, state);

CREATE INDEX IF NOT EXISTS job_assignments_by_job
    ON public.job_assignments (job_id);

-- The two questions the controller asks constantly: what is this node to do, and what is
-- still outstanding on this job. Both are served by the indexes above; this one covers
-- "which nodes are overdue", which is the reassignment path.
CREATE INDEX IF NOT EXISTS job_assignments_lease
    ON public.job_assignments (lease_expires_at)
    WHERE state IN ('assigned', 'running');

-- ---------------------------------------------------------------------------
-- 3. Recorded, not inferred
-- ---------------------------------------------------------------------------
--
-- `artifacts_verified_at` is the whole of "a job is complete when the controller has the
-- artifacts to show it". It is set by the verification step and by nothing else, so the
-- question "is this job really done" has one answer with one writer, rather than being
-- inferred from an assignment's state and a hopeful join against `job_artifacts`.

COMMENT ON COLUMN public.job_assignments.artifacts_verified_at IS
    'Set only by artifact verification. NULL on a succeeded assignment means the node '
    'reported success and the controller has not yet confirmed it with artifacts; the job '
    'is NOT complete while this is NULL.';