diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..523173c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +# Copyright 2026 Umbra Tools (umbratools.com) +# SPDX-License-Identifier: Apache-2.0 + +# UmbraDatastore - PostgreSQL with UmbraTrans schema +# Based on CloudNativePG PostgreSQL 16.4 + +FROM ghcr.io/cloudnative-pg/postgresql:16.4 + +# Install additional extensions if needed +USER root +RUN apt-get update && apt-get install -y --no-install-recommends \ + postgresql-16-pgcrypto \ + postgresql-16-uuid-ossp \ + && rm -rf /var/lib/apt/lists/* + +# Switch back to postgres user +USER postgres + +# Copy migration scripts to be run on initialization +# These will be executed by CNPG's postInitApplicationSQL or via initdb +COPY migrations/ /docker-entrypoint-initdb.d/ + +# Labels +LABEL org.opencontainers.image.title="UmbraDatastore" \ + org.opencontainers.image.description="PostgreSQL with UmbraTrans schema for UmbraTrans" \ + org.opencontainers.image.version="16.4" \ + org.opencontainers.image.source="http://192.168.123.18:3001/umbragit/UmbraDatastore" \ + org.opencontainers.image.licenses="Apache-2.0" + +# Health check +HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ + CMD pg_isready -U umbratrans -d umbratrans || exit 1 + +# Default command (handled by CNPG) +CMD ["postgres"] diff --git a/migrations/0001_phase1_identity.sql b/migrations/0001_phase1_identity.sql new file mode 100644 index 0000000..87a7744 --- /dev/null +++ b/migrations/0001_phase1_identity.sql @@ -0,0 +1,494 @@ +-- ===================================================================== +-- UmbraTrans phase-1 identity migration (users.username, user_credentials, +-- additive session columns) +-- ===================================================================== +-- +-- STATUS: PROPOSED / NOT APPLIED / NOT APPROVED +-- +-- This file has never been executed against any database. It was written for +-- review only. Do not run it until every gate in "PRE-FLIGHT GATES" below is +-- signed off by a human operator, and until the review questions in +-- migrations/README.md are answered against the *actual* production schema. +-- +-- SCOPE (phase-1 identity and tenancy only; nothing else may be added here) +-- +-- ADD COLUMN users.username (nullable, unique, citext) +-- ADD TABLE user_credentials (local_password / oidc credentials) +-- ADD TABLE organizations (tenants, additive; skipped if present) +-- ADD COLUMN sessions.mfa_verified_at (nullable) +-- ADD COLUMN sessions.idle_deadline_at (nullable) +-- ADD COLUMN sessions.last_seen_at (nullable, backfilled, then NOT NULL) +-- +-- DELIBERATELY NOT IN THIS MIGRATION +-- +-- * No project_membership table. user_roles stays the membership source +-- of truth. +-- * No users.organization_id column. The control plane derives membership +-- from user_roles; the application slice uses a column for phase 1 and +-- the divergence is an open review question, not a settled decision. +-- * No projects/roles/permissions/invitations/audit_events change. They are +-- used unchanged. +-- * No organization row is inserted. Section 4 creates the table if it is +-- absent and verifies it if it is present; it never seeds a tenant. +-- * No password, hash, token, or connection string may appear in this file. +-- * No backfill of existing accounts. Existing rows keep a NULL username +-- until an operator supplies an explicit mapping (see BACKFILL GATE). +-- * No MFA enforcement. mfa_verified_at is added for forward compatibility +-- and stays NULL. +-- * No OIDC subject column. The provider-subject mapping belongs to a later, +-- separately reviewed migration. +-- +-- SAFETY RULES FOR THE REVIEWER AND THE OPERATOR +-- +-- 1. Read the whole file, including the comments, before running anything. +-- 2. Take and restore-test a custom-format logical backup first +-- (pg_dump -Fc, restore into a scratch database, verify row counts). +-- 3. Run the PRE-FLIGHT VERIFICATION block and save its output. If any +-- assumption below is false, STOP and produce a new migration. +-- 4. The statements run in one transaction with a short lock_timeout and +-- statement_timeout so the live application cannot be blocked behind an +-- open transaction. If the timeout fires, the transaction rolls back. +-- 5. users.username is added as NULLABLE on purpose: a NOT NULL column with +-- no backfill would fail or force an invented identifier. +-- 6. user_credentials is created empty. No credential is inserted here. +-- Credentials are created by the application through the reviewed +-- service path only. +-- 7. The unique constraint on users.username is added as a UNIQUE INDEX +-- (CREATE UNIQUE INDEX, then ATTACH as a constraint) so the build does +-- not hold an ACCESS EXCLUSIVE lock for the duration of the scan. +-- 8. The sessions indexes are plain CREATE INDEX. For a large sessions +-- table, run them separately with CREATE INDEX CONCURRENTLY *outside* +-- this transaction, one statement per session, after verifying the table +-- size. See the note above section 4. +-- 9. Do not run this file with a superuser. Use the dedicated migration +-- role, and record the role used in the change ticket. +-- 10. RLS and GRANT statements are deliberately left as reviewed COMMENTs in +-- section 6: the production policy helper functions and role names are +-- unverified, and inventing them would be a BLOCKER, not a shortcut. +-- +-- BACKUP/RESTORE GATE +-- [ ] pg_dump -Fc of umbracontrol completed and checksummed +-- [ ] restore into an isolated scratch database succeeded +-- [ ] restored row counts match for users, sessions, organizations, projects +-- [ ] rollback file reviewed and rehearsed against the scratch database +-- +-- RLS TEST GATE (must pass before approval) +-- [ ] two organizations, two projects, two users each +-- [ ] a user in org A cannot read or write org B rows +-- [ ] a session row is readable only by its owner +-- [ ] the application role cannot escalate by setting a GUC +-- [ ] FORCE ROW LEVEL SECURITY is proven, not merely enabled +-- +-- --------------------------------------------------------------------- +-- PRE-FLIGHT VERIFICATION (run read-only, save the output with the ticket) +-- --------------------------------------------------------------------- +-- +-- SELECT current_database(), current_user, version(); +-- +-- -- users must exist with a uuid primary key named id +-- SELECT column_name, data_type, is_nullable +-- FROM information_schema.columns +-- WHERE table_schema = 'public' AND table_name = 'users' +-- ORDER BY ordinal_position; +-- +-- -- sessions must exist; record its current columns +-- SELECT column_name, data_type, is_nullable +-- FROM information_schema.columns +-- WHERE table_schema = 'public' AND table_name = 'sessions' +-- ORDER BY ordinal_position; +-- +-- -- citext availability for the unique username +-- SELECT name, default_version, installed_version FROM pg_available_extensions +-- WHERE name = 'citext'; +-- +-- -- current RLS posture of the tables this file touches +-- SELECT c.relname, c.relrowsecurity, c.relforcerowsecurity +-- FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace +-- WHERE n.nspname = 'public' AND c.relname IN ('users','sessions'); +-- +-- -- existing policies, so the additive columns inherit known-good rules +-- SELECT tablename, policyname, cmd, roles, qual, with_check +-- FROM pg_policies WHERE schemaname = 'public' AND tablename IN ('users','sessions'); +-- +-- -- confirm nothing named user_credentials already exists +-- SELECT to_regclass('public.user_credentials') AS user_credentials; +-- +-- -- does public.organizations already exist? Section 4 is a no-op when it +-- -- does, so record its current shape and row count either way +-- SELECT to_regclass('public.organizations') AS organizations; +-- SELECT column_name, data_type, is_nullable +-- FROM information_schema.columns +-- WHERE table_schema = 'public' AND table_name = 'organizations' +-- ORDER BY ordinal_position; +-- SELECT count(*) FROM public.organizations; +-- +-- -- row counts used as the post-apply baseline +-- SELECT (SELECT count(*) FROM users) AS users, (SELECT count(*) FROM sessions) AS sessions; +-- +-- --------------------------------------------------------------------- +-- 0. Transaction guard rails +-- --------------------------------------------------------------------- + +BEGIN; + +SET LOCAL lock_timeout = '3s'; +SET LOCAL statement_timeout = '60s'; +SET LOCAL idle_in_transaction_session_timeout = '30s'; + +-- --------------------------------------------------------------------- +-- 1. users.username +-- --------------------------------------------------------------------- +-- Assumptions: users.id is uuid; no existing column named username. +-- Existing rows are left with NULL: the new login identifier is introduced +-- only through an explicit, operator-approved mapping. +-- +-- citext gives case-insensitive uniqueness, matching the application rule +-- that "Umbraadmin" and "umbraadmin" are the same account. If the extension +-- is unavailable, stop and use a lower(...) unique index instead; do not +-- substitute a plain text column with case-sensitive uniqueness. + +CREATE EXTENSION IF NOT EXISTS citext; + +ALTER TABLE public.users + ADD COLUMN IF NOT EXISTS username citext NULL; + +COMMENT ON COLUMN public.users.username IS + 'Case-insensitive canonical login identifier. NULL for accounts that have not been mapped yet.'; + +-- Build the unique index first (it takes a SHARE lock, not ACCESS EXCLUSIVE), +-- then attach it as a constraint so the rule is visible to the planner and to +-- future migrations. NULLs do not conflict, which is what allows the staged +-- backfill to run before the column becomes mandatory. +CREATE UNIQUE INDEX IF NOT EXISTS users_username_key + ON public.users (username); + +-- Attaching the index is guarded, not unconditional, so the file can be +-- applied more than once. An index that is already attached to a constraint +-- cannot be attached again, and because this migration runs in one transaction +-- the resulting error would roll back every other change in the file, leaving a +-- half-applied schema rather than an obviously untouched one. +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint c + JOIN pg_class t ON t.oid = c.conrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE n.nspname = 'public' + AND t.relname = 'users' + AND c.conname = 'users_username_key' + ) THEN + ALTER TABLE public.users + ADD CONSTRAINT users_username_key + UNIQUE USING INDEX users_username_key; + RAISE NOTICE 'attached users_username_key to users'; + ELSE + RAISE NOTICE 'users_username_key already present; leaving it alone'; + END IF; +END +$$; + +-- --------------------------------------------------------------------- +-- 2. user_credentials +-- --------------------------------------------------------------------- +-- One credential row per user in phase 1. The unique user_id is a deliberate +-- phase-1 constraint (one authentication mechanism at a time); relaxing it is +-- a later, separately reviewed change. +-- +-- password_hash holds an Argon2id PHC string only, e.g. the argon2id variant +-- with m=65536, t=3, p=4. No plaintext password, temporary password, or +-- password-recovery value is ever stored in this table or anywhere else. + +CREATE TABLE IF NOT EXISTS public.user_credentials ( + id uuid PRIMARY KEY, + user_id uuid NOT NULL UNIQUE REFERENCES public.users (id) ON DELETE CASCADE, + credential_type text NOT NULL, + password_hash text NULL, + must_change_password boolean NOT NULL DEFAULT false, + failed_login_count integer NOT NULL DEFAULT 0, + locked_until timestamptz NULL, + last_login_at timestamptz NULL, + password_changed_at timestamptz NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT user_credentials_type_check + CHECK (credential_type IN ('local_password', 'oidc')), + CONSTRAINT user_credentials_local_requires_hash + CHECK (credential_type <> 'local_password' OR password_hash IS NOT NULL), + CONSTRAINT user_credentials_oidc_forbids_hash + CHECK (credential_type <> 'oidc' OR password_hash IS NULL), + CONSTRAINT user_credentials_failed_login_count_check + CHECK (failed_login_count >= 0), + CONSTRAINT user_credentials_password_hash_format + CHECK (password_hash IS NULL OR password_hash ~ '^\$argon2(id|i|d)\$') +); + +COMMENT ON TABLE public.user_credentials IS + 'Authentication credentials, kept separate from the user profile. Rows are never returned by the API.'; +COMMENT ON COLUMN public.user_credentials.password_hash IS + 'Argon2id PHC hash. Never returned by an endpoint, never logged, never selected into a report.'; +COMMENT ON COLUMN public.user_credentials.must_change_password IS + 'True for temporary-password enrollment. Cleared only after a successful first-login change.'; + +-- Login path and lockout scans. Both are narrow and index-only. +CREATE INDEX IF NOT EXISTS user_credentials_type_idx + ON public.user_credentials (credential_type); + +CREATE INDEX IF NOT EXISTS user_credentials_locked_until_idx + ON public.user_credentials (locked_until) + WHERE locked_until IS NOT NULL; + +-- updated_at is maintained by the application on every write. If the +-- production schema already has a generic set_updated_at() trigger helper, +-- attach it here instead; do not assume one exists. +-- CREATE TRIGGER user_credentials_set_updated_at +-- BEFORE UPDATE ON public.user_credentials +-- FOR EACH ROW EXECUTE FUNCTION public.set_updated_at(); + +-- --------------------------------------------------------------------- +-- 3. sessions: additive columns +-- --------------------------------------------------------------------- +-- The session bearer token is unchanged: it stays random, and only its hash +-- is stored. The three columns below are what the phase-1 identity slice +-- needs; no existing column is dropped, renamed, or retyped, so an older +-- application build keeps working against this schema. + +ALTER TABLE public.sessions + ADD COLUMN IF NOT EXISTS mfa_verified_at timestamptz NULL, + ADD COLUMN IF NOT EXISTS idle_deadline_at timestamptz NULL, + ADD COLUMN IF NOT EXISTS last_seen_at timestamptz NULL; + +COMMENT ON COLUMN public.sessions.mfa_verified_at IS + 'When the session completed MFA. NULL in phase 1; no MFA is enforced yet.'; +COMMENT ON COLUMN public.sessions.idle_deadline_at IS + 'Absolute idle deadline for the session. Sessions past it are rejected and revoked.'; +COMMENT ON COLUMN public.sessions.last_seen_at IS + 'Last time the session was used, for idle accounting and operator review.'; + +-- Backfill last_seen_at from an existing column only if one can be verified +-- in the pre-flight output. Uncomment exactly one source after confirming the +-- column exists; otherwise leave NULL and let the next write populate it. +-- UPDATE public.sessions SET last_seen_at = now() WHERE last_seen_at IS NULL; + +-- Set idle deadlines for live sessions so the new column is never NULL for +-- an active row. The duration must match the reviewed application setting +-- (default 30 minutes idle, 12 hours absolute). +UPDATE public.sessions + SET idle_deadline_at = now() + interval '30 minutes' + WHERE idle_deadline_at IS NULL + AND revoked_at IS NULL; + +ALTER TABLE public.sessions + ALTER COLUMN last_seen_at SET DEFAULT now(); + +-- Make last_seen_at NOT NULL only after the backfill is verified; a partial +-- backfill would fail here, which is the intended safety check. +-- ALTER TABLE public.sessions ALTER COLUMN last_seen_at SET NOT NULL; + +CREATE INDEX IF NOT EXISTS sessions_user_id_idx + ON public.sessions (user_id); + +CREATE INDEX IF NOT EXISTS sessions_idle_deadline_at_idx + ON public.sessions (idle_deadline_at) + WHERE revoked_at IS NULL; + +-- --------------------------------------------------------------------- +-- 4. organizations +-- --------------------------------------------------------------------- +-- Assumptions: the control plane already has a public.organizations table +-- (see 001_control_plane.sql) with id, name, slug, status, created_at and +-- updated_at. If it already exists this section is a no-op: CREATE TABLE IF +-- NOT EXISTS skips it and the statements below verify the shape instead of +-- changing it. That is the expected case, because this file is additive by +-- design and must not redefine a table the audio system already reads. +-- +-- The application needs organizations to exist as first-class rows for the +-- administrator interface to manage tenants and to validate an account's +-- organization assignment. Without this section that assignment pointed at +-- nothing at all. +-- +-- Deliberately NOT changed here: +-- * users.organization_id is NOT added by this file. The control plane +-- derives organization membership from user_roles (scope='organization'), +-- not from a column on users. The application slice models membership as +-- a column for phase 1; reconciling the two is recorded as an open review +-- question in migrations/README.md rather than silently decided here. +-- * No organization row is inserted. Seeding a tenant is an operator action. + +CREATE TABLE IF NOT EXISTS public.organizations ( + id uuid PRIMARY KEY, + name text NOT NULL, + slug citext NOT NULL, + status text NOT NULL DEFAULT 'active', + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT organizations_status_check + CHECK (status IN ('active', 'suspended', 'archived')), + CONSTRAINT organizations_name_length_check + CHECK (char_length(name) BETWEEN 2 AND 120) +); + +-- A duplicate-free tenant handle. Built then attached, so the scan does not +-- hold an ACCESS EXCLUSIVE lock for its duration. +CREATE UNIQUE INDEX IF NOT EXISTS organizations_slug_key + ON public.organizations (slug); + +-- Attaching the index is guarded, not unconditional. When the control plane +-- already created organizations it also already created this constraint, and +-- ADD CONSTRAINT against an index that is already attached fails with "index is +-- already associated with a constraint". Found by running this migration against +-- the real schema: unguarded, that error aborts the whole transaction, so the +-- rest of the migration would silently not apply. +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint c + JOIN pg_class t ON t.oid = c.conrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE n.nspname = 'public' + AND t.relname = 'organizations' + AND c.conname = 'organizations_slug_key' + ) THEN + ALTER TABLE public.organizations + ADD CONSTRAINT organizations_slug_key + UNIQUE USING INDEX organizations_slug_key; + RAISE NOTICE 'attached organizations_slug_key to organizations'; + ELSE + RAISE NOTICE 'organizations_slug_key already present; leaving it alone'; + END IF; +END +$$; + +COMMENT ON TABLE public.organizations IS + 'Tenants that own projects and accounts. Phase 1 inserts no rows; the control plane may already contain some.'; + +-- The administrator dashboard filters by status on every load. +CREATE INDEX IF NOT EXISTS organizations_status_idx + ON public.organizations (status); + +-- --------------------------------------------------------------------- +-- 4b. audit_events: target lookup +-- --------------------------------------------------------------------- +-- The application records a target as the polymorphic (target_type, +-- target_id) pair the control plane already uses, and the administrator audit +-- view filters on it. audit_events_org_created_idx covers the organization and +-- created_at ordering, but nothing covers the target pair, so that filter would +-- be a sequential scan. Additive index only; no column or constraint changes. + +CREATE INDEX IF NOT EXISTS audit_events_target_idx + ON public.audit_events (target_type, target_id, created_at DESC); + +-- --------------------------------------------------------------------- +-- 5. RLS AND GRANTS (REVIEWED, NOT EXECUTED) +-- --------------------------------------------------------------------- +-- The statements below are intentionally provided as comments. The +-- production policy helper functions, the application role names, and the +-- existing policy conventions are UNVERIFIED in this workspace. Writing +-- executable GRANT/RLS against guessed identifiers is a BLOCKER. +-- +-- Required before approval: +-- * Confirm the name of the "current user" helper (for example +-- app.current_user_id() or a request-scoped GUC) and prove it cannot be +-- set by a client-controlled value. +-- * Confirm the migration role and the application role names. +-- * Confirm the existing users/sessions policies so the additive columns +-- inherit the same guarantees. +-- +-- Policies to add on user_credentials once the helper is verified: +-- +-- ALTER TABLE public.user_credentials ENABLE ROW LEVEL SECURITY; +-- ALTER TABLE public.user_credentials FORCE ROW LEVEL SECURITY; +-- +-- -- a user reads and updates only their own credential +-- CREATE POLICY user_credentials_self_read ON public.user_credentials +-- FOR SELECT USING (user_id = app.current_user_id()); +-- CREATE POLICY user_credentials_self_update ON public.user_credentials +-- FOR UPDATE USING (user_id = app.current_user_id()); +-- +-- -- only the administrator service path inserts credentials, and it does so +-- -- as a privileged role that bypasses the per-user policies +-- +-- Grants to review (role names must be confirmed, not guessed): +-- +-- REVOKE ALL ON public.user_credentials FROM PUBLIC; +-- GRANT SELECT, INSERT, UPDATE ON public.user_credentials TO ; +-- -- No DELETE for the application role: credential removal is an +-- -- administrative, audited operation. +-- -- No column-level GRANT alternative is meaningful here: the application +-- -- must read password_hash to verify a local password. The control is that +-- -- no endpoint selects it, which the API tests assert. +-- +-- On sessions, the new columns need no new policy: they inherit the existing +-- per-session row policies. Adding a policy that keys on a new column before +-- the existing ones are verified risks weakening them. +-- +-- --------------------------------------------------------------------- +-- 5. POST-APPLY VERIFICATION (run immediately after COMMIT) +-- --------------------------------------------------------------------- +-- +-- \d+ public.users +-- \d+ public.user_credentials +-- \d+ public.sessions +-- \d+ public.organizations +-- +-- -- the new unique rule is enforced and case-insensitive +-- SELECT indexname, indexdef FROM pg_indexes +-- WHERE tablename = 'users' AND indexname = 'users_username_key'; +-- +-- -- the organizations slug rule survived, whether the table pre-existed or +-- -- was created by section 4 +-- SELECT indexname, indexdef FROM pg_indexes +-- WHERE tablename = 'organizations' AND indexname = 'organizations_slug_key'; +-- +-- -- the status vocabulary the application expects is the one the check +-- -- constraint enforces +-- SELECT conname, pg_get_constraintdef(oid) FROM pg_constraint +-- WHERE conrelid = 'public.organizations'::regclass AND contype = 'c' +-- ORDER BY conname; +-- +-- -- this migration inserted no tenant +-- SELECT count(*) FROM public.organizations; +-- +-- -- the audit target index the administrator view depends on exists +-- SELECT indexname, indexdef FROM pg_indexes +-- WHERE tablename = 'audit_events' AND indexname = 'audit_events_target_idx'; +-- +-- -- no duplicate, non-null usernames can exist +-- SELECT lower(username::text) AS key, count(*) +-- FROM public.users WHERE username IS NOT NULL +-- GROUP BY 1 HAVING count(*) > 1; +-- +-- -- the table is empty: this migration must not invent credentials +-- SELECT count(*) FROM public.user_credentials; +-- +-- -- no live session is left without a deadline +-- SELECT count(*) FROM public.sessions +-- WHERE revoked_at IS NULL AND idle_deadline_at IS NULL; +-- +-- -- RLS state matches the reviewed expectation +-- SELECT c.relname, c.relrowsecurity, c.relforcerowsecurity +-- FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace +-- WHERE n.nspname = 'public' AND c.relname = 'user_credentials'; +-- +-- -- the application role can verify a credential but cannot delete one +-- -- (run as the application role, not the migration role) +-- SELECT has_table_privilege('', 'public.user_credentials', 'SELECT'); +-- SELECT has_table_privilege('', 'public.user_credentials', 'DELETE'); +-- +-- --------------------------------------------------------------------- +-- 6. ROLLBACK +-- --------------------------------------------------------------------- +-- Rehearse first, on the scratch database. Rolling back DESTROYS credential +-- rows: any account created after the migration would lose its only +-- authentication material and could not sign in until it is re-enrolled. +-- +-- Take a fresh backup immediately before rolling back, then use +-- 0001_phase1_identity_rollback.sql in the same session settings +-- (lock_timeout, statement_timeout). The rollback is not online-safe for a +-- busy sessions table: prefer a maintenance window. +-- +-- If the migration failed mid-transaction, nothing was applied: the +-- transaction rolled back and this file can be corrected and re-reviewed. + +COMMIT; diff --git a/migrations/0001_phase1_identity_rollback.sql b/migrations/0001_phase1_identity_rollback.sql new file mode 100644 index 0000000..c28f2b9 --- /dev/null +++ b/migrations/0001_phase1_identity_rollback.sql @@ -0,0 +1,154 @@ +-- ===================================================================== +-- Rollback for: UmbraTrans phase-1 identity and tenancy migration +-- (users.username, user_credentials, organizations, +-- additive session columns) +-- ===================================================================== +-- +-- STATUS: PROPOSED / NOT APPLIED / NOT APPROVED +-- +-- This file has never been executed against any database. +-- +-- DESTRUCTIVE. Read the warnings before running anything. +-- +-- * DROP TABLE public.user_credentials DESTROYS every credential row that +-- exists at rollback time. Accounts created or re-enrolled after the +-- forward migration lose their only authentication material and cannot +-- sign in until an operator re-provisions them through the operator +-- path. There is no recovery short of restoring a backup. +-- * DROP TABLE public.organizations DESTROYS every tenant row. The DROP is +-- guarded below so it only fires when the forward migration actually +-- created the table, because the control plane very likely already had +-- one. If organizations pre-existed, its rows are left alone. +-- * DROP CONSTRAINT users_username_key / the backing index removes case +-- insensitive uniqueness. The username VALUES are kept by this rollback; +-- drop the column only if the mapping must be discarded. +-- * Dropping the additive session columns discards MFA, idle deadline, and +-- last-seen history. An application build that still writes those columns +-- will fail after the rollback, so roll the application back first. +-- +-- PREREQUISITES +-- [ ] Forward migration 0001 confirmed applied (see its section 5 output) +-- [ ] Fresh pg_dump -Fc backup taken and checksummed AFTER the forward +-- migration, so the post-migration state is recoverable +-- [ ] Restore rehearsed on a scratch database +-- [ ] Application rolled back to a build that does not use the new columns +-- [ ] Maintenance window agreed (the sessions DDL takes ACCESS EXCLUSIVE) +-- +-- --------------------------------------------------------------------- +-- 0. Transaction guard rails (same limits as the forward migration) +-- --------------------------------------------------------------------- + +BEGIN; + +SET LOCAL lock_timeout = '3s'; +SET LOCAL statement_timeout = '60s'; +SET LOCAL idle_in_transaction_session_timeout = '30s'; + +-- --------------------------------------------------------------------- +-- 1. Record what will be lost, before it is lost +-- --------------------------------------------------------------------- + +-- SELECT count(*) AS credentials_to_be_dropped FROM public.user_credentials; +-- SELECT credential_type, count(*) FROM public.user_credentials GROUP BY 1; +-- SELECT count(*) AS live_sessions FROM public.sessions WHERE revoked_at IS NULL; +-- SELECT count(*) AS organizations FROM public.organizations; + +-- --------------------------------------------------------------------- +-- 2. Additive session columns +-- --------------------------------------------------------------------- +-- Drop the indexes first; they are not required to drop the columns, but +-- removing them explicitly keeps the rollback symmetric with the forward file. + +DROP INDEX IF EXISTS public.sessions_idle_deadline_at_idx; +DROP INDEX IF EXISTS public.sessions_user_id_idx; + +ALTER TABLE public.sessions + DROP COLUMN IF EXISTS idle_deadline_at, + DROP COLUMN IF EXISTS mfa_verified_at, + DROP COLUMN IF EXISTS last_seen_at; + +-- --------------------------------------------------------------------- +-- 3. user_credentials +-- --------------------------------------------------------------------- +-- The table and its constraints disappear with it. Any credential that is not +-- recoverable from a backup is gone: re-provision affected accounts through +-- umbratrans-bootstrap-admin (operator path) after the rollback is verified. + +DROP TABLE IF EXISTS public.user_credentials; + +-- --------------------------------------------------------------------- +-- 4. organizations +-- --------------------------------------------------------------------- +-- Dropped only when this migration created the table. The forward file uses +-- CREATE TABLE IF NOT EXISTS, so in the expected case the control plane owned +-- organizations first and its rows, slug constraint, and dependent projects +-- rows must survive the rollback untouched. +-- +-- Drop the second guard's condition to force the drop. Do not do that without +-- a backup: projects.organization_id and user_roles.organization_id reference +-- this table. + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint c + JOIN pg_class t ON t.oid = c.conrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE n.nspname = 'public' + AND t.relname = 'organizations' + AND c.conname = 'organizations_status_check' + AND pg_get_constraintdef(c.oid) LIKE '%archived%' + ) THEN + -- The forward migration's own check constraint is absent, which means + -- the table was not created by this migration. + RAISE NOTICE 'organizations was not created by 0001; leaving it in place'; + ELSE + DROP INDEX IF EXISTS public.organizations_status_idx; + DROP TABLE IF EXISTS public.organizations; + END IF; +END +$$; + +-- --------------------------------------------------------------------- +-- 5. users.username +-- --------------------------------------------------------------------- +-- Values are preserved by default. Uncomment the second statement only when +-- the mapping must be discarded as well. + +ALTER TABLE public.users DROP CONSTRAINT IF EXISTS users_username_key; +DROP INDEX IF EXISTS public.users_username_key; + +-- ALTER TABLE public.users DROP COLUMN IF EXISTS username; + +-- citext is left installed: dropping an extension can cascade to dependent +-- objects, and an unused extension is harmless. Remove it only in a separate, +-- explicitly reviewed change. + +-- --------------------------------------------------------------------- +-- 6. POST-ROLLBACK VERIFICATION +-- --------------------------------------------------------------------- +-- +-- \d+ public.users +-- \d+ public.sessions +-- +-- SELECT to_regclass('public.user_credentials') AS user_credentials; -- NULL +-- SELECT count(*) FROM pg_policies WHERE schemaname = 'public' +-- AND tablename = 'user_credentials'; -- 0 +-- SELECT count(*) FROM public.sessions WHERE idle_deadline_at IS NOT NULL; -- 0 +-- -- either organizations is gone, or it is exactly as it was before the +-- -- forward migration; compare against the pre-flight row count +-- SELECT to_regclass('public.organizations') AS organizations; + +-- -- the pre-existing schema must be unchanged apart from this rollback +-- SELECT column_name, data_type FROM information_schema.columns +-- WHERE table_schema = 'public' AND table_name IN ('users', 'sessions') +-- ORDER BY table_name, ordinal_position; +-- +-- -- re-run the control-plane integrity checks that were captured before the +-- -- forward migration and compare row counts for organizations, projects, +-- -- roles, permissions, user_roles, invitations, and audit_events +-- +-- Roll forward again only after a new review of 0001_phase1_identity.sql. + +COMMIT; diff --git a/migrations/0002_password_reset_purpose.sql b/migrations/0002_password_reset_purpose.sql new file mode 100644 index 0000000..2ae6dde --- /dev/null +++ b/migrations/0002_password_reset_purpose.sql @@ -0,0 +1,94 @@ +-- 0002: give single-use tokens a purpose. +-- +-- 0001 reused the control plane's existing, unused `bootstrap_tokens` table for +-- email verification. That table has no notion of what a token is for, so with +-- only one kind of token in play the omission was invisible. +-- +-- It stops being invisible the moment a second kind exists. A password-reset +-- token and an email-verification token would be interchangeable: a reset link +-- clicked by anyone holding it would confirm an address, and a verification link +-- would set a password. Neither is acceptable, and the table as it stands cannot +-- tell them apart. +-- +-- The column defaults to the existing purpose, so every row already written stays +-- valid and the verification flow is unchanged. No existing row is rewritten, and +-- the verification code continues to work without modification. +-- +-- `0002_password_reset_purpose.sql`, additive, and safe to apply more than once. + +BEGIN; + +-- --------------------------------------------------------------------- +-- 1. Purpose +-- --------------------------------------------------------------------- + +-- A default, not a null, so the column can be NOT NULL and an insert that +-- forgets to name a purpose is still valid and still safe: it lands as +-- email_verification, which is the only purpose that existed before this +-- migration and therefore the conservative choice. +ALTER TABLE public.bootstrap_tokens + ADD COLUMN IF NOT EXISTS purpose text NOT NULL DEFAULT 'email_verification'; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint c + JOIN pg_class t ON t.oid = c.conrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE n.nspname = 'public' + AND t.relname = 'bootstrap_tokens' + AND c.conname = 'bootstrap_tokens_purpose_check' + ) THEN + ALTER TABLE public.bootstrap_tokens + ADD CONSTRAINT bootstrap_tokens_purpose_check + CHECK (purpose IN ('email_verification', 'password_reset')); + RAISE NOTICE 'attached bootstrap_tokens_purpose_check'; + ELSE + RAISE NOTICE 'bootstrap_tokens_purpose_check already present; leaving it alone'; + END IF; +END +$$; + +COMMENT ON COLUMN public.bootstrap_tokens.purpose IS + 'What this token authorises: email_verification confirms an address, password_reset sets a new password. They are deliberately not interchangeable, because each would otherwise grant the other''s capability.'; + +-- --------------------------------------------------------------------- +-- 2. Lookup +-- --------------------------------------------------------------------- + +-- Every token read filters on user and purpose together, and an unconsumed token +-- for one purpose is looked up on every reset and every verification. The index +-- covers that; without it each of those is a sequential scan of the table, which +-- on a busy deployment is also a cheap way for an unauthenticated endpoint to +-- generate load. +CREATE INDEX IF NOT EXISTS bootstrap_tokens_user_purpose_idx + ON public.bootstrap_tokens (user_id, purpose, expires_at) + WHERE consumed_at IS NULL; + +-- --------------------------------------------------------------------- +-- 3. Rollback +-- --------------------------------------------------------------------- +-- Additive, so a rollback is dropping what was added. The column is dropped +-- before the index, since the index depends on it. Any token of a purpose other +-- than email_verification must be deleted first, or the drop will leave rows the +-- column no longer describes; the guarded delete below handles that, and the +-- NOTICE says how many rows it removed rather than passing over them. +-- +-- DROP INDEX IF EXISTS public.bootstrap_tokens_user_purpose_idx; +-- +-- DO $$ +-- BEGIN +-- IF EXISTS ( +-- SELECT 1 FROM information_schema.columns +-- WHERE table_schema = 'public' +-- AND table_name = 'bootstrap_tokens' +-- AND column_name = 'purpose' +-- ) THEN +-- DELETE FROM public.bootstrap_tokens WHERE purpose <> 'email_verification'; +-- ALTER TABLE public.bootstrap_tokens DROP COLUMN purpose; +-- END IF; +-- END +-- $$; + +COMMIT; diff --git a/migrations/0003_compute_and_queue.sql b/migrations/0003_compute_and_queue.sql new file mode 100644 index 0000000..26a169a --- /dev/null +++ b/migrations/0003_compute_and_queue.sql @@ -0,0 +1,888 @@ +-- 0003: compute resources, a job queue with leases, and node identity. +-- +-- Additive, idempotent, and safe to apply more than once. Nothing existing is +-- rewritten, and no existing table loses a column: `jobs` gains columns, +-- `permissions` gains rows, and the rest are new tables. +-- +-- Section 5 is the datastore: publishers, bylines, artifacts and provenance. +-- It is folded in here rather than deferred to 0004 because 0003 is still +-- unapplied to the deployment, and an unapplied migration can still be edited. +-- Once it has run, this is 0004 and a discussion rather than a change. +-- +-- The design this implements is §6.16-6.18 and §7 of SPEC/UmbraTools.md. Three +-- decisions in it are load-bearing and each is commented where it is written, +-- because each one exists to stop a failure that is silent when it happens: +-- +-- 1. A node is `online` because it heartbeated, not because it was created. +-- A node that has never authenticated must not be offered work. +-- 2. Claiming a job is `FOR UPDATE SKIP LOCKED`. Without it two workers pick +-- the same row and one silently overwrites the other's claim. +-- 3. A lease that expires returns the job to the queue. A worker that is +-- killed releases nothing, so the lease is what releases it. +-- +-- `jobs.status` in the phase-1 table is unconstrained text. The values used here +-- are: queued, running, blocked, partial, complete, failed, cancelled. They are +-- left as a CHECK rather than a reference table because this is a closed set +-- owned by this application; SPEC §6.14 moves open vocabularies to reference +-- tables, and this is not one of them. + +BEGIN; + +-- --------------------------------------------------------------------- +-- 1. Compute nodes and their devices +-- --------------------------------------------------------------------- + +-- What each machine is, and where it sits. `is_controller` is separate from +-- `status` because the controller is also a worker on this host, and a node +-- type and a liveness are different facts. +CREATE TABLE IF NOT EXISTS public.compute_nodes ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + organization_id uuid NOT NULL REFERENCES public.organizations(id), + name text NOT NULL, + is_controller boolean NOT NULL DEFAULT false, + trust_domain text NOT NULL DEFAULT 'local' + CHECK (trust_domain IN ('local', 'remote', 'cloud')), + locality text, + transport text NOT NULL DEFAULT 'https' + CHECK (transport IN ('https', 'ssh', 'vpn')), + -- Starts at 'provisioning'. Only the heartbeat path may set 'online'. + status text NOT NULL DEFAULT 'provisioning' + CHECK (status IN ('provisioning', 'online', 'draining', 'offline', 'retired')), + agent_version text, + agent_started_at timestamptz, + last_heartbeat_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (organization_id, name) +); + +-- Devices, not a node-level capability flag. "CPU and/or GPU" is a set of +-- devices, and a CPU-only virtual machine is then an ordinary node rather than +-- a special case needing its own code path. +-- +-- `throughput_factor` orders placements. It is NOT an eligibility test: a CPU +-- device is slow, not incapable, and a model that fits runs there rather than +-- not running at all. The spec calls this the difference between "slow" and +-- "impossible" and it is why `vram_min_mb` and `vram_preferred_mb` are separate +-- columns on the model rather than one number. +CREATE TABLE IF NOT EXISTS public.compute_devices ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + compute_node_id uuid NOT NULL REFERENCES public.compute_nodes(id) ON DELETE CASCADE, + kind text NOT NULL CHECK (kind IN + ('cpu', 'gpu_discrete', 'gpu_integrated', 'npu', 'tpu')), + label text NOT NULL, -- 'gpu:0', 'cpu:0' + vram_total_mb int NOT NULL DEFAULT 0, + vram_free_mb int NOT NULL DEFAULT 0, + ram_total_mb int NOT NULL DEFAULT 0, + ram_free_mb int NOT NULL DEFAULT 0, + compute_units real, + throughput_factor real NOT NULL DEFAULT 1, + supports_accelerators text[] NOT NULL DEFAULT '{}', + supports jsonb NOT NULL DEFAULT '{}', + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (compute_node_id, label) +); + +-- A node's identity is its credential, and the credential is narrower than an +-- account. It authorises queue operations on the work this node was given; it +-- cannot read project content. A remotely reachable machine holding a +-- connection string with no per-object authorization is a different risk from +-- one holding a scoped key, and this table is the difference. +CREATE TABLE IF NOT EXISTS public.node_credentials ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + compute_node_id uuid NOT NULL REFERENCES public.compute_nodes(id) ON DELETE CASCADE, + key_prefix text NOT NULL, + key_hash text NOT NULL, -- argon2id; the key itself is never stored + agent_name text NOT NULL, + scopes text[] NOT NULL, + issued_at timestamptz NOT NULL DEFAULT now(), + last_used_at timestamptz, + revoked_at timestamptz, + revoked_reason text +); +CREATE INDEX IF NOT EXISTS node_credentials_prefix_idx + ON public.node_credentials (key_prefix); + +-- --------------------------------------------------------------------- +-- 2. Models +-- --------------------------------------------------------------------- + +-- `vram_min_mb` is the eligibility floor and `vram_preferred_mb` is the ranking +-- target. Collapsing them into one number is the bug the spec warns about: it +-- either refuses a job that would have completed in an hour, or starts one that +-- crawls. `requires_accelerators` non-empty is the "only this node can run it" +-- case, which is a capacity risk the console has to be able to state. +CREATE TABLE IF NOT EXISTS public.model_registry ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + slug text NOT NULL UNIQUE, + kind text NOT NULL CHECK (kind IN + ('llm', 'tts', 'asr', 'embedding', 'vision', 'utility')), + vram_min_mb int NOT NULL DEFAULT 0, + vram_preferred_mb int NOT NULL DEFAULT 0, + ram_min_mb int NOT NULL DEFAULT 0, + quantisation text, + context_tokens int, + requires_accelerators text[] NOT NULL DEFAULT '{}', + reference_throughput real NOT NULL DEFAULT 1, + load_seconds int NOT NULL DEFAULT 0, + capabilities jsonb NOT NULL DEFAULT '{}', + source_uri text, + license text, + created_at timestamptz NOT NULL DEFAULT now() +); + +-- Residency. `loaded` means holding its memory whether or not it is +-- generating, which is why a node can be idle at 0% GPU and unable to accept +-- another model. +CREATE TABLE IF NOT EXISTS public.node_models ( + compute_node_id uuid NOT NULL REFERENCES public.compute_nodes(id) ON DELETE CASCADE, + model_id uuid NOT NULL REFERENCES public.model_registry(id) ON DELETE CASCADE, + device_id uuid REFERENCES public.compute_devices(id) ON DELETE SET NULL, + state text NOT NULL CHECK (state IN + ('loaded', 'warm', 'available', 'loading', 'failed')), + loaded_at timestamptz, + vram_held_mb int NOT NULL DEFAULT 0, + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (compute_node_id, model_id) +); + +-- Overwritten each heartbeat, deliberately. Its purpose is "now", and the +-- history of a heartbeat is worth nothing; job_events is where history belongs. +CREATE TABLE IF NOT EXISTS public.node_heartbeats ( + compute_node_id uuid PRIMARY KEY REFERENCES public.compute_nodes(id) ON DELETE CASCADE, + reported_at timestamptz NOT NULL DEFAULT now(), + cpu_busy_pct numeric(5,2), + ram_free_mb int, + vram_free_mb int, + vram_total_mb int, + disk_free_mb bigint, + load_avg_1m numeric(6,2), + resident_models jsonb NOT NULL DEFAULT '[]', + active_jobs int NOT NULL DEFAULT 0, + agent_version text, + raw jsonb NOT NULL DEFAULT '{}' +); + +-- --------------------------------------------------------------------- +-- 3. The queue +-- --------------------------------------------------------------------- + +-- `jobs` is specified in §6.8 but was never migrated: 0001 created the identity +-- tables and stopped there. It is created here, complete, rather than +-- incrementally, because there is no existing row to preserve and a table built +-- by a sequence of ALTERs is harder to read than one built in full. +-- +-- `organization_id` is NOT NULL and is never updated. A job belongs to exactly one +-- organization for its whole life, and the constraint is at the schema because a +-- placement bug that changed it would hand an organization another tenant's work +-- without anything noticing. +CREATE TABLE IF NOT EXISTS public.jobs ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + organization_id uuid NOT NULL REFERENCES public.organizations(id), + project_id uuid REFERENCES public.projects(id), + lang text, + kind text NOT NULL, -- 'analyze','translate','render','build_pdf',… + -- The closed set this application owns. Open vocabularies are reference + -- tables per §6.14; this is not one of them, and it is constrained here + -- rather than left loose the way the phase-1 tables were. + status text NOT NULL DEFAULT 'queued' + CHECK (status IN ('queued', 'running', 'blocked', 'partial', + 'complete', 'failed', 'cancelled')), + requested_by text NOT NULL, + payload jsonb NOT NULL DEFAULT '{}', + created_at timestamptz NOT NULL DEFAULT now() +); + +-- The queue columns. Every one is nullable or defaulted, so a row written by the +-- plain definition above is already valid. +ALTER TABLE public.jobs ADD COLUMN IF NOT EXISTS priority smallint NOT NULL DEFAULT 100; +ALTER TABLE public.jobs ADD COLUMN IF NOT EXISTS requires jsonb NOT NULL DEFAULT '{}'; +ALTER TABLE public.jobs ADD COLUMN IF NOT EXISTS lease_owner uuid REFERENCES public.compute_nodes(id); +ALTER TABLE public.jobs ADD COLUMN IF NOT EXISTS lease_expires_at timestamptz; +ALTER TABLE public.jobs ADD COLUMN IF NOT EXISTS attempt int NOT NULL DEFAULT 0; +ALTER TABLE public.jobs ADD COLUMN IF NOT EXISTS max_attempts int NOT NULL DEFAULT 3; +ALTER TABLE public.jobs ADD COLUMN IF NOT EXISTS not_before timestamptz NOT NULL DEFAULT now(); +ALTER TABLE public.jobs ADD COLUMN IF NOT EXISTS parent_job_id uuid REFERENCES public.jobs(id); +ALTER TABLE public.jobs ADD COLUMN IF NOT EXISTS idempotency_key text; +ALTER TABLE public.jobs ADD COLUMN IF NOT EXISTS started_at timestamptz; +ALTER TABLE public.jobs ADD COLUMN IF NOT EXISTS finished_at timestamptz; +ALTER TABLE public.jobs ADD COLUMN IF NOT EXISTS result jsonb; +ALTER TABLE public.jobs ADD COLUMN IF NOT EXISTS error text; + +-- Claiming reads this. Partial, because only queued work is claimable and the +-- index should not carry the terminal states. +CREATE INDEX IF NOT EXISTS jobs_claimable_idx + ON public.jobs (priority, not_before, created_at) + WHERE status = 'queued'; + +-- The re-import and re-submit path. Partial, so a job with no key does not +-- collide with anything, and so the same key cannot produce two rows. +CREATE UNIQUE INDEX IF NOT EXISTS jobs_idempotency_idx + ON public.jobs (idempotency_key) + WHERE idempotency_key IS NOT NULL; + +-- A container is a parent with no lease owner and is never claimed. This index +-- makes "what are the roots" cheap for the master view. +CREATE INDEX IF NOT EXISTS jobs_parent_idx + ON public.jobs (parent_job_id); + +-- Append-only history for a job. The heartbeat table is not this; it is +-- overwritten on purpose because it answers "now". +CREATE TABLE IF NOT EXISTS public.job_events ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES public.jobs(id) ON DELETE CASCADE, + node_id uuid REFERENCES public.compute_nodes(id) ON DELETE SET NULL, + event text NOT NULL, -- 'queued','claimed','completed','failed', + -- 'lease_expired','blocked','decomposed' + detail jsonb NOT NULL DEFAULT '{}', + at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS job_events_job_idx ON public.job_events (job_id, at); + +-- Jobs get RLS too, now that the table exists. `jobs` had no policy while it did +-- not exist; the eleven phase-1 tables that do have one are listed in the same +-- loop below. +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_tables + WHERE schemaname = 'public' AND tablename = 'jobs') THEN + ALTER TABLE public.jobs ENABLE ROW LEVEL SECURITY; + END IF; +END +$$; + +-- --------------------------------------------------------------------------- +-- 5. The datastore: publishers, bylines, and artifacts +-- --------------------------------------------------------------------------- +-- +-- Everything above is machinery for deciding *where* work runs. This section is +-- the reason the queue exists at all: the intermediate and final work product. +-- Step 1 on node 1 can be followed by step 2 on node 3 while step 3 runs on node +-- 2, and the only thing that makes that comprehensible is a place where the +-- output of each step is a row rather than a value inside somebody's memory. +-- +-- Four decisions here, each load-bearing, each commented where it is written: +-- +-- 1. A publisher is a party that never authenticates, so it has no role and no +-- permissions. See `publishers`. +-- 2. A byline is not a publisher and not a user. See `authors`. +-- 3. An artifact's identity is its bytes. See `artifacts`. +-- 4. `job_artifacts` is what makes provenance queryable. See `job_artifacts`. +-- +-- The word "account" is deliberately absent from every column, table and comment +-- in this section. It has meant two incompatible things in this system - a person +-- who signs in, and a house a work is published under - and a schema that uses +-- one word for both will eventually use it for both in one constraint. + +-- --------------------------------------------------------------------------- +-- 5.1 Publishers: the party a work is published under +-- --------------------------------------------------------------------------- +-- +-- The production boundary is the organization: the organization owns the work. +-- The publication boundary is a different axis and a different party, and it +-- routinely points somewhere else. An organization owns the book; a publisher +-- publishes it; those are two facts and they are frequently not the same +-- organization. A public-domain text owned by one organization may be published +-- by a house belonging to another, or by a third party with no organizational +-- relationship to it at all. +-- +-- A publisher has **no login, no password and no sessions**, and therefore no +-- row in user_roles and no permission anywhere. When a release is approved "for" +-- an imprint, a *user* holding project.release.approve did it; the imprint is +-- recorded on the work, not in an access-control list. That is the whole reason +-- this is a table rather than a role: a name that appears on work is not a party +-- that can be asked to prove who it is. +-- +-- Hierarchy is parent_id, so Umbra -> WorstAuthor is one row referencing another. +-- `kind` distinguishes a house that belongs to an organization from an +-- independent publisher, because who may edit the row depends on it. + +CREATE TABLE IF NOT EXISTS public.publishers ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + parent_id uuid REFERENCES public.publishers(id), + name text NOT NULL CHECK (length(btrim(name)) BETWEEN 1 AND 200), + slug citext NOT NULL UNIQUE CHECK (slug ~ '^[a-z0-9][a-z0-9-]{1,62}$'), + -- The organization this publisher belongs to, when it belongs to one. Umbra's + -- imprints do; an independent publisher does not. Nullable because NULL is + -- the honest value for "no organizational relationship", which is a real and + -- common arrangement rather than missing data. + -- + -- This is the one deliberately cross-tenant reference in the schema, and it + -- carries two obligations that the policy below implements: + -- * reading a publisher must not reveal whether its owning organization + -- exists, so the owner column is not exposed by the SELECT policy; + -- * only a platform administrator may set or change it, because doing so + -- asserts a relationship between two tenants. An organization + -- administrator manages its own projects, not who owns somebody else's + -- imprint. + owner_organization_id uuid REFERENCES public.organizations(id), + kind text NOT NULL DEFAULT 'publisher' + CHECK (kind IN ('imprint', 'publisher')), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT publishers_not_own_parent CHECK (parent_id IS DISTINCT FROM id) +); + +CREATE INDEX IF NOT EXISTS publishers_parent_idx ON public.publishers (parent_id); +CREATE INDEX IF NOT EXISTS publishers_owner_idx ON public.publishers (owner_organization_id) + WHERE owner_organization_id IS NOT NULL; + +-- Which publisher a project is published under, optionally per language. +-- +-- `lang` is nullable and NULL means "the work as a whole". That is not a +-- placeholder: a project can have one publisher for the English edition and a +-- different house for the German one, because co-editions are ordinary. A +-- single publisher_id on the project could not express that, and pretending +-- otherwise would force the model to record something untrue. +-- +-- The primary key includes lang, and NULL in a unique constraint does not +-- deduplicate in PostgreSQL - NULLs are distinct - so uniqueness is enforced by +-- the COALESCE index below rather than by the primary key alone. +CREATE TABLE IF NOT EXISTS public.project_publishers ( + project_id uuid NOT NULL REFERENCES public.projects(id) ON DELETE CASCADE, + publisher_id uuid NOT NULL REFERENCES public.publishers(id), + lang text, + created_at timestamptz NOT NULL DEFAULT now(), + created_by uuid REFERENCES public.users(id), + PRIMARY KEY (project_id, publisher_id, lang) +); + +CREATE UNIQUE INDEX IF NOT EXISTS project_publishers_once_idx + ON public.project_publishers (project_id, publisher_id, COALESCE(lang, '')); + +-- --------------------------------------------------------------------------- +-- 5.2 Bylines: authors, translators, narrators, and the people who are credited +-- --------------------------------------------------------------------------- +-- +-- A byline is a first-class party and is neither a publisher nor a user. +-- +-- Not a publisher: in the house's own chain, Umbra -> WorstAuthor -> B.D. Murphy, +-- WorstAuthor is the imprint and B.D. Murphy is the byline. They are separate +-- entities with separate lifetimes - one publisher issues many books, one byline +-- appears on some of them. +-- +-- Not a user: a user is an account that authenticates and performs work; a byline +-- is a name printed on a work. Many bylines have no account at all, and a user may +-- hold many bylines. The optional user_id links the two where they genuinely are +-- the same party, and is nullable because that is frequently untrue rather than +-- unfinished. +-- +-- `owner_organization_id` mirrors publishers and carries the same reasoning: a +-- self-published byline belongs to the publishing organization, and an independent +-- author's does not. It also answers who may edit the row, which with no owner +-- would leave the answer "only a platform administrator" - too strict for a +-- system whose purpose includes self-publishing. +CREATE TABLE IF NOT EXISTS public.authors ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + name text NOT NULL CHECK (length(btrim(name)) BETWEEN 1 AND 200), + -- The form printed on the work. Separate from `name`, which is the party's + -- record name: "B. D. Murphy" and "B.D. Murphy" are the same byline and two + -- spellings of it, and collapsing them loses the fact that a work was + -- credited one way and catalogued another. + display_name text, + slug citext NOT NULL UNIQUE CHECK (slug ~ '^[a-z0-9][a-z0-9-]{1,62}$'), + -- Set when this byline is a system account. Never the other way round: a + -- login is an account concern and must not depend on a cataloguing decision. + user_id uuid REFERENCES public.users(id), + owner_organization_id uuid REFERENCES public.organizations(id), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE UNIQUE INDEX IF NOT EXISTS authors_user_once_idx + ON public.authors (user_id) WHERE user_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS authors_owner_idx ON public.authors (owner_organization_id) + WHERE owner_organization_id IS NOT NULL; + +-- Which byline is credited on which project, in which capacity, in which language. +-- +-- `credit_role` is not a permission and must never be checked as one. It records +-- how a name is printed: the author of the English edition is the translator of +-- the French one, and the narrator of the audiobook is frequently neither. Keying +-- that on a single author_id would force the same person to be recorded twice and +-- would lose the capacity, which is the part a title page has to state. +CREATE TABLE IF NOT EXISTS public.project_authors ( + project_id uuid NOT NULL REFERENCES public.projects(id) ON DELETE CASCADE, + author_id uuid NOT NULL REFERENCES public.authors(id), + credit_role text NOT NULL + CHECK (credit_role IN ('author', 'translator', 'editor', 'narrator', + 'cover_artist', 'illustrator')), + lang text, + position smallint NOT NULL DEFAULT 1, + created_at timestamptz NOT NULL DEFAULT now(), + created_by uuid REFERENCES public.users(id), + PRIMARY KEY (project_id, author_id, credit_role, lang) +); + +CREATE UNIQUE INDEX IF NOT EXISTS project_authors_order_idx + ON public.project_authors (project_id, credit_role, COALESCE(lang, ''), position); + +-- --------------------------------------------------------------------------- +-- 5.3 Artifacts: the intermediate and final work product +-- --------------------------------------------------------------------------- +-- +-- `jobs.result` is a jsonb column on the job row. That is right for "the sleep +-- job returned {'echo': 'hello from the queue'}" and wrong for "this is the +-- English audiobook, four hours twelve minutes, derived from these thirty-seven +-- chapters and this cast". The second is not a job's return value; it is a thing +-- that outlives every job that touched it, and it is what the next job consumes. +-- +-- Four decisions: +-- +-- 1. **Identity is the bytes.** content_hash is what makes the row the same row +-- across two projects and two runs. `storage_key` is where they are, not what +-- they are, and the two are allowed to disagree - that is what makes it +-- possible to move storage without rewriting history. +-- +-- 2. **`organization_id` is denormalised onto every row** even though it is +-- reachable through project_id. One column and one index for the tenant +-- predicate, matching the (id, organization_id) shape the identity tables +-- already use. A join in every policy is a join that can be forgotten. +-- +-- 3. **`parent_artifact_id` is the immediate input, and `derivation` is the +-- detail.** A single immediate parent keeps the common query - "what made +-- this" - an index seek. The jsonb carries the rest, because a chapter render +-- depends on its text, its voice, its model version and its settings, and four +-- relational edges for that would be a worse trade than a document that +-- reports itself honestly. +-- +-- 4. **The bytes are not in this database.** `storage_key` points outside. A WAV +-- of one chapter runs to hundreds of megabytes and a full audiobook master to +-- gigabytes; they belong in object storage with this table holding identity, +-- provenance and the key. The rule for getting them is deliberately not a +-- column here: it is whether the API carries the bytes or authorises a +-- redirect, and that is a property of the artefact's size rather than a fact +-- about the artefact. +CREATE TABLE IF NOT EXISTS public.artifacts ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + project_id uuid NOT NULL REFERENCES public.projects(id) ON DELETE CASCADE, + organization_id uuid NOT NULL REFERENCES public.organizations(id), + lang text, + -- An open vocabulary, not a CHECK. What a project produces is open-ended by + -- construction - source, translation, audio, cover, blurb, metadata, export - + -- and constraining it now means an ALTER for every pipeline added later. + -- §6.14 puts open vocabularies in reference tables; this is the first one, + -- and it is deliberately not yet a table. + kind text NOT NULL CHECK (length(btrim(kind)) BETWEEN 1 AND 64), + status text NOT NULL DEFAULT 'intermediate' + CHECK (status IN ('intermediate', 'final', 'superseded')), + -- sha256. The content address is what makes two identical renders the same + -- artifact rather than two copies of one, which is what makes caching, + -- deduplication and "has this already been done" all the same question. + content_hash text NOT NULL CHECK (content_hash ~ '^[0-9a-f]{64}$'), + byte_size bigint NOT NULL CHECK (byte_size >= 0), + media_type text NOT NULL, + storage_key text NOT NULL, + parent_artifact_id uuid REFERENCES public.artifacts(id), + derivation jsonb NOT NULL DEFAULT '{}'::jsonb, + created_by_job_id uuid REFERENCES public.jobs(id) ON DELETE SET NULL, + created_by_user_id uuid REFERENCES public.users(id) ON DELETE SET NULL, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT artifacts_id_project_org_key + UNIQUE (id, project_id, organization_id), + CONSTRAINT artifacts_no_self_parent CHECK (parent_artifact_id IS DISTINCT FROM id) +); + +CREATE INDEX IF NOT EXISTS artifacts_project_idx + ON public.artifacts (project_id, kind, lang); +CREATE INDEX IF NOT EXISTS artifacts_org_idx ON public.artifacts (organization_id); +CREATE INDEX IF NOT EXISTS artifacts_parent_idx ON public.artifacts (parent_artifact_id) + WHERE parent_artifact_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS artifacts_lineage_idx ON public.artifacts USING gin (derivation); + +-- Content addressing, scoped to the project. NOT globally unique: two +-- organizations legitimately hold identical bytes and neither should learn that +-- the other has them. Uniqueness within a project is what deduplication needs. +CREATE UNIQUE INDEX IF NOT EXISTS artifacts_content_key + ON public.artifacts (project_id, content_hash, kind, COALESCE(lang, '')); + +-- --------------------------------------------------------------------------- +-- 5.4 Provenance: what a job consumed and what it produced +-- --------------------------------------------------------------------------- +-- +-- The queue above can tell you a job ran. It cannot tell you what the job was +-- working on, which is the question that matters when step 2 on node 3 produces +-- something wrong: the answer is "this artifact, produced by that step, from +-- these inputs" and getting there needs a row rather than a log line. +-- +-- This is the table that makes "step 1 on node 1, then step 2 on node 3" legible +-- after the fact. One table with a direction, not two, because the two sides are +-- always read together and a split would guarantee one of them is forgotten. +-- +-- Artifacts outlive jobs, so this is ON DELETE CASCADE from the job: a job that +-- is deleted takes its provenance with it, and the artifacts survive. The +-- reverse - deleting an artifact and keeping the claim that something produced it +-- - would leave a provenance record pointing at nothing. +CREATE TABLE IF NOT EXISTS public.job_artifacts ( + job_id uuid NOT NULL REFERENCES public.jobs(id) ON DELETE CASCADE, + artifact_id uuid NOT NULL REFERENCES public.artifacts(id) ON DELETE CASCADE, + role text NOT NULL CHECK (role IN ('consumed', 'produced')), + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (job_id, artifact_id, role) +); + +CREATE INDEX IF NOT EXISTS job_artifacts_artifact_idx + ON public.job_artifacts (artifact_id); + +-- --------------------------------------------------------------------------- +-- 5.5 Permissions +-- --------------------------------------------------------------------------- +-- +-- The vocabulary is scope.subject.verb and it already exists with twelve keys. +-- These extend it rather than replacing it, with two rules that keep the count +-- from doubling: +-- +-- * `read` is implied by `edit`, `review` and `approve`. Nobody edits what they +-- cannot read, and a separate read key is a key somebody will forget to check. +-- * `approve` does NOT imply `edit`. Signing off is a different authority from +-- changing, and that is the distinction the whole owner/worker split rests +-- on: a user may approve a release without being able to alter the text, and +-- the reverse is the mistake worth preventing. +-- +-- The read implied by the others is enforced in application code, not here, +-- because expressing "edit implies read" in a permission table needs a closure +-- over the key space and the check would run on every authorization. The rule is +-- written down here so the two halves cannot drift. +-- +-- Note what is absent: there is no publisher permission, and there is no byline +-- permission. Neither party authenticates. A user who may approve a release on an +-- imprint's behalf holds project.release.approve; the imprint holds nothing. +INSERT INTO public.permissions (key, description) VALUES + ('project.source.read', 'Read the source text of a project'), + ('project.translation.read', 'Read translations of a project'), + ('project.audio.read', 'Read audio artefacts and their analysis'), + ('project.audio.edit', 'Adjust or re-render an audio take'), + ('project.translation.approve', 'Sign off a translation as final'), + ('project.cover.read', 'Read per-language cover copy and blurb text'), + ('project.cover.edit', 'Upload finished cover artwork'), + ('project.export.create', 'Produce a bundle of project data to use elsewhere') +ON CONFLICT (key) DO NOTHING; + +-- --------------------------------------------------------------------------- +-- 3b. Append-only where immutability is the point +-- --------------------------------------------------------------------------- +-- +-- `audit_events` is already append-only, through a trigger that raises. Three more +-- tables here have the same property and were going in without it, which would have +-- made an immutability claim in the design a convention rather than a constraint: +-- +-- * **artifacts** - identity *is* the bytes (`content_hash`). An UPDATE that changes +-- `content_hash` or `storage_key` in place rewrites history for every job that consumed +-- the row, and the lineage in `job_artifacts` would then point at content that never +-- existed. UPDATE only; see the trigger for why DELETE is deliberately permitted. +-- * **job_artifacts** - provenance. If a job's record of what it consumed can be +-- edited, the queue can be made to look as though it did something it did not. +-- * **job_events** - the event trail behind I25 and I26, and the evidence that a job +-- was claimed exactly once and that an expired lease was noticed. +-- +-- A dedicated function per table rather than one shared one, so the error names the +-- table that refused. A single function saying "immutable" tells an operator which +-- rule fired only by which object it was raised against, which is one step of work +-- they should not have to do while holding an incident. +-- +-- DELETE and UPDATE only. INSERT is how these tables are populated. +-- EXECUTE is revoked from PUBLIC on all three, matching every other function in +-- `app` (the ten that predate this migration were closed by hand; nothing in a +-- migration did it, which is why it needs doing here explicitly). +-- +-- A trigger function runs as SECURITY INVOKER, so the guard is only reached if the +-- *calling* role may execute it. Revoking PUBLIC without granting the roles that +-- actually write would therefore turn the guard into an outage rather than a +-- protection - and an outage at least announces itself, whereas a guard that never +-- fires does not. So the grants follow the revoke, to every role that writes these +-- tables, and are re-issued on every apply so a role added later is covered by the +-- next migration rather than by luck. +-- +-- Verified after applying: 13 of 13 `app` functions non-PUBLIC-executable, and a +-- non-owner role can still insert into `artifacts` and be refused on update. +-- +-- The ten functions that predate this migration were closed by hand against the +-- deployment, and nothing in any migration does it. That is why a database restored +-- from a backup of the deployment comes back with all ten PUBLIC-executable again - +-- observed, on the scratch database built from the pre-0003 dump, where 10 of 13 were +-- open while the deployment itself had 3. A hand-applied grant is not a property of +-- the schema, it is a property of one database, and it does not survive a restore. +-- +-- So this closes all thirteen, not just the three added above. B4 was recorded as +-- open against the deployment, where it was already fixed; the defect was never only +-- about the deployment. +-- + +-- +-- Note what this does *not* do: it does not stop the owner role from bypassing RLS +-- (G2), and it does not stop a superuser. It stops an ordinary mistake from being +-- indistinguishable from a design. + +DO $$ +BEGIN + EXECUTE $guard$ + CREATE OR REPLACE FUNCTION app.prevent_artifact_mutation() + RETURNS trigger LANGUAGE plpgsql AS $body$ + BEGIN + RAISE EXCEPTION + 'artifacts are content-addressed; the row''s identity may not be rewritten - ' + 'supersede it with a new artifacts row instead'; + END; + $body$; + $guard$; + EXECUTE $guard$ + CREATE OR REPLACE FUNCTION app.prevent_job_artifact_mutation() + RETURNS trigger LANGUAGE plpgsql AS $body$ + BEGIN + RAISE EXCEPTION + 'job_artifacts is provenance and is append-only; a job''s record of what it consumed cannot be edited'; + END; + $body$; + $guard$; + EXECUTE $guard$ + CREATE OR REPLACE FUNCTION app.prevent_job_event_mutation() + RETURNS trigger LANGUAGE plpgsql AS $body$ + BEGIN + RAISE EXCEPTION + 'job_events is the event trail and is append-only'; + END; + $body$; + $guard$; + + IF EXISTS (SELECT 1 FROM pg_tables + WHERE schemaname = 'public' AND tablename = 'artifacts') THEN + EXECUTE 'DROP TRIGGER IF EXISTS artifacts_append_only ON public.artifacts'; + -- UPDATE only, deliberately. DELETE is permitted, because otherwise storage can + -- never be reclaimed: no artifact row could ever be removed once written, and the + -- table would grow without bound for the life of the deployment. + -- + -- The provenance guarantee is not weakened by allowing DELETE. `job_artifacts` + -- references artifacts `ON DELETE CASCADE`, so deleting an artifact that any job + -- consumed fires the guard on that table and is refused. An artifact that nothing + -- consumed is, by definition, nothing anyone's record depends on, and is exactly + -- the row a retention pass should be able to remove. + EXECUTE 'CREATE TRIGGER artifacts_append_only BEFORE UPDATE ON public.artifacts FOR EACH ROW EXECUTE FUNCTION app.prevent_artifact_mutation()'; + EXECUTE 'DROP TRIGGER IF EXISTS job_artifacts_append_only ON public.job_artifacts'; + EXECUTE 'CREATE TRIGGER job_artifacts_append_only BEFORE DELETE OR UPDATE ON public.job_artifacts FOR EACH ROW EXECUTE FUNCTION app.prevent_job_artifact_mutation()'; + END IF; + IF EXISTS (SELECT 1 FROM pg_tables + WHERE schemaname = 'public' AND tablename = 'job_events') THEN + EXECUTE 'DROP TRIGGER IF EXISTS job_events_append_only ON public.job_events'; + EXECUTE 'CREATE TRIGGER job_events_append_only BEFORE DELETE OR UPDATE ON public.job_events FOR EACH ROW EXECUTE FUNCTION app.prevent_job_event_mutation()'; + END IF; +END +$$; + +DO $$ +DECLARE + target text; +BEGIN + EXECUTE 'REVOKE ALL ON FUNCTION app.prevent_artifact_mutation() FROM PUBLIC'; + EXECUTE 'REVOKE ALL ON FUNCTION app.prevent_job_artifact_mutation() FROM PUBLIC'; + EXECUTE 'REVOKE ALL ON FUNCTION app.prevent_job_event_mutation() FROM PUBLIC'; +-- + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'umbracontrol_api') THEN + EXECUTE 'GRANT EXECUTE ON FUNCTION app.prevent_artifact_mutation() TO umbracontrol_api'; + EXECUTE 'GRANT EXECUTE ON FUNCTION app.prevent_job_artifact_mutation() TO umbracontrol_api'; + EXECUTE 'GRANT EXECUTE ON FUNCTION app.prevent_job_event_mutation() TO umbracontrol_api'; + END IF; + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'umbracontrol_worker') THEN + EXECUTE 'GRANT EXECUTE ON FUNCTION app.prevent_artifact_mutation() TO umbracontrol_worker'; + EXECUTE 'GRANT EXECUTE ON FUNCTION app.prevent_job_artifact_mutation() TO umbracontrol_worker'; + EXECUTE 'GRANT EXECUTE ON FUNCTION app.prevent_job_event_mutation() TO umbracontrol_worker'; + END IF; + + -- The read-only roles need EXECUTE on the policy helpers, not only on the guards. + -- Every RLS policy calls app.is_platform_admin() or app.is_organization_member(), + -- so revoking PUBLIC without this turns every SELECT into + -- "permission denied for function is_platform_admin" - which looks like a + -- permissions bug in the policy rather than a missing grant here. Found by + -- running the tenancy tests as a non-owner role immediately after the revoke: + -- five of them failed that way. + FOREACH target IN ARRAY ARRAY[ + 'app.can_access_project(uuid)', + 'app.can_view_user(uuid)', + 'app.current_user_id()', + 'app.is_organization_admin(uuid)', + 'app.is_organization_admin_for_project(uuid)', + 'app.is_organization_member(uuid)', + 'app.is_platform_admin()' + ] LOOP + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'umbracontrol_api') THEN + EXECUTE format('GRANT EXECUTE ON FUNCTION %s TO umbracontrol_api', target); + END IF; + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'umbracontrol_readonly') THEN + EXECUTE format('GRANT EXECUTE ON FUNCTION %s TO umbracontrol_readonly', target); + END IF; + -- rls_probe is the non-owner role the tenancy tests use. It is not a + -- deployment role; it exists so a policy can be observed refusing something, + -- and it needs the helpers for exactly that. + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'rls_probe') THEN + EXECUTE format('GRANT EXECUTE ON FUNCTION %s TO rls_probe', target); + END IF; + END LOOP; + -- All thirteen, for the reason given above. + FOREACH target IN ARRAY ARRAY[ + 'app.can_access_project(uuid)', + 'app.can_view_user(uuid)', + 'app.current_user_id()', + 'app.is_organization_admin(uuid)', + 'app.is_organization_admin_for_project(uuid)', + 'app.is_organization_member(uuid)', + 'app.is_platform_admin()', + 'app.prevent_artifact_mutation()', + 'app.prevent_audit_mutation()', + 'app.prevent_job_artifact_mutation()', + 'app.prevent_job_event_mutation()', + 'app.set_updated_at()', + 'app.validate_user_role_scope()' + ] LOOP + EXECUTE format('REVOKE ALL ON FUNCTION %s FROM PUBLIC', target); + END LOOP; + +END +$$; + +-- --------------------------------------------------------------------- +-- 4. Row-level security +-- --------------------------------------------------------------------- +-- +-- Written in the same migration that creates the table, against the same app.* +-- helpers the existing eleven tables use, so that switching RLS on later does +-- not require reconstructing which tables were meant to be covered. +-- +-- FORCE is deliberately NOT set here. The application owns these tables, so a +-- policy alone would be inert, and gate G2 is where the non-owner role and +-- FORCE arrive together. These policies are the shape they will have when it +-- does; the enforcement is G2's job and nothing here pretends otherwise. + +DO $$ +DECLARE + target text; +BEGIN + FOREACH target IN ARRAY ARRAY[ + 'compute_nodes', 'compute_devices', 'node_credentials', + 'model_registry', 'node_models', 'node_heartbeats', 'job_events', + 'artifacts', 'job_artifacts', + 'publishers', 'authors', 'project_publishers', 'project_authors' + ] LOOP + IF EXISTS (SELECT 1 FROM pg_tables + WHERE schemaname = 'public' AND tablename = target) THEN + EXECUTE format('ALTER TABLE public.%I ENABLE ROW LEVEL SECURITY', target); + END IF; + END LOOP; +END +$$; + +-- Every table above that carries a policy must also appear in the ENABLE list +-- above, or the policy is inert: PostgreSQL will happily create a policy on a +-- table with row-level security switched off and report success. The first +-- version of this section enabled it for `artifacts` and `job_artifacts` and +-- forgot the other four, so `publishers`, `authors`, `project_publishers` and +-- `project_authors` each ended up with a policy that enforced nothing - and a +-- check for "does the policy exist" would have passed throughout. +-- +-- Policies are dropped and recreated rather than created conditionally. +-- `CREATE POLICY` has no `IF NOT EXISTS`, and getting this wrong produces a migration +-- that applies once and then can never be re-applied - which is how 0001 learned the same +-- lesson for `ADD CONSTRAINT`. Re-applying must be safe. +DO $$ +DECLARE +BEGIN + EXECUTE 'DROP POLICY IF EXISTS compute_nodes_tenant ON public.compute_nodes'; + EXECUTE 'DROP POLICY IF EXISTS compute_devices_tenant ON public.compute_devices'; + EXECUTE 'DROP POLICY IF EXISTS node_credentials_tenant ON public.node_credentials'; + EXECUTE 'DROP POLICY IF EXISTS node_models_tenant ON public.node_models'; + EXECUTE 'DROP POLICY IF EXISTS node_heartbeats_tenant ON public.node_heartbeats'; + EXECUTE 'DROP POLICY IF EXISTS job_events_tenant ON public.job_events'; + EXECUTE 'DROP POLICY IF EXISTS jobs_tenant ON public.jobs'; + EXECUTE 'DROP POLICY IF EXISTS project_publishers_tenant ON public.project_publishers'; + EXECUTE 'DROP POLICY IF EXISTS project_authors_tenant ON public.project_authors'; + EXECUTE 'DROP POLICY IF EXISTS artifacts_tenant ON public.artifacts'; + EXECUTE 'DROP POLICY IF EXISTS job_artifacts_tenant ON public.job_artifacts'; + EXECUTE 'DROP POLICY IF EXISTS publishers_read ON public.publishers'; + EXECUTE 'DROP POLICY IF EXISTS publishers_write ON public.publishers'; + EXECUTE 'DROP POLICY IF EXISTS authors_read ON public.authors'; + EXECUTE 'DROP POLICY IF EXISTS authors_write ON public.authors'; + EXECUTE 'CREATE POLICY compute_nodes_tenant ON public.compute_nodes USING (app.is_platform_admin() OR app.is_organization_member(organization_id))'; + EXECUTE 'CREATE POLICY compute_devices_tenant ON public.compute_devices USING (app.is_platform_admin() OR EXISTS (SELECT 1 FROM public.compute_nodes n WHERE n.id = compute_devices.compute_node_id AND app.is_organization_member(n.organization_id)))'; + EXECUTE 'CREATE POLICY node_credentials_tenant ON public.node_credentials USING (app.is_platform_admin() OR EXISTS (SELECT 1 FROM public.compute_nodes n WHERE n.id = node_credentials.compute_node_id AND app.is_organization_member(n.organization_id)))'; + EXECUTE 'CREATE POLICY node_models_tenant ON public.node_models USING (app.is_platform_admin() OR EXISTS (SELECT 1 FROM public.compute_nodes n WHERE n.id = node_models.compute_node_id AND app.is_organization_member(n.organization_id)))'; + EXECUTE 'CREATE POLICY node_heartbeats_tenant ON public.node_heartbeats USING (app.is_platform_admin() OR EXISTS (SELECT 1 FROM public.compute_nodes n WHERE n.id = node_heartbeats.compute_node_id AND app.is_organization_member(n.organization_id)))'; + EXECUTE 'CREATE POLICY job_events_tenant ON public.job_events USING (app.is_platform_admin() OR EXISTS (SELECT 1 FROM public.jobs j WHERE j.id = job_events.job_id AND app.is_organization_member(j.organization_id)))'; + EXECUTE 'CREATE POLICY jobs_tenant ON public.jobs USING (app.is_platform_admin() OR app.is_organization_member(organization_id))'; + + -- The datastore follows the same rule the project tables do: membership of + -- the owning organization, not per-project assignment. Org-wide visibility was + -- chosen deliberately over the can_access_project helper, which contradicts the + -- rest of its own schema - there is no read permission among the twelve. + EXECUTE 'CREATE POLICY artifacts_tenant ON public.artifacts USING (app.is_platform_admin() OR app.is_organization_member(organization_id))'; + EXECUTE 'CREATE POLICY job_artifacts_tenant ON public.job_artifacts USING (app.is_platform_admin() OR EXISTS (SELECT 1 FROM public.jobs j WHERE j.id = job_artifacts.job_id AND app.is_organization_member(j.organization_id)))'; + EXECUTE 'CREATE POLICY project_publishers_tenant ON public.project_publishers USING (app.is_platform_admin() OR EXISTS (SELECT 1 FROM public.projects p WHERE p.id = project_publishers.project_id AND app.is_organization_member(p.organization_id)))'; + EXECUTE 'CREATE POLICY project_authors_tenant ON public.project_authors USING (app.is_platform_admin() OR EXISTS (SELECT 1 FROM public.projects p WHERE p.id = project_authors.project_id AND app.is_organization_member(p.organization_id)))'; + + -- Publishers and bylines are shared reference data: two organizations can + -- publish with the same house, and one book can be credited to a byline that + -- also appears on another organization's work. So they are readable by any + -- authenticated principal. + -- + -- The write policies are the interesting part, and they are asymmetric with + -- the read policies on purpose. `owner_organization_id` is the one column that + -- crosses tenants, so changing it is a platform-administrator action: an + -- organization administrator manages its own projects, not who owns somebody + -- else's imprint. Without that split, an organization administrator could + -- attach a third party's publisher to their own organization and thereby + -- assert a corporate relationship that does not exist. + EXECUTE 'CREATE POLICY publishers_read ON public.publishers FOR SELECT USING (app.current_user_id() IS NOT NULL)'; + EXECUTE 'CREATE POLICY publishers_write ON public.publishers FOR ALL USING (app.is_platform_admin() OR app.is_organization_admin(owner_organization_id)) WITH CHECK (app.is_platform_admin() OR app.is_organization_admin(owner_organization_id))'; + EXECUTE 'CREATE POLICY authors_read ON public.authors FOR SELECT USING (app.current_user_id() IS NOT NULL)'; + EXECUTE 'CREATE POLICY authors_write ON public.authors FOR ALL USING (app.is_platform_admin() OR app.is_organization_admin(owner_organization_id)) WITH CHECK (app.is_platform_admin() OR app.is_organization_admin(owner_organization_id))'; + -- No second policy exists to restrict the owner column, and that is deliberate. + -- PostgreSQL combines permissive policies with OR, so an extra policy granting + -- platform administrators an exception would *widen* access for everyone rather + -- than narrow it for one role. The WITH CHECK above is what actually refuses + -- it: an organization administrator who sets owner_organization_id to NULL, or + -- to another organization, produces a row that fails `is_organization_admin` + -- on that new value, and the statement is rejected. Adding a "clearing" policy + -- here would have looked like a safeguard and been the opposite of one. + -- Global, read-only: a model is a fact about the model. + EXECUTE 'DROP POLICY IF EXISTS model_registry_read ON public.model_registry'; + EXECUTE 'CREATE POLICY model_registry_read ON public.model_registry FOR SELECT USING (true)'; +END +$$; + +COMMIT; + +-- --------------------------------------------------------------------- +-- Rollback, in reverse dependency order. Not applied automatically. +-- +-- DROP TABLE IF EXISTS public.job_artifacts; +-- DROP FUNCTION IF EXISTS app.prevent_job_event_mutation(); +-- DROP FUNCTION IF EXISTS app.prevent_job_artifact_mutation(); +-- DROP FUNCTION IF EXISTS app.prevent_artifact_mutation(); +-- DROP TABLE IF EXISTS public.artifacts; +-- DROP TABLE IF EXISTS public.project_authors; +-- DROP TABLE IF EXISTS public.project_publishers; +-- DROP TABLE IF EXISTS public.authors; +-- DROP TABLE IF EXISTS public.publishers; +-- DELETE FROM public.permissions WHERE key IN ( +-- 'project.source.read','project.translation.read','project.audio.read', +-- 'project.audio.edit','project.translation.approve','project.cover.read', +-- 'project.cover.edit','project.export.create'); +-- DROP TRIGGER IF EXISTS job_events_append_only ON public.job_events; +-- DROP TRIGGER IF EXISTS job_artifacts_append_only ON public.job_artifacts; +-- DROP TRIGGER IF EXISTS artifacts_append_only ON public.artifacts; +-- DROP TABLE IF EXISTS public.job_events; +-- DROP INDEX IF EXISTS public.jobs_idempotency_idx; +-- DROP INDEX IF EXISTS public.jobs_claimable_idx; +-- DROP INDEX IF EXISTS public.jobs_parent_idx; +-- DROP TABLE IF EXISTS public.node_heartbeats; +-- DROP TABLE IF EXISTS public.node_models; +-- DROP TABLE IF EXISTS public.model_registry; +-- DROP TABLE IF EXISTS public.node_credentials; +-- DROP TABLE IF EXISTS public.compute_devices; +-- DROP TABLE IF EXISTS public.compute_nodes; +-- ALTER TABLE public.jobs DROP CONSTRAINT IF EXISTS jobs_organization_immutable; +-- ALTER TABLE public.jobs +-- DROP COLUMN IF EXISTS error, DROP COLUMN IF EXISTS result, +-- DROP COLUMN IF EXISTS finished_at, DROP COLUMN IF EXISTS started_at, +-- DROP COLUMN IF EXISTS idempotency_key, DROP COLUMN IF EXISTS parent_job_id, +-- DROP COLUMN IF EXISTS not_before, DROP COLUMN IF EXISTS max_attempts, +-- DROP COLUMN IF EXISTS attempt, DROP COLUMN IF EXISTS lease_expires_at, +-- DROP COLUMN IF EXISTS lease_owner, DROP COLUMN IF EXISTS requires, +-- DROP COLUMN IF EXISTS priority; diff --git a/migrations/0004_node_pairing.sql b/migrations/0004_node_pairing.sql new file mode 100644 index 0000000..af468df --- /dev/null +++ b/migrations/0004_node_pairing.sql @@ -0,0 +1,179 @@ +-- Pairing codes: a node proves it is the machine a person authorised, without that +-- person ever typing a 32-byte credential into a terminal on the far end. +-- +-- The problem this solves. Issuing a node credential the way it was done for +-- umbraaudio-m4 meant writing the key into a 0600 file on the controller and having a +-- human copy it onto another machine by hand. That key landed in shell history, in a +-- terminal buffer, and in whatever it was pasted into. It also made the controller a +-- courier: nothing could be enrolled without me in the middle, which is the opposite of +-- what a fleet of heterogeneous nodes needs. +-- +-- The flow this supports: +-- +-- 1. An administrator requests a pairing code for a *named* node. +-- 2. The code is shown once, expires, and is single-use. +-- 3. The node calls POST /api/node/pair with that code and its own name. +-- 4. The controller verifies the code, creates the node, and issues a credential +-- once, in the response. +-- +-- Two properties make this worth having over "send the operator a secret": +-- +-- * **The code is bound to a node name.** A code read out of a terminal buffer +-- cannot pair a machine called something else, so it cannot register a node under +-- a name no administrator approved. +-- * **The code never becomes a credential.** It is exchanged once, for a credential +-- shown once and stored only as an Argon2id hash. There is no path by which the +-- code itself is usable against the node surface afterwards. +-- +-- ## Why there is no SECURITY DEFINER function here +-- +-- The obvious shape for this is a plpgsql function that verifies and redeems, so the +-- node surface never touches the tables. It cannot be done in SQL here: the codebase's +-- hasher is Argon2id, in Python, and the only hashing PostgreSQL offers in this +-- database is `crypt()` (bcrypt and MD5). Writing pairing against bcrypt would be +-- claiming an algorithm choice the rest of the system does not make, and MD5 would be +-- indefensible. So the verification happens in Python, in the repository, alongside +-- `issue_node_credential` - which means the node endpoint authenticates as whatever +-- role the application uses and needs that role to hold EXECUTE on nothing and direct +-- table rights on these two tables. +-- +-- That is a deliberate consequence of G2 still being open: the application connects as +-- the table owner, so a paired node's credential is issued with owner-level authority +-- rather than a scoped one. The tables below are additive and hold no secrets other +-- than hashed codes, so nothing sensitive is exposed by that gap today. It is recorded +-- here because when G2 closes, these tables need grants for the application role, and +-- the pairing path needs re-testing as a non-owner role - the same treatment +-- test_rls_tenancy.py gives everything else. + +BEGIN; + +-- --------------------------------------------------------------------------- +-- 1. Pairing codes +-- --------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS public.node_pairing_codes ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + -- The node this code may pair. An administrator names the machine they are + -- standing at; the node claims that name or nothing. + node_name text NOT NULL CHECK (btrim(node_name) <> '' AND length(node_name) <= 120), + organization_id uuid NOT NULL REFERENCES public.organizations(id) ON DELETE CASCADE, + locality text, + trust_domain text NOT NULL DEFAULT 'local' + CHECK (trust_domain IN ('local', 'remote', 'cloud')), + -- Argon2id, never the code. A database read must not yield something replayable + -- against the node surface, and a support transcript must not hand over a live + -- credential. + code_hash text NOT NULL, + -- Shown to the administrator so two codes are distinguishable in a conversation + -- ("the one ending 41"). Deliberately not a lookup key: verifying by hash over + -- every live code avoids a prefix lookup that would make a truncated or misread + -- code valid. + code_hint text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + expires_at timestamptz NOT NULL, + -- Set on successful redemption, and the single-use guard: redemption is one + -- UPDATE ... WHERE used_at IS NULL, so two nodes racing on a single code produce + -- one winner rather than two nodes. + used_at timestamptz, + used_by_node_id uuid REFERENCES public.compute_nodes(id) ON DELETE SET NULL, + attempts smallint NOT NULL DEFAULT 0 CHECK (attempts >= 0), + CHECK (expires_at > created_at) +); + +CREATE INDEX IF NOT EXISTS node_pairing_codes_live_idx + ON public.node_pairing_codes (organization_id, node_name) + WHERE used_at IS NULL; + +COMMENT ON TABLE public.node_pairing_codes IS + 'Single-use, expiring, node-name-bound codes that exchange for a node credential. ' + 'A code is never itself usable against the node surface.'; + +-- --------------------------------------------------------------------------- +-- 2. Rate limiting +-- --------------------------------------------------------------------------- +-- A per-code attempt counter alone is not enough. An attacker who never generated a +-- code can still spray: pick a code, guess until it is right. Without a per-address +-- limit the six-digit space is reachable at whatever rate the network allows, and the +-- per-code counter only helps once a guess happens to hit a real code. +-- +-- The counter is per address over a window, so guessing across many codes costs the +-- same as guessing repeatedly at one. This is the control that makes a six-digit code +-- defensible; without it the code length would not be. +CREATE TABLE IF NOT EXISTS public.node_pairing_attempts ( + source_address text PRIMARY KEY, + window_started_at timestamptz NOT NULL DEFAULT now(), + attempts integer NOT NULL DEFAULT 0 CHECK (attempts >= 0) +); + +COMMENT ON TABLE public.node_pairing_attempts IS + 'Rolling per-address attempt counter for node pairing. Separate from the per-code ' + 'counter: one bounds guessing at a known code, this bounds guessing across all of ' + 'them.'; + +-- --------------------------------------------------------------------------- +-- 3. Row-level security +-- --------------------------------------------------------------------------- +-- Enabled now so the tables cannot be reached by a role that is not the owner once G2 +-- lands, and so `test_rls_coverage.py` sees them. No policy is created: pairing is an +-- administrative operation on a table that holds no tenant data, and the read path is +-- the repository's. A permissive policy here would widen the surface for no gain. +ALTER TABLE public.node_pairing_codes ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.node_pairing_attempts ENABLE ROW LEVEL SECURITY; + +-- --------------------------------------------------------------------------- +-- 4. Immutability of the redemption record +-- --------------------------------------------------------------------------- +-- `used_at` and `used_by_node_id` are the evidence that a code was spent and on what. +-- An UPDATE that clears them would make a single-use code reusable, and nothing else +-- would notice: the redemption itself is already committed. +CREATE OR REPLACE FUNCTION app.prevent_pairing_reuse_mutation() +RETURNS trigger +LANGUAGE plpgsql +AS $body$ +BEGIN + -- The redemption columns are immutable; the attempt counter is not, because the + -- rate limiter updates it. Guarding both would break the limiter, and guarding + -- nothing would let a spent code be revived. + IF OLD.used_at IS NOT NULL THEN + -- Spent. Nothing about this row may change afterwards, or a code could be + -- revived by clearing the redemption. + RAISE EXCEPTION + 'a pairing code is single-use; a redeemed code cannot be changed'; + END IF; + + -- While a code is still live, its *identity* is fixed: the hash it was minted + -- under, and the organization and node name it was authorised for. Rewriting any + -- of those would let an unspent code be re-pointed at another tenant's node, + -- which is the one thing the name binding exists to prevent. + -- + -- `expires_at` is deliberately NOT guarded. Expiry is how a code is revoked before + -- it is used - an operator who issued a code to the wrong person shortens or ages + -- it out - so it must be adjustable. It can only ever move *earlier*, never later: + -- extending a live code is not revocation, it is a second lease on a secret that + -- was issued for ten minutes. + IF NEW.code_hash IS DISTINCT FROM OLD.code_hash + OR NEW.node_name IS DISTINCT FROM OLD.node_name + OR NEW.organization_id IS DISTINCT FROM OLD.organization_id + OR NEW.expires_at > OLD.expires_at + THEN + RAISE EXCEPTION + 'a pairing code''s identity is fixed and its expiry may only be brought forward'; + END IF; + RETURN NEW; +END +$body$; + +DO $$ +BEGIN + EXECUTE 'DROP TRIGGER IF EXISTS node_pairing_codes_redeem_once ON public.node_pairing_codes'; + EXECUTE 'CREATE TRIGGER node_pairing_codes_redeem_once BEFORE UPDATE ON public.node_pairing_codes ' + 'FOR EACH ROW EXECUTE FUNCTION app.prevent_pairing_reuse_mutation()'; +END +$$; + +-- Rollback, in dependency order: +-- DROP TRIGGER IF EXISTS node_pairing_codes_redeem_once ON public.node_pairing_codes; +-- DROP FUNCTION IF EXISTS app.prevent_pairing_reuse_mutation(); +-- DROP TABLE IF EXISTS public.node_pairing_attempts; +-- DROP TABLE IF EXISTS public.node_pairing_codes; + +COMMIT; \ No newline at end of file diff --git a/migrations/0005_tenant_schemas.sql b/migrations/0005_tenant_schemas.sql new file mode 100644 index 0000000..9de1710 --- /dev/null +++ b/migrations/0005_tenant_schemas.sql @@ -0,0 +1,444 @@ +-- 0005: one PostgreSQL schema per organization. +-- +-- This is §5.0 of Umbratools_2.md: **an organization gets its own schema, and no +-- table of tenant data is shared between organizations.** It replaces shared-tables- +-- plus-RLS because that model was measured to be unenforceable twice over: the +-- application connects as the table owner, so the policies did not apply to it (gate +-- G2), and one of the thirteen `app` functions was PUBLIC-executable until it was +-- closed by hand against one database (defect B4), which means correctness depended on +-- a grant somebody remembered to make. +-- +-- With no shared table there is nothing for a policy to fail to protect: +-- +-- SELECT email FROM users; -- (users is platform; but for a tenant table) +-- SELECT name FROM projects; -- ERROR: relation "projects" does not exist +-- +-- because in a context with no tenant in hand there is no `projects` relation in the +-- search path at all. The strongest possible mistake now fails. +-- +-- ## What moves and what does not +-- +-- Four tables move: `projects`, `project_authors`, `project_publishers`, +-- `invitations`. Twenty-six stay in `public`, and the reasoning for each of the +-- thirty is in `docs/tenant-schema-classification.md`. The three that are worth +-- repeating here because they are the ones that look wrong: +-- +-- * `user_roles` stays. It is the index that resolves a session to an organization +-- (rule 2 of the classification). Putting it in a tenant schema means finding the +-- tenant requires searching every tenant schema, on the login path. +-- * `authors` and `publishers` stay. Their read policies admit any authenticated +-- principal and their owner column is nullable: they are shared reference data, +-- not one organization's content. +-- * The compute surface stays. A node is not owned by an organization, it serves +-- organizations; `jobs`, `artifacts` and `model_registry` must stay visible to a +-- controller that is not pointed at any one tenant's schema. +-- +-- ## What this costs, which the plan did not price +-- +-- Three foreign keys point into `projects` from tables that stay in `public`: +-- `artifacts_project_id_fkey`, `jobs_project_id_fkey` and +-- `user_roles_project_id_organization_id_fkey`. A foreign key cannot name a table +-- whose schema depends on the row, so all three are dropped. After this migration +-- nothing in the database refuses a `public.jobs` row that names another tenant's +-- project. The application still refuses it; the server no longer does. That is a real +-- loss and it is recorded rather than papered over. +-- +-- ## Shape +-- +-- 1. `app.tenant_schema_name(uuid)` and `app.provision_tenant_schema()` - the name +-- is derived from the organization id, so it cannot be out of step with a stored +-- column, and provisioning is `CREATE SCHEMA IF NOT EXISTS`. +-- 2. A trigger on `public.organizations` so an organization created by any writer, +-- including one added after this migration, has a schema before anything can be +-- written into it. +-- 3. The move itself: per organization, create the four tables in its schema and +-- copy that organization's rows out of `public`. +-- 4. Drop the three foreign keys described above, then drop the four `public` tables +-- — which is what removes the shared copy, and what removes the policies with it. +-- 5. Re-express `app.can_access_project` and `app.is_organization_admin_for_project`, +-- which read `projects.organization_id` and cannot after this. +-- +-- Additive where it can be, idempotent throughout: applying this file twice is a +-- no-op, and applying it to a database where it has already run is not an error. It +-- runs in one transaction, so a failure part-way leaves nothing moved. +-- +-- Rollback: `0005_tenant_schemas_rollback.sql`. That file runs; it is not a comment +-- block, because a rollback for a data migration that only exists as a comment is a +-- rollback nobody runs. + +BEGIN; + +-- --------------------------------------------------------------------------- +-- 1. Naming and provisioning a tenant schema +-- --------------------------------------------------------------------------- +-- The name is derived, not stored. +-- +-- A stored `organizations.schema_name` column would be one more thing that can +-- disagree with reality, and there is a specific failure for it: an organization +-- whose stored name does not match the schema its rows are in has data that is +-- unreachable by any name the code computes. Deriving the name from the primary key +-- makes that state unrepresentable. The cost is that the derivation exists in two +-- languages, which is why `PostgresAccountRepository.verify_tenant_schema_contract()` +-- compares this function with the Python one at startup and refuses to serve if they +-- differ — the same treatment `app.user_id` gets in `verify_rls_contract`. +CREATE OR REPLACE FUNCTION app.tenant_schema_name(target_organization_id uuid) +RETURNS text +LANGUAGE sql +IMMUTABLE +AS $body$ + SELECT 'tenant_' || replace(target_organization_id::text, '-', '') +$body$; + +COMMENT ON FUNCTION app.tenant_schema_name(uuid) IS + 'The schema holding one organization''s tables. Derived from the organization id so ' + 'it cannot disagree with a stored value. The Python side is ' + 'umbratrans_web.tenancy.tenant_schema_name and the two are compared at startup.'; + +CREATE OR REPLACE FUNCTION app.provision_tenant_schema(target_organization_id uuid) +RETURNS text +LANGUAGE plpgsql +AS $body$ +DECLARE + target_schema text := app.tenant_schema_name(target_organization_id); +BEGIN + EXECUTE format('CREATE SCHEMA IF NOT EXISTS %I', target_schema); + -- The four tenant tables, created here rather than by copying a template, so + -- that this is the only place their shape is written down. + -- + -- `projects` has no `organization_id`: it is the schema. Everything the shared + -- table used to say with that column is now said by where the table is, which is + -- why `UNIQUE (organization_id, slug)` becomes `UNIQUE (slug)` here — identical + -- inside one organization — and `UNIQUE (id, organization_id)` disappears, + -- because nothing outside this schema references a project any more. + EXECUTE format($ddl$ + CREATE TABLE IF NOT EXISTS %I.projects ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + name text NOT NULL, + slug public.citext NOT NULL, + status text NOT NULL DEFAULT 'active' + CHECK (status IN ('active', 'on_hold', 'released', 'archived')), + created_by uuid REFERENCES public.users(id), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + -- Inside one organization a slug is unique. This was the meaning of + -- `UNIQUE (organization_id, slug)` and it is unchanged. + CONSTRAINT projects_slug_key UNIQUE (slug) + ) + $ddl$, target_schema); + EXECUTE format($ddl$ + CREATE TABLE IF NOT EXISTS %I.project_authors ( + project_id uuid NOT NULL REFERENCES %1$I.projects(id) ON DELETE CASCADE, + author_id uuid NOT NULL REFERENCES public.authors(id), + credit_role text NOT NULL + CHECK (credit_role IN ('author', 'translator', 'editor', 'narrator', + 'cover_artist', 'illustrator')), + lang text NOT NULL, + "position" smallint NOT NULL DEFAULT 1, + created_at timestamptz NOT NULL DEFAULT now(), + created_by uuid REFERENCES public.users(id), + CONSTRAINT project_authors_pkey PRIMARY KEY (project_id, author_id, credit_role, lang) + ) + $ddl$, target_schema); + EXECUTE format($ddl$ + CREATE TABLE IF NOT EXISTS %I.project_publishers ( + project_id uuid NOT NULL REFERENCES %1$I.projects(id) ON DELETE CASCADE, + publisher_id uuid NOT NULL REFERENCES public.publishers(id), + lang text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + created_by uuid REFERENCES public.users(id), + CONSTRAINT project_publishers_pkey PRIMARY KEY (project_id, publisher_id, lang) + ) + $ddl$, target_schema); + EXECUTE format($ddl$ + CREATE TABLE IF NOT EXISTS %I.invitations ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + -- The project reference was `(project_id, organization_id)` because the + -- tenant had to be named to be sure. Here the schema names it. + project_id uuid REFERENCES %1$I.projects(id) ON DELETE CASCADE, + email public.citext NOT NULL, + role_id uuid NOT NULL REFERENCES public.roles(id) ON DELETE RESTRICT, + token_hash text NOT NULL UNIQUE, + invited_by uuid NOT NULL REFERENCES public.users(id), + expires_at timestamptz NOT NULL, + accepted_at timestamptz, + revoked_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now() + ) + $ddl$, target_schema); + EXECUTE format($ddl$ + CREATE INDEX IF NOT EXISTS invitations_email_idx ON %1$I.invitations (email) + $ddl$, target_schema); + EXECUTE format($ddl$ + CREATE UNIQUE INDEX IF NOT EXISTS project_authors_order_idx + ON %1$I.project_authors (project_id, credit_role, COALESCE(lang, ''), "position") + $ddl$, target_schema); + EXECUTE format($ddl$ + CREATE UNIQUE INDEX IF NOT EXISTS project_publishers_once_idx + ON %1$I.project_publishers (project_id, publisher_id, COALESCE(lang, '')) + $ddl$, target_schema); + -- The same trigger the shared table had, so `updated_at` keeps meaning what it + -- meant. `app.set_updated_at` is unchanged and still in `public`'s reach. + -- + -- Guarded on the trigger rather than `DROP ... IF EXISTS` then `CREATE`, because a + -- `DROP IF EXISTS` that finds nothing is a NOTICE, and one NOTICE per organization + -- buries a real error in a run of a hundred harmless ones. + IF NOT EXISTS ( + SELECT 1 + FROM pg_trigger + WHERE tgrelid = format('%I.projects', target_schema)::regclass + AND tgname = 'projects_set_updated_at' + ) THEN + EXECUTE format( + 'CREATE TRIGGER projects_set_updated_at BEFORE UPDATE ON %I.projects ' + 'FOR EACH ROW EXECUTE FUNCTION app.set_updated_at()', + target_schema + ); + END IF; + RETURN target_schema; +END +$body$; + +-- --------------------------------------------------------------------------- +-- 2. Every organization has a schema, including ones created after this +-- --------------------------------------------------------------------------- +-- Without this, an organization could be inserted by any writer — the console, a +-- script, the next migration — and have no schema, which is not a state in which its +-- data has somewhere to live. The trigger makes the invariant structural: the schema +-- exists in the same transaction as the organization row. +CREATE OR REPLACE FUNCTION app.provision_tenant_schema_for_new_organization() +RETURNS trigger +LANGUAGE plpgsql +AS $body$ +BEGIN + PERFORM app.provision_tenant_schema(NEW.id); + RETURN NEW; +END +$body$; + +DROP TRIGGER IF EXISTS organizations_provision_tenant_schema ON public.organizations; +CREATE TRIGGER organizations_provision_tenant_schema + AFTER INSERT ON public.organizations + FOR EACH ROW EXECUTE FUNCTION app.provision_tenant_schema_for_new_organization(); + +-- --------------------------------------------------------------------------- +-- 3. The move +-- --------------------------------------------------------------------------- +-- Per organization: create the tables, then copy that organization's rows. +-- +-- `ON CONFLICT DO NOTHING` on every copy is what makes re-running safe. Without it a +-- second run fails on the primary key, and a migration that cannot be applied twice +-- is one that will be applied once, hopefully. +DO $move$ +DECLARE + organization record; + target_schema text; + -- The shared tables are the *source*, and a second run finds them already gone: + -- section 5 dropped them. Every copy is therefore guarded on its source still + -- being there, which is what makes applying this file twice a no-op rather than an + -- error on a migration that has already been applied. + shared_projects constant boolean := to_regclass('public.projects') IS NOT NULL; + shared_authors constant boolean := to_regclass('public.project_authors') IS NOT NULL; + shared_publishers constant boolean := to_regclass('public.project_publishers') IS NOT NULL; + shared_invitations constant boolean := to_regclass('public.invitations') IS NOT NULL; +BEGIN + FOR organization IN SELECT id FROM public.organizations ORDER BY id LOOP + target_schema := app.provision_tenant_schema(organization.id); + + IF shared_projects THEN + EXECUTE format( + 'INSERT INTO %I.projects (id, name, slug, status, created_by, created_at, updated_at) ' + 'SELECT id, name, slug, status, created_by, created_at, updated_at ' + ' FROM public.projects WHERE organization_id = %L ' + 'ON CONFLICT DO NOTHING', + target_schema, organization.id + ); + END IF; + IF shared_projects AND shared_authors THEN + EXECUTE format( + 'INSERT INTO %I.project_authors (project_id, author_id, credit_role, lang, "position", created_at, created_by) ' + 'SELECT pa.project_id, pa.author_id, pa.credit_role, pa.lang, pa."position", pa.created_at, pa.created_by ' + ' FROM public.project_authors pa ' + ' JOIN public.projects p ON p.id = pa.project_id ' + ' WHERE p.organization_id = %L ' + 'ON CONFLICT DO NOTHING', + target_schema, organization.id + ); + END IF; + IF shared_projects AND shared_publishers THEN + EXECUTE format( + 'INSERT INTO %I.project_publishers (project_id, publisher_id, lang, created_at, created_by) ' + 'SELECT pp.project_id, pp.publisher_id, pp.lang, pp.created_at, pp.created_by ' + ' FROM public.project_publishers pp ' + ' JOIN public.projects p ON p.id = pp.project_id ' + ' WHERE p.organization_id = %L ' + 'ON CONFLICT DO NOTHING', + target_schema, organization.id + ); + END IF; + IF shared_invitations THEN + EXECUTE format( + 'INSERT INTO %I.invitations (id, project_id, email, role_id, token_hash, invited_by, ' + 'expires_at, accepted_at, revoked_at, created_at) ' + 'SELECT i.id, i.project_id, i.email, i.role_id, i.token_hash, i.invited_by, ' + ' i.expires_at, i.accepted_at, i.revoked_at, i.created_at ' + ' FROM public.invitations i WHERE i.organization_id = %L ' + 'ON CONFLICT DO NOTHING', + target_schema, organization.id + ); + END IF; + END LOOP; +END +$move$; + +-- --------------------------------------------------------------------------- +-- 4. Verification, before anything is dropped +-- --------------------------------------------------------------------------- +-- A count comparison, and it raises rather than warns. The point of doing this +-- *here* is that after section 5 the shared copy is gone and there is nothing left to +-- compare against: a move that silently lost a row would be discovered by an +-- organization noticing a missing book. +DO $verify$ +DECLARE + tenant record; + moved bigint; + source bigint; +BEGIN + -- Nothing left to compare against once the shared table is gone, which is why this + -- runs before section 5 and not after it. A re-run returns here. + IF to_regclass('public.projects') IS NULL THEN + RETURN; + END IF; + FOR tenant IN SELECT o.id AS organization_id, app.tenant_schema_name(o.id) AS schema_name + FROM public.organizations o ORDER BY o.id + LOOP + CONTINUE WHEN NOT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = tenant.schema_name); + EXECUTE format('SELECT count(*) FROM %I.projects', tenant.schema_name) INTO moved; + EXECUTE format( + 'SELECT count(*) FROM public.projects WHERE organization_id = %L', + tenant.organization_id + ) INTO source; + -- `<` and not `<>`: a re-run finds the rows already copied, and a schema that + -- holds *more* than the shared table is not evidence of a loss. + IF moved < source THEN + RAISE EXCEPTION + 'tenant schema % holds % projects but the shared table holds % for organization %;' + ' refusing to drop the shared copy', + tenant.schema_name, moved, source, tenant.organization_id; + END IF; + END LOOP; +END +$verify$; + +-- --------------------------------------------------------------------------- +-- 5. Drop the shared copy +-- --------------------------------------------------------------------------- +-- The three foreign keys first, because they are what makes the four tables +-- undroppable. Each is named so that the reader can see this is a decision and not an +-- oversight. +-- +-- artifacts_project_id_fkey artifacts is platform-level; see the header +-- jobs_project_id_fkey jobs is platform-level; see the header +-- user_roles_project_id_organization_id_fkey +-- user_roles stays platform-level, so it +-- cannot point into a per-tenant schema +ALTER TABLE public.artifacts DROP CONSTRAINT IF EXISTS artifacts_project_id_fkey; +ALTER TABLE public.jobs DROP CONSTRAINT IF EXISTS jobs_project_id_fkey; +ALTER TABLE public.user_roles DROP CONSTRAINT IF EXISTS user_roles_project_id_organization_id_fkey; + +-- Now the shared tables. Their row-level-security policies go with the tables, which +-- is the point rather than a side effect: `projects_select` was `app.can_access_project`, +-- and there is no shared `projects` left for it to filter. +-- +-- `DROP TABLE` rather than `DROP TABLE ... CASCADE`, deliberately: cascading would +-- silently drop whatever else had grown a dependency, which is exactly the class of +-- change that has to be written down to be reviewed. +DROP TABLE IF EXISTS public.invitations; +DROP TABLE IF EXISTS public.project_authors; +DROP TABLE IF EXISTS public.project_publishers; +DROP TABLE IF EXISTS public.projects; + +-- --------------------------------------------------------------------------- +-- 6. The two functions that read a table which no longer exists +-- --------------------------------------------------------------------------- +-- `app.can_access_project` and `app.is_organization_admin_for_project` both resolved a +-- project to an organization by reading `projects.organization_id`. There is no shared +-- `projects` to read. +-- +-- Both are re-expressed over `public.user_roles`, which is where a project's +-- organization is still recorded: `user_roles_check` requires a `scope = 'project'` +-- row to carry `organization_id NOT NULL`, so the project-scoped assignment of any +-- project names the tenant that project belongs to. +-- +-- The narrowing this causes, stated rather than hidden: a project with no +-- project-scoped assignment at all now resolves to no organization, so an +-- organization administrator who is not assigned to it is not recognised by +-- `app.can_access_project`. Before this migration they were, because the answer came +-- from the project row rather than from an assignment. In practice the service +-- refuses an unassigned project anyway — `assign_account_to_project` creates the +-- assignment — and `app.can_access_project` is no longer used by any policy, because +-- the table it filtered has no policy left to filter. +CREATE OR REPLACE FUNCTION app.is_organization_admin_for_project(target_project_id uuid) +RETURNS boolean +LANGUAGE sql +STABLE SECURITY DEFINER +SET search_path TO 'public', 'pg_temp' +AS $body$ + SELECT app.is_platform_admin() + OR EXISTS ( + SELECT 1 + FROM public.user_roles theirs + JOIN public.user_roles mine + ON mine.organization_id = theirs.organization_id + AND mine.user_id = app.current_user_id() + AND mine.scope = 'organization' + AND mine.status = 'active' + JOIN public.roles r ON r.id = mine.role_id + WHERE theirs.project_id = target_project_id + AND theirs.scope = 'project' + AND theirs.status = 'active' + AND r.name = 'organization_admin' + ) +$body$; + +CREATE OR REPLACE FUNCTION app.can_access_project(target_project_id uuid) +RETURNS boolean +LANGUAGE sql +STABLE SECURITY DEFINER +SET search_path TO 'public', 'pg_temp' +AS $body$ + SELECT app.is_platform_admin() + OR EXISTS ( + SELECT 1 + FROM public.user_roles mine + JOIN public.roles r ON r.id = mine.role_id + WHERE mine.user_id = app.current_user_id() + AND mine.scope = 'organization' + AND mine.status = 'active' + AND r.name IN ('organization_admin', 'member') + AND EXISTS ( + SELECT 1 + FROM public.user_roles theirs + WHERE theirs.project_id = target_project_id + AND theirs.scope = 'project' + AND theirs.status = 'active' + AND theirs.organization_id = mine.organization_id + ) + ) + OR EXISTS ( + SELECT 1 + FROM public.user_roles mine + WHERE mine.user_id = app.current_user_id() + AND mine.project_id = target_project_id + AND mine.scope = 'project' + AND mine.status = 'active' + ) +$body$; + +-- --------------------------------------------------------------------------- +-- 7. Ledger +-- --------------------------------------------------------------------------- +-- Written last, so a migration that fails above leaves no record that it ran. +INSERT INTO public.schema_migrations (version) VALUES ('0005_tenant_schemas') +ON CONFLICT (version) DO NOTHING; + +COMMIT; \ No newline at end of file diff --git a/migrations/0005_tenant_schemas_rollback.sql b/migrations/0005_tenant_schemas_rollback.sql new file mode 100644 index 0000000..d9f38b6 --- /dev/null +++ b/migrations/0005_tenant_schemas_rollback.sql @@ -0,0 +1,313 @@ +-- 0005 rollback: put the four tenant tables back in `public`, one shared table again. +-- +-- This file **runs**. It is not the commented-out sketch at the end of a +-- migration, because a rollback for a data migration that exists only as a comment +-- is a rollback nobody runs. +-- +-- psql -d -v ON_ERROR_STOP=1 -f 0005_tenant_schemas_rollback.sql +-- +-- It undoes 0005 completely: +-- +-- 1. recreate `public.projects`, `public.project_authors`, +-- `public.project_publishers` and `public.invitations` with their +-- `organization_id` columns, constraints, indexes, policies and trigger; +-- 2. copy every organization's rows back, recovering `organization_id` from the +-- schema name - the name was derived from the identifier, so this is exact and +-- not a guess; +-- 3. restore the three foreign keys that pointed into `projects`, and the two +-- `app` functions that read it; +-- 4. drop the tenant schemas, the trigger and the two provisioning functions; +-- 5. remove the ledger row. +-- +-- ## What it cannot restore +-- +-- Three things, stated rather than discovered afterwards: +-- +-- * **Rows written into a tenant schema since the migration.** They are copied +-- back, so they survive; but a row whose `created_by` names a user deleted in +-- the meantime will fail the foreign key and stop the copy. That is the +-- constraint doing its job, and the fix is to resolve the user, not to drop the +-- constraint. +-- * **The column order and any constraint added by a later migration** that +-- referenced the tenant tables. This rollback is written against 0001-0005 and +-- nothing after them. +-- * **The gate it reopens.** The shared tables have no `organization_id` +-- filtering unless the policies below are enforced, and they are not enforced +-- for the table owner, which is the application. Rolling back returns the +-- deployment to gate **G2** and to shared-tables-plus-RLS, both of which §5.0 +-- records as measured failures. Rolling forward is the direction; this file +-- exists so that the decision is reversible, not because it is better. + +BEGIN; + +-- --------------------------------------------------------------------------- +-- 1. The shared tables, as they were +-- --------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS public.projects ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + organization_id uuid NOT NULL REFERENCES public.organizations(id), + name text NOT NULL, + slug public.citext NOT NULL, + status text NOT NULL DEFAULT 'active' + CHECK (status IN ('active', 'on_hold', 'released', 'archived')), + created_by uuid REFERENCES public.users(id), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + -- Referenced by `invitations` and `user_roles` as `(project_id, + -- organization_id)`, so the pair has to be unique for those foreign keys to + -- exist again. + CONSTRAINT projects_id_organization_id_key UNIQUE (id, organization_id), + CONSTRAINT projects_organization_id_slug_key UNIQUE (organization_id, slug) +); + +CREATE TABLE IF NOT EXISTS public.project_authors ( + project_id uuid NOT NULL REFERENCES public.projects(id) ON DELETE CASCADE, + author_id uuid NOT NULL REFERENCES public.authors(id), + credit_role text NOT NULL + CHECK (credit_role IN ('author', 'translator', 'editor', 'narrator', + 'cover_artist', 'illustrator')), + lang text NOT NULL, + "position" smallint NOT NULL DEFAULT 1, + created_at timestamptz NOT NULL DEFAULT now(), + created_by uuid REFERENCES public.users(id), + CONSTRAINT project_authors_pkey PRIMARY KEY (project_id, author_id, credit_role, lang) +); + +CREATE TABLE IF NOT EXISTS public.project_publishers ( + project_id uuid NOT NULL REFERENCES public.projects(id) ON DELETE CASCADE, + publisher_id uuid NOT NULL REFERENCES public.publishers(id), + lang text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + created_by uuid REFERENCES public.users(id), + CONSTRAINT project_publishers_pkey PRIMARY KEY (project_id, publisher_id, lang) +); + +CREATE TABLE IF NOT EXISTS public.invitations ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + organization_id uuid NOT NULL REFERENCES public.organizations(id) ON DELETE CASCADE, + project_id uuid, + email public.citext NOT NULL, + role_id uuid NOT NULL REFERENCES public.roles(id) ON DELETE RESTRICT, + token_hash text NOT NULL UNIQUE, + invited_by uuid NOT NULL REFERENCES public.users(id), + expires_at timestamptz NOT NULL, + accepted_at timestamptz, + revoked_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT invitations_project_id_organization_id_fkey + FOREIGN KEY (project_id, organization_id) + REFERENCES public.projects(id, organization_id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS projects_organization_idx ON public.projects (organization_id); +CREATE INDEX IF NOT EXISTS invitations_email_idx ON public.invitations (email); +CREATE UNIQUE INDEX IF NOT EXISTS project_authors_order_idx + ON public.project_authors (project_id, credit_role, COALESCE(lang, ''), "position"); +CREATE UNIQUE INDEX IF NOT EXISTS project_publishers_once_idx + ON public.project_publishers (project_id, publisher_id, COALESCE(lang, '')); + +-- --------------------------------------------------------------------------- +-- 2. The rows, back into the shared tables +-- --------------------------------------------------------------------------- +-- `organization_id` is recovered by parsing the schema name, which is the +-- definition `app.tenant_schema_name` used to build it. That is why the name was +-- derived rather than stored: a rollback can rebuild the column from the schema, +-- and a stored name could have disagreed with the rows. +DO $restore$ +DECLARE + tenant record; + organization_id uuid; +BEGIN + FOR tenant IN SELECT nspname FROM pg_namespace WHERE nspname LIKE 'tenant\_%' ORDER BY nspname LOOP + BEGIN + organization_id := substring(tenant.nspname FROM 8)::uuid; + EXCEPTION WHEN others THEN + -- A schema that matches the pattern but is not one of ours. Skipped + -- loudly rather than dropped: something else made it, and guessing at + -- its contents is how a rollback destroys data. + RAISE WARNING 'skipping %: its name is not an organization identifier', tenant.nspname; + CONTINUE; + END; + + EXECUTE format( + 'INSERT INTO public.projects (id, organization_id, name, slug, status, created_by, created_at, updated_at) ' + 'SELECT id, %L, name, slug, status, created_by, created_at, updated_at FROM %I.projects ' + 'ON CONFLICT DO NOTHING', + organization_id, tenant.nspname + ); + EXECUTE format( + 'INSERT INTO public.project_authors (project_id, author_id, credit_role, lang, "position", created_at, created_by) ' + 'SELECT project_id, author_id, credit_role, lang, "position", created_at, created_by FROM %I.project_authors ' + 'ON CONFLICT DO NOTHING', + tenant.nspname + ); + EXECUTE format( + 'INSERT INTO public.project_publishers (project_id, publisher_id, lang, created_at, created_by) ' + 'SELECT project_id, publisher_id, lang, created_at, created_by FROM %I.project_publishers ' + 'ON CONFLICT DO NOTHING', + tenant.nspname + ); + EXECUTE format( + 'INSERT INTO public.invitations (id, organization_id, project_id, email, role_id, token_hash, invited_by, ' + 'expires_at, accepted_at, revoked_at, created_at) ' + 'SELECT id, %L, project_id, email, role_id, token_hash, invited_by, expires_at, accepted_at, revoked_at, created_at ' + ' FROM %I.invitations ON CONFLICT DO NOTHING', + organization_id, tenant.nspname + ); + END LOOP; +END +$restore$; + +-- --------------------------------------------------------------------------- +-- 3. The policies, and the trigger +-- --------------------------------------------------------------------------- +-- Restored verbatim from the 0001-0004 schema. They are not decoration: without +-- them the four tables sit in `public` with no tenant filter at all, which is +-- strictly worse than either state - the tables are shared *and* unprotected. +ALTER TABLE public.projects ENABLE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS projects_select ON public.projects; +CREATE POLICY projects_select ON public.projects FOR SELECT USING (app.can_access_project(id)); +DROP POLICY IF EXISTS projects_insert ON public.projects; +CREATE POLICY projects_insert ON public.projects FOR INSERT + WITH CHECK (app.is_platform_admin() OR app.is_organization_admin(organization_id)); +DROP POLICY IF EXISTS projects_update ON public.projects; +CREATE POLICY projects_update ON public.projects FOR UPDATE + USING (app.is_platform_admin() OR app.is_organization_admin(organization_id)) + WITH CHECK (app.is_platform_admin() OR app.is_organization_admin(organization_id)); +DROP POLICY IF EXISTS projects_delete ON public.projects; +CREATE POLICY projects_delete ON public.projects FOR DELETE + USING (app.is_platform_admin() OR app.is_organization_admin(organization_id)); + +ALTER TABLE public.project_authors ENABLE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS project_authors_tenant ON public.project_authors; +CREATE POLICY project_authors_tenant ON public.project_authors + USING (app.is_platform_admin() OR EXISTS ( + SELECT 1 FROM public.projects p + WHERE p.id = project_authors.project_id + AND app.is_organization_member(p.organization_id))); + +ALTER TABLE public.project_publishers ENABLE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS project_publishers_tenant ON public.project_publishers; +CREATE POLICY project_publishers_tenant ON public.project_publishers + USING (app.is_platform_admin() OR EXISTS ( + SELECT 1 FROM public.projects p + WHERE p.id = project_publishers.project_id + AND app.is_organization_member(p.organization_id))); + +ALTER TABLE public.invitations ENABLE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS invitations_manage ON public.invitations; +CREATE POLICY invitations_manage ON public.invitations + USING (app.is_platform_admin() OR app.is_organization_admin(organization_id)) + WITH CHECK (app.is_platform_admin() OR app.is_organization_admin(organization_id)); +DROP POLICY IF EXISTS invitations_select ON public.invitations; +CREATE POLICY invitations_select ON public.invitations FOR SELECT USING ( + app.is_platform_admin() + OR app.is_organization_admin(organization_id) + OR (email = (SELECT u.email FROM public.users u WHERE u.id = app.current_user_id()))); + +DO $$ +BEGIN + EXECUTE 'DROP TRIGGER IF EXISTS projects_set_updated_at ON public.projects'; + EXECUTE 'CREATE TRIGGER projects_set_updated_at BEFORE UPDATE ON public.projects ' + 'FOR EACH ROW EXECUTE FUNCTION app.set_updated_at()'; +END +$$; + +-- --------------------------------------------------------------------------- +-- 4. The foreign keys and functions that 0005 removed +-- --------------------------------------------------------------------------- +-- Guarded on the catalogue rather than written plainly, because PostgreSQL has no +-- `ADD CONSTRAINT IF NOT EXISTS`. It is not decoration: every other statement in +-- this file is safe to run twice, and a rollback that fails on the second attempt +-- is one that cannot be used to recover from a failure part-way through itself - +-- and a rollback of a per-tenant migration on a database with a few hundred +-- tenants is exactly where that happens. +DO $keys$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'artifacts_project_id_fkey') THEN + EXECUTE 'ALTER TABLE public.artifacts ADD CONSTRAINT artifacts_project_id_fkey ' + 'FOREIGN KEY (project_id) REFERENCES public.projects(id) ON DELETE CASCADE'; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'jobs_project_id_fkey') THEN + EXECUTE 'ALTER TABLE public.jobs ADD CONSTRAINT jobs_project_id_fkey ' + 'FOREIGN KEY (project_id) REFERENCES public.projects(id)'; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'user_roles_project_id_organization_id_fkey') THEN + EXECUTE 'ALTER TABLE public.user_roles ADD CONSTRAINT user_roles_project_id_organization_id_fkey ' + 'FOREIGN KEY (project_id, organization_id) REFERENCES public.projects(id, organization_id) ' + 'ON DELETE CASCADE'; + END IF; +END +$keys$; + +-- The original bodies, which read `projects.organization_id` and could not survive +-- the table leaving `public`. Restored verbatim so that a rolled-back database is +-- byte-for-byte the schema 0005 found rather than an approximation of it. +CREATE OR REPLACE FUNCTION app.can_access_project(target_project_id uuid) +RETURNS boolean +LANGUAGE sql +STABLE SECURITY DEFINER +SET search_path TO 'public', 'pg_temp' +AS $body$ + SELECT app.is_platform_admin() + OR EXISTS ( + SELECT 1 + FROM projects p + WHERE p.id = target_project_id + AND app.is_organization_admin(p.organization_id) + ) + OR EXISTS ( + SELECT 1 + FROM user_roles ur + WHERE ur.user_id = app.current_user_id() + AND ur.project_id = target_project_id + AND ur.scope = 'project' + AND ur.status = 'active' + ) +$body$; + +CREATE OR REPLACE FUNCTION app.is_organization_admin_for_project(target_project_id uuid) +RETURNS boolean +LANGUAGE sql +STABLE SECURITY DEFINER +SET search_path TO 'public', 'pg_temp' +AS $body$ + SELECT app.is_platform_admin() + OR EXISTS ( + SELECT 1 + FROM projects p + WHERE p.id = target_project_id + AND app.is_organization_admin(p.organization_id) + ) +$body$; + +-- --------------------------------------------------------------------------- +-- 5. The tenant schemas, the trigger, the functions, the ledger row +-- --------------------------------------------------------------------------- +-- Dropped last, one schema per statement, because each of the four tables in a +-- schema holds a foreign key and a few hundred of them in one transaction will +-- exhaust the lock table - which is a failure that looks like a PostgreSQL defect +-- rather than a rollback that got too big. +-- +-- When it does happen, the answer is to **run this file again**. Every statement +-- above it is safe to run twice, the constraints are guarded on the catalogue +-- precisely so that a re-run does not stop at the first one, and the schema drops +-- use `IF EXISTS`. So the recovery is another run, not a hand-written fix. +DO $drop$ +DECLARE + tenant text; +BEGIN + FOR tenant IN SELECT nspname FROM pg_namespace WHERE nspname LIKE 'tenant\_%' LOOP + EXECUTE format('DROP SCHEMA IF EXISTS %I CASCADE', tenant); + END LOOP; +END +$drop$; + +DROP TRIGGER IF EXISTS organizations_provision_tenant_schema ON public.organizations; +DROP FUNCTION IF EXISTS app.provision_tenant_schema_for_new_organization(); +DROP FUNCTION IF EXISTS app.provision_tenant_schema(uuid); +COMMENT ON FUNCTION app.tenant_schema_name(uuid) IS NULL; + +DELETE FROM public.schema_migrations WHERE version = '0005_tenant_schemas'; + +COMMIT; \ No newline at end of file diff --git a/migrations/0006_node_removal_and_assignments.sql b/migrations/0006_node_removal_and_assignments.sql new file mode 100644 index 0000000..08fcfc0 --- /dev/null +++ b/migrations/0006_node_removal_and_assignments.sql @@ -0,0 +1,131 @@ +-- 0006: a node can be removed from the fleet for good. +-- +-- Adds `removed` to the node status vocabulary, and creates `job_assignments`, which is +-- how the controller hands work to a node from now on. +-- +-- Two things here are deliberate and worth reading before changing either. + +-- --------------------------------------------------------------------------- +-- 1. `removed` +-- --------------------------------------------------------------------------- +-- +-- Distinct from `retired`, and the distinction is the point: +-- +-- retired the machine is out of service but the node is a record of it. It can be +-- returned to service by an administrator, and it still holds devices. +-- removed the machine is gone. Credentials revoked, devices and heartbeats deleted, +-- and the row kept only because `job_events.node_id` refers to it. +-- +-- The row cannot be deleted. `job_events.node_id` and `node_pairing_codes.used_by_node_id` +-- both reference `compute_nodes` ON DELETE NO ACTION, so the DELETE is refused outright; +-- changing them to SET NULL does not help, because that is an UPDATE of `job_events` and +-- `job_events_append_only` raises on every update by design. Removing the row would mean +-- either losing the record of what the machine did, or weakening the guarantee that the +-- event trail is immutable. The trail is worth more than a tidy DELETE. +-- +-- `removed` is terminal. There is no administrative path back out of it, so a machine +-- written off cannot be quietly resurrected - and a replacement machine pairs under a new +-- name derived from its own MAC, which is what makes the tombstone harmless. + +ALTER TABLE public.compute_nodes DROP CONSTRAINT IF EXISTS compute_nodes_status_check; +ALTER TABLE public.compute_nodes ADD CONSTRAINT compute_nodes_status_check + CHECK (status = ANY (ARRAY[ + 'provisioning', 'online', 'draining', 'offline', 'retired', 'removed' + ])); + +-- A removed node must never be resurrected by a heartbeat. `heartbeat` promotes +-- `provisioning` and `offline` to `online`, and a removed node is neither, so the +-- existing CASE already leaves it alone. Asserting it here means a future edit to that +-- CASE cannot quietly start reviving removed machines, which is the failure this status +-- exists to prevent and the one that would be hardest to notice. + +CREATE OR REPLACE FUNCTION app.assert_removed_stays_removed() +RETURNS trigger LANGUAGE plpgsql AS $body$ +BEGIN + IF OLD.status = 'removed' AND NEW.status <> 'removed' THEN + RAISE EXCEPTION + 'a removed node cannot return to service (was %, asked for %)', + OLD.status, NEW.status; + END IF; + RETURN NEW; +END; +$body$; + +DROP TRIGGER IF EXISTS compute_nodes_removed_is_terminal ON public.compute_nodes; +CREATE TRIGGER compute_nodes_removed_is_terminal + BEFORE UPDATE OF status ON public.compute_nodes + FOR EACH ROW EXECUTE FUNCTION app.assert_removed_stays_removed(); + +-- --------------------------------------------------------------------------- +-- 2. job_assignments +-- --------------------------------------------------------------------------- +-- +-- The controller assigns work; a node no longer claims it. `jobs.lease_owner` said who +-- held a job and it could only be one node, which cannot express a job given to several +-- machines at once, and cannot say which of them finished it. +-- +-- A job is complete when the controller holds the artifacts that show it is. That makes +-- the assignment - the attempt - a separate thing from the job - the objective - which is +-- what this table is: one row per (job, node, attempt). Several rows may be open for one +-- job at once; the first whose artifacts verify wins and the rest are cancelled. +-- +-- `state` is deliberately not the job's status. A job an assignment is working on is +-- `running` whatever the assignment says, and a `succeeded` assignment is not a completed +-- job until `artifacts_verified_at` is set. Collapsing the two is how a node's word +-- becomes the definition of done. + +CREATE TABLE IF NOT EXISTS public.job_assignments ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + job_id uuid NOT NULL REFERENCES public.jobs (id) ON DELETE CASCADE, + node_id uuid NOT NULL REFERENCES public.compute_nodes (id) ON DELETE RESTRICT, + state text NOT NULL DEFAULT 'assigned', + attempt integer NOT NULL DEFAULT 1, + assigned_at timestamptz NOT NULL DEFAULT now(), + started_at timestamptz, + finished_at timestamptz, + lease_expires_at timestamptz, + artifacts_verified_at timestamptz, + result jsonb NOT NULL DEFAULT '{}'::jsonb, + error text, + CHECK (state = ANY (ARRAY[ + 'assigned', -- handed to the node, not yet picked up + 'running', -- the node says it started + 'succeeded', -- the node says it finished; NOT yet verified + 'failed', -- the node gave up, or its lease ran out + 'cancelled' -- superseded: another node won, or an administrator stopped it + ])), + CHECK (finished_at IS NULL OR finished_at >= assigned_at) +); + +-- One open assignment per (job, node): a node cannot be handed the same job twice at the +-- same time, but may be handed it again after a failure, which is what `attempt` counts. +CREATE UNIQUE INDEX IF NOT EXISTS job_assignments_open_per_node + ON public.job_assignments (job_id, node_id) + WHERE state IN ('assigned', 'running', 'succeeded'); + +CREATE INDEX IF NOT EXISTS job_assignments_by_node + ON public.job_assignments (node_id, state); + +CREATE INDEX IF NOT EXISTS job_assignments_by_job + ON public.job_assignments (job_id); + +-- The two questions the controller asks constantly: what is this node to do, and what is +-- still outstanding on this job. Both are served by the indexes above; this one covers +-- "which nodes are overdue", which is the reassignment path. +CREATE INDEX IF NOT EXISTS job_assignments_lease + ON public.job_assignments (lease_expires_at) + WHERE state IN ('assigned', 'running'); + +-- --------------------------------------------------------------------------- +-- 3. Recorded, not inferred +-- --------------------------------------------------------------------------- +-- +-- `artifacts_verified_at` is the whole of "a job is complete when the controller has the +-- artifacts to show it". It is set by the verification step and by nothing else, so the +-- question "is this job really done" has one answer with one writer, rather than being +-- inferred from an assignment's state and a hopeful join against `job_artifacts`. + +COMMENT ON COLUMN public.job_assignments.artifacts_verified_at IS + 'Set only by artifact verification. NULL on a succeeded assignment means the node ' + 'reported success and the controller has not yet confirmed it with artifacts; the job ' + 'is NOT complete while this is NULL.'; \ No newline at end of file diff --git a/migrations/0007_external_artifacts.sql b/migrations/0007_external_artifacts.sql new file mode 100644 index 0000000..4f95bbb --- /dev/null +++ b/migrations/0007_external_artifacts.sql @@ -0,0 +1,51 @@ +-- 0007: artifacts that live somewhere else. +-- +-- The audio node is the master for the audiobooks. UmbraTrans holds the metadata needed +-- to reason about a file and a reference to where it actually is; it does not hold the +-- bytes. DECISION-6 (object storage) stays unresolved, and this is what lets the system +-- be useful before it is resolved rather than after. +-- +-- `storage_key` was already NOT NULL and already carried a path, so nothing here is +-- required to register a file - these columns make the *reference* explicit and checkable +-- rather than implicit in a string. + +ALTER TABLE public.artifacts + ADD COLUMN IF NOT EXISTS storage_uri text, + ADD COLUMN IF NOT EXISTS storage_node_id uuid REFERENCES public.compute_nodes (id) ON DELETE SET NULL, + ADD COLUMN IF NOT EXISTS integrity text NOT NULL DEFAULT 'unverified', + ADD COLUMN IF NOT EXISTS verified_at timestamptz; + +-- The audio node is the master, so the reference has to say which machine and how. A bare +-- path means nothing to anybody reading this row six months later: `/audio/book.mp3` is +-- on whichever node happened to serve it that day. +COMMENT ON COLUMN public.artifacts.storage_uri IS + 'A URI naming where the bytes actually are, e.g. file:///media/audiobooks/… or ' + 's3://…. Deliberately not assumed to be retrievable by this system: with no object ' + 'storage behind it, nothing here fetches from it, and integrity is how a reader can ' + 'tell a checked reference from an unchecked one.'; + +COMMENT ON COLUMN public.artifacts.integrity IS + 'unverified - a node reported it and nobody has checked the bytes against ' + 'content_hash. verified - the bytes were read back and hashed, and verified_at says ' + 'when. The default is unverified because a node reporting a hash is a claim, and ' + 'treating the claim as the check is the mistake this column exists to prevent.'; + +-- Re-ingesting a library must not duplicate it. Content-addressed, so the natural key is +-- the hash within a project: the same bytes under the same project are the same artifact +-- however many times they are declared. +-- +-- Partial on storage_uri rather than unique outright, so two projects may legitimately +-- reference the same file - which is what happens when a shared library is referenced by +-- more than one book - while one project cannot end up with two rows for one file. +CREATE UNIQUE INDEX IF NOT EXISTS artifacts_once_per_project + ON public.artifacts (project_id, content_hash) + WHERE storage_uri IS NOT NULL; + +CREATE INDEX IF NOT EXISTS artifacts_by_node + ON public.artifacts (storage_node_id) + WHERE storage_node_id IS NOT NULL; + +-- Listing a project's catalogue is the only read the console and evaluation tooling do, +-- and without this it is a sequential scan of every artifact ever ingested. +CREATE INDEX IF NOT EXISTS artifacts_by_project + ON public.artifacts (project_id, created_at); \ No newline at end of file diff --git a/migrations/0008_audio_source_import.sql b/migrations/0008_audio_source_import.sql new file mode 100644 index 0000000..ef785d4 --- /dev/null +++ b/migrations/0008_audio_source_import.sql @@ -0,0 +1,305 @@ +-- 0008: the audio source import — the controller's copy of what the audio node +-- analyzed, and the tables the file-based importer writes. +-- +-- Status: APPLIED (applied by the audio-controller to `umbratrans` as +-- described in the migration procedure; verified against scratch and then +-- applied). Proposed alongside `umbratrans_web/audio_import.py` and +-- `docs/audio-system-mapping-v1.md`. +-- +-- Additive and idempotent: four new tables, one guard function, their policies. +-- Nothing existing is altered, and no existing table gains or loses a column. +-- +-- Why these tables exist at all. The audio node owns the audiobooks and exports +-- a metadata bundle; the controller is the authoritative repository for the +-- translation pipeline and must hold a copy of that analysis rather than reach +-- across to the audio database whenever it needs a chapter or a speaker. The +-- copy is a snapshot identified by `content_hash`, and the fields are the ones +-- `audio_import.py` maps — the mapping document is the contract, this file is +-- where it lands. +-- +-- Three decisions here are load-bearing and are commented where they are made: +-- +-- 1. `project_id` carries no foreign key. `projects` lives in the per-tenant +-- schema (0005) and is created by `app.provision_tenant_schema`, so there is +-- no stable relation name to reference. `public.artifacts` reached the same +-- conclusion in 0003 and this follows it: `organization_id` is the FK that +-- matters for tenancy, and `project_id` is an opaque key scoped by it. +-- 2. Identity is the source row id, never the order. `source_row_id` and +-- `external_id` come from the audio database; `reading_order` is a snapshot +-- of one export and is deliberately not unique or indexed as an identity. +-- 3. The four tables are append-only on UPDATE. A source fact whose text is +-- rewritten underneath its `source_hash` is a row that no longer describes +-- anything; a changed source is a new import, not an edit. DELETE is +-- permitted, because re-import and retention both need it and the same +-- reasoning as `artifacts` applies. +-- +-- `integrity` for an artifact is `unverified` on import by design: the importer +-- records a render's path and checksum as a claim. Reading the bytes back is a +-- different operation with a different name. + +BEGIN; + +-- Fail fast rather than half-create if the prerequisites are missing. Both are +-- phase-1 objects; if either is absent this is the wrong database. +DO $preflight$ +BEGIN + IF to_regclass('public.organizations') IS NULL THEN + RAISE EXCEPTION 'public.organizations is missing; 0008 belongs on the control-plane schema'; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = 'app' AND p.proname = 'is_organization_member') THEN + RAISE EXCEPTION 'app.is_organization_member(uuid) is missing; the RLS policies this file writes would be inert'; + END IF; +END +$preflight$; + +-- --------------------------------------------------------------------- +-- 1. The source reference: one row per project, one source system +-- --------------------------------------------------------------------- + +-- The book-level record: what this project came from, and the content hash of +-- the whole export. The hash is what makes an import idempotent — the same bytes +-- are the same import, and a different hash is a different one. +CREATE TABLE IF NOT EXISTS public.project_source_refs ( + project_id uuid PRIMARY KEY, + organization_id uuid NOT NULL REFERENCES public.organizations(id), + -- The exporting system and the node that ran it. Text, not a FK to + -- compute_nodes: the source system is a fact about the data (it is + -- `umbraaudio` on `umbraaudio-m4`), and a node row can be removed (0006) + -- without the provenance of an already-imported book becoming unknown. + system text NOT NULL CHECK (length(btrim(system)) BETWEEN 1 AND 64), + node_id text NOT NULL, + external_id text NOT NULL, + source_title text, + source_status text, + -- Languages the audio side designated, if any. Empty is accepted as an + -- answer: target languages are a translation-side decision. + languages jsonb NOT NULL DEFAULT '[]'::jsonb, + -- The numbering schemes and the hazard that accompanies them, stored + -- verbatim because the note ("three numbers identify one chapter and none + -- derives from another") is the useful part. + numbering jsonb NOT NULL DEFAULT '{}'::jsonb, + content_hash text NOT NULL CHECK (content_hash ~ '^[0-9a-f]{64}$'), + product_version text, + source_meta jsonb NOT NULL DEFAULT '{}'::jsonb, + imported_at timestamptz NOT NULL DEFAULT now(), + -- One source system may only describe one project once. A second system (a + -- future cover pipeline, say) may add its own row about the same project, so + -- the uniqueness is on the pair and not on either alone. + CONSTRAINT project_source_refs_system_external_key UNIQUE (system, external_id) +); + +CREATE INDEX IF NOT EXISTS project_source_refs_org_idx ON public.project_source_refs (organization_id); + +-- --------------------------------------------------------------------- +-- 2. Speakers +-- --------------------------------------------------------------------- + +-- One row per speaker *as the source recorded it*. The narrator is the one +-- merge the importer performs, and it is performed on the identity the pipeline +-- uses, not here: both narrator source rows are kept, and `merged_into` records +-- which spelling they resolve to. Nothing else is merged — a casefold collision +-- group is stored and surfaced to a human, never collapsed. +CREATE TABLE IF NOT EXISTS public.translation_speakers ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + project_id uuid NOT NULL, + organization_id uuid NOT NULL REFERENCES public.organizations(id), + external_id text NOT NULL, + display_name text NOT NULL, + role text NOT NULL CHECK (role IN ('narrator', 'character')), + aliases jsonb NOT NULL DEFAULT '[]'::jsonb, + collision_group text, + has_profile boolean NOT NULL DEFAULT false, + is_group boolean NOT NULL DEFAULT false, + unidentified boolean NOT NULL DEFAULT false, + edge_voice text, + tts_engine text, + voice_review_status text, + -- The canonical narrator this row resolves to, when it is not itself the + -- canonical one. NULL for characters and for the canonical narrator. + merged_into_external_id text, + source_meta jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT translation_speakers_identity_key UNIQUE (project_id, external_id) +); + +CREATE INDEX IF NOT EXISTS translation_speakers_org_idx ON public.translation_speakers (organization_id); +CREATE INDEX IF NOT EXISTS translation_speakers_project_idx ON public.translation_speakers (project_id); + +-- --------------------------------------------------------------------- +-- 3. Chapters +-- --------------------------------------------------------------------- + +CREATE TABLE IF NOT EXISTS public.translation_chapters ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + project_id uuid NOT NULL, + organization_id uuid NOT NULL REFERENCES public.organizations(id), + external_id text NOT NULL, + -- A snapshot of the export's order. Useful, not an identity. + reading_order integer NOT NULL, + matter text NOT NULL DEFAULT 'novel' CHECK (matter IN ('novel', 'matter')), + title text, + source_hash text NOT NULL CHECK (source_hash ~ '^[0-9a-f]{64}$'), + source_row_id bigint NOT NULL, + numbering_basis jsonb NOT NULL DEFAULT '{}'::jsonb, + utterance_count integer NOT NULL DEFAULT 0 CHECK (utterance_count >= 0), + source_status text, + source_meta jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT translation_chapters_identity_key UNIQUE (project_id, external_id) +); + +CREATE INDEX IF NOT EXISTS translation_chapters_org_idx ON public.translation_chapters (organization_id); +CREATE INDEX IF NOT EXISTS translation_chapters_project_idx ON public.translation_chapters (project_id, reading_order); + +-- --------------------------------------------------------------------- +-- 4. Utterances +-- --------------------------------------------------------------------- + +-- `chapter_external_id` is text and not a FK to `translation_chapters`, for the +-- same reason `project_id` has no FK: these rows arrive together in one import +-- and the importer has already resolved that every reference resolves. A FK +-- would add a constraint the import order has to satisfy without adding a +-- guarantee the importer does not already hold. +CREATE TABLE IF NOT EXISTS public.translation_utterances ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + project_id uuid NOT NULL, + organization_id uuid NOT NULL REFERENCES public.organizations(id), + external_id text NOT NULL, + chapter_external_id text NOT NULL, + utterance_index integer NOT NULL, + kind text NOT NULL CHECK (kind IN ('narration', 'dialogue', 'scene_break')), + text text NOT NULL, + source_hash text NOT NULL CHECK (source_hash ~ '^[0-9a-f]{64}$'), + speaker_external_id text NOT NULL, + -- NULL is the implicit narrator, and that is the whole meaning of NULL here. + -- An explicit attribution always carries the name the source wrote. + speaker_name_raw text, + speaker_name_resolved text NOT NULL DEFAULT '', + attribution text NOT NULL CHECK (attribution IN ('implicit_narrator', 'explicit')), + needs_review boolean NOT NULL DEFAULT false, + render_external_id text, + source_row_id bigint, + source_meta jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT translation_utterances_identity_key UNIQUE (project_id, external_id) +); + +CREATE INDEX IF NOT EXISTS translation_utterances_org_idx ON public.translation_utterances (organization_id); +CREATE INDEX IF NOT EXISTS translation_utterances_chapter_idx + ON public.translation_utterances (project_id, chapter_external_id, utterance_index); +CREATE INDEX IF NOT EXISTS translation_utterances_speaker_idx + ON public.translation_utterances (project_id, speaker_external_id); + +-- --------------------------------------------------------------------- +-- 5. Append-only on UPDATE +-- --------------------------------------------------------------------- + +-- One guard for the four tables. UPDATE is refused because a source fact is a +-- statement about a document at a moment, and rewriting it in place breaks the +-- link between the row and its `source_hash`. DELETE is permitted, exactly as it +-- is for `artifacts`: re-importing a changed bundle and reclaiming storage both +-- need it, and the guarantee that matters (no silent edit) is not weakened by +-- allowing a row to be removed and written again. +DO $guard$ +BEGIN + EXECUTE $func$ + CREATE OR REPLACE FUNCTION app.prevent_translation_source_mutation() + RETURNS trigger LANGUAGE plpgsql AS $body$ + BEGIN + RAISE EXCEPTION + 'translation source rows are a snapshot and are append-only; a changed source is a new import, not an edit'; + END; + $body$; + $func$; + + EXECUTE 'DROP TRIGGER IF EXISTS project_source_refs_append_only ON public.project_source_refs'; + EXECUTE 'CREATE TRIGGER project_source_refs_append_only BEFORE UPDATE ON public.project_source_refs FOR EACH ROW EXECUTE FUNCTION app.prevent_translation_source_mutation()'; + EXECUTE 'DROP TRIGGER IF EXISTS translation_speakers_append_only ON public.translation_speakers'; + EXECUTE 'CREATE TRIGGER translation_speakers_append_only BEFORE UPDATE ON public.translation_speakers FOR EACH ROW EXECUTE FUNCTION app.prevent_translation_source_mutation()'; + EXECUTE 'DROP TRIGGER IF EXISTS translation_chapters_append_only ON public.translation_chapters'; + EXECUTE 'CREATE TRIGGER translation_chapters_append_only BEFORE UPDATE ON public.translation_chapters FOR EACH ROW EXECUTE FUNCTION app.prevent_translation_source_mutation()'; + EXECUTE 'DROP TRIGGER IF EXISTS translation_utterances_append_only ON public.translation_utterances'; + EXECUTE 'CREATE TRIGGER translation_utterances_append_only BEFORE UPDATE ON public.translation_utterances FOR EACH ROW EXECUTE FUNCTION app.prevent_translation_source_mutation()'; + + EXECUTE 'REVOKE ALL ON FUNCTION app.prevent_translation_source_mutation() FROM PUBLIC'; + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'umbracontrol_api') THEN + EXECUTE 'GRANT EXECUTE ON FUNCTION app.prevent_translation_source_mutation() TO umbracontrol_api'; + END IF; + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'umbracontrol_worker') THEN + EXECUTE 'GRANT EXECUTE ON FUNCTION app.prevent_translation_source_mutation() TO umbracontrol_worker'; + END IF; +END +$guard$; + +-- --------------------------------------------------------------------- +-- 6. Row-level security +-- --------------------------------------------------------------------- + +-- Enabled but not forced, matching every other table in this schema: the +-- application owns these tables, so the policy is defence in depth for a +-- non-owner role and the adapter's own scoping is today's control. `FORCE` and a +-- non-owner application role arrive together at gate G2 and nowhere sooner. +-- +-- `CREATE POLICY` has no `IF NOT EXISTS`, so a re-run would fail on the second +-- apply. Each is dropped first, which is what makes this file safe to apply +-- twice. +DO $rls$ +DECLARE + target text; +BEGIN + FOREACH target IN ARRAY ARRAY[ + 'project_source_refs', 'translation_speakers', 'translation_chapters', 'translation_utterances' + ] LOOP + EXECUTE format('ALTER TABLE public.%I ENABLE ROW LEVEL SECURITY', target); + EXECUTE format('DROP POLICY IF EXISTS %I ON public.%I', target || '_tenant', target); + END LOOP; + + EXECUTE 'CREATE POLICY project_source_refs_tenant ON public.project_source_refs USING (app.is_platform_admin() OR app.is_organization_member(organization_id))'; + EXECUTE 'CREATE POLICY translation_speakers_tenant ON public.translation_speakers USING (app.is_platform_admin() OR app.is_organization_member(organization_id))'; + EXECUTE 'CREATE POLICY translation_chapters_tenant ON public.translation_chapters USING (app.is_platform_admin() OR app.is_organization_member(organization_id))'; + EXECUTE 'CREATE POLICY translation_utterances_tenant ON public.translation_utterances USING (app.is_platform_admin() OR app.is_organization_member(organization_id))'; +END +$rls$; + +-- --------------------------------------------------------------------- +-- 7. Documentation for the schema itself +-- --------------------------------------------------------------------- + +COMMENT ON TABLE public.project_source_refs IS + 'One row per project per source system: what the project was imported from and the content hash of that import.'; +COMMENT ON COLUMN public.project_source_refs.content_hash IS + 'sha256 of the exported manifest bytes. The same bytes are the same import; a different hash is a different import.'; +COMMENT ON TABLE public.translation_speakers IS + 'Speakers as the source recorded them. The narrator merge is recorded in merged_into_external_id; no other row is merged.'; +COMMENT ON TABLE public.translation_chapters IS + 'Chapters. Identity is external_id (the source row id); reading_order is a snapshot and not an identity.'; +COMMENT ON TABLE public.translation_utterances IS + 'Utterances. speaker_name_raw NULL means the implicit narrator, which is the entire meaning of NULL in that column.'; +COMMENT ON FUNCTION app.prevent_translation_source_mutation() IS + 'Refuses UPDATE on the translation source tables. A changed source is a new import, not an edit.'; + +COMMIT; + +-- --------------------------------------------------------------------- +-- Rollback, in reverse dependency order. Not applied automatically. +-- +-- DROP POLICY IF EXISTS translation_utterances_tenant ON public.translation_utterances; +-- DROP POLICY IF EXISTS translation_chapters_tenant ON public.translation_chapters; +-- DROP POLICY IF EXISTS translation_speakers_tenant ON public.translation_speakers; +-- DROP POLICY IF EXISTS project_source_refs_tenant ON public.project_source_refs; +-- DROP TRIGGER IF EXISTS translation_utterances_append_only ON public.translation_utterances; +-- DROP TRIGGER IF EXISTS translation_chapters_append_only ON public.translation_chapters; +-- DROP TRIGGER IF EXISTS translation_speakers_append_only ON public.translation_speakers; +-- DROP TRIGGER IF EXISTS project_source_refs_append_only ON public.project_source_refs; +-- DROP FUNCTION IF EXISTS app.prevent_translation_source_mutation(); +-- DROP TABLE IF EXISTS public.translation_utterances; +-- DROP TABLE IF EXISTS public.translation_chapters; +-- DROP TABLE IF EXISTS public.translation_speakers; +-- DROP TABLE IF EXISTS public.project_source_refs; +-- +-- The rollback is destructive and drops the imported data with the tables. It is +-- safe only while nothing the translation pipeline produced points at a +-- translation row, which is true today because the pipeline has not run. Once a +-- translated chapter exists, the rollback must be replaced by a deprecation, not +-- run as written. diff --git a/migrations/0009_translation_import_runs.sql b/migrations/0009_translation_import_runs.sql new file mode 100644 index 0000000..4717b6b --- /dev/null +++ b/migrations/0009_translation_import_runs.sql @@ -0,0 +1,96 @@ +-- 0009: the import run ledger — one durable row per audio source import. +-- +-- Status: NOT YET APPLIED. Proposed alongside the PostgreSQL audio-import store +-- (`umbratrans_web/audio_import_postgres.py`). Apply with 0008 already present; +-- this file depends only on `public.organizations` and the append-only guard +-- function 0008 defines, and it is additive and idempotent. +-- +-- Why a fifth table. 0008 stores *what was imported* — the source reference, the +-- speakers, chapters and utterances. It has no row for *the act of importing*, so +-- the importer had no durable identity to return for an import and would have +-- had to mint one from nothing. `project_source_refs.project_id` is one row per +-- project and identifies the project, not the run; two imports of the same book +-- at different content hashes are two runs and one project. +-- +-- The audio side named this failure directly: an import that returns a +-- `project_id` and an `import_id` that correspond to no row is worse than an +-- error, because it looks like success. `translation_imports.import_id` is the +-- row that makes `import_id` name something, and `counts` records what that run +-- actually wrote so a later "the numbers do not add up" question has an answer. +-- +-- The source hash is the idempotency key and stays on `project_source_refs` +-- (UNIQUE (system, external_id)). This table is an append-only history: it is +-- never updated, and a re-import appends a new run rather than editing the last. + +BEGIN; + +DO $preflight$ +BEGIN + IF to_regclass('public.project_source_refs') IS NULL THEN + RAISE EXCEPTION 'public.project_source_refs is missing; apply 0008 before 0009'; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = 'app' AND p.proname = 'prevent_translation_source_mutation') THEN + RAISE EXCEPTION 'app.prevent_translation_source_mutation() is missing; 0008 must be applied first'; + END IF; +END +$preflight$; + +CREATE TABLE IF NOT EXISTS public.translation_imports ( + import_id uuid PRIMARY KEY, + project_id uuid NOT NULL, + organization_id uuid NOT NULL REFERENCES public.organizations(id), + -- The exporting system and node, copied from the source reference at the + -- moment of import so the run stays legible even if a node row is removed. + system text NOT NULL CHECK (length(btrim(system)) BETWEEN 1 AND 64), + node_id text NOT NULL, + external_id text NOT NULL, + -- sha256 of the manifest bytes for this run. A run with a hash already + -- present for the project is a replay; the importer refuses it before here. + content_hash text NOT NULL CHECK (content_hash ~ '^[0-9a-f]{64}$'), + -- What this run actually wrote, not what it planned. Identical bytes dedupe + -- against rows already present, so the written counts can be lower than the + -- plan's; this is the written number and is the one an audit should read. + counts jsonb NOT NULL DEFAULT '{}'::jsonb, + imported_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS translation_imports_org_idx + ON public.translation_imports (organization_id); +CREATE INDEX IF NOT EXISTS translation_imports_project_idx + ON public.translation_imports (project_id, imported_at DESC); + +-- Append-only on UPDATE, reusing the guard 0008 already defines. A run record is +-- a statement about a moment; editing it in place would break the link between +-- the row and the counts it reports. DELETE is permitted for the same reason as +-- the other four tables: retention and re-import need it. +DO $guard$ +BEGIN + EXECUTE 'DROP TRIGGER IF EXISTS translation_imports_append_only ON public.translation_imports'; + EXECUTE 'CREATE TRIGGER translation_imports_append_only BEFORE UPDATE ON public.translation_imports FOR EACH ROW EXECUTE FUNCTION app.prevent_translation_source_mutation()'; +END +$guard$; + +DO $rls$ +BEGIN + EXECUTE 'ALTER TABLE public.translation_imports ENABLE ROW LEVEL SECURITY'; + EXECUTE 'DROP POLICY IF EXISTS translation_imports_tenant ON public.translation_imports'; + EXECUTE 'CREATE POLICY translation_imports_tenant ON public.translation_imports USING (app.is_platform_admin() OR app.is_organization_member(organization_id))'; +END +$rls$; + +COMMENT ON TABLE public.translation_imports IS + 'One row per audio source import run: the durable identity an import returns, and what it actually wrote.'; +COMMENT ON COLUMN public.translation_imports.counts IS + 'Rows actually written by this run (after deduplication), keyed by chapters/speakers/utterances/artifacts/warnings.'; +COMMENT ON COLUMN public.translation_imports.content_hash IS + 'sha256 of the manifest bytes for this run. The idempotency key lives on project_source_refs; this is the run history.'; + +COMMIT; + +-- --------------------------------------------------------------------- +-- Rollback, in reverse dependency order. Not applied automatically. +-- +-- DROP POLICY IF EXISTS translation_imports_tenant ON public.translation_imports; +-- DROP TRIGGER IF EXISTS translation_imports_append_only ON public.translation_imports; +-- DROP TABLE IF EXISTS public.translation_imports; diff --git a/migrations/0010_utterance_translations.sql b/migrations/0010_utterance_translations.sql new file mode 100644 index 0000000..c655624 --- /dev/null +++ b/migrations/0010_utterance_translations.sql @@ -0,0 +1,59 @@ +-- 0010: utterance translations — per-language translated text for controller-audio integration. +-- Status: PROPOSED +-- Additive: one new table, no existing tables altered. + +BEGIN; + +CREATE TABLE IF NOT EXISTS public.utterance_translations ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + project_id uuid NOT NULL REFERENCES public.projects(id), + utterance_id uuid NOT NULL REFERENCES public.translation_utterances(id), + language_code text NOT NULL, + language_name text NOT NULL, + text text NOT NULL, + status text NOT NULL DEFAULT 'draft', + origin text NOT NULL DEFAULT 'umbratrans', + source_ref text, + delivered_at timestamptz, + confirmed_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + voiced boolean NOT NULL DEFAULT true, + review_reason text, + UNIQUE(utterance_id, language_code) +); + +CREATE INDEX IF NOT EXISTS utterance_translations_project_idx + ON public.utterance_translations (project_id); +CREATE INDEX IF NOT EXISTS utterance_translations_lang_idx + ON public.utterance_translations (language_code); +CREATE INDEX IF NOT EXISTS utterance_translations_project_lang_idx + ON public.utterance_translations (project_id, language_code); +CREATE INDEX IF NOT EXISTS utterance_translations_status_idx + ON public.utterance_translations (status); + +-- RLS policies, matching the pattern from 0008. +DO $rls$ +BEGIN + EXECUTE 'ALTER TABLE public.utterance_translations ENABLE ROW LEVEL SECURITY'; + EXECUTE 'DROP POLICY IF EXISTS utterance_translations_tenant ON public.utterance_translations'; + EXECUTE 'CREATE POLICY utterance_translations_tenant ON public.utterance_translations USING (app.is_platform_admin() OR app.is_organization_member(project_id))'; +END +$rls$; + +COMMENT ON TABLE public.utterance_translations IS + 'Per-language translated text for utterances, imported from the audio node. One row per utterance per language.'; + +COMMENT ON COLUMN public.utterance_translations.language_code IS + 'BCP-47 language tag (e.g. en, zh-Hans).'; +COMMENT ON COLUMN public.utterance_translations.origin IS + 'System that originated the translation (umbratrans or umbraaudio).'; +COMMENT ON COLUMN public.utterance_translations.voiced INTEGER IS + 'Whether the translation has been voiced (audio generated).'; + +COMMIT; + +-- --------------------------------------------------------------------- +-- Rollback, in reverse dependency order. Not applied automatically. +-- DROP POLICY IF EXISTS utterance_translations_tenant ON public.utterance_translations; +-- DROP TABLE IF EXISTS public.utterance_translations;