26de44feb1
The UmbraAudio side of the cross-machine contract: this node registers with the UmbraTrans controller, polls for jobs, claims them under a lease, and writes artifacts back over NFS. * `umbraaudio/cluster/` - configuration, PostgreSQL access, node registration and heartbeat, the job queue, artifact storage and NFS mount management. * `scripts/cluster_worker.py` - the CLI: `worker`, `check-cluster`, `register-node`. * `scripts/backup_system.sh` / `scripts/verify_backup.sh` - whole-system backup and integrity verification. * `.env` - the `UMBRA_CLUSTER__*` settings. `UMBRA_CLUSTER__PG_PASSWORD` is deliberately empty; the password comes from the environment, not from a committed file. `k3s-agent.yaml` is NOT committed: it contains a K3s server join token. It is now gitignored, and the token should be rotated if that file has ever left this machine. ruff clean on the whole package.
141 lines
4.5 KiB
Python
141 lines
4.5 KiB
Python
"""NFS mount management for UmbraAudio cluster integration."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import subprocess
|
|
from typing import TYPE_CHECKING
|
|
|
|
if TYPE_CHECKING:
|
|
from pathlib import Path
|
|
|
|
from .config import ClusterConfig
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class NFSMounts:
|
|
"""Manages NFS mounts for cluster shared storage."""
|
|
|
|
def __init__(self, config: ClusterConfig) -> None:
|
|
self.config = config
|
|
|
|
def is_mounted(self, mount_point: Path) -> bool:
|
|
"""Check if a path is mounted."""
|
|
try:
|
|
result = subprocess.run(
|
|
["findmnt", "-n", "-o", "TARGET", str(mount_point)],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=5,
|
|
)
|
|
return result.returncode == 0 and result.stdout.strip() == str(mount_point)
|
|
except Exception:
|
|
return False
|
|
|
|
def mount(self, export: str, mount_point: Path, options: str | None = None) -> bool:
|
|
"""Mount an NFS export."""
|
|
mount_point.mkdir(parents=True, exist_ok=True)
|
|
|
|
if self.is_mounted(mount_point):
|
|
logger.debug("Already mounted: %s", mount_point)
|
|
return True
|
|
|
|
opts = options or "rw,hard,intr,rsize=1048576,wsize=1048576,timeo=600,retrans=2,vers=4.2"
|
|
cmd = ["mount", "-t", "nfs4", "-o", opts, export, str(mount_point)]
|
|
|
|
try:
|
|
logger.info("Mounting %s → %s", export, mount_point)
|
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
|
|
if result.returncode == 0:
|
|
logger.info("Mounted: %s", mount_point)
|
|
return True
|
|
else:
|
|
logger.error("Mount failed: %s", result.stderr)
|
|
return False
|
|
except subprocess.TimeoutExpired:
|
|
logger.error("Mount timed out: %s", mount_point)
|
|
return False
|
|
except Exception as e:
|
|
logger.error("Mount error: %s", e)
|
|
return False
|
|
|
|
def unmount(self, mount_point: Path) -> bool:
|
|
"""Unmount an NFS mount."""
|
|
if not self.is_mounted(mount_point):
|
|
return True
|
|
|
|
try:
|
|
result = subprocess.run(
|
|
["umount", str(mount_point)],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=10,
|
|
)
|
|
if result.returncode == 0:
|
|
logger.info("Unmounted: %s", mount_point)
|
|
return True
|
|
else:
|
|
logger.error("Unmount failed: %s", result.stderr)
|
|
return False
|
|
except Exception as e:
|
|
logger.error("Unmount error: %s", e)
|
|
return False
|
|
|
|
def ensure_all(self) -> dict[str, bool]:
|
|
"""Ensure all configured NFS mounts are present."""
|
|
results = {}
|
|
for name, (export, mount_point) in self.config.nfs_mounts.items():
|
|
results[name] = self.mount(export, mount_point)
|
|
return results
|
|
|
|
|
|
def ensure_nfs_mounts(config: ClusterConfig) -> dict[str, bool]:
|
|
"""Ensure all NFS mounts are present (convenience function)."""
|
|
mounts = NFSMounts(config)
|
|
return mounts.ensure_all()
|
|
|
|
|
|
def check_nfs_server(config: ClusterConfig) -> bool:
|
|
"""Check if NFS server is reachable and exporting expected paths."""
|
|
try:
|
|
# Check rpcbind
|
|
result = subprocess.run(
|
|
["rpcinfo", "-p", config.nfs_server],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=5,
|
|
)
|
|
if result.returncode != 0:
|
|
logger.warning("rpcinfo failed: %s", result.stderr)
|
|
return False
|
|
|
|
# Check for NFS exports
|
|
result = subprocess.run(
|
|
["showmount", "-e", config.nfs_server],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=5,
|
|
)
|
|
if result.returncode != 0:
|
|
logger.warning("showmount failed: %s", result.stderr)
|
|
return False
|
|
|
|
exports = result.stdout
|
|
required = [
|
|
config.nfs_productions_path,
|
|
config.nfs_artifacts_path,
|
|
config.nfs_covers_path,
|
|
config.nfs_audio_path,
|
|
]
|
|
missing = [p for p in required if p not in exports]
|
|
if missing:
|
|
logger.warning("Missing NFS exports: %s", missing)
|
|
return False
|
|
|
|
logger.info("NFS server %s exports verified", config.nfs_server)
|
|
return True
|
|
except Exception as e:
|
|
logger.error("NFS server check failed: %s", e)
|
|
return False
|