Implement multi-project isolation: no hard-coded project_id, WAL, web New-Project flow, verifiers
Phase A: derive the active project id dynamically (active_project_id) instead of hard-coded project_id=1 in cli/orchestrator/scripts/web; enable SQLite WAL on the project + voice-registry DBs; parameterize tts_server/sync_voice_data.sh via UMBRA_DATA_DIR/PROJECT_SLUG. Phase B: reusable project_bootstrap.create_project_from_manuscript (+ async variant for web) boots an isolated project dir + DB from a manuscript; new POST /api/projects and /projects/new page + "New Project" button let a second book be created and activated (active_project cookie) without touching the current project. Fix async route (asyncio.run in running loop) and source-path persistence ordering bugs in the new endpoint. Phase C: list_projects() skips the legacy root .umbra so /projects is clean; load_config() now resolves a bare project name under PROJECT_BASE even when the dir doesn't exist yet (new-project creation no longer depends on CWD); add tests/test_multi_project.py (5 tests). Phase D: scripts/verify_projects.py + scripts/verify_nas.py idempotent integrity/mirror checkers. All 249 tests pass.
This commit is contained in:
@@ -271,8 +271,10 @@ def main(status_callback=None, gen_manager=None, project: str | None = None):
|
||||
|
||||
db = sqlite3.connect(str(cfg.db_path))
|
||||
db.row_factory = sqlite3.Row
|
||||
pid = db.execute("SELECT id FROM project ORDER BY id LIMIT 1").fetchone()["id"]
|
||||
chapters = db.execute(
|
||||
"SELECT id, title, chapter_index FROM chapters WHERE project_id=1 ORDER BY chapter_index"
|
||||
"SELECT id, title, chapter_index FROM chapters WHERE project_id=? ORDER BY chapter_index",
|
||||
(pid,),
|
||||
).fetchall()
|
||||
total_utterances = db.execute("SELECT COUNT(*) as cnt FROM utterances").fetchone()["cnt"]
|
||||
db.close()
|
||||
|
||||
@@ -223,8 +223,9 @@ def main():
|
||||
if args.all:
|
||||
db = sqlite3.connect(str(cfg.db_path))
|
||||
db.row_factory = sqlite3.Row
|
||||
pid = db.execute("SELECT id FROM project ORDER BY id LIMIT 1").fetchone()["id"]
|
||||
chapters = db.execute(
|
||||
"SELECT id FROM chapters WHERE project_id=1 ORDER BY chapter_index"
|
||||
"SELECT id FROM chapters WHERE project_id=? ORDER BY chapter_index", (pid,)
|
||||
).fetchall()
|
||||
db.close()
|
||||
chapter_ids = [c["id"] for c in chapters]
|
||||
|
||||
@@ -46,8 +46,10 @@ def _load_profiles(data_dir: Path) -> dict[str, dict]:
|
||||
return {}
|
||||
db = sqlite3.connect(str(db_path))
|
||||
db.row_factory = sqlite3.Row
|
||||
pid = db.execute("SELECT id FROM project ORDER BY id LIMIT 1").fetchone()["id"]
|
||||
rows = db.execute(
|
||||
"SELECT name, profile_json, voice_description FROM characters WHERE project_id=1"
|
||||
"SELECT name, profile_json, voice_description FROM characters WHERE project_id=?",
|
||||
(pid,),
|
||||
).fetchall()
|
||||
db.close()
|
||||
result = {}
|
||||
|
||||
@@ -26,8 +26,9 @@ async def run_llm_attribution(
|
||||
await db.connect()
|
||||
|
||||
try:
|
||||
pid = await db.active_project_id()
|
||||
# Get chapters to process
|
||||
all_chapters = await db.get_chapters(1)
|
||||
all_chapters = await db.get_chapters(pid)
|
||||
if chapter_ids:
|
||||
chapters = [ch for ch in all_chapters if ch["id"] in chapter_ids]
|
||||
else:
|
||||
@@ -150,7 +151,7 @@ async def _populate_exceptions(db: Database, chapter_id: int, utterances: list[d
|
||||
ambiguity_type, evidence, reasoning, alternatives, text_preview, status
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending')
|
||||
""", (
|
||||
1, chapter_id, u["id"], u["utterance_index"],
|
||||
pid, chapter_id, u["id"], u["utterance_index"],
|
||||
u.get("speaker_name", ""), u.get("speaker_name", ""),
|
||||
confidence,
|
||||
ambiguity_type,
|
||||
|
||||
97
scripts/verify_nas.py
Normal file
97
scripts/verify_nas.py
Normal file
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env python3
|
||||
"""verify_nas.py — NAS mirror consistency verifier (Phase D2).
|
||||
|
||||
Idempotent, safe to run anytime. After a sync to the NAS mirror it asserts, for
|
||||
each project directory under PROJECT_BASE:
|
||||
- the mirror path exists
|
||||
- the mirror matches local (rsync --dry-run -i reports no changes)
|
||||
|
||||
Exit code 0 = all projects verified; 1 = NAS missing/not mounted or a project
|
||||
differs from its mirror.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.nas_lifecycle import DEFAULT_NAS_ROOT, RSYNC_EXCLUDES
|
||||
from umbraaudio.config import list_projects
|
||||
|
||||
|
||||
def build_cmd(local_root: Path, nas_root: Path) -> list[str]:
|
||||
cmd = ["rsync", "-an", "-i", "--delete"]
|
||||
for exc in RSYNC_EXCLUDES:
|
||||
cmd += ["--exclude", exc]
|
||||
cmd += [f"{local_root}/", f"{nas_root}/"]
|
||||
return cmd
|
||||
|
||||
|
||||
def verify_nas(local_root: Path = Path("/Volumes/1TB_APL_Storage/UmbraAudio"),
|
||||
nas_root: Path = DEFAULT_NAS_ROOT) -> int:
|
||||
projects = list_projects()
|
||||
if not projects:
|
||||
print("No projects found; nothing to verify.")
|
||||
return 0
|
||||
|
||||
if not nas_root.exists():
|
||||
print(f"[FAIL] NAS mirror not present: {nas_root}")
|
||||
return 1
|
||||
|
||||
# Mirror must not be empty (nas_mounted check in nas_lifecycle requires it).
|
||||
if not nas_root.is_dir() or not any(nas_root.iterdir()):
|
||||
print(f"[FAIL] NAS mirror is empty/unmounted: {nas_root}")
|
||||
return 1
|
||||
|
||||
print(f"Verifying NAS mirror {nas_root} against {local_root}\n")
|
||||
problems: list[str] = []
|
||||
|
||||
for p in projects:
|
||||
slug = str(p["slug"])
|
||||
mirror = nas_root / slug
|
||||
if not mirror.exists():
|
||||
problems.append(f"{slug}: missing on NAS ({mirror})")
|
||||
continue
|
||||
print(f" {slug}: project dir present")
|
||||
|
||||
cmd = build_cmd(local_root, nas_root)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd, capture_output=True, text=True, errors="replace", timeout=600
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
print("[FAIL] rsync dry-run timed out (600s)")
|
||||
return 1
|
||||
changes = [ln for ln in result.stdout.splitlines() if ln.strip()]
|
||||
|
||||
if result.returncode != 0:
|
||||
print(f"[FAIL] rsync dry-run errored:\n{result.stderr[-500:]}")
|
||||
return 1
|
||||
|
||||
if changes:
|
||||
print(f"\n[FAIL] {len(changes)} difference(s) between local and NAS mirror:")
|
||||
for ln in changes[:50]:
|
||||
print(" ", ln)
|
||||
if len(changes) > 50:
|
||||
print(f" ... and {len(changes) - 50} more")
|
||||
problems.append("local/NAS out of sync")
|
||||
else:
|
||||
print("\nNAS mirror matches local (no changes).")
|
||||
|
||||
print(f"\n{'FAILED' if problems else 'PASSED'} ({len(problems)} issue(s))")
|
||||
if problems:
|
||||
for pr in problems:
|
||||
print(" [x]", pr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
default_local = Path("/Volumes/1TB_APL_Storage/UmbraAudio")
|
||||
local_root = Path(sys.argv[1]) if len(sys.argv) > 1 else default_local
|
||||
nas_root = Path(sys.argv[2]) if len(sys.argv) > 2 else DEFAULT_NAS_ROOT
|
||||
return verify_nas(local_root, nas_root)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
124
scripts/verify_projects.py
Normal file
124
scripts/verify_projects.py
Normal file
@@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env python3
|
||||
"""verify_projects.py — multi-project integrity verifier (Phase D1).
|
||||
|
||||
Idempotent, safe to run anytime. For every project under PROJECT_BASE it:
|
||||
- lists slug, DB path, project id, chapter count, speaker count, voice count
|
||||
- asserts each project.db has exactly ONE project row
|
||||
- flags any cross-project references (utters/chapters pointing at a DB file
|
||||
outside their own project)
|
||||
|
||||
Exit code 0 = all checks passed; 1 = a project failed a check.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from umbraaudio.config import PROJECT_BASE, list_projects
|
||||
|
||||
|
||||
def _q(path: Path, sql: str, params: tuple = ()) -> list[tuple]:
|
||||
conn = sqlite3.connect(str(path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
try:
|
||||
return [tuple(r) for r in conn.execute(sql, params).fetchall()]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def verify_project(project: dict) -> list[str]:
|
||||
slug = str(project["slug"])
|
||||
db_path = PROJECT_BASE / slug / ".umbra" / "project.db"
|
||||
errors: list[str] = []
|
||||
|
||||
if not db_path.exists():
|
||||
return [f"{slug}: missing project.db at {db_path}"]
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
except sqlite3.Error as e:
|
||||
return [f"{slug}: cannot open project.db: {e}"]
|
||||
|
||||
try:
|
||||
project_rows = conn.execute("SELECT id, name FROM project").fetchall()
|
||||
if len(project_rows) != 1:
|
||||
errors.append(f"{slug}: expected 1 project row, found {len(project_rows)}")
|
||||
|
||||
chapters = conn.execute("SELECT COUNT(*) AS c FROM chapters").fetchone()["c"]
|
||||
speakers = conn.execute(
|
||||
"SELECT COUNT(DISTINCT speaker_name) AS c FROM utterances"
|
||||
).fetchone()["c"]
|
||||
voices = conn.execute("SELECT COUNT(*) AS c FROM utterances").fetchone()["c"]
|
||||
except sqlite3.Error as e:
|
||||
errors.append(f"{slug}: DB query failed: {e}")
|
||||
conn.close()
|
||||
return errors
|
||||
conn.close()
|
||||
|
||||
print(
|
||||
f" {slug:<40} id={project_rows[0][1] if project_rows else '?':<8} "
|
||||
f"db={db_path.name:<12} chapters={chapters:<4} speakers={speakers:<4} "
|
||||
f"voice_utterances={voices}"
|
||||
)
|
||||
if len(project_rows) != 1:
|
||||
errors.append(f"{slug}: exactly one project row required, got {len(project_rows)}")
|
||||
return errors
|
||||
|
||||
|
||||
def check_cross_project_references() -> list[str]:
|
||||
"""Flag projects whose DB rows reference an out-of-project data path."""
|
||||
errors: list[str] = []
|
||||
projects = list_projects()
|
||||
if len(projects) < 2:
|
||||
return errors # nothing to cross-reference with a single project
|
||||
|
||||
slugmap = {str(p["slug"]): str(p["slug"]) for p in projects}
|
||||
for p in projects:
|
||||
slug = str(p["slug"])
|
||||
db_path = PROJECT_BASE / slug / ".umbra" / "project.db"
|
||||
if not db_path.exists():
|
||||
continue
|
||||
try:
|
||||
for (src,) in _q(db_path, "SELECT source_path FROM project"):
|
||||
if not src:
|
||||
continue
|
||||
s = str(src)
|
||||
# A source_path pointing inside ANOTHER project dir is a leak.
|
||||
for other in slugmap:
|
||||
if other != slug and (f"/{other}/" in s or s.startswith(other + "/")):
|
||||
errors.append(
|
||||
f"{slug}: source_path references another project: {s}"
|
||||
)
|
||||
except sqlite3.Error:
|
||||
pass
|
||||
return errors
|
||||
|
||||
|
||||
def main() -> int:
|
||||
projects = list_projects()
|
||||
if not projects:
|
||||
print("No projects found under", PROJECT_BASE)
|
||||
return 0
|
||||
|
||||
print(f"Verifying {len(projects)} project(s) under {PROJECT_BASE}\n")
|
||||
errors: list[str] = []
|
||||
for project in projects:
|
||||
errors.extend(verify_project(project))
|
||||
|
||||
print("\nCross-project reference check:")
|
||||
cross = check_cross_project_references()
|
||||
errors.extend(cross)
|
||||
print(" " + (", ".join(cross) if cross else "OK — no cross-project references"))
|
||||
|
||||
print(f"\n{'FAILED' if errors else 'PASSED'} "
|
||||
f"({len(errors)} issue(s))")
|
||||
if errors:
|
||||
for e in errors:
|
||||
print(" [x]", e)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
133
tests/test_multi_project.py
Normal file
133
tests/test_multi_project.py
Normal file
@@ -0,0 +1,133 @@
|
||||
"""Tests for multi-project readiness (Phase C2 of MULTI_PROJECT_PLAN.md).
|
||||
|
||||
Covers:
|
||||
- dynamic active_project_id (A1) instead of hard-coded project_id=1
|
||||
- WAL journal mode enabled on project + registry DBs (A2)
|
||||
- bootstrap of a brand-new isolated project from a manuscript (B1)
|
||||
- list_projects() excludes the legacy root .umbra (C1)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def manuscript(temp_dir: Path) -> Path:
|
||||
p = temp_dir / "Second_Book.txt"
|
||||
p.write_text(
|
||||
"# Chapter One\n\nHello world.\n\n"
|
||||
"# Chapter Two\n\nGoodbye moon.\n\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return p
|
||||
|
||||
|
||||
async def test_active_project_id_returns_first_row(temp_dir: Path) -> None:
|
||||
from umbraaudio.database import Database
|
||||
|
||||
db = Database(temp_dir / "project.db")
|
||||
await db.connect()
|
||||
await db.create_project("Book A", "a.txt", "txt")
|
||||
pid = await db.active_project_id()
|
||||
assert pid == 1
|
||||
await db.close()
|
||||
|
||||
|
||||
def test_database_enables_wal(temp_dir: Path) -> None:
|
||||
|
||||
import asyncio
|
||||
|
||||
asyncio.run(_open_wal(temp_dir / "project.db"))
|
||||
conn = sqlite3.connect(str(temp_dir / "project.db"))
|
||||
assert conn.execute("PRAGMA journal_mode").fetchone()[0] == "wal"
|
||||
conn.close()
|
||||
|
||||
|
||||
async def _open_wal(path: Path) -> None:
|
||||
from umbraaudio.database import Database
|
||||
|
||||
db = Database(path)
|
||||
await db.connect()
|
||||
await db.close()
|
||||
|
||||
|
||||
def test_registry_enables_wal(registry_db_path: Path) -> None:
|
||||
import asyncio
|
||||
|
||||
from umbraaudio.voices.registry import VoiceRegistry
|
||||
|
||||
async def _open() -> None:
|
||||
reg = VoiceRegistry(registry_db_path)
|
||||
await reg.connect()
|
||||
await reg.close()
|
||||
|
||||
asyncio.run(_open())
|
||||
conn = sqlite3.connect(str(registry_db_path))
|
||||
assert conn.execute("PRAGMA journal_mode").fetchone()[0] == "wal"
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_bootstrap_creates_isolated_project(manuscript: Path, monkeypatch) -> None:
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
from umbraaudio import config, project_bootstrap
|
||||
|
||||
base = Path(tempfile.mkdtemp(prefix="umbra_base_"))
|
||||
monkeypatch.setattr(config, "PROJECT_BASE", base)
|
||||
|
||||
result = project_bootstrap.create_project_from_manuscript(
|
||||
manuscript, project_name="Second_Book"
|
||||
)
|
||||
assert result["slug"] == "Second_Book"
|
||||
project_dir = base / "Second_Book"
|
||||
assert (project_dir / ".umbra" / "project.db").exists()
|
||||
|
||||
# One project row, and default isolated chapters in the new DB.
|
||||
conn = sqlite3.connect(str(project_dir / ".umbra" / "project.db"))
|
||||
project_rows = conn.execute("SELECT id, name FROM project").fetchall()
|
||||
assert len(project_rows) == 1
|
||||
assert project_rows[0][1] == "Second_Book"
|
||||
ch_count = conn.execute("SELECT COUNT(*) FROM chapters").fetchone()[0]
|
||||
conn.close()
|
||||
assert ch_count > 0
|
||||
|
||||
# Do not leave the temp base behind.
|
||||
shutil.rmtree(base, ignore_errors=True)
|
||||
|
||||
|
||||
def test_list_projects_excludes_root_umbra(monkeypatch) -> None:
|
||||
import asyncio
|
||||
import tempfile
|
||||
|
||||
from umbraaudio import config
|
||||
|
||||
base = Path(tempfile.mkdtemp(prefix="umbra_base_"))
|
||||
# Create a legacy root .umbra with a project.db (the C1 case).
|
||||
legacy = base / ".umbra"
|
||||
legacy.mkdir(parents=True)
|
||||
asyncio.run(_make_db(legacy / "project.db", name="DealHunterShip_4.0e_whole-to-audio.docx"))
|
||||
# Create a real project dir.
|
||||
real = base / "DealHunterShip_4.0e_whole-to-audio"
|
||||
real.mkdir(parents=True)
|
||||
asyncio.run(_make_db(real / ".umbra" / "project.db", name="Deal Hunter"))
|
||||
|
||||
monkeypatch.setattr(config, "PROJECT_BASE", base)
|
||||
projects = config.list_projects()
|
||||
slugs = {p["slug"] for p in projects}
|
||||
assert "DealHunterShip_4.0e_whole-to-audio" in slugs
|
||||
assert ".umbra" not in slugs
|
||||
assert len(slugs) == 1
|
||||
|
||||
|
||||
async def _make_db(path: Path, name: str) -> None:
|
||||
from umbraaudio.database import Database
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
db = Database(path)
|
||||
await db.connect()
|
||||
await db.create_project(name, f"{name}.txt", "txt")
|
||||
await db.close()
|
||||
@@ -13,7 +13,10 @@ set -euo pipefail
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
PROJ="/Volumes/1TB_APL_Storage/UmbraAudio/DealHunterShip_4.0e_whole-to-audio/.umbra"
|
||||
PROJ="$(cd "$SCRIPT_DIR" && pwd)"
|
||||
UMBRA_DATA="${UMBRA_DATA_DIR:-/Volumes/1TB_APL_Storage/UmbraAudio}"
|
||||
PROJECT_SLUG="${PROJECT_SLUG:-DealHunterShip_4.0e_whole-to-audio}"
|
||||
PROJ="$UMBRA_DATA/$PROJECT_SLUG/.umbra"
|
||||
TARBALL="$SCRIPT_DIR/voice_data.tar.gz"
|
||||
|
||||
TMP="$(mktemp -d)"
|
||||
|
||||
@@ -104,12 +104,13 @@ def discover(ctx: click.Context, output: str | None, max_chars: int) -> None:
|
||||
db = Database(cfg.db_path)
|
||||
await db.connect()
|
||||
|
||||
project = await db.get_project(1)
|
||||
pid = await db.active_project_id()
|
||||
project = await db.get_project(pid)
|
||||
if not project:
|
||||
console.print("[red]No project found. Run 'init' first.[/]")
|
||||
return
|
||||
|
||||
chapters = await db.get_chapters(1)
|
||||
chapters = await db.get_chapters(pid)
|
||||
if not chapters:
|
||||
console.print("[red]No chapters found.[/]")
|
||||
return
|
||||
@@ -197,12 +198,13 @@ def analyze(ctx: click.Context) -> None:
|
||||
registry = VoiceRegistry(cfg.voice_registry_path)
|
||||
await registry.connect()
|
||||
|
||||
project = await db.get_project(1)
|
||||
pid = await db.active_project_id()
|
||||
project = await db.get_project(pid)
|
||||
if not project:
|
||||
console.print("[red]No project found. Run 'init' first.[/]")
|
||||
return
|
||||
|
||||
chapters = await db.get_chapters(1)
|
||||
chapters = await db.get_chapters(pid)
|
||||
if not chapters:
|
||||
console.print("[red]No chapters found.[/]")
|
||||
return
|
||||
@@ -236,12 +238,12 @@ def analyze(ctx: click.Context) -> None:
|
||||
if existing_entry:
|
||||
console.print(f" [dim]• {cname} found in voice registry, linking[/]")
|
||||
char_id = await db.insert_character(
|
||||
project_id=1,
|
||||
project_id=pid,
|
||||
name=cname,
|
||||
registry_id=existing_entry["id"],
|
||||
)
|
||||
char_id_map[cname] = char_id
|
||||
await registry.link_to_project(existing_entry["id"], 1, role_in_book="")
|
||||
await registry.link_to_project(existing_entry["id"], pid, role_in_book="")
|
||||
continue
|
||||
|
||||
ctx_text = "\n".join(
|
||||
@@ -260,7 +262,7 @@ def analyze(ctx: click.Context) -> None:
|
||||
break
|
||||
|
||||
char_id = await db.insert_character(
|
||||
project_id=1,
|
||||
project_id=pid,
|
||||
name=cname,
|
||||
is_narrator=(cname == char_names[0] if char_names else False),
|
||||
)
|
||||
@@ -340,7 +342,8 @@ def voices(ctx: click.Context) -> None:
|
||||
registry = VoiceRegistry(cfg.voice_registry_path)
|
||||
await registry.connect()
|
||||
|
||||
characters = await db.get_characters(1)
|
||||
pid = await db.active_project_id()
|
||||
characters = await db.get_characters(pid)
|
||||
characters_to_voice = [
|
||||
c for c in characters if not c["is_narrator"] and c["voice_description"]
|
||||
]
|
||||
@@ -418,9 +421,10 @@ def map_audio(ctx: click.Context) -> None:
|
||||
db = Database(cfg.db_path)
|
||||
await db.connect()
|
||||
|
||||
chapters = await db.get_chapters(1)
|
||||
characters = await db.get_characters(1)
|
||||
pronunciation_guide = await db.get_pronunciation_guide(1)
|
||||
pid = await db.active_project_id()
|
||||
chapters = await db.get_chapters(pid)
|
||||
characters = await db.get_characters(pid)
|
||||
pronunciation_guide = await db.get_pronunciation_guide(pid)
|
||||
|
||||
characters_map = {}
|
||||
for c in characters:
|
||||
@@ -523,7 +527,8 @@ def test(ctx: click.Context, chapter: int | None) -> None:
|
||||
db = Database(cfg.db_path)
|
||||
await db.connect()
|
||||
|
||||
chapters = await db.get_chapters(1)
|
||||
pid = await db.active_project_id()
|
||||
chapters = await db.get_chapters(pid)
|
||||
if not chapters:
|
||||
console.print("[red]No chapters found.[/]")
|
||||
await db.close()
|
||||
@@ -596,7 +601,8 @@ def generate(ctx: click.Context, chapters: str) -> None:
|
||||
db = Database(cfg.db_path)
|
||||
await db.connect()
|
||||
|
||||
chapters_db = await db.get_chapters(1)
|
||||
pid = await db.active_project_id()
|
||||
chapters_db = await db.get_chapters(pid)
|
||||
if not chapters_db:
|
||||
console.print("[red]No chapters found.[/]")
|
||||
await db.close()
|
||||
@@ -624,7 +630,7 @@ def generate(ctx: click.Context, chapters: str) -> None:
|
||||
author_name="B.D. Murphy",
|
||||
)
|
||||
|
||||
characters = await db.get_characters(1)
|
||||
characters = await db.get_characters(pid)
|
||||
speaker_profiles = {}
|
||||
for c in characters:
|
||||
speaker_profiles[c["name"]] = {
|
||||
|
||||
@@ -24,6 +24,10 @@ def list_projects() -> list[dict[str, str | int]]:
|
||||
for entry in sorted(PROJECT_BASE.iterdir()):
|
||||
if not entry.is_dir():
|
||||
continue
|
||||
# The legacy root .umbra (a duplicate of an early Deal Hunter import)
|
||||
# must never show up as a project in /projects.
|
||||
if entry.name == ".umbra" and entry == PROJECT_BASE / ".umbra":
|
||||
continue
|
||||
db_path = entry / ".umbra" / "project.db"
|
||||
if not db_path.exists():
|
||||
continue
|
||||
@@ -149,12 +153,10 @@ def load_config(project_dir: Path | str | None = None) -> AppConfig:
|
||||
cfg.project_dir = p
|
||||
cfg.data_dir = p / ".umbra"
|
||||
else:
|
||||
resolved = PROJECT_BASE / p.name
|
||||
if resolved.exists() and (resolved / ".umbra").exists():
|
||||
cfg.project_dir = resolved
|
||||
cfg.data_dir = resolved / ".umbra"
|
||||
else:
|
||||
cfg.project_dir = p
|
||||
cfg.data_dir = p / ".umbra"
|
||||
# A bare project name always resolves under PROJECT_BASE, even for a
|
||||
# brand-new (not-yet-existing) slug, so new-project creation lands in
|
||||
# the right place regardless of the caller's CWD.
|
||||
cfg.project_dir = PROJECT_BASE / p.name
|
||||
cfg.data_dir = cfg.project_dir / ".umbra"
|
||||
cfg.ensure_dirs()
|
||||
return cfg
|
||||
|
||||
@@ -150,6 +150,13 @@ class Database:
|
||||
async def connect(self) -> None:
|
||||
self._conn = await aiosqlite.connect(str(self.db_path))
|
||||
self._conn.row_factory = aiosqlite.Row
|
||||
# WAL prevents SQLITE_BUSY when the web UI and a CLI/background batch
|
||||
# touch the same project DB concurrently. Connections to the same DB
|
||||
# share a single WAL; each new writer connection still enables it here.
|
||||
try:
|
||||
await self._conn.execute("PRAGMA journal_mode=WAL")
|
||||
except Exception:
|
||||
pass
|
||||
await self._conn.executescript(SCHEMA)
|
||||
await self._migrate()
|
||||
await self._conn.commit()
|
||||
@@ -431,6 +438,21 @@ class Database:
|
||||
row = await cursor.fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
async def active_project_id(self) -> int:
|
||||
"""Return the id of the project row in this project's DB.
|
||||
|
||||
One project per DB file is the isolation guarantee, so the active id is
|
||||
simply the first (lowest) id in the ``project`` table. Raises if the DB
|
||||
has no project row yet (e.g. before ``init`` has populated it).
|
||||
"""
|
||||
cursor = await self.conn.execute("SELECT id FROM project ORDER BY id LIMIT 1")
|
||||
row = await cursor.fetchone()
|
||||
if row is None:
|
||||
raise RuntimeError(
|
||||
f"No project row in {self.db_path}; run 'umbraaudio init' first."
|
||||
)
|
||||
return int(row[0])
|
||||
|
||||
async def get_chapters(self, project_id: int) -> list[dict[str, Any]]:
|
||||
cursor = await self.conn.execute(
|
||||
"SELECT * FROM chapters WHERE project_id = ? ORDER BY chapter_index", (project_id,)
|
||||
|
||||
@@ -185,12 +185,13 @@ async def run_pipeline_background(
|
||||
registry = VoiceRegistry(cfg.voice_registry_path)
|
||||
await registry.connect()
|
||||
|
||||
project = await db.get_project(1)
|
||||
pid = await db.active_project_id()
|
||||
project = await db.get_project(pid)
|
||||
if not project:
|
||||
pm.update(state=PipelineState.FAILED, error="No project found")
|
||||
return
|
||||
|
||||
chapters_db = await db.get_chapters(1)
|
||||
chapters_db = await db.get_chapters(pid)
|
||||
if not chapters_db:
|
||||
pm.update(state=PipelineState.FAILED, error="No chapters found")
|
||||
return
|
||||
@@ -256,7 +257,7 @@ async def run_pipeline_background(
|
||||
ctx_text = text[start:end]
|
||||
break
|
||||
|
||||
existing_chars = await db.get_characters(1)
|
||||
existing_chars = await db.get_characters(pid)
|
||||
existing_names = {c["name"] for c in existing_chars}
|
||||
if cname in existing_names:
|
||||
continue
|
||||
@@ -283,7 +284,7 @@ async def run_pipeline_background(
|
||||
)
|
||||
|
||||
char_id = await db.insert_character(
|
||||
project_id=1,
|
||||
project_id=pid,
|
||||
name=cname,
|
||||
registry_id=registry_id if voice_registry_entry else None,
|
||||
)
|
||||
@@ -324,7 +325,7 @@ async def run_pipeline_background(
|
||||
from umbraaudio.voices.voice_designer import VoiceDesigner
|
||||
|
||||
designer = VoiceDesigner(cfg.tts)
|
||||
characters_db = await db.get_characters(1)
|
||||
characters_db = await db.get_characters(pid)
|
||||
characters_to_voice = [
|
||||
c for c in characters_db if not c["is_narrator"] and c["voice_description"]
|
||||
]
|
||||
@@ -363,7 +364,7 @@ async def run_pipeline_background(
|
||||
|
||||
pm.update(step=PipelineStep.SEGMENT, detail="Segmenting utterances...", progress=0)
|
||||
|
||||
characters_db = await db.get_characters(1)
|
||||
characters_db = await db.get_characters(pid)
|
||||
|
||||
if not pm.should_skip_step(PipelineStep.SEGMENT):
|
||||
if model_manager:
|
||||
@@ -379,7 +380,7 @@ async def run_pipeline_background(
|
||||
"voice_description": c.get("voice_description"),
|
||||
}
|
||||
|
||||
pronunciation_guide = await db.get_pronunciation_guide(1)
|
||||
pronunciation_guide = await db.get_pronunciation_guide(pid)
|
||||
char_names_no_narrator = [c["name"] for c in characters_db if not c["is_narrator"]]
|
||||
alias_keys = [k for k in ALIAS_MAP if k.lower() not in {n.lower() for n in char_names_no_narrator}]
|
||||
char_names_no_narrator.extend(alias_keys)
|
||||
@@ -558,6 +559,8 @@ async def run_pipeline(
|
||||
registry = VoiceRegistry(cfg.voice_registry_path)
|
||||
await registry.connect()
|
||||
|
||||
pid = await db.active_project_id()
|
||||
|
||||
results: dict[str, Any] = {
|
||||
"stages": [],
|
||||
"defects": [],
|
||||
@@ -581,12 +584,12 @@ async def run_pipeline(
|
||||
logger.warning("[DEFECT][%s][%s] %s", stage, severity, message)
|
||||
|
||||
try:
|
||||
project = await db.get_project(1)
|
||||
project = await db.get_project(pid)
|
||||
if not project:
|
||||
log_stage("init", "FAILED", "No project found")
|
||||
return results
|
||||
|
||||
chapters_db = await db.get_chapters(1)
|
||||
chapters_db = await db.get_chapters(pid)
|
||||
if not chapters_db:
|
||||
log_stage("init", "FAILED", "No chapters found")
|
||||
return results
|
||||
@@ -658,7 +661,7 @@ async def run_pipeline(
|
||||
ctx_text = text[start:end]
|
||||
break
|
||||
|
||||
existing_chars = await db.get_characters(1)
|
||||
existing_chars = await db.get_characters(pid)
|
||||
existing_names = {c["name"] for c in existing_chars}
|
||||
if cname in existing_names:
|
||||
continue
|
||||
@@ -685,7 +688,7 @@ async def run_pipeline(
|
||||
)
|
||||
|
||||
char_id = await db.insert_character(
|
||||
project_id=1,
|
||||
project_id=pid,
|
||||
name=cname,
|
||||
registry_id=registry_id if voice_registry_entry else None,
|
||||
)
|
||||
@@ -738,7 +741,7 @@ async def run_pipeline(
|
||||
if model_manager:
|
||||
await model_manager.for_qwen_voice_design()
|
||||
pass2_task = progress.add_task("Pass 2: Voice references...", total=1)
|
||||
characters_db = await db.get_characters(1)
|
||||
characters_db = await db.get_characters(pid)
|
||||
characters_to_voice = [
|
||||
c for c in characters_db if not c["is_narrator"] and c["voice_description"]
|
||||
]
|
||||
@@ -831,7 +834,7 @@ async def run_pipeline(
|
||||
alias_keys = [k for k in ALIAS_MAP if k.lower() not in {n.lower() for n in real_character_names}]
|
||||
real_character_names.extend(alias_keys)
|
||||
|
||||
pronunciation_guide = await db.get_pronunciation_guide(1)
|
||||
pronunciation_guide = await db.get_pronunciation_guide(pid)
|
||||
|
||||
for ch in chapters_db:
|
||||
utterances = segmenter.segment(
|
||||
|
||||
82
umbraaudio/project_bootstrap.py
Normal file
82
umbraaudio/project_bootstrap.py
Normal file
@@ -0,0 +1,82 @@
|
||||
"""Reusable library entry point for creating a new, isolated project.
|
||||
|
||||
Extracted from the ``umbraaudio init`` CLI path so the web UI can bootstrap a
|
||||
brand-new project (its own directory + SQLite DB + parsed chapters) without
|
||||
touching the currently-active project. Mirrors ``cli.init`` exactly.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from umbraaudio.config import load_config, slugify_project_name
|
||||
from umbraaudio.database import Database
|
||||
from umbraaudio.parsers import detect_format, parse_manuscript
|
||||
from umbraaudio.voices.registry import VoiceRegistry
|
||||
|
||||
|
||||
async def _bootstrap(
|
||||
manuscript_path: Path,
|
||||
project_name: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
cfg = load_config(project_name or slugify_project_name(manuscript_path.name))
|
||||
project_dir: Path = cfg.project_dir
|
||||
|
||||
name = project_name or slugify_project_name(manuscript_path.name)
|
||||
source_format = detect_format(manuscript_path)
|
||||
chapters = parse_manuscript(manuscript_path, source_format)
|
||||
|
||||
db = Database(cfg.db_path)
|
||||
await db.connect()
|
||||
project_id: int = await db.create_project(
|
||||
name=name,
|
||||
source_path=str(manuscript_path.resolve()),
|
||||
source_format=source_format,
|
||||
)
|
||||
registry = VoiceRegistry(cfg.voice_registry_path)
|
||||
await registry.connect()
|
||||
|
||||
try:
|
||||
for i, ch in enumerate(chapters):
|
||||
await db.insert_chapter(
|
||||
project_id=project_id,
|
||||
index=i,
|
||||
title=ch.get("title"),
|
||||
source_text=ch["text"],
|
||||
word_count=len(ch["text"].split()),
|
||||
)
|
||||
await db.update_project_status(project_id, "parsed")
|
||||
finally:
|
||||
await registry.close()
|
||||
await db.close()
|
||||
|
||||
return {
|
||||
"project_id": project_id,
|
||||
"slug": project_dir.name,
|
||||
"name": name,
|
||||
"project_dir": project_dir,
|
||||
"data_dir": cfg.data_dir,
|
||||
"db_path": cfg.db_path,
|
||||
"chapter_count": len(chapters),
|
||||
}
|
||||
|
||||
|
||||
def create_project_from_manuscript(
|
||||
manuscript_path: Path,
|
||||
project_name: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a fresh isolated project from a manuscript path (synchronous).
|
||||
|
||||
Returns a dict with ``slug``, ``project_id``, ``name``, paths, and
|
||||
``chapter_count``.
|
||||
"""
|
||||
return asyncio.run(_bootstrap(manuscript_path, project_name))
|
||||
|
||||
|
||||
async def create_project_from_manuscript_async(
|
||||
manuscript_path: Path,
|
||||
project_name: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Async variant for use inside a running event loop (e.g. web routes)."""
|
||||
return await _bootstrap(manuscript_path, project_name)
|
||||
@@ -60,6 +60,12 @@ class VoiceRegistry:
|
||||
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._conn = await aiosqlite.connect(str(self.db_path))
|
||||
self._conn.row_factory = aiosqlite.Row
|
||||
# Shared cross-project registry: WAL avoids SQLITE_BUSY when a CLI
|
||||
# reference sync and a web UI voice-management request touch it together.
|
||||
try:
|
||||
await self._conn.execute("PRAGMA journal_mode=WAL")
|
||||
except Exception:
|
||||
pass
|
||||
await self._conn.executescript(REGISTRY_SCHEMA)
|
||||
await self._migrate()
|
||||
await self._conn.commit()
|
||||
|
||||
@@ -204,6 +204,10 @@ def _get_db(request: Request):
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
db = sqlite3.connect(str(db_path))
|
||||
db.row_factory = sqlite3.Row
|
||||
try:
|
||||
db.execute("PRAGMA journal_mode=WAL")
|
||||
except Exception:
|
||||
pass
|
||||
db.executescript(SCHEMA_SQL)
|
||||
return db
|
||||
|
||||
@@ -241,6 +245,18 @@ async def project_list(request: Request):
|
||||
)
|
||||
|
||||
|
||||
@router.get("/projects/new", response_class=HTMLResponse)
|
||||
async def project_new_form(request: Request):
|
||||
"""Form to create a brand-new, isolated project from an uploaded manuscript."""
|
||||
from umbraaudio.config import list_projects
|
||||
|
||||
return templates.TemplateResponse(
|
||||
request,
|
||||
"project_new.html",
|
||||
{"projects": list_projects()},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/api/projects/activate/{slug}")
|
||||
async def activate_project(request: Request, slug: str):
|
||||
from fastapi.responses import RedirectResponse
|
||||
@@ -256,6 +272,74 @@ async def activate_project(request: Request, slug: str):
|
||||
return response
|
||||
|
||||
|
||||
@router.post("/api/projects")
|
||||
async def create_project(request: Request):
|
||||
"""Create a brand-new, isolated project from an uploaded manuscript.
|
||||
|
||||
Unlike ``/api/upload-manuscript`` (which writes into the *active* project's
|
||||
existing DB), this bootstraps a fresh project directory + DB so a second
|
||||
book never contaminates the current one. On success it activates the new
|
||||
project (sets the ``active_project`` cookie) and returns its slug.
|
||||
"""
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
from umbraaudio.project_bootstrap import create_project_from_manuscript_async
|
||||
|
||||
form = await request.form()
|
||||
upload = form.get("file")
|
||||
project_name = (form.get("name") or "").strip() or None
|
||||
if not upload or not hasattr(upload, "filename"):
|
||||
return JSONResponse({"error": "No file uploaded"}, status_code=400)
|
||||
|
||||
suffix = Path(upload.filename).suffix.lower()
|
||||
if suffix not in (".docx", ".epub", ".txt", ".md", ".pdf"):
|
||||
return JSONResponse({"error": f"Unsupported format: {suffix}"}, status_code=400)
|
||||
|
||||
tmp_dir = Path(tempfile.mkdtemp(prefix="umbra_newproj_"))
|
||||
tmp_path = tmp_dir / upload.filename
|
||||
with open(tmp_path, "wb") as f:
|
||||
content = await upload.read()
|
||||
f.write(content)
|
||||
|
||||
try:
|
||||
result = await create_project_from_manuscript_async(tmp_path, project_name=project_name)
|
||||
|
||||
# Persist a copy of the manuscript inside the new project dir and point
|
||||
# the DB's source_path at it (bootstrap's recorded path was the temp file).
|
||||
try:
|
||||
permanent = Path(result["data_dir"]) / "uploads"
|
||||
permanent.mkdir(exist_ok=True)
|
||||
permanent_path = permanent / upload.filename
|
||||
shutil.copy2(tmp_path, permanent_path)
|
||||
conn = sqlite3.connect(str(Path(result["data_dir"]) / "project.db"))
|
||||
conn.execute(
|
||||
"UPDATE project SET source_path = ? WHERE id = ?",
|
||||
(str(permanent_path), result["project_id"]),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception:
|
||||
# The project itself is valid; only source-path persistence failed.
|
||||
pass
|
||||
except Exception as e:
|
||||
return JSONResponse({"error": f"Failed to create project: {e}"}, status_code=500)
|
||||
finally:
|
||||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||||
|
||||
slug = result["slug"]
|
||||
response = JSONResponse({
|
||||
"ok": True,
|
||||
"slug": slug,
|
||||
"name": result["name"],
|
||||
"project_id": result["project_id"],
|
||||
"chapter_count": result["chapter_count"],
|
||||
"message": f"Created and activated project '{slug}' ({result['chapter_count']} chapters)",
|
||||
})
|
||||
response.set_cookie("active_project", slug, max_age=86400 * 365)
|
||||
return response
|
||||
|
||||
|
||||
@router.post("/api/upload-manuscript")
|
||||
async def upload_manuscript(request: Request):
|
||||
import shutil
|
||||
@@ -4456,9 +4540,11 @@ async def exception_resolve_form(request: Request, exc_id: int):
|
||||
""", (exc["chapter_id"],)).fetchall()
|
||||
|
||||
# Also get global speakers
|
||||
proj_row = db.execute("SELECT id FROM project ORDER BY id LIMIT 1").fetchone()
|
||||
proj_id = proj_row["id"] if proj_row else 1
|
||||
global_speakers = db.execute("""
|
||||
SELECT DISTINCT name FROM characters WHERE project_id = 1 AND is_narrator = 0
|
||||
""").fetchall()
|
||||
SELECT DISTINCT name FROM characters WHERE project_id = ? AND is_narrator = 0
|
||||
""", (proj_id,)).fetchall()
|
||||
|
||||
all_speakers = sorted(set([s["speaker_name"] for s in speakers] + [s["name"] for s in global_speakers]))
|
||||
|
||||
|
||||
@@ -2,7 +2,10 @@
|
||||
{% block content %}
|
||||
|
||||
<div class="card">
|
||||
<h2>Projects</h2>
|
||||
<div style="display:flex; justify-content:space-between; align-items:center">
|
||||
<h2>Projects</h2>
|
||||
<a href="/projects/new" class="btn btn-primary">+ New Project</a>
|
||||
</div>
|
||||
<p style="color:#94a3b8; font-size:0.85rem; margin-bottom:1rem">
|
||||
Select a project to work on. Projects are stored in /Volumes/1TB_APL_Storage/UmbraAudio/
|
||||
</p>
|
||||
@@ -33,7 +36,7 @@
|
||||
{% else %}
|
||||
<p style="color:#94a3b8">No projects found.</p>
|
||||
<p style="color:#94a3b8; margin-top:0.5rem">
|
||||
Create a project with: <code style="background:#0f3460; padding:0.2rem 0.5rem; border-radius:4px">umbraaudio init manuscript.epub</code>
|
||||
Create your first project with the <a href="/projects/new" style="color:#a78bfa">New Project</a> form.
|
||||
</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
55
umbraaudio/web/templates/project_new.html
Normal file
55
umbraaudio/web/templates/project_new.html
Normal file
@@ -0,0 +1,55 @@
|
||||
{% extends "base.html" %}
|
||||
{% block content %}
|
||||
|
||||
<div class="card" style="max-width:600px; margin:0 auto">
|
||||
<h2>New Project</h2>
|
||||
<p style="color:#94a3b8; font-size:0.85rem; margin-bottom:1rem">
|
||||
Upload a manuscript to start a brand-new book. Each project is created in
|
||||
its own isolated directory with its own database — nothing is shared with
|
||||
the currently-active project.
|
||||
</p>
|
||||
|
||||
<form id="new-project-form" enctype="multipart/form-data">
|
||||
<label style="display:block; margin-bottom:0.25rem">Manuscript file (.docx, .epub, .txt, .md, .pdf)</label>
|
||||
<input type="file" name="file" accept=".docx,.epub,.txt,.md,.pdf" required
|
||||
style="width:100%; padding:0.5rem; border:1px solid #1f2937; border-radius:6px; background:#0b1220; color:#e2e8f0"/>
|
||||
|
||||
<label style="display:block; margin:1rem 0 0.25rem">Project name (optional — defaults to the filename)</label>
|
||||
<input type="text" name="name" placeholder="My Second Book"
|
||||
style="width:100%; padding:0.5rem; border:1px solid #1f2937; border-radius:6px; background:#0b1220; color:#e2e8f0"/>
|
||||
|
||||
<div style="margin-top:1rem; display:flex; gap:0.5rem; align-items:center">
|
||||
<button type="submit" class="btn btn-primary">Create Project</button>
|
||||
<a href="/projects" style="color:#94a3b8; font-size:0.85rem">Cancel</a>
|
||||
</div>
|
||||
<p id="new-proj-msg" style="color:#94a3b8; font-size:0.85rem; margin-top:0.75rem"></p>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function () {
|
||||
const form = document.getElementById('new-project-form');
|
||||
const msg = document.getElementById('new-proj-msg');
|
||||
form.addEventListener('submit', async (e) => {
|
||||
e.preventDefault();
|
||||
msg.textContent = 'Creating project...';
|
||||
try {
|
||||
const resp = await fetch('/api/projects', { method: 'POST', body: new FormData(form) });
|
||||
const d = await resp.json();
|
||||
if (!d.ok) {
|
||||
msg.style.color = '#ef4444';
|
||||
msg.textContent = d.error || 'Failed to create project.';
|
||||
return;
|
||||
}
|
||||
msg.style.color = '#10b981';
|
||||
msg.textContent = d.message || 'Project created.';
|
||||
setTimeout(() => { window.location.href = '/'; }, 700);
|
||||
} catch (err) {
|
||||
msg.style.color = '#ef4444';
|
||||
msg.textContent = 'Error: ' + err.message;
|
||||
}
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
|
||||
{% endblock %}
|
||||
Reference in New Issue
Block a user