c86e323d7a
Quality-first multi-pass audiobook generation from English manuscripts. Includes: - umbraaudio/ core package (pipeline, config, voices, web review UI) - scripts/ CLI utilities + model download script - tts_server/ remote TTS server - specs/ design docs, tests/, pyproject.toml + uv.lock - Qwen3-TTS git submodule (pinned 022e286) with local demo.py UI patch saved at scripts/qwen3_tts_demo_umbra_patch.patch - .env with runtime config/secrets Excluded (see .gitignore): models/ (downloaded via scripts/download_models.sh), output/, logs/, .venv, .umbra runtime data, and book projects (DealHunterShip_*), which are committed separately.
259 lines
8.2 KiB
Python
259 lines
8.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Generate audio for chapters using Qwen3-TTS with voice cloning from references."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import sqlite3
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent.parent))
|
|
|
|
from umbraaudio.config import load_config
|
|
from umbraaudio.pronunciation import load_phonetic_names
|
|
from umbraaudio.voices.lmstudio_lifecycle import get_loaded_model, get_vram_usage_mb, is_running
|
|
from umbraaudio.voices.model_coordinator import ModelCoordinator, PipelineStage
|
|
from umbraaudio.voices.qwen_tts import VOICE_REFERENCE_TEXT, QwenTTSEngine
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def load_voice_references(voices_dir: Path) -> dict[str, Path]:
|
|
"""Load voice reference files, mapping character name -> ref audio path."""
|
|
refs: dict[str, Path] = {}
|
|
if not voices_dir.exists():
|
|
return refs
|
|
for f in voices_dir.glob("*_ref.wav"):
|
|
name = f.stem.replace("_ref", "").replace("_", " ")
|
|
refs[name] = f
|
|
return refs
|
|
|
|
|
|
def load_speakers(data_dir: Path) -> dict[str, dict]:
|
|
"""Load speakers from JSON file."""
|
|
path = data_dir / "speakers.json"
|
|
if not path.exists():
|
|
return {}
|
|
with open(path) as f:
|
|
data = json.load(f)
|
|
return {s["primary_name"]: s for s in data.get("speakers", [])}
|
|
|
|
|
|
def get_speaker_name_map(speakers: dict[str, dict]) -> dict[str, str]:
|
|
"""Build a mapping from all aliases to primary name."""
|
|
alias_map: dict[str, str] = {}
|
|
for name, info in speakers.items():
|
|
alias_map[name.lower()] = name
|
|
for alias in info.get("aliases", []):
|
|
alias_map[alias.lower()] = name
|
|
return alias_map
|
|
|
|
|
|
def _apply_phonetics(text: str, phonetic: dict[str, str]) -> str:
|
|
if not phonetic:
|
|
return text
|
|
for name, phonetic_spelling in sorted(phonetic.items(), key=lambda x: -len(x[0])):
|
|
text = text.replace(name, phonetic_spelling)
|
|
return text
|
|
|
|
|
|
def resolve_reference(
|
|
speaker_name: str,
|
|
voice_refs: dict[str, Path],
|
|
alias_map: dict[str, str],
|
|
narrator_ref: Path | None = None,
|
|
) -> Path | None:
|
|
"""Find the voice reference file for a speaker."""
|
|
if speaker_name in voice_refs:
|
|
return voice_refs[speaker_name]
|
|
for name, path in voice_refs.items():
|
|
if name.lower() == speaker_name.lower():
|
|
return path
|
|
canonical = alias_map.get(speaker_name.lower())
|
|
if canonical and canonical in voice_refs:
|
|
return voice_refs[canonical]
|
|
return narrator_ref
|
|
|
|
|
|
def generate_chapter_audio(
|
|
chapter_id: int,
|
|
engine: QwenTTSEngine,
|
|
voice_refs: dict[str, Path],
|
|
alias_map: dict[str, str],
|
|
output_base: Path,
|
|
db_path: Path,
|
|
narrator_ref: Path | None = None,
|
|
) -> int:
|
|
"""Generate audio for all utterances in a chapter using voice cloning."""
|
|
db = sqlite3.connect(str(db_path))
|
|
db.row_factory = sqlite3.Row
|
|
|
|
chapter = db.execute("SELECT * FROM chapters WHERE id = ?", (chapter_id,)).fetchone()
|
|
if not chapter:
|
|
logger.error("Chapter %d not found", chapter_id)
|
|
return 0
|
|
|
|
title = chapter["title"] or f"Chapter {chapter['chapter_index']+1}"
|
|
logger.info("Generating audio for: %s", title)
|
|
|
|
utterances = db.execute(
|
|
"""
|
|
SELECT u.id, u.speaker_name, u.text, u.utterance_index
|
|
FROM utterances u
|
|
WHERE u.chapter_id = ?
|
|
ORDER BY u.utterance_index
|
|
""",
|
|
(chapter_id,),
|
|
).fetchall()
|
|
logger.info("Found %d utterances", len(utterances))
|
|
|
|
chapter_dir = output_base / f"ch{chapter_id}"
|
|
chapter_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
generated = 0
|
|
for utt in utterances:
|
|
utt_id = utt["id"]
|
|
speaker = utt["speaker_name"] or "narrator"
|
|
text = utt["text"]
|
|
idx = utt["utterance_index"]
|
|
|
|
if not text or not text.strip():
|
|
continue
|
|
|
|
ref_path = resolve_reference(speaker, voice_refs, alias_map, narrator_ref)
|
|
|
|
filename = f"utt_{idx:04d}.wav"
|
|
output_path = chapter_dir / filename
|
|
|
|
if output_path.exists():
|
|
generated += 1
|
|
continue
|
|
|
|
spoken_text = _apply_phonetics(text, phonetic)
|
|
|
|
try:
|
|
if ref_path:
|
|
engine.generate(
|
|
text=spoken_text,
|
|
output_path=output_path,
|
|
voice_ref=ref_path,
|
|
ref_text=VOICE_REFERENCE_TEXT,
|
|
)
|
|
else:
|
|
engine.generate(
|
|
text=text,
|
|
output_path=output_path,
|
|
instruct="A calm, neutral voice speaking clearly.",
|
|
)
|
|
|
|
if output_path.exists():
|
|
generated += 1
|
|
if generated % 20 == 0:
|
|
logger.info(" Generated %d/%d utterances", generated, len(utterances))
|
|
else:
|
|
logger.warning(" Failed: utt_%04d (%s)", idx, speaker)
|
|
|
|
except Exception as e:
|
|
logger.error(" Error: utt_%04d (%s): %s", idx, speaker, e)
|
|
|
|
logger.info("Generated %d/%d utterances for %s", generated, len(utterances), title)
|
|
db.close()
|
|
return generated
|
|
|
|
|
|
def main():
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser(description="Generate chapter audio with Qwen3-TTS")
|
|
parser.add_argument("chapters", nargs="*", type=int, help="Chapter IDs to generate")
|
|
parser.add_argument("--force", action="store_true", help="Overwrite existing audio")
|
|
parser.add_argument("--all", action="store_true", help="Generate all chapters")
|
|
parser.add_argument(
|
|
"--project", type=str, default=None, help="Project name or path"
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
|
|
|
cfg = load_config(args.project)
|
|
|
|
coordinator = ModelCoordinator(cfg)
|
|
if not coordinator.switch_to(PipelineStage.UTTERANCE_GENERATION).get("ok"):
|
|
logger.error("Failed to switch models to UTTERANCE_GENERATION stage")
|
|
sys.exit(1)
|
|
|
|
voices_dir = cfg.voices_dir
|
|
|
|
voice_refs = load_voice_references(voices_dir)
|
|
speakers = load_speakers(cfg.data_dir)
|
|
alias_map = get_speaker_name_map(speakers)
|
|
|
|
phonetic = load_phonetic_names(cfg.data_dir)
|
|
logger.info("Loaded %d confirmed phonetic name mappings", len(phonetic))
|
|
|
|
narrator_ref = None
|
|
for name, path in voice_refs.items():
|
|
if name.lower() == "narrator":
|
|
narrator_ref = path
|
|
break
|
|
if not narrator_ref:
|
|
for name, path in voice_refs.items():
|
|
if name.lower() == "kainda":
|
|
narrator_ref = path
|
|
break
|
|
|
|
logger.info("Loaded %d voice references", len(voice_refs))
|
|
logger.info("Loaded %d speakers", len(speakers))
|
|
|
|
vram = get_vram_usage_mb()
|
|
logger.info("VRAM: %dMB used, %dMB free", vram["used_mb"], vram["free_mb"])
|
|
|
|
if is_running():
|
|
model = get_loaded_model()
|
|
if model:
|
|
logger.warning("LM Studio has model loaded: %s", model)
|
|
logger.warning("TTS needs ~4GB VRAM. If OOM occurs, unload the model in LM Studio UI.")
|
|
|
|
engine = QwenTTSEngine(device="cuda", sample_rate=24000)
|
|
|
|
output_base = cfg.data_dir / "chapter_audio"
|
|
output_base.mkdir(parents=True, exist_ok=True)
|
|
|
|
if args.all:
|
|
db = sqlite3.connect(str(cfg.db_path))
|
|
db.row_factory = sqlite3.Row
|
|
chapters = db.execute(
|
|
"SELECT id FROM chapters WHERE project_id=1 ORDER BY chapter_index"
|
|
).fetchall()
|
|
db.close()
|
|
chapter_ids = [c["id"] for c in chapters]
|
|
elif args.chapters:
|
|
chapter_ids = args.chapters
|
|
else:
|
|
chapter_ids = [5]
|
|
|
|
if args.force:
|
|
for ch_id in chapter_ids:
|
|
ch_dir = output_base / f"ch{ch_id}"
|
|
if ch_dir.exists():
|
|
for f in ch_dir.glob("*.wav"):
|
|
f.unlink()
|
|
logger.info("Cleared existing audio for chapter %d", ch_id)
|
|
|
|
total_generated = 0
|
|
for ch_id in chapter_ids:
|
|
count = generate_chapter_audio(
|
|
ch_id, engine, voice_refs, alias_map, output_base, cfg.db_path, narrator_ref
|
|
)
|
|
total_generated += count
|
|
|
|
engine.unload_all()
|
|
coordinator.switch_to(PipelineStage.IDLE)
|
|
|
|
logger.info("Total generated: %d utterances across %d chapters", total_generated, len(chapter_ids))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|