UmbraAudio/scripts/extract_embeddings.py

#!/usr/bin/env python3
"""Extract Qwen TTS speaker embeddings from existing voice reference WAVs.

Usage:
    python scripts/extract_embeddings.py
    python scripts/extract_embeddings.py --speaker narrator
    python scripts/extract_embeddings.py --force  # re-extract all
    python scripts/extract_embeddings.py --design "A deep male voice" --slug new_speaker

Loads the Base model once, iterates over voice ref WAVs in .umbra/voices/,
extracts the 2048-dim speaker embedding via Qwen TTS's internal encoder,
and saves each as a .npy file in .umbra/voice_embeddings/.
"""
from __future__ import annotations

import argparse
import os
import sys
import time
from pathlib import Path

import numpy as np
import soundfile as sf
import torch

project_root = Path(__file__).resolve().parent.parent
if str(project_root) not in sys.path:
    sys.path.insert(0, str(project_root))

from umbraaudio.config import load_config  # noqa: E402
from umbraaudio.voices.qwen_embeddings import QwenEmbeddingCache  # noqa: E402

BASE_MODEL_PATH = os.getenv(
    "QWEN_BASE_MODEL",
    "/Volumes/1TB_APL_Storage/UmbraAudio/models/Qwen3-TTS-12Hz-1.7B-Base",
)
VOICEDESIGN_MODEL_PATH = os.getenv(
    "QWEN_VOICEDESIGN_MODEL",
    "/Volumes/1TB_APL_Storage/UmbraAudio/models/Qwen3-TTS-12Hz-1.7B-VoiceDesign",
)


def extract_all_embeddings(
    project_name: str,
    speaker_filter: str | None = None,
    force: bool = False,
) -> None:
    config = load_config(project_name)
    voices_dir = config.data_dir / "voices"
    embeddings_dir = config.data_dir / "voice_embeddings"

    if not voices_dir.is_dir():
        print(f"ERROR: No voices directory at {voices_dir}")
        return

    cache = QwenEmbeddingCache(embeddings_dir)
    wav_files = sorted(voices_dir.glob("*_ref.wav"))
    print(f"Found {len(wav_files)} voice ref WAVs in {voices_dir}")

    if speaker_filter:
        wav_files = [w for w in wav_files if speaker_filter.lower() in w.stem.lower()]
        print(f"Filtered to {len(wav_files)} WAVs matching '{speaker_filter}'")

    to_extract = []
    for wav_path in wav_files:
        slug = wav_path.stem.replace("_ref", "")
        if force or not cache.has(slug):
            to_extract.append((slug, wav_path))

    if not to_extract:
        print("All embeddings already cached. Use --force to re-extract.")
        return

    print(f"Will extract {len(to_extract)} embeddings")

    print("Loading Base model...")
    from qwen_tts import Qwen3TTSModel

    model = Qwen3TTSModel.from_pretrained(
        BASE_MODEL_PATH,
        device_map="mps",
        dtype=torch.bfloat16,
    )
    print("Base model loaded")

    extracted = 0
    failed = 0
    t0 = time.time()

    for slug, wav_path in to_extract:
        try:
            audio, sr = sf.read(str(wav_path), dtype="float32")
            if sr != 24000:
                print(f"  SKIP {slug}: sample rate {sr} != 24000")
                failed += 1
                continue

            emb = cache.extract_and_save(
                slug, audio, sr, model.model
            )
            print(
                f"  OK {slug}: shape={emb.shape}, "
                f"norm={np.linalg.norm(emb):.2f}"
            )
            extracted += 1
        except Exception as e:
            print(f"  FAIL {slug}: {e}")
            failed += 1

    elapsed = time.time() - t0
    print(f"\nDone: {extracted} extracted, {failed} failed, {elapsed:.1f}s")
    print(f"Cached embeddings: {cache.count()} total in {embeddings_dir}")


def create_embedding_from_design(
    project_name: str,
    instruct: str,
    slug: str,
    seed: int = 42,
    force: bool = False,
) -> None:
    """Generate a voice via VoiceDesign, extract its embedding, and cache it."""
    config = load_config(project_name)
    embeddings_dir = config.data_dir / "voice_embeddings"
    cache = QwenEmbeddingCache(embeddings_dir)

    if cache.has(slug) and not force:
        print(f"Embedding for '{slug}' already exists. Use --force to regenerate.")
        return

    print("Loading VoiceDesign model...")
    from qwen_tts import Qwen3TTSModel

    vd_model = Qwen3TTSModel.from_pretrained(
        VOICEDESIGN_MODEL_PATH,
        device_map="mps",
        dtype=torch.bfloat16,
    )
    print("VoiceDesign model loaded")

    test_text = "When the sunlight strikes raindrops in the air, they act as a prism."
    print(f"Generating voice with instruct: {instruct[:80]}...")
    wavs, sr = vd_model.generate_voice_design(
        text=test_text,
        instruct=instruct,
        language="english",
        temperature=0.5,
        do_sample=True,
        seed=seed,
    )
    audio = wavs[0]
    if isinstance(audio, list):
        audio = audio[0]
    audio_np = np.array(audio)
    print(f"Generated {len(audio_np)/sr:.2f}s of audio")

    del vd_model
    torch.mps.empty_cache()

    print("Loading Base model for embedding extraction...")
    base_model = Qwen3TTSModel.from_pretrained(
        BASE_MODEL_PATH,
        device_map="mps",
        dtype=torch.bfloat16,
    )
    print("Base model loaded")

    emb = cache.extract_and_save(slug, audio_np, sr, base_model.model)
    print(f"Saved embedding for '{slug}': shape={emb.shape}, norm={np.linalg.norm(emb):.2f}")

    wav_path = embeddings_dir / f"{slug}_design_ref.wav"
    sf.write(str(wav_path), audio_np, sr)
    print(f"Saved reference audio to {wav_path}")


def main() -> None:
    parser = argparse.ArgumentParser(
        description="Extract Qwen TTS speaker embeddings from voice refs"
    )
    parser.add_argument(
        "--project",
        default="DealHunterShip_4.0e_whole-to-audio",
        help="Project name (default: DealHunterShip_4.0e_whole-to-audio)",
    )
    parser.add_argument("--speaker", help="Extract only this speaker (substring match)")
    parser.add_argument("--force", action="store_true", help="Re-extract even if cached")
    parser.add_argument("--design", help="VoiceDesign instruct text (create new voice)")
    parser.add_argument("--slug", help="Speaker slug for --design mode")
    parser.add_argument("--seed", type=int, default=42, help="Random seed for --design")
    args = parser.parse_args()

    if args.design:
        if not args.slug:
            parser.error("--slug is required with --design")
        create_embedding_from_design(args.project, args.design, args.slug, args.seed, args.force)
    else:
        extract_all_embeddings(args.project, args.speaker, args.force)


if __name__ == "__main__":
    main()