0c9bea72e0
Two groups of changes, committed together as the follow-up to the initial push: 1. .gitmodules — submodule Qwen3-TTS now points at the internal Gitea fork (ssh://git@mini-eq:2222/umbragit/Qwen3-TTS.git) instead of upstream GitHub, so a clone actually gets the UmbraAudio branding/integration commit 012e452 rather than vanilla upstream. 2. New infra/ tooling for the cross-machine work in spec 21: agent_bridge, remote_credential_broker (Keychain-backed, secrets read at runtime and generated via read_secret/random_secret — no hardcoded credentials), the enterprise bootstrap scripts, and the oldbox SSH/auth prompts. Plus backup/ restore scripts, two specs (18 UmbraTrans, 19 enterprise control plane), umbraaudio/voices/lmstudio_gate.py, and in-flight edits to the discovery/attribution/profiling modules, config, chapter_builder, emotion, pronunciation, qwen_tts and routes. Also adds *.db to .gitignore: SQLite project databases are runtime state and no .db file was tracked before. Verified before staging: .env's only change is UMBRA_TTS__QWEN_TTS_HELPER_URL and the committed copy has zero secret-looking values; the broker ships only config.example.json.
304 lines
10 KiB
Python
304 lines
10 KiB
Python
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
from dataclasses import dataclass
|
||
from typing import Any
|
||
|
||
from umbraaudio.analysis import (
|
||
TAXONOMY_ENUMS,
|
||
CharacterProfile,
|
||
)
|
||
from umbraaudio.config import LLMConfig
|
||
from umbraaudio.voices.lmstudio_gate import queued_chat_completion
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
CHARACTER_PROFILE_PROMPT = """You are an expert audiobook director. Analyze the following character
|
||
from a novel and create a detailed voice profile for text-to-speech generation.
|
||
|
||
Character name: {name}
|
||
Known aliases: {aliases}
|
||
Gender (inferred): {gender}
|
||
Context from the book:
|
||
{context}
|
||
|
||
Already-cast voices in this book (avoid sounding like any of these):
|
||
{existing_cast_summary}
|
||
|
||
Create a JSON profile with these fields:
|
||
- personality: Brief personality description (1-2 sentences)
|
||
- voice_description: Detailed voice description for TTS voice design
|
||
(include gender, age, accent, tone, pitch characteristics)
|
||
- emotion_tendencies: List of 3-5 emotions this character commonly expresses
|
||
- speech_patterns: How this character speaks (formal/casual, sentence structure, etc.)
|
||
- narrator_style: If this character were narrating, what style would they use?
|
||
- pitch_register: one of [very_low, low, mid, high, very_high]
|
||
- pace_baseline: one of [very_slow, slow, moderate, brisk, fast]
|
||
- energy: one of [subdued, soft_spoken, measured, assertive, commanding, volatile]
|
||
- timbre: one of [warm, breathy, nasal, gravelly, clear, resonant, thin]
|
||
- texture: one of [smooth, raspy, clipped, melodious, monotone]
|
||
- formality: one of [casual, conversational, formal, archaic]
|
||
- emotional_baseline: short tag, e.g. "weary_cynical"
|
||
- distinguishing_trait: one short phrase describing a distinctive speech habit
|
||
- accent_marker: only set if the text gives direct evidence; otherwise "none"
|
||
|
||
Return ONLY valid JSON, no other text."""
|
||
|
||
EDIT_PROMPT_TEMPLATE = """You are adjusting a TTS voice profile based on director feedback.
|
||
|
||
Current profile:
|
||
{taxonomy_json}
|
||
|
||
Director's note: "{prompt}"
|
||
|
||
Return ONLY a JSON object containing the fields that should change, from:
|
||
pitch_register, pace_baseline, energy, timbre, texture, formality,
|
||
emotional_baseline, distinguishing_trait, voice_description.
|
||
Only include a field if the note actually justifies changing it.
|
||
Do not invent unrelated changes."""
|
||
|
||
|
||
@dataclass
|
||
class DistinctivenessResult:
|
||
is_distinct: bool
|
||
colliding_with: list[str]
|
||
matched_dimensions: list[str]
|
||
|
||
|
||
class CharacterProfiler:
|
||
def __init__(self, llm_config: LLMConfig) -> None:
|
||
self.config = llm_config
|
||
self._client = None
|
||
|
||
def _get_client(self) -> Any:
|
||
if self._client is not None:
|
||
return self._client
|
||
|
||
if self.config.provider in ("lmstudio", "auto"):
|
||
try:
|
||
import httpx
|
||
|
||
resp = httpx.get(f"{self.config.lmstudio_base_url}/models", timeout=5)
|
||
if resp.status_code == 200:
|
||
from openai import OpenAI
|
||
|
||
self._client = OpenAI(
|
||
base_url=self.config.lmstudio_base_url,
|
||
api_key="lm-studio",
|
||
)
|
||
return self._client
|
||
except Exception:
|
||
pass
|
||
|
||
if self.config.provider in ("openai", "auto"):
|
||
from openai import OpenAI
|
||
|
||
self._client = OpenAI()
|
||
return self._client
|
||
|
||
raise RuntimeError("No LLM provider available")
|
||
|
||
def _build_cast_summary(self, existing: list[CharacterProfile]) -> str:
|
||
if not existing:
|
||
return "(no other characters profiled yet)"
|
||
lines = []
|
||
for p in existing:
|
||
lines.append(
|
||
f"- {p.name}: pitch={p.pitch_register}, energy={p.energy}, "
|
||
f"timbre={p.timbre}, texture={p.texture}"
|
||
)
|
||
return "\n".join(lines)
|
||
|
||
async def profile_character(
|
||
self,
|
||
name: str,
|
||
aliases: list[str],
|
||
gender: str | None,
|
||
context: str,
|
||
existing_cast: list[CharacterProfile] | None = None,
|
||
retry_hint: DistinctivenessResult | None = None,
|
||
) -> CharacterProfile:
|
||
client = self._get_client()
|
||
|
||
cast_summary = self._build_cast_summary(existing_cast or [])
|
||
|
||
prompt = CHARACTER_PROFILE_PROMPT.format(
|
||
name=name,
|
||
aliases=", ".join(aliases) if aliases else "None",
|
||
gender=gender or "Unknown",
|
||
context=context[:2000],
|
||
existing_cast_summary=cast_summary,
|
||
)
|
||
|
||
if retry_hint and retry_hint.colliding_with:
|
||
collision_names = ", ".join(retry_hint.colliding_with)
|
||
dims = ", ".join(retry_hint.matched_dimensions)
|
||
prompt += (
|
||
f"\n\nIMPORTANT: Your previous profile collided with "
|
||
f"{collision_names} on dimensions: {dims}. "
|
||
f"Revise so this character differs in at least two of those "
|
||
f"dimensions while staying consistent with the text evidence."
|
||
)
|
||
|
||
response = queued_chat_completion(
|
||
client,
|
||
model=self.config.openai_model,
|
||
messages=[{"role": "user", "content": prompt}],
|
||
temperature=self.config.temperature,
|
||
max_tokens=self.config.max_tokens,
|
||
)
|
||
|
||
content = response.choices[0].message.content or "{}"
|
||
|
||
try:
|
||
data = json.loads(content)
|
||
except json.JSONDecodeError:
|
||
json_start = content.find("{")
|
||
json_end = content.rfind("}") + 1
|
||
if json_start >= 0 and json_end > json_start:
|
||
data = json.loads(content[json_start:json_end])
|
||
else:
|
||
logger.warning("Failed to parse LLM response for character %s", name)
|
||
data = {}
|
||
|
||
return CharacterProfile.from_dict(
|
||
{
|
||
"name": name,
|
||
"aliases": aliases,
|
||
"gender": gender,
|
||
**data,
|
||
}
|
||
)
|
||
|
||
async def check_distinctiveness(
|
||
self,
|
||
profile: CharacterProfile,
|
||
existing: list[CharacterProfile],
|
||
co_occurrence: dict[str, set[str]] | None = None,
|
||
) -> DistinctivenessResult:
|
||
new_tuple = profile.taxonomy_tuple()
|
||
colliding = []
|
||
matched = []
|
||
|
||
for other in existing:
|
||
if other.name == profile.name:
|
||
continue
|
||
if other.is_narrator:
|
||
continue
|
||
|
||
if co_occurrence and profile.name in co_occurrence:
|
||
if other.name not in co_occurrence[profile.name]:
|
||
continue
|
||
|
||
other_tuple = other.taxonomy_tuple()
|
||
dims = [
|
||
"pitch_register",
|
||
"pace_baseline",
|
||
"energy",
|
||
"timbre",
|
||
"texture",
|
||
"formality",
|
||
]
|
||
match_count = sum(1 for a, b in zip(new_tuple, other_tuple) if a == b)
|
||
if match_count >= 4:
|
||
match_dims = [dims[i] for i in range(6) if new_tuple[i] == other_tuple[i]]
|
||
colliding.append(other.name)
|
||
matched.extend(match_dims)
|
||
|
||
return DistinctivenessResult(
|
||
is_distinct=len(colliding) == 0,
|
||
colliding_with=colliding,
|
||
matched_dimensions=list(set(matched)),
|
||
)
|
||
|
||
async def apply_edit_prompt(
|
||
self,
|
||
profile: CharacterProfile,
|
||
prompt: str,
|
||
) -> CharacterProfile:
|
||
client = self._get_client()
|
||
|
||
taxonomy = {
|
||
"pitch_register": profile.pitch_register,
|
||
"pace_baseline": profile.pace_baseline,
|
||
"energy": profile.energy,
|
||
"timbre": profile.timbre,
|
||
"texture": profile.texture,
|
||
"formality": profile.formality,
|
||
"emotional_baseline": profile.emotional_baseline,
|
||
"distinguishing_trait": profile.distinguishing_trait,
|
||
"voice_description": profile.voice_description,
|
||
}
|
||
|
||
llm_prompt = EDIT_PROMPT_TEMPLATE.format(
|
||
taxonomy_json=json.dumps(taxonomy, indent=2),
|
||
prompt=prompt,
|
||
)
|
||
|
||
response = queued_chat_completion(
|
||
client,
|
||
model=self.config.openai_model,
|
||
messages=[{"role": "user", "content": llm_prompt}],
|
||
temperature=self.config.temperature,
|
||
max_tokens=self.config.max_tokens,
|
||
)
|
||
|
||
content = response.choices[0].message.content or "{}"
|
||
try:
|
||
deltas = json.loads(content)
|
||
except json.JSONDecodeError:
|
||
json_start = content.find("{")
|
||
json_end = content.rfind("}") + 1
|
||
if json_start >= 0 and json_end > json_start:
|
||
deltas = json.loads(content[json_start:json_end])
|
||
else:
|
||
logger.warning("Failed to parse edit prompt response")
|
||
return profile
|
||
|
||
new_data = profile.to_dict()
|
||
for key, value in deltas.items():
|
||
if key in TAXONOMY_ENUMS:
|
||
if value in TAXONOMY_ENUMS[key]:
|
||
new_data[key] = value
|
||
else:
|
||
logger.warning(
|
||
"Invalid taxonomy value for %s: %s, keeping %s",
|
||
key,
|
||
value,
|
||
getattr(profile, key),
|
||
)
|
||
elif key in ("emotional_baseline", "distinguishing_trait", "voice_description"):
|
||
new_data[key] = value
|
||
|
||
return CharacterProfile.from_dict(new_data)
|
||
|
||
async def generate_pronunciation_guide(self, terms: list[str], context: str) -> dict[str, str]:
|
||
client = self._get_client()
|
||
|
||
prompt = f"""Given these unusual words/names from a novel, provide IPA pronunciations.
|
||
|
||
Words: {", ".join(terms)}
|
||
|
||
Context snippet:
|
||
{context[:1000]}
|
||
|
||
Return a JSON object mapping each word to its IPA pronunciation.
|
||
Example: {{"Eilean": "eɪ.lən", "Fjord": "fjɔːrd"}}
|
||
|
||
Return ONLY valid JSON."""
|
||
|
||
response = queued_chat_completion(
|
||
client,
|
||
model=self.config.openai_model,
|
||
messages=[{"role": "user", "content": prompt}],
|
||
temperature=0.1,
|
||
max_tokens=2048,
|
||
)
|
||
|
||
content = response.choices[0].message.content or "{}"
|
||
try:
|
||
return json.loads(content)
|
||
except json.JSONDecodeError:
|
||
return {}
|