224 lines
8.9 KiB
Python
224 lines
8.9 KiB
Python
# Copyright 2026 Umbra Tools (umbratools.com)
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
"""voice_lab — measure what the Author voice matrix can become.
|
|
|
|
One fixed text for every sample (comparisons are only valid at constant text).
|
|
Every sample records: F0 median/sd/range, spectral-envelope centroid and its
|
|
semitone offset (the "vocal tract" proxy), brightness, flatness, level, whisper
|
|
WER, and elapsed time.
|
|
|
|
Axes:
|
|
A pitch pitch_shift_pv, -12..+12 st -> where is the intelligibility cliff
|
|
B tract neural instruct, 8 specs -> how far will identity move
|
|
C hybrid instruct + DSP trim -> close the gap instruct leaves
|
|
D variance seeds x temperatures -> intra-voice spread (cast-size limit)
|
|
|
|
Outputs WAVs + results.json + results.md under OUT.
|
|
Nothing in the project tree is read except the reference; nothing is written to it.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import json
|
|
import re
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import librosa
|
|
import numpy as np
|
|
import soundfile as sf
|
|
|
|
sys.path.insert(0, "/Volumes/1TB_APL_Storage/UmbraAudio")
|
|
from umbraaudio.generation.audio_effects import EFFECT_FN # noqa: E402
|
|
|
|
import httpx # noqa: E402
|
|
|
|
PROJECT = Path("/Volumes/1TB_APL_Storage/UmbraAudio")
|
|
REF = PROJECT / "DealHunterShip_4.0e_whole-to-audio/.umbra/voices/author_ref.wav"
|
|
OUT = Path("/private/var/folders/y8/cbvhds297918lrxt2wd3btsr0000gn/T/opencode/author_lab")
|
|
URL = "http://127.0.0.1:8881"
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
(OUT / "wav").mkdir(exist_ok=True)
|
|
|
|
# ONE text for every sample.
|
|
TEXT = ("The lantern guttered as she pushed the door open, and the cold air of the "
|
|
"cavern rolled over her like the breath of something waiting.")
|
|
|
|
REF_F0 = None # set from the reference wav at startup
|
|
_pv = EFFECT_FN["pitch_shift_pv"]
|
|
_sat = EFFECT_FN["tube_sat"]
|
|
_ring = EFFECT_FN["ring_modulate"]
|
|
_chorus = EFFECT_FN["chorus"]
|
|
_jitter = EFFECT_FN["pitch_jitter"]
|
|
|
|
|
|
# ---------------------------------------------------------------- measurement
|
|
def measure(y: np.ndarray, sr: int) -> dict:
|
|
y = y.astype(np.float32)
|
|
f0, _, _ = librosa.pyin(y, fmin=60.0, fmax=400.0, sr=sr)
|
|
v = f0[~np.isnan(f0)]
|
|
S = np.abs(librosa.stft(y, n_fft=2048, hop_length=512))
|
|
f = librosa.fft_frequencies(sr=sr, n_fft=2048)
|
|
band = f <= 4000
|
|
ftr = float((S[band] * f[band][:, None]).sum() / S[band].sum())
|
|
m = {
|
|
"f0_median": round(float(np.median(v)), 1) if len(v) else None,
|
|
"f0_sd": round(float(v.std()), 1) if len(v) else None,
|
|
"envelope_hz": round(ftr),
|
|
"centroid_hz": round(float(librosa.feature.spectral_centroid(S=S, sr=sr).mean())),
|
|
"flatness": round(float(librosa.feature.spectral_flatness(S=S).mean()), 5),
|
|
"dur_s": round(len(y) / sr, 2),
|
|
"rms_db": round(float(20 * np.log10(np.sqrt((y**2).mean()) + 1e-12)), 1),
|
|
"peak_db": round(float(20 * np.log10(np.max(np.abs(y)) + 1e-12)), 1),
|
|
}
|
|
if m["f0_median"] and REF_F0:
|
|
m["d_f0_st"] = round(12 * np.log2(m["f0_median"] / REF_F0), 1)
|
|
return m
|
|
|
|
|
|
def wer(model, ref_words: list[str], y: np.ndarray) -> float | None:
|
|
import difflib
|
|
|
|
try:
|
|
txt = model.transcribe(y.astype(np.float32), language="en", fp16=False,
|
|
temperature=0.0)["text"]
|
|
except Exception:
|
|
return None
|
|
hyp = re.sub(r"[^a-z ]", "", txt.lower()).split()
|
|
n = sum(b.size for b in difflib.SequenceMatcher(None, ref_words, hyp).get_matching_blocks())
|
|
return round((len(ref_words) - n) / max(1, len(ref_words)), 3)
|
|
|
|
|
|
# ---------------------------------------------------------------- generation
|
|
def synth(label: str, *, instruct: str | None = None, temperature: float = 0.7,
|
|
seed: int = 1234, dsp: list | None = None, want_wer: bool = True,
|
|
whisper_model=None) -> dict:
|
|
data = {"text": TEXT, "sample_rate": 24000, "temperature": temperature,
|
|
"do_sample": True, "seed": seed}
|
|
if instruct:
|
|
data["voice_description"] = instruct
|
|
t0 = time.time()
|
|
r = httpx.post(f"{URL}/v1/tts", data=data,
|
|
files={"ref_audio": ("ref.wav", REF.read_bytes(), "audio/wav")},
|
|
timeout=900)
|
|
if r.status_code != 200:
|
|
return {"label": label, "error": f"HTTP {r.status_code}: {r.text[:160]}"}
|
|
y, sr = sf.read(io.BytesIO(r.content), dtype="float32")
|
|
if y.ndim > 1:
|
|
y = y.mean(axis=1)
|
|
gen_s = round(time.time() - t0, 1)
|
|
|
|
chain_desc = None
|
|
if dsp:
|
|
for name, params in dsp:
|
|
y = EFFECT_FN[name](y, sr, **params)
|
|
p = float(np.max(np.abs(y)))
|
|
if p > 1.0:
|
|
y = y / p
|
|
chain_desc = [n for n, _ in dsp]
|
|
|
|
m = measure(y, sr)
|
|
m.update(label=label, gen_s=gen_s, instruct=instruct, temperature=temperature,
|
|
seed=seed, dsp=chain_desc,
|
|
d_env_st=(round(12 * np.log2(m["envelope_hz"] / REF_ENV), 1)
|
|
if REF_ENV else None))
|
|
if want_wer and whisper_model is not None:
|
|
m["wer"] = wer(whisper_model, REF_WORDS, y)
|
|
sf.write(OUT / "wav" / f"{label}.wav", y, sr, subtype="PCM_16")
|
|
return m
|
|
|
|
|
|
# ---------------------------------------------------------------- axes
|
|
TRACTS = {
|
|
"B_female_young": "A young woman's voice, bright and light, with a higher and clearer tone.",
|
|
"B_female_mature": "A mature woman's voice, low and calm for a woman, unhurried and steady.",
|
|
"B_boy": "A young boy's voice, light and eager, not yet broken.",
|
|
"B_elderly_m": "An elderly man's voice, very deep, gravelly and slow.",
|
|
"B_young_m": "A young man's voice, light, energetic and clear.",
|
|
"B_large_m": "A very large man's voice, heavy and booming, with a deep chest.",
|
|
"B_small_w": "A small woman's voice, petite and light, with a narrow bright tone.",
|
|
"B_androgynous": "A neutral, androgynous voice, neither distinctly male nor female.",
|
|
}
|
|
|
|
GRIT = {
|
|
"grit_sat": [("tube_sat", {"drive_db": 12.0, "mix": 1.0})],
|
|
"grit_ring": [("ring_modulate", {"freq": 38.0})],
|
|
"grit_chorus": [("chorus", {"rate_hz": 1.6, "depth": 0.3, "centre_delay_ms": 8.0,
|
|
"feedback": 0.2, "mix": 0.4})],
|
|
"grit_full": [("tube_sat", {"drive_db": 12.0, "mix": 1.0}),
|
|
("ring_modulate", {"freq": 38.0}),
|
|
("chorus", {"rate_hz": 1.6, "depth": 0.3, "centre_delay_ms": 8.0,
|
|
"feedback": 0.2, "mix": 0.4}),
|
|
("pitch_jitter", {"jitter_hz": 6.0, "depth": 0.03})],
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
global REF_F0, REF_ENV, REF_WORDS
|
|
import whisper
|
|
|
|
ref_y, ref_sr = sf.read(str(REF), dtype="float32")
|
|
if ref_y.ndim > 1:
|
|
ref_y = ref_y.mean(axis=1)
|
|
S = np.abs(librosa.stft(ref_y.astype(np.float32), n_fft=2048, hop_length=512))
|
|
f = librosa.fft_frequencies(sr=ref_sr, n_fft=2048)
|
|
band = f <= 4000
|
|
REF_ENV = float((S[band] * f[band][:, None]).sum() / S[band].sum())
|
|
f0r, _, _ = librosa.pyin(ref_y.astype(np.float32), fmin=60.0, fmax=400.0, sr=ref_sr)
|
|
REF_F0 = float(np.median(f0r[~np.isnan(f0r)]))
|
|
REF_WORDS = re.sub(r"[^a-z ]", "", TEXT.lower()).split()
|
|
print(f"REFERENCE (author_ref.wav): F0 {REF_F0:.1f} Hz envelope {REF_ENV:.0f} Hz "
|
|
f"{len(ref_y)/ref_sr:.1f}s")
|
|
|
|
wm = whisper.load_model("base", device="cpu")
|
|
rows: list[dict] = []
|
|
|
|
def run(label, **kw):
|
|
m = synth(label, whisper_model=wm, **kw)
|
|
rows.append(m)
|
|
print(f" {label:26} F0={m.get('f0_median')} "
|
|
f"({m.get('d_f0_st')}st) env={m.get('d_env_st')}st "
|
|
f"WER={m.get('wer')} {m.get('gen_s')}s", flush=True)
|
|
return m
|
|
|
|
print("\n[baseline]")
|
|
run("A00_baseline", temperature=0.7, seed=1234)
|
|
|
|
print("\n[A] pitch axis -12..+12 step 2")
|
|
for st in range(-12, 13, 2):
|
|
run(f"A_pitch{st:+03d}", dsp=[("pitch_shift_pv", {"semitones": float(st)})])
|
|
|
|
print("\n[B] neural tract morphs")
|
|
for label, instr in TRACTS.items():
|
|
run(label, instruct=instr)
|
|
|
|
print("\n[C] hybrids")
|
|
for trim in (4, 8):
|
|
run(f"C_female+{trim}st", instruct=TRACTS["B_female_young"],
|
|
dsp=[("pitch_shift_pv", {"semitones": float(trim)})])
|
|
run("C_monster_pitch-10", dsp=[("pitch_shift_pv", {"semitones": -10.0})])
|
|
for gname, gchain in GRIT.items():
|
|
run(f"C_monster_{gname}",
|
|
instruct="A huge ancient dragon: deep, rumbling, growling, with a metallic "
|
|
"edge and a resonant echoing chest tone.",
|
|
dsp=gchain)
|
|
|
|
print("\n[D] intra-voice variance (seeds x temperature)")
|
|
for temp in (0.3, 0.7, 0.9):
|
|
for seed in (11, 22, 33, 44, 55):
|
|
run(f"D_seed{seed}_t{temp}", temperature=temp, seed=seed, want_wer=False)
|
|
|
|
(OUT / "results.json").write_text(json.dumps(
|
|
{"reference": {"f0_median": round(REF_F0, 1), "envelope_hz": round(REF_ENV),
|
|
"path": str(REF)}, "rows": rows}, indent=2) + "\n")
|
|
print(f"\nwrote {OUT/'results.json'} ({len(rows)} samples)")
|
|
|
|
|
|
REF_ENV = None
|
|
REF_WORDS: list[str] = []
|
|
|
|
if __name__ == "__main__":
|
|
main()
|