Add REST endpoints and frontend wiring for per-language project actions:
- POST /api/workspace/{project}/translation/{lang}/queue — queue translation
- POST /api/workspace/{project}/audio/{lang}/queue — queue audio generation
- GET/POST /api/workspace/{project}/format/{lang} — format metadata (EPUB/PDF)
- GET /api/workspace/{project}/review/{lang}/chapters/{chapter} — utterance review
- GET/POST /api/workspace/{project}/front-matter/{lang} — front matter editor
- GET /api/workspace/{project}/chapters/{lang} — chapter list
- GET /api/workspace/{project}/chapter-content/{lang}/{chapter} — chapter text
Frontend fixes:
- Review page shows per-language text (English source, Chinese translation, etc.)
- Non-translated languages show 'Queue translation' button only
- Speaker data from English source matched by paragraph ID
- UmbraAudio column layout: Speaker, Variation, Text, Actions
- Chapter tree in left sidebar: Front matter, Back matter, Body
- Audio Format label at top of sidebar
- Pandoc formatting toolbar for front matter text editing
705 lines
28 KiB
Python
705 lines
28 KiB
Python
# Copyright 2026 Umbra Tools (umbratools.com)
|
||
# SPDX-License-Identifier: Apache-2.0
|
||
|
||
"""Multi-pass translation orchestration.
|
||
|
||
Pass 0 is local: paragraphs are grouped and chunked with source indices kept.
|
||
Pass 1 analyses intent, Pass 2 drafts, Pass 4 checks meaning with a different
|
||
model. Pass 3 (chapter voice consistency) is optional and runs after the
|
||
chapter's chunks are drafted.
|
||
|
||
Every artefact is written to disk and reused on the next run, so a multi-hour
|
||
book run can be stopped and resumed.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import re
|
||
import sys
|
||
import time
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||
|
||
from llm import DEFAULT_ANALYST, DEFAULT_CRITIC, DEFAULT_MODEL, LLM, LLMError, Usage # noqa: E402
|
||
from prompts import ( # noqa: E402
|
||
SYSTEM_ANALYST,
|
||
SYSTEM_CRITIC,
|
||
SYSTEM_DRAFTER,
|
||
analysis_prompt,
|
||
consistency_prompt,
|
||
critic_prompt,
|
||
draft_prompt,
|
||
load_glossary,
|
||
)
|
||
|
||
CHARACTER_NOTES = """- 凯因达:聪明、直率、好奇,带一点叛逆;对权威不耐烦,对朋友温暖。不要把她写成娇滴滴的公主。
|
||
- Hunter / Deal Hunter:干练、简洁、有冷幽默;是船,也是生意人;不用敬语堆砌。
|
||
- 南妮:温和、认真、字面化,偶尔因为太认真而好笑;不要写成卖萌。
|
||
- 维多利亚公爵夫人:端庄、克制、关切;对凯因达既有长辈的威严,也有家人的柔软。
|
||
- 罗曼诺:热情、商人腔、爱谈价格,但不油滑。
|
||
- 旁白:克制、清晰、有画面感;不替人物抒情。"""
|
||
|
||
# Proper nouns that may legitimately stay in Latin script. Anything else that
|
||
# survives in the Chinese text is reported, not silently accepted.
|
||
LATIN_ALLOW = {
|
||
"ai", "audio", "authorbdmurphy", "b", "blazes", "brutus", "com", "d",
|
||
"deal", "dial", "eva", "felucca", "gisfrid", "goodreads", "hunter",
|
||
"http", "https", "isbn", "kabir", "kainda", "konrad", "llc", "l2",
|
||
"maul", "murphy", "nanny", "niñera", "romano", "sao", "umbra", "vince",
|
||
"wakam", "wake", "weston", "willamina", "witold", "worstauthors",
|
||
"worstauthor", "www", "zheng", "yi", "adonai", "adric", "albert",
|
||
}
|
||
|
||
|
||
def _normalise_quotes(text: str) -> str:
|
||
text = re.sub(r'"([^"\n]*)"', r"“\1”", text)
|
||
if '"' in text:
|
||
parts = text.split('"')
|
||
out = parts[0]
|
||
for i, part in enumerate(parts[1:]):
|
||
out += ("“" if i % 2 == 0 else "”") + part
|
||
text = out
|
||
if "“" not in text and text.endswith("”"):
|
||
text = text[:-1]
|
||
opens, closes = text.count("“"), text.count("”")
|
||
if opens > closes:
|
||
text += "”" * (opens - closes)
|
||
elif closes > opens:
|
||
for _ in range(closes - opens):
|
||
idx = text.rfind("”")
|
||
if idx > text.rfind("“"):
|
||
text = text[:idx] + text[idx + 1 :]
|
||
else:
|
||
break
|
||
return text
|
||
|
||
|
||
def sanitize(text: str) -> str:
|
||
text = text.strip()
|
||
text = re.sub(r"^\[[^\]]{1,40}\]\s*", "", text)
|
||
text = re.sub(r"^((?:对话|旁白|叙述)[^)]{0,60})\s*", "", text)
|
||
text = re.sub(r"^\((?:dialogue|narration)[^)]{0,60}\)\s*", "", text, flags=re.I)
|
||
text = text.replace("《交易猎人》", "《Deal Hunter》")
|
||
text = text.replace("交易猎人", "Deal Hunter")
|
||
text = text.replace("“猎人”", "Hunter")
|
||
text = text.replace("猎人号", "Deal Hunter")
|
||
text = text.replace("亨特", "Hunter")
|
||
text = text.replace("B.D. 墨菲", "B·D·墨菲")
|
||
text = text.replace("B. D. 墨菲", "B·D·墨菲")
|
||
text = text.replace("B.D.墨菲", "B·D·墨菲")
|
||
text = text.replace("郑义少", "郑一嫂")
|
||
text = text.replace("乌姆布拉音频系统", "Umbra Audio")
|
||
text = text.replace("乌姆布拉音频", "Umbra Audio")
|
||
text = text.replace("Umbra Audio 星系系统", "Umbra Audio 系统")
|
||
text = text.replace("Umbra Audio星系系统", "Umbra Audio 系统")
|
||
text = text.replace("音频星系系统", "音频系统")
|
||
text = text.replace("在小时前", "一小时前")
|
||
text = text.replace("...", "……").replace("--", "——")
|
||
return _normalise_quotes(text).strip()
|
||
|
||
|
||
def latin_residue(text: str) -> tuple[list[str], list[str]]:
|
||
words = re.findall(r"[A-Za-zÀ-ÿ][A-Za-zÀ-ÿ'’\-]{1,}", text)
|
||
common: set[str] = set()
|
||
proper: set[str] = set()
|
||
for word in words:
|
||
base = word.strip("'’-")
|
||
if not base or base.lower() in LATIN_ALLOW:
|
||
continue
|
||
if base.isupper() and len(base) <= 5:
|
||
continue
|
||
if word[0].isupper():
|
||
proper.add(word)
|
||
else:
|
||
common.add(word)
|
||
return sorted(common), sorted(proper)
|
||
|
||
|
||
@dataclass
|
||
class Chunk:
|
||
id: str
|
||
scene_index: int
|
||
chunks_in_scene: int
|
||
paragraphs: list[dict]
|
||
|
||
@property
|
||
def words(self) -> int:
|
||
return sum(len(p.get("text", "").split()) for p in self.paragraphs)
|
||
|
||
|
||
def load_book(path: Path) -> dict:
|
||
return json.loads(path.read_text(encoding="utf-8"))
|
||
|
||
|
||
def select_chapters(book: dict, selector: str) -> list[dict]:
|
||
chapters = book["chapters"]
|
||
if selector in {"all", "*"}:
|
||
return chapters
|
||
if selector == "front":
|
||
return [c for c in chapters if c["matter"].startswith("front")]
|
||
if selector == "body":
|
||
return [c for c in chapters if c["matter"] == "body"]
|
||
if selector == "back":
|
||
return [c for c in chapters if c["matter"] == "back"]
|
||
wanted = {s.strip() for s in selector.split(",") if s.strip()}
|
||
out = []
|
||
for c in chapters:
|
||
if str(c["index"]) in wanted or c["id"] in wanted or c["title"] in wanted:
|
||
out.append(c)
|
||
missing = wanted - {str(c["index"]) for c in out} - {c["id"] for c in out} - {c["title"] for c in out}
|
||
if missing:
|
||
raise SystemExit(f"chapter selector did not match: {sorted(missing)}")
|
||
return out
|
||
|
||
|
||
def chunk_chapter(chapter: dict, max_words: int = 260, max_paras: int = 14) -> list[Chunk]:
|
||
paras = chapter["paragraphs"]
|
||
# Assign scene numbers.
|
||
scene_of: dict[str, int] = {}
|
||
scene = 0
|
||
for p in paras:
|
||
if p.get("type") == "scene_break":
|
||
scene += 1
|
||
continue
|
||
scene_of[p["id"]] = scene
|
||
|
||
raw: list[list[dict]] = []
|
||
current: list[dict] = []
|
||
current_words = 0
|
||
current_scene = None
|
||
for p in paras:
|
||
if p.get("type") == "scene_break":
|
||
if current:
|
||
raw.append(current)
|
||
current, current_words, current_scene = [], 0, None
|
||
continue
|
||
p_scene = scene_of.get(p["id"], 0)
|
||
if current and (p_scene != current_scene or current_words + len(p["text"].split()) > max_words or len(current) >= max_paras):
|
||
raw.append(current)
|
||
current, current_words = [], 0
|
||
current.append(p)
|
||
current_words += len(p["text"].split())
|
||
current_scene = p_scene
|
||
if current:
|
||
raw.append(current)
|
||
|
||
# Count chunks per scene.
|
||
per_scene: dict[int, int] = {}
|
||
for group in raw:
|
||
s = scene_of.get(group[0]["id"], 0)
|
||
per_scene[s] = per_scene.get(s, 0) + 1
|
||
|
||
chunks: list[Chunk] = []
|
||
seen: dict[int, int] = {}
|
||
for i, group in enumerate(raw):
|
||
s = scene_of.get(group[0]["id"], 0)
|
||
seen[s] = seen.get(s, 0) + 1
|
||
chunks.append(
|
||
Chunk(
|
||
id=f"{chapter['id']}-c{i:03d}",
|
||
scene_index=s,
|
||
chunks_in_scene=per_scene[s],
|
||
paragraphs=group,
|
||
)
|
||
)
|
||
return chunks
|
||
|
||
|
||
class Runner:
|
||
def __init__(
|
||
self,
|
||
*,
|
||
book_path: Path,
|
||
glossary_path: Path,
|
||
run_dir: Path,
|
||
model: str = DEFAULT_MODEL,
|
||
analyst_model: str = DEFAULT_ANALYST,
|
||
critic_model: str = DEFAULT_CRITIC,
|
||
base_url: str | None = None,
|
||
force: bool = False,
|
||
critic: bool = True,
|
||
consistency: bool = False,
|
||
max_words: int = 260,
|
||
max_paras: int = 14,
|
||
critic_limit: int = 0,
|
||
) -> None:
|
||
self.book = load_book(book_path)
|
||
self.glossary = load_glossary(glossary_path)
|
||
self.run_dir = run_dir
|
||
self.run_dir.mkdir(parents=True, exist_ok=True)
|
||
self.model = model
|
||
self.analyst_model = analyst_model
|
||
self.critic_model = critic_model
|
||
self.force = force
|
||
self.do_critic = critic
|
||
self.do_consistency = consistency
|
||
self.max_words = max_words
|
||
self.max_paras = max_paras
|
||
self.critic_limit = critic_limit
|
||
kwargs = {}
|
||
if base_url:
|
||
kwargs["base_url"] = base_url
|
||
self.llm = LLM(
|
||
cache_dir=run_dir / "cache",
|
||
usage_log=run_dir / "usage.jsonl",
|
||
**kwargs,
|
||
)
|
||
|
||
# ----------------------------------------------------------------- files
|
||
def chapter_dir(self, chapter: dict) -> Path:
|
||
d = self.run_dir / chapter["id"]
|
||
d.mkdir(parents=True, exist_ok=True)
|
||
return d
|
||
|
||
def _read(self, path: Path) -> dict | None:
|
||
if path.exists() and not self.force:
|
||
return json.loads(path.read_text(encoding="utf-8"))
|
||
return None
|
||
|
||
def _write(self, path: Path, value: object) -> None:
|
||
path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
|
||
# ----------------------------------------------------------------- passes
|
||
def analyse(self, chapter: dict, chunk: Chunk, previous_summary: str) -> dict:
|
||
path = self.chapter_dir(chapter) / f"{chunk.id}.analysis.json"
|
||
cached = self._read(path)
|
||
if cached is not None:
|
||
return cached
|
||
prompt = analysis_prompt(
|
||
chapter_title=f"{chapter['title']} / {chapter['title_zh']}",
|
||
scene_index=chunk.scene_index,
|
||
chunk_index=int(chunk.id.rsplit("c", 1)[-1]),
|
||
chunks_in_scene=chunk.chunks_in_scene,
|
||
paragraphs=chunk.paragraphs,
|
||
previous_summary=previous_summary,
|
||
glossary=self.glossary,
|
||
)
|
||
value, usage = self.llm.chat_json(
|
||
SYSTEM_ANALYST,
|
||
prompt,
|
||
model=self.analyst_model,
|
||
temperature=0.2,
|
||
max_tokens=1400,
|
||
cache_key=f"{chunk.id}.analysis.{self.analyst_model}",
|
||
# The small analyst model rejects LM Studio's JSON-schema grammar;
|
||
# its answer is parsed from the text instead.
|
||
json_mode=False,
|
||
)
|
||
if isinstance(value, list) and value and isinstance(value[0], dict):
|
||
value = value[0]
|
||
# Sometimes wrapped as {"analysis": {...}} or similar
|
||
if isinstance(value, dict):
|
||
for k in ("analysis", "result", "data"):
|
||
if isinstance(value.get(k), dict):
|
||
value = value[k]
|
||
break
|
||
if not isinstance(value, dict):
|
||
# The analyst is a small model asked for a JSON object, and it
|
||
# sometimes answers with a list or a bare string instead. Analysis is
|
||
# guidance for the drafter, not a requirement for a valid translation,
|
||
# so a malformed answer must not abort a chunk that could otherwise be
|
||
# translated. Record that the analysis is degraded and continue with a
|
||
# minimal one; the drafter falls back to the source and glossary.
|
||
print(
|
||
f" {chunk.id}: analyst returned {type(value).__name__}, "
|
||
"using a minimal analysis",
|
||
flush=True,
|
||
)
|
||
value = {"degraded": True, "note": "analyst returned non-object JSON"}
|
||
value["_usage"] = usage.__dict__
|
||
self._write(path, value)
|
||
return value
|
||
|
||
def draft(self, chapter: dict, chunk: Chunk, analysis: dict, previous_summary: str) -> dict:
|
||
path = self.chapter_dir(chapter) / f"{chunk.id}.draft.json"
|
||
cached = self._read(path)
|
||
if cached is not None:
|
||
return cached
|
||
expected = [p["id"] for p in chunk.paragraphs if p.get("type") != "scene_break"]
|
||
prompt = draft_prompt(
|
||
chapter_title=f"{chapter['title']} / {chapter['title_zh']}",
|
||
scene_index=chunk.scene_index,
|
||
chunk_index=int(chunk.id.rsplit("c", 1)[-1]),
|
||
chunks_in_scene=chunk.chunks_in_scene,
|
||
paragraphs=chunk.paragraphs,
|
||
analysis={k: v for k, v in analysis.items() if k != "_usage"},
|
||
previous_summary=previous_summary,
|
||
glossary=self.glossary,
|
||
)
|
||
value, usage = self.llm.chat_json(
|
||
SYSTEM_DRAFTER,
|
||
prompt,
|
||
model=self.model,
|
||
temperature=0.45,
|
||
max_tokens=6000,
|
||
cache_key=f"{chunk.id}.draft",
|
||
)
|
||
out = self._normalise_draft(value, expected)
|
||
missing = [i for i in expected if not out.get(i)]
|
||
if missing:
|
||
out.update(self._draft_missing(chapter, chunk, analysis, missing))
|
||
out = self._repair_residue(chapter, chunk, out)
|
||
out = self._fix_paragraph_quotes(chunk, out)
|
||
warnings: dict[str, dict[str, list[str]]] = {}
|
||
for pid, zh in out.items():
|
||
common, proper = latin_residue(zh)
|
||
if common or proper:
|
||
warnings[pid] = {"common_english": common, "proper_names": proper}
|
||
result = {
|
||
"paragraphs": out,
|
||
"analysis": analysis,
|
||
"warnings": warnings,
|
||
"_usage": usage.__dict__,
|
||
}
|
||
self._write(path, result)
|
||
return result
|
||
|
||
def _normalise_draft(self, value: object, expected: list[str]) -> dict[str, str]:
|
||
out: dict[str, str] = {}
|
||
if isinstance(value, dict) and isinstance(value.get("paragraphs"), list):
|
||
items = value["paragraphs"]
|
||
elif isinstance(value, list):
|
||
items = value
|
||
elif isinstance(value, dict) and value.get("id") and value.get("zh"):
|
||
items = [value]
|
||
elif isinstance(value, dict):
|
||
# maybe mapping id->zh
|
||
for k, v in value.items():
|
||
if isinstance(v, str) and k.startswith("ch-"):
|
||
out[k] = sanitize(v)
|
||
return {k: v for k, v in out.items() if k in expected and v}
|
||
else:
|
||
raise LLMError(f"drafter returned unexpected JSON: {str(value)[:200]}")
|
||
for item in items:
|
||
if isinstance(item, dict) and item.get("id") and item.get("zh"):
|
||
out[str(item["id"])] = sanitize(str(item["zh"]))
|
||
return {k: v for k, v in out.items() if k in expected and v}
|
||
|
||
def _fix_paragraph_quotes(self, chunk: Chunk, drafts: dict[str, str]) -> dict[str, str]:
|
||
"""Remove quotes the model wrapped around unquoted narration."""
|
||
for p in chunk.paragraphs:
|
||
zh = drafts.get(p["id"])
|
||
source = (p.get("text") or "").strip()
|
||
if (
|
||
zh
|
||
and p.get("type") == "narration"
|
||
and not source.startswith("“")
|
||
and zh.startswith("“")
|
||
and zh.endswith("”")
|
||
and zh.count("“") == 1
|
||
and zh.count("”") == 1
|
||
):
|
||
drafts[p["id"]] = zh[1:-1].strip()
|
||
return drafts
|
||
|
||
def _repair_residue(self, chapter: dict, chunk: Chunk, drafts: dict[str, str]) -> dict[str, str]:
|
||
"""Translate stray lowercase English words in place, without touching names."""
|
||
by_id = {p["id"]: p for p in chunk.paragraphs}
|
||
for pid, zh in list(drafts.items()):
|
||
common, _proper = latin_residue(zh)
|
||
if not common:
|
||
continue
|
||
source = by_id.get(pid, {}).get("text", "")
|
||
prompt = f"""下面这段中文译文里混入了英文单词:{", ".join(common)}。
|
||
请只把这些英文单词译成中文,其余内容尽量保持不变;不要解释,不要改写剧情。
|
||
输出 JSON:{{"zh": "修正后的中文译文"}}
|
||
|
||
英文原文:
|
||
{source}
|
||
|
||
当前译文:
|
||
{zh}"""
|
||
try:
|
||
value, _ = self.llm.chat_json(
|
||
SYSTEM_DRAFTER,
|
||
prompt,
|
||
model=self.model,
|
||
temperature=0.2,
|
||
max_tokens=1500,
|
||
cache_key=f"{chunk.id}.{pid}.residue",
|
||
)
|
||
fixed = value.get("zh") if isinstance(value, dict) else None
|
||
if fixed:
|
||
drafts[pid] = sanitize(str(fixed))
|
||
except LLMError as exc:
|
||
print(f" {chunk.id}/{pid}: residue repair failed: {exc}", flush=True)
|
||
return drafts
|
||
|
||
def _draft_missing(self, chapter: dict, chunk: Chunk, analysis: dict, missing: list[str]) -> dict[str, str]:
|
||
"""Last-resort one-paragraph calls. Keeps a chunk failure from losing the run."""
|
||
result: dict[str, str] = {}
|
||
by_id = {p["id"]: p for p in chunk.paragraphs}
|
||
for pid in missing:
|
||
p = by_id[pid]
|
||
try:
|
||
prompt = draft_prompt(
|
||
chapter_title=f"{chapter['title']} / {chapter['title_zh']}",
|
||
scene_index=chunk.scene_index,
|
||
chunk_index=0,
|
||
chunks_in_scene=1,
|
||
paragraphs=[p],
|
||
analysis=analysis,
|
||
previous_summary="",
|
||
glossary=self.glossary,
|
||
)
|
||
value, _ = self.llm.chat_json(
|
||
SYSTEM_DRAFTER,
|
||
prompt,
|
||
model=self.model,
|
||
temperature=0.4,
|
||
max_tokens=1500,
|
||
cache_key=f"{chunk.id}.{pid}.draft",
|
||
)
|
||
one = self._normalise_draft(value, [pid])
|
||
if one.get(pid):
|
||
result[pid] = one[pid]
|
||
continue
|
||
except Exception:
|
||
pass
|
||
# Fallback to source text or empty to avoid blocking
|
||
result[pid] = sanitize(str(p.get("text", "")))
|
||
return result
|
||
|
||
def criticise(self, chapter: dict, chunk: Chunk, draft: dict[str, str]) -> dict:
|
||
path = self.chapter_dir(chapter) / f"{chunk.id}.critic.json"
|
||
cached = self._read(path)
|
||
if cached is not None:
|
||
return cached
|
||
prompt = critic_prompt(
|
||
chapter_title=f"{chapter['title']} / {chapter['title_zh']}",
|
||
paragraphs=chunk.paragraphs,
|
||
draft=draft,
|
||
glossary=self.glossary,
|
||
)
|
||
try:
|
||
value, usage = self.llm.chat_json(
|
||
SYSTEM_CRITIC,
|
||
prompt,
|
||
model=self.critic_model,
|
||
temperature=0.15,
|
||
max_tokens=6000,
|
||
cache_key=f"{chunk.id}.critic.{self.critic_model}",
|
||
)
|
||
except LLMError as exc:
|
||
# A reasoning-only critic produces empty content. Fall back to the
|
||
# drafter model rather than silently skipping the check.
|
||
print(f" critic {self.critic_model} failed ({exc}); falling back to {self.model}", flush=True)
|
||
try:
|
||
value, usage = self.llm.chat_json(
|
||
SYSTEM_CRITIC,
|
||
prompt,
|
||
model=self.model,
|
||
temperature=0.15,
|
||
max_tokens=6000,
|
||
cache_key=f"{chunk.id}.critic.{self.model}",
|
||
)
|
||
except LLMError as exc2:
|
||
# Do not abort a multi-hour run because one critique failed.
|
||
print(f" critic fallback failed ({exc2}); recording a skipped critique", flush=True)
|
||
value = {"issues": [], "verdict": "skipped", "_error": str(exc2)}
|
||
usage = Usage(model=self.model, prompt_tokens=0, completion_tokens=0, elapsed_s=0.0)
|
||
if isinstance(value, list):
|
||
value = {"issues": value, "verdict": "revise" if value else "pass"}
|
||
if not isinstance(value, dict):
|
||
raise LLMError(f"{chunk.id}: critic returned {type(value).__name__}")
|
||
# DeepSeek sometimes invents its own key names. Normalise the common
|
||
# aliases so the report and the review queue stay machine-readable.
|
||
normalised = []
|
||
confirmations = (
|
||
"被正确翻译",
|
||
"翻译正确",
|
||
"已正确",
|
||
"无需修改",
|
||
"没有问题",
|
||
"不存在问题",
|
||
"未改变理解",
|
||
"英文原文存在",
|
||
"原文存在语法错误",
|
||
"建议英文",
|
||
"英文原文表达",
|
||
"原文错误",
|
||
"无需修改中文",
|
||
)
|
||
for raw in value.get("issues", []) or []:
|
||
if not isinstance(raw, dict):
|
||
continue
|
||
problem_text = str(raw.get("problem") or raw.get("explanation") or "")
|
||
if any(marker in problem_text for marker in confirmations):
|
||
continue
|
||
fix_text = str(raw.get("fix") or raw.get("suggestion") or "").strip()
|
||
if not fix_text and str(raw.get("severity") or "major").lower() == "minor":
|
||
continue
|
||
normalised.append(
|
||
{
|
||
"id": raw.get("id") or raw.get("paragraph_id") or raw.get("sentence_id") or "",
|
||
"severity": raw.get("severity") or "major",
|
||
"type": raw.get("type") or "meaning",
|
||
"problem": raw.get("problem") or raw.get("explanation") or "",
|
||
"source_excerpt": raw.get("source_excerpt") or raw.get("english_sentence_part") or "",
|
||
"target_excerpt": raw.get("target_excerpt") or raw.get("chinese_sentence_part") or "",
|
||
"fix": raw.get("fix") or raw.get("suggestion") or "",
|
||
}
|
||
)
|
||
value["issues"] = normalised
|
||
verdict = str(value.get("verdict", "")).lower()
|
||
if verdict == "skipped":
|
||
value["verdict"] = "skipped"
|
||
elif normalised or verdict in {"fail", "revise"}:
|
||
value["verdict"] = "revise"
|
||
else:
|
||
value["verdict"] = "pass"
|
||
value["_usage"] = usage.__dict__
|
||
self._write(path, value)
|
||
return value
|
||
|
||
def consistency(self, chapter: dict, drafts: dict[str, str], paragraphs: list[dict]) -> dict:
|
||
path = self.chapter_dir(chapter) / "consistency.json"
|
||
cached = self._read(path)
|
||
if cached is not None:
|
||
return cached
|
||
prompt = consistency_prompt(
|
||
chapter_title=f"{chapter['title']} / {chapter['title_zh']}",
|
||
paragraphs=paragraphs,
|
||
draft=drafts,
|
||
glossary=self.glossary,
|
||
character_notes=CHARACTER_NOTES,
|
||
)
|
||
value, usage = self.llm.chat_json(
|
||
SYSTEM_CRITIC,
|
||
prompt,
|
||
model=self.critic_model,
|
||
temperature=0.15,
|
||
max_tokens=6000,
|
||
cache_key=f"{chapter['id']}.consistency",
|
||
)
|
||
if isinstance(value, list):
|
||
value = {"issues": value}
|
||
if not isinstance(value, dict):
|
||
raise LLMError(f"{chapter['id']}: consistency returned {type(value).__name__}")
|
||
value["_usage"] = usage.__dict__
|
||
self._write(path, value)
|
||
return value
|
||
|
||
# ----------------------------------------------------------------- driver
|
||
def run(self, chapters: list[dict]) -> None:
|
||
started = time.time()
|
||
for ci, chapter in enumerate(chapters, 1):
|
||
chunks = chunk_chapter(chapter, self.max_words, self.max_paras)
|
||
print(
|
||
f"[{ci}/{len(chapters)}] {chapter['id']} {chapter['title']} / {chapter['title_zh']}: "
|
||
f"{len(chapter['paragraphs'])} paragraphs, {len(chunks)} chunks",
|
||
flush=True,
|
||
)
|
||
previous_summary = ""
|
||
drafts: dict[str, str] = {}
|
||
issues: list[dict] = []
|
||
for chunk in chunks:
|
||
analysis = self.analyse(chapter, chunk, previous_summary)
|
||
previous_summary = str(analysis.get("summary_zh") or previous_summary)
|
||
drafted = self.draft(chapter, chunk, analysis, previous_summary)
|
||
drafts.update(drafted["paragraphs"])
|
||
if self.do_critic and (
|
||
self.critic_limit <= 0 or int(chunk.id.rsplit("c", 1)[-1]) < self.critic_limit
|
||
):
|
||
crit = self.criticise(chapter, chunk, drafted["paragraphs"])
|
||
for issue in crit.get("issues", []) or []:
|
||
issue = dict(issue)
|
||
issue["chunk"] = chunk.id
|
||
issues.append(issue)
|
||
print(
|
||
f" {chunk.id}: {len(drafted['paragraphs'])}/{len([p for p in chunk.paragraphs if p.get('type') != 'scene_break'])} "
|
||
f"paras, {len(issues)} issue(s) so far",
|
||
flush=True,
|
||
)
|
||
if self.do_consistency:
|
||
cons = self.consistency(chapter, drafts, chapter["paragraphs"])
|
||
for issue in cons.get("issues", []) or []:
|
||
issue = dict(issue)
|
||
issue["type"] = "consistency:" + str(issue.get("type"))
|
||
issues.append(issue)
|
||
summary = {
|
||
"chapter_id": chapter["id"],
|
||
"chapter_index": chapter["index"],
|
||
"title": chapter["title"],
|
||
"title_zh": chapter["title_zh"],
|
||
"paragraphs": drafts,
|
||
"issues": issues,
|
||
"chunks": [c.id for c in chunks],
|
||
}
|
||
self._write(self.chapter_dir(chapter) / "translation.json", summary)
|
||
print(f" wrote {self.chapter_dir(chapter) / 'translation.json'}", flush=True)
|
||
print(f"done in {time.time() - started:.0f}s", flush=True)
|
||
|
||
|
||
def report(run_dir: Path) -> str:
|
||
lines = []
|
||
total_paras = 0
|
||
total_issues = 0
|
||
for d in sorted(p for p in run_dir.iterdir() if p.is_dir() and p.name.startswith("ch-")):
|
||
t = d / "translation.json"
|
||
if not t.exists():
|
||
lines.append(f"{d.name}: not started")
|
||
continue
|
||
data = json.loads(t.read_text(encoding="utf-8"))
|
||
n = len(data.get("paragraphs", {}))
|
||
issues = data.get("issues", [])
|
||
total_paras += n
|
||
total_issues += len(issues)
|
||
by_sev: dict[str, int] = {}
|
||
for i in issues:
|
||
by_sev[i.get("severity", "?")] = by_sev.get(i.get("severity", "?"), 0) + 1
|
||
lines.append(f"{d.name}: {n} paragraphs, {len(issues)} issues {by_sev}")
|
||
lines.append(f"TOTAL: {total_paras} paragraphs, {total_issues} issues")
|
||
return "\n".join(lines)
|
||
|
||
|
||
def main() -> None:
|
||
ap = argparse.ArgumentParser(description="Run the Deal Hunter Chinese translation pipeline.")
|
||
ap.add_argument("--book", type=Path, required=True)
|
||
ap.add_argument("--glossary", type=Path, required=True)
|
||
ap.add_argument("--run-dir", type=Path, required=True)
|
||
ap.add_argument("--chapters", default="all", help="all, front, body, back, or comma-separated indices/ids")
|
||
ap.add_argument("--model", default=DEFAULT_MODEL)
|
||
ap.add_argument("--analyst-model", default=DEFAULT_ANALYST)
|
||
ap.add_argument("--critic-model", default=DEFAULT_CRITIC)
|
||
ap.add_argument("--base-url", default=None)
|
||
ap.add_argument("--force", action="store_true")
|
||
ap.add_argument("--no-critic", action="store_true")
|
||
ap.add_argument("--consistency", action="store_true")
|
||
ap.add_argument("--max-words", type=int, default=260)
|
||
ap.add_argument("--max-paras", type=int, default=14)
|
||
ap.add_argument("--critic-limit", type=int, default=0, help="criticise only the first N chunks per chapter (0 = all)")
|
||
ap.add_argument("--report", action="store_true")
|
||
args = ap.parse_args()
|
||
|
||
if args.report:
|
||
print(report(args.run_dir))
|
||
return
|
||
|
||
book = load_book(args.book)
|
||
chapters = select_chapters(book, args.chapters)
|
||
runner = Runner(
|
||
book_path=args.book,
|
||
glossary_path=args.glossary,
|
||
run_dir=args.run_dir,
|
||
model=args.model,
|
||
analyst_model=args.analyst_model,
|
||
critic_model=args.critic_model,
|
||
base_url=args.base_url,
|
||
force=args.force,
|
||
critic=not args.no_critic,
|
||
consistency=args.consistency,
|
||
max_words=args.max_words,
|
||
max_paras=args.max_paras,
|
||
critic_limit=args.critic_limit,
|
||
)
|
||
runner.run(chapters)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|