3f05489e25
The pipeline was hard-coded to Simplified Chinese in three places: the drafter was told to translate into Chinese, the critic was told to compare English with Chinese, and the glossary's translated forms were read from a "zh" key. A production that declared target_language: es-ES was therefore translated into Chinese and filed under a Spanish name - measured on oldbox, deal-hunter-es held 3,038 paragraphs of Chinese and was reported complete. A new registry, tools/translation/languages.py, is the single source of truth for what a target language is: its names, its script, its punctuation, its glossary key, its typography and its chapter-numbering convention. Ten languages are registered. The prompt scaffolding stays in Chinese - that is what the models read best - and the target is named with its Chinese, English and native names so there is no room to misread it. The registry now reaches every stage: - prompts.py: the analyst, drafter, critic and glossary blocks take the target. The hard rules about this book - Deal Hunter is the ship's name, the AI's name and the book's title, so it stays in English - are the language's own table for Chinese and the global rules for everything else, so a language with no table does not lose them. The Chinese rules are byte-for-byte what they were. - pipeline.py: --target-language, a run-directory guard that refuses to reuse a directory belonging to another language (from the manifest, or by measuring the text it already holds), a draft-keyed-by-target reader, and a refusal of a draft that came back in the wrong script. - book.py: per-language chapter-title and subtitle tables. A language with no table falls back to the English title, never to another language's; a language with a table and a missing chapter gets a visible marker in its own language. - qa.py: the target's quote marks, punctuation, script count and forbidden title forms. The Latin-residue check is skipped for a Latin target, where it would flag the whole book. - align.py: the aligner and verifier prompts name the target, and the sentence and clause splitting uses the target's own terminators. - render.py: the fonts, the language and country codes, the writing direction, the chapter headings and the fixed editorial strings around the translation are the target's. The Latin-run span is skipped for a Latin target, where marking the text as English would tell the renderer it is English. - project_status.py: the dashboard measures a sample of a production's text and reports a production whose text is not in the language it declares as "mismatched", not "complete". A chapter count is a claim about the files; the script check is a measurement of the text. Also fixes: the dashboard gated every language's production on the source production existing, so a Spanish production could never be reported even when it was there. The Chinese render is byte-identical to before on the real book, and the Chinese rules are unchanged. Suite: 620 passed, the same 68 pre-existing environment failures, +49 new tests.
916 lines
36 KiB
Python
916 lines
36 KiB
Python
# Copyright 2026 Umbra Tools (umbratools.com)
|
||
# SPDX-License-Identifier: Apache-2.0
|
||
|
||
"""Render the translated book as AsciiDoc and as an OpenDocument text file.
|
||
|
||
The AsciiDoc file is the canonical text output. The ODT is produced from a
|
||
Flat ODF (.fodt) document so that styles, page breaks and the table of
|
||
contents are under our control, then converted with LibreOffice. The TOC is
|
||
updated through UNO, because a plain ``--convert-to`` leaves an empty index.
|
||
|
||
The typography is the target language's: its fonts, its language and country
|
||
codes, its chapter-numbering convention, its quotation marks, and the fixed
|
||
editorial strings the renderer writes around the translation. A Spanish
|
||
edition typeset in Chinese fonts with Chinese chapter headings is the same
|
||
defect as a Chinese draft under a Spanish name, one level down.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import re
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
from pathlib import Path
|
||
from xml.sax.saxutils import escape
|
||
|
||
from book import SUBTITLES
|
||
from languages import SCRIPT_LATIN, TargetLanguage, resolve
|
||
from llm import DEFAULT_ANALYST, DEFAULT_CRITIC, DEFAULT_MODEL
|
||
|
||
CHINESE = resolve("zh-Hans")
|
||
|
||
|
||
def _esc(text: str) -> str:
|
||
return escape(text, {'"': """})
|
||
|
||
|
||
LATIN_RUN = re.compile(r"[A-Za-z][A-Za-z0-9'’.\-·]*(?:\s+[A-Za-z][A-Za-z0-9'’.\-·]*)*")
|
||
|
||
|
||
def _rich_esc(text: str, target: TargetLanguage | None = None) -> str:
|
||
"""Escape text and mark Latin runs as English.
|
||
|
||
A Chinese paragraph tells LibreOffice to use Chinese line-breaking for the
|
||
whole run, which can split an English name in the middle of a word
|
||
("Deal Hu / nter"). A character style with fo:language="en" keeps the
|
||
Latin run on Western line-breaking rules without changing its font.
|
||
|
||
For a target written in Latin script this is not applied at all: the whole
|
||
text is Latin, and marking it as English would tell the renderer that a
|
||
Spanish paragraph is English.
|
||
"""
|
||
target = target or CHINESE
|
||
if SCRIPT_LATIN in target.scripts:
|
||
return _esc(text)
|
||
out: list[str] = []
|
||
last = 0
|
||
for m in LATIN_RUN.finditer(text):
|
||
out.append(_esc(text[last : m.start()]))
|
||
out.append(f'<text:span text:style-name="Latin">{_esc(m.group(0))}</text:span>')
|
||
last = m.end()
|
||
out.append(_esc(text[last:]))
|
||
return "".join(out)
|
||
|
||
|
||
def load_run(run_dir: Path, chapter_id: str) -> dict | None:
|
||
path = run_dir / chapter_id / "translation.json"
|
||
if not path.exists():
|
||
return None
|
||
return json.loads(path.read_text(encoding="utf-8"))
|
||
|
||
|
||
def chapter_drafts(run_dir: Path | None, chapter_id: str) -> dict[str, str]:
|
||
if not run_dir:
|
||
return {}
|
||
data = load_run(run_dir, chapter_id)
|
||
return dict(data.get("paragraphs", {})) if data else {}
|
||
|
||
|
||
def _ordered_chapters(book: dict, run_dir: Path | None, include_missing: bool = False) -> list[dict]:
|
||
chapters = book["chapters"]
|
||
out = []
|
||
for c in chapters:
|
||
if c["matter"].startswith("front") and c["matter"] != "front_audio":
|
||
out.append(c)
|
||
out += [c for c in chapters if c["matter"] == "front_audio"]
|
||
out += [c for c in chapters if c["matter"] == "body"]
|
||
out += [c for c in chapters if c["matter"] == "back"]
|
||
if include_missing:
|
||
return out
|
||
return [c for c in out if chapter_drafts(run_dir, c["id"])]
|
||
|
||
|
||
def _title_of(chapter: dict) -> str:
|
||
"""The chapter's title in the target language.
|
||
|
||
The field was called ``title_zh`` while the pipeline was Chinese-only
|
||
and is now called ``title_target``. The old name is still read, so a book
|
||
file written by an earlier version keeps working.
|
||
"""
|
||
return chapter.get("title_target") or chapter.get("title_zh") or chapter.get("title") or ""
|
||
|
||
|
||
def _subtitle_of(book: dict, target: TargetLanguage | None = None) -> str:
|
||
"""The book's subtitle in the target language.
|
||
|
||
The old book format had no subtitle field: it stored the subtitle in the
|
||
book-level ``title_zh``, so that is read as a fallback. The table is the
|
||
last resort, and a language with no entry in it gets no subtitle rather
|
||
than another language's.
|
||
"""
|
||
target = target or CHINESE
|
||
return (
|
||
book.get("subtitle_target") or book.get("subtitle_zh") or book.get("title_zh") or SUBTITLES.get(target.base, "")
|
||
)
|
||
|
||
|
||
def _book_title(book: dict) -> str:
|
||
"""The book's title.
|
||
|
||
The title stays in English for every language: "Deal Hunter" is also the
|
||
ship's name and the AI's name, so translating it would rename a
|
||
character.
|
||
"""
|
||
return book.get("title_target") or book.get("title") or "Deal Hunter"
|
||
|
||
|
||
def _body_chapter_number(book: dict, chapter: dict) -> int:
|
||
body = [c for c in book["chapters"] if c["matter"] == "body"]
|
||
body.sort(key=lambda c: c["index"])
|
||
return body.index(chapter) + 1
|
||
|
||
|
||
def _paragraph_style(paragraph: dict, matter: str) -> str:
|
||
if matter == "front_copyright":
|
||
return "Copyright"
|
||
if matter == "front_note":
|
||
return "Quote" if _is_author_note_code(paragraph.get("text", "")) else "FrontText"
|
||
if matter.startswith("front"):
|
||
return "FrontText"
|
||
if paragraph.get("type") == "dialogue":
|
||
return "Dialogue"
|
||
return "BodyText"
|
||
|
||
|
||
def _is_author_note_code(text: str) -> bool:
|
||
keys = (
|
||
"One of those codes was",
|
||
"All women captives",
|
||
"Those who rape",
|
||
)
|
||
return any(text.startswith(k) for k in keys)
|
||
|
||
|
||
# --------------------------------------------------------------------- AsciiDoc
|
||
def build_asciidoc(
|
||
book: dict,
|
||
run_dir: Path,
|
||
*,
|
||
target: TargetLanguage | None = None,
|
||
sample: bool = False,
|
||
date: str = "",
|
||
) -> str:
|
||
target = target or CHINESE
|
||
chapters = _ordered_chapters(book, run_dir)
|
||
title = _book_title(book)
|
||
subtitle = _subtitle_of(book, target)
|
||
lines: list[str] = []
|
||
lines.append(f"= {title}: {subtitle}" if subtitle else f"= {title}")
|
||
lines.append("B. D. Murphy")
|
||
lines.append(":doctype: book")
|
||
lines.append(f":lang: {target.asciidoc_lang}")
|
||
lines.append(":toc: left")
|
||
lines.append(":toclevels: 2")
|
||
lines.append(f":revdate: {date or time.strftime('%Y-%m-%d')}")
|
||
if sample:
|
||
lines.append(f":revremark: {target.string('review_sample')}")
|
||
lines.append(":icons: font")
|
||
lines.append("")
|
||
lines.append(f"// {target.string('translator_credit')}")
|
||
if sample:
|
||
lines.append("")
|
||
lines.append(f"NOTE: {target.string('sample_note')}")
|
||
lines.append("")
|
||
|
||
seen_audio = False
|
||
for c in chapters:
|
||
drafts = chapter_drafts(run_dir, c["id"])
|
||
if not drafts:
|
||
continue
|
||
if c["matter"] == "front_audio":
|
||
if seen_audio:
|
||
continue
|
||
seen_audio = True
|
||
lines.append(f"// source: {c['id']} (audio front matter)")
|
||
lines.append(f"== {target.string('audio_preface')}")
|
||
lines.append("")
|
||
lines.append(target.string("audio_preface_intro"))
|
||
lines.append("")
|
||
for ac in [x for x in book["chapters"] if x["matter"] == "front_audio"]:
|
||
adrafts = chapter_drafts(run_dir, ac["id"])
|
||
if not adrafts:
|
||
continue
|
||
lines.append(f"=== {_title_of(ac)}")
|
||
lines.append("")
|
||
for p in ac["paragraphs"]:
|
||
zh = adrafts.get(p["id"])
|
||
if not zh:
|
||
continue
|
||
if p.get("type") == "scene_break":
|
||
lines.append("'''")
|
||
lines.append("")
|
||
continue
|
||
lines.append(zh)
|
||
lines.append("")
|
||
continue
|
||
|
||
if c["matter"] == "body":
|
||
n = _body_chapter_number(book, c)
|
||
heading = target.chapter_heading(n, _title_of(c))
|
||
else:
|
||
heading = _title_of(c)
|
||
lines.append(f"// source: {c['id']} (database row {c['row']}), {len(c['paragraphs'])} paragraphs")
|
||
lines.append(f"== {heading}")
|
||
lines.append("")
|
||
for p in c["paragraphs"]:
|
||
zh = drafts.get(p["id"])
|
||
if not zh:
|
||
continue
|
||
if p.get("type") == "scene_break":
|
||
lines.append("'''")
|
||
lines.append("")
|
||
continue
|
||
style = _paragraph_style(p, c["matter"])
|
||
if style in {"Quote", "Dialogue"}:
|
||
lines.append(f"[.{style.lower()}]" if style == "Dialogue" else "[quote]")
|
||
if style == "Quote":
|
||
lines.append("____")
|
||
lines.append(zh)
|
||
lines.append("____")
|
||
else:
|
||
lines.append(zh)
|
||
lines.append("")
|
||
return "\n".join(lines).rstrip() + "\n"
|
||
|
||
|
||
# --------------------------------------------------------------------- Flat ODF
|
||
def build_styles(target: TargetLanguage | None = None) -> str:
|
||
"""The document stylesheet for the target language.
|
||
|
||
The fonts, the language and country codes and the writing direction are
|
||
the target's. A document that declares ``fo:language="zh"`` while its
|
||
text is Spanish tells LibreOffice to hyphenate and shape it as Chinese,
|
||
which is how a Latin-script edition ends up with the wrong line breaks.
|
||
"""
|
||
target = target or CHINESE
|
||
direction = ' fo:direction="rtl"' if target.rtl else ""
|
||
return f"""
|
||
<office:font-face-decls>
|
||
<style:font-face style:name="{target.font_serif}" svg:font-family="'{target.font_serif}'" style:font-family-generic="roman" style:font-pitch="variable"/>
|
||
<style:font-face style:name="{target.font_sans}" svg:font-family="'{target.font_sans}'" style:font-family-generic="swiss" style:font-pitch="variable"/>
|
||
</office:font-face-decls>
|
||
<office:styles>
|
||
<style:default-style style:family="paragraph">
|
||
<style:paragraph-properties fo:orphans="2" fo:widows="2" fo:hyphenation-ladder-count="no-limit"/>
|
||
<style:text-properties style:font-name="{target.font_serif}" fo:font-size="11pt" fo:language="{target.odf_language}" fo:country="{target.odf_country}"{direction}/>
|
||
</style:default-style>
|
||
<style:style style:name="Title" style:family="paragraph" style:parent-style-name="Standard" style:class="text" style:master-page-name="FirstPage">
|
||
<style:paragraph-properties fo:text-align="center" fo:margin-top="2.1in" fo:margin-bottom="0.15in"/>
|
||
<style:text-properties fo:font-size="30pt" fo:font-weight="bold"/>
|
||
</style:style>
|
||
<style:style style:name="Subtitle" style:family="paragraph" style:parent-style-name="Standard" style:class="text">
|
||
<style:paragraph-properties fo:text-align="center" fo:margin-bottom="0.5in"/>
|
||
<style:text-properties fo:font-size="18pt" fo:color="#444444"/>
|
||
</style:style>
|
||
<style:style style:name="Author" style:family="paragraph" style:parent-style-name="Standard" style:class="text">
|
||
<style:paragraph-properties fo:text-align="center" fo:margin-top="0.4in" fo:margin-bottom="0.15in"/>
|
||
<style:text-properties fo:font-size="14pt"/>
|
||
</style:style>
|
||
<style:style style:name="Small" style:family="paragraph" style:parent-style-name="Standard" style:class="text">
|
||
<style:paragraph-properties fo:text-align="center" fo:margin-bottom="0.08in"/>
|
||
<style:text-properties fo:font-size="9.5pt" fo:color="#666666"/>
|
||
</style:style>
|
||
<style:style style:name="PageBreak" style:family="paragraph" style:parent-style-name="Standard" style:master-page-name="Standard">
|
||
<style:paragraph-properties fo:break-before="page"/>
|
||
</style:style>
|
||
<style:style style:name="ChapterHeading" style:family="paragraph" style:parent-style-name="Standard" style:class="text">
|
||
<style:paragraph-properties fo:break-before="page" fo:margin-top="0.25in" fo:margin-bottom="0.3in" fo:keep-with-next="always"/>
|
||
<style:text-properties fo:font-size="20pt" fo:font-weight="bold"/>
|
||
</style:style>
|
||
<style:style style:name="SectionHeading" style:family="paragraph" style:parent-style-name="Standard" style:class="text">
|
||
<style:paragraph-properties fo:margin-top="0.25in" fo:margin-bottom="0.12in" fo:keep-with-next="always"/>
|
||
<style:text-properties fo:font-size="14pt" fo:font-weight="bold"/>
|
||
</style:style>
|
||
<style:style style:name="BodyText" style:family="paragraph" style:parent-style-name="Standard" style:class="text">
|
||
<style:paragraph-properties fo:text-align="justify" fo:text-indent="0.31in" fo:margin-bottom="0.06in" fo:line-height="150%"/>
|
||
<style:text-properties fo:font-size="11pt"/>
|
||
</style:style>
|
||
<style:style style:name="Dialogue" style:family="paragraph" style:parent-style-name="Standard" style:class="text">
|
||
<style:paragraph-properties fo:text-align="justify" fo:text-indent="0.31in" fo:margin-bottom="0.06in" fo:line-height="150%"/>
|
||
<style:text-properties fo:font-size="11pt"/>
|
||
</style:style>
|
||
<style:style style:name="Latin" style:family="text">
|
||
<style:text-properties fo:language="en" fo:country="US"/>
|
||
</style:style>
|
||
<style:style style:name="FrontText" style:family="paragraph" style:parent-style-name="Standard" style:class="text">
|
||
<style:paragraph-properties fo:text-align="left" fo:margin-bottom="0.10in" fo:line-height="140%"/>
|
||
<style:text-properties fo:font-size="10.5pt"/>
|
||
</style:style>
|
||
<style:style style:name="Copyright" style:family="paragraph" style:parent-style-name="Standard" style:class="text">
|
||
<style:paragraph-properties fo:text-align="left" fo:margin-bottom="0.10in" fo:line-height="135%"/>
|
||
<style:text-properties fo:font-size="9.5pt" fo:color="#333333"/>
|
||
</style:style>
|
||
<style:style style:name="Quote" style:family="paragraph" style:parent-style-name="Standard" style:class="text">
|
||
<style:paragraph-properties fo:margin-left="0.5in" fo:margin-right="0.4in" fo:margin-bottom="0.12in" fo:line-height="145%" fo:background-color="#f4f4f4"/>
|
||
<style:text-properties fo:font-size="10.5pt"/>
|
||
</style:style>
|
||
<style:style style:name="SceneBreak" style:family="paragraph" style:parent-style-name="Standard" style:class="text">
|
||
<style:paragraph-properties fo:text-align="center" fo:margin-top="0.12in" fo:margin-bottom="0.12in"/>
|
||
<style:text-properties fo:font-size="12pt"/>
|
||
</style:style>
|
||
<style:style style:name="TOCTitle" style:family="paragraph" style:parent-style-name="Standard" style:class="index">
|
||
<style:paragraph-properties fo:text-align="center" fo:margin-bottom="0.3in"/>
|
||
<style:text-properties fo:font-size="18pt" fo:font-weight="bold"/>
|
||
</style:style>
|
||
<style:style style:name="Contents1" style:family="paragraph" style:parent-style-name="Standard" style:class="index">
|
||
<style:paragraph-properties fo:margin-left="0in" fo:margin-bottom="0.08in"/>
|
||
<style:text-properties fo:font-size="11pt"/>
|
||
</style:style>
|
||
<style:style style:name="Header" style:family="paragraph" style:parent-style-name="Standard" style:class="extra">
|
||
<style:paragraph-properties fo:text-align="center"/>
|
||
<style:text-properties fo:font-size="8.5pt" fo:color="#777777"/>
|
||
</style:style>
|
||
<style:style style:name="Footer" style:family="paragraph" style:parent-style-name="Standard" style:class="extra">
|
||
<style:paragraph-properties fo:text-align="center"/>
|
||
<style:text-properties fo:font-size="8.5pt" fo:color="#777777"/>
|
||
</style:style>
|
||
<text:outline-style style:name="Outline">
|
||
<text:outline-level-style text:level="1" style:num-format=""/>
|
||
<text:outline-level-style text:level="2" style:num-format=""/>
|
||
</text:outline-style>
|
||
</office:styles>
|
||
"""
|
||
|
||
|
||
def _odt_paragraph(
|
||
style: str,
|
||
text: str,
|
||
outline: int | None = None,
|
||
target: TargetLanguage | None = None,
|
||
) -> str:
|
||
body = _rich_esc(text, target)
|
||
if outline:
|
||
return f'<text:h text:style-name="{style}" text:outline-level="{outline}">{body}</text:h>'
|
||
return f'<text:p text:style-name="{style}">{body}</text:p>'
|
||
|
||
|
||
def build_fodt(
|
||
book: dict,
|
||
run_dir: Path,
|
||
*,
|
||
target: TargetLanguage | None = None,
|
||
sample: bool = False,
|
||
date: str = "",
|
||
) -> str:
|
||
target = target or CHINESE
|
||
chapters = _ordered_chapters(book, run_dir)
|
||
body: list[str] = []
|
||
title = _book_title(book)
|
||
subtitle = _subtitle_of(book, target)
|
||
|
||
# Title page
|
||
body.append(_odt_paragraph("Title", title, target=target))
|
||
if subtitle:
|
||
body.append(_odt_paragraph("Subtitle", subtitle, target=target))
|
||
body.append(_odt_paragraph("Author", target.string("author_line"), target=target))
|
||
body.append(_odt_paragraph("Small", target.string("translator_line"), target=target))
|
||
if sample:
|
||
body.append(_odt_paragraph("Small", target.string("edition_label"), target=target))
|
||
body.append(_odt_paragraph("Small", date or time.strftime("%Y-%m-%d"), target=target))
|
||
body.append('<text:p text:style-name="PageBreak"/>')
|
||
|
||
# Table of contents
|
||
body.append(
|
||
'<text:table-of-content text:name="TableOfContents" text:style-name="Contents1" text:protected="false">'
|
||
'<text:table-of-content-source text:outline-level="1">'
|
||
f'<text:index-title-template text:style-name="TOCTitle">{target.string("toc_title")}</text:index-title-template>'
|
||
'<text:table-of-content-entry-template text:outline-level="1" text:style-name="Contents1">'
|
||
"<text:index-entry-text/>"
|
||
'<text:index-entry-tab-stop style:type="right" style:leader-char="."/>'
|
||
"<text:index-entry-page-number/>"
|
||
"</text:table-of-content-entry-template>"
|
||
"</text:table-of-content-source>"
|
||
"</text:table-of-content>"
|
||
)
|
||
|
||
if sample:
|
||
body.append(_odt_paragraph("ChapterHeading", target.string("sample_heading"), 1, target=target))
|
||
body.append(_odt_paragraph("FrontText", target.string("sample_note"), target=target))
|
||
body.append(_odt_paragraph("FrontText", target.string("sample_process"), target=target))
|
||
|
||
seen_audio = False
|
||
for c in chapters:
|
||
drafts = chapter_drafts(run_dir, c["id"])
|
||
if not drafts:
|
||
continue
|
||
if c["matter"] == "front_audio":
|
||
if seen_audio:
|
||
continue
|
||
seen_audio = True
|
||
body.append(_odt_paragraph("ChapterHeading", target.string("audio_preface"), 1, target=target))
|
||
body.append(_odt_paragraph("FrontText", target.string("audio_preface_intro"), target=target))
|
||
for ac in [x for x in book["chapters"] if x["matter"] == "front_audio"]:
|
||
adrafts = chapter_drafts(run_dir, ac["id"])
|
||
if not adrafts:
|
||
continue
|
||
body.append(_odt_paragraph("SectionHeading", _title_of(ac), 2, target=target))
|
||
for p in ac["paragraphs"]:
|
||
zh = adrafts.get(p["id"])
|
||
if not zh:
|
||
continue
|
||
if p.get("type") == "scene_break":
|
||
body.append(_odt_paragraph("SceneBreak", "* * *", target=target))
|
||
else:
|
||
body.append(_odt_paragraph("FrontText", zh, target=target))
|
||
continue
|
||
|
||
if c["matter"] == "body":
|
||
n = _body_chapter_number(book, c)
|
||
heading = target.chapter_heading(n, _title_of(c))
|
||
else:
|
||
heading = _title_of(c)
|
||
body.append(_odt_paragraph("ChapterHeading", heading, 1, target=target))
|
||
for p in c["paragraphs"]:
|
||
zh = drafts.get(p["id"])
|
||
if not zh:
|
||
continue
|
||
if p.get("type") == "scene_break":
|
||
body.append(_odt_paragraph("SceneBreak", "* * *", target=target))
|
||
continue
|
||
body.append(_odt_paragraph(_paragraph_style(p, c["matter"]), zh, target=target))
|
||
|
||
if sample:
|
||
body.append(_odt_paragraph("ChapterHeading", target.string("sample_end"), 1, target=target))
|
||
body.append(_odt_paragraph("FrontText", target.string("sample_end_note"), target=target))
|
||
|
||
return f"""<?xml version="1.0" encoding="UTF-8"?>
|
||
<office:document xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0"
|
||
xmlns:style="urn:oasis:names:tc:opendocument:xmlns:style:1.0"
|
||
xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0"
|
||
xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0"
|
||
xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0"
|
||
xmlns:fo="urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0"
|
||
xmlns:xlink="http://www.w3.org/1999/xlink"
|
||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||
xmlns:meta="urn:oasis:names:tc:opendocument:xmlns:meta:1.0"
|
||
xmlns:number="urn:oasis:names:tc:opendocument:xmlns:datastyle:1.0"
|
||
xmlns:svg="urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0"
|
||
xmlns:of="urn:oasis:names:tc:opendocument:xmlns:of:1.2"
|
||
office:version="1.3" office:mimetype="application/vnd.oasis.opendocument.text">
|
||
{build_styles(target)}
|
||
<office:automatic-styles>
|
||
<style:page-layout style:name="pm1">
|
||
<style:page-layout-properties fo:page-width="8.27in" fo:page-height="11.69in"
|
||
fo:margin-top="0.9in" fo:margin-bottom="0.9in" fo:margin-left="1.0in" fo:margin-right="1.0in"
|
||
style:print-orientation="portrait" style:first-page-number="1"/>
|
||
<style:header-style><style:header-footer-properties fo:min-height="0.3in" fo:margin-bottom="0.2in"/></style:header-style>
|
||
<style:footer-style><style:header-footer-properties fo:min-height="0.3in" fo:margin-top="0.2in"/></style:footer-style>
|
||
</style:page-layout>
|
||
</office:automatic-styles>
|
||
<office:master-styles>
|
||
<style:master-page style:name="Standard" style:page-layout-name="pm1" style:next-style-name="Standard">
|
||
<style:header><text:p text:style-name="Header">{_esc(title + (f" · {subtitle}" if subtitle else ""))}</text:p></style:header>
|
||
<style:footer><text:p text:style-name="Footer"><text:page-number text:select-page="current">1</text:page-number></text:p></style:footer>
|
||
</style:master-page>
|
||
<style:master-page style:name="FirstPage" style:page-layout-name="pm1"/>
|
||
</office:master-styles>
|
||
<office:body>
|
||
<office:text>
|
||
<text:sequence-decls>
|
||
<text:sequence-decl text:display-outline-level="0" text:name="Illustration"/>
|
||
<text:sequence-decl text:display-outline-level="0" text:name="Table"/>
|
||
<text:sequence-decl text:display-outline-level="0" text:name="Text"/>
|
||
<text:sequence-decl text:display-outline-level="0" text:name="Drawing"/>
|
||
</text:sequence-decls>
|
||
{chr(10).join(body)}
|
||
</office:text>
|
||
</office:body>
|
||
</office:document>
|
||
"""
|
||
|
||
|
||
# --------------------------------------------------------------------- LibreOffice
|
||
def _soffice_binary() -> str:
|
||
for name in ("soffice", "libreoffice"):
|
||
path = shutil.which(name)
|
||
if path:
|
||
return path
|
||
raise SystemExit("LibreOffice not found; cannot produce ODT/PDF")
|
||
|
||
|
||
def convert_with_uno(fodt: Path, odt: Path, pdf: Path | None, port: int = 2002) -> None:
|
||
"""Convert FODT -> ODT (and PDF) with an updated TOC.
|
||
|
||
``--convert-to`` leaves a text:table-of-content empty. Opening the document
|
||
through UNO lets us call update() on the index before storing.
|
||
"""
|
||
fodt = fodt.resolve()
|
||
odt = odt.resolve()
|
||
if pdf:
|
||
pdf = pdf.resolve()
|
||
soffice = _soffice_binary()
|
||
profile = Path(tempfile.mkdtemp(prefix="lo-profile-"))
|
||
proc = subprocess.Popen(
|
||
[
|
||
soffice,
|
||
"--headless",
|
||
"--invisible",
|
||
"--norestore",
|
||
"--nofirststartwizard",
|
||
f"-env:UserInstallation={profile.as_uri()}",
|
||
f"--accept=socket,host=127.0.0.1,port={port};urp;",
|
||
],
|
||
stdout=subprocess.DEVNULL,
|
||
stderr=subprocess.DEVNULL,
|
||
)
|
||
try:
|
||
import uno # type: ignore
|
||
from com.sun.star.beans import PropertyValue # type: ignore
|
||
|
||
local = uno.getComponentContext()
|
||
resolver = local.ServiceManager.createInstanceWithContext(
|
||
"com.sun.star.bridge.UnoUrlResolver", local
|
||
)
|
||
ctx = None
|
||
for _ in range(80):
|
||
try:
|
||
ctx = resolver.resolve(
|
||
f"uno:socket,host=127.0.0.1,port={port};urp;StarOffice.ComponentContext"
|
||
)
|
||
break
|
||
except Exception:
|
||
time.sleep(0.25)
|
||
if ctx is None:
|
||
raise RuntimeError("could not connect to LibreOffice")
|
||
|
||
desktop = ctx.ServiceManager.createInstanceWithContext("com.sun.star.frame.Desktop", ctx)
|
||
doc = desktop.loadComponentFromURL(uno.systemPathToFileUrl(str(fodt)), "_blank", 0, ())
|
||
indexes = doc.getDocumentIndexes()
|
||
for i in range(indexes.getCount()):
|
||
indexes.getByIndex(i).update()
|
||
|
||
def prop(name: str, value: object) -> object:
|
||
p = PropertyValue()
|
||
p.Name = name
|
||
p.Value = value
|
||
return p
|
||
|
||
doc.storeToURL(uno.systemPathToFileUrl(str(odt)), (prop("FilterName", "writer8"),))
|
||
if pdf:
|
||
doc.storeToURL(uno.systemPathToFileUrl(str(pdf)), (prop("FilterName", "writer_pdf_Export"),))
|
||
doc.close(True)
|
||
finally:
|
||
proc.terminate()
|
||
try:
|
||
proc.wait(timeout=10)
|
||
except subprocess.TimeoutExpired:
|
||
proc.kill()
|
||
shutil.rmtree(profile, ignore_errors=True)
|
||
|
||
|
||
def render(
|
||
*,
|
||
book_path: Path,
|
||
run_dir: Path,
|
||
out_dir: Path,
|
||
sample: bool = False,
|
||
date: str = "",
|
||
pdf: bool = True,
|
||
target_language: str = "zh-Hans",
|
||
) -> dict[str, Path]:
|
||
target = resolve(target_language)
|
||
book = json.loads(book_path.read_text(encoding="utf-8"))
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
stem = f"deal-hunter-{target.base}-sample" if sample else f"deal-hunter-{target.base}"
|
||
asciidoc = out_dir / f"{stem}.asciidoc"
|
||
fodt = out_dir / f"{stem}.fodt"
|
||
odt = out_dir / f"{stem}.odt"
|
||
pdf_path = out_dir / f"{stem}.pdf" if pdf else None
|
||
|
||
asciidoc.write_text(
|
||
build_asciidoc(book, run_dir, target=target, sample=sample, date=date), encoding="utf-8"
|
||
)
|
||
fodt.write_text(build_fodt(book, run_dir, target=target, sample=sample, date=date), encoding="utf-8")
|
||
convert_with_uno(fodt, odt, pdf_path)
|
||
out = {"asciidoc": asciidoc, "fodt": fodt, "odt": odt}
|
||
if pdf_path:
|
||
out["pdf"] = pdf_path
|
||
|
||
# Enhance EPUB with cover image and metadata
|
||
epub_path = out_dir / f"{stem}.epub"
|
||
cover_path = book_path.parent / "covers" / "cover.png"
|
||
enhance_epub(epub_path, book, cover_path if cover_path.exists() else None, target)
|
||
out["epub"] = epub_path
|
||
return out
|
||
|
||
|
||
def enhance_epub(
|
||
epub_path: Path,
|
||
book: dict,
|
||
cover_path: Path | None = None,
|
||
target: TargetLanguage | None = None,
|
||
) -> None:
|
||
"""Add cover image, ISBN, publisher, and translator metadata to EPUB.
|
||
|
||
Pandoc's ODT->EPUB conversion loses some metadata. This function post-processes
|
||
the EPUB to add cover image and complete metadata.
|
||
"""
|
||
target = target or CHINESE
|
||
import zipfile
|
||
import tempfile
|
||
import shutil
|
||
from lxml import etree
|
||
|
||
epub_path = epub_path.resolve()
|
||
tmpdir = Path(tempfile.mkdtemp(prefix="epub-enhance-"))
|
||
try:
|
||
with zipfile.ZipFile(epub_path, "r") as zin:
|
||
zin.extractall(tmpdir)
|
||
|
||
# Find content.opf
|
||
opf_path = None
|
||
for f in tmpdir.rglob("*.opf"):
|
||
if f.name == "content.opf":
|
||
opf_path = f
|
||
break
|
||
if opf_path is None:
|
||
return
|
||
|
||
tree = etree.parse(str(opf_path))
|
||
root = tree.getroot()
|
||
ns = {
|
||
"opf": "http://www.idpf.org/2007/opf",
|
||
"dc": "http://purl.org/dc/elements/1.1/",
|
||
}
|
||
|
||
# Add ISBN metadata if present in book
|
||
# ISBN is stored in copyright paragraph text; extract it
|
||
import re
|
||
for ch in book.get("chapters", []):
|
||
if ch.get("matter") == "front_copyright":
|
||
for p in ch.get("paragraphs", []):
|
||
text = p.get("text", "")
|
||
isbn_match = re.search(r"ISBN:\s*([\d-]+)", text)
|
||
if isbn_match:
|
||
isbn = isbn_match.group(1)
|
||
# Add ISBN identifier
|
||
id_elem = etree.SubElement(root.find("opf:metadata", ns), "{http://purl.org/dc/elements/1.1/}identifier")
|
||
id_elem.set("{http://www.idpf.org/2007/opf}scheme", "ISBN")
|
||
id_elem.set("id", "epub-isbn")
|
||
id_elem.text = isbn
|
||
break
|
||
|
||
# Add cover image if provided
|
||
if cover_path and cover_path.exists():
|
||
# Add cover image to manifest
|
||
cover_href = "images/cover.png"
|
||
cover_id = "cover-image"
|
||
|
||
# Copy cover image to EPUB images dir
|
||
images_dir = tmpdir / "EPUB" / "images"
|
||
images_dir.mkdir(parents=True, exist_ok=True)
|
||
shutil.copy2(cover_path, images_dir / "cover.png")
|
||
|
||
# Add manifest item
|
||
manifest = root.find("opf:manifest", ns)
|
||
cover_item = etree.SubElement(manifest, "{http://www.idpf.org/2007/opf}item")
|
||
cover_item.set("id", cover_id)
|
||
cover_item.set("href", cover_href)
|
||
cover_item.set("media-type", "image/png")
|
||
cover_item.set("properties", "cover-image")
|
||
|
||
# Add cover meta tag
|
||
metadata = root.find("opf:metadata", ns)
|
||
cover_meta = etree.SubElement(metadata, "{http://www.idpf.org/2007/opf}meta")
|
||
cover_meta.set("name", "cover")
|
||
cover_meta.text = cover_id
|
||
|
||
# Add the target language's typography to the stylesheet
|
||
css_path = tmpdir / "EPUB" / "styles" / "stylesheet1.css"
|
||
if css_path.exists():
|
||
css_content = css_path.read_text(encoding="utf-8")
|
||
lang = target.asciidoc_lang
|
||
font_stack = target.epub_font_stack
|
||
if target.cjk:
|
||
css_extra = f"""
|
||
/* CJK typography for {lang} body text */
|
||
html {{
|
||
font-family: {font_stack};
|
||
}}
|
||
html[lang="{lang}"], body[lang="{lang}"] {{
|
||
font-family: {font_stack};
|
||
line-break: strict;
|
||
word-break: normal;
|
||
}}
|
||
p[lang="{lang}"] {{
|
||
font-family: {font_stack};
|
||
line-break: strict;
|
||
word-break: normal;
|
||
}}"""
|
||
else:
|
||
css_extra = f"""
|
||
/* Typography for {lang} body text */
|
||
html {{
|
||
font-family: {font_stack};
|
||
}}
|
||
html[lang="{lang}"], body[lang="{lang}"] {{
|
||
font-family: {font_stack};
|
||
}}
|
||
p[lang="{lang}"] {{
|
||
font-family: {font_stack};
|
||
}}"""
|
||
target_css = css_extra + '''
|
||
/* Dialogue styling */
|
||
.utterance__text {
|
||
font-size: 1.1em;
|
||
line-height: 1.6;
|
||
}
|
||
/* Chapter headings */
|
||
h1, h2 {
|
||
page-break-before: always;
|
||
page-break-after: avoid;
|
||
}
|
||
/* Speaker labels */
|
||
.utterance__speaker {
|
||
font-size: 0.85em;
|
||
color: #666;
|
||
font-weight: bold;
|
||
}
|
||
/* Variation dropdowns */
|
||
.utterance__variation select {
|
||
font-size: 0.8em;
|
||
}
|
||
'''
|
||
css_path.write_text(css_content + chr(10) + target_css, encoding="utf-8")
|
||
|
||
# Add publisher metadata
|
||
metadata = root.find("opf:metadata", ns)
|
||
publisher = etree.SubElement(metadata, "{http://purl.org/dc/elements/1.1/}publisher")
|
||
publisher.text = "Umbra Audio"
|
||
|
||
# Add translator contributor
|
||
translator = etree.SubElement(metadata, "{http://purl.org/dc/elements/1.1/}contributor")
|
||
translator.set("{http://www.idpf.org/2007/opf}role", "trl")
|
||
translator.text = "Umbra Tools Translation Team"
|
||
language = etree.SubElement(metadata, "{http://purl.org/dc/elements/1.1/}language")
|
||
language.text = target.asciidoc_lang
|
||
|
||
tree.write(str(opf_path), xml_declaration=True, encoding="UTF-8", standalone=True)
|
||
|
||
# Repackage EPUB
|
||
with zipfile.ZipFile(epub_path, "w", zipfile.ZIP_DEFLATED) as zout:
|
||
# mimetype must be first and uncompressed
|
||
zout.write(tmpdir / "mimetype", "mimetype", compress_type=zipfile.ZIP_STORED)
|
||
for f in tmpdir.rglob("*"):
|
||
if f.is_file() and f.name != "mimetype":
|
||
arcname = f.relative_to(tmpdir)
|
||
zout.write(f, arcname)
|
||
finally:
|
||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||
|
||
|
||
def _sha256(path: Path) -> str:
|
||
h = hashlib.sha256()
|
||
with path.open("rb") as f:
|
||
for block in iter(lambda: f.read(1024 * 1024), b""):
|
||
h.update(block)
|
||
return h.hexdigest()
|
||
|
||
|
||
def _repo_commit(start: Path) -> str | None:
|
||
for p in [start, *start.parents]:
|
||
if (p / ".git").exists():
|
||
try:
|
||
return subprocess.check_output(
|
||
["git", "-C", str(p), "rev-parse", "HEAD"],
|
||
text=True,
|
||
stderr=subprocess.DEVNULL,
|
||
).strip()
|
||
except (OSError, subprocess.CalledProcessError):
|
||
return None
|
||
return None
|
||
|
||
|
||
def write_manifest(
|
||
*,
|
||
book_path: Path,
|
||
run_dir: Path,
|
||
out_dir: Path,
|
||
sample: bool,
|
||
files: dict[str, Path],
|
||
target: TargetLanguage | None = None,
|
||
) -> Path:
|
||
"""Write the reproducibility manifest next to the rendered files.
|
||
|
||
It contains hashes, counts and model names, never the book text.
|
||
"""
|
||
target = target or CHINESE
|
||
book = json.loads(book_path.read_text(encoding="utf-8"))
|
||
chapters: list[dict] = []
|
||
total = translated = issues = scene_breaks = 0
|
||
for c in book["chapters"]:
|
||
data = load_run(run_dir, c["id"])
|
||
expected = len([p for p in c["paragraphs"] if p.get("type") != "scene_break"])
|
||
scene_breaks += len([p for p in c["paragraphs"] if p.get("type") == "scene_break"])
|
||
n = len(data.get("paragraphs", {})) if data else 0
|
||
total += expected
|
||
translated += n
|
||
if not data:
|
||
continue
|
||
chapter_issues = len(data.get("issues", []) or [])
|
||
issues += chapter_issues
|
||
chapters.append(
|
||
{
|
||
"id": c["id"],
|
||
"index": c["index"],
|
||
"matter": c["matter"],
|
||
"title": c["title"],
|
||
"title_target": _title_of(c),
|
||
"paragraphs_expected": expected,
|
||
"paragraphs_translated": n,
|
||
"critic_issues": chapter_issues,
|
||
}
|
||
)
|
||
glossary = run_dir.parent / "glossary" / "glossary.json"
|
||
manifest = {
|
||
"title": book.get("title"),
|
||
"title_target": _book_title(book),
|
||
"subtitle_target": _subtitle_of(book, target),
|
||
"target_language": target.code,
|
||
"author": book.get("author"),
|
||
"generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||
"sample": sample,
|
||
"source": {
|
||
"bundle": book.get("bundle"),
|
||
"bundle_sha256": book.get("bundle_sha256"),
|
||
"book_json_sha256": _sha256(book_path),
|
||
},
|
||
"glossary_sha256": _sha256(glossary) if glossary.exists() else None,
|
||
"prompts_sha256": _sha256(Path(__file__).with_name("prompts.py")),
|
||
"git_commit": _repo_commit(book_path),
|
||
"models": {
|
||
"drafter": DEFAULT_MODEL,
|
||
"analyst": DEFAULT_ANALYST,
|
||
"critic": DEFAULT_CRITIC,
|
||
},
|
||
"counts": {
|
||
"chapters": len(chapters),
|
||
"paragraphs_expected": total,
|
||
"paragraphs_translated": translated,
|
||
"scene_breaks": scene_breaks,
|
||
"critic_issues": issues,
|
||
},
|
||
"chapters": chapters,
|
||
"outputs": {k: str(v) for k, v in files.items()},
|
||
}
|
||
path = out_dir / "run-manifest.json"
|
||
path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
return path
|
||
|
||
|
||
def main() -> None:
|
||
ap = argparse.ArgumentParser(description="Render Deal Hunter AsciiDoc + ODT in the target language.")
|
||
ap.add_argument("--book", type=Path, required=True)
|
||
ap.add_argument("--run-dir", type=Path, required=True)
|
||
ap.add_argument("--out-dir", type=Path, required=True)
|
||
ap.add_argument(
|
||
"--target-language",
|
||
default=None,
|
||
help="the language the translations are in. Defaults to the book's own.",
|
||
)
|
||
ap.add_argument("--sample", action="store_true")
|
||
ap.add_argument("--date", default="")
|
||
ap.add_argument("--no-pdf", action="store_true")
|
||
args = ap.parse_args()
|
||
book = json.loads(args.book.read_text(encoding="utf-8"))
|
||
target_language = args.target_language or book.get("target_language") or "zh-Hans"
|
||
out = render(
|
||
book_path=args.book,
|
||
run_dir=args.run_dir,
|
||
out_dir=args.out_dir,
|
||
sample=args.sample,
|
||
date=args.date,
|
||
pdf=not args.no_pdf,
|
||
target_language=target_language,
|
||
)
|
||
out["manifest"] = write_manifest(
|
||
book_path=args.book,
|
||
run_dir=args.run_dir,
|
||
out_dir=args.out_dir,
|
||
sample=args.sample,
|
||
files=out,
|
||
target=resolve(target_language),
|
||
)
|
||
for k, v in out.items():
|
||
print(f"{k:9} {v}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|