UmbraTrans/tools/translation/names.py

# Copyright 2026 Umbra Tools (umbratools.com)
# SPDX-License-Identifier: Apache-2.0

"""Collect proper-name and term candidates for the glossary.

Two sources are merged:

1. Terms proposed by the scene analyst (``proposed_terms`` in every
   ``*.analysis.json``).
2. Capitalised words in the English source that appear mid-sentence, which is
   a decent signal for a name but filters out sentence-initial function words.

Nothing is written into the glossary automatically. The output is a proposal
list for the operator to accept, edit or reject, because an unapproved
transliteration is worse than an English name left in place.
"""

from __future__ import annotations

import argparse
import json
import re
import sys
from collections import defaultdict
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))

from pipeline import LATIN_ALLOW  # noqa: E402

STOPWORDS = {
    "a", "about", "after", "again", "against", "all", "also", "am", "an", "and",
    "any", "are", "aren", "as", "at", "back", "be", "because", "been", "before",
    "being", "below", "between", "both", "but", "by", "can", "could", "did",
    "didn", "do", "does", "doesn", "doing", "don", "down", "during", "each",
    "even", "ever", "every", "few", "for", "from", "get", "gets", "getting",
    "give", "go", "going", "gone", "good", "got", "had", "has", "have",
    "having", "he", "her", "here", "hers", "herself", "him", "himself", "his",
    "how", "however", "i", "if", "in", "into", "is", "isn", "it", "its",
    "itself", "just", "know", "let", "like", "look", "looking", "made", "make",
    "many", "may", "maybe", "me", "might", "mine", "more", "most", "much",
    "must", "my", "myself", "need", "no", "nor", "not", "now", "of", "off",
    "on", "once", "one", "only", "or", "other", "ought", "our", "ours",
    "ourselves", "out", "over", "own", "perhaps", "please", "put", "quite",
    "rather", "really", "right", "said", "same", "say", "saying", "see",
    "seen", "she", "should", "since", "so", "some", "something", "still",
    "such", "sure", "take", "than", "thank", "thanks", "that", "the", "their",
    "theirs", "them", "themselves", "then", "there", "these", "they", "thing",
    "things", "think", "this", "those", "though", "through", "to", "too",
    "under", "until", "up", "upon", "us", "very", "was", "wasn", "we", "well",
    "were", "weren", "what", "when", "where", "whether", "which", "while",
    "who", "whom", "why", "will", "with", "within", "without", "would",
    "yes", "yet", "you", "your", "yours", "yourself", "yourselves",
    "okay", "ok", "hi", "hello", "hey", "oh", "ah", "um", "uh", "hmm",
    "mr", "mrs", "ms", "dr", "sir", "ma", "am", "captain", "princess",
    "doctor", "aunt", "uncle", "nanny", "hunter",
    "everyone", "everything", "someone", "something", "nothing", "anyone",
    "anything", "people", "one", "two", "three", "four", "five", "six",
    "seven", "eight", "nine", "ten", "eleven", "twelve", "first", "second",
    "third", "fourth", "fifth", "next", "last", "inside", "outside", "based",
    "turning", "checking", "looking", "going", "trying", "waiting",
    "standing", "sitting", "walking", "talking", "several", "tell",
    "additionally", "besides", "however", "therefore", "meanwhile",
    "suddenly", "finally", "actually", "apparently", "obviously",
    "unfortunately", "luckily", "instead", "as", "at", "if", "is", "it",
    "its", "be", "by", "do", "does", "did", "for", "from", "had", "has",
    "he", "her", "him", "his", "how", "into", "me", "my", "of", "on", "or",
    "our", "she", "so", "than", "that", "the", "their", "them", "then",
    "there", "these", "they", "this", "those", "to", "up", "us", "was",
    "we", "were", "what", "when", "where", "which", "who", "why", "will",
    "with", "would", "you", "your",
}

SENTENCE_BOUNDARY = set('.!?…"“”\n')


def _load(path: Path) -> dict:
    return json.loads(path.read_text(encoding="utf-8"))


def _glossary_keys(glossary: dict) -> set[str]:
    keys: set[str] = set()
    for section in ("characters", "roles", "places", "terms"):
        for key in glossary.get(section, {}) or {}:
            keys.add(key.lower())
    for extra in (glossary.get("title") or {}).values():
        if isinstance(extra, str):
            keys.add(extra.lower())
    return keys


def _analyst_terms(run_dir: Path) -> dict[str, dict]:
    out: dict[str, dict] = {}
    for path in sorted(run_dir.glob("*/*.analysis.json")):
        try:
            data = _load(path)
        except (OSError, json.JSONDecodeError):
            continue
        for term in data.get("proposed_terms", []) or []:
            if not isinstance(term, dict):
                continue
            en = str(term.get("en") or "").strip().lower()
            if not en:
                continue
            entry = out.setdefault(en, {"en": term.get("en"), "zh": term.get("zh"), "why": [], "count": 0})
            entry["count"] += 1
            why = term.get("why")
            if why and why not in entry["why"]:
                entry["why"].append(why)
            if term.get("zh") and not entry.get("zh"):
                entry["zh"] = term.get("zh")
    return out


def _source_names(book: dict, only: set[str] | None) -> dict[str, dict]:
    counts: dict[str, dict] = defaultdict(lambda: {"count": 0, "contexts": []})
    pattern = re.compile(r"\b([A-Z][a-zA-Z'’-]+(?:\s+[A-Z][a-zA-Z'’-]+){0,2})\b")
    for chapter in book["chapters"]:
        if only and chapter["id"] not in only:
            continue
        for p in chapter["paragraphs"]:
            text = p.get("text") or ""
            for m in pattern.finditer(text):
                name = m.group(1).strip()
                name = re.sub(r"[’']s$", "", name).strip("-'")
                if not name or "’" in name or "'" in name:
                    continue
                words = name.split()
                if all(w.lower() in STOPWORDS for w in words):
                    continue
                if len(words) == 1 and words[0].lower().endswith("ly"):
                    continue
                first = words[0].lower()
                if first in STOPWORDS and len(words) == 1:
                    continue
                start = m.start()
                prev = (text[:start].rstrip() or " ")[-1]
                mid_sentence = prev not in SENTENCE_BOUNDARY
                if not mid_sentence and len(words) == 1:
                    # A single capitalised word at the start of a sentence is
                    # too weak a signal; require at least one mid-sentence use.
                    counts[name]["_initial"] = counts[name].get("_initial", 0) + 1
                    continue
                counts[name]["count"] += 1
                if len(counts[name]["contexts"]) < 3:
                    lo = max(0, start - 40)
                    hi = min(len(text), m.end() + 40)
                    counts[name]["contexts"].append("…" + text[lo:hi].replace("\n", " ") + "…")
    return {k: v for k, v in counts.items() if v["count"] > 0}


def main() -> None:
    ap = argparse.ArgumentParser(description="Propose glossary entries from the source and analyses.")
    ap.add_argument("--book", type=Path, required=True)
    ap.add_argument("--run-dir", type=Path, required=True)
    ap.add_argument("--glossary", type=Path, required=True)
    ap.add_argument("--out", type=Path, required=True)
    ap.add_argument("--chapters", help="comma-separated chapter ids (default: all)")
    args = ap.parse_args()

    book = _load(args.book)
    glossary = _load(args.glossary)
    only = {c.strip() for c in args.chapters.split(",")} if args.chapters else None
    known = _glossary_keys(glossary)
    analyst = _analyst_terms(args.run_dir)
    names = _source_names(book, only)

    lines = ["# Deal Hunter glossary candidates", "", "未自动写入 glossary.json；请人工批准。", ""]
    lines.append("## 分析器提议的术语 (proposed_terms)")
    lines.append("")
    lines.append("| en | zh | 次数 | 说明 |")
    lines.append("|---|---|---|---|")
    for en, entry in sorted(analyst.items()):
        if en in known:
            continue
        why = " / ".join(entry["why"])[:120]
        lines.append(f"| {entry['en']} | {entry.get('zh') or ''} | {entry['count']} | {why} |")
    lines.append("")
    lines.append("## 源码中的专名候选 (mid-sentence capitalised)")
    lines.append("")
    lines.append("| name | 次数 | 上下文 |")
    lines.append("|---|---|---|")
    for name, entry in sorted(names.items(), key=lambda kv: (-kv[1]["count"], kv[0])):
        low = name.lower()
        if low in known or low in LATIN_ALLOW:
            continue
        ctx = " <br> ".join(entry["contexts"][:2])
        lines.append(f"| {name} | {entry['count']} | {ctx} |")
    lines.append("")

    args.out.parent.mkdir(parents=True, exist_ok=True)
    args.out.write_text("\n".join(lines) + "\n", encoding="utf-8")
    print(f"wrote {args.out}")
    print(f"  analyst proposals: {len(analyst)}  source candidates: {len(names)}")


if __name__ == "__main__":
    main()