# Copyright 2026 Umbra Tools (umbratools.com) # SPDX-License-Identifier: Apache-2.0 """Collect proper-name and term candidates for the glossary. Two sources are merged: 1. Terms proposed by the scene analyst (``proposed_terms`` in every ``*.analysis.json``). 2. Capitalised words in the English source that appear mid-sentence, which is a decent signal for a name but filters out sentence-initial function words. Nothing is written into the glossary automatically. The output is a proposal list for the operator to accept, edit or reject, because an unapproved transliteration is worse than an English name left in place. """ from __future__ import annotations import argparse import json import re import sys from collections import defaultdict from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from pipeline import LATIN_ALLOW # noqa: E402 STOPWORDS = { "a", "about", "after", "again", "against", "all", "also", "am", "an", "and", "any", "are", "aren", "as", "at", "back", "be", "because", "been", "before", "being", "below", "between", "both", "but", "by", "can", "could", "did", "didn", "do", "does", "doesn", "doing", "don", "down", "during", "each", "even", "ever", "every", "few", "for", "from", "get", "gets", "getting", "give", "go", "going", "gone", "good", "got", "had", "has", "have", "having", "he", "her", "here", "hers", "herself", "him", "himself", "his", "how", "however", "i", "if", "in", "into", "is", "isn", "it", "its", "itself", "just", "know", "let", "like", "look", "looking", "made", "make", "many", "may", "maybe", "me", "might", "mine", "more", "most", "much", "must", "my", "myself", "need", "no", "nor", "not", "now", "of", "off", "on", "once", "one", "only", "or", "other", "ought", "our", "ours", "ourselves", "out", "over", "own", "perhaps", "please", "put", "quite", "rather", "really", "right", "said", "same", "say", "saying", "see", "seen", "she", "should", "since", "so", "some", "something", "still", "such", "sure", "take", "than", "thank", "thanks", "that", "the", "their", "theirs", "them", "themselves", "then", "there", "these", "they", "thing", "things", "think", "this", "those", "though", "through", "to", "too", "under", "until", "up", "upon", "us", "very", "was", "wasn", "we", "well", "were", "weren", "what", "when", "where", "whether", "which", "while", "who", "whom", "why", "will", "with", "within", "without", "would", "yes", "yet", "you", "your", "yours", "yourself", "yourselves", "okay", "ok", "hi", "hello", "hey", "oh", "ah", "um", "uh", "hmm", "mr", "mrs", "ms", "dr", "sir", "ma", "am", "captain", "princess", "doctor", "aunt", "uncle", "nanny", "hunter", "everyone", "everything", "someone", "something", "nothing", "anyone", "anything", "people", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven", "twelve", "first", "second", "third", "fourth", "fifth", "next", "last", "inside", "outside", "based", "turning", "checking", "looking", "going", "trying", "waiting", "standing", "sitting", "walking", "talking", "several", "tell", "additionally", "besides", "however", "therefore", "meanwhile", "suddenly", "finally", "actually", "apparently", "obviously", "unfortunately", "luckily", "instead", "as", "at", "if", "is", "it", "its", "be", "by", "do", "does", "did", "for", "from", "had", "has", "he", "her", "him", "his", "how", "into", "me", "my", "of", "on", "or", "our", "she", "so", "than", "that", "the", "their", "them", "then", "there", "these", "they", "this", "those", "to", "up", "us", "was", "we", "were", "what", "when", "where", "which", "who", "why", "will", "with", "would", "you", "your", } SENTENCE_BOUNDARY = set('.!?…"“”\n') def _load(path: Path) -> dict: return json.loads(path.read_text(encoding="utf-8")) def _glossary_keys(glossary: dict) -> set[str]: keys: set[str] = set() for section in ("characters", "roles", "places", "terms"): for key in glossary.get(section, {}) or {}: keys.add(key.lower()) for extra in (glossary.get("title") or {}).values(): if isinstance(extra, str): keys.add(extra.lower()) return keys def _analyst_terms(run_dir: Path) -> dict[str, dict]: out: dict[str, dict] = {} for path in sorted(run_dir.glob("*/*.analysis.json")): try: data = _load(path) except (OSError, json.JSONDecodeError): continue for term in data.get("proposed_terms", []) or []: if not isinstance(term, dict): continue en = str(term.get("en") or "").strip().lower() if not en: continue entry = out.setdefault(en, {"en": term.get("en"), "zh": term.get("zh"), "why": [], "count": 0}) entry["count"] += 1 why = term.get("why") if why and why not in entry["why"]: entry["why"].append(why) if term.get("zh") and not entry.get("zh"): entry["zh"] = term.get("zh") return out def _source_names(book: dict, only: set[str] | None) -> dict[str, dict]: counts: dict[str, dict] = defaultdict(lambda: {"count": 0, "contexts": []}) pattern = re.compile(r"\b([A-Z][a-zA-Z'’-]+(?:\s+[A-Z][a-zA-Z'’-]+){0,2})\b") for chapter in book["chapters"]: if only and chapter["id"] not in only: continue for p in chapter["paragraphs"]: text = p.get("text") or "" for m in pattern.finditer(text): name = m.group(1).strip() name = re.sub(r"[’']s$", "", name).strip("-'") if not name or "’" in name or "'" in name: continue words = name.split() if all(w.lower() in STOPWORDS for w in words): continue if len(words) == 1 and words[0].lower().endswith("ly"): continue first = words[0].lower() if first in STOPWORDS and len(words) == 1: continue start = m.start() prev = (text[:start].rstrip() or " ")[-1] mid_sentence = prev not in SENTENCE_BOUNDARY if not mid_sentence and len(words) == 1: # A single capitalised word at the start of a sentence is # too weak a signal; require at least one mid-sentence use. counts[name]["_initial"] = counts[name].get("_initial", 0) + 1 continue counts[name]["count"] += 1 if len(counts[name]["contexts"]) < 3: lo = max(0, start - 40) hi = min(len(text), m.end() + 40) counts[name]["contexts"].append("…" + text[lo:hi].replace("\n", " ") + "…") return {k: v for k, v in counts.items() if v["count"] > 0} def main() -> None: ap = argparse.ArgumentParser(description="Propose glossary entries from the source and analyses.") ap.add_argument("--book", type=Path, required=True) ap.add_argument("--run-dir", type=Path, required=True) ap.add_argument("--glossary", type=Path, required=True) ap.add_argument("--out", type=Path, required=True) ap.add_argument("--chapters", help="comma-separated chapter ids (default: all)") args = ap.parse_args() book = _load(args.book) glossary = _load(args.glossary) only = {c.strip() for c in args.chapters.split(",")} if args.chapters else None known = _glossary_keys(glossary) analyst = _analyst_terms(args.run_dir) names = _source_names(book, only) lines = ["# Deal Hunter glossary candidates", "", "未自动写入 glossary.json；请人工批准。", ""] lines.append("## 分析器提议的术语 (proposed_terms)") lines.append("") lines.append("| en | zh | 次数 | 说明 |") lines.append("|---|---|---|---|") for en, entry in sorted(analyst.items()): if en in known: continue why = " / ".join(entry["why"])[:120] lines.append(f"| {entry['en']} | {entry.get('zh') or ''} | {entry['count']} | {why} |") lines.append("") lines.append("## 源码中的专名候选 (mid-sentence capitalised)") lines.append("") lines.append("| name | 次数 | 上下文 |") lines.append("|---|---|---|") for name, entry in sorted(names.items(), key=lambda kv: (-kv[1]["count"], kv[0])): low = name.lower() if low in known or low in LATIN_ALLOW: continue ctx = "
".join(entry["contexts"][:2]) lines.append(f"| {name} | {entry['count']} | {ctx} |") lines.append("") args.out.parent.mkdir(parents=True, exist_ok=True) args.out.write_text("\n".join(lines) + "\n", encoding="utf-8") print(f"wrote {args.out}") print(f" analyst proposals: {len(analyst)} source candidates: {len(names)}") if __name__ == "__main__": main()