UmbraAudio/umbraaudio/languages.py

# Copyright 2026 Umbra Tools (umbratools.com)
# SPDX-License-Identifier: Apache-2.0

"""The set of languages this system knows about, in one place.

Two groups, and the split is a hard capability boundary, not a preference:

* **Voiceable** - the ten languages Qwen3-TTS can synthesise. The model's own
  ``codec_language_id`` keys are the authority for the name string it expects.
* **Gap** - languages we can tag and store but cannot speak. Recording the tag
  is useful (it excludes the line from translation and explains why a hole
  exists); emitting audio for it would be wrong, so the render is refused.

Spec 31 section 5. ``/v1/tts`` validation, the export filter and the review UI
all read this module, so they cannot disagree about what is supported.
"""

from __future__ import annotations

from dataclasses import dataclass

# Where a source-language tag came from. Kept as plain strings rather than an
# enum so the value round-trips through JSON and SQLite without conversion.
ORIGIN_UNKNOWN = "unknown"
ORIGIN_UMBRATRANS = "umbratrans"
ORIGIN_OPERATOR = "operator"
ORIGINS: tuple[str, ...] = (ORIGIN_UNKNOWN, ORIGIN_UMBRATRANS, ORIGIN_OPERATOR)


# The script a language is written in. A delivery that claims one language
# and arrives written in another is not a translation of that language - it is
# the exact "declared value disagrees with the measured value" defect class
# that the release work already had to fix for audio padding.
SCRIPT_LATIN = "latin"
SCRIPT_CYRILLIC = "cyrillic"
SCRIPT_HAN = "han"
SCRIPT_KANA = "kana"
SCRIPT_HANGUL = "hangul"
SCRIPT_ARABIC = "arabic"
SCRIPT_DEVANAGARI = "devanagari"

SCRIPT_NAMES: dict[str, str] = {
    SCRIPT_LATIN: "Latin script",
    SCRIPT_CYRILLIC: "Cyrillic script",
    SCRIPT_HAN: "Han characters",
    SCRIPT_KANA: "Japanese kana",
    SCRIPT_HANGUL: "Hangul",
    SCRIPT_ARABIC: "Arabic script",
    SCRIPT_DEVANAGARI: "Devanagari",
}

# The Unicode blocks that count as each script. Latin is split so that the
# accented forms are counted with it rather than falling through.
_SCRIPT_RANGES: tuple[tuple[str, int, int], ...] = (
    (SCRIPT_LATIN, 0x0041, 0x005A),
    (SCRIPT_LATIN, 0x0061, 0x007A),
    (SCRIPT_LATIN, 0x00C0, 0x024F),
    (SCRIPT_CYRILLIC, 0x0400, 0x04FF),
    (SCRIPT_HAN, 0x3400, 0x4DBF),
    (SCRIPT_HAN, 0x4E00, 0x9FFF),
    (SCRIPT_HAN, 0xF900, 0xFAFF),
    (SCRIPT_KANA, 0x3040, 0x309F),
    (SCRIPT_KANA, 0x30A0, 0x30FF),
    (SCRIPT_KANA, 0x31F0, 0x31FF),
    (SCRIPT_HANGUL, 0x1100, 0x11FF),
    (SCRIPT_HANGUL, 0x3130, 0x318F),
    (SCRIPT_HANGUL, 0xAC00, 0xD7AF),
    (SCRIPT_ARABIC, 0x0600, 0x06FF),
    (SCRIPT_ARABIC, 0x0750, 0x077F),
    (SCRIPT_DEVANAGARI, 0x0900, 0x097F),
)


@dataclass(frozen=True)
class Language:
    """One language. ``tts_name`` is None when it cannot be voiced."""

    code: str
    name: str
    tts_name: str | None = None
    # The scripts this language is normally written in. A delivery written
    # mostly in something else is a defect, not a translation.
    scripts: tuple[str, ...] = (SCRIPT_LATIN,)
    # A script that has to be present for the text to be this language at all:
    # Japanese without kana is Chinese, and Korean without Hangul is Chinese.
    required_script: str | None = None
    # Scripts that would mean the delivery is a different language. Chinese
    # text carrying kana is a Japanese translation filed under the wrong lane.
    forbidden_scripts: tuple[str, ...] = ()


# The ten Qwen3-TTS languages. ``tts_name`` is copied from the model's own
# codec_language_id keys via scripts/render_lang_test.py, which is the working
# reference - do not retype these by hand.
QWEN_LANGUAGES: tuple[Language, ...] = (
    Language("en", "English", "english"),
    Language(
        "zh",
        "Chinese",
        "chinese",
        scripts=(SCRIPT_HAN,),
        forbidden_scripts=(SCRIPT_KANA, SCRIPT_HANGUL),
    ),
    Language(
        "ja",
        "Japanese",
        "japanese",
        scripts=(SCRIPT_HAN, SCRIPT_KANA),
        required_script=SCRIPT_KANA,
    ),
    Language(
        "ko",
        "Korean",
        "korean",
        scripts=(SCRIPT_HANGUL,),
        required_script=SCRIPT_HANGUL,
    ),
    Language("de", "German", "german"),
    Language("es", "Spanish", "spanish"),
    Language("fr", "French", "french"),
    Language("ru", "Russian", "russian", scripts=(SCRIPT_CYRILLIC,)),
    Language("it", "Italian", "italian"),
    Language("pt", "Portuguese", "portuguese"),
)

# Taggable, storable, and deliberately not speakable (spec 31 section 4.3).
GAP_LANGUAGES: tuple[Language, ...] = (
    Language("nl", "Dutch"),
    Language("no", "Norwegian"),
    Language("pl", "Polish"),
    Language("ar", "Arabic", None, scripts=(SCRIPT_ARABIC,)),
    Language("hi", "Hindi", None, scripts=(SCRIPT_DEVANAGARI,)),
)

ALL_LANGUAGES: tuple[Language, ...] = QWEN_LANGUAGES + GAP_LANGUAGES

_BY_CODE: dict[str, Language] = {lang.code: lang for lang in ALL_LANGUAGES}

# Accepts a code ('DE'), a name ('German') or the model string ('german').
_BY_ALIAS: dict[str, Language] = {}
for _lang in ALL_LANGUAGES:
    _BY_ALIAS[_lang.code] = _lang
    _BY_ALIAS[_lang.name.casefold()] = _lang
    if _lang.tts_name is not None:
        _BY_ALIAS[_lang.tts_name] = _lang


def by_code(value: str | None) -> Language | None:
    """Look up by code, display name or model string. None when unknown."""
    if not value:
        return None
    return _BY_ALIAS.get(str(value).strip().casefold())


def normalize(value: str | None) -> str | None:
    """Return the canonical lowercase code for a code or name, else None.

    ``None`` and empty input return ``None``. An unrecognised value also
    returns ``None`` rather than raising: a language we do not know is a
    finding to record, not a crash (spec 31 section 8).
    """
    lang = by_code(value)
    return lang.code if lang is not None else None


def is_known(value: str | None) -> bool:
    """True when we recognise the language at all (voiceable or gap)."""
    return by_code(value) is not None


def is_voiceable(value: str | None) -> bool:
    """True when Qwen3-TTS can synthesise this language."""
    lang = by_code(value)
    return lang is not None and lang.tts_name is not None


def tts_name(value: str | None) -> str | None:
    """The string to hand the TTS server, or None when not voiceable."""
    lang = by_code(value)
    return lang.tts_name if lang is not None else None


def script_counts(text: str) -> dict[str, int]:
    """How many letters of *text* belong to each script.

    Only letters count. Punctuation, digits and whitespace say nothing about
    which language a passage is written in, and a name kept in English inside
    a Chinese paragraph must not pull the count towards Latin.
    """
    counts: dict[str, int] = {}
    for ch in text:
        point = ord(ch)
        for script, low, high in _SCRIPT_RANGES:
            if low <= point <= high:
                counts[script] = counts.get(script, 0) + 1
                break
    return counts


def script_mismatch(code: str | None, text: str) -> str | None:
    """Why *text* cannot be a translation into *code*, or None when it can.

    A delivery that claims Spanish and arrives written in Chinese is not a
    Spanish translation. It is the "declared value disagrees with the measured
    value" defect class, and it has to be caught at the boundary: once the
    rows are in the database, the review lane looks populated and a reviewer
    approves the wrong thing.

    The check is deliberately about the *script*, not the language. It cannot
    tell Spanish from Portuguese, but it can tell Chinese from Spanish, which
    is the mistake that actually happens - a pipeline configured for one
    language that silently produces another.
    """
    language = by_code(code)
    if language is None:
        # An unknown language has no script expectation to check against.
        return None

    counts = script_counts(text)
    total = sum(counts.values())
    if total == 0:
        return None

    for forbidden in language.forbidden_scripts:
        if counts.get(forbidden, 0) / total >= 0.10:
            return f"the text is written in {SCRIPT_NAMES[forbidden]}, so it is not {language.name}"

    if language.required_script and not counts.get(language.required_script):
        return (
            f"the text has no {SCRIPT_NAMES[language.required_script]}, "
            f"so it is not {language.name}"
        )

    acceptable = sum(counts.get(script, 0) for script in language.scripts)
    if acceptable / total < 0.5:
        dominant = max(counts, key=lambda key: counts[key])
        return f"the text is mostly {SCRIPT_NAMES[dominant]}, so it is not {language.name}"
    return None


def language_name(value: str | None) -> str:
    """Display name for a code. Falls back to the raw value, then 'unknown'."""
    lang = by_code(value)
    if lang is not None:
        return lang.name
    return str(value) if value else "unknown"


def refusal_reason(value: str | None) -> str | None:
    """Why this language cannot be voiced, or None when it can.

    The wording is used verbatim in the review UI, so a refused render always
    says *why* rather than leaving an unexplained gap (spec 31 section 11).
    """
    if not value:
        return None
    lang = by_code(value)
    if lang is None:
        return f"unrecognised language {value!r} - no voice available"
    if lang.tts_name is None:
        return f"no Qwen3-TTS voice for {lang.code!r} ({lang.name})"
    return None