303 lines
11 KiB
Python
303 lines
11 KiB
Python
# Copyright 2026 Umbra Tools (umbratools.com)
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
from __future__ import annotations
|
|
|
|
from umbraaudio.analysis.clarity import (
|
|
assess_utterance_clarity,
|
|
evidence_kind_for,
|
|
extract_mechanical_evidence,
|
|
)
|
|
|
|
ALIAS_MAP = {
|
|
"hunter": "Deal Hunter",
|
|
"deal hunter": "Deal Hunter",
|
|
"kainda": "Kainda Walters Foster Murphy",
|
|
"kainda walters foster murphy": "Kainda Walters Foster Murphy",
|
|
"princess kainda": "Kainda Walters Foster Murphy",
|
|
"kabir": "Kabir",
|
|
"captain": "Captain Weston",
|
|
"captain weston": "Captain Weston",
|
|
"chief of the guard": "Chief of the Guard",
|
|
"narrator": "Narrator",
|
|
}
|
|
|
|
PROFILES = [
|
|
{
|
|
"primary_name": "Deal Hunter",
|
|
"aliases": ["Hunter"],
|
|
"demographics": {"gender": "Non-binary"},
|
|
},
|
|
{
|
|
"primary_name": "Kainda Walters Foster Murphy",
|
|
"aliases": ["Kainda", "Princess Kainda"],
|
|
"demographics": {"gender": "Female"},
|
|
},
|
|
{
|
|
"primary_name": "Kabir",
|
|
"aliases": [],
|
|
"demographics": {"gender": "Male"},
|
|
},
|
|
{
|
|
"primary_name": "Captain Weston",
|
|
"aliases": ["Captain"],
|
|
"demographics": {"gender": "Male"},
|
|
},
|
|
{
|
|
"primary_name": "Chief of the Guard",
|
|
"aliases": [],
|
|
"demographics": {"gender": "Male"},
|
|
},
|
|
]
|
|
|
|
|
|
def _utt(idx: int, text: str, speaker: str, type_: str = "narration", **kw) -> dict:
|
|
row = {
|
|
"id": idx + 1000,
|
|
"utterance_index": idx,
|
|
"text": text,
|
|
"type": type_,
|
|
"speaker_name": speaker,
|
|
}
|
|
row.update(kw)
|
|
return row
|
|
|
|
|
|
class TestEvidenceKind:
|
|
def test_name_kind(self) -> None:
|
|
assert evidence_kind_for({"tag_kind": "name", "addressees": []}) == "name"
|
|
|
|
def test_role_kind(self) -> None:
|
|
assert evidence_kind_for({"tag_kind": "role", "addressees": []}) == "role"
|
|
|
|
def test_pronoun_kind(self) -> None:
|
|
assert evidence_kind_for({"tag_kind": "pronoun", "addressees": []}) == "pronoun"
|
|
|
|
def test_address_only_kind(self) -> None:
|
|
assert (
|
|
evidence_kind_for({"tag_kind": None, "addressees": ["Captain"]})
|
|
== "address_only"
|
|
)
|
|
|
|
def test_none_kind(self) -> None:
|
|
assert evidence_kind_for({"tag_kind": None, "addressees": []}) == "none"
|
|
|
|
|
|
class TestExtractMechanicalEvidence:
|
|
def test_name_tag(self) -> None:
|
|
utts = [
|
|
_utt(0, "Kainda says,", "narrator"),
|
|
_utt(1, "We are ready.", "Kainda Walters Foster Murphy", "dialogue"),
|
|
]
|
|
ev = extract_mechanical_evidence(utts, ALIAS_MAP)
|
|
assert ev[1]["tag_kind"] == "name"
|
|
assert ev[1]["tag"] == "Kainda Walters Foster Murphy"
|
|
|
|
def test_role_tag(self) -> None:
|
|
utts = [
|
|
_utt(0, "The lead guard said,", "narrator"),
|
|
_utt(1, "Stop there.", "Chief of the Guard", "dialogue"),
|
|
]
|
|
ev = extract_mechanical_evidence(utts, ALIAS_MAP)
|
|
assert ev[1]["tag_kind"] == "role"
|
|
|
|
def test_pronoun_tag(self) -> None:
|
|
utts = [
|
|
_utt(0, "He said,", "narrator"),
|
|
_utt(1, "I will go.", "Kabir", "dialogue"),
|
|
]
|
|
ev = extract_mechanical_evidence(utts, ALIAS_MAP)
|
|
assert ev[1]["tag_kind"] == "pronoun"
|
|
assert ev[1]["tag"] == "he"
|
|
|
|
def test_no_evidence(self) -> None:
|
|
utts = [
|
|
_utt(0, "The ship hummed quietly.", "narrator"),
|
|
_utt(1, "But where is the coat?", "Kabir", "dialogue"),
|
|
]
|
|
ev = extract_mechanical_evidence(utts, ALIAS_MAP)
|
|
assert ev[1]["tag_kind"] is None
|
|
assert ev[1]["addressees"] == []
|
|
|
|
def test_addressee_only(self) -> None:
|
|
utts = [
|
|
_utt(0, "Captain, we are taking fire.", "Kabir", "dialogue"),
|
|
]
|
|
ev = extract_mechanical_evidence(utts, ALIAS_MAP)
|
|
assert ev[0]["tag_kind"] is None
|
|
assert "Captain Weston" in ev[0]["addressees"]
|
|
|
|
|
|
class TestAssessUtteranceClarity:
|
|
def test_high_confidence_name_tag_auto_accept(self) -> None:
|
|
utts = [
|
|
_utt(0, "Kainda says,", "narrator"),
|
|
_utt(
|
|
1,
|
|
"We are ready.",
|
|
"Kainda Walters Foster Murphy",
|
|
"dialogue",
|
|
attribution_confidence=0.95,
|
|
),
|
|
]
|
|
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
|
|
dialogue = out[1]
|
|
assert dialogue["attribution_evidence_kind"] == "name"
|
|
assert dialogue["needs_review"] == 0
|
|
assert dialogue["review_reason"] == ""
|
|
|
|
def test_speech_tag_mismatch_flagged(self) -> None:
|
|
utts = [
|
|
_utt(0, "Kainda says,", "narrator"),
|
|
_utt(1, "We are ready.", "Kabir", "dialogue", attribution_confidence=0.95),
|
|
]
|
|
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
|
|
assert out[1]["needs_review"] == 1
|
|
assert "tag" in out[1]["review_reason"].lower()
|
|
|
|
def test_pronoun_unique_candidate_correct(self) -> None:
|
|
utts = [
|
|
_utt(0, "Kabir says,", "Kabir"),
|
|
_utt(1, "I am going.", "Kabir", "dialogue", attribution_confidence=0.9),
|
|
_utt(2, "He said,", "narrator"),
|
|
_utt(3, "I will return soon.", "Kabir", "dialogue", attribution_confidence=0.9),
|
|
]
|
|
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
|
|
assert out[3]["attribution_evidence_kind"] == "pronoun"
|
|
assert out[3]["needs_review"] == 0
|
|
|
|
def test_pronoun_no_candidate_flagged(self) -> None:
|
|
utts = [
|
|
_utt(0, "He said,", "narrator"),
|
|
_utt(1, "I will return soon.", "Kabir", "dialogue", attribution_confidence=0.9),
|
|
]
|
|
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
|
|
assert out[1]["needs_review"] == 1
|
|
assert "0 candidate" in out[1]["review_reason"]
|
|
|
|
def test_pronoun_ambiguous_two_candidates_flagged(self) -> None:
|
|
# Two males present (Kabir + Captain Weston) → pronoun referent ambiguous.
|
|
utts = [
|
|
_utt(0, "Kabir says,", "Kabir"),
|
|
_utt(1, "I am going.", "Kabir", "dialogue", attribution_confidence=0.9),
|
|
_utt(2, "Captain Weston says,", "Captain Weston"),
|
|
_utt(3, "Stand down.", "Captain Weston", "dialogue", attribution_confidence=0.9),
|
|
_utt(4, "He said,", "narrator"),
|
|
_utt(5, "Enough of this.", "Kabir", "dialogue", attribution_confidence=0.9),
|
|
]
|
|
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
|
|
assert out[5]["needs_review"] == 1
|
|
assert "2 candidate" in out[5]["review_reason"]
|
|
|
|
def test_role_tag_no_profile_flagged(self) -> None:
|
|
utts = [
|
|
_utt(0, "The lead guard said,", "narrator"),
|
|
# "Lead Guard" is not a profile entry → inventoried role without profile.
|
|
_utt(1, "Stop there.", "Lead Guard", "dialogue", attribution_confidence=0.9),
|
|
]
|
|
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
|
|
assert out[1]["needs_review"] == 1
|
|
assert "no profile" in out[1]["review_reason"]
|
|
|
|
def test_role_tag_with_profile_auto_accept(self) -> None:
|
|
utts = [
|
|
_utt(0, "The lead guard said,", "narrator"),
|
|
_utt(1, "Stop there.", "Chief of the Guard", "dialogue", attribution_confidence=0.95),
|
|
]
|
|
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
|
|
assert out[1]["needs_review"] == 0
|
|
|
|
def test_no_evidence_low_confidence_flagged(self) -> None:
|
|
utts = [
|
|
_utt(0, "The ship hummed quietly.", "narrator"),
|
|
_utt(1, "But where is the coat?", "Kabir", "dialogue", attribution_confidence=0.65),
|
|
]
|
|
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
|
|
assert out[1]["attribution_evidence_kind"] == "none"
|
|
assert out[1]["needs_review"] == 1
|
|
assert "no narration evidence" in out[1]["review_reason"]
|
|
|
|
def test_no_evidence_high_confidence_accepted(self) -> None:
|
|
utts = [
|
|
_utt(0, "The ship hummed quietly.", "narrator"),
|
|
_utt(1, "But where is the coat?", "Kabir", "dialogue", attribution_confidence=0.97),
|
|
]
|
|
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
|
|
assert out[1]["needs_review"] == 0
|
|
|
|
def test_narrator_dialogue_flagged(self) -> None:
|
|
utts = [
|
|
_utt(0, "Some narration.", "narrator"),
|
|
_utt(1, "I am talking.", "narrator", "dialogue", attribution_confidence=1.0),
|
|
]
|
|
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
|
|
assert out[1]["needs_review"] == 1
|
|
assert "narrator" in out[1]["review_reason"]
|
|
|
|
def test_uncertain_persisted_and_flagged_when_subhigh(self) -> None:
|
|
utts = [
|
|
_utt(0, "The ship hummed quietly.", "narrator"),
|
|
_utt(
|
|
1,
|
|
"But where is the coat?",
|
|
"Kabir",
|
|
"dialogue",
|
|
attribution_confidence=0.8,
|
|
attribution_uncertain=True,
|
|
),
|
|
]
|
|
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
|
|
assert out[1]["attribution_uncertain"] == 1
|
|
assert out[1]["needs_review"] == 1
|
|
assert "uncertain" in out[1]["review_reason"]
|
|
|
|
def test_uncertain_no_longer_flagged_after_verify_high(self) -> None:
|
|
utts = [
|
|
_utt(0, "The ship hummed quietly.", "narrator"),
|
|
_utt(
|
|
1,
|
|
"But where is the coat?",
|
|
"Kabir",
|
|
"dialogue",
|
|
attribution_confidence=0.95,
|
|
attribution_uncertain=True,
|
|
),
|
|
]
|
|
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
|
|
assert out[1]["attribution_uncertain"] == 1
|
|
assert out[1]["needs_review"] == 0
|
|
|
|
def test_low_confidence_flagged(self) -> None:
|
|
utts = [
|
|
_utt(0, "Kainda says,", "narrator"),
|
|
_utt(
|
|
1,
|
|
"We are ready.",
|
|
"Kainda Walters Foster Murphy",
|
|
"dialogue",
|
|
attribution_confidence=0.6,
|
|
),
|
|
]
|
|
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
|
|
assert out[1]["needs_review"] == 1
|
|
assert "low confidence" in out[1]["review_reason"]
|
|
|
|
def test_originals_not_mutated(self) -> None:
|
|
utts = [
|
|
_utt(0, "Kainda says,", "narrator"),
|
|
_utt(1, "We are ready.", "Kabir", "dialogue", attribution_confidence=0.95),
|
|
]
|
|
assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
|
|
assert "needs_review" not in utts[1]
|
|
assert "attribution_evidence_kind" not in utts[1]
|
|
|
|
def test_non_dialogue_pass_through(self) -> None:
|
|
utts = [
|
|
_utt(0, "Kainda says,", "narrator"),
|
|
_utt(1, "We are ready.", "Kainda Walters Foster Murphy", "dialogue"),
|
|
_utt(2, "<<<>>>", "scene_break", "scene_break"),
|
|
]
|
|
out = assess_utterance_clarity(utts, 6, ALIAS_MAP, PROFILES)
|
|
assert "needs_review" not in out[2]
|
|
assert out[2]["type"] == "scene_break"
|