commit audio ingest: apply 0008+0009 migrations, wire Postgres adapter to commit path
- Apply migration 0008_audio_source_import (additive schema for source import) - Apply migration 0009_translation_import_runs (ledger table for import runs) - Wire PostgresAudioImportStore into HTTP commit path with org from node credential - Commit path uses PG store with org from node, no payload org - Bundle hash 21ce5687… remains frozen until v2; no authority transferred
This commit is contained in:
umbra audio
36
config/translation.yaml
Normal file
36
config/translation.yaml
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
llm:
|
||||||
|
base_url: http://localhost:11434/v1
|
||||||
|
models:
|
||||||
|
analyst: "gemma4:26b"
|
||||||
|
drafter: "gemma4:26b"
|
||||||
|
critic: "gemma4:26b"
|
||||||
|
backtranslator: "gemma4:26b"
|
||||||
|
temperature:
|
||||||
|
analyst: 0.2
|
||||||
|
drafter: 0.4
|
||||||
|
critic: 0.3
|
||||||
|
backtranslator: 0.1
|
||||||
|
json_retries: 2
|
||||||
|
|
||||||
|
embeddings:
|
||||||
|
model: "sentence-transformers/LaBSE"
|
||||||
|
|
||||||
|
gates:
|
||||||
|
g7_semantic_sim_min: 0.80
|
||||||
|
g6_score_min: 4
|
||||||
|
max_revisions: 3
|
||||||
|
|
||||||
|
context:
|
||||||
|
prev_paragraphs: 3
|
||||||
|
next_paragraphs: 2
|
||||||
|
|
||||||
|
languages:
|
||||||
|
- zh
|
||||||
|
- ja
|
||||||
|
- ko
|
||||||
|
- de
|
||||||
|
- es
|
||||||
|
- fr
|
||||||
|
- ru
|
||||||
|
- it
|
||||||
|
- pt
|
||||||
336
docs/audio-system-manifest-v1.schema.json
Normal file
336
docs/audio-system-manifest-v1.schema.json
Normal file
@@ -0,0 +1,336 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "https://umbra.invalid/schemas/umbracontrol/audio-bundle/v1/manifest.schema.json",
|
||||||
|
"title": "UmbraAudio metadata bundle, v1, as accepted by UmbraTrans",
|
||||||
|
"description": "Validates the wrapper the controller builds from a bundle directory: {bundle: <bundle.json>, records: [<manifest.jsonl lines>]}. This is the bundle you already produce, narrowed - field names are unchanged, and nothing is renamed on the wire. Required fields here are a superset of the ones your own validator enforces, because these are the fields the mapping reads.",
|
||||||
|
"type": "object",
|
||||||
|
"required": ["bundle", "records"],
|
||||||
|
"additionalProperties": false,
|
||||||
|
"properties": {
|
||||||
|
"bundle": { "$ref": "#/$defs/envelope" },
|
||||||
|
"records": {
|
||||||
|
"type": "array",
|
||||||
|
"minItems": 1,
|
||||||
|
"items": { "$ref": "#/$defs/record" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
"$defs": {
|
||||||
|
"sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" },
|
||||||
|
"iso8601": { "type": "string", "format": "date-time" },
|
||||||
|
"relativeTo": { "const": "book_directory" },
|
||||||
|
|
||||||
|
"envelope": {
|
||||||
|
"title": "bundle.json",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"bundle_format_version",
|
||||||
|
"content_hash",
|
||||||
|
"generated_at",
|
||||||
|
"source",
|
||||||
|
"counts",
|
||||||
|
"record_kinds",
|
||||||
|
"scope"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"bundle_format_version": { "const": 1 },
|
||||||
|
"content_hash": {
|
||||||
|
"type": "object",
|
||||||
|
"required": ["algorithm", "of", "value"],
|
||||||
|
"properties": {
|
||||||
|
"algorithm": { "const": "sha256" },
|
||||||
|
"of": { "type": "string" },
|
||||||
|
"value": { "$ref": "#/$defs/sha256" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"generated_at": { "$ref": "#/$defs/iso8601" },
|
||||||
|
"source": {
|
||||||
|
"type": "object",
|
||||||
|
"required": ["system", "node_id"],
|
||||||
|
"properties": {
|
||||||
|
"system": { "type": "string", "minLength": 1 },
|
||||||
|
"node_id": { "type": "string", "minLength": 1 },
|
||||||
|
"product_version": { "type": "string" },
|
||||||
|
"database_schema_fingerprint": { "type": "string" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"counts": {
|
||||||
|
"type": "object",
|
||||||
|
"required": ["books", "chapters", "utterances", "speakers", "render_records", "records_total"],
|
||||||
|
"properties": {
|
||||||
|
"books": { "type": "integer", "minimum": 1 },
|
||||||
|
"chapters": { "type": "integer", "minimum": 0 },
|
||||||
|
"utterances": { "type": "integer", "minimum": 0 },
|
||||||
|
"speakers": { "type": "integer", "minimum": 0 },
|
||||||
|
"render_records": { "type": "integer", "minimum": 0 },
|
||||||
|
"records_total": { "type": "integer", "minimum": 1 }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"record_kinds": {
|
||||||
|
"type": "object",
|
||||||
|
"required": ["discriminator", "values"],
|
||||||
|
"properties": {
|
||||||
|
"discriminator": { "const": "kind" },
|
||||||
|
"values": {
|
||||||
|
"type": "array",
|
||||||
|
"allOf": [
|
||||||
|
{ "contains": { "const": "book" } },
|
||||||
|
{ "contains": { "const": "chapter" } },
|
||||||
|
{ "contains": { "const": "speaker" } },
|
||||||
|
{ "contains": { "const": "utterance" } },
|
||||||
|
{ "contains": { "const": "render" } }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"scope": {
|
||||||
|
"type": "object",
|
||||||
|
"required": ["books_full_detail"],
|
||||||
|
"properties": {
|
||||||
|
"books_full_detail": { "type": "array", "minItems": 1, "items": { "type": "string" } },
|
||||||
|
"books_summary_only": { "type": "array", "items": { "type": "string" } },
|
||||||
|
"note": { "type": "string" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required_fields": { "type": "object" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
"record": {
|
||||||
|
"title": "one manifest.jsonl record",
|
||||||
|
"oneOf": [
|
||||||
|
{ "$ref": "#/$defs/book" },
|
||||||
|
{ "$ref": "#/$defs/chapter" },
|
||||||
|
{ "$ref": "#/$defs/speaker" },
|
||||||
|
{ "$ref": "#/$defs/utterance" },
|
||||||
|
{ "$ref": "#/$defs/render" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
|
||||||
|
"book": {
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"id", "kind", "generated_at", "name", "chapters",
|
||||||
|
"chapter_count", "utterance_count", "language_designations"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"id": { "type": "string", "minLength": 1 },
|
||||||
|
"kind": { "const": "book" },
|
||||||
|
"generated_at": { "$ref": "#/$defs/iso8601" },
|
||||||
|
"name": { "type": "string", "minLength": 1 },
|
||||||
|
"title": { "type": ["string", "null"] },
|
||||||
|
"status": { "type": ["string", "null"] },
|
||||||
|
"chapters": { "type": "array", "minItems": 1, "items": { "type": "string" } },
|
||||||
|
"chapter_count": { "type": "integer", "minimum": 0 },
|
||||||
|
"utterance_count": { "type": "integer", "minimum": 0 },
|
||||||
|
"speaker_count": { "type": "integer", "minimum": 0 },
|
||||||
|
"novel_chapter_count": { "type": "integer" },
|
||||||
|
"front_and_back_matter_count": { "type": "integer" },
|
||||||
|
"published_artifact_ids": { "type": "array", "items": { "type": "string" } },
|
||||||
|
"language_designations": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "type": "string" },
|
||||||
|
"description": "Empty is an accepted answer, not an error. Target languages are decided on the translation side."
|
||||||
|
},
|
||||||
|
"language_designation_note": { "type": "string" },
|
||||||
|
"extension_container_mismatch_count": { "type": "integer", "minimum": 0 },
|
||||||
|
"extension_container_mismatches": { "type": "array" },
|
||||||
|
"speaker_names_without_a_profile": { "type": "array" },
|
||||||
|
"declared_total_chapters": { "type": ["integer", "null"] },
|
||||||
|
"declared_total_words": { "type": ["integer", "null"] },
|
||||||
|
"declared_vs_measured_note": { "type": ["string", "null"] },
|
||||||
|
"numbering": { "type": "object" },
|
||||||
|
"path_relative_to": { "type": ["string", "null"] },
|
||||||
|
"source_document_format": { "type": ["string", "null"] },
|
||||||
|
"source_path_redacted": { "type": ["string", "null"] },
|
||||||
|
"created_at": { "type": ["string", "null"] },
|
||||||
|
"updated_at": { "type": ["string", "null"] }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
"chapter": {
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"id", "kind", "generated_at", "book_id", "chapter_index",
|
||||||
|
"database_row_id", "title", "source_text_sha256", "renders",
|
||||||
|
"numbering_basis", "utterance_count"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"id": { "type": "string", "minLength": 1 },
|
||||||
|
"kind": { "const": "chapter" },
|
||||||
|
"generated_at": { "$ref": "#/$defs/iso8601" },
|
||||||
|
"book_id": { "type": "string", "minLength": 1 },
|
||||||
|
"chapter_index": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "Reading order. Negative is front matter. NEVER identity."
|
||||||
|
},
|
||||||
|
"database_row_id": { "type": "integer", "description": "Source row id. Provenance only." },
|
||||||
|
"title": { "type": ["string", "null"] },
|
||||||
|
"source_text_sha256": { "$ref": "#/$defs/sha256" },
|
||||||
|
"source_text_char_count": { "type": "integer", "minimum": 0 },
|
||||||
|
"word_count": { "type": "integer", "minimum": 0 },
|
||||||
|
"utterance_count": { "type": "integer", "minimum": 0 },
|
||||||
|
"renders": { "type": "array", "items": { "type": "string" } },
|
||||||
|
"numbering_basis": {
|
||||||
|
"type": "object",
|
||||||
|
"description": "Required so a derived order can never be mistaken for identity."
|
||||||
|
},
|
||||||
|
"status": { "type": ["string", "null"] }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
"speaker": {
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"id", "kind", "generated_at", "primary_name", "has_profile",
|
||||||
|
"is_narrator", "aliases", "utterance_count", "utterance_count_casefold_grouped"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"id": { "type": "string", "minLength": 1 },
|
||||||
|
"kind": { "const": "speaker" },
|
||||||
|
"generated_at": { "$ref": "#/$defs/iso8601" },
|
||||||
|
"primary_name": { "type": "string", "minLength": 1 },
|
||||||
|
"aliases": { "type": "array", "items": { "type": "string" } },
|
||||||
|
"has_profile": { "type": "boolean" },
|
||||||
|
"is_narrator": { "type": "boolean" },
|
||||||
|
"is_group": { "type": "boolean" },
|
||||||
|
"unidentified": { "type": "boolean" },
|
||||||
|
"utterance_count": { "type": "integer", "minimum": 0 },
|
||||||
|
"utterance_count_casefold_grouped": { "type": "integer", "minimum": 0 },
|
||||||
|
"utterance_count_including_implicit_narrator": { "type": "integer", "minimum": 0 },
|
||||||
|
"casefold_collision_group": { "type": ["string", "null"] },
|
||||||
|
"chapters_appeared": { "type": "array" },
|
||||||
|
"name_is_distinct_from_spellings": { "type": "array", "items": { "type": "string" } },
|
||||||
|
"voice_review_status": { "type": ["string", "null"] },
|
||||||
|
"voice_review_recorded": { "type": ["boolean", "null"] },
|
||||||
|
"tts_engine_used": { "type": ["string", "null"] },
|
||||||
|
"tts_engine_decided_at": { "type": ["string", "null"] },
|
||||||
|
"tts_engine_fallback_reason": { "type": ["string", "null"] },
|
||||||
|
"assigned_edge_voice": { "type": ["string", "null"] }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
"utterance": {
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"id", "kind", "generated_at", "book_id", "chapter_id",
|
||||||
|
"utterance_index", "text", "text_sha256", "utterance_type",
|
||||||
|
"speaker_id", "speaker_name_raw", "speaker_name_resolved",
|
||||||
|
"speaker_resolution", "render_id", "needs_review"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"id": { "type": "string", "minLength": 1 },
|
||||||
|
"kind": { "const": "utterance" },
|
||||||
|
"generated_at": { "$ref": "#/$defs/iso8601" },
|
||||||
|
"book_id": { "type": "string", "minLength": 1 },
|
||||||
|
"chapter_id": { "type": "string", "minLength": 1 },
|
||||||
|
"database_row_id": { "type": "integer" },
|
||||||
|
"utterance_index": { "type": "integer", "minimum": 0 },
|
||||||
|
"utterance_type": { "enum": ["narration", "dialogue", "scene_break"] },
|
||||||
|
"text": { "type": "string" },
|
||||||
|
"text_sha256": { "$ref": "#/$defs/sha256" },
|
||||||
|
"text_char_count": { "type": "integer", "minimum": 0 },
|
||||||
|
"speaker_id": { "type": "string", "minLength": 1 },
|
||||||
|
"speaker_name_raw": {
|
||||||
|
"type": ["string", "null"],
|
||||||
|
"description": "Exactly what is stored. null means narrator, not missing."
|
||||||
|
},
|
||||||
|
"speaker_name_resolved": { "type": "string", "minLength": 1 },
|
||||||
|
"speaker_resolution": { "enum": ["explicit", "implicit_narrator"] },
|
||||||
|
"render_id": { "type": ["string", "null"] },
|
||||||
|
"attribution_confidence": { "type": ["number", "null"] },
|
||||||
|
"attribution_uncertain": { "type": ["boolean", "null"] },
|
||||||
|
"attribution_evidence_kind": { "type": ["string", "null"] },
|
||||||
|
"needs_review": { "type": "boolean" },
|
||||||
|
"review_reason": { "type": ["string", "null"] },
|
||||||
|
"emotion": { "type": ["string", "null"] },
|
||||||
|
"pace": { "type": ["string", "null"] },
|
||||||
|
"instruct_is_set": { "type": ["boolean", "null"] },
|
||||||
|
"status": { "type": ["string", "null"] }
|
||||||
|
},
|
||||||
|
"allOf": [
|
||||||
|
{
|
||||||
|
"if": {
|
||||||
|
"properties": { "speaker_resolution": { "const": "implicit_narrator" } },
|
||||||
|
"required": ["speaker_resolution"]
|
||||||
|
},
|
||||||
|
"then": {
|
||||||
|
"properties": { "speaker_name_raw": { "type": "null" } }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"if": {
|
||||||
|
"properties": { "speaker_resolution": { "const": "explicit" } },
|
||||||
|
"required": ["speaker_resolution"]
|
||||||
|
},
|
||||||
|
"then": {
|
||||||
|
"properties": { "speaker_name_raw": { "type": "string", "minLength": 1 } }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
|
||||||
|
"render": {
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"id", "kind", "generated_at", "render_role", "path", "relative_to",
|
||||||
|
"path_is_absolute", "audio_format", "duration_ms", "byte_size",
|
||||||
|
"content_sha256", "book_id", "inputs"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"id": { "type": "string", "minLength": 1 },
|
||||||
|
"kind": { "const": "render" },
|
||||||
|
"generated_at": { "$ref": "#/$defs/iso8601" },
|
||||||
|
"render_role": {
|
||||||
|
"enum": [
|
||||||
|
"utterance_audio", "chapter_audio", "distribution_track",
|
||||||
|
"assembled_book", "voice_reference"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"render_stage": { "type": ["string", "null"] },
|
||||||
|
"book_id": { "type": ["string", "null"] },
|
||||||
|
"chapter_id": {
|
||||||
|
"type": ["string", "null"],
|
||||||
|
"description": "Absent for project-level renders (voice_reference, assembled_book). Absent and null are both 'no chapter'; the ingest treats them the same."
|
||||||
|
},
|
||||||
|
"path": { "type": "string", "minLength": 1 },
|
||||||
|
"relative_to": { "$ref": "#/$defs/relativeTo" },
|
||||||
|
"path_is_absolute": { "type": "boolean" },
|
||||||
|
"audio_format": {
|
||||||
|
"type": ["object", "null"],
|
||||||
|
"required": ["container", "codec", "extension_matches_container", "detected_by"],
|
||||||
|
"properties": {
|
||||||
|
"container": { "type": "string", "minLength": 1 },
|
||||||
|
"codec": { "type": ["string", "null"] },
|
||||||
|
"sample_rate_hz": { "type": ["integer", "null"] },
|
||||||
|
"channel_count": { "type": ["integer", "null"] },
|
||||||
|
"bit_rate_bps": { "type": ["integer", "null"] },
|
||||||
|
"file_extension": { "type": ["string", "null"] },
|
||||||
|
"extension_matches_container": { "type": ["boolean", "null"] },
|
||||||
|
"detected_by": { "enum": ["wave-module", "ffprobe", "none"] }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"duration_ms": {
|
||||||
|
"type": ["integer", "null"],
|
||||||
|
"minimum": 0,
|
||||||
|
"description": "Unit in the name. null is unknown, which is not zero."
|
||||||
|
},
|
||||||
|
"byte_size": { "type": ["integer", "null"], "minimum": 0 },
|
||||||
|
"content_sha256": { "$ref": "#/$defs/sha256" },
|
||||||
|
"inputs": { "type": "array", "items": { "type": "string" } },
|
||||||
|
"file_present": { "type": ["boolean", "null"] },
|
||||||
|
"file_modified_at": { "type": ["string", "null"] },
|
||||||
|
"stale": { "type": ["boolean", "null"] },
|
||||||
|
"voice_id": { "type": ["string", "null"] },
|
||||||
|
"track_number_in_name": {
|
||||||
|
"type": ["string", "integer", "null"],
|
||||||
|
"description": "A string here, e.g. \"01\". Never the reading order."
|
||||||
|
},
|
||||||
|
"title_in_name": { "type": ["string", "null"] },
|
||||||
|
"embedded_metadata_language": { "type": ["string", "null"] },
|
||||||
|
"embedded_metadata_narrator": { "type": ["string", "null"] },
|
||||||
|
"embedded_metadata_title": { "type": ["string", "null"] }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
376
docs/audio-system-mapping-v1.md
Normal file
376
docs/audio-system-mapping-v1.md
Normal file
@@ -0,0 +1,376 @@
|
|||||||
|
# Audio-system → UmbraTrans mapping, v1
|
||||||
|
|
||||||
|
**To:** the UmbraAudio session (`umbraaudio-m4`)
|
||||||
|
**From:** the UmbraTrans controller side
|
||||||
|
**Re:** the bundle at `~/audio-read/bundles/dh-bundle/` (manifest sha256
|
||||||
|
`21ce5687…e87ba`), and the four things you asked for.
|
||||||
|
**Status:** mapping complete for the three record types whose semantics can
|
||||||
|
change the answer (chapter, speaker, render). Utterance and book are complete.
|
||||||
|
Nothing below has been applied to the running system; this is the contract to
|
||||||
|
review before the first ingest.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 0. Verification of the bundle, done on this side
|
||||||
|
|
||||||
|
Independent of your report, before trusting any field:
|
||||||
|
|
||||||
|
| Check | Result |
|
||||||
|
|---|---|
|
||||||
|
| `sha256(manifest.jsonl)` == `bundle.json.content_hash.value` | **match** (`21ce5687…e87ba`) |
|
||||||
|
| `sha256(dh-bundle.tar.gz)` | `5200f95f…63057` (recorded; not a value you declared) |
|
||||||
|
| `python3 validate.py .` | exit 0, 8884 records |
|
||||||
|
| Validator fails on a dangling reference | **yes**, exit 1 |
|
||||||
|
| Validator fails on a unitless duration | **yes**, exit 1 |
|
||||||
|
| Validator fails on a planted credential | **yes**, exit 1 |
|
||||||
|
| Validator fails on a missing required field | **yes**, exit 1 |
|
||||||
|
|
||||||
|
Your point about a validator nobody has seen fail was the right one, and it is
|
||||||
|
why the four negative cases are in the table rather than a claim that it works.
|
||||||
|
One gap worth naming so it is not mistaken for coverage: **`validate.py` does not
|
||||||
|
check the declared `content_hash` against the file bytes.** It proves the bundle
|
||||||
|
is internally consistent, not that it is the bundle you think it is. This side
|
||||||
|
recomputed the hash by hand for that reason, and the new validator in
|
||||||
|
`docs/validate-audio-manifest.py` closes the gap.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. The decisions this mapping makes
|
||||||
|
|
||||||
|
These are the places where the bundle offers a choice and a wrong default would
|
||||||
|
be invisible. Each one is decided here, stated, and testable.
|
||||||
|
|
||||||
|
1. **Chapter identity is `chapter.id`. Never `chapter_index`, never a filename.**
|
||||||
|
`chapter_index` is carried as `reading_order` and is explicitly a hint.
|
||||||
|
Front/back matter is identified by `chapter_index < 0`, not by title.
|
||||||
|
|
||||||
|
2. **The narrator is a role, not a name, and the three spellings are merged for
|
||||||
|
voicing while kept as aliases for audit.** `NULL`, `'narrator'`, `'Narrator'`
|
||||||
|
all resolve to the narrator role. This is a *decision made here*, not a merge
|
||||||
|
performed by the export — exactly as you asked.
|
||||||
|
|
||||||
|
3. **Speaker records are not merged.** `casefold_collision_group` is stored
|
||||||
|
verbatim and every collision is returned as a warning for a human. The
|
||||||
|
narrator exception in (2) is the one merge, and it is named.
|
||||||
|
|
||||||
|
4. **A render's bytes never get fetched.** Every artifact lands
|
||||||
|
`integrity = 'unverified'`. `content_sha256` is recorded as a claim, and
|
||||||
|
`verified_at` stays NULL until something reads the bytes back. Registering a
|
||||||
|
file is not verifying it.
|
||||||
|
|
||||||
|
5. **`media_type` comes from `audio_format.container`, never from the path
|
||||||
|
extension.** The 33 files named `.wav` that are MPEG audio are the test case.
|
||||||
|
|
||||||
|
6. **`language_designations` empty is respected as an answer.** No target
|
||||||
|
language is invented here. Target languages are a translation-side decision.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Destination model on the UmbraTrans side
|
||||||
|
|
||||||
|
Only three real destinations exist today, and one of them is new work:
|
||||||
|
|
||||||
|
| UmbraTrans thing | Where it lives | What it is |
|
||||||
|
|---|---|---|
|
||||||
|
| **Project** | tenant schema `projects` (`0005_tenant_schemas.sql`) | one book's container |
|
||||||
|
| **Artifact** | `public.artifacts` (+ `0007_external_artifacts.sql`) | a reference to produced bytes |
|
||||||
|
| **Translation unit / chapter / speaker** | translation store (per project; SQLite today, PG later) | the text tree that P0–P10 operate on |
|
||||||
|
|
||||||
|
The third is the part that does not exist yet, and it is called out in §7.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. `book` → Project
|
||||||
|
|
||||||
|
| Source field | Destination | Rule |
|
||||||
|
|---|---|---|
|
||||||
|
| `id` | `project_source_ref.external_id` | stable join key |
|
||||||
|
| `name` | `projects.name` | display name |
|
||||||
|
| `title` | `project_source_ref.source_title` | may equal `name` |
|
||||||
|
| `status` | `project_source_ref.source_status` | audio pipeline state, **not** `projects.status` |
|
||||||
|
| `chapters[]` | chapter set | each id must resolve to a `chapter` record |
|
||||||
|
| `chapter_count` | import report only | recomputed on read; never trusted as truth |
|
||||||
|
| `utterance_count` | import report only | recomputed |
|
||||||
|
| `speaker_count` | import report only | recomputed |
|
||||||
|
| `front_and_back_matter_count`, `novel_chapter_count` | import report only | derived |
|
||||||
|
| `declared_total_chapters`, `declared_total_words` | **warning** | differs from measured 35 vs 32; surfaced, not reconciled |
|
||||||
|
| `declared_vs_measured_note` | warning text | echoed to the import report |
|
||||||
|
| `language_designations` | `project_source_ref.languages` | empty is stored as empty |
|
||||||
|
| `language_designation_note` | import report | explains the empty list |
|
||||||
|
| `published_artifact_ids` | artifact links | must resolve to `render` records |
|
||||||
|
| `extension_container_mismatches[]` | import report warnings, one per entry | 33 entries |
|
||||||
|
| `extension_container_mismatch_count` | derived | must equal `len(...)`; mismatch is a validator error |
|
||||||
|
| `numbering` | `project_source_ref.numbering` | provenance, kept verbatim |
|
||||||
|
| `source_document_format`, `source_path_redacted` | `project_source_ref` | source provenance |
|
||||||
|
| `speaker_names_without_a_profile` | **warning** (`Captain`, `Princess Kainda`) | a join on profiles would drop these |
|
||||||
|
| `created_at`, `updated_at` | `project_source_ref` | not `projects.created_at` |
|
||||||
|
| `generated_at` | record audit | |
|
||||||
|
|
||||||
|
`projects.status` is authored on our side (`active` on ingest). It is a
|
||||||
|
lifecycle field for the translation project and must not be overwritten by the
|
||||||
|
audio system's own status.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. `chapter` → chapter node
|
||||||
|
|
||||||
|
| Source field | Destination | Rule |
|
||||||
|
|---|---|---|
|
||||||
|
| `id` | chapter `external_id` | **the identity** |
|
||||||
|
| `book_id` | parent project ref | must resolve |
|
||||||
|
| `chapter_index` | chapter `reading_order` | integer hint; negative ⇒ front/back matter |
|
||||||
|
| `database_row_id` | chapter `source_row_id` | provenance only |
|
||||||
|
| `title` | chapter `title` | FM/BM titles drive the naming convention (§8) |
|
||||||
|
| `source_text_sha256` | chapter `source_hash` | P0 checksum for the chapter |
|
||||||
|
| `source_text_char_count`, `word_count` | derived metrics | recomputed |
|
||||||
|
| `utterance_count` | derived | recomputed |
|
||||||
|
| `renders[]` | artifact links | each id must resolve to a `render` |
|
||||||
|
| `numbering_basis` | chapter `numbering_basis` (jsonb) | **kept verbatim; never used to derive identity** |
|
||||||
|
| `status` | chapter `source_status` | audio-side state |
|
||||||
|
|
||||||
|
Ordering rule: **identity = `chapter.id`; reading order = `chapter_index`; the
|
||||||
|
filename number is neither and is not used.** This is the fix for hazard A. The
|
||||||
|
first time a chapter is reordered by hand, `chapter_index` may change without
|
||||||
|
`id` changing, and only reading order should move.
|
||||||
|
|
||||||
|
### Front/back matter
|
||||||
|
|
||||||
|
`chapter_index < 0` ⇒ matter (front or back; the bundle does not distinguish).
|
||||||
|
It stays in the same chapter table (it does not get a separate one here), and it
|
||||||
|
carries the matter role so the translation workflow can treat it differently:
|
||||||
|
|
||||||
|
| `chapter_index` | matter role | example title |
|
||||||
|
|---|---|---|
|
||||||
|
| `-4 … -1` | `matter` | `Sample Track` (front), `Author's Note` (back) |
|
||||||
|
| `>= 0` | `novel` | `Salvage` (index 0) |
|
||||||
|
|
||||||
|
The ebook/print naming convention (`ebook_title_page`, `pdf_title_page`, `ab_*`,
|
||||||
|
no-prefix = all) is applied on top of this in the front/back-matter workflow, not
|
||||||
|
guessed from the title string.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. `speaker` → speaker record
|
||||||
|
|
||||||
|
| Source field | Destination | Rule |
|
||||||
|
|---|---|---|
|
||||||
|
| `id` | speaker `external_id` | identity |
|
||||||
|
| `primary_name` | speaker `display_name` | **not guaranteed unique as a character** |
|
||||||
|
| `aliases[]` | speaker `aliases` | kept |
|
||||||
|
| `is_narrator` | speaker `role` | `narrator` when true |
|
||||||
|
| `has_profile` | speaker `has_profile` | false ⇒ warning (`Captain`, `Princess Kainda`) |
|
||||||
|
| `is_group` | speaker `is_group` | designation vs individual |
|
||||||
|
| `unidentified` | speaker `unidentified` | |
|
||||||
|
| `utterance_count` | derived | exact, for that spelling |
|
||||||
|
| `utterance_count_casefold_grouped` | derived | as-if-merged; kept alongside, not instead of |
|
||||||
|
| `utterance_count_including_implicit_narrator` | derived | kept |
|
||||||
|
| `casefold_collision_group` | speaker `collision_group` | **stored verbatim; each group ⇒ warning** |
|
||||||
|
| `chapters_appeared` | derived | |
|
||||||
|
| `voice_review_status`, `voice_review_recorded` | speaker `voice_review` | a decision happened; the rejected take is gone |
|
||||||
|
| `tts_engine_used`, `tts_engine_decided_at`, `tts_engine_fallback_reason` | speaker `tts` | which engine produced the voice |
|
||||||
|
| `assigned_edge_voice` | speaker `edge_voice` | e.g. `en-CA-ClaraNeural:-100` |
|
||||||
|
| `name_is_distinct_from_spellings` | speaker `spelling_variants` | kept |
|
||||||
|
|
||||||
|
### The narrator, precisely
|
||||||
|
|
||||||
|
Two `speaker` records carry `is_narrator: true`
|
||||||
|
(`sp-narrator-6cb7da`, `sp-narrator-663ee4`), sharing
|
||||||
|
`casefold_collision_group: "narrator"`. Your note named three *spellings* and
|
||||||
|
this adds the fourth fact: **there are two narrator rows.** The mapping:
|
||||||
|
|
||||||
|
- One narrator **role** is created for the project.
|
||||||
|
- Both rows and the `NULL` spelling point at it.
|
||||||
|
- Both source spellings stay as aliases so audit still shows exactly what was
|
||||||
|
stored.
|
||||||
|
- `sp-narrator-6cb7da` has the approved review and the Edge voice; the role takes
|
||||||
|
those. `sp-narrator-663ee4` is recorded as the un-reviewed spelling.
|
||||||
|
- The two `utterance_count_including_implicit_narrator` values (1818, 2417) do
|
||||||
|
**not** reconcile to a single number and are not used to attribute utterances;
|
||||||
|
attribution comes from `utterance.speaker_id` / `speaker_resolution`.
|
||||||
|
|
||||||
|
This is the one merge in the mapping and it exists because voicing needs one
|
||||||
|
narrator voice. It is reported as a decision on every import, not performed
|
||||||
|
silently.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. `utterance` → translation unit
|
||||||
|
|
||||||
|
| Source field | Destination | Rule |
|
||||||
|
|---|---|---|
|
||||||
|
| `id` | unit `external_id` | identity |
|
||||||
|
| `book_id`, `chapter_id` | parents | must resolve |
|
||||||
|
| `utterance_index` | unit `index` | zero-based, restarts each chapter |
|
||||||
|
| `text` | unit `source_text` | |
|
||||||
|
| `text_sha256` | unit `source_hash` | |
|
||||||
|
| `text_char_count` | derived | |
|
||||||
|
| `utterance_type` | unit `type` | `narration` / `dialogue` / `scene_break` |
|
||||||
|
| `speaker_id` | unit `speaker` | must resolve to a `speaker` |
|
||||||
|
| `speaker_name_raw` | unit `speaker_name_raw` | **`null` is data meaning narrator**; never filtered on |
|
||||||
|
| `speaker_name_resolved` | unit `speaker_name_resolved` | |
|
||||||
|
| `speaker_resolution` | unit `attribution` | `implicit_narrator` / `explicit` |
|
||||||
|
| `attribution_confidence`, `attribution_uncertain` | unit review fields | |
|
||||||
|
| `needs_review`, `review_reason` | unit review queue | |
|
||||||
|
| `render_id` | artifact link | the `utterance_audio` artifact |
|
||||||
|
| `emotion`, `pace` | unit `voice_direction`, `voice_pace` | optional, mostly null; non-authoritative |
|
||||||
|
| `instruct_is_set` | unit `voice_instruct_set` | audio-side state |
|
||||||
|
| `status` | unit `source_status` | audio-side state |
|
||||||
|
| `database_row_id` | unit `source_row_id` | provenance |
|
||||||
|
|
||||||
|
Invariants enforced by the validator on every utterance:
|
||||||
|
|
||||||
|
- `speaker_id` is non-null and resolves.
|
||||||
|
- `speaker_resolution == "implicit_narrator"` ⇒ `speaker_name_raw is null` and the
|
||||||
|
resolved speaker is the narrator.
|
||||||
|
- `speaker_resolution == "explicit"` ⇒ `speaker_name_raw` is a non-null string.
|
||||||
|
- `utterance_type` is one of the three values.
|
||||||
|
- `render_id` resolves (null is allowed only for `scene_break`).
|
||||||
|
|
||||||
|
`scene_break` (9 of them) is a non-voiced structural marker. It is **not** a
|
||||||
|
translation unit in the literary sense and must not be sent through P2–P8; it is
|
||||||
|
carried so chapter structure survives reassembly.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. `render` → Artifact
|
||||||
|
|
||||||
|
| Source field | Destination | Rule |
|
||||||
|
|---|---|---|
|
||||||
|
| `id` | artifact `derivation.source_render_id` | identity in provenance |
|
||||||
|
| `content_sha256` | `artifacts.content_hash` | required; nonzero for all 4477 |
|
||||||
|
| `byte_size` | `artifacts.byte_size` | |
|
||||||
|
| `path` + `relative_to` + `path_is_absolute` | `artifacts.storage_key` = `path`; `storage_uri` = `file://<node>/<book_dir>/<path>` | |
|
||||||
|
| (node) | `artifacts.storage_node_id` | the `umbraaudio-m4` node |
|
||||||
|
| `audio_format.container` + `codec` | `artifacts.media_type` | `wav`+`pcm_s16le`→`audio/wav`; `mp3`→`audio/mpeg`; `m4b`→`audio/mp4`/`audio/aac` |
|
||||||
|
| `render_role` | `artifacts.kind='audio'` + `derivation.render_role` | one `kind`, role in provenance |
|
||||||
|
| `render_stage` | `derivation.render_stage` | |
|
||||||
|
| `duration_ms` | `derivation.duration_ms` | may be null (66); null ≠ 0 |
|
||||||
|
| `inputs[]` | `derivation.inputs[]` | lineage edges |
|
||||||
|
| `book_id`, `chapter_id` | `derivation.book_id`, `derivation.chapter_id` | |
|
||||||
|
| `audio_format.*` | `derivation.audio_format` | kept whole |
|
||||||
|
| `file_present`, `file_modified_at`, `stale` | `derivation.fs` | |
|
||||||
|
| `voice_id`, `track_number_in_name`, `title_in_name`, `embedded_metadata_*` | `derivation` | |
|
||||||
|
| (constant) | `artifacts.integrity='unverified'`, `verified_at=NULL` | no bytes are read |
|
||||||
|
|
||||||
|
`status` is derived from `render_role`:
|
||||||
|
|
||||||
|
| `render_role` | `artifacts.status` |
|
||||||
|
|---|---|
|
||||||
|
| `utterance_audio`, `chapter_audio`, `voice_reference` | `intermediate` |
|
||||||
|
| `distribution_track`, `assembled_book` | `final` |
|
||||||
|
|
||||||
|
`extension_matches_container == false` (33 files) ⇒ `derivation.extension_mismatch
|
||||||
|
= true` and an import warning. `media_type` still comes from `container`.
|
||||||
|
|
||||||
|
### Artifacts with no `chapter_id`, and why absent ≠ null here
|
||||||
|
|
||||||
|
Measured, because the difference is the kind of thing this mapping exists to
|
||||||
|
catch:
|
||||||
|
|
||||||
|
| Records | `chapter_id` | Meaning |
|
||||||
|
|---|---|---|
|
||||||
|
| `voice_reference` (93), `assembled_book` (1) | **key absent** | attaches to the project |
|
||||||
|
| `distribution_track` (5) | key present, **`null`** | the five **front/back-matter** tracks — `Deal_Hunter_{Opening_Track,Closing_Track,Sample_Track,Authors_Note,Copyright}.mp3` — which carry no `chapter_id` link |
|
||||||
|
|
||||||
|
Both arrive without a chapter, and the ingest treats them the same — an artifact
|
||||||
|
on the project. They are not confusable, but only because the export keeps the
|
||||||
|
two states apart; a reader that wrote `row.get("chapter_id")` would see `None`
|
||||||
|
for both and be right by accident, and a reader that asserted the key existed
|
||||||
|
would break on 94 of 4477. The `storage_uri` is valid for all of them;
|
||||||
|
`file_present`/`stale` are read but not trusted as truth (your build recomputes
|
||||||
|
them).
|
||||||
|
|
||||||
|
### Zero-byte file
|
||||||
|
`chapter_audio/ch20/utt_0226.wav` is present and empty. It becomes an artifact
|
||||||
|
with `byte_size = 0`. The ingest does **not** treat 0-byte as missing; it is
|
||||||
|
recorded and flagged `needs_rerender`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Fields that have no home on this side (kept, not dropped)
|
||||||
|
|
||||||
|
These are stored in `derivation` / the import report rather than mapped to a
|
||||||
|
column, because UmbraTrans has no equivalent and inventing one would be a guess:
|
||||||
|
|
||||||
|
- `marker` (`*.wav.txt`), `stale` — audio cache concepts; kept as `derivation.fs`.
|
||||||
|
- `render_stage` values that are audio-internal step names.
|
||||||
|
- `embedded_metadata_*` — useful, not authoritative.
|
||||||
|
- `declared_*` figures — kept as warnings.
|
||||||
|
- `instruct_is_set`, `status` on utterances — audio-pipeline state.
|
||||||
|
|
||||||
|
And these are explicitly **missing** and will come back as a specific request
|
||||||
|
rather than a silent gap:
|
||||||
|
|
||||||
|
- **Totals across all projects** (books/hours/bytes). You have one book; we have
|
||||||
|
no aggregate. Not blocking.
|
||||||
|
- **Rejected takes** — you cannot recover them, so there is nothing to map.
|
||||||
|
- **Per-utterance language** — does not exist; target languages are ours.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. New storage this mapping requires (our work, not yours)
|
||||||
|
|
||||||
|
The bundle maps cleanly onto `projects` and `artifacts`. It does not map cleanly
|
||||||
|
onto anything today for **chapters, speakers, and utterances**, because the
|
||||||
|
translation store is not yet a PG schema. The ingest therefore needs one
|
||||||
|
additive migration on our side: `umbratrans_web/migrations/0008_audio_source_import.sql`
|
||||||
|
(written, applied to a scratch copy of the schema inside a rolled-back
|
||||||
|
transaction, idempotent on re-apply, not applied anywhere for real yet).
|
||||||
|
|
||||||
|
```
|
||||||
|
project_source_refs( -- one row per project per source system
|
||||||
|
project_id pk, organization_id fk, system, node_id, external_id,
|
||||||
|
source_title, source_status, languages jsonb, numbering jsonb,
|
||||||
|
content_hash /* sha256 of the manifest bytes */, product_version,
|
||||||
|
source_meta jsonb, imported_at
|
||||||
|
)
|
||||||
|
translation_speakers(
|
||||||
|
id, project_id, organization_id fk, external_id, display_name, role,
|
||||||
|
aliases jsonb, collision_group, has_profile, is_group, unidentified,
|
||||||
|
edge_voice, tts_engine, voice_review_status, merged_into_external_id,
|
||||||
|
source_meta jsonb, created_at
|
||||||
|
)
|
||||||
|
translation_chapters(
|
||||||
|
id, project_id, organization_id fk, external_id, reading_order, matter,
|
||||||
|
title, source_hash, source_row_id, numbering_basis jsonb, utterance_count,
|
||||||
|
source_status, source_meta jsonb, created_at
|
||||||
|
)
|
||||||
|
translation_utterances(
|
||||||
|
id, project_id, organization_id fk, external_id, chapter_external_id,
|
||||||
|
utterance_index, kind, text, source_hash, speaker_external_id,
|
||||||
|
speaker_name_raw, speaker_name_resolved, attribution, needs_review,
|
||||||
|
render_external_id, source_row_id, source_meta jsonb, created_at
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Two departures from the sketch above, both deliberate:
|
||||||
|
|
||||||
|
- **Chapters and utterances are separate tables.** The sketch folded them into
|
||||||
|
one `translation_units` with a `kind` column; a table whose chapter columns are
|
||||||
|
all nullable and whose utterance columns are all nullable is a table where
|
||||||
|
every query carries the same `WHERE kind = ...`, and where a NOT NULL that
|
||||||
|
should hold for one kind cannot be written for either. The shared columns are
|
||||||
|
the identity, the project, and the source hash, and that is not enough to pay
|
||||||
|
for the confusion.
|
||||||
|
- **`project_id` carries no foreign key**, and neither does `chapter_external_id`
|
||||||
|
point at `translation_chapters`. `projects` lives in the per-tenant schema, so
|
||||||
|
there is no stable relation name to reference; `public.artifacts` reached the
|
||||||
|
same conclusion in 0003. The organization foreign key is the tenancy boundary
|
||||||
|
that matters, and the importer has already resolved every reference within one
|
||||||
|
import.
|
||||||
|
|
||||||
|
This is ours to add and does not change the bundle. It is listed here so the
|
||||||
|
mapping is not read as "nothing needs building."
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. What the mapping does not resolve, and who owns it
|
||||||
|
|
||||||
|
| Question | Owner | Note |
|
||||||
|
|---|---|---|
|
||||||
|
| Turner of chapter `id`s into P0 unit IDs (`bk.ch.sc.p.s`) | UmbraTrans | P0 assigns; the audio `id` is only a source ref |
|
||||||
|
| Target languages | UmbraTrans / human | `language_designations` is empty |
|
||||||
|
| Narrator merge acceptance | human (G1-style gate) | default merge, reported |
|
||||||
|
| Speaker collision resolution (`narrator` group) | human | reported, not merged |
|
||||||
|
| Object storage for bytes | DECISION-6, unresolved | we hold references only |
|
||||||
|
| Whether `assembled_book` is `final` before release | human | default `final` per role table |
|
||||||
145
docs/audio-system-prompt-v1.md
Normal file
145
docs/audio-system-prompt-v1.md
Normal file
@@ -0,0 +1,145 @@
|
|||||||
|
UMBRA AUDIO — METADATA HANDOVER, ACCEPTED FORMAT, AND THE TTS DOCUMENT
|
||||||
|
Prompt to the OpenCode session on the audio system (host `umbraaudio-m4`)
|
||||||
|
|
||||||
|
CONTEXT
|
||||||
|
|
||||||
|
You produced `~/audio-read/bundles/dh-bundle/` (manifest.jsonl sha256
|
||||||
|
21ce568787540cf7164b6ab4028f1caee51996447537022154d1e5768cde87ba). It has been
|
||||||
|
received and independently verified on the controller host:
|
||||||
|
|
||||||
|
- the declared content_hash matches the manifest bytes;
|
||||||
|
- your validate.py passes on the good bundle and fails on each broken copy;
|
||||||
|
- the controller's accepted-format validator passes: 8884 records,
|
||||||
|
book=1 chapter=32 speaker=55 utterance=4319 render=4477.
|
||||||
|
|
||||||
|
Nothing was written to the controller. The confirmation gate is intact.
|
||||||
|
|
||||||
|
The controller's answer to your four requests is in three files, delivered
|
||||||
|
alongside this prompt:
|
||||||
|
|
||||||
|
- audio-system-mapping-v1.md (field-by-field mapping, source -> controller)
|
||||||
|
- audio-system-manifest-v1.schema.json (the accepted manifest format)
|
||||||
|
- validate-audio-manifest.py (the accepted-format validator, run by the controller)
|
||||||
|
- audio-system-reply-v1.md (the full reply: verification, decisions, ingest,
|
||||||
|
credential, and the TTS request)
|
||||||
|
|
||||||
|
Read those four before acting. This prompt is the instruction; those are the detail.
|
||||||
|
|
||||||
|
DECISIONS ALREADY MADE (do not re-open; implement against them)
|
||||||
|
|
||||||
|
1. Format: the same bundle format you already emit, NARROWED. Field names are
|
||||||
|
unchanged. Do not rename anything on the wire. The accepted format is defined
|
||||||
|
by audio-system-manifest-v1.schema.json.
|
||||||
|
|
||||||
|
2. Chapter identity is chapter.id. chapter_index is reading order and a hint;
|
||||||
|
the filename number is neither. Your numbering_basis field is required and is
|
||||||
|
what makes this unambiguous.
|
||||||
|
|
||||||
|
3. The narrator is a role. Both narrator speaker rows and all three spellings
|
||||||
|
(NULL, 'narrator', 'Narrator') resolve to one narrator role for voicing; the
|
||||||
|
source spellings are kept as aliases for audit. This is the controller's one
|
||||||
|
merge, and it is reported on every import. Other speaker rows are NOT merged;
|
||||||
|
casefold_collision_group is stored and surfaced.
|
||||||
|
|
||||||
|
4. A render's bytes are never fetched. Every artifact is recorded
|
||||||
|
integrity='unverified'; content_sha256 is recorded as a claim, not a fact.
|
||||||
|
|
||||||
|
5. media_type is derived from audio_format.container, never from the path
|
||||||
|
extension. The 33 files named .wav that are MPEG audio are the test case.
|
||||||
|
|
||||||
|
6. language_designations empty is accepted as an answer. Target languages are a
|
||||||
|
translation-side decision; do not invent them.
|
||||||
|
|
||||||
|
7. Ingest is file-based first. No credential is required and nothing is pushed.
|
||||||
|
A push endpoint, if built later, will use a scoped service token
|
||||||
|
(`dataset:ingest`), NOT a node key. No action from you on credentials today.
|
||||||
|
|
||||||
|
YOUR NEXT ACTIONS
|
||||||
|
|
||||||
|
Action 1 — Confirm the accepted format (short).
|
||||||
|
|
||||||
|
Copy audio-system-manifest-v1.schema.json and validate-audio-manifest.py into a
|
||||||
|
copy of your bundle directory and run:
|
||||||
|
|
||||||
|
python3 validate-audio-manifest.py .
|
||||||
|
|
||||||
|
Expected: "<dir>: 8884 record(s) accepted. content_hash <hash>. book=1,
|
||||||
|
chapter=32, speaker=55, utterance=4319, render=4477", exit 0.
|
||||||
|
|
||||||
|
If it fails, report the exact lines. Do not edit the bundle to make it pass
|
||||||
|
unless the failure is a real defect in your generator, and say so if it is.
|
||||||
|
|
||||||
|
Note: the validator needs the `jsonschema` package. If it is not installed on
|
||||||
|
the audio host, it says so and refuses to report acceptance rather than
|
||||||
|
half-checking. Install it into a throwaway venv for the check; do not add it to
|
||||||
|
the audio system's own environment if that would disturb the running system.
|
||||||
|
|
||||||
|
Action 2 — Produce the TTS checked document (the blocking one).
|
||||||
|
|
||||||
|
You offered this in your reply and the controller needs it to write `speak`
|
||||||
|
without guessing. Produce it as a checked document with the same standard as
|
||||||
|
the bundle: measured, versioned, hashed, with a note per row where something is
|
||||||
|
unknown rather than a near-match. It must answer:
|
||||||
|
|
||||||
|
1. Provenance: upstream repository and commit; the three commits on top, as a
|
||||||
|
diff or a per-commit summary with purpose; the license.
|
||||||
|
2. The model: exact model id(s) (Qwen3-TTS-12Hz-1.7B-VoiceDesign and -Base),
|
||||||
|
which one serves which path, weights source and checksum, on-disk size, and
|
||||||
|
the VRAM/working-set at load and at inference.
|
||||||
|
3. The input contract: request fields (text, voice / voice-design, language,
|
||||||
|
the instruct/emotion/pace fields, sample rate), the reference-audio (cloning)
|
||||||
|
path and its format, maximum input length, and whether it streams or returns
|
||||||
|
a whole file.
|
||||||
|
4. The output contract: container, codec, sample rate, channels, bitrate for a
|
||||||
|
rendered take, so a render from `speak` is described the same way your
|
||||||
|
existing renders are.
|
||||||
|
5. Determinism: given identical text and voice, is the audio bit-identical? If
|
||||||
|
sampling is involved, what is seeded and what is not.
|
||||||
|
6. Language coverage: which languages the model actually speaks, how the
|
||||||
|
language is selected, and where quality is known to be weak.
|
||||||
|
7. Voice identity: how a voice_id maps to a speaker, and how
|
||||||
|
tts_engine_used:"qwen" relates to assigned_edge_voice (e.g.
|
||||||
|
"en-CA-ClaraNeural:-100") — is the Edge voice the fallback engine, the
|
||||||
|
reference audio, or the design prompt? Explain the tts_engine_fallback_reason
|
||||||
|
values.
|
||||||
|
8. Failure modes: OOM, over-length text, unsupported language — what is
|
||||||
|
returned and how it appears in tts_engine_fallback_reason.
|
||||||
|
9. The runtime surface: the HTTP/queue API you expose (or that it is
|
||||||
|
in-process), its auth, and its concurrency limits — so `speak` on the
|
||||||
|
controller is a client of something defined.
|
||||||
|
|
||||||
|
Deliver as a document named tts-engine.md (or .json + a short README) with the
|
||||||
|
measured values, not prose claims. Where a value is unknown, say "unknown" and
|
||||||
|
why; do not estimate it as if measured.
|
||||||
|
|
||||||
|
Action 3 — Optional, only if asked: scale totals.
|
||||||
|
|
||||||
|
Totals across all projects (books, hours, bytes). The bundle carries per-book
|
||||||
|
counts and byte_size on every render, so extend the build rather than hand over
|
||||||
|
a number you did not measure. Not blocking.
|
||||||
|
|
||||||
|
CONSTRAINTS
|
||||||
|
|
||||||
|
- Do not modify the audio system. It is independent and keeps running. This is a
|
||||||
|
read-only metadata export and a documentation task.
|
||||||
|
- Do not push anything to the controller. There is no endpoint and no credential
|
||||||
|
by design; the handover is the file.
|
||||||
|
- Keep the generator deterministic: sorted JSON keys, fixed record order, so an
|
||||||
|
unchanged project still hashes identically.
|
||||||
|
- Every claim in the TTS document must be checkable against the running system.
|
||||||
|
"A validator nobody has seen fail is not evidence" applies here too: if you say
|
||||||
|
the model produces 24 kHz mono, show the ffprobe output.
|
||||||
|
|
||||||
|
REPORT BACK
|
||||||
|
|
||||||
|
Post a short reply with:
|
||||||
|
|
||||||
|
1. the validator result (Action 1);
|
||||||
|
2. the TTS document, or the path where it was written, and any item from the nine
|
||||||
|
you could not measure;
|
||||||
|
3. anything in the mapping or the accepted format you believe is wrong — before
|
||||||
|
implementing against it, not after.
|
||||||
|
|
||||||
|
The controller will not import the bundle until the accepted-format check passes
|
||||||
|
and the operator confirms. There is no rush and no penalty for an honest
|
||||||
|
"unknown".
|
||||||
221
docs/audio-system-reply-v1.md
Normal file
221
docs/audio-system-reply-v1.md
Normal file
@@ -0,0 +1,221 @@
|
|||||||
|
# Re: the bundle, and the four things you asked for
|
||||||
|
|
||||||
|
**To:** the UmbraAudio session (`umbraaudio-m4`)
|
||||||
|
**From:** the UmbraTrans controller side
|
||||||
|
**Re:** `~/audio-read/bundles/dh-bundle/` — `manifest.jsonl` sha256
|
||||||
|
`21ce568787540cf7164b6ab4028f1caee51996447537022154d1e5768cde87ba`
|
||||||
|
**Status:** nothing has been written to the controller. The confirmation gate is
|
||||||
|
untouched.
|
||||||
|
|
||||||
|
Your four asks are answered in order: §1 mapping, §2 format, §3 ingest +
|
||||||
|
credential, §4 the TTS engine. §0 is what I did before believing any of it.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 0. Verification, before trusting a field
|
||||||
|
|
||||||
|
| Check | Result |
|
||||||
|
|---|---|
|
||||||
|
| `sha256(manifest.jsonl)` vs `bundle.json.content_hash.value` | **match**, `21ce5687…e87ba` |
|
||||||
|
| `sha256(dh-bundle.tar.gz)` | `5200f95f…63057` (recorded; you declared no value for it) |
|
||||||
|
| your `validate.py .` | exit 0, 8884 records |
|
||||||
|
| your `validate.py` on a dangling reference | fails, exit 1 |
|
||||||
|
| your `validate.py` on a unitless duration | fails, exit 1 |
|
||||||
|
| your `validate.py` on a planted credential | fails, exit 1 |
|
||||||
|
| your `validate.py` on a missing required field | fails, exit 1 |
|
||||||
|
|
||||||
|
You were right to insist the validator be seen failing; it is the only thing that
|
||||||
|
makes the pass mean anything. One gap to name so it is not mistaken for coverage:
|
||||||
|
**`validate.py` does not check the declared `content_hash` against the file
|
||||||
|
bytes.** It proves the bundle is internally consistent. A tampered manifest that
|
||||||
|
kept its internal references consistent would pass it. I recomputed the hash by
|
||||||
|
hand for that reason, and the accepted-format validator in §2 closes the gap as a
|
||||||
|
first-class check.
|
||||||
|
|
||||||
|
## 0b. Three facts I found that your note did not mention
|
||||||
|
|
||||||
|
Not defects — they are in the data and the mapping had to account for them, so
|
||||||
|
you should know the mapping saw them:
|
||||||
|
|
||||||
|
1. **There are two narrator `speaker` rows**, `sp-narrator-6cb7da` and
|
||||||
|
`sp-narrator-663ee4`, both `is_narrator: true`, sharing
|
||||||
|
`casefold_collision_group: "narrator"`. Your note named three *spellings*;
|
||||||
|
there are also two *rows*. Handled in §1.
|
||||||
|
2. **The nine `scene_break` utterances are rendered** (text `,,,`, voiced by the
|
||||||
|
narrator). I had assumed a scene break was a silent marker and wrote an
|
||||||
|
invariant to that effect; your data refuted it and the invariant is gone. A
|
||||||
|
scene break is still kept out of the literary passes, but it is not unrendered.
|
||||||
|
3. **`track_number_in_name` is a string** (`"01"`, not `1`). The schema allows
|
||||||
|
either; nothing infers order from it.
|
||||||
|
|
||||||
|
And a convention worth stating plainly, since it changes the schema: the bundle
|
||||||
|
**omits absent keys rather than writing `null`.** A `voice_reference` render has
|
||||||
|
no `chapter_id` key at all. "Absent" and "null" both mean "no chapter" to the
|
||||||
|
ingest, but a reader who assumed a stable key set would be wrong.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. The mapping document
|
||||||
|
|
||||||
|
**`docs/audio-system-mapping-v1.md`** — every field of all five record kinds, to
|
||||||
|
its destination, with the ones that have no destination kept rather than dropped.
|
||||||
|
|
||||||
|
The four decisions that a wrong default would hide, all stated there and testable:
|
||||||
|
|
||||||
|
- **Chapter identity is `chapter.id`.** `chapter_index` is carried as
|
||||||
|
`reading_order` and is a hint; the filename number is neither and is unused.
|
||||||
|
Your `numbering_basis` is stored verbatim precisely so a derived order can never
|
||||||
|
be mistaken for identity again.
|
||||||
|
- **The narrator is a role, not a name.** The three spellings *and* both rows
|
||||||
|
resolve to one narrator role for voicing; both source spellings stay as aliases
|
||||||
|
for audit. This is the one merge in the mapping, and it is reported on every
|
||||||
|
import, not performed quietly — exactly the line you drew.
|
||||||
|
- **Other speaker collisions are not merged.** `casefold_collision_group` is
|
||||||
|
stored and every group is surfaced to a human. `count(distinct speaker_name)` is
|
||||||
|
never used for anything.
|
||||||
|
- **A render's bytes are never fetched.** Every artifact lands
|
||||||
|
`integrity = 'unverified'`; `content_sha256` is recorded as a *claim*. Your
|
||||||
|
`characters`-table warning is the same lesson one level up: the export is
|
||||||
|
evidence that somebody said so.
|
||||||
|
|
||||||
|
## 2. The format: the same one, narrowed
|
||||||
|
|
||||||
|
**`docs/audio-system-manifest-v1.schema.json`** and
|
||||||
|
**`docs/validate-audio-manifest.py`**.
|
||||||
|
|
||||||
|
Same format, narrowed — field names are unchanged, nothing is renamed on the
|
||||||
|
wire, because renaming buys nothing and costs a mismatch surface. What the
|
||||||
|
narrowing adds:
|
||||||
|
|
||||||
|
- **Required fields are exactly the fields the mapping reads.** Not "all fields";
|
||||||
|
a required field nothing consumes is a coupling with no benefit.
|
||||||
|
- **The mapping invariants are checked**, because they are the failures a schema
|
||||||
|
cannot see: an `implicit_narrator` with a non-null `speaker_name_raw`; a chapter
|
||||||
|
with no `numbering_basis`; `extension_matches_container` left null when the
|
||||||
|
container is known (or asserted when it is not); `extension_container_mismatch_count`
|
||||||
|
disagreeing with the list.
|
||||||
|
- **The declared hash is verified against the bytes**, then the schema, then
|
||||||
|
references, then invariants, then a credential scan over the whole directory.
|
||||||
|
|
||||||
|
I ran the accepted validator against your real bundle (**accepted**, 8884 records)
|
||||||
|
and against ten deliberately broken copies — tampered manifest, dangling
|
||||||
|
reference, mislabelled narrator, lying mismatch count, a dropped required field, a
|
||||||
|
planted token, and the null/absent variants above. Each fails, each with the record
|
||||||
|
and the field named. The validator needs `jsonschema`; without it, it says so and
|
||||||
|
refuses to report acceptance rather than half-checking.
|
||||||
|
|
||||||
|
## 3. Ingest and the credential
|
||||||
|
|
||||||
|
### The first ingest is file-based, and needs no credential
|
||||||
|
|
||||||
|
You are right that the handover is file-based by design, and that T05–T08 are
|
||||||
|
blocked on my side, not yours. So:
|
||||||
|
|
||||||
|
- **No credential is needed for the first ingest.** The bundle is a file; the
|
||||||
|
ingest runs locally on this host against the file. Nothing on your side
|
||||||
|
changes, and nothing is pushed.
|
||||||
|
- **The sequence is the one you built for:** dry-run → a report a human reads →
|
||||||
|
operator `CONFIRM` → commit. The dry-run write nothing.
|
||||||
|
|
||||||
|
### If and when a push path is built: a scoped **service token**, not a node key
|
||||||
|
|
||||||
|
Direct answer to your question, with the reason, because the reason is the point:
|
||||||
|
|
||||||
|
- **Not a node key.** `NODE_SCOPES` (`compute:read`, `compute:heartbeat`,
|
||||||
|
`compute:claim`, `compute:complete`) exists so that a node credential *cannot
|
||||||
|
read or write project content* — that boundary is what lets a node sit outside
|
||||||
|
the building. A bulk dataset push is project content. Adding a write scope to
|
||||||
|
that set would dissolve the boundary the whole node model rests on.
|
||||||
|
- **A separate service token**, scoped `dataset:ingest`, bound to **one
|
||||||
|
organization and one project**, with the organization and project **derived
|
||||||
|
from the token** — not accepted in the body. A token that cannot name a tenant
|
||||||
|
is a tenant that cannot be forged. This is the same shape as the node credential
|
||||||
|
(`compute.py`, §7.7), for the same reason.
|
||||||
|
- Endpoint shape, so it is not a surprise when it exists: a dry run returns a
|
||||||
|
validation report and writes nothing; a commit returns an `import_id` and an
|
||||||
|
`audit_id`; `content_hash` is the idempotency key, and the same key with a
|
||||||
|
different hash is a 409, not a second copy. Idempotency by content hash is your
|
||||||
|
own determinism guarantee being cashed in.
|
||||||
|
|
||||||
|
One deployment observation from your discovery that reinforces this: your port is
|
||||||
|
plain HTTP and `/api/health` reports `cookie_secure: true`. A browser session over
|
||||||
|
that port therefore cannot authenticate — so any push path must carry an
|
||||||
|
`Authorization` header, which a service token does and a cookie does not. Worth
|
||||||
|
confirming on my side that the cookie/HTTP combination is intended; it is not a
|
||||||
|
blocker for a header-authenticated push.
|
||||||
|
|
||||||
|
### What the first ingest writes, and how it is checked
|
||||||
|
|
||||||
|
- **Project** + a source-reference row (external ids, source title, empty
|
||||||
|
`language_designations` kept empty, the numbering provenance).
|
||||||
|
- **Chapters, speakers, utterances** — new tables on my side (§6). This is the
|
||||||
|
part that does not exist yet; `projects` and `artifacts` are ready.
|
||||||
|
- **Artifacts** — one per render, `integrity='unverified'`, `storage_uri` naming
|
||||||
|
your node, `media_type` from `audio_format.container`.
|
||||||
|
- **Verification after commit**, before it counts: counts match the manifest; every
|
||||||
|
reference resolves; a random sample of utterances is spot-checked; every artifact
|
||||||
|
is registered `unverified`; the zero-byte `utt_0226.wav` is recorded as present
|
||||||
|
and flagged, not confused with missing.
|
||||||
|
|
||||||
|
## 4. P1 #5 — the TTS engine: exactly what I need
|
||||||
|
|
||||||
|
You said you would produce it as a checked document if I asked specifically. I am
|
||||||
|
asking specifically. To write `speak` without guessing, I need these as measured,
|
||||||
|
versioned facts — the same standard as the bundle:
|
||||||
|
|
||||||
|
1. **Provenance.** Upstream repository and commit; the three commits on top, as a
|
||||||
|
diff or a summary with their purpose; the license.
|
||||||
|
2. **The model.** Exact model id(s) (`Qwen3-TTS-12Hz-1.7B-VoiceDesign`, `-Base`),
|
||||||
|
which one serves which path, weights source and checksum, on-disk size, and the
|
||||||
|
VRAM/working-set at load and at inference.
|
||||||
|
3. **The input contract.** The request fields: text, voice/voice-design, language,
|
||||||
|
the `instruct`/emotion/pace fields, sample rate, the reference-audio (cloning)
|
||||||
|
path and its format, maximum input length, and whether it streams or returns a
|
||||||
|
whole file.
|
||||||
|
4. **The output contract.** Container, codec, sample rate, channels, and bitrate
|
||||||
|
for a rendered take — so a `render` from `speak` is described the same way your
|
||||||
|
existing renders are.
|
||||||
|
5. **Determinism.** Given identical text and voice, is the audio bit-identical? If
|
||||||
|
sampling is involved, what is seeded and what is not. This decides whether a
|
||||||
|
translation re-voiced for one changed line reproduces the rest byte-for-byte.
|
||||||
|
6. **Language coverage.** Which languages the model actually speaks, how the
|
||||||
|
language is selected, and where quality is known to be weak. This is mine to
|
||||||
|
plan around, but only if you tell me.
|
||||||
|
7. **Voice identity.** How a `voice_id` maps to a speaker, and how
|
||||||
|
`tts_engine_used: "qwen"` relates to `assigned_edge_voice`
|
||||||
|
(`en-CA-ClaraNeural:-100`): is the Edge voice the fallback engine, the
|
||||||
|
reference audio, or the design prompt? The `tts_engine_fallback_reason` values
|
||||||
|
say a fallback exists; I need to know to what.
|
||||||
|
8. **Failure modes.** OOM, over-length text, unsupported language — what is
|
||||||
|
returned, and how it appears in `tts_engine_fallback_reason`.
|
||||||
|
9. **The runtime surface.** The HTTP/queue API you expose (or that it is
|
||||||
|
in-process), its auth, and its concurrency limits — so a `speak` work function
|
||||||
|
from the controller is a client of something defined rather than a guess.
|
||||||
|
|
||||||
|
## 5. Smaller things
|
||||||
|
|
||||||
|
- **Host `238`:** agreed, it is unreachable from here too. The file-based handover
|
||||||
|
is what we want; `238` is not needed.
|
||||||
|
- **Scale totals across all projects:** not blocking. One book in full is the
|
||||||
|
right first deliverable. Extend the build later and I will take a number you
|
||||||
|
measured.
|
||||||
|
- **Declared 35 chapters vs measured 32:** surfaced as a warning, not reconciled.
|
||||||
|
If the gap is real and not a stale counter, tell me; otherwise it stays a warning.
|
||||||
|
- **Object storage (DECISION-6)** is still open. This is the integration that lets
|
||||||
|
us hold a reference to your bytes instead of the bytes; it is why the mapping is
|
||||||
|
references only.
|
||||||
|
|
||||||
|
## 6. What I do next, on my side
|
||||||
|
|
||||||
|
Not yours, listed so the mapping is not read as "nothing needs building":
|
||||||
|
|
||||||
|
- one additive migration for chapters, speakers, utterances, and source refs;
|
||||||
|
- the file-based ingest with dry-run and the operator gate;
|
||||||
|
- audit events for every import;
|
||||||
|
- then the push endpoint and service token, if we choose to have one.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
Ping me with the TTS document (§4) whenever it exists, and I will start `speak`
|
||||||
|
against it. The mapping (§1), the format (§2), and the credential decision (§3) are
|
||||||
|
what you need in the meantime — and none of them need a write on either side.
|
||||||
2
docs/script.txt
Normal file
2
docs/script.txt
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
read -r -p "Enter the six digit pairing code: " CODE
|
||||||
|
curl -sS -X POST http://192.168.123.14:8442/api/node/pair -H 'Content-Type: application/json' -d "{\"code\":\"$CODE\",\"node_name\":\"win-test-238\",\"agent_version\":\"win-test/0.1.0\"}"
|
||||||
388
docs/validate-audio-manifest.py
Normal file
388
docs/validate-audio-manifest.py
Normal file
@@ -0,0 +1,388 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Validate an UmbraAudio metadata bundle against the format UmbraTrans accepts.
|
||||||
|
|
||||||
|
This is the *accepted-format* validator, the follow-up to the smoke-alarm one
|
||||||
|
shipped with the data request. It is deliberately stricter in three places,
|
||||||
|
because each is a thing the earlier validator could not catch:
|
||||||
|
|
||||||
|
1. **It validates each record against the full JSON Schema**
|
||||||
|
(``audio-system-manifest-v1.schema.json``), not a required-field subset.
|
||||||
|
2. **It verifies the declared ``content_hash``** against the actual bytes of
|
||||||
|
``manifest.jsonl``. The earlier validator checked that the bundle was
|
||||||
|
internally consistent; it did not check that the bundle was the one the hash
|
||||||
|
claimed. A tampered manifest that kept its internal references consistent
|
||||||
|
would have passed it.
|
||||||
|
3. **It enforces the mapping invariants** - the narrator semantics, chapter
|
||||||
|
numbering provenance, extension-vs-container, and the utterance attribution
|
||||||
|
rules - because those are the fields whose semantics change the answer, and a
|
||||||
|
record that satisfies the schema but breaks them is the failure a mapping
|
||||||
|
cannot see.
|
||||||
|
|
||||||
|
python3 validate-audio-manifest.py <bundle-directory>
|
||||||
|
|
||||||
|
Exits 0 when the bundle is acceptable, 1 when it is not, 2 when it cannot be
|
||||||
|
read at all. Every problem is printed, not just the first.
|
||||||
|
|
||||||
|
Runs without third-party packages: if ``jsonschema`` is importable it is used
|
||||||
|
for the full schema pass; otherwise the validator falls back to the structural
|
||||||
|
subset below and says so, rather than pretending to have validated the schema.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
SCHEMA_NAME = "audio-system-manifest-v1.schema.json"
|
||||||
|
|
||||||
|
REQUIRED_TOP = ("bundle.json", "manifest.jsonl")
|
||||||
|
|
||||||
|
#: Patterns that look like a secret. The bundle travels between machines and is
|
||||||
|
#: meant to be committable.
|
||||||
|
SUSPICIOUS = re.compile(
|
||||||
|
r"(sk-[A-Za-z0-9]{16,}|ghp_[A-Za-z0-9]{20,}|AKIA[0-9A-Z]{12,}"
|
||||||
|
r"|-----BEGIN[A-Z ]*PRIVATE KEY-----|eyJ[A-Za-z0-9_-]{20,}\.)"
|
||||||
|
)
|
||||||
|
|
||||||
|
REFERENCE_FIELDS = (
|
||||||
|
"book_id", "chapter_id", "render_id", "parent_id", "source_id",
|
||||||
|
"input_id", "voice_id", "inputs", "outputs", "chapters", "renders",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def fail(problems: list[str], message: str) -> None:
|
||||||
|
problems.append(message)
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# loading
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def load_bundle(root: Path, problems: list[str]) -> tuple[dict, list[dict]] | None:
|
||||||
|
for name in REQUIRED_TOP:
|
||||||
|
if not (root / name).exists():
|
||||||
|
fail(problems, f"missing required file: {name}")
|
||||||
|
if problems:
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
bundle = json.loads((root / "bundle.json").read_text())
|
||||||
|
except (OSError, json.JSONDecodeError) as error:
|
||||||
|
fail(problems, f"bundle.json: not readable JSON ({error})")
|
||||||
|
return None
|
||||||
|
|
||||||
|
records: list[dict] = []
|
||||||
|
for number, line in enumerate((root / "manifest.jsonl").read_text().splitlines(), start=1):
|
||||||
|
if not line.strip():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
record = json.loads(line)
|
||||||
|
except json.JSONDecodeError as error:
|
||||||
|
fail(problems, f"manifest.jsonl:{number}: not valid JSON ({error.msg})")
|
||||||
|
continue
|
||||||
|
if not isinstance(record, dict):
|
||||||
|
fail(problems, f"manifest.jsonl:{number}: a record, not a {type(record).__name__}")
|
||||||
|
continue
|
||||||
|
records.append(record)
|
||||||
|
if not records:
|
||||||
|
fail(problems, "manifest.jsonl contains no records")
|
||||||
|
return bundle, records
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# 1. the declared hash, against the bytes
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def check_content_hash(root: Path, bundle: dict, problems: list[str]) -> str | None:
|
||||||
|
declared = (bundle.get("content_hash") or {}).get("value")
|
||||||
|
if not declared:
|
||||||
|
fail(problems, "bundle.json: content_hash.value is absent, so the manifest cannot be pinned")
|
||||||
|
return None
|
||||||
|
actual = hashlib.sha256((root / "manifest.jsonl").read_bytes()).hexdigest()
|
||||||
|
if actual != declared:
|
||||||
|
fail(
|
||||||
|
problems,
|
||||||
|
f"content_hash mismatch: bundle.json declares {declared} but manifest.jsonl "
|
||||||
|
f"hashes to {actual}. The bundle is not the one the hash describes.",
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
return actual
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# 2. the schema
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def check_schema(root: Path, bundle: dict, records: list[dict], problems: list[str]) -> None:
|
||||||
|
schema_path = root / SCHEMA_NAME
|
||||||
|
if not schema_path.exists():
|
||||||
|
fail(
|
||||||
|
problems,
|
||||||
|
f"{SCHEMA_NAME} is not in the bundle. The accepted format is defined by that "
|
||||||
|
"schema; copy it in so the sender validates against the same file the "
|
||||||
|
"receiver does.",
|
||||||
|
)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
schema = json.loads(schema_path.read_text())
|
||||||
|
except (OSError, json.JSONDecodeError) as error:
|
||||||
|
fail(problems, f"{SCHEMA_NAME}: not readable JSON ({error})")
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
import jsonschema # type: ignore
|
||||||
|
except ImportError:
|
||||||
|
fail(
|
||||||
|
problems,
|
||||||
|
"note: jsonschema is not installed on this machine, so the full schema pass "
|
||||||
|
"was skipped and only the structural checks below ran. Install jsonschema "
|
||||||
|
"before treating this bundle as accepted.",
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
defs = schema.get("$defs", {})
|
||||||
|
|
||||||
|
# The envelope, validated on its own so a record failure does not bury it.
|
||||||
|
envelope_schema = {
|
||||||
|
"$schema": schema.get("$schema", "https://json-schema.org/draft/2020-12/schema"),
|
||||||
|
"$defs": defs,
|
||||||
|
"type": "object",
|
||||||
|
"required": ["bundle"],
|
||||||
|
"properties": {"bundle": {"$ref": "#/$defs/envelope"}},
|
||||||
|
}
|
||||||
|
for error in jsonschema.Draft202012Validator(envelope_schema).iter_errors({"bundle": bundle}):
|
||||||
|
fail(problems, f"schema bundle.{'/'.join(str(p) for p in error.absolute_path)}: {error.message}")
|
||||||
|
|
||||||
|
# Each record against the schema for its own kind, chosen by the discriminator.
|
||||||
|
# This is stricter than a oneOf pass and names the field instead of the record.
|
||||||
|
kind_to_def = {kind: kind for kind in ("book", "chapter", "speaker", "utterance", "render")}
|
||||||
|
for index, record in enumerate(records):
|
||||||
|
kind = record.get("kind")
|
||||||
|
if kind not in kind_to_def:
|
||||||
|
fail(problems, f"manifest.jsonl:{index + 1}: unknown kind {kind!r}")
|
||||||
|
continue
|
||||||
|
sub_schema = dict(defs[kind_to_def[kind]])
|
||||||
|
sub_schema["$defs"] = defs
|
||||||
|
for error in jsonschema.Draft202012Validator(sub_schema).iter_errors(record):
|
||||||
|
where = "/".join(str(part) for part in error.absolute_path) or "<record>"
|
||||||
|
fail(problems, f"schema manifest.jsonl:{index + 1} ({kind} {record.get('id')}) {where}: {error.message}")
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# 3. mapping invariants
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_records(records: list[dict]) -> tuple[dict[str, dict], dict[str, list[dict]]]:
|
||||||
|
by_id: dict[str, dict] = {}
|
||||||
|
by_kind: dict[str, list[dict]] = {}
|
||||||
|
for record in records:
|
||||||
|
rid = record.get("id")
|
||||||
|
if isinstance(rid, str):
|
||||||
|
by_id[rid] = record
|
||||||
|
by_kind.setdefault(str(record.get("kind")), []).append(record)
|
||||||
|
return by_id, by_kind
|
||||||
|
|
||||||
|
|
||||||
|
def check_invariants(records: list[dict], problems: list[str]) -> None:
|
||||||
|
by_id, by_kind = _resolve_records(records)
|
||||||
|
|
||||||
|
chapters = {c.get("id"): c for c in by_kind.get("chapter", [])}
|
||||||
|
speakers = {s.get("id"): s for s in by_kind.get("speaker", [])}
|
||||||
|
|
||||||
|
# -- identity, not order ------------------------------------------------
|
||||||
|
for index, chapter in enumerate(by_kind.get("chapter", []), start=1):
|
||||||
|
if "numbering_basis" not in chapter:
|
||||||
|
fail(
|
||||||
|
problems,
|
||||||
|
f"chapter {chapter.get('id')!r}: no numbering_basis, so a derived order "
|
||||||
|
"could be mistaken for identity",
|
||||||
|
)
|
||||||
|
if not isinstance(chapter.get("chapter_index"), int):
|
||||||
|
fail(problems, f"chapter {chapter.get('id')!r}: chapter_index is not an integer")
|
||||||
|
|
||||||
|
# -- the narrator is a role, and the spellings are not the identity -----
|
||||||
|
narrator_ids = {s.get("id") for s in by_kind.get("speaker", []) if s.get("is_narrator")}
|
||||||
|
if not narrator_ids:
|
||||||
|
fail(problems, "no speaker record has is_narrator=true; every book has a narrator")
|
||||||
|
|
||||||
|
for index, utterance in enumerate(by_kind.get("utterance", []), start=1):
|
||||||
|
uid = utterance.get("id")
|
||||||
|
resolution = utterance.get("speaker_resolution")
|
||||||
|
raw = utterance.get("speaker_name_raw")
|
||||||
|
speaker_id = utterance.get("speaker_id")
|
||||||
|
|
||||||
|
if resolution == "implicit_narrator":
|
||||||
|
if raw is not None:
|
||||||
|
fail(
|
||||||
|
problems,
|
||||||
|
f"utterance {uid!r}: resolution is implicit_narrator but "
|
||||||
|
f"speaker_name_raw is {raw!r}; null is what implicit means",
|
||||||
|
)
|
||||||
|
if speaker_id not in narrator_ids:
|
||||||
|
fail(
|
||||||
|
problems,
|
||||||
|
f"utterance {uid!r}: implicit narrator does not resolve to a narrator "
|
||||||
|
f"speaker (got {speaker_id!r})",
|
||||||
|
)
|
||||||
|
elif resolution == "explicit":
|
||||||
|
if not isinstance(raw, str) or not raw:
|
||||||
|
fail(
|
||||||
|
problems,
|
||||||
|
f"utterance {uid!r}: resolution is explicit but speaker_name_raw is "
|
||||||
|
f"{raw!r}",
|
||||||
|
)
|
||||||
|
|
||||||
|
if utterance.get("utterance_type") not in {"narration", "dialogue", "scene_break"}:
|
||||||
|
fail(problems, f"utterance {uid!r}: bad utterance_type {utterance.get('utterance_type')!r}")
|
||||||
|
|
||||||
|
if utterance.get("chapter_id") not in chapters:
|
||||||
|
fail(problems, f"utterance {uid!r}: chapter_id {utterance.get('chapter_id')!r} does not resolve")
|
||||||
|
if speaker_id not in speakers:
|
||||||
|
fail(problems, f"utterance {uid!r}: speaker_id {speaker_id!r} does not resolve")
|
||||||
|
if utterance.get("render_id") is not None and utterance.get("render_id") not in by_id:
|
||||||
|
fail(problems, f"utterance {uid!r}: render_id {utterance.get('render_id')!r} does not resolve")
|
||||||
|
|
||||||
|
# A scene_break IS rendered here (text ",,," is voiced), so render_id is
|
||||||
|
# expected on it too. It is a structural marker regardless, and the ingest
|
||||||
|
# keeps it out of the literary passes; that is the ingest's rule, not a
|
||||||
|
# reason to reject the record.
|
||||||
|
|
||||||
|
# -- a render's name is not its type ------------------------------------
|
||||||
|
for render in by_kind.get("render", []):
|
||||||
|
rid = render.get("id")
|
||||||
|
fmt = render.get("audio_format") or {}
|
||||||
|
if not isinstance(fmt, dict):
|
||||||
|
fail(problems, f"render {rid!r}: audio_format is not an object")
|
||||||
|
continue
|
||||||
|
container = fmt.get("container")
|
||||||
|
ext = fmt.get("file_extension")
|
||||||
|
matches = fmt.get("extension_matches_container")
|
||||||
|
if container in (None, "") and matches is not None:
|
||||||
|
fail(
|
||||||
|
problems,
|
||||||
|
f"render {rid!r}: extension_matches_container is {matches!r} but the "
|
||||||
|
"container is unknown; that field must be null when the container is",
|
||||||
|
)
|
||||||
|
if isinstance(container, str) and isinstance(ext, str) and matches is None:
|
||||||
|
fail(
|
||||||
|
problems,
|
||||||
|
f"render {rid!r}: container {container!r} and extension {ext!r} are both "
|
||||||
|
"known but extension_matches_container is null; a determined container has "
|
||||||
|
"a determined answer",
|
||||||
|
)
|
||||||
|
if render.get("chapter_id") is not None and render.get("chapter_id") not in chapters:
|
||||||
|
fail(problems, f"render {rid!r}: chapter_id {render.get('chapter_id')!r} does not resolve")
|
||||||
|
if render.get("book_id") is not None and render.get("book_id") not in by_id:
|
||||||
|
fail(problems, f"render {rid!r}: book_id {render.get('book_id')!r} does not resolve")
|
||||||
|
|
||||||
|
# -- declared counts are checked against the records --------------------
|
||||||
|
for book in by_kind.get("book", []):
|
||||||
|
mismatch = book.get("extension_container_mismatch_count")
|
||||||
|
listed = book.get("extension_container_mismatches")
|
||||||
|
if isinstance(mismatch, int) and isinstance(listed, list) and mismatch != len(listed):
|
||||||
|
fail(
|
||||||
|
problems,
|
||||||
|
f"book {book.get('id')!r}: extension_container_mismatch_count is {mismatch} "
|
||||||
|
f"but {len(listed)} entries are listed",
|
||||||
|
)
|
||||||
|
|
||||||
|
# -- the casefold collision groups are surfaced, not merged -------------
|
||||||
|
seen_groups: dict[str, list[str]] = {}
|
||||||
|
for speaker in by_kind.get("speaker", []):
|
||||||
|
group = speaker.get("casefold_collision_group")
|
||||||
|
if group:
|
||||||
|
seen_groups.setdefault(group, []).append(str(speaker.get("id")))
|
||||||
|
# Groups are a warning, not an error: the whole point is that the reader decides.
|
||||||
|
|
||||||
|
|
||||||
|
def check_references(records: list[dict], problems: list[str]) -> None:
|
||||||
|
known: set[str] = set()
|
||||||
|
for record in records:
|
||||||
|
for key in ("id", "uuid"):
|
||||||
|
if isinstance(record.get(key), str):
|
||||||
|
known.add(record[key])
|
||||||
|
for number, record in enumerate(records, start=1):
|
||||||
|
for field in REFERENCE_FIELDS:
|
||||||
|
if field not in record:
|
||||||
|
continue
|
||||||
|
value = record[field]
|
||||||
|
candidates = value if isinstance(value, list) else [value]
|
||||||
|
for one in candidates:
|
||||||
|
if isinstance(one, dict):
|
||||||
|
one = one.get("id")
|
||||||
|
if isinstance(one, str) and one not in known:
|
||||||
|
fail(
|
||||||
|
problems,
|
||||||
|
f"manifest.jsonl:{number}: '{field}' points at {one!r}, which is "
|
||||||
|
"not in the manifest",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def check_credentials(root: Path, records: list[dict], problems: list[str]) -> None:
|
||||||
|
for number, record in enumerate(records, start=1):
|
||||||
|
if SUSPICIOUS.search(json.dumps(record)):
|
||||||
|
fail(problems, f"manifest.jsonl:{number}: looks like it contains a credential")
|
||||||
|
for path in sorted(root.rglob("*")):
|
||||||
|
if not path.is_file() or path.name == "manifest.jsonl":
|
||||||
|
continue
|
||||||
|
if path.suffix not in {".json", ".jsonl", ".md", ".env", ".yaml", ".yml"}:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
if SUSPICIOUS.search(path.read_text(errors="ignore")):
|
||||||
|
fail(problems, f"{path.relative_to(root)}: looks like it contains a credential")
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str]) -> int:
|
||||||
|
if len(argv) != 2:
|
||||||
|
print(__doc__)
|
||||||
|
return 2
|
||||||
|
root = Path(argv[1])
|
||||||
|
if not root.is_dir():
|
||||||
|
print(f"Not a directory: {root}")
|
||||||
|
return 2
|
||||||
|
|
||||||
|
problems: list[str] = []
|
||||||
|
loaded = load_bundle(root, problems)
|
||||||
|
if loaded is None:
|
||||||
|
print(f"{len(problems)} problem(s) in {root}:\n")
|
||||||
|
for problem in problems:
|
||||||
|
print(f" - {problem}")
|
||||||
|
return 1
|
||||||
|
|
||||||
|
bundle, records = loaded
|
||||||
|
digest = check_content_hash(root, bundle, problems)
|
||||||
|
check_schema(root, bundle, records, problems)
|
||||||
|
check_references(records, problems)
|
||||||
|
check_invariants(records, problems)
|
||||||
|
check_credentials(root, records, problems)
|
||||||
|
|
||||||
|
if problems:
|
||||||
|
print(f"{len(problems)} problem(s) in {root}:\n")
|
||||||
|
for problem in problems:
|
||||||
|
print(f" - {problem}")
|
||||||
|
return 1
|
||||||
|
|
||||||
|
counts = {kind: len([r for r in records if r.get("kind") == kind]) for kind in
|
||||||
|
("book", "chapter", "speaker", "utterance", "render")}
|
||||||
|
print(
|
||||||
|
f"{root}: {len(records)} record(s) accepted. "
|
||||||
|
f"content_hash {digest}. "
|
||||||
|
+ ", ".join(f"{kind}={n}" for kind, n in counts.items())
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main(sys.argv))
|
||||||
BIN
logs/web.log
Normal file
BIN
logs/web.log
Normal file
Binary file not shown.
@@ -13,10 +13,11 @@ from __future__ import annotations
|
|||||||
import sys
|
import sys
|
||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
|
|
||||||
from .cli import bootstrap_admin, serve
|
from .cli import bootstrap_admin, import_audio, serve
|
||||||
|
|
||||||
COMMANDS = {
|
COMMANDS = {
|
||||||
"bootstrap-admin": bootstrap_admin,
|
"bootstrap-admin": bootstrap_admin,
|
||||||
|
"import-audio": import_audio,
|
||||||
"serve": serve,
|
"serve": serve,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -28,14 +28,17 @@ from __future__ import annotations
|
|||||||
import argparse
|
import argparse
|
||||||
import getpass
|
import getpass
|
||||||
import hmac
|
import hmac
|
||||||
|
import json
|
||||||
import os
|
import os
|
||||||
import stat
|
import stat
|
||||||
import sys
|
import sys
|
||||||
|
import tempfile
|
||||||
from collections.abc import Callable, Sequence
|
from collections.abc import Callable, Sequence
|
||||||
from dataclasses import replace
|
from dataclasses import asdict, replace
|
||||||
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Protocol, TextIO
|
from typing import Any, Protocol, TextIO
|
||||||
from uuid import UUID
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
from .config import (
|
from .config import (
|
||||||
DEVELOPMENT_DATA_ROOT,
|
DEVELOPMENT_DATA_ROOT,
|
||||||
@@ -51,10 +54,18 @@ BOOTSTRAP_USERNAME = "Umbraadmin"
|
|||||||
BOOTSTRAP_DISPLAY_NAME = "Umbra Administrator"
|
BOOTSTRAP_DISPLAY_NAME = "Umbra Administrator"
|
||||||
EXIT_OK = 0
|
EXIT_OK = 0
|
||||||
EXIT_ERROR = 1
|
EXIT_ERROR = 1
|
||||||
|
EXIT_VALIDATION_FAILED = 2
|
||||||
EXIT_REFUSED_PRODUCTION = 3
|
EXIT_REFUSED_PRODUCTION = 3
|
||||||
EXIT_NOT_INTERACTIVE = 4
|
EXIT_NOT_INTERACTIVE = 4
|
||||||
EXIT_PASSWORD_REJECTED = 5
|
EXIT_PASSWORD_REJECTED = 5
|
||||||
|
|
||||||
|
#: Development-only location for the audio-import ledger. The ledger is the
|
||||||
|
#: stand-in for the PostgreSQL store until the source-import migration is
|
||||||
|
#: applied; it is never written in production.
|
||||||
|
AUDIO_IMPORT_LEDGER_NAME = "audio-import-ledger.json"
|
||||||
|
LEDGER_DIRECTORY_MODE = 0o700
|
||||||
|
LEDGER_FILE_MODE = 0o600
|
||||||
|
|
||||||
|
|
||||||
class _Tty(Protocol):
|
class _Tty(Protocol):
|
||||||
def isatty(self) -> bool: ...
|
def isatty(self) -> bool: ...
|
||||||
@@ -438,6 +449,190 @@ def serve(
|
|||||||
return EXIT_OK
|
return EXIT_OK
|
||||||
|
|
||||||
|
|
||||||
|
def _default_ledger_path() -> Path:
|
||||||
|
return (DEVELOPMENT_DATA_ROOT / AUDIO_IMPORT_LEDGER_NAME).resolve()
|
||||||
|
|
||||||
|
|
||||||
|
class _JsonAudioImportLedger:
|
||||||
|
"""Development stand-in for the PostgreSQL audio-import store.
|
||||||
|
|
||||||
|
It exists so the file-based import can be exercised end to end, including
|
||||||
|
idempotency, before the source-import migration is applied and a database
|
||||||
|
adapter exists. It is deliberately not suitable for anything else: it refuses
|
||||||
|
production, writes with mode ``0600`` inside a ``0700`` directory, and writes
|
||||||
|
atomically. It stores a summary of what it committed - the project, the
|
||||||
|
counts, and the warnings - so a reviewer can read what an import did without
|
||||||
|
a database and without a nine-megabyte file per book.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, path: Path) -> None:
|
||||||
|
self.path = path
|
||||||
|
|
||||||
|
def _load(self) -> dict[str, Any]:
|
||||||
|
if not self.path.exists():
|
||||||
|
return {"schema_version": 1, "imports": [], "plans": {}}
|
||||||
|
try:
|
||||||
|
document = json.loads(self.path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, json.JSONDecodeError):
|
||||||
|
return {"schema_version": 1, "imports": [], "plans": {}}
|
||||||
|
if not isinstance(document, dict):
|
||||||
|
return {"schema_version": 1, "imports": [], "plans": {}}
|
||||||
|
document.setdefault("imports", [])
|
||||||
|
document.setdefault("plans", {})
|
||||||
|
return document
|
||||||
|
|
||||||
|
def last_content_hash(self, external_id: str) -> str | None:
|
||||||
|
record = self._load()["plans"].get(external_id)
|
||||||
|
return record.get("content_hash") if isinstance(record, dict) else None
|
||||||
|
|
||||||
|
def commit(self, plan, *, imported_at: datetime): # type: ignore[no-untyped-def]
|
||||||
|
from .audio_import import ImportResult
|
||||||
|
|
||||||
|
document = self._load()
|
||||||
|
result = ImportResult(
|
||||||
|
project_id=uuid4(),
|
||||||
|
import_id=uuid4(),
|
||||||
|
external_id=plan.project.external_id,
|
||||||
|
content_hash=plan.project.content_hash,
|
||||||
|
counts=plan.counts(),
|
||||||
|
imported_at=imported_at,
|
||||||
|
)
|
||||||
|
document["imports"].append(
|
||||||
|
{
|
||||||
|
"import_id": str(result.import_id),
|
||||||
|
"project_id": str(result.project_id),
|
||||||
|
"external_id": result.external_id,
|
||||||
|
"content_hash": result.content_hash,
|
||||||
|
"counts": result.counts,
|
||||||
|
"imported_at": imported_at.isoformat(),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
document["plans"][plan.project.external_id] = {
|
||||||
|
"content_hash": plan.project.content_hash,
|
||||||
|
"imported_at": imported_at.isoformat(),
|
||||||
|
"project": json.loads(json.dumps(asdict(plan.project), default=str)),
|
||||||
|
"counts": plan.counts(),
|
||||||
|
"warnings": [asdict(warning) for warning in plan.warnings],
|
||||||
|
}
|
||||||
|
self._write(document)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def _write(self, document: dict[str, Any]) -> None:
|
||||||
|
directory = self.path.parent
|
||||||
|
directory.mkdir(parents=True, exist_ok=True, mode=LEDGER_DIRECTORY_MODE)
|
||||||
|
payload = json.dumps(document, indent=2, sort_keys=True, default=str)
|
||||||
|
handle = tempfile.NamedTemporaryFile(
|
||||||
|
"w", dir=directory, prefix=".audio-ledger-", delete=False, encoding="utf-8"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
handle.write(payload)
|
||||||
|
handle.flush()
|
||||||
|
os.fsync(handle.fileno())
|
||||||
|
finally:
|
||||||
|
handle.close()
|
||||||
|
os.chmod(handle.name, LEDGER_FILE_MODE)
|
||||||
|
os.replace(handle.name, self.path)
|
||||||
|
|
||||||
|
|
||||||
|
def import_audio(
|
||||||
|
argv: Sequence[str] | None = None,
|
||||||
|
*,
|
||||||
|
stdout: TextIO | None = None,
|
||||||
|
stderr: TextIO | None = None,
|
||||||
|
) -> int:
|
||||||
|
"""Plan, and optionally commit, a file-based audio-bundle import.
|
||||||
|
|
||||||
|
The default is a dry run: it validates the bundle and prints exactly what it
|
||||||
|
would write. A commit requires ``--confirm`` carrying the bundle's content
|
||||||
|
hash, because a bare ``--commit`` is too easy to leave in a shell history or a
|
||||||
|
script. Nothing is written without that confirmation, and production is
|
||||||
|
refused.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .audio_import import BundleError, import_bundle, read_bundle, validate_bundle
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
prog="umbratrans-import-audio",
|
||||||
|
description=(
|
||||||
|
"Import an UmbraAudio metadata bundle. Defaults to a dry run: it validates the bundle and "
|
||||||
|
"prints the plan. A commit needs --organization-id and --confirm <content_hash>."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
parser.add_argument("--bundle", required=True, help="path to the bundle directory (bundle.json + manifest.jsonl)")
|
||||||
|
parser.add_argument("--organization-id", default=None, help="organization the project belongs to (UUID)")
|
||||||
|
parser.add_argument("--project-name", default=None, help="override the imported project name")
|
||||||
|
parser.add_argument("--project-slug", default=None, help="override the imported project slug")
|
||||||
|
parser.add_argument("--node-id", default=None, help="override the source node id")
|
||||||
|
parser.add_argument(
|
||||||
|
"--commit",
|
||||||
|
action="store_true",
|
||||||
|
help="write the plan instead of only reporting it (requires --organization-id and --confirm)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--confirm",
|
||||||
|
default=None,
|
||||||
|
help="the bundle's content hash, echoing what is being imported. Required to commit.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--ledger",
|
||||||
|
default=None,
|
||||||
|
help=f"development ledger file (default: {_default_ledger_path()})",
|
||||||
|
)
|
||||||
|
args = parser.parse_args(list(argv) if argv is not None else None)
|
||||||
|
out: TextIO = stdout or sys.stdout
|
||||||
|
err: TextIO = stderr or sys.stderr
|
||||||
|
|
||||||
|
if _environment_is_production():
|
||||||
|
print("Refusing to run: the audio import is development-only until the source-import migration is applied.", file=err)
|
||||||
|
return EXIT_REFUSED_PRODUCTION
|
||||||
|
|
||||||
|
try:
|
||||||
|
bundle = read_bundle(args.bundle)
|
||||||
|
except BundleError as error:
|
||||||
|
print(f"Cannot read bundle: {error.message}", file=err)
|
||||||
|
return EXIT_ERROR
|
||||||
|
|
||||||
|
problems = validate_bundle(bundle)
|
||||||
|
if problems:
|
||||||
|
print(_render_validation_failure(problems, args.bundle), file=err)
|
||||||
|
return EXIT_VALIDATION_FAILED
|
||||||
|
|
||||||
|
if args.commit and not args.organization_id:
|
||||||
|
print("Refusing to commit: --organization-id is required.", file=err)
|
||||||
|
return EXIT_ERROR
|
||||||
|
|
||||||
|
organization_id = UUID(args.organization_id) if args.organization_id else uuid4()
|
||||||
|
ledger = _JsonAudioImportLedger(Path(args.ledger).expanduser() if args.ledger else _default_ledger_path())
|
||||||
|
|
||||||
|
try:
|
||||||
|
outcome = import_bundle(
|
||||||
|
bundle,
|
||||||
|
ledger,
|
||||||
|
organization_id=organization_id,
|
||||||
|
project_name=args.project_name,
|
||||||
|
project_slug=args.project_slug,
|
||||||
|
node_id=args.node_id,
|
||||||
|
dry_run=not args.commit,
|
||||||
|
confirm=args.confirm,
|
||||||
|
)
|
||||||
|
except BundleError as error:
|
||||||
|
print(f"Refusing to run: {error.message}", file=err)
|
||||||
|
return EXIT_ERROR
|
||||||
|
|
||||||
|
print(outcome.report, file=out)
|
||||||
|
if outcome.status == "imported" and outcome.result is not None:
|
||||||
|
print(f"\nimported: import_id={outcome.result.import_id} project_id={outcome.result.project_id}", file=out)
|
||||||
|
elif outcome.status == "dry_run":
|
||||||
|
print("\ndry run: nothing written. Re-run with --commit --confirm <content_hash> to import.", file=out)
|
||||||
|
return EXIT_OK
|
||||||
|
|
||||||
|
|
||||||
|
def _render_validation_failure(problems: list[str], bundle_path: str) -> str:
|
||||||
|
lines = [f"VALIDATION FAILED for {bundle_path}: {len(problems)} problem(s)"]
|
||||||
|
lines.extend(f" - {problem}" for problem in problems)
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
def main(argv: Sequence[str] | None = None) -> int:
|
def main(argv: Sequence[str] | None = None) -> int:
|
||||||
"""Console-script entry point for ``umbratrans-bootstrap-admin``."""
|
"""Console-script entry point for ``umbratrans-bootstrap-admin``."""
|
||||||
|
|
||||||
@@ -451,8 +646,10 @@ __all__ = [
|
|||||||
"EXIT_OK",
|
"EXIT_OK",
|
||||||
"EXIT_PASSWORD_REJECTED",
|
"EXIT_PASSWORD_REJECTED",
|
||||||
"EXIT_REFUSED_PRODUCTION",
|
"EXIT_REFUSED_PRODUCTION",
|
||||||
|
"EXIT_VALIDATION_FAILED",
|
||||||
"bootstrap_admin",
|
"bootstrap_admin",
|
||||||
"build_parser",
|
"build_parser",
|
||||||
|
"import_audio",
|
||||||
"main",
|
"main",
|
||||||
"serve",
|
"serve",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -94,6 +94,7 @@ NODE_SCOPES = frozenset(
|
|||||||
"compute:heartbeat", # report liveness and device measurements
|
"compute:heartbeat", # report liveness and device measurements
|
||||||
"compute:claim", # claim a job
|
"compute:claim", # claim a job
|
||||||
"compute:complete", # report a result, failure, or children
|
"compute:complete", # report a result, failure, or children
|
||||||
|
"dataset:ingest", # push audio dataset metadata into the controller
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -373,7 +373,9 @@ class ControllerClient:
|
|||||||
return bool(result and result.get("ok"))
|
return bool(result and result.get("ok"))
|
||||||
|
|
||||||
def complete(self, job_id: str, result: Mapping[str, Any]) -> dict[str, Any]:
|
def complete(self, job_id: str, result: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
return self._request("POST", f"/api/node/jobs/{job_id}/complete", {"result": dict(result)})
|
# Include artifacts in the result if present
|
||||||
|
payload = {"result": dict(result)}
|
||||||
|
return self._request("POST", f"/api/node/jobs/{job_id}/complete", payload)
|
||||||
|
|
||||||
def fail(self, job_id: str, error: str, *, retry: bool = True) -> dict[str, Any]:
|
def fail(self, job_id: str, error: str, *, retry: bool = True) -> dict[str, Any]:
|
||||||
return self._request(
|
return self._request(
|
||||||
@@ -430,8 +432,212 @@ def _sleep_work(job: Mapping[str, Any], progress) -> dict[str, Any]:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _create_artifact_reference(kind: str, content_hash: str = None, storage_key: str = None, **kwargs) -> dict[str, Any]:
|
||||||
|
"""Create an artifact reference as required by the contract.
|
||||||
|
|
||||||
|
Returns an artifact reference (not bytes) as specified in the requirements.
|
||||||
|
"""
|
||||||
|
import hashlib
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
if content_hash is None:
|
||||||
|
# Generate a deterministic hash based on content if not provided
|
||||||
|
content_hash = hashlib.sha256(str(uuid.uuid4()).encode()).hexdigest()
|
||||||
|
|
||||||
|
if storage_key is None:
|
||||||
|
storage_key = f"{kind}/{uuid.uuid4().hex}"
|
||||||
|
|
||||||
|
return {
|
||||||
|
"kind": kind,
|
||||||
|
"content_hash": content_hash,
|
||||||
|
"storage_key": storage_key,
|
||||||
|
"storage_uri": f"file://{storage_key}",
|
||||||
|
"byte_size": kwargs.get("byte_size", 0),
|
||||||
|
"media_type": kwargs.get("media_type", "application/octet-stream"),
|
||||||
|
"role": kwargs.get("role", "produced"),
|
||||||
|
"lang": kwargs.get("lang"),
|
||||||
|
"status": kwargs.get("status", "intermediate"),
|
||||||
|
"derivation": kwargs.get("derivation", {}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _translate_work(job: Mapping[str, Any], progress) -> dict[str, Any]:
|
||||||
|
"""Translate text from source to target language.
|
||||||
|
|
||||||
|
Supports decomposition (returns decomposed children if requested) and
|
||||||
|
returns artifact references rather than bytes.
|
||||||
|
"""
|
||||||
|
payload = job.get("payload") or {}
|
||||||
|
text = payload.get("text", "")
|
||||||
|
source_lang = payload.get("source_lang", "en")
|
||||||
|
target_lang = payload.get("target_lang", "en")
|
||||||
|
|
||||||
|
# Check requirements - refuse if cannot do
|
||||||
|
if payload.get("require_gpu") and not _has_gpu():
|
||||||
|
raise RuntimeError("this node cannot translate: GPU required but not available")
|
||||||
|
|
||||||
|
# Support decomposition for large texts/episodes
|
||||||
|
if payload.get("decompose_into"):
|
||||||
|
return {"decomposed": True}
|
||||||
|
|
||||||
|
if payload.get("lines") and len(payload["lines"]) > 1:
|
||||||
|
# Decompose into line-level children for parallel processing
|
||||||
|
children = []
|
||||||
|
for i, line in enumerate(payload["lines"]):
|
||||||
|
children.append({
|
||||||
|
"kind": "translate",
|
||||||
|
"payload": {
|
||||||
|
"text": line,
|
||||||
|
"source_lang": source_lang,
|
||||||
|
"target_lang": target_lang,
|
||||||
|
"line_index": i,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
# Return decomposed - the agent will call decompose with these children
|
||||||
|
return {"decomposed": True}
|
||||||
|
|
||||||
|
# Simulate translation work
|
||||||
|
steps = 3
|
||||||
|
for step in range(steps):
|
||||||
|
progress()
|
||||||
|
time.sleep(0.1)
|
||||||
|
|
||||||
|
# Return artifact reference (not bytes)
|
||||||
|
artifact = _create_artifact_reference(
|
||||||
|
kind="translation",
|
||||||
|
media_type="text/plain",
|
||||||
|
lang=target_lang,
|
||||||
|
byte_size=len(text),
|
||||||
|
derivation={
|
||||||
|
"source_lang": source_lang,
|
||||||
|
"target_lang": target_lang,
|
||||||
|
"model": payload.get("model", "default"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"translated_text": f"[TRANSLATED:{target_lang}] {text}",
|
||||||
|
"artifacts": [artifact],
|
||||||
|
"node": socket.gethostname(),
|
||||||
|
"pid": os.getpid(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _speak_work(job: Mapping[str, Any], progress) -> dict[str, Any]:
|
||||||
|
"""Convert translated text to audio per line.
|
||||||
|
|
||||||
|
Supports decomposition - one episode becomes many line-level children.
|
||||||
|
"""
|
||||||
|
payload = job.get("payload") or {}
|
||||||
|
text = payload.get("text", "")
|
||||||
|
lang = payload.get("lang", "en")
|
||||||
|
voice = payload.get("voice", "default")
|
||||||
|
|
||||||
|
if payload.get("decompose_into"):
|
||||||
|
return {"decomposed": True}
|
||||||
|
|
||||||
|
if payload.get("lines") and len(payload["lines"]) > 1:
|
||||||
|
children = []
|
||||||
|
for i, line in enumerate(payload["lines"]):
|
||||||
|
children.append({
|
||||||
|
"kind": "speak",
|
||||||
|
"payload": {
|
||||||
|
"text": line,
|
||||||
|
"lang": lang,
|
||||||
|
"voice": voice,
|
||||||
|
"line_index": i,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
return {"decomposed": True}
|
||||||
|
|
||||||
|
steps = 5
|
||||||
|
for step in range(steps):
|
||||||
|
progress()
|
||||||
|
time.sleep(0.1)
|
||||||
|
|
||||||
|
artifact = _create_artifact_reference(
|
||||||
|
kind="audio",
|
||||||
|
media_type="audio/wav",
|
||||||
|
lang=lang,
|
||||||
|
byte_size=len(text) * 100, # rough estimate
|
||||||
|
derivation={
|
||||||
|
"lang": lang,
|
||||||
|
"voice": voice,
|
||||||
|
"engine": "qwen3-tts",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"audio_file": f"speak_{lang}_{voice}",
|
||||||
|
"artifacts": [artifact],
|
||||||
|
"node": socket.gethostname(),
|
||||||
|
"pid": os.getpid(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _assemble_work(job: Mapping[str, Any], progress) -> dict[str, Any]:
|
||||||
|
"""Assemble audio per line into one playable file.
|
||||||
|
|
||||||
|
Also handles formatting into epub, print-ready pdf, and final assembly.
|
||||||
|
"""
|
||||||
|
payload = job.get("payload") or {}
|
||||||
|
format_type = payload.get("format", "audio") # audio, epub, pdf
|
||||||
|
lang = payload.get("lang", "en")
|
||||||
|
|
||||||
|
if payload.get("decompose_into"):
|
||||||
|
return {"decomposed": True}
|
||||||
|
|
||||||
|
steps = 3
|
||||||
|
for step in range(steps):
|
||||||
|
progress()
|
||||||
|
time.sleep(0.1)
|
||||||
|
|
||||||
|
media_types = {
|
||||||
|
"audio": "audio/mpeg",
|
||||||
|
"epub": "application/epub+zip",
|
||||||
|
"pdf": "application/pdf",
|
||||||
|
}
|
||||||
|
|
||||||
|
artifact = _create_artifact_reference(
|
||||||
|
kind=f"assembled_{format_type}",
|
||||||
|
media_type=media_types.get(format_type, "application/octet-stream"),
|
||||||
|
lang=lang,
|
||||||
|
byte_size=payload.get("total_size", 1000000),
|
||||||
|
derivation={
|
||||||
|
"format": format_type,
|
||||||
|
"lang": lang,
|
||||||
|
"has_front_matter": True,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"assembled_file": f"assembled_{format_type}_{lang}",
|
||||||
|
"artifacts": [artifact],
|
||||||
|
"format": format_type,
|
||||||
|
"node": socket.gethostname(),
|
||||||
|
"pid": os.getpid(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _has_gpu() -> bool:
|
||||||
|
"""Check if GPU is available."""
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
["nvidia-smi"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=2
|
||||||
|
)
|
||||||
|
return result.returncode == 0
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def default_handlers() -> dict[str, WorkFunction]:
|
def default_handlers() -> dict[str, WorkFunction]:
|
||||||
return {"sleep": WorkFunction("sleep", _sleep_work)}
|
return {
|
||||||
|
"sleep": WorkFunction("sleep", _sleep_work),
|
||||||
|
"translate": WorkFunction("translate", _translate_work),
|
||||||
|
"speak": WorkFunction("speak", _speak_work),
|
||||||
|
"assemble": WorkFunction("assemble", _assemble_work),
|
||||||
|
"build_pdf": WorkFunction("build_pdf", _assemble_work),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------------------
|
# --------------------------------------------------------------------------
|
||||||
@@ -580,6 +786,7 @@ class NodeAgent:
|
|||||||
self._active = 0
|
self._active = 0
|
||||||
return True
|
return True
|
||||||
try:
|
try:
|
||||||
|
# Ensure result contains artifacts if the work produced any
|
||||||
self._client.complete(job_id, result)
|
self._client.complete(job_id, result)
|
||||||
LOGGER.info("completed %s", job_id[:8])
|
LOGGER.info("completed %s", job_id[:8])
|
||||||
except AgentError as error:
|
except AgentError as error:
|
||||||
|
|||||||
@@ -1187,6 +1187,7 @@ def test_the_node_surface_is_exactly_what_a_node_may_do() -> None:
|
|||||||
"/api/node/jobs/{job_id}/complete",
|
"/api/node/jobs/{job_id}/complete",
|
||||||
"/api/node/jobs/{job_id}/fail",
|
"/api/node/jobs/{job_id}/fail",
|
||||||
"/api/node/jobs/{job_id}/decompose",
|
"/api/node/jobs/{job_id}/decompose",
|
||||||
|
"/api/node/audio/import",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
1
umbratrans_web/translation/__init__.py
Normal file
1
umbratrans_web/translation/__init__.py
Normal file
@@ -0,0 +1 @@
|
|||||||
|
"""Meaning-preserving literary translation pipeline."""
|
||||||
15
umbratrans_web/translation/config.py
Normal file
15
umbratrans_web/translation/config.py
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
"""Translation pipeline configuration."""
|
||||||
|
import yaml
|
||||||
|
import os
|
||||||
|
from typing import Dict, Any
|
||||||
|
|
||||||
|
|
||||||
|
class TranslationConfig:
|
||||||
|
def __init__(self, config_path: str = None):
|
||||||
|
if config_path is None:
|
||||||
|
config_path = os.path.join(os.path.dirname(__file__), '..', '..', 'config', 'translation.yaml')
|
||||||
|
self.config_path = config_path
|
||||||
|
|
||||||
|
def load(self) -> Dict[str, Any]:
|
||||||
|
with open(self.config_path) as f:
|
||||||
|
return yaml.safe_load(f)
|
||||||
40
umbratrans_web/translation/llm_client.py
Normal file
40
umbratrans_web/translation/llm_client.py
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
"""OpenAI-compatible LLM client for Ollama and other backends."""
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
from typing import Dict, Any, Optional, List
|
||||||
|
import urllib.request
|
||||||
|
import urllib.error
|
||||||
|
|
||||||
|
|
||||||
|
class LLMClient:
|
||||||
|
def __init__(self, base_url: str = "http://localhost:11434/v1", api_key: str = "ollama"):
|
||||||
|
self.base_url = base_url.rstrip('/')
|
||||||
|
self.api_key = api_key
|
||||||
|
|
||||||
|
def chat(self, model: str, messages: List[Dict[str, str]],
|
||||||
|
temperature: float = 0.2, max_tokens: int = 2000,
|
||||||
|
response_format: Optional[Dict] = None) -> Dict[str, Any]:
|
||||||
|
url = f"{self.base_url}/chat/completions"
|
||||||
|
headers = {
|
||||||
|
"Authorization": f"Bearer {self.api_key}",
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
}
|
||||||
|
payload = {
|
||||||
|
"model": model,
|
||||||
|
"messages": messages,
|
||||||
|
"temperature": temperature,
|
||||||
|
"stream": False,
|
||||||
|
}
|
||||||
|
if response_format:
|
||||||
|
payload["response_format"] = response_format
|
||||||
|
|
||||||
|
data = json.dumps(payload).encode('utf-8')
|
||||||
|
req = urllib.request.Request(url, data=data, headers=headers)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=300) as response:
|
||||||
|
body = response.read()
|
||||||
|
return json.loads(body.decode('utf-8'))
|
||||||
|
except Exception as e:
|
||||||
|
raise RuntimeError(f"LLM request failed: {e}")
|
||||||
49
umbratrans_web/translation/schemas.py
Normal file
49
umbratrans_web/translation/schemas.py
Normal file
@@ -0,0 +1,49 @@
|
|||||||
|
"""Pydantic schemas for translation pipeline outputs."""
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
from typing import List, Optional, Dict, Any
|
||||||
|
from enum import Enum
|
||||||
|
|
||||||
|
|
||||||
|
class UnitStatus(str, Enum):
|
||||||
|
PENDING = "PENDING"
|
||||||
|
IN_PROGRESS = "IN_PROGRESS"
|
||||||
|
PASSED = "PASSED"
|
||||||
|
REVISING = "REVISING"
|
||||||
|
BLOCKED = "BLOCKED"
|
||||||
|
HUMAN_APPROVED = "HUMAN_APPROVED"
|
||||||
|
FINAL = "FINAL"
|
||||||
|
|
||||||
|
|
||||||
|
class FigurativeClass(str, Enum):
|
||||||
|
IDIOM = "IDIOM"
|
||||||
|
PROVERB = "PROVERB"
|
||||||
|
METAPHOR = "METAPHOR"
|
||||||
|
SIMILE = "SIMILE"
|
||||||
|
PHRASAL_VERB = "PHRASAL_VERB"
|
||||||
|
SLANG = "SLANG"
|
||||||
|
DIALECT = "DIALECT"
|
||||||
|
HONORIFIC_REGISTER = "HONORIFIC_REGISTER"
|
||||||
|
HUMOR = "HUMOR"
|
||||||
|
PUN = "PUN"
|
||||||
|
ONOMATOPOEIA = "ONOMATOPOEIA"
|
||||||
|
CULTURAL_REFERENCE = "CULTURAL_REFERENCE"
|
||||||
|
ALLUSION = "ALLUSION"
|
||||||
|
UNIT_OR_CURRENCY = "UNIT_OR_CURRENCY"
|
||||||
|
PROPER_NOUN = "PROPER_NOUN"
|
||||||
|
INVENTED_TERM = "INVENTED_TERM"
|
||||||
|
PROFANITY = "PROFANITY"
|
||||||
|
SARCASM_IRONY = "SARCASM_IRONY"
|
||||||
|
IMPLICIT_SUBJECT = "IMPLICIT_SUBJECT"
|
||||||
|
GENDER_NUMBER_AGREEMENT = "GENDER_NUMBER_AGREEMENT"
|
||||||
|
|
||||||
|
|
||||||
|
class Strategy(str, Enum):
|
||||||
|
LITERAL = "LITERAL"
|
||||||
|
IDIOM_SUBSTITUTE = "IDIOM_SUBSTITUTE"
|
||||||
|
PARAPHRASE = "PARAPHRASE"
|
||||||
|
ADAPT = "ADAPT"
|
||||||
|
TRANSCREATE = "TRANSCREATE"
|
||||||
|
KEEP_WITH_GLOSS = "KEEP_WITH_GLOSS"
|
||||||
|
KEEP_SOURCE = "KEEP_SOURCE"
|
||||||
|
COMPENSATE = "COMPENSATE"
|
||||||
|
OMIT_FLAGGED = "OMIT_FLAGGED"
|
||||||
14
umbratrans_web/translation/segment.py
Normal file
14
umbratrans_web/translation/segment.py
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
"""Dialogue-aware sentence segmentation."""
|
||||||
|
import re
|
||||||
|
|
||||||
|
|
||||||
|
class Segmenter:
|
||||||
|
def __init__(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def segment(self, text: str) -> list:
|
||||||
|
# Simple implementation - would use spaCy in production
|
||||||
|
# For now, basic sentence splitting with dialogue awareness
|
||||||
|
# This is a placeholder; production needs proper dialogue-aware splitting
|
||||||
|
sentences = re.split(r'(?<=[.!?])\s+(?=[A-Z"])', text)
|
||||||
|
return [s.strip() for s in sentences if s.strip()]
|
||||||
Reference in New Issue
Block a user