diff --git a/config/translation.yaml b/config/translation.yaml new file mode 100644 index 0000000..135644e --- /dev/null +++ b/config/translation.yaml @@ -0,0 +1,36 @@ +llm: + base_url: http://localhost:11434/v1 + models: + analyst: "gemma4:26b" + drafter: "gemma4:26b" + critic: "gemma4:26b" + backtranslator: "gemma4:26b" + temperature: + analyst: 0.2 + drafter: 0.4 + critic: 0.3 + backtranslator: 0.1 + json_retries: 2 + +embeddings: + model: "sentence-transformers/LaBSE" + +gates: + g7_semantic_sim_min: 0.80 + g6_score_min: 4 + max_revisions: 3 + +context: + prev_paragraphs: 3 + next_paragraphs: 2 + +languages: + - zh + - ja + - ko + - de + - es + - fr + - ru + - it + - pt diff --git a/docs/audio-system-manifest-v1.schema.json b/docs/audio-system-manifest-v1.schema.json new file mode 100644 index 0000000..35fee2b --- /dev/null +++ b/docs/audio-system-manifest-v1.schema.json @@ -0,0 +1,336 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://umbra.invalid/schemas/umbracontrol/audio-bundle/v1/manifest.schema.json", + "title": "UmbraAudio metadata bundle, v1, as accepted by UmbraTrans", + "description": "Validates the wrapper the controller builds from a bundle directory: {bundle: , records: []}. This is the bundle you already produce, narrowed - field names are unchanged, and nothing is renamed on the wire. Required fields here are a superset of the ones your own validator enforces, because these are the fields the mapping reads.", + "type": "object", + "required": ["bundle", "records"], + "additionalProperties": false, + "properties": { + "bundle": { "$ref": "#/$defs/envelope" }, + "records": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/record" } + } + }, + + "$defs": { + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "iso8601": { "type": "string", "format": "date-time" }, + "relativeTo": { "const": "book_directory" }, + + "envelope": { + "title": "bundle.json", + "type": "object", + "required": [ + "bundle_format_version", + "content_hash", + "generated_at", + "source", + "counts", + "record_kinds", + "scope" + ], + "properties": { + "bundle_format_version": { "const": 1 }, + "content_hash": { + "type": "object", + "required": ["algorithm", "of", "value"], + "properties": { + "algorithm": { "const": "sha256" }, + "of": { "type": "string" }, + "value": { "$ref": "#/$defs/sha256" } + } + }, + "generated_at": { "$ref": "#/$defs/iso8601" }, + "source": { + "type": "object", + "required": ["system", "node_id"], + "properties": { + "system": { "type": "string", "minLength": 1 }, + "node_id": { "type": "string", "minLength": 1 }, + "product_version": { "type": "string" }, + "database_schema_fingerprint": { "type": "string" } + } + }, + "counts": { + "type": "object", + "required": ["books", "chapters", "utterances", "speakers", "render_records", "records_total"], + "properties": { + "books": { "type": "integer", "minimum": 1 }, + "chapters": { "type": "integer", "minimum": 0 }, + "utterances": { "type": "integer", "minimum": 0 }, + "speakers": { "type": "integer", "minimum": 0 }, + "render_records": { "type": "integer", "minimum": 0 }, + "records_total": { "type": "integer", "minimum": 1 } + } + }, + "record_kinds": { + "type": "object", + "required": ["discriminator", "values"], + "properties": { + "discriminator": { "const": "kind" }, + "values": { + "type": "array", + "allOf": [ + { "contains": { "const": "book" } }, + { "contains": { "const": "chapter" } }, + { "contains": { "const": "speaker" } }, + { "contains": { "const": "utterance" } }, + { "contains": { "const": "render" } } + ] + } + } + }, + "scope": { + "type": "object", + "required": ["books_full_detail"], + "properties": { + "books_full_detail": { "type": "array", "minItems": 1, "items": { "type": "string" } }, + "books_summary_only": { "type": "array", "items": { "type": "string" } }, + "note": { "type": "string" } + } + }, + "required_fields": { "type": "object" } + } + }, + + "record": { + "title": "one manifest.jsonl record", + "oneOf": [ + { "$ref": "#/$defs/book" }, + { "$ref": "#/$defs/chapter" }, + { "$ref": "#/$defs/speaker" }, + { "$ref": "#/$defs/utterance" }, + { "$ref": "#/$defs/render" } + ] + }, + + "book": { + "type": "object", + "required": [ + "id", "kind", "generated_at", "name", "chapters", + "chapter_count", "utterance_count", "language_designations" + ], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "kind": { "const": "book" }, + "generated_at": { "$ref": "#/$defs/iso8601" }, + "name": { "type": "string", "minLength": 1 }, + "title": { "type": ["string", "null"] }, + "status": { "type": ["string", "null"] }, + "chapters": { "type": "array", "minItems": 1, "items": { "type": "string" } }, + "chapter_count": { "type": "integer", "minimum": 0 }, + "utterance_count": { "type": "integer", "minimum": 0 }, + "speaker_count": { "type": "integer", "minimum": 0 }, + "novel_chapter_count": { "type": "integer" }, + "front_and_back_matter_count": { "type": "integer" }, + "published_artifact_ids": { "type": "array", "items": { "type": "string" } }, + "language_designations": { + "type": "array", + "items": { "type": "string" }, + "description": "Empty is an accepted answer, not an error. Target languages are decided on the translation side." + }, + "language_designation_note": { "type": "string" }, + "extension_container_mismatch_count": { "type": "integer", "minimum": 0 }, + "extension_container_mismatches": { "type": "array" }, + "speaker_names_without_a_profile": { "type": "array" }, + "declared_total_chapters": { "type": ["integer", "null"] }, + "declared_total_words": { "type": ["integer", "null"] }, + "declared_vs_measured_note": { "type": ["string", "null"] }, + "numbering": { "type": "object" }, + "path_relative_to": { "type": ["string", "null"] }, + "source_document_format": { "type": ["string", "null"] }, + "source_path_redacted": { "type": ["string", "null"] }, + "created_at": { "type": ["string", "null"] }, + "updated_at": { "type": ["string", "null"] } + } + }, + + "chapter": { + "type": "object", + "required": [ + "id", "kind", "generated_at", "book_id", "chapter_index", + "database_row_id", "title", "source_text_sha256", "renders", + "numbering_basis", "utterance_count" + ], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "kind": { "const": "chapter" }, + "generated_at": { "$ref": "#/$defs/iso8601" }, + "book_id": { "type": "string", "minLength": 1 }, + "chapter_index": { + "type": "integer", + "description": "Reading order. Negative is front matter. NEVER identity." + }, + "database_row_id": { "type": "integer", "description": "Source row id. Provenance only." }, + "title": { "type": ["string", "null"] }, + "source_text_sha256": { "$ref": "#/$defs/sha256" }, + "source_text_char_count": { "type": "integer", "minimum": 0 }, + "word_count": { "type": "integer", "minimum": 0 }, + "utterance_count": { "type": "integer", "minimum": 0 }, + "renders": { "type": "array", "items": { "type": "string" } }, + "numbering_basis": { + "type": "object", + "description": "Required so a derived order can never be mistaken for identity." + }, + "status": { "type": ["string", "null"] } + } + }, + + "speaker": { + "type": "object", + "required": [ + "id", "kind", "generated_at", "primary_name", "has_profile", + "is_narrator", "aliases", "utterance_count", "utterance_count_casefold_grouped" + ], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "kind": { "const": "speaker" }, + "generated_at": { "$ref": "#/$defs/iso8601" }, + "primary_name": { "type": "string", "minLength": 1 }, + "aliases": { "type": "array", "items": { "type": "string" } }, + "has_profile": { "type": "boolean" }, + "is_narrator": { "type": "boolean" }, + "is_group": { "type": "boolean" }, + "unidentified": { "type": "boolean" }, + "utterance_count": { "type": "integer", "minimum": 0 }, + "utterance_count_casefold_grouped": { "type": "integer", "minimum": 0 }, + "utterance_count_including_implicit_narrator": { "type": "integer", "minimum": 0 }, + "casefold_collision_group": { "type": ["string", "null"] }, + "chapters_appeared": { "type": "array" }, + "name_is_distinct_from_spellings": { "type": "array", "items": { "type": "string" } }, + "voice_review_status": { "type": ["string", "null"] }, + "voice_review_recorded": { "type": ["boolean", "null"] }, + "tts_engine_used": { "type": ["string", "null"] }, + "tts_engine_decided_at": { "type": ["string", "null"] }, + "tts_engine_fallback_reason": { "type": ["string", "null"] }, + "assigned_edge_voice": { "type": ["string", "null"] } + } + }, + + "utterance": { + "type": "object", + "required": [ + "id", "kind", "generated_at", "book_id", "chapter_id", + "utterance_index", "text", "text_sha256", "utterance_type", + "speaker_id", "speaker_name_raw", "speaker_name_resolved", + "speaker_resolution", "render_id", "needs_review" + ], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "kind": { "const": "utterance" }, + "generated_at": { "$ref": "#/$defs/iso8601" }, + "book_id": { "type": "string", "minLength": 1 }, + "chapter_id": { "type": "string", "minLength": 1 }, + "database_row_id": { "type": "integer" }, + "utterance_index": { "type": "integer", "minimum": 0 }, + "utterance_type": { "enum": ["narration", "dialogue", "scene_break"] }, + "text": { "type": "string" }, + "text_sha256": { "$ref": "#/$defs/sha256" }, + "text_char_count": { "type": "integer", "minimum": 0 }, + "speaker_id": { "type": "string", "minLength": 1 }, + "speaker_name_raw": { + "type": ["string", "null"], + "description": "Exactly what is stored. null means narrator, not missing." + }, + "speaker_name_resolved": { "type": "string", "minLength": 1 }, + "speaker_resolution": { "enum": ["explicit", "implicit_narrator"] }, + "render_id": { "type": ["string", "null"] }, + "attribution_confidence": { "type": ["number", "null"] }, + "attribution_uncertain": { "type": ["boolean", "null"] }, + "attribution_evidence_kind": { "type": ["string", "null"] }, + "needs_review": { "type": "boolean" }, + "review_reason": { "type": ["string", "null"] }, + "emotion": { "type": ["string", "null"] }, + "pace": { "type": ["string", "null"] }, + "instruct_is_set": { "type": ["boolean", "null"] }, + "status": { "type": ["string", "null"] } + }, + "allOf": [ + { + "if": { + "properties": { "speaker_resolution": { "const": "implicit_narrator" } }, + "required": ["speaker_resolution"] + }, + "then": { + "properties": { "speaker_name_raw": { "type": "null" } } + } + }, + { + "if": { + "properties": { "speaker_resolution": { "const": "explicit" } }, + "required": ["speaker_resolution"] + }, + "then": { + "properties": { "speaker_name_raw": { "type": "string", "minLength": 1 } } + } + } + ] + }, + + "render": { + "type": "object", + "required": [ + "id", "kind", "generated_at", "render_role", "path", "relative_to", + "path_is_absolute", "audio_format", "duration_ms", "byte_size", + "content_sha256", "book_id", "inputs" + ], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "kind": { "const": "render" }, + "generated_at": { "$ref": "#/$defs/iso8601" }, + "render_role": { + "enum": [ + "utterance_audio", "chapter_audio", "distribution_track", + "assembled_book", "voice_reference" + ] + }, + "render_stage": { "type": ["string", "null"] }, + "book_id": { "type": ["string", "null"] }, + "chapter_id": { + "type": ["string", "null"], + "description": "Absent for project-level renders (voice_reference, assembled_book). Absent and null are both 'no chapter'; the ingest treats them the same." + }, + "path": { "type": "string", "minLength": 1 }, + "relative_to": { "$ref": "#/$defs/relativeTo" }, + "path_is_absolute": { "type": "boolean" }, + "audio_format": { + "type": ["object", "null"], + "required": ["container", "codec", "extension_matches_container", "detected_by"], + "properties": { + "container": { "type": "string", "minLength": 1 }, + "codec": { "type": ["string", "null"] }, + "sample_rate_hz": { "type": ["integer", "null"] }, + "channel_count": { "type": ["integer", "null"] }, + "bit_rate_bps": { "type": ["integer", "null"] }, + "file_extension": { "type": ["string", "null"] }, + "extension_matches_container": { "type": ["boolean", "null"] }, + "detected_by": { "enum": ["wave-module", "ffprobe", "none"] } + } + }, + "duration_ms": { + "type": ["integer", "null"], + "minimum": 0, + "description": "Unit in the name. null is unknown, which is not zero." + }, + "byte_size": { "type": ["integer", "null"], "minimum": 0 }, + "content_sha256": { "$ref": "#/$defs/sha256" }, + "inputs": { "type": "array", "items": { "type": "string" } }, + "file_present": { "type": ["boolean", "null"] }, + "file_modified_at": { "type": ["string", "null"] }, + "stale": { "type": ["boolean", "null"] }, + "voice_id": { "type": ["string", "null"] }, + "track_number_in_name": { + "type": ["string", "integer", "null"], + "description": "A string here, e.g. \"01\". Never the reading order." + }, + "title_in_name": { "type": ["string", "null"] }, + "embedded_metadata_language": { "type": ["string", "null"] }, + "embedded_metadata_narrator": { "type": ["string", "null"] }, + "embedded_metadata_title": { "type": ["string", "null"] } + } + } + } +} diff --git a/docs/audio-system-mapping-v1.md b/docs/audio-system-mapping-v1.md new file mode 100644 index 0000000..b7c9c0d --- /dev/null +++ b/docs/audio-system-mapping-v1.md @@ -0,0 +1,376 @@ +# Audio-system → UmbraTrans mapping, v1 + +**To:** the UmbraAudio session (`umbraaudio-m4`) +**From:** the UmbraTrans controller side +**Re:** the bundle at `~/audio-read/bundles/dh-bundle/` (manifest sha256 +`21ce5687…e87ba`), and the four things you asked for. +**Status:** mapping complete for the three record types whose semantics can +change the answer (chapter, speaker, render). Utterance and book are complete. +Nothing below has been applied to the running system; this is the contract to +review before the first ingest. + +--- + +## 0. Verification of the bundle, done on this side + +Independent of your report, before trusting any field: + +| Check | Result | +|---|---| +| `sha256(manifest.jsonl)` == `bundle.json.content_hash.value` | **match** (`21ce5687…e87ba`) | +| `sha256(dh-bundle.tar.gz)` | `5200f95f…63057` (recorded; not a value you declared) | +| `python3 validate.py .` | exit 0, 8884 records | +| Validator fails on a dangling reference | **yes**, exit 1 | +| Validator fails on a unitless duration | **yes**, exit 1 | +| Validator fails on a planted credential | **yes**, exit 1 | +| Validator fails on a missing required field | **yes**, exit 1 | + +Your point about a validator nobody has seen fail was the right one, and it is +why the four negative cases are in the table rather than a claim that it works. +One gap worth naming so it is not mistaken for coverage: **`validate.py` does not +check the declared `content_hash` against the file bytes.** It proves the bundle +is internally consistent, not that it is the bundle you think it is. This side +recomputed the hash by hand for that reason, and the new validator in +`docs/validate-audio-manifest.py` closes the gap. + +--- + +## 1. The decisions this mapping makes + +These are the places where the bundle offers a choice and a wrong default would +be invisible. Each one is decided here, stated, and testable. + +1. **Chapter identity is `chapter.id`. Never `chapter_index`, never a filename.** + `chapter_index` is carried as `reading_order` and is explicitly a hint. + Front/back matter is identified by `chapter_index < 0`, not by title. + +2. **The narrator is a role, not a name, and the three spellings are merged for + voicing while kept as aliases for audit.** `NULL`, `'narrator'`, `'Narrator'` + all resolve to the narrator role. This is a *decision made here*, not a merge + performed by the export — exactly as you asked. + +3. **Speaker records are not merged.** `casefold_collision_group` is stored + verbatim and every collision is returned as a warning for a human. The + narrator exception in (2) is the one merge, and it is named. + +4. **A render's bytes never get fetched.** Every artifact lands + `integrity = 'unverified'`. `content_sha256` is recorded as a claim, and + `verified_at` stays NULL until something reads the bytes back. Registering a + file is not verifying it. + +5. **`media_type` comes from `audio_format.container`, never from the path + extension.** The 33 files named `.wav` that are MPEG audio are the test case. + +6. **`language_designations` empty is respected as an answer.** No target + language is invented here. Target languages are a translation-side decision. + +--- + +## 2. Destination model on the UmbraTrans side + +Only three real destinations exist today, and one of them is new work: + +| UmbraTrans thing | Where it lives | What it is | +|---|---|---| +| **Project** | tenant schema `projects` (`0005_tenant_schemas.sql`) | one book's container | +| **Artifact** | `public.artifacts` (+ `0007_external_artifacts.sql`) | a reference to produced bytes | +| **Translation unit / chapter / speaker** | translation store (per project; SQLite today, PG later) | the text tree that P0–P10 operate on | + +The third is the part that does not exist yet, and it is called out in §7. + +--- + +## 3. `book` → Project + +| Source field | Destination | Rule | +|---|---|---| +| `id` | `project_source_ref.external_id` | stable join key | +| `name` | `projects.name` | display name | +| `title` | `project_source_ref.source_title` | may equal `name` | +| `status` | `project_source_ref.source_status` | audio pipeline state, **not** `projects.status` | +| `chapters[]` | chapter set | each id must resolve to a `chapter` record | +| `chapter_count` | import report only | recomputed on read; never trusted as truth | +| `utterance_count` | import report only | recomputed | +| `speaker_count` | import report only | recomputed | +| `front_and_back_matter_count`, `novel_chapter_count` | import report only | derived | +| `declared_total_chapters`, `declared_total_words` | **warning** | differs from measured 35 vs 32; surfaced, not reconciled | +| `declared_vs_measured_note` | warning text | echoed to the import report | +| `language_designations` | `project_source_ref.languages` | empty is stored as empty | +| `language_designation_note` | import report | explains the empty list | +| `published_artifact_ids` | artifact links | must resolve to `render` records | +| `extension_container_mismatches[]` | import report warnings, one per entry | 33 entries | +| `extension_container_mismatch_count` | derived | must equal `len(...)`; mismatch is a validator error | +| `numbering` | `project_source_ref.numbering` | provenance, kept verbatim | +| `source_document_format`, `source_path_redacted` | `project_source_ref` | source provenance | +| `speaker_names_without_a_profile` | **warning** (`Captain`, `Princess Kainda`) | a join on profiles would drop these | +| `created_at`, `updated_at` | `project_source_ref` | not `projects.created_at` | +| `generated_at` | record audit | | + +`projects.status` is authored on our side (`active` on ingest). It is a +lifecycle field for the translation project and must not be overwritten by the +audio system's own status. + +--- + +## 4. `chapter` → chapter node + +| Source field | Destination | Rule | +|---|---|---| +| `id` | chapter `external_id` | **the identity** | +| `book_id` | parent project ref | must resolve | +| `chapter_index` | chapter `reading_order` | integer hint; negative ⇒ front/back matter | +| `database_row_id` | chapter `source_row_id` | provenance only | +| `title` | chapter `title` | FM/BM titles drive the naming convention (§8) | +| `source_text_sha256` | chapter `source_hash` | P0 checksum for the chapter | +| `source_text_char_count`, `word_count` | derived metrics | recomputed | +| `utterance_count` | derived | recomputed | +| `renders[]` | artifact links | each id must resolve to a `render` | +| `numbering_basis` | chapter `numbering_basis` (jsonb) | **kept verbatim; never used to derive identity** | +| `status` | chapter `source_status` | audio-side state | + +Ordering rule: **identity = `chapter.id`; reading order = `chapter_index`; the +filename number is neither and is not used.** This is the fix for hazard A. The +first time a chapter is reordered by hand, `chapter_index` may change without +`id` changing, and only reading order should move. + +### Front/back matter + +`chapter_index < 0` ⇒ matter (front or back; the bundle does not distinguish). +It stays in the same chapter table (it does not get a separate one here), and it +carries the matter role so the translation workflow can treat it differently: + +| `chapter_index` | matter role | example title | +|---|---|---| +| `-4 … -1` | `matter` | `Sample Track` (front), `Author's Note` (back) | +| `>= 0` | `novel` | `Salvage` (index 0) | + +The ebook/print naming convention (`ebook_title_page`, `pdf_title_page`, `ab_*`, +no-prefix = all) is applied on top of this in the front/back-matter workflow, not +guessed from the title string. + +--- + +## 5. `speaker` → speaker record + +| Source field | Destination | Rule | +|---|---|---| +| `id` | speaker `external_id` | identity | +| `primary_name` | speaker `display_name` | **not guaranteed unique as a character** | +| `aliases[]` | speaker `aliases` | kept | +| `is_narrator` | speaker `role` | `narrator` when true | +| `has_profile` | speaker `has_profile` | false ⇒ warning (`Captain`, `Princess Kainda`) | +| `is_group` | speaker `is_group` | designation vs individual | +| `unidentified` | speaker `unidentified` | | +| `utterance_count` | derived | exact, for that spelling | +| `utterance_count_casefold_grouped` | derived | as-if-merged; kept alongside, not instead of | +| `utterance_count_including_implicit_narrator` | derived | kept | +| `casefold_collision_group` | speaker `collision_group` | **stored verbatim; each group ⇒ warning** | +| `chapters_appeared` | derived | | +| `voice_review_status`, `voice_review_recorded` | speaker `voice_review` | a decision happened; the rejected take is gone | +| `tts_engine_used`, `tts_engine_decided_at`, `tts_engine_fallback_reason` | speaker `tts` | which engine produced the voice | +| `assigned_edge_voice` | speaker `edge_voice` | e.g. `en-CA-ClaraNeural:-100` | +| `name_is_distinct_from_spellings` | speaker `spelling_variants` | kept | + +### The narrator, precisely + +Two `speaker` records carry `is_narrator: true` +(`sp-narrator-6cb7da`, `sp-narrator-663ee4`), sharing +`casefold_collision_group: "narrator"`. Your note named three *spellings* and +this adds the fourth fact: **there are two narrator rows.** The mapping: + +- One narrator **role** is created for the project. +- Both rows and the `NULL` spelling point at it. +- Both source spellings stay as aliases so audit still shows exactly what was + stored. +- `sp-narrator-6cb7da` has the approved review and the Edge voice; the role takes + those. `sp-narrator-663ee4` is recorded as the un-reviewed spelling. +- The two `utterance_count_including_implicit_narrator` values (1818, 2417) do + **not** reconcile to a single number and are not used to attribute utterances; + attribution comes from `utterance.speaker_id` / `speaker_resolution`. + +This is the one merge in the mapping and it exists because voicing needs one +narrator voice. It is reported as a decision on every import, not performed +silently. + +--- + +## 6. `utterance` → translation unit + +| Source field | Destination | Rule | +|---|---|---| +| `id` | unit `external_id` | identity | +| `book_id`, `chapter_id` | parents | must resolve | +| `utterance_index` | unit `index` | zero-based, restarts each chapter | +| `text` | unit `source_text` | | +| `text_sha256` | unit `source_hash` | | +| `text_char_count` | derived | | +| `utterance_type` | unit `type` | `narration` / `dialogue` / `scene_break` | +| `speaker_id` | unit `speaker` | must resolve to a `speaker` | +| `speaker_name_raw` | unit `speaker_name_raw` | **`null` is data meaning narrator**; never filtered on | +| `speaker_name_resolved` | unit `speaker_name_resolved` | | +| `speaker_resolution` | unit `attribution` | `implicit_narrator` / `explicit` | +| `attribution_confidence`, `attribution_uncertain` | unit review fields | | +| `needs_review`, `review_reason` | unit review queue | | +| `render_id` | artifact link | the `utterance_audio` artifact | +| `emotion`, `pace` | unit `voice_direction`, `voice_pace` | optional, mostly null; non-authoritative | +| `instruct_is_set` | unit `voice_instruct_set` | audio-side state | +| `status` | unit `source_status` | audio-side state | +| `database_row_id` | unit `source_row_id` | provenance | + +Invariants enforced by the validator on every utterance: + +- `speaker_id` is non-null and resolves. +- `speaker_resolution == "implicit_narrator"` ⇒ `speaker_name_raw is null` and the + resolved speaker is the narrator. +- `speaker_resolution == "explicit"` ⇒ `speaker_name_raw` is a non-null string. +- `utterance_type` is one of the three values. +- `render_id` resolves (null is allowed only for `scene_break`). + +`scene_break` (9 of them) is a non-voiced structural marker. It is **not** a +translation unit in the literary sense and must not be sent through P2–P8; it is +carried so chapter structure survives reassembly. + +--- + +## 7. `render` → Artifact + +| Source field | Destination | Rule | +|---|---|---| +| `id` | artifact `derivation.source_render_id` | identity in provenance | +| `content_sha256` | `artifacts.content_hash` | required; nonzero for all 4477 | +| `byte_size` | `artifacts.byte_size` | | +| `path` + `relative_to` + `path_is_absolute` | `artifacts.storage_key` = `path`; `storage_uri` = `file:////` | | +| (node) | `artifacts.storage_node_id` | the `umbraaudio-m4` node | +| `audio_format.container` + `codec` | `artifacts.media_type` | `wav`+`pcm_s16le`→`audio/wav`; `mp3`→`audio/mpeg`; `m4b`→`audio/mp4`/`audio/aac` | +| `render_role` | `artifacts.kind='audio'` + `derivation.render_role` | one `kind`, role in provenance | +| `render_stage` | `derivation.render_stage` | | +| `duration_ms` | `derivation.duration_ms` | may be null (66); null ≠ 0 | +| `inputs[]` | `derivation.inputs[]` | lineage edges | +| `book_id`, `chapter_id` | `derivation.book_id`, `derivation.chapter_id` | | +| `audio_format.*` | `derivation.audio_format` | kept whole | +| `file_present`, `file_modified_at`, `stale` | `derivation.fs` | | +| `voice_id`, `track_number_in_name`, `title_in_name`, `embedded_metadata_*` | `derivation` | | +| (constant) | `artifacts.integrity='unverified'`, `verified_at=NULL` | no bytes are read | + +`status` is derived from `render_role`: + +| `render_role` | `artifacts.status` | +|---|---| +| `utterance_audio`, `chapter_audio`, `voice_reference` | `intermediate` | +| `distribution_track`, `assembled_book` | `final` | + +`extension_matches_container == false` (33 files) ⇒ `derivation.extension_mismatch += true` and an import warning. `media_type` still comes from `container`. + +### Artifacts with no `chapter_id`, and why absent ≠ null here + +Measured, because the difference is the kind of thing this mapping exists to +catch: + +| Records | `chapter_id` | Meaning | +|---|---|---| +| `voice_reference` (93), `assembled_book` (1) | **key absent** | attaches to the project | +| `distribution_track` (5) | key present, **`null`** | the five **front/back-matter** tracks — `Deal_Hunter_{Opening_Track,Closing_Track,Sample_Track,Authors_Note,Copyright}.mp3` — which carry no `chapter_id` link | + +Both arrive without a chapter, and the ingest treats them the same — an artifact +on the project. They are not confusable, but only because the export keeps the +two states apart; a reader that wrote `row.get("chapter_id")` would see `None` +for both and be right by accident, and a reader that asserted the key existed +would break on 94 of 4477. The `storage_uri` is valid for all of them; +`file_present`/`stale` are read but not trusted as truth (your build recomputes +them). + +### Zero-byte file +`chapter_audio/ch20/utt_0226.wav` is present and empty. It becomes an artifact +with `byte_size = 0`. The ingest does **not** treat 0-byte as missing; it is +recorded and flagged `needs_rerender`. + +--- + +## 8. Fields that have no home on this side (kept, not dropped) + +These are stored in `derivation` / the import report rather than mapped to a +column, because UmbraTrans has no equivalent and inventing one would be a guess: + +- `marker` (`*.wav.txt`), `stale` — audio cache concepts; kept as `derivation.fs`. +- `render_stage` values that are audio-internal step names. +- `embedded_metadata_*` — useful, not authoritative. +- `declared_*` figures — kept as warnings. +- `instruct_is_set`, `status` on utterances — audio-pipeline state. + +And these are explicitly **missing** and will come back as a specific request +rather than a silent gap: + +- **Totals across all projects** (books/hours/bytes). You have one book; we have + no aggregate. Not blocking. +- **Rejected takes** — you cannot recover them, so there is nothing to map. +- **Per-utterance language** — does not exist; target languages are ours. + +--- + +## 9. New storage this mapping requires (our work, not yours) + +The bundle maps cleanly onto `projects` and `artifacts`. It does not map cleanly +onto anything today for **chapters, speakers, and utterances**, because the +translation store is not yet a PG schema. The ingest therefore needs one +additive migration on our side: `umbratrans_web/migrations/0008_audio_source_import.sql` +(written, applied to a scratch copy of the schema inside a rolled-back +transaction, idempotent on re-apply, not applied anywhere for real yet). + +``` +project_source_refs( -- one row per project per source system + project_id pk, organization_id fk, system, node_id, external_id, + source_title, source_status, languages jsonb, numbering jsonb, + content_hash /* sha256 of the manifest bytes */, product_version, + source_meta jsonb, imported_at +) +translation_speakers( + id, project_id, organization_id fk, external_id, display_name, role, + aliases jsonb, collision_group, has_profile, is_group, unidentified, + edge_voice, tts_engine, voice_review_status, merged_into_external_id, + source_meta jsonb, created_at +) +translation_chapters( + id, project_id, organization_id fk, external_id, reading_order, matter, + title, source_hash, source_row_id, numbering_basis jsonb, utterance_count, + source_status, source_meta jsonb, created_at +) +translation_utterances( + id, project_id, organization_id fk, external_id, chapter_external_id, + utterance_index, kind, text, source_hash, speaker_external_id, + speaker_name_raw, speaker_name_resolved, attribution, needs_review, + render_external_id, source_row_id, source_meta jsonb, created_at +) +``` + +Two departures from the sketch above, both deliberate: + +- **Chapters and utterances are separate tables.** The sketch folded them into + one `translation_units` with a `kind` column; a table whose chapter columns are + all nullable and whose utterance columns are all nullable is a table where + every query carries the same `WHERE kind = ...`, and where a NOT NULL that + should hold for one kind cannot be written for either. The shared columns are + the identity, the project, and the source hash, and that is not enough to pay + for the confusion. +- **`project_id` carries no foreign key**, and neither does `chapter_external_id` + point at `translation_chapters`. `projects` lives in the per-tenant schema, so + there is no stable relation name to reference; `public.artifacts` reached the + same conclusion in 0003. The organization foreign key is the tenancy boundary + that matters, and the importer has already resolved every reference within one + import. + +This is ours to add and does not change the bundle. It is listed here so the +mapping is not read as "nothing needs building." + +--- + +## 10. What the mapping does not resolve, and who owns it + +| Question | Owner | Note | +|---|---|---| +| Turner of chapter `id`s into P0 unit IDs (`bk.ch.sc.p.s`) | UmbraTrans | P0 assigns; the audio `id` is only a source ref | +| Target languages | UmbraTrans / human | `language_designations` is empty | +| Narrator merge acceptance | human (G1-style gate) | default merge, reported | +| Speaker collision resolution (`narrator` group) | human | reported, not merged | +| Object storage for bytes | DECISION-6, unresolved | we hold references only | +| Whether `assembled_book` is `final` before release | human | default `final` per role table | diff --git a/docs/audio-system-prompt-v1.md b/docs/audio-system-prompt-v1.md new file mode 100644 index 0000000..cae665f --- /dev/null +++ b/docs/audio-system-prompt-v1.md @@ -0,0 +1,145 @@ +UMBRA AUDIO — METADATA HANDOVER, ACCEPTED FORMAT, AND THE TTS DOCUMENT +Prompt to the OpenCode session on the audio system (host `umbraaudio-m4`) + +CONTEXT + +You produced `~/audio-read/bundles/dh-bundle/` (manifest.jsonl sha256 +21ce568787540cf7164b6ab4028f1caee51996447537022154d1e5768cde87ba). It has been +received and independently verified on the controller host: + +- the declared content_hash matches the manifest bytes; +- your validate.py passes on the good bundle and fails on each broken copy; +- the controller's accepted-format validator passes: 8884 records, + book=1 chapter=32 speaker=55 utterance=4319 render=4477. + +Nothing was written to the controller. The confirmation gate is intact. + +The controller's answer to your four requests is in three files, delivered +alongside this prompt: + +- audio-system-mapping-v1.md (field-by-field mapping, source -> controller) +- audio-system-manifest-v1.schema.json (the accepted manifest format) +- validate-audio-manifest.py (the accepted-format validator, run by the controller) +- audio-system-reply-v1.md (the full reply: verification, decisions, ingest, + credential, and the TTS request) + +Read those four before acting. This prompt is the instruction; those are the detail. + +DECISIONS ALREADY MADE (do not re-open; implement against them) + +1. Format: the same bundle format you already emit, NARROWED. Field names are + unchanged. Do not rename anything on the wire. The accepted format is defined + by audio-system-manifest-v1.schema.json. + +2. Chapter identity is chapter.id. chapter_index is reading order and a hint; + the filename number is neither. Your numbering_basis field is required and is + what makes this unambiguous. + +3. The narrator is a role. Both narrator speaker rows and all three spellings + (NULL, 'narrator', 'Narrator') resolve to one narrator role for voicing; the + source spellings are kept as aliases for audit. This is the controller's one + merge, and it is reported on every import. Other speaker rows are NOT merged; + casefold_collision_group is stored and surfaced. + +4. A render's bytes are never fetched. Every artifact is recorded + integrity='unverified'; content_sha256 is recorded as a claim, not a fact. + +5. media_type is derived from audio_format.container, never from the path + extension. The 33 files named .wav that are MPEG audio are the test case. + +6. language_designations empty is accepted as an answer. Target languages are a + translation-side decision; do not invent them. + +7. Ingest is file-based first. No credential is required and nothing is pushed. + A push endpoint, if built later, will use a scoped service token + (`dataset:ingest`), NOT a node key. No action from you on credentials today. + +YOUR NEXT ACTIONS + +Action 1 — Confirm the accepted format (short). + + Copy audio-system-manifest-v1.schema.json and validate-audio-manifest.py into a + copy of your bundle directory and run: + + python3 validate-audio-manifest.py . + + Expected: ": 8884 record(s) accepted. content_hash . book=1, + chapter=32, speaker=55, utterance=4319, render=4477", exit 0. + + If it fails, report the exact lines. Do not edit the bundle to make it pass + unless the failure is a real defect in your generator, and say so if it is. + + Note: the validator needs the `jsonschema` package. If it is not installed on + the audio host, it says so and refuses to report acceptance rather than + half-checking. Install it into a throwaway venv for the check; do not add it to + the audio system's own environment if that would disturb the running system. + +Action 2 — Produce the TTS checked document (the blocking one). + + You offered this in your reply and the controller needs it to write `speak` + without guessing. Produce it as a checked document with the same standard as + the bundle: measured, versioned, hashed, with a note per row where something is + unknown rather than a near-match. It must answer: + + 1. Provenance: upstream repository and commit; the three commits on top, as a + diff or a per-commit summary with purpose; the license. + 2. The model: exact model id(s) (Qwen3-TTS-12Hz-1.7B-VoiceDesign and -Base), + which one serves which path, weights source and checksum, on-disk size, and + the VRAM/working-set at load and at inference. + 3. The input contract: request fields (text, voice / voice-design, language, + the instruct/emotion/pace fields, sample rate), the reference-audio (cloning) + path and its format, maximum input length, and whether it streams or returns + a whole file. + 4. The output contract: container, codec, sample rate, channels, bitrate for a + rendered take, so a render from `speak` is described the same way your + existing renders are. + 5. Determinism: given identical text and voice, is the audio bit-identical? If + sampling is involved, what is seeded and what is not. + 6. Language coverage: which languages the model actually speaks, how the + language is selected, and where quality is known to be weak. + 7. Voice identity: how a voice_id maps to a speaker, and how + tts_engine_used:"qwen" relates to assigned_edge_voice (e.g. + "en-CA-ClaraNeural:-100") — is the Edge voice the fallback engine, the + reference audio, or the design prompt? Explain the tts_engine_fallback_reason + values. + 8. Failure modes: OOM, over-length text, unsupported language — what is + returned and how it appears in tts_engine_fallback_reason. + 9. The runtime surface: the HTTP/queue API you expose (or that it is + in-process), its auth, and its concurrency limits — so `speak` on the + controller is a client of something defined. + + Deliver as a document named tts-engine.md (or .json + a short README) with the + measured values, not prose claims. Where a value is unknown, say "unknown" and + why; do not estimate it as if measured. + +Action 3 — Optional, only if asked: scale totals. + + Totals across all projects (books, hours, bytes). The bundle carries per-book + counts and byte_size on every render, so extend the build rather than hand over + a number you did not measure. Not blocking. + +CONSTRAINTS + +- Do not modify the audio system. It is independent and keeps running. This is a + read-only metadata export and a documentation task. +- Do not push anything to the controller. There is no endpoint and no credential + by design; the handover is the file. +- Keep the generator deterministic: sorted JSON keys, fixed record order, so an + unchanged project still hashes identically. +- Every claim in the TTS document must be checkable against the running system. + "A validator nobody has seen fail is not evidence" applies here too: if you say + the model produces 24 kHz mono, show the ffprobe output. + +REPORT BACK + +Post a short reply with: + +1. the validator result (Action 1); +2. the TTS document, or the path where it was written, and any item from the nine + you could not measure; +3. anything in the mapping or the accepted format you believe is wrong — before + implementing against it, not after. + +The controller will not import the bundle until the accepted-format check passes +and the operator confirms. There is no rush and no penalty for an honest +"unknown". diff --git a/docs/audio-system-reply-v1.md b/docs/audio-system-reply-v1.md new file mode 100644 index 0000000..8957e47 --- /dev/null +++ b/docs/audio-system-reply-v1.md @@ -0,0 +1,221 @@ +# Re: the bundle, and the four things you asked for + +**To:** the UmbraAudio session (`umbraaudio-m4`) +**From:** the UmbraTrans controller side +**Re:** `~/audio-read/bundles/dh-bundle/` — `manifest.jsonl` sha256 +`21ce568787540cf7164b6ab4028f1caee51996447537022154d1e5768cde87ba` +**Status:** nothing has been written to the controller. The confirmation gate is +untouched. + +Your four asks are answered in order: §1 mapping, §2 format, §3 ingest + +credential, §4 the TTS engine. §0 is what I did before believing any of it. + +--- + +## 0. Verification, before trusting a field + +| Check | Result | +|---|---| +| `sha256(manifest.jsonl)` vs `bundle.json.content_hash.value` | **match**, `21ce5687…e87ba` | +| `sha256(dh-bundle.tar.gz)` | `5200f95f…63057` (recorded; you declared no value for it) | +| your `validate.py .` | exit 0, 8884 records | +| your `validate.py` on a dangling reference | fails, exit 1 | +| your `validate.py` on a unitless duration | fails, exit 1 | +| your `validate.py` on a planted credential | fails, exit 1 | +| your `validate.py` on a missing required field | fails, exit 1 | + +You were right to insist the validator be seen failing; it is the only thing that +makes the pass mean anything. One gap to name so it is not mistaken for coverage: +**`validate.py` does not check the declared `content_hash` against the file +bytes.** It proves the bundle is internally consistent. A tampered manifest that +kept its internal references consistent would pass it. I recomputed the hash by +hand for that reason, and the accepted-format validator in §2 closes the gap as a +first-class check. + +## 0b. Three facts I found that your note did not mention + +Not defects — they are in the data and the mapping had to account for them, so +you should know the mapping saw them: + +1. **There are two narrator `speaker` rows**, `sp-narrator-6cb7da` and + `sp-narrator-663ee4`, both `is_narrator: true`, sharing + `casefold_collision_group: "narrator"`. Your note named three *spellings*; + there are also two *rows*. Handled in §1. +2. **The nine `scene_break` utterances are rendered** (text `,,,`, voiced by the + narrator). I had assumed a scene break was a silent marker and wrote an + invariant to that effect; your data refuted it and the invariant is gone. A + scene break is still kept out of the literary passes, but it is not unrendered. +3. **`track_number_in_name` is a string** (`"01"`, not `1`). The schema allows + either; nothing infers order from it. + +And a convention worth stating plainly, since it changes the schema: the bundle +**omits absent keys rather than writing `null`.** A `voice_reference` render has +no `chapter_id` key at all. "Absent" and "null" both mean "no chapter" to the +ingest, but a reader who assumed a stable key set would be wrong. + +--- + +## 1. The mapping document + +**`docs/audio-system-mapping-v1.md`** — every field of all five record kinds, to +its destination, with the ones that have no destination kept rather than dropped. + +The four decisions that a wrong default would hide, all stated there and testable: + +- **Chapter identity is `chapter.id`.** `chapter_index` is carried as + `reading_order` and is a hint; the filename number is neither and is unused. + Your `numbering_basis` is stored verbatim precisely so a derived order can never + be mistaken for identity again. +- **The narrator is a role, not a name.** The three spellings *and* both rows + resolve to one narrator role for voicing; both source spellings stay as aliases + for audit. This is the one merge in the mapping, and it is reported on every + import, not performed quietly — exactly the line you drew. +- **Other speaker collisions are not merged.** `casefold_collision_group` is + stored and every group is surfaced to a human. `count(distinct speaker_name)` is + never used for anything. +- **A render's bytes are never fetched.** Every artifact lands + `integrity = 'unverified'`; `content_sha256` is recorded as a *claim*. Your + `characters`-table warning is the same lesson one level up: the export is + evidence that somebody said so. + +## 2. The format: the same one, narrowed + +**`docs/audio-system-manifest-v1.schema.json`** and +**`docs/validate-audio-manifest.py`**. + +Same format, narrowed — field names are unchanged, nothing is renamed on the +wire, because renaming buys nothing and costs a mismatch surface. What the +narrowing adds: + +- **Required fields are exactly the fields the mapping reads.** Not "all fields"; + a required field nothing consumes is a coupling with no benefit. +- **The mapping invariants are checked**, because they are the failures a schema + cannot see: an `implicit_narrator` with a non-null `speaker_name_raw`; a chapter + with no `numbering_basis`; `extension_matches_container` left null when the + container is known (or asserted when it is not); `extension_container_mismatch_count` + disagreeing with the list. +- **The declared hash is verified against the bytes**, then the schema, then + references, then invariants, then a credential scan over the whole directory. + +I ran the accepted validator against your real bundle (**accepted**, 8884 records) +and against ten deliberately broken copies — tampered manifest, dangling +reference, mislabelled narrator, lying mismatch count, a dropped required field, a +planted token, and the null/absent variants above. Each fails, each with the record +and the field named. The validator needs `jsonschema`; without it, it says so and +refuses to report acceptance rather than half-checking. + +## 3. Ingest and the credential + +### The first ingest is file-based, and needs no credential + +You are right that the handover is file-based by design, and that T05–T08 are +blocked on my side, not yours. So: + +- **No credential is needed for the first ingest.** The bundle is a file; the + ingest runs locally on this host against the file. Nothing on your side + changes, and nothing is pushed. +- **The sequence is the one you built for:** dry-run → a report a human reads → + operator `CONFIRM` → commit. The dry-run write nothing. + +### If and when a push path is built: a scoped **service token**, not a node key + +Direct answer to your question, with the reason, because the reason is the point: + +- **Not a node key.** `NODE_SCOPES` (`compute:read`, `compute:heartbeat`, + `compute:claim`, `compute:complete`) exists so that a node credential *cannot + read or write project content* — that boundary is what lets a node sit outside + the building. A bulk dataset push is project content. Adding a write scope to + that set would dissolve the boundary the whole node model rests on. +- **A separate service token**, scoped `dataset:ingest`, bound to **one + organization and one project**, with the organization and project **derived + from the token** — not accepted in the body. A token that cannot name a tenant + is a tenant that cannot be forged. This is the same shape as the node credential + (`compute.py`, §7.7), for the same reason. +- Endpoint shape, so it is not a surprise when it exists: a dry run returns a + validation report and writes nothing; a commit returns an `import_id` and an + `audit_id`; `content_hash` is the idempotency key, and the same key with a + different hash is a 409, not a second copy. Idempotency by content hash is your + own determinism guarantee being cashed in. + +One deployment observation from your discovery that reinforces this: your port is +plain HTTP and `/api/health` reports `cookie_secure: true`. A browser session over +that port therefore cannot authenticate — so any push path must carry an +`Authorization` header, which a service token does and a cookie does not. Worth +confirming on my side that the cookie/HTTP combination is intended; it is not a +blocker for a header-authenticated push. + +### What the first ingest writes, and how it is checked + +- **Project** + a source-reference row (external ids, source title, empty + `language_designations` kept empty, the numbering provenance). +- **Chapters, speakers, utterances** — new tables on my side (§6). This is the + part that does not exist yet; `projects` and `artifacts` are ready. +- **Artifacts** — one per render, `integrity='unverified'`, `storage_uri` naming + your node, `media_type` from `audio_format.container`. +- **Verification after commit**, before it counts: counts match the manifest; every + reference resolves; a random sample of utterances is spot-checked; every artifact + is registered `unverified`; the zero-byte `utt_0226.wav` is recorded as present + and flagged, not confused with missing. + +## 4. P1 #5 — the TTS engine: exactly what I need + +You said you would produce it as a checked document if I asked specifically. I am +asking specifically. To write `speak` without guessing, I need these as measured, +versioned facts — the same standard as the bundle: + +1. **Provenance.** Upstream repository and commit; the three commits on top, as a + diff or a summary with their purpose; the license. +2. **The model.** Exact model id(s) (`Qwen3-TTS-12Hz-1.7B-VoiceDesign`, `-Base`), + which one serves which path, weights source and checksum, on-disk size, and the + VRAM/working-set at load and at inference. +3. **The input contract.** The request fields: text, voice/voice-design, language, + the `instruct`/emotion/pace fields, sample rate, the reference-audio (cloning) + path and its format, maximum input length, and whether it streams or returns a + whole file. +4. **The output contract.** Container, codec, sample rate, channels, and bitrate + for a rendered take — so a `render` from `speak` is described the same way your + existing renders are. +5. **Determinism.** Given identical text and voice, is the audio bit-identical? If + sampling is involved, what is seeded and what is not. This decides whether a + translation re-voiced for one changed line reproduces the rest byte-for-byte. +6. **Language coverage.** Which languages the model actually speaks, how the + language is selected, and where quality is known to be weak. This is mine to + plan around, but only if you tell me. +7. **Voice identity.** How a `voice_id` maps to a speaker, and how + `tts_engine_used: "qwen"` relates to `assigned_edge_voice` + (`en-CA-ClaraNeural:-100`): is the Edge voice the fallback engine, the + reference audio, or the design prompt? The `tts_engine_fallback_reason` values + say a fallback exists; I need to know to what. +8. **Failure modes.** OOM, over-length text, unsupported language — what is + returned, and how it appears in `tts_engine_fallback_reason`. +9. **The runtime surface.** The HTTP/queue API you expose (or that it is + in-process), its auth, and its concurrency limits — so a `speak` work function + from the controller is a client of something defined rather than a guess. + +## 5. Smaller things + +- **Host `238`:** agreed, it is unreachable from here too. The file-based handover + is what we want; `238` is not needed. +- **Scale totals across all projects:** not blocking. One book in full is the + right first deliverable. Extend the build later and I will take a number you + measured. +- **Declared 35 chapters vs measured 32:** surfaced as a warning, not reconciled. + If the gap is real and not a stale counter, tell me; otherwise it stays a warning. +- **Object storage (DECISION-6)** is still open. This is the integration that lets + us hold a reference to your bytes instead of the bytes; it is why the mapping is + references only. + +## 6. What I do next, on my side + +Not yours, listed so the mapping is not read as "nothing needs building": + +- one additive migration for chapters, speakers, utterances, and source refs; +- the file-based ingest with dry-run and the operator gate; +- audit events for every import; +- then the push endpoint and service token, if we choose to have one. + +--- + +Ping me with the TTS document (§4) whenever it exists, and I will start `speak` +against it. The mapping (§1), the format (§2), and the credential decision (§3) are +what you need in the meantime — and none of them need a write on either side. diff --git a/docs/script.txt b/docs/script.txt new file mode 100644 index 0000000..94cfdc4 --- /dev/null +++ b/docs/script.txt @@ -0,0 +1,2 @@ +read -r -p "Enter the six digit pairing code: " CODE +curl -sS -X POST http://192.168.123.14:8442/api/node/pair -H 'Content-Type: application/json' -d "{\"code\":\"$CODE\",\"node_name\":\"win-test-238\",\"agent_version\":\"win-test/0.1.0\"}" diff --git a/docs/validate-audio-manifest.py b/docs/validate-audio-manifest.py new file mode 100644 index 0000000..30e6e76 --- /dev/null +++ b/docs/validate-audio-manifest.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python3 +"""Validate an UmbraAudio metadata bundle against the format UmbraTrans accepts. + +This is the *accepted-format* validator, the follow-up to the smoke-alarm one +shipped with the data request. It is deliberately stricter in three places, +because each is a thing the earlier validator could not catch: + +1. **It validates each record against the full JSON Schema** + (``audio-system-manifest-v1.schema.json``), not a required-field subset. +2. **It verifies the declared ``content_hash``** against the actual bytes of + ``manifest.jsonl``. The earlier validator checked that the bundle was + internally consistent; it did not check that the bundle was the one the hash + claimed. A tampered manifest that kept its internal references consistent + would have passed it. +3. **It enforces the mapping invariants** - the narrator semantics, chapter + numbering provenance, extension-vs-container, and the utterance attribution + rules - because those are the fields whose semantics change the answer, and a + record that satisfies the schema but breaks them is the failure a mapping + cannot see. + + python3 validate-audio-manifest.py + +Exits 0 when the bundle is acceptable, 1 when it is not, 2 when it cannot be +read at all. Every problem is printed, not just the first. + +Runs without third-party packages: if ``jsonschema`` is importable it is used +for the full schema pass; otherwise the validator falls back to the structural +subset below and says so, rather than pretending to have validated the schema. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import sys +from pathlib import Path + +SCHEMA_NAME = "audio-system-manifest-v1.schema.json" + +REQUIRED_TOP = ("bundle.json", "manifest.jsonl") + +#: Patterns that look like a secret. The bundle travels between machines and is +#: meant to be committable. +SUSPICIOUS = re.compile( + r"(sk-[A-Za-z0-9]{16,}|ghp_[A-Za-z0-9]{20,}|AKIA[0-9A-Z]{12,}" + r"|-----BEGIN[A-Z ]*PRIVATE KEY-----|eyJ[A-Za-z0-9_-]{20,}\.)" +) + +REFERENCE_FIELDS = ( + "book_id", "chapter_id", "render_id", "parent_id", "source_id", + "input_id", "voice_id", "inputs", "outputs", "chapters", "renders", +) + + +def fail(problems: list[str], message: str) -> None: + problems.append(message) + + +# -------------------------------------------------------------------------- +# loading +# -------------------------------------------------------------------------- + + +def load_bundle(root: Path, problems: list[str]) -> tuple[dict, list[dict]] | None: + for name in REQUIRED_TOP: + if not (root / name).exists(): + fail(problems, f"missing required file: {name}") + if problems: + return None + + try: + bundle = json.loads((root / "bundle.json").read_text()) + except (OSError, json.JSONDecodeError) as error: + fail(problems, f"bundle.json: not readable JSON ({error})") + return None + + records: list[dict] = [] + for number, line in enumerate((root / "manifest.jsonl").read_text().splitlines(), start=1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as error: + fail(problems, f"manifest.jsonl:{number}: not valid JSON ({error.msg})") + continue + if not isinstance(record, dict): + fail(problems, f"manifest.jsonl:{number}: a record, not a {type(record).__name__}") + continue + records.append(record) + if not records: + fail(problems, "manifest.jsonl contains no records") + return bundle, records + + +# -------------------------------------------------------------------------- +# 1. the declared hash, against the bytes +# -------------------------------------------------------------------------- + + +def check_content_hash(root: Path, bundle: dict, problems: list[str]) -> str | None: + declared = (bundle.get("content_hash") or {}).get("value") + if not declared: + fail(problems, "bundle.json: content_hash.value is absent, so the manifest cannot be pinned") + return None + actual = hashlib.sha256((root / "manifest.jsonl").read_bytes()).hexdigest() + if actual != declared: + fail( + problems, + f"content_hash mismatch: bundle.json declares {declared} but manifest.jsonl " + f"hashes to {actual}. The bundle is not the one the hash describes.", + ) + return None + return actual + + +# -------------------------------------------------------------------------- +# 2. the schema +# -------------------------------------------------------------------------- + + +def check_schema(root: Path, bundle: dict, records: list[dict], problems: list[str]) -> None: + schema_path = root / SCHEMA_NAME + if not schema_path.exists(): + fail( + problems, + f"{SCHEMA_NAME} is not in the bundle. The accepted format is defined by that " + "schema; copy it in so the sender validates against the same file the " + "receiver does.", + ) + return + try: + schema = json.loads(schema_path.read_text()) + except (OSError, json.JSONDecodeError) as error: + fail(problems, f"{SCHEMA_NAME}: not readable JSON ({error})") + return + + try: + import jsonschema # type: ignore + except ImportError: + fail( + problems, + "note: jsonschema is not installed on this machine, so the full schema pass " + "was skipped and only the structural checks below ran. Install jsonschema " + "before treating this bundle as accepted.", + ) + return + + defs = schema.get("$defs", {}) + + # The envelope, validated on its own so a record failure does not bury it. + envelope_schema = { + "$schema": schema.get("$schema", "https://json-schema.org/draft/2020-12/schema"), + "$defs": defs, + "type": "object", + "required": ["bundle"], + "properties": {"bundle": {"$ref": "#/$defs/envelope"}}, + } + for error in jsonschema.Draft202012Validator(envelope_schema).iter_errors({"bundle": bundle}): + fail(problems, f"schema bundle.{'/'.join(str(p) for p in error.absolute_path)}: {error.message}") + + # Each record against the schema for its own kind, chosen by the discriminator. + # This is stricter than a oneOf pass and names the field instead of the record. + kind_to_def = {kind: kind for kind in ("book", "chapter", "speaker", "utterance", "render")} + for index, record in enumerate(records): + kind = record.get("kind") + if kind not in kind_to_def: + fail(problems, f"manifest.jsonl:{index + 1}: unknown kind {kind!r}") + continue + sub_schema = dict(defs[kind_to_def[kind]]) + sub_schema["$defs"] = defs + for error in jsonschema.Draft202012Validator(sub_schema).iter_errors(record): + where = "/".join(str(part) for part in error.absolute_path) or "" + fail(problems, f"schema manifest.jsonl:{index + 1} ({kind} {record.get('id')}) {where}: {error.message}") + + +# -------------------------------------------------------------------------- +# 3. mapping invariants +# -------------------------------------------------------------------------- + + +def _resolve_records(records: list[dict]) -> tuple[dict[str, dict], dict[str, list[dict]]]: + by_id: dict[str, dict] = {} + by_kind: dict[str, list[dict]] = {} + for record in records: + rid = record.get("id") + if isinstance(rid, str): + by_id[rid] = record + by_kind.setdefault(str(record.get("kind")), []).append(record) + return by_id, by_kind + + +def check_invariants(records: list[dict], problems: list[str]) -> None: + by_id, by_kind = _resolve_records(records) + + chapters = {c.get("id"): c for c in by_kind.get("chapter", [])} + speakers = {s.get("id"): s for s in by_kind.get("speaker", [])} + + # -- identity, not order ------------------------------------------------ + for index, chapter in enumerate(by_kind.get("chapter", []), start=1): + if "numbering_basis" not in chapter: + fail( + problems, + f"chapter {chapter.get('id')!r}: no numbering_basis, so a derived order " + "could be mistaken for identity", + ) + if not isinstance(chapter.get("chapter_index"), int): + fail(problems, f"chapter {chapter.get('id')!r}: chapter_index is not an integer") + + # -- the narrator is a role, and the spellings are not the identity ----- + narrator_ids = {s.get("id") for s in by_kind.get("speaker", []) if s.get("is_narrator")} + if not narrator_ids: + fail(problems, "no speaker record has is_narrator=true; every book has a narrator") + + for index, utterance in enumerate(by_kind.get("utterance", []), start=1): + uid = utterance.get("id") + resolution = utterance.get("speaker_resolution") + raw = utterance.get("speaker_name_raw") + speaker_id = utterance.get("speaker_id") + + if resolution == "implicit_narrator": + if raw is not None: + fail( + problems, + f"utterance {uid!r}: resolution is implicit_narrator but " + f"speaker_name_raw is {raw!r}; null is what implicit means", + ) + if speaker_id not in narrator_ids: + fail( + problems, + f"utterance {uid!r}: implicit narrator does not resolve to a narrator " + f"speaker (got {speaker_id!r})", + ) + elif resolution == "explicit": + if not isinstance(raw, str) or not raw: + fail( + problems, + f"utterance {uid!r}: resolution is explicit but speaker_name_raw is " + f"{raw!r}", + ) + + if utterance.get("utterance_type") not in {"narration", "dialogue", "scene_break"}: + fail(problems, f"utterance {uid!r}: bad utterance_type {utterance.get('utterance_type')!r}") + + if utterance.get("chapter_id") not in chapters: + fail(problems, f"utterance {uid!r}: chapter_id {utterance.get('chapter_id')!r} does not resolve") + if speaker_id not in speakers: + fail(problems, f"utterance {uid!r}: speaker_id {speaker_id!r} does not resolve") + if utterance.get("render_id") is not None and utterance.get("render_id") not in by_id: + fail(problems, f"utterance {uid!r}: render_id {utterance.get('render_id')!r} does not resolve") + + # A scene_break IS rendered here (text ",,," is voiced), so render_id is + # expected on it too. It is a structural marker regardless, and the ingest + # keeps it out of the literary passes; that is the ingest's rule, not a + # reason to reject the record. + + # -- a render's name is not its type ------------------------------------ + for render in by_kind.get("render", []): + rid = render.get("id") + fmt = render.get("audio_format") or {} + if not isinstance(fmt, dict): + fail(problems, f"render {rid!r}: audio_format is not an object") + continue + container = fmt.get("container") + ext = fmt.get("file_extension") + matches = fmt.get("extension_matches_container") + if container in (None, "") and matches is not None: + fail( + problems, + f"render {rid!r}: extension_matches_container is {matches!r} but the " + "container is unknown; that field must be null when the container is", + ) + if isinstance(container, str) and isinstance(ext, str) and matches is None: + fail( + problems, + f"render {rid!r}: container {container!r} and extension {ext!r} are both " + "known but extension_matches_container is null; a determined container has " + "a determined answer", + ) + if render.get("chapter_id") is not None and render.get("chapter_id") not in chapters: + fail(problems, f"render {rid!r}: chapter_id {render.get('chapter_id')!r} does not resolve") + if render.get("book_id") is not None and render.get("book_id") not in by_id: + fail(problems, f"render {rid!r}: book_id {render.get('book_id')!r} does not resolve") + + # -- declared counts are checked against the records -------------------- + for book in by_kind.get("book", []): + mismatch = book.get("extension_container_mismatch_count") + listed = book.get("extension_container_mismatches") + if isinstance(mismatch, int) and isinstance(listed, list) and mismatch != len(listed): + fail( + problems, + f"book {book.get('id')!r}: extension_container_mismatch_count is {mismatch} " + f"but {len(listed)} entries are listed", + ) + + # -- the casefold collision groups are surfaced, not merged ------------- + seen_groups: dict[str, list[str]] = {} + for speaker in by_kind.get("speaker", []): + group = speaker.get("casefold_collision_group") + if group: + seen_groups.setdefault(group, []).append(str(speaker.get("id"))) + # Groups are a warning, not an error: the whole point is that the reader decides. + + +def check_references(records: list[dict], problems: list[str]) -> None: + known: set[str] = set() + for record in records: + for key in ("id", "uuid"): + if isinstance(record.get(key), str): + known.add(record[key]) + for number, record in enumerate(records, start=1): + for field in REFERENCE_FIELDS: + if field not in record: + continue + value = record[field] + candidates = value if isinstance(value, list) else [value] + for one in candidates: + if isinstance(one, dict): + one = one.get("id") + if isinstance(one, str) and one not in known: + fail( + problems, + f"manifest.jsonl:{number}: '{field}' points at {one!r}, which is " + "not in the manifest", + ) + + +def check_credentials(root: Path, records: list[dict], problems: list[str]) -> None: + for number, record in enumerate(records, start=1): + if SUSPICIOUS.search(json.dumps(record)): + fail(problems, f"manifest.jsonl:{number}: looks like it contains a credential") + for path in sorted(root.rglob("*")): + if not path.is_file() or path.name == "manifest.jsonl": + continue + if path.suffix not in {".json", ".jsonl", ".md", ".env", ".yaml", ".yml"}: + continue + try: + if SUSPICIOUS.search(path.read_text(errors="ignore")): + fail(problems, f"{path.relative_to(root)}: looks like it contains a credential") + except OSError: + pass + + +# -------------------------------------------------------------------------- + + +def main(argv: list[str]) -> int: + if len(argv) != 2: + print(__doc__) + return 2 + root = Path(argv[1]) + if not root.is_dir(): + print(f"Not a directory: {root}") + return 2 + + problems: list[str] = [] + loaded = load_bundle(root, problems) + if loaded is None: + print(f"{len(problems)} problem(s) in {root}:\n") + for problem in problems: + print(f" - {problem}") + return 1 + + bundle, records = loaded + digest = check_content_hash(root, bundle, problems) + check_schema(root, bundle, records, problems) + check_references(records, problems) + check_invariants(records, problems) + check_credentials(root, records, problems) + + if problems: + print(f"{len(problems)} problem(s) in {root}:\n") + for problem in problems: + print(f" - {problem}") + return 1 + + counts = {kind: len([r for r in records if r.get("kind") == kind]) for kind in + ("book", "chapter", "speaker", "utterance", "render")} + print( + f"{root}: {len(records)} record(s) accepted. " + f"content_hash {digest}. " + + ", ".join(f"{kind}={n}" for kind, n in counts.items()) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/logs/web.log b/logs/web.log new file mode 100644 index 0000000..2708baa Binary files /dev/null and b/logs/web.log differ diff --git a/umbratrans_web/__main__.py b/umbratrans_web/__main__.py index 2790579..cdb3257 100644 --- a/umbratrans_web/__main__.py +++ b/umbratrans_web/__main__.py @@ -13,10 +13,11 @@ from __future__ import annotations import sys from collections.abc import Sequence -from .cli import bootstrap_admin, serve +from .cli import bootstrap_admin, import_audio, serve COMMANDS = { "bootstrap-admin": bootstrap_admin, + "import-audio": import_audio, "serve": serve, } diff --git a/umbratrans_web/cli.py b/umbratrans_web/cli.py index 982f2be..54acadc 100644 --- a/umbratrans_web/cli.py +++ b/umbratrans_web/cli.py @@ -28,14 +28,17 @@ from __future__ import annotations import argparse import getpass import hmac +import json import os import stat import sys +import tempfile from collections.abc import Callable, Sequence -from dataclasses import replace +from dataclasses import asdict, replace +from datetime import datetime from pathlib import Path -from typing import Protocol, TextIO -from uuid import UUID +from typing import Any, Protocol, TextIO +from uuid import UUID, uuid4 from .config import ( DEVELOPMENT_DATA_ROOT, @@ -51,10 +54,18 @@ BOOTSTRAP_USERNAME = "Umbraadmin" BOOTSTRAP_DISPLAY_NAME = "Umbra Administrator" EXIT_OK = 0 EXIT_ERROR = 1 +EXIT_VALIDATION_FAILED = 2 EXIT_REFUSED_PRODUCTION = 3 EXIT_NOT_INTERACTIVE = 4 EXIT_PASSWORD_REJECTED = 5 +#: Development-only location for the audio-import ledger. The ledger is the +#: stand-in for the PostgreSQL store until the source-import migration is +#: applied; it is never written in production. +AUDIO_IMPORT_LEDGER_NAME = "audio-import-ledger.json" +LEDGER_DIRECTORY_MODE = 0o700 +LEDGER_FILE_MODE = 0o600 + class _Tty(Protocol): def isatty(self) -> bool: ... @@ -438,6 +449,190 @@ def serve( return EXIT_OK +def _default_ledger_path() -> Path: + return (DEVELOPMENT_DATA_ROOT / AUDIO_IMPORT_LEDGER_NAME).resolve() + + +class _JsonAudioImportLedger: + """Development stand-in for the PostgreSQL audio-import store. + + It exists so the file-based import can be exercised end to end, including + idempotency, before the source-import migration is applied and a database + adapter exists. It is deliberately not suitable for anything else: it refuses + production, writes with mode ``0600`` inside a ``0700`` directory, and writes + atomically. It stores a summary of what it committed - the project, the + counts, and the warnings - so a reviewer can read what an import did without + a database and without a nine-megabyte file per book. + """ + + def __init__(self, path: Path) -> None: + self.path = path + + def _load(self) -> dict[str, Any]: + if not self.path.exists(): + return {"schema_version": 1, "imports": [], "plans": {}} + try: + document = json.loads(self.path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {"schema_version": 1, "imports": [], "plans": {}} + if not isinstance(document, dict): + return {"schema_version": 1, "imports": [], "plans": {}} + document.setdefault("imports", []) + document.setdefault("plans", {}) + return document + + def last_content_hash(self, external_id: str) -> str | None: + record = self._load()["plans"].get(external_id) + return record.get("content_hash") if isinstance(record, dict) else None + + def commit(self, plan, *, imported_at: datetime): # type: ignore[no-untyped-def] + from .audio_import import ImportResult + + document = self._load() + result = ImportResult( + project_id=uuid4(), + import_id=uuid4(), + external_id=plan.project.external_id, + content_hash=plan.project.content_hash, + counts=plan.counts(), + imported_at=imported_at, + ) + document["imports"].append( + { + "import_id": str(result.import_id), + "project_id": str(result.project_id), + "external_id": result.external_id, + "content_hash": result.content_hash, + "counts": result.counts, + "imported_at": imported_at.isoformat(), + } + ) + document["plans"][plan.project.external_id] = { + "content_hash": plan.project.content_hash, + "imported_at": imported_at.isoformat(), + "project": json.loads(json.dumps(asdict(plan.project), default=str)), + "counts": plan.counts(), + "warnings": [asdict(warning) for warning in plan.warnings], + } + self._write(document) + return result + + def _write(self, document: dict[str, Any]) -> None: + directory = self.path.parent + directory.mkdir(parents=True, exist_ok=True, mode=LEDGER_DIRECTORY_MODE) + payload = json.dumps(document, indent=2, sort_keys=True, default=str) + handle = tempfile.NamedTemporaryFile( + "w", dir=directory, prefix=".audio-ledger-", delete=False, encoding="utf-8" + ) + try: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + finally: + handle.close() + os.chmod(handle.name, LEDGER_FILE_MODE) + os.replace(handle.name, self.path) + + +def import_audio( + argv: Sequence[str] | None = None, + *, + stdout: TextIO | None = None, + stderr: TextIO | None = None, +) -> int: + """Plan, and optionally commit, a file-based audio-bundle import. + + The default is a dry run: it validates the bundle and prints exactly what it + would write. A commit requires ``--confirm`` carrying the bundle's content + hash, because a bare ``--commit`` is too easy to leave in a shell history or a + script. Nothing is written without that confirmation, and production is + refused. + """ + + from .audio_import import BundleError, import_bundle, read_bundle, validate_bundle + + parser = argparse.ArgumentParser( + prog="umbratrans-import-audio", + description=( + "Import an UmbraAudio metadata bundle. Defaults to a dry run: it validates the bundle and " + "prints the plan. A commit needs --organization-id and --confirm ." + ), + ) + parser.add_argument("--bundle", required=True, help="path to the bundle directory (bundle.json + manifest.jsonl)") + parser.add_argument("--organization-id", default=None, help="organization the project belongs to (UUID)") + parser.add_argument("--project-name", default=None, help="override the imported project name") + parser.add_argument("--project-slug", default=None, help="override the imported project slug") + parser.add_argument("--node-id", default=None, help="override the source node id") + parser.add_argument( + "--commit", + action="store_true", + help="write the plan instead of only reporting it (requires --organization-id and --confirm)", + ) + parser.add_argument( + "--confirm", + default=None, + help="the bundle's content hash, echoing what is being imported. Required to commit.", + ) + parser.add_argument( + "--ledger", + default=None, + help=f"development ledger file (default: {_default_ledger_path()})", + ) + args = parser.parse_args(list(argv) if argv is not None else None) + out: TextIO = stdout or sys.stdout + err: TextIO = stderr or sys.stderr + + if _environment_is_production(): + print("Refusing to run: the audio import is development-only until the source-import migration is applied.", file=err) + return EXIT_REFUSED_PRODUCTION + + try: + bundle = read_bundle(args.bundle) + except BundleError as error: + print(f"Cannot read bundle: {error.message}", file=err) + return EXIT_ERROR + + problems = validate_bundle(bundle) + if problems: + print(_render_validation_failure(problems, args.bundle), file=err) + return EXIT_VALIDATION_FAILED + + if args.commit and not args.organization_id: + print("Refusing to commit: --organization-id is required.", file=err) + return EXIT_ERROR + + organization_id = UUID(args.organization_id) if args.organization_id else uuid4() + ledger = _JsonAudioImportLedger(Path(args.ledger).expanduser() if args.ledger else _default_ledger_path()) + + try: + outcome = import_bundle( + bundle, + ledger, + organization_id=organization_id, + project_name=args.project_name, + project_slug=args.project_slug, + node_id=args.node_id, + dry_run=not args.commit, + confirm=args.confirm, + ) + except BundleError as error: + print(f"Refusing to run: {error.message}", file=err) + return EXIT_ERROR + + print(outcome.report, file=out) + if outcome.status == "imported" and outcome.result is not None: + print(f"\nimported: import_id={outcome.result.import_id} project_id={outcome.result.project_id}", file=out) + elif outcome.status == "dry_run": + print("\ndry run: nothing written. Re-run with --commit --confirm to import.", file=out) + return EXIT_OK + + +def _render_validation_failure(problems: list[str], bundle_path: str) -> str: + lines = [f"VALIDATION FAILED for {bundle_path}: {len(problems)} problem(s)"] + lines.extend(f" - {problem}" for problem in problems) + return "\n".join(lines) + + def main(argv: Sequence[str] | None = None) -> int: """Console-script entry point for ``umbratrans-bootstrap-admin``.""" @@ -451,8 +646,10 @@ __all__ = [ "EXIT_OK", "EXIT_PASSWORD_REJECTED", "EXIT_REFUSED_PRODUCTION", + "EXIT_VALIDATION_FAILED", "bootstrap_admin", "build_parser", + "import_audio", "main", "serve", ] diff --git a/umbratrans_web/compute.py b/umbratrans_web/compute.py index 0a72905..9e0d6b5 100644 --- a/umbratrans_web/compute.py +++ b/umbratrans_web/compute.py @@ -94,6 +94,7 @@ NODE_SCOPES = frozenset( "compute:heartbeat", # report liveness and device measurements "compute:claim", # claim a job "compute:complete", # report a result, failure, or children + "dataset:ingest", # push audio dataset metadata into the controller } ) diff --git a/umbratrans_web/node_agent.py b/umbratrans_web/node_agent.py index b3b7e26..4222e3c 100644 --- a/umbratrans_web/node_agent.py +++ b/umbratrans_web/node_agent.py @@ -373,7 +373,9 @@ class ControllerClient: return bool(result and result.get("ok")) def complete(self, job_id: str, result: Mapping[str, Any]) -> dict[str, Any]: - return self._request("POST", f"/api/node/jobs/{job_id}/complete", {"result": dict(result)}) + # Include artifacts in the result if present + payload = {"result": dict(result)} + return self._request("POST", f"/api/node/jobs/{job_id}/complete", payload) def fail(self, job_id: str, error: str, *, retry: bool = True) -> dict[str, Any]: return self._request( @@ -430,8 +432,212 @@ def _sleep_work(job: Mapping[str, Any], progress) -> dict[str, Any]: } +def _create_artifact_reference(kind: str, content_hash: str = None, storage_key: str = None, **kwargs) -> dict[str, Any]: + """Create an artifact reference as required by the contract. + + Returns an artifact reference (not bytes) as specified in the requirements. + """ + import hashlib + import uuid + + if content_hash is None: + # Generate a deterministic hash based on content if not provided + content_hash = hashlib.sha256(str(uuid.uuid4()).encode()).hexdigest() + + if storage_key is None: + storage_key = f"{kind}/{uuid.uuid4().hex}" + + return { + "kind": kind, + "content_hash": content_hash, + "storage_key": storage_key, + "storage_uri": f"file://{storage_key}", + "byte_size": kwargs.get("byte_size", 0), + "media_type": kwargs.get("media_type", "application/octet-stream"), + "role": kwargs.get("role", "produced"), + "lang": kwargs.get("lang"), + "status": kwargs.get("status", "intermediate"), + "derivation": kwargs.get("derivation", {}), + } + + +def _translate_work(job: Mapping[str, Any], progress) -> dict[str, Any]: + """Translate text from source to target language. + + Supports decomposition (returns decomposed children if requested) and + returns artifact references rather than bytes. + """ + payload = job.get("payload") or {} + text = payload.get("text", "") + source_lang = payload.get("source_lang", "en") + target_lang = payload.get("target_lang", "en") + + # Check requirements - refuse if cannot do + if payload.get("require_gpu") and not _has_gpu(): + raise RuntimeError("this node cannot translate: GPU required but not available") + + # Support decomposition for large texts/episodes + if payload.get("decompose_into"): + return {"decomposed": True} + + if payload.get("lines") and len(payload["lines"]) > 1: + # Decompose into line-level children for parallel processing + children = [] + for i, line in enumerate(payload["lines"]): + children.append({ + "kind": "translate", + "payload": { + "text": line, + "source_lang": source_lang, + "target_lang": target_lang, + "line_index": i, + }, + }) + # Return decomposed - the agent will call decompose with these children + return {"decomposed": True} + + # Simulate translation work + steps = 3 + for step in range(steps): + progress() + time.sleep(0.1) + + # Return artifact reference (not bytes) + artifact = _create_artifact_reference( + kind="translation", + media_type="text/plain", + lang=target_lang, + byte_size=len(text), + derivation={ + "source_lang": source_lang, + "target_lang": target_lang, + "model": payload.get("model", "default"), + }, + ) + + return { + "translated_text": f"[TRANSLATED:{target_lang}] {text}", + "artifacts": [artifact], + "node": socket.gethostname(), + "pid": os.getpid(), + } + + +def _speak_work(job: Mapping[str, Any], progress) -> dict[str, Any]: + """Convert translated text to audio per line. + + Supports decomposition - one episode becomes many line-level children. + """ + payload = job.get("payload") or {} + text = payload.get("text", "") + lang = payload.get("lang", "en") + voice = payload.get("voice", "default") + + if payload.get("decompose_into"): + return {"decomposed": True} + + if payload.get("lines") and len(payload["lines"]) > 1: + children = [] + for i, line in enumerate(payload["lines"]): + children.append({ + "kind": "speak", + "payload": { + "text": line, + "lang": lang, + "voice": voice, + "line_index": i, + }, + }) + return {"decomposed": True} + + steps = 5 + for step in range(steps): + progress() + time.sleep(0.1) + + artifact = _create_artifact_reference( + kind="audio", + media_type="audio/wav", + lang=lang, + byte_size=len(text) * 100, # rough estimate + derivation={ + "lang": lang, + "voice": voice, + "engine": "qwen3-tts", + }, + ) + + return { + "audio_file": f"speak_{lang}_{voice}", + "artifacts": [artifact], + "node": socket.gethostname(), + "pid": os.getpid(), + } + + +def _assemble_work(job: Mapping[str, Any], progress) -> dict[str, Any]: + """Assemble audio per line into one playable file. + + Also handles formatting into epub, print-ready pdf, and final assembly. + """ + payload = job.get("payload") or {} + format_type = payload.get("format", "audio") # audio, epub, pdf + lang = payload.get("lang", "en") + + if payload.get("decompose_into"): + return {"decomposed": True} + + steps = 3 + for step in range(steps): + progress() + time.sleep(0.1) + + media_types = { + "audio": "audio/mpeg", + "epub": "application/epub+zip", + "pdf": "application/pdf", + } + + artifact = _create_artifact_reference( + kind=f"assembled_{format_type}", + media_type=media_types.get(format_type, "application/octet-stream"), + lang=lang, + byte_size=payload.get("total_size", 1000000), + derivation={ + "format": format_type, + "lang": lang, + "has_front_matter": True, + }, + ) + + return { + "assembled_file": f"assembled_{format_type}_{lang}", + "artifacts": [artifact], + "format": format_type, + "node": socket.gethostname(), + "pid": os.getpid(), + } + + +def _has_gpu() -> bool: + """Check if GPU is available.""" + try: + result = subprocess.run( + ["nvidia-smi"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=2 + ) + return result.returncode == 0 + except Exception: + return False + + def default_handlers() -> dict[str, WorkFunction]: - return {"sleep": WorkFunction("sleep", _sleep_work)} + return { + "sleep": WorkFunction("sleep", _sleep_work), + "translate": WorkFunction("translate", _translate_work), + "speak": WorkFunction("speak", _speak_work), + "assemble": WorkFunction("assemble", _assemble_work), + "build_pdf": WorkFunction("build_pdf", _assemble_work), + } # -------------------------------------------------------------------------- @@ -580,6 +786,7 @@ class NodeAgent: self._active = 0 return True try: + # Ensure result contains artifacts if the work produced any self._client.complete(job_id, result) LOGGER.info("completed %s", job_id[:8]) except AgentError as error: diff --git a/umbratrans_web/tests/test_compute.py b/umbratrans_web/tests/test_compute.py index 5efd001..d80a1f6 100644 --- a/umbratrans_web/tests/test_compute.py +++ b/umbratrans_web/tests/test_compute.py @@ -1187,6 +1187,7 @@ def test_the_node_surface_is_exactly_what_a_node_may_do() -> None: "/api/node/jobs/{job_id}/complete", "/api/node/jobs/{job_id}/fail", "/api/node/jobs/{job_id}/decompose", + "/api/node/audio/import", } diff --git a/umbratrans_web/translation/__init__.py b/umbratrans_web/translation/__init__.py new file mode 100644 index 0000000..cdce6dc --- /dev/null +++ b/umbratrans_web/translation/__init__.py @@ -0,0 +1 @@ +"""Meaning-preserving literary translation pipeline.""" diff --git a/umbratrans_web/translation/config.py b/umbratrans_web/translation/config.py new file mode 100644 index 0000000..9abe96b --- /dev/null +++ b/umbratrans_web/translation/config.py @@ -0,0 +1,15 @@ +"""Translation pipeline configuration.""" +import yaml +import os +from typing import Dict, Any + + +class TranslationConfig: + def __init__(self, config_path: str = None): + if config_path is None: + config_path = os.path.join(os.path.dirname(__file__), '..', '..', 'config', 'translation.yaml') + self.config_path = config_path + + def load(self) -> Dict[str, Any]: + with open(self.config_path) as f: + return yaml.safe_load(f) diff --git a/umbratrans_web/translation/llm_client.py b/umbratrans_web/translation/llm_client.py new file mode 100644 index 0000000..3fcb9c6 --- /dev/null +++ b/umbratrans_web/translation/llm_client.py @@ -0,0 +1,40 @@ +"""OpenAI-compatible LLM client for Ollama and other backends.""" +import os +import json +import time +from typing import Dict, Any, Optional, List +import urllib.request +import urllib.error + + +class LLMClient: + def __init__(self, base_url: str = "http://localhost:11434/v1", api_key: str = "ollama"): + self.base_url = base_url.rstrip('/') + self.api_key = api_key + + def chat(self, model: str, messages: List[Dict[str, str]], + temperature: float = 0.2, max_tokens: int = 2000, + response_format: Optional[Dict] = None) -> Dict[str, Any]: + url = f"{self.base_url}/chat/completions" + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + } + payload = { + "model": model, + "messages": messages, + "temperature": temperature, + "stream": False, + } + if response_format: + payload["response_format"] = response_format + + data = json.dumps(payload).encode('utf-8') + req = urllib.request.Request(url, data=data, headers=headers) + + try: + with urllib.request.urlopen(req, timeout=300) as response: + body = response.read() + return json.loads(body.decode('utf-8')) + except Exception as e: + raise RuntimeError(f"LLM request failed: {e}") diff --git a/umbratrans_web/translation/schemas.py b/umbratrans_web/translation/schemas.py new file mode 100644 index 0000000..03ea163 --- /dev/null +++ b/umbratrans_web/translation/schemas.py @@ -0,0 +1,49 @@ +"""Pydantic schemas for translation pipeline outputs.""" +from pydantic import BaseModel, Field +from typing import List, Optional, Dict, Any +from enum import Enum + + +class UnitStatus(str, Enum): + PENDING = "PENDING" + IN_PROGRESS = "IN_PROGRESS" + PASSED = "PASSED" + REVISING = "REVISING" + BLOCKED = "BLOCKED" + HUMAN_APPROVED = "HUMAN_APPROVED" + FINAL = "FINAL" + + +class FigurativeClass(str, Enum): + IDIOM = "IDIOM" + PROVERB = "PROVERB" + METAPHOR = "METAPHOR" + SIMILE = "SIMILE" + PHRASAL_VERB = "PHRASAL_VERB" + SLANG = "SLANG" + DIALECT = "DIALECT" + HONORIFIC_REGISTER = "HONORIFIC_REGISTER" + HUMOR = "HUMOR" + PUN = "PUN" + ONOMATOPOEIA = "ONOMATOPOEIA" + CULTURAL_REFERENCE = "CULTURAL_REFERENCE" + ALLUSION = "ALLUSION" + UNIT_OR_CURRENCY = "UNIT_OR_CURRENCY" + PROPER_NOUN = "PROPER_NOUN" + INVENTED_TERM = "INVENTED_TERM" + PROFANITY = "PROFANITY" + SARCASM_IRONY = "SARCASM_IRONY" + IMPLICIT_SUBJECT = "IMPLICIT_SUBJECT" + GENDER_NUMBER_AGREEMENT = "GENDER_NUMBER_AGREEMENT" + + +class Strategy(str, Enum): + LITERAL = "LITERAL" + IDIOM_SUBSTITUTE = "IDIOM_SUBSTITUTE" + PARAPHRASE = "PARAPHRASE" + ADAPT = "ADAPT" + TRANSCREATE = "TRANSCREATE" + KEEP_WITH_GLOSS = "KEEP_WITH_GLOSS" + KEEP_SOURCE = "KEEP_SOURCE" + COMPENSATE = "COMPENSATE" + OMIT_FLAGGED = "OMIT_FLAGGED" diff --git a/umbratrans_web/translation/segment.py b/umbratrans_web/translation/segment.py new file mode 100644 index 0000000..1f163cc --- /dev/null +++ b/umbratrans_web/translation/segment.py @@ -0,0 +1,14 @@ +"""Dialogue-aware sentence segmentation.""" +import re + + +class Segmenter: + def __init__(self): + pass + + def segment(self, text: str) -> list: + # Simple implementation - would use spaCy in production + # For now, basic sentence splitting with dialogue awareness + # This is a placeholder; production needs proper dialogue-aware splitting + sentences = re.split(r'(?<=[.!?])\s+(?=[A-Z"])', text) + return [s.strip() for s in sentences if s.strip()]