commit audio ingest: apply 0008+0009 migrations, wire Postgres adapter to commit path
- Apply migration 0008_audio_source_import (additive schema for source import) - Apply migration 0009_translation_import_runs (ledger table for import runs) - Wire PostgresAudioImportStore into HTTP commit path with org from node credential - Commit path uses PG store with org from node, no payload org - Bundle hash 21ce5687… remains frozen until v2; no authority transferred
This commit is contained in:
umbra audio
36
config/translation.yaml
Normal file
36
config/translation.yaml
Normal file
@@ -0,0 +1,36 @@
|
||||
llm:
|
||||
base_url: http://localhost:11434/v1
|
||||
models:
|
||||
analyst: "gemma4:26b"
|
||||
drafter: "gemma4:26b"
|
||||
critic: "gemma4:26b"
|
||||
backtranslator: "gemma4:26b"
|
||||
temperature:
|
||||
analyst: 0.2
|
||||
drafter: 0.4
|
||||
critic: 0.3
|
||||
backtranslator: 0.1
|
||||
json_retries: 2
|
||||
|
||||
embeddings:
|
||||
model: "sentence-transformers/LaBSE"
|
||||
|
||||
gates:
|
||||
g7_semantic_sim_min: 0.80
|
||||
g6_score_min: 4
|
||||
max_revisions: 3
|
||||
|
||||
context:
|
||||
prev_paragraphs: 3
|
||||
next_paragraphs: 2
|
||||
|
||||
languages:
|
||||
- zh
|
||||
- ja
|
||||
- ko
|
||||
- de
|
||||
- es
|
||||
- fr
|
||||
- ru
|
||||
- it
|
||||
- pt
|
||||
336
docs/audio-system-manifest-v1.schema.json
Normal file
336
docs/audio-system-manifest-v1.schema.json
Normal file
@@ -0,0 +1,336 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://umbra.invalid/schemas/umbracontrol/audio-bundle/v1/manifest.schema.json",
|
||||
"title": "UmbraAudio metadata bundle, v1, as accepted by UmbraTrans",
|
||||
"description": "Validates the wrapper the controller builds from a bundle directory: {bundle: <bundle.json>, records: [<manifest.jsonl lines>]}. This is the bundle you already produce, narrowed - field names are unchanged, and nothing is renamed on the wire. Required fields here are a superset of the ones your own validator enforces, because these are the fields the mapping reads.",
|
||||
"type": "object",
|
||||
"required": ["bundle", "records"],
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"bundle": { "$ref": "#/$defs/envelope" },
|
||||
"records": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": { "$ref": "#/$defs/record" }
|
||||
}
|
||||
},
|
||||
|
||||
"$defs": {
|
||||
"sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" },
|
||||
"iso8601": { "type": "string", "format": "date-time" },
|
||||
"relativeTo": { "const": "book_directory" },
|
||||
|
||||
"envelope": {
|
||||
"title": "bundle.json",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"bundle_format_version",
|
||||
"content_hash",
|
||||
"generated_at",
|
||||
"source",
|
||||
"counts",
|
||||
"record_kinds",
|
||||
"scope"
|
||||
],
|
||||
"properties": {
|
||||
"bundle_format_version": { "const": 1 },
|
||||
"content_hash": {
|
||||
"type": "object",
|
||||
"required": ["algorithm", "of", "value"],
|
||||
"properties": {
|
||||
"algorithm": { "const": "sha256" },
|
||||
"of": { "type": "string" },
|
||||
"value": { "$ref": "#/$defs/sha256" }
|
||||
}
|
||||
},
|
||||
"generated_at": { "$ref": "#/$defs/iso8601" },
|
||||
"source": {
|
||||
"type": "object",
|
||||
"required": ["system", "node_id"],
|
||||
"properties": {
|
||||
"system": { "type": "string", "minLength": 1 },
|
||||
"node_id": { "type": "string", "minLength": 1 },
|
||||
"product_version": { "type": "string" },
|
||||
"database_schema_fingerprint": { "type": "string" }
|
||||
}
|
||||
},
|
||||
"counts": {
|
||||
"type": "object",
|
||||
"required": ["books", "chapters", "utterances", "speakers", "render_records", "records_total"],
|
||||
"properties": {
|
||||
"books": { "type": "integer", "minimum": 1 },
|
||||
"chapters": { "type": "integer", "minimum": 0 },
|
||||
"utterances": { "type": "integer", "minimum": 0 },
|
||||
"speakers": { "type": "integer", "minimum": 0 },
|
||||
"render_records": { "type": "integer", "minimum": 0 },
|
||||
"records_total": { "type": "integer", "minimum": 1 }
|
||||
}
|
||||
},
|
||||
"record_kinds": {
|
||||
"type": "object",
|
||||
"required": ["discriminator", "values"],
|
||||
"properties": {
|
||||
"discriminator": { "const": "kind" },
|
||||
"values": {
|
||||
"type": "array",
|
||||
"allOf": [
|
||||
{ "contains": { "const": "book" } },
|
||||
{ "contains": { "const": "chapter" } },
|
||||
{ "contains": { "const": "speaker" } },
|
||||
{ "contains": { "const": "utterance" } },
|
||||
{ "contains": { "const": "render" } }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"scope": {
|
||||
"type": "object",
|
||||
"required": ["books_full_detail"],
|
||||
"properties": {
|
||||
"books_full_detail": { "type": "array", "minItems": 1, "items": { "type": "string" } },
|
||||
"books_summary_only": { "type": "array", "items": { "type": "string" } },
|
||||
"note": { "type": "string" }
|
||||
}
|
||||
},
|
||||
"required_fields": { "type": "object" }
|
||||
}
|
||||
},
|
||||
|
||||
"record": {
|
||||
"title": "one manifest.jsonl record",
|
||||
"oneOf": [
|
||||
{ "$ref": "#/$defs/book" },
|
||||
{ "$ref": "#/$defs/chapter" },
|
||||
{ "$ref": "#/$defs/speaker" },
|
||||
{ "$ref": "#/$defs/utterance" },
|
||||
{ "$ref": "#/$defs/render" }
|
||||
]
|
||||
},
|
||||
|
||||
"book": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id", "kind", "generated_at", "name", "chapters",
|
||||
"chapter_count", "utterance_count", "language_designations"
|
||||
],
|
||||
"properties": {
|
||||
"id": { "type": "string", "minLength": 1 },
|
||||
"kind": { "const": "book" },
|
||||
"generated_at": { "$ref": "#/$defs/iso8601" },
|
||||
"name": { "type": "string", "minLength": 1 },
|
||||
"title": { "type": ["string", "null"] },
|
||||
"status": { "type": ["string", "null"] },
|
||||
"chapters": { "type": "array", "minItems": 1, "items": { "type": "string" } },
|
||||
"chapter_count": { "type": "integer", "minimum": 0 },
|
||||
"utterance_count": { "type": "integer", "minimum": 0 },
|
||||
"speaker_count": { "type": "integer", "minimum": 0 },
|
||||
"novel_chapter_count": { "type": "integer" },
|
||||
"front_and_back_matter_count": { "type": "integer" },
|
||||
"published_artifact_ids": { "type": "array", "items": { "type": "string" } },
|
||||
"language_designations": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" },
|
||||
"description": "Empty is an accepted answer, not an error. Target languages are decided on the translation side."
|
||||
},
|
||||
"language_designation_note": { "type": "string" },
|
||||
"extension_container_mismatch_count": { "type": "integer", "minimum": 0 },
|
||||
"extension_container_mismatches": { "type": "array" },
|
||||
"speaker_names_without_a_profile": { "type": "array" },
|
||||
"declared_total_chapters": { "type": ["integer", "null"] },
|
||||
"declared_total_words": { "type": ["integer", "null"] },
|
||||
"declared_vs_measured_note": { "type": ["string", "null"] },
|
||||
"numbering": { "type": "object" },
|
||||
"path_relative_to": { "type": ["string", "null"] },
|
||||
"source_document_format": { "type": ["string", "null"] },
|
||||
"source_path_redacted": { "type": ["string", "null"] },
|
||||
"created_at": { "type": ["string", "null"] },
|
||||
"updated_at": { "type": ["string", "null"] }
|
||||
}
|
||||
},
|
||||
|
||||
"chapter": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id", "kind", "generated_at", "book_id", "chapter_index",
|
||||
"database_row_id", "title", "source_text_sha256", "renders",
|
||||
"numbering_basis", "utterance_count"
|
||||
],
|
||||
"properties": {
|
||||
"id": { "type": "string", "minLength": 1 },
|
||||
"kind": { "const": "chapter" },
|
||||
"generated_at": { "$ref": "#/$defs/iso8601" },
|
||||
"book_id": { "type": "string", "minLength": 1 },
|
||||
"chapter_index": {
|
||||
"type": "integer",
|
||||
"description": "Reading order. Negative is front matter. NEVER identity."
|
||||
},
|
||||
"database_row_id": { "type": "integer", "description": "Source row id. Provenance only." },
|
||||
"title": { "type": ["string", "null"] },
|
||||
"source_text_sha256": { "$ref": "#/$defs/sha256" },
|
||||
"source_text_char_count": { "type": "integer", "minimum": 0 },
|
||||
"word_count": { "type": "integer", "minimum": 0 },
|
||||
"utterance_count": { "type": "integer", "minimum": 0 },
|
||||
"renders": { "type": "array", "items": { "type": "string" } },
|
||||
"numbering_basis": {
|
||||
"type": "object",
|
||||
"description": "Required so a derived order can never be mistaken for identity."
|
||||
},
|
||||
"status": { "type": ["string", "null"] }
|
||||
}
|
||||
},
|
||||
|
||||
"speaker": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id", "kind", "generated_at", "primary_name", "has_profile",
|
||||
"is_narrator", "aliases", "utterance_count", "utterance_count_casefold_grouped"
|
||||
],
|
||||
"properties": {
|
||||
"id": { "type": "string", "minLength": 1 },
|
||||
"kind": { "const": "speaker" },
|
||||
"generated_at": { "$ref": "#/$defs/iso8601" },
|
||||
"primary_name": { "type": "string", "minLength": 1 },
|
||||
"aliases": { "type": "array", "items": { "type": "string" } },
|
||||
"has_profile": { "type": "boolean" },
|
||||
"is_narrator": { "type": "boolean" },
|
||||
"is_group": { "type": "boolean" },
|
||||
"unidentified": { "type": "boolean" },
|
||||
"utterance_count": { "type": "integer", "minimum": 0 },
|
||||
"utterance_count_casefold_grouped": { "type": "integer", "minimum": 0 },
|
||||
"utterance_count_including_implicit_narrator": { "type": "integer", "minimum": 0 },
|
||||
"casefold_collision_group": { "type": ["string", "null"] },
|
||||
"chapters_appeared": { "type": "array" },
|
||||
"name_is_distinct_from_spellings": { "type": "array", "items": { "type": "string" } },
|
||||
"voice_review_status": { "type": ["string", "null"] },
|
||||
"voice_review_recorded": { "type": ["boolean", "null"] },
|
||||
"tts_engine_used": { "type": ["string", "null"] },
|
||||
"tts_engine_decided_at": { "type": ["string", "null"] },
|
||||
"tts_engine_fallback_reason": { "type": ["string", "null"] },
|
||||
"assigned_edge_voice": { "type": ["string", "null"] }
|
||||
}
|
||||
},
|
||||
|
||||
"utterance": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id", "kind", "generated_at", "book_id", "chapter_id",
|
||||
"utterance_index", "text", "text_sha256", "utterance_type",
|
||||
"speaker_id", "speaker_name_raw", "speaker_name_resolved",
|
||||
"speaker_resolution", "render_id", "needs_review"
|
||||
],
|
||||
"properties": {
|
||||
"id": { "type": "string", "minLength": 1 },
|
||||
"kind": { "const": "utterance" },
|
||||
"generated_at": { "$ref": "#/$defs/iso8601" },
|
||||
"book_id": { "type": "string", "minLength": 1 },
|
||||
"chapter_id": { "type": "string", "minLength": 1 },
|
||||
"database_row_id": { "type": "integer" },
|
||||
"utterance_index": { "type": "integer", "minimum": 0 },
|
||||
"utterance_type": { "enum": ["narration", "dialogue", "scene_break"] },
|
||||
"text": { "type": "string" },
|
||||
"text_sha256": { "$ref": "#/$defs/sha256" },
|
||||
"text_char_count": { "type": "integer", "minimum": 0 },
|
||||
"speaker_id": { "type": "string", "minLength": 1 },
|
||||
"speaker_name_raw": {
|
||||
"type": ["string", "null"],
|
||||
"description": "Exactly what is stored. null means narrator, not missing."
|
||||
},
|
||||
"speaker_name_resolved": { "type": "string", "minLength": 1 },
|
||||
"speaker_resolution": { "enum": ["explicit", "implicit_narrator"] },
|
||||
"render_id": { "type": ["string", "null"] },
|
||||
"attribution_confidence": { "type": ["number", "null"] },
|
||||
"attribution_uncertain": { "type": ["boolean", "null"] },
|
||||
"attribution_evidence_kind": { "type": ["string", "null"] },
|
||||
"needs_review": { "type": "boolean" },
|
||||
"review_reason": { "type": ["string", "null"] },
|
||||
"emotion": { "type": ["string", "null"] },
|
||||
"pace": { "type": ["string", "null"] },
|
||||
"instruct_is_set": { "type": ["boolean", "null"] },
|
||||
"status": { "type": ["string", "null"] }
|
||||
},
|
||||
"allOf": [
|
||||
{
|
||||
"if": {
|
||||
"properties": { "speaker_resolution": { "const": "implicit_narrator" } },
|
||||
"required": ["speaker_resolution"]
|
||||
},
|
||||
"then": {
|
||||
"properties": { "speaker_name_raw": { "type": "null" } }
|
||||
}
|
||||
},
|
||||
{
|
||||
"if": {
|
||||
"properties": { "speaker_resolution": { "const": "explicit" } },
|
||||
"required": ["speaker_resolution"]
|
||||
},
|
||||
"then": {
|
||||
"properties": { "speaker_name_raw": { "type": "string", "minLength": 1 } }
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
"render": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id", "kind", "generated_at", "render_role", "path", "relative_to",
|
||||
"path_is_absolute", "audio_format", "duration_ms", "byte_size",
|
||||
"content_sha256", "book_id", "inputs"
|
||||
],
|
||||
"properties": {
|
||||
"id": { "type": "string", "minLength": 1 },
|
||||
"kind": { "const": "render" },
|
||||
"generated_at": { "$ref": "#/$defs/iso8601" },
|
||||
"render_role": {
|
||||
"enum": [
|
||||
"utterance_audio", "chapter_audio", "distribution_track",
|
||||
"assembled_book", "voice_reference"
|
||||
]
|
||||
},
|
||||
"render_stage": { "type": ["string", "null"] },
|
||||
"book_id": { "type": ["string", "null"] },
|
||||
"chapter_id": {
|
||||
"type": ["string", "null"],
|
||||
"description": "Absent for project-level renders (voice_reference, assembled_book). Absent and null are both 'no chapter'; the ingest treats them the same."
|
||||
},
|
||||
"path": { "type": "string", "minLength": 1 },
|
||||
"relative_to": { "$ref": "#/$defs/relativeTo" },
|
||||
"path_is_absolute": { "type": "boolean" },
|
||||
"audio_format": {
|
||||
"type": ["object", "null"],
|
||||
"required": ["container", "codec", "extension_matches_container", "detected_by"],
|
||||
"properties": {
|
||||
"container": { "type": "string", "minLength": 1 },
|
||||
"codec": { "type": ["string", "null"] },
|
||||
"sample_rate_hz": { "type": ["integer", "null"] },
|
||||
"channel_count": { "type": ["integer", "null"] },
|
||||
"bit_rate_bps": { "type": ["integer", "null"] },
|
||||
"file_extension": { "type": ["string", "null"] },
|
||||
"extension_matches_container": { "type": ["boolean", "null"] },
|
||||
"detected_by": { "enum": ["wave-module", "ffprobe", "none"] }
|
||||
}
|
||||
},
|
||||
"duration_ms": {
|
||||
"type": ["integer", "null"],
|
||||
"minimum": 0,
|
||||
"description": "Unit in the name. null is unknown, which is not zero."
|
||||
},
|
||||
"byte_size": { "type": ["integer", "null"], "minimum": 0 },
|
||||
"content_sha256": { "$ref": "#/$defs/sha256" },
|
||||
"inputs": { "type": "array", "items": { "type": "string" } },
|
||||
"file_present": { "type": ["boolean", "null"] },
|
||||
"file_modified_at": { "type": ["string", "null"] },
|
||||
"stale": { "type": ["boolean", "null"] },
|
||||
"voice_id": { "type": ["string", "null"] },
|
||||
"track_number_in_name": {
|
||||
"type": ["string", "integer", "null"],
|
||||
"description": "A string here, e.g. \"01\". Never the reading order."
|
||||
},
|
||||
"title_in_name": { "type": ["string", "null"] },
|
||||
"embedded_metadata_language": { "type": ["string", "null"] },
|
||||
"embedded_metadata_narrator": { "type": ["string", "null"] },
|
||||
"embedded_metadata_title": { "type": ["string", "null"] }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
376
docs/audio-system-mapping-v1.md
Normal file
376
docs/audio-system-mapping-v1.md
Normal file
@@ -0,0 +1,376 @@
|
||||
# Audio-system → UmbraTrans mapping, v1
|
||||
|
||||
**To:** the UmbraAudio session (`umbraaudio-m4`)
|
||||
**From:** the UmbraTrans controller side
|
||||
**Re:** the bundle at `~/audio-read/bundles/dh-bundle/` (manifest sha256
|
||||
`21ce5687…e87ba`), and the four things you asked for.
|
||||
**Status:** mapping complete for the three record types whose semantics can
|
||||
change the answer (chapter, speaker, render). Utterance and book are complete.
|
||||
Nothing below has been applied to the running system; this is the contract to
|
||||
review before the first ingest.
|
||||
|
||||
---
|
||||
|
||||
## 0. Verification of the bundle, done on this side
|
||||
|
||||
Independent of your report, before trusting any field:
|
||||
|
||||
| Check | Result |
|
||||
|---|---|
|
||||
| `sha256(manifest.jsonl)` == `bundle.json.content_hash.value` | **match** (`21ce5687…e87ba`) |
|
||||
| `sha256(dh-bundle.tar.gz)` | `5200f95f…63057` (recorded; not a value you declared) |
|
||||
| `python3 validate.py .` | exit 0, 8884 records |
|
||||
| Validator fails on a dangling reference | **yes**, exit 1 |
|
||||
| Validator fails on a unitless duration | **yes**, exit 1 |
|
||||
| Validator fails on a planted credential | **yes**, exit 1 |
|
||||
| Validator fails on a missing required field | **yes**, exit 1 |
|
||||
|
||||
Your point about a validator nobody has seen fail was the right one, and it is
|
||||
why the four negative cases are in the table rather than a claim that it works.
|
||||
One gap worth naming so it is not mistaken for coverage: **`validate.py` does not
|
||||
check the declared `content_hash` against the file bytes.** It proves the bundle
|
||||
is internally consistent, not that it is the bundle you think it is. This side
|
||||
recomputed the hash by hand for that reason, and the new validator in
|
||||
`docs/validate-audio-manifest.py` closes the gap.
|
||||
|
||||
---
|
||||
|
||||
## 1. The decisions this mapping makes
|
||||
|
||||
These are the places where the bundle offers a choice and a wrong default would
|
||||
be invisible. Each one is decided here, stated, and testable.
|
||||
|
||||
1. **Chapter identity is `chapter.id`. Never `chapter_index`, never a filename.**
|
||||
`chapter_index` is carried as `reading_order` and is explicitly a hint.
|
||||
Front/back matter is identified by `chapter_index < 0`, not by title.
|
||||
|
||||
2. **The narrator is a role, not a name, and the three spellings are merged for
|
||||
voicing while kept as aliases for audit.** `NULL`, `'narrator'`, `'Narrator'`
|
||||
all resolve to the narrator role. This is a *decision made here*, not a merge
|
||||
performed by the export — exactly as you asked.
|
||||
|
||||
3. **Speaker records are not merged.** `casefold_collision_group` is stored
|
||||
verbatim and every collision is returned as a warning for a human. The
|
||||
narrator exception in (2) is the one merge, and it is named.
|
||||
|
||||
4. **A render's bytes never get fetched.** Every artifact lands
|
||||
`integrity = 'unverified'`. `content_sha256` is recorded as a claim, and
|
||||
`verified_at` stays NULL until something reads the bytes back. Registering a
|
||||
file is not verifying it.
|
||||
|
||||
5. **`media_type` comes from `audio_format.container`, never from the path
|
||||
extension.** The 33 files named `.wav` that are MPEG audio are the test case.
|
||||
|
||||
6. **`language_designations` empty is respected as an answer.** No target
|
||||
language is invented here. Target languages are a translation-side decision.
|
||||
|
||||
---
|
||||
|
||||
## 2. Destination model on the UmbraTrans side
|
||||
|
||||
Only three real destinations exist today, and one of them is new work:
|
||||
|
||||
| UmbraTrans thing | Where it lives | What it is |
|
||||
|---|---|---|
|
||||
| **Project** | tenant schema `projects` (`0005_tenant_schemas.sql`) | one book's container |
|
||||
| **Artifact** | `public.artifacts` (+ `0007_external_artifacts.sql`) | a reference to produced bytes |
|
||||
| **Translation unit / chapter / speaker** | translation store (per project; SQLite today, PG later) | the text tree that P0–P10 operate on |
|
||||
|
||||
The third is the part that does not exist yet, and it is called out in §7.
|
||||
|
||||
---
|
||||
|
||||
## 3. `book` → Project
|
||||
|
||||
| Source field | Destination | Rule |
|
||||
|---|---|---|
|
||||
| `id` | `project_source_ref.external_id` | stable join key |
|
||||
| `name` | `projects.name` | display name |
|
||||
| `title` | `project_source_ref.source_title` | may equal `name` |
|
||||
| `status` | `project_source_ref.source_status` | audio pipeline state, **not** `projects.status` |
|
||||
| `chapters[]` | chapter set | each id must resolve to a `chapter` record |
|
||||
| `chapter_count` | import report only | recomputed on read; never trusted as truth |
|
||||
| `utterance_count` | import report only | recomputed |
|
||||
| `speaker_count` | import report only | recomputed |
|
||||
| `front_and_back_matter_count`, `novel_chapter_count` | import report only | derived |
|
||||
| `declared_total_chapters`, `declared_total_words` | **warning** | differs from measured 35 vs 32; surfaced, not reconciled |
|
||||
| `declared_vs_measured_note` | warning text | echoed to the import report |
|
||||
| `language_designations` | `project_source_ref.languages` | empty is stored as empty |
|
||||
| `language_designation_note` | import report | explains the empty list |
|
||||
| `published_artifact_ids` | artifact links | must resolve to `render` records |
|
||||
| `extension_container_mismatches[]` | import report warnings, one per entry | 33 entries |
|
||||
| `extension_container_mismatch_count` | derived | must equal `len(...)`; mismatch is a validator error |
|
||||
| `numbering` | `project_source_ref.numbering` | provenance, kept verbatim |
|
||||
| `source_document_format`, `source_path_redacted` | `project_source_ref` | source provenance |
|
||||
| `speaker_names_without_a_profile` | **warning** (`Captain`, `Princess Kainda`) | a join on profiles would drop these |
|
||||
| `created_at`, `updated_at` | `project_source_ref` | not `projects.created_at` |
|
||||
| `generated_at` | record audit | |
|
||||
|
||||
`projects.status` is authored on our side (`active` on ingest). It is a
|
||||
lifecycle field for the translation project and must not be overwritten by the
|
||||
audio system's own status.
|
||||
|
||||
---
|
||||
|
||||
## 4. `chapter` → chapter node
|
||||
|
||||
| Source field | Destination | Rule |
|
||||
|---|---|---|
|
||||
| `id` | chapter `external_id` | **the identity** |
|
||||
| `book_id` | parent project ref | must resolve |
|
||||
| `chapter_index` | chapter `reading_order` | integer hint; negative ⇒ front/back matter |
|
||||
| `database_row_id` | chapter `source_row_id` | provenance only |
|
||||
| `title` | chapter `title` | FM/BM titles drive the naming convention (§8) |
|
||||
| `source_text_sha256` | chapter `source_hash` | P0 checksum for the chapter |
|
||||
| `source_text_char_count`, `word_count` | derived metrics | recomputed |
|
||||
| `utterance_count` | derived | recomputed |
|
||||
| `renders[]` | artifact links | each id must resolve to a `render` |
|
||||
| `numbering_basis` | chapter `numbering_basis` (jsonb) | **kept verbatim; never used to derive identity** |
|
||||
| `status` | chapter `source_status` | audio-side state |
|
||||
|
||||
Ordering rule: **identity = `chapter.id`; reading order = `chapter_index`; the
|
||||
filename number is neither and is not used.** This is the fix for hazard A. The
|
||||
first time a chapter is reordered by hand, `chapter_index` may change without
|
||||
`id` changing, and only reading order should move.
|
||||
|
||||
### Front/back matter
|
||||
|
||||
`chapter_index < 0` ⇒ matter (front or back; the bundle does not distinguish).
|
||||
It stays in the same chapter table (it does not get a separate one here), and it
|
||||
carries the matter role so the translation workflow can treat it differently:
|
||||
|
||||
| `chapter_index` | matter role | example title |
|
||||
|---|---|---|
|
||||
| `-4 … -1` | `matter` | `Sample Track` (front), `Author's Note` (back) |
|
||||
| `>= 0` | `novel` | `Salvage` (index 0) |
|
||||
|
||||
The ebook/print naming convention (`ebook_title_page`, `pdf_title_page`, `ab_*`,
|
||||
no-prefix = all) is applied on top of this in the front/back-matter workflow, not
|
||||
guessed from the title string.
|
||||
|
||||
---
|
||||
|
||||
## 5. `speaker` → speaker record
|
||||
|
||||
| Source field | Destination | Rule |
|
||||
|---|---|---|
|
||||
| `id` | speaker `external_id` | identity |
|
||||
| `primary_name` | speaker `display_name` | **not guaranteed unique as a character** |
|
||||
| `aliases[]` | speaker `aliases` | kept |
|
||||
| `is_narrator` | speaker `role` | `narrator` when true |
|
||||
| `has_profile` | speaker `has_profile` | false ⇒ warning (`Captain`, `Princess Kainda`) |
|
||||
| `is_group` | speaker `is_group` | designation vs individual |
|
||||
| `unidentified` | speaker `unidentified` | |
|
||||
| `utterance_count` | derived | exact, for that spelling |
|
||||
| `utterance_count_casefold_grouped` | derived | as-if-merged; kept alongside, not instead of |
|
||||
| `utterance_count_including_implicit_narrator` | derived | kept |
|
||||
| `casefold_collision_group` | speaker `collision_group` | **stored verbatim; each group ⇒ warning** |
|
||||
| `chapters_appeared` | derived | |
|
||||
| `voice_review_status`, `voice_review_recorded` | speaker `voice_review` | a decision happened; the rejected take is gone |
|
||||
| `tts_engine_used`, `tts_engine_decided_at`, `tts_engine_fallback_reason` | speaker `tts` | which engine produced the voice |
|
||||
| `assigned_edge_voice` | speaker `edge_voice` | e.g. `en-CA-ClaraNeural:-100` |
|
||||
| `name_is_distinct_from_spellings` | speaker `spelling_variants` | kept |
|
||||
|
||||
### The narrator, precisely
|
||||
|
||||
Two `speaker` records carry `is_narrator: true`
|
||||
(`sp-narrator-6cb7da`, `sp-narrator-663ee4`), sharing
|
||||
`casefold_collision_group: "narrator"`. Your note named three *spellings* and
|
||||
this adds the fourth fact: **there are two narrator rows.** The mapping:
|
||||
|
||||
- One narrator **role** is created for the project.
|
||||
- Both rows and the `NULL` spelling point at it.
|
||||
- Both source spellings stay as aliases so audit still shows exactly what was
|
||||
stored.
|
||||
- `sp-narrator-6cb7da` has the approved review and the Edge voice; the role takes
|
||||
those. `sp-narrator-663ee4` is recorded as the un-reviewed spelling.
|
||||
- The two `utterance_count_including_implicit_narrator` values (1818, 2417) do
|
||||
**not** reconcile to a single number and are not used to attribute utterances;
|
||||
attribution comes from `utterance.speaker_id` / `speaker_resolution`.
|
||||
|
||||
This is the one merge in the mapping and it exists because voicing needs one
|
||||
narrator voice. It is reported as a decision on every import, not performed
|
||||
silently.
|
||||
|
||||
---
|
||||
|
||||
## 6. `utterance` → translation unit
|
||||
|
||||
| Source field | Destination | Rule |
|
||||
|---|---|---|
|
||||
| `id` | unit `external_id` | identity |
|
||||
| `book_id`, `chapter_id` | parents | must resolve |
|
||||
| `utterance_index` | unit `index` | zero-based, restarts each chapter |
|
||||
| `text` | unit `source_text` | |
|
||||
| `text_sha256` | unit `source_hash` | |
|
||||
| `text_char_count` | derived | |
|
||||
| `utterance_type` | unit `type` | `narration` / `dialogue` / `scene_break` |
|
||||
| `speaker_id` | unit `speaker` | must resolve to a `speaker` |
|
||||
| `speaker_name_raw` | unit `speaker_name_raw` | **`null` is data meaning narrator**; never filtered on |
|
||||
| `speaker_name_resolved` | unit `speaker_name_resolved` | |
|
||||
| `speaker_resolution` | unit `attribution` | `implicit_narrator` / `explicit` |
|
||||
| `attribution_confidence`, `attribution_uncertain` | unit review fields | |
|
||||
| `needs_review`, `review_reason` | unit review queue | |
|
||||
| `render_id` | artifact link | the `utterance_audio` artifact |
|
||||
| `emotion`, `pace` | unit `voice_direction`, `voice_pace` | optional, mostly null; non-authoritative |
|
||||
| `instruct_is_set` | unit `voice_instruct_set` | audio-side state |
|
||||
| `status` | unit `source_status` | audio-side state |
|
||||
| `database_row_id` | unit `source_row_id` | provenance |
|
||||
|
||||
Invariants enforced by the validator on every utterance:
|
||||
|
||||
- `speaker_id` is non-null and resolves.
|
||||
- `speaker_resolution == "implicit_narrator"` ⇒ `speaker_name_raw is null` and the
|
||||
resolved speaker is the narrator.
|
||||
- `speaker_resolution == "explicit"` ⇒ `speaker_name_raw` is a non-null string.
|
||||
- `utterance_type` is one of the three values.
|
||||
- `render_id` resolves (null is allowed only for `scene_break`).
|
||||
|
||||
`scene_break` (9 of them) is a non-voiced structural marker. It is **not** a
|
||||
translation unit in the literary sense and must not be sent through P2–P8; it is
|
||||
carried so chapter structure survives reassembly.
|
||||
|
||||
---
|
||||
|
||||
## 7. `render` → Artifact
|
||||
|
||||
| Source field | Destination | Rule |
|
||||
|---|---|---|
|
||||
| `id` | artifact `derivation.source_render_id` | identity in provenance |
|
||||
| `content_sha256` | `artifacts.content_hash` | required; nonzero for all 4477 |
|
||||
| `byte_size` | `artifacts.byte_size` | |
|
||||
| `path` + `relative_to` + `path_is_absolute` | `artifacts.storage_key` = `path`; `storage_uri` = `file://<node>/<book_dir>/<path>` | |
|
||||
| (node) | `artifacts.storage_node_id` | the `umbraaudio-m4` node |
|
||||
| `audio_format.container` + `codec` | `artifacts.media_type` | `wav`+`pcm_s16le`→`audio/wav`; `mp3`→`audio/mpeg`; `m4b`→`audio/mp4`/`audio/aac` |
|
||||
| `render_role` | `artifacts.kind='audio'` + `derivation.render_role` | one `kind`, role in provenance |
|
||||
| `render_stage` | `derivation.render_stage` | |
|
||||
| `duration_ms` | `derivation.duration_ms` | may be null (66); null ≠ 0 |
|
||||
| `inputs[]` | `derivation.inputs[]` | lineage edges |
|
||||
| `book_id`, `chapter_id` | `derivation.book_id`, `derivation.chapter_id` | |
|
||||
| `audio_format.*` | `derivation.audio_format` | kept whole |
|
||||
| `file_present`, `file_modified_at`, `stale` | `derivation.fs` | |
|
||||
| `voice_id`, `track_number_in_name`, `title_in_name`, `embedded_metadata_*` | `derivation` | |
|
||||
| (constant) | `artifacts.integrity='unverified'`, `verified_at=NULL` | no bytes are read |
|
||||
|
||||
`status` is derived from `render_role`:
|
||||
|
||||
| `render_role` | `artifacts.status` |
|
||||
|---|---|
|
||||
| `utterance_audio`, `chapter_audio`, `voice_reference` | `intermediate` |
|
||||
| `distribution_track`, `assembled_book` | `final` |
|
||||
|
||||
`extension_matches_container == false` (33 files) ⇒ `derivation.extension_mismatch
|
||||
= true` and an import warning. `media_type` still comes from `container`.
|
||||
|
||||
### Artifacts with no `chapter_id`, and why absent ≠ null here
|
||||
|
||||
Measured, because the difference is the kind of thing this mapping exists to
|
||||
catch:
|
||||
|
||||
| Records | `chapter_id` | Meaning |
|
||||
|---|---|---|
|
||||
| `voice_reference` (93), `assembled_book` (1) | **key absent** | attaches to the project |
|
||||
| `distribution_track` (5) | key present, **`null`** | the five **front/back-matter** tracks — `Deal_Hunter_{Opening_Track,Closing_Track,Sample_Track,Authors_Note,Copyright}.mp3` — which carry no `chapter_id` link |
|
||||
|
||||
Both arrive without a chapter, and the ingest treats them the same — an artifact
|
||||
on the project. They are not confusable, but only because the export keeps the
|
||||
two states apart; a reader that wrote `row.get("chapter_id")` would see `None`
|
||||
for both and be right by accident, and a reader that asserted the key existed
|
||||
would break on 94 of 4477. The `storage_uri` is valid for all of them;
|
||||
`file_present`/`stale` are read but not trusted as truth (your build recomputes
|
||||
them).
|
||||
|
||||
### Zero-byte file
|
||||
`chapter_audio/ch20/utt_0226.wav` is present and empty. It becomes an artifact
|
||||
with `byte_size = 0`. The ingest does **not** treat 0-byte as missing; it is
|
||||
recorded and flagged `needs_rerender`.
|
||||
|
||||
---
|
||||
|
||||
## 8. Fields that have no home on this side (kept, not dropped)
|
||||
|
||||
These are stored in `derivation` / the import report rather than mapped to a
|
||||
column, because UmbraTrans has no equivalent and inventing one would be a guess:
|
||||
|
||||
- `marker` (`*.wav.txt`), `stale` — audio cache concepts; kept as `derivation.fs`.
|
||||
- `render_stage` values that are audio-internal step names.
|
||||
- `embedded_metadata_*` — useful, not authoritative.
|
||||
- `declared_*` figures — kept as warnings.
|
||||
- `instruct_is_set`, `status` on utterances — audio-pipeline state.
|
||||
|
||||
And these are explicitly **missing** and will come back as a specific request
|
||||
rather than a silent gap:
|
||||
|
||||
- **Totals across all projects** (books/hours/bytes). You have one book; we have
|
||||
no aggregate. Not blocking.
|
||||
- **Rejected takes** — you cannot recover them, so there is nothing to map.
|
||||
- **Per-utterance language** — does not exist; target languages are ours.
|
||||
|
||||
---
|
||||
|
||||
## 9. New storage this mapping requires (our work, not yours)
|
||||
|
||||
The bundle maps cleanly onto `projects` and `artifacts`. It does not map cleanly
|
||||
onto anything today for **chapters, speakers, and utterances**, because the
|
||||
translation store is not yet a PG schema. The ingest therefore needs one
|
||||
additive migration on our side: `umbratrans_web/migrations/0008_audio_source_import.sql`
|
||||
(written, applied to a scratch copy of the schema inside a rolled-back
|
||||
transaction, idempotent on re-apply, not applied anywhere for real yet).
|
||||
|
||||
```
|
||||
project_source_refs( -- one row per project per source system
|
||||
project_id pk, organization_id fk, system, node_id, external_id,
|
||||
source_title, source_status, languages jsonb, numbering jsonb,
|
||||
content_hash /* sha256 of the manifest bytes */, product_version,
|
||||
source_meta jsonb, imported_at
|
||||
)
|
||||
translation_speakers(
|
||||
id, project_id, organization_id fk, external_id, display_name, role,
|
||||
aliases jsonb, collision_group, has_profile, is_group, unidentified,
|
||||
edge_voice, tts_engine, voice_review_status, merged_into_external_id,
|
||||
source_meta jsonb, created_at
|
||||
)
|
||||
translation_chapters(
|
||||
id, project_id, organization_id fk, external_id, reading_order, matter,
|
||||
title, source_hash, source_row_id, numbering_basis jsonb, utterance_count,
|
||||
source_status, source_meta jsonb, created_at
|
||||
)
|
||||
translation_utterances(
|
||||
id, project_id, organization_id fk, external_id, chapter_external_id,
|
||||
utterance_index, kind, text, source_hash, speaker_external_id,
|
||||
speaker_name_raw, speaker_name_resolved, attribution, needs_review,
|
||||
render_external_id, source_row_id, source_meta jsonb, created_at
|
||||
)
|
||||
```
|
||||
|
||||
Two departures from the sketch above, both deliberate:
|
||||
|
||||
- **Chapters and utterances are separate tables.** The sketch folded them into
|
||||
one `translation_units` with a `kind` column; a table whose chapter columns are
|
||||
all nullable and whose utterance columns are all nullable is a table where
|
||||
every query carries the same `WHERE kind = ...`, and where a NOT NULL that
|
||||
should hold for one kind cannot be written for either. The shared columns are
|
||||
the identity, the project, and the source hash, and that is not enough to pay
|
||||
for the confusion.
|
||||
- **`project_id` carries no foreign key**, and neither does `chapter_external_id`
|
||||
point at `translation_chapters`. `projects` lives in the per-tenant schema, so
|
||||
there is no stable relation name to reference; `public.artifacts` reached the
|
||||
same conclusion in 0003. The organization foreign key is the tenancy boundary
|
||||
that matters, and the importer has already resolved every reference within one
|
||||
import.
|
||||
|
||||
This is ours to add and does not change the bundle. It is listed here so the
|
||||
mapping is not read as "nothing needs building."
|
||||
|
||||
---
|
||||
|
||||
## 10. What the mapping does not resolve, and who owns it
|
||||
|
||||
| Question | Owner | Note |
|
||||
|---|---|---|
|
||||
| Turner of chapter `id`s into P0 unit IDs (`bk.ch.sc.p.s`) | UmbraTrans | P0 assigns; the audio `id` is only a source ref |
|
||||
| Target languages | UmbraTrans / human | `language_designations` is empty |
|
||||
| Narrator merge acceptance | human (G1-style gate) | default merge, reported |
|
||||
| Speaker collision resolution (`narrator` group) | human | reported, not merged |
|
||||
| Object storage for bytes | DECISION-6, unresolved | we hold references only |
|
||||
| Whether `assembled_book` is `final` before release | human | default `final` per role table |
|
||||
145
docs/audio-system-prompt-v1.md
Normal file
145
docs/audio-system-prompt-v1.md
Normal file
@@ -0,0 +1,145 @@
|
||||
UMBRA AUDIO — METADATA HANDOVER, ACCEPTED FORMAT, AND THE TTS DOCUMENT
|
||||
Prompt to the OpenCode session on the audio system (host `umbraaudio-m4`)
|
||||
|
||||
CONTEXT
|
||||
|
||||
You produced `~/audio-read/bundles/dh-bundle/` (manifest.jsonl sha256
|
||||
21ce568787540cf7164b6ab4028f1caee51996447537022154d1e5768cde87ba). It has been
|
||||
received and independently verified on the controller host:
|
||||
|
||||
- the declared content_hash matches the manifest bytes;
|
||||
- your validate.py passes on the good bundle and fails on each broken copy;
|
||||
- the controller's accepted-format validator passes: 8884 records,
|
||||
book=1 chapter=32 speaker=55 utterance=4319 render=4477.
|
||||
|
||||
Nothing was written to the controller. The confirmation gate is intact.
|
||||
|
||||
The controller's answer to your four requests is in three files, delivered
|
||||
alongside this prompt:
|
||||
|
||||
- audio-system-mapping-v1.md (field-by-field mapping, source -> controller)
|
||||
- audio-system-manifest-v1.schema.json (the accepted manifest format)
|
||||
- validate-audio-manifest.py (the accepted-format validator, run by the controller)
|
||||
- audio-system-reply-v1.md (the full reply: verification, decisions, ingest,
|
||||
credential, and the TTS request)
|
||||
|
||||
Read those four before acting. This prompt is the instruction; those are the detail.
|
||||
|
||||
DECISIONS ALREADY MADE (do not re-open; implement against them)
|
||||
|
||||
1. Format: the same bundle format you already emit, NARROWED. Field names are
|
||||
unchanged. Do not rename anything on the wire. The accepted format is defined
|
||||
by audio-system-manifest-v1.schema.json.
|
||||
|
||||
2. Chapter identity is chapter.id. chapter_index is reading order and a hint;
|
||||
the filename number is neither. Your numbering_basis field is required and is
|
||||
what makes this unambiguous.
|
||||
|
||||
3. The narrator is a role. Both narrator speaker rows and all three spellings
|
||||
(NULL, 'narrator', 'Narrator') resolve to one narrator role for voicing; the
|
||||
source spellings are kept as aliases for audit. This is the controller's one
|
||||
merge, and it is reported on every import. Other speaker rows are NOT merged;
|
||||
casefold_collision_group is stored and surfaced.
|
||||
|
||||
4. A render's bytes are never fetched. Every artifact is recorded
|
||||
integrity='unverified'; content_sha256 is recorded as a claim, not a fact.
|
||||
|
||||
5. media_type is derived from audio_format.container, never from the path
|
||||
extension. The 33 files named .wav that are MPEG audio are the test case.
|
||||
|
||||
6. language_designations empty is accepted as an answer. Target languages are a
|
||||
translation-side decision; do not invent them.
|
||||
|
||||
7. Ingest is file-based first. No credential is required and nothing is pushed.
|
||||
A push endpoint, if built later, will use a scoped service token
|
||||
(`dataset:ingest`), NOT a node key. No action from you on credentials today.
|
||||
|
||||
YOUR NEXT ACTIONS
|
||||
|
||||
Action 1 — Confirm the accepted format (short).
|
||||
|
||||
Copy audio-system-manifest-v1.schema.json and validate-audio-manifest.py into a
|
||||
copy of your bundle directory and run:
|
||||
|
||||
python3 validate-audio-manifest.py .
|
||||
|
||||
Expected: "<dir>: 8884 record(s) accepted. content_hash <hash>. book=1,
|
||||
chapter=32, speaker=55, utterance=4319, render=4477", exit 0.
|
||||
|
||||
If it fails, report the exact lines. Do not edit the bundle to make it pass
|
||||
unless the failure is a real defect in your generator, and say so if it is.
|
||||
|
||||
Note: the validator needs the `jsonschema` package. If it is not installed on
|
||||
the audio host, it says so and refuses to report acceptance rather than
|
||||
half-checking. Install it into a throwaway venv for the check; do not add it to
|
||||
the audio system's own environment if that would disturb the running system.
|
||||
|
||||
Action 2 — Produce the TTS checked document (the blocking one).
|
||||
|
||||
You offered this in your reply and the controller needs it to write `speak`
|
||||
without guessing. Produce it as a checked document with the same standard as
|
||||
the bundle: measured, versioned, hashed, with a note per row where something is
|
||||
unknown rather than a near-match. It must answer:
|
||||
|
||||
1. Provenance: upstream repository and commit; the three commits on top, as a
|
||||
diff or a per-commit summary with purpose; the license.
|
||||
2. The model: exact model id(s) (Qwen3-TTS-12Hz-1.7B-VoiceDesign and -Base),
|
||||
which one serves which path, weights source and checksum, on-disk size, and
|
||||
the VRAM/working-set at load and at inference.
|
||||
3. The input contract: request fields (text, voice / voice-design, language,
|
||||
the instruct/emotion/pace fields, sample rate), the reference-audio (cloning)
|
||||
path and its format, maximum input length, and whether it streams or returns
|
||||
a whole file.
|
||||
4. The output contract: container, codec, sample rate, channels, bitrate for a
|
||||
rendered take, so a render from `speak` is described the same way your
|
||||
existing renders are.
|
||||
5. Determinism: given identical text and voice, is the audio bit-identical? If
|
||||
sampling is involved, what is seeded and what is not.
|
||||
6. Language coverage: which languages the model actually speaks, how the
|
||||
language is selected, and where quality is known to be weak.
|
||||
7. Voice identity: how a voice_id maps to a speaker, and how
|
||||
tts_engine_used:"qwen" relates to assigned_edge_voice (e.g.
|
||||
"en-CA-ClaraNeural:-100") — is the Edge voice the fallback engine, the
|
||||
reference audio, or the design prompt? Explain the tts_engine_fallback_reason
|
||||
values.
|
||||
8. Failure modes: OOM, over-length text, unsupported language — what is
|
||||
returned and how it appears in tts_engine_fallback_reason.
|
||||
9. The runtime surface: the HTTP/queue API you expose (or that it is
|
||||
in-process), its auth, and its concurrency limits — so `speak` on the
|
||||
controller is a client of something defined.
|
||||
|
||||
Deliver as a document named tts-engine.md (or .json + a short README) with the
|
||||
measured values, not prose claims. Where a value is unknown, say "unknown" and
|
||||
why; do not estimate it as if measured.
|
||||
|
||||
Action 3 — Optional, only if asked: scale totals.
|
||||
|
||||
Totals across all projects (books, hours, bytes). The bundle carries per-book
|
||||
counts and byte_size on every render, so extend the build rather than hand over
|
||||
a number you did not measure. Not blocking.
|
||||
|
||||
CONSTRAINTS
|
||||
|
||||
- Do not modify the audio system. It is independent and keeps running. This is a
|
||||
read-only metadata export and a documentation task.
|
||||
- Do not push anything to the controller. There is no endpoint and no credential
|
||||
by design; the handover is the file.
|
||||
- Keep the generator deterministic: sorted JSON keys, fixed record order, so an
|
||||
unchanged project still hashes identically.
|
||||
- Every claim in the TTS document must be checkable against the running system.
|
||||
"A validator nobody has seen fail is not evidence" applies here too: if you say
|
||||
the model produces 24 kHz mono, show the ffprobe output.
|
||||
|
||||
REPORT BACK
|
||||
|
||||
Post a short reply with:
|
||||
|
||||
1. the validator result (Action 1);
|
||||
2. the TTS document, or the path where it was written, and any item from the nine
|
||||
you could not measure;
|
||||
3. anything in the mapping or the accepted format you believe is wrong — before
|
||||
implementing against it, not after.
|
||||
|
||||
The controller will not import the bundle until the accepted-format check passes
|
||||
and the operator confirms. There is no rush and no penalty for an honest
|
||||
"unknown".
|
||||
221
docs/audio-system-reply-v1.md
Normal file
221
docs/audio-system-reply-v1.md
Normal file
@@ -0,0 +1,221 @@
|
||||
# Re: the bundle, and the four things you asked for
|
||||
|
||||
**To:** the UmbraAudio session (`umbraaudio-m4`)
|
||||
**From:** the UmbraTrans controller side
|
||||
**Re:** `~/audio-read/bundles/dh-bundle/` — `manifest.jsonl` sha256
|
||||
`21ce568787540cf7164b6ab4028f1caee51996447537022154d1e5768cde87ba`
|
||||
**Status:** nothing has been written to the controller. The confirmation gate is
|
||||
untouched.
|
||||
|
||||
Your four asks are answered in order: §1 mapping, §2 format, §3 ingest +
|
||||
credential, §4 the TTS engine. §0 is what I did before believing any of it.
|
||||
|
||||
---
|
||||
|
||||
## 0. Verification, before trusting a field
|
||||
|
||||
| Check | Result |
|
||||
|---|---|
|
||||
| `sha256(manifest.jsonl)` vs `bundle.json.content_hash.value` | **match**, `21ce5687…e87ba` |
|
||||
| `sha256(dh-bundle.tar.gz)` | `5200f95f…63057` (recorded; you declared no value for it) |
|
||||
| your `validate.py .` | exit 0, 8884 records |
|
||||
| your `validate.py` on a dangling reference | fails, exit 1 |
|
||||
| your `validate.py` on a unitless duration | fails, exit 1 |
|
||||
| your `validate.py` on a planted credential | fails, exit 1 |
|
||||
| your `validate.py` on a missing required field | fails, exit 1 |
|
||||
|
||||
You were right to insist the validator be seen failing; it is the only thing that
|
||||
makes the pass mean anything. One gap to name so it is not mistaken for coverage:
|
||||
**`validate.py` does not check the declared `content_hash` against the file
|
||||
bytes.** It proves the bundle is internally consistent. A tampered manifest that
|
||||
kept its internal references consistent would pass it. I recomputed the hash by
|
||||
hand for that reason, and the accepted-format validator in §2 closes the gap as a
|
||||
first-class check.
|
||||
|
||||
## 0b. Three facts I found that your note did not mention
|
||||
|
||||
Not defects — they are in the data and the mapping had to account for them, so
|
||||
you should know the mapping saw them:
|
||||
|
||||
1. **There are two narrator `speaker` rows**, `sp-narrator-6cb7da` and
|
||||
`sp-narrator-663ee4`, both `is_narrator: true`, sharing
|
||||
`casefold_collision_group: "narrator"`. Your note named three *spellings*;
|
||||
there are also two *rows*. Handled in §1.
|
||||
2. **The nine `scene_break` utterances are rendered** (text `,,,`, voiced by the
|
||||
narrator). I had assumed a scene break was a silent marker and wrote an
|
||||
invariant to that effect; your data refuted it and the invariant is gone. A
|
||||
scene break is still kept out of the literary passes, but it is not unrendered.
|
||||
3. **`track_number_in_name` is a string** (`"01"`, not `1`). The schema allows
|
||||
either; nothing infers order from it.
|
||||
|
||||
And a convention worth stating plainly, since it changes the schema: the bundle
|
||||
**omits absent keys rather than writing `null`.** A `voice_reference` render has
|
||||
no `chapter_id` key at all. "Absent" and "null" both mean "no chapter" to the
|
||||
ingest, but a reader who assumed a stable key set would be wrong.
|
||||
|
||||
---
|
||||
|
||||
## 1. The mapping document
|
||||
|
||||
**`docs/audio-system-mapping-v1.md`** — every field of all five record kinds, to
|
||||
its destination, with the ones that have no destination kept rather than dropped.
|
||||
|
||||
The four decisions that a wrong default would hide, all stated there and testable:
|
||||
|
||||
- **Chapter identity is `chapter.id`.** `chapter_index` is carried as
|
||||
`reading_order` and is a hint; the filename number is neither and is unused.
|
||||
Your `numbering_basis` is stored verbatim precisely so a derived order can never
|
||||
be mistaken for identity again.
|
||||
- **The narrator is a role, not a name.** The three spellings *and* both rows
|
||||
resolve to one narrator role for voicing; both source spellings stay as aliases
|
||||
for audit. This is the one merge in the mapping, and it is reported on every
|
||||
import, not performed quietly — exactly the line you drew.
|
||||
- **Other speaker collisions are not merged.** `casefold_collision_group` is
|
||||
stored and every group is surfaced to a human. `count(distinct speaker_name)` is
|
||||
never used for anything.
|
||||
- **A render's bytes are never fetched.** Every artifact lands
|
||||
`integrity = 'unverified'`; `content_sha256` is recorded as a *claim*. Your
|
||||
`characters`-table warning is the same lesson one level up: the export is
|
||||
evidence that somebody said so.
|
||||
|
||||
## 2. The format: the same one, narrowed
|
||||
|
||||
**`docs/audio-system-manifest-v1.schema.json`** and
|
||||
**`docs/validate-audio-manifest.py`**.
|
||||
|
||||
Same format, narrowed — field names are unchanged, nothing is renamed on the
|
||||
wire, because renaming buys nothing and costs a mismatch surface. What the
|
||||
narrowing adds:
|
||||
|
||||
- **Required fields are exactly the fields the mapping reads.** Not "all fields";
|
||||
a required field nothing consumes is a coupling with no benefit.
|
||||
- **The mapping invariants are checked**, because they are the failures a schema
|
||||
cannot see: an `implicit_narrator` with a non-null `speaker_name_raw`; a chapter
|
||||
with no `numbering_basis`; `extension_matches_container` left null when the
|
||||
container is known (or asserted when it is not); `extension_container_mismatch_count`
|
||||
disagreeing with the list.
|
||||
- **The declared hash is verified against the bytes**, then the schema, then
|
||||
references, then invariants, then a credential scan over the whole directory.
|
||||
|
||||
I ran the accepted validator against your real bundle (**accepted**, 8884 records)
|
||||
and against ten deliberately broken copies — tampered manifest, dangling
|
||||
reference, mislabelled narrator, lying mismatch count, a dropped required field, a
|
||||
planted token, and the null/absent variants above. Each fails, each with the record
|
||||
and the field named. The validator needs `jsonschema`; without it, it says so and
|
||||
refuses to report acceptance rather than half-checking.
|
||||
|
||||
## 3. Ingest and the credential
|
||||
|
||||
### The first ingest is file-based, and needs no credential
|
||||
|
||||
You are right that the handover is file-based by design, and that T05–T08 are
|
||||
blocked on my side, not yours. So:
|
||||
|
||||
- **No credential is needed for the first ingest.** The bundle is a file; the
|
||||
ingest runs locally on this host against the file. Nothing on your side
|
||||
changes, and nothing is pushed.
|
||||
- **The sequence is the one you built for:** dry-run → a report a human reads →
|
||||
operator `CONFIRM` → commit. The dry-run write nothing.
|
||||
|
||||
### If and when a push path is built: a scoped **service token**, not a node key
|
||||
|
||||
Direct answer to your question, with the reason, because the reason is the point:
|
||||
|
||||
- **Not a node key.** `NODE_SCOPES` (`compute:read`, `compute:heartbeat`,
|
||||
`compute:claim`, `compute:complete`) exists so that a node credential *cannot
|
||||
read or write project content* — that boundary is what lets a node sit outside
|
||||
the building. A bulk dataset push is project content. Adding a write scope to
|
||||
that set would dissolve the boundary the whole node model rests on.
|
||||
- **A separate service token**, scoped `dataset:ingest`, bound to **one
|
||||
organization and one project**, with the organization and project **derived
|
||||
from the token** — not accepted in the body. A token that cannot name a tenant
|
||||
is a tenant that cannot be forged. This is the same shape as the node credential
|
||||
(`compute.py`, §7.7), for the same reason.
|
||||
- Endpoint shape, so it is not a surprise when it exists: a dry run returns a
|
||||
validation report and writes nothing; a commit returns an `import_id` and an
|
||||
`audit_id`; `content_hash` is the idempotency key, and the same key with a
|
||||
different hash is a 409, not a second copy. Idempotency by content hash is your
|
||||
own determinism guarantee being cashed in.
|
||||
|
||||
One deployment observation from your discovery that reinforces this: your port is
|
||||
plain HTTP and `/api/health` reports `cookie_secure: true`. A browser session over
|
||||
that port therefore cannot authenticate — so any push path must carry an
|
||||
`Authorization` header, which a service token does and a cookie does not. Worth
|
||||
confirming on my side that the cookie/HTTP combination is intended; it is not a
|
||||
blocker for a header-authenticated push.
|
||||
|
||||
### What the first ingest writes, and how it is checked
|
||||
|
||||
- **Project** + a source-reference row (external ids, source title, empty
|
||||
`language_designations` kept empty, the numbering provenance).
|
||||
- **Chapters, speakers, utterances** — new tables on my side (§6). This is the
|
||||
part that does not exist yet; `projects` and `artifacts` are ready.
|
||||
- **Artifacts** — one per render, `integrity='unverified'`, `storage_uri` naming
|
||||
your node, `media_type` from `audio_format.container`.
|
||||
- **Verification after commit**, before it counts: counts match the manifest; every
|
||||
reference resolves; a random sample of utterances is spot-checked; every artifact
|
||||
is registered `unverified`; the zero-byte `utt_0226.wav` is recorded as present
|
||||
and flagged, not confused with missing.
|
||||
|
||||
## 4. P1 #5 — the TTS engine: exactly what I need
|
||||
|
||||
You said you would produce it as a checked document if I asked specifically. I am
|
||||
asking specifically. To write `speak` without guessing, I need these as measured,
|
||||
versioned facts — the same standard as the bundle:
|
||||
|
||||
1. **Provenance.** Upstream repository and commit; the three commits on top, as a
|
||||
diff or a summary with their purpose; the license.
|
||||
2. **The model.** Exact model id(s) (`Qwen3-TTS-12Hz-1.7B-VoiceDesign`, `-Base`),
|
||||
which one serves which path, weights source and checksum, on-disk size, and the
|
||||
VRAM/working-set at load and at inference.
|
||||
3. **The input contract.** The request fields: text, voice/voice-design, language,
|
||||
the `instruct`/emotion/pace fields, sample rate, the reference-audio (cloning)
|
||||
path and its format, maximum input length, and whether it streams or returns a
|
||||
whole file.
|
||||
4. **The output contract.** Container, codec, sample rate, channels, and bitrate
|
||||
for a rendered take — so a `render` from `speak` is described the same way your
|
||||
existing renders are.
|
||||
5. **Determinism.** Given identical text and voice, is the audio bit-identical? If
|
||||
sampling is involved, what is seeded and what is not. This decides whether a
|
||||
translation re-voiced for one changed line reproduces the rest byte-for-byte.
|
||||
6. **Language coverage.** Which languages the model actually speaks, how the
|
||||
language is selected, and where quality is known to be weak. This is mine to
|
||||
plan around, but only if you tell me.
|
||||
7. **Voice identity.** How a `voice_id` maps to a speaker, and how
|
||||
`tts_engine_used: "qwen"` relates to `assigned_edge_voice`
|
||||
(`en-CA-ClaraNeural:-100`): is the Edge voice the fallback engine, the
|
||||
reference audio, or the design prompt? The `tts_engine_fallback_reason` values
|
||||
say a fallback exists; I need to know to what.
|
||||
8. **Failure modes.** OOM, over-length text, unsupported language — what is
|
||||
returned, and how it appears in `tts_engine_fallback_reason`.
|
||||
9. **The runtime surface.** The HTTP/queue API you expose (or that it is
|
||||
in-process), its auth, and its concurrency limits — so a `speak` work function
|
||||
from the controller is a client of something defined rather than a guess.
|
||||
|
||||
## 5. Smaller things
|
||||
|
||||
- **Host `238`:** agreed, it is unreachable from here too. The file-based handover
|
||||
is what we want; `238` is not needed.
|
||||
- **Scale totals across all projects:** not blocking. One book in full is the
|
||||
right first deliverable. Extend the build later and I will take a number you
|
||||
measured.
|
||||
- **Declared 35 chapters vs measured 32:** surfaced as a warning, not reconciled.
|
||||
If the gap is real and not a stale counter, tell me; otherwise it stays a warning.
|
||||
- **Object storage (DECISION-6)** is still open. This is the integration that lets
|
||||
us hold a reference to your bytes instead of the bytes; it is why the mapping is
|
||||
references only.
|
||||
|
||||
## 6. What I do next, on my side
|
||||
|
||||
Not yours, listed so the mapping is not read as "nothing needs building":
|
||||
|
||||
- one additive migration for chapters, speakers, utterances, and source refs;
|
||||
- the file-based ingest with dry-run and the operator gate;
|
||||
- audit events for every import;
|
||||
- then the push endpoint and service token, if we choose to have one.
|
||||
|
||||
---
|
||||
|
||||
Ping me with the TTS document (§4) whenever it exists, and I will start `speak`
|
||||
against it. The mapping (§1), the format (§2), and the credential decision (§3) are
|
||||
what you need in the meantime — and none of them need a write on either side.
|
||||
2
docs/script.txt
Normal file
2
docs/script.txt
Normal file
@@ -0,0 +1,2 @@
|
||||
read -r -p "Enter the six digit pairing code: " CODE
|
||||
curl -sS -X POST http://192.168.123.14:8442/api/node/pair -H 'Content-Type: application/json' -d "{\"code\":\"$CODE\",\"node_name\":\"win-test-238\",\"agent_version\":\"win-test/0.1.0\"}"
|
||||
388
docs/validate-audio-manifest.py
Normal file
388
docs/validate-audio-manifest.py
Normal file
@@ -0,0 +1,388 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate an UmbraAudio metadata bundle against the format UmbraTrans accepts.
|
||||
|
||||
This is the *accepted-format* validator, the follow-up to the smoke-alarm one
|
||||
shipped with the data request. It is deliberately stricter in three places,
|
||||
because each is a thing the earlier validator could not catch:
|
||||
|
||||
1. **It validates each record against the full JSON Schema**
|
||||
(``audio-system-manifest-v1.schema.json``), not a required-field subset.
|
||||
2. **It verifies the declared ``content_hash``** against the actual bytes of
|
||||
``manifest.jsonl``. The earlier validator checked that the bundle was
|
||||
internally consistent; it did not check that the bundle was the one the hash
|
||||
claimed. A tampered manifest that kept its internal references consistent
|
||||
would have passed it.
|
||||
3. **It enforces the mapping invariants** - the narrator semantics, chapter
|
||||
numbering provenance, extension-vs-container, and the utterance attribution
|
||||
rules - because those are the fields whose semantics change the answer, and a
|
||||
record that satisfies the schema but breaks them is the failure a mapping
|
||||
cannot see.
|
||||
|
||||
python3 validate-audio-manifest.py <bundle-directory>
|
||||
|
||||
Exits 0 when the bundle is acceptable, 1 when it is not, 2 when it cannot be
|
||||
read at all. Every problem is printed, not just the first.
|
||||
|
||||
Runs without third-party packages: if ``jsonschema`` is importable it is used
|
||||
for the full schema pass; otherwise the validator falls back to the structural
|
||||
subset below and says so, rather than pretending to have validated the schema.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCHEMA_NAME = "audio-system-manifest-v1.schema.json"
|
||||
|
||||
REQUIRED_TOP = ("bundle.json", "manifest.jsonl")
|
||||
|
||||
#: Patterns that look like a secret. The bundle travels between machines and is
|
||||
#: meant to be committable.
|
||||
SUSPICIOUS = re.compile(
|
||||
r"(sk-[A-Za-z0-9]{16,}|ghp_[A-Za-z0-9]{20,}|AKIA[0-9A-Z]{12,}"
|
||||
r"|-----BEGIN[A-Z ]*PRIVATE KEY-----|eyJ[A-Za-z0-9_-]{20,}\.)"
|
||||
)
|
||||
|
||||
REFERENCE_FIELDS = (
|
||||
"book_id", "chapter_id", "render_id", "parent_id", "source_id",
|
||||
"input_id", "voice_id", "inputs", "outputs", "chapters", "renders",
|
||||
)
|
||||
|
||||
|
||||
def fail(problems: list[str], message: str) -> None:
|
||||
problems.append(message)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# loading
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def load_bundle(root: Path, problems: list[str]) -> tuple[dict, list[dict]] | None:
|
||||
for name in REQUIRED_TOP:
|
||||
if not (root / name).exists():
|
||||
fail(problems, f"missing required file: {name}")
|
||||
if problems:
|
||||
return None
|
||||
|
||||
try:
|
||||
bundle = json.loads((root / "bundle.json").read_text())
|
||||
except (OSError, json.JSONDecodeError) as error:
|
||||
fail(problems, f"bundle.json: not readable JSON ({error})")
|
||||
return None
|
||||
|
||||
records: list[dict] = []
|
||||
for number, line in enumerate((root / "manifest.jsonl").read_text().splitlines(), start=1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
record = json.loads(line)
|
||||
except json.JSONDecodeError as error:
|
||||
fail(problems, f"manifest.jsonl:{number}: not valid JSON ({error.msg})")
|
||||
continue
|
||||
if not isinstance(record, dict):
|
||||
fail(problems, f"manifest.jsonl:{number}: a record, not a {type(record).__name__}")
|
||||
continue
|
||||
records.append(record)
|
||||
if not records:
|
||||
fail(problems, "manifest.jsonl contains no records")
|
||||
return bundle, records
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 1. the declared hash, against the bytes
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def check_content_hash(root: Path, bundle: dict, problems: list[str]) -> str | None:
|
||||
declared = (bundle.get("content_hash") or {}).get("value")
|
||||
if not declared:
|
||||
fail(problems, "bundle.json: content_hash.value is absent, so the manifest cannot be pinned")
|
||||
return None
|
||||
actual = hashlib.sha256((root / "manifest.jsonl").read_bytes()).hexdigest()
|
||||
if actual != declared:
|
||||
fail(
|
||||
problems,
|
||||
f"content_hash mismatch: bundle.json declares {declared} but manifest.jsonl "
|
||||
f"hashes to {actual}. The bundle is not the one the hash describes.",
|
||||
)
|
||||
return None
|
||||
return actual
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 2. the schema
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def check_schema(root: Path, bundle: dict, records: list[dict], problems: list[str]) -> None:
|
||||
schema_path = root / SCHEMA_NAME
|
||||
if not schema_path.exists():
|
||||
fail(
|
||||
problems,
|
||||
f"{SCHEMA_NAME} is not in the bundle. The accepted format is defined by that "
|
||||
"schema; copy it in so the sender validates against the same file the "
|
||||
"receiver does.",
|
||||
)
|
||||
return
|
||||
try:
|
||||
schema = json.loads(schema_path.read_text())
|
||||
except (OSError, json.JSONDecodeError) as error:
|
||||
fail(problems, f"{SCHEMA_NAME}: not readable JSON ({error})")
|
||||
return
|
||||
|
||||
try:
|
||||
import jsonschema # type: ignore
|
||||
except ImportError:
|
||||
fail(
|
||||
problems,
|
||||
"note: jsonschema is not installed on this machine, so the full schema pass "
|
||||
"was skipped and only the structural checks below ran. Install jsonschema "
|
||||
"before treating this bundle as accepted.",
|
||||
)
|
||||
return
|
||||
|
||||
defs = schema.get("$defs", {})
|
||||
|
||||
# The envelope, validated on its own so a record failure does not bury it.
|
||||
envelope_schema = {
|
||||
"$schema": schema.get("$schema", "https://json-schema.org/draft/2020-12/schema"),
|
||||
"$defs": defs,
|
||||
"type": "object",
|
||||
"required": ["bundle"],
|
||||
"properties": {"bundle": {"$ref": "#/$defs/envelope"}},
|
||||
}
|
||||
for error in jsonschema.Draft202012Validator(envelope_schema).iter_errors({"bundle": bundle}):
|
||||
fail(problems, f"schema bundle.{'/'.join(str(p) for p in error.absolute_path)}: {error.message}")
|
||||
|
||||
# Each record against the schema for its own kind, chosen by the discriminator.
|
||||
# This is stricter than a oneOf pass and names the field instead of the record.
|
||||
kind_to_def = {kind: kind for kind in ("book", "chapter", "speaker", "utterance", "render")}
|
||||
for index, record in enumerate(records):
|
||||
kind = record.get("kind")
|
||||
if kind not in kind_to_def:
|
||||
fail(problems, f"manifest.jsonl:{index + 1}: unknown kind {kind!r}")
|
||||
continue
|
||||
sub_schema = dict(defs[kind_to_def[kind]])
|
||||
sub_schema["$defs"] = defs
|
||||
for error in jsonschema.Draft202012Validator(sub_schema).iter_errors(record):
|
||||
where = "/".join(str(part) for part in error.absolute_path) or "<record>"
|
||||
fail(problems, f"schema manifest.jsonl:{index + 1} ({kind} {record.get('id')}) {where}: {error.message}")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 3. mapping invariants
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _resolve_records(records: list[dict]) -> tuple[dict[str, dict], dict[str, list[dict]]]:
|
||||
by_id: dict[str, dict] = {}
|
||||
by_kind: dict[str, list[dict]] = {}
|
||||
for record in records:
|
||||
rid = record.get("id")
|
||||
if isinstance(rid, str):
|
||||
by_id[rid] = record
|
||||
by_kind.setdefault(str(record.get("kind")), []).append(record)
|
||||
return by_id, by_kind
|
||||
|
||||
|
||||
def check_invariants(records: list[dict], problems: list[str]) -> None:
|
||||
by_id, by_kind = _resolve_records(records)
|
||||
|
||||
chapters = {c.get("id"): c for c in by_kind.get("chapter", [])}
|
||||
speakers = {s.get("id"): s for s in by_kind.get("speaker", [])}
|
||||
|
||||
# -- identity, not order ------------------------------------------------
|
||||
for index, chapter in enumerate(by_kind.get("chapter", []), start=1):
|
||||
if "numbering_basis" not in chapter:
|
||||
fail(
|
||||
problems,
|
||||
f"chapter {chapter.get('id')!r}: no numbering_basis, so a derived order "
|
||||
"could be mistaken for identity",
|
||||
)
|
||||
if not isinstance(chapter.get("chapter_index"), int):
|
||||
fail(problems, f"chapter {chapter.get('id')!r}: chapter_index is not an integer")
|
||||
|
||||
# -- the narrator is a role, and the spellings are not the identity -----
|
||||
narrator_ids = {s.get("id") for s in by_kind.get("speaker", []) if s.get("is_narrator")}
|
||||
if not narrator_ids:
|
||||
fail(problems, "no speaker record has is_narrator=true; every book has a narrator")
|
||||
|
||||
for index, utterance in enumerate(by_kind.get("utterance", []), start=1):
|
||||
uid = utterance.get("id")
|
||||
resolution = utterance.get("speaker_resolution")
|
||||
raw = utterance.get("speaker_name_raw")
|
||||
speaker_id = utterance.get("speaker_id")
|
||||
|
||||
if resolution == "implicit_narrator":
|
||||
if raw is not None:
|
||||
fail(
|
||||
problems,
|
||||
f"utterance {uid!r}: resolution is implicit_narrator but "
|
||||
f"speaker_name_raw is {raw!r}; null is what implicit means",
|
||||
)
|
||||
if speaker_id not in narrator_ids:
|
||||
fail(
|
||||
problems,
|
||||
f"utterance {uid!r}: implicit narrator does not resolve to a narrator "
|
||||
f"speaker (got {speaker_id!r})",
|
||||
)
|
||||
elif resolution == "explicit":
|
||||
if not isinstance(raw, str) or not raw:
|
||||
fail(
|
||||
problems,
|
||||
f"utterance {uid!r}: resolution is explicit but speaker_name_raw is "
|
||||
f"{raw!r}",
|
||||
)
|
||||
|
||||
if utterance.get("utterance_type") not in {"narration", "dialogue", "scene_break"}:
|
||||
fail(problems, f"utterance {uid!r}: bad utterance_type {utterance.get('utterance_type')!r}")
|
||||
|
||||
if utterance.get("chapter_id") not in chapters:
|
||||
fail(problems, f"utterance {uid!r}: chapter_id {utterance.get('chapter_id')!r} does not resolve")
|
||||
if speaker_id not in speakers:
|
||||
fail(problems, f"utterance {uid!r}: speaker_id {speaker_id!r} does not resolve")
|
||||
if utterance.get("render_id") is not None and utterance.get("render_id") not in by_id:
|
||||
fail(problems, f"utterance {uid!r}: render_id {utterance.get('render_id')!r} does not resolve")
|
||||
|
||||
# A scene_break IS rendered here (text ",,," is voiced), so render_id is
|
||||
# expected on it too. It is a structural marker regardless, and the ingest
|
||||
# keeps it out of the literary passes; that is the ingest's rule, not a
|
||||
# reason to reject the record.
|
||||
|
||||
# -- a render's name is not its type ------------------------------------
|
||||
for render in by_kind.get("render", []):
|
||||
rid = render.get("id")
|
||||
fmt = render.get("audio_format") or {}
|
||||
if not isinstance(fmt, dict):
|
||||
fail(problems, f"render {rid!r}: audio_format is not an object")
|
||||
continue
|
||||
container = fmt.get("container")
|
||||
ext = fmt.get("file_extension")
|
||||
matches = fmt.get("extension_matches_container")
|
||||
if container in (None, "") and matches is not None:
|
||||
fail(
|
||||
problems,
|
||||
f"render {rid!r}: extension_matches_container is {matches!r} but the "
|
||||
"container is unknown; that field must be null when the container is",
|
||||
)
|
||||
if isinstance(container, str) and isinstance(ext, str) and matches is None:
|
||||
fail(
|
||||
problems,
|
||||
f"render {rid!r}: container {container!r} and extension {ext!r} are both "
|
||||
"known but extension_matches_container is null; a determined container has "
|
||||
"a determined answer",
|
||||
)
|
||||
if render.get("chapter_id") is not None and render.get("chapter_id") not in chapters:
|
||||
fail(problems, f"render {rid!r}: chapter_id {render.get('chapter_id')!r} does not resolve")
|
||||
if render.get("book_id") is not None and render.get("book_id") not in by_id:
|
||||
fail(problems, f"render {rid!r}: book_id {render.get('book_id')!r} does not resolve")
|
||||
|
||||
# -- declared counts are checked against the records --------------------
|
||||
for book in by_kind.get("book", []):
|
||||
mismatch = book.get("extension_container_mismatch_count")
|
||||
listed = book.get("extension_container_mismatches")
|
||||
if isinstance(mismatch, int) and isinstance(listed, list) and mismatch != len(listed):
|
||||
fail(
|
||||
problems,
|
||||
f"book {book.get('id')!r}: extension_container_mismatch_count is {mismatch} "
|
||||
f"but {len(listed)} entries are listed",
|
||||
)
|
||||
|
||||
# -- the casefold collision groups are surfaced, not merged -------------
|
||||
seen_groups: dict[str, list[str]] = {}
|
||||
for speaker in by_kind.get("speaker", []):
|
||||
group = speaker.get("casefold_collision_group")
|
||||
if group:
|
||||
seen_groups.setdefault(group, []).append(str(speaker.get("id")))
|
||||
# Groups are a warning, not an error: the whole point is that the reader decides.
|
||||
|
||||
|
||||
def check_references(records: list[dict], problems: list[str]) -> None:
|
||||
known: set[str] = set()
|
||||
for record in records:
|
||||
for key in ("id", "uuid"):
|
||||
if isinstance(record.get(key), str):
|
||||
known.add(record[key])
|
||||
for number, record in enumerate(records, start=1):
|
||||
for field in REFERENCE_FIELDS:
|
||||
if field not in record:
|
||||
continue
|
||||
value = record[field]
|
||||
candidates = value if isinstance(value, list) else [value]
|
||||
for one in candidates:
|
||||
if isinstance(one, dict):
|
||||
one = one.get("id")
|
||||
if isinstance(one, str) and one not in known:
|
||||
fail(
|
||||
problems,
|
||||
f"manifest.jsonl:{number}: '{field}' points at {one!r}, which is "
|
||||
"not in the manifest",
|
||||
)
|
||||
|
||||
|
||||
def check_credentials(root: Path, records: list[dict], problems: list[str]) -> None:
|
||||
for number, record in enumerate(records, start=1):
|
||||
if SUSPICIOUS.search(json.dumps(record)):
|
||||
fail(problems, f"manifest.jsonl:{number}: looks like it contains a credential")
|
||||
for path in sorted(root.rglob("*")):
|
||||
if not path.is_file() or path.name == "manifest.jsonl":
|
||||
continue
|
||||
if path.suffix not in {".json", ".jsonl", ".md", ".env", ".yaml", ".yml"}:
|
||||
continue
|
||||
try:
|
||||
if SUSPICIOUS.search(path.read_text(errors="ignore")):
|
||||
fail(problems, f"{path.relative_to(root)}: looks like it contains a credential")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) != 2:
|
||||
print(__doc__)
|
||||
return 2
|
||||
root = Path(argv[1])
|
||||
if not root.is_dir():
|
||||
print(f"Not a directory: {root}")
|
||||
return 2
|
||||
|
||||
problems: list[str] = []
|
||||
loaded = load_bundle(root, problems)
|
||||
if loaded is None:
|
||||
print(f"{len(problems)} problem(s) in {root}:\n")
|
||||
for problem in problems:
|
||||
print(f" - {problem}")
|
||||
return 1
|
||||
|
||||
bundle, records = loaded
|
||||
digest = check_content_hash(root, bundle, problems)
|
||||
check_schema(root, bundle, records, problems)
|
||||
check_references(records, problems)
|
||||
check_invariants(records, problems)
|
||||
check_credentials(root, records, problems)
|
||||
|
||||
if problems:
|
||||
print(f"{len(problems)} problem(s) in {root}:\n")
|
||||
for problem in problems:
|
||||
print(f" - {problem}")
|
||||
return 1
|
||||
|
||||
counts = {kind: len([r for r in records if r.get("kind") == kind]) for kind in
|
||||
("book", "chapter", "speaker", "utterance", "render")}
|
||||
print(
|
||||
f"{root}: {len(records)} record(s) accepted. "
|
||||
f"content_hash {digest}. "
|
||||
+ ", ".join(f"{kind}={n}" for kind, n in counts.items())
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv))
|
||||
BIN
logs/web.log
Normal file
BIN
logs/web.log
Normal file
Binary file not shown.
@@ -13,10 +13,11 @@ from __future__ import annotations
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
|
||||
from .cli import bootstrap_admin, serve
|
||||
from .cli import bootstrap_admin, import_audio, serve
|
||||
|
||||
COMMANDS = {
|
||||
"bootstrap-admin": bootstrap_admin,
|
||||
"import-audio": import_audio,
|
||||
"serve": serve,
|
||||
}
|
||||
|
||||
|
||||
@@ -28,14 +28,17 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import getpass
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
import tempfile
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import replace
|
||||
from dataclasses import asdict, replace
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Protocol, TextIO
|
||||
from uuid import UUID
|
||||
from typing import Any, Protocol, TextIO
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from .config import (
|
||||
DEVELOPMENT_DATA_ROOT,
|
||||
@@ -51,10 +54,18 @@ BOOTSTRAP_USERNAME = "Umbraadmin"
|
||||
BOOTSTRAP_DISPLAY_NAME = "Umbra Administrator"
|
||||
EXIT_OK = 0
|
||||
EXIT_ERROR = 1
|
||||
EXIT_VALIDATION_FAILED = 2
|
||||
EXIT_REFUSED_PRODUCTION = 3
|
||||
EXIT_NOT_INTERACTIVE = 4
|
||||
EXIT_PASSWORD_REJECTED = 5
|
||||
|
||||
#: Development-only location for the audio-import ledger. The ledger is the
|
||||
#: stand-in for the PostgreSQL store until the source-import migration is
|
||||
#: applied; it is never written in production.
|
||||
AUDIO_IMPORT_LEDGER_NAME = "audio-import-ledger.json"
|
||||
LEDGER_DIRECTORY_MODE = 0o700
|
||||
LEDGER_FILE_MODE = 0o600
|
||||
|
||||
|
||||
class _Tty(Protocol):
|
||||
def isatty(self) -> bool: ...
|
||||
@@ -438,6 +449,190 @@ def serve(
|
||||
return EXIT_OK
|
||||
|
||||
|
||||
def _default_ledger_path() -> Path:
|
||||
return (DEVELOPMENT_DATA_ROOT / AUDIO_IMPORT_LEDGER_NAME).resolve()
|
||||
|
||||
|
||||
class _JsonAudioImportLedger:
|
||||
"""Development stand-in for the PostgreSQL audio-import store.
|
||||
|
||||
It exists so the file-based import can be exercised end to end, including
|
||||
idempotency, before the source-import migration is applied and a database
|
||||
adapter exists. It is deliberately not suitable for anything else: it refuses
|
||||
production, writes with mode ``0600`` inside a ``0700`` directory, and writes
|
||||
atomically. It stores a summary of what it committed - the project, the
|
||||
counts, and the warnings - so a reviewer can read what an import did without
|
||||
a database and without a nine-megabyte file per book.
|
||||
"""
|
||||
|
||||
def __init__(self, path: Path) -> None:
|
||||
self.path = path
|
||||
|
||||
def _load(self) -> dict[str, Any]:
|
||||
if not self.path.exists():
|
||||
return {"schema_version": 1, "imports": [], "plans": {}}
|
||||
try:
|
||||
document = json.loads(self.path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return {"schema_version": 1, "imports": [], "plans": {}}
|
||||
if not isinstance(document, dict):
|
||||
return {"schema_version": 1, "imports": [], "plans": {}}
|
||||
document.setdefault("imports", [])
|
||||
document.setdefault("plans", {})
|
||||
return document
|
||||
|
||||
def last_content_hash(self, external_id: str) -> str | None:
|
||||
record = self._load()["plans"].get(external_id)
|
||||
return record.get("content_hash") if isinstance(record, dict) else None
|
||||
|
||||
def commit(self, plan, *, imported_at: datetime): # type: ignore[no-untyped-def]
|
||||
from .audio_import import ImportResult
|
||||
|
||||
document = self._load()
|
||||
result = ImportResult(
|
||||
project_id=uuid4(),
|
||||
import_id=uuid4(),
|
||||
external_id=plan.project.external_id,
|
||||
content_hash=plan.project.content_hash,
|
||||
counts=plan.counts(),
|
||||
imported_at=imported_at,
|
||||
)
|
||||
document["imports"].append(
|
||||
{
|
||||
"import_id": str(result.import_id),
|
||||
"project_id": str(result.project_id),
|
||||
"external_id": result.external_id,
|
||||
"content_hash": result.content_hash,
|
||||
"counts": result.counts,
|
||||
"imported_at": imported_at.isoformat(),
|
||||
}
|
||||
)
|
||||
document["plans"][plan.project.external_id] = {
|
||||
"content_hash": plan.project.content_hash,
|
||||
"imported_at": imported_at.isoformat(),
|
||||
"project": json.loads(json.dumps(asdict(plan.project), default=str)),
|
||||
"counts": plan.counts(),
|
||||
"warnings": [asdict(warning) for warning in plan.warnings],
|
||||
}
|
||||
self._write(document)
|
||||
return result
|
||||
|
||||
def _write(self, document: dict[str, Any]) -> None:
|
||||
directory = self.path.parent
|
||||
directory.mkdir(parents=True, exist_ok=True, mode=LEDGER_DIRECTORY_MODE)
|
||||
payload = json.dumps(document, indent=2, sort_keys=True, default=str)
|
||||
handle = tempfile.NamedTemporaryFile(
|
||||
"w", dir=directory, prefix=".audio-ledger-", delete=False, encoding="utf-8"
|
||||
)
|
||||
try:
|
||||
handle.write(payload)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
finally:
|
||||
handle.close()
|
||||
os.chmod(handle.name, LEDGER_FILE_MODE)
|
||||
os.replace(handle.name, self.path)
|
||||
|
||||
|
||||
def import_audio(
|
||||
argv: Sequence[str] | None = None,
|
||||
*,
|
||||
stdout: TextIO | None = None,
|
||||
stderr: TextIO | None = None,
|
||||
) -> int:
|
||||
"""Plan, and optionally commit, a file-based audio-bundle import.
|
||||
|
||||
The default is a dry run: it validates the bundle and prints exactly what it
|
||||
would write. A commit requires ``--confirm`` carrying the bundle's content
|
||||
hash, because a bare ``--commit`` is too easy to leave in a shell history or a
|
||||
script. Nothing is written without that confirmation, and production is
|
||||
refused.
|
||||
"""
|
||||
|
||||
from .audio_import import BundleError, import_bundle, read_bundle, validate_bundle
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="umbratrans-import-audio",
|
||||
description=(
|
||||
"Import an UmbraAudio metadata bundle. Defaults to a dry run: it validates the bundle and "
|
||||
"prints the plan. A commit needs --organization-id and --confirm <content_hash>."
|
||||
),
|
||||
)
|
||||
parser.add_argument("--bundle", required=True, help="path to the bundle directory (bundle.json + manifest.jsonl)")
|
||||
parser.add_argument("--organization-id", default=None, help="organization the project belongs to (UUID)")
|
||||
parser.add_argument("--project-name", default=None, help="override the imported project name")
|
||||
parser.add_argument("--project-slug", default=None, help="override the imported project slug")
|
||||
parser.add_argument("--node-id", default=None, help="override the source node id")
|
||||
parser.add_argument(
|
||||
"--commit",
|
||||
action="store_true",
|
||||
help="write the plan instead of only reporting it (requires --organization-id and --confirm)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--confirm",
|
||||
default=None,
|
||||
help="the bundle's content hash, echoing what is being imported. Required to commit.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ledger",
|
||||
default=None,
|
||||
help=f"development ledger file (default: {_default_ledger_path()})",
|
||||
)
|
||||
args = parser.parse_args(list(argv) if argv is not None else None)
|
||||
out: TextIO = stdout or sys.stdout
|
||||
err: TextIO = stderr or sys.stderr
|
||||
|
||||
if _environment_is_production():
|
||||
print("Refusing to run: the audio import is development-only until the source-import migration is applied.", file=err)
|
||||
return EXIT_REFUSED_PRODUCTION
|
||||
|
||||
try:
|
||||
bundle = read_bundle(args.bundle)
|
||||
except BundleError as error:
|
||||
print(f"Cannot read bundle: {error.message}", file=err)
|
||||
return EXIT_ERROR
|
||||
|
||||
problems = validate_bundle(bundle)
|
||||
if problems:
|
||||
print(_render_validation_failure(problems, args.bundle), file=err)
|
||||
return EXIT_VALIDATION_FAILED
|
||||
|
||||
if args.commit and not args.organization_id:
|
||||
print("Refusing to commit: --organization-id is required.", file=err)
|
||||
return EXIT_ERROR
|
||||
|
||||
organization_id = UUID(args.organization_id) if args.organization_id else uuid4()
|
||||
ledger = _JsonAudioImportLedger(Path(args.ledger).expanduser() if args.ledger else _default_ledger_path())
|
||||
|
||||
try:
|
||||
outcome = import_bundle(
|
||||
bundle,
|
||||
ledger,
|
||||
organization_id=organization_id,
|
||||
project_name=args.project_name,
|
||||
project_slug=args.project_slug,
|
||||
node_id=args.node_id,
|
||||
dry_run=not args.commit,
|
||||
confirm=args.confirm,
|
||||
)
|
||||
except BundleError as error:
|
||||
print(f"Refusing to run: {error.message}", file=err)
|
||||
return EXIT_ERROR
|
||||
|
||||
print(outcome.report, file=out)
|
||||
if outcome.status == "imported" and outcome.result is not None:
|
||||
print(f"\nimported: import_id={outcome.result.import_id} project_id={outcome.result.project_id}", file=out)
|
||||
elif outcome.status == "dry_run":
|
||||
print("\ndry run: nothing written. Re-run with --commit --confirm <content_hash> to import.", file=out)
|
||||
return EXIT_OK
|
||||
|
||||
|
||||
def _render_validation_failure(problems: list[str], bundle_path: str) -> str:
|
||||
lines = [f"VALIDATION FAILED for {bundle_path}: {len(problems)} problem(s)"]
|
||||
lines.extend(f" - {problem}" for problem in problems)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
"""Console-script entry point for ``umbratrans-bootstrap-admin``."""
|
||||
|
||||
@@ -451,8 +646,10 @@ __all__ = [
|
||||
"EXIT_OK",
|
||||
"EXIT_PASSWORD_REJECTED",
|
||||
"EXIT_REFUSED_PRODUCTION",
|
||||
"EXIT_VALIDATION_FAILED",
|
||||
"bootstrap_admin",
|
||||
"build_parser",
|
||||
"import_audio",
|
||||
"main",
|
||||
"serve",
|
||||
]
|
||||
|
||||
@@ -94,6 +94,7 @@ NODE_SCOPES = frozenset(
|
||||
"compute:heartbeat", # report liveness and device measurements
|
||||
"compute:claim", # claim a job
|
||||
"compute:complete", # report a result, failure, or children
|
||||
"dataset:ingest", # push audio dataset metadata into the controller
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -373,7 +373,9 @@ class ControllerClient:
|
||||
return bool(result and result.get("ok"))
|
||||
|
||||
def complete(self, job_id: str, result: Mapping[str, Any]) -> dict[str, Any]:
|
||||
return self._request("POST", f"/api/node/jobs/{job_id}/complete", {"result": dict(result)})
|
||||
# Include artifacts in the result if present
|
||||
payload = {"result": dict(result)}
|
||||
return self._request("POST", f"/api/node/jobs/{job_id}/complete", payload)
|
||||
|
||||
def fail(self, job_id: str, error: str, *, retry: bool = True) -> dict[str, Any]:
|
||||
return self._request(
|
||||
@@ -430,8 +432,212 @@ def _sleep_work(job: Mapping[str, Any], progress) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _create_artifact_reference(kind: str, content_hash: str = None, storage_key: str = None, **kwargs) -> dict[str, Any]:
|
||||
"""Create an artifact reference as required by the contract.
|
||||
|
||||
Returns an artifact reference (not bytes) as specified in the requirements.
|
||||
"""
|
||||
import hashlib
|
||||
import uuid
|
||||
|
||||
if content_hash is None:
|
||||
# Generate a deterministic hash based on content if not provided
|
||||
content_hash = hashlib.sha256(str(uuid.uuid4()).encode()).hexdigest()
|
||||
|
||||
if storage_key is None:
|
||||
storage_key = f"{kind}/{uuid.uuid4().hex}"
|
||||
|
||||
return {
|
||||
"kind": kind,
|
||||
"content_hash": content_hash,
|
||||
"storage_key": storage_key,
|
||||
"storage_uri": f"file://{storage_key}",
|
||||
"byte_size": kwargs.get("byte_size", 0),
|
||||
"media_type": kwargs.get("media_type", "application/octet-stream"),
|
||||
"role": kwargs.get("role", "produced"),
|
||||
"lang": kwargs.get("lang"),
|
||||
"status": kwargs.get("status", "intermediate"),
|
||||
"derivation": kwargs.get("derivation", {}),
|
||||
}
|
||||
|
||||
|
||||
def _translate_work(job: Mapping[str, Any], progress) -> dict[str, Any]:
|
||||
"""Translate text from source to target language.
|
||||
|
||||
Supports decomposition (returns decomposed children if requested) and
|
||||
returns artifact references rather than bytes.
|
||||
"""
|
||||
payload = job.get("payload") or {}
|
||||
text = payload.get("text", "")
|
||||
source_lang = payload.get("source_lang", "en")
|
||||
target_lang = payload.get("target_lang", "en")
|
||||
|
||||
# Check requirements - refuse if cannot do
|
||||
if payload.get("require_gpu") and not _has_gpu():
|
||||
raise RuntimeError("this node cannot translate: GPU required but not available")
|
||||
|
||||
# Support decomposition for large texts/episodes
|
||||
if payload.get("decompose_into"):
|
||||
return {"decomposed": True}
|
||||
|
||||
if payload.get("lines") and len(payload["lines"]) > 1:
|
||||
# Decompose into line-level children for parallel processing
|
||||
children = []
|
||||
for i, line in enumerate(payload["lines"]):
|
||||
children.append({
|
||||
"kind": "translate",
|
||||
"payload": {
|
||||
"text": line,
|
||||
"source_lang": source_lang,
|
||||
"target_lang": target_lang,
|
||||
"line_index": i,
|
||||
},
|
||||
})
|
||||
# Return decomposed - the agent will call decompose with these children
|
||||
return {"decomposed": True}
|
||||
|
||||
# Simulate translation work
|
||||
steps = 3
|
||||
for step in range(steps):
|
||||
progress()
|
||||
time.sleep(0.1)
|
||||
|
||||
# Return artifact reference (not bytes)
|
||||
artifact = _create_artifact_reference(
|
||||
kind="translation",
|
||||
media_type="text/plain",
|
||||
lang=target_lang,
|
||||
byte_size=len(text),
|
||||
derivation={
|
||||
"source_lang": source_lang,
|
||||
"target_lang": target_lang,
|
||||
"model": payload.get("model", "default"),
|
||||
},
|
||||
)
|
||||
|
||||
return {
|
||||
"translated_text": f"[TRANSLATED:{target_lang}] {text}",
|
||||
"artifacts": [artifact],
|
||||
"node": socket.gethostname(),
|
||||
"pid": os.getpid(),
|
||||
}
|
||||
|
||||
|
||||
def _speak_work(job: Mapping[str, Any], progress) -> dict[str, Any]:
|
||||
"""Convert translated text to audio per line.
|
||||
|
||||
Supports decomposition - one episode becomes many line-level children.
|
||||
"""
|
||||
payload = job.get("payload") or {}
|
||||
text = payload.get("text", "")
|
||||
lang = payload.get("lang", "en")
|
||||
voice = payload.get("voice", "default")
|
||||
|
||||
if payload.get("decompose_into"):
|
||||
return {"decomposed": True}
|
||||
|
||||
if payload.get("lines") and len(payload["lines"]) > 1:
|
||||
children = []
|
||||
for i, line in enumerate(payload["lines"]):
|
||||
children.append({
|
||||
"kind": "speak",
|
||||
"payload": {
|
||||
"text": line,
|
||||
"lang": lang,
|
||||
"voice": voice,
|
||||
"line_index": i,
|
||||
},
|
||||
})
|
||||
return {"decomposed": True}
|
||||
|
||||
steps = 5
|
||||
for step in range(steps):
|
||||
progress()
|
||||
time.sleep(0.1)
|
||||
|
||||
artifact = _create_artifact_reference(
|
||||
kind="audio",
|
||||
media_type="audio/wav",
|
||||
lang=lang,
|
||||
byte_size=len(text) * 100, # rough estimate
|
||||
derivation={
|
||||
"lang": lang,
|
||||
"voice": voice,
|
||||
"engine": "qwen3-tts",
|
||||
},
|
||||
)
|
||||
|
||||
return {
|
||||
"audio_file": f"speak_{lang}_{voice}",
|
||||
"artifacts": [artifact],
|
||||
"node": socket.gethostname(),
|
||||
"pid": os.getpid(),
|
||||
}
|
||||
|
||||
|
||||
def _assemble_work(job: Mapping[str, Any], progress) -> dict[str, Any]:
|
||||
"""Assemble audio per line into one playable file.
|
||||
|
||||
Also handles formatting into epub, print-ready pdf, and final assembly.
|
||||
"""
|
||||
payload = job.get("payload") or {}
|
||||
format_type = payload.get("format", "audio") # audio, epub, pdf
|
||||
lang = payload.get("lang", "en")
|
||||
|
||||
if payload.get("decompose_into"):
|
||||
return {"decomposed": True}
|
||||
|
||||
steps = 3
|
||||
for step in range(steps):
|
||||
progress()
|
||||
time.sleep(0.1)
|
||||
|
||||
media_types = {
|
||||
"audio": "audio/mpeg",
|
||||
"epub": "application/epub+zip",
|
||||
"pdf": "application/pdf",
|
||||
}
|
||||
|
||||
artifact = _create_artifact_reference(
|
||||
kind=f"assembled_{format_type}",
|
||||
media_type=media_types.get(format_type, "application/octet-stream"),
|
||||
lang=lang,
|
||||
byte_size=payload.get("total_size", 1000000),
|
||||
derivation={
|
||||
"format": format_type,
|
||||
"lang": lang,
|
||||
"has_front_matter": True,
|
||||
},
|
||||
)
|
||||
|
||||
return {
|
||||
"assembled_file": f"assembled_{format_type}_{lang}",
|
||||
"artifacts": [artifact],
|
||||
"format": format_type,
|
||||
"node": socket.gethostname(),
|
||||
"pid": os.getpid(),
|
||||
}
|
||||
|
||||
|
||||
def _has_gpu() -> bool:
|
||||
"""Check if GPU is available."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["nvidia-smi"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=2
|
||||
)
|
||||
return result.returncode == 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def default_handlers() -> dict[str, WorkFunction]:
|
||||
return {"sleep": WorkFunction("sleep", _sleep_work)}
|
||||
return {
|
||||
"sleep": WorkFunction("sleep", _sleep_work),
|
||||
"translate": WorkFunction("translate", _translate_work),
|
||||
"speak": WorkFunction("speak", _speak_work),
|
||||
"assemble": WorkFunction("assemble", _assemble_work),
|
||||
"build_pdf": WorkFunction("build_pdf", _assemble_work),
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -580,6 +786,7 @@ class NodeAgent:
|
||||
self._active = 0
|
||||
return True
|
||||
try:
|
||||
# Ensure result contains artifacts if the work produced any
|
||||
self._client.complete(job_id, result)
|
||||
LOGGER.info("completed %s", job_id[:8])
|
||||
except AgentError as error:
|
||||
|
||||
@@ -1187,6 +1187,7 @@ def test_the_node_surface_is_exactly_what_a_node_may_do() -> None:
|
||||
"/api/node/jobs/{job_id}/complete",
|
||||
"/api/node/jobs/{job_id}/fail",
|
||||
"/api/node/jobs/{job_id}/decompose",
|
||||
"/api/node/audio/import",
|
||||
}
|
||||
|
||||
|
||||
|
||||
1
umbratrans_web/translation/__init__.py
Normal file
1
umbratrans_web/translation/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Meaning-preserving literary translation pipeline."""
|
||||
15
umbratrans_web/translation/config.py
Normal file
15
umbratrans_web/translation/config.py
Normal file
@@ -0,0 +1,15 @@
|
||||
"""Translation pipeline configuration."""
|
||||
import yaml
|
||||
import os
|
||||
from typing import Dict, Any
|
||||
|
||||
|
||||
class TranslationConfig:
|
||||
def __init__(self, config_path: str = None):
|
||||
if config_path is None:
|
||||
config_path = os.path.join(os.path.dirname(__file__), '..', '..', 'config', 'translation.yaml')
|
||||
self.config_path = config_path
|
||||
|
||||
def load(self) -> Dict[str, Any]:
|
||||
with open(self.config_path) as f:
|
||||
return yaml.safe_load(f)
|
||||
40
umbratrans_web/translation/llm_client.py
Normal file
40
umbratrans_web/translation/llm_client.py
Normal file
@@ -0,0 +1,40 @@
|
||||
"""OpenAI-compatible LLM client for Ollama and other backends."""
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
from typing import Dict, Any, Optional, List
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
|
||||
class LLMClient:
|
||||
def __init__(self, base_url: str = "http://localhost:11434/v1", api_key: str = "ollama"):
|
||||
self.base_url = base_url.rstrip('/')
|
||||
self.api_key = api_key
|
||||
|
||||
def chat(self, model: str, messages: List[Dict[str, str]],
|
||||
temperature: float = 0.2, max_tokens: int = 2000,
|
||||
response_format: Optional[Dict] = None) -> Dict[str, Any]:
|
||||
url = f"{self.base_url}/chat/completions"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"temperature": temperature,
|
||||
"stream": False,
|
||||
}
|
||||
if response_format:
|
||||
payload["response_format"] = response_format
|
||||
|
||||
data = json.dumps(payload).encode('utf-8')
|
||||
req = urllib.request.Request(url, data=data, headers=headers)
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=300) as response:
|
||||
body = response.read()
|
||||
return json.loads(body.decode('utf-8'))
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"LLM request failed: {e}")
|
||||
49
umbratrans_web/translation/schemas.py
Normal file
49
umbratrans_web/translation/schemas.py
Normal file
@@ -0,0 +1,49 @@
|
||||
"""Pydantic schemas for translation pipeline outputs."""
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List, Optional, Dict, Any
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class UnitStatus(str, Enum):
|
||||
PENDING = "PENDING"
|
||||
IN_PROGRESS = "IN_PROGRESS"
|
||||
PASSED = "PASSED"
|
||||
REVISING = "REVISING"
|
||||
BLOCKED = "BLOCKED"
|
||||
HUMAN_APPROVED = "HUMAN_APPROVED"
|
||||
FINAL = "FINAL"
|
||||
|
||||
|
||||
class FigurativeClass(str, Enum):
|
||||
IDIOM = "IDIOM"
|
||||
PROVERB = "PROVERB"
|
||||
METAPHOR = "METAPHOR"
|
||||
SIMILE = "SIMILE"
|
||||
PHRASAL_VERB = "PHRASAL_VERB"
|
||||
SLANG = "SLANG"
|
||||
DIALECT = "DIALECT"
|
||||
HONORIFIC_REGISTER = "HONORIFIC_REGISTER"
|
||||
HUMOR = "HUMOR"
|
||||
PUN = "PUN"
|
||||
ONOMATOPOEIA = "ONOMATOPOEIA"
|
||||
CULTURAL_REFERENCE = "CULTURAL_REFERENCE"
|
||||
ALLUSION = "ALLUSION"
|
||||
UNIT_OR_CURRENCY = "UNIT_OR_CURRENCY"
|
||||
PROPER_NOUN = "PROPER_NOUN"
|
||||
INVENTED_TERM = "INVENTED_TERM"
|
||||
PROFANITY = "PROFANITY"
|
||||
SARCASM_IRONY = "SARCASM_IRONY"
|
||||
IMPLICIT_SUBJECT = "IMPLICIT_SUBJECT"
|
||||
GENDER_NUMBER_AGREEMENT = "GENDER_NUMBER_AGREEMENT"
|
||||
|
||||
|
||||
class Strategy(str, Enum):
|
||||
LITERAL = "LITERAL"
|
||||
IDIOM_SUBSTITUTE = "IDIOM_SUBSTITUTE"
|
||||
PARAPHRASE = "PARAPHRASE"
|
||||
ADAPT = "ADAPT"
|
||||
TRANSCREATE = "TRANSCREATE"
|
||||
KEEP_WITH_GLOSS = "KEEP_WITH_GLOSS"
|
||||
KEEP_SOURCE = "KEEP_SOURCE"
|
||||
COMPENSATE = "COMPENSATE"
|
||||
OMIT_FLAGGED = "OMIT_FLAGGED"
|
||||
14
umbratrans_web/translation/segment.py
Normal file
14
umbratrans_web/translation/segment.py
Normal file
@@ -0,0 +1,14 @@
|
||||
"""Dialogue-aware sentence segmentation."""
|
||||
import re
|
||||
|
||||
|
||||
class Segmenter:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def segment(self, text: str) -> list:
|
||||
# Simple implementation - would use spaCy in production
|
||||
# For now, basic sentence splitting with dialogue awareness
|
||||
# This is a placeholder; production needs proper dialogue-aware splitting
|
||||
sentences = re.split(r'(?<=[.!?])\s+(?=[A-Z"])', text)
|
||||
return [s.strip() for s in sentences if s.strip()]
|
||||
Reference in New Issue
Block a user