diff --git a/hosts/mars/hermes-agent.nix b/hosts/mars/hermes-agent.nix index 53b0950..e4fa364 100644 --- a/hosts/mars/hermes-agent.nix +++ b/hosts/mars/hermes-agent.nix @@ -301,6 +301,11 @@ in # Authentik rejects. Safe to trust any peer: 9119 is already scoped to # loopback + tailscale0 only. FORWARDED_ALLOW_IPS = "*"; + + # KittenTTS: point huggingface_hub at the pre-seeded offline cache and + # forbid network — no model drift, no boot-time fetch (review #4). + HF_HOME = "/opt/data/kittentts-hf"; + HF_HUB_OFFLINE = "1"; }; environmentFiles = [ config.sops.templates."hermes-agent.env".path ]; cmd = [ "gateway" "run" ]; @@ -486,26 +491,42 @@ in # .pth-registered stub (kitten-misaki-stub.py) that fails loudly if misaki # is ever actually used. # - # Provisioner invariants (shaped by the Mnemosyne-round review): - # - `uv venv --clear`: re-provision cannot wedge on an existing dir. - # - Root runs NO python from this venv: the unit itself does only fs - # writes; the venv is chowned to the container uid before Hermes ever - # imports from it. (Hermes executes provider code as uid 986.) - # - HF model cache is PRE-SEEDED from hash-pinned store paths, so - # HF_HUB_OFFLINE=1 gives zero boot-time network and no drift. - # - Idempotent via a stamp keyed on the requirements hash; checked - # against BOTH the venv python and model files being intact. + # Provisioner invariants (shaped by the Mnemosyne + KittenTTS review rounds): + # - Build in a ROOT-OWNED staging dir (/var/lib/hermes-kittentts, 0700): + # stamp, staging venv, uv cache and staging model copies all live there. + # Root never reads or executes anything the container can write — the + # container cannot symlink-takeover the stamp (review #1) or plant a + # wheel in the uv cache (review #2); only FINISHED artifacts are copied + # into hermesHome, and the stamp compares the delivered copy against + # staging byte-for-byte. + # - Root runs python ONLY from the staging venv (never from the delivered + # uid-986-owned tree in hermesHome) — after a root-side offline import + # check; a failing build aborts before anything is delivered. + # - HF model cache delivered to hermesHome from staging (refs/main -> + # snapshots/), and the container env pins + # HF_HOME=/opt/data/kittentts-hf + HF_HUB_OFFLINE=1: zero boot-time + # network, no drift. + # - Idempotency stamp keyed on the FULL input set (requirements + wheel + + # stub + model files), not just requirements (review #5); a deleted + # delivered file falls out of the fast path and re-provisions cheaply. + # - Order after prepare-dirs (review #6) — mirrors the mnemosyne unit. # - # Trust boundary: the venv lives inside hermesHome (HERMES_WRITE_SAFE_ROOT), - # so luna can technically alter her own TTS engine. Deliberate: it's her - # voice, not her jail — the webhook filter scripts remain the only + # Trust boundary: the DELIVERED venv lives inside hermesHome + # (HERMES_WRITE_SAFE_ROOT), so luna can alter her own TTS engine — and a + # deleted/modified copy just triggers a fresh delivery from the root-owned + # staging area on next boot (self-heals instead of wedging). Deliberate: it's + # her voice, not her jail — the webhook filter scripts remain the only # write-protected-but-load-bearing items. systemd.services.hermes-agent-kittentts-provision = { description = "Provision KittenTTS voice provider (side venv + offline HF cache)"; before = [ "podman-hermes-agent.service" ]; wantedBy = [ "podman-hermes-agent.service" ]; wants = [ "network-online.target" ]; - after = [ "network-online.target" ]; + # after prepare-dirs (review #6): on a fresh state dir this unit must not + # create hermesHome root-owned before prepare-dirs sets the ownership + # layout — same ordering contract the mnemosyne unit has. + after = [ "hermes-agent-prepare-dirs.service" ]; + requires = [ "hermes-agent-prepare-dirs.service" ]; path = [ pkgs.uv pkgs.coreutils ]; serviceConfig = { Type = "oneshot"; @@ -516,51 +537,110 @@ in venv=${hermesHome}/kittentts-venv hubDir=${hermesHome}/kittentts-hf/hub/models--KittenML--kitten-tts-mini-0.8 - snap=$hubDir/snapshots/kitten - stampFile=${hermesHome}/kittentts-provision.stamp - reqHash=$(sha256sum ${kittenttsReqs} | cut -d' ' -f1) + # Real upstream commit SHA as snapshot dir: hf_hub_download resolves + # refs/main -> snapshots/; "kitten" (review #3) is never found + # offline and silently triggers a re-download. + modelSha=c02725660cea441db4c383af69f1f26f5cd00947 + snap=$hubDir/snapshots/$modelSha + # REVIEW #1/#2: nothing root touches lives in hermesHome. Stamp, staging + # venv and uv cache live root-owned under /var/lib/hermes-kittentts (a + # path the container can not pathwrite or symlinks into its own tree); + # the FINISHED staging venv and model files are the only things copied + # into hermesHome, and only after being validated. Root doesn't follow + # any uid-986-writable path while running as root. + stageDir=/var/lib/hermes-kittentts + # Root-side staging of the model files (hash-pinned store paths; the + # delivered copy under hermesHome is compared against these). + stageSnap=$stageDir/hf-model + # Input key: requirements + wheel + stub + model files + resolved script. + # Review #5 — a miss on the old requirements-only stamp let a changed + # wheel/stub/model URL keep running the stale install forever. + inputHash=$(cat ${kittenttsReqs} ${kittenttsWheel} ${kittenttsStub} \ + ${kittenttsModelOnnx} ${kittenttsModelVoices} ${kittenttsModelConfig} \ + | sha256sum | cut -d' ' -f1) + stampFile=$stageDir/provision.stamp - # Idempotent early exit: stamp + venv + all three model files intact. - if [ -f "$stampFile" ] && [ "$(cat "$stampFile")" = "$reqHash" ] \ - && [ -x "$venv/bin/python" ] \ - && [ -f "$snap/kitten_tts_mini_v0_8.onnx" ] \ - && [ -f "$snap/voices.npz" ] \ - && [ -f "$venv/lib/python3.13/site-packages/kitten_tts_stub.py" ]; then - exit 0 + # Idempotent early exit: stamp matches the full input hash, staging venv + # validated, delivered copy intact check runs below. + if [ -f "$stampFile" ] && [ "$(cat "$stampFile")" = "$inputHash" ] \ + && [ -x "$stageVenv/bin/python" ] \ + && [ -f "$stageSnap/config.json" ]; then + # Delivered artifacts in hermesHome must ALSO match the staging copy: + # luna can rewrite her copy, that's fine — but then the missing file + # forces a re-provision (cheap copy, not a rebuild) so deletions + # cannot wedge the gateway without a voice. + if [ -x "$venv/bin/python" ] \ + && cmp -s "$stageSnap/config.json" "$snap/config.json" 2>/dev/null \ + && cmp -s "$stageSnap/kitten_tts_mini_v0_8.onnx" "$snap/kitten_tts_mini_v0_8.onnx" 2>/dev/null \ + && cmp -s "$stageSnap/voices.npz" "$snap/voices.npz" 2>/dev/null; then + exit 0 + fi fi - # Venv (rebuilt rather than broken on --no-clear: uv exits 2 otherwise). - uv venv "$venv" --python ${pkgs.python313}/bin/python3 --clear --quiet - UV_CACHE_DIR=${hermesHome}/kittentts-uv-cache \ - uv pip install --python "$venv/bin/python" --quiet \ + # ---- staging venv: root-owned path, uv cache included. Root runs + # python from HERE (container can write nothing in this tree), then the + # FINISHED result is copied into hermesHome. Review #1: a stamp symlink + # into /etc/shadow can't redirect us — the stamp dir is root-only. ---- + stageVenv=$stageDir/venv + mkdir -p "$stageDir" "$stageSnap" + chmod 0700 "$stageDir" + install -m 0444 ${kittenttsModelConfig} "$stageSnap/config.json" + install -m 0444 ${kittenttsModelOnnx} "$stageSnap/kitten_tts_mini_v0_8.onnx" + install -m 0444 ${kittenttsModelVoices} "$stageSnap/voices.npz" + + rm -rf "$stageVenv" + UV_CACHE_DIR=$stageDir/uv-cache \ + uv venv "$stageVenv" --python ${pkgs.python313}/bin/python3 --quiet + UV_CACHE_DIR=$stageDir/uv-cache \ + uv pip install --python "$stageVenv/bin/python" --quiet \ --requirement ${kittenttsReqs} # kittentts --no-deps: its overlay of spacy/misaki[en] is what drags in # the CUDA tree; the requirements freeze already covers its real needs. - UV_CACHE_DIR=${hermesHome}/kittentts-uv-cache \ - uv pip install --python "$venv/bin/python" --quiet --no-deps \ + UV_CACHE_DIR=$stageDir/uv-cache \ + uv pip install --python "$stageVenv/bin/python" --quiet --no-deps \ ${kittenttsWheel} # Dead-import shim: .pth auto-loads kitten_tts_stub at interpreter start # so `from misaki import en, espeak` resolves without the real misaki.en. - siteDir=$("$venv/bin/python" -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])') + siteDir=$("$stageVenv/bin/python" -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])') cp ${kittenttsStub} "$siteDir/kitten_tts_stub.py" printf 'import kitten_tts_stub\n' > "$siteDir/zz-kitten-stub.pth" - # Seed the HF cache with the hashed model files (exact hub layout; - # hf_hub_download scans refs/snapshots on disk offline). - mkdir -p "$hubDir/refs" "$snap" + # Root-side sanity: the venv python must import the plugin entrypoint + # cleanly offline; a failing build aborts BEFORE anything lands in + # hermesHome (fail closed, not half-delivered). + HF_HOME=$stageDir/hf-cache \ + HF_HUB_OFFLINE=1 \ + PHONEMIZER_ESPEAK_LIBRARY="$("$stageVenv/bin/python" -c 'import espeakng_loader,pathlib;print(pathlib.Path(espeakng_loader.get_library_path()))')" \ + PHONEMIZER_ESPEAK_DATA_PATH=$("$stageVenv/bin/python" -c 'import espeakng_loader,pathlib;print(pathlib.Path(espeakng_loader.get_data_path()))') \ + "$stageVenv/bin/python" -c 'from kittentts import KittenTTS; print("kittentts import ok")' >/dev/null + + # ---- deliver: only finished artifacts into hermesHome. Copy, not + # move — staging stays root-owned for the integrity check above. ---- + hubDir=${hermesHome}/kittentts-hf/hub/models--KittenML--kitten-tts-mini-0.8 + snap=$hubDir/snapshots/$modelSha + rm -rf "$venv.tmp" + cp -a "$stageVenv" "$venv.tmp" + mv -Tf "$venv.tmp" "$venv" # atomic-ish swap of the delivered venv + + mkdir -p "$hubDir/refs" "$snap" "$hubDir/blobs" + printf '%s' "$modelSha" > "$hubDir/refs/main" # review #3: resolves offline install -m 0444 ${kittenttsModelOnnx} "$snap/kitten_tts_mini_v0_8.onnx" install -m 0444 ${kittenttsModelVoices} "$snap/voices.npz" install -m 0444 ${kittenttsModelConfig} "$snap/config.json" + # HF hub layout puts the blob under blobs/ with snapshot + # symlinks pointing back — but kittentts reads config/model paths + # RETURNED by hf_hub_download, which serves the resolved snapshot file + # directly. Files-as-plain-content suffices (no LFS indirection here: + # pre-materialized), no blob indirection needed. - # Hand ownership to the container uid BEFORE any python runs in this - # tree (root never imports from it — that was Mnemosyne review #3). + # Hand ownership to the container uid BEFORE any python runs in + # hermesHome's copy — root does its python in $stageVenv, not here. chown -R ${hermesUid}:${hermesGid} "$venv" "$hubDir" - # Stamp LAST — a half-provisioned venv fails the integrity check and - # re-provisions on next boot rather than being trusted. - printf '%s' "$reqHash" > "$stampFile" - chown ${hermesUid}:${hermesGid} "$stampFile" + # Stamp LAST, root-owned outside hermesHome — luna can delete it (which + # just forces a cheap re-provision copy), not tamper via symlink. + printf '%s' "$inputHash" > "$stampFile" ''; }; }