homelab/hosts/mars/hermes-agent.nix

{ config, pkgs, ... }:

# Hermes Agent — moved here from jupiter (hosts/jupiter/hermes-agent.nix,
# see its git history / b5fa599 / 713d91d for the terra->jupiter->mars
# lineage). mars is dedicated to this one service, on-site, with no big
# data array of its own — unlike jupiter it has nothing under /mnt/data, so
# state lives on the local OS disk and the shared dropbox rides jupiter's
# samba share as a CIFS client instead of being served locally.
#
# Runs the OFFICIAL published image (docker.io/nousresearch/hermes-agent —
# real and actively maintained, contrary to what the checked-out repo's own
# README/docker-compose.yml suggested; verified directly on Docker Hub) as a
# plain podman container. It never sets HERMES_MANAGED or writes .managed, so
# Hermes fully self-manages config.yaml, profiles, memories and skills at
# runtime — no redeploy needed except to bump the pinned digest below.
#
# Security posture:
#   - Reachable paths: its own local state dir, the small shared "dropbox"
#     (via the jupiter samba mount) for darman to hand files to Hermes, and
#     `git`/`tea`, logged in as the `luna` gitea account (PR-tier only —
#     see services/dev/gitea.nix). No working copy of this repo is
#     provisioned for her: an earlier version cloned one into
#     ${hermesHome}/workspace/homelab, dropped again because nothing ever
#     told her at runtime where it was (she self-manages config/profiles/
#     memories, so a host-side path in this file never reached her) — she
#     searched /opt/data/homelab and /workspace, found neither, and
#     concluded she had no repo at all. She can clone one herself if she
#     wants; the credentials below are what actually grants the access.
#     Nothing else on jupiter's array or the host is reachable if a
#     command goes wrong or gets injected via Telegram/tool output.
#   - Its own Telegram bot (own token, in secrets.nix) with an EXPLICIT
#     TELEGRAM_ALLOWED_USERS.
#   - Runs as a rootful podman container (services/containers.nix) with its
#     OWN numeric uid/gid — not darman, who is in the "hermes" group for
#     host-level debugging only (`hermes ...` alias below, needs sudo since
#     the container itself runs under root's podman, not darman's rootless
#     one).
#   - git/tea access is direct CLI, not a narrow wrapper: darman explicitly
#     chose this over a purpose-built MCP server (tried first, scrapped —
#     see git history) in favor of simplicity. The backstop is entirely
#     server-side: gitea's branch protection on `master` (only darman can
#     push/merge/approve there) is what actually keeps a bad or injected
#     command from reaching the base branch, not anything client-side here.
#
# Dashboard (HERMES_DASHBOARD=1) is gated behind Authentik, same setup as on
# jupiter. Its default bind (0.0.0.0:9119) fails closed without an auth
# provider registered, and 0.0.0.0 (not loopback) is required so neptun's
# Caddy can reach it over tailscale0 — reachability itself stays LAN-closed
# (no networking.firewall.allowedTCPPorts entry; tailscale0 is already a
# trustedInterface, services/vpn/tailscale.nix). Public route: neptun's
# hermes.mgaction.town vhost (hosts/neptun/configuration.nix) proxies to this
# over the tailnet. mars runs no Caddy of its own (single-purpose box), so
# there is no LAN vhost — reach the dashboard directly via mars's tailnet
# name (mars.orbit.sol:9119) or LAN IP:9119 for local debugging.
#
# Uses upstream's generic self-hosted OIDC plugin, same Authentik
# application as before (slug `hermes`) — the client ID/secret didn't need
# to change since the public redirect URI (hermes.mgaction.town) didn't.
#
# Data migration: this starts with a FRESH state dir. jupiter's instance was
# itself reset to fresh on 2026-08-21 (see its old hermes-agent.nix), so
# there was nothing irreplaceable to carry forward; if that turns out to be
# wrong, jupiter's old data is backed up at
# /mnt/data/AppData/hermes.bak-2026-08-21 and can be rsynced into
# ${hermesHome} below before the first switch on mars.
let
  stateDir = "/var/lib/hermes";
  hermesHome = "${stateDir}/.hermes";
  # Shared drop-in folder: darman can put files here from any host. Lives on
  # jupiter's array (reachable at /mnt/jupiter, the samba mount below) rather
  # than locally, so it's the same physical location it always was — only
  # the container reading it moved. Mounted under /opt/data so it falls
  # inside Hermes's own sealed write-safe root (HERMES_WRITE_SAFE_ROOT=
  # /opt/data) rather than a path its own tooling would treat as untrusted.
  dropboxDir = "/mnt/jupiter/AppData/hermes-dropbox";

  # Pinned by digest (captured 2026-08-21 via `podman image inspect
  # docker.io/nousresearch/hermes-agent:latest --format '{{.Digest}}'` on
  # jupiter) rather than floating `:latest`, so a redeploy is reproducible —
  # bumping Hermes is an explicit edit here, not silent drift on next pull.
  hermesImage = "docker.io/nousresearch/hermes-agent@sha256:5342e518734a08f6c66b89b4262434813c28a77abbc59c230c8f1637df71a259";

  # Kept identical to jupiter's instance purely so nothing else needs to
  # change if state ever gets migrated over.
  hermesUid = "986";
  hermesGid = "983";

  # luna's gitea identity (account + PR-tier repo access provisioned in
  # services/dev/gitea.nix). Only the server is pinned here — any checkout
  # is hers to make, anywhere inside HERMES_WRITE_SAFE_ROOT=/opt/data.
  giteaHost = "git.mgaction.town";

  # luna's webhook filter, mounted READ-ONLY below. It lives in the nix store
  # rather than being written into hermesHome because hermesHome IS
  # HERMES_WRITE_SAFE_ROOT: a filter dropped there is a loop guard sitting
  # inside the writable root of the agent it constrains, and she could edit
  # it back out. Deleting it would fail closed (Hermes treats a missing
  # script as "ignore"), but rewriting it to always-allow would silently
  # restore the reply loop. Read-only from the store makes that impossible
  # and keeps the guard versioned in git — same reasoning as the git/tea
  # binaries mounted below.
  prCommentFilter = pkgs.writeText "gitea-pr-comment-filter.py" (
    builtins.readFile ./gitea-pr-comment-filter.py
  );

  # The route prompt, mounted read-only for the same reason as the filter and
  # kept in a file rather than inline in the subscribe command: it is 60 lines
  # of markdown containing apostrophes and {placeholders}, which would have to
  # survive nix string escaping, the systemd unit, and `podman exec sh -c`
  # quoting. A file crosses all three untouched and stays diffable in git.
  prCommentPrompt = pkgs.writeText "gitea-pr-comment-prompt.md" (
    builtins.readFile ./gitea-pr-comment-prompt.md
  );

  # hermesHome as the CONTAINER sees it (the bind mount below). Anything
  # written host-side that gets READ back inside the container must use this
  # prefix, not hermesHome — see the credential.helper below, which was
  # broken exactly that way from 3c1f3e5 until 2026-08-23.
  containerHome = "/opt/data";
in
{
  # Browsing convenience (ssh access to the bind-mounted local state) — does
  # NOT touch the container, which keeps using HERMES_UID/GID above
  # regardless of what's declared here.
  users.groups.hermes.gid = 983;
  users.users.darman.extraGroups = [ "hermes" ];

  # `hermes <args>` on mars == `sudo podman exec -it hermes-agent hermes <args>`.
  # sudo is required: virtualisation.oci-containers runs rootful (system)
  # podman, a separate namespace from darman's own rootless `podman`/`docker`
  # — darman's "hermes"/"docker" group membership only grants filesystem
  # access to the bind-mounted state dir, not to root's container socket.
  programs.zsh.shellAliases.hermes = "sudo podman exec -it hermes-agent hermes";

  systemd.tmpfiles.rules = [
    "d ${stateDir} 0750 root hermes -"
  ];

  # podman requires the bind-mount source to already exist (no auto-create),
  # and the dropbox lives on the CIFS mount below — mkdir there works fine
  # over cifs, no server-side (jupiter) config needed.
  #
  # Also provisions luna's git/tea access: writes a git credential-store file
  # and runs `tea logins add` INTO hermesHome (i.e. paths that appear at
  # /opt/data/... once the container is up). Both run on the HOST as root,
  # before the container starts, and both therefore have to chown what they
  # write themselves — see the chown at the end of the script. Do NOT assume
  # the image's cont-init fixes ownership under hermesHome: it does not
  # recurse into what this oneshot drops there, even though it runs after it.
  #
  # It deliberately does NOT clone the repo for her any more (see the
  # header). The stale ${hermesHome}/workspace/homelab left behind by the
  # version that did is not cleaned up here either — it just stops being
  # managed, and stops being updated. Remove it by hand if you want it gone.
  #
  # Delete-then-add for the tea login (not a "does it exist" check): tea can
  # leave a login entry behind even when `add` reports failure (e.g. a token
  # missing a scope errors out AFTER the entry is written — observed
  # directly against the real instance during the first version of this
  # setup). Delete-then-add is idempotent either way and picks up a rotated
  # token for free.
  systemd.services.hermes-agent-prepare-dirs = {
    description = "Create Hermes state dirs + luna's git/tea access before the container starts";
    before = [ "podman-hermes-agent.service" ];
    wantedBy = [ "podman-hermes-agent.service" ];
    unitConfig.RequiresMountsFor = [ "/mnt/jupiter" ];
    path = [ pkgs.git pkgs.tea ];
    serviceConfig.Type = "oneshot";
    script = ''
      mkdir -p ${hermesHome}
      mkdir -p ${dropboxDir}
      # Parent for the read-only filter bind-mounted at
      # /opt/data/scripts/gitea-pr-comment-filter.py. /opt/data is itself a
      # bind mount of hermesHome, so this directory has to exist HOST-side
      # before podman can mount a file inside it.
      mkdir -p ${hermesHome}/scripts
      mkdir -p ${hermesHome}/prompts

      export HOME=${hermesHome}
      export GIT_CONFIG_GLOBAL=${hermesHome}/.gitconfig
      export XDG_CONFIG_HOME=${hermesHome}/.config
      token_file=${config.sops.secrets.gitea_luna_token.path}

      # Never embed the token in a remote URL (it would land in that
      # clone's .git/config in plaintext) — the credential helper reads it
      # from this file instead.
      install -m 0600 /dev/null ${hermesHome}/.git-credentials
      printf 'https://luna:%s@${giteaHost}\n' "$(cat "$token_file")" \
        > ${hermesHome}/.git-credentials
      # containerHome, NOT hermesHome: git reads this .gitconfig from INSIDE
      # the container, where the host path does not exist. Nothing host-side
      # consumes these credentials any more (the clone that used to is gone),
      # so the container's view is the only one that has to be right.
      git config --global credential.helper "store --file=${containerHome}/.git-credentials"
      git config --global user.name "luna"
      git config --global user.email "luna@${giteaHost}"

      tea logins delete luna 2>/dev/null || true
      GITEA_SERVER_TOKEN="$(cat "$token_file")" tea logins add \
        --name luna --url "https://${giteaHost}" --no-version-check

      # Hand everything written above to the container's uid/gid. This does
      # NOT happen by itself: the image's cont-init only chowns hermesHome's
      # top level and its own state, so root-owned 0600 files dropped here by
      # this oneshot (.git-credentials, and tea's config.yml — tea writes it
      # 0600 too) are simply unreadable to uid ${hermesUid}. Symptom is not an
      # error but an absence: git reports no credential helper and tea reports
      # no login, i.e. "they're missing". Confirmed on the real instance
      # 2026-08-23 — cont-init ran AFTER these files were written and left
      # them root-owned regardless.
      #
      # `if`, not `[ -d x ] && chown`: this script runs under `set -e`, where
      # a false test as the left side of an && list takes the whole list's
      # non-zero status and aborts the unit.
      chown ${hermesUid}:${hermesGid} \
        ${hermesHome}/.gitconfig \
        ${hermesHome}/.git-credentials
      # Same cont-init caveat as the files above: the directory is created
      # here as root, and Hermes reads its scripts as uid ${hermesUid}. The
      # mounted filter itself is world-readable 0444 from the store, so only
      # the directory needs handing over.
      chown ${hermesUid}:${hermesGid} ${hermesHome}/scripts ${hermesHome}/prompts

      if [ -d ${hermesHome}/.config ]; then
        chown ${hermesUid}:${hermesGid} ${hermesHome}/.config
      fi
      if [ -d ${hermesHome}/.config/tea ]; then
        chown -R ${hermesUid}:${hermesGid} ${hermesHome}/.config/tea
      fi
    '';
  };

  virtualisation.oci-containers.containers.hermes-agent = {
    image = hermesImage;
    autoStart = true;
    # Host networking: Hermes only long-polls Telegram outbound, no inbound
    # ports to publish (same reasoning as clonarr on jupiter).
    extraOptions = [ "--network=host" ];
    # Upstream's own documented single-mount pattern (docker/docker-compose.yml):
    # ~/.hermes:/opt/data.
    volumes = [
      "${hermesHome}:/opt/data"
      "${dropboxDir}:/opt/data/dropbox"

      # git/tea for luna: the image doesn't ship `tea` (and shouldn't be
      # trusted to have a known-good `git` either), so both come from this
      # host's Nix store instead — mounted read-only at fixed PATH-visible
      # locations. /nix/store itself has to come along too since both
      # binaries are dynamically linked against paths inside it; the store
      # is read-only content-addressed build output, not a source of
      # secrets, so mounting the whole thing read-only costs nothing beyond
      # the two specific binaries actually being reachable.
      # Read-only: see prCommentFilter above. Hermes resolves route scripts
      # under ~/.hermes/scripts, which is /opt/data/scripts in here.
      "${prCommentFilter}:/opt/data/scripts/gitea-pr-comment-filter.py:ro"
      "${prCommentPrompt}:/opt/data/prompts/gitea-pr-comment.md:ro"

      "/nix/store:/nix/store:ro"
      "${pkgs.git}/bin/git:/usr/local/bin/git:ro"
      "${pkgs.tea}/bin/tea:/usr/local/bin/tea:ro"
    ];
    environment = {
      HERMES_UID = hermesUid;
      HERMES_GID = hermesGid;
      TZ = "Europe/Berlin";

      # Point git/tea at the config the prepare-dirs oneshot wrote into
      # hermesHome (visible here as /opt/data/...) — the credential-store
      # helper, the luna gitea login, and (implicitly, via HOME not being
      # overridden) darman's Hermes state stays wherever it already was.
      GIT_CONFIG_GLOBAL = "/opt/data/.gitconfig";
      XDG_CONFIG_HOME = "/opt/data/.config";
      # HERMES_TIMEZONE is the highest-priority source hermes_time.py checks
      # (ahead of config.yaml's `timezone` key) — the container has no host
      # /etc/localtime bind-mount, so it defaults to UTC otherwise (fixed in
      # 9403122 on jupiter; carried forward here).
      HERMES_TIMEZONE = "Europe/Berlin";

      # Dashboard + Authentik OIDC gate — see the file-level comment above.
      HERMES_DASHBOARD = "1";
      HERMES_DASHBOARD_HOST = "0.0.0.0"; # must be tailscale0-reachable, not just loopback
      HERMES_DASHBOARD_OIDC_ISSUER = "https://auth.mgaction.town/application/o/hermes/";
      HERMES_DASHBOARD_OIDC_CLIENT_ID = "4BqdJu3htnMtSZnyEu5zHnsSOvlEbw3Ie3mYVlh6";
      # uvicorn's proxy_headers=True (web_server.py) only trusts
      # X-Forwarded-Proto from forwarded_allow_ips, which defaults to
      # 127.0.0.1 — neptun's Caddy reaches this over the tailnet (a real
      # routed IP), so without this the dashboard sees the raw scheme (http)
      # and builds an http:// redirect_uri that Authentik rejects against its
      # registered https:// one. Safe to trust any peer here: 9119 is already
      # scoped to loopback + tailscale0 only (no LAN firewall rule), so
      # nothing untrusted can reach this process to begin with.
      FORWARDED_ALLOW_IPS = "*";
    };
    environmentFiles = [ config.sops.templates."hermes-agent.env".path ];
    cmd = [ "gateway" "run" ];
  };

  systemd.services.podman-hermes-agent = {
    after = [
      "hermes-agent-prepare-dirs.service"
      "systemd-tmpfiles-setup.service"
    ];
    requires = [ "hermes-agent-prepare-dirs.service" ];
    unitConfig.RequiresMountsFor = [ "/mnt/jupiter" ];
  };

  # The Gitea PR-comment route. Gitea posts straight here (jupiter's
  # gitea-hermes-webhook-provision registers the hook at
  # http://mars.orbit.sol:8644/webhooks/gitea-pr-comments) -- there is no relay
  # in between. Gitea's addDefaultHeaders sends X-Hub-Signature-256 in GitHub's
  # exact format AND X-GitHub-Event, unconditionally, for every webhook type,
  # which is precisely what Hermes validates and reads the event name from.
  #
  # --events issue_comment, NOT pull_request_comment. Gitea uses the same
  # strings in two different namespaces and they collide:
  #
  #   subscription name          wire name (X-GitHub-Event)   what it is
  #   -----------------------    --------------------------   ----------------
  #   pull_request_comment       issue_comment                comment on a PR
  #   issue_comment              issue_comment                comment on an issue
  #   pull_request_review_comment  pull_request_comment       review on a PR
  #
  # The hook's `events` array (services/dev/gitea.nix) takes the SUBSCRIPTION
  # name; Hermes matches --events against X-GitHub-Event, i.e. the WIRE name,
  # which comes from HookEventType.Event() in modules/webhook/type.go. So
  # "pull_request_comment" here would match review submissions and never a
  # comment -- the exact inversion of what it reads like. X-GitHub-Event-Type
  # carries the subscription name, but Hermes does not look at it.
  #
  # issue_comment on the wire covers comments on plain issues too; the hook
  # does not subscribe those, and the filter's is_pull check drops them anyway
  # if the hook is ever widened.
  #
  # A route carries exactly one prompt, so another event means either branching
  # on {action} in the prompt or a second subscription plus a second Gitea hook
  # at /webhooks/<name>. Review comments would need that: they arrive as a
  # PullRequestPayload with action "reviewed" and no comment object at all.
  #
  # No --deliver: it defaults to `log`. The prompt tells her to answer in the
  # pull request, so the PR comment IS the delivery.
  #
  # --script is the selection that MUST NOT be retunable at runtime.
  # gitea-pr-comment-filter.py drops luna's own comments before any LLM call,
  # which is what stops the reply loop: the prompt tells her to answer on the
  # PR, and her answer is itself a pull_request_comment. Both it and the prompt
  # are bind-mounted read-only from the store above so the agent cannot edit
  # its own guard out. Hermes resolves both names relative to ~/.hermes, hence
  # the bare filename.
  #
  # What read-only does NOT buy: it protects the sources, and this unit
  # re-subscribes from them on every start, so a restart restores the intended
  # prompt, filter and event list. The live subscription lives in
  # webhook_subscriptions.json under /opt/data and is hot-reloaded, which is
  # inside the agent's own write-safe root -- a self-modification sticks until
  # this unit next runs.
  #
  # The secret comes from the CONTAINER's environment, injected via
  # sops.templates."hermes-agent.env", which is why secrets.nix restarts
  # podman-hermes-agent BEFORE this unit on rotation: re-subscribing against a
  # container still holding the old value would silently pin the stale secret.
  systemd.services.hermes-agent-webhook-route = {
    description = "Configure Hermes Gitea PR-comment webhook route";
    wantedBy = [ "multi-user.target" ];
    after = [ "podman-hermes-agent.service" ];
    requires = [ "podman-hermes-agent.service" ];
    path = [ pkgs.podman ];
    serviceConfig = {
      Type = "oneshot";
      RemainAfterExit = true;
    };
    script = ''
      set -euo pipefail

      # The container unit is ordered before us, but its gateway may still be
      # warming up while the image initializes its persistent state directory.
      for _ in $(seq 1 60); do
        if podman exec hermes-agent hermes webhook list >/dev/null 2>&1; then
          break
        fi
        sleep 1
      done

      # Idempotency for the subscribe below, not cleanup: this removes only the
      # route this unit owns. Retiring an old route is a one-off done by hand,
      # so that a redeploy never silently deletes one added on purpose.
      podman exec hermes-agent hermes webhook remove gitea-pr-comments >/dev/null 2>&1 || true

      # `set -eu` plus both emptiness checks are load-bearing. Without them a
      # missing prompt file or an unset secret yields an empty string, and the
      # subscription is created with an empty prompt or -- worse -- an empty
      # secret, which silently fails EVERY delivery signature check afterwards
      # while the unit still looks healthy. Fail loudly here instead.
      podman exec hermes-agent sh -c '
        set -eu
        [ -n "''${GITEA_HERMES_WEBHOOK_SECRET:-}" ] || {
          echo "GITEA_HERMES_WEBHOOK_SECRET is unset in the container" >&2; exit 1; }
        prompt="$(cat /opt/data/prompts/gitea-pr-comment.md)"
        [ -n "$prompt" ] || { echo "gitea-pr-comment prompt is empty" >&2; exit 1; }
        hermes webhook subscribe gitea-pr-comments \
          --secret "$GITEA_HERMES_WEBHOOK_SECRET" \
          --description "Gitea PR comments -> L.U.N.A." \
          --events issue_comment \
          --script gitea-pr-comment-filter.py \
          --prompt "$prompt"
      '
    '';
  };
}