WIP

hosts/mars/hermes-agent.nix

@@ -49,8 +49,8 @@
 # (no networking.firewall.allowedTCPPorts entry; tailscale0 is already a
 # trustedInterface, services/vpn/tailscale.nix). Public route: neptun's
 # hermes.mgaction.town vhost (hosts/neptun/configuration.nix) proxies to this
-# over the tailnet. mars runs no Caddy of its own (single-purpose box), so
-# there is no LAN vhost — reach the dashboard directly via mars's tailnet
+# over the tailnet. mars's own Caddy (luna-sites.nix) only serves luna's apps
+# and has no vhost for this — reach the dashboard directly via mars's tailnet
 # name (mars.orbit.sol:9119) or LAN IP:9119 for local debugging.
 #
 # Uses upstream's generic self-hosted OIDC plugin, same Authentik
@@ -196,13 +196,31 @@ in
   # directly against the real instance during the first version of this
   # setup). Delete-then-add is idempotent either way and picks up a rotated
   # token for free.
+  #
+  # `tea logins add` is the ONLY step in here that touches the network, and
+  # ordering is what makes it survivable. switch-to-configuration restarts
+  # NetworkManager and starts this unit in the SAME pass: on 2026-09-11 the
+  # two landed in the same second, tea's connect went out over an interface
+  # that was still coming back, and the kernel spent 2m48s on SYN retries
+  # before reporting "connection timed out". That failed this unit, which
+  # podman-hermes-agent Requires=, so a five-second network blip took the
+  # whole container down and returned 4 from the deploy. Hence
+  # network-online.target below, the bounded reachability probe in the script,
+  # and TimeoutStartSec as the backstop — no single blocking call in here may
+  # outlive the deploy that started it.
   systemd.services.hermes-agent-prepare-dirs = {
     description = "Create Hermes state dirs + luna's git/tea access before the container starts";
     before = [ "podman-hermes-agent.service" ];
     wantedBy = [ "podman-hermes-agent.service" ];
+    wants = [ "network-online.target" ];
+    after = [ "network-online.target" ];
     unitConfig.RequiresMountsFor = [ "/mnt/jupiter" ];
-    path = [ pkgs.git pkgs.tea ];
+    path = [ pkgs.git pkgs.tea pkgs.curl pkgs.coreutils ];
     serviceConfig.Type = "oneshot";
+    # Everything here is either local or bounded to ~30s by the probe loop, so
+    # anything past two minutes is a hang, not slowness. Failing at that point
+    # is strictly better than holding the deploy open.
+    serviceConfig.TimeoutStartSec = "120";
     script = ''
       mkdir -p ${hermesHome}
       mkdir -p ${dropboxDir}
@@ -231,9 +249,43 @@ in
       git config --global user.name "luna"
       git config --global user.email "luna@${giteaHost}"
 
-      tea logins delete luna 2>/dev/null || true
-      GITEA_SERVER_TOKEN="$(cat "$token_file")" tea logins add \
-        --name luna --url "https://${giteaHost}" --no-version-check
+      # Probe before touching the login, with a hard per-attempt timeout: a
+      # bare TCP connect to an interface that is still coming up hangs for
+      # ~3 minutes on kernel SYN retries, and tea has no timeout flag of its
+      # own. /api/v1/version is unauthenticated, so this says "is gitea
+      # reachable", never "is the token good" — the token is the add's job.
+      #
+      # Probing FIRST (rather than retrying the add) is what protects the
+      # login that is already there. delete-then-add is not atomic: an add
+      # that fails because the network is down leaves luna with no login at
+      # all, strictly worse than the stale-but-working one we started with.
+      # Unreachable therefore means skip the refresh entirely and warn.
+      gitea_up=0
+      for attempt in 1 2 3; do
+        if curl -fsS --max-time 5 -o /dev/null "https://${giteaHost}/api/v1/version"; then
+          gitea_up=1
+          break
+        fi
+        echo "${giteaHost} unreachable (attempt $attempt/3); retrying in 5s" >&2
+        sleep 5
+      done
+
+      if [ "$gitea_up" = 1 ]; then
+        # Reachable but the add still fails == a real problem (revoked or
+        # under-scoped token, gitea rejecting the login), and that stays
+        # fatal: it is a config error, it will not fix itself on the next
+        # boot, and it should be loud.
+        tea logins delete luna 2>/dev/null || true
+        GITEA_SERVER_TOKEN="$(cat "$token_file")" timeout 60 tea logins add \
+          --name luna --url "https://${giteaHost}" --no-version-check
+      else
+        # Deliberately not fatal. Every other thing this unit does is local,
+        # and podman-hermes-agent Requires= it — failing here would take
+        # Telegram and the dashboard down over a transient blip. luna keeps
+        # git (the credential helper above needs no network to be written)
+        # and loses only the tea CLI until the next start re-runs this.
+        echo "WARNING: ${giteaHost} unreachable; left luna's tea login untouched." >&2
+      fi
 
       # Hand everything written above to the container's uid/gid. This does
       # NOT happen by itself: the image's cont-init only chowns hermesHome's