README.md

@@ -40,16 +40,31 @@ run. Each service module opens its own firewall ports.
 ## Gitea event relay
 
 Mars includes a small HMAC-validating relay for Gitea webhooks. It forwards the
-authenticated request body unchanged, along with Gitea event and delivery
-headers, to Hermes over localhost. The relay has no event, repository, action,
-payload, or prompt policy; Hermes owns interpretation and response behavior.
-Jupiter's Gitea provisioning service registers the webhook idempotently at
-`http://mars.orbit.sol:8645/gitea`.
+authenticated request body and Gitea's own signature to Hermes over localhost
+completely unchanged, and copies `X-Gitea-Event` into `X-GitHub-Event`. It has
+no event, repository, action, payload, or prompt policy; Hermes owns
+interpretation and response behavior. Jupiter's Gitea provisioning service
+registers the webhook idempotently at `http://mars.orbit.sol:8645/gitea`.
+
+That one header copy is the entire reason the relay exists. Gitea signs every
+webhook with `X-Hub-Signature-256` in GitHub's exact format, which Hermes
+already accepts on any route — so authentication would work pointing Gitea
+straight at Hermes on 8644. But Hermes reads the event name only from
+`X-GitHub-Event`/`X-GitLab-Event` (then `event_type`/`type` in the payload,
+then the literal `"unknown"`), and Gitea sends none of those. Without the copy
+every delivery arrives as `unknown` and `hermes webhook subscribe --events ...`
+can never match anything.
+
+Run `python3 services/dev/gitea-hermes-webhook-relay-test.py` to exercise the
+relay end to end (signature acceptance and rejection, byte-identical body
+forwarding, and the event-header copy).
 
 Before deploying either host, add the same random
 `gitea_hermes_webhook_secret` value to both `secrets/mars.yaml` and
-`secrets/jupiter.yaml` with `sops --set`. The value is intentionally not
-included in the repository.
+`secrets/jupiter.yaml` using `scripts/edit_secrets`, with no trailing newline
+— the value reaches Hermes through an env-file template, where a newline both
+corrupts the file and changes the key the HMAC is computed with. The value is
+intentionally not included in the repository.
 
 ## Test in VirtualBox (no hardware needed)
 

hosts/mars/secrets.nix

@@ -28,10 +28,24 @@
   sops.secrets.opencode_go_api_key = { };
   sops.secrets.telegram_bot_token = { };
   sops.secrets.hermes_dashboard_oidc_client_secret = { };
+  # Add the same value to secrets/mars.yaml before deploying Mars, and store
+  # it WITHOUT a trailing newline: it reaches Hermes through the env template
+  # below, where a newline would both corrupt the env file and change the key
+  # the HMAC is computed with. `scripts/edit_secrets` writes a bare value.
+  #
+  # podman-hermes-agent is in restartUnits for a reason that is easy to miss:
+  # the secret reaches the container only through sops.templates, whose
+  # rendered PATH never changes, so the container unit's definition is
+  # identical before and after the secret is added and systemd will NOT
+  # restart it on its own. Without this line the very first deploy leaves the
+  # container holding an empty GITEA_HERMES_WEBHOOK_SECRET, and
+  # hermes-agent-webhook-route (which reads it back out of the running
+  # container) subscribes with an empty secret — every relayed delivery then
+  # fails signature validation inside Hermes with no obvious cause.
   sops.secrets.gitea_hermes_webhook_secret = {
-    # Add the same value to secrets/mars.yaml before deploying Mars.
     restartUnits = [
       "gitea-hermes-webhook-relay.service"
+      "podman-hermes-agent.service"
       "hermes-agent-webhook-route.service"
     ];
   };

services/dev/gitea-hermes-webhook-relay-test.py

@@ -0,0 +1,126 @@
+"""Integration test for gitea-hermes-webhook-relay.py.
+
+Spawns the real relay as a subprocess against a stub Hermes and drives it over
+real HTTP. Run it directly:  python3 services/dev/gitea-hermes-webhook-relay-test.py
+
+The assertion that matters is `X-GitHub-Event injected`: Hermes derives the
+event name it matches `--events` against from X-GitHub-Event, and Gitea only
+ever sends X-Gitea-Event. If that copy regresses, every delivery silently
+becomes event "unknown" and no Hermes-side event selection can work.
+"""
+import hashlib, hmac, json, os, pathlib, subprocess, sys, threading, time, urllib.request, urllib.error
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+SECRET = b"s3cr3t-test-value"
+RELAY_PORT, HERMES_PORT = 18645, 18644
+received = []
+
+class Hermes(BaseHTTPRequestHandler):
+    def do_POST(self):
+        n = int(self.headers.get("Content-Length", 0))
+        # lower-cased keys: HTTP headers are case-insensitive and urllib
+        # normalises them with .title() on the wire ("X-GitHub-Event" leaves
+        # as "X-Github-Event"). aiohttp reads them into a case-insensitive
+        # CIMultiDict, so matching case-insensitively here is the correct
+        # assertion, not a workaround.
+        received.append({"path": self.path, "body": self.rfile.read(n),
+                         "headers": {k.lower(): v for k, v in self.headers.items()}})
+        self.send_response(200); self.send_header("Content-Length", "2")
+        self.end_headers(); self.wfile.write(b"ok")
+    def log_message(self, *a): pass
+
+hermes = HTTPServer(("127.0.0.1", HERMES_PORT), Hermes)
+threading.Thread(target=hermes.serve_forever, daemon=True).start()
+
+import tempfile
+creds = os.path.join(tempfile.mkdtemp(), "creds"); os.makedirs(creds, exist_ok=True)
+# trailing newline on purpose: mimics a sops secret file
+open(os.path.join(creds, "webhook_secret"), "wb").write(SECRET + b"\n")
+
+env = {**os.environ, "CREDENTIALS_DIRECTORY": creds, "LISTEN_HOST": "127.0.0.1",
+       "LISTEN_PORT": str(RELAY_PORT),
+       "HERMES_WEBHOOK_URL": f"http://127.0.0.1:{HERMES_PORT}/webhooks/gitea-events"}
+RELAY = str(pathlib.Path(__file__).with_name('gitea-hermes-webhook-relay.py'))
+relay = subprocess.Popen([sys.executable, RELAY],
+                         env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+
+for _ in range(50):
+    try:
+        urllib.request.urlopen(f"http://127.0.0.1:{RELAY_PORT}/health", timeout=1); break
+    except Exception: time.sleep(0.1)
+
+def post(body, headers):
+    req = urllib.request.Request(f"http://127.0.0.1:{RELAY_PORT}/gitea", data=body,
+                                 headers=headers, method="POST")
+    try:
+        with urllib.request.urlopen(req, timeout=5) as r: return r.status, json.load(r)
+    except urllib.error.HTTPError as e: return e.code, json.load(e)
+
+fails = []
+def check(name, cond, detail=""):
+    print(("PASS " if cond else "FAIL ") + name + ("" if cond else f"  <- {detail}"))
+    if not cond: fails.append(name)
+
+payload = json.dumps({"action": "opened", "number": 7}).encode()
+sig = hmac.new(SECRET, payload, hashlib.sha256).hexdigest()
+
+# 1. health
+with urllib.request.urlopen(f"http://127.0.0.1:{RELAY_PORT}/health") as r:
+    check("health endpoint", json.load(r)["status"] == "ok")
+
+# 2. happy path with BOTH gitea headers (what Gitea really sends)
+received.clear()
+st, resp = post(payload, {"Content-Type": "application/json",
+                          "X-Gitea-Event": "pull_request_comment",
+                          "X-Gitea-Event-Type": "pull_request_comment",
+                          "X-Gitea-Delivery": "abc-123",
+                          "X-Gitea-Signature": sig,
+                          "X-Hub-Signature-256": "sha256=" + sig})
+check("valid delivery accepted", st == 200, f"got {st} {resp}")
+check("forwarded to hermes", len(received) == 1)
+fwd = received[0]
+check("body forwarded byte-identical", fwd["body"] == payload)
+check("X-GitHub-Event injected (THE fix)",
+      fwd["headers"].get("x-github-event") == "pull_request_comment",
+      f"got {fwd['headers'].get('X-GitHub-Event')!r}")
+check("X-Hub-Signature-256 forwarded unchanged",
+      fwd["headers"].get("x-hub-signature-256") == "sha256=" + sig)
+check("signature still valid over forwarded body",
+      hmac.compare_digest(
+          fwd["headers"]["x-hub-signature-256"].removeprefix("sha256="),
+          hmac.new(SECRET, fwd["body"], hashlib.sha256).hexdigest()))
+check("delivery id propagated", fwd["headers"].get("x-request-id") == "abc-123")
+check("path preserved", fwd["path"] == "/webhooks/gitea-events")
+
+# 3. gitea-only signature header (no X-Hub-Signature-256)
+received.clear()
+st, _ = post(payload, {"Content-Type": "application/json", "X-Gitea-Event": "push",
+                       "X-Gitea-Signature": sig})
+check("bare X-Gitea-Signature accepted", st == 200)
+check("relay signs when hub header absent",
+      received and received[0]["headers"].get("x-hub-signature-256") == "sha256=" + sig)
+
+# 4. rejections
+received.clear()
+st, _ = post(payload, {"Content-Type": "application/json", "X-Hub-Signature-256": "sha256=" + "0"*64})
+check("bad signature -> 401", st == 401)
+st, _ = post(payload, {"Content-Type": "application/json"})
+check("missing signature -> 401", st == 401)
+st, _ = post(payload + b"x", {"Content-Type": "application/json", "X-Hub-Signature-256": "sha256=" + sig})
+check("tampered body -> 401", st == 401)
+check("nothing leaked to hermes on rejection", len(received) == 0, f"{len(received)} forwarded")
+
+# 5. oversize
+big = b"x" * 200
+st, _ = post(big, {"Content-Type": "application/json", "MAX": "1",
+                   "X-Hub-Signature-256": "sha256=" + hmac.new(SECRET, big, hashlib.sha256).hexdigest()})
+check("normal-size body still ok", st == 200)
+
+# 6. unknown path
+st, _ = post(payload, {"X-Hub-Signature-256": "sha256=" + sig})
+check("POST /gitea ok baseline", st == 200)
+
+relay.terminate(); relay.wait(timeout=5); hermes.shutdown()
+print()
+print(f"{'ALL PASSED' if not fails else 'FAILURES: ' + ', '.join(fails)}")
+sys.exit(1 if fails else 0)

services/dev/gitea-hermes-webhook-relay.nix

@@ -1,5 +1,28 @@
 { config, pkgs, ... }:
 
+# Gitea -> Hermes webhook relay.
+#
+# Why this exists at all, since Gitea could POST straight at Hermes's own
+# webhook port (8644, already tailnet-reachable — tailscale0 is a
+# trustedInterface): AUTH would work directly. Gitea's addDefaultHeaders()
+# signs every webhook type with `X-Hub-Signature-256: sha256=<hmac>`, the
+# exact GitHub scheme, and Hermes accepts that header on any route with no
+# per-route provider gating. What does NOT work directly is EVENT SELECTION.
+# Hermes reads the event name from `X-GitHub-Event`/`X-GitLab-Event`, then
+# the payload's `event_type`/`type` keys, then gives up and calls it
+# "unknown". Gitea sends `X-Gitea-Event` and no such payload key, so a direct
+# hook authenticates fine and then arrives as "unknown" forever — which makes
+# `hermes webhook subscribe --events ...` unable to select anything, i.e. the
+# "Hermes owns event policy" split this module is built around cannot exist
+# without something copying that one header.
+#
+# So that is all this does: verify the signature, copy X-Gitea-Event into
+# X-GitHub-Event, forward body and signature untouched. No re-signing, no
+# payload rewriting, no event/repo/action filtering.
+#
+# It binds 0.0.0.0 but gets no allowedTCPPorts entry, so it is reachable over
+# tailscale0 only — same posture as the Hermes dashboard on 9119.
+
 let
   relayScript = pkgs.writeText "gitea-hermes-webhook-relay.py" (
     builtins.readFile ./gitea-hermes-webhook-relay.py
@@ -7,7 +30,7 @@ let
 in
 {
   systemd.services.gitea-hermes-webhook-relay = {
-    description = "Normalize Gitea PR webhooks for Hermes Agent";
+    description = "Relay Gitea webhooks to Hermes with a Hermes-readable event header";
     wantedBy = [ "multi-user.target" ];
     wants = [ "network-online.target" ];
     after = [
@@ -45,6 +68,18 @@ in
   # The relay forwards into a generic Hermes webhook subscription. Keep the
   # subscription declaratively present without putting event policy or prompt
   # text in this transport unit. Hermes owns interpretation and response policy.
+  #
+  # `--events` is deliberately omitted: an empty events list means "accept
+  # everything", and the selection is Hermes-side policy that darman can
+  # retune with `hermes webhook subscribe` at runtime without a redeploy.
+  # That only works because the relay supplies X-GitHub-Event — see the
+  # header comment above.
+  #
+  # The secret is read from the CONTAINER's environment ($GITEA_HERMES_
+  # WEBHOOK_SECRET, injected via sops.templates."hermes-agent.env"), which is
+  # why hosts/mars/secrets.nix restarts podman-hermes-agent BEFORE this unit
+  # on rotation — re-subscribing against a container still holding the old
+  # value would silently pin the stale secret.
   systemd.services.hermes-agent-webhook-route = {
     description = "Configure Hermes Gitea event webhook route";
     wantedBy = [ "multi-user.target" ];

services/dev/gitea-hermes-webhook-relay.py

@@ -1,5 +1,25 @@
 #!/usr/bin/env python3
-"""Relay authenticated Gitea webhook requests to Hermes Agent."""
+"""Relay authenticated Gitea webhook requests to Hermes Agent.
+
+This service exists for exactly one reason: Hermes derives the event name it
+matches a subscription's `events` filter against from `X-GitHub-Event` /
+`X-GitLab-Event`, falling back to the payload's `event_type`/`type` keys and
+then to the literal string "unknown" (gateway/platforms/webhook.py). Gitea
+never sends any of those — its event name rides `X-Gitea-Event`, and its
+payloads carry no `event_type`/`type` key — so a Gitea webhook pointed
+straight at Hermes authenticates fine but arrives as "unknown" forever, which
+makes `hermes webhook subscribe --events ...` unable to select anything.
+
+Everything else about a Gitea delivery already speaks Hermes natively:
+Gitea's addDefaultHeaders() signs EVERY webhook type with
+`X-Hub-Signature-256: sha256=<hmac-sha256(body)>`, byte-identical to GitHub's
+scheme, and Hermes accepts that header on any route with no per-route
+provider gating. So the body and the signature are forwarded untouched — this
+process re-signs nothing and rewrites no payload. It copies one header.
+
+It still verifies the signature itself rather than forwarding blindly, so an
+unauthenticated caller that reaches this port never reaches the agent.
+"""
 from __future__ import annotations
 
 import hashlib
@@ -25,6 +45,13 @@ CREDENTIAL_NAME = os.environ.get("WEBHOOK_CREDENTIAL_NAME", "webhook_secret")
 
 
 def load_secret() -> bytes:
+    """Read the shared secret, preferring systemd's credential store.
+
+    Both sources are stripped: the sops secret file usually ends in a newline,
+    while the value Gitea signs with comes from `$(cat ...)` in the
+    provisioning unit, which drops trailing newlines. Stripping here is what
+    keeps those two in agreement.
+    """
     credentials_dir = os.environ.get("CREDENTIALS_DIRECTORY")
     if credentials_dir:
         path = Path(credentials_dir) / CREDENTIAL_NAME
@@ -32,7 +59,7 @@ def load_secret() -> bytes:
             return path.read_bytes().strip()
     value = os.environ.get("GITEA_HERMES_WEBHOOK_SECRET", "")
     if value:
-        return value.encode()
+        return value.strip().encode()
     raise RuntimeError("webhook secret is not available")
 
 
@@ -40,6 +67,25 @@ def json_bytes(payload: dict) -> bytes:
     return json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode()
 
 
+def signature_matches(secret: bytes, body: bytes, headers) -> bool:
+    """Check the body against whichever signature header Gitea supplied.
+
+    Gitea sends both on every delivery: `X-Hub-Signature-256` (GitHub format,
+    `sha256=` prefixed) and `X-Gitea-Signature` (bare lowercase hex). Either is
+    accepted so the relay keeps working if one is ever dropped upstream.
+    """
+    expected = hmac.new(secret, body, hashlib.sha256).hexdigest()
+    for header in ("X-Hub-Signature-256", "X-Gitea-Signature"):
+        provided = headers.get(header, "").strip()
+        if not provided:
+            continue
+        if provided.startswith("sha256="):
+            provided = provided.removeprefix("sha256=")
+        if hmac.compare_digest(provided, expected):
+            return True
+    return False
+
+
 class Handler(BaseHTTPRequestHandler):
     server_version = "gitea-hermes-relay/1.0"
 
@@ -65,12 +111,19 @@ class Handler(BaseHTTPRequestHandler):
             self.send_json(404, {"status": "not_found"})
             return
 
+        raw_length = self.headers.get("Content-Length")
+        if raw_length is None:
+            self.send_json(411, {"status": "length_required"})
+            return
         try:
-            content_length = int(self.headers.get("Content-Length", "-1"))
+            content_length = int(raw_length)
         except ValueError:
             self.send_json(400, {"status": "invalid_content_length"})
             return
-        if content_length < 0 or content_length > MAX_BODY_BYTES:
+        if content_length < 0:
+            self.send_json(400, {"status": "invalid_content_length"})
+            return
+        if content_length > MAX_BODY_BYTES:
             self.send_json(413, {"status": "payload_too_large"})
             return
 
@@ -82,29 +135,34 @@ class Handler(BaseHTTPRequestHandler):
             self.send_json(503, {"status": "relay_not_ready"})
             return
 
-        provided = self.headers.get("X-Gitea-Signature", "").strip()
-        if provided.startswith("sha256="):
-            provided = provided.removeprefix("sha256=")
-        expected = hmac.new(secret, body, hashlib.sha256).hexdigest()
-        if not provided or not hmac.compare_digest(provided, expected):
+        if not signature_matches(secret, body, self.headers):
             LOG.warning("rejected webhook with invalid signature")
             self.send_json(401, {"status": "invalid_signature"})
             return
 
-        # Keep the incoming body unchanged. Event interpretation and policy
-        # belong to Hermes, not to this transport service.
-        forwarded_body = body
-        forwarded_signature = hmac.new(
-            secret, forwarded_body, hashlib.sha256
-        ).hexdigest()
         gitea_event = self.headers.get("X-Gitea-Event", "")
         gitea_event_type = self.headers.get("X-Gitea-Event-Type", "")
         delivery_id = self.headers.get("X-Gitea-Delivery", "")
+        hub_signature = self.headers.get("X-Hub-Signature-256", "")
+
+        # The body is forwarded byte-for-byte, so Gitea's own signature stays
+        # valid — nothing is re-signed here. If Gitea ever stops sending the
+        # GitHub-format header, sign the unchanged body ourselves so Hermes
+        # still has something its GitHub branch can verify.
+        if not hub_signature:
+            hub_signature = "sha256=" + hmac.new(secret, body, hashlib.sha256).hexdigest()
+
         forwarded_headers = {
             "Content-Type": "application/json",
-            "X-Webhook-Signature": forwarded_signature,
+            "X-Hub-Signature-256": hub_signature,
         }
+        # The one transformation this service performs. Gitea's event names are
+        # passed through verbatim rather than mapped onto GitHub's vocabulary:
+        # Hermes only string-matches them against the subscription's `events`
+        # list, and Gitea has events (pull_request_comment, pull_request_sync,
+        # pull_request_review_approved, ...) with no GitHub equivalent to map to.
         if gitea_event:
+            forwarded_headers["X-GitHub-Event"] = gitea_event
             forwarded_headers["X-Gitea-Event"] = gitea_event
         if gitea_event_type:
             forwarded_headers["X-Gitea-Event-Type"] = gitea_event_type
@@ -114,27 +172,22 @@ class Handler(BaseHTTPRequestHandler):
 
         request = Request(
             HERMES_URL,
-            data=forwarded_body,
+            data=body,
             headers=forwarded_headers,
             method="POST",
         )
         try:
             with urlopen(request, timeout=15) as response:
                 response.read()
-                status = response.status
         except HTTPError as exc:
             LOG.error("Hermes returned HTTP %s", exc.code)
-            self.send_json(502, {"status": "hermes_error"})
+            self.send_json(502, {"status": "hermes_error", "http_status": exc.code})
             return
         except (URLError, TimeoutError, OSError) as exc:
             LOG.error("failed to forward webhook to Hermes: %s", exc)
             self.send_json(502, {"status": "hermes_unreachable"})
             return
 
-        if status < 200 or status >= 300:
-            self.send_json(502, {"status": "hermes_error", "http_status": status})
-            return
-
         LOG.info(
             "forwarded Gitea event=%s delivery=%s",
             gitea_event or gitea_event_type or "unknown",

services/dev/gitea.nix

@@ -321,6 +321,20 @@ in
       secret="$(cat "$SECRET_FILE")"
       auth=(-H "Authorization: token $admin_token")
       target="http://mars.orbit.sol:8645/gitea"
+
+      # Same readiness gate as gitea-ci-bot-provision / gitea-luna-provision
+      # above: After=gitea.service only means the process started, not that it
+      # is serving HTTP yet. Without this the first curl below fails under
+      # `set -e`, and a Type=oneshot with no Restart= stays failed — leaving
+      # the webhook silently unregistered until someone restarts the unit.
+      for _ in $(seq 1 30); do
+        curl -fs "$api/version" >/dev/null 2>&1 && break
+        sleep 1
+      done
+
+      # The secret goes to curl on stdin (--data @-), never in argv: this unit
+      # runs as the gitea user on a multi-user box, and a request body passed
+      # with -d is world-readable in /proc/<pid>/cmdline for its lifetime.
       body="$(jq -n --arg url "$target" --arg secret "$secret" \
         --argjson events '${builtins.toJSON giteaWebhookEvents}' \
         '{type: "gitea", config: {content_type: "json", url: $url, secret: $secret}, events: $events, active: true}')"
@@ -328,11 +342,11 @@ in
       hook_id="$(curl -fsS "''${auth[@]}" "$api/repos/darman/homelab/hooks" \
         | jq -r --arg url "$target" 'first(.[] | select(.type == "gitea" and .config.url == $url)) | .id // empty')"
       if [ -n "$hook_id" ]; then
-        curl -fsS "''${auth[@]}" -H 'Content-Type: application/json' \
-          -X PATCH "$api/repos/darman/homelab/hooks/$hook_id" -d "$body" >/dev/null
+        printf '%s' "$body" | curl -fsS "''${auth[@]}" -H 'Content-Type: application/json' \
+          -X PATCH "$api/repos/darman/homelab/hooks/$hook_id" --data @- >/dev/null
       else
-        curl -fsS "''${auth[@]}" -H 'Content-Type: application/json' \
-          -X POST "$api/repos/darman/homelab/hooks" -d "$body" >/dev/null
+        printf '%s' "$body" | curl -fsS "''${auth[@]}" -H 'Content-Type: application/json' \
+          -X POST "$api/repos/darman/homelab/hooks" --data @- >/dev/null
       fi
     '';
   };