homelab/services/monitoring/victoriametrics.nix

{ ... }:

# VictoriaMetrics single-node store for the homelab dashboard on Jupiter. It
# listens on all interfaces, but tailscale.nix makes tailscale0 the only trusted ingress;
# the host firewall therefore keeps :8428 off the LAN and public interfaces.
#
# The scrape targets are the node_exporter instances enabled by
# services/monitoring/node-exporter.nix on every real host. MagicDNS names use
# the tailnet's orbit.sol suffix (see services/vpn/headscale.nix).
{
  services.victoriametrics = {
    enable = true;
    retentionPeriod = "15d";
    listenAddress = ":8428";

    prometheusConfig = {
      global.scrape_interval = "5s";
      # Explicit and equal to the interval on purpose: VictoriaMetrics silently
      # clamps scrape_timeout down to scrape_interval, so leaving the Prometheus
      # default (10s) here would misstate what actually happens.
      global.scrape_timeout = "5s";

      scrape_configs = [
        {
          job_name = "node-exporter";
          static_configs = [
            {
              targets = [ "127.0.0.1:9100" ];
              labels.host = "jupiter";
            }
            {
              targets = [ "mars.orbit.sol:9100" ];
              labels.host = "mars";
            }
            {
              targets = [ "neptun.orbit.sol:9100" ];
              labels.host = "neptun";
            }
            {
              targets = [ "terra.orbit.sol:9100" ];
              labels.host = "terra";
            }
          ];
        }
        # mercury is a Pi scraped over the tailnet, so it gets its own job at a
        # slower cadence to avoid timing out at the 5s global. A separate cadence
        # requires a separate job (scrape_interval is per-job), so mercury's
        # `job` label differs from every other host's — select on `host` in
        # dashboards/alerts, not job="node-exporter", or mercury drops out silently.
        {
          job_name = "node-exporter-mercury";
          scrape_interval = "15s";
          scrape_timeout = "10s";
          static_configs = [
            {
              targets = [ "mercury.orbit.sol:9100" ];
              labels.host = "mercury";
            }
          ];
        }
        {
          job_name = "victoriametrics";
          static_configs = [
            {
              targets = [ "127.0.0.1:8428" ];
              labels.host = "jupiter";
            }
          ];
        }
      ];
    };
  };

  # Start after Tailscale has had a chance to establish MagicDNS. This is only
  # ordering, not a hard dependency: VictoriaMetrics still starts locally if
  # another host or the tailnet is temporarily unavailable.
  systemd.services.victoriametrics.after = [ "tailscaled-autoconnect.service" ];

  # Keep the TSDB off jupiter's 29G eMMC: the module hardcodes
  # -storageDataPath=/var/lib/<stateDir> under DynamicUser, so without this bind
  # a continuous small-write workload lands on the one disk with no headroom.
  # Same /var/lib/private bind pattern as prowlarr.nix and seerr.nix — see
  # prowlarr.nix for why it targets the private path, and why `nofail` here is
  # not optional.
  fileSystems."/var/lib/private/victoriametrics" = {
    device = "/mnt/data/AppData/victoriametrics";
    fsType = "none";
    options = [ "bind" "nofail" ];
  };

  # The bind above needs its source dir to exist or it quietly fails (`nofail`)
  # and VictoriaMetrics falls through to writing the eMMC anyway — this is a
  # fresh service so, unlike prowlarr.nix's pre-existing dir, it must create its
  # own (same as seerr.nix). 0755 darman:users matches the other AppData dirs.
  systemd.tmpfiles.rules = [
    "d /mnt/data/AppData/victoriametrics 0755 darman users -"
  ];

  # The service path is under /var/lib/private, so systemd would otherwise
  # derive its mount dependency from the eMMC-backed path alone.
  systemd.services.victoriametrics.unitConfig.RequiresMountsFor = [ "/mnt/data" ];
}