diff --git a/services/monitoring/victoriametrics.nix b/services/monitoring/victoriametrics.nix index e6c6aeb..c38fb75 100644 --- a/services/monitoring/victoriametrics.nix +++ b/services/monitoring/victoriametrics.nix @@ -15,6 +15,11 @@ prometheusConfig = { global.scrape_interval = "5s"; + # Explicit, and equal to the interval on purpose. The Prometheus default + # is 10s, and VictoriaMetrics silently clamps scrape_timeout down to + # scrape_interval rather than erroring — so leaving it implicit means the + # config says 10s while the scraper uses 5s. Say what actually happens. + global.scrape_timeout = "5s"; scrape_configs = [ { @@ -38,6 +43,15 @@ } ]; } + # mercury is a Pi scraped over the tailnet, so it gets its own job at a + # slower cadence: at the 5s global it would time out (see above) and + # the series would show gaps rather than late samples. + # + # A separate cadence REQUIRES a separate job — scrape_interval is a + # per-job setting and job_name has to be unique — which means mercury's + # `job` label differs from every other host's. Select on `host` (set on + # every target below) rather than job="node-exporter" in dashboards and + # alerts, or mercury drops out of them silently. { job_name = "node-exporter-mercury"; scrape_interval = "15s"; @@ -67,12 +81,35 @@ # another host or the tailnet is temporarily unavailable. systemd.services.victoriametrics.after = [ "tailscaled-autoconnect.service" ]; + # Keep the TSDB off jupiter's 29G eMMC. The module hardcodes + # -storageDataPath=/var/lib/ and runs DynamicUser, so without this + # the data lands on the OS disk — a continuous small-write workload aimed at + # the one disk here with no headroom and finite write endurance. Same + # bind-onto-/var/lib/private pattern as prowlarr.nix and seerr.nix; see + # prowlarr.nix for why the mount targets the private path and not the public + # /var/lib/victoriametrics. + # + # `nofail` is NOT optional — again see prowlarr.nix: without it this bind is + # RequiredBy local-fs.target, so an unassembled array drops jupiter into an + # emergency shell that a headless box cannot be rescued from. fileSystems."/var/lib/private/victoriametrics" = { device = "/mnt/data/AppData/victoriametrics"; fsType = "none"; options = [ "bind" "nofail" ]; }; + # The bind above needs its SOURCE to exist or the mount fails — and because + # it is `nofail` that failure is quiet: RequiresMountsFor below is satisfied + # by /mnt/data itself, so VictoriaMetrics would start regardless and write to + # the eMMC, which is the exact thing the bind exists to prevent. prowlarr.nix + # gets away without this only because its directory predates the module + # (migrated from ZimaOS). This is a fresh service, so it creates its own, + # same as seerr.nix. 0755 darman:users matches the other AppData dirs, which + # matters because /mnt/data/AppData itself is drwx--x--- darman:users. + systemd.tmpfiles.rules = [ + "d /mnt/data/AppData/victoriametrics 0755 darman users -" + ]; + # The service path is under /var/lib/private, so systemd would otherwise # derive its mount dependency from the eMMC-backed path alone. systemd.services.victoriametrics.unitConfig.RequiresMountsFor = [ "/mnt/data" ];