From ea9be6fb8ad761533b0fc402c0443c1fc633d7ae Mon Sep 17 00:00:00 2001 From: luna Date: Sun, 23 Aug 2026 00:55:07 +0000 Subject: [PATCH 1/6] mars: add VictoriaMetrics monitoring --- hosts/mars/configuration.nix | 1 + services/monitoring/victoriametrics.nix | 58 +++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 services/monitoring/victoriametrics.nix diff --git a/hosts/mars/configuration.nix b/hosts/mars/configuration.nix index 13a0d3f..5379921 100644 --- a/hosts/mars/configuration.nix +++ b/hosts/mars/configuration.nix @@ -12,6 +12,7 @@ ../../services/containers.nix ../../services/vpn/tailscale.nix ../../services/monitoring/node-exporter.nix + ../../services/monitoring/victoriametrics.nix ]; networking.hostName = "mars"; diff --git a/services/monitoring/victoriametrics.nix b/services/monitoring/victoriametrics.nix new file mode 100644 index 0000000..6e377d5 --- /dev/null +++ b/services/monitoring/victoriametrics.nix @@ -0,0 +1,58 @@ +{ ... }: + +# VictoriaMetrics single-node store for the homelab dashboard. It listens on +# all interfaces, but tailscale.nix makes tailscale0 the only trusted ingress; +# the host firewall therefore keeps :8428 off the LAN and public interfaces. +# +# The scrape targets are the node_exporter instances enabled by +# services/monitoring/node-exporter.nix on every real host. MagicDNS names use +# the tailnet's orbit.sol suffix (see services/vpn/headscale.nix). +{ + services.victoriametrics = { + enable = true; + retentionPeriod = "30d"; + listenAddress = ":8428"; + + prometheusConfig = { + global.scrape_interval = "60s"; + + scrape_configs = [ + { + job_name = "node-exporter"; + static_configs = [ + { + targets = [ "127.0.0.1:9100" ]; + labels.host = "mars"; + } + { + targets = [ "jupiter.orbit.sol:9100" ]; + labels.host = "jupiter"; + } + { + targets = [ "neptun.orbit.sol:9100" ]; + labels.host = "neptun"; + } + { + targets = [ "mercury.orbit.sol:9100" ]; + labels.host = "mercury"; + } + ]; + } + { + job_name = "victoriametrics"; + static_configs = [ + { + targets = [ "127.0.0.1:8428" ]; + labels.host = "mars"; + } + ]; + } + ]; + }; + }; + + # Start after Tailscale has had a chance to establish MagicDNS. This is only + # ordering, not a hard dependency: VictoriaMetrics still starts locally if + # another host or the tailnet is temporarily unavailable. + systemd.services.victoriametrics.after = [ "tailscaled-autoconnect.service" ]; +} From 082cbaff2a4a120b001cd8e92182faed4640127b Mon Sep 17 00:00:00 2001 From: luna Date: Mon, 24 Aug 2026 01:12:50 +0000 Subject: [PATCH 2/6] mars: reduce VictoriaMetrics retention to 15 days --- services/monitoring/victoriametrics.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/monitoring/victoriametrics.nix b/services/monitoring/victoriametrics.nix index 6e377d5..937705a 100644 --- a/services/monitoring/victoriametrics.nix +++ b/services/monitoring/victoriametrics.nix @@ -10,7 +10,7 @@ { services.victoriametrics = { enable = true; - retentionPeriod = "30d"; + retentionPeriod = "15d"; listenAddress = ":8428"; prometheusConfig = { From 5764e6c64412fc73a76152f68bf74cae4d189503 Mon Sep 17 00:00:00 2001 From: luna Date: Mon, 24 Aug 2026 01:30:37 +0000 Subject: [PATCH 3/6] mars: tune VictoriaMetrics scrape targets --- services/monitoring/victoriametrics.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/monitoring/victoriametrics.nix b/services/monitoring/victoriametrics.nix index 937705a..97665c7 100644 --- a/services/monitoring/victoriametrics.nix +++ b/services/monitoring/victoriametrics.nix @@ -14,7 +14,7 @@ listenAddress = ":8428"; prometheusConfig = { - global.scrape_interval = "60s"; + global.scrape_interval = "5s"; scrape_configs = [ { @@ -36,6 +36,10 @@ targets = [ "mercury.orbit.sol:9100" ]; labels.host = "mercury"; } + { + targets = [ "terra.orbit.sol:9100" ]; + labels.host = "terra"; + } ]; } { From dfe8504402b887a2457be33f32b66facc0974b8e Mon Sep 17 00:00:00 2001 From: luna Date: Mon, 24 Aug 2026 01:42:07 +0000 Subject: [PATCH 4/6] monitoring: move VictoriaMetrics to Jupiter --- hosts/jupiter/configuration.nix | 1 + hosts/mars/configuration.nix | 1 - services/monitoring/victoriametrics.nix | 8 ++++---- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hosts/jupiter/configuration.nix b/hosts/jupiter/configuration.nix index ba947f7..98c1861 100644 --- a/hosts/jupiter/configuration.nix +++ b/hosts/jupiter/configuration.nix @@ -14,6 +14,7 @@ ../../services/network/caddy.nix ../../services/vpn/tailscale.nix ../../services/monitoring/node-exporter.nix + ../../services/monitoring/victoriametrics.nix ../../services/media/jellyfin.nix ../../services/media/sabnzbd.nix ../../services/media/prowlarr.nix diff --git a/hosts/mars/configuration.nix b/hosts/mars/configuration.nix index 5379921..13a0d3f 100644 --- a/hosts/mars/configuration.nix +++ b/hosts/mars/configuration.nix @@ -12,7 +12,6 @@ ../../services/containers.nix ../../services/vpn/tailscale.nix ../../services/monitoring/node-exporter.nix - ../../services/monitoring/victoriametrics.nix ]; networking.hostName = "mars"; diff --git a/services/monitoring/victoriametrics.nix b/services/monitoring/victoriametrics.nix index 97665c7..f8a175f 100644 --- a/services/monitoring/victoriametrics.nix +++ b/services/monitoring/victoriametrics.nix @@ -1,7 +1,7 @@ { ... }: -# VictoriaMetrics single-node store for the homelab dashboard. It listens on -# all interfaces, but tailscale.nix makes tailscale0 the only trusted ingress; +# VictoriaMetrics single-node store for the homelab dashboard on Jupiter. It +# listens on all interfaces, but tailscale.nix makes tailscale0 the only trusted ingress; # the host firewall therefore keeps :8428 off the LAN and public interfaces. # # The scrape targets are the node_exporter instances enabled by @@ -22,7 +22,7 @@ static_configs = [ { targets = [ "127.0.0.1:9100" ]; - labels.host = "mars"; + labels.host = "jupiter"; } { targets = [ "jupiter.orbit.sol:9100" ]; @@ -47,7 +47,7 @@ static_configs = [ { targets = [ "127.0.0.1:8428" ]; - labels.host = "mars"; + labels.host = "jupiter"; } ]; } From b6aec1e307c8c19a9e7b38e9eb18e57e0a9b9616 Mon Sep 17 00:00:00 2001 From: luna Date: Mon, 24 Aug 2026 01:45:00 +0000 Subject: [PATCH 5/6] monitoring: scrape mars node exporter --- services/monitoring/victoriametrics.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/monitoring/victoriametrics.nix b/services/monitoring/victoriametrics.nix index f8a175f..eec749e 100644 --- a/services/monitoring/victoriametrics.nix +++ b/services/monitoring/victoriametrics.nix @@ -25,8 +25,8 @@ labels.host = "jupiter"; } { - targets = [ "jupiter.orbit.sol:9100" ]; - labels.host = "jupiter"; + targets = [ "mars.orbit.sol:9100" ]; + labels.host = "mars"; } { targets = [ "neptun.orbit.sol:9100" ]; From eec17b77df1f3e305d5f1b41d957a350e3502cea Mon Sep 17 00:00:00 2001 From: luna Date: Mon, 24 Aug 2026 02:03:51 +0000 Subject: [PATCH 6/6] monitoring: move metrics state to the data array --- services/monitoring/victoriametrics.nix | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/services/monitoring/victoriametrics.nix b/services/monitoring/victoriametrics.nix index eec749e..e6c6aeb 100644 --- a/services/monitoring/victoriametrics.nix +++ b/services/monitoring/victoriametrics.nix @@ -32,16 +32,23 @@ targets = [ "neptun.orbit.sol:9100" ]; labels.host = "neptun"; } - { - targets = [ "mercury.orbit.sol:9100" ]; - labels.host = "mercury"; - } { targets = [ "terra.orbit.sol:9100" ]; labels.host = "terra"; } ]; } + { + job_name = "node-exporter-mercury"; + scrape_interval = "15s"; + scrape_timeout = "10s"; + static_configs = [ + { + targets = [ "mercury.orbit.sol:9100" ]; + labels.host = "mercury"; + } + ]; + } { job_name = "victoriametrics"; static_configs = [ @@ -59,4 +66,14 @@ # ordering, not a hard dependency: VictoriaMetrics still starts locally if # another host or the tailnet is temporarily unavailable. systemd.services.victoriametrics.after = [ "tailscaled-autoconnect.service" ]; + + fileSystems."/var/lib/private/victoriametrics" = { + device = "/mnt/data/AppData/victoriametrics"; + fsType = "none"; + options = [ "bind" "nofail" ]; + }; + + # The service path is under /var/lib/private, so systemd would otherwise + # derive its mount dependency from the eMMC-backed path alone. + systemd.services.victoriametrics.unitConfig.RequiresMountsFor = [ "/mnt/data" ]; }