add node_exporter host vitals + quickshell HUD
Prometheus node_exporter enabled on every host, plus a quickshell widget (SUPER+CTRL+V on terra) to view live CPU/mem/disk/net/uptime without a separate dashboard. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011FHr5ug9pu8q4XPrRkFnzJ
This commit is contained in:
Claude Sonnet 5
@@ -0,0 +1,321 @@
|
||||
pragma ComponentBehavior: Bound
|
||||
|
||||
import Quickshell
|
||||
import Quickshell.Io
|
||||
import QtQuick
|
||||
|
||||
// Vitals source: scrapes this host's own node_exporter over loopback.
|
||||
//
|
||||
// The exporter comes from services/monitoring/node-exporter.nix — the same
|
||||
// collection layer the homelab dashboard scrapes over the tailnet — so what
|
||||
// this panel shows and what the dashboard graphs can never drift apart.
|
||||
// :9100 is firewalled to tailscale0 for everyone else, but loopback is always
|
||||
// reachable, so no extra hole is opened for this.
|
||||
//
|
||||
// CPU busy and network throughput are counter DELTAS: the first sample after
|
||||
// `active` flips on only primes the counters, and `ratesReady` stays false
|
||||
// until a second one gives them an interval to divide by.
|
||||
Scope {
|
||||
id: root
|
||||
|
||||
// Poll only while the panel is on screen — no cost when hidden.
|
||||
property bool active: false
|
||||
property int interval: 2000
|
||||
property int port: 9100
|
||||
|
||||
// ── Readings ───────────────────────────────────────────────────────────
|
||||
property bool ready: false // one successful scrape happened
|
||||
property bool ratesReady: false // two — so deltas are meaningful
|
||||
property bool failed: false // exporter unreachable / no metrics
|
||||
|
||||
property string host: ""
|
||||
property real uptime: 0 // seconds
|
||||
|
||||
property real cpu: 0 // 0..1 busy
|
||||
property int cpuThreads: 0
|
||||
property real load1: 0
|
||||
property real load5: 0
|
||||
property real load15: 0
|
||||
property real cpuTemp: NaN // °C
|
||||
|
||||
property real memUsed: 0 // bytes
|
||||
property real memTotal: 0
|
||||
|
||||
property real gpuTemp: NaN // edge
|
||||
property real gpuHotspot: NaN // junction
|
||||
property real gpuPower: NaN // W
|
||||
property real gpuPowerCap: NaN
|
||||
property real gpuClock: NaN // MHz, shader clock
|
||||
property real nvmeTemp: NaN
|
||||
|
||||
property var disks: [] // [{ mount, used, size }]
|
||||
|
||||
property string netIface: ""
|
||||
property real netRx: 0 // bytes/s
|
||||
property real netTx: 0
|
||||
|
||||
// ── Delta state ────────────────────────────────────────────────────────
|
||||
property double _prevTime: 0
|
||||
property double _prevIdle: 0
|
||||
property double _prevTotal: 0
|
||||
property double _prevRx: 0
|
||||
property double _prevTx: 0
|
||||
|
||||
onActiveChanged: {
|
||||
if (!root.active) {
|
||||
// Drop the counters so reopening the panel doesn't average a rate
|
||||
// across however long it sat hidden.
|
||||
root._prevTime = 0;
|
||||
root.ratesReady = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Local filesystems worth showing; everything else (tmpfs, ramfs, and the
|
||||
// cifs mount of jupiter, which is another host's disk, not terra's) is out.
|
||||
readonly property var _fsTypes: ["ext4", "btrfs", "xfs", "vfat", "f2fs"]
|
||||
|
||||
// Virtual/overlay interfaces that would drown out the real NIC.
|
||||
readonly property var _skipIface: ["lo", "docker", "podman", "veth", "br-", "virbr", "cni"]
|
||||
|
||||
function _label(s, key) {
|
||||
const m = s.match(new RegExp(key + '="([^"]*)"'));
|
||||
return m ? m[1] : "";
|
||||
}
|
||||
|
||||
function _parse(text) {
|
||||
const lines = text.split("\n");
|
||||
|
||||
let idle = 0, total = 0;
|
||||
const seenCpu = {};
|
||||
let memTotal = 0, memAvail = 0, bootTime = 0;
|
||||
let l1 = 0, l5 = 0, l15 = 0;
|
||||
let host = "";
|
||||
|
||||
// hwmon is keyed by an opaque chip id; node_hwmon_chip_names maps it to
|
||||
// the driver (amdgpu/k10temp/nvme) but is NOT guaranteed to be emitted
|
||||
// before the readings, so collect raw and resolve after the loop.
|
||||
const chipName = {};
|
||||
const tempRaw = {}, powerRaw = {}, freqRaw = {};
|
||||
|
||||
const fsSize = {}, fsAvail = {};
|
||||
const rxByIface = {}, txByIface = {};
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
if (line.length === 0 || line.charCodeAt(0) === 35 /* '#' */)
|
||||
continue;
|
||||
|
||||
const sp = line.lastIndexOf(" ");
|
||||
if (sp < 0)
|
||||
continue;
|
||||
|
||||
const key = line.substring(0, sp);
|
||||
const val = parseFloat(line.substring(sp + 1));
|
||||
if (!isFinite(val))
|
||||
continue;
|
||||
|
||||
if (key.startsWith("node_cpu_seconds_total{")) {
|
||||
total += val;
|
||||
const mode = root._label(key, "mode");
|
||||
if (mode === "idle")
|
||||
idle += val;
|
||||
seenCpu[root._label(key, "cpu")] = true;
|
||||
} else if (key === "node_memory_MemTotal_bytes") {
|
||||
memTotal = val;
|
||||
} else if (key === "node_memory_MemAvailable_bytes") {
|
||||
memAvail = val;
|
||||
} else if (key === "node_load1") {
|
||||
l1 = val;
|
||||
} else if (key === "node_load5") {
|
||||
l5 = val;
|
||||
} else if (key === "node_load15") {
|
||||
l15 = val;
|
||||
} else if (key === "node_boot_time_seconds") {
|
||||
bootTime = val;
|
||||
} else if (key.startsWith("node_uname_info{")) {
|
||||
host = root._label(key, "nodename");
|
||||
} else if (key.startsWith("node_hwmon_chip_names{")) {
|
||||
chipName[root._label(key, "chip")] = root._label(key, "chip_name");
|
||||
} else if (key.startsWith("node_hwmon_temp_celsius{")) {
|
||||
const c = root._label(key, "chip");
|
||||
(tempRaw[c] = tempRaw[c] || {})[root._label(key, "sensor")] = val;
|
||||
} else if (key.startsWith("node_hwmon_power_average_watt{")) {
|
||||
powerRaw[root._label(key, "chip")] = val;
|
||||
} else if (key.startsWith("node_hwmon_power_cap_watt{")) {
|
||||
const c = root._label(key, "chip");
|
||||
(freqRaw[c] = freqRaw[c] || {})["cap"] = val;
|
||||
} else if (key.startsWith("node_hwmon_freq_freq_mhz{")) {
|
||||
const c = root._label(key, "chip");
|
||||
(freqRaw[c] = freqRaw[c] || {})[root._label(key, "sensor")] = val;
|
||||
} else if (key.startsWith("node_filesystem_size_bytes{")) {
|
||||
const mp = root._label(key, "mountpoint");
|
||||
if (root._fsTypes.indexOf(root._label(key, "fstype")) >= 0)
|
||||
fsSize[mp] = val;
|
||||
} else if (key.startsWith("node_filesystem_avail_bytes{")) {
|
||||
fsAvail[root._label(key, "mountpoint")] = val;
|
||||
} else if (key.startsWith("node_network_receive_bytes_total{")) {
|
||||
rxByIface[root._label(key, "device")] = val;
|
||||
} else if (key.startsWith("node_network_transmit_bytes_total{")) {
|
||||
txByIface[root._label(key, "device")] = val;
|
||||
}
|
||||
}
|
||||
|
||||
if (memTotal <= 0) {
|
||||
// Reachable but not serving node metrics — treat as a failure
|
||||
// rather than rendering a panel full of zeroes.
|
||||
root.failed = true;
|
||||
return;
|
||||
}
|
||||
|
||||
// Resolve hwmon chips by driver name.
|
||||
const byDriver = {};
|
||||
for (const chip in tempRaw)
|
||||
byDriver[chipName[chip] || chip] = { temp: tempRaw[chip], chip: chip };
|
||||
|
||||
const cpuChip = byDriver["k10temp"] || byDriver["coretemp"] || byDriver["zenpower"];
|
||||
root.cpuTemp = cpuChip ? (cpuChip.temp["temp1"] ?? NaN) : NaN;
|
||||
|
||||
const gpu = byDriver["amdgpu"];
|
||||
if (gpu) {
|
||||
root.gpuTemp = gpu.temp["temp1"] ?? NaN; // edge
|
||||
root.gpuHotspot = gpu.temp["temp2"] ?? NaN; // junction
|
||||
root.gpuPower = powerRaw[gpu.chip] ?? NaN;
|
||||
root.gpuPowerCap = freqRaw[gpu.chip] ? (freqRaw[gpu.chip]["cap"] ?? NaN) : NaN;
|
||||
root.gpuClock = freqRaw[gpu.chip] ? (freqRaw[gpu.chip]["sclk"] ?? NaN) : NaN;
|
||||
} else {
|
||||
root.gpuTemp = NaN;
|
||||
root.gpuHotspot = NaN;
|
||||
root.gpuPower = NaN;
|
||||
root.gpuPowerCap = NaN;
|
||||
root.gpuClock = NaN;
|
||||
}
|
||||
|
||||
const nvme = byDriver["nvme"];
|
||||
root.nvmeTemp = nvme ? (nvme.temp["temp1"] ?? NaN) : NaN;
|
||||
|
||||
// Filesystems. /nix/store is the same device as / on every host here,
|
||||
// so listing it twice would just be noise.
|
||||
const mounts = [];
|
||||
for (const mp in fsSize) {
|
||||
if (mp === "/nix/store")
|
||||
continue;
|
||||
const size = fsSize[mp];
|
||||
const avail = fsAvail[mp];
|
||||
if (!(size > 0) || avail === undefined)
|
||||
continue;
|
||||
mounts.push({ mount: mp, used: size - avail, size: size });
|
||||
}
|
||||
mounts.sort((a, b) => a.mount === "/" ? -1 : b.mount === "/" ? 1 : a.mount.localeCompare(b.mount));
|
||||
root.disks = mounts;
|
||||
|
||||
// Busiest real interface.
|
||||
let iface = "", best = -1;
|
||||
for (const dev in rxByIface) {
|
||||
let skip = false;
|
||||
for (let s = 0; s < root._skipIface.length; s++) {
|
||||
if (dev === root._skipIface[s] || dev.indexOf(root._skipIface[s]) === 0) {
|
||||
skip = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (skip)
|
||||
continue;
|
||||
if (rxByIface[dev] > best) {
|
||||
best = rxByIface[dev];
|
||||
iface = dev;
|
||||
}
|
||||
}
|
||||
root.netIface = iface;
|
||||
|
||||
const rx = iface ? (rxByIface[iface] ?? 0) : 0;
|
||||
const tx = iface ? (txByIface[iface] ?? 0) : 0;
|
||||
|
||||
// Rates.
|
||||
const now = Date.now() / 1000;
|
||||
const dt = now - root._prevTime;
|
||||
if (root._prevTime > 0 && dt > 0) {
|
||||
const dTotal = total - root._prevTotal;
|
||||
if (dTotal > 0)
|
||||
root.cpu = Math.max(0, Math.min(1, 1 - (idle - root._prevIdle) / dTotal));
|
||||
root.netRx = Math.max(0, (rx - root._prevRx) / dt);
|
||||
root.netTx = Math.max(0, (tx - root._prevTx) / dt);
|
||||
root.ratesReady = true;
|
||||
}
|
||||
root._prevTime = now;
|
||||
root._prevIdle = idle;
|
||||
root._prevTotal = total;
|
||||
root._prevRx = rx;
|
||||
root._prevTx = tx;
|
||||
|
||||
root.cpuThreads = Object.keys(seenCpu).length;
|
||||
root.memTotal = memTotal;
|
||||
root.memUsed = memTotal - memAvail;
|
||||
root.load1 = l1;
|
||||
root.load5 = l5;
|
||||
root.load15 = l15;
|
||||
root.host = host;
|
||||
root.uptime = bootTime > 0 ? (Date.now() / 1000 - bootTime) : 0;
|
||||
|
||||
root.failed = false;
|
||||
root.ready = true;
|
||||
}
|
||||
|
||||
// ── Formatting helpers, shared with the panel ──────────────────────────
|
||||
function fmtBytes(b) {
|
||||
if (!isFinite(b))
|
||||
return "--";
|
||||
const u = ["B", "K", "M", "G", "T"];
|
||||
let i = 0;
|
||||
while (b >= 1024 && i < u.length - 1) {
|
||||
b /= 1024;
|
||||
i++;
|
||||
}
|
||||
return (b >= 100 || i === 0 ? Math.round(b) : b.toFixed(1)) + u[i];
|
||||
}
|
||||
|
||||
function fmtRate(b) {
|
||||
return root.ratesReady ? root.fmtBytes(b) + "/S" : "--";
|
||||
}
|
||||
|
||||
function fmtTemp(c) {
|
||||
return isFinite(c) ? Math.round(c) + "°" : "--";
|
||||
}
|
||||
|
||||
function fmtUptime(s) {
|
||||
if (!(s > 0))
|
||||
return "--";
|
||||
const d = Math.floor(s / 86400);
|
||||
const h = Math.floor(s % 86400 / 3600);
|
||||
const m = Math.floor(s % 3600 / 60);
|
||||
return d > 0 ? d + "D " + h + "H" : h > 0 ? h + "H " + m + "M" : m + "M";
|
||||
}
|
||||
|
||||
// ── Polling ────────────────────────────────────────────────────────────
|
||||
Process {
|
||||
id: scrape
|
||||
|
||||
// Filtered at the source: the full endpoint is ~1400 lines and only
|
||||
// these families are drawn.
|
||||
command: ["sh", "-c", "curl -s --max-time 2 http://127.0.0.1:" + root.port + "/metrics | grep -E '^node_(cpu_seconds_total|memory_MemTotal_bytes|memory_MemAvailable_bytes|load1|load5|load15|boot_time_seconds|uname_info|hwmon_chip_names|hwmon_temp_celsius|hwmon_power_average_watt|hwmon_power_cap_watt|hwmon_freq_freq_mhz|filesystem_avail_bytes|filesystem_size_bytes|network_receive_bytes_total|network_transmit_bytes_total)[ {]'"]
|
||||
|
||||
stdout: StdioCollector {
|
||||
onStreamFinished: {
|
||||
if (this.text.length === 0)
|
||||
root.failed = true;
|
||||
else
|
||||
root._parse(this.text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Timer {
|
||||
interval: root.interval
|
||||
running: root.active
|
||||
repeat: true
|
||||
triggeredOnStart: true
|
||||
onTriggered: {
|
||||
if (!scrape.running)
|
||||
scrape.running = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user