Prometheus node_exporter enabled on every host, plus a quickshell widget (SUPER+CTRL+V on terra) to view live CPU/mem/disk/net/uptime without a separate dashboard. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011FHr5ug9pu8q4XPrRkFnzJ
322 lines
12 KiB
QML
322 lines
12 KiB
QML
pragma ComponentBehavior: Bound
|
|
|
|
import Quickshell
|
|
import Quickshell.Io
|
|
import QtQuick
|
|
|
|
// Vitals source: scrapes this host's own node_exporter over loopback.
|
|
//
|
|
// The exporter comes from services/monitoring/node-exporter.nix — the same
|
|
// collection layer the homelab dashboard scrapes over the tailnet — so what
|
|
// this panel shows and what the dashboard graphs can never drift apart.
|
|
// :9100 is firewalled to tailscale0 for everyone else, but loopback is always
|
|
// reachable, so no extra hole is opened for this.
|
|
//
|
|
// CPU busy and network throughput are counter DELTAS: the first sample after
|
|
// `active` flips on only primes the counters, and `ratesReady` stays false
|
|
// until a second one gives them an interval to divide by.
|
|
Scope {
|
|
id: root
|
|
|
|
// Poll only while the panel is on screen — no cost when hidden.
|
|
property bool active: false
|
|
property int interval: 2000
|
|
property int port: 9100
|
|
|
|
// ── Readings ───────────────────────────────────────────────────────────
|
|
property bool ready: false // one successful scrape happened
|
|
property bool ratesReady: false // two — so deltas are meaningful
|
|
property bool failed: false // exporter unreachable / no metrics
|
|
|
|
property string host: ""
|
|
property real uptime: 0 // seconds
|
|
|
|
property real cpu: 0 // 0..1 busy
|
|
property int cpuThreads: 0
|
|
property real load1: 0
|
|
property real load5: 0
|
|
property real load15: 0
|
|
property real cpuTemp: NaN // °C
|
|
|
|
property real memUsed: 0 // bytes
|
|
property real memTotal: 0
|
|
|
|
property real gpuTemp: NaN // edge
|
|
property real gpuHotspot: NaN // junction
|
|
property real gpuPower: NaN // W
|
|
property real gpuPowerCap: NaN
|
|
property real gpuClock: NaN // MHz, shader clock
|
|
property real nvmeTemp: NaN
|
|
|
|
property var disks: [] // [{ mount, used, size }]
|
|
|
|
property string netIface: ""
|
|
property real netRx: 0 // bytes/s
|
|
property real netTx: 0
|
|
|
|
// ── Delta state ────────────────────────────────────────────────────────
|
|
property double _prevTime: 0
|
|
property double _prevIdle: 0
|
|
property double _prevTotal: 0
|
|
property double _prevRx: 0
|
|
property double _prevTx: 0
|
|
|
|
onActiveChanged: {
|
|
if (!root.active) {
|
|
// Drop the counters so reopening the panel doesn't average a rate
|
|
// across however long it sat hidden.
|
|
root._prevTime = 0;
|
|
root.ratesReady = false;
|
|
}
|
|
}
|
|
|
|
// Local filesystems worth showing; everything else (tmpfs, ramfs, and the
|
|
// cifs mount of jupiter, which is another host's disk, not terra's) is out.
|
|
readonly property var _fsTypes: ["ext4", "btrfs", "xfs", "vfat", "f2fs"]
|
|
|
|
// Virtual/overlay interfaces that would drown out the real NIC.
|
|
readonly property var _skipIface: ["lo", "docker", "podman", "veth", "br-", "virbr", "cni"]
|
|
|
|
function _label(s, key) {
|
|
const m = s.match(new RegExp(key + '="([^"]*)"'));
|
|
return m ? m[1] : "";
|
|
}
|
|
|
|
function _parse(text) {
|
|
const lines = text.split("\n");
|
|
|
|
let idle = 0, total = 0;
|
|
const seenCpu = {};
|
|
let memTotal = 0, memAvail = 0, bootTime = 0;
|
|
let l1 = 0, l5 = 0, l15 = 0;
|
|
let host = "";
|
|
|
|
// hwmon is keyed by an opaque chip id; node_hwmon_chip_names maps it to
|
|
// the driver (amdgpu/k10temp/nvme) but is NOT guaranteed to be emitted
|
|
// before the readings, so collect raw and resolve after the loop.
|
|
const chipName = {};
|
|
const tempRaw = {}, powerRaw = {}, freqRaw = {};
|
|
|
|
const fsSize = {}, fsAvail = {};
|
|
const rxByIface = {}, txByIface = {};
|
|
|
|
for (let i = 0; i < lines.length; i++) {
|
|
const line = lines[i];
|
|
if (line.length === 0 || line.charCodeAt(0) === 35 /* '#' */)
|
|
continue;
|
|
|
|
const sp = line.lastIndexOf(" ");
|
|
if (sp < 0)
|
|
continue;
|
|
|
|
const key = line.substring(0, sp);
|
|
const val = parseFloat(line.substring(sp + 1));
|
|
if (!isFinite(val))
|
|
continue;
|
|
|
|
if (key.startsWith("node_cpu_seconds_total{")) {
|
|
total += val;
|
|
const mode = root._label(key, "mode");
|
|
if (mode === "idle")
|
|
idle += val;
|
|
seenCpu[root._label(key, "cpu")] = true;
|
|
} else if (key === "node_memory_MemTotal_bytes") {
|
|
memTotal = val;
|
|
} else if (key === "node_memory_MemAvailable_bytes") {
|
|
memAvail = val;
|
|
} else if (key === "node_load1") {
|
|
l1 = val;
|
|
} else if (key === "node_load5") {
|
|
l5 = val;
|
|
} else if (key === "node_load15") {
|
|
l15 = val;
|
|
} else if (key === "node_boot_time_seconds") {
|
|
bootTime = val;
|
|
} else if (key.startsWith("node_uname_info{")) {
|
|
host = root._label(key, "nodename");
|
|
} else if (key.startsWith("node_hwmon_chip_names{")) {
|
|
chipName[root._label(key, "chip")] = root._label(key, "chip_name");
|
|
} else if (key.startsWith("node_hwmon_temp_celsius{")) {
|
|
const c = root._label(key, "chip");
|
|
(tempRaw[c] = tempRaw[c] || {})[root._label(key, "sensor")] = val;
|
|
} else if (key.startsWith("node_hwmon_power_average_watt{")) {
|
|
powerRaw[root._label(key, "chip")] = val;
|
|
} else if (key.startsWith("node_hwmon_power_cap_watt{")) {
|
|
const c = root._label(key, "chip");
|
|
(freqRaw[c] = freqRaw[c] || {})["cap"] = val;
|
|
} else if (key.startsWith("node_hwmon_freq_freq_mhz{")) {
|
|
const c = root._label(key, "chip");
|
|
(freqRaw[c] = freqRaw[c] || {})[root._label(key, "sensor")] = val;
|
|
} else if (key.startsWith("node_filesystem_size_bytes{")) {
|
|
const mp = root._label(key, "mountpoint");
|
|
if (root._fsTypes.indexOf(root._label(key, "fstype")) >= 0)
|
|
fsSize[mp] = val;
|
|
} else if (key.startsWith("node_filesystem_avail_bytes{")) {
|
|
fsAvail[root._label(key, "mountpoint")] = val;
|
|
} else if (key.startsWith("node_network_receive_bytes_total{")) {
|
|
rxByIface[root._label(key, "device")] = val;
|
|
} else if (key.startsWith("node_network_transmit_bytes_total{")) {
|
|
txByIface[root._label(key, "device")] = val;
|
|
}
|
|
}
|
|
|
|
if (memTotal <= 0) {
|
|
// Reachable but not serving node metrics — treat as a failure
|
|
// rather than rendering a panel full of zeroes.
|
|
root.failed = true;
|
|
return;
|
|
}
|
|
|
|
// Resolve hwmon chips by driver name.
|
|
const byDriver = {};
|
|
for (const chip in tempRaw)
|
|
byDriver[chipName[chip] || chip] = { temp: tempRaw[chip], chip: chip };
|
|
|
|
const cpuChip = byDriver["k10temp"] || byDriver["coretemp"] || byDriver["zenpower"];
|
|
root.cpuTemp = cpuChip ? (cpuChip.temp["temp1"] ?? NaN) : NaN;
|
|
|
|
const gpu = byDriver["amdgpu"];
|
|
if (gpu) {
|
|
root.gpuTemp = gpu.temp["temp1"] ?? NaN; // edge
|
|
root.gpuHotspot = gpu.temp["temp2"] ?? NaN; // junction
|
|
root.gpuPower = powerRaw[gpu.chip] ?? NaN;
|
|
root.gpuPowerCap = freqRaw[gpu.chip] ? (freqRaw[gpu.chip]["cap"] ?? NaN) : NaN;
|
|
root.gpuClock = freqRaw[gpu.chip] ? (freqRaw[gpu.chip]["sclk"] ?? NaN) : NaN;
|
|
} else {
|
|
root.gpuTemp = NaN;
|
|
root.gpuHotspot = NaN;
|
|
root.gpuPower = NaN;
|
|
root.gpuPowerCap = NaN;
|
|
root.gpuClock = NaN;
|
|
}
|
|
|
|
const nvme = byDriver["nvme"];
|
|
root.nvmeTemp = nvme ? (nvme.temp["temp1"] ?? NaN) : NaN;
|
|
|
|
// Filesystems. /nix/store is the same device as / on every host here,
|
|
// so listing it twice would just be noise.
|
|
const mounts = [];
|
|
for (const mp in fsSize) {
|
|
if (mp === "/nix/store")
|
|
continue;
|
|
const size = fsSize[mp];
|
|
const avail = fsAvail[mp];
|
|
if (!(size > 0) || avail === undefined)
|
|
continue;
|
|
mounts.push({ mount: mp, used: size - avail, size: size });
|
|
}
|
|
mounts.sort((a, b) => a.mount === "/" ? -1 : b.mount === "/" ? 1 : a.mount.localeCompare(b.mount));
|
|
root.disks = mounts;
|
|
|
|
// Busiest real interface.
|
|
let iface = "", best = -1;
|
|
for (const dev in rxByIface) {
|
|
let skip = false;
|
|
for (let s = 0; s < root._skipIface.length; s++) {
|
|
if (dev === root._skipIface[s] || dev.indexOf(root._skipIface[s]) === 0) {
|
|
skip = true;
|
|
break;
|
|
}
|
|
}
|
|
if (skip)
|
|
continue;
|
|
if (rxByIface[dev] > best) {
|
|
best = rxByIface[dev];
|
|
iface = dev;
|
|
}
|
|
}
|
|
root.netIface = iface;
|
|
|
|
const rx = iface ? (rxByIface[iface] ?? 0) : 0;
|
|
const tx = iface ? (txByIface[iface] ?? 0) : 0;
|
|
|
|
// Rates.
|
|
const now = Date.now() / 1000;
|
|
const dt = now - root._prevTime;
|
|
if (root._prevTime > 0 && dt > 0) {
|
|
const dTotal = total - root._prevTotal;
|
|
if (dTotal > 0)
|
|
root.cpu = Math.max(0, Math.min(1, 1 - (idle - root._prevIdle) / dTotal));
|
|
root.netRx = Math.max(0, (rx - root._prevRx) / dt);
|
|
root.netTx = Math.max(0, (tx - root._prevTx) / dt);
|
|
root.ratesReady = true;
|
|
}
|
|
root._prevTime = now;
|
|
root._prevIdle = idle;
|
|
root._prevTotal = total;
|
|
root._prevRx = rx;
|
|
root._prevTx = tx;
|
|
|
|
root.cpuThreads = Object.keys(seenCpu).length;
|
|
root.memTotal = memTotal;
|
|
root.memUsed = memTotal - memAvail;
|
|
root.load1 = l1;
|
|
root.load5 = l5;
|
|
root.load15 = l15;
|
|
root.host = host;
|
|
root.uptime = bootTime > 0 ? (Date.now() / 1000 - bootTime) : 0;
|
|
|
|
root.failed = false;
|
|
root.ready = true;
|
|
}
|
|
|
|
// ── Formatting helpers, shared with the panel ──────────────────────────
|
|
function fmtBytes(b) {
|
|
if (!isFinite(b))
|
|
return "--";
|
|
const u = ["B", "K", "M", "G", "T"];
|
|
let i = 0;
|
|
while (b >= 1024 && i < u.length - 1) {
|
|
b /= 1024;
|
|
i++;
|
|
}
|
|
return (b >= 100 || i === 0 ? Math.round(b) : b.toFixed(1)) + u[i];
|
|
}
|
|
|
|
function fmtRate(b) {
|
|
return root.ratesReady ? root.fmtBytes(b) + "/S" : "--";
|
|
}
|
|
|
|
function fmtTemp(c) {
|
|
return isFinite(c) ? Math.round(c) + "°" : "--";
|
|
}
|
|
|
|
function fmtUptime(s) {
|
|
if (!(s > 0))
|
|
return "--";
|
|
const d = Math.floor(s / 86400);
|
|
const h = Math.floor(s % 86400 / 3600);
|
|
const m = Math.floor(s % 3600 / 60);
|
|
return d > 0 ? d + "D " + h + "H" : h > 0 ? h + "H " + m + "M" : m + "M";
|
|
}
|
|
|
|
// ── Polling ────────────────────────────────────────────────────────────
|
|
Process {
|
|
id: scrape
|
|
|
|
// Filtered at the source: the full endpoint is ~1400 lines and only
|
|
// these families are drawn.
|
|
command: ["sh", "-c", "curl -s --max-time 2 http://127.0.0.1:" + root.port + "/metrics | grep -E '^node_(cpu_seconds_total|memory_MemTotal_bytes|memory_MemAvailable_bytes|load1|load5|load15|boot_time_seconds|uname_info|hwmon_chip_names|hwmon_temp_celsius|hwmon_power_average_watt|hwmon_power_cap_watt|hwmon_freq_freq_mhz|filesystem_avail_bytes|filesystem_size_bytes|network_receive_bytes_total|network_transmit_bytes_total)[ {]'"]
|
|
|
|
stdout: StdioCollector {
|
|
onStreamFinished: {
|
|
if (this.text.length === 0)
|
|
root.failed = true;
|
|
else
|
|
root._parse(this.text);
|
|
}
|
|
}
|
|
}
|
|
|
|
Timer {
|
|
interval: root.interval
|
|
running: root.active
|
|
repeat: true
|
|
triggeredOnStart: true
|
|
onTriggered: {
|
|
if (!scrape.running)
|
|
scrape.running = true;
|
|
}
|
|
}
|
|
}
|