darmanandClaude Sonnet 5
969bd69d8d
Points Nous Research's Hermes Agent at terra's own ROCm ollama server (gemma4:12b) as a custom OpenAI-compatible provider instead of a cloud key. Native systemd mode via the hermes-agent flake's own NixOS module — simpler than container mode, avoids the podman-rootful-sudo dance its docs call out. Also bumps OLLAMA_CONTEXT_LENGTH (and Hermes' mirrored model.context_length) from ollama's ~4k default to 131072, load-tested with real multi-ten- thousand-token prompts rather than just idle `ollama ps` checks — chosen as the practical ceiling where VRAM headroom and prefill throughput both start visibly degrading, not just the largest number that technically fit. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
542 lines
27 KiB
Nix
542 lines
27 KiB
Nix
{
|
|
description = "Homelab NixOS configuration";
|
|
|
|
inputs = {
|
|
nixpkgs.url = "github:NixOS/nixpkgs/nixos-26.05";
|
|
nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
|
|
disko = {
|
|
url = "github:nix-community/disko";
|
|
inputs.nixpkgs.follows = "nixpkgs";
|
|
};
|
|
sops-nix = {
|
|
url = "github:Mic92/sops-nix";
|
|
inputs.nixpkgs.follows = "nixpkgs";
|
|
};
|
|
nixos-images = {
|
|
url = "github:nix-community/nixos-images";
|
|
inputs.nixos-stable.follows = "nixpkgs";
|
|
};
|
|
nixos-anywhere = {
|
|
url = "github:nix-community/nixos-anywhere";
|
|
inputs.nixpkgs.follows = "nixpkgs";
|
|
inputs.nixos-stable.follows = "nixpkgs"; # 26.05 already IS stable
|
|
inputs.disko.follows = "disko";
|
|
inputs.nixos-images.follows = "nixos-images";
|
|
};
|
|
home-manager = {
|
|
url = "github:nix-community/home-manager/release-26.05";
|
|
inputs.nixpkgs.follows = "nixpkgs";
|
|
};
|
|
mediamanager-nix = {
|
|
url = "github:strangeglyph/mediamanager-nix";
|
|
inputs.nixpkgs.follows = "nixpkgs";
|
|
};
|
|
authentik-nix.url = "github:nix-community/authentik-nix";
|
|
nix-flatpak.url = "github:gmodena/nix-flatpak";
|
|
# Hermes Agent (see services/desktop/hermes-agent.nix) — Tier 2 platform
|
|
# per its own docs (best-effort Nix support, can break on any upstream
|
|
# commit), so pinned like everything else via flake.lock rather than
|
|
# followed loosely.
|
|
hermes-agent.url = "github:NousResearch/hermes-agent";
|
|
# Own Hyprland plugin (border + title bar), public repo, fetched over
|
|
# https (no credentials needed, unlike tome below). `nixpkgs.follows` is
|
|
# what makes its packaged build ABI-correct — Hyprland plugins are
|
|
# ABI-locked to the exact Hyprland build they load into, so it has to be
|
|
# built against THIS flake's own nixpkgs, not whatever hypr-chrome's own
|
|
# flake.lock happens to pin standalone.
|
|
hypr-chrome = {
|
|
url = "git+https://git.mgaction.town/darman/hypr-chrome.git";
|
|
inputs.nixpkgs.follows = "nixpkgs";
|
|
};
|
|
# Tome (formerly AudibleLibrary) — darman's own .NET/Photino desktop app.
|
|
# Private repo on our own gitea; fetched over ssh with darman's ambient key,
|
|
# same as any other git flake input. `flake = false`: it's a plain source
|
|
# tree, not itself a flake. See pkgs/tome.nix.
|
|
#
|
|
# NOTE: the credential-less installer-iso can't fetch this (git+ssh needs
|
|
# darman's key), so `./scripts/deploy install terra localhost` will fail
|
|
# at nixos-install (post-disko) while this input is present. Known
|
|
# tradeoff — re-removed this once before (4f79ec7) for the same reason.
|
|
tome = {
|
|
url = "git+ssh://gitea@git.mgaction.town:2222/darman/TOME.git";
|
|
flake = false;
|
|
};
|
|
};
|
|
|
|
outputs = { self, nixpkgs, disko, nixos-anywhere, sops-nix, nixos-images, home-manager, mediamanager-nix, authentik-nix, ... }@inputs:
|
|
let
|
|
system = "x86_64-linux";
|
|
in
|
|
{
|
|
packages.${system} = {
|
|
# Re-exported so `./scripts/deploy` can run them as `nix run .#disko` /
|
|
# `nix run .#nixos-anywhere`, at the revision flake.lock pins. See the
|
|
# nixos-anywhere input above for why that matters.
|
|
disko = disko.packages.${system}.disko;
|
|
nixos-anywhere = nixos-anywhere.packages.${system}.nixos-anywhere;
|
|
};
|
|
|
|
nixosConfigurations = {
|
|
# Real host — install on the ZimaBlade.
|
|
# disko owns the OS-disk partitioning + filesystems (see disk-config.nix).
|
|
jupiter = nixpkgs.lib.nixosSystem {
|
|
inherit system;
|
|
specialArgs = { inherit inputs; };
|
|
modules = [
|
|
disko.nixosModules.disko
|
|
sops-nix.nixosModules.sops
|
|
./hosts/jupiter/configuration.nix
|
|
];
|
|
};
|
|
|
|
# netcup VPS — public reverse proxy + tailnet node.
|
|
neptun = nixpkgs.lib.nixosSystem {
|
|
inherit system;
|
|
specialArgs = { inherit inputs; };
|
|
modules = [
|
|
disko.nixosModules.disko
|
|
sops-nix.nixosModules.sops
|
|
./hosts/neptun/configuration.nix
|
|
];
|
|
};
|
|
|
|
# terra — Ryzen 9 5900X desktop (MSI MS-7A32). Replaces CachyOS on the
|
|
# OS SSD; Hyprland desktop + tailnet node. See hosts/terra/*.
|
|
terra = nixpkgs.lib.nixosSystem {
|
|
inherit system;
|
|
specialArgs = { inherit inputs; };
|
|
modules = [
|
|
disko.nixosModules.disko
|
|
sops-nix.nixosModules.sops
|
|
home-manager.nixosModules.home-manager
|
|
inputs.nix-flatpak.nixosModules.nix-flatpak
|
|
inputs.hermes-agent.nixosModules.default
|
|
./hosts/terra/configuration.nix
|
|
];
|
|
};
|
|
|
|
# mercury — Raspberry Pi 3B+ (aarch64), DNS/DHCP. Boots from an SD image:
|
|
# nix build .#nixosConfigurations.mercury.config.system.build.sdImage
|
|
# (aarch64 build — needs binfmt/qemu on this x86 host, or a remote/aarch64
|
|
# builder; substitutes most paths from cache.nixos.org.)
|
|
mercury = nixpkgs.lib.nixosSystem {
|
|
system = "aarch64-linux";
|
|
specialArgs = { inherit inputs; };
|
|
modules = [
|
|
(nixpkgs + "/nixos/modules/installer/sd-card/sd-image-aarch64.nix")
|
|
sops-nix.nixosModules.sops
|
|
./hosts/mercury/configuration.nix
|
|
];
|
|
};
|
|
|
|
# x86_64 QEMU VM to runtime-test mercury's DNS/DHCP stack (pihole +
|
|
# unbound) before flashing the aarch64 SD. Build + run:
|
|
# nix build .#nixosConfigurations.mercury-vm.config.system.build.vm
|
|
# ./result/bin/run-mercury-vm-vm
|
|
mercury-vm = nixpkgs.lib.nixosSystem {
|
|
inherit system; # x86_64-linux, fast to build/boot with KVM
|
|
modules = [
|
|
(nixpkgs + "/nixos/modules/virtualisation/qemu-vm.nix")
|
|
./common.nix
|
|
./services/network/unbound.nix
|
|
./services/network/pihole.nix
|
|
({ lib, ... }: {
|
|
networking.hostName = "mercury-vm";
|
|
networking.nameservers = [ "1.1.1.1" "9.9.9.9" ]; # host resolver (not pihole)
|
|
users.users.darman.initialPassword = "test";
|
|
users.users.root.initialPassword = "test";
|
|
services.openssh.settings.PasswordAuthentication = lib.mkForce true;
|
|
virtualisation.graphics = false;
|
|
virtualisation.memorySize = 2048;
|
|
virtualisation.forwardPorts = [
|
|
{ from = "host"; host.port = 2223; guest.port = 22; }
|
|
{ from = "host"; host.port = 8081; guest.port = 80; }
|
|
];
|
|
system.stateVersion = "26.05";
|
|
})
|
|
];
|
|
};
|
|
|
|
# VirtualBox test image. Build the OVA with:
|
|
# nix build .#nixosConfigurations.jupiter-vbox.config.system.build.virtualBoxOVA
|
|
# NOTE: no disko here — the virtualbox-image module supplies the disk.
|
|
jupiter-vbox = nixpkgs.lib.nixosSystem {
|
|
inherit system;
|
|
specialArgs = { inherit inputs; };
|
|
modules = [ ./hosts/jupiter/vm.nix ];
|
|
};
|
|
|
|
# Custom kexec installer with our SSH key baked in, for headless install
|
|
# onto a box with a read-only root (ZimaOS) where nixos-anywhere can't
|
|
# ssh-copy-id. Build the tarball:
|
|
# nix build .#nixosConfigurations.kexec.config.system.build.kexecInstallerTarball
|
|
# then scp it to the target's writable /tmp and run kexec/run (see README).
|
|
kexec = nixpkgs.lib.nixosSystem {
|
|
inherit system;
|
|
modules = [
|
|
nixos-images.nixosModules.kexec-installer
|
|
({ ... }: {
|
|
users.users.root.openssh.authorizedKeys.keys = [
|
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGZpkPVhzi1zG5JI9hWyUgdyvNIQbp4ts4jw3idpMhhN erik@laptop"
|
|
];
|
|
})
|
|
];
|
|
};
|
|
|
|
# Bootable USB recovery installer with our SSH key + sshd + DHCP. Clones
|
|
# the (now public) homelab repo fresh at every boot to /root/homelab —
|
|
# always current master, so the same USB stick stays useful across
|
|
# install/rescue occasions without ever needing a rebuild. No
|
|
# rsync/copy-the-repo-over step: boot it, ssh in,
|
|
# `cd /root/homelab && ./scripts/deploy install ...`.
|
|
# Reusable for any host's manual-USB install path (jupiter, terra, ...).
|
|
# Build the ISO:
|
|
# nix build .#nixosConfigurations.installer-iso.config.system.build.isoImage
|
|
# dd it to a USB stick, boot the target from it, SSH in, ./deploy install.
|
|
installer-iso = nixpkgs.lib.nixosSystem {
|
|
inherit system;
|
|
modules = [
|
|
(nixpkgs + "/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix")
|
|
({ pkgs, lib, ... }: {
|
|
services.openssh.enable = true;
|
|
services.openssh.settings.PermitRootLogin = "prohibit-password";
|
|
users.users.root.openssh.authorizedKeys.keys = [
|
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGZpkPVhzi1zG5JI9hWyUgdyvNIQbp4ts4jw3idpMhhN erik@laptop"
|
|
];
|
|
networking.hostName = "homelab-installer";
|
|
environment.systemPackages = [ pkgs.git ];
|
|
|
|
# findiso= is a SCRIPT-stage-1 feature (stage-1-init.sh) only. The
|
|
# systemd initrd — the default since 26.05 — has no findiso path
|
|
# at all: it mounts /iso straight from
|
|
# /dev/disk/by-label/<volumeID> (iso-image.nix), which only exists
|
|
# when the ISO is the physical boot medium. Booted as a kernel +
|
|
# initrd off the ESP with the iso as a plain file elsewhere, that
|
|
# label never appears and stage 1 times out into an emergency
|
|
# shell (mounts /sysroot fine, then fails /sysroot/nix/.ro-store).
|
|
# Script stage 1 instead loop-mounts the file findiso= points at
|
|
# and symlinks it to /dev/root — which is the whole mechanism this
|
|
# install path relies on. So force it off here.
|
|
boot.initrd.systemd.enable = false;
|
|
|
|
# installation-cd-minimal leaves experimental-features unset, so
|
|
# the ISO's nix.conf has no `nix-command`/`flakes` at all (unlike
|
|
# the nixos-images kexec installer, which sets
|
|
# extra-experimental-features itself — which is why the same
|
|
# `install <config> localhost` branch works after kexec-local but
|
|
# not here). Without this, both `nix run .#disko` and
|
|
# `nixos-install --flake` die with "experimental Nix feature
|
|
# 'nix-command' is disabled".
|
|
nix.settings.experimental-features = [ "nix-command" "flakes" ];
|
|
|
|
# Fresh clone of a PUBLIC repo — no credentials baked into the
|
|
# ISO. require_tracked() in scripts/deploy still works fine here
|
|
# (this IS a real git checkout, unlike the old baked-`self`
|
|
# approach), but retry manually with `systemctl restart
|
|
# homelab-checkout` if DHCP was still coming up at boot.
|
|
systemd.services.homelab-checkout = {
|
|
description = "Clone the homelab repo to /root/homelab";
|
|
after = [ "network-online.target" ];
|
|
wants = [ "network-online.target" ];
|
|
wantedBy = [ "multi-user.target" ];
|
|
path = [ pkgs.git ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
};
|
|
script = ''
|
|
rm -rf /root/homelab
|
|
git clone --depth 1 https://git.mgaction.town/darman/homelab.git /root/homelab
|
|
'';
|
|
};
|
|
|
|
# Finishes a local_install_prepare_and_reboot() run (scripts/deploy)
|
|
# unattended: that function stages this ISO, points a systemd-boot
|
|
# one-shot entry at it with `homelab.install=<config>` on the kernel
|
|
# cmdline, and reboots. Once booted here, this re-runs the exact same
|
|
# `./scripts/deploy install <config> localhost` command — now genuinely
|
|
# inside the installer (hostname homelab-installer), so is_live_installer
|
|
# takes the disko+nixos-install branch instead of preparing again.
|
|
# A manual boot of this ISO with no such cmdline param is a no-op.
|
|
systemd.services.homelab-auto-install = {
|
|
description = "Auto-run the homelab install if homelab.install= was passed on the kernel cmdline";
|
|
after = [ "homelab-checkout.service" ];
|
|
requires = [ "homelab-checkout.service" ];
|
|
wantedBy = [ "multi-user.target" ];
|
|
serviceConfig.Type = "oneshot";
|
|
# Full system PATH, not the restricted default a `path = [...]`
|
|
# produces: this unit execs `./scripts/deploy`, whose
|
|
# `#!/usr/bin/env bash` needs bash, and which then reaches for
|
|
# nix / nixos-install / git / sudo / efibootmgr. The default
|
|
# service PATH gave "env: 'bash': No such file or directory"
|
|
# (status 127) before the script even started.
|
|
# /run/current-system/sw/bin carries all of it on the installer;
|
|
# /run/wrappers/bin for sudo. mkForce because NixOS otherwise
|
|
# derives environment.PATH from `path` and that line would win.
|
|
#
|
|
# HOME too: systemd sets no $HOME for a service without User=
|
|
# (systemd.exec(5): SetLoginEnvironment= defaults false), and
|
|
# scripts/deploy runs under `set -u`, so a bare $HOME aborted the
|
|
# whole run with an "unbound variable" that read like a bug.
|
|
environment = {
|
|
HOME = "/root";
|
|
PATH = lib.mkForce "/run/current-system/sw/bin:/run/wrappers/bin";
|
|
};
|
|
script = ''
|
|
cfg=$(grep -o 'homelab\.install=[^ ]*' /proc/cmdline | cut -d= -f2 || true)
|
|
if [ -z "$cfg" ]; then
|
|
echo "no homelab.install= on the kernel cmdline — nothing to auto-install"
|
|
exit 0
|
|
fi
|
|
|
|
# Persist this whole run to a file that OUTLIVES the install.
|
|
# The systemd journal is on the installer's tmpfs and dies with
|
|
# the reboot, and by the time anything interesting fails disko
|
|
# has already wiped the OS disk — so a failed attempt used to
|
|
# leave nothing to debug. local_install_prepare_and_reboot()
|
|
# (scripts/deploy) passes the STAGING partition's PARTUUID as
|
|
# homelab.logpart=; that partition holds the iso and is on a
|
|
# different disk from the one disko wipes, so it survives. The
|
|
# actual install runs inside do_install() below so one tee at
|
|
# the end captures all of it. Every step here is best-effort:
|
|
# logging must never be the thing that breaks an install.
|
|
logfile=""
|
|
logpart=$(grep -o 'homelab\.logpart=[^ ]*' /proc/cmdline | cut -d= -f2 || true)
|
|
if [ -n "$logpart" ]; then
|
|
dev="/dev/disk/by-partuuid/$logpart"
|
|
logdir=""
|
|
mkdir -p /run/homelab-log
|
|
if mount -o rw "$dev" /run/homelab-log 2>/dev/null; then
|
|
logdir=/run/homelab-log
|
|
elif where=$(findmnt -fno TARGET "$dev" 2>/dev/null) && [ -n "$where" ]; then
|
|
# stage-1's findiso already holds this partition mounted
|
|
# (that is how it reached the iso) — write into the existing
|
|
# mount rather than trying to stack a second one on it.
|
|
mount -o remount,rw "$where" 2>/dev/null || true
|
|
logdir="$where"
|
|
fi
|
|
if [ -n "$logdir" ]; then
|
|
# Next to the iso: findiso= is its path on this partition.
|
|
iso=$(grep -o 'findiso=[^ ]*' /proc/cmdline | cut -d= -f2 || true)
|
|
dest="$logdir/$(dirname "$iso" 2>/dev/null || echo /)"
|
|
if mkdir -p "$dest" 2>/dev/null; then
|
|
logfile="$dest/homelab-install-$cfg.log"
|
|
else
|
|
logfile="$logdir/homelab-install-$cfg.log"
|
|
fi
|
|
echo "logging this install to $logfile (on the staging disk — survives the wipe)"
|
|
else
|
|
echo "warning: could not mount PARTUUID=$logpart to log to — continuing without a persistent log" >&2
|
|
fi
|
|
fi
|
|
|
|
do_install() {
|
|
# The host key scripts/deploy seeds /etc/ssh with (so sops can
|
|
# decrypt on boot #1) cannot live in this ISO: it is built from
|
|
# a PUBLIC repo and the private keys are deliberately off-repo.
|
|
# local_install_prepare_and_reboot() therefore drops it on the
|
|
# boot partition and passes that partition's PARTUUID here.
|
|
# That copy dies with the disko wipe a few minutes later.
|
|
keypart=$(grep -o 'homelab\.keypart=[^ ]*' /proc/cmdline | cut -d= -f2 || true)
|
|
if [ -n "$keypart" ]; then
|
|
mkdir -p /run/homelab-key
|
|
if mount -o ro "/dev/disk/by-partuuid/$keypart" /run/homelab-key; then
|
|
src=/run/homelab-key/homelab-installer
|
|
if [ -f "$src/ssh_host_ed25519_key" ]; then
|
|
echo "picking up $cfg's host key from PARTUUID=$keypart"
|
|
install -Dm600 "$src/ssh_host_ed25519_key" \
|
|
"/root/.config/homelab/$cfg/ssh_host_ed25519_key"
|
|
install -Dm644 "$src/ssh_host_ed25519_key.pub" \
|
|
"/root/.config/homelab/$cfg/ssh_host_ed25519_key.pub"
|
|
else
|
|
echo "warning: no host key at $src — the install will refuse" >&2
|
|
fi
|
|
umount /run/homelab-key
|
|
else
|
|
echo "warning: could not mount PARTUUID=$keypart for the host key" >&2
|
|
fi
|
|
fi
|
|
|
|
# On a box whose old bootloader had no one-shot (Limine on
|
|
# terra), scripts/deploy got us here via a temporary UEFI
|
|
# entry + BootNext (arm_efi_bootnext). BootNext is already
|
|
# spent, but the entry itself would linger in NVRAM pointing
|
|
# at a partition disko is about to reformat. Drop it now, so
|
|
# even an install that fails later leaves NVRAM clean.
|
|
for n in $(efibootmgr 2>/dev/null \
|
|
| sed -n 's/^Boot\([0-9A-Fa-f]\{4\}\)\*\?[[:space:]]Homelab Installer[[:space:]].*/\1/p'); do
|
|
echo "removing temporary UEFI entry Boot$n"
|
|
efibootmgr -q -B -b "$n" || true
|
|
done
|
|
|
|
echo "auto-installing $cfg (homelab.install= on the kernel cmdline)"
|
|
cd /root/homelab
|
|
./scripts/deploy install "$cfg" localhost --yes
|
|
}
|
|
|
|
# tee, not exec: we need the exit status back to sync the log
|
|
# to the platter before the box possibly drops to a shell.
|
|
if [ -n "$logfile" ]; then
|
|
{ echo "=== homelab auto-install: $cfg ($(date -u 2>/dev/null || true)) ==="; do_install; } 2>&1 | tee -a "$logfile"
|
|
status=''${PIPESTATUS[0]}
|
|
else
|
|
do_install
|
|
status=$?
|
|
fi
|
|
sync 2>/dev/null || true
|
|
exit "$status"
|
|
'';
|
|
};
|
|
})
|
|
];
|
|
};
|
|
};
|
|
|
|
# VM test for `./scripts/deploy kexec-local`. Run:
|
|
# nix build .#checks.x86_64-linux.kexec-local -L
|
|
#
|
|
# Worth having because kexec-local is the one command that cannot be
|
|
# rehearsed on real hardware: it jumps the machine you are typing at, and
|
|
# a failure looks exactly like a slow boot. It regression-tests the
|
|
# subtle one — kexec-run.sh backgrounds `sleep 6 && kexec -e` and returns,
|
|
# so anything that cleans up the staging dir on exit deletes the binary
|
|
# that performs the jump and the box silently never leaves the old kernel.
|
|
#
|
|
# After the jump the test driver's backdoor is gone with the old kernel,
|
|
# so the installer is driven over a forwarded ssh port instead (the same
|
|
# approach nixos-images uses in its own kexec test).
|
|
checks.${system}.kexec-local =
|
|
let
|
|
pkgs = nixpkgs.legacyPackages.${system};
|
|
tarball = self.nixosConfigurations.kexec.config.system.build.kexecInstallerTarball;
|
|
sshKey = nixos-images + "/nix/kexec-installer/ssh-keys/id_ed25519";
|
|
in
|
|
pkgs.testers.runNixOSTest {
|
|
name = "deploy-kexec-local";
|
|
|
|
nodes.machine = { modulesPath, ... }: {
|
|
imports = [ (modulesPath + "/profiles/minimal.nix") ];
|
|
virtualisation.vlans = [ ];
|
|
# kexec-local refuses to run if RAM < 3x the installer image, and
|
|
# the staging dir needs ~3x the tarball on /var/tmp.
|
|
virtualisation.memorySize = 4 * 1024;
|
|
virtualisation.diskSize = 12 * 1024;
|
|
virtualisation.forwardPorts = [{ host.port = 2222; guest.port = 22; }];
|
|
|
|
services.openssh.enable = true;
|
|
users.users.root.openssh.authorizedKeys.keyFiles = [ "${sshKey}.pub" ];
|
|
|
|
# Everything the script shells out to, minus nix — the test uses the
|
|
# HOMELAB_KEXEC_* hook so no build happens inside the VM.
|
|
environment.systemPackages = with pkgs; [
|
|
bash gnutar coreutils findutils util-linux cpio gzip
|
|
];
|
|
system.extraDependencies = [ tarball pkgs.cpio pkgs.gzip ];
|
|
|
|
environment.etc."deploy".source = ./scripts/deploy;
|
|
};
|
|
|
|
testScript = /* python */ ''
|
|
import os, shutil, subprocess, tempfile, time
|
|
|
|
start_all()
|
|
machine.wait_for_unit("sshd.service")
|
|
|
|
# ssh refuses a private key that is group/world readable, and nix
|
|
# store paths are 0444 — copy it out and tighten the mode.
|
|
keydir = tempfile.mkdtemp()
|
|
key = os.path.join(keydir, "id_ed25519")
|
|
shutil.copyfile("${sshKey}", key)
|
|
os.chmod(key, 0o600)
|
|
|
|
def ssh(cmd, check=True, stdout=None):
|
|
return subprocess.run(
|
|
[ "${pkgs.openssh}/bin/ssh",
|
|
"-o", "StrictHostKeyChecking=no",
|
|
"-o", "UserKnownHostsFile=/dev/null",
|
|
"-o", "ConnectTimeout=1",
|
|
"-i", key,
|
|
"-p", "2222", "root@127.0.0.1", "--" ] + cmd,
|
|
text=True, check=check, stdout=stdout)
|
|
|
|
machine.succeed("install -Dm755 /etc/deploy /root/deploy")
|
|
|
|
# systemd-run starts units with a bare PATH that lacks
|
|
# /run/current-system/sw/bin, so `#!/usr/bin/env bash` cannot even
|
|
# resolve bash, let alone tar/findmnt/nohup. Set it explicitly.
|
|
env = (
|
|
" --setenv=PATH=/run/wrappers/bin:/run/current-system/sw/bin"
|
|
" --setenv=HOMELAB_KEXEC_TARBALL=${tarball}/nixos-kexec-installer-${system}.tar.gz"
|
|
" --setenv=HOMELAB_KEXEC_CPIO=${pkgs.cpio}/bin/cpio"
|
|
" --setenv=HOMELAB_KEXEC_GZIP=${pkgs.gzip}/bin/gzip"
|
|
)
|
|
# Same values for the foreground (non-systemd-run) invocation below.
|
|
envsh = (
|
|
"HOMELAB_KEXEC_TARBALL=${tarball}/nixos-kexec-installer-${system}.tar.gz"
|
|
" HOMELAB_KEXEC_CPIO=${pkgs.cpio}/bin/cpio"
|
|
" HOMELAB_KEXEC_GZIP=${pkgs.gzip}/bin/gzip"
|
|
)
|
|
|
|
# Marker on a tmpfs: it must NOT survive the jump, proving we really
|
|
# booted a new kernel rather than just restarting a service.
|
|
machine.succeed("touch /run/pre-kexec-marker")
|
|
host_key_before = machine.succeed("cat /etc/ssh/ssh_host_ed25519_key.pub").strip()
|
|
|
|
while ssh(["true"], check=False).returncode != 0:
|
|
time.sleep(1)
|
|
|
|
# Refuses without --yes when stdin is not a tty (read gets EOF).
|
|
# Must reach the confirmation prompt, so it needs the same env —
|
|
# otherwise it just dies early on the nix build and proves nothing.
|
|
out = machine.fail(f"{envsh} /root/deploy kexec-local </dev/null 2>&1")
|
|
assert "using prebuilt kexec installer" in out, \
|
|
f"never reached the prompt, so the refusal proves nothing:\n{out}"
|
|
|
|
# systemd-run so the call returns immediately: the script stays
|
|
# alive ~60s on purpose, outliving kexec-run.sh's `sleep 6`.
|
|
machine.succeed(f"systemd-run --collect --unit=kexec-local{env} /root/deploy kexec-local --yes")
|
|
|
|
print("waiting for the jump...")
|
|
deadline = time.time() + 300
|
|
while ssh(["true"], check=False).returncode == 0:
|
|
# Surface a dead unit immediately instead of stalling until the
|
|
# deadline and blaming "never left the old kernel".
|
|
st = ssh(["systemctl", "is-active", "kexec-local"],
|
|
check=False, stdout=subprocess.PIPE).stdout or ""
|
|
if st.strip() in ("failed", "inactive"):
|
|
# NB: not `log` — the driver already binds that name to its
|
|
# AbstractLogger and the type check rejects the shadowing.
|
|
unit_log = ssh(["journalctl", "-u", "kexec-local", "--no-pager"],
|
|
check=False, stdout=subprocess.PIPE).stdout or ""
|
|
raise AssertionError(
|
|
f"kexec-local.service ended ({st.strip()}) without jumping:\n{unit_log}")
|
|
assert time.time() < deadline, "machine never left the old kernel"
|
|
time.sleep(1)
|
|
|
|
print("waiting for the installer...")
|
|
deadline = time.time() + 300
|
|
while ssh(["true"], check=False).returncode != 0:
|
|
assert time.time() < deadline, "installer never came up"
|
|
time.sleep(1)
|
|
|
|
# It really is the RAM installer, not the old system.
|
|
host = ssh(["hostname"], stdout=subprocess.PIPE).stdout.strip()
|
|
assert host == "nixos-installer", f"hostname is {host}, not nixos-installer"
|
|
|
|
assert ssh(["ls", "/run/pre-kexec-marker"], check=False).returncode != 0, \
|
|
"old /run survived — this was not a fresh kernel"
|
|
|
|
# The host key is carried across (kexec-run.sh copies /etc/ssh into
|
|
# the appended initrd), which is why `kexec` does no ssh-keygen -R.
|
|
host_key_after = ssh(
|
|
["cat", "/etc/ssh/ssh_host_ed25519_key.pub"], stdout=subprocess.PIPE
|
|
).stdout.strip()
|
|
assert host_key_before == host_key_after, \
|
|
f"host key changed: {host_key_before} != {host_key_after}"
|
|
|
|
machine.crash()
|
|
'';
|
|
};
|
|
};
|
|
}
|