From ff92e24ff75aeee0919d4a03091fffbdaec10cd3 Mon Sep 17 00:00:00 2001 From: Erik Simon Date: Fri, 24 Jul 2026 19:43:43 +0200 Subject: [PATCH] installer-iso: persist auto-install logs to the staging disk --- flake.nix | 123 +++++++++++++++++++++++++++++++++++-------------- scripts/deploy | 19 ++++++++ 2 files changed, 108 insertions(+), 34 deletions(-) diff --git a/flake.nix b/flake.nix index 4ab7407..95fe771 100644 --- a/flake.nix +++ b/flake.nix @@ -291,47 +291,102 @@ exit 0 fi - # The host key scripts/deploy seeds /etc/ssh with (so sops can - # decrypt on boot #1) cannot live in this ISO: it is built from - # a PUBLIC repo and the private keys are deliberately off-repo. - # local_install_prepare_and_reboot() therefore drops it on the - # boot partition and passes that partition's PARTUUID here. - # That copy dies with the disko wipe a few minutes later. - keypart=$(grep -o 'homelab\.keypart=[^ ]*' /proc/cmdline | cut -d= -f2 || true) - if [ -n "$keypart" ]; then - mkdir -p /run/homelab-key - if mount -o ro "/dev/disk/by-partuuid/$keypart" /run/homelab-key; then - src=/run/homelab-key/homelab-installer - if [ -f "$src/ssh_host_ed25519_key" ]; then - echo "picking up $cfg's host key from PARTUUID=$keypart" - install -Dm600 "$src/ssh_host_ed25519_key" \ - "/root/.config/homelab/$cfg/ssh_host_ed25519_key" - install -Dm644 "$src/ssh_host_ed25519_key.pub" \ - "/root/.config/homelab/$cfg/ssh_host_ed25519_key.pub" + # Persist this whole run to a file that OUTLIVES the install. + # The systemd journal is on the installer's tmpfs and dies with + # the reboot, and by the time anything interesting fails disko + # has already wiped the OS disk — so a failed attempt used to + # leave nothing to debug. local_install_prepare_and_reboot() + # (scripts/deploy) passes the STAGING partition's PARTUUID as + # homelab.logpart=; that partition holds the iso and is on a + # different disk from the one disko wipes, so it survives. The + # actual install runs inside do_install() below so one tee at + # the end captures all of it. Every step here is best-effort: + # logging must never be the thing that breaks an install. + logfile="" + logpart=$(grep -o 'homelab\.logpart=[^ ]*' /proc/cmdline | cut -d= -f2 || true) + if [ -n "$logpart" ]; then + dev="/dev/disk/by-partuuid/$logpart" + logdir="" + mkdir -p /run/homelab-log + if mount -o rw "$dev" /run/homelab-log 2>/dev/null; then + logdir=/run/homelab-log + elif where=$(findmnt -fno TARGET "$dev" 2>/dev/null) && [ -n "$where" ]; then + # stage-1's findiso already holds this partition mounted + # (that is how it reached the iso) — write into the existing + # mount rather than trying to stack a second one on it. + mount -o remount,rw "$where" 2>/dev/null || true + logdir="$where" + fi + if [ -n "$logdir" ]; then + # Next to the iso: findiso= is its path on this partition. + iso=$(grep -o 'findiso=[^ ]*' /proc/cmdline | cut -d= -f2 || true) + dest="$logdir/$(dirname "$iso" 2>/dev/null || echo /)" + if mkdir -p "$dest" 2>/dev/null; then + logfile="$dest/homelab-install-$cfg.log" else - echo "warning: no host key at $src — the install will refuse" >&2 + logfile="$logdir/homelab-install-$cfg.log" fi - umount /run/homelab-key + echo "logging this install to $logfile (on the staging disk — survives the wipe)" else - echo "warning: could not mount PARTUUID=$keypart for the host key" >&2 + echo "warning: could not mount PARTUUID=$logpart to log to — continuing without a persistent log" >&2 fi fi - # On a box whose old bootloader had no one-shot (Limine on - # terra), scripts/deploy got us here via a temporary UEFI - # entry + BootNext (arm_efi_bootnext). BootNext is already - # spent, but the entry itself would linger in NVRAM pointing - # at a partition disko is about to reformat. Drop it now, so - # even an install that fails later leaves NVRAM clean. - for n in $(efibootmgr 2>/dev/null \ - | sed -n 's/^Boot\([0-9A-Fa-f]\{4\}\)\*\?[[:space:]]Homelab Installer[[:space:]].*/\1/p'); do - echo "removing temporary UEFI entry Boot$n" - efibootmgr -q -B -b "$n" || true - done + do_install() { + # The host key scripts/deploy seeds /etc/ssh with (so sops can + # decrypt on boot #1) cannot live in this ISO: it is built from + # a PUBLIC repo and the private keys are deliberately off-repo. + # local_install_prepare_and_reboot() therefore drops it on the + # boot partition and passes that partition's PARTUUID here. + # That copy dies with the disko wipe a few minutes later. + keypart=$(grep -o 'homelab\.keypart=[^ ]*' /proc/cmdline | cut -d= -f2 || true) + if [ -n "$keypart" ]; then + mkdir -p /run/homelab-key + if mount -o ro "/dev/disk/by-partuuid/$keypart" /run/homelab-key; then + src=/run/homelab-key/homelab-installer + if [ -f "$src/ssh_host_ed25519_key" ]; then + echo "picking up $cfg's host key from PARTUUID=$keypart" + install -Dm600 "$src/ssh_host_ed25519_key" \ + "/root/.config/homelab/$cfg/ssh_host_ed25519_key" + install -Dm644 "$src/ssh_host_ed25519_key.pub" \ + "/root/.config/homelab/$cfg/ssh_host_ed25519_key.pub" + else + echo "warning: no host key at $src — the install will refuse" >&2 + fi + umount /run/homelab-key + else + echo "warning: could not mount PARTUUID=$keypart for the host key" >&2 + fi + fi - echo "auto-installing $cfg (homelab.install= on the kernel cmdline)" - cd /root/homelab - exec ./scripts/deploy install "$cfg" localhost --yes + # On a box whose old bootloader had no one-shot (Limine on + # terra), scripts/deploy got us here via a temporary UEFI + # entry + BootNext (arm_efi_bootnext). BootNext is already + # spent, but the entry itself would linger in NVRAM pointing + # at a partition disko is about to reformat. Drop it now, so + # even an install that fails later leaves NVRAM clean. + for n in $(efibootmgr 2>/dev/null \ + | sed -n 's/^Boot\([0-9A-Fa-f]\{4\}\)\*\?[[:space:]]Homelab Installer[[:space:]].*/\1/p'); do + echo "removing temporary UEFI entry Boot$n" + efibootmgr -q -B -b "$n" || true + done + + echo "auto-installing $cfg (homelab.install= on the kernel cmdline)" + cd /root/homelab + ./scripts/deploy install "$cfg" localhost --yes + } + + # tee, not exec: we need the exit status back to sync the log + # to the platter before the box possibly drops to a shell. + if [ -n "$logfile" ]; then + { echo "=== homelab auto-install: $cfg ($(date -u 2>/dev/null || true)) ==="; do_install; } 2>&1 | tee -a "$logfile" + status=''${PIPESTATUS[0]} + else + do_install + status=$? + fi + sync 2>/dev/null || true + exit "$status" ''; }; }) diff --git a/scripts/deploy b/scripts/deploy index 722e179..47be9e0 100755 --- a/scripts/deploy +++ b/scripts/deploy @@ -342,6 +342,17 @@ local_install_prepare_and_reboot() { [ "$stage_fstype" != btrfs ] \ || die "$stagedir is btrfs: findiso= mounts the volume's top level, so a path inside a subvolume never resolves. Stage on a non-btrfs partition (ext4/vfat/ntfs)." + # PARTUUID of the staging partition. Handed to the installer as + # homelab.logpart= so it can mount this partition rw and persist its whole + # run — disko + nixos-install output included — to a file next to the iso. + # This partition is on a DIFFERENT disk from the one disko wipes (guarded + # above), so unlike $boot it SURVIVES the install: a failed attempt otherwise + # leaves nothing to debug, its journal having died on tmpfs at the reboot. + # Best-effort — an LVM/mdraid stage_src has no PARTUUID, in which case logging + # is simply skipped rather than blocking the install. + local stage_partuuid + stage_partuuid="$(lsblk -no PARTUUID "$stage_src" 2>/dev/null | head -1 | tr -d ' ' || true)" + # Last chance to back out. This is the most destructive command in the # script — it reboots the machine you are typing at and the wipe that # follows is unattended — so it confirms just like `flash` and `kexec-local` @@ -354,6 +365,11 @@ local_install_prepare_and_reboot() { echo " -> $osdisk_real ** WIPED, unattended, after the reboot **" # Unquoted on purpose: collapses the one-per-line list onto one line. echo " staging: $stagedir (on $(echo $stage_disks))" + if [ -n "$stage_partuuid" ]; then + echo " logs: $stagedir/homelab-install-$config.log (on the staging disk — survives the wipe)" + else + echo " logs: (none — $stagedir has no PARTUUID; installer output won't survive the wipe)" + fi echo " one-shot: $boot_mode" read -rp ">> type 'yes' to build the installer, reboot into it and wipe $osdisk_real: " ok [ "$ok" = yes ] || die "aborted" @@ -426,6 +442,9 @@ local_install_prepare_and_reboot() { local cmdline volumeID volumeID="$(nix eval --raw .#nixosConfigurations.installer-iso.config.isoImage.volumeID)" cmdline="init=$toplevel/init nohibernate root=LABEL=$volumeID boot.shell_on_fail loglevel=4 lsm=landlock,yama,bpf findiso=$iso_relpath homelab.install=$config homelab.keypart=$boot_partuuid" + # Only when the staging partition has a PARTUUID (see stage_partuuid). Points + # the installer's homelab-auto-install.service at the surviving disk to log to. + [ -n "$stage_partuuid" ] && cmdline="$cmdline homelab.logpart=$stage_partuuid" case "$boot_mode" in systemd-boot)