installer-iso: persist auto-install logs to the staging disk

flake.nix

@@ -291,6 +291,48 @@
                     exit 0
                   fi
 
+                  # Persist this whole run to a file that OUTLIVES the install.
+                  # The systemd journal is on the installer's tmpfs and dies with
+                  # the reboot, and by the time anything interesting fails disko
+                  # has already wiped the OS disk — so a failed attempt used to
+                  # leave nothing to debug. local_install_prepare_and_reboot()
+                  # (scripts/deploy) passes the STAGING partition's PARTUUID as
+                  # homelab.logpart=; that partition holds the iso and is on a
+                  # different disk from the one disko wipes, so it survives. The
+                  # actual install runs inside do_install() below so one tee at
+                  # the end captures all of it. Every step here is best-effort:
+                  # logging must never be the thing that breaks an install.
+                  logfile=""
+                  logpart=$(grep -o 'homelab\.logpart=[^ ]*' /proc/cmdline | cut -d= -f2 || true)
+                  if [ -n "$logpart" ]; then
+                    dev="/dev/disk/by-partuuid/$logpart"
+                    logdir=""
+                    mkdir -p /run/homelab-log
+                    if mount -o rw "$dev" /run/homelab-log 2>/dev/null; then
+                      logdir=/run/homelab-log
+                    elif where=$(findmnt -fno TARGET "$dev" 2>/dev/null) && [ -n "$where" ]; then
+                      # stage-1's findiso already holds this partition mounted
+                      # (that is how it reached the iso) — write into the existing
+                      # mount rather than trying to stack a second one on it.
+                      mount -o remount,rw "$where" 2>/dev/null || true
+                      logdir="$where"
+                    fi
+                    if [ -n "$logdir" ]; then
+                      # Next to the iso: findiso= is its path on this partition.
+                      iso=$(grep -o 'findiso=[^ ]*' /proc/cmdline | cut -d= -f2 || true)
+                      dest="$logdir/$(dirname "$iso" 2>/dev/null || echo /)"
+                      if mkdir -p "$dest" 2>/dev/null; then
+                        logfile="$dest/homelab-install-$cfg.log"
+                      else
+                        logfile="$logdir/homelab-install-$cfg.log"
+                      fi
+                      echo "logging this install to $logfile (on the staging disk — survives the wipe)"
+                    else
+                      echo "warning: could not mount PARTUUID=$logpart to log to — continuing without a persistent log" >&2
+                    fi
+                  fi
+
+                  do_install() {
                     # The host key scripts/deploy seeds /etc/ssh with (so sops can
                     # decrypt on boot #1) cannot live in this ISO: it is built from
                     # a PUBLIC repo and the private keys are deliberately off-repo.
@@ -331,7 +373,20 @@
 
                     echo "auto-installing $cfg (homelab.install= on the kernel cmdline)"
                     cd /root/homelab
-                  exec ./scripts/deploy install "$cfg" localhost --yes
+                    ./scripts/deploy install "$cfg" localhost --yes
+                  }
+
+                  # tee, not exec: we need the exit status back to sync the log
+                  # to the platter before the box possibly drops to a shell.
+                  if [ -n "$logfile" ]; then
+                    { echo "=== homelab auto-install: $cfg ($(date -u 2>/dev/null || true)) ==="; do_install; } 2>&1 | tee -a "$logfile"
+                    status=''${PIPESTATUS[0]}
+                  else
+                    do_install
+                    status=$?
+                  fi
+                  sync 2>/dev/null || true
+                  exit "$status"
                 '';
               };
             })

scripts/deploy

@@ -342,6 +342,17 @@ local_install_prepare_and_reboot() {
   [ "$stage_fstype" != btrfs ] \
     || die "$stagedir is btrfs: findiso= mounts the volume's top level, so a path inside a subvolume never resolves. Stage on a non-btrfs partition (ext4/vfat/ntfs)."
 
+  # PARTUUID of the staging partition. Handed to the installer as
+  # homelab.logpart= so it can mount this partition rw and persist its whole
+  # run — disko + nixos-install output included — to a file next to the iso.
+  # This partition is on a DIFFERENT disk from the one disko wipes (guarded
+  # above), so unlike $boot it SURVIVES the install: a failed attempt otherwise
+  # leaves nothing to debug, its journal having died on tmpfs at the reboot.
+  # Best-effort — an LVM/mdraid stage_src has no PARTUUID, in which case logging
+  # is simply skipped rather than blocking the install.
+  local stage_partuuid
+  stage_partuuid="$(lsblk -no PARTUUID "$stage_src" 2>/dev/null | head -1 | tr -d ' ' || true)"
+
   # Last chance to back out. This is the most destructive command in the
   # script — it reboots the machine you are typing at and the wipe that
   # follows is unattended — so it confirms just like `flash` and `kexec-local`
@@ -354,6 +365,11 @@ local_install_prepare_and_reboot() {
     echo "              -> $osdisk_real  ** WIPED, unattended, after the reboot **"
     # Unquoted on purpose: collapses the one-per-line list onto one line.
     echo "   staging:   $stagedir  (on $(echo $stage_disks))"
+    if [ -n "$stage_partuuid" ]; then
+      echo "   logs:      $stagedir/homelab-install-$config.log  (on the staging disk — survives the wipe)"
+    else
+      echo "   logs:      (none — $stagedir has no PARTUUID; installer output won't survive the wipe)"
+    fi
     echo "   one-shot:  $boot_mode"
     read -rp ">> type 'yes' to build the installer, reboot into it and wipe $osdisk_real: " ok
     [ "$ok" = yes ] || die "aborted"
@@ -426,6 +442,9 @@ local_install_prepare_and_reboot() {
   local cmdline volumeID
   volumeID="$(nix eval --raw .#nixosConfigurations.installer-iso.config.isoImage.volumeID)"
   cmdline="init=$toplevel/init nohibernate root=LABEL=$volumeID boot.shell_on_fail loglevel=4 lsm=landlock,yama,bpf findiso=$iso_relpath homelab.install=$config homelab.keypart=$boot_partuuid"
+  # Only when the staging partition has a PARTUUID (see stage_partuuid). Points
+  # the installer's homelab-auto-install.service at the surviving disk to log to.
+  [ -n "$stage_partuuid" ] && cmdline="$cmdline homelab.logpart=$stage_partuuid"
 
   case "$boot_mode" in
     systemd-boot)