diff --git a/CLAUDE.md b/CLAUDE.md index 53a0648..a64c86e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -131,6 +131,20 @@ kept its ssh host key. Run it after ANY change to the kexec paths. `HOMELAB_INSTALLER_STAGE_DIR` skips the prompt for scripted use. Both this and `kexec-local` self-elevate via `sudo` (`require_root()`) rather than requiring you to prefix the command yourself. +- **terra runs Limine, not systemd-boot** — `bootctl set-oneshot` is useless there + (`bootctl status` lists `✗ One-shot entry control`, and CachyOS's pacman hooks + regenerate `limine.conf` anyway). `local_install_prepare_and_reboot()` therefore + picks its one-shot mechanism at runtime: a systemd-boot loader entry when + `$BOOT/loader/entries` exists, otherwise `arm_efi_bootnext()` — a temporary UEFI + entry that EFI-stub-boots the kernel off the ESP, armed via the firmware's + `BootNext`. Created with `efibootmgr --create-only` (NOT `--create`, which pushes + it to the front of `BootOrder` and would make a wiped installer the permanent + default if anything went wrong). BootNext is spent by that one boot, so a failed + attempt still comes back on the normal bootloader. The EFI-stub path needs the + kernel on the **ESP** itself, not on a separate XBOOTLDR — hence `--print-esp-path` + rather than `--print-boot-path` in that mode. `homelab-auto-install.service` deletes + the leftover NVRAM entry as soon as it boots; both it and the script match the label + `Homelab Installer` EXACTLY (a prefix match would delete the Windows or Limine entry). - **The staging-dir guard must fail CLOSED, and `findmnt` needs `--nofsroot`**: on btrfs `findmnt -no SOURCE` prints `/dev/sdb2[/@]`, which `lsblk` cannot open, so a naive parent-device lookup comes back empty. Treating empty as "different disk" silently diff --git a/README.md b/README.md index 014f95d..8718fe1 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,8 @@ Manual alternative (USB ISO): boot installer, `disko` the disk, then ## First install on terra — no-USB findiso reinstall (replacing CachyOS) terra is a Ryzen 9 5900X / Radeon RX 6800 XT desktop, currently running -CachyOS with a writable root and systemd-boot. Everything is already prepped +CachyOS with a writable root and **Limine** as its bootloader (not +systemd-boot — see step 4). Everything is already prepped in this repo: real OS-disk id in `disk-config.nix`, real login pubkey in `common.nix`, terra's age recipient in `.sops.yaml`, its host key pre-generated at `~/.config/homelab/terra/`, and `secrets/terra.yaml` already @@ -137,10 +138,20 @@ self-elevates: pre-generated ssh host key** to the boot partition (found via `bootctl --print-boot-path`, not assumed to be `/boot`) and the iso to the staging dir. -4. Writes a systemd-boot **one-shot** entry (`bootctl set-oneshot` — reverts to - the normal default after this one boot, even if it fails) with `findiso=` + - `homelab.install=terra` + `homelab.keypart=` on its kernel - cmdline, and reboots — a real `systemctl reboot`, not kexec. +4. Arms a **one-shot** boot of it with `findiso=` + `homelab.install=terra` + + `homelab.keypart=` on the kernel cmdline, and reboots — a real + `systemctl reboot`, not kexec. Two mechanisms, picked automatically: + - **systemd-boot** (jupiter, neptun, and terra once NixOS is on it): a + `bootctl set-oneshot` loader entry. + - **anything else** — terra today runs Limine, which reports `One-shot entry + control: ✗` and has no equivalent: the firmware's own **`BootNext`**, + pointing at a temporary UEFI entry that EFI-stub-boots the kernel straight + off the ESP. Created with `--create-only` so it never enters `BootOrder`, + which means it is reachable exactly once and nothing else changes. + + Either way the box falls back to its normal bootloader if the attempt + fails — nothing is made permanent before the install succeeds. The + temporary UEFI entry is deleted by the installer as soon as it boots. The booted installer clones the repo (`homelab-checkout.service`, needs network — it's public now, no credentials involved) and then diff --git a/flake.nix b/flake.nix index 65e7c0d..027abdb 100644 --- a/flake.nix +++ b/flake.nix @@ -252,7 +252,7 @@ after = [ "homelab-checkout.service" ]; requires = [ "homelab-checkout.service" ]; wantedBy = [ "multi-user.target" ]; - path = [ pkgs.gnugrep pkgs.util-linux pkgs.coreutils pkgs.git ]; + path = [ pkgs.gnugrep pkgs.gnused pkgs.util-linux pkgs.coreutils pkgs.git pkgs.efibootmgr ]; serviceConfig.Type = "oneshot"; # systemd does NOT set $HOME for a system service without User= # (systemd.exec(5): SetLoginEnvironment= "defaults to true if @@ -294,6 +294,18 @@ fi fi + # On a box whose old bootloader had no one-shot (Limine on + # terra), scripts/deploy got us here via a temporary UEFI + # entry + BootNext (arm_efi_bootnext). BootNext is already + # spent, but the entry itself would linger in NVRAM pointing + # at a partition disko is about to reformat. Drop it now, so + # even an install that fails later leaves NVRAM clean. + for n in $(efibootmgr 2>/dev/null \ + | sed -n 's/^Boot\([0-9A-Fa-f]\{4\}\)\*\?[[:space:]]Homelab Installer[[:space:]].*/\1/p'); do + echo "removing temporary UEFI entry Boot$n" + efibootmgr -q -B -b "$n" || true + done + echo "auto-installing $cfg (homelab.install= on the kernel cmdline)" cd /root/homelab exec ./scripts/deploy install "$cfg" localhost --yes diff --git a/scripts/deploy b/scripts/deploy index 45bfb14..6afb64f 100755 --- a/scripts/deploy +++ b/scripts/deploy @@ -132,6 +132,72 @@ disks_backing() { lsblk -rnso NAME,TYPE "$1" 2>/dev/null | awk '$2 == "disk" { print "/dev/" $1 }' } +# Label of the temporary UEFI boot entry arm_efi_bootnext() creates. Also the +# key the ISO uses to delete it again once it has booted (see flake.nix). +EFI_LABEL="Homelab Installer" + +# Boot numbers of every UEFI entry with exactly this label, one per line. +# efibootmgr prints `Boot0002* LimineHD(1,GPT,...)/\EFI\...`, so the +# label runs from past the "Boot####* " prefix up to the first TAB. +# (Character classes spelled out rather than {4}: mawk predates ERE intervals.) +efi_entries_named() { + efibootmgr 2>/dev/null | awk -v want="$1" ' + /^Boot[0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f]/ { + num = substr($0, 5, 4) + rest = substr($0, 9) + sub(/^\*/, "", rest); sub(/^ +/, "", rest) + split(rest, parts, "\t") + if (parts[1] == want) print num + }' +} + +# Arm a genuine one-shot boot of the staged installer WITHOUT any help from the +# bootloader: create a UEFI boot entry that EFI-stub-boots the kernel straight +# off the ESP, and point BootNext at it. +# +# Needed because "boot this once, then go back to normal" is not something +# every bootloader can do. systemd-boot has it; terra's CachyOS runs Limine, +# which reports `One-shot entry control: ✗` and has no equivalent, and whose +# limine.conf is regenerated by pacman hooks anyway. BootNext is a firmware +# feature, so it works underneath all of them — and the firmware clears it +# after that one boot, which is what keeps the "a failed attempt still comes +# back on the normal bootloader" property that makes this safe to try. +arm_efi_bootnext() { + local esp="$1" cmdline="$2" + local esp_src esp_disk esp_part num n + need efibootmgr + esp_src="$(findmnt -no SOURCE --nofsroot --target "$esp")" \ + || die "couldn't resolve $esp to a device" + esp_disk="$(disks_backing "$esp_src" | head -1 || true)" + esp_part="$(cat "/sys/class/block/$(basename "$esp_src")/partition" 2>/dev/null || true)" + { [ -n "$esp_disk" ] && [ -n "$esp_part" ]; } \ + || die "couldn't work out the disk + partition number of the ESP ($esp -> $esp_src)" + + # Clear anything left by an earlier attempt first, so repeated runs don't + # slowly fill NVRAM with dead entries pointing at a wiped partition. + for n in $(efi_entries_named "$EFI_LABEL"); do + echo ">> removing stale UEFI entry Boot$n ($EFI_LABEL)" + efibootmgr -q -B -b "$n" + done + + # --create-only, NOT --create: the latter also pushes the entry to the front + # of BootOrder, which would make a wiped installer the permanent default if + # anything went wrong. This way the entry is reachable through BootNext and + # nothing else, i.e. exactly once. + # + # The EFI stub loads `initrd=` off the volume it was itself loaded from, so + # the path is relative to the ESP root and uses backslashes. + efibootmgr -q --create-only --disk "$esp_disk" --part "$esp_part" \ + --label "$EFI_LABEL" \ + --loader '\homelab-installer\bzImage' \ + --unicode "initrd=\\homelab-installer\\initrd $cmdline" + + num="$(efi_entries_named "$EFI_LABEL" | head -1)" + [ -n "$num" ] || die "efibootmgr did not create a '$EFI_LABEL' entry" + efibootmgr -q --bootnext "$num" + echo ">> UEFI BootNext -> Boot$num ($EFI_LABEL); BootOrder untouched" +} + # Sets tb / cpio / bbox — the kexec tarball plus the static cpio+gzip that # kexec-run.sh needs on PATH to rebuild its initrd. # @@ -194,15 +260,30 @@ local_install_prepare_and_reboot() { need stat need df + # Where to stage the installer, and how to make the box boot it exactly once. + # # systemd-boot keeps its entries on $BOOT — the XBOOTLDR partition when there # is one, the ESP otherwise — which is not always /boot. Hardcoding /boot on # a box that mounts its ESP elsewhere just creates a directory on the root - # filesystem, and then reboots into an entry the firmware never sees. - local boot - boot="$(bootctl --print-boot-path 2>/dev/null)" \ - || die "bootctl couldn't locate the boot partition — is systemd-boot installed here?" - [ -d "$boot/loader/entries" ] \ - || die "$boot/loader/entries doesn't exist — systemd-boot isn't installed on this box" + # filesystem and then reboots into an entry the firmware never sees. + # + # No systemd-boot (terra's CachyOS runs Limine) means no `bootctl set-oneshot`, + # so fall back to the firmware's own BootNext — see arm_efi_bootnext(). That + # path EFI-stub-boots the kernel directly, which requires it to sit on the ESP + # itself rather than on a separate XBOOTLDR. + local boot boot_mode esp + esp="$(bootctl --print-esp-path 2>/dev/null)" \ + || die "bootctl couldn't locate the ESP — is this box actually UEFI-booted?" + boot="$(bootctl --print-boot-path 2>/dev/null || echo "$esp")" + if [ -d "$boot/loader/entries" ]; then + boot_mode=systemd-boot + else + boot_mode=efi-bootnext + boot="$esp" + need efibootmgr + echo ">> no systemd-boot entries at $boot/loader/entries — arming the firmware's" + echo " own BootNext instead (bootloader in charge here: $(bootctl status 2>/dev/null | awk '/Product:/ {$1=""; print substr($0,2); exit}' || echo unknown))" + fi # No default/auto-picked location — the wrong disk here is destroyed # mid-install (see the OS-disk check below), so this always asks rather @@ -273,7 +354,7 @@ local_install_prepare_and_reboot() { echo " -> $osdisk_real ** WIPED, unattended, after the reboot **" # Unquoted on purpose: collapses the one-per-line list onto one line. echo " staging: $stagedir (on $(echo $stage_disks))" - echo " entry: $boot/loader/entries/homelab-installer.conf (one-shot)" + echo " one-shot: $boot_mode" read -rp ">> type 'yes' to build the installer, reboot into it and wipe $osdisk_real: " ok [ "$ok" = yes ] || die "aborted" fi @@ -330,15 +411,28 @@ local_install_prepare_and_reboot() { mnt_point="$(findmnt -no TARGET --target "$stagedir")" iso_relpath="$(printf '/%s/%s' "${stagedir#"$mnt_point"}" homelab-installer.iso | tr -s /)" - cat >"$boot/loader/entries/homelab-installer.conf" <"$boot/loader/entries/homelab-installer.conf" <> systemd-boot one-shot entry armed" + ;; + efi-bootnext) + arm_efi_bootnext "$boot" "$cmdline" + ;; + esac - echo ">> one-shot boot into the installer, then rebooting — it will finish this install itself" - bootctl set-oneshot homelab-installer.conf + echo ">> rebooting into the installer — it will finish this install itself" systemctl reboot }