darmanandClaude Sonnet 5
6f24ab69ad
Comments had drifted into multi-paragraph narrative (git commit lineage, debugging stories, restated code) in several hot spots (scripts/deploy, hermes-agent.nix, flake.nix, gitea.nix, headscale.nix). Trim every comment to its load-bearing "why" — gotchas, safety warnings, and non-obvious rationale survive verbatim in substance, just tightened to 1-2 sentences; historical narrative and anything already covered in CLAUDE.md is cut. No code/logic changed. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UJqEmY1y3AYX3JoX4Y6b21
769 lines
38 KiB
Bash
Executable File
769 lines
38 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Deploy a NixOS host from this flake. ALL arguments are mandatory (no defaults).
|
|
#
|
|
# ./deploy kexec <config> <host> ZimaOS/RO-root box: kexec into a RAM installer, ships the ssh key, then run `install`.
|
|
# ./deploy kexec-local [--yes] kexec THIS machine (no ssh) into the RAM installer; disks untouched. Then `install <config> localhost`.
|
|
# ./deploy install <config> <host> [--yes]
|
|
# first install (wipes the OS disk, ships the host's sops key). localhost only
|
|
# runs disko/nixos-install directly once already inside a live installer;
|
|
# from a real running OS it stages installer-iso and reboots into that instead.
|
|
# See CLAUDE.md.
|
|
# ./deploy switch <config> <host> rebuild + activate on a running host.
|
|
# ./deploy boot <config> <host> stage for next boot, don't activate now.
|
|
# ./deploy test <config> <host> activate without adding a boot entry.
|
|
# ./deploy image <config> build an SD-card image (e.g. rpi mercury).
|
|
# ./deploy flash <config> <dev> build SD image, write to <dev>, and drop the sops age key onto it if one exists.
|
|
#
|
|
# <config> = a nixosConfigurations name. Its pre-generated SSH host key must be
|
|
# at ~/.config/homelab/<config>/ssh_host_ed25519_key. Runs from a non-NixOS host too.
|
|
#
|
|
# Password prompts auto-fill from the "HomeLab" Proton Pass vault, keyed by
|
|
# <config> not <host> (darman@<config> for sudo, root@<config> for ssh).
|
|
# Override with HOMELAB_PASS_ITEM / HOMELAB_PASS_ROOT_ITEM / HOMELAB_PASS_VAULT.
|
|
set -euo pipefail
|
|
shopt -s nullglob
|
|
|
|
# Captured before $@ is parsed, so require_root() can re-exec the ORIGINAL
|
|
# invocation under sudo (inside a function, "$@" is the function's own args).
|
|
SCRIPT_ARGS=("$@")
|
|
|
|
# Locate the repo root (flake dir) regardless of where this script lives on disk.
|
|
SCRIPT_PATH="$(realpath "$0")"
|
|
SCRIPT_DIR="$(dirname "$SCRIPT_PATH")"
|
|
REPO="$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel 2>/dev/null || dirname "$SCRIPT_DIR")"
|
|
cd "$REPO"
|
|
export PATH="/nix/var/nix/profiles/default/bin:$PATH"
|
|
|
|
# A stock NixOS box (unlike a Determinate-Nix laptop) leaves nix-command/flakes
|
|
# disabled, and that's exactly the prepare host for `install <config> localhost`.
|
|
# Enable them additively so root gets them too after the require_root() re-exec.
|
|
export NIX_CONFIG="$(printf 'extra-experimental-features = nix-command flakes\n%s' "${NIX_CONFIG:-}")"
|
|
|
|
# Off-repo material keyed by <config>: pre-generated SSH host keys (install)
|
|
# and per-config sops age keys (flash). Resolved defensively, not as a bare
|
|
# $HOME, since systemd doesn't set $HOME for a service without User= — this
|
|
# also runs unattended from installer-iso's homelab-auto-install.service.
|
|
KEYDIR="${HOMELAB_KEY_DIR:-${HOME:-/root}/.config/homelab}"
|
|
|
|
die() { echo "error: $*" >&2; exit 1; }
|
|
|
|
need() { command -v "$1" >/dev/null 2>&1 || die "missing required tool: $1"; }
|
|
|
|
# Self-elevate instead of dying: re-exec this exact invocation under sudo.
|
|
# -E preserves HOMELAB_*/vault env vars; pin HOMELAB_KEY_DIR too since whether
|
|
# sudo carries $HOME across depends on the local sudoers policy. No-op if already root.
|
|
require_root() {
|
|
[ "$(id -u)" = 0 ] && return 0
|
|
echo ">> $1 needs root — re-executing under sudo" >&2
|
|
export HOMELAB_KEY_DIR="$KEYDIR"
|
|
exec sudo -E -- "$SCRIPT_PATH" "${SCRIPT_ARGS[@]}"
|
|
}
|
|
|
|
# Exactly one path matching a glob, or die. `ls glob | head -1` silently yields
|
|
# an empty string when nothing matches (head exits 0, so set -e never fires).
|
|
one_match() {
|
|
local what="$1"; shift
|
|
local f=("$@") # caller expands the glob (nullglob is on)
|
|
[ "${#f[@]}" -gt 0 ] || die "no $what found — did the build actually produce one?"
|
|
# A stale result-sd/ symlink from an earlier config is how you'd otherwise
|
|
# flash the wrong image without a word — warn instead of silently taking [0].
|
|
[ "${#f[@]}" -eq 1 ] \
|
|
|| echo ">> warning: ${#f[@]} candidates for $what, using ${f[0]} (rm the stale ones)" >&2
|
|
printf '%s\n' "${f[0]}"
|
|
}
|
|
|
|
# Every whole-disk device backing a block device or mounted path, one per line.
|
|
# LVM/RAID/LUKS can span several disks at once (e.g. terra's /mnt/ssd_01),
|
|
# so callers must treat empty output as "unknown", not "safe".
|
|
disks_backing() {
|
|
lsblk -rnso NAME,TYPE "$1" 2>/dev/null | awk '$2 == "disk" { print "/dev/" $1 }'
|
|
}
|
|
|
|
# Label of the temporary UEFI boot entry arm_efi_bootnext() creates; also what
|
|
# the booted ISO matches to delete it again (flake.nix) — must match EXACTLY.
|
|
EFI_LABEL="Homelab Installer"
|
|
|
|
# Boot numbers of every UEFI entry with exactly this label, one per line.
|
|
# (Character classes spelled out rather than {4}: mawk predates ERE intervals.)
|
|
efi_entries_named() {
|
|
efibootmgr 2>/dev/null | awk -v want="$1" '
|
|
/^Boot[0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f]/ {
|
|
num = substr($0, 5, 4)
|
|
rest = substr($0, 9)
|
|
sub(/^\*/, "", rest); sub(/^ +/, "", rest)
|
|
split(rest, parts, "\t")
|
|
if (parts[1] == want) print num
|
|
}'
|
|
}
|
|
|
|
# Arm a genuine one-shot boot of the staged installer without bootloader help:
|
|
# create a UEFI entry that EFI-stub-boots the kernel off the ESP and point
|
|
# BootNext at it. Needed because Limine (terra's CachyOS) has no one-shot
|
|
# entry support; BootNext is a firmware feature so it works underneath any
|
|
# bootloader, and the firmware clears it after one boot either way.
|
|
arm_efi_bootnext() {
|
|
local esp="$1" cmdline="$2"
|
|
local esp_src esp_disk esp_part num n
|
|
need efibootmgr
|
|
esp_src="$(findmnt -no SOURCE --nofsroot --target "$esp")" \
|
|
|| die "couldn't resolve $esp to a device"
|
|
esp_disk="$(disks_backing "$esp_src" | head -1 || true)"
|
|
esp_part="$(cat "/sys/class/block/$(basename "$esp_src")/partition" 2>/dev/null || true)"
|
|
{ [ -n "$esp_disk" ] && [ -n "$esp_part" ]; } \
|
|
|| die "couldn't work out the disk + partition number of the ESP ($esp -> $esp_src)"
|
|
|
|
# Clear anything left by an earlier attempt so NVRAM doesn't slowly fill
|
|
# with dead entries pointing at a wiped partition.
|
|
for n in $(efi_entries_named "$EFI_LABEL"); do
|
|
echo ">> removing stale UEFI entry Boot$n ($EFI_LABEL)"
|
|
efibootmgr -q -B -b "$n"
|
|
done
|
|
|
|
# --create-only, NOT --create: --create also pushes the entry to the front of
|
|
# BootOrder, which would make a wiped installer the permanent default on any
|
|
# failure. This way it's reachable only via BootNext, exactly once. The EFI
|
|
# stub loads `initrd=` relative to the ESP root, hence the backslash path.
|
|
efibootmgr -q --create-only --disk "$esp_disk" --part "$esp_part" \
|
|
--label "$EFI_LABEL" \
|
|
--loader '\homelab-installer\bzImage' \
|
|
--unicode "initrd=\\homelab-installer\\initrd $cmdline"
|
|
|
|
num="$(efi_entries_named "$EFI_LABEL" | head -1)"
|
|
[ -n "$num" ] || die "efibootmgr did not create a '$EFI_LABEL' entry"
|
|
efibootmgr -q --bootnext "$num"
|
|
echo ">> UEFI BootNext -> Boot$num ($EFI_LABEL); BootOrder untouched"
|
|
}
|
|
|
|
# Sets tb / cpio / bbox — the kexec tarball plus the static cpio+gzip that
|
|
# kexec-run.sh needs on PATH to rebuild its initrd. HOMELAB_KEXEC_TARBALL (+
|
|
# _CPIO/_GZIP) short-circuits the build to reuse a prebuilt installer instead.
|
|
kexec_artifacts() {
|
|
if [ -n "${HOMELAB_KEXEC_TARBALL:-}" ]; then
|
|
tb="$HOMELAB_KEXEC_TARBALL"
|
|
[ -f "$tb" ] || die "HOMELAB_KEXEC_TARBALL=$tb is not a file"
|
|
cpio="${HOMELAB_KEXEC_CPIO:-$(command -v cpio || true)}"
|
|
bbox="${HOMELAB_KEXEC_GZIP:-$(command -v gzip || true)}"
|
|
{ [ -n "$cpio" ] && [ -n "$bbox" ]; } \
|
|
|| die "set HOMELAB_KEXEC_CPIO / HOMELAB_KEXEC_GZIP, or put cpio+gzip on PATH"
|
|
echo ">> using prebuilt kexec installer: $tb"
|
|
else
|
|
need nix
|
|
echo ">> building kexec installer + static tools"
|
|
nix build .#nixosConfigurations.kexec.config.system.build.kexecInstallerTarball \
|
|
-o result-kexec
|
|
tb="$(one_match 'kexec tarball' result-kexec/*.tar.gz)"
|
|
cpio="$(nix build --no-link --print-out-paths nixpkgs#pkgsStatic.cpio)/bin/cpio"
|
|
bbox="$(nix build --no-link --print-out-paths nixpkgs#pkgsStatic.busybox)/bin/busybox"
|
|
fi
|
|
}
|
|
|
|
# True inside one of this repo's throwaway live-installer environments
|
|
# (nixos-installer from kexec, or homelab-installer from installer-iso) —
|
|
# i.e. `install <config> localhost` should wipe/install right here, not
|
|
# prepare-and-reboot (see local_install_prepare_and_reboot).
|
|
is_live_installer() {
|
|
case "$(uname -n)" in
|
|
nixos-installer | homelab-installer) return 0 ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
# `install <config> localhost` on a REAL running OS (not yet inside a live
|
|
# installer): stages installer-iso's kernel/initrd + host key on the boot
|
|
# partition, arms a one-shot boot with homelab.install=<config> on its
|
|
# cmdline, and does a real ACPI reboot — deliberately not a kexec jump, per
|
|
# terra's kexec-local gotcha in CLAUDE.md. The booted installer re-runs this
|
|
# same command itself once its repo checkout succeeds, finishing unattended.
|
|
local_install_prepare_and_reboot() {
|
|
local config="$1" hostkey="$2" assume_yes="$3"
|
|
require_root "preparing a local reinstall"
|
|
[ -d /sys/firmware/efi ] || die "not booted UEFI — the one-shot boot entry needs systemd-boot"
|
|
need bootctl
|
|
need nix
|
|
need lsblk
|
|
need findmnt
|
|
need awk
|
|
need realpath
|
|
need stat
|
|
need df
|
|
|
|
# Where to stage the installer: use bootctl's reported $BOOT (XBOOTLDR or the
|
|
# ESP), not a hardcoded /boot, since that's not always where the ESP mounts.
|
|
# No systemd-boot (terra's CachyOS runs Limine) means no `bootctl
|
|
# set-oneshot`, so fall back to firmware BootNext (arm_efi_bootnext()) —
|
|
# which EFI-stub-boots the kernel directly and needs it on the ESP itself,
|
|
# not a separate XBOOTLDR.
|
|
local boot boot_mode esp
|
|
esp="$(bootctl --print-esp-path 2>/dev/null)" \
|
|
|| die "bootctl couldn't locate the ESP — is this box actually UEFI-booted?"
|
|
boot="$(bootctl --print-boot-path 2>/dev/null || echo "$esp")"
|
|
if [ -d "$boot/loader/entries" ]; then
|
|
boot_mode=systemd-boot
|
|
else
|
|
boot_mode=efi-bootnext
|
|
boot="$esp"
|
|
need efibootmgr
|
|
echo ">> no systemd-boot entries at $boot/loader/entries — arming the firmware's"
|
|
echo " own BootNext instead (bootloader in charge here: $(bootctl status 2>/dev/null | awk '/Product:/ {$1=""; print substr($0,2); exit}' || echo unknown))"
|
|
fi
|
|
|
|
# No default/auto-picked location: the wrong disk here is destroyed
|
|
# mid-install (see the OS-disk check below), so this always asks unless
|
|
# HOMELAB_INSTALLER_STAGE_DIR is set for scripted use.
|
|
local stagedir="${HOMELAB_INSTALLER_STAGE_DIR:-}"
|
|
if [ -z "$stagedir" ]; then
|
|
echo ">> currently mounted filesystems:"
|
|
lsblk -o NAME,SIZE,FSTYPE,MOUNTPOINT
|
|
read -rp ">> path to stage the installer iso on (must NOT be on the OS disk being wiped): " stagedir
|
|
fi
|
|
[ -n "$stagedir" ] || die "no staging path given"
|
|
[ -d "$stagedir" ] \
|
|
|| die "staging dir $stagedir doesn't exist — needs to be an existing partition that is NOT the OS disk being wiped"
|
|
# Absolute + symlink-free: findiso= below is computed by stripping the
|
|
# mountpoint prefix off this, and a relative answer at the prompt would
|
|
# produce a path the initrd can never resolve.
|
|
stagedir="$(realpath "$stagedir")"
|
|
|
|
# Refuse if the staging partition turns out to live on the same disk
|
|
# disko is about to wipe — the iso file (and the running installer
|
|
# loopback-mounted from it) would be destroyed mid-install.
|
|
local osdisk osdisk_real stage_src stage_fstype stage_disks d
|
|
osdisk="$(nix eval --raw ".#nixosConfigurations.$config.config.disko.devices.disk" \
|
|
--apply 'd: (builtins.head (builtins.attrValues d)).device' 2>/dev/null)" \
|
|
|| die "couldn't read the OS disk device from hosts/$config/disk-config.nix"
|
|
osdisk_real="$(readlink -f "$osdisk")"
|
|
|
|
# --nofsroot matters: on btrfs findmnt prints the subvolume as
|
|
# `/dev/sdb2[/@]`, which lsblk can't open, silently skipping the guard
|
|
# below and allowing staging on the disk about to be wiped (terra's layout).
|
|
stage_src="$(findmnt -no SOURCE --nofsroot --target "$stagedir")" \
|
|
|| die "$stagedir doesn't resolve to a mounted filesystem"
|
|
# `|| true` so the fail-closed check below reports the problem, rather than
|
|
# `set -e`/pipefail silently killing the script on lsblk's nonzero exit.
|
|
stage_disks="$(disks_backing "$stage_src" || true)"
|
|
# Fail closed. "Couldn't determine the disk" is not "different disk".
|
|
[ -n "$stage_disks" ] \
|
|
|| die "couldn't determine which physical disk $stagedir ($stage_src) is on — refusing to guess, since being wrong destroys the install mid-flight"
|
|
for d in $stage_disks; do
|
|
if [ "$d" = "$osdisk_real" ]; then
|
|
die "$stagedir is on the OS disk ($osdisk -> $osdisk_real) that install would wipe — re-run and pick a different disk"
|
|
fi
|
|
done
|
|
|
|
# stage-1 mounts a btrfs volume's TOP level to resolve findiso=, so a path
|
|
# inside a subvolume is unreachable and the box boots to an emergency shell
|
|
# after it's already left the working OS. Refuse btrfs staging outright.
|
|
stage_fstype="$(findmnt -no FSTYPE --target "$stagedir")"
|
|
[ "$stage_fstype" != btrfs ] \
|
|
|| die "$stagedir is btrfs: findiso= mounts the volume's top level, so a path inside a subvolume never resolves. Stage on a non-btrfs partition (ext4/vfat/ntfs)."
|
|
|
|
# PARTUUID of the staging partition, handed to the installer as
|
|
# homelab.logpart= so it can persist the whole install's log there — it's on
|
|
# a different disk than the one disko wipes, so it survives a failed
|
|
# install. Best-effort: an LVM/mdraid stage_src has no PARTUUID, so logging
|
|
# is simply skipped rather than blocking the install.
|
|
local stage_partuuid
|
|
stage_partuuid="$(lsblk -no PARTUUID "$stage_src" 2>/dev/null | head -1 | tr -d ' ' || true)"
|
|
|
|
# Last chance to back out: this reboots the machine you're typing at into an
|
|
# unattended wipe, so it confirms like `flash`/`kexec-local` do.
|
|
if [ "$assume_yes" != "--yes" ]; then
|
|
echo ">> about to REINSTALL this machine from scratch:"
|
|
echo " hostname: $(uname -n)"
|
|
echo " config: $config"
|
|
echo " OS disk: $osdisk"
|
|
echo " -> $osdisk_real ** WIPED, unattended, after the reboot **"
|
|
# Unquoted on purpose: collapses the one-per-line list onto one line.
|
|
echo " staging: $stagedir (on $(echo $stage_disks))"
|
|
if [ -n "$stage_partuuid" ]; then
|
|
echo " logs: $stagedir/homelab-install-$config.log (on the staging disk — survives the wipe)"
|
|
else
|
|
echo " logs: (none — $stagedir has no PARTUUID; installer output won't survive the wipe)"
|
|
fi
|
|
echo " one-shot: $boot_mode"
|
|
read -rp ">> type 'yes' to build the installer, reboot into it and wipe $osdisk_real: " ok
|
|
[ "$ok" = yes ] || die "aborted"
|
|
fi
|
|
|
|
echo ">> building installer-iso (kernel + initrd + iso image)"
|
|
local kernel initrd isodir iso toplevel mnt_point iso_relpath boot_src boot_partuuid
|
|
kernel="$(nix build --no-link --print-out-paths .#nixosConfigurations.installer-iso.config.system.build.kernel)/bzImage"
|
|
initrd="$(nix build --no-link --print-out-paths .#nixosConfigurations.installer-iso.config.system.build.initialRamdisk)/initrd"
|
|
isodir="$(nix build --no-link --print-out-paths .#nixosConfigurations.installer-iso.config.system.build.isoImage)"
|
|
iso="$(one_match 'installer iso' "$isodir"/iso/*.iso)"
|
|
# The grub/isolinux menu normally supplies init=<toplevel>/init; EFI-stub
|
|
# booting our own cmdline means we must pass it too, or stage 1 dies on
|
|
# "stage 2 init script (/mnt-root//init) not found".
|
|
toplevel="$(nix build --no-link --print-out-paths .#nixosConfigurations.installer-iso.config.system.build.toplevel)"
|
|
|
|
# Check space before writing: a short write isn't visible until reboot,
|
|
# when findiso finds a truncated ~1GB iso and drops to an emergency shell.
|
|
local need_stage need_boot avail_stage avail_boot
|
|
need_stage="$(stat -Lc %s "$iso")"
|
|
need_boot="$(( $(stat -Lc %s "$kernel") + $(stat -Lc %s "$initrd") + $(stat -Lc %s "$hostkey") ))"
|
|
avail_stage="$(df -B1 --output=avail "$stagedir" | tail -1 | tr -d ' ')"
|
|
avail_boot="$(df -B1 --output=avail "$boot" | tail -1 | tr -d ' ')"
|
|
[ "$avail_stage" -ge "$(( need_stage + 64 * 1024 * 1024 ))" ] \
|
|
|| die "$stagedir has $(( avail_stage / 1024 / 1024 ))MB free, the iso needs $(( need_stage / 1024 / 1024 ))MB — pick another partition"
|
|
[ "$avail_boot" -ge "$(( need_boot + 16 * 1024 * 1024 ))" ] \
|
|
|| die "$boot has $(( avail_boot / 1024 / 1024 ))MB free, kernel+initrd need $(( need_boot / 1024 / 1024 ))MB"
|
|
|
|
echo ">> staging kernel/initrd/host key on $boot, iso image on $stagedir"
|
|
install -Dm644 "$kernel" "$boot/homelab-installer/bzImage"
|
|
install -Dm644 "$initrd" "$boot/homelab-installer/initrd"
|
|
install -Dm644 "$iso" "$stagedir/homelab-installer.iso"
|
|
|
|
# The ISO is built from a public repo with no credentials, so the host key
|
|
# must travel with the staged installer or sops can't decrypt on boot #1
|
|
# (README). $boot is on the OS disk, so disko destroys this copy minutes later.
|
|
install -Dm600 "$hostkey" "$boot/homelab-installer/ssh_host_ed25519_key"
|
|
install -Dm644 "$hostkey.pub" "$boot/homelab-installer/ssh_host_ed25519_key.pub"
|
|
boot_src="$(findmnt -no SOURCE --nofsroot --target "$boot")" \
|
|
|| die "couldn't resolve $boot to a device"
|
|
boot_partuuid="$(lsblk -no PARTUUID "$boot_src" 2>/dev/null | head -1 | tr -d ' ' || true)"
|
|
[ -n "$boot_partuuid" ] \
|
|
|| die "couldn't read a PARTUUID for $boot ($boot_src) — the installer needs it to find the host key"
|
|
|
|
# findiso= is relative to whichever partition the initrd finds it on, and
|
|
# must KEEP its leading slash: stage-1 tests `-e /findiso$isoPath`, so a bare
|
|
# `var/tmp/x.iso` becomes `/findisovar/tmp/x.iso` and never matches.
|
|
mnt_point="$(findmnt -no TARGET --target "$stagedir")"
|
|
iso_relpath="$(printf '/%s/%s' "${stagedir#"$mnt_point"}" homelab-installer.iso | tr -s /)"
|
|
|
|
# root=LABEL=<volumeID> matches what the ISO menu passes (findiso overwrites
|
|
# /dev/root regardless); boot.shell_on_fail gives a shell instead of a
|
|
# reboot/ignore prompt if stage 1 fails again.
|
|
local cmdline volumeID
|
|
volumeID="$(nix eval --raw .#nixosConfigurations.installer-iso.config.isoImage.volumeID)"
|
|
cmdline="init=$toplevel/init nohibernate root=LABEL=$volumeID boot.shell_on_fail loglevel=4 lsm=landlock,yama,bpf findiso=$iso_relpath homelab.install=$config homelab.keypart=$boot_partuuid"
|
|
# Only when the staging partition has a PARTUUID (see stage_partuuid). Points
|
|
# the installer's homelab-auto-install.service at the surviving disk to log to.
|
|
[ -n "$stage_partuuid" ] && cmdline="$cmdline homelab.logpart=$stage_partuuid"
|
|
|
|
case "$boot_mode" in
|
|
systemd-boot)
|
|
cat >"$boot/loader/entries/homelab-installer.conf" <<EOF
|
|
title Homelab Installer ($config, findiso)
|
|
linux /homelab-installer/bzImage
|
|
initrd /homelab-installer/initrd
|
|
options $cmdline
|
|
EOF
|
|
bootctl set-oneshot homelab-installer.conf
|
|
echo ">> systemd-boot one-shot entry armed"
|
|
;;
|
|
efi-bootnext)
|
|
arm_efi_bootnext "$boot" "$cmdline"
|
|
;;
|
|
esac
|
|
|
|
echo ">> rebooting into the installer — it will finish this install itself"
|
|
systemctl reboot
|
|
}
|
|
|
|
# Flakes only see git-tracked files: an untracked hosts/<config>/ is silently
|
|
# invisible to `nix build`/`nixos-install`, which then fails obscurely or builds
|
|
# a stale config. Check before doing anything destructive.
|
|
require_tracked() {
|
|
local config="$1" cfgfile="hosts/$1/configuration.nix" f
|
|
[ -e "$cfgfile" ] || die "no $cfgfile in the repo"
|
|
# No .git or no working tree (e.g. a tarball export) means nothing CAN be
|
|
# untracked — skip only on that, not on any other git failure.
|
|
command -v git >/dev/null 2>&1 || return 0
|
|
git -C "$REPO" rev-parse --is-inside-work-tree >/dev/null 2>&1 || return 0
|
|
# Every .nix in hosts/<config>/, not just configuration.nix — an untracked
|
|
# disk-config.nix decides which disk gets wiped and is just as invisible.
|
|
for f in "hosts/$config"/*.nix; do
|
|
git -C "$REPO" ls-files --error-unmatch "$f" >/dev/null 2>&1 \
|
|
|| die "$f is untracked — 'git add hosts/$config' first (flakes ignore untracked files)"
|
|
done
|
|
}
|
|
|
|
# The password field of a Proton Pass item, or empty if pass-cli is missing /
|
|
# logged out / has no such item — callers then fall back to an interactive
|
|
# prompt. Resolves the title among ACTIVE items first, because `item view
|
|
# --item-title` has no state filter and can silently match a trashed item of
|
|
# the same title instead, returning an empty password with exit 0 (hit for
|
|
# real on darman@neptun, which had both an Active and a Trashed copy).
|
|
proton_pass_password() {
|
|
local title="$1" vault="${HOMELAB_PASS_VAULT:-HomeLab}" id pw
|
|
command -v pass-cli >/dev/null 2>&1 || return 0
|
|
|
|
# Lines look like: - [ITEM_ID]: the title (state=Active)
|
|
id="$(pass-cli item list --vault-name "$vault" --filter-state active \
|
|
--output human 2>/dev/null \
|
|
| awk -v t="$title" '
|
|
{ i = index($0, "]: "); if (i == 0) next
|
|
id = substr($0, 4, i - 4)
|
|
rest = substr($0, i + 3)
|
|
sub(/ \(state=[^)]*\)$/, "", rest)
|
|
if (rest == t) { print id; exit } }' || true)"
|
|
|
|
if [ -n "$id" ]; then
|
|
pw="$(pass-cli item view --vault-name "$vault" --item-id "$id" \
|
|
--field password --output human 2>/dev/null | head -1 || true)"
|
|
# Resolved an active item but its password field is blank: worth saying so,
|
|
# otherwise this looks identical to having no vault entry at all.
|
|
[ -n "$pw" ] || echo ">> vault item '$title' resolved but its password field is empty" >&2
|
|
else
|
|
# No active match — fall back to the title lookup so an older pass-cli
|
|
# without --filter-state still works exactly as it used to.
|
|
pw="$(pass-cli item view --vault-name "$vault" --item-title "$title" \
|
|
--field password --output human 2>/dev/null | head -1 || true)"
|
|
fi
|
|
|
|
printf '%s' "$pw"
|
|
}
|
|
|
|
# Path to an sshpass binary (system one, else built from nixpkgs). Empty if
|
|
# neither is available.
|
|
sshpass_bin() {
|
|
command -v sshpass 2>/dev/null && return 0
|
|
nix build --no-link --print-out-paths nixpkgs#sshpass 2>/dev/null \
|
|
| sed 's|$|/bin/sshpass|'
|
|
}
|
|
|
|
cmd="${1:-}"; [ -n "$cmd" ] || die "usage: ./deploy <kexec|kexec-local|install|switch|boot|test|image|flash> ..."
|
|
|
|
case "$cmd" in
|
|
kexec)
|
|
config="${2:-}"; host="${3:-}"
|
|
{ [ -n "$config" ] && [ -n "$host" ]; } || die "usage: ./deploy kexec <config> <host>"
|
|
|
|
need ssh; need scp
|
|
|
|
# kexec/run rebuilds an initrd with `cpio` + `gzip` from PATH — ZimaOS lacks
|
|
# both. Ship static ones: GNU cpio (reliable -o -H newc), busybox as gzip.
|
|
kexec_artifacts
|
|
|
|
# One password prompt: multiplex scp + ssh over a shared control connection.
|
|
cm="/tmp/homelab-cm-%r@%h:%p"
|
|
o=(-o ControlMaster=auto -o "ControlPath=$cm" -o ControlPersist=300 \
|
|
-o StrictHostKeyChecking=accept-new)
|
|
|
|
# Root's password from Proton Pass, fed to ssh/scp via sshpass -e; only the
|
|
# first (master) connection authenticates, the rest ride the control socket.
|
|
# Exported rather than `env SSHPASS=... sshpass` to close the sub-millisecond
|
|
# argv-exposure race before exec (either way the secret only lives in environ).
|
|
sp=()
|
|
root_item="${HOMELAB_PASS_ROOT_ITEM:-root@$config}"
|
|
root_pw="$(proton_pass_password "$root_item" || true)"
|
|
if [ -n "$root_pw" ]; then
|
|
sshpass="$(sshpass_bin || true)"
|
|
if [ -n "$sshpass" ]; then
|
|
export SSHPASS="$root_pw"
|
|
sp=("$sshpass" -e)
|
|
# sshpass drives the password prompt; don't let a key/agent short-circuit
|
|
# into an interactive one for a host that only accepts passwords.
|
|
o+=(-o PreferredAuthentications=password -o PubkeyAuthentication=no)
|
|
else
|
|
echo ">> sshpass unavailable — falling back to the interactive prompt" >&2
|
|
fi
|
|
fi
|
|
unset root_pw
|
|
|
|
if [ "${#sp[@]}" -gt 0 ]; then
|
|
echo ">> connecting to root@$host (password from Proton Pass: $root_item)"
|
|
else
|
|
echo ">> connecting to root@$host (enter the root password once)"
|
|
fi
|
|
"${sp[@]+"${sp[@]}"}" ssh "${o[@]}" "root@$host" 'mkdir -p /tmp/bin'
|
|
scp "${o[@]}" "$cpio" "root@$host:/tmp/bin/cpio"
|
|
scp "${o[@]}" "$bbox" "root@$host:/tmp/bin/gzip" # busybox as gzip (argv0)
|
|
|
|
echo ">> streaming installer + kexec-ing. SSH drops as the box jumps into the"
|
|
echo " RAM installer. Disks are untouched."
|
|
rc=0
|
|
ssh "${o[@]}" "root@$host" \
|
|
'chmod +x /tmp/bin/*; mkdir -p /tmp/k && tar -C /tmp/k -xzf - && PATH=/tmp/bin:$PATH /tmp/k/kexec/run' \
|
|
< "$tb" || rc=$?
|
|
# A dropped connection IS the success case here, so this can't be fatal —
|
|
# but a full /tmp, a tar error or a missing static tool looks identical from
|
|
# the outside, so at least surface the code instead of swallowing it.
|
|
[ "$rc" -eq 0 ] || echo ">> ssh exited $rc — expected if the box jumped; suspect this if install can't connect" >&2
|
|
|
|
ssh "${o[@]}" -O exit "root@$host" 2>/dev/null || true # close control socket
|
|
unset SSHPASS
|
|
|
|
# NB: no ssh-keygen -R here on purpose — the kexec installer keeps the box's
|
|
# ssh host key (restore-remote-access.nix), so known_hosts is still valid.
|
|
|
|
echo ">> box is kexec-ing. Wait ~1-2 min for the installer + network, then:"
|
|
echo " ./deploy install $config $host"
|
|
;;
|
|
|
|
kexec-local)
|
|
# Build the same RAM installer as `kexec`, but run it directly on this box
|
|
# (no ssh/second machine). One-way trip on the machine you're typing at, so
|
|
# every check that can fail runs BEFORE the point of no return (see the
|
|
# trap discussion below).
|
|
require_root "kexec-local"
|
|
|
|
assume_yes=""
|
|
[ "${2:-}" = "--yes" ] && assume_yes=1
|
|
|
|
need tar; need install; need mktemp; need find; need sync; need nohup
|
|
|
|
# --- preflight: reasons the jump would fail, checked while it's still safe --
|
|
# CONFIG_KEXEC. Without it kexec --load fails and you've built ~1GB for
|
|
# nothing; with lockdown it fails at load time too (Secure Boot blocks the
|
|
# kexec_load syscall unless the image is signed).
|
|
[ -e /sys/kernel/kexec_loaded ] \
|
|
|| die "this kernel has no kexec support (CONFIG_KEXEC) — boot install media instead"
|
|
if [ -r /sys/kernel/security/lockdown ] \
|
|
&& ! grep -q '\[none\]' /sys/kernel/security/lockdown 2>/dev/null; then
|
|
die "kernel lockdown is active — kexec_load is blocked. Disable Secure Boot, or boot install media."
|
|
fi
|
|
|
|
kexec_artifacts
|
|
|
|
# Staging lives on /var/tmp, not /tmp: kexec-run.sh APPENDS a fresh cpio to
|
|
# kexec/initrd in place and runs binaries out of this directory, so it needs
|
|
# real space and exec permission. A tmpfs /tmp is both size-capped (ENOSPC
|
|
# mid-append = a half-written initrd) and frequently noexec.
|
|
stage="$(TMPDIR="${TMPDIR:-/var/tmp}" mktemp -d)"
|
|
trap 'rm -rf "$stage"' EXIT
|
|
|
|
# noexec would only surface as a cryptic "Permission denied" on kexec/run.
|
|
printf '#!/bin/sh\nexit 0\n' > "$stage/.execmove"
|
|
chmod +x "$stage/.execmove"
|
|
"$stage/.execmove" 2>/dev/null \
|
|
|| die "$stage is mounted noexec — set TMPDIR to an exec-capable filesystem"
|
|
rm -f "$stage/.execmove"
|
|
|
|
# Need room for the tarball, its extracted contents, and the appended cpio.
|
|
tb_sz="$(stat -Lc %s "$tb")"
|
|
avail="$(df -B1 --output=avail "$stage" | tail -1 | tr -d ' ')"
|
|
[ "$avail" -ge $(( tb_sz * 3 )) ] \
|
|
|| die "only $(( avail / 1024 / 1024 ))MB free on $stage, need ~$(( tb_sz * 3 / 1024 / 1024 ))MB — set TMPDIR elsewhere"
|
|
|
|
install -Dm755 "$cpio" "$stage/bin/cpio"
|
|
install -Dm755 "$bbox" "$stage/bin/gzip" # busybox as gzip (argv0)
|
|
mkdir -p "$stage/k" && tar -C "$stage/k" -xzf "$tb"
|
|
|
|
# Fail loudly here rather than with a bare "not found" from kexec/run.
|
|
for f in run kexec bzImage initrd ip; do
|
|
[ -f "$stage/k/kexec/$f" ] || die "kexec tarball is missing kexec/$f — bad build?"
|
|
done
|
|
|
|
# The installer root is a RAM disk: the initrd has to fit in memory with
|
|
# room to unpack. Refuse rather than OOM halfway into the new kernel, where
|
|
# there is no way back except the power button.
|
|
img_sz="$(( $(stat -Lc %s "$stage/k/kexec/initrd") + $(stat -Lc %s "$stage/k/kexec/bzImage") ))"
|
|
mem_kb="$(awk '/^MemTotal:/{print $2}' /proc/meminfo)"
|
|
[ "$(( mem_kb * 1024 ))" -ge "$(( img_sz * 3 ))" ] \
|
|
|| die "only $(( mem_kb / 1024 ))MB RAM for a $(( img_sz / 1024 / 1024 ))MB RAM-disk installer — too tight, boot install media instead"
|
|
|
|
# --- confirmation: this script normally runs on the LAPTOP ------------------
|
|
# A stray `kexec-local` here would jump the machine you develop on, not the
|
|
# target. Name it explicitly so a wrong-terminal mistake is visible.
|
|
echo ">> about to kexec THIS machine:"
|
|
echo " hostname: $(uname -n)"
|
|
echo " kernel: $(uname -r)"
|
|
echo " root: $(findmnt -no SOURCE / 2>/dev/null || echo '?')"
|
|
echo " Disks are untouched; only the running kernel changes. Console drops"
|
|
echo " for ~1-2 min, then comes back as the installer."
|
|
if [ -z "$assume_yes" ]; then
|
|
read -rp ">> type 'yes' to kexec $(uname -n) into the RAM installer: " ok
|
|
[ "$ok" = yes ] || die "aborted"
|
|
fi
|
|
|
|
# kexec -e jumps straight to the new kernel: no unmount, no journal flush,
|
|
# no systemd shutdown. Anything still in page cache is lost and the OS disk
|
|
# is left dirty. Cheap insurance, since the box may yet be rebooted back.
|
|
sync
|
|
|
|
echo ">> loading the new kernel"
|
|
if ! PATH="$stage/bin:$PATH" "$stage/k/kexec/run"; then
|
|
rm -rf "$stage"
|
|
die "kexec/run failed — machine untouched, still on the old kernel"
|
|
fi
|
|
|
|
# kexec --load succeeded iff the kernel now reports a loaded image. If this
|
|
# is 0 the background `kexec -e` below will do nothing and we'd hang forever
|
|
# waiting for a jump that cannot happen.
|
|
[ "$(cat /sys/kernel/kexec_loaded 2>/dev/null || echo 0)" = 1 ] \
|
|
|| { rm -rf "$stage"; die "kexec reported success but no image is loaded — aborting"; }
|
|
|
|
# THE trap MUST GO NOW: kexec-run.sh backgrounds the actual jump ~6s in the
|
|
# future, so an EXIT trap rm -rf'ing $stage here would delete the binary
|
|
# that performs it and the machine would silently never jump.
|
|
trap - EXIT
|
|
|
|
sync
|
|
echo ">> kernel loaded; jumping in ~6s. (staging left at $stage on purpose —"
|
|
echo " the backgrounded kexec still needs it; it's gone after the jump.)"
|
|
# Outlive the sleep 6 so the jump happens while this script is still alive.
|
|
sleep 60
|
|
die "still here after 60s — the jump did not happen. Check dmesg; the old system is intact."
|
|
;;
|
|
|
|
install)
|
|
config="${2:-}"; host="${3:-}"; assume_yes="${4:-}"
|
|
{ [ -n "$config" ] && [ -n "$host" ]; } || die "usage: ./deploy install <config> <host> [--yes]"
|
|
# $KEYDIR, not a bare $HOME — see its definition (also runs inside
|
|
# installer-iso, which has no $HOME).
|
|
hostkey="$KEYDIR/$config/ssh_host_ed25519_key"
|
|
[ -f "$hostkey" ] || die "missing host key: $hostkey"
|
|
[ -d "./hosts/$config" ] || die "no ./hosts/$config directory in the repo"
|
|
require_tracked "$config"
|
|
|
|
if [ "$host" = "localhost" ] || [ "$host" = "127.0.0.1" ]; then
|
|
if ! is_live_installer; then
|
|
# Not already inside a live installer: build one, stage it, one-shot
|
|
# boot into it, and let it finish this exact command itself. See
|
|
# local_install_prepare_and_reboot above and CLAUDE.md.
|
|
local_install_prepare_and_reboot "$config" "$hostkey" "$assume_yes"
|
|
exit 0
|
|
fi
|
|
|
|
# Local install: no ssh, no nixos-anywhere. Run after `kexec-local` (or
|
|
# from a live ISO) so /mnt is free to wipe — this IS the box, no second
|
|
# machine in the loop, so skip straight to disko + nixos-install.
|
|
require_root "local install"
|
|
[ -f "./hosts/$config/disk-config.nix" ] || die "no ./hosts/$config/disk-config.nix"
|
|
|
|
echo ">> disko .#$config onto this box's OS disk (WILL be wiped)"
|
|
# `.#disko`, not github:nix-community/disko: pins to this repo's
|
|
# flake.lock revision instead of upstream master-of-the-day.
|
|
nix run ".#disko" -- \
|
|
--mode disko "./hosts/$config/disk-config.nix"
|
|
|
|
echo ">> installing sops host key so it can decrypt on boot #1"
|
|
install -Dm600 "$hostkey" /mnt/etc/ssh/ssh_host_ed25519_key
|
|
install -Dm644 "$hostkey.pub" /mnt/etc/ssh/ssh_host_ed25519_key.pub
|
|
|
|
echo ">> nixos-install .#$config into /mnt"
|
|
nixos-install --root /mnt --flake ".#$config"
|
|
else
|
|
# Stage the pre-generated SSH host key so sops can decrypt on boot #1.
|
|
stage="$(mktemp -d)"
|
|
trap 'rm -rf "$stage"' EXIT
|
|
install -Dm600 "$hostkey" "$stage/etc/ssh/ssh_host_ed25519_key"
|
|
install -Dm644 "$hostkey.pub" "$stage/etc/ssh/ssh_host_ed25519_key.pub"
|
|
|
|
echo ">> nixos-anywhere .#$config onto root@$host (OS disk WILL be wiped)"
|
|
anywhere=(--flake ".#$config"
|
|
--extra-files "$stage"
|
|
--generate-hardware-config nixos-generate-config "./hosts/$config/hardware-configuration.nix"
|
|
--target-host "root@$host")
|
|
|
|
# nixos-anywhere's --env-password reads root's ssh password from $SSHPASS
|
|
# (its own bundled sshpass), so a vault hit skips the ssh-copy-id prompt.
|
|
root_item="${HOMELAB_PASS_ROOT_ITEM:-root@$config}"
|
|
root_pw="$(proton_pass_password "$root_item" || true)"
|
|
if [ -n "$root_pw" ]; then
|
|
echo ">> root ssh password from Proton Pass ($root_item)"
|
|
export SSHPASS="$root_pw"
|
|
unset root_pw
|
|
nix run ".#nixos-anywhere" -- \
|
|
--env-password "${anywhere[@]}"
|
|
unset SSHPASS
|
|
else
|
|
nix run ".#nixos-anywhere" -- "${anywhere[@]}"
|
|
fi
|
|
fi
|
|
;;
|
|
|
|
switch|boot|test)
|
|
config="${2:-}"; host="${3:-}"
|
|
{ [ -n "$config" ] && [ -n "$host" ]; } || die "usage: ./deploy $cmd <config> <host>"
|
|
require_tracked "$config"
|
|
|
|
echo ">> nixos-rebuild $cmd .#$config on darman@$host"
|
|
# --ask-sudo-password, not the deprecated --use-remote-sudo: common.nix sets
|
|
# wheelNeedsPassword = true, and --use-remote-sudo never actually prompts.
|
|
rebuild=(nix run nixpkgs#nixos-rebuild -- "$cmd"
|
|
--flake ".#$config"
|
|
--target-host "darman@$host"
|
|
--ask-sudo-password)
|
|
|
|
item="${HOMELAB_PASS_ITEM:-darman@$config}"
|
|
pw="$(proton_pass_password "$item" || true)"
|
|
if [ -n "$pw" ] && command -v setsid >/dev/null 2>&1; then
|
|
# nixos-rebuild's getpass() reads /dev/tty and ignores piped stdin; setsid
|
|
# drops the controlling terminal so it falls back to stdin instead. Every
|
|
# prompt in the subtree now reads that stdin, so the password line is fed
|
|
# a few times to survive a retry — anything else that prompts still fails.
|
|
echo ">> sudo password from Proton Pass ($item)"
|
|
printf '%s\n%s\n%s\n' "$pw" "$pw" "$pw" | setsid -w "${rebuild[@]}"
|
|
else
|
|
"${rebuild[@]}"
|
|
fi
|
|
|
|
# jupiter's 29G eMMC has already filled up once waiting for the weekly gc
|
|
# (common.nix). configurationLimit=5 only drops old generations as GC
|
|
# roots, so collect explicitly here rather than waiting up to a week.
|
|
if [ "$cmd" = switch ] && [ "$config" = jupiter ]; then
|
|
echo ">> jupiter: collecting garbage post-switch (keeps the eMMC under the 5-generation cap)"
|
|
need ssh
|
|
if [ -n "$pw" ]; then
|
|
printf '%s\n' "$pw" | ssh "darman@$host" 'sudo -S nix-collect-garbage -d' \
|
|
|| echo ">> warning: post-switch gc on jupiter failed — check disk space by hand" >&2
|
|
else
|
|
ssh -t "darman@$host" 'sudo nix-collect-garbage -d' \
|
|
|| echo ">> warning: post-switch gc on jupiter failed — check disk space by hand" >&2
|
|
fi
|
|
fi
|
|
unset pw
|
|
;;
|
|
|
|
image|flash)
|
|
if [ "$cmd" = flash ]; then
|
|
config="${2:-}"; dev="${3:-}"
|
|
{ [ -n "$config" ] && [ -n "$dev" ]; } \
|
|
|| die "usage: ./deploy flash <config> <dev> (e.g. /dev/sdX)"
|
|
# Validate the device and the tools BEFORE building, so a bad `flash`
|
|
# fails fast instead of dying halfway through writing the card.
|
|
[ -b "$dev" ] || die "$dev is not a block device"
|
|
need zstdcat; need dd; need lsblk
|
|
else
|
|
config="${2:-}"; [ -n "$config" ] || die "usage: ./deploy image <config>"
|
|
fi
|
|
require_tracked "$config"
|
|
|
|
echo ">> building SD image for .#$config (aarch64 needs qemu binfmt)"
|
|
nix build ".#nixosConfigurations.$config.config.system.build.sdImage" -o result-sd
|
|
img="$(one_match 'SD image' result-sd/sd-image/*.img.zst)"
|
|
echo ">> image: $img"
|
|
[ "$cmd" = image ] && exit 0
|
|
|
|
echo ">> TARGET DEVICE — everything on it will be ERASED:"
|
|
lsblk -o NAME,SIZE,MODEL,TRAN,MOUNTPOINTS "$dev"
|
|
read -rp ">> type 'yes' to write $config to $dev: " ok
|
|
[ "$ok" = yes ] || die "aborted"
|
|
zstdcat "$img" | sudo dd of="$dev" bs=4M status=progress oflag=sync
|
|
sync
|
|
|
|
# If this config has a dedicated sops age key, drop it on the ROOT ext4
|
|
# partition (the Pi's vfat one isn't mounted at runtime) so sops decrypts
|
|
# on first boot. Key stays off-repo, out of the nix store and the image.
|
|
keyfile="$KEYDIR/$config/age.txt"
|
|
if [ -f "$keyfile" ]; then
|
|
echo ">> installing sops age key onto the root partition"
|
|
sudo partprobe "$dev" 2>/dev/null || sudo blockdev --rereadpt "$dev" 2>/dev/null || true
|
|
sudo udevadm settle 2>/dev/null || true
|
|
# Largest ext4 partition = the NixOS root. -P (KEY="value" pairs) instead
|
|
# of a columnar listing: an empty FSTYPE collapses under whitespace
|
|
# splitting and shifts every later field.
|
|
rootpart="$(lsblk -bPo PATH,FSTYPE,SIZE "$dev" \
|
|
| sed -n 's/^PATH="\([^"]*\)" FSTYPE="ext4" SIZE="\([0-9]*\)"$/\2 \1/p' \
|
|
| sort -rn | head -1 | cut -d' ' -f2)"
|
|
[ -n "$rootpart" ] || die "no ext4 root partition found on $dev — place $keyfile at /var/lib/sops-nix/age.txt manually"
|
|
mnt="$(mktemp -d)"
|
|
# Unmount + remove even if the install fails, so a retry doesn't trip
|
|
# over the card still being mounted on a stale temp dir.
|
|
trap 'sudo umount "$mnt" 2>/dev/null || true; rmdir "$mnt" 2>/dev/null || true' EXIT
|
|
sudo mount "$rootpart" "$mnt"
|
|
sudo install -Dm600 "$keyfile" "$mnt/var/lib/sops-nix/age.txt"
|
|
sudo sync
|
|
sudo umount "$mnt"; rmdir "$mnt"; trap - EXIT
|
|
echo ">> age key installed (/var/lib/sops-nix/age.txt)"
|
|
fi
|
|
echo ">> done — insert the card into the Pi and boot."
|
|
;;
|
|
|
|
*)
|
|
die "unknown command '$cmd' (kexec|kexec-local|install|switch|boot|test|image|flash)"
|
|
;;
|
|
esac
|