deploy: harden kexec-local, key vault items by config, add VM test

kexec-local could never actually jump. nixos-images' kexec-run.sh ends with
`nohup sh -c "sleep 6 && $SCRIPT_DIR/kexec -e" &` and returns immediately, so
the EXIT trap's `rm -rf "$stage"` deleted the kexec binary out from under the
sleeping shell. The box stayed on the old kernel and it looked like a slow boot.
Clear the trap before jumping, verify /sys/kernel/kexec_loaded, then sleep past
the timer.

Preflight everything before the point of no return, since this jumps the machine
you are typing at: CONFIG_KEXEC, kernel lockdown, exec-capable staging dir, free
space, RAM vs image size, and that the tarball holds all five expected files.
Stage on /var/tmp rather than /tmp because kexec-run.sh appends to initrd in
place and execs from that directory. sync before jumping (kexec -e skips
unmount). Confirmation prompt naming the host, since run in the wrong terminal
this kexecs the laptop; --yes skips it.

Drop the ssh-keygen -R added to the remote kexec path: kexec-run.sh copies
/etc/ssh/ssh_host_* into the appended initrd and restore-remote-access.nix
installs them back, so the host key survives the jump.

Proton Pass items are now keyed by <config> instead of <host>, since the address
is incidental and the config name is stable. kexec therefore takes <config>
<host>. Resolve titles among --filter-state active items first: a trashed item
with the same title shadowed the active one and returned an empty password,
which is indistinguishable from "no entry" and silently fell back to prompting
(hit on darman@neptun).

Other fixes: replace `ls glob | head -1` (returns empty with exit 0 on no match)
with a helper that dies; guard against untracked hosts/<config> since flakes
ignore untracked files; feed the sudo password more than once under setsid;
handle empty arrays under set -u; tolerate empty FSTYPE in the SD-card root
partition lookup; preflight zstdcat/dd/lsblk before the destructive dd; list
image and flash in the usage strings.

Add checks.x86_64-linux.kexec-local, a VM test driving the real script. It is
the only way to exercise kexec-local, which cannot be rehearsed on hardware. It
asserts the box left the old kernel, returned as nixos-installer, lost its old
/run, and kept its ssh host key. HOMELAB_KEXEC_TARBALL lets it reuse a prebuilt
installer instead of building ~500MB inside the guest.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

flake.nix

@@ -20,6 +20,10 @@
       url = "github:nix-community/nixos-images";
       inputs.nixpkgs.follows = "nixpkgs";
     };
+    home-manager = {
+      url = "github:nix-community/home-manager/release-26.05";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
     mediamanager-nix = {
       url = "github:strangeglyph/mediamanager-nix";
       inputs.nixpkgs.follows = "nixpkgs";
@@ -30,7 +34,7 @@
     authentik-nix.url = "github:nix-community/authentik-nix";
   };
 
-  outputs = { self, nixpkgs, disko, sops-nix, nixos-images, mediamanager-nix, authentik-nix, ... }@inputs:
+  outputs = { self, nixpkgs, disko, sops-nix, nixos-images, home-manager, mediamanager-nix, authentik-nix, ... }@inputs:
     let
       system = "x86_64-linux";
     in
@@ -59,6 +63,19 @@
           ];
         };
 
+        # terra — Ryzen 9 5900X desktop (MSI MS-7A32). Replaces CachyOS on the
+        # OS SSD; Hyprland desktop + tailnet node. See hosts/terra/*.
+        terra = nixpkgs.lib.nixosSystem {
+          inherit system;
+          specialArgs = { inherit inputs; };
+          modules = [
+            disko.nixosModules.disko
+            sops-nix.nixosModules.sops
+            home-manager.nixosModules.home-manager
+            ./hosts/terra/configuration.nix
+          ];
+        };
+
         # mercury — Raspberry Pi 3B+ (aarch64), DNS/DHCP. Boots from an SD image:
         #   nix build .#nixosConfigurations.mercury.config.system.build.sdImage
         # (aarch64 build — needs binfmt/qemu on this x86 host, or a remote/aarch64
@@ -146,5 +163,151 @@
           ];
         };
       };
+
+      # VM test for `./scripts/deploy kexec-local`. Run:
+      #   nix build .#checks.x86_64-linux.kexec-local -L
+      #
+      # Worth having because kexec-local is the one command that cannot be
+      # rehearsed on real hardware: it jumps the machine you are typing at, and
+      # a failure looks exactly like a slow boot. It regression-tests the
+      # subtle one — kexec-run.sh backgrounds `sleep 6 && kexec -e` and returns,
+      # so anything that cleans up the staging dir on exit deletes the binary
+      # that performs the jump and the box silently never leaves the old kernel.
+      #
+      # After the jump the test driver's backdoor is gone with the old kernel,
+      # so the installer is driven over a forwarded ssh port instead (the same
+      # approach nixos-images uses in its own kexec test).
+      checks.${system}.kexec-local =
+        let
+          pkgs = nixpkgs.legacyPackages.${system};
+          tarball = self.nixosConfigurations.kexec.config.system.build.kexecInstallerTarball;
+          sshKey = nixos-images + "/nix/kexec-installer/ssh-keys/id_ed25519";
+        in
+        pkgs.testers.runNixOSTest {
+          name = "deploy-kexec-local";
+
+          nodes.machine = { modulesPath, ... }: {
+            imports = [ (modulesPath + "/profiles/minimal.nix") ];
+            virtualisation.vlans = [ ];
+            # kexec-local refuses to run if RAM < 3x the installer image, and
+            # the staging dir needs ~3x the tarball on /var/tmp.
+            virtualisation.memorySize = 4 * 1024;
+            virtualisation.diskSize = 12 * 1024;
+            virtualisation.forwardPorts = [{ host.port = 2222; guest.port = 22; }];
+
+            services.openssh.enable = true;
+            users.users.root.openssh.authorizedKeys.keyFiles = [ "${sshKey}.pub" ];
+
+            # Everything the script shells out to, minus nix — the test uses the
+            # HOMELAB_KEXEC_* hook so no build happens inside the VM.
+            environment.systemPackages = with pkgs; [
+              bash gnutar coreutils findutils util-linux cpio gzip
+            ];
+            system.extraDependencies = [ tarball pkgs.cpio pkgs.gzip ];
+
+            environment.etc."deploy".source = ./scripts/deploy;
+          };
+
+          testScript = /* python */ ''
+            import os, shutil, subprocess, tempfile, time
+
+            start_all()
+            machine.wait_for_unit("sshd.service")
+
+            # ssh refuses a private key that is group/world readable, and nix
+            # store paths are 0444 — copy it out and tighten the mode.
+            keydir = tempfile.mkdtemp()
+            key = os.path.join(keydir, "id_ed25519")
+            shutil.copyfile("${sshKey}", key)
+            os.chmod(key, 0o600)
+
+            def ssh(cmd, check=True, stdout=None):
+                return subprocess.run(
+                    [ "${pkgs.openssh}/bin/ssh",
+                      "-o", "StrictHostKeyChecking=no",
+                      "-o", "UserKnownHostsFile=/dev/null",
+                      "-o", "ConnectTimeout=1",
+                      "-i", key,
+                      "-p", "2222", "root@127.0.0.1", "--" ] + cmd,
+                    text=True, check=check, stdout=stdout)
+
+            machine.succeed("install -Dm755 /etc/deploy /root/deploy")
+
+            # systemd-run starts units with a bare PATH that lacks
+            # /run/current-system/sw/bin, so `#!/usr/bin/env bash` cannot even
+            # resolve bash, let alone tar/findmnt/nohup. Set it explicitly.
+            env = (
+                " --setenv=PATH=/run/wrappers/bin:/run/current-system/sw/bin"
+                " --setenv=HOMELAB_KEXEC_TARBALL=${tarball}/nixos-kexec-installer-${system}.tar.gz"
+                " --setenv=HOMELAB_KEXEC_CPIO=${pkgs.cpio}/bin/cpio"
+                " --setenv=HOMELAB_KEXEC_GZIP=${pkgs.gzip}/bin/gzip"
+            )
+            # Same values for the foreground (non-systemd-run) invocation below.
+            envsh = (
+                "HOMELAB_KEXEC_TARBALL=${tarball}/nixos-kexec-installer-${system}.tar.gz"
+                " HOMELAB_KEXEC_CPIO=${pkgs.cpio}/bin/cpio"
+                " HOMELAB_KEXEC_GZIP=${pkgs.gzip}/bin/gzip"
+            )
+
+            # Marker on a tmpfs: it must NOT survive the jump, proving we really
+            # booted a new kernel rather than just restarting a service.
+            machine.succeed("touch /run/pre-kexec-marker")
+            host_key_before = machine.succeed("cat /etc/ssh/ssh_host_ed25519_key.pub").strip()
+
+            while ssh(["true"], check=False).returncode != 0:
+                time.sleep(1)
+
+            # Refuses without --yes when stdin is not a tty (read gets EOF).
+            # Must reach the confirmation prompt, so it needs the same env —
+            # otherwise it just dies early on the nix build and proves nothing.
+            out = machine.fail(f"{envsh} /root/deploy kexec-local </dev/null 2>&1")
+            assert "using prebuilt kexec installer" in out, \
+                f"never reached the prompt, so the refusal proves nothing:\n{out}"
+
+            # systemd-run so the call returns immediately: the script stays
+            # alive ~60s on purpose, outliving kexec-run.sh's `sleep 6`.
+            machine.succeed(f"systemd-run --collect --unit=kexec-local{env} /root/deploy kexec-local --yes")
+
+            print("waiting for the jump...")
+            deadline = time.time() + 300
+            while ssh(["true"], check=False).returncode == 0:
+                # Surface a dead unit immediately instead of stalling until the
+                # deadline and blaming "never left the old kernel".
+                st = ssh(["systemctl", "is-active", "kexec-local"],
+                         check=False, stdout=subprocess.PIPE).stdout or ""
+                if st.strip() in ("failed", "inactive"):
+                    # NB: not `log` — the driver already binds that name to its
+                    # AbstractLogger and the type check rejects the shadowing.
+                    unit_log = ssh(["journalctl", "-u", "kexec-local", "--no-pager"],
+                                   check=False, stdout=subprocess.PIPE).stdout or ""
+                    raise AssertionError(
+                        f"kexec-local.service ended ({st.strip()}) without jumping:\n{unit_log}")
+                assert time.time() < deadline, "machine never left the old kernel"
+                time.sleep(1)
+
+            print("waiting for the installer...")
+            deadline = time.time() + 300
+            while ssh(["true"], check=False).returncode != 0:
+                assert time.time() < deadline, "installer never came up"
+                time.sleep(1)
+
+            # It really is the RAM installer, not the old system.
+            host = ssh(["hostname"], stdout=subprocess.PIPE).stdout.strip()
+            assert host == "nixos-installer", f"hostname is {host}, not nixos-installer"
+
+            assert ssh(["ls", "/run/pre-kexec-marker"], check=False).returncode != 0, \
+                "old /run survived — this was not a fresh kernel"
+
+            # The host key is carried across (kexec-run.sh copies /etc/ssh into
+            # the appended initrd), which is why `kexec` does no ssh-keygen -R.
+            host_key_after = ssh(
+                ["cat", "/etc/ssh/ssh_host_ed25519_key.pub"], stdout=subprocess.PIPE
+            ).stdout.strip()
+            assert host_key_before == host_key_after, \
+                f"host key changed: {host_key_before} != {host_key_after}"
+
+            machine.crash()
+          '';
+        };
     };
 }