WIP
This commit is contained in:
@@ -111,8 +111,8 @@
|
||||
];
|
||||
};
|
||||
|
||||
# mars — on-site x86_64 box, single-purpose: Hermes Agent only.
|
||||
# See hosts/mars/*.
|
||||
# mars — on-site x86_64 box: Hermes Agent, plus the LAN web apps luna
|
||||
# hosts herself (hosts/mars/luna-sites.nix). See hosts/mars/*.
|
||||
mars = nixpkgs.lib.nixosSystem {
|
||||
inherit system;
|
||||
specialArgs = { inherit inputs; };
|
||||
@@ -420,138 +420,146 @@
|
||||
# After the jump the test driver's backdoor is gone with the old kernel,
|
||||
# so the installer is driven over a forwarded ssh port instead (the same
|
||||
# approach nixos-images uses in its own kexec test).
|
||||
checks.${system}.kexec-local =
|
||||
let
|
||||
pkgs = nixpkgs.legacyPackages.${system};
|
||||
tarball = self.nixosConfigurations.kexec.config.system.build.kexecInstallerTarball;
|
||||
sshKey = nixos-images + "/nix/kexec-installer/ssh-keys/id_ed25519";
|
||||
in
|
||||
pkgs.testers.runNixOSTest {
|
||||
name = "deploy-kexec-local";
|
||||
checks.${system} = {
|
||||
kexec-local =
|
||||
let
|
||||
pkgs = nixpkgs.legacyPackages.${system};
|
||||
tarball = self.nixosConfigurations.kexec.config.system.build.kexecInstallerTarball;
|
||||
sshKey = nixos-images + "/nix/kexec-installer/ssh-keys/id_ed25519";
|
||||
in
|
||||
pkgs.testers.runNixOSTest {
|
||||
name = "deploy-kexec-local";
|
||||
|
||||
nodes.machine = { modulesPath, ... }: {
|
||||
imports = [ (modulesPath + "/profiles/minimal.nix") ];
|
||||
virtualisation.vlans = [ ];
|
||||
# kexec-local refuses to run if RAM < 3x the installer image, and
|
||||
# the staging dir needs ~3x the tarball on /var/tmp.
|
||||
virtualisation.memorySize = 4 * 1024;
|
||||
virtualisation.diskSize = 12 * 1024;
|
||||
virtualisation.forwardPorts = [{ host.port = 2222; guest.port = 22; }];
|
||||
nodes.machine = { modulesPath, ... }: {
|
||||
imports = [ (modulesPath + "/profiles/minimal.nix") ];
|
||||
virtualisation.vlans = [ ];
|
||||
# kexec-local refuses to run if RAM < 3x the installer image, and
|
||||
# the staging dir needs ~3x the tarball on /var/tmp.
|
||||
virtualisation.memorySize = 4 * 1024;
|
||||
virtualisation.diskSize = 12 * 1024;
|
||||
virtualisation.forwardPorts = [{ host.port = 2222; guest.port = 22; }];
|
||||
|
||||
services.openssh.enable = true;
|
||||
users.users.root.openssh.authorizedKeys.keyFiles = [ "${sshKey}.pub" ];
|
||||
services.openssh.enable = true;
|
||||
users.users.root.openssh.authorizedKeys.keyFiles = [ "${sshKey}.pub" ];
|
||||
|
||||
# Everything the script shells out to, minus nix — the test uses the
|
||||
# HOMELAB_KEXEC_* hook so no build happens inside the VM.
|
||||
environment.systemPackages = with pkgs; [
|
||||
bash gnutar coreutils findutils util-linux cpio gzip
|
||||
];
|
||||
system.extraDependencies = [ tarball pkgs.cpio pkgs.gzip ];
|
||||
# Everything the script shells out to, minus nix — the test uses the
|
||||
# HOMELAB_KEXEC_* hook so no build happens inside the VM.
|
||||
environment.systemPackages = with pkgs; [
|
||||
bash gnutar coreutils findutils util-linux cpio gzip
|
||||
];
|
||||
system.extraDependencies = [ tarball pkgs.cpio pkgs.gzip ];
|
||||
|
||||
environment.etc."deploy".source = ./scripts/deploy;
|
||||
environment.etc."deploy".source = ./scripts/deploy;
|
||||
};
|
||||
|
||||
testScript = /* python */ ''
|
||||
import os, shutil, subprocess, tempfile, time
|
||||
|
||||
start_all()
|
||||
machine.wait_for_unit("sshd.service")
|
||||
|
||||
# ssh refuses a private key that is group/world readable, and nix
|
||||
# store paths are 0444 — copy it out and tighten the mode.
|
||||
keydir = tempfile.mkdtemp()
|
||||
key = os.path.join(keydir, "id_ed25519")
|
||||
shutil.copyfile("${sshKey}", key)
|
||||
os.chmod(key, 0o600)
|
||||
|
||||
def ssh(cmd, check=True, stdout=None):
|
||||
return subprocess.run(
|
||||
[ "${pkgs.openssh}/bin/ssh",
|
||||
"-o", "StrictHostKeyChecking=no",
|
||||
"-o", "UserKnownHostsFile=/dev/null",
|
||||
"-o", "ConnectTimeout=1",
|
||||
"-i", key,
|
||||
"-p", "2222", "root@127.0.0.1", "--" ] + cmd,
|
||||
text=True, check=check, stdout=stdout)
|
||||
|
||||
machine.succeed("install -Dm755 /etc/deploy /root/deploy")
|
||||
|
||||
# systemd-run starts units with a bare PATH that lacks
|
||||
# /run/current-system/sw/bin, so `#!/usr/bin/env bash` cannot even
|
||||
# resolve bash, let alone tar/findmnt/nohup. Set it explicitly.
|
||||
env = (
|
||||
" --setenv=PATH=/run/wrappers/bin:/run/current-system/sw/bin"
|
||||
" --setenv=HOMELAB_KEXEC_TARBALL=${tarball}/nixos-kexec-installer-${system}.tar.gz"
|
||||
" --setenv=HOMELAB_KEXEC_CPIO=${pkgs.cpio}/bin/cpio"
|
||||
" --setenv=HOMELAB_KEXEC_GZIP=${pkgs.gzip}/bin/gzip"
|
||||
)
|
||||
# Same values for the foreground (non-systemd-run) invocation below.
|
||||
envsh = (
|
||||
"HOMELAB_KEXEC_TARBALL=${tarball}/nixos-kexec-installer-${system}.tar.gz"
|
||||
" HOMELAB_KEXEC_CPIO=${pkgs.cpio}/bin/cpio"
|
||||
" HOMELAB_KEXEC_GZIP=${pkgs.gzip}/bin/gzip"
|
||||
)
|
||||
|
||||
# Marker on a tmpfs: it must NOT survive the jump, proving we really
|
||||
# booted a new kernel rather than just restarting a service.
|
||||
machine.succeed("touch /run/pre-kexec-marker")
|
||||
host_key_before = machine.succeed("cat /etc/ssh/ssh_host_ed25519_key.pub").strip()
|
||||
|
||||
while ssh(["true"], check=False).returncode != 0:
|
||||
time.sleep(1)
|
||||
|
||||
# Refuses without --yes when stdin is not a tty (read gets EOF).
|
||||
# Must reach the confirmation prompt, so it needs the same env —
|
||||
# otherwise it just dies early on the nix build and proves nothing.
|
||||
out = machine.fail(f"{envsh} /root/deploy kexec-local </dev/null 2>&1")
|
||||
assert "using prebuilt kexec installer" in out, \
|
||||
f"never reached the prompt, so the refusal proves nothing:\n{out}"
|
||||
|
||||
# systemd-run so the call returns immediately: the script stays
|
||||
# alive ~60s on purpose, outliving kexec-run.sh's `sleep 6`.
|
||||
machine.succeed(f"systemd-run --collect --unit=kexec-local{env} /root/deploy kexec-local --yes")
|
||||
|
||||
print("waiting for the jump...")
|
||||
deadline = time.time() + 300
|
||||
while ssh(["true"], check=False).returncode == 0:
|
||||
# Surface a dead unit immediately instead of stalling until the
|
||||
# deadline and blaming "never left the old kernel".
|
||||
st = ssh(["systemctl", "is-active", "kexec-local"],
|
||||
check=False, stdout=subprocess.PIPE).stdout or ""
|
||||
if st.strip() in ("failed", "inactive"):
|
||||
# NB: not `log` — the driver already binds that name to its
|
||||
# AbstractLogger and the type check rejects the shadowing.
|
||||
unit_log = ssh(["journalctl", "-u", "kexec-local", "--no-pager"],
|
||||
check=False, stdout=subprocess.PIPE).stdout or ""
|
||||
raise AssertionError(
|
||||
f"kexec-local.service ended ({st.strip()}) without jumping:\n{unit_log}")
|
||||
assert time.time() < deadline, "machine never left the old kernel"
|
||||
time.sleep(1)
|
||||
|
||||
print("waiting for the installer...")
|
||||
deadline = time.time() + 300
|
||||
while ssh(["true"], check=False).returncode != 0:
|
||||
assert time.time() < deadline, "installer never came up"
|
||||
time.sleep(1)
|
||||
|
||||
# It really is the RAM installer, not the old system.
|
||||
host = ssh(["hostname"], stdout=subprocess.PIPE).stdout.strip()
|
||||
assert host == "nixos-installer", f"hostname is {host}, not nixos-installer"
|
||||
|
||||
assert ssh(["ls", "/run/pre-kexec-marker"], check=False).returncode != 0, \
|
||||
"old /run survived — this was not a fresh kernel"
|
||||
|
||||
# The host key is carried across (kexec-run.sh copies /etc/ssh into
|
||||
# the appended initrd), which is why `kexec` does no ssh-keygen -R.
|
||||
host_key_after = ssh(
|
||||
["cat", "/etc/ssh/ssh_host_ed25519_key.pub"], stdout=subprocess.PIPE
|
||||
).stdout.strip()
|
||||
assert host_key_before == host_key_after, \
|
||||
f"host key changed: {host_key_before} != {host_key_after}"
|
||||
|
||||
machine.crash()
|
||||
'';
|
||||
};
|
||||
|
||||
testScript = /* python */ ''
|
||||
import os, shutil, subprocess, tempfile, time
|
||||
|
||||
start_all()
|
||||
machine.wait_for_unit("sshd.service")
|
||||
|
||||
# ssh refuses a private key that is group/world readable, and nix
|
||||
# store paths are 0444 — copy it out and tighten the mode.
|
||||
keydir = tempfile.mkdtemp()
|
||||
key = os.path.join(keydir, "id_ed25519")
|
||||
shutil.copyfile("${sshKey}", key)
|
||||
os.chmod(key, 0o600)
|
||||
|
||||
def ssh(cmd, check=True, stdout=None):
|
||||
return subprocess.run(
|
||||
[ "${pkgs.openssh}/bin/ssh",
|
||||
"-o", "StrictHostKeyChecking=no",
|
||||
"-o", "UserKnownHostsFile=/dev/null",
|
||||
"-o", "ConnectTimeout=1",
|
||||
"-i", key,
|
||||
"-p", "2222", "root@127.0.0.1", "--" ] + cmd,
|
||||
text=True, check=check, stdout=stdout)
|
||||
|
||||
machine.succeed("install -Dm755 /etc/deploy /root/deploy")
|
||||
|
||||
# systemd-run starts units with a bare PATH that lacks
|
||||
# /run/current-system/sw/bin, so `#!/usr/bin/env bash` cannot even
|
||||
# resolve bash, let alone tar/findmnt/nohup. Set it explicitly.
|
||||
env = (
|
||||
" --setenv=PATH=/run/wrappers/bin:/run/current-system/sw/bin"
|
||||
" --setenv=HOMELAB_KEXEC_TARBALL=${tarball}/nixos-kexec-installer-${system}.tar.gz"
|
||||
" --setenv=HOMELAB_KEXEC_CPIO=${pkgs.cpio}/bin/cpio"
|
||||
" --setenv=HOMELAB_KEXEC_GZIP=${pkgs.gzip}/bin/gzip"
|
||||
)
|
||||
# Same values for the foreground (non-systemd-run) invocation below.
|
||||
envsh = (
|
||||
"HOMELAB_KEXEC_TARBALL=${tarball}/nixos-kexec-installer-${system}.tar.gz"
|
||||
" HOMELAB_KEXEC_CPIO=${pkgs.cpio}/bin/cpio"
|
||||
" HOMELAB_KEXEC_GZIP=${pkgs.gzip}/bin/gzip"
|
||||
)
|
||||
|
||||
# Marker on a tmpfs: it must NOT survive the jump, proving we really
|
||||
# booted a new kernel rather than just restarting a service.
|
||||
machine.succeed("touch /run/pre-kexec-marker")
|
||||
host_key_before = machine.succeed("cat /etc/ssh/ssh_host_ed25519_key.pub").strip()
|
||||
|
||||
while ssh(["true"], check=False).returncode != 0:
|
||||
time.sleep(1)
|
||||
|
||||
# Refuses without --yes when stdin is not a tty (read gets EOF).
|
||||
# Must reach the confirmation prompt, so it needs the same env —
|
||||
# otherwise it just dies early on the nix build and proves nothing.
|
||||
out = machine.fail(f"{envsh} /root/deploy kexec-local </dev/null 2>&1")
|
||||
assert "using prebuilt kexec installer" in out, \
|
||||
f"never reached the prompt, so the refusal proves nothing:\n{out}"
|
||||
|
||||
# systemd-run so the call returns immediately: the script stays
|
||||
# alive ~60s on purpose, outliving kexec-run.sh's `sleep 6`.
|
||||
machine.succeed(f"systemd-run --collect --unit=kexec-local{env} /root/deploy kexec-local --yes")
|
||||
|
||||
print("waiting for the jump...")
|
||||
deadline = time.time() + 300
|
||||
while ssh(["true"], check=False).returncode == 0:
|
||||
# Surface a dead unit immediately instead of stalling until the
|
||||
# deadline and blaming "never left the old kernel".
|
||||
st = ssh(["systemctl", "is-active", "kexec-local"],
|
||||
check=False, stdout=subprocess.PIPE).stdout or ""
|
||||
if st.strip() in ("failed", "inactive"):
|
||||
# NB: not `log` — the driver already binds that name to its
|
||||
# AbstractLogger and the type check rejects the shadowing.
|
||||
unit_log = ssh(["journalctl", "-u", "kexec-local", "--no-pager"],
|
||||
check=False, stdout=subprocess.PIPE).stdout or ""
|
||||
raise AssertionError(
|
||||
f"kexec-local.service ended ({st.strip()}) without jumping:\n{unit_log}")
|
||||
assert time.time() < deadline, "machine never left the old kernel"
|
||||
time.sleep(1)
|
||||
|
||||
print("waiting for the installer...")
|
||||
deadline = time.time() + 300
|
||||
while ssh(["true"], check=False).returncode != 0:
|
||||
assert time.time() < deadline, "installer never came up"
|
||||
time.sleep(1)
|
||||
|
||||
# It really is the RAM installer, not the old system.
|
||||
host = ssh(["hostname"], stdout=subprocess.PIPE).stdout.strip()
|
||||
assert host == "nixos-installer", f"hostname is {host}, not nixos-installer"
|
||||
|
||||
assert ssh(["ls", "/run/pre-kexec-marker"], check=False).returncode != 0, \
|
||||
"old /run survived — this was not a fresh kernel"
|
||||
|
||||
# The host key is carried across (kexec-run.sh copies /etc/ssh into
|
||||
# the appended initrd), which is why `kexec` does no ssh-keygen -R.
|
||||
host_key_after = ssh(
|
||||
["cat", "/etc/ssh/ssh_host_ed25519_key.pub"], stdout=subprocess.PIPE
|
||||
).stdout.strip()
|
||||
assert host_key_before == host_key_after, \
|
||||
f"host key changed: {host_key_before} != {host_key_after}"
|
||||
|
||||
machine.crash()
|
||||
'';
|
||||
# VM test for hosts/mars/luna-sites.nix (header of luna-sites-test.nix):
|
||||
# nix build .#checks.x86_64-linux.luna-sites -L
|
||||
luna-sites = import ./hosts/mars/luna-sites-test.nix {
|
||||
pkgs = nixpkgs.legacyPackages.${system};
|
||||
};
|
||||
};
|
||||
|
||||
# `nix develop` — hot-reload loop for dotfiles/quickshell.
|
||||
#
|
||||
|
||||
Reference in New Issue
Block a user