From 0ec3c6413c74944a96173c0837065d29b6b60f64 Mon Sep 17 00:00:00 2001 From: Erik Simon Date: Thu, 6 Aug 2026 03:32:42 +0200 Subject: [PATCH] jupiter: keep booting when the data array is missing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the rack move one of the RAID0 disks failed to enumerate, and jupiter boot-looped into an emergency shell nobody could use — root is locked, so sulogin offers a prompt with no answer, and there is no ssh from there: Timed out waiting for device /dev/disk/by-uuid/dadbff6f-... Dependency failed for /mnt/data. Dependency failed for /var/lib/private/prowlarr. Dependency failed for Local File Systems. local-fs.target: Job local-fs.target/start failed with result 'dependency' Reached target Emergency Mode. `nofail` on /mnt/data did not help, because the prowlarr and seerr bind mounts layered on top of it had none: without it a mount is RequiredBy local-fs.target, so those two failed the target on the array's behalf. Give them `nofail` too and let them fail alone. `systemd.enableEmergencyMode = false` then keeps a bad array from costing a reachable box at all — far more useful on a headless host than a console prompt. Booting further is only safe if nothing quietly relocates onto the 29G eMMC, so pin the array-backed services to the mount. systemd derives RequiresMountsFor from a unit's own paths, which for these is somewhere under /var/lib (eMMC) — nothing pointed immich at mediaLocation or sabnzbd at its configFile, so with the array gone they would have started and written to the OS disk, into directories that go invisible the moment /mnt/data mounts over them. jellyfin, sonarr, radarr and gitea already had a real dependency and are untouched. Co-Authored-By: Claude Opus 5 --- hosts/jupiter/configuration.nix | 12 ++++++++++++ services/media/audiobookshelf.nix | 5 +++++ services/media/clonarr.nix | 6 ++++++ services/media/immich.nix | 6 ++++++ services/media/prowlarr.nix | 14 +++++++++++++- services/media/sabnzbd.nix | 6 ++++++ services/media/seerr.nix | 8 +++++++- 7 files changed, 55 insertions(+), 2 deletions(-) diff --git a/hosts/jupiter/configuration.nix b/hosts/jupiter/configuration.nix index 99573f5..3672de7 100644 --- a/hosts/jupiter/configuration.nix +++ b/hosts/jupiter/configuration.nix @@ -63,6 +63,18 @@ options = [ "nofail" ]; # don't block boot if the array is degraded/absent }; + # `nofail` above is necessary but NOT sufficient — any mount layered on the + # array (prowlarr/seerr binds) is RequiredBy local-fs.target and will fail it + # regardless, and emergency mode on this box is a dead end: root is locked, so + # sulogin drops you at a prompt you cannot answer, with no ssh. 2026-08-06: a + # drive that failed to enumerate after the rack move did exactly this — + # "Timed out waiting for device /dev/disk/by-uuid/dadbff6f-…" -> Dependency + # failed for Local File Systems -> Reached target Emergency Mode, twice. + # Boot as far as possible instead and leave the failed units to be read over + # ssh. The array-backed services carry RequiresMountsFor=/mnt/data so they + # still refuse to start rather than writing to the eMMC. + systemd.enableEmergencyMode = false; + # ---- Caddy vhosts (LAN) ---- # Reached via pihole local-DNS names -> jupiter IP. services.caddy.virtualHosts = { diff --git a/services/media/audiobookshelf.nix b/services/media/audiobookshelf.nix index a7177f9..28801a9 100644 --- a/services/media/audiobookshelf.nix +++ b/services/media/audiobookshelf.nix @@ -13,4 +13,9 @@ port = 8000; }; users.users.audiobookshelf.extraGroups = [ "users" ]; + + # Its state dir is on the eMMC, so systemd sees no reason to wait for the + # array — but every library path points into /mnt/data. Starting without it + # means an empty library and rescans against nothing. + systemd.services.audiobookshelf.unitConfig.RequiresMountsFor = [ "/mnt/data" ]; } diff --git a/services/media/clonarr.nix b/services/media/clonarr.nix index 8479a18..a844d78 100644 --- a/services/media/clonarr.nix +++ b/services/media/clonarr.nix @@ -25,4 +25,10 @@ systemd.tmpfiles.rules = [ "d /mnt/data/AppData/clonarr 0755 darman users -" ]; + + # podman bind-mounts /mnt/data/AppData/clonarr into the container, but the + # generated unit only knows about /run/clonarr — with the array absent podman + # would create the source path on the eMMC and the container would run + # against an empty config. + systemd.services.podman-clonarr.unitConfig.RequiresMountsFor = [ "/mnt/data" ]; } diff --git a/services/media/immich.nix b/services/media/immich.nix index e3b8041..f23a611 100644 --- a/services/media/immich.nix +++ b/services/media/immich.nix @@ -119,4 +119,10 @@ in systemd.tmpfiles.rules = [ "d /mnt/data/AppData/immich 0700 immich immich -" ]; + + # The unit's automatic RequiresMountsFor covers /run/immich and /var/lib/immich + # only — nothing points it at mediaLocation. Without this immich starts with + # the array missing and writes uploaded photos onto the 29G eMMC, into a + # directory that becomes invisible the moment /mnt/data mounts over it. + systemd.services.immich-server.unitConfig.RequiresMountsFor = [ "/mnt/data" ]; } diff --git a/services/media/prowlarr.nix b/services/media/prowlarr.nix index 01b42e2..aa24dd9 100644 --- a/services/media/prowlarr.nix +++ b/services/media/prowlarr.nix @@ -15,9 +15,21 @@ { services.prowlarr.enable = true; + # `nofail` is NOT optional here: without it this bind is RequiredBy + # local-fs.target, so an unassembled RAID array fails that target and drops + # jupiter into emergency mode — which is a dead end, since root is locked and + # sulogin has nothing to offer on a headless box. It defeats the `nofail` on + # /mnt/data itself (a mount layered on the array is what actually took the + # target down). Let this bind fail alone instead. fileSystems."/var/lib/private/prowlarr" = { device = "/mnt/data/AppData/prowlarr/config"; fsType = "none"; - options = [ "bind" ]; + options = [ "bind" "nofail" ]; }; + + # systemd derives RequiresMountsFor from the unit's own paths, which here is + # only /var/lib/prowlarr on the eMMC — so without this prowlarr starts happily + # with the array absent and writes its state onto the 29G OS disk. Pin it to + # the array so it fails loudly instead. + systemd.services.prowlarr.unitConfig.RequiresMountsFor = [ "/mnt/data" ]; } diff --git a/services/media/sabnzbd.nix b/services/media/sabnzbd.nix index 7ce40a8..ac4a818 100644 --- a/services/media/sabnzbd.nix +++ b/services/media/sabnzbd.nix @@ -16,6 +16,12 @@ # Write access to the shared downloads dir (owned darman:users on disk). users.users.sabnzbd.extraGroups = [ "users" ]; + # configFile and the downloads dir both live on the array, but systemd only + # derives RequiresMountsFor from /var/lib/sabnzbd (eMMC) — so with the array + # absent sabnzbd would start and download onto the 29G OS disk. + systemd.services.sabnzbd.unitConfig.RequiresMountsFor = [ "/mnt/data" ]; + systemd.services.fix-downloads-perms.unitConfig.RequiresMountsFor = [ "/mnt/data" ]; + # SABnzbd hardcodes completed job folders to 0700 on every job, ignoring # the ini's `umask` (that only covers files during unpack, not the job # dir itself). setgid on Downloads keeps the group as "users" but perm diff --git a/services/media/seerr.nix b/services/media/seerr.nix index 9738da9..836500e 100644 --- a/services/media/seerr.nix +++ b/services/media/seerr.nix @@ -11,12 +11,18 @@ { services.seerr.enable = true; + # `nofail` for the same reason as prowlarr.nix — see the comment there: an + # array-backed bind without it fails local-fs.target and boots to an + # unusable emergency shell. fileSystems."/var/lib/private/seerr" = { device = "/mnt/data/AppData/seerr"; fsType = "none"; - options = [ "bind" ]; + options = [ "bind" "nofail" ]; }; + # Only /var/lib/seerr (eMMC) is picked up automatically; pin to the array. + systemd.services.seerr.unitConfig.RequiresMountsFor = [ "/mnt/data" ]; + systemd.tmpfiles.rules = [ "d /mnt/data/AppData/seerr 0755 darman users -" ];