jupiter: keep booting when the data array is missing

After the rack move one of the RAID0 disks failed to enumerate, and jupiter
boot-looped into an emergency shell nobody could use — root is locked, so
sulogin offers a prompt with no answer, and there is no ssh from there:

  Timed out waiting for device /dev/disk/by-uuid/dadbff6f-...
  Dependency failed for /mnt/data.
  Dependency failed for /var/lib/private/prowlarr.
  Dependency failed for Local File Systems.
  local-fs.target: Job local-fs.target/start failed with result 'dependency'
  Reached target Emergency Mode.

`nofail` on /mnt/data did not help, because the prowlarr and seerr bind
mounts layered on top of it had none: without it a mount is RequiredBy
local-fs.target, so those two failed the target on the array's behalf. Give
them `nofail` too and let them fail alone. `systemd.enableEmergencyMode =
false` then keeps a bad array from costing a reachable box at all — far more
useful on a headless host than a console prompt.

Booting further is only safe if nothing quietly relocates onto the 29G eMMC,
so pin the array-backed services to the mount. systemd derives
RequiresMountsFor from a unit's own paths, which for these is somewhere under
/var/lib (eMMC) — nothing pointed immich at mediaLocation or sabnzbd at its
configFile, so with the array gone they would have started and written to the
OS disk, into directories that go invisible the moment /mnt/data mounts over
them. jellyfin, sonarr, radarr and gitea already had a real dependency and
are untouched.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

hosts/jupiter/configuration.nix

@@ -63,6 +63,18 @@
     options = [ "nofail" ];  # don't block boot if the array is degraded/absent
   };
 
+  # `nofail` above is necessary but NOT sufficient — any mount layered on the
+  # array (prowlarr/seerr binds) is RequiredBy local-fs.target and will fail it
+  # regardless, and emergency mode on this box is a dead end: root is locked, so
+  # sulogin drops you at a prompt you cannot answer, with no ssh. 2026-08-06: a
+  # drive that failed to enumerate after the rack move did exactly this —
+  # "Timed out waiting for device /dev/disk/by-uuid/dadbff6f-…" -> Dependency
+  # failed for Local File Systems -> Reached target Emergency Mode, twice.
+  # Boot as far as possible instead and leave the failed units to be read over
+  # ssh. The array-backed services carry RequiresMountsFor=/mnt/data so they
+  # still refuse to start rather than writing to the eMMC.
+  systemd.enableEmergencyMode = false;
+
   # ---- Caddy vhosts (LAN) ----
   # Reached via pihole local-DNS names -> jupiter IP.
   services.caddy.virtualHosts = {

services/media/audiobookshelf.nix

@@ -13,4 +13,9 @@
     port = 8000;
   };
   users.users.audiobookshelf.extraGroups = [ "users" ];
+
+  # Its state dir is on the eMMC, so systemd sees no reason to wait for the
+  # array — but every library path points into /mnt/data. Starting without it
+  # means an empty library and rescans against nothing.
+  systemd.services.audiobookshelf.unitConfig.RequiresMountsFor = [ "/mnt/data" ];
 }

services/media/clonarr.nix

@@ -25,4 +25,10 @@
   systemd.tmpfiles.rules = [
     "d /mnt/data/AppData/clonarr 0755 darman users -"
   ];
+
+  # podman bind-mounts /mnt/data/AppData/clonarr into the container, but the
+  # generated unit only knows about /run/clonarr — with the array absent podman
+  # would create the source path on the eMMC and the container would run
+  # against an empty config.
+  systemd.services.podman-clonarr.unitConfig.RequiresMountsFor = [ "/mnt/data" ];
 }

services/media/immich.nix

@@ -119,4 +119,10 @@ in
   systemd.tmpfiles.rules = [
     "d /mnt/data/AppData/immich 0700 immich immich -"
   ];
+
+  # The unit's automatic RequiresMountsFor covers /run/immich and /var/lib/immich
+  # only — nothing points it at mediaLocation. Without this immich starts with
+  # the array missing and writes uploaded photos onto the 29G eMMC, into a
+  # directory that becomes invisible the moment /mnt/data mounts over it.
+  systemd.services.immich-server.unitConfig.RequiresMountsFor = [ "/mnt/data" ];
 }

services/media/prowlarr.nix

@@ -15,9 +15,21 @@
 {
   services.prowlarr.enable = true;
 
+  # `nofail` is NOT optional here: without it this bind is RequiredBy
+  # local-fs.target, so an unassembled RAID array fails that target and drops
+  # jupiter into emergency mode — which is a dead end, since root is locked and
+  # sulogin has nothing to offer on a headless box. It defeats the `nofail` on
+  # /mnt/data itself (a mount layered on the array is what actually took the
+  # target down). Let this bind fail alone instead.
   fileSystems."/var/lib/private/prowlarr" = {
     device = "/mnt/data/AppData/prowlarr/config";
     fsType = "none";
-    options = [ "bind" ];
+    options = [ "bind" "nofail" ];
   };
+
+  # systemd derives RequiresMountsFor from the unit's own paths, which here is
+  # only /var/lib/prowlarr on the eMMC — so without this prowlarr starts happily
+  # with the array absent and writes its state onto the 29G OS disk. Pin it to
+  # the array so it fails loudly instead.
+  systemd.services.prowlarr.unitConfig.RequiresMountsFor = [ "/mnt/data" ];
 }

services/media/sabnzbd.nix

@@ -16,6 +16,12 @@
   # Write access to the shared downloads dir (owned darman:users on disk).
   users.users.sabnzbd.extraGroups = [ "users" ];
 
+  # configFile and the downloads dir both live on the array, but systemd only
+  # derives RequiresMountsFor from /var/lib/sabnzbd (eMMC) — so with the array
+  # absent sabnzbd would start and download onto the 29G OS disk.
+  systemd.services.sabnzbd.unitConfig.RequiresMountsFor = [ "/mnt/data" ];
+  systemd.services.fix-downloads-perms.unitConfig.RequiresMountsFor = [ "/mnt/data" ];
+
   # SABnzbd hardcodes completed job folders to 0700 on every job, ignoring
   # the ini's `umask` (that only covers files during unpack, not the job
   # dir itself). setgid on Downloads keeps the group as "users" but perm

services/media/seerr.nix

@@ -11,12 +11,18 @@
 {
   services.seerr.enable = true;
 
+  # `nofail` for the same reason as prowlarr.nix — see the comment there: an
+  # array-backed bind without it fails local-fs.target and boots to an
+  # unusable emergency shell.
   fileSystems."/var/lib/private/seerr" = {
     device = "/mnt/data/AppData/seerr";
     fsType = "none";
-    options = [ "bind" ];
+    options = [ "bind" "nofail" ];
   };
 
+  # Only /var/lib/seerr (eMMC) is picked up automatically; pin to the array.
+  systemd.services.seerr.unitConfig.RequiresMountsFor = [ "/mnt/data" ];
+
   systemd.tmpfiles.rules = [
     "d /mnt/data/AppData/seerr 0755 darman users -"
   ];