diff --git a/common.nix b/common.nix index 32cf979..c148422 100644 --- a/common.nix +++ b/common.nix @@ -54,10 +54,9 @@ environment.systemPackages = with pkgs; [ git btop tmux curl wget zsh-powerlevel10k lsd jq ]; # ---- home-manager (user-level config for darman, all hosts) ---- - # Requires home-manager.nixosModules.home-manager in the host's own - # `modules` list (flake.nix) — this only sets values for options that - # module declares, it doesn't import it, so every nixosSystem using - # common.nix needs that line too (mirrors terra's original setup). + # Only sets values for options declared by home-manager.nixosModules.home-manager; + # it doesn't import that module, so every nixosSystem using common.nix must + # also list it in flake.nix's `modules`. home-manager.useGlobalPkgs = true; home-manager.useUserPackages = true; # Protects activation if a plain (non-symlink) ~/.zshrc etc. already @@ -91,10 +90,9 @@ boot.loader.systemd-boot.configurationLimit = 5; boot.loader.generic-extlinux-compatible.configurationLimit = 5; - # Stock journald defaults to ~10% of the filesystem (up to 4G) before it - # rotates — no scheduled vacuum, just a ceiling it grows into. On jupiter's - # 29G eMMC that's ~2.9G it could silently accumulate. Cap it well below that - # everywhere instead of only noticing when a disk fills up again. + # Stock journald grows unbounded up to ~10% of the filesystem (4G cap, no + # scheduled vacuum) — on jupiter's 29G eMMC that's ~2.9G it could silently + # fill. Cap it well below that everywhere. services.journald.extraConfig = '' SystemMaxUse=200M ''; diff --git a/flake.nix b/flake.nix index 96fe694..a986132 100644 --- a/flake.nix +++ b/flake.nix @@ -31,39 +31,32 @@ url = "github:strangeglyph/mediamanager-nix"; inputs.nixpkgs.follows = "nixpkgs"; }; - # livesync-bridge — headless CouchDB <-> filesystem sync for Obsidian - # LiveSync, used on mars to give luna a real directory of markdown - # (hosts/mars/livesync-bridge.nix). Not a flake and not in nixpkgs, so it - # comes in as plain source pinned by flake.lock; the service copies it out - # and runs it under deno. Pinning matters more than usual here — this is a - # small third-party project with open bugs on the storage->couchdb path, - # so an unreviewed bump could quietly change how the agent's notes are - # written back. + # Headless CouchDB<->filesystem sync for Obsidian LiveSync + # (hosts/mars/livesync-bridge.nix); not a flake/not in nixpkgs, so plain + # source pinned by flake.lock. Pin carefully — it's a small third-party + # project with open storage->couchdb bugs, so an unreviewed bump could + # silently change how notes get written back. livesync-bridge = { url = "github:vrtmrz/livesync-bridge"; flake = false; }; authentik-nix.url = "github:nix-community/authentik-nix"; nix-flatpak.url = "github:gmodena/nix-flatpak"; - # Own Hyprland plugin (border + title bar), public repo, fetched over - # https (no credentials needed, unlike tome below). `nixpkgs.follows` is - # what makes its packaged build ABI-correct — Hyprland plugins are - # ABI-locked to the exact Hyprland build they load into, so it has to be - # built against THIS flake's own nixpkgs, not whatever hypr-chrome's own - # flake.lock happens to pin standalone. + # Own Hyprland plugin (border + title bar), public repo over https. + # `nixpkgs.follows` is required since Hyprland plugins are ABI-locked to + # the exact Hyprland build — it must share this flake's nixpkgs, not + # whatever hypr-chrome's own lock pins standalone. hypr-chrome = { url = "git+https://git.mgaction.town/darman/hypr-chrome.git"; inputs.nixpkgs.follows = "nixpkgs"; }; # Tome (formerly AudibleLibrary) — darman's own .NET/Photino desktop app. - # Private repo on our own gitea; fetched over ssh with darman's ambient key, - # same as any other git flake input. `flake = false`: it's a plain source - # tree, not itself a flake. See pkgs/tome.nix. + # Private repo on our own gitea, fetched over ssh with darman's ambient + # key; plain source tree (`flake = false`), see pkgs/tome.nix. # - # NOTE: the credential-less installer-iso can't fetch this (git+ssh needs - # darman's key), so `./scripts/deploy install terra localhost` will fail - # at nixos-install (post-disko) while this input is present. Known - # tradeoff — re-removed this once before (4f79ec7) for the same reason. + # NOTE: the credential-less installer-iso can't fetch this, so + # `./scripts/deploy install terra localhost` fails at nixos-install + # (post-disko) while this input is present — a known tradeoff. tome = { url = "git+ssh://gitea@git.mgaction.town:2222/darman/TOME.git"; flake = false; @@ -136,10 +129,9 @@ ]; }; - # mercury — Raspberry Pi 3B+ (aarch64), DNS/DHCP. Boots from an SD image: + # mercury — Raspberry Pi 3B+ (aarch64), DNS/DHCP; SD image via: # nix build .#nixosConfigurations.mercury.config.system.build.sdImage - # (aarch64 build — needs binfmt/qemu on this x86 host, or a remote/aarch64 - # builder; substitutes most paths from cache.nixos.org.) + # Needs binfmt/qemu for the aarch64 build on this x86 host (or a remote aarch64 builder). mercury = nixpkgs.lib.nixosSystem { system = "aarch64-linux"; specialArgs = { inherit inputs; }; @@ -209,16 +201,12 @@ ]; }; - # Bootable USB recovery installer with our SSH key + sshd + DHCP. Clones - # the (now public) homelab repo fresh at every boot to /root/homelab — - # always current master, so the same USB stick stays useful across - # install/rescue occasions without ever needing a rebuild. No - # rsync/copy-the-repo-over step: boot it, ssh in, - # `cd /root/homelab && ./scripts/deploy install ...`. - # Reusable for any host's manual-USB install path (jupiter, terra, ...). - # Build the ISO: - # nix build .#nixosConfigurations.installer-iso.config.system.build.isoImage - # dd it to a USB stick, boot the target from it, SSH in, ./deploy install. + # Bootable USB recovery installer with our SSH key + sshd + DHCP; clones + # the public homelab repo fresh at every boot to /root/homelab, so the + # same stick stays current without a rebuild. Reusable for any host's + # manual-USB install path. + # Build: nix build .#nixosConfigurations.installer-iso.config.system.build.isoImage, + # dd to USB, boot the target, ssh in, ./scripts/deploy install ... installer-iso = nixpkgs.lib.nixosSystem { inherit system; modules = [ @@ -233,34 +221,18 @@ console.keyMap = "de"; # matches common.nix's real hosts environment.systemPackages = [ pkgs.git ]; - # findiso= is a SCRIPT-stage-1 feature (stage-1-init.sh) only. The - # systemd initrd — the default since 26.05 — has no findiso path - # at all: it mounts /iso straight from - # /dev/disk/by-label/ (iso-image.nix), which only exists - # when the ISO is the physical boot medium. Booted as a kernel + - # initrd off the ESP with the iso as a plain file elsewhere, that - # label never appears and stage 1 times out into an emergency - # shell (mounts /sysroot fine, then fails /sysroot/nix/.ro-store). - # Script stage 1 instead loop-mounts the file findiso= points at - # and symlinks it to /dev/root — which is the whole mechanism this - # install path relies on. So force it off here. + # The systemd initrd (default since 26.05) has no findiso= path — only + # the legacy script stage-1 does — so this install method needs it off. boot.initrd.systemd.enable = false; - # installation-cd-minimal leaves experimental-features unset, so - # the ISO's nix.conf has no `nix-command`/`flakes` at all (unlike - # the nixos-images kexec installer, which sets - # extra-experimental-features itself — which is why the same - # `install localhost` branch works after kexec-local but - # not here). Without this, both `nix run .#disko` and - # `nixos-install --flake` die with "experimental Nix feature - # 'nix-command' is disabled". + # installation-cd-minimal ships with experimental-features unset; + # without this, both `nix run .#disko` and `nixos-install --flake` + # die with "experimental Nix feature 'nix-command' is disabled". nix.settings.experimental-features = [ "nix-command" "flakes" ]; - # Fresh clone of a PUBLIC repo — no credentials baked into the - # ISO. require_tracked() in scripts/deploy still works fine here - # (this IS a real git checkout, unlike the old baked-`self` - # approach), but retry manually with `systemctl restart - # homelab-checkout` if DHCP was still coming up at boot. + # Fresh clone of the public repo (no credentials baked in) so + # scripts/deploy's require_tracked() sees a real checkout; retry + # with `systemctl restart homelab-checkout` if DHCP wasn't up yet. systemd.services.homelab-checkout = { description = "Clone the homelab repo to /root/homelab"; after = [ "network-online.target" ]; @@ -277,34 +249,24 @@ ''; }; - # Finishes a local_install_prepare_and_reboot() run (scripts/deploy) - # unattended: that function stages this ISO, points a systemd-boot - # one-shot entry at it with `homelab.install=` on the kernel - # cmdline, and reboots. Once booted here, this re-runs the exact same - # `./scripts/deploy install localhost` command — now genuinely - # inside the installer (hostname homelab-installer), so is_live_installer - # takes the disko+nixos-install branch instead of preparing again. - # A manual boot of this ISO with no such cmdline param is a no-op. + # Completes an unattended local_install_prepare_and_reboot() run: + # re-runs `./scripts/deploy install localhost`, now genuinely + # inside the installer so it takes the disko+nixos-install branch. + # No-op if homelab.install= isn't on the kernel cmdline. systemd.services.homelab-auto-install = { description = "Auto-run the homelab install if homelab.install= was passed on the kernel cmdline"; after = [ "homelab-checkout.service" ]; requires = [ "homelab-checkout.service" ]; wantedBy = [ "multi-user.target" ]; serviceConfig.Type = "oneshot"; - # Full system PATH, not the restricted default a `path = [...]` - # produces: this unit execs `./scripts/deploy`, whose - # `#!/usr/bin/env bash` needs bash, and which then reaches for - # nix / nixos-install / git / sudo / efibootmgr. The default - # service PATH gave "env: 'bash': No such file or directory" - # (status 127) before the script even started. - # /run/current-system/sw/bin carries all of it on the installer; - # /run/wrappers/bin for sudo. mkForce because NixOS otherwise - # derives environment.PATH from `path` and that line would win. + # Needs the full system PATH: scripts/deploy execs bash then shells + # out to nix/nixos-install/git/sudo/efibootmgr, none of which a + # restricted `path = [...]` PATH provides. mkForce overrides NixOS's + # default PATH derivation from `path`. # - # HOME too: systemd sets no $HOME for a service without User= - # (systemd.exec(5): SetLoginEnvironment= defaults false), and - # scripts/deploy runs under `set -u`, so a bare $HOME aborted the - # whole run with an "unbound variable" that read like a bug. + # HOME too: systemd sets no $HOME without User= (SetLoginEnvironment= + # defaults false), and scripts/deploy runs under `set -u`, so a + # missing $HOME aborted with a confusing "unbound variable". environment = { HOME = "/root"; PATH = lib.mkForce "/run/current-system/sw/bin:/run/wrappers/bin"; @@ -316,17 +278,11 @@ exit 0 fi - # Persist this whole run to a file that OUTLIVES the install. - # The systemd journal is on the installer's tmpfs and dies with - # the reboot, and by the time anything interesting fails disko - # has already wiped the OS disk — so a failed attempt used to - # leave nothing to debug. local_install_prepare_and_reboot() - # (scripts/deploy) passes the STAGING partition's PARTUUID as - # homelab.logpart=; that partition holds the iso and is on a - # different disk from the one disko wipes, so it survives. The - # actual install runs inside do_install() below so one tee at - # the end captures all of it. Every step here is best-effort: - # logging must never be the thing that breaks an install. + # Persist this run to a file that outlives the install: the journal + # dies with the reboot and disko wipes the OS disk before a failure + # can be read back. homelab.logpart= points at the staging partition + # (survives the wipe); every step here is best-effort so logging + # itself can't break an install. logfile="" logpart=$(grep -o 'homelab\.logpart=[^ ]*' /proc/cmdline | cut -d= -f2 || true) if [ -n "$logpart" ]; then @@ -336,9 +292,8 @@ if mount -o rw "$dev" /run/homelab-log 2>/dev/null; then logdir=/run/homelab-log elif where=$(findmnt -fno TARGET "$dev" 2>/dev/null) && [ -n "$where" ]; then - # stage-1's findiso already holds this partition mounted - # (that is how it reached the iso) — write into the existing - # mount rather than trying to stack a second one on it. + # stage-1's findiso already has this partition mounted (how it + # reached the iso) — reuse that mount instead of a second one. mount -o remount,rw "$where" 2>/dev/null || true logdir="$where" fi @@ -358,12 +313,10 @@ fi do_install() { - # The host key scripts/deploy seeds /etc/ssh with (so sops can - # decrypt on boot #1) cannot live in this ISO: it is built from - # a PUBLIC repo and the private keys are deliberately off-repo. - # local_install_prepare_and_reboot() therefore drops it on the - # boot partition and passes that partition's PARTUUID here. - # That copy dies with the disko wipe a few minutes later. + # The host key (so sops can decrypt on first boot) can't live in + # this public-repo ISO; local_install_prepare_and_reboot() drops it + # on the boot partition instead and passes that PARTUUID here — the + # copy dies with disko's wipe minutes later. keypart=$(grep -o 'homelab\.keypart=[^ ]*' /proc/cmdline | cut -d= -f2 || true) if [ -n "$keypart" ]; then mkdir -p /run/homelab-key @@ -384,12 +337,10 @@ fi fi - # On a box whose old bootloader had no one-shot (Limine on - # terra), scripts/deploy got us here via a temporary UEFI - # entry + BootNext (arm_efi_bootnext). BootNext is already - # spent, but the entry itself would linger in NVRAM pointing - # at a partition disko is about to reformat. Drop it now, so - # even an install that fails later leaves NVRAM clean. + # On bootloaders with no one-shot (Limine on terra), scripts/deploy + # got here via a temporary UEFI entry + BootNext (arm_efi_bootnext); + # BootNext is spent but the entry would linger pointing at a + # partition disko is about to wipe, so remove it now. for n in $(efibootmgr 2>/dev/null \ | sed -n 's/^Boot\([0-9A-Fa-f]\{4\}\)\*\?[[:space:]]Homelab Installer[[:space:]].*/\1/p'); do echo "removing temporary UEFI entry Boot$n" @@ -419,19 +370,11 @@ }; }; - # VM test for `./scripts/deploy kexec-local`. Run: - # nix build .#checks.x86_64-linux.kexec-local -L - # - # Worth having because kexec-local is the one command that cannot be - # rehearsed on real hardware: it jumps the machine you are typing at, and - # a failure looks exactly like a slow boot. It regression-tests the - # subtle one — kexec-run.sh backgrounds `sleep 6 && kexec -e` and returns, - # so anything that cleans up the staging dir on exit deletes the binary - # that performs the jump and the box silently never leaves the old kernel. - # - # After the jump the test driver's backdoor is gone with the old kernel, - # so the installer is driven over a forwarded ssh port instead (the same - # approach nixos-images uses in its own kexec test). + # VM test for `./scripts/deploy kexec-local` (nix build .#checks.x86_64-linux.kexec-local -L) + # — the one command that can't be rehearsed on real hardware since it jumps + # the machine you're on. Regression-tests kexec-run.sh's backgrounded + # `sleep 6 && kexec -e`: cleaning up the staging dir on exit would delete + # the jump binary and the box would silently stay on the old kernel. checks.${system} = { kexec-local = let @@ -489,9 +432,9 @@ machine.succeed("install -Dm755 /etc/deploy /root/deploy") - # systemd-run starts units with a bare PATH that lacks - # /run/current-system/sw/bin, so `#!/usr/bin/env bash` cannot even - # resolve bash, let alone tar/findmnt/nohup. Set it explicitly. + # systemd-run starts units with a bare PATH lacking + # /run/current-system/sw/bin, so bash (and tar/findmnt/nohup) + # can't resolve — set it explicitly. env = ( " --setenv=PATH=/run/wrappers/bin:/run/current-system/sw/bin" " --setenv=HOMELAB_KEXEC_TARBALL=${tarball}/nixos-kexec-installer-${system}.tar.gz" @@ -513,9 +456,9 @@ while ssh(["true"], check=False).returncode != 0: time.sleep(1) - # Refuses without --yes when stdin is not a tty (read gets EOF). - # Must reach the confirmation prompt, so it needs the same env — - # otherwise it just dies early on the nix build and proves nothing. + # Refuses without --yes when stdin isn't a tty; needs the same env to + # reach the confirmation prompt, else it dies early on the nix build + # and proves nothing. out = machine.fail(f"{envsh} /root/deploy kexec-local &1") assert "using prebuilt kexec installer" in out, \ f"never reached the prompt, so the refusal proves nothing:\n{out}" @@ -575,27 +518,23 @@ # `nix develop` — hot-reload loop for dotfiles/quickshell. # - # hosts/terra/home.nix ships the shell via `xdg.configFile."quickshell"`, - # which COPIES the tree into the store, so ~/.config/quickshell is a - # read-only symlink into /nix/store and every QML tweak costs a - # nixos-rebuild. quickshell DOES hot-reload on file save — but only for - # the files it is watching, which are those frozen store copies. Pointing - # it at the working tree with `qs -p` restores edit-save-see, no rebuild. + # hosts/terra/home.nix ships the shell as a store copy (`xdg.configFile`), + # which only hot-reloads its own frozen files; pointing at the working + # tree with `qs -p` restores edit-save-see without a rebuild. # - # quickshell keys instance identity on the CONFIG PATH, so a working-tree - # instance and the store-backed one are two different instances that would - # both map layer-shell bars onto every output. Hence a swap, not a second - # instance — and the swap starts dev FIRST, killing the packaged shell - # only once dev is confirmed up, so a QML error in the working tree leaves - # you on your normal bar instead of no bar at all. + # quickshell keys instance identity on the config path, so the + # working-tree and store-backed shells are different instances that + # would both claim every output — hence a swap, not a second instance. + # The swap starts dev first and only kills the packaged shell once dev + # is confirmed up, so a QML error leaves you on your normal bar. # - # Every kill is scoped to one config (`qs kill` = default only, `qs kill - # -p` = that path only). A blanket kill would also take out unrelated - # quickshell instances — pkgs/rishot.nix is one. + # Every kill is scoped to one config (`qs kill` = default, `qs kill -p + # ` = that path) since a blanket kill would also take out + # unrelated instances like pkgs/rishot.nix. # - # Deliberately NOT wired to direnv (no .envrc in this repo): programs.direnv - # is enabled for this user, so a `use flake` would swap the running desktop - # shell on every `cd` into the checkout, including over ssh. + # Deliberately not wired to direnv: programs.direnv is enabled for this + # user, so a `use flake` would swap the desktop shell on every `cd` + # into the checkout, including over ssh. devShells.${system}.default = let pkgs = nixpkgs.legacyPackages.${system}; @@ -652,10 +591,9 @@ echo "qs-dev: live on $cfg — edits there now hot-reload" ''; - # qs log -f prints everything the instance logs; WARN and ERROR are the - # two that mean something is wrong with the QML in front of you. A - # binding loop or a failed binding is a WARN and easy to miss when it - # scrolls past inside a reload's worth of chatter. + # qs log -f prints everything the instance logs; WARN/ERROR are what + # mean something is actually wrong with the QML (a binding loop or + # failed binding is a WARN, easy to miss in the reload chatter). qs-log = pkgs.writeShellScriptBin "qs-log" '' set -uo pipefail ${preamble} @@ -665,12 +603,10 @@ -a|--all) filter='.' ;; esac - # -t 1: `qs log -f` replays the whole backlog first, which would dump - # every historical warning into the terminal on shell entry. - # - # `qs log -f` ends when the instance it attached to exits, and the dev - # shell outlives individual instances — a QML error kills one, `qs-dev` - # starts another. Re-attach instead of going quiet for the session. + # -t 1: `qs log -f` otherwise replays the whole backlog on shell entry. + # It also ends when the attached instance exits, and the dev shell + # outlives individual instances (a QML error kills one, qs-dev starts + # another) — so re-attach in a loop instead of going quiet for the session. while :; do if running "$cfg"; then ${qs} log -p "$cfg" -t 1 -f 2>/dev/null | ${grep} --line-buffered -E "$filter" >&2 diff --git a/home/common.nix b/home/common.nix index cd05528..e43d1d0 100644 --- a/home/common.nix +++ b/home/common.nix @@ -8,10 +8,9 @@ programs.home-manager.enable = true; # Matches terra's baseline (compinit, deduped/shared history, HISTFILE - # under $HOME). home-manager owns ~/.zshrc + ~/.zshenv as real files, which - # also means zsh's built-in zsh-newuser-install wizard never fires on - # first interactive login (it only triggers when none of - # .zshenv/.zprofile/.zshrc/.zlogin exist) — that used to happen on every - # host except terra. + # under $HOME). home-manager owning ~/.zshrc + ~/.zshenv as real files also + # means zsh's newuser-install wizard never fires (it only triggers when + # none of those dotfiles exist) — previously an issue on every host except + # terra. programs.zsh.enable = true; } diff --git a/hosts/jupiter/configuration.nix b/hosts/jupiter/configuration.nix index a634d26..ae51e38 100644 --- a/hosts/jupiter/configuration.nix +++ b/hosts/jupiter/configuration.nix @@ -40,16 +40,11 @@ # systemd-boot for UEFI. If ZimaBlade boots legacy/BIOS, switch to grub. boot.loader.systemd-boot.enable = true; boot.loader.efi.canTouchEfiVariables = true; - # common.nix's cap of 5 comes from this box's own 34-generation incident, - # but at ~5G free on a 29G eMMC even 5 is too many — override down to 2. + # common.nix's default of 5 is still too many boot entries for a 29G eMMC — override down to 2. boot.loader.systemd-boot.configurationLimit = lib.mkForce 2; - # A `switch` pins the old generation as a GC root until the box reboots onto - # the new one (booted-system vs current-system) — common.nix's nix.gc is - # weekly, far too slow to catch that on a 29G eMMC. 2026-08-19: one switch - # alone took 14G -> 19G used; only reboot (releases the old root) + this GC - # brought it back to 14G. Run a full collect right after every boot instead - # of waiting on the weekly timer. + # A `switch` pins the old generation as a GC root until reboot; common.nix's weekly + # nix.gc is too slow for a 29G eMMC, so collect garbage on every boot instead. systemd.services.gc-on-boot = { description = "Full nix-collect-garbage on every boot"; wantedBy = [ "multi-user.target" ]; @@ -70,46 +65,24 @@ boot.kernelParams = [ "reboot=pci" ]; # ---- GPU (jellyfin hardware transcoding) ---- - # Apollo Lake N3450 / HD Graphics 500 (Gen9, pci 8086:5A85). The i915 KERNEL - # driver binds on its own — /dev/dri/{card1,renderD128} exist without this — - # but the libva USERSPACE driver only ships when hardware.graphics is on, and - # nothing else here pulled it in. Without it VAAPI init fails with "unknown - # libva error" and jellyfin-ffmpeg exits 251 on EVERY transcode, which the - # client shows as generic playback failure: the server log only says "FFmpeg - # exited with code 251", never that a driver is missing. Verified on the box: - # the same h264_vaapi encode goes 251 -> 0 once iHD is on LIBVA_DRIVERS_PATH. - # - # iHD (intel-media-driver) is the right one for Gen9; i965 is for Gen8 and - # older. Note the render node is 0666 but card1 is 0660 root:video, so the - # group membership in services/media/jellyfin.nix matters for the card node. + # Apollo Lake N3450 / HD Graphics 500 (Gen9). i915 binds on its own, but VAAPI needs + # the iHD userspace driver (Gen9; i965 is Gen8-only) or jellyfin-ffmpeg exits 251 on + # every transcode with no clearer error than "FFmpeg exited with code 251" in the log. hardware.graphics = { enable = true; extraPackages = [ pkgs.intel-media-driver ]; }; - # ⚠️ This buys VAAPI only — jellyfin must be set to VAAPI, NOT QSV, in its - # web UI (Dashboard -> Playback -> Transcoding). QSV needs an MFX runtime on - # top of the libva driver: ffmpeg's `-init_hw_device qsv=qs@va` dies with - # "Error creating a MFX session: -9" -> exit 171, the SECOND failure hiding - # behind the first (fixing the missing driver only moved 251 -> 171). - # There is no good way to provide it here: vpl-gpu-rt is Gen12+, and the - # Gen9 runtime `intel-media-sdk` is marked INSECURE in nixpkgs (EOL, 5 CVEs - # incl. local privilege escalation) — not worth it when VAAPI does the same - # job on this chip at ~3.5x realtime for 1080p->720p. - # - # Also: 4K HDR (the 2160p HEVC/DV remuxes) can NOT be tone-mapped here. - # tonemap_opencl needs OpenCL, which has no platform on this box, and - # tonemap_vaapi is Gen11+ — both fail. Only a plain scale_vaapi=format=nv12 - # succeeds, which drops HDR without tone-mapping (washed-out picture). - # Those files need to direct-play, or be kept as 1080p SDR versions. + # ⚠️ Use VAAPI, not QSV, in jellyfin's UI — QSV needs an MFX runtime not safely + # available for this Gen9 chip (only insecure/EOL options) and fails with exit 171. + # 4K HDR remuxes also can't be tone-mapped here (needs OpenCL or Gen11+); keep those + # as 1080p SDR or let them direct-play. # ---- NAS data array ---- - # Existing ext4 on the mdadm RAID0 over sda+sdb (md0, 29.1T). - # Mounted, NOT formatted; kept out of disko so it is never wiped. - # ⚠️ RAID0 = no redundancy: either 16TB disk failing loses ALL data. - boot.swraid.enable = true; # assemble the mdadm array at boot - # Silences "mdmon service will crash" eval warning. RAID0 here uses native - # superblocks so mdmon (external-metadata arrays only) never actually runs, - # but the module warns unconditionally without SOME MAILADDR/PROGRAM set. + # Existing ext4 on mdadm RAID0 (sda+sdb, md0, 29.1T) — mounted, not formatted, kept + # out of disko. ⚠️ RAID0 has no redundancy: either disk failing loses ALL data. + boot.swraid.enable = true; + # Silences the "mdmon service will crash" eval warning — mdmon never actually runs + # here (native superblocks, not external-metadata) but the module warns regardless. boot.swraid.mdadmConf = "MAILADDR root"; fileSystems."/mnt/data" = { # fs UUID (stable) — the array may enumerate as /dev/md127, so avoid /dev/md0. @@ -118,69 +91,48 @@ options = [ "nofail" ]; # don't block boot if the array is degraded/absent }; - # `nofail` above is necessary but NOT sufficient — any mount layered on the - # array (prowlarr/seerr binds) is RequiredBy local-fs.target and will fail it - # regardless, and emergency mode on this box is a dead end: root is locked, so - # sulogin drops you at a prompt you cannot answer, with no ssh. 2026-08-06: a - # drive that failed to enumerate after the rack move did exactly this — - # "Timed out waiting for device /dev/disk/by-uuid/dadbff6f-…" -> Dependency - # failed for Local File Systems -> Reached target Emergency Mode, twice. - # Boot as far as possible instead and leave the failed units to be read over - # ssh. The array-backed services carry RequiresMountsFor=/mnt/data so they - # still refuse to start rather than writing to the eMMC. + # `nofail` alone isn't enough — mounts layered on the array (prowlarr/seerr binds) + # are RequiredBy local-fs.target and can still trip Emergency Mode, which is a dead + # end here (root locked, no ssh). Boot as far as possible instead; the array-backed + # services carry RequiresMountsFor=/mnt/data so they still won't write to the eMMC. systemd.enableEmergencyMode = false; # ---- Heavy state moved off the eMMC ---- - # A deploy holds TWO full closures (~9G each) on a 29G disk at once, so the - # OS disk has no room for state that grows on its own. 2026-08-09: it hit 0 - # bytes free with both gen 39 and gen 40 resident, and postgres died on - # "No space left on device" — note ext4 reserves 5% for root, so non-root - # services see zero while df still shows ~300M free. - # - # Paths live under /mnt/data/AppData like every other service's state. Both - # settings below are jupiter-only on purpose: services/containers.nix stays - # engine- and host-agnostic (mercury runs pihole on podman with no array). + # A deploy holds two full closures (~9G each) on this 29G disk at once, so state + # that grows on its own can't live there — moved under /mnt/data/AppData like every + # other service's state. Settings below are jupiter-only; services/containers.nix + # stays engine/host-agnostic (mercury runs podman with no array). - # podman: CI images dominate and keep growing — the gitea runner's - # act-latest is 1.7G, and the act-22.04 label in services/dev/gitea.nix - # pulls another ~1.7G the first time a job requests it. - # runroot stays on /run: it is per-boot tmpfs state, not a growing store. + # runroot stays on /run (per-boot tmpfs, doesn't grow); graphroot moves to the array + # since the gitea runner's CI images alone run several GB. virtualisation.containers.storage.settings.storage = { driver = "overlay"; graphroot = "/mnt/data/AppData/containers/storage"; runroot = "/run/containers/storage"; }; - # immich's postgres cluster. Version component mirrors the upstream default - # (`/var/lib/postgresql/${psqlSchema}`) so a major bump gets its own dir - # instead of silently reusing the old cluster's files. - # ⚠️ This puts the DB in the SAME failure domain as the photos it indexes: - # /mnt/data is RAID0, so either 16TB disk now loses both, where before an - # eMMC failure and an array failure each took only one. Chosen deliberately - # — the two are useless apart — but neither is backed up. + # immich's postgres cluster. Version-qualified path (matches upstream default) so a + # major bump gets a fresh dir instead of reusing the old cluster's files. + # ⚠️ Puts the DB in the same RAID0 failure domain as the photos it indexes — + # deliberate (the two are useless apart) but neither is backed up. services.postgresql.dataDir = "/mnt/data/AppData/postgresql/${config.services.postgresql.package.psqlSchema}"; - # /mnt/data/AppData is drwx--x--- darman:users, so postgres needs group - # "users" just to TRAVERSE into its own dataDir — exactly the reason immich - # has the same line. The cluster dir itself keeps the mode it was initdb'd - # with (0750 postgres:postgres) — postgres only accepts 0700, or 0750 when - # the cluster was created with group access, and refuses to start otherwise. + # /mnt/data/AppData is drwx--x--- darman:users, so postgres needs the "users" group + # just to traverse into its dataDir (same reason immich needs it) — postgres itself + # refuses to start unless the cluster dir is 0700 or 0750. users.users.postgres.extraGroups = [ "users" ]; - # Neither path is under /var/lib, so no module creates it: the postgresql - # module's own tmpfiles entry only adjusts a dataDir that already exists, - # the same way immich's mediaLocation rule does. + # Neither path is under /var/lib, so no module creates it automatically — same + # reason immich needs its own mediaLocation tmpfiles rule. systemd.tmpfiles.rules = [ "d /mnt/data/AppData/postgresql 0750 postgres postgres -" "d /mnt/data/AppData/containers 0700 root root -" ]; - # graphroot is not a systemd path dependency the way dataDir is, so nothing - # derives a mount ordering from it. Without these, podman would recreate an - # empty store on the eMMC under the mountpoint when the array is late or - # absent, and the runner would re-pull every image into it. - # (podman-clonarr already carries this from services/media/clonarr.nix.) + # Without this, podman would recreate an empty store on the eMMC if the array mounts + # late or is absent, and the runner would re-pull every image. + # (podman-clonarr already sets this in services/media/clonarr.nix.) systemd.services.podman.unitConfig.RequiresMountsFor = [ "/mnt/data" ]; systemd.services.gitea-runner-jupiter.unitConfig.RequiresMountsFor = [ "/mnt/data" ]; diff --git a/hosts/jupiter/secrets.nix b/hosts/jupiter/secrets.nix index 034e15e..b50def3 100644 --- a/hosts/jupiter/secrets.nix +++ b/hosts/jupiter/secrets.nix @@ -1,14 +1,8 @@ { config, ... }: -# sops-nix secret wiring (real host only; not imported by vm.nix). -# Encrypted values live in ../../secrets/jupiter.yaml, decrypted at activation to -# /run/secrets/. -# -# The host decrypts with its OWN SSH host key (age identity derived via -# ssh-to-age, recipient listed in ../../.sops.yaml). The key is pre-generated on -# the laptop and shipped once at install as /etc/ssh/ssh_host_ed25519_key -# (nixos-anywhere --extra-files) — so decryption works on boot #1 and there is -# no separate sops-only key to manage. +# sops-nix secret wiring (real host only; not imported by vm.nix). Decrypts with the +# host's own SSH host key (ssh-to-age), shipped once at install via nixos-anywhere +# --extra-files, so there's no separate sops-only key to manage. { sops.defaultSopsFile = ../../secrets/jupiter.yaml; sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ]; @@ -26,18 +20,14 @@ # Headscale pre-auth key for tailscale auto-registration (see configuration.nix). sops.secrets.tailscale_authkey = { }; - # Immich's OIDC client secret, from its Authentik application (a SEPARATE - # app from headscale's and headplane's — see hosts/neptun/secrets.nix). - # Referenced as settings.oauth.clientSecret._secret in - # services/media/immich.nix; the module resolves it through systemd - # LoadCredential, which reads as root before dropping privileges, so the - # sops default of root:root 0400 is correct — do NOT set `owner`. + # Immich's OIDC client secret (separate Authentik app from headscale/headplane, see + # hosts/neptun/secrets.nix). Resolved via systemd LoadCredential as root before + # privilege drop, so sops's default root:root 0400 is correct — do NOT set `owner`. sops.secrets.immich_oauth_client_secret = { }; - # Gitea Actions runner registration token (services/dev/gitea.nix). Gitea - # generates this itself once Actions is enabled — it is not a password - # chosen up front. Rendered into a `TOKEN=...` env file because - # gitea-actions-runner takes an EnvironmentFile, not a raw secret path. + # Gitea Actions runner registration token — gitea generates this itself once Actions + # is enabled. Rendered into an env file since gitea-actions-runner takes an + # EnvironmentFile, not a raw secret path. sops.secrets.gitea_runner_token = { }; sops.templates."gitea-runner.env".content = "TOKEN=${config.sops.placeholder.gitea_runner_token}"; @@ -53,15 +43,11 @@ owner = "gitea"; }; - # SABnzbd credentials (web UI login, API keys, eweka.nl usenet server) — - # migrated off the reused ini in services/media/sabnzbd.nix into - # services.sabnzbd.settings + secretValues. sabnzbd_api_key predates this - # migration (provisioned for mediamanager's future use, services/experimental/ - # mediamanager.nix — not currently imported by any host); reused here as the - # same single source of truth rather than duplicating it. - # owner = sabnzbd: the module's preStart (replace-secret) runs as the - # service's own User=/Group=, and sops secrets default to root:root 0400 — - # without this, replace-secret gets Permission denied reading /run/secrets. + # SABnzbd credentials (web UI login, API keys, eweka.nl usenet server) for + # services/media/sabnzbd.nix; sabnzbd_api_key is shared with + # services/experimental/mediamanager.nix rather than duplicated. + # owner = sabnzbd because the module's preStart runs as that user, and sops secrets + # default to root:root 0400. sops.secrets.sabnzbd_web_username.owner = "sabnzbd"; sops.secrets.sabnzbd_web_password.owner = "sabnzbd"; sops.secrets.sabnzbd_api_key.owner = "sabnzbd"; @@ -69,15 +55,12 @@ sops.secrets.sabnzbd_eweka_username.owner = "sabnzbd"; sops.secrets.sabnzbd_eweka_password.owner = "sabnzbd"; - # CouchDB admin account for Obsidian LiveSync - # (services/dev/obsidian-livesync.nix). Rendered into an [admins] ini - # fragment rather than passed as services.couchdb.adminPass, which would put - # the plaintext in the world-readable store. - # - # owner = couchdb on BOTH: couchdb re-reads its ini chain as its own - # User=/Group= after systemd drops privileges, and sops defaults to - # root:root 0400 — without this it comes up with no admin configured, which - # under require_valid_user means every request 401s. + # CouchDB admin account for Obsidian LiveSync — rendered into an [admins] ini + # fragment instead of services.couchdb.adminPass, which would put the plaintext in + # the world-readable store. + # owner = couchdb on both: couchdb re-reads the ini as its own user after privilege + # drop, and without this sops's default root:root 0400 leaves it with no admin + # configured (every request 401s). sops.secrets.couchdb_admin_password.owner = "couchdb"; sops.templates."couchdb-admins.ini" = { owner = "couchdb"; diff --git a/hosts/mars/configuration.nix b/hosts/mars/configuration.nix index e2fa8cc..4a69aff 100644 --- a/hosts/mars/configuration.nix +++ b/hosts/mars/configuration.nix @@ -25,13 +25,12 @@ boot.loader.systemd-boot.enable = true; boot.loader.efi.canTouchEfiVariables = true; - # jupiter's samba share (services/network/samba.nix) — mounted on demand so - # mars doesn't stall boot/login when jupiter is off or unreachable. This is - # also where Hermes's shared dropbox lives now (hermes-agent.nix). Modes are - # tighter than terra's equivalent mount (0770 not 0755, gid=hermes not - # gid=users) since the hermes-agent container (uid 986, gid 983 — no podman - # userns remapping, see services/network/pihole.nix) needs group write into - # it, not just darman. + # jupiter's samba share (services/network/samba.nix), mounted on demand so + # mars doesn't stall when jupiter is off — also where Hermes's shared + # dropbox lives (hermes-agent.nix). Tighter modes than terra's equivalent + # mount (0770/gid=hermes, not 0755/gid=users) since the hermes-agent + # container (uid 986/gid 983, no podman userns remapping) needs group + # write here, not just darman. fileSystems."/mnt/jupiter" = { device = "//jupiter/data"; fsType = "cifs"; @@ -43,11 +42,9 @@ "dir_mode=0770" "nofail" "x-systemd.automount" # lazy-mount so boot doesn't stall if jupiter's down - # NO idle-timeout here (unlike terra's equivalent mount): hermes-agent's - # podman-hermes-agent.service RequiresMountsFor this path, so an idle - # auto-unmount tears the container down with it — confirmed the hard - # way, it killed the service ~60-70s after every start with no crash - # or error, just "Unmounting /mnt/jupiter" right before the stop. + # NO idle-timeout here (unlike terra's): podman-hermes-agent.service + # RequiresMountsFor this path, so an idle auto-unmount silently kills + # the container with it — confirmed the hard way (~60-70s per start). "x-systemd.mount-timeout=10s" "_netdev" ]; diff --git a/hosts/mars/hermes-agent.nix b/hosts/mars/hermes-agent.nix index 845b07a..f100b5d 100644 --- a/hosts/mars/hermes-agent.nix +++ b/hosts/mars/hermes-agent.nix @@ -1,83 +1,45 @@ { config, pkgs, ... }: -# Hermes Agent — moved here from jupiter (hosts/jupiter/hermes-agent.nix, -# see its git history / b5fa599 / 713d91d for the terra->jupiter->mars -# lineage). mars is dedicated to this one service, on-site, with no big -# data array of its own — unlike jupiter it has nothing under /mnt/data, so -# state lives on the local OS disk and the shared dropbox rides jupiter's -# samba share as a CIFS client instead of being served locally. +# Hermes Agent runs on mars, which has no big data array — state lives on the +# local OS disk, and the shared dropbox reaches jupiter's array as a CIFS +# client instead of being served locally. # -# Runs the OFFICIAL published image (docker.io/nousresearch/hermes-agent — -# real and actively maintained, contrary to what the checked-out repo's own -# README/docker-compose.yml suggested; verified directly on Docker Hub) as a -# plain podman container. It never sets HERMES_MANAGED or writes .managed, so -# Hermes fully self-manages config.yaml, profiles, memories and skills at -# runtime — no redeploy needed except to bump the pinned digest below. +# Runs the official docker.io/nousresearch/hermes-agent image (verified on +# Docker Hub) as a plain podman container. It never sets HERMES_MANAGED, so +# Hermes fully self-manages config.yaml, profiles, memories and skills. # -# Security posture: -# - Reachable paths: its own local state dir, the small shared "dropbox" -# (via the jupiter samba mount) for darman to hand files to Hermes, and -# `git`/`tea`, logged in as the `luna` gitea account (PR-tier only — -# see services/dev/gitea.nix). No working copy of this repo is -# provisioned for her: an earlier version cloned one into -# ${hermesHome}/workspace/homelab, dropped again because nothing ever -# told her at runtime where it was (she self-manages config/profiles/ -# memories, so a host-side path in this file never reached her) — she -# searched /opt/data/homelab and /workspace, found neither, and -# concluded she had no repo at all. She can clone one herself if she -# wants; the credentials below are what actually grants the access. -# Nothing else on jupiter's array or the host is reachable if a -# command goes wrong or gets injected via Telegram/tool output. -# - Its own Telegram bot (own token, in secrets.nix) with an EXPLICIT -# TELEGRAM_ALLOWED_USERS. -# - Runs as a rootful podman container (services/containers.nix) with its -# OWN numeric uid/gid — not darman, who is in the "hermes" group for -# host-level debugging only (`hermes ...` alias below, needs sudo since -# the container itself runs under root's podman, not darman's rootless -# one). -# - git/tea access is direct CLI, not a narrow wrapper: darman explicitly -# chose this over a purpose-built MCP server (tried first, scrapped — -# see git history) in favor of simplicity. The backstop is entirely -# server-side: gitea's branch protection on `master` (only darman can -# push/merge/approve there) is what actually keeps a bad or injected -# command from reaching the base branch, not anything client-side here. +# Security posture: reachable paths are only Hermes's own state dir, the +# shared dropbox, and git/tea as the PR-tier `luna` gitea account (see +# services/dev/gitea.nix) — no working copy of this repo is provisioned, and +# nothing else on jupiter's array or host is reachable if a command goes +# wrong or gets injected via Telegram/tool output. It runs its own Telegram +# bot with an explicit TELEGRAM_ALLOWED_USERS, and as a rootful podman +# container under its own uid/gid (not darman's). git/tea access is direct +# CLI rather than a wrapper; the real backstop is server-side gitea branch +# protection on `master` (only darman can push/merge/approve), not anything +# client-side here. # -# Dashboard (HERMES_DASHBOARD=1) is gated behind Authentik, same setup as on -# jupiter. Its default bind (0.0.0.0:9119) fails closed without an auth -# provider registered, and 0.0.0.0 (not loopback) is required so neptun's -# Caddy can reach it over tailscale0 — reachability itself stays LAN-closed -# (no networking.firewall.allowedTCPPorts entry; tailscale0 is already a -# trustedInterface, services/vpn/tailscale.nix). Public route: neptun's -# hermes.mgaction.town vhost (hosts/neptun/configuration.nix) proxies to this -# over the tailnet. mars's own Caddy (luna-sites.nix) only serves luna's apps -# and has no vhost for this — reach the dashboard directly via mars's tailnet -# name (mars.orbit.sol:9119) or LAN IP:9119 for local debugging. +# Dashboard (HERMES_DASHBOARD=1) is gated behind Authentik like jupiter's; it +# fails closed without a registered auth provider. Binds 0.0.0.0:9119 (not +# loopback) so neptun's Caddy can reach it over tailscale0, but stays +# LAN-closed since there's no firewall rule opening it — reach it directly at +# mars.orbit.sol:9119 or via the public hermes.mgaction.town vhost on neptun. +# Uses upstream's generic self-hosted OIDC plugin against the same Authentik +# application (slug `hermes`) as before. # -# Uses upstream's generic self-hosted OIDC plugin, same Authentik -# application as before (slug `hermes`) — the client ID/secret didn't need -# to change since the public redirect URI (hermes.mgaction.town) didn't. -# -# Data migration: this starts with a FRESH state dir. jupiter's instance was -# itself reset to fresh on 2026-08-21 (see its old hermes-agent.nix), so -# there was nothing irreplaceable to carry forward; if that turns out to be -# wrong, jupiter's old data is backed up at -# /mnt/data/AppData/hermes.bak-2026-08-21 and can be rsynced into -# ${hermesHome} below before the first switch on mars. +# Starts with a fresh state dir — jupiter's instance was already reset to +# fresh on 2026-08-21, so nothing needed carrying forward. Its old data is +# backed up at /mnt/data/AppData/hermes.bak-2026-08-21 if that's ever wrong. let stateDir = "/var/lib/hermes"; hermesHome = "${stateDir}/.hermes"; - # Shared drop-in folder: darman can put files here from any host. Lives on - # jupiter's array (reachable at /mnt/jupiter, the samba mount below) rather - # than locally, so it's the same physical location it always was — only - # the container reading it moved. Mounted under /opt/data so it falls - # inside Hermes's own sealed write-safe root (HERMES_WRITE_SAFE_ROOT= - # /opt/data) rather than a path its own tooling would treat as untrusted. + # Shared drop-in folder for darman to hand files to Hermes, on jupiter's + # array (CIFS mount below) rather than locally. Mounted under /opt/data so + # it's inside Hermes's own write-safe root (HERMES_WRITE_SAFE_ROOT). dropboxDir = "/mnt/jupiter/AppData/hermes-dropbox"; - # Pinned by digest (captured 2026-08-21 via `podman image inspect - # docker.io/nousresearch/hermes-agent:latest --format '{{.Digest}}'` on - # jupiter) rather than floating `:latest`, so a redeploy is reproducible — - # bumping Hermes is an explicit edit here, not silent drift on next pull. + # Pinned by digest (captured 2026-08-21 from jupiter) rather than floating + # :latest, so bumping Hermes is an explicit edit here, not silent drift. hermesImage = "docker.io/nousresearch/hermes-agent@sha256:5342e518734a08f6c66b89b4262434813c28a77abbc59c230c8f1637df71a259"; # Kept identical to jupiter's instance purely so nothing else needs to @@ -90,15 +52,10 @@ let # is hers to make, anywhere inside HERMES_WRITE_SAFE_ROOT=/opt/data. giteaHost = "git.mgaction.town"; - # luna's webhook filters, mounted READ-ONLY below. They live in the nix store - # rather than being written into hermesHome because hermesHome IS - # HERMES_WRITE_SAFE_ROOT: a filter dropped there is a loop guard sitting - # inside the writable root of the agent it constrains, and she could edit - # it back out. Deleting it would fail closed (Hermes treats a missing - # script as "ignore"), but rewriting it to always-allow would silently - # restore the reply loop. Read-only from the store makes that impossible - # and keeps the guard versioned in git — same reasoning as the git/tea - # binaries mounted below. + # luna's webhook filters, mounted READ-ONLY from the nix store rather than + # written into hermesHome: that IS her write-safe root, so a writable copy + # would let her edit her own loop guard back out. A missing script fails + # closed (Hermes ignores it); read-only from the store rules out a rewrite. prCommentFilter = pkgs.writeText "gitea-pr-comment-filter.py" ( builtins.readFile ./gitea-pr-comment-filter.py ); @@ -106,13 +63,10 @@ let builtins.readFile ./gitea-pr-review-filter.py ); - # The route prompts. These are NOT mounted into the container: the route - # config below embeds them as strings, and jq reads them from these store - # paths host-side with --rawfile. Keeping them in files rather than inline - # nix strings is still what makes that work — they are ~60 lines of markdown - # full of apostrophes and {placeholders} that would otherwise have to - # survive nix string escaping on the way into a shell command. --rawfile - # crosses all of that untouched, and they stay diffable in git. + # Route prompts: not mounted into the container, but embedded as strings by + # the route config below via jq --rawfile, which lets ~60 lines of markdown + # full of apostrophes/{placeholders} skip nix string escaping and stay + # diffable in git. prCommentPrompt = pkgs.writeText "gitea-pr-comment-prompt.md" ( builtins.readFile ./gitea-pr-comment-prompt.md ); @@ -126,88 +80,55 @@ let prCommentEvents = [ "issue_comment" ]; prReviewEvents = [ "pull_request_comment" "pull_request_rejected" ]; - # Toolsets granted to both routes' agent runs. - # - # Hermes defaults webhook runs to a deliberately narrow set (web_search, - # web_extract, vision_analyze, clarify) because a webhook payload is - # third-party content. That default cannot clone, edit or push, so neither - # prompt was executable under it: the run would be woken, read the comment, - # and have no way to act on it. - # - # This list REPLACES the platform default for these routes rather than - # merging with it, so anything the default provided has to be re-listed — - # "web" is here for that reason, not because the prompts ask for research. - # - # Upstream's stated boundary is that `hermes webhook subscribe` has no - # --toolsets flag, so "an agent creating its own subscription at runtime - # cannot self-grant terminal". That boundary does NOT hold here and must not - # be relied on: webhook_subscriptions.json lives under /opt/data, which is - # HERMES_WRITE_SAFE_ROOT, so luna can edit her own grant — she already did - # once, which is why this moved into nix. What this buys is that the grant - # is deliberate, reviewable and re-asserted on every restart, not that it is - # unforgeable. The real backstop stays server-side: gitea's branch - # protection on master. + # Toolsets granted to both routes' agent runs. Hermes's webhook default + # (web_search, web_extract, vision_analyze, clarify) has no shell/file/edit + # access, so neither prompt could act without this — and it REPLACES the + # default rather than merging, hence "web" being re-listed. luna could in + # principle self-grant via webhook_subscriptions.json (it's under her own + # HERMES_WRITE_SAFE_ROOT, and she has edited it before), so this only makes + # the grant reviewable and reasserted on restart, not unforgeable — the + # real backstop stays gitea's branch protection on master. routeToolsets = [ "terminal" "file" "web" ]; - # hermesHome as the CONTAINER sees it (the bind mount below). Anything - # written host-side that gets READ back inside the container must use this - # prefix, not hermesHome — see the credential.helper below, which was - # broken exactly that way from 3c1f3e5 until 2026-08-23. + # hermesHome as the CONTAINER sees it. Anything written host-side that gets + # READ back inside the container must use this prefix, not hermesHome. containerHome = "/opt/data"; in { - # Browsing convenience (ssh access to the bind-mounted local state) — does - # NOT touch the container, which keeps using HERMES_UID/GID above - # regardless of what's declared here. + # ssh browsing convenience only — the container still uses HERMES_UID/GID + # above regardless of this. users.groups.hermes.gid = 983; users.users.darman.extraGroups = [ "hermes" ]; - # `hermes ` on mars == `sudo podman exec -it hermes-agent hermes `. - # sudo is required: virtualisation.oci-containers runs rootful (system) - # podman, a separate namespace from darman's own rootless `podman`/`docker` - # — darman's "hermes"/"docker" group membership only grants filesystem - # access to the bind-mounted state dir, not to root's container socket. + # `hermes ` == `sudo podman exec -it hermes-agent hermes `. sudo + # is needed because oci-containers runs rootful podman, a separate + # namespace from darman's own rootless one. programs.zsh.shellAliases.hermes = "sudo podman exec -it hermes-agent hermes"; systemd.tmpfiles.rules = [ "d ${stateDir} 0750 root hermes -" ]; - # podman requires the bind-mount source to already exist (no auto-create), - # and the dropbox lives on the CIFS mount below — mkdir there works fine - # over cifs, no server-side (jupiter) config needed. + # podman needs the bind-mount sources to exist first; the dropbox lives on + # the CIFS mount below, which is fine to mkdir into directly. # - # Also provisions luna's git/tea access: writes a git credential-store file - # and runs `tea logins add` INTO hermesHome (i.e. paths that appear at - # /opt/data/... once the container is up). Both run on the HOST as root, - # before the container starts, and both therefore have to chown what they - # write themselves — see the chown at the end of the script. Do NOT assume - # the image's cont-init fixes ownership under hermesHome: it does not - # recurse into what this oneshot drops there, even though it runs after it. + # Also provisions luna's git/tea access as root, before the container + # starts, and chowns what it writes itself — the image's cont-init only + # fixes ownership of hermesHome's top level, not what this oneshot drops + # into it. No longer clones the repo for her (see the header); the version + # that did left a stale ${hermesHome}/workspace/homelab that this does not + # clean up. # - # It deliberately does NOT clone the repo for her any more (see the - # header). The stale ${hermesHome}/workspace/homelab left behind by the - # version that did is not cleaned up here either — it just stops being - # managed, and stops being updated. Remove it by hand if you want it gone. - # - # Delete-then-add for the tea login (not a "does it exist" check): tea can - # leave a login entry behind even when `add` reports failure (e.g. a token - # missing a scope errors out AFTER the entry is written — observed - # directly against the real instance during the first version of this - # setup). Delete-then-add is idempotent either way and picks up a rotated + # Delete-then-add for the tea login, not an existence check: tea can leave + # a login entry behind even when `add` itself reports failure, so + # delete-then-add is the only idempotent option and picks up a rotated # token for free. # - # `tea logins add` is the ONLY step in here that touches the network, and - # ordering is what makes it survivable. switch-to-configuration restarts - # NetworkManager and starts this unit in the SAME pass: on 2026-09-11 the - # two landed in the same second, tea's connect went out over an interface - # that was still coming back, and the kernel spent 2m48s on SYN retries - # before reporting "connection timed out". That failed this unit, which - # podman-hermes-agent Requires=, so a five-second network blip took the - # whole container down and returned 4 from the deploy. Hence - # network-online.target below, the bounded reachability probe in the script, - # and TimeoutStartSec as the backstop — no single blocking call in here may - # outlive the deploy that started it. + # `tea logins add` is the only network call here, and ordering matters: + # switch-to-configuration restarts NetworkManager in the same pass as this + # unit, and on 2026-09-11 that raced badly enough to hang the unit for + # minutes and take the whole container down. Hence network-online.target, + # the bounded probe below, and TimeoutStartSec as a backstop. systemd.services.hermes-agent-prepare-dirs = { description = "Create Hermes state dirs + luna's git/tea access before the container starts"; before = [ "podman-hermes-agent.service" ]; @@ -218,16 +139,13 @@ in path = [ pkgs.git pkgs.tea pkgs.curl pkgs.coreutils ]; serviceConfig.Type = "oneshot"; # Everything here is either local or bounded to ~30s by the probe loop, so - # anything past two minutes is a hang, not slowness. Failing at that point - # is strictly better than holding the deploy open. + # anything past two minutes is a hang, not slowness. serviceConfig.TimeoutStartSec = "120"; script = '' mkdir -p ${hermesHome} mkdir -p ${dropboxDir} - # Parent for the read-only filters bind-mounted at - # /opt/data/scripts/gitea-pr-*-filter.py. /opt/data is itself a bind - # mount of hermesHome, so this directory has to exist HOST-side before - # podman can mount a file inside it. + # Parent dir for the read-only filters bind-mounted below; must exist + # host-side first since /opt/data is itself a bind mount of hermesHome. mkdir -p ${hermesHome}/scripts export HOME=${hermesHome} @@ -241,25 +159,18 @@ in install -m 0600 /dev/null ${hermesHome}/.git-credentials printf 'https://luna:%s@${giteaHost}\n' "$(cat "$token_file")" \ > ${hermesHome}/.git-credentials - # containerHome, NOT hermesHome: git reads this .gitconfig from INSIDE - # the container, where the host path does not exist. Nothing host-side - # consumes these credentials any more (the clone that used to is gone), - # so the container's view is the only one that has to be right. + # containerHome, not hermesHome: git reads this .gitconfig from inside + # the container, and nothing host-side needs it any more. git config --global credential.helper "store --file=${containerHome}/.git-credentials" git config --global user.name "luna" git config --global user.email "luna@${giteaHost}" - # Probe before touching the login, with a hard per-attempt timeout: a - # bare TCP connect to an interface that is still coming up hangs for - # ~3 minutes on kernel SYN retries, and tea has no timeout flag of its - # own. /api/v1/version is unauthenticated, so this says "is gitea - # reachable", never "is the token good" — the token is the add's job. - # - # Probing FIRST (rather than retrying the add) is what protects the - # login that is already there. delete-then-add is not atomic: an add - # that fails because the network is down leaves luna with no login at - # all, strictly worse than the stale-but-working one we started with. - # Unreachable therefore means skip the refresh entirely and warn. + # A bare TCP connect to an interface still coming up can hang ~3min on + # kernel SYN retries, and tea has no timeout flag, so probe first with a + # hard per-attempt timeout. /api/v1/version is unauthenticated (tests + # reachability only). Probing before touching the login (rather than + # retrying the add) protects it: delete-then-add isn't atomic, so an add + # that fails on a down network would leave luna with no login at all. gitea_up=0 for attempt in 1 2 3; do if curl -fsS --max-time 5 -o /dev/null "https://${giteaHost}/api/v1/version"; then @@ -271,42 +182,29 @@ in done if [ "$gitea_up" = 1 ]; then - # Reachable but the add still fails == a real problem (revoked or - # under-scoped token, gitea rejecting the login), and that stays - # fatal: it is a config error, it will not fix itself on the next - # boot, and it should be loud. + # Reachable but still failing means a real problem (revoked/under- + # scoped token) — stays fatal since it won't fix itself on reboot. tea logins delete luna 2>/dev/null || true GITEA_SERVER_TOKEN="$(cat "$token_file")" timeout 60 tea logins add \ --name luna --url "https://${giteaHost}" --no-version-check else - # Deliberately not fatal. Every other thing this unit does is local, - # and podman-hermes-agent Requires= it — failing here would take - # Telegram and the dashboard down over a transient blip. luna keeps - # git (the credential helper above needs no network to be written) - # and loses only the tea CLI until the next start re-runs this. + # Not fatal: everything else here is local, and podman-hermes-agent + # Requires= this unit — failing here would take Telegram/dashboard + # down over a transient blip instead of just the tea CLI. echo "WARNING: ${giteaHost} unreachable; left luna's tea login untouched." >&2 fi - # Hand everything written above to the container's uid/gid. This does - # NOT happen by itself: the image's cont-init only chowns hermesHome's - # top level and its own state, so root-owned 0600 files dropped here by - # this oneshot (.git-credentials, and tea's config.yml — tea writes it - # 0600 too) are simply unreadable to uid ${hermesUid}. Symptom is not an - # error but an absence: git reports no credential helper and tea reports - # no login, i.e. "they're missing". Confirmed on the real instance - # 2026-08-23 — cont-init ran AFTER these files were written and left - # them root-owned regardless. + # Hand written files to the container's uid/gid: the image's cont-init + # only chowns hermesHome's top level, so root-owned files dropped here + # (confirmed on 2026-08-23) are otherwise unreadable to Hermes. # - # `if`, not `[ -d x ] && chown`: this script runs under `set -e`, where - # a false test as the left side of an && list takes the whole list's - # non-zero status and aborts the unit. + # `if`, not `[ -d x ] && chown`: this script runs under `set -e`, and a + # false test on the left of && would abort the whole unit. chown ${hermesUid}:${hermesGid} \ ${hermesHome}/.gitconfig \ ${hermesHome}/.git-credentials - # Same cont-init caveat as the files above: the directory is created - # here as root, and Hermes reads its scripts as uid ${hermesUid}. The - # mounted filters themselves are world-readable 0444 from the store, so - # only the directory needs handing over. + # Same cont-init caveat: this dir is created as root, and Hermes reads + # scripts as uid ${hermesUid}. chown ${hermesUid}:${hermesGid} ${hermesHome}/scripts if [ -d ${hermesHome}/.config ]; then @@ -330,25 +228,18 @@ in "${hermesHome}:/opt/data" "${dropboxDir}:/opt/data/dropbox" - # luna's Obsidian vault, kept in sync with CouchDB on jupiter by - # livesync-bridge.nix. Under /opt/data so it lands inside - # HERMES_WRITE_SAFE_ROOT and she can write notes, not just read them — - # same reasoning as the dropbox above. The bridge runs as this very - # uid/gid, so no ownership fixup is needed on either side. + # luna's Obsidian vault, synced with CouchDB on jupiter by + # livesync-bridge.nix. Under /opt/data so she can write notes, not just + # read them; the bridge runs as this same uid/gid so no chown is needed. "/var/lib/livesync-bridge/vault:/opt/data/vault" - # git/tea for luna: the image doesn't ship `tea` (and shouldn't be - # trusted to have a known-good `git` either), so both come from this - # host's Nix store instead — mounted read-only at fixed PATH-visible - # locations. /nix/store itself has to come along too since both - # binaries are dynamically linked against paths inside it; the store - # is read-only content-addressed build output, not a source of - # secrets, so mounting the whole thing read-only costs nothing beyond - # the two specific binaries actually being reachable. - # Read-only: see prCommentFilter above. Hermes resolves route scripts - # under ~/.hermes/scripts, which is /opt/data/scripts in here. The route - # prompts are NOT mounted — they are embedded in the route config the - # unit below writes, so nothing inside the container reads them. + # git/tea for luna: the image ships neither (and its own git shouldn't + # be trusted), so both come from this host's Nix store, read-only. + # /nix/store must come along too since both binaries are dynamically + # linked against it. + # Filters mounted read-only (see prCommentFilter above), where Hermes + # resolves route scripts (~/.hermes/scripts). Prompts are NOT mounted — + # they're embedded directly in the route config the unit below writes. "${prCommentFilter}:/opt/data/scripts/gitea-pr-comment-filter.py:ro" "${prReviewFilter}:/opt/data/scripts/gitea-pr-review-filter.py:ro" @@ -361,16 +252,12 @@ in HERMES_GID = hermesGid; TZ = "Europe/Berlin"; - # Point git/tea at the config the prepare-dirs oneshot wrote into - # hermesHome (visible here as /opt/data/...) — the credential-store - # helper, the luna gitea login, and (implicitly, via HOME not being - # overridden) darman's Hermes state stays wherever it already was. + # Points git/tea at the config prepare-dirs wrote into hermesHome + # (visible here as /opt/data/...). GIT_CONFIG_GLOBAL = "/opt/data/.gitconfig"; XDG_CONFIG_HOME = "/opt/data/.config"; - # HERMES_TIMEZONE is the highest-priority source hermes_time.py checks - # (ahead of config.yaml's `timezone` key) — the container has no host - # /etc/localtime bind-mount, so it defaults to UTC otherwise (fixed in - # 9403122 on jupiter; carried forward here). + # Highest-priority source hermes_time.py checks; without it the + # container defaults to UTC (no /etc/localtime bind-mount). HERMES_TIMEZONE = "Europe/Berlin"; # Dashboard + Authentik OIDC gate — see the file-level comment above. @@ -378,14 +265,11 @@ in HERMES_DASHBOARD_HOST = "0.0.0.0"; # must be tailscale0-reachable, not just loopback HERMES_DASHBOARD_OIDC_ISSUER = "https://auth.mgaction.town/application/o/hermes/"; HERMES_DASHBOARD_OIDC_CLIENT_ID = "4BqdJu3htnMtSZnyEu5zHnsSOvlEbw3Ie3mYVlh6"; - # uvicorn's proxy_headers=True (web_server.py) only trusts - # X-Forwarded-Proto from forwarded_allow_ips, which defaults to - # 127.0.0.1 — neptun's Caddy reaches this over the tailnet (a real - # routed IP), so without this the dashboard sees the raw scheme (http) - # and builds an http:// redirect_uri that Authentik rejects against its - # registered https:// one. Safe to trust any peer here: 9119 is already - # scoped to loopback + tailscale0 only (no LAN firewall rule), so - # nothing untrusted can reach this process to begin with. + # uvicorn only trusts X-Forwarded-Proto from forwarded_allow_ips + # (default 127.0.0.1); neptun's Caddy reaches this over a real routed + # tailnet IP, so without this it builds an http:// redirect_uri that + # Authentik rejects. Safe to trust any peer: 9119 is already scoped to + # loopback + tailscale0 only. FORWARDED_ALLOW_IPS = "*"; }; environmentFiles = [ config.sops.templates."hermes-agent.env".path ]; @@ -401,35 +285,19 @@ in unitConfig.RequiresMountsFor = [ "/mnt/jupiter" ]; }; - # The two Gitea webhook routes, written as config rather than created with - # `hermes webhook subscribe`. + # The two Gitea webhook routes, written as config (not via `hermes webhook + # subscribe`, which has no --toolsets flag — see routeToolsets above). + # Gitea posts directly to Hermes with X-Hub-Signature-256 and + # X-GitHub-Event, which is what Hermes validates against and reads the + # event name from. # - # Gitea posts straight at Hermes (jupiter's gitea-hermes-webhook-provision - # registers one hook per route at http://mars.orbit.sol:8644/webhooks/) - # — there is no relay in between. Gitea's addDefaultHeaders sends - # X-Hub-Signature-256 in GitHub's exact format AND X-GitHub-Event, - # unconditionally, for every webhook type, which is precisely what Hermes - # validates and reads the event name from. + # Written host-side into hermesHome (bind-mounted at /opt/data), so the + # webhook adapter hot-reloads it on the next delivery — no container + # restart needed. # - # WHY NOT `hermes webhook subscribe`: it has no --toolsets flag, and without - # a toolset override a webhook run gets Hermes's constrained default - # (web_search, web_extract, vision_analyze, clarify) — no shell, no file - # access, so neither prompt below can actually be carried out. Upstream's - # documented answer is to write the `toolsets` key into - # webhook_subscriptions.json by hand. Doing that by hand does not survive - # this unit, which re-provisions on every start, so the whole route - # definition moves here instead and the CLI is not used at all. See - # routeToolsets above for what that costs. - # - # This writes the file HOST-side. hermesHome is bind-mounted at /opt/data, - # so the container sees the same inode, and the webhook adapter hot-reloads - # the file (mtime-gated) on the next delivery — no container restart, and no - # `podman exec` quoting chain between nix and the prompt text. - # - # Events are WIRE names (X-GitHub-Event). Gitea spells the same events three - # different ways and two of the spellings collide — from - # HookEventType.Event() in modules/webhook/type.go, and updateHookEvents in - # routers/api/v1/utils/hook.go for the api column: + # Events below are WIRE names (X-GitHub-Event), not the api names + # gitea.nix's hooks use — gitea spells the same events three ways and two + # spellings collide: # # HookEventType wire name (here) api name (gitea.nix) # --------------------------- ---------------------- -------------------- @@ -439,46 +307,31 @@ in # pull_request_review_rejected pull_request_rejected pull_request_review # pull_request_review_approved pull_request_approved pull_request_review # - # Hermes matches these against X-GitHub-Event, i.e. the WIRE name. So - # "pull_request_comment" HERE means a review and "issue_comment" HERE means - # a comment — the exact inversion of how they read. X-GitHub-Event-Type - # carries the HookEventType, but Hermes does not look at it. This file and - # services/dev/gitea.nix therefore name the same event differently on - # purpose; neither is a typo. + # So "pull_request_comment" HERE means a review and "issue_comment" HERE + # means a comment — neither this file nor gitea.nix has a typo. # - # The api column is not a third alias but a coarser set: HasEvent - # (models/webhook/webhook.go) collapses all three review types onto - # pull_request_review, so the gitea hook cannot subscribe them separately. - # Approvals arrive here as a result and are dropped by NOT being in - # prReviewEvents — Hermes answers {"status": "ignored"} on the event match, - # before the filter script and before any LLM call. Widening to approvals is - # a mars-side change only: add "pull_request_approved" to prReviewEvents and - # "pull_request_review_approved" to the filter's ALLOWED_REVIEW_TYPES. + # api names collapse all three review types onto pull_request_review, so + # approvals can't be subscribed separately — they arrive here and are + # dropped by omission from prReviewEvents. Widen by adding + # "pull_request_approved" here and to the filter's ALLOWED_REVIEW_TYPES. # - # issue_comment on the wire covers comments on plain issues too; the hook - # does not subscribe those, and the comment filter's is_pull check drops - # them anyway if the hook is ever widened. + # issue_comment on the wire also covers plain-issue comments; the comment + # filter's is_pull check drops those if the hook is ever widened. # - # deliver is "log", not a chat target: both prompts tell her to answer in - # the pull request, so the PR comment IS the delivery. + # deliver is "log", not a chat target — both prompts answer directly in the + # pull request. # - # `script` is the selection that MUST NOT be retunable at runtime. - # gitea-pr-comment-filter.py drops luna's own comments before any LLM call, - # which is what stops the reply loop: the prompt tells her to answer on the - # PR, and her answer is itself a pull_request_comment. Both filters are - # bind-mounted read-only from the store above so the agent cannot edit her - # own guard out. Hermes resolves the name relative to ~/.hermes/scripts, - # hence the bare filename. + # `script` must not be retunable at runtime: the filter drops luna's own + # comments before any LLM call (what stops the reply loop, since her PR + # answer is itself a pull_request_comment), and is mounted read-only so she + # can't edit her own guard out. # - # What read-only does NOT buy: it protects the sources, and this unit - # re-asserts prompt, filter, events and toolsets from them on every start, - # so a restart restores the intended config. The live file is inside the - # agent's own write-safe root, so a self-modification sticks until this unit - # next runs. + # Read-only protects the source only — this unit re-asserts prompt, filter, + # events and toolsets on every start, so a live self-modification only + # sticks until the next restart. # - # Routes this unit does not name are left alone (the merge below is - # per-key), so retiring an old one stays a deliberate one-off: - # sudo podman exec hermes-agent hermes webhook remove + # Routes not named here are left alone (the merge below is per-key); + # retire one with `sudo podman exec hermes-agent hermes webhook remove `. systemd.services.hermes-agent-webhook-routes = { description = "Write Hermes's Gitea webhook route config"; wantedBy = [ "multi-user.target" ]; @@ -504,26 +357,19 @@ in tmp="$conf.new" trap 'rm -f "$tmp"' EXIT - # --slurpfile below cannot read a file that does not exist. Creating it - # empty is safe: this only ever happens before the first run, when there - # are no routes to lose. If it exists but is not valid JSON, slurpfile - # fails the unit loudly and leaves it untouched, which is the right - # direction — better a failed unit than silently discarded routes. + # --slurpfile needs the file to exist; empty is safe pre-first-run. + # Invalid JSON fails the unit loudly and leaves it untouched — better a + # failed unit than silently discarded routes. [ -e "$conf" ] || printf '%s\n' '{}' > "$conf" - # The secret reaches jq via --rawfile, never argv: /proc//cmdline - # is world-readable, so `--arg secret "$(cat ...)"` would publish it to - # every user on the box for the lifetime of the process. Same reason the - # prompts come in by path rather than by value. - # - # sops stores this one without a trailing newline (see secrets.nix), but - # rtrimstr is kept anyway: a stray newline would silently change the key - # the HMAC is computed with and fail every delivery afterwards. - # - # The emptiness guards are load-bearing. Without them a truncated secret - # file or an unreadable prompt yields "", and the route is written with - # an empty secret — which fails EVERY signature check while the unit - # still reports success. + # Secret goes to jq via --rawfile, never argv (cmdline is world + # readable) — same reason the prompts come in by path, not value. + # sops stores this without a trailing newline, but rtrimstr guards + # against one anyway: it would silently change the HMAC key. + # The emptiness guards are load-bearing: without them a truncated + # secret or unreadable prompt yields "", and the route is written with + # an empty secret that fails every signature check while reporting + # success. jq -n \ --slurpfile existing "$conf" \ --rawfile rawSecret "$SECRET_FILE" \ @@ -547,12 +393,9 @@ in deliver: "log", toolsets: $toolsets }; - # created_at is cosmetic (hermes webhook list prints it) and is the - # one key carried over from whatever is already there, so it keeps - # reading as when the route first appeared rather than as the last - # deploy. Everything else is replaced outright: a leftover key from - # an earlier definition — or from a hand edit — would otherwise - # survive here forever. + # created_at is cosmetic and the only key carried over from any + # existing route; everything else is replaced outright so a + # leftover key from an earlier definition can't survive here. def upsert($name; $r): .[$name] = ($r + { created_at: (.[$name].created_at // (now | todate)) }); @@ -566,10 +409,9 @@ in $reviewEvents; $reviewPrompt; "gitea-pr-review-filter.py")) ' > "$tmp" - # 0600 because the file holds the HMAC secret in cleartext, and owned by - # the container's uid because Hermes rewrites it itself whenever anything - # calls `hermes webhook subscribe`. mv is an atomic rename within the - # same directory, so a delivery landing mid-write never reads a half + # 0600: holds the HMAC secret in cleartext. Owned by the container's + # uid since Hermes rewrites this file itself on `webhook subscribe`. + # mv is an atomic rename, so a delivery mid-write never sees a half # written config. chmod 0600 "$tmp" chown ${hermesUid}:${hermesGid} "$tmp" diff --git a/hosts/mars/livesync-bridge.nix b/hosts/mars/livesync-bridge.nix index 3b763ba..6f42daf 100644 --- a/hosts/mars/livesync-bridge.nix +++ b/hosts/mars/livesync-bridge.nix @@ -1,59 +1,47 @@ { config, pkgs, inputs, ... }: # livesync-bridge (vrtmrz) — mirrors an Obsidian LiveSync vault out of CouchDB -# on jupiter (services/dev/obsidian-livesync.nix) into a real directory of -# markdown here, so luna can read and write the vault as files. Obsidian itself -# is an Electron GUI with no headless mode, and an agent wants files anyway. -# -# ⚠️ THE WRITE-BACK PATH IS THE RISKY ONE. Upstream has three open, unanswered -# issues on storage->couchdb — #50 (Jun 2026, writes detected and logged as -# uploaded, database never updated), #23 (only lowercase filenames transmitted -# from storage), #46 (silent stall on files over ~30KB). All fail QUIETLY: the -# log says success and the note never arrives. So do not treat this directory -# as durable storage for anything luna cannot regenerate, and check that her -# edits actually reach your devices before trusting it. (E2EE itself is fine — -# PeerCouchDB.ts hard-errors if a passphrase is missing for an encrypted -# remote, so it is a deliberate code path. The one issue claiming E2EE breaks -# bridging, #12, is a single unreproduced report with no maintainer reply.) +# on jupiter (services/dev/obsidian-livesync.nix) into real markdown files +# here, since Obsidian itself is a GUI-only Electron app and luna needs files. # +# ⚠️ THE WRITE-BACK PATH IS THE RISKY ONE: upstream has open bugs where a +# write is logged as uploaded but the database is never updated (#50), only +# lowercase filenames sync from storage (#23), and files over ~30KB silently +# stall (#46) — all fail quietly with no error in the log. Don't treat this +# directory as durable for anything luna can't regenerate, and verify her +# edits actually reach your devices. (E2EE itself is fine — it hard-errors on +# a missing passphrase rather than failing silently.) # # EXPECTED NOISE ON FIRST SYNC: a stack trace per historically-deleted file — -# NotFound: ... remove '/Welcome.md' at PeerStorage.delete -# CouchDB keeps deletion tombstones, and the bridge replays them against a -# directory where the file never existed. PeerStorage.ts:33-40 catches it, -# logs, and returns false, so nothing is wrong; it only LOOKS fatal because -# main.ts pins the logger to LOG_LEVEL_DEBUG, which prints exception dumps -# that are otherwise verbose-level. It stops once the initial catch-up ends. -# Talks to CouchDB over the TAILNET (jupiter.orbit.sol:5984), not through -# neptun: mars is a tailnet node, so the public vhost, its TLS and its path -# allowlist are all irrelevant here. +# CouchDB replays deletion tombstones against a directory where the file +# never existed. Harmless, caught and logged, and stops once the initial +# catch-up ends. +# +# Talks to CouchDB over the tailnet (jupiter.orbit.sol:5984) directly — mars +# is a tailnet node, so neptun's public vhost/TLS/allowlist don't apply here. let stateDir = "/var/lib/livesync-bridge"; appDir = "${stateDir}/app"; vaultDir = "${stateDir}/vault"; - # The same uid/gid the hermes-agent container runs as (hermes-agent.nix). - # Deliberate: the bridge and luna both read and write these files, and - # sharing one uid removes any dependence on the container's umask. Two - # different uids in a shared group only works while every file stays - # group-writable, and a single 0644 file dropped by the agent would stall - # sync on that path with nothing but a permission error in the log. + # The same uid/gid hermes-agent runs as (hermes-agent.nix), so both peers + # share files without depending on umask — two uids in a shared group only + # works while every file stays group-writable, and one 0644 file from the + # agent would silently stall sync. hermesUid = 986; - # Which vault. `group` is what pairs the two peers — both must match or the - # bridge starts cleanly and simply never syncs anything. + # `group` pairs the two peers — mismatched and the bridge starts but never + # syncs. # - # ⚠️ `database` must be the name entered in the Obsidian plugin for luna's - # vault. Get it wrong and nothing errors: the credential below is CouchDB's - # admin, so PouchDB CREATES the misnamed database and replicates an empty - # vault into it quite happily. + # ⚠️ `database` must match the name entered in the Obsidian plugin exactly: + # get it wrong and nothing errors, since the admin credential below lets + # PouchDB just create the misnamed database and replicate an empty vault. peerGroup = "luna"; database = "luna_wiki"; in { - # hermes-agent.nix declares the GROUP (gid 983) but no user: the container - # brings its own uid and needs no host account. The bridge does need one to - # run as, so the matching user is declared here. + # hermes-agent.nix declares the group (gid 983) but no user — the container + # needs no host account, but this service does, so it's declared here. users.users.hermes = { uid = hermesUid; group = "hermes"; @@ -62,27 +50,23 @@ in description = "Hermes agent uid, shared with the livesync-bridge service"; }; - # Created here rather than by the service so they exist before anything - # tries to use them: - # - vaultDir before podman-hermes-agent starts, because a bind-mount - # source that does not exist is created by podman as root:root and the - # bridge then cannot write into its own vault; - # - appDir because WorkingDirectory applies to ExecStartPre as well, so a - # missing one fails the unit before preStart ever gets to create it. + # Created here, not by the service, so they exist before anything needs + # them: vaultDir before podman-hermes-agent starts (else podman creates it + # as root:root), and appDir before ExecStartPre runs (WorkingDirectory + # applies to it too). systemd.tmpfiles.rules = [ "d ${vaultDir} 0770 hermes hermes -" "d ${appDir} 0750 hermes hermes -" "d ${stateDir}/deno 0750 hermes hermes -" ]; - # The bridge's peer config, rendered by sops because it carries three - # secrets inline (CouchDB password + both passphrases) and the file format - # has no include mechanism. + # Rendered by sops (three inline secrets: CouchDB password + both + # passphrases; the json format has no include mechanism). # - # ⚠️ sops substitutes placeholders into the ALREADY-RENDERED json, so a - # secret containing a double quote or a backslash produces an invalid config - # and the bridge logs "Could not parse configuration!" and then sits there - # with zero peers — it does not exit. Keep all three values alphanumeric. + # ⚠️ sops substitutes into the ALREADY-RENDERED json, so a secret with a + # quote or backslash yields invalid config — the bridge then just sits with + # zero peers logging "Could not parse configuration!" instead of exiting. + # Keep all three values alphanumeric. sops.templates."livesync-bridge.json" = { owner = "hermes"; content = builtins.toJSON { @@ -96,11 +80,10 @@ in username = "obsidian"; password = config.sops.placeholder.couchdb_luna_password; passphrase = config.sops.placeholder.obsidian_luna_passphrase; - # The plugin derives path obfuscation from the same passphrase it - # uses for content, so this is the same secret. Split into its own - # field because the bridge takes them separately — if paths come - # back as garbage while contents decode fine, this is the field that - # is wrong. + # Same secret as the content passphrase — the plugin derives path + # obfuscation from it too, but the bridge takes them as separate + # fields. If paths come back as garbage while contents decode fine, + # this is the field to check. obfuscatePassphrase = config.sops.placeholder.obsidian_luna_passphrase; # Reads the chunking tweaks the plugin stored in the remote, instead # of guessing sizes that then disagree with every other client. @@ -137,22 +120,17 @@ in HOME = stateDir; }; - # Copy the pinned source out of the store and install its locked deps. - # It cannot run from /nix/store directly: deno.jsonc sets - # `nodeModulesDir: manual` with byonm, so `deno install` must write a - # node_modules/ next to the sources. + # Copies the pinned source out of the store and installs locked deps, + # since deno.jsonc's `nodeModulesDir: manual` (byonm) needs to write + # node_modules/ next to the sources — it can't run from /nix/store directly. # - # The copy target is a FIXED path on purpose. Deno keys localStorage — - # which is where the bridge records per-file sync state (Peer.ts:119) — by - # the main module's origin, and stores it under - # DENO_DIR/location_data/. VERIFIED by running the same - # source from two paths against one DENO_DIR: two separate origin dirs - # appear. Running straight from /nix/store would therefore change the - # origin on every input bump and silently reset the bridge to a full - # rescan of both peers. + # The copy target is a FIXED path on purpose: Deno keys its localStorage + # (where the bridge tracks per-file sync state) by the main module's + # origin, so running straight from /nix/store would change that origin — + # and reset the bridge to a full rescan of both peers — on every input bump. # - # Guarded by a stamp file so this is a no-op on ordinary restarts; only a - # flake input bump pays for the re-install (which needs network). + # Guarded by a stamp file: a no-op on ordinary restarts, only a flake + # input bump pays for the (networked) re-install. preStart = '' set -eu stamp=${stateDir}/.src diff --git a/hosts/mars/luna-sites-test.nix b/hosts/mars/luna-sites-test.nix index bf9bb07..1c22b1f 100644 --- a/hosts/mars/luna-sites-test.nix +++ b/hosts/mars/luna-sites-test.nix @@ -29,9 +29,9 @@ let cmd = [ "/bin/sleep" "infinity" ]; }; - # The "app" luna builds on top of. No network in the VM, so it is loaded - # from the store instead of pulled. Runs under luna-apps, which has no - # /nix/store mount — hence the closure inside the image. + # The "app" luna builds on top of, loaded from the store since the VM has + # no network. Runs under luna-apps, which has no /nix/store mount — hence + # the closure baked into the image. app = busyboxImage { name = "testapp"; extraCommands = "mkdir -p www && echo hello > www/index.html"; diff --git a/hosts/mars/luna-sites.nix b/hosts/mars/luna-sites.nix index 2e3e527..2977f47 100644 --- a/hosts/mars/luna-sites.nix +++ b/hosts/mars/luna-sites.nix @@ -13,31 +13,22 @@ # /var/lib/luna-sites/live/.caddy root-owned, imported by caddy # /opt/data/sites-status.txt what was accepted, and why not # -# Why a registry of {name, port} instead of letting her drop Caddyfile -# snippets: a snippet can proxy to anything on this box (the dashboard on -# 9119, the webhook listener on 8644, node-exporter) or file_server anything -# caddy can read, and one syntax error keeps caddy from coming up on the next -# boot. The generator only ever emits one fixed shape from a validated name -# and a port inside portMin..portMax, so none of that is expressible. +# A registry of {name, port}, not raw Caddyfile snippets from her: a snippet +# could proxy to anything on the box or break caddy on the next boot, while +# the generator only ever emits one validated shape. # -# Why paths, not .mars.sol: mars has no fixed DHCP lease, and a wildcard -# needs one. `address=/…/` takes an IP, and pihole-FTL's dnsmasq skips -# wildcard --cname entries outside authoritative zones (cache_reload(): -# `if (a->alias[1] != '*' …)`). Moving to subdomains later only changes the -# fragment the generator writes; the registry format stays. +# Paths, not .mars.sol: mars has no fixed DHCP lease, and pihole-FTL's +# dnsmasq can't wildcard-CNAME without one. # -# Why a podman socket instead of ssh: what she needs is long-running processes -# OUTSIDE her own container (anything started inside it dies with the -# container, and sits next to her Telegram/gitea tokens). The socket gives -# exactly that and no host shell. It is not a strong boundary on its own — -# rootless podman socket access is code execution as luna-apps, which can read -# whatever that user can — but luna-apps owns nothing and cannot enter -# /var/lib/hermes (0750 root:hermes), so the apps cannot reach her tokens. +# A podman socket, not ssh: gives her long-running processes outside her own +# container (which dies on restart and holds her tokens) with no host shell. +# It's not a strong boundary by itself — socket access is code execution as +# luna-apps — but luna-apps can't enter /var/lib/hermes (0750 root:hermes), so +# her apps can't reach her tokens. # # She learns all this from a read-only README mounted at -# /opt/data/sites-README.md (luna-sites-README.md). She self-manages her -# memories, so nothing in this file reaches her otherwise — see the dropped -# repo clone in hermes-agent.nix's header for what happens when it doesn't. +# /opt/data/sites-README.md (luna-sites-README.md) — she self-manages her own +# memory, so nothing else in this file reaches her. # # VM test: nix build .#checks.x86_64-linux.luna-sites -L (luna-sites-test.nix) let @@ -77,29 +68,26 @@ in isNormalUser = true; inherit uid; description = "luna's hosted web apps (rootless podman)"; - # Nothing ever logs in as this user. Only its systemd user manager runs, - # kept up without a session by linger, which is what brings the podman - # socket and podman-restart back after a reboot. + # No interactive login; linger keeps its systemd user manager (and thus + # the podman socket) running across reboots without a session. linger = true; autoSubUidGidRange = true; # rootless podman's user namespace hashedPassword = "!"; shell = "${pkgs.shadow}/bin/nologin"; }; - # `--restart=always` containers only come back after a reboot through this - # unit — rootless podman has no daemon to remember them. The podman module - # already enables podman.socket for every user's manager; this one is - # scoped to luna-apps. + # Rootless podman has no daemon to bring `--restart=always` containers back + # after a reboot; the podman module enables this for every user, scoped + # here to luna-apps. systemd.user.services.podman-restart = { wantedBy = [ "default.target" ]; unitConfig.ConditionUser = user; }; # ---- the socket luna's container talks to ---- - # luna-apps's own socket lives under /run/user/1001 (0700), which the - # container's uid cannot enter. This re-exposes it to group hermes, and the - # proxy behind it runs as luna-apps, so it holds no access beyond the socket - # it forwards to. + # luna-apps's own socket lives under /run/user/1001 (0700), unreachable to + # the container's uid; this re-exposes it to group hermes via a proxy that + # itself runs as luna-apps, so it holds no more access than the socket. systemd.sockets.luna-apps-podman = { wantedBy = [ "sockets.target" ]; listenStreams = [ "${socketDir}/podman.sock" ]; @@ -124,9 +112,9 @@ in # Merges into hermes-agent.nix's container definition. virtualisation.oci-containers.containers.hermes-agent = { volumes = [ - # The directory, not the socket file: the socket is created by systemd - # at boot, and a file bind mount would pin whatever inode was there when - # the container started. Read-only still permits connect(). + # Mounts the directory, not the socket file — a file bind mount would + # pin the inode present at container start, before systemd creates the + # socket. Read-only still permits connect(). "${socketDir}:${socketDir}:ro" "${config.virtualisation.podman.package}/bin/podman:/usr/local/bin/podman:ro" "${readme}:/opt/data/sites-README.md:ro" @@ -163,16 +151,13 @@ in description = "Turn luna's site registry into caddy routes"; # Also runs once at boot, for edits made while nothing was watching. wantedBy = [ "multi-user.target" ]; - # After caddy, so the reload below never races caddy's own start. Nothing - # orders caddy after THIS unit, which is what keeps the blocking - # `systemctl reload caddy` from waiting on its own start job. + # After caddy, so the reload below can't race caddy's own start; nothing + # orders caddy after this unit, so that reload never waits on its own. after = [ "caddy.service" ]; - # No start rate limit. The default (5 starts in 10s) is hit by nothing - # more than a handful of quick writes — the VM test does exactly that — - # and when it is, systemd also fails luna-sites.path for good - # (unit-start-limit-hit): every later registration is silently ignored - # until someone runs reset-failed. Bursts are absorbed by the debounce at - # the top of the script instead. + # No start rate limit: the default (5/10s) trips from just a handful of + # quick writes and permanently disables luna-sites.path (unit-start- + # limit-hit) until someone runs reset-failed. Bursts are absorbed by the + # script's own debounce instead. startLimitIntervalSec = 0; path = [ pkgs.jq pkgs.util-linux pkgs.diffutils config.services.caddy.package ]; # caddy validate wants somewhere to write its data/config dirs. @@ -195,9 +180,9 @@ in script = '' set -euo pipefail - # Everything that touches luna's tree runs as the container's uid, never - # as root: she controls every path under it, including swapping one for - # a symlink into /etc between a check here and its use. + # Runs as the container's uid, never root — she controls every path + # under it, including swapping one for a symlink between a check here + # and its use. as_luna() { setpriv --reuid=${hermesUid} --regid=${hermesGid} --clear-groups -- "$@"; } if [ ! -d ${hermesHome} ]; then @@ -313,15 +298,14 @@ in publish_report } - # Debounce: writes usually come in bursts (several files, or an editor's - # write-then-rename), and every trigger that lands while this oneshot - # is still activating merges into this same start job instead of - # queuing another. One second collapses a burst into one run. + # Debounce: any trigger landing while this oneshot is still activating + # merges into the same start job, so one second collapses a burst of + # writes (several files, an editor's write-then-rename) into one run. sleep 1 - # That merging also means an entry written mid-run would otherwise wait - # for the next unrelated change. Compare the registry before and after, - # and go again. Bounded, so a writer in a loop cannot pin the unit. + # That same merging means an entry written mid-run would otherwise wait + # for the next unrelated trigger, so compare the registry before/after + # and rerun if it changed — bounded, so a writer in a loop can't pin it. for attempt in 1 2 3 4 5; do before=$(entries) generate diff --git a/hosts/mars/secrets.nix b/hosts/mars/secrets.nix index dd87e49..4f86df0 100644 --- a/hosts/mars/secrets.nix +++ b/hosts/mars/secrets.nix @@ -22,25 +22,16 @@ password=${config.sops.placeholder.samba_password} ''; - # Hermes Agent (hermes-agent.nix) — moved here from jupiter (see that - # host's git history); same Telegram bot token, opencode key, and - # Authentik OIDC client secret, so no new bot/app to provision. + # Hermes Agent (hermes-agent.nix) — same Telegram bot token, opencode key, + # and Authentik OIDC client secret as it used before moving here from + # jupiter, so no new bot/app to provision. sops.secrets.opencode_go_api_key = { }; sops.secrets.telegram_bot_token = { }; sops.secrets.hermes_dashboard_oidc_client_secret = { }; - # Same value as in secrets/jupiter.yaml (the sending side), stored WITHOUT a - # trailing newline — a stray newline would change the key the HMAC is - # computed with and fail every delivery. `scripts/edit_secrets` writes a - # bare value. hermes-agent.nix trims one anyway, belt and braces. - # - # This is NOT in the container's env any more. It used to be, because - # hermes-agent-webhook-route ran `hermes webhook subscribe` inside the - # container and read the secret back out of its environment — which meant - # podman-hermes-agent had to be restarted first on rotation, or the - # subscription silently pinned the stale value. The route config is now - # written host-side (hermes-agent-webhook-routes reads this file directly), - # so that ordering constraint is gone and the secret no longer sits in an - # env var luna can read with `env`. + # Same value as secrets/jupiter.yaml (the sending side), stored WITHOUT a + # trailing newline — a stray newline would change the HMAC key and fail + # every delivery. Written host-side by hermes-agent-webhook-routes, so it + # no longer needs to sit in the container's env where luna could read it. sops.secrets.gitea_hermes_webhook_secret = { restartUnits = [ "hermes-agent-webhook-routes.service" ]; }; @@ -54,47 +45,31 @@ HERMES_DASHBOARD_OIDC_CLIENT_SECRET=${config.sops.placeholder.hermes_dashboard_oidc_client_secret} ''; - # luna's own gitea push token (services/dev/gitea.nix provisions the - # account + PR-tier repo access on jupiter; this is the per-user token - # generated once via `gitea admin user generate-access-token --username - # luna --scopes write:repository,read:user` on jupiter — read:user is - # required, `tea logins add` fails without it). Read directly by - # hermes-agent.nix's prepare-dirs oneshot (default root:root owner is - # fine — that oneshot already runs as root) to set up a git - # credential-store file and a `tea` login, both written into hermesHome - # so they're visible inside the container at /opt/data/.... - # restartUnits re-provisions both on rotation, without a full mars deploy. + # luna's gitea push token (services/dev/gitea.nix provisions the account + + # PR-tier access), generated once via `gitea admin user generate-access-token + # --username luna --scopes write:repository,read:user` on jupiter — read:user + # is required or `tea logins add` fails. restartUnits re-provisions the git + # credential-store file and `tea` login on rotation, without a full deploy. sops.secrets.gitea_luna_token.restartUnits = [ "hermes-agent-prepare-dirs.service" ]; # livesync-bridge (livesync-bridge.nix) — luna's Obsidian vault, mirrored - # out of CouchDB on jupiter. Both values are consumed by the rendered - # config.json rather than read directly, so the sops default of root:root - # 0400 is correct here; only the TEMPLATE needs an owner (set where it is - # defined, next to the vault path it references). + # from CouchDB on jupiter. Consumed only via the rendered config.json, so + # the sops default of root:root 0400 is fine here. # - # couchdb_luna_password holds jupiter's `obsidian` ADMIN password — the same - # value as secrets/jupiter.yaml's couchdb_admin_password — and - # obsidian_luna_passphrase is the same passphrase as the personal vault. - # That is a deliberate choice to reuse what already existed, but it is worth - # being clear about what it costs: mars can decrypt and read EVERY vault - # database, not just luna's, and mars is the box running an autonomous - # agent. The two are independent to fix, cheapest first: - # - # 1. A vault-specific passphrase (re-encrypts luna's remote database, but - # leaves the personal vault's contents unreadable from here). - # 2. A CouchDB account scoped to luna's database via _security (three curl - # calls, in README -> "Obsidian vaults"), which also stops mars from - # reaching the other databases at all. - # - # Neither is required for the bridge to work; both shrink the blast radius - # if mars is ever compromised. + # ⚠️ couchdb_luna_password is jupiter's `obsidian` ADMIN password (same as + # secrets/jupiter.yaml's couchdb_admin_password) and obsidian_luna_passphrase + # reuses the personal vault's passphrase — reusing what already existed, but + # it means mars (running an autonomous agent) can decrypt and read EVERY + # vault database, not just luna's. To shrink that blast radius: give luna's + # vault its own passphrase, and/or scope a CouchDB account to her database + # via _security (README -> "Obsidian vaults"). Neither is required for the + # bridge to work. sops.secrets.couchdb_luna_password = { }; # The E2EE passphrase for luna's vault, as entered in the Obsidian plugin. - # Vault passphrases otherwise never leave the clients (see the note in - # services/dev/obsidian-livesync.nix) — this one has to be here because mars - # IS a client: it decrypts in order to write real markdown to disk. Path - # obfuscation uses the same passphrase in the plugin, so the bridge's - # separate obfuscatePassphrase field is fed from this one value. + # Vault passphrases otherwise never leave the clients (obsidian-livesync.nix) + # — this has to be here because mars IS a client, decrypting to write real + # markdown to disk. Also feeds the bridge's separate obfuscatePassphrase + # field, since the plugin derives path obfuscation from the same value. sops.secrets.obsidian_luna_passphrase = { }; } diff --git a/hosts/mercury/configuration.nix b/hosts/mercury/configuration.nix index 8062ae3..a9abd3d 100644 --- a/hosts/mercury/configuration.nix +++ b/hosts/mercury/configuration.nix @@ -16,26 +16,25 @@ networking.hostName = "mercury"; # ---- Static networking ---- - # A DNS/DHCP server must have a fixed address. Fill in the Pi's real values - # (from `ip -brief a` / `ip route` on the running Pi). eth0 = the Pi's NIC. + # A DNS/DHCP server needs a fixed address (values from `ip -brief a` / `ip + # route` on the running Pi; eth0 is its NIC). networking.useDHCP = false; networking.usePredictableInterfaceNames = false; # keep it named eth0 networking.interfaces.eth0.ipv4.addresses = [ { address = "10.0.0.10"; prefixLength = 24; } # the Pi's current IP ]; - # Stable IPv6 (FRITZ!Box ULA prefix) so mercury is a fixed IPv6 DNS target. - # SLAAC still provides the GUA + default route. Announce THIS address as the - # DNSv6 server in the FRITZ!Box so IPv6 clients resolve .sol via pihole. + # Stable IPv6 (FRITZ!Box ULA prefix) so mercury is a fixed IPv6 DNS target — + # SLAAC still handles the GUA + default route. Announce this address as the + # FRITZ!Box's DNSv6 server so IPv6 clients resolve .sol via pihole. networking.interfaces.eth0.ipv6.addresses = [ { address = "fd18:df17:9078:0::10"; prefixLength = 64; } ]; networking.defaultGateway = { address = "10.0.0.1"; interface = "eth0"; }; networking.nameservers = [ "1.1.1.1" "9.9.9.9" ]; - # Never take the tailnet's DNS on THIS host: headscale points every node at - # pihole, which runs here — mercury would be resolving through itself. Keep - # the public resolvers above for the Pi's own lookups, exactly as the - # unbound resolveLocalQueries note in CLAUDE.md requires. + # Never take the tailnet's DNS here: headscale points every node at pihole, + # which runs on this host, so mercury would resolve through itself — keep + # the public resolvers above for its own lookups. services.tailscale.extraUpFlags = [ "--accept-dns=false" ]; # ---- pihole web admin password (from sops) ---- diff --git a/hosts/mercury/secrets.nix b/hosts/mercury/secrets.nix index 3c1bc5e..6e6e5af 100644 --- a/hosts/mercury/secrets.nix +++ b/hosts/mercury/secrets.nix @@ -2,11 +2,10 @@ # sops-nix wiring for mercury. Encrypted values in ../../secrets/mercury.yaml. # -# SD images have no `--extra-files` step, so mercury uses a DEDICATED age key -# placed on the ROOT filesystem (the Pi's vfat partition isn't mounted at -# runtime — u-boot reads it pre-boot). `./deploy flash mercury ` drops -# ~/.config/homelab/mercury/age.txt there automatically. -# The key never enters the repo, the nix store, or the image itself. +# SD images get no `--extra-files` step, so mercury uses a dedicated age key +# on the root filesystem instead of the admin key — the Pi's vfat boot +# partition isn't mounted at runtime (u-boot reads it pre-boot), so the key +# can't live there. { sops.defaultSopsFile = ../../secrets/mercury.yaml; sops.age.keyFile = "/var/lib/sops-nix/age.txt"; diff --git a/hosts/neptun/configuration.nix b/hosts/neptun/configuration.nix index 5226657..a29223a 100644 --- a/hosts/neptun/configuration.nix +++ b/hosts/neptun/configuration.nix @@ -43,30 +43,19 @@ # default via fe80::1 dev eth0 metric 1024 onlink networking.defaultGateway6 = { address = "fe80::1"; interface = "eth0"; }; networking.nameservers = [ "9.9.9.9" "1.1.1.1" "2620:fe::fe" ]; - # Addressing is fully static above, but netcup's router still sends periodic - # RAs on this segment; the kernel then tries (and fails, since the static - # route already exists) to install its own default route from them, spamming - # "ndisc_router_discovery failed to add default route" on the console. Stop - # it from processing RAs on eth0 at all rather than just live with the noise. + # netcup's router still sends periodic RAs on this segment despite fully static + # addressing, spamming "ndisc_router_discovery failed to add default route" on the + # console. Stop processing RAs on eth0 entirely instead of living with the noise. boot.kernel.sysctl."net.ipv6.conf.eth0.accept_ra" = 0; # ---- Local split-DNS stub ---- - # neptun must NOT take the tailnet's DNS: headscale points every node at - # pihole on mercury, and making a public reverse proxy's name resolution - # depend on a Pi behind a domestic line would take ACME renewals — and so - # the certs for the control server every node needs — down with it. It is - # also circular, since tailscaled has to resolve vpn.mgaction.town to - # connect in the first place. - # - # So neptun opts out with --accept-dns=false and does its own split DNS. - # tailscaled still answers MagicDNS on 100.100.100.100 whenever it is - # running (--accept-dns only governs whether it rewrites resolv.conf), so - # dnsmasq forwards just the tailnet suffix there and everything else to the - # public resolvers above. jupiter's address is therefore resolved live and - # never pinned — nothing to update when the tailnet is rebuilt. - # - # resolveLocalQueries (default) points resolv.conf at 127.0.0.1 and feeds - # networking.nameservers to dnsmasq as upstreams via resolvconf. + # neptun must NOT take the tailnet's DNS: headscale points every node at pihole on + # mercury, and a public reverse proxy depending on a Pi on a domestic line for name + # resolution (and thus for its own ACME renewals) would be fragile and circular. + # It opts out (--accept-dns=false) and runs its own split DNS instead: dnsmasq + # forwards the tailnet suffix to MagicDNS (100.100.100.100, still answered by + # tailscaled) and everything else to the public resolvers above — jupiter's address + # is resolved live, never pinned. services.tailscale.extraUpFlags = [ "--accept-dns=false" ]; services.dnsmasq = { enable = true; @@ -112,49 +101,20 @@ ''; # ---- Obsidian LiveSync (CouchDB on jupiter) ---- - # Obsidian's mobile apps refuse cleartext HTTP and *.jupiter.sol cannot hold - # a publicly trusted cert, so the vault database is published here instead of - # staying on the LAN. That means a credentialed database on the open - # internet; two things keep it sane: + # Published publicly (mobile apps refuse cleartext HTTP; *.jupiter.sol has no public + # cert), kept safe by the plugin's end-to-end encryption (jupiter stores only + # ciphertext) plus this allowlist — CouchDB otherwise exposes Fauxton, /_all_dbs and + # /_node/_local/_config, the last of which can rewrite the server's config with admin + # creds. Use the tailnet directly for those: `curl http://jupiter.orbit.sol:5984/_utils/`. # - # 1. The plugin's end-to-end encryption, switched on BEFORE the first sync. - # jupiter then stores only ciphertext, so a breach here is not a leak of - # the notes themselves. - # 2. This allowlist. CouchDB serves far more than the replication API — - # Fauxton (/_utils), /_all_dbs, and /_node/_local/_config, the last of - # which REWRITES the server's config given admin credentials. Only the - # paths the plugin actually speaks are proxied; everything else is - # answered here and never reaches jupiter. Use the tailnet for the rest: - # `curl http://jupiter.orbit.sol:5984/_utils/`. + # The regex keys off CouchDB's own naming rule (system paths start with `_`, user + # databases can't) rather than listing vaults, plus `_session` for cookie auth — so a + # mistyped-but-legal name reaches CouchDB (real 404) while an illegal one gets + # caddy's 404 with no CORS, which Obsidian shows as a silent connection failure. + # Never point two vaults at the same database (LiveSync merges them, not reversibly). # - # ONE DATABASE PER VAULT, and the matcher keys off CouchDB's own naming rule - # rather than listing them: every system endpoint begins with `_`, and a - # user-creatable database never can (CouchDB requires a lowercase letter - # first). So adding a vault needs no edit here. `_session` is the single - # underscore path let through, for cookie auth. - # - # The flip side of not listing them: a mistyped but otherwise LEGAL database - # name is proxied through and reaches CouchDB, which answers a real 404 the - # plugin can report. An ILLEGAL one — anything starting with a capital or an - # underscore — fails the matcher instead and gets caddy's 404, which carries - # no CORS headers and surfaces in Obsidian as a connection failure with no - # error message at all. If a new vault refuses to connect and the plugin - # says nothing, check the database name is lowercase first. - # - # Never point two vaults at one database: LiveSync merges them into a single - # file tree, which is not cleanly reversible. - # - # Known consequence: LiveSync's "Check database configuration" panel reads - # /_node/_local/_config and so reports the server as unconfigured from - # outside. Expected — that config is declarative in - # services/dev/obsidian-livesync.nix and is not the plugin's to patch. - # - # `flush_interval -1` is required, not tuning: replication rides a - # continuous _changes feed, which caddy would otherwise buffer — sync then - # stalls until the buffer fills (same reason vpn.mgaction.town sets it). - # - # No netcup edge-firewall change: this rides the 443 the other vhosts - # already use, unlike gitea's :2222. + # `flush_interval -1` is required, not tuning — replication rides a continuous + # _changes feed that caddy would otherwise buffer, stalling sync. services.caddy.virtualHosts."notes.mgaction.town".extraConfig = '' @livesync path_regexp ^/(_session|[a-z][a-z0-9_$()+-]*)?(/.*)?$ handle @livesync { diff --git a/hosts/neptun/secrets.nix b/hosts/neptun/secrets.nix index 0a1ad0f..c067ba7 100644 --- a/hosts/neptun/secrets.nix +++ b/hosts/neptun/secrets.nix @@ -14,13 +14,11 @@ sops.secrets.darman_password.neededForUsers = true; users.users.darman.hashedPasswordFile = config.sops.secrets.darman_password.path; - # Authentik takes a single systemd EnvironmentFile (services/identity/authentik.nix). - # No `owner` here on purpose: systemd reads EnvironmentFile as root before - # dropping to the service's DynamicUser, so root:root 0400 is what we want. - # - # AUTHENTIK_SECRET_KEY signs sessions/tokens — rotating it logs everyone out. - # The BOOTSTRAP_* vars only take effect on the very first start, where they - # create the `akadmin` superuser; they're inert on every boot after that. + # Authentik takes a single systemd EnvironmentFile (services/identity/authentik.nix); + # no `owner` here on purpose, since systemd reads it as root before dropping to + # DynamicUser. AUTHENTIK_SECRET_KEY signs sessions (rotating it logs everyone out); + # the BOOTSTRAP_* vars only matter on the very first start (create `akadmin`) and are + # inert after. sops.secrets.authentik_secret_key = { }; sops.secrets.authentik_bootstrap_password = { }; sops.secrets.authentik_bootstrap_email = { }; @@ -38,17 +36,12 @@ ACME_EMAIL=${config.sops.placeholder.caddy_acme_email} ''; - # Headplane: cookie_secret_path takes a path natively (no store leak). - # oidc.client_secret + the headscale API key are still REPLACE_ME - # placeholders (see services/vpn/headplane.nix) until Authentik/headscale are - # actually deployed and those get created for real. - # - # owner: unlike authentik's EnvironmentFile above, headscale and headplane - # open these paths themselves, already running as the headscale user — so - # the root:root 0400 default would fail and each needs an explicit owner. - # - # headscale's OIDC client is a SEPARATE Authentik application from - # headplane's (services/vpn/headscale.nix), hence the second client secret. + # Headplane's cookie_secret_path takes a path natively (no store leak); oidc.client_secret + # and the headscale API key are still REPLACE_ME placeholders (services/vpn/headplane.nix) + # until Authentik/headscale are deployed for real. Unlike authentik's EnvironmentFile, + # headscale/headplane open these paths themselves as the headscale user, so each needs + # an explicit owner — and headscale's OIDC client is a separate Authentik app from + # headplane's, hence the second client secret. sops.secrets.headscale_oidc_client_secret.owner = "headscale"; sops.secrets.headplane_cookie_secret.owner = "headscale"; diff --git a/hosts/terra/configuration.nix b/hosts/terra/configuration.nix index ac74d69..30f04b7 100644 --- a/hosts/terra/configuration.nix +++ b/hosts/terra/configuration.nix @@ -44,17 +44,9 @@ in # https://nix.dev/permalink/stub-ld ---- programs.nix-ld.enable = true; - # The default set above is deliberately minimal and carries no X11, - # freetype, wayland or xkbcommon, so a prebuilt *graphical* binary dies - # before it draws anything. JetBrains IDEs installed through Toolbox are the - # case that surfaced this: their bundled JBR aborts with `libX11.so.6: - # cannot open shared object file` unless the Toolbox GUI — itself an FHS - # wrapper — is what launches them, which makes them unusable from a terminal - # or from a per-repo devShell. These are the libraries `ldd` reports missing - # across a JBR's own .so files, plus the three it resolves by dlopen rather - # than DT_NEEDED: fontconfig for font discovery, libGL, and libsecret for - # the credential store. Definitions merge, so this adds to the module's base - # list rather than replacing it (zlib is already there). + # JetBrains IDEs installed via Toolbox bundle a JBR that aborts with + # `libX11.so.6: cannot open shared object file` under the default (X11-less) + # nix-ld set. Additive — merges with the module's own base list (zlib etc). programs.nix-ld.libraries = with pkgs; [ freetype fontconfig @@ -72,28 +64,19 @@ in libxinerama libxcb - # CLion Nova's C++ backend (the clion-radler plugin) is a .NET 10 - # application bundling its own runtime, and .NET refuses to start - # without ICU: libSystem.Globalization.Native.so dlopens libicuuc.so - # and libicui18n.so, and failing that the IDE reports "Couldn't find a - # valid ICU package installed on the system" and comes up degraded. + # CLion Nova's C++ backend is a .NET 10 app that needs ICU or reports + # "Couldn't find a valid ICU package installed on the system". icu ]; - # ---- envfs: serves /bin and /usr/bin from the calling process's PATH ---- - # NixOS ships only /bin/sh, but plenty of third-party tooling writes scripts - # with a hardcoded interpreter. JetBrains Toolbox is the standing example: - # it generates ~/.local/share/JetBrains/Toolbox/scripts/{clion,rider,...} - # with `#!/bin/bash`, so every one of those shims fails with `bad - # interpreter` in any shell. envfs resolves such shebangs against PATH, - # which fixes them all at once instead of per-IDE wrappers. + # NixOS only ships /bin/sh; envfs serves /bin and /usr/bin from PATH so + # third-party scripts hardcoding `#!/bin/bash` (e.g. JetBrains Toolbox's + # generated launchers) still resolve. services.envfs.enable = true; # ---- home-manager (user-level config for darman) ---- - # Base settings (useGlobalPkgs/useUserPackages/backupFileExtension) and the - # shared zsh baseline now live in common.nix + home/common.nix, applied to - # every host. This just layers terra's desktop/dev-specific profile on top - # — home-manager.users.darman.imports merges additively across modules. + # Base settings + shared zsh baseline live in common.nix + home/common.nix + # (every host); this layers terra's desktop profile on top (imports merge). home-manager.extraSpecialArgs = { inherit unstable inputs; }; home-manager.users.darman.imports = [ ./home.nix ]; @@ -102,71 +85,47 @@ in boot.loader.efi.canTouchEfiVariables = true; hardware.cpu.amd.updateMicrocode = true; - # mercury (aarch64) is built/flashed from here. Without this, `nix build` - # for it dies with "platform mismatch" — no qemu binfmt handler registered - # and aarch64-linux missing from nix.settings.extra-platforms. This module - # sets up both (see CLAUDE.md's aarch64 gotcha). + # Lets `nix build` target mercury (aarch64) from here — see CLAUDE.md's + # aarch64 gotcha. boot.binfmt.emulatedSystems = [ "aarch64-linux" ]; # ---- GPU (Radeon RX 6800 XT / Navi 21) ---- hardware.enableRedistributableFirmware = true; boot.initrd.kernelModules = [ "amdgpu" ]; - # /dev/dri/renderD128 is root:render 0660, so rootless podman containers can - # only reach the GPU if the *host* user is in render. Needed by the Vulkan - # whisper.cpp/llama.cpp containers in ~/Data/Dev/repos/content-trigger-scanner. + # /dev/dri/renderD128 is root:render 0660 — host user needs render group for + # rootless podman GPU containers (Vulkan whisper.cpp/llama.cpp). users.users.darman.extraGroups = [ "render" "video" ]; # ---- ollama (local LLM server, ROCm on the 6800 XT) ---- - # Navi 21 is gfx1030 — officially supported by ROCm, so no - # rocmOverrideGfx/HSA_OVERRIDE_GFX_VERSION needed (that's for gpus ROCm - # doesn't recognize, e.g. RDNA1/gfx101x). The upstream module runs the - # service under DynamicUser with SupplementaryGroups=["render"] and - # DeviceAllow for char-kfd/char-drm/char-fb already, so unlike jellyfin's - # static user it needs no extraGroups wiring here. + # Navi 21 (gfx1030) is officially ROCm-supported, so no + # HSA_OVERRIDE_GFX_VERSION needed. Upstream module already runs under + # DynamicUser with render/kfd/drm access wired, unlike jellyfin's static user. services.ollama = { enable = true; package = pkgs.ollama-rocm; - # keep in sync with services/desktop/librechat.nix's endpoints.custom - # default model — LibreChat's config schema needs a non-empty default - # even though fetch=true replaces it with whatever's actually pulled. - # gemma4:12b: general chat/coding daily driver, fits fully in 16G VRAM — - # also doubles as the memory-extraction agent (see librechat.nix): a - # 3b model (llama3.2:3b, dropped) couldn't reliably tell the user's - # stated facts apart from its own boilerplate, e.g. saving "I am an AI - # assistant with tool calling capabilities" as the user's personal_info - # after "Hi I'm Erik Simon". Reusing gemma4:12b for both roles also means - # no second model needs to swap into VRAM while it's already the active - # chat model. - # qwen3.6:35b-a3b: MoE (3B active/36B total), ~24GB Q4_K_M — doesn't fit - # in VRAM alone, so ollama offloads the inactive experts to CPU RAM. - # Sparse activation makes that far less painful than it'd be for a dense - # model this size, but still expect it to run slower than the two above. - # VladimirGav/qwen3.8-27B-14GB-IQ4: dense 27B at IQ4, ~14GB of weights — - # nominally fits the 6800 XT's 16G, but that leaves only ~2G for the KV - # cache and the compositor, so expect partial CPU offload as context grows - # (OLLAMA_CONTEXT_LENGTH below applies to every model on this server). + # keep default model in sync with services/desktop/librechat.nix's + # endpoints.custom default (its schema needs a non-empty value even + # though fetch=true overrides it). + # gemma4:12b: daily-driver chat/coding model, fits fully in 16G VRAM; also + # doubles as LibreChat's memory-extraction agent (librechat.nix) since a + # smaller model confused the user's stated facts with its own boilerplate. + # qwen3.6:35b-a3b: MoE (3B active/36B total, ~24GB Q4_K_M) — doesn't fit + # in VRAM alone, so ollama offloads inactive experts to CPU RAM; sparsity + # makes that less painful than for a dense model this size, but still slower. + # VladimirGav/qwen3.8-27B-14GB-IQ4: dense 27B at IQ4 (~14GB) — nominally + # fits the 16G card but leaves little headroom, so expect partial CPU + # offload as context grows. loadModels = [ "gemma4:12b" "qwen3.6:35b-a3b" "VladimirGav/qwen3.8-27B-14GB-IQ4" ]; - # Ollama truncates context far below the model's real window unless - # told otherwise (the OpenAI-compat /v1 route it's reached through has - # no way to set this per-request). 131072 chosen as the practical - # ceiling after load-testing with real prompts, not just idle - # `ollama ps` checks: - # 32768 (31.6k-token prompt) and 65536 (40.8k-token prompt) both stayed - # 100% GPU with VRAM barely moving (~10.1G / ~10.67G of 16G) — KV cache - # cost barely grows with context, likely sliding-window/local attention - # on most of gemma4:12b's layers. At 131072 that stopped being true: a - # ~108k-token prompt pushed VRAM to ~11.4G/16G (still 100% GPU, no CPU - # spillover, negligible GTT) but with visibly shrinking headroom, and - # prefill throughput measurably dropped (~490 -> ~460 tok/s) over just - # the last 13k tokens — filling the full window would take minutes of - # pure prompt processing. Stopped here rather than push further: next - # doubling would risk CPU spillover under any concurrent GPU load - # (desktop compositor, jellyfin transcode) for diminishing benefit. + # Ollama truncates context far below a model's real window unless told + # otherwise. 131072 is the practical ceiling from load-testing: VRAM stays + # 100% GPU with no CPU spillover up to here, but headroom and prefill + # throughput both degrade near the top — going higher risks CPU spillover + # under concurrent GPU load (compositor, jellyfin transcode) for little gain. environmentVariables.OLLAMA_CONTEXT_LENGTH = "131072"; }; diff --git a/hosts/terra/disk-config.nix b/hosts/terra/disk-config.nix index 949a3e6..1333d45 100644 --- a/hosts/terra/disk-config.nix +++ b/hosts/terra/disk-config.nix @@ -5,27 +5,14 @@ # `fileSystems.*` entries, so hardware-configuration.nix must NOT define # fileSystems for "/" or "/boot". # -# ⚠️ disko's `mkfs` create step SKIPS formatting when `blkid` still detects a -# filesystem signature on the freshly-cut partition: -# -# if ! (blkid "$device" -o export | grep -q '^TYPE='); then -# mkfs.btrfs "$device" -f # ← -f only runs WHEN this line runs -# fi -# -# The disk previously held a CachyOS btrfs root. The whole-disk `wipefs` -# disko runs before partitioning clears the signature at the OLD layout's -# offsets, but `sgdisk --clear --align-end` then re-cuts the partitions, so -# a stale btrfs superblock survives at the NEW root partition's own 64 KiB -# offset. `blkid` sees TYPE=btrfs, `mkfs` is skipped entirely, and the -# later `mount` fails on the leftover bytes ("wrong fs type / bad -# superblock"). Switching ext4→btrfs did NOT fix this: `mkfs.btrfs -f` is -# never reached, because the guard is on whether `mkfs` runs at all, not on -# its flags. The ESP hits the same trap (its `mkfs.vfat` gets skipped too). -# -# Fix: `preCreateHook = wipefs --all --force "$device"` on each partition's -# content. The hook runs AFTER sgdisk re-cuts the partition but BEFORE the -# `blkid` guard, so it erases the stale signature at the FINAL offset; -# `blkid` then comes back empty and `mkfs` actually runs. +# ⚠️ disko's `mkfs` step skips formatting if `blkid` still detects a +# filesystem signature on the partition. Repartitioning doesn't erase +# signatures at the new offsets, so this disk's old CachyOS btrfs +# superblock survived, causing mkfs (and the ESP's mkfs.vfat) to be +# skipped and the later mount to fail on the stale superblock. +# Fix: `preCreateHook = wipefs --all --force "$device"` on each +# partition — it runs after sgdisk re-cuts the partition but before the +# `blkid` guard, so the guard sees no signature and `mkfs` actually runs. # # ⚠️ This disk is WIPED on install. This is the Kingston SA400 SSD that # currently holds CachyOS (btrfs root+subvols on sdb2, ESP on sdb1). @@ -58,8 +45,7 @@ type = "btrfs"; extraArgs = [ "-f" ]; mountpoint = "/"; - # erase the stale CachyOS btrfs superblock before disko's blkid - # format-guard, otherwise mkfs.btrfs is skipped (see header comment) + # same wipefs fix as the ESP above (see header comment) preCreateHook = ''wipefs --all --force "$device"''; }; }; diff --git a/hosts/terra/home.nix b/hosts/terra/home.nix index b61e6a5..d47b196 100644 --- a/hosts/terra/home.nix +++ b/hosts/terra/home.nix @@ -2,23 +2,17 @@ let tome = pkgs.callPackage ../../pkgs/tome.nix { src = inputs.tome; }; - # SUDO_ASKPASS helper: renders sudo's password prompt in the quickshell - # shell (HyprChrome/Widgets/Askpass) instead of on the terminal. + # SUDO_ASKPASS helper: shows sudo's password prompt in quickshell + # (HyprChrome/Widgets/Askpass) instead of the terminal. sudo doesn't speak + # polkit (setuid + PAM reading the tty), so this reuses the polkit dialog's + # look via the askpass mechanism instead — `run0` is the actual polkit-native + # alternative. # - # sudo does NOT speak polkit — it is setuid + PAM reading the tty, and no - # sudoers option bridges the two — so this is the askpass mechanism, a - # separate path that happens to reuse the polkit dialog's look. `run0` is the - # polkit-native alternative if you want the agent itself. + # Must be a package, not a dotfiles file: SUDO_ASKPASS needs an executable, + # and xdg.configFile copies keep store-copy permissions. # - # A package rather than a file in dotfiles/quickshell because SUDO_ASKPASS - # must point at something EXECUTABLE, and xdg.configFile copies keep their - # store mode — which is why open_launcher.sh has to be invoked as - # `bash ` rather than run directly. - # - # The secret comes back over a 0600 fifo, never in argv or the environment, - # so it is not visible in /proc to anything. Cancelling closes the fifo - # without writing: `cat` reads nothing, this exits non-zero, and sudo aborts - # instead of burning a retry on an empty password. + # The secret returns over a 0600 fifo (never argv/env, so not visible in + # /proc); cancelling closes the fifo unwritten so sudo aborts cleanly. qs-askpass = pkgs.writeShellApplication { name = "qs-askpass"; runtimeInputs = [ pkgs.quickshell pkgs.coreutils ]; @@ -74,11 +68,9 @@ in nix-direnv.enable = true; }; - # Rootless podman: containers run as darman, not root. services/containers.nix - # gives us the `docker` CLI shim (dockerCompat), but compose v2 is a separate - # binary and talks to a socket rather than the CLI — the NixOS podman module - # enables the *user* socket (systemd.user.sockets.podman), so point compose at - # it instead of the root /var/run/docker.sock. + # Rootless podman runs containers as darman; compose v2 talks to a socket + # rather than the docker CLI shim, so point it at the user podman socket + # instead of the root one. home.sessionVariables.DOCKER_HOST = "unix:///run/user/1000/podman/podman.sock"; # Only sets WHICH helper sudo uses; it still only calls it when asked with diff --git a/hosts/terra/home/hyprland.nix b/hosts/terra/home/hyprland.nix index 4dc20e3..ffe3ef2 100644 --- a/hosts/terra/home/hyprland.nix +++ b/hosts/terra/home/hyprland.nix @@ -1,32 +1,16 @@ { lib, pkgs, config, inputs, ... }: -# Hyprland config migrated from github.com/darman96/hyprland-dotfiles (the -# hyprlang `hypr/*.conf` files) into the home-manager lua-style `settings` -# (configType defaults to "lua" on stateVersion 26.05). Each top-level -# `settings` attr becomes an `hl.(...)` call in ~/.config/hypr/hyprland.lua; -# `_args` lists become multi-arg calls, `_var` locals become `local x = ...`, and -# `lib.generators.mkLuaInline` values render as raw Lua expressions. +# Hyprland config migrated from github.com/darman96/hyprland-dotfiles into +# home-manager's lua-style `settings` (each attr becomes an `hl.(...)` +# call in hyprland.lua). Imported by home.nix; system-level enable lives in +# ../../services/desktop/desktop-hyprland.nix. # -# Imported by home.nix. System-level Hyprland enable (session entry, portals) -# lives in ../../services/desktop/desktop-hyprland.nix; this manages the user's -# own hyprland.lua. -# -# Deliberately NOT migrated: -# - hyprbars.conf: config for the third-party `hyprbevelbars` plugin, which -# isn't packaged in nixpkgs. Load it via -# `wayland.windowManager.hyprland.plugins` and re-add its config once -# available. (hyprredsquare.conf's plugin was renamed hypr-chrome and -# rewritten since - it's wired in below via the `hypr-chrome` flake -# input instead, with its own `plugin.hyprchrome` config.) -# - hyprqt6engine.conf + `QT_QPA_PLATFORMTHEME=hyprqt6engine`: terra themes Qt -# through qtct/Dracula in home.nix, so that env var is left off to avoid a conflict. -# - hyprlock.conf: a separate program (use `programs.hyprlock` if wanted). -# - the duplicate pamixer/amixer + `.wob` volume binds: kept only the clean -# pipewire `wpctl`/`playerctl` set (no wob overlay is configured here). -# - `XDG_MENU_PREFIX=arch-` and `VCPKG_ROOT`: Arch-/user-specific. -# Many binds reference apps/scripts not packaged on terra yet (vivaldi-stable, -# dolphin, vicinae, grimblast, waypaper, discord, gitkraken, qbz, -# ~/.config/scripts/start-communications.sh); add them separately. +# Not migrated: hyprbars (unpackaged plugin; its successor hypr-chrome is +# wired in below instead), hyprqt6engine (conflicts with home.nix's qtct/ +# Dracula Qt theming), hyprlock (use programs.hyprlock), the old wob volume +# overlay (kept only wpctl/playerctl), and Arch-specific env vars. Several +# binds reference apps not yet packaged here (vivaldi-stable, dolphin, +# vicinae, grimblast, waypaper, discord, gitkraken, qbz). let lua = lib.generators.mkLuaInline; @@ -37,11 +21,9 @@ let cursorName = config.home.pointerCursor.name; cursorSize = toString config.home.pointerCursor.size; - # Wallpaper images aren't checked into this repo (binary blobs) — pulled - # from the existing Wallhaven library on /mnt/hdd_01 instead. Picked once - # here rather than at runtime, since hyprpaper has no built-in "random" - # mode; re-pick and rebuild (or swap in real per-monitor selection) when - # this stops being a placeholder. + # Wallpapers aren't checked into this repo (binaries) — pulled from the + # Wallhaven library on /mnt/hdd_01. Picked once here since hyprpaper has + # no built-in "random" mode. # wallpaper = "/mnt/hdd_01/data/Pictures/Wallhaven/wallhaven-ym81rl.png"; wallpaper = "/mnt/hdd_01/data/Pictures/Wallhaven/wallhaven-mlwz78.png"; @@ -317,11 +299,8 @@ in "hyprland.start" (lua '' function() - -- No polkit agent is started here: quickshell registers one - -- itself (HyprChrome/Widgets/Polkit), and a session admits only - -- one. The hyprpolkitagent line this replaces had been dead for - -- a while anyway — the unit was never installed, so the start - -- failed silently and the session ran with no agent at all. + -- No polkit agent started here: quickshell registers its own + -- (HyprChrome/Widgets/Polkit), and a session admits only one. hl.exec_cmd("cosmic-settings-daemon") hl.exec_cmd("quickshell") hl.exec_cmd("alacritty", { workspace = "special:terminal silent" }) diff --git a/hosts/terra/home/theme.nix b/hosts/terra/home/theme.nix index 2dc2b5b..cac4704 100644 --- a/hosts/terra/home/theme.nix +++ b/hosts/terra/home/theme.nix @@ -23,14 +23,11 @@ in }; }; - # The cursor theme. XCURSOR_THEME alone is not enough for Steam: the client - # UI (steamwebhelper) runs inside a pressure-vessel container that rebuilds - # /etc, so the /etc/profiles/per-user/darman/share/icons entry of - # XCURSOR_PATH does not exist in there and libXcursor finds no theme by - # that name — it falls back to the built-in core X11 cursor. $HOME and - # /nix are bind-mounted into the container, so the ~/.icons symlink that - # `dotIcons` (on by default) drops does resolve. Same class of problem as - # the ~/.themes/~/.icons flatpak workaround above. + # XCURSOR_THEME alone isn't enough for Steam: steamwebhelper runs inside a + # pressure-vessel container with its own /etc, so XCURSOR_PATH doesn't + # resolve there and it falls back to the core X11 cursor. $HOME and /nix are + # bind-mounted in though, so the ~/.icons symlink `dotIcons` drops still + # resolves — same fix as the flatpak workaround below. home.pointerCursor = { name = "Bibata-Modern-Classic"; package = pkgs.bibata-cursors; @@ -39,11 +36,9 @@ in hyprcursor.enable = true; }; - # Flatpak apps are sandboxed and can't see XDG_DATA_DIRS/nix-store theme - # paths, so the portal-reported GTK theme / icon theme names resolve to - # nothing inside the sandbox and they fall back to Adwaita. Flatpak - # auto-exposes ~/.themes and ~/.icons read-only to every sandboxed app - # specifically for this case. + # Flatpak apps can't see XDG_DATA_DIRS/nix-store theme paths, so the + # portal-reported theme names resolve to nothing and fall back to Adwaita; + # Flatpak auto-exposes ~/.themes and ~/.icons read-only as the workaround. home.file.".themes/Dracula".source = "${pkgs.dracula-theme}/share/themes/Dracula"; home.file.".icons/${iconTheme}".source = iconThemeFolder; diff --git a/pkgs/azure-glassy-dark-icons.nix b/pkgs/azure-glassy-dark-icons.nix index ca6cee4..af7bdb4 100644 --- a/pkgs/azure-glassy-dark-icons.nix +++ b/pkgs/azure-glassy-dark-icons.nix @@ -17,13 +17,11 @@ stdenvNoCC.mkDerivation { dontBuild = true; - # Upstream ships a handful of dangling symlinks under mimetypes/16 (e.g. - # libreoffice-spreadsheet.svg -> libreoffice-oasis-spreadsheet.svg, which - # doesn't exist in that size dir) — a minor packaging bug in the theme - # itself. Harmless: GTK's icon lookup just falls through to the theme's - # own Inherits= chain (breeze-dark, breeze, Adwaita, hicolor) for those few - # mimetypes. Nixpkgs' default noBrokenSymlinks fixup check would otherwise - # fail the whole build over it. + # Upstream ships a handful of dangling symlinks under mimetypes/16 (e.g. a + # target that doesn't exist in that size dir) — harmless, GTK's own + # Inherits= chain (breeze-dark, breeze, Adwaita, hicolor) covers the + # fallback. Nixpkgs' default noBrokenSymlinks check would otherwise fail + # the build over it. dontCheckForBrokenSymlinks = true; # gtk3's setup hook strips icon-theme.cache from $out by default diff --git a/pkgs/slot-beauty-dark-icons.nix b/pkgs/slot-beauty-dark-icons.nix index 8c65478..066248f 100644 --- a/pkgs/slot-beauty-dark-icons.nix +++ b/pkgs/slot-beauty-dark-icons.nix @@ -21,15 +21,11 @@ stdenvNoCC.mkDerivation { # the theme's own Inherits= chain (breeze-dark, Adwaita, hicolor) for those. dontCheckForBrokenSymlinks = true; - # index.theme's Directories= lists panel/16@2, panel/22@2, panel/24@2 (with - # Scale=2), but the actual on-disk dirs are named 16@2x/22@2x/24@2x (the - # correct freedesktop-spec suffix) — an upstream index.theme typo. That - # mismatch makes `gtk-update-icon-cache` refuse to emit ANY cache at all - # (exits 1, "The generated cache was invalid"), so unlike the other vendored - # themes here, this one ships with no icon-theme.cache and relies on GTK's - # live directory-scan lookup instead — functionally fine, just not - # cache-accelerated. gtk3's default postFixup hook (dropIconThemeCache) - # would strip a cache anyway, so there's nothing to opt out of. + # index.theme's Directories= names panel/16@2 etc. (Scale=2) but the + # on-disk dirs are 16@2x etc. (the correct suffix) — an upstream typo that + # makes `gtk-update-icon-cache` exit 1, so this theme ships uncached and + # relies on GTK's live directory scan instead (functionally fine, just not + # cache-accelerated). installPhase = '' runHook preInstall mkdir -p "$out/share/icons" diff --git a/pkgs/tome.nix b/pkgs/tome.nix index 7ca758d..666d646 100644 --- a/pkgs/tome.nix +++ b/pkgs/tome.nix @@ -56,13 +56,11 @@ buildDotnetModule (finalAttrs: { executables = [ "Tome.App" ]; - # wrapGAppsHook3: buildDotnetModule sets dontWrapGApps = true by default (to - # avoid double-wrapping) but its own wrap step still splices gappsWrapperArgs - # in when the hook is present (see nixpkgs' libation package, same pattern). - # Without it the binary never gets XDG_DATA_DIRS/GSETTINGS_SCHEMA_DIR set, so - # GTK/WebKitGTK can't find the icon theme or GTK settings from the desktop - # session — symptoms: missing icons and a denser default UI font/size than - # when launched from an already-fully-initialized session (e.g. via Rider). + # buildDotnetModule sets dontWrapGApps = true by default, but wrapGAppsHook3's + # own wrap step still splices gappsWrapperArgs in when present (same pattern + # as nixpkgs' libation). Without it XDG_DATA_DIRS/GSETTINGS_SCHEMA_DIR never + # get set, so GTK/WebKitGTK can't find the icon theme or settings — missing + # icons, denser default UI font. nativeBuildInputs = [ copyDesktopItems wrapGAppsHook3 ]; runtimeDeps = [ diff --git a/scripts/deploy b/scripts/deploy index ef2feac..000d9b4 100755 --- a/scripts/deploy +++ b/scripts/deploy @@ -1,106 +1,48 @@ #!/usr/bin/env bash # Deploy a NixOS host from this flake. ALL arguments are mandatory (no defaults). # -# ./deploy kexec headless kexec into a RAM installer, for a -# read-only-root box (ZimaOS) where -# nixos-anywhere can't ssh-copy-id. Ships our -# SSH login key. Then run `install`. -# is only used to look up the vault item. -# ./deploy kexec-local [--yes] kexec THIS machine into the RAM installer, -# no ssh/second machine involved. Run as root, -# locally, on the box you're installing onto. -# Disks are untouched; console drops for -# ~1-2 min then comes back as the installer. -# Prompts for confirmation (--yes skips it), -# because run on the wrong terminal this -# kexecs your laptop. TMPDIR (default -# /var/tmp) must be exec-capable and hold -# ~3x the tarball. -# Then run `install localhost`. +# ./deploy kexec ZimaOS/RO-root box: kexec into a RAM installer, ships the ssh key, then run `install`. +# ./deploy kexec-local [--yes] kexec THIS machine (no ssh) into the RAM installer; disks untouched. Then `install localhost`. # ./deploy install [--yes] -# first install. Wipes the OS disk. Ships the -# host's sops key. =localhost/127.0.0.1 -# skips nixos-anywhere/ssh and runs disko + -# nixos-install directly against /mnt — but -# ONLY once actually inside a live installer -# (hostname nixos-installer, from kexec, or -# homelab-installer, from installer-iso). -# Run from the REAL running OS instead (e.g. -# a box where kexec-local doesn't work), -# it builds installer-iso, stages its -# kernel/initrd + the host key on the boot -# partition and the iso file on a non-OS-disk -# partition, sets a systemd-boot one-shot -# entry with homelab.install= + -# homelab.keypart= on its kernel -# cmdline, and reboots — a real ACPI reboot, -# not a kexec jump. The booted installer's -# homelab-auto-install.service reads those -# cmdline params, picks the host key back up -# and re-runs this exact command itself once -# its repo checkout (homelab-checkout.service) -# succeeds, finishing the install unattended. -# It confirms before rebooting; --yes skips -# that (it is what the ISO passes itself). +# first install (wipes the OS disk, ships the host's sops key). localhost only +# runs disko/nixos-install directly once already inside a live installer; +# from a real running OS it stages installer-iso and reboots into that instead. # See CLAUDE.md. # ./deploy switch rebuild + activate on a running host. # ./deploy boot stage for next boot, don't activate now. # ./deploy test activate without adding a boot entry. # ./deploy image build an SD-card image (e.g. rpi mercury). -# ./deploy flash build SD image, write to , and (if -# ~/.config/homelab//age.txt exists) -# drop the sops key on its boot partition. +# ./deploy flash build SD image, write to , and drop the sops age key onto it if one exists. # -# = a nixosConfigurations name (e.g. jupiter, vps). Its pre-generated -# SSH host key must be at ~/.config/homelab//ssh_host_ed25519_key. +# = a nixosConfigurations name. Its pre-generated SSH host key must be +# at ~/.config/homelab//ssh_host_ed25519_key. Runs from a non-NixOS host too. # -# Runs from a non-NixOS host too (nixos-rebuild / nixos-anywhere via `nix run`). -# -# Password prompts are auto-filled from the "HomeLab" Proton Pass vault when -# `pass-cli` is installed and logged in; otherwise every command prompts exactly -# as before. Both items are keyed by , never by : the address is -# incidental (DHCP, a new box, localhost) while the config name is the stable -# identity of the machine being built. -# darman@ darman's sudo password (switch/boot/test) -# root@ root's ssh password (kexec/install) +# Password prompts auto-fill from the "HomeLab" Proton Pass vault, keyed by +# not (darman@ for sudo, root@ for ssh). # Override with HOMELAB_PASS_ITEM / HOMELAB_PASS_ROOT_ITEM / HOMELAB_PASS_VAULT. set -euo pipefail shopt -s nullglob -# Captured before anything shifts/parses $@, so require_root() below can -# re-exec the ORIGINAL invocation under sudo — inside a function, "$@"/"$1" -# refer to the function's own args (empty here), not the script's, so this -# has to be a global array instead of relying on positional-parameter scoping. +# Captured before $@ is parsed, so require_root() can re-exec the ORIGINAL +# invocation under sudo (inside a function, "$@" is the function's own args). SCRIPT_ARGS=("$@") # Locate the repo root (flake dir) regardless of where this script lives on disk. -SCRIPT_PATH="$(realpath "$0")" # absolute — "$0" itself may be relative, - # and require_root() re-execs after cd "$REPO" +SCRIPT_PATH="$(realpath "$0")" SCRIPT_DIR="$(dirname "$SCRIPT_PATH")" REPO="$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel 2>/dev/null || dirname "$SCRIPT_DIR")" cd "$REPO" export PATH="/nix/var/nix/profiles/default/bin:$PATH" -# Every `nix` call below assumes `nix-command` + `flakes`. Those are ambient on -# a Determinate-Nix laptop, but a STOCK NixOS box leaves both experimental -# features OFF — so bare `nix eval`/`build`/`run` die with "experimental Nix -# feature 'nix-command' is disabled". That box is exactly the prepare host for -# `install localhost` (a fresh NixOS the reinstall runs from), and it -# is why the installer-iso already sets these itself (flake.nix). Enable them -# additively via NIX_CONFIG (extra-, so anything already configured is kept). -# This runs again at the top of the sudo re-exec in require_root(), so root -# gets it too regardless of whether `sudo -E` carries the env across. +# A stock NixOS box (unlike a Determinate-Nix laptop) leaves nix-command/flakes +# disabled, and that's exactly the prepare host for `install localhost`. +# Enable them additively so root gets them too after the require_root() re-exec. export NIX_CONFIG="$(printf 'extra-experimental-features = nix-command flakes\n%s' "${NIX_CONFIG:-}")" # Off-repo material keyed by : pre-generated SSH host keys (install) -# and per-config sops age keys (flash). -# -# Resolved defensively rather than as a bare $HOME, because this script also -# runs from installer-iso's homelab-auto-install.service, and systemd does not -# set $HOME for a system service without User= (systemd.exec(5): -# SetLoginEnvironment= "defaults to true if User=, DynamicUser= or PAMName= are -# set, false otherwise"). Under `set -u` that aborted the whole unattended run -# with an "unbound variable" that read like a bug in this script. +# and per-config sops age keys (flash). Resolved defensively, not as a bare +# $HOME, since systemd doesn't set $HOME for a service without User= — this +# also runs unattended from installer-iso's homelab-auto-install.service. KEYDIR="${HOMELAB_KEY_DIR:-${HOME:-/root}/.config/homelab}" die() { echo "error: $*" >&2; exit 1; } @@ -108,48 +50,40 @@ die() { echo "error: $*" >&2; exit 1; } need() { command -v "$1" >/dev/null 2>&1 || die "missing required tool: $1"; } # Self-elevate instead of dying: re-exec this exact invocation under sudo. -# -E preserves the environment (HOMELAB_* overrides, Proton Pass vault vars) -# across the re-exec. A no-op once already root. +# -E preserves HOMELAB_*/vault env vars; pin HOMELAB_KEY_DIR too since whether +# sudo carries $HOME across depends on the local sudoers policy. No-op if already root. require_root() { [ "$(id -u)" = 0 ] && return 0 echo ">> $1 needs root — re-executing under sudo" >&2 - # $KEYDIR is derived from $HOME, and whether sudo carries $HOME across - # depends on the local sudoers policy (env_reset/always_set_home). Pin the - # resolved value so the re-exec looks for host keys where the invoking user - # has them, not under /root. export HOMELAB_KEY_DIR="$KEYDIR" exec sudo -E -- "$SCRIPT_PATH" "${SCRIPT_ARGS[@]}" } # Exactly one path matching a glob, or die. `ls glob | head -1` silently yields -# an empty string when nothing matches (head exits 0, so set -e never fires) and -# the failure only surfaces later as a confusing tar/dd error. +# an empty string when nothing matches (head exits 0, so set -e never fires). one_match() { local what="$1"; shift local f=("$@") # caller expands the glob (nullglob is on) [ "${#f[@]}" -gt 0 ] || die "no $what found — did the build actually produce one?" - # Say so instead of silently taking [0]: a stale result-sd/ symlink from an - # earlier config is exactly how you flash the wrong image without a word. + # A stale result-sd/ symlink from an earlier config is how you'd otherwise + # flash the wrong image without a word — warn instead of silently taking [0]. [ "${#f[@]}" -eq 1 ] \ || echo ">> warning: ${#f[@]} candidates for $what, using ${f[0]} (rm the stale ones)" >&2 printf '%s\n' "${f[0]}" } -# Every whole-disk device backing a block device or a mounted path, one per -# line. LVM/RAID/LUKS can sit on several at once (verified on terra: -# /mnt/ssd_01 -> sdd AND sde), so a single lookup is not enough. Empty output -# means "could not determine" — which callers must treat as unsafe, not as OK. +# Every whole-disk device backing a block device or mounted path, one per line. +# LVM/RAID/LUKS can span several disks at once (e.g. terra's /mnt/ssd_01), +# so callers must treat empty output as "unknown", not "safe". disks_backing() { lsblk -rnso NAME,TYPE "$1" 2>/dev/null | awk '$2 == "disk" { print "/dev/" $1 }' } -# Label of the temporary UEFI boot entry arm_efi_bootnext() creates. Also the -# key the ISO uses to delete it again once it has booted (see flake.nix). +# Label of the temporary UEFI boot entry arm_efi_bootnext() creates; also what +# the booted ISO matches to delete it again (flake.nix) — must match EXACTLY. EFI_LABEL="Homelab Installer" # Boot numbers of every UEFI entry with exactly this label, one per line. -# efibootmgr prints `Boot0002* LimineHD(1,GPT,...)/\EFI\...`, so the -# label runs from past the "Boot####* " prefix up to the first TAB. # (Character classes spelled out rather than {4}: mawk predates ERE intervals.) efi_entries_named() { efibootmgr 2>/dev/null | awk -v want="$1" ' @@ -162,17 +96,11 @@ efi_entries_named() { }' } -# Arm a genuine one-shot boot of the staged installer WITHOUT any help from the -# bootloader: create a UEFI boot entry that EFI-stub-boots the kernel straight -# off the ESP, and point BootNext at it. -# -# Needed because "boot this once, then go back to normal" is not something -# every bootloader can do. systemd-boot has it; terra's CachyOS runs Limine, -# which reports `One-shot entry control: ✗` and has no equivalent, and whose -# limine.conf is regenerated by pacman hooks anyway. BootNext is a firmware -# feature, so it works underneath all of them — and the firmware clears it -# after that one boot, which is what keeps the "a failed attempt still comes -# back on the normal bootloader" property that makes this safe to try. +# Arm a genuine one-shot boot of the staged installer without bootloader help: +# create a UEFI entry that EFI-stub-boots the kernel off the ESP and point +# BootNext at it. Needed because Limine (terra's CachyOS) has no one-shot +# entry support; BootNext is a firmware feature so it works underneath any +# bootloader, and the firmware clears it after one boot either way. arm_efi_bootnext() { local esp="$1" cmdline="$2" local esp_src esp_disk esp_part num n @@ -184,20 +112,17 @@ arm_efi_bootnext() { { [ -n "$esp_disk" ] && [ -n "$esp_part" ]; } \ || die "couldn't work out the disk + partition number of the ESP ($esp -> $esp_src)" - # Clear anything left by an earlier attempt first, so repeated runs don't - # slowly fill NVRAM with dead entries pointing at a wiped partition. + # Clear anything left by an earlier attempt so NVRAM doesn't slowly fill + # with dead entries pointing at a wiped partition. for n in $(efi_entries_named "$EFI_LABEL"); do echo ">> removing stale UEFI entry Boot$n ($EFI_LABEL)" efibootmgr -q -B -b "$n" done - # --create-only, NOT --create: the latter also pushes the entry to the front - # of BootOrder, which would make a wiped installer the permanent default if - # anything went wrong. This way the entry is reachable through BootNext and - # nothing else, i.e. exactly once. - # - # The EFI stub loads `initrd=` off the volume it was itself loaded from, so - # the path is relative to the ESP root and uses backslashes. + # --create-only, NOT --create: --create also pushes the entry to the front of + # BootOrder, which would make a wiped installer the permanent default on any + # failure. This way it's reachable only via BootNext, exactly once. The EFI + # stub loads `initrd=` relative to the ESP root, hence the backslash path. efibootmgr -q --create-only --disk "$esp_disk" --part "$esp_part" \ --label "$EFI_LABEL" \ --loader '\homelab-installer\bzImage' \ @@ -210,11 +135,8 @@ arm_efi_bootnext() { } # Sets tb / cpio / bbox — the kexec tarball plus the static cpio+gzip that -# kexec-run.sh needs on PATH to rebuild its initrd. -# -# HOMELAB_KEXEC_TARBALL (with _CPIO / _GZIP) short-circuits the build and uses a -# prebuilt installer instead. That lets the VM test in flake.nix drive this -# script offline, and lets you re-kexec a box without rebuilding ~500MB. +# kexec-run.sh needs on PATH to rebuild its initrd. HOMELAB_KEXEC_TARBALL (+ +# _CPIO/_GZIP) short-circuits the build to reuse a prebuilt installer instead. kexec_artifacts() { if [ -n "${HOMELAB_KEXEC_TARBALL:-}" ]; then tb="$HOMELAB_KEXEC_TARBALL" @@ -235,11 +157,10 @@ kexec_artifacts() { fi } -# True inside one of the throwaway live-installer environments this repo -# produces (kexec's nixos-installer, or installer-iso's homelab-installer) — -# i.e. `install localhost` should wipe/install right here. False on -# any real running OS, where the same command instead means "prepare and -# reboot into an installer for THIS box" (see local_install_prepare_and_reboot). +# True inside one of this repo's throwaway live-installer environments +# (nixos-installer from kexec, or homelab-installer from installer-iso) — +# i.e. `install localhost` should wipe/install right here, not +# prepare-and-reboot (see local_install_prepare_and_reboot). is_live_installer() { case "$(uname -n)" in nixos-installer | homelab-installer) return 0 ;; @@ -247,17 +168,12 @@ is_live_installer() { esac } -# `install localhost` run on a REAL running OS (not already inside a -# live installer): builds installer-iso, stages its kernel/initrd + the host's -# pre-generated ssh key on the boot partition and the iso file on a non-OS -# disk, points a systemd-boot one-shot entry at them with -# homelab.install= + homelab.keypart= on the kernel cmdline, -# and reboots — a real ACPI reboot through firmware POST, deliberately NOT a -# kexec jump (see terra's kexec-local gotcha in CLAUDE.md). The booted -# installer's homelab-auto-install.service reads those params, picks the host -# key back up and re-runs this exact `install localhost` command -# itself (now genuinely inside the installer) once homelab-checkout.service has -# fetched the repo, finishing the job unattended. +# `install localhost` on a REAL running OS (not yet inside a live +# installer): stages installer-iso's kernel/initrd + host key on the boot +# partition, arms a one-shot boot with homelab.install= on its +# cmdline, and does a real ACPI reboot — deliberately not a kexec jump, per +# terra's kexec-local gotcha in CLAUDE.md. The booted installer re-runs this +# same command itself once its repo checkout succeeds, finishing unattended. local_install_prepare_and_reboot() { local config="$1" hostkey="$2" assume_yes="$3" require_root "preparing a local reinstall" @@ -271,17 +187,12 @@ local_install_prepare_and_reboot() { need stat need df - # Where to stage the installer, and how to make the box boot it exactly once. - # - # systemd-boot keeps its entries on $BOOT — the XBOOTLDR partition when there - # is one, the ESP otherwise — which is not always /boot. Hardcoding /boot on - # a box that mounts its ESP elsewhere just creates a directory on the root - # filesystem and then reboots into an entry the firmware never sees. - # - # No systemd-boot (terra's CachyOS runs Limine) means no `bootctl set-oneshot`, - # so fall back to the firmware's own BootNext — see arm_efi_bootnext(). That - # path EFI-stub-boots the kernel directly, which requires it to sit on the ESP - # itself rather than on a separate XBOOTLDR. + # Where to stage the installer: use bootctl's reported $BOOT (XBOOTLDR or the + # ESP), not a hardcoded /boot, since that's not always where the ESP mounts. + # No systemd-boot (terra's CachyOS runs Limine) means no `bootctl + # set-oneshot`, so fall back to firmware BootNext (arm_efi_bootnext()) — + # which EFI-stub-boots the kernel directly and needs it on the ESP itself, + # not a separate XBOOTLDR. local boot boot_mode esp esp="$(bootctl --print-esp-path 2>/dev/null)" \ || die "bootctl couldn't locate the ESP — is this box actually UEFI-booted?" @@ -296,10 +207,9 @@ local_install_prepare_and_reboot() { echo " own BootNext instead (bootloader in charge here: $(bootctl status 2>/dev/null | awk '/Product:/ {$1=""; print substr($0,2); exit}' || echo unknown))" fi - # No default/auto-picked location — the wrong disk here is destroyed - # mid-install (see the OS-disk check below), so this always asks rather - # than guessing. HOMELAB_INSTALLER_STAGE_DIR skips the prompt for scripted - # use, but is otherwise just as explicit a choice as typing it in. + # No default/auto-picked location: the wrong disk here is destroyed + # mid-install (see the OS-disk check below), so this always asks unless + # HOMELAB_INSTALLER_STAGE_DIR is set for scripted use. local stagedir="${HOMELAB_INSTALLER_STAGE_DIR:-}" if [ -z "$stagedir" ]; then echo ">> currently mounted filesystems:" @@ -323,17 +233,13 @@ local_install_prepare_and_reboot() { || die "couldn't read the OS disk device from hosts/$config/disk-config.nix" osdisk_real="$(readlink -f "$osdisk")" - # --nofsroot matters: on btrfs, findmnt prints the subvolume as - # `/dev/sdb2[/@]`, which is not a path lsblk can open. Without it the lookup - # came back empty and the guard below was skipped entirely — i.e. it silently - # allowed staging on the very disk about to be wiped. terra's current - # CachyOS root is exactly that layout. + # --nofsroot matters: on btrfs findmnt prints the subvolume as + # `/dev/sdb2[/@]`, which lsblk can't open, silently skipping the guard + # below and allowing staging on the disk about to be wiped (terra's layout). stage_src="$(findmnt -no SOURCE --nofsroot --target "$stagedir")" \ || die "$stagedir doesn't resolve to a mounted filesystem" - # `|| true` so the explicit check below is what reports the problem: lsblk - # exits nonzero on a device it can't parse, and under `set -e` + pipefail a - # bare assignment from a failing substitution kills the script silently, - # right past the fail-closed message. + # `|| true` so the fail-closed check below reports the problem, rather than + # `set -e`/pipefail silently killing the script on lsblk's nonzero exit. stage_disks="$(disks_backing "$stage_src" || true)" # Fail closed. "Couldn't determine the disk" is not "different disk". [ -n "$stage_disks" ] \ @@ -344,30 +250,23 @@ local_install_prepare_and_reboot() { fi done - # stage-1 resolves findiso= by mounting each blkid-visible partition and - # testing `-e /findiso$isoPath` (nixos/modules/system/boot/stage-1-init.sh). - # For btrfs it mounts the volume's TOP level, so a path that lives inside a - # subvolume (/@/...) is simply not there and the box boots to an emergency - # shell — after it has already rebooted out of the working OS. + # stage-1 mounts a btrfs volume's TOP level to resolve findiso=, so a path + # inside a subvolume is unreachable and the box boots to an emergency shell + # after it's already left the working OS. Refuse btrfs staging outright. stage_fstype="$(findmnt -no FSTYPE --target "$stagedir")" [ "$stage_fstype" != btrfs ] \ || die "$stagedir is btrfs: findiso= mounts the volume's top level, so a path inside a subvolume never resolves. Stage on a non-btrfs partition (ext4/vfat/ntfs)." - # PARTUUID of the staging partition. Handed to the installer as - # homelab.logpart= so it can mount this partition rw and persist its whole - # run — disko + nixos-install output included — to a file next to the iso. - # This partition is on a DIFFERENT disk from the one disko wipes (guarded - # above), so unlike $boot it SURVIVES the install: a failed attempt otherwise - # leaves nothing to debug, its journal having died on tmpfs at the reboot. - # Best-effort — an LVM/mdraid stage_src has no PARTUUID, in which case logging + # PARTUUID of the staging partition, handed to the installer as + # homelab.logpart= so it can persist the whole install's log there — it's on + # a different disk than the one disko wipes, so it survives a failed + # install. Best-effort: an LVM/mdraid stage_src has no PARTUUID, so logging # is simply skipped rather than blocking the install. local stage_partuuid stage_partuuid="$(lsblk -no PARTUUID "$stage_src" 2>/dev/null | head -1 | tr -d ' ' || true)" - # Last chance to back out. This is the most destructive command in the - # script — it reboots the machine you are typing at and the wipe that - # follows is unattended — so it confirms just like `flash` and `kexec-local` - # do, both of which are less final than this. + # Last chance to back out: this reboots the machine you're typing at into an + # unattended wipe, so it confirms like `flash`/`kexec-local` do. if [ "$assume_yes" != "--yes" ]; then echo ">> about to REINSTALL this machine from scratch:" echo " hostname: $(uname -n)" @@ -392,16 +291,13 @@ local_install_prepare_and_reboot() { initrd="$(nix build --no-link --print-out-paths .#nixosConfigurations.installer-iso.config.system.build.initialRamdisk)/initrd" isodir="$(nix build --no-link --print-out-paths .#nixosConfigurations.installer-iso.config.system.build.isoImage)" iso="$(one_match 'installer iso' "$isodir"/iso/*.iso)" - # The live ISO's root is a tmpfs; stage 1 finds the real system's init via - # init=/init, which the grub/isolinux menu supplies on a normal - # boot (iso-image.nix). EFI-stub-booting our own cmdline, we must pass it too - # — omit it and stage 1 loop-mounts the iso fine, then dies on + # The grub/isolinux menu normally supplies init=/init; EFI-stub + # booting our own cmdline means we must pass it too, or stage 1 dies on # "stage 2 init script (/mnt-root//init) not found". toplevel="$(nix build --no-link --print-out-paths .#nixosConfigurations.installer-iso.config.system.build.toplevel)" - # A short write is not visible until the reboot, when findiso finds a - # truncated iso and drops to an emergency shell. Check first — `install` - # prints no progress and the iso is ~1GB. + # Check space before writing: a short write isn't visible until reboot, + # when findiso finds a truncated ~1GB iso and drops to an emergency shell. local need_stage need_boot avail_stage avail_boot need_stage="$(stat -Lc %s "$iso")" need_boot="$(( $(stat -Lc %s "$kernel") + $(stat -Lc %s "$initrd") + $(stat -Lc %s "$hostkey") ))" @@ -417,15 +313,9 @@ local_install_prepare_and_reboot() { install -Dm644 "$initrd" "$boot/homelab-installer/initrd" install -Dm644 "$iso" "$stagedir/homelab-installer.iso" - # The ISO is built from a PUBLIC repo and deliberately carries no - # credentials, so the host key has to travel with the staged installer or - # the auto-install run has nothing to seed /etc/ssh with — and without that, - # sops can't decrypt on boot #1, /etc/shadow gets written once with a locked - # darman, and no later `deploy switch` can fix it (README). - # - # $boot lives on the OS disk, so disko destroys this copy minutes later. The - # mode is advisory on vfat (permissions come from the mount's fmask, 0077 on - # a NixOS/systemd-boot ESP) — it is the wipe, not the mode, doing the work. + # The ISO is built from a public repo with no credentials, so the host key + # must travel with the staged installer or sops can't decrypt on boot #1 + # (README). $boot is on the OS disk, so disko destroys this copy minutes later. install -Dm600 "$hostkey" "$boot/homelab-installer/ssh_host_ed25519_key" install -Dm644 "$hostkey.pub" "$boot/homelab-installer/ssh_host_ed25519_key.pub" boot_src="$(findmnt -no SOURCE --nofsroot --target "$boot")" \ @@ -434,22 +324,15 @@ local_install_prepare_and_reboot() { [ -n "$boot_partuuid" ] \ || die "couldn't read a PARTUUID for $boot ($boot_src) — the installer needs it to find the host key" - # findiso= is a path relative to whatever partition the initrd finds it on - # (it mounts every blkid-visible partition looking for it), not to `/`, if - # $stagedir is a subdirectory of a bigger filesystem rather than a mountpoint - # itself. It must KEEP its leading slash: stage-1 tests `-e /findiso$isoPath`, - # so a bare `var/tmp/x.iso` becomes `/findisovar/tmp/x.iso` and never matches. - # Prefixing then squeezing handles both ends: stagedir == the mountpoint - # (strip leaves "") and mnt_point == "/" (strip leaves a relative path). + # findiso= is relative to whichever partition the initrd finds it on, and + # must KEEP its leading slash: stage-1 tests `-e /findiso$isoPath`, so a bare + # `var/tmp/x.iso` becomes `/findisovar/tmp/x.iso` and never matches. mnt_point="$(findmnt -no TARGET --target "$stagedir")" iso_relpath="$(printf '/%s/%s' "${stagedir#"$mnt_point"}" homelab-installer.iso | tr -s /)" - # Identical either way — only the mechanism that gets the kernel booted with - # it differs. - # root=LABEL= matches what the ISO menu passes; findiso overwrites - # /dev/root with the loop-mounted iso regardless, but keep it honest. - # boot.shell_on_fail gives a shell instead of the reboot/ignore prompt if - # stage 1 ever fails again. init= is the one that actually made this work. + # root=LABEL= matches what the ISO menu passes (findiso overwrites + # /dev/root regardless); boot.shell_on_fail gives a shell instead of a + # reboot/ignore prompt if stage 1 fails again. local cmdline volumeID volumeID="$(nix eval --raw .#nixosConfigurations.installer-iso.config.isoImage.volumeID)" cmdline="init=$toplevel/init nohibernate root=LABEL=$volumeID boot.shell_on_fail loglevel=4 lsm=landlock,yama,bpf findiso=$iso_relpath homelab.install=$config homelab.keypart=$boot_partuuid" @@ -483,31 +366,24 @@ EOF require_tracked() { local config="$1" cfgfile="hosts/$1/configuration.nix" f [ -e "$cfgfile" ] || die "no $cfgfile in the repo" - # No .git at all (e.g. a tarball export of the repo, no working tree), or no - # git binary, means there's nothing that CAN be untracked — nothing to check. - # Only skip on that, not on any other git failure. + # No .git or no working tree (e.g. a tarball export) means nothing CAN be + # untracked — skip only on that, not on any other git failure. command -v git >/dev/null 2>&1 || return 0 git -C "$REPO" rev-parse --is-inside-work-tree >/dev/null 2>&1 || return 0 - # Every .nix in hosts//, not just configuration.nix: an untracked - # disk-config.nix is exactly as invisible to the flake, and it is the file - # that decides which disk gets wiped. + # Every .nix in hosts//, not just configuration.nix — an untracked + # disk-config.nix decides which disk gets wiped and is just as invisible. for f in "hosts/$config"/*.nix; do git -C "$REPO" ls-files --error-unmatch "$f" >/dev/null 2>&1 \ || die "$f is untracked — 'git add hosts/$config' first (flakes ignore untracked files)" done } -# The password field of a Proton Pass item ("--field password" prints the bare -# value, one line), or empty if pass-cli is missing / logged out / has no such -# item — every caller then falls back to the normal interactive prompt. -# -# Resolve the title to an item id among ACTIVE items first, because `item view -# --item-title` has no state filter: Proton Pass keeps deleted items in the -# trash, and if a trashed item shares the title, view can match THAT one and -# return an empty password with exit 0. Empty is indistinguishable from "no such -# item", so the only symptom is a silent fall back to the interactive prompt -# even though the vault clearly holds the entry. (Hit for real on darman@neptun, -# which had an Active and a Trashed copy.) +# The password field of a Proton Pass item, or empty if pass-cli is missing / +# logged out / has no such item — callers then fall back to an interactive +# prompt. Resolves the title among ACTIVE items first, because `item view +# --item-title` has no state filter and can silently match a trashed item of +# the same title instead, returning an empty password with exit 0 (hit for +# real on darman@neptun, which had both an Active and a Trashed copy). proton_pass_password() { local title="$1" vault="${HOMELAB_PASS_VAULT:-HomeLab}" id pw command -v pass-cli >/dev/null 2>&1 || return 0 @@ -564,15 +440,10 @@ case "$cmd" in o=(-o ControlMaster=auto -o "ControlPath=$cm" -o ControlPersist=300 \ -o StrictHostKeyChecking=accept-new) - # Root's password from Proton Pass, fed to ssh/scp via sshpass -e. Only the - # first (master) connection authenticates; the rest ride the control socket. - # - # SSHPASS is exported here rather than passed as `env SSHPASS=... sshpass`. - # Both end up equally safe at rest: `env` execs its target immediately, so - # the assignment is only in argv for the sub-millisecond before exec, after - # which /proc/PID/cmdline reads plain `sshpass -e`. Exporting just closes - # that race window and drops a process. Either way the secret lives in the - # child's environ, which is readable by the owner and root only. + # Root's password from Proton Pass, fed to ssh/scp via sshpass -e; only the + # first (master) connection authenticates, the rest ride the control socket. + # Exported rather than `env SSHPASS=... sshpass` to close the sub-millisecond + # argv-exposure race before exec (either way the secret only lives in environ). sp=() root_item="${HOMELAB_PASS_ROOT_ITEM:-root@$config}" root_pw="$(proton_pass_password "$root_item" || true)" @@ -613,24 +484,18 @@ case "$cmd" in ssh "${o[@]}" -O exit "root@$host" 2>/dev/null || true # close control socket unset SSHPASS - # NB: no ssh-keygen -R here on purpose. kexec-run.sh copies /etc/ssh/ssh_host_* - # into the appended initrd and restore-remote-access.nix installs them back - # into the installer's /etc/ssh, so the host key SURVIVES the jump. Clearing - # known_hosts would just throw away the TOFU record for no reason. + # NB: no ssh-keygen -R here on purpose — the kexec installer keeps the box's + # ssh host key (restore-remote-access.nix), so known_hosts is still valid. echo ">> box is kexec-ing. Wait ~1-2 min for the installer + network, then:" echo " ./deploy install $config $host" ;; kexec-local) - # No ssh, no second machine: build the same RAM installer as `kexec`, but - # run it directly on this box (you're sitting at it). The current shell - # drops when the kernel switches, same as any reboot — that's expected, - # not a failure. Disks are untouched; only the running kernel changes. - # - # This is a one-way trip on the machine you are typing at, so every check - # that can fail is done BEFORE the point of no return, and nothing that the - # jump depends on is cleaned up behind it (see the trap discussion below). + # Build the same RAM installer as `kexec`, but run it directly on this box + # (no ssh/second machine). One-way trip on the machine you're typing at, so + # every check that can fail runs BEFORE the point of no return (see the + # trap discussion below). require_root "kexec-local" assume_yes="" @@ -719,11 +584,9 @@ case "$cmd" in [ "$(cat /sys/kernel/kexec_loaded 2>/dev/null || echo 0)" = 1 ] \ || { rm -rf "$stage"; die "kexec reported success but no image is loaded — aborting"; } - # THE trap MUST GO NOW. kexec-run.sh backgrounds `nohup sh -c "sleep 6 && - # $SCRIPT_DIR/kexec -e"` and returns immediately, so the binary that - # performs the jump still has to exist ~6s after this script would normally - # exit. Letting the EXIT trap rm -rf "$stage" deletes it out from under that - # sleeping shell and the machine silently never jumps. + # THE trap MUST GO NOW: kexec-run.sh backgrounds the actual jump ~6s in the + # future, so an EXIT trap rm -rf'ing $stage here would delete the binary + # that performs it and the machine would silently never jump. trap - EXIT sync @@ -737,9 +600,8 @@ case "$cmd" in install) config="${2:-}"; host="${3:-}"; assume_yes="${4:-}" { [ -n "$config" ] && [ -n "$host" ]; } || die "usage: ./deploy install [--yes]" - # $KEYDIR, not a bare $HOME — see its definition. This same check runs - # inside installer-iso, where homelab-auto-install.service has no $HOME and - # has just dropped the key into /root/.config/homelab//. + # $KEYDIR, not a bare $HOME — see its definition (also runs inside + # installer-iso, which has no $HOME). hostkey="$KEYDIR/$config/ssh_host_ed25519_key" [ -f "$hostkey" ] || die "missing host key: $hostkey" [ -d "./hosts/$config" ] || die "no ./hosts/$config directory in the repo" @@ -761,9 +623,8 @@ case "$cmd" in [ -f "./hosts/$config/disk-config.nix" ] || die "no ./hosts/$config/disk-config.nix" echo ">> disko .#$config onto this box's OS disk (WILL be wiped)" - # `.#disko`, not github:nix-community/disko — the revision comes from this - # repo's flake.lock rather than upstream master-of-the-day, and resolves - # from the local store. See the nixos-anywhere input in flake.nix. + # `.#disko`, not github:nix-community/disko: pins to this repo's + # flake.lock revision instead of upstream master-of-the-day. nix run ".#disko" -- \ --mode disko "./hosts/$config/disk-config.nix" @@ -787,9 +648,7 @@ case "$cmd" in --target-host "root@$host") # nixos-anywhere's --env-password reads root's ssh password from $SSHPASS - # (it ships its own sshpass), so a vault hit skips the ssh-copy-id prompt. - # Exported rather than `env SSHPASS=...` for consistency with `kexec`; - # see the note there — it's a marginal win, not a leak fix. + # (its own bundled sshpass), so a vault hit skips the ssh-copy-id prompt. root_item="${HOMELAB_PASS_ROOT_ITEM:-root@$config}" root_pw="$(proton_pass_password "$root_item" || true)" if [ -n "$root_pw" ]; then @@ -812,9 +671,7 @@ case "$cmd" in echo ">> nixos-rebuild $cmd .#$config on darman@$host" # --ask-sudo-password, not the deprecated --use-remote-sudo: common.nix sets - # security.sudo.wheelNeedsPassword = true, and --use-remote-sudo only - # prefixes with sudo without ever prompting. Asks for darman's password - # (the darman_password hash in each host's sops file). + # wheelNeedsPassword = true, and --use-remote-sudo never actually prompts. rebuild=(nix run nixpkgs#nixos-rebuild -- "$cmd" --flake ".#$config" --target-host "darman@$host" @@ -823,30 +680,19 @@ case "$cmd" in item="${HOMELAB_PASS_ITEM:-darman@$config}" pw="$(proton_pass_password "$item" || true)" if [ -n "$pw" ] && command -v setsid >/dev/null 2>&1; then - # nixos-rebuild prompts with getpass(), which reads /dev/tty and ignores a - # piped stdin. setsid drops the controlling terminal, so getpass falls back - # to stdin and takes the vault password (it warns about echo — harmless, - # nothing is echoed since the password never reaches the terminal). - # - # Caveat of dropping the tty: EVERY prompt in the subtree now reads this - # stdin, not just the sudo one. Feed the line a few times so a retry or a - # second sudo ask doesn't hit EOF and hang. Anything else that prompts - # (an ssh key passphrase, a host-key confirmation) will still fail — fix - # those out of band rather than by feeding more lines here. + # nixos-rebuild's getpass() reads /dev/tty and ignores piped stdin; setsid + # drops the controlling terminal so it falls back to stdin instead. Every + # prompt in the subtree now reads that stdin, so the password line is fed + # a few times to survive a retry — anything else that prompts still fails. echo ">> sudo password from Proton Pass ($item)" printf '%s\n%s\n%s\n' "$pw" "$pw" "$pw" | setsid -w "${rebuild[@]}" else "${rebuild[@]}" fi - # jupiter's 29G eMMC has no room to just let generations pile up between - # gc.dates=weekly runs (common.nix) — that's exactly how it filled up - # once already. configurationLimit=5 (also common.nix) makes - # switch-to-configuration prune generations beyond 5 as part of the - # switch above, but pruning a generation only drops it as a GC root — - # the store paths themselves still need an actual collect to free the - # disk. So do that here, right after every switch, rather than waiting - # up to a week for it to matter again. + # jupiter's 29G eMMC has already filled up once waiting for the weekly gc + # (common.nix). configurationLimit=5 only drops old generations as GC + # roots, so collect explicitly here rather than waiting up to a week. if [ "$cmd" = switch ] && [ "$config" = jupiter ]; then echo ">> jupiter: collecting garbage post-switch (keeps the eMMC under the 5-generation cap)" need ssh @@ -889,9 +735,8 @@ case "$cmd" in sync # If this config has a dedicated sops age key, drop it on the ROOT ext4 - # partition at /var/lib/sops-nix/age.txt so sops decrypts on first boot. - # (The Pi's vfat partition isn't mounted at runtime, so the key can't live - # there.) Key stays off-repo, out of the nix store, and out of the image. + # partition (the Pi's vfat one isn't mounted at runtime) so sops decrypts + # on first boot. Key stays off-repo, out of the nix store and the image. keyfile="$KEYDIR/$config/age.txt" if [ -f "$keyfile" ]; then echo ">> installing sops age key onto the root partition" diff --git a/scripts/edit_secrets b/scripts/edit_secrets index d7310a3..499b45a 100755 --- a/scripts/edit_secrets +++ b/scripts/edit_secrets @@ -36,10 +36,9 @@ if [ "$show" -eq 1 ]; then exec nix shell nixpkgs#sops -c sops --decrypt "$file" fi -# sops opens $EDITOR on a temp file and re-encrypts only if it changed. -# Pitfalls that cause "File has not changed, exiting": -# - $EDITOR unset: no editor is on the `nix shell` PATH -> bundle one. -# - GUI editor (code/zed) forks and returns instantly -> force --wait. +# sops re-encrypts only if the $EDITOR session actually changed the temp file. +# GUI editors (code/zed) return instantly unless forced to --wait, and if +# $EDITOR is unset no editor exists on the `nix shell` PATH, so bundle one. editor="${VISUAL:-${EDITOR:-}}" extra=() case "$editor" in diff --git a/scripts/immich-import-legacy-db b/scripts/immich-import-legacy-db index 24b5bc2..69a8571 100755 --- a/scripts/immich-import-legacy-db +++ b/scripts/immich-import-legacy-db @@ -1,50 +1,28 @@ #!/usr/bin/env bash # Import the OLD ZimaOS/CasaOS Immich database into the NixOS-managed one. -# Run this ON jupiter, as root, ONCE, AFTER the first `./deploy switch jupiter` -# that ships services/media/immich.nix (the empty `immich` DB must exist). -# -# The media files are moved separately — do that FIRST, it is a rename on the -# same filesystem, so instant even at 9.1G. Move the CONTENTS, not the dir: -# systemd.tmpfiles already created /mnt/data/AppData/immich on the first -# deploy, so `mv ` would nest it as .../immich/upload/ and every -# thumbnail lookup would ENOENT. +# Run ONCE on jupiter, as root, after the first `./deploy switch jupiter` that +# ships services/media/immich.nix (the empty `immich` DB must already exist). # +# Move the media files separately FIRST (a same-filesystem rename, instant +# even at 9.1G) — move the CONTENTS of /mnt/data/Immich/upload into +# /mnt/data/AppData/immich, not the directory itself, or it nests under +# .../immich/upload and every thumbnail lookup ENOENTs: # systemctl stop immich-server immich-machine-learning # mv /mnt/data/Immich/upload/* /mnt/data/AppData/immich/ -# chown -R immich:immich /mnt/data/AppData/immich -# chmod 700 /mnt/data/AppData/immich +# chown -R immich:immich /mnt/data/AppData/immich && chmod 700 /mnt/data/AppData/immich # -# Expected afterwards: library/ upload/ thumbs/ encoded-video/ profile/ backups/ -# -# The legacy cluster turned out to be Postgres 14 running VectorChord 0.3.0 + -# pgvector 0.8.1 (NOT pgvecto.rs), the same extensions nixpkgs ships — so this -# is a plain version-upgrade dump/restore and the smart-search and face -# embeddings come across intact. No re-running the ML jobs over the library. -# Upstream's accepted VectorChord range is >= 0.3, < 2.0, so 0.3.0 -> 1.1.1 is -# a supported jump; the REINDEX at the end is what upstream asks for after a -# version change. -# -# What this script does: -# 1. cp -a the legacy PGDATA to a scratch dir (the original is never touched, -# never even mounted rw — postgres would replay WAL into it). -# 2. Boots that copy under immich's own PG14 image, pinned to the SAME -# VectorChord version nixpkgs has (1.1.1), and runs `ALTER EXTENSION -# vchord UPDATE` so the catalog matches the loaded library. -# 3. Dumps it with the LOCAL pg_dump (17.x) over TCP, not the container's -# pg_dump (14.x) — dumping with the newer tool is the supported direction. -# 4. Restores into a scratch DB, hands ownership to the immich role, shows -# you the row counts, and only swaps it into place after you confirm. -# -# Afterwards Immich runs its own schema migrations up to 2.7.5 on first start. +# The legacy cluster is Postgres 14 + VectorChord 0.3.0 + pgvector 0.8.1 (the +# same extensions nixpkgs ships), so this is a plain version-upgrade +# dump/restore — smart-search and face embeddings come across intact with no +# ML rerun needed. set -euo pipefail LEGACY="${LEGACY:-/mnt/data/Immich/pg-data}" WORK="${WORK:-/var/tmp/immich-import}" -# Pinned to EXACTLY what the legacy cluster records in pg_extension — -# vchord 0.3.0 + pgvector 0.8.1 — so the old server reads its own indexes -# without any in-place extension upgrade. The target side is vchord 1.1.1 / -# pgvector 0.8.2, which is fine: a dump/restore rebuilds every index from -# scratch, so only the index DEFINITION has to still be valid there. +# Pinned to exactly what the legacy cluster's pg_extension records (vchord +# 0.3.0/pgvector 0.8.1) so it reads its own indexes unmodified; the dump/ +# restore rebuilds indexes from scratch on the target's newer versions, so +# only the index definitions need to stay valid. IMAGE="${IMAGE:-ghcr.io/immich-app/postgres:14-vectorchord0.3.0-pgvector0.8.1}" CTR=immich-legacy-pg PORT="${PORT:-15432}" @@ -71,13 +49,10 @@ cp -a "$LEGACY" "$WORK/pgdata" # A crashed cluster leaves this behind; it makes the container refuse to start. rm -f "$WORK/pgdata/postmaster.pid" -# The dump runs over TCP (local pg_dump 17 -> published port), and this -# cluster's own pg_hba wants a password for host connections — the marketplace -# app's POSTGRES_PASSWORD is long gone, and POSTGRES_HOST_AUTH_METHOD only -# applies when the image INITIALISES a cluster, not to an existing one. This is -# a scratch copy bound to 127.0.0.1 for the length of one dump, so trust it. -# REPLACE the file rather than appending: pg_hba is first-match-wins, and the -# image's existing scram-sha-256 line would shadow anything added below it. +# The marketplace app's original POSTGRES_PASSWORD is long gone, and +# POSTGRES_HOST_AUTH_METHOD only applies when the image initializes a cluster +# (not an existing one) — so pg_hba is REPLACED outright (not appended, since +# it's first-match-wins) to trust this scratch copy while it's dumped. cat > "$WORK/pgdata/pg_hba.conf" <<'EOF' local all all trust host all all 0.0.0.0/0 trust @@ -101,10 +76,10 @@ for _ in $(seq 1 60); do done [ "${ready:-}" = 1 ] || { podman logs --tail 30 "$CTR"; die "legacy postgres never became ready"; } -# The compose stack's POSTGRES_USER is not recorded anywhere on disk and is NOT -# necessarily "postgres" — the ZimaOS/CasaOS marketplace app used "casaos". -# pg_isready reports "accepting connections" even for a role that doesn't -# exist, so probe for one that can actually log in. +# The original POSTGRES_USER isn't recorded on disk and wasn't necessarily +# "postgres" (this marketplace app used "casaos"), and pg_isready reports +# ready even for a role that doesn't exist — so probe for one that can +# actually log in. if [ -z "$LEGACY_USER" ] || ! podman exec "$CTR" psql -U "$LEGACY_USER" -lqt >/dev/null 2>&1; then for candidate in casaos immich postgres; do if podman exec "$CTR" psql -U "$candidate" -lqt >/dev/null 2>&1; then @@ -160,13 +135,11 @@ echo ">> errors logged: $(grep -c '^ERROR' "$WORK/restore.log" || true) (see $W grep '^ERROR' "$WORK/restore.log" | sort -u | head -10 | sed 's/^/ /' || true step "handing ownership to the immich role" -# --no-owner made everything owned by the restoring role (postgres); immich -# connects as "immich" and its startup migrations run ALTER TABLE, so it must -# own its own schema. NOT `REASSIGN OWNED BY postgres` — that also sweeps up -# system objects and fails with "cannot reassign ownership of objects owned by -# role postgres because they are required by the database system". Extension- -# owned routines/types are excluded for the same reason; immich never alters -# those, and they correctly stay with postgres. +# immich's own ALTER TABLE migrations need it to own its schema, but plain +# `REASSIGN OWNED BY postgres` also sweeps up system objects and fails on ones +# the database system requires — so ownership is walked table-by-table +# instead, skipping extension-owned routines/types, which correctly stay with +# postgres. sudo -u postgres psql -qd "$STAGING_DB" <<'SQL' ALTER SCHEMA public OWNER TO immich; DO $$ diff --git a/services/desktop/librechat.nix b/services/desktop/librechat.nix index ca683de..df716c8 100644 --- a/services/desktop/librechat.nix +++ b/services/desktop/librechat.nix @@ -9,11 +9,10 @@ enable = true; enableLocalDB = true; # spins up a local, unauthenticated-on-localhost mongodb - # LibreChat's isEnabled() treats an UNSET var as false, not true — so - # registration is closed unless this is explicit, despite .env.example - # suggesting true is the default. Only reachable over the tailnet - # (trusted interface, see module comment below), so leaving it open is - # fine; flip to false once your account exists if you want it locked down. + # LibreChat's isEnabled() treats an unset var as false, not true (despite + # .env.example suggesting true is the default), so this must be explicit. + # Fine to leave open since it's tailnet-only; flip to false once your + # account exists to lock it down. env.ALLOW_REGISTRATION = true; credentials = { @@ -32,10 +31,9 @@ apiKey = "ollama"; baseURL = "http://127.0.0.1:11434/v1"; models = { - # schema requires >=1 entry even though fetch=true overwrites it - # at runtime with whatever's pulled (see loadModels in - # hosts/terra/configuration.nix) — kept roughly in sync anyway - # so the UI has sane names before the first fetch completes. + # Schema requires >=1 entry even though fetch=true overwrites this at + # runtime with whatever's pulled (hosts/terra/configuration.nix) — + # kept roughly in sync so the UI has sane names before the first fetch. default = [ "gemma4:12b" "qwen3.6:35b-a3b" "VladimirGav/qwen3.8-27B-14GB-IQ4:latest" ]; fetch = true; # pull the model list from ollama at startup }; @@ -43,22 +41,15 @@ } ]; - # Persistent memory is opt-in at the CONFIG level — omitting this block - # (as before) leaves the feature entirely off, no matter what a user - # toggles in Settings > Personalization. `agent.provider` must match - # endpoints.custom[].name above exactly ("Ollama"), which is how the - # memory-extraction agent picks a backend/model. + # Persistent memory is opt-in at the config level — omitting this block + # leaves it off regardless of the user's Settings > Personalization toggle. + # `agent.provider` must match endpoints.custom[].name above exactly. memory = { personalize = true; # still needs a per-user opt-in toggle in the UI - # instructions REPLACES the default extraction prompt entirely (not - # appended to it) — the 3b model (llama3.2:3b, dropped) was - # defaulting to saving things like its own "I am a helpful - # assistant..." boilerplate under an invented "user_conversation" - # key, and even after adding this prompt, still saved "I am an AI - # assistant with tool calling capabilities" as personal_info after - # the user introduced THEMSELVES — a capability ceiling, not a - # prompting problem. validKeys constrains it to a fixed whitelist - # and instructions spells out the bar for each one. + # instructions REPLACES the default extraction prompt, not appends to it — + # needed because the smaller llama3.2:3b (since dropped) kept saving its + # own assistant boilerplate as memories, a capability ceiling rather than + # a prompting gap. validKeys whitelists what can be stored. validKeys = [ "user_preferences" "personal_info" "ongoing_projects" "technical_context" ]; agent = { enabled = true; diff --git a/services/dev/gitea.nix b/services/dev/gitea.nix index 67f7ff1..47f8a01 100644 --- a/services/dev/gitea.nix +++ b/services/dev/gitea.nix @@ -1,14 +1,12 @@ { config, lib, pkgs, ... }: -# Gitea — self-hosted git. stateDir/repositories were migrated from the old -# ZimaOS docker instance straight into stateDir's default layout, so no -# import step is needed — just chown it to the gitea user after first deploy -# (currently darman:users from the CIFS copy): -# chown -R gitea:gitea /mnt/data/AppData/gitea +# Gitea — self-hosted git. Repos were migrated from the old ZimaOS docker +# instance straight into stateDir's default layout, so after first deploy +# just: chown -R gitea:gitea /mnt/data/AppData/gitea # # HTTP is reverse-proxied through Caddy (hosts/jupiter/configuration.nix). -# SSH uses gitea's own built-in server on :2222 (not the host's :22, and not -# :222 — the unpriv gitea user can't bind <1024). +# SSH uses gitea's own server on :2222, since the unprivileged gitea user +# can't bind :22 or :222 (<1024). let # Repos where the ci-bot account (see below) should be a Write collaborator # and whitelisted to push past branch protection. Add a repo here and @@ -21,40 +19,15 @@ let # nothing she does lands without darman clicking merge. lunaRepos = [ "darman/homelab" ]; - # One gitea webhook per Hermes route. `route` is the path segment Hermes - # dispatches on (http://mars.orbit.sol:8644/webhooks/), so it must - # match a key in the route config that hosts/mars/hermes-agent.nix writes. + # One gitea webhook per Hermes route; `route` must match a key in the route + # config hosts/mars/hermes-agent.nix writes. # - # `events` are the strings gitea's HOOK API accepts. That set is coarser - # than gitea's internal HookEventType set, and both collide on spelling with - # the wire names Hermes matches on — three namespaces, one of which is a - # trap. From routers/api/v1/utils/hook.go (updateHookEvents), - # models/webhook/webhook.go (HasEvent) and modules/webhook/type.go (Event()): - # - # api event (here) delivers wire name (mars route) - # -------------------- ------------------- ---------------------- - # pull_request_comment comment on a PR issue_comment - # pull_request_review review with a body pull_request_comment - # changes requested pull_request_rejected - # approval pull_request_approved - # - # So this file and hosts/mars/hermes-agent.nix name the same event - # differently on purpose, and neither is a typo. - # - # THE TRAP: updateHookEvents silently ignores strings it does not recognise, - # so a plausible-looking but non-API name leaves the hook registered with no - # events at all, delivering nothing and reporting no error. That is exactly - # what "pull_request_review_comment" did here — a real HookEventType, and a - # real value of X-GitHub-Event-Type, but not an API event name. - # - # There is no narrower name for reviews: HasEvent collapses approved, - # rejected and review-comment onto HookEventPullRequestReview, so - # `pull_request_review` is a single switch for all three. Approvals - # therefore cannot be excluded here. They are dropped on the mars side - # instead — the route's event list has no "pull_request_approved", so Hermes - # answers {"status": "ignored"} without running the filter or spending a - # token. Expect approvals in gitea's delivery log, answered 200 and ignored; - # that is the design, not a failure. + # `events` must be gitea's HOOK API event names, which gitea silently drops + # if unrecognized — registering with no events and no error ("pull_request_ + # review_comment" did this: a real HookEventType, but not an API name). + # `pull_request_review` also covers approvals with no narrower option, so + # those are filtered on the mars side instead (answered 200 and ignored — + # expected, not a failure). giteaHermesHooks = [ { name = "PR comments Hermes"; @@ -81,9 +54,8 @@ in server = { DOMAIN = "git.mgaction.town"; SSH_DOMAIN = "git.mgaction.town"; - # https, not http: neptun's Caddy terminates TLS for this name. Gitea - # builds its absolute URLs (clone buttons, redirects, webhooks) from - # ROOT_URL, so an http:// value hands out downgraded links. + # https, not http: neptun's Caddy terminates TLS here, and gitea builds + # its absolute URLs (clone buttons, webhooks) from ROOT_URL. ROOT_URL = "https://git.mgaction.town/"; HTTP_PORT = 3000; START_SSH_SERVER = true; @@ -94,20 +66,11 @@ in DISABLE_REGISTRATION = true; }; security = { - # Gitea refuses to deliver a webhook to any host outside this list, - # which defaults to `external` — "a valid non-private unicast IP". - # Tailscale addresses are 100.64.0.0/10 (RFC 6598 carrier-grade NAT), - # which is neither RFC1918 private nor, as far as gitea's matcher is - # concerned, external — so the hermes relay on mars was refused with - # deny 'mars.orbit.sol(100.64.0.6:8644)' - # even though nothing here is private in the RFC1918 sense. Adding - # the tailnet CIDR is what makes tailnet-internal webhook targets - # deliverable at all; `external` is kept so a future webhook to a - # public service (discord, slack) still works without another edit. - # - # This lives in [security], not [webhook]: the webhook-section key is - # deprecated and now just falls back to this one, which is the name - # the delivery error itself reports. + # Gitea's default `external` webhook target filter treats tailnet + # addresses (100.64.0.0/10, CGNAT) as neither private nor external, so + # the mars hermes relay was refused until the CIDR was added here. + # Lives under [security], not the deprecated [webhook] key it falls + # back to. ALLOWED_HOST_LIST = "external,100.64.0.0/10"; }; actions = { @@ -118,14 +81,10 @@ in networking.firewall.allowedTCPPorts = [ 2222 ]; - # `gitea ` == the admin CLI, as the gitea user, against the real - # state dir — mirrors the `hermes` alias on mars. Worth having because none - # of that is discoverable: the package is not in systemPackages (so `gitea` - # is not otherwise on PATH at all), every admin subcommand needs - # GITEA_WORK_DIR pointed at a stateDir that is not the module default, and - # it has to run as the gitea user or it writes root-owned files into that - # directory. Both paths come from the config rather than being spelled out, - # so a package bump or a stateDir move cannot leave this stale. + # `gitea ` == the admin CLI as the gitea user against the real state + # dir. Not otherwise usable: the package isn't on PATH, and admin + # subcommands need GITEA_WORK_DIR set and root-owned files avoided by + # running as gitea. # # Handy ones: # gitea admin user generate-access-token --username luna \ @@ -138,15 +97,13 @@ in users.users.gitea.extraGroups = [ "users" ]; - # Runner instance registered against this same gitea. Jobs run in containers - # (podman, via services/containers.nix — already enabled on jupiter), one - # image per requested `runs-on` label using the catthehacker act-compatible - # images (same ones upstream `act`/Forgejo docs recommend). + # Runner instance registered against this same gitea. Jobs run in podman + # containers (services/containers.nix), one image per `runs-on` label, using + # the catthehacker act-compatible images. # - # tokenFile points at an env file rendered by sops (TOKEN=, see hosts/jupiter/secrets.nix) rather than a plain `token`, so the - # secret never lands in the Nix store. The registration token itself is NOT - # generated by this module — it comes from gitea once Actions is enabled: + # tokenFile (not `token`) keeps the sops-rendered secret out of the Nix + # store. The registration token isn't generated by this module — get it + # from gitea once Actions is enabled: # su gitea -s /bin/sh -c \ # 'GITEA_WORK_DIR=/mnt/data/AppData/gitea gitea actions generate-runner-token' # then written into secrets/jupiter.yaml as gitea_runner_token. @@ -161,23 +118,18 @@ in ]; }; - # ci-bot: dedicated account CI workflows push as (kept separate from any - # human account so its own PAT can be scoped/rotated/revoked independently). - # Collaborator access + branch-protection push-whitelisting have no CLI or - # config-file surface in gitea — only the HTTP API — so this is the one - # part of the setup that stays imperative even though it's nix-triggered: - # a oneshot that PUTs/PATCHes the API into the desired state on every - # deploy where its script changed (adding a repo to `ciBotRepos` and - # redeploying is enough to pick it up; it won't self-heal a manual revert - # done via the web UI unless the unit is also restarted). + # ci-bot: dedicated account CI workflows push as, so its PAT can be scoped + # and rotated independently of any human account. Collaborator access and + # branch-protection whitelisting have no CLI/config-file surface in gitea — + # only the HTTP API — so this oneshot re-applies the desired state via + # PUT/PATCH on every deploy (won't self-heal a manual UI revert unless + # restarted). # - # Auth for those API calls is darman's OWN token (named - # "jupiter-ci-bot-provisioning" in gitea, scopes write:repository + - # write:user — see hosts/jupiter/secrets.nix), since darman owns the repos - # in ciBotRepos and only an owner-scoped token clears the reqOwnerCheck on - # the collaborator/branch-protection endpoints; write:user is additionally - # needed to push ci-bot's token below as a secret on darman's own account. - # It is NOT ci-bot's own push token — ci-bot can't grant itself access. + # Auth is darman's own token (write:repository + write:user, see + # hosts/jupiter/secrets.nix): an owner-scoped token is required by the + # collaborator/branch-protection endpoints, and write:user is needed to + # push ci-bot's token as a secret on darman's account — ci-bot can't grant + # itself access. # # ci-bot's own push token (separate secret, ci_bot_token) is generated # once via: @@ -255,47 +207,26 @@ in ''; }; - # luna: Hermes Agent's own gitea identity (Hermes was renamed L.U.N.A., - # 2026-08-22). Deliberately PR-tier only, not push-tier like ci-bot: - # Hermes runs on mars, takes instructions over Telegram, and can be - # prompt-injected via tool output — a dedicated account with its own - # scoped, revocable token keeps that blast radius off darman's own - # credentials, and the branch-protection whitelists below keep it off - # `master` entirely regardless of what the token can technically do. - # She gets Write collaborator access (needed to push a branch and open a - # PR against the same repo — this instance has no fork workflow), but: - # - enable_push + enable_push_whitelist(darman only): nobody but darman - # can push straight to master; luna can only land on a side branch. - # - enable_merge_whitelist(darman only): opening a PR is not the same - # as merging one — only darman can click merge. - # - required_approvals=1 + enable_approvals_whitelist(darman only): - # an approval has to come from darman specifically, not luna - # rubber-stamping her own PR from a second identity. - # This covers the SERVER side only (account + collaborator + branch - # protection). The client side — git/tea inside the hermes-agent container, - # and the token below — lives in hosts/mars/hermes-agent.nix. + # luna: Hermes Agent's gitea identity, deliberately PR-tier only (not + # push-tier like ci-bot) — Hermes runs on mars, takes Telegram instructions, + # and can be prompt-injected via tool output, so branch protection below + # keeps her off `master` regardless of what her token can technically do: + # - enable_push_whitelist(darman only): nobody but darman pushes to master. + # - enable_merge_whitelist(darman only): opening a PR isn't merging one. + # - required_approvals=1 + enable_approvals_whitelist(darman only): no + # self-approval from a second identity. + # This is the server side only; the client side (git/tea, token) is in + # hosts/mars/hermes-agent.nix. # - # luna's own push token is generated once, the same way ci-bot's was: - # su gitea -s /bin/sh -c \ - # 'GITEA_WORK_DIR=/mnt/data/AppData/gitea gitea admin user generate-access-token \ - # --username luna --scopes write:repository,write:issue,read:user' - # then stored as a secret (e.g. secrets/mars.yaml's gitea_luna_token) — - # NOT pushed into gitea itself as an Actions secret like ci-bot's is, - # since luna isn't a CI workflow running inside gitea, she's an external - # agent calling out to it. + # luna's push token is generated once (same as ci-bot's, username luna, + # scopes write:repository,write:issue,read:user) and stored as a secret — + # NOT pushed into gitea as an Actions secret, since she's an external agent + # calling in, not a CI workflow. # - # **write:issue is NOT optional and is easy to miss**: this token started - # life as `write:repository` alone, which clones, fetches and pushes - # branches perfectly well — so everything looks fine right up until the - # first `tea pr create`, which gitea rejects with - # token scope=write:repository,read:user required=read:issue - # A pull request IS an issue in gitea's data model, so every /pulls - # endpoint is gated on the *issue* scope category, not the repository one. - # write:issue covers it (in gitea's scope model write:X implies read:X); - # read:issue alone would satisfy the GET half and then fail the POST that - # actually opens the PR. The error names read:issue only because that's - # the first check tea trips on. Rotating the token is free — the prepare - # oneshot on mars does delete-then-add for the tea login on every start. + # write:issue is required, not optional: a PR is an issue in gitea's data + # model, so `tea pr create` needs it even though push/fetch work fine on + # write:repository alone. The resulting error misleadingly names read:issue + # (the first check tea trips), not write:issue. systemd.services.gitea-luna-provision = { description = "Provision luna (Hermes Agent) gitea account + PR-tier repo access"; after = [ "gitea.service" ]; @@ -359,14 +290,10 @@ in ''; }; - # Register one Gitea webhook per Hermes route (giteaHermesHooks above). - # Idempotent: each target URL is updated if a hook for it already exists and - # created otherwise. - # - # It deliberately does NOT delete anything, including hooks for routes that - # were removed from the list above. Retiring one is a one-off, done by hand - # in the repo's Settings -> Webhooks, so that a redeploy can never silently - # unregister a hook someone added on purpose. + # Register one Gitea webhook per Hermes route (giteaHermesHooks above), + # idempotently (update if the target URL exists, else create). Deliberately + # never deletes — a hook for a route removed from the list is retired by + # hand in Settings -> Webhooks, not silently by a redeploy. systemd.services.gitea-hermes-webhook-provision = { description = "Provision Gitea webhooks for Hermes routes"; after = [ "gitea.service" ]; @@ -386,24 +313,17 @@ in set -euo pipefail api=http://127.0.0.1:${toString config.services.gitea.settings.server.HTTP_PORT}/api/v1 - # Neither secret is ever passed as an argument. This unit runs as the - # gitea user on a multi-user box, where /proc//cmdline is - # world-readable for the lifetime of the process — so `-H "Authorization: - # token $t"` would publish the admin token, and `jq --arg secret "$s"` - # the webhook secret. The token goes into a 0600 curl config file - # instead (printf is a shell builtin, so the substitution below never - # reaches an argv), the webhook secret into jq via --rawfile, and the - # request body into curl on stdin with --data @-. + # Secrets never go on argv, since /proc//cmdline is world-readable + # on this multi-user box: the token goes into a 0600 curl config file + # (printf avoids argv entirely), the webhook secret into jq via + # --rawfile, and the body into curl via stdin. authcfg="$(mktemp)" trap 'rm -f "$authcfg"' EXIT chmod 0600 "$authcfg" printf 'header = "Authorization: token %s"\n' "$(cat "$TOKEN_FILE")" > "$authcfg" - # Same readiness gate as gitea-ci-bot-provision / gitea-luna-provision - # above: After=gitea.service only means the process started, not that it - # is serving HTTP yet. Without this the first curl below fails under - # `set -e`, and a Type=oneshot with no Restart= stays failed — leaving - # the webhooks silently unregistered until someone restarts the unit. + # Same readiness gate as the other provisioning units: After=gitea.service + # only means the process started, not that it's serving HTTP yet. for _ in $(seq 1 30); do curl -fs "$api/version" >/dev/null 2>&1 && break sleep 1 @@ -413,10 +333,9 @@ in local name="$1" route="$2" events="$3" url body hook_id url="http://mars.orbit.sol:8644/webhooks/$route" - # rtrimstr: sops stores this without a trailing newline, but one - # slipping in would change the key the HMAC is computed with and make - # every delivery fail signature validation on the Hermes side. The - # same trim happens there, so both ends agree either way. + # rtrimstr: a stray trailing newline would change the HMAC key and + # break signature validation on the Hermes side, which trims the same + # way. body="$(jq -n --rawfile rawSecret "$SECRET_FILE" \ --arg url "$url" --arg name "$name" --argjson events "$events" \ '{type: "gitea", name: $name, active: true, events: $events, diff --git a/services/dev/obsidian-livesync.nix b/services/dev/obsidian-livesync.nix index 7bb37e3..27d0136 100644 --- a/services/dev/obsidian-livesync.nix +++ b/services/dev/obsidian-livesync.nix @@ -1,66 +1,50 @@ { config, ... }: -# CouchDB, tuned as the backend for Obsidian Self-hosted LiveSync -# (vrtmrz/obsidian-livesync). The plugin replicates the vault into CouchDB -# chunk-by-chunk over PouchDB's replication protocol, so this is a plain -# CouchDB 3 node — nothing Obsidian-specific runs here. +# Plain CouchDB 3 node, tuned as the backend for Obsidian Self-hosted LiveSync +# (vrtmrz/obsidian-livesync), which replicates the vault into it via PouchDB. # -# Published PUBLICLY as https://notes.mgaction.town via neptun's caddy (see -# hosts/neptun/configuration.nix), because Obsidian's mobile apps refuse -# cleartext HTTP and jupiter's *.jupiter.sol names cannot get a real cert. -# That makes the settings below security-relevant, not cosmetic: +# Published PUBLICLY as https://notes.mgaction.town via neptun's caddy, since +# Obsidian's mobile apps refuse cleartext HTTP and jupiter's *.jupiter.sol +# names can't get a real cert — so the settings below are security-relevant: +# - `require_valid_user` in both [chttpd] and [chttpd_auth], else CouchDB +# answers unauthenticated GETs on the open internet. +# - neptun's vhost allowlists only the plugin's endpoints; Fauxton and +# cluster/config are reachable only over the tailnet. +# - Turn on the plugin's end-to-end encryption (+ "Obfuscate Properties"), +# so this server only ever holds ciphertext — what makes a +# publicly-reachable credentialed database an acceptable trade. # -# - `require_valid_user` in BOTH [chttpd] and [chttpd_auth]: without it -# CouchDB answers unauthenticated GETs on the open internet. -# - neptun's vhost allowlists only the endpoints the plugin uses, so Fauxton -# (/_utils) and the cluster/config endpoints are not reachable from -# outside at all — reach them over the tailnet instead. -# - Turn ON end-to-end encryption in the plugin (Settings → Remote Database -# → End-to-End Encryption, plus "Obfuscate Properties", which covers the -# paths and timestamps that E2EE alone leaves readable). Then this server -# only ever holds ciphertext, which is what makes a publicly-reachable -# credentialed database an acceptable trade rather than a bad one. -# -# Its passphrase is a SEPARATE secret from couchdb_admin_password below — -# deliberately, and it must stay that way. The couchdb password -# authenticates to this server and is stored here (hashed) and in -# secrets/jupiter.yaml; the E2EE passphrase never leaves the Obsidian -# clients and CouchDB has no idea it exists. Reusing one string for both -# hands whoever obtains that credential the decryption key as well, which -# is precisely the failure E2EE is here to prevent. The passphrase is -# therefore NOT in sops (nothing on this host consumes it) — it lives in -# the HomeLab Proton Pass vault, with the deploy credentials. -# -# Losing it costs the remote database, not the notes: wipe it and +# Its passphrase must stay a SEPARATE secret from couchdb_admin_password: +# the CouchDB password is stored here and in secrets/jupiter.yaml, while +# the E2EE passphrase never leaves the clients (kept in the HomeLab Proton +# Pass vault, not sops) — reusing one string for both would hand the +# decryption key to whoever gets the CouchDB credential. Losing the +# passphrase costs the remote database, not the notes: wipe and # re-initialize from a device that still holds the plaintext vault. { services.couchdb = { enable = true; - # Listens on all interfaces, same reasoning as immich: :5984 is NOT opened - # in the firewall, so it is reachable over tailscale0 (trusted in - # common.nix) and localhost only. That is the path neptun's caddy takes. + # Listens on all interfaces, but :5984 is not opened in the firewall, so + # it's reachable only over tailscale0 (trusted) and localhost — the path + # neptun's caddy takes. bindAddress = "0.0.0.0"; port = 5984; # The vault database is the ONLY copy of the notes once LiveSync is the - # source of truth, so it belongs on the array, not the 29G eMMC. All three - # of these default under /var/lib/couchdb and have to move together — - # configFile especially, since CouchDB writes to it at runtime (below). + # source of truth, so it belongs on the array, not the 29G eMMC — all + # three default under /var/lib/couchdb and must move together. databaseDir = "/mnt/data/AppData/couchdb"; viewIndexDir = "/mnt/data/AppData/couchdb"; configFile = "/mnt/data/AppData/couchdb/local.ini"; - # The admin password, as an [admins] ini fragment from sops. - # services.couchdb.adminPass would render it into the world-readable - # store; extraConfigFiles is the module's own documented hook for this - # (hosts/jupiter/secrets.nix renders the template). + # [admins] ini fragment from sops; services.couchdb.adminPass would render + # into the world-readable store instead. # - # ⚠️ CouchDB hashes a plaintext admin password at startup and persists the - # hash to the LAST, writable file in its ini chain — local.ini above, - # which then takes precedence over this fragment. So changing the sops - # value alone does NOT rotate the password: delete the `[admins]` line - # from /mnt/data/AppData/couchdb/local.ini and restart as well. + # ⚠️ CouchDB hashes the password at startup and persists it to local.ini + # (above), which then takes precedence — so changing the sops value alone + # does NOT rotate it. Also delete the `[admins]` line from + # /mnt/data/AppData/couchdb/local.ini and restart. extraConfigFiles = [ config.sops.templates."couchdb-admins.ini".path ]; # Values taken from LiveSync's own CouchDB setup documentation; the plugin diff --git a/services/experimental/cinephage.nix b/services/experimental/cinephage.nix index 677c548..45b411d 100644 --- a/services/experimental/cinephage.nix +++ b/services/experimental/cinephage.nix @@ -1,12 +1,11 @@ { config, ... }: -# Cinephage — indexer search + streaming/library manager. Runs the official -# container image, not upstream's nix flake module: its npmDepsHash is stale -# against its own package-lock.json, and a transitive dep hard-enforces pnpm, -# breaking the nix-sandboxed npm build regardless. Docker is the actually- -# maintained path. BETTER_AUTH_SECRET (paired sops secret in -# hosts/jupiter/secrets.nix) signs sessions/encrypts stored API keys — must -# be static, not app-generated, or losing it invalidates everything. +# Cinephage — indexer search + streaming/library manager, run as the official +# container image rather than upstream's nix flake module (its npmDepsHash is +# stale and a transitive dep hard-enforces pnpm, breaking the sandboxed npm +# build). BETTER_AUTH_SECRET (paired sops secret, hosts/jupiter/secrets.nix) +# signs sessions and encrypts stored API keys — keep it static, since losing +# it invalidates everything. { virtualisation.oci-containers.containers.cinephage = { image = "ghcr.io/moldytaint/cinephage:latest"; diff --git a/services/experimental/mediamanager.nix b/services/experimental/mediamanager.nix index 8f00424..ae3fe67 100644 --- a/services/experimental/mediamanager.nix +++ b/services/experimental/mediamanager.nix @@ -1,10 +1,10 @@ { config, ... }: -# MediaManager — media request/library manager. Module comes from the -# community flake input `mediamanager-nix`, not nixpkgs. Paired sops secret -# in hosts/jupiter/secrets.nix — without it the module mints+discards a -# random auth token_secret on every restart, logging everyone out. -# Port 8010: 8000 is taken by audiobookshelf on this host. +# MediaManager — media request/library manager (module from the +# `mediamanager-nix` flake input, not nixpkgs). The paired sops secret +# (hosts/jupiter/secrets.nix) is required — without it the module mints a +# random token_secret every restart, logging everyone out; port 8010 since +# audiobookshelf already holds 8000. { services.media-manager = { enable = true; @@ -45,9 +45,9 @@ MEDIAMANAGER_INDEXERS__PROWLARR__API_KEY=${config.sops.placeholder.prowlarr_api_key} ''; - # HighSeas/{Movies,Shows,images,Downloads} are darman:users 755 on disk — - # group has no write bit. media-manager is in "users" (below); the dirs - # themselves were chmod g+w by hand once (not declarative — see CLAUDE.md - # gotchas), since this is pre-existing data, not something tmpfiles owns. + # HighSeas/{Movies,Shows,images,Downloads} are darman:users 755 (no group + # write bit); media-manager is in "users" (below), and the dirs were + # chmod g+w by hand once since this is pre-existing data, not something + # tmpfiles owns. users.users.media-manager.extraGroups = [ "users" ]; } diff --git a/services/identity/authentik.nix b/services/identity/authentik.nix index 98970cb..c04c916 100644 --- a/services/identity/authentik.nix +++ b/services/identity/authentik.nix @@ -1,30 +1,22 @@ { config, pkgs, inputs, ... }: -# Authentik — self-hosted identity/OIDC provider. +# Authentik — self-hosted identity/OIDC provider. Replaced Zitadel because +# nixpkgs is stuck on 2.71 (no login-v2 split) with a forward-only db +# migration; authentik-nix tracks upstream closely instead. # -# Replaced Zitadel: nixpkgs only carries Zitadel 2.71 (no login-v2 split, and -# a v3/v4 database migrates forward only, so an existing instance can't be -# moved onto it). authentik-nix tracks upstream closely instead. +# The upstream module owns postgres and its unit ordering, and needs no redis +# (channels/cache run on postgres). TLS terminates at Caddy; every listener +# below is pinned to loopback since only tailscale0 is trusted. # -# The upstream module owns postgres (createDatabase) AND orders the units -# against postgresql.target, so no manual After= is needed here. No redis — -# recent authentik runs channels/cache on postgres. -# -# TLS terminates at Caddy; every listener is pinned to loopback below so -# nothing is reachable from the tailnet (hosts trust tailscale0). -# -# Needs, wired via sops in the host's secrets.nix: an environmentFile carrying -# - AUTHENTIK_SECRET_KEY (`openssl rand -base64 60`) — signs sessions -# - AUTHENTIK_BOOTSTRAP_PASSWORD first-run akadmin password -# systemd reads EnvironmentFile as root before dropping to the service's -# DynamicUser, so the sops default root:root 0400 is correct — do NOT set -# `owner` on it the way the headplane secrets need. +# Needs an environmentFile from sops (host's secrets.nix) carrying +# AUTHENTIK_SECRET_KEY and AUTHENTIK_BOOTSTRAP_PASSWORD. Keep it root:root +# 0400 (systemd reads it as root before dropping to DynamicUser) — don't set +# `owner` the way headplane's secrets need. { imports = [ inputs.authentik-nix.nixosModules.default ]; # Pinned explicitly: the default tracks system.stateVersion, so editing that - # line would silently demand a pg_upgrade of the identity store. Bump this - # deliberately, with a dump in hand. + # would silently demand a pg_upgrade of the identity store. services.postgresql.package = pkgs.postgresql_17; services.authentik = { diff --git a/services/media/audiobookshelf.nix b/services/media/audiobookshelf.nix index 28801a9..28b750a 100644 --- a/services/media/audiobookshelf.nix +++ b/services/media/audiobookshelf.nix @@ -1,11 +1,9 @@ { ... }: -# Audiobookshelf audiobook/podcast server. -# Listens on all interfaces: :8000 stays closed on the LAN (no openFirewall), -# but reachable over the trusted tailscale0 interface and via localhost (caddy). -# Library/media paths are set in the web UI — point them at /mnt/data/... -# Runs as user `audiobookshelf`; added to `users` so it can read group-owned -# library dirs on the RAID. +# Audiobookshelf audiobook/podcast server, listening on all interfaces but +# reachable only via tailscale0 or local caddy (no openFirewall) — library +# paths are set in the web UI, pointed at /mnt/data/... In the "users" group +# so it can read the RAID's group-owned library dirs. { services.audiobookshelf = { enable = true; diff --git a/services/media/immich.nix b/services/media/immich.nix index f23a611..ec3fdf5 100644 --- a/services/media/immich.nix +++ b/services/media/immich.nix @@ -1,30 +1,22 @@ { config, pkgs, inputs, ... }: -# Immich photo/video library. Native nixpkgs module (not the upstream compose -# stack) — it owns its own postgres (with the pgvector + vectorchord extensions -# it needs for search) and a unix-socket redis, so nothing else is required here. +# Immich photo/video library. Native nixpkgs module, not the upstream compose +# stack — it owns its own postgres (pgvector + vectorchord) and a unix-socket redis. # -# Storage: everything lives under /mnt/data/AppData/immich, which is the media -# store MIGRATED from the old ZimaOS/CasaOS install's UPLOAD_LOCATION -# (/mnt/data/Immich/upload — same layout: library/ upload/ thumbs/ -# encoded-video/ profile/ backups/). See scripts/immich-import-legacy-db for the -# matching database import. The postgres cluster itself stays on the OS disk. +# Storage lives under /mnt/data/AppData/immich, migrated from the old ZimaOS/CasaOS +# UPLOAD_LOCATION (same subfolder layout); see scripts/immich-import-legacy-db for +# the matching DB import. The postgres cluster itself stays on the OS disk. # # ⚠️ The immich DB is the only copy of albums/faces/dates — the files alone # can't rebuild it. It joins the other unbacked databases on this network. let - # The PACKAGE comes from nixpkgs-unstable (3.0.3); the MODULE comes from the - # 26.05 pin (which ships 2.7.5). That combination is safe because the two - # module files are byte-identical — verified by diffing them at the revisions - # in flake.lock. RE-CHECK THAT DIFF on any input bump: + # Package pinned to nixpkgs-unstable (3.0.3) while the module stays on the 26.05 + # pin (2.7.5) — safe only because the two module files are byte-identical + # (verified by diff; re-check on any input bump). Needed because immich's + # migrations are forward-only and jupiter's imported DB was last written by + # 3.0.0, which 2.7.5 refuses to start against; drop once the pin ships >= 3.0.0. # diff <(nixpkgs)/nixos/modules/services/web-apps/immich.nix \ # <(unstable)/nixos/modules/services/web-apps/immich.nix - # - # Why: jupiter's imported database was last written by immich 3.0.0, and - # immich runs its migrations forward only — 2.7.5 refuses to start against it - # with "corrupted migrations: previously executed migration - # 1776217577402-DropAuditTable is missing". Drop this override once nixos-26.11 - # (or whatever the pin becomes) ships >= 3.0.0. unstable = import inputs.nixpkgs-unstable { inherit (pkgs.stdenv.hostPlatform) system; }; @@ -42,27 +34,20 @@ in mediaLocation = "/mnt/data/AppData/immich"; machine-learning.enable = true; - # ⚠️ Setting `settings` at all switches immich to IMMICH_CONFIG_FILE, and - # that is ALL-OR-NOTHING (dist/utils/config.js: the config is - # `configFile ? loadFromFile(...) : metadataRepo.get(SystemConfig)` — the - # database copy is IGNORED, not merged). Two consequences: - # 1. Anything not declared here falls back to immich's DEFAULTS, not to - # whatever the admin UI had. The old settings stay in the - # system_metadata table, so deleting this block restores them. - # 2. The admin settings UI goes read-only — saving throws "Cannot update - # configuration while IMMICH_CONFIG_FILE is in use". Change settings - # HERE and redeploy. - # An unknown/misspelled key is a HARD startup failure under a config file - # (the same code path only logs a warning without one), so keys below are - # taken verbatim from `defaults` in immich's dist/config.js. + # ⚠️ Setting `settings` at all switches immich to IMMICH_CONFIG_FILE mode, + # which is all-or-nothing: undeclared keys fall back to immich's defaults, not + # the admin UI's saved values (which stay in system_metadata and return if + # this block is deleted), and the admin settings UI goes read-only. An + # unknown/misspelled key is a hard startup failure here (just a warning + # without a config file), so keys are copied verbatim from `defaults` in + # immich's dist/config.js. settings = { server.externalDomain = "https://immich.mgaction.town"; newVersionCheck.enabled = false; # nixpkgs pins the version, not immich - # OIDC via Authentik on neptun. The Authentik application/provider is - # created BY HAND in its UI — same as headscale's and headplane's, which - # are also separate apps (hosts/neptun/secrets.nix). Only the client - # secret is managed here. + # OIDC via Authentik on neptun; the application/provider is created by hand + # in its UI (like headscale's and headplane's, separate apps) — only the + # client secret is managed here (hosts/neptun/secrets.nix). oauth = { enabled = true; # Authentik's per-application issuer. Trailing slash matters: immich @@ -76,24 +61,18 @@ in clientSecret._secret = config.sops.secrets.immich_oauth_client_secret.path; scope = "openid email profile"; buttonText = "Login with Authentik"; - # Existing accounts (the 2 imported users) keep working: matching is by - # email, so an Authentik user with the same address adopts that account - # rather than creating a second one. + # Matches by email, so the 2 imported users adopt their Authentik account + # instead of getting a duplicate. autoRegister = true; # Leave the password form reachable — autoLaunch would bounce straight # to Authentik, locking everyone out if the OIDC app is misconfigured. autoLaunch = false; - # Land back on immich's own login page after logout. Without this, - # immich falls back to the IdP's discovered end_session_endpoint - # (auth.service.js:320-326) and logout dumps you on Authentik's - # "you've been logged out" page instead. Must be an ABSOLUTE url — - # the config schema rejects a relative path — and mirrors immich's - # internal LOGIN_URL, including autoLaunch=0. - # - # Note this ends the IMMICH session only; the Authentik SSO session - # survives, so the next "Login with Authentik" click signs straight - # back in without a credential prompt. To end both, drop this line and - # let the IdP endpoint take over again. + # Without this, immich falls back to the IdP's discovered + # end_session_endpoint and logout dumps you on Authentik's own page + # instead of back here — must be an absolute url, mirroring immich's + # internal LOGIN_URL. This ends the immich session only; the Authentik + # SSO session survives, so the next login skips the credential prompt — + # drop this line to end both. endSessionEndpoint = "https://auth.mgaction.town/application/o/immich/end-session?post_logout_redirect_url=https://immich.mgaction.town"; # The mobile app can't follow a browser redirect back to a custom # scheme through Authentik, so immich bounces it via this endpoint. @@ -101,28 +80,24 @@ in mobileRedirectUri = "https://immich.mgaction.town/api/oauth/mobile-redirect"; }; }; - # Hardware transcoding would need the iGPU passed in explicitly, e.g. - # accelerationDevices = [ "/dev/dri/renderD128" ]; the default [ ] means - # PrivateDevices=yes and CPU-only transcode. The ZimaBlade's Celeron does - # this slowly but it only runs on upload. + # Hardware transcoding needs accelerationDevices set explicitly (e.g. + # "/dev/dri/renderD128"); default CPU-only transcode is slow on the + # ZimaBlade's Celeron but only runs on upload. }; - # /mnt/data/AppData is drwx--x--- darman:users — immich needs group "users" - # just to TRAVERSE into its own media dir. The dir itself stays 0700 - # immich:immich (the module's tmpfiles rule re-asserts that every rebuild, - # and UMask=0077 keeps new files private), so this grants nothing else. + # /mnt/data/AppData is drwx--x--- darman:users; immich only needs group "users" + # to traverse into it — the dir itself stays 0700 immich:immich (tmpfiles + + # UMask=0077 reassert that), so this grants nothing else. users.users.immich.extraGroups = [ "users" ]; - # mediaLocation is outside /var/lib, so the module won't create it — its own - # tmpfiles entry only ADJUSTS an existing dir. Harmless no-op after the - # legacy import, which puts the real store here. + # mediaLocation is outside /var/lib, so the module won't create it — this rule + # only adjusts perms on the dir the legacy import already created. systemd.tmpfiles.rules = [ "d /mnt/data/AppData/immich 0700 immich immich -" ]; - # The unit's automatic RequiresMountsFor covers /run/immich and /var/lib/immich - # only — nothing points it at mediaLocation. Without this immich starts with - # the array missing and writes uploaded photos onto the 29G eMMC, into a - # directory that becomes invisible the moment /mnt/data mounts over it. + # The unit's automatic RequiresMountsFor doesn't cover mediaLocation — without + # this, immich starts before /mnt/data mounts and writes uploads onto the 29G + # eMMC, invisibly, under the future mountpoint. systemd.services.immich-server.unitConfig.RequiresMountsFor = [ "/mnt/data" ]; } diff --git a/services/media/jellyfin.nix b/services/media/jellyfin.nix index 798b919..2223924 100644 --- a/services/media/jellyfin.nix +++ b/services/media/jellyfin.nix @@ -6,20 +6,17 @@ dataDir = "/mnt/data/AppData/jellyfin"; cacheDir = "/mnt/data/AppData/jellyfin/cache"; }; - # "users" so the shared library stays readable (see the UMask note below); - # "video"/"render" for the DRI nodes used by hardware transcoding. renderD128 - # happens to be 0666 so VAAPI alone would work without this, but card1 is - # 0660 root:video — and neither mode is guaranteed, so don't rely on it. The - # groups are harmless on a host with no GPU: they exist regardless, and this - # module stays host-agnostic (the DRIVER is enabled per-host, e.g. jupiter's - # hardware.graphics + intel-media-driver). + # "users" keeps the shared library readable (see the UMask note below); + # "video"/"render" cover the DRI nodes for hardware transcoding — card1 is + # 0660 root:video (not guaranteed 0666 like renderD128), so don't rely on + # device perms alone. Harmless on a GPU-less host: the driver itself is + # enabled per-host (e.g. jupiter's hardware.graphics + intel-media-driver). users.users.jellyfin.extraGroups = [ "users" "video" "render" ]; - # The upstream module hardcodes UMask=0077 — root cause of jellyfin writing - # trickplay thumbnails into stray new show folders it invented itself, - # owned jellyfin:jellyfin 700, invisible to every other service sharing - # the library (cinephage, mediamanager, ...). New files/dirs it creates - # from here on inherit group "users" (library roots are setgid, see the - # one-time chmod g+s done by hand) and stay group-writable. + # The upstream module hardcodes UMask=0077, which made jellyfin write + # trickplay thumbnails into new folders owned jellyfin:jellyfin 700 — + # invisible to every other service sharing the library (cinephage, + # mediamanager). Forcing 0002 makes new files inherit group "users" + # (library roots are setgid via a one-time chmod g+s) and stay group-writable. systemd.services.jellyfin.serviceConfig.UMask = lib.mkForce "0002"; } diff --git a/services/media/prowlarr.nix b/services/media/prowlarr.nix index aa24dd9..2e79a8b 100644 --- a/services/media/prowlarr.nix +++ b/services/media/prowlarr.nix @@ -15,21 +15,16 @@ { services.prowlarr.enable = true; - # `nofail` is NOT optional here: without it this bind is RequiredBy - # local-fs.target, so an unassembled RAID array fails that target and drops - # jupiter into emergency mode — which is a dead end, since root is locked and - # sulogin has nothing to offer on a headless box. It defeats the `nofail` on - # /mnt/data itself (a mount layered on the array is what actually took the - # target down). Let this bind fail alone instead. + # `nofail` is not optional: without it this bind is RequiredBy local-fs.target, + # so an unassembled array drops jupiter into emergency mode — a dead end on a + # headless box with root locked. Let this bind fail alone instead. fileSystems."/var/lib/private/prowlarr" = { device = "/mnt/data/AppData/prowlarr/config"; fsType = "none"; options = [ "bind" "nofail" ]; }; - # systemd derives RequiresMountsFor from the unit's own paths, which here is - # only /var/lib/prowlarr on the eMMC — so without this prowlarr starts happily - # with the array absent and writes its state onto the 29G OS disk. Pin it to - # the array so it fails loudly instead. + # systemd derives RequiresMountsFor only from /var/lib/prowlarr (eMMC) — pin + # it to the array too, or prowlarr starts happily and writes state to the OS disk. systemd.services.prowlarr.unitConfig.RequiresMountsFor = [ "/mnt/data" ]; } diff --git a/services/media/radarr.nix b/services/media/radarr.nix index 3bd9e7f..69efa6c 100644 --- a/services/media/radarr.nix +++ b/services/media/radarr.nix @@ -1,11 +1,10 @@ { ... }: -# Radarr — movie library manager, feeds off SABnzbd/Prowlarr. dataDir points -# at the config migrated from the old ZimaOS docker stack (indexers/download -# client/history already set up). Unlike prowlarr, this module uses a static -# `radarr` user (no DynamicUser) and only auto-chowns dataDir when it's the -# module's own default path — since we point at a pre-existing migrated dir, -# chown it by hand once after first deploy: +# Radarr — movie library manager, feeds off SABnzbd/Prowlarr; dataDir points +# at config migrated from the old ZimaOS docker stack. Unlike prowlarr, this +# module uses a static `radarr` user (no DynamicUser) and only auto-chowns +# dataDir at its own default path, so the migrated dir needs a manual +# one-time chown after first deploy: # chown -R radarr:radarr /mnt/data/AppData/radarr/config { services.radarr = { diff --git a/services/media/sabnzbd.nix b/services/media/sabnzbd.nix index e67ffb7..a88b02a 100644 --- a/services/media/sabnzbd.nix +++ b/services/media/sabnzbd.nix @@ -1,18 +1,13 @@ { config, ... }: -# SABnzbd — usenet downloader. Migrated off a reused hand-authored ini -# (servers/API key/history originally imported from the old ZimaOS docker -# stack) onto NixOS-managed `settings`, per the module's own deprecation -# notice for `configFile`. Only the values that differ from SABnzbd's own -# built-in defaults are declared here — everything else falls back to the -# same defaults SABnzbd was already using. +# SABnzbd — usenet downloader, migrated off a hand-authored ini (imported from +# the old ZimaOS docker stack) onto NixOS-managed `settings`. Only values that +# differ from SABnzbd's own defaults are declared here. # -# `admin_dir`/`log_dir` MUST stay absolute: the module writes the merged ini -# to /var/lib/sabnzbd/sabnzbd.ini (eMMC), and both dirs are otherwise -# relative to wherever the ini lives. Pointing them back at the ORIGINAL -# /mnt/data location keeps the existing download queue/history database -# (admin_dir) intact — a relative default here would silently "reset" -# SABnzbd to an empty queue on first switch, even though nothing was deleted. +# `admin_dir`/`log_dir` must stay absolute: the module writes the merged ini to +# /var/lib/sabnzbd/sabnzbd.ini (eMMC), so a relative default would resolve +# there instead of the original /mnt/data location — silently "resetting" +# SABnzbd to an empty queue/history on first switch, without deleting anything. { services.sabnzbd = { enable = true; @@ -73,17 +68,15 @@ # Write access to the shared downloads dir (owned darman:users on disk). users.users.sabnzbd.extraGroups = [ "users" ]; - # download/complete/admin dirs all live on the array, but systemd only - # derives RequiresMountsFor from /var/lib/sabnzbd (eMMC) — so with the array - # absent sabnzbd would start and download onto the 29G OS disk. + # download/complete/admin dirs live on the array, but systemd only derives + # RequiresMountsFor from /var/lib/sabnzbd (eMMC) — without this, a missing + # array lets sabnzbd start and download onto the 29G OS disk instead. systemd.services.sabnzbd.unitConfig.RequiresMountsFor = [ "/mnt/data" ]; systemd.services.fix-downloads-perms.unitConfig.RequiresMountsFor = [ "/mnt/data" ]; - # SABnzbd hardcodes completed job folders to 0700 on every job, ignoring - # the ini's `umask` (that only covers files during unpack, not the job - # dir itself). setgid on Downloads keeps the group as "users" but perm - # bits still come back zeroed, locking out cinephage/mediamanager — sweep - # it clean instead of fighting SABnzbd. + # SABnzbd hardcodes completed job folders to 0700, ignoring the ini's `umask` + # (unpack-only) — setgid keeps the group but perm bits still zero out and + # lock out cinephage/mediamanager, so sweep it clean on a timer instead. systemd.services.fix-downloads-perms = { description = "Fix group perms SABnzbd resets on completed downloads"; serviceConfig.Type = "oneshot"; diff --git a/services/media/seerr.nix b/services/media/seerr.nix index 836500e..889f2d0 100644 --- a/services/media/seerr.nix +++ b/services/media/seerr.nix @@ -1,13 +1,10 @@ { ... }: -# Seerr (formerly Jellyseerr) — request manager for Jellyfin, talks to -# Sonarr/Radarr to fulfill requests. Fresh install, no migrated data. -# -# configDir stays at the module default; bind-mount AppData onto it instead -# of overriding configDir, so data lives on the RAID array and survives an -# OS-disk reinstall (same DynamicUser/StateDirectory issue as prowlarr.nix — -# see that file for why, and why the mount targets /var/lib/private/seerr -# rather than the public path). +# Seerr (formerly Jellyseerr) — request manager for Jellyfin, talking to +# Sonarr/Radarr; fresh install, no migrated data. configDir stays at the +# module default, with AppData bind-mounted onto it instead (same +# DynamicUser/StateDirectory issue as prowlarr.nix — see that file for why, +# and why the mount targets /var/lib/private/seerr rather than the public path). { services.seerr.enable = true; diff --git a/services/media/sonarr.nix b/services/media/sonarr.nix index 01ac41a..dfd9c3f 100644 --- a/services/media/sonarr.nix +++ b/services/media/sonarr.nix @@ -1,11 +1,10 @@ { ... }: -# Sonarr — TV library manager, feeds off SABnzbd/Prowlarr. dataDir points at -# the config migrated from the old ZimaOS docker stack (indexers/download -# client/history already set up). Unlike prowlarr, this module uses a static -# `sonarr` user (no DynamicUser) and only auto-chowns dataDir when it's the -# module's own default path — since we point at a pre-existing migrated dir, -# chown it by hand once after first deploy: +# Sonarr — TV library manager, feeds off SABnzbd/Prowlarr; dataDir points at +# config migrated from the old ZimaOS docker stack. Unlike prowlarr, this +# module uses a static `sonarr` user (no DynamicUser) and only auto-chowns +# dataDir at its own default path, so the migrated dir needs a manual +# one-time chown after first deploy: # chown -R sonarr:sonarr /mnt/data/AppData/sonarr/config { services.sonarr = { diff --git a/services/monitoring/victoriametrics.nix b/services/monitoring/victoriametrics.nix index c38fb75..1c95820 100644 --- a/services/monitoring/victoriametrics.nix +++ b/services/monitoring/victoriametrics.nix @@ -15,10 +15,9 @@ prometheusConfig = { global.scrape_interval = "5s"; - # Explicit, and equal to the interval on purpose. The Prometheus default - # is 10s, and VictoriaMetrics silently clamps scrape_timeout down to - # scrape_interval rather than erroring — so leaving it implicit means the - # config says 10s while the scraper uses 5s. Say what actually happens. + # Explicit and equal to the interval on purpose: VictoriaMetrics silently + # clamps scrape_timeout down to scrape_interval, so leaving the Prometheus + # default (10s) here would misstate what actually happens. global.scrape_timeout = "5s"; scrape_configs = [ @@ -44,14 +43,10 @@ ]; } # mercury is a Pi scraped over the tailnet, so it gets its own job at a - # slower cadence: at the 5s global it would time out (see above) and - # the series would show gaps rather than late samples. - # - # A separate cadence REQUIRES a separate job — scrape_interval is a - # per-job setting and job_name has to be unique — which means mercury's - # `job` label differs from every other host's. Select on `host` (set on - # every target below) rather than job="node-exporter" in dashboards and - # alerts, or mercury drops out of them silently. + # slower cadence to avoid timing out at the 5s global. A separate cadence + # requires a separate job (scrape_interval is per-job), so mercury's + # `job` label differs from every other host's — select on `host` in + # dashboards/alerts, not job="node-exporter", or mercury drops out silently. { job_name = "node-exporter-mercury"; scrape_interval = "15s"; @@ -81,31 +76,22 @@ # another host or the tailnet is temporarily unavailable. systemd.services.victoriametrics.after = [ "tailscaled-autoconnect.service" ]; - # Keep the TSDB off jupiter's 29G eMMC. The module hardcodes - # -storageDataPath=/var/lib/ and runs DynamicUser, so without this - # the data lands on the OS disk — a continuous small-write workload aimed at - # the one disk here with no headroom and finite write endurance. Same - # bind-onto-/var/lib/private pattern as prowlarr.nix and seerr.nix; see - # prowlarr.nix for why the mount targets the private path and not the public - # /var/lib/victoriametrics. - # - # `nofail` is NOT optional — again see prowlarr.nix: without it this bind is - # RequiredBy local-fs.target, so an unassembled array drops jupiter into an - # emergency shell that a headless box cannot be rescued from. + # Keep the TSDB off jupiter's 29G eMMC: the module hardcodes + # -storageDataPath=/var/lib/ under DynamicUser, so without this bind + # a continuous small-write workload lands on the one disk with no headroom. + # Same /var/lib/private bind pattern as prowlarr.nix and seerr.nix — see + # prowlarr.nix for why it targets the private path, and why `nofail` here is + # not optional. fileSystems."/var/lib/private/victoriametrics" = { device = "/mnt/data/AppData/victoriametrics"; fsType = "none"; options = [ "bind" "nofail" ]; }; - # The bind above needs its SOURCE to exist or the mount fails — and because - # it is `nofail` that failure is quiet: RequiresMountsFor below is satisfied - # by /mnt/data itself, so VictoriaMetrics would start regardless and write to - # the eMMC, which is the exact thing the bind exists to prevent. prowlarr.nix - # gets away without this only because its directory predates the module - # (migrated from ZimaOS). This is a fresh service, so it creates its own, - # same as seerr.nix. 0755 darman:users matches the other AppData dirs, which - # matters because /mnt/data/AppData itself is drwx--x--- darman:users. + # The bind above needs its source dir to exist or it quietly fails (`nofail`) + # and VictoriaMetrics falls through to writing the eMMC anyway — this is a + # fresh service so, unlike prowlarr.nix's pre-existing dir, it must create its + # own (same as seerr.nix). 0755 darman:users matches the other AppData dirs. systemd.tmpfiles.rules = [ "d /mnt/data/AppData/victoriametrics 0755 darman users -" ]; diff --git a/services/network/pihole.nix b/services/network/pihole.nix index e14ead9..3009a7f 100644 --- a/services/network/pihole.nix +++ b/services/network/pihole.nix @@ -53,17 +53,11 @@ in }; }; - # Bind-mount source must exist (podman won't create it), and it must be - # owned by 1000 — the `pihole` user FTL drops to after the entrypoint's root - # phase. Podman here is rootful with no userns remapping, so that number is - # the same inside and out (on the host it collides with darman, harmlessly). - # - # Ownership of gravity.db alone is not enough: sqlite creates a sibling - # gravity.db-journal for every write transaction, so FTL needs to CREATE - # files in this directory. Root-owned, it fails with - # open(/etc/pihole/gravity.db-journal) - (14) - # attempt to write a readonly database - # which reads like a corrupt or read-only database and is neither. + # Bind-mount source must exist (podman won't create it) and be owned by 1000, + # the `pihole` user FTL drops to (rootful podman, no userns remapping, so the + # uid is the same inside and out). Must be the whole DIRECTORY, not just + # gravity.db — sqlite needs to create a sibling gravity.db-journal per write, + # and a root-owned dir makes that fail with a misleading "readonly database". systemd.tmpfiles.rules = [ "d /var/lib/pihole 0750 1000 1000 -" ]; # Seed the adlists above into gravity. `INSERT OR IGNORE` keyed on the URL diff --git a/services/network/samba.nix b/services/network/samba.nix index 57132d7..6522709 100644 --- a/services/network/samba.nix +++ b/services/network/samba.nix @@ -22,14 +22,13 @@ }; }; - # Samba keeps its own NTLM password DB, separate from the system password; - # `services.samba` never sets it, so logins fail until provisioned. Runs - # AFTER samba-smbd so its state dir exists — an activation script runs too - # early and smbpasswd fails to init the passdb. Reads a single-line - # password from the first file that exists: + # Samba keeps its own NTLM password DB, separate from the system password — + # `services.samba` never sets it, and this runs as a service (not an + # activation script, which fires too early for smbpasswd's passdb) after + # samba-smbd. Reads a single-line password from the first existing file, + # feeding it twice since smbpasswd prompts new+confirm: # Real host: /run/secrets/samba_password (sops-nix, see secrets.nix) # VM test: /etc/samba/smb-password (plaintext, see vm.nix) - # smbpasswd prompts new + confirm, so the value is fed twice. systemd.services.samba-smbpasswd = { description = "Provision Samba password for darman"; after = [ "samba-smbd.service" ]; diff --git a/services/network/unbound.nix b/services/network/unbound.nix index baaefda..60296f2 100644 --- a/services/network/unbound.nix +++ b/services/network/unbound.nix @@ -1,8 +1,8 @@ { ... }: -# Local recursive DNS resolver (privacy + DNSSEC). Your adblock DNS -# (pihole/AdGuard) forwards to this instead of a public upstream. -# Listens on 127.0.0.1:5335 — point the adblock engine's upstream there: +# Local recursive DNS resolver (privacy + DNSSEC) that the adblock DNS +# (pihole/AdGuard) forwards to instead of a public upstream — listens on +# 127.0.0.1:5335, so point the adblock engine's upstream there: # AdGuard: dns.upstream_dns = [ "127.0.0.1:5335" ]; # pihole: upstream = "127.0.0.1#5335"; { diff --git a/services/vpn/headplane.nix b/services/vpn/headplane.nix index 600cd5c..b8b86e9 100644 --- a/services/vpn/headplane.nix +++ b/services/vpn/headplane.nix @@ -1,25 +1,20 @@ { config, ... }: -# Headplane — web UI for headscale (services/vpn/headscale.nix; must be enabled -# first), running as headscale's own OS user. +# Headplane — web UI for headscale (services/vpn/headscale.nix; enable first), +# running as headscale's OS user. # -# It reads headscale's config from the nix store, which is read-only — so the -# UI DISPLAYS the settings but can't change them. That's the intended shape -# for a declaratively-configured box (config_strict already defaults off -# upstream for exactly this reason); edit them here and rebuild instead. -# DNS extra-records are the one thing worth making editable, since they're -# data rather than config — hence the writable extra_records file below, -# which also spares headplane from restarting headscale on every change. +# It reads headscale's config from the nix store, so the UI DISPLAYS settings +# but can't change them (edit here and rebuild instead) — except DNS +# extra-records, which are data rather than config, hence the writable +# extra_records file below. # -# Served at vpn.mgaction.town/admin (path-routed alongside headscale itself, -# see hosts/neptun/configuration.nix). base_url is the site root WITHOUT the -# /admin prefix — Headplane appends that itself, including for the OIDC -# callback. +# Served at vpn.mgaction.town/admin (path-routed with headscale, see +# hosts/neptun/configuration.nix); base_url excludes the /admin prefix, which +# Headplane appends itself including for the OIDC callback. # -# Auth is Authentik (services/identity/authentik.nix) via OIDC. client_id, -# client_secret, and the headscale API key can't be known until -# Authentik/headscale are actually deployed, so they're placeholders below; -# direct API-key login still works as a fallback until then. Once live: +# Auth is Authentik via OIDC; client_id/client_secret/API key are placeholders +# until Authentik/headscale are deployed (direct API-key login works as a +# fallback until then). Once live: # 1. In Authentik: create an OAuth2/OpenID Provider + Application with slug # `headplane` and redirect URI # https://vpn.mgaction.town/admin/oidc/callback. Copy the generated @@ -28,9 +23,6 @@ # headplane_oidc_client_secret with the provider's client secret. # 3. `headscale apikeys create` on the box, and replace # headplane_headscale_api_key the same way. -# -# NOTE: Authentik issues per-application, so the issuer carries the app slug — -# it is NOT the bare host the way Zitadel's was. { # Writable DNS extra-records, shared by both services (they run as the same # user). tmpfiles seeds an empty JSON array — headscale won't start against diff --git a/services/vpn/headscale.nix b/services/vpn/headscale.nix index fd7559a..31adad1 100644 --- a/services/vpn/headscale.nix +++ b/services/vpn/headscale.nix @@ -1,14 +1,10 @@ { config, ... }: -# Headscale — self-hosted control server for the tailnet. Every host's -# services/vpn/tailscale.nix points --login-server at https://vpn.mgaction.town -# (this host). MagicDNS base_domain "orbit.sol" matches the -# "jupiter.orbit.sol" names used in this repo's Caddy vhosts -# (hosts/neptun/configuration.nix) — changing base_domain means changing -# those too, and re-pointing neptun's dnsmasq stub at the new suffix. -# -# TLS terminates at Caddy (see the host's configuration.nix); headscale -# itself only listens on localhost. +# Headscale — self-hosted control server for the tailnet; every host's +# services/vpn/tailscale.nix points --login-server at https://vpn.mgaction.town. +# TLS terminates at Caddy; headscale itself only listens on localhost. Changing +# base_domain below also means updating this repo's Caddy vhosts and neptun's +# dnsmasq stub, which assume "orbit.sol". { services.headscale = { enable = true; @@ -18,89 +14,55 @@ server_url = "https://vpn.mgaction.town"; dns = { - # Deliberately OUTSIDE mgaction.town. That zone has a wildcard A+AAAA - # pointing at neptun, and DNS wildcards match multi-label names — so - # with base_domain = hosts.mgaction.town, `jupiter.hosts.mgaction.town` - # resolved publicly to NEPTUN and Caddy proxied to itself: a silent - # loop rather than a lookup failure. - # - # `.sol` is the LAN domain pihole serves, so this nests the tailnet - # inside it: planets sit on the LAN as jupiter.sol, and reach each - # other in orbit as jupiter.orbit.sol. Resolution is unambiguous - # because tailscale matches routes by LONGEST suffix, so orbit.sol - # goes to MagicDNS even when everything else funnels to pihole. - # - # Never give a LAN host the name `orbit`: pihole's - # `address=/.sol/` lines match a name AND everything under - # it, so an `orbit` host would swallow this entire zone. + # Deliberately outside mgaction.town: that zone has a wildcard A+AAAA at + # neptun, so a name under it would resolve publicly to neptun and Caddy + # would proxy to itself. Nested under `.sol` (pihole's LAN domain) so + # jupiter.sol (LAN) and jupiter.orbit.sol (tailnet) resolve unambiguously + # — tailscale matches by longest suffix. Never name a LAN host `orbit`: + # pihole's `address=/.sol/` would swallow this whole zone. base_domain = "orbit.sol"; - # pihole on mercury, over the tailnet — so every roaming device gets - # ad blocking and .sol names wherever it is, not just on the LAN. - # Deliberately NO public fallback: tailscale treats the list as a set, - # so adding 9.9.9.9 here would let queries slip past the filter - # whenever mercury is briefly slow. Strict blocking, at the cost of - # mercury being a single point of failure for tailnet DNS. - # - # ⚠️ A hardcoded tailnet address, so it changes if mercury re-enrols - # — check `headscale nodes list` if DNS dies tailnet-wide. + # pihole on mercury, over the tailnet, so roaming devices get ad blocking + # and .sol names everywhere. Deliberately no public fallback — tailscale + # treats this as a set, so adding one would let queries slip past the + # filter whenever mercury is briefly slow, at the cost of mercury being a + # single point of failure for tailnet DNS. + # ⚠️ Hardcoded tailnet address — check `headscale nodes list` if it + # changes (mercury re-enrolled) and DNS dies tailnet-wide. nameservers.global = [ "100.64.0.7" ]; - # Must be set, and must be HERE rather than via the module's - # `dns.split` option. nixpkgs renders that option one level too high - # (a sibling of `nameservers:`), but headscale reads - # dns.nameservers.split (hscontrol/types/config.go:722) and so does - # headplane. So the module's option is dead, and the missing key makes - # headplane's DNS page die with - # TypeError: Cannot convert undefined or null to object - # from Object.keys(config.dns.nameservers.split). + # Must be set here, not via the module's `dns.split` option — nixpkgs + # renders that one level too high, but headscale (and headplane) read + # dns.nameservers.split; the missing key crashes headplane's DNS page. nameservers.split = { }; - # Point every node's resolver at MagicDNS, which forwards on to the - # global nameserver above. That is the only way to get pihole onto a - # roaming device: with this false, globalResolvers land in the - # netmap's FallbackResolvers (hscontrol/types/config.go:826-830) and a - # phone with carrier DNS never consults them. - # - # The cost is that every node's DNS now depends on mercury and on the - # home connection, so mercury going down costs name resolution - # everywhere, not just `.sol`. neptun and mercury opt out of this - # individually with --accept-dns=false — see their configuration.nix. + # Routes every node's resolver through MagicDNS to the global nameserver + # above — the only way pihole reaches a roaming device (otherwise it + # lands in netmap's FallbackResolvers and carrier DNS never consults it). + # Cost: all DNS now depends on mercury and the home connection; neptun + # and mercury opt out individually with --accept-dns=false. override_local_dns = true; }; - # Authentik as the login provider, so `tailscale up --login-server ...` - # sends you to a browser instead of needing a pre-auth key. This is a - # SEPARATE Authentik application from headplane's — its own provider, - # slug `headscale`, redirect https://vpn.mgaction.town/oidc/callback - # (headscale's own callback; headplane's is under /admin). - # - # ⚠️ headscale performs OIDC discovery at STARTUP and a failure is - # FATAL ("creating OIDC provider from issuer config: 404 Not Found") — - # it will not boot, taking the whole tailnet's control plane with it. - # Never point `issuer` at an application that doesn't exist yet; verify - # with: + # Authentik as the login provider (own application, slug `headscale`, + # separate from headplane's) so `tailscale up --login-server ...` opens a + # browser instead of needing a pre-auth key; headless hosts still use those. + # ⚠️ headscale does OIDC discovery at startup and a failure is fatal — it + # won't boot, taking the whole control plane with it. Never point `issuer` + # at an application that doesn't exist yet; verify with # curl -s .well-known/openid-configuration - # - # Headless hosts still enrol with pre-auth keys. Note also that users - # created here are distinct from `headscale users create` ones: matching - # is by the OIDC `sub` claim against the user's providerId, and 0.28 - # dropped map_legacy_users, so CLI-made users never gain one. + # Users created here are matched by OIDC `sub`, so `headscale users + # create`-made users never link to one (0.28 dropped map_legacy_users). oidc = { issuer = "https://auth.mgaction.town/application/o/headscale/"; client_id = "14vhRYaLiONHmI2YFIxbQEveJDLu5cCvzSkTb9oq"; client_secret_path = config.sops.secrets.headscale_oidc_client_secret.path; }; - # Run our own DERP relay instead of pulling Tailscale's map. - # - # With the default (urls = [controlplane.tailscale.com/derpmap/default], - # auto_update_enabled = true) headscale fetches that map at startup and - # treats failure as FATAL — so a DNS blip or a Tailscale outage stops the - # control server from booting at all. A self-hosted control plane that - # can't start without Tailscale's infrastructure rather misses the point. - # - # The relay itself rides Caddy on :443 (hence the flush_interval -1 on - # that vhost); only STUN needs its own UDP port. + # Run our own DERP relay instead of pulling Tailscale's map: the default + # fetches that map at startup and treats a failure as fatal, so a DNS blip + # or Tailscale outage would stop this control server from booting at all. + # The relay rides Caddy on :443 (hence flush_interval -1 on that vhost); + # only STUN needs its own UDP port. derp = { urls = [ ]; auto_update_enabled = false; diff --git a/services/vpn/tailscale.nix b/services/vpn/tailscale.nix index 86e93d7..7232481 100644 --- a/services/vpn/tailscale.nix +++ b/services/vpn/tailscale.nix @@ -1,9 +1,9 @@ { config, ... }: -# Tailscale node joined to the self-hosted headscale control server. -# Auto-registers on boot from a sops pre-auth key. Requires the importing host -# to declare `sops.secrets.tailscale_authkey` (see each host's secrets.nix). -# Not for the VM (no sops). +# Tailscale node joined to the self-hosted headscale control server, +# auto-registering on boot from a sops pre-auth key — importing hosts must +# declare `sops.secrets.tailscale_authkey` (see each host's secrets.nix). +# Not used by the VM target (no sops there). { services.tailscale = { enable = true; @@ -14,11 +14,11 @@ # Reach the host's services over the tailnet without opening LAN ports. networking.firewall.trustedInterfaces = [ "tailscale0" ]; - # The upstream unit is a one-shot with no Restart, so a login attempt made - # before the control server is reachable fails permanently until someone - # starts it by hand. That's the norm on a first boot — neptun hosts headscale - # itself, and the other hosts race it. 30s spacing also keeps restarts clear - # of systemd's default start limit (5 within 10s). + # The upstream unit is a one-shot with no Restart, so a login attempted + # before the control server is up fails permanently until restarted by + # hand — the norm on first boot, since neptun hosts headscale itself and + # other hosts race it. 30s spacing keeps retries clear of systemd's default + # start limit (5 within 10s). systemd.services.tailscaled-autoconnect.serviceConfig = { Restart = "on-failure"; RestartSec = 30;