darmanandClaude Sonnet 5
7904433d34
Local LLM server on the 6800 XT (ollama-rocm, gfx1030 needs no HSA_OVERRIDE_GFX_VERSION) fronted by a LibreChat web UI, talking to it over the OpenAI-compatible /v1 route. Also wires up LibreChat's persistent-memory feature, which needed its own agent+model plus a custom extraction prompt: the default 3b model couldn't reliably tell the user's stated facts apart from its own boilerplate, and even a tuned prompt didn't fix that — so memory extraction now reuses gemma4:12b, the same model as the daily-driver chat endpoint. flake.lock bump pulls in the ollama and librechat NixOS modules. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
97 lines
4.7 KiB
Nix
97 lines
4.7 KiB
Nix
{ config, ... }:
|
|
|
|
# LibreChat — web chat UI, talking to the local ollama server (see
|
|
# hosts/terra/configuration.nix) over its OpenAI-compatible /v1 route.
|
|
# Only reachable over the tailnet (networking.firewall.trustedInterfaces =
|
|
# [ "tailscale0" ] in services/vpn/tailscale.nix) — openFirewall stays off.
|
|
{
|
|
services.librechat = {
|
|
enable = true;
|
|
enableLocalDB = true; # spins up a local, unauthenticated-on-localhost mongodb
|
|
|
|
# LibreChat's isEnabled() treats an UNSET var as false, not true — so
|
|
# registration is closed unless this is explicit, despite .env.example
|
|
# suggesting true is the default. Only reachable over the tailnet
|
|
# (trusted interface, see module comment below), so leaving it open is
|
|
# fine; flip to false once your account exists if you want it locked down.
|
|
env.ALLOW_REGISTRATION = true;
|
|
|
|
credentials = {
|
|
CREDS_KEY = config.sops.secrets.librechat_creds_key.path;
|
|
CREDS_IV = config.sops.secrets.librechat_creds_iv.path;
|
|
JWT_SECRET = config.sops.secrets.librechat_jwt_secret.path;
|
|
JWT_REFRESH_SECRET = config.sops.secrets.librechat_jwt_refresh_secret.path;
|
|
};
|
|
|
|
settings = {
|
|
version = "1.2.1";
|
|
endpoints.custom = [
|
|
{
|
|
name = "Ollama";
|
|
# required field but unchecked by ollama's OpenAI-compat shim
|
|
apiKey = "ollama";
|
|
baseURL = "http://127.0.0.1:11434/v1";
|
|
models = {
|
|
# schema requires >=1 entry even though fetch=true overwrites it
|
|
# at runtime with whatever's pulled (see loadModels in
|
|
# hosts/terra/configuration.nix) — kept roughly in sync anyway
|
|
# so the UI has sane names before the first fetch completes.
|
|
default = [ "gemma4:12b" "qwen3.6:35b-a3b" ];
|
|
fetch = true; # pull the model list from ollama at startup
|
|
};
|
|
titleConvo = true;
|
|
}
|
|
];
|
|
|
|
# Persistent memory is opt-in at the CONFIG level — omitting this block
|
|
# (as before) leaves the feature entirely off, no matter what a user
|
|
# toggles in Settings > Personalization. `agent.provider` must match
|
|
# endpoints.custom[].name above exactly ("Ollama"), which is how the
|
|
# memory-extraction agent picks a backend/model.
|
|
memory = {
|
|
personalize = true; # still needs a per-user opt-in toggle in the UI
|
|
# instructions REPLACES the default extraction prompt entirely (not
|
|
# appended to it) — the 3b model (llama3.2:3b, dropped) was
|
|
# defaulting to saving things like its own "I am a helpful
|
|
# assistant..." boilerplate under an invented "user_conversation"
|
|
# key, and even after adding this prompt, still saved "I am an AI
|
|
# assistant with tool calling capabilities" as personal_info after
|
|
# the user introduced THEMSELVES — a capability ceiling, not a
|
|
# prompting problem. validKeys constrains it to a fixed whitelist
|
|
# and instructions spells out the bar for each one.
|
|
validKeys = [ "user_preferences" "personal_info" "ongoing_projects" "technical_context" ];
|
|
agent = {
|
|
enabled = true;
|
|
provider = "Ollama";
|
|
# same model as the chat endpoint's primary driver — when that's
|
|
# the active chat model, extraction needs no second model swapped
|
|
# into VRAM alongside it.
|
|
model = "gemma4:12b";
|
|
instructions = ''
|
|
Save memory ONLY using the keys below, and only when the user's
|
|
message states something durable and genuinely useful to recall
|
|
in a LATER, unrelated conversation. Small talk, greetings, and
|
|
anything about what the assistant said or is capable of are NOT
|
|
memories — if nothing meets the bar, save nothing.
|
|
|
|
set_memory REPLACES the entire value stored at a key — it does
|
|
NOT append to it. Before calling set_memory for a key, check the
|
|
"Existing memory" section below. If that key already has a
|
|
value, your new value MUST merge the old and new information
|
|
into one complete sentence or short paragraph — calling
|
|
set_memory with only the newest fact silently ERASES everything
|
|
already stored under that key. Only drop prior details if the
|
|
user is explicitly correcting or replacing them.
|
|
|
|
- user_preferences: explicitly stated preferences (tools, formats, style).
|
|
- personal_info: durable facts about the user (name, role, timezone).
|
|
- ongoing_projects: projects or tasks the user is actively working on.
|
|
- technical_context: durable facts about the user's setup/stack
|
|
relevant to future answers (e.g. "runs NixOS", "GPU is AMD ROCm").
|
|
'';
|
|
};
|
|
};
|
|
};
|
|
};
|
|
}
|