The key is a toggle rather than push-to-talk, because "voice mode is active" has to be a state something can display; a held key cannot sustain one. So the daemon grew a VAD loop: speech starts a capture, `silenceMs` of quiet ends it. A turn is awaited inline, which is also what stops her hearing herself — the tick cannot fire while she speaks. The keybind runs the CLI, not a quickshell GlobalShortcut, so the mic can still be switched off while the shell is reloading and a QML error can never leave it open with nothing on screen saying so. Every threshold had to be resized for terra's mic, which is QUIET: 0.001-0.002 ambient against 0.0045 speaking, about 6dB apart. The VAD floor of 0.006 and whisper's own 0.015 gate both sat ABOVE the user's voice, and a 3.5x multiplier cannot separate signals that close — three independent reasons nothing could ever trigger. Hence vadMult 1.8, vadFloor 0.0025, and stt::normalize lifting the clip (capped at 30x, so near-silence is not amplified into words) before whisper sees it. The floor tracker also had to stop feeding on everything: a rising sound dragged its own threshold up faster than it rose, and a tone peaking at 0.029 was outrun by a threshold that reached 0.018. Only samples below the current trigger update it now. Four diagnostics, because none of the above was visible while it was failing: `watch` prints the socket protocol, `say` runs a turn from typed text (a broken mic vs a broken pipeline), `level` is a live meter with the trigger marked, `devices` lists the inputs. Amplitude is now emitted on every tick, not only while she speaks — a meter that moves only during playback cannot answer "is it hearing me". Also: connect-time replay of mode/state, so restarting the shell while voice mode is on does not leave it blind until the next event, which in an idle conversation is never. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
148 lines
5.1 KiB
Nix
148 lines
5.1 KiB
Nix
{ config, pkgs, lib, ... }:
|
|
|
|
# Voice loop for luna: hold a key, talk, hear her answer.
|
|
#
|
|
# mic -> whisper-cpp (local, vulkan) -> Hermes' OpenAI-compatible API on mars
|
|
# -> ElevenLabs streaming TTS -> speaker. State, amplitude and the viseme
|
|
# timeline are published on a unix socket for the quickshell avatar to read
|
|
# (nothing consumes it yet — the widget is the next phase).
|
|
#
|
|
# A USER service, not a system one: it needs darman's pipewire session and his
|
|
# XDG_RUNTIME_DIR for the socket. That is also why the two secrets it reads
|
|
# are owner-darman in hosts/terra/secrets.nix — a user manager cannot read a
|
|
# root-owned 0400 file, and LoadCredential does not bridge that gap.
|
|
#
|
|
# STT is LOCAL on purpose: mars already runs faster-whisper for Telegram voice
|
|
# notes, but routing the mic through it would put every spoken word on the
|
|
# network for no latency win. This is the client-direct path Hermes itself
|
|
# documents.
|
|
|
|
let
|
|
cfg = config.homelab.lunaVoice;
|
|
luna-voice = pkgs.callPackage ../../pkgs/luna-voice.nix { };
|
|
|
|
# Pinned like the KittenTTS models on mars: a model that re-downloads at
|
|
# first use is a service that fails on a fresh install with no network.
|
|
whisperModel = pkgs.fetchurl {
|
|
url = "https://huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-base.en.bin";
|
|
sha256 = "sha256-oDd5yG3zMjB19eeWyyzlAp8A7Ihp7uP9+4l6/jbG0AI=";
|
|
};
|
|
in
|
|
{
|
|
options.homelab.lunaVoice = {
|
|
enable = lib.mkEnableOption "luna voice daemon";
|
|
|
|
user = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "darman";
|
|
description = "User whose session runs the daemon.";
|
|
};
|
|
|
|
hermesUrl = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "http://mars.orbit.sol:8642";
|
|
description = "Base URL of luna's Hermes API server (tailnet-only).";
|
|
};
|
|
|
|
model = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "hermes-agent";
|
|
description = "Model name to request; Hermes routes it to her configured provider.";
|
|
};
|
|
|
|
voiceId = lib.mkOption {
|
|
type = lib.types.str;
|
|
description = "ElevenLabs voice id — luna's voice.";
|
|
};
|
|
|
|
ttsModel = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "eleven_flash_v2_5";
|
|
description = "~75ms model, the one ElevenLabs recommends for live conversation.";
|
|
};
|
|
|
|
hermesKeyFile = lib.mkOption {
|
|
type = lib.types.path;
|
|
description = "File holding the API_SERVER_KEY bearer token.";
|
|
};
|
|
|
|
elevenLabsKeyFile = lib.mkOption {
|
|
type = lib.types.path;
|
|
description = "File holding the ElevenLabs API key.";
|
|
};
|
|
|
|
# Mic tuning. Defaults are set for terra's AIR 192|4, which is QUIET:
|
|
# ~0.002 ambient against ~0.0045 for normal speech. Raising the gain on
|
|
# the interface is worth more than any value here — everything downstream,
|
|
# whisper included, works better with headroom.
|
|
inputDevice = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
description = "Substring of the input device name; null uses the system default.";
|
|
};
|
|
|
|
vadMult = lib.mkOption {
|
|
type = lib.types.float;
|
|
default = 1.8;
|
|
description = "Speech triggers above noise_floor * this.";
|
|
};
|
|
|
|
vadFloor = lib.mkOption {
|
|
type = lib.types.float;
|
|
default = 0.0025;
|
|
description = "Absolute lower bound on the trigger, whatever the floor says.";
|
|
};
|
|
|
|
silenceMs = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 2000;
|
|
description = "Silence that ends an utterance.";
|
|
};
|
|
|
|
systemPrompt = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
description = ''
|
|
Prepended to the conversation. Worth setting to something like "You
|
|
are being spoken to out loud; keep replies short" — her Telegram
|
|
register is too long-form to listen to.
|
|
'';
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
environment.systemPackages = [ luna-voice ];
|
|
|
|
systemd.user.services.luna-voiced = {
|
|
description = "luna voice loop";
|
|
wantedBy = [ "graphical-session.target" ];
|
|
after = [ "graphical-session.target" "pipewire.service" ];
|
|
partOf = [ "graphical-session.target" ];
|
|
|
|
environment = {
|
|
LUNA_HERMES_URL = cfg.hermesUrl;
|
|
LUNA_MODEL = cfg.model;
|
|
LUNA_VOICE_ID = cfg.voiceId;
|
|
LUNA_TTS_MODEL = cfg.ttsModel;
|
|
LUNA_HERMES_KEY_FILE = toString cfg.hermesKeyFile;
|
|
LUNA_ELEVENLABS_KEY_FILE = toString cfg.elevenLabsKeyFile;
|
|
LUNA_WHISPER_BIN = "${pkgs.whisper-cpp-vulkan}/bin/whisper-cli";
|
|
LUNA_WHISPER_MODEL = whisperModel;
|
|
LUNA_VAD_MULT = toString cfg.vadMult;
|
|
LUNA_VAD_FLOOR = toString cfg.vadFloor;
|
|
LUNA_SILENCE_MS = toString cfg.silenceMs;
|
|
} // lib.optionalAttrs (cfg.systemPrompt != null) {
|
|
LUNA_SYSTEM_PROMPT = cfg.systemPrompt;
|
|
} // lib.optionalAttrs (cfg.inputDevice != null) {
|
|
LUNA_INPUT_DEVICE = cfg.inputDevice;
|
|
};
|
|
|
|
serviceConfig = {
|
|
ExecStart = "${luna-voice}/bin/luna-voice daemon";
|
|
Restart = "on-failure";
|
|
RestartSec = 3;
|
|
};
|
|
};
|
|
};
|
|
}
|