diff --git a/pkgs/luna-voice/src/config.rs b/pkgs/luna-voice/src/config.rs index 60cc044..ae5ad6f 100644 --- a/pkgs/luna-voice/src/config.rs +++ b/pkgs/luna-voice/src/config.rs @@ -27,6 +27,8 @@ pub struct Config { pub whisper_bin: String, pub whisper_model: PathBuf, + /// whisper -l; "auto" detects per utterance. + pub whisper_language: String, pub socket_path: PathBuf, /// Silence that ends an utterance in hands-free mode. Hermes' own default. @@ -84,6 +86,7 @@ impl Config { whisper_bin: env_or("LUNA_WHISPER_BIN", "whisper-cli"), whisper_model: PathBuf::from(env("LUNA_WHISPER_MODEL")?), + whisper_language: env_or("LUNA_WHISPER_LANGUAGE", "auto"), socket_path: PathBuf::from(env_or( "LUNA_SOCKET", diff --git a/pkgs/luna-voice/src/main.rs b/pkgs/luna-voice/src/main.rs index e0d1138..019ba85 100644 --- a/pkgs/luna-voice/src/main.rs +++ b/pkgs/luna-voice/src/main.rs @@ -272,7 +272,7 @@ async fn daemon() -> Result<()> { capturing = false; let samples = capture.stop(); ipc.emit(Event::State { value: "thinking" }); - match stt::transcribe(&cfg.whisper_bin, &cfg.whisper_model, &samples).await { + match stt::transcribe(&cfg.whisper_bin, &cfg.whisper_model, &cfg.whisper_language, &samples).await { Ok(heard) if !heard.is_empty() => { run_turn( &cfg, &http, &ipc, &capture, &playback, &mut cmd_rx, @@ -336,7 +336,7 @@ async fn daemon() -> Result<()> { ipc.emit(Event::State { value: "thinking" }); let heard = - match stt::transcribe(&cfg.whisper_bin, &cfg.whisper_model, &samples).await { + match stt::transcribe(&cfg.whisper_bin, &cfg.whisper_model, &cfg.whisper_language, &samples).await { Ok(h) => h, Err(e) => { eprintln!("luna-voice: {e:#}"); diff --git a/pkgs/luna-voice/src/stt.rs b/pkgs/luna-voice/src/stt.rs index cf77afb..0cc4054 100644 --- a/pkgs/luna-voice/src/stt.rs +++ b/pkgs/luna-voice/src/stt.rs @@ -6,7 +6,12 @@ use tokio::process::Command; /// Shells out to whisper-cli rather than linking whisper.cpp: nixpkgs already /// ships whisper-cpp-vulkan built against this box's GPU stack, and a bundled /// build would have to rediscover that at cargo time. -pub async fn transcribe(bin: &str, model: &Path, samples: &[f32]) -> Result { +pub async fn transcribe( + bin: &str, + model: &Path, + language: &str, + samples: &[f32], +) -> Result { // Whisper hallucinates confident words into silence — "you", "Thank // you.", "Bye." — which then reach luna as if spoken. Duration alone does // not catch it (a held key over a quiet room is long AND empty), so gate @@ -26,7 +31,7 @@ pub async fn transcribe(bin: &str, model: &Path, samples: &[f32]) -> Result Result bool { }) } +/// Multilingual whisper fills silence with the subtitle credits it was trained +/// on ("Untertitel im Auftrag des ZDF, 2017", "Untertitel der Amara.org-Community"). +/// Nobody says these to luna, so the whole transcript is dropped. +fn subtitle_credit(text: &str) -> bool { + let t = text.to_lowercase(); + t.contains("untertitel") || t.contains("amara.org") +} + /// whisper emits bracketed non-speech annotations ([BLANK_AUDIO], (wind /// blowing)) that are not words and must not reach the agent. fn clean(raw: &str) -> String { @@ -97,7 +114,7 @@ fn clean(raw: &str) -> String { #[cfg(test)] mod tests { - use super::{clean, has_speech, normalize}; + use super::{clean, has_speech, normalize, subtitle_credit}; use crate::audio::STT_RATE; fn peak(v: &[f32]) -> f32 { @@ -165,6 +182,13 @@ mod tests { assert_eq!(clean("[ music ] real words"), "real words"); } + #[test] + fn drops_subtitle_credits() { + assert!(subtitle_credit("Untertitel im Auftrag des ZDF, 2017")); + assert!(subtitle_credit("Untertitel der Amara.org-Community")); + assert!(!subtitle_credit("Ist das Backup auf jupiter gelaufen?")); + } + #[test] fn collapses_whitespace() { assert_eq!(clean(" a b \n c "), "a b c"); diff --git a/services/desktop/luna-voice.nix b/services/desktop/luna-voice.nix index a9a3ace..d0f4e0c 100644 --- a/services/desktop/luna-voice.nix +++ b/services/desktop/luna-voice.nix @@ -24,9 +24,12 @@ let # Pinned like the KittenTTS models on mars: a model that re-downloads at # first use is a service that fails on a fresh install with no network. + # small, multilingual: base.en heard "backup" as "Mac up" and turned German + # into English nonsense. ~1s per turn on the 6800 XT vs ~0.4s; turbo was + # ~2.3s for no visible gain. whisperModel = pkgs.fetchurl { - url = "https://huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-base.en.bin"; - sha256 = "sha256-oDd5yG3zMjB19eeWyyzlAp8A7Ihp7uP9+4l6/jbG0AI="; + url = "https://huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-small.bin"; + sha256 = "sha256-G+OpsgY4Z7k35k4ux0gzZKeZF+FX+pjF2UtcH//qmHs="; }; in { @@ -87,6 +90,16 @@ in description = "Substring of the input device name; null uses the system default."; }; + whisperLanguage = lib.mkOption { + type = lib.types.str; + default = "auto"; + description = '' + whisper's -l. "auto" detects per utterance, which costs a second + encoder pass; a fixed code like "en" translates other languages + instead of transcribing them. + ''; + }; + vadMult = lib.mkOption { type = lib.types.float; default = 1.8; @@ -147,6 +160,7 @@ in LUNA_ELEVENLABS_KEY_FILE = toString cfg.elevenLabsKeyFile; LUNA_WHISPER_BIN = "${pkgs.whisper-cpp-vulkan}/bin/whisper-cli"; LUNA_WHISPER_MODEL = whisperModel; + LUNA_WHISPER_LANGUAGE = cfg.whisperLanguage; LUNA_VAD_MULT = toString cfg.vadMult; LUNA_VAD_FLOOR = toString cfg.vadFloor; LUNA_SILENCE_MS = toString cfg.silenceMs;