feat(terra): multilingual whisper small for luna's voice
base.en heard "backup" as "Mac up" and turned German into English nonsense. small with -l auto gets both right at ~1s per turn (vs ~0.4s); turbo was ~2.3s for no visible gain. Drops the subtitle credits multilingual whisper invents out of silence. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011XkDJkdDzneVVdnkZfrvTY
This commit is contained in:
@@ -27,6 +27,8 @@ pub struct Config {
|
|||||||
|
|
||||||
pub whisper_bin: String,
|
pub whisper_bin: String,
|
||||||
pub whisper_model: PathBuf,
|
pub whisper_model: PathBuf,
|
||||||
|
/// whisper -l; "auto" detects per utterance.
|
||||||
|
pub whisper_language: String,
|
||||||
|
|
||||||
pub socket_path: PathBuf,
|
pub socket_path: PathBuf,
|
||||||
/// Silence that ends an utterance in hands-free mode. Hermes' own default.
|
/// Silence that ends an utterance in hands-free mode. Hermes' own default.
|
||||||
@@ -84,6 +86,7 @@ impl Config {
|
|||||||
|
|
||||||
whisper_bin: env_or("LUNA_WHISPER_BIN", "whisper-cli"),
|
whisper_bin: env_or("LUNA_WHISPER_BIN", "whisper-cli"),
|
||||||
whisper_model: PathBuf::from(env("LUNA_WHISPER_MODEL")?),
|
whisper_model: PathBuf::from(env("LUNA_WHISPER_MODEL")?),
|
||||||
|
whisper_language: env_or("LUNA_WHISPER_LANGUAGE", "auto"),
|
||||||
|
|
||||||
socket_path: PathBuf::from(env_or(
|
socket_path: PathBuf::from(env_or(
|
||||||
"LUNA_SOCKET",
|
"LUNA_SOCKET",
|
||||||
|
|||||||
@@ -272,7 +272,7 @@ async fn daemon() -> Result<()> {
|
|||||||
capturing = false;
|
capturing = false;
|
||||||
let samples = capture.stop();
|
let samples = capture.stop();
|
||||||
ipc.emit(Event::State { value: "thinking" });
|
ipc.emit(Event::State { value: "thinking" });
|
||||||
match stt::transcribe(&cfg.whisper_bin, &cfg.whisper_model, &samples).await {
|
match stt::transcribe(&cfg.whisper_bin, &cfg.whisper_model, &cfg.whisper_language, &samples).await {
|
||||||
Ok(heard) if !heard.is_empty() => {
|
Ok(heard) if !heard.is_empty() => {
|
||||||
run_turn(
|
run_turn(
|
||||||
&cfg, &http, &ipc, &capture, &playback, &mut cmd_rx,
|
&cfg, &http, &ipc, &capture, &playback, &mut cmd_rx,
|
||||||
@@ -336,7 +336,7 @@ async fn daemon() -> Result<()> {
|
|||||||
ipc.emit(Event::State { value: "thinking" });
|
ipc.emit(Event::State { value: "thinking" });
|
||||||
|
|
||||||
let heard =
|
let heard =
|
||||||
match stt::transcribe(&cfg.whisper_bin, &cfg.whisper_model, &samples).await {
|
match stt::transcribe(&cfg.whisper_bin, &cfg.whisper_model, &cfg.whisper_language, &samples).await {
|
||||||
Ok(h) => h,
|
Ok(h) => h,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
eprintln!("luna-voice: {e:#}");
|
eprintln!("luna-voice: {e:#}");
|
||||||
|
|||||||
@@ -6,7 +6,12 @@ use tokio::process::Command;
|
|||||||
/// Shells out to whisper-cli rather than linking whisper.cpp: nixpkgs already
|
/// Shells out to whisper-cli rather than linking whisper.cpp: nixpkgs already
|
||||||
/// ships whisper-cpp-vulkan built against this box's GPU stack, and a bundled
|
/// ships whisper-cpp-vulkan built against this box's GPU stack, and a bundled
|
||||||
/// build would have to rediscover that at cargo time.
|
/// build would have to rediscover that at cargo time.
|
||||||
pub async fn transcribe(bin: &str, model: &Path, samples: &[f32]) -> Result<String> {
|
pub async fn transcribe(
|
||||||
|
bin: &str,
|
||||||
|
model: &Path,
|
||||||
|
language: &str,
|
||||||
|
samples: &[f32],
|
||||||
|
) -> Result<String> {
|
||||||
// Whisper hallucinates confident words into silence — "you", "Thank
|
// Whisper hallucinates confident words into silence — "you", "Thank
|
||||||
// you.", "Bye." — which then reach luna as if spoken. Duration alone does
|
// you.", "Bye." — which then reach luna as if spoken. Duration alone does
|
||||||
// not catch it (a held key over a quiet room is long AND empty), so gate
|
// not catch it (a held key over a quiet room is long AND empty), so gate
|
||||||
@@ -26,7 +31,7 @@ pub async fn transcribe(bin: &str, model: &Path, samples: &[f32]) -> Result<Stri
|
|||||||
.arg(&wav)
|
.arg(&wav)
|
||||||
// -nt: transcript only, no [00:00:00] prefixes. -np: no progress
|
// -nt: transcript only, no [00:00:00] prefixes. -np: no progress
|
||||||
// banner. Together they make stdout exactly the text.
|
// banner. Together they make stdout exactly the text.
|
||||||
.args(["-nt", "-np"])
|
.args(["-nt", "-np", "-l", language])
|
||||||
.output()
|
.output()
|
||||||
.await?;
|
.await?;
|
||||||
let _ = std::fs::remove_file(&wav);
|
let _ = std::fs::remove_file(&wav);
|
||||||
@@ -37,7 +42,11 @@ pub async fn transcribe(bin: &str, model: &Path, samples: &[f32]) -> Result<Stri
|
|||||||
String::from_utf8_lossy(&out.stderr).trim()
|
String::from_utf8_lossy(&out.stderr).trim()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
Ok(clean(&String::from_utf8_lossy(&out.stdout)))
|
let text = clean(&String::from_utf8_lossy(&out.stdout));
|
||||||
|
if subtitle_credit(&text) {
|
||||||
|
return Ok(String::new());
|
||||||
|
}
|
||||||
|
Ok(text)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Scales a quiet recording up before whisper sees it.
|
/// Scales a quiet recording up before whisper sees it.
|
||||||
@@ -79,6 +88,14 @@ fn has_speech(samples: &[f32]) -> bool {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Multilingual whisper fills silence with the subtitle credits it was trained
|
||||||
|
/// on ("Untertitel im Auftrag des ZDF, 2017", "Untertitel der Amara.org-Community").
|
||||||
|
/// Nobody says these to luna, so the whole transcript is dropped.
|
||||||
|
fn subtitle_credit(text: &str) -> bool {
|
||||||
|
let t = text.to_lowercase();
|
||||||
|
t.contains("untertitel") || t.contains("amara.org")
|
||||||
|
}
|
||||||
|
|
||||||
/// whisper emits bracketed non-speech annotations ([BLANK_AUDIO], (wind
|
/// whisper emits bracketed non-speech annotations ([BLANK_AUDIO], (wind
|
||||||
/// blowing)) that are not words and must not reach the agent.
|
/// blowing)) that are not words and must not reach the agent.
|
||||||
fn clean(raw: &str) -> String {
|
fn clean(raw: &str) -> String {
|
||||||
@@ -97,7 +114,7 @@ fn clean(raw: &str) -> String {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{clean, has_speech, normalize};
|
use super::{clean, has_speech, normalize, subtitle_credit};
|
||||||
use crate::audio::STT_RATE;
|
use crate::audio::STT_RATE;
|
||||||
|
|
||||||
fn peak(v: &[f32]) -> f32 {
|
fn peak(v: &[f32]) -> f32 {
|
||||||
@@ -165,6 +182,13 @@ mod tests {
|
|||||||
assert_eq!(clean("[ music ] real words"), "real words");
|
assert_eq!(clean("[ music ] real words"), "real words");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn drops_subtitle_credits() {
|
||||||
|
assert!(subtitle_credit("Untertitel im Auftrag des ZDF, 2017"));
|
||||||
|
assert!(subtitle_credit("Untertitel der Amara.org-Community"));
|
||||||
|
assert!(!subtitle_credit("Ist das Backup auf jupiter gelaufen?"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn collapses_whitespace() {
|
fn collapses_whitespace() {
|
||||||
assert_eq!(clean(" a b \n c "), "a b c");
|
assert_eq!(clean(" a b \n c "), "a b c");
|
||||||
|
|||||||
@@ -24,9 +24,12 @@ let
|
|||||||
|
|
||||||
# Pinned like the KittenTTS models on mars: a model that re-downloads at
|
# Pinned like the KittenTTS models on mars: a model that re-downloads at
|
||||||
# first use is a service that fails on a fresh install with no network.
|
# first use is a service that fails on a fresh install with no network.
|
||||||
|
# small, multilingual: base.en heard "backup" as "Mac up" and turned German
|
||||||
|
# into English nonsense. ~1s per turn on the 6800 XT vs ~0.4s; turbo was
|
||||||
|
# ~2.3s for no visible gain.
|
||||||
whisperModel = pkgs.fetchurl {
|
whisperModel = pkgs.fetchurl {
|
||||||
url = "https://huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-base.en.bin";
|
url = "https://huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-small.bin";
|
||||||
sha256 = "sha256-oDd5yG3zMjB19eeWyyzlAp8A7Ihp7uP9+4l6/jbG0AI=";
|
sha256 = "sha256-G+OpsgY4Z7k35k4ux0gzZKeZF+FX+pjF2UtcH//qmHs=";
|
||||||
};
|
};
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
@@ -87,6 +90,16 @@ in
|
|||||||
description = "Substring of the input device name; null uses the system default.";
|
description = "Substring of the input device name; null uses the system default.";
|
||||||
};
|
};
|
||||||
|
|
||||||
|
whisperLanguage = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "auto";
|
||||||
|
description = ''
|
||||||
|
whisper's -l. "auto" detects per utterance, which costs a second
|
||||||
|
encoder pass; a fixed code like "en" translates other languages
|
||||||
|
instead of transcribing them.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
vadMult = lib.mkOption {
|
vadMult = lib.mkOption {
|
||||||
type = lib.types.float;
|
type = lib.types.float;
|
||||||
default = 1.8;
|
default = 1.8;
|
||||||
@@ -147,6 +160,7 @@ in
|
|||||||
LUNA_ELEVENLABS_KEY_FILE = toString cfg.elevenLabsKeyFile;
|
LUNA_ELEVENLABS_KEY_FILE = toString cfg.elevenLabsKeyFile;
|
||||||
LUNA_WHISPER_BIN = "${pkgs.whisper-cpp-vulkan}/bin/whisper-cli";
|
LUNA_WHISPER_BIN = "${pkgs.whisper-cpp-vulkan}/bin/whisper-cli";
|
||||||
LUNA_WHISPER_MODEL = whisperModel;
|
LUNA_WHISPER_MODEL = whisperModel;
|
||||||
|
LUNA_WHISPER_LANGUAGE = cfg.whisperLanguage;
|
||||||
LUNA_VAD_MULT = toString cfg.vadMult;
|
LUNA_VAD_MULT = toString cfg.vadMult;
|
||||||
LUNA_VAD_FLOOR = toString cfg.vadFloor;
|
LUNA_VAD_FLOOR = toString cfg.vadFloor;
|
||||||
LUNA_SILENCE_MS = toString cfg.silenceMs;
|
LUNA_SILENCE_MS = toString cfg.silenceMs;
|
||||||
|
|||||||
Reference in New Issue
Block a user