Files
homelab/pkgs/luna-voice/src/stt.rs
T
darmanandClaude Opus 5.5 4b366f952e feat(terra): multilingual whisper small for luna's voice
base.en heard "backup" as "Mac up" and turned German into English nonsense.
small with -l auto gets both right at ~1s per turn (vs ~0.4s); turbo was
~2.3s for no visible gain. Drops the subtitle credits multilingual whisper
invents out of silence.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011XkDJkdDzneVVdnkZfrvTY
2026-10-03 01:48:29 +02:00

206 lines
7.1 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
use crate::audio;
use anyhow::{bail, Result};
use std::path::Path;
use tokio::process::Command;
/// Shells out to whisper-cli rather than linking whisper.cpp: nixpkgs already
/// ships whisper-cpp-vulkan built against this box's GPU stack, and a bundled
/// build would have to rediscover that at cargo time.
pub async fn transcribe(
bin: &str,
model: &Path,
language: &str,
samples: &[f32],
) -> Result<String> {
// Whisper hallucinates confident words into silence — "you", "Thank
// you.", "Bye." — which then reach luna as if spoken. Duration alone does
// not catch it (a held key over a quiet room is long AND empty), so gate
// on level too.
if samples.len() < (audio::STT_RATE as usize) / 3 || !has_speech(samples) {
return Ok(String::new());
}
let dir = std::env::var("XDG_RUNTIME_DIR").unwrap_or_else(|_| "/tmp".into());
let wav = Path::new(&dir).join(format!("luna-stt-{}.wav", std::process::id()));
audio::write_wav(&wav, &normalize(samples), audio::STT_RATE)?;
let out = Command::new(bin)
.args(["-m"])
.arg(model)
.args(["-f"])
.arg(&wav)
// -nt: transcript only, no [00:00:00] prefixes. -np: no progress
// banner. Together they make stdout exactly the text.
.args(["-nt", "-np", "-l", language])
.output()
.await?;
let _ = std::fs::remove_file(&wav);
if !out.status.success() {
bail!(
"whisper-cli failed: {}",
String::from_utf8_lossy(&out.stderr).trim()
);
}
let text = clean(&String::from_utf8_lossy(&out.stdout));
if subtitle_credit(&text) {
return Ok(String::new());
}
Ok(text)
}
/// Scales a quiet recording up before whisper sees it.
///
/// Whisper is trained on normalised audio and degrades badly on very quiet
/// input, which is precisely what a low-gain interface produces — speech at a
/// 0.03 peak is intelligible to a person and mush to the model. The gain is
/// capped so a recording of near-silence is not amplified into hiss the model
/// will confidently transcribe as words.
fn normalize(samples: &[f32]) -> Vec<f32> {
const TARGET_PEAK: f32 = 0.6;
const MAX_GAIN: f32 = 30.0;
let peak = samples.iter().fold(0.0f32, |m, s| m.max(s.abs()));
if peak <= f32::EPSILON {
return samples.to_vec();
}
let gain = (TARGET_PEAK / peak).min(MAX_GAIN);
// Never attenuate: a loud recording is already fine, and turning it down
// would only lose resolution in the 16-bit conversion.
if gain <= 1.0 {
return samples.to_vec();
}
samples.iter().map(|s| (s * gain).clamp(-1.0, 1.0)).collect()
}
/// True if any 100ms window is loud enough to be a voice. Peak rather than
/// overall RMS: a short sentence inside a long silent recording averages down
/// to nothing, and that is the common case for a held key.
fn has_speech(samples: &[f32]) -> bool {
// Matches the VAD's lower clamp rather than a "typical" mic: a quiet
// interface puts normal speech at ~0.004, and gating at 0.015 threw away
// every word it captured.
const FLOOR: f32 = 0.0025;
let window = (audio::STT_RATE as usize) / 10;
samples.chunks(window).any(|w| {
let sum: f32 = w.iter().map(|s| s * s).sum();
(sum / w.len().max(1) as f32).sqrt() > FLOOR
})
}
/// Multilingual whisper fills silence with the subtitle credits it was trained
/// on ("Untertitel im Auftrag des ZDF, 2017", "Untertitel der Amara.org-Community").
/// Nobody says these to luna, so the whole transcript is dropped.
fn subtitle_credit(text: &str) -> bool {
let t = text.to_lowercase();
t.contains("untertitel") || t.contains("amara.org")
}
/// whisper emits bracketed non-speech annotations ([BLANK_AUDIO], (wind
/// blowing)) that are not words and must not reach the agent.
fn clean(raw: &str) -> String {
let mut out = String::new();
let mut depth = 0i32;
for c in raw.chars() {
match c {
'[' | '(' => depth += 1,
']' | ')' => depth = (depth - 1).max(0),
c if depth == 0 => out.push(c),
_ => {}
}
}
out.split_whitespace().collect::<Vec<_>>().join(" ")
}
#[cfg(test)]
mod tests {
use super::{clean, has_speech, normalize, subtitle_credit};
use crate::audio::STT_RATE;
fn peak(v: &[f32]) -> f32 {
v.iter().fold(0.0f32, |m, s| m.max(s.abs()))
}
#[test]
fn normalize_lifts_a_quiet_recording() {
// 0.004 RMS speech peaks around 0.02, which is 30x under target —
// exactly at the gain cap, so it just reaches full scale.
let quiet: Vec<f32> = (0..1000).map(|i| 0.02 * ((i % 7) as f32 - 3.0) / 3.0).collect();
assert!((peak(&normalize(&quiet)) - 0.6).abs() < 0.01);
}
#[test]
fn normalize_leaves_a_loud_recording_alone() {
let loud: Vec<f32> = vec![0.9, -0.8, 0.7];
assert_eq!(normalize(&loud), loud);
}
#[test]
fn normalize_caps_the_gain_so_silence_stays_silent() {
let hiss: Vec<f32> = vec![0.0001, -0.0001];
// 0.6/0.0001 would be 6000x; the cap keeps it at 30x.
assert!(peak(&normalize(&hiss)) <= 0.0001 * 30.0 + 1e-9);
}
#[test]
fn normalize_handles_pure_silence() {
assert_eq!(normalize(&[0.0, 0.0]), vec![0.0, 0.0]);
}
#[test]
fn silence_is_not_speech() {
assert!(!has_speech(&vec![0.0f32; STT_RATE as usize]));
// Terra's measured room tone. Speech on the same mic reaches
// 0.004–0.005, so the gate sits between the two.
assert!(!has_speech(&vec![0.001f32; STT_RATE as usize]));
}
#[test]
fn quiet_speech_still_counts() {
// 0.004 is what normal speech measures on terra's interface; gating
// it out discarded every word the VAD had just captured.
let mut s = vec![0.001f32; STT_RATE as usize];
for v in s.iter_mut().take(STT_RATE as usize / 5) {
*v = 0.004;
}
assert!(has_speech(&s));
}
#[test]
fn a_short_utterance_in_a_long_silence_counts() {
let mut s = vec![0.0f32; STT_RATE as usize];
for v in s.iter_mut().take(STT_RATE as usize / 5) {
*v = 0.2;
}
assert!(has_speech(&s));
}
#[test]
fn strips_bracketed_annotations() {
assert_eq!(clean("[BLANK_AUDIO]"), "");
assert_eq!(clean(" hello (wind blowing) there \n"), "hello there");
assert_eq!(clean("[ music ] real words"), "real words");
}
#[test]
fn drops_subtitle_credits() {
assert!(subtitle_credit("Untertitel im Auftrag des ZDF, 2017"));
assert!(subtitle_credit("Untertitel der Amara.org-Community"));
assert!(!subtitle_credit("Ist das Backup auf jupiter gelaufen?"));
}
#[test]
fn collapses_whitespace() {
assert_eq!(clean(" a b \n c "), "a b c");
}
#[test]
fn unbalanced_brackets_do_not_eat_the_rest() {
// The stray closer is dropped (depth floors at 0) but the words
// around it survive — the failure mode that matters is a lone
// bracket swallowing the whole transcript.
assert_eq!(clean("ok] fine"), "ok fine");
assert_eq!(clean("[ oops real words"), "");
}
}