base.en heard "backup" as "Mac up" and turned German into English nonsense. small with -l auto gets both right at ~1s per turn (vs ~0.4s); turbo was ~2.3s for no visible gain. Drops the subtitle credits multilingual whisper invents out of silence. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011XkDJkdDzneVVdnkZfrvTY
206 lines
7.1 KiB
Rust
206 lines
7.1 KiB
Rust
use crate::audio;
|
||
use anyhow::{bail, Result};
|
||
use std::path::Path;
|
||
use tokio::process::Command;
|
||
|
||
/// Shells out to whisper-cli rather than linking whisper.cpp: nixpkgs already
|
||
/// ships whisper-cpp-vulkan built against this box's GPU stack, and a bundled
|
||
/// build would have to rediscover that at cargo time.
|
||
pub async fn transcribe(
|
||
bin: &str,
|
||
model: &Path,
|
||
language: &str,
|
||
samples: &[f32],
|
||
) -> Result<String> {
|
||
// Whisper hallucinates confident words into silence — "you", "Thank
|
||
// you.", "Bye." — which then reach luna as if spoken. Duration alone does
|
||
// not catch it (a held key over a quiet room is long AND empty), so gate
|
||
// on level too.
|
||
if samples.len() < (audio::STT_RATE as usize) / 3 || !has_speech(samples) {
|
||
return Ok(String::new());
|
||
}
|
||
|
||
let dir = std::env::var("XDG_RUNTIME_DIR").unwrap_or_else(|_| "/tmp".into());
|
||
let wav = Path::new(&dir).join(format!("luna-stt-{}.wav", std::process::id()));
|
||
audio::write_wav(&wav, &normalize(samples), audio::STT_RATE)?;
|
||
|
||
let out = Command::new(bin)
|
||
.args(["-m"])
|
||
.arg(model)
|
||
.args(["-f"])
|
||
.arg(&wav)
|
||
// -nt: transcript only, no [00:00:00] prefixes. -np: no progress
|
||
// banner. Together they make stdout exactly the text.
|
||
.args(["-nt", "-np", "-l", language])
|
||
.output()
|
||
.await?;
|
||
let _ = std::fs::remove_file(&wav);
|
||
|
||
if !out.status.success() {
|
||
bail!(
|
||
"whisper-cli failed: {}",
|
||
String::from_utf8_lossy(&out.stderr).trim()
|
||
);
|
||
}
|
||
let text = clean(&String::from_utf8_lossy(&out.stdout));
|
||
if subtitle_credit(&text) {
|
||
return Ok(String::new());
|
||
}
|
||
Ok(text)
|
||
}
|
||
|
||
/// Scales a quiet recording up before whisper sees it.
|
||
///
|
||
/// Whisper is trained on normalised audio and degrades badly on very quiet
|
||
/// input, which is precisely what a low-gain interface produces — speech at a
|
||
/// 0.03 peak is intelligible to a person and mush to the model. The gain is
|
||
/// capped so a recording of near-silence is not amplified into hiss the model
|
||
/// will confidently transcribe as words.
|
||
fn normalize(samples: &[f32]) -> Vec<f32> {
|
||
const TARGET_PEAK: f32 = 0.6;
|
||
const MAX_GAIN: f32 = 30.0;
|
||
|
||
let peak = samples.iter().fold(0.0f32, |m, s| m.max(s.abs()));
|
||
if peak <= f32::EPSILON {
|
||
return samples.to_vec();
|
||
}
|
||
let gain = (TARGET_PEAK / peak).min(MAX_GAIN);
|
||
// Never attenuate: a loud recording is already fine, and turning it down
|
||
// would only lose resolution in the 16-bit conversion.
|
||
if gain <= 1.0 {
|
||
return samples.to_vec();
|
||
}
|
||
samples.iter().map(|s| (s * gain).clamp(-1.0, 1.0)).collect()
|
||
}
|
||
|
||
/// True if any 100ms window is loud enough to be a voice. Peak rather than
|
||
/// overall RMS: a short sentence inside a long silent recording averages down
|
||
/// to nothing, and that is the common case for a held key.
|
||
fn has_speech(samples: &[f32]) -> bool {
|
||
// Matches the VAD's lower clamp rather than a "typical" mic: a quiet
|
||
// interface puts normal speech at ~0.004, and gating at 0.015 threw away
|
||
// every word it captured.
|
||
const FLOOR: f32 = 0.0025;
|
||
let window = (audio::STT_RATE as usize) / 10;
|
||
samples.chunks(window).any(|w| {
|
||
let sum: f32 = w.iter().map(|s| s * s).sum();
|
||
(sum / w.len().max(1) as f32).sqrt() > FLOOR
|
||
})
|
||
}
|
||
|
||
/// Multilingual whisper fills silence with the subtitle credits it was trained
|
||
/// on ("Untertitel im Auftrag des ZDF, 2017", "Untertitel der Amara.org-Community").
|
||
/// Nobody says these to luna, so the whole transcript is dropped.
|
||
fn subtitle_credit(text: &str) -> bool {
|
||
let t = text.to_lowercase();
|
||
t.contains("untertitel") || t.contains("amara.org")
|
||
}
|
||
|
||
/// whisper emits bracketed non-speech annotations ([BLANK_AUDIO], (wind
|
||
/// blowing)) that are not words and must not reach the agent.
|
||
fn clean(raw: &str) -> String {
|
||
let mut out = String::new();
|
||
let mut depth = 0i32;
|
||
for c in raw.chars() {
|
||
match c {
|
||
'[' | '(' => depth += 1,
|
||
']' | ')' => depth = (depth - 1).max(0),
|
||
c if depth == 0 => out.push(c),
|
||
_ => {}
|
||
}
|
||
}
|
||
out.split_whitespace().collect::<Vec<_>>().join(" ")
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::{clean, has_speech, normalize, subtitle_credit};
|
||
use crate::audio::STT_RATE;
|
||
|
||
fn peak(v: &[f32]) -> f32 {
|
||
v.iter().fold(0.0f32, |m, s| m.max(s.abs()))
|
||
}
|
||
|
||
#[test]
|
||
fn normalize_lifts_a_quiet_recording() {
|
||
// 0.004 RMS speech peaks around 0.02, which is 30x under target —
|
||
// exactly at the gain cap, so it just reaches full scale.
|
||
let quiet: Vec<f32> = (0..1000).map(|i| 0.02 * ((i % 7) as f32 - 3.0) / 3.0).collect();
|
||
assert!((peak(&normalize(&quiet)) - 0.6).abs() < 0.01);
|
||
}
|
||
|
||
#[test]
|
||
fn normalize_leaves_a_loud_recording_alone() {
|
||
let loud: Vec<f32> = vec![0.9, -0.8, 0.7];
|
||
assert_eq!(normalize(&loud), loud);
|
||
}
|
||
|
||
#[test]
|
||
fn normalize_caps_the_gain_so_silence_stays_silent() {
|
||
let hiss: Vec<f32> = vec![0.0001, -0.0001];
|
||
// 0.6/0.0001 would be 6000x; the cap keeps it at 30x.
|
||
assert!(peak(&normalize(&hiss)) <= 0.0001 * 30.0 + 1e-9);
|
||
}
|
||
|
||
#[test]
|
||
fn normalize_handles_pure_silence() {
|
||
assert_eq!(normalize(&[0.0, 0.0]), vec![0.0, 0.0]);
|
||
}
|
||
|
||
#[test]
|
||
fn silence_is_not_speech() {
|
||
assert!(!has_speech(&vec![0.0f32; STT_RATE as usize]));
|
||
// Terra's measured room tone. Speech on the same mic reaches
|
||
// 0.004–0.005, so the gate sits between the two.
|
||
assert!(!has_speech(&vec![0.001f32; STT_RATE as usize]));
|
||
}
|
||
|
||
#[test]
|
||
fn quiet_speech_still_counts() {
|
||
// 0.004 is what normal speech measures on terra's interface; gating
|
||
// it out discarded every word the VAD had just captured.
|
||
let mut s = vec![0.001f32; STT_RATE as usize];
|
||
for v in s.iter_mut().take(STT_RATE as usize / 5) {
|
||
*v = 0.004;
|
||
}
|
||
assert!(has_speech(&s));
|
||
}
|
||
|
||
#[test]
|
||
fn a_short_utterance_in_a_long_silence_counts() {
|
||
let mut s = vec![0.0f32; STT_RATE as usize];
|
||
for v in s.iter_mut().take(STT_RATE as usize / 5) {
|
||
*v = 0.2;
|
||
}
|
||
assert!(has_speech(&s));
|
||
}
|
||
|
||
#[test]
|
||
fn strips_bracketed_annotations() {
|
||
assert_eq!(clean("[BLANK_AUDIO]"), "");
|
||
assert_eq!(clean(" hello (wind blowing) there \n"), "hello there");
|
||
assert_eq!(clean("[ music ] real words"), "real words");
|
||
}
|
||
|
||
#[test]
|
||
fn drops_subtitle_credits() {
|
||
assert!(subtitle_credit("Untertitel im Auftrag des ZDF, 2017"));
|
||
assert!(subtitle_credit("Untertitel der Amara.org-Community"));
|
||
assert!(!subtitle_credit("Ist das Backup auf jupiter gelaufen?"));
|
||
}
|
||
|
||
#[test]
|
||
fn collapses_whitespace() {
|
||
assert_eq!(clean(" a b \n c "), "a b c");
|
||
}
|
||
|
||
#[test]
|
||
fn unbalanced_brackets_do_not_eat_the_rest() {
|
||
// The stray closer is dropped (depth floors at 0) but the words
|
||
// around it survive — the failure mode that matters is a lone
|
||
// bracket swallowing the whole transcript.
|
||
assert_eq!(clean("ok] fine"), "ok fine");
|
||
assert_eq!(clean("[ oops real words"), "");
|
||
}
|
||
}
|