Files
homelab/pkgs/luna-voice/src/audio.rs
T
darmanandClaude Opus 5 67dd30eb72 feat(terra): hands-free voice mode, toggled with SUPER+V
The key is a toggle rather than push-to-talk, because "voice mode is
active" has to be a state something can display; a held key cannot
sustain one. So the daemon grew a VAD loop: speech starts a capture,
`silenceMs` of quiet ends it. A turn is awaited inline, which is also
what stops her hearing herself — the tick cannot fire while she speaks.

The keybind runs the CLI, not a quickshell GlobalShortcut, so the mic can
still be switched off while the shell is reloading and a QML error can
never leave it open with nothing on screen saying so.

Every threshold had to be resized for terra's mic, which is QUIET:
0.001-0.002 ambient against 0.0045 speaking, about 6dB apart. The VAD
floor of 0.006 and whisper's own 0.015 gate both sat ABOVE the user's
voice, and a 3.5x multiplier cannot separate signals that close — three
independent reasons nothing could ever trigger. Hence vadMult 1.8,
vadFloor 0.0025, and stt::normalize lifting the clip (capped at 30x, so
near-silence is not amplified into words) before whisper sees it.

The floor tracker also had to stop feeding on everything: a rising sound
dragged its own threshold up faster than it rose, and a tone peaking at
0.029 was outrun by a threshold that reached 0.018. Only samples below
the current trigger update it now.

Four diagnostics, because none of the above was visible while it was
failing: `watch` prints the socket protocol, `say` runs a turn from typed
text (a broken mic vs a broken pipeline), `level` is a live meter with
the trigger marked, `devices` lists the inputs. Amplitude is now emitted
on every tick, not only while she speaks — a meter that moves only during
playback cannot answer "is it hearing me".

Also: connect-time replay of mode/state, so restarting the shell while
voice mode is on does not leave it blind until the next event, which in
an idle conversation is never.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-20 23:37:51 +02:00

312 lines
11 KiB
Rust

use anyhow::{anyhow, Result};
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
use std::collections::VecDeque;
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use std::sync::{Arc, Mutex};
/// What whisper wants. Everything captured is resampled to this before it
/// reaches disk.
pub const STT_RATE: u32 = 16_000;
/// What we ask ElevenLabs for (pcm_24000), resampled to the device on the way out.
pub const TTS_RATE: u32 = 24_000;
/// Mic. The stream runs for the whole session, not just while recording:
/// barge-in needs a live RMS while she is speaking, and restarting a cpal
/// stream per utterance costs ~100ms of dropped samples on pipewire.
///
/// cpal::Stream is !Send, so it is owned by a parked thread rather than by
/// this handle — which then holds nothing but atomics and a mutex and can be
/// shared across tokio tasks freely.
pub struct Capture {
pub rms: Arc<AtomicU32>,
recording: Arc<AtomicBool>,
buffer: Arc<Mutex<Vec<f32>>>,
rate: u32,
}
impl Capture {
pub fn new() -> Result<Self> {
let rms = Arc::new(AtomicU32::new(0));
let recording = Arc::new(AtomicBool::new(false));
let buffer = Arc::new(Mutex::new(Vec::new()));
let (r, rec, buf) = (rms.clone(), recording.clone(), buffer.clone());
let rate = spawn_stream(move || {
let device = input_device()?;
// Named in the log because "the mic does not work" is usually
// "the default source is the wrong device", and nothing else in
// the pipeline can tell you that.
eprintln!(
"luna-voice: input device {:?}",
device.name().unwrap_or_else(|_| "<unnamed>".into())
);
let config = device.default_input_config()?;
let rate = config.sample_rate().0;
let channels = config.channels() as usize;
let stream = device.build_input_stream(
&config.into(),
move |data: &[f32], _: &cpal::InputCallbackInfo| {
// Downmix to mono up front; nothing downstream cares about stereo.
let mono: Vec<f32> = data
.chunks(channels)
.map(|f| f.iter().sum::<f32>() / channels as f32)
.collect();
let sum: f32 = mono.iter().map(|s| s * s).sum();
let level = (sum / mono.len().max(1) as f32).sqrt();
r.store(level.to_bits(), Ordering::Relaxed);
if rec.load(Ordering::Relaxed) {
if let Ok(mut b) = buf.lock() {
b.extend_from_slice(&mono);
}
}
},
|e| eprintln!("luna-voice: capture error: {e}"),
None,
)?;
stream.play()?;
Ok((stream, rate))
})?;
Ok(Capture { rms, recording, buffer, rate })
}
pub fn level(&self) -> f32 {
f32::from_bits(self.rms.load(Ordering::Relaxed))
}
pub fn start(&self) {
self.buffer.lock().map(|mut b| b.clear()).ok();
self.recording.store(true, Ordering::Relaxed);
}
/// Stops recording and returns what was captured, resampled to STT_RATE.
pub fn stop(&self) -> Vec<f32> {
self.recording.store(false, Ordering::Relaxed);
let raw = self
.buffer
.lock()
.map(|mut b| std::mem::take(&mut *b))
.unwrap_or_default();
resample(&raw, self.rate, STT_RATE)
}
}
/// Speaker. Fed from the TTS websocket; `clear` is what makes barge-in
/// instant rather than "after the current sentence".
pub struct Playback {
queue: Arc<Mutex<VecDeque<f32>>>,
rate: u32,
}
impl Playback {
pub fn new() -> Result<Self> {
let queue: Arc<Mutex<VecDeque<f32>>> = Arc::new(Mutex::new(VecDeque::new()));
let q = queue.clone();
let rate = spawn_stream(move || {
let device = cpal::default_host()
.default_output_device()
.ok_or_else(|| anyhow!("no output device"))?;
let config = device.default_output_config()?;
let rate = config.sample_rate().0;
let channels = config.channels() as usize;
let stream = device.build_output_stream(
&config.into(),
move |data: &mut [f32], _: &cpal::OutputCallbackInfo| {
let mut q = match q.lock() {
Ok(q) => q,
Err(_) => return,
};
for frame in data.chunks_mut(channels) {
let s = q.pop_front().unwrap_or(0.0);
for out in frame.iter_mut() {
*out = s;
}
}
},
|e| eprintln!("luna-voice: playback error: {e}"),
None,
)?;
stream.play()?;
Ok((stream, rate))
})?;
Ok(Playback { queue, rate })
}
pub fn push(&self, pcm: &[f32]) {
let out = resample(pcm, TTS_RATE, self.rate);
if let Ok(mut q) = self.queue.lock() {
q.extend(out);
}
}
pub fn clear(&self) {
self.queue.lock().map(|mut q| q.clear()).ok();
}
pub fn pending(&self) -> usize {
self.queue.lock().map(|q| q.len()).unwrap_or(0)
}
/// Short sine burst, so you know the key registered without looking at
/// anything. Hermes' voice mode has `beep_enabled` for the same reason:
/// push-to-talk with no acknowledgement is indistinguishable from broken.
///
/// Raised-cosine envelope — a bare sine starting at full amplitude clicks.
pub fn beep(&self, freq: f32, ms: u32) {
let n = (self.rate as f32 * ms as f32 / 1000.0) as usize;
let samples: Vec<f32> = (0..n)
.map(|i| {
let t = i as f32 / self.rate as f32;
let progress = i as f32 / n as f32;
let envelope = (std::f32::consts::PI * progress).sin();
(t * freq * std::f32::consts::TAU).sin() * envelope * 0.15
})
.collect();
if let Ok(mut q) = self.queue.lock() {
q.extend(samples);
}
}
}
/// The microphone, honouring LUNA_INPUT_DEVICE (a case-insensitive substring
/// of the device name) before falling back to the system default.
///
/// Worth having because the default source is a system-wide setting that has
/// nothing to do with which input you actually speak into: a box with an
/// audio interface and onboard audio has several, and picking the silent one
/// looks exactly like a broken pipeline.
fn input_device() -> Result<cpal::Device> {
let host = cpal::default_host();
if let Ok(want) = std::env::var("LUNA_INPUT_DEVICE") {
let want = want.to_lowercase();
if let Ok(devices) = host.input_devices() {
for device in devices {
if device
.name()
.map(|n| n.to_lowercase().contains(&want))
.unwrap_or(false)
{
return Ok(device);
}
}
}
eprintln!("luna-voice: no input device matching {want:?}; using the default");
}
host.default_input_device()
.ok_or_else(|| anyhow!("no input device"))
}
/// Every input cpal can see, for `luna-voice devices`.
pub fn list_inputs() -> Result<Vec<String>> {
let host = cpal::default_host();
let default = host.default_input_device().and_then(|d| d.name().ok());
let mut out = Vec::new();
for device in host.input_devices()? {
let name = device.name().unwrap_or_else(|_| "<unnamed>".into());
let marker = if Some(&name) == default.as_ref() { " (default)" } else { "" };
out.push(format!("{name}{marker}"));
}
Ok(out)
}
/// Builds a cpal stream on a dedicated thread that then parks forever,
/// keeping it alive. Returns the device's sample rate, or the build error —
/// a stream that fails here must surface at startup, not as silence later.
fn spawn_stream<F>(build: F) -> Result<u32>
where
F: FnOnce() -> Result<(cpal::Stream, u32)> + Send + 'static,
{
let (tx, rx) = std::sync::mpsc::channel::<Result<u32, String>>();
std::thread::spawn(move || match build() {
Ok((_stream, rate)) => {
let _ = tx.send(Ok(rate));
// Dropping `stream` would stop the device, so the thread holds it
// for the life of the process.
loop {
std::thread::park();
}
#[allow(unreachable_code)]
drop(_stream);
}
Err(e) => {
let _ = tx.send(Err(e.to_string()));
}
});
rx.recv()
.map_err(|_| anyhow!("audio thread died during setup"))?
.map_err(|e| anyhow!(e))
}
/// Linear resampling. Speech at these rates does not justify a windowed sinc,
/// and rubato would add a dependency for inaudible gain.
pub fn resample(input: &[f32], from: u32, to: u32) -> Vec<f32> {
if from == to || input.is_empty() {
return input.to_vec();
}
let ratio = to as f64 / from as f64;
let out_len = (input.len() as f64 * ratio).round() as usize;
(0..out_len)
.map(|i| {
let src = i as f64 / ratio;
let j = src.floor() as usize;
let frac = (src - j as f64) as f32;
let a = input.get(j).copied().unwrap_or(0.0);
let b = input.get(j + 1).copied().unwrap_or(a);
a + (b - a) * frac
})
.collect()
}
/// 16-bit mono WAV, which is the only thing whisper-cli accepts without
/// converting first.
pub fn write_wav(path: &std::path::Path, samples: &[f32], rate: u32) -> Result<()> {
let spec = hound::WavSpec {
channels: 1,
sample_rate: rate,
bits_per_sample: 16,
sample_format: hound::SampleFormat::Int,
};
let mut w = hound::WavWriter::create(path, spec)?;
for s in samples {
w.write_sample((s.clamp(-1.0, 1.0) * i16::MAX as f32) as i16)?;
}
w.finalize()?;
Ok(())
}
/// Signed 16-bit little-endian, as ElevenLabs sends pcm_*.
pub fn pcm16_to_f32(bytes: &[u8]) -> Vec<f32> {
bytes
.chunks_exact(2)
.map(|b| i16::from_le_bytes([b[0], b[1]]) as f32 / i16::MAX as f32)
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn resample_keeps_length_ratio() {
let input = vec![0.0f32; 1000];
assert_eq!(resample(&input, 48_000, 16_000).len(), 333);
assert_eq!(resample(&input, 24_000, 48_000).len(), 2000);
}
#[test]
fn resample_is_identity_at_equal_rates() {
let input = vec![0.1, -0.4, 0.9];
assert_eq!(resample(&input, 16_000, 16_000), input);
}
#[test]
fn pcm16_roundtrips_through_f32() {
let bytes = [0x00, 0x00, 0xff, 0x7f, 0x01, 0x80];
let got = pcm16_to_f32(&bytes);
assert_eq!(got.len(), 3);
assert!((got[0]).abs() < 1e-6);
assert!((got[1] - 1.0).abs() < 1e-3);
assert!((got[2] + 1.0).abs() < 1e-3);
}
}