use anyhow::{anyhow, Result}; use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; use std::collections::VecDeque; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::{Arc, Mutex}; /// What whisper wants. Everything captured is resampled to this before it /// reaches disk. pub const STT_RATE: u32 = 16_000; /// What we ask ElevenLabs for (pcm_24000), resampled to the device on the way out. pub const TTS_RATE: u32 = 24_000; /// Mic. The stream runs for the whole session, not just while recording: /// barge-in needs a live RMS while she is speaking, and restarting a cpal /// stream per utterance costs ~100ms of dropped samples on pipewire. /// /// cpal::Stream is !Send, so it is owned by a parked thread rather than by /// this handle — which then holds nothing but atomics and a mutex and can be /// shared across tokio tasks freely. pub struct Capture { pub rms: Arc, recording: Arc, buffer: Arc>>, rate: u32, } impl Capture { pub fn new() -> Result { let rms = Arc::new(AtomicU32::new(0)); let recording = Arc::new(AtomicBool::new(false)); let buffer = Arc::new(Mutex::new(Vec::new())); let (r, rec, buf) = (rms.clone(), recording.clone(), buffer.clone()); let rate = spawn_stream(move || { let device = input_device()?; // Named in the log because "the mic does not work" is usually // "the default source is the wrong device", and nothing else in // the pipeline can tell you that. eprintln!( "luna-voice: input device {:?}", device.name().unwrap_or_else(|_| "".into()) ); let config = device.default_input_config()?; let rate = config.sample_rate().0; let channels = config.channels() as usize; let stream = device.build_input_stream( &config.into(), move |data: &[f32], _: &cpal::InputCallbackInfo| { // Downmix to mono up front; nothing downstream cares about stereo. let mono: Vec = data .chunks(channels) .map(|f| f.iter().sum::() / channels as f32) .collect(); let sum: f32 = mono.iter().map(|s| s * s).sum(); let level = (sum / mono.len().max(1) as f32).sqrt(); r.store(level.to_bits(), Ordering::Relaxed); if rec.load(Ordering::Relaxed) { if let Ok(mut b) = buf.lock() { b.extend_from_slice(&mono); } } }, |e| eprintln!("luna-voice: capture error: {e}"), None, )?; stream.play()?; Ok((stream, rate)) })?; Ok(Capture { rms, recording, buffer, rate }) } pub fn level(&self) -> f32 { f32::from_bits(self.rms.load(Ordering::Relaxed)) } pub fn start(&self) { self.buffer.lock().map(|mut b| b.clear()).ok(); self.recording.store(true, Ordering::Relaxed); } /// Stops recording and returns what was captured, resampled to STT_RATE. pub fn stop(&self) -> Vec { self.recording.store(false, Ordering::Relaxed); let raw = self .buffer .lock() .map(|mut b| std::mem::take(&mut *b)) .unwrap_or_default(); resample(&raw, self.rate, STT_RATE) } } /// Speaker. Fed from the TTS websocket; `clear` is what makes barge-in /// instant rather than "after the current sentence". pub struct Playback { queue: Arc>>, rate: u32, } impl Playback { pub fn new() -> Result { let queue: Arc>> = Arc::new(Mutex::new(VecDeque::new())); let q = queue.clone(); let rate = spawn_stream(move || { let device = cpal::default_host() .default_output_device() .ok_or_else(|| anyhow!("no output device"))?; let config = device.default_output_config()?; let rate = config.sample_rate().0; let channels = config.channels() as usize; let stream = device.build_output_stream( &config.into(), move |data: &mut [f32], _: &cpal::OutputCallbackInfo| { let mut q = match q.lock() { Ok(q) => q, Err(_) => return, }; for frame in data.chunks_mut(channels) { let s = q.pop_front().unwrap_or(0.0); for out in frame.iter_mut() { *out = s; } } }, |e| eprintln!("luna-voice: playback error: {e}"), None, )?; stream.play()?; Ok((stream, rate)) })?; Ok(Playback { queue, rate }) } pub fn push(&self, pcm: &[f32]) { let out = resample(pcm, TTS_RATE, self.rate); if let Ok(mut q) = self.queue.lock() { q.extend(out); } } pub fn clear(&self) { self.queue.lock().map(|mut q| q.clear()).ok(); } pub fn pending(&self) -> usize { self.queue.lock().map(|q| q.len()).unwrap_or(0) } /// Short sine burst, so you know the key registered without looking at /// anything. Hermes' voice mode has `beep_enabled` for the same reason: /// push-to-talk with no acknowledgement is indistinguishable from broken. /// /// Raised-cosine envelope — a bare sine starting at full amplitude clicks. pub fn beep(&self, freq: f32, ms: u32) { let n = (self.rate as f32 * ms as f32 / 1000.0) as usize; let samples: Vec = (0..n) .map(|i| { let t = i as f32 / self.rate as f32; let progress = i as f32 / n as f32; let envelope = (std::f32::consts::PI * progress).sin(); (t * freq * std::f32::consts::TAU).sin() * envelope * 0.15 }) .collect(); if let Ok(mut q) = self.queue.lock() { q.extend(samples); } } } /// The microphone, honouring LUNA_INPUT_DEVICE (a case-insensitive substring /// of the device name) before falling back to the system default. /// /// Worth having because the default source is a system-wide setting that has /// nothing to do with which input you actually speak into: a box with an /// audio interface and onboard audio has several, and picking the silent one /// looks exactly like a broken pipeline. fn input_device() -> Result { let host = cpal::default_host(); if let Ok(want) = std::env::var("LUNA_INPUT_DEVICE") { let want = want.to_lowercase(); if let Ok(devices) = host.input_devices() { for device in devices { if device .name() .map(|n| n.to_lowercase().contains(&want)) .unwrap_or(false) { return Ok(device); } } } eprintln!("luna-voice: no input device matching {want:?}; using the default"); } host.default_input_device() .ok_or_else(|| anyhow!("no input device")) } /// Every input cpal can see, for `luna-voice devices`. pub fn list_inputs() -> Result> { let host = cpal::default_host(); let default = host.default_input_device().and_then(|d| d.name().ok()); let mut out = Vec::new(); for device in host.input_devices()? { let name = device.name().unwrap_or_else(|_| "".into()); let marker = if Some(&name) == default.as_ref() { " (default)" } else { "" }; out.push(format!("{name}{marker}")); } Ok(out) } /// Builds a cpal stream on a dedicated thread that then parks forever, /// keeping it alive. Returns the device's sample rate, or the build error — /// a stream that fails here must surface at startup, not as silence later. fn spawn_stream(build: F) -> Result where F: FnOnce() -> Result<(cpal::Stream, u32)> + Send + 'static, { let (tx, rx) = std::sync::mpsc::channel::>(); std::thread::spawn(move || match build() { Ok((_stream, rate)) => { let _ = tx.send(Ok(rate)); // Dropping `stream` would stop the device, so the thread holds it // for the life of the process. loop { std::thread::park(); } #[allow(unreachable_code)] drop(_stream); } Err(e) => { let _ = tx.send(Err(e.to_string())); } }); rx.recv() .map_err(|_| anyhow!("audio thread died during setup"))? .map_err(|e| anyhow!(e)) } /// Linear resampling. Speech at these rates does not justify a windowed sinc, /// and rubato would add a dependency for inaudible gain. pub fn resample(input: &[f32], from: u32, to: u32) -> Vec { if from == to || input.is_empty() { return input.to_vec(); } let ratio = to as f64 / from as f64; let out_len = (input.len() as f64 * ratio).round() as usize; (0..out_len) .map(|i| { let src = i as f64 / ratio; let j = src.floor() as usize; let frac = (src - j as f64) as f32; let a = input.get(j).copied().unwrap_or(0.0); let b = input.get(j + 1).copied().unwrap_or(a); a + (b - a) * frac }) .collect() } /// 16-bit mono WAV, which is the only thing whisper-cli accepts without /// converting first. pub fn write_wav(path: &std::path::Path, samples: &[f32], rate: u32) -> Result<()> { let spec = hound::WavSpec { channels: 1, sample_rate: rate, bits_per_sample: 16, sample_format: hound::SampleFormat::Int, }; let mut w = hound::WavWriter::create(path, spec)?; for s in samples { w.write_sample((s.clamp(-1.0, 1.0) * i16::MAX as f32) as i16)?; } w.finalize()?; Ok(()) } /// Signed 16-bit little-endian, as ElevenLabs sends pcm_*. pub fn pcm16_to_f32(bytes: &[u8]) -> Vec { bytes .chunks_exact(2) .map(|b| i16::from_le_bytes([b[0], b[1]]) as f32 / i16::MAX as f32) .collect() } #[cfg(test)] mod tests { use super::*; #[test] fn resample_keeps_length_ratio() { let input = vec![0.0f32; 1000]; assert_eq!(resample(&input, 48_000, 16_000).len(), 333); assert_eq!(resample(&input, 24_000, 48_000).len(), 2000); } #[test] fn resample_is_identity_at_equal_rates() { let input = vec![0.1, -0.4, 0.9]; assert_eq!(resample(&input, 16_000, 16_000), input); } #[test] fn pcm16_roundtrips_through_f32() { let bytes = [0x00, 0x00, 0xff, 0x7f, 0x01, 0x80]; let got = pcm16_to_f32(&bytes); assert_eq!(got.len(), 3); assert!((got[0]).abs() < 1e-6); assert!((got[1] - 1.0).abs() < 1e-3); assert!((got[2] + 1.0).abs() < 1e-3); } }