feat(terra): talk to luna with push-to-talk
luna-voiced: mic -> whisper-cpp (local, vulkan) -> Hermes' API on mars ->
ElevenLabs streaming TTS -> speaker, bound to SUPER+V. State, amplitude
and a viseme timeline are published on a unix socket; nothing consumes
that yet, but it is the data source the quickshell avatar will read.
STT stays local on purpose. mars already runs faster-whisper for Telegram
voice notes, but routing the mic through it would put every spoken word
on the network for no latency win.
Shape notes, all of them things the first live runs decided:
- Viseme offsets are relative to speech_start (emitted when the first
PCM frame reaches the speaker), never wall clock, so the shell and
the daemon never have to compare clocks. ElevenLabs sends character
alignment, not visemes, so the mapping is ours; the glTF morph target
names are the contract.
- clean_for_speech is a whitelist. She writes "(¬‿¬)", whose characters
live in Latin-1 and punctuation blocks rather than any emoji range,
so enumerating what to drop never converges. Em dashes become commas
because they are a pause — dropping one ran two clauses together and
changed what a sentence meant.
- whisper invents words out of silence ("you", "Thank you."), and a
held key over a quiet room is long AND empty, so has_speech gates on
the loudest 100ms window rather than on duration.
- Beeps on key down/up and a distinct low tone for "heard nothing":
push-to-talk with no acknowledgement is indistinguishable from broken,
and a dead mic must not feel like a slow reply.
- A warmup request at startup: her first reply after the container
starts costs ~16s against ~5s warm, and the first one is always the
one you are waiting for.
- `luna-voice watch` prints the socket protocol and `luna-voice say`
runs a turn from typed text, which is how a broken mic is told apart
from a broken pipeline.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,265 @@
|
||||
use anyhow::{anyhow, Result};
|
||||
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
/// What whisper wants. Everything captured is resampled to this before it
|
||||
/// reaches disk.
|
||||
pub const STT_RATE: u32 = 16_000;
|
||||
/// What we ask ElevenLabs for (pcm_24000), resampled to the device on the way out.
|
||||
pub const TTS_RATE: u32 = 24_000;
|
||||
|
||||
/// Mic. The stream runs for the whole session, not just while recording:
|
||||
/// barge-in needs a live RMS while she is speaking, and restarting a cpal
|
||||
/// stream per utterance costs ~100ms of dropped samples on pipewire.
|
||||
///
|
||||
/// cpal::Stream is !Send, so it is owned by a parked thread rather than by
|
||||
/// this handle — which then holds nothing but atomics and a mutex and can be
|
||||
/// shared across tokio tasks freely.
|
||||
pub struct Capture {
|
||||
pub rms: Arc<AtomicU32>,
|
||||
recording: Arc<AtomicBool>,
|
||||
buffer: Arc<Mutex<Vec<f32>>>,
|
||||
rate: u32,
|
||||
}
|
||||
|
||||
impl Capture {
|
||||
pub fn new() -> Result<Self> {
|
||||
let rms = Arc::new(AtomicU32::new(0));
|
||||
let recording = Arc::new(AtomicBool::new(false));
|
||||
let buffer = Arc::new(Mutex::new(Vec::new()));
|
||||
|
||||
let (r, rec, buf) = (rms.clone(), recording.clone(), buffer.clone());
|
||||
let rate = spawn_stream(move || {
|
||||
let device = cpal::default_host()
|
||||
.default_input_device()
|
||||
.ok_or_else(|| anyhow!("no input device"))?;
|
||||
let config = device.default_input_config()?;
|
||||
let rate = config.sample_rate().0;
|
||||
let channels = config.channels() as usize;
|
||||
let stream = device.build_input_stream(
|
||||
&config.into(),
|
||||
move |data: &[f32], _: &cpal::InputCallbackInfo| {
|
||||
// Downmix to mono up front; nothing downstream cares about stereo.
|
||||
let mono: Vec<f32> = data
|
||||
.chunks(channels)
|
||||
.map(|f| f.iter().sum::<f32>() / channels as f32)
|
||||
.collect();
|
||||
let sum: f32 = mono.iter().map(|s| s * s).sum();
|
||||
let level = (sum / mono.len().max(1) as f32).sqrt();
|
||||
r.store(level.to_bits(), Ordering::Relaxed);
|
||||
if rec.load(Ordering::Relaxed) {
|
||||
if let Ok(mut b) = buf.lock() {
|
||||
b.extend_from_slice(&mono);
|
||||
}
|
||||
}
|
||||
},
|
||||
|e| eprintln!("luna-voice: capture error: {e}"),
|
||||
None,
|
||||
)?;
|
||||
stream.play()?;
|
||||
Ok((stream, rate))
|
||||
})?;
|
||||
|
||||
Ok(Capture { rms, recording, buffer, rate })
|
||||
}
|
||||
|
||||
pub fn level(&self) -> f32 {
|
||||
f32::from_bits(self.rms.load(Ordering::Relaxed))
|
||||
}
|
||||
|
||||
pub fn start(&self) {
|
||||
self.buffer.lock().map(|mut b| b.clear()).ok();
|
||||
self.recording.store(true, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Stops recording and returns what was captured, resampled to STT_RATE.
|
||||
pub fn stop(&self) -> Vec<f32> {
|
||||
self.recording.store(false, Ordering::Relaxed);
|
||||
let raw = self
|
||||
.buffer
|
||||
.lock()
|
||||
.map(|mut b| std::mem::take(&mut *b))
|
||||
.unwrap_or_default();
|
||||
resample(&raw, self.rate, STT_RATE)
|
||||
}
|
||||
}
|
||||
|
||||
/// Speaker. Fed from the TTS websocket; `clear` is what makes barge-in
|
||||
/// instant rather than "after the current sentence".
|
||||
pub struct Playback {
|
||||
queue: Arc<Mutex<VecDeque<f32>>>,
|
||||
rate: u32,
|
||||
}
|
||||
|
||||
impl Playback {
|
||||
pub fn new() -> Result<Self> {
|
||||
let queue: Arc<Mutex<VecDeque<f32>>> = Arc::new(Mutex::new(VecDeque::new()));
|
||||
let q = queue.clone();
|
||||
let rate = spawn_stream(move || {
|
||||
let device = cpal::default_host()
|
||||
.default_output_device()
|
||||
.ok_or_else(|| anyhow!("no output device"))?;
|
||||
let config = device.default_output_config()?;
|
||||
let rate = config.sample_rate().0;
|
||||
let channels = config.channels() as usize;
|
||||
let stream = device.build_output_stream(
|
||||
&config.into(),
|
||||
move |data: &mut [f32], _: &cpal::OutputCallbackInfo| {
|
||||
let mut q = match q.lock() {
|
||||
Ok(q) => q,
|
||||
Err(_) => return,
|
||||
};
|
||||
for frame in data.chunks_mut(channels) {
|
||||
let s = q.pop_front().unwrap_or(0.0);
|
||||
for out in frame.iter_mut() {
|
||||
*out = s;
|
||||
}
|
||||
}
|
||||
},
|
||||
|e| eprintln!("luna-voice: playback error: {e}"),
|
||||
None,
|
||||
)?;
|
||||
stream.play()?;
|
||||
Ok((stream, rate))
|
||||
})?;
|
||||
Ok(Playback { queue, rate })
|
||||
}
|
||||
|
||||
pub fn push(&self, pcm: &[f32]) {
|
||||
let out = resample(pcm, TTS_RATE, self.rate);
|
||||
if let Ok(mut q) = self.queue.lock() {
|
||||
q.extend(out);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn clear(&self) {
|
||||
self.queue.lock().map(|mut q| q.clear()).ok();
|
||||
}
|
||||
|
||||
pub fn pending(&self) -> usize {
|
||||
self.queue.lock().map(|q| q.len()).unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Short sine burst, so you know the key registered without looking at
|
||||
/// anything. Hermes' voice mode has `beep_enabled` for the same reason:
|
||||
/// push-to-talk with no acknowledgement is indistinguishable from broken.
|
||||
///
|
||||
/// Raised-cosine envelope — a bare sine starting at full amplitude clicks.
|
||||
pub fn beep(&self, freq: f32, ms: u32) {
|
||||
let n = (self.rate as f32 * ms as f32 / 1000.0) as usize;
|
||||
let samples: Vec<f32> = (0..n)
|
||||
.map(|i| {
|
||||
let t = i as f32 / self.rate as f32;
|
||||
let progress = i as f32 / n as f32;
|
||||
let envelope = (std::f32::consts::PI * progress).sin();
|
||||
(t * freq * std::f32::consts::TAU).sin() * envelope * 0.15
|
||||
})
|
||||
.collect();
|
||||
if let Ok(mut q) = self.queue.lock() {
|
||||
q.extend(samples);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds a cpal stream on a dedicated thread that then parks forever,
|
||||
/// keeping it alive. Returns the device's sample rate, or the build error —
|
||||
/// a stream that fails here must surface at startup, not as silence later.
|
||||
fn spawn_stream<F>(build: F) -> Result<u32>
|
||||
where
|
||||
F: FnOnce() -> Result<(cpal::Stream, u32)> + Send + 'static,
|
||||
{
|
||||
let (tx, rx) = std::sync::mpsc::channel::<Result<u32, String>>();
|
||||
std::thread::spawn(move || match build() {
|
||||
Ok((_stream, rate)) => {
|
||||
let _ = tx.send(Ok(rate));
|
||||
// Dropping `stream` would stop the device, so the thread holds it
|
||||
// for the life of the process.
|
||||
loop {
|
||||
std::thread::park();
|
||||
}
|
||||
#[allow(unreachable_code)]
|
||||
drop(_stream);
|
||||
}
|
||||
Err(e) => {
|
||||
let _ = tx.send(Err(e.to_string()));
|
||||
}
|
||||
});
|
||||
rx.recv()
|
||||
.map_err(|_| anyhow!("audio thread died during setup"))?
|
||||
.map_err(|e| anyhow!(e))
|
||||
}
|
||||
|
||||
/// Linear resampling. Speech at these rates does not justify a windowed sinc,
|
||||
/// and rubato would add a dependency for inaudible gain.
|
||||
pub fn resample(input: &[f32], from: u32, to: u32) -> Vec<f32> {
|
||||
if from == to || input.is_empty() {
|
||||
return input.to_vec();
|
||||
}
|
||||
let ratio = to as f64 / from as f64;
|
||||
let out_len = (input.len() as f64 * ratio).round() as usize;
|
||||
(0..out_len)
|
||||
.map(|i| {
|
||||
let src = i as f64 / ratio;
|
||||
let j = src.floor() as usize;
|
||||
let frac = (src - j as f64) as f32;
|
||||
let a = input.get(j).copied().unwrap_or(0.0);
|
||||
let b = input.get(j + 1).copied().unwrap_or(a);
|
||||
a + (b - a) * frac
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// 16-bit mono WAV, which is the only thing whisper-cli accepts without
|
||||
/// converting first.
|
||||
pub fn write_wav(path: &std::path::Path, samples: &[f32], rate: u32) -> Result<()> {
|
||||
let spec = hound::WavSpec {
|
||||
channels: 1,
|
||||
sample_rate: rate,
|
||||
bits_per_sample: 16,
|
||||
sample_format: hound::SampleFormat::Int,
|
||||
};
|
||||
let mut w = hound::WavWriter::create(path, spec)?;
|
||||
for s in samples {
|
||||
w.write_sample((s.clamp(-1.0, 1.0) * i16::MAX as f32) as i16)?;
|
||||
}
|
||||
w.finalize()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Signed 16-bit little-endian, as ElevenLabs sends pcm_*.
|
||||
pub fn pcm16_to_f32(bytes: &[u8]) -> Vec<f32> {
|
||||
bytes
|
||||
.chunks_exact(2)
|
||||
.map(|b| i16::from_le_bytes([b[0], b[1]]) as f32 / i16::MAX as f32)
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn resample_keeps_length_ratio() {
|
||||
let input = vec![0.0f32; 1000];
|
||||
assert_eq!(resample(&input, 48_000, 16_000).len(), 333);
|
||||
assert_eq!(resample(&input, 24_000, 48_000).len(), 2000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resample_is_identity_at_equal_rates() {
|
||||
let input = vec![0.1, -0.4, 0.9];
|
||||
assert_eq!(resample(&input, 16_000, 16_000), input);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pcm16_roundtrips_through_f32() {
|
||||
let bytes = [0x00, 0x00, 0xff, 0x7f, 0x01, 0x80];
|
||||
let got = pcm16_to_f32(&bytes);
|
||||
assert_eq!(got.len(), 3);
|
||||
assert!((got[0]).abs() < 1e-6);
|
||||
assert!((got[1] - 1.0).abs() < 1e-3);
|
||||
assert!((got[2] + 1.0).abs() < 1e-3);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user