Files
homelab/pkgs/luna-voice/src/ipc.rs
T
darmanandClaude Opus 5 67dd30eb72 feat(terra): hands-free voice mode, toggled with SUPER+V
The key is a toggle rather than push-to-talk, because "voice mode is
active" has to be a state something can display; a held key cannot
sustain one. So the daemon grew a VAD loop: speech starts a capture,
`silenceMs` of quiet ends it. A turn is awaited inline, which is also
what stops her hearing herself — the tick cannot fire while she speaks.

The keybind runs the CLI, not a quickshell GlobalShortcut, so the mic can
still be switched off while the shell is reloading and a QML error can
never leave it open with nothing on screen saying so.

Every threshold had to be resized for terra's mic, which is QUIET:
0.001-0.002 ambient against 0.0045 speaking, about 6dB apart. The VAD
floor of 0.006 and whisper's own 0.015 gate both sat ABOVE the user's
voice, and a 3.5x multiplier cannot separate signals that close — three
independent reasons nothing could ever trigger. Hence vadMult 1.8,
vadFloor 0.0025, and stt::normalize lifting the clip (capped at 30x, so
near-silence is not amplified into words) before whisper sees it.

The floor tracker also had to stop feeding on everything: a rising sound
dragged its own threshold up faster than it rose, and a tone peaking at
0.029 was outrun by a threshold that reached 0.018. Only samples below
the current trigger update it now.

Four diagnostics, because none of the above was visible while it was
failing: `watch` prints the socket protocol, `say` runs a turn from typed
text (a broken mic vs a broken pipeline), `level` is a live meter with
the trigger marked, `devices` lists the inputs. Amplitude is now emitted
on every tick, not only while she speaks — a meter that moves only during
playback cannot answer "is it hearing me".

Also: connect-time replay of mode/state, so restarting the shell while
voice mode is on does not leave it blind until the next event, which in
an idle conversation is never.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-20 23:37:51 +02:00

240 lines
8.2 KiB
Rust

use anyhow::Result;
use serde::Serialize;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
use tokio::net::{UnixListener, UnixStream};
use tokio::sync::{broadcast, mpsc};
use crate::visemes::Viseme;
/// Daemon -> shell. Newline-delimited JSON; the shell reads it with a
/// SplitParser and never has to spawn anything.
#[derive(Serialize, Clone, Debug)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum Event {
/// Whether voice mode is on at all. Distinct from `state`: the widget is
/// only on screen while this is true, and every state inside it is a
/// phase of an active conversation.
Mode { active: bool },
State { value: &'static str },
/// Emitted when the first PCM frame reaches the speaker. Viseme offsets
/// are relative to THIS, so the two processes never compare clocks.
SpeechStart,
Visemes { seq: Vec<Viseme> },
Amplitude { rms: f32 },
/// What was said, and by whom. Her side is emitted sentence by sentence
/// as it goes to the synthesiser, so subtitles can lead the audio the
/// same way the visemes do.
Caption { speaker: &'static str, text: String },
SpeechStop { reason: &'static str },
Error { message: String },
}
/// Shell -> daemon.
#[derive(Debug, Clone)]
pub enum Command {
Ptt { down: bool },
Cancel,
/// Run a turn from typed text, skipping the microphone entirely. This is
/// how the brain -> TTS -> speaker half gets tested without speaking, and
/// how you tell a broken mic from a broken pipeline.
Say { text: String },
/// Turn hands-free voice mode on, off, or (None) to the opposite of what
/// it is now — the keybind sends None so one key does both.
Mode { on: Option<bool> },
}
pub struct Ipc {
pub events: broadcast::Sender<Event>,
path: PathBuf,
/// Replayed to every client the moment it connects. The shell can be
/// restarted at any time — including while voice mode is on — and a
/// broadcast-only socket would leave it showing nothing until the next
/// event happened to fire, which in an idle conversation is never.
sticky: Arc<Mutex<Vec<Event>>>,
}
impl Ipc {
/// Binds the socket and serves it until dropped. A stale socket from a
/// crashed run is removed first — bind() would fail with EADDRINUSE on a
/// file nothing is listening to.
pub fn bind(path: &Path, commands: mpsc::Sender<Command>) -> Result<Self> {
if path.exists() {
let _ = std::fs::remove_file(path);
}
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
let listener = UnixListener::bind(path)?;
let (events, _) = broadcast::channel(256);
let sticky: Arc<Mutex<Vec<Event>>> = Arc::new(Mutex::new(Vec::new()));
let tx = events.clone();
let accept_sticky = sticky.clone();
tokio::spawn(async move {
loop {
match listener.accept().await {
Ok((stream, _)) => {
let rx = tx.subscribe();
let cmds = commands.clone();
let snapshot = accept_sticky
.lock()
.map(|s| s.clone())
.unwrap_or_default();
tokio::spawn(client(stream, rx, cmds, snapshot));
}
Err(e) => {
eprintln!("luna-voice: accept failed: {e}");
return;
}
}
}
});
Ok(Ipc { events, path: path.to_path_buf(), sticky })
}
pub fn emit(&self, event: Event) {
// Only the two events that describe a lasting condition are kept;
// replaying a viseme timeline or an old caption to a fresh client
// would animate a conversation that already finished.
if matches!(event, Event::Mode { .. } | Event::State { .. }) {
if let Ok(mut sticky) = self.sticky.lock() {
sticky.retain(|e| std::mem::discriminant(e) != std::mem::discriminant(&event));
sticky.push(event.clone());
}
}
// An error here only means nobody is connected, which is normal.
let _ = self.events.send(event);
}
}
impl Drop for Ipc {
fn drop(&mut self) {
let _ = std::fs::remove_file(&self.path);
}
}
async fn client(
stream: UnixStream,
mut rx: broadcast::Receiver<Event>,
commands: mpsc::Sender<Command>,
snapshot: Vec<Event>,
) {
let (read, mut write) = stream.into_split();
let mut lines = BufReader::new(read).lines();
// Failure here must NOT abandon the connection: the one-shot clients
// (`luna-voice on`, `toggle`, …) write a command and close immediately,
// so this write hits a closed socket and errors — and returning on that
// would drop the command unread, which is exactly what it did once.
for event in snapshot {
let Ok(mut line) = serde_json::to_string(&event) else { continue };
line.push('\n');
if write.write_all(line.as_bytes()).await.is_err() {
break;
}
}
loop {
tokio::select! {
event = rx.recv() => {
let Ok(event) = event else { return };
let Ok(mut line) = serde_json::to_string(&event) else { continue };
line.push('\n');
if write.write_all(line.as_bytes()).await.is_err() {
return;
}
}
line = lines.next_line() => {
match line {
Ok(Some(line)) => {
if let Some(cmd) = parse_command(&line) {
let _ = commands.send(cmd).await;
}
}
_ => return,
}
}
}
}
}
fn parse_command(line: &str) -> Option<Command> {
let v: serde_json::Value = serde_json::from_str(line).ok()?;
match v.get("type")?.as_str()? {
"ptt" => Some(Command::Ptt {
down: v.get("down").and_then(|d| d.as_bool()).unwrap_or(false),
}),
"cancel" => Some(Command::Cancel),
"say" => Some(Command::Say {
text: v.get("text")?.as_str()?.to_string(),
}),
"mode" => Some(Command::Mode {
on: v.get("on").and_then(|o| o.as_bool()),
}),
_ => None,
}
}
/// Client side of the same socket, for the `luna-voice ptt` subcommand that
/// Hyprland binds a key to before the quickshell widget exists.
pub async fn send_command(path: &Path, line: &str) -> Result<()> {
let mut stream = UnixStream::connect(path).await?;
stream.write_all(format!("{line}\n").as_bytes()).await?;
stream.flush().await?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_ptt_commands() {
assert!(matches!(
parse_command(r#"{"type":"ptt","down":true}"#),
Some(Command::Ptt { down: true })
));
assert!(matches!(
parse_command(r#"{"type":"ptt","down":false}"#),
Some(Command::Ptt { down: false })
));
}
#[test]
fn ptt_without_down_defaults_to_release() {
assert!(matches!(
parse_command(r#"{"type":"ptt"}"#),
Some(Command::Ptt { down: false })
));
}
#[test]
fn mode_without_a_value_means_toggle() {
assert!(matches!(
parse_command(r#"{"type":"mode"}"#),
Some(Command::Mode { on: None })
));
assert!(matches!(
parse_command(r#"{"type":"mode","on":true}"#),
Some(Command::Mode { on: Some(true) })
));
}
#[test]
fn ignores_unknown_and_malformed() {
assert!(parse_command(r#"{"type":"nope"}"#).is_none());
assert!(parse_command("garbage").is_none());
}
#[test]
fn events_serialise_with_a_type_tag() {
let json = serde_json::to_string(&Event::State { value: "listening" }).unwrap();
assert_eq!(json, r#"{"type":"state","value":"listening"}"#);
let json = serde_json::to_string(&Event::SpeechStart).unwrap();
assert_eq!(json, r#"{"type":"speech_start"}"#);
}
}