feat(terra): talk to luna with push-to-talk
luna-voiced: mic -> whisper-cpp (local, vulkan) -> Hermes' API on mars ->
ElevenLabs streaming TTS -> speaker, bound to SUPER+V. State, amplitude
and a viseme timeline are published on a unix socket; nothing consumes
that yet, but it is the data source the quickshell avatar will read.
STT stays local on purpose. mars already runs faster-whisper for Telegram
voice notes, but routing the mic through it would put every spoken word
on the network for no latency win.
Shape notes, all of them things the first live runs decided:
- Viseme offsets are relative to speech_start (emitted when the first
PCM frame reaches the speaker), never wall clock, so the shell and
the daemon never have to compare clocks. ElevenLabs sends character
alignment, not visemes, so the mapping is ours; the glTF morph target
names are the contract.
- clean_for_speech is a whitelist. She writes "(¬‿¬)", whose characters
live in Latin-1 and punctuation blocks rather than any emoji range,
so enumerating what to drop never converges. Em dashes become commas
because they are a pause — dropping one ran two clauses together and
changed what a sentence meant.
- whisper invents words out of silence ("you", "Thank you."), and a
held key over a quiet room is long AND empty, so has_speech gates on
the loudest 100ms window rather than on duration.
- Beeps on key down/up and a distinct low tone for "heard nothing":
push-to-talk with no acknowledgement is indistinguishable from broken,
and a dead mic must not feel like a slow reply.
- A warmup request at startup: her first reply after the container
starts costs ~16s against ~5s warm, and the first one is always the
one you are waiting for.
- `luna-voice watch` prints the socket protocol and `luna-voice say`
runs a turn from typed text, which is how a broken mic is told apart
from a broken pipeline.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,183 @@
|
||||
use anyhow::Result;
|
||||
use serde::Serialize;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||
use tokio::net::{UnixListener, UnixStream};
|
||||
use tokio::sync::{broadcast, mpsc};
|
||||
|
||||
use crate::visemes::Viseme;
|
||||
|
||||
/// Daemon -> shell. Newline-delimited JSON; the shell reads it with a
|
||||
/// SplitParser and never has to spawn anything.
|
||||
#[derive(Serialize, Clone, Debug)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum Event {
|
||||
State { value: &'static str },
|
||||
/// Emitted when the first PCM frame reaches the speaker. Viseme offsets
|
||||
/// are relative to THIS, so the two processes never compare clocks.
|
||||
SpeechStart,
|
||||
Visemes { seq: Vec<Viseme> },
|
||||
Amplitude { rms: f32 },
|
||||
/// What was said, and by whom. Her side is emitted sentence by sentence
|
||||
/// as it goes to the synthesiser, so subtitles can lead the audio the
|
||||
/// same way the visemes do.
|
||||
Caption { speaker: &'static str, text: String },
|
||||
SpeechStop { reason: &'static str },
|
||||
Error { message: String },
|
||||
}
|
||||
|
||||
/// Shell -> daemon.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum Command {
|
||||
Ptt { down: bool },
|
||||
Cancel,
|
||||
/// Run a turn from typed text, skipping the microphone entirely. This is
|
||||
/// how the brain -> TTS -> speaker half gets tested without speaking, and
|
||||
/// how you tell a broken mic from a broken pipeline.
|
||||
Say { text: String },
|
||||
}
|
||||
|
||||
pub struct Ipc {
|
||||
pub events: broadcast::Sender<Event>,
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl Ipc {
|
||||
/// Binds the socket and serves it until dropped. A stale socket from a
|
||||
/// crashed run is removed first — bind() would fail with EADDRINUSE on a
|
||||
/// file nothing is listening to.
|
||||
pub fn bind(path: &Path, commands: mpsc::Sender<Command>) -> Result<Self> {
|
||||
if path.exists() {
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
let listener = UnixListener::bind(path)?;
|
||||
let (events, _) = broadcast::channel(256);
|
||||
|
||||
let tx = events.clone();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
match listener.accept().await {
|
||||
Ok((stream, _)) => {
|
||||
let rx = tx.subscribe();
|
||||
let cmds = commands.clone();
|
||||
tokio::spawn(client(stream, rx, cmds));
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("luna-voice: accept failed: {e}");
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Ipc { events, path: path.to_path_buf() })
|
||||
}
|
||||
|
||||
pub fn emit(&self, event: Event) {
|
||||
// An error here only means nobody is connected, which is normal.
|
||||
let _ = self.events.send(event);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Ipc {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_file(&self.path);
|
||||
}
|
||||
}
|
||||
|
||||
async fn client(
|
||||
stream: UnixStream,
|
||||
mut rx: broadcast::Receiver<Event>,
|
||||
commands: mpsc::Sender<Command>,
|
||||
) {
|
||||
let (read, mut write) = stream.into_split();
|
||||
let mut lines = BufReader::new(read).lines();
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
event = rx.recv() => {
|
||||
let Ok(event) = event else { return };
|
||||
let Ok(mut line) = serde_json::to_string(&event) else { continue };
|
||||
line.push('\n');
|
||||
if write.write_all(line.as_bytes()).await.is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
line = lines.next_line() => {
|
||||
match line {
|
||||
Ok(Some(line)) => {
|
||||
if let Some(cmd) = parse_command(&line) {
|
||||
let _ = commands.send(cmd).await;
|
||||
}
|
||||
}
|
||||
_ => return,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_command(line: &str) -> Option<Command> {
|
||||
let v: serde_json::Value = serde_json::from_str(line).ok()?;
|
||||
match v.get("type")?.as_str()? {
|
||||
"ptt" => Some(Command::Ptt {
|
||||
down: v.get("down").and_then(|d| d.as_bool()).unwrap_or(false),
|
||||
}),
|
||||
"cancel" => Some(Command::Cancel),
|
||||
"say" => Some(Command::Say {
|
||||
text: v.get("text")?.as_str()?.to_string(),
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Client side of the same socket, for the `luna-voice ptt` subcommand that
|
||||
/// Hyprland binds a key to before the quickshell widget exists.
|
||||
pub async fn send_command(path: &Path, line: &str) -> Result<()> {
|
||||
let mut stream = UnixStream::connect(path).await?;
|
||||
stream.write_all(format!("{line}\n").as_bytes()).await?;
|
||||
stream.flush().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_ptt_commands() {
|
||||
assert!(matches!(
|
||||
parse_command(r#"{"type":"ptt","down":true}"#),
|
||||
Some(Command::Ptt { down: true })
|
||||
));
|
||||
assert!(matches!(
|
||||
parse_command(r#"{"type":"ptt","down":false}"#),
|
||||
Some(Command::Ptt { down: false })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ptt_without_down_defaults_to_release() {
|
||||
assert!(matches!(
|
||||
parse_command(r#"{"type":"ptt"}"#),
|
||||
Some(Command::Ptt { down: false })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignores_unknown_and_malformed() {
|
||||
assert!(parse_command(r#"{"type":"nope"}"#).is_none());
|
||||
assert!(parse_command("garbage").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn events_serialise_with_a_type_tag() {
|
||||
let json = serde_json::to_string(&Event::State { value: "listening" }).unwrap();
|
||||
assert_eq!(json, r#"{"type":"state","value":"listening"}"#);
|
||||
let json = serde_json::to_string(&Event::SpeechStart).unwrap();
|
||||
assert_eq!(json, r#"{"type":"speech_start"}"#);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user