feat(terra): hands-free voice mode, toggled with SUPER+V
The key is a toggle rather than push-to-talk, because "voice mode is active" has to be a state something can display; a held key cannot sustain one. So the daemon grew a VAD loop: speech starts a capture, `silenceMs` of quiet ends it. A turn is awaited inline, which is also what stops her hearing herself — the tick cannot fire while she speaks. The keybind runs the CLI, not a quickshell GlobalShortcut, so the mic can still be switched off while the shell is reloading and a QML error can never leave it open with nothing on screen saying so. Every threshold had to be resized for terra's mic, which is QUIET: 0.001-0.002 ambient against 0.0045 speaking, about 6dB apart. The VAD floor of 0.006 and whisper's own 0.015 gate both sat ABOVE the user's voice, and a 3.5x multiplier cannot separate signals that close — three independent reasons nothing could ever trigger. Hence vadMult 1.8, vadFloor 0.0025, and stt::normalize lifting the clip (capped at 30x, so near-silence is not amplified into words) before whisper sees it. The floor tracker also had to stop feeding on everything: a rising sound dragged its own threshold up faster than it rose, and a tone peaking at 0.029 was outrun by a threshold that reached 0.018. Only samples below the current trigger update it now. Four diagnostics, because none of the above was visible while it was failing: `watch` prints the socket protocol, `say` runs a turn from typed text (a broken mic vs a broken pipeline), `level` is a live meter with the trigger marked, `devices` lists the inputs. Amplitude is now emitted on every tick, not only while she speaks — a meter that moves only during playback cannot answer "is it hearing me". Also: connect-time replay of mode/state, so restarting the shell while voice mode is on does not leave it blind until the next event, which in an idle conversation is never. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -195,11 +195,10 @@ in
|
||||
# expand / collapse the hyprchrome bar as a whole
|
||||
(bind "SUPER + A" (dsp.global "quickshell:chrome"))
|
||||
|
||||
# push-to-talk for luna (services/desktop/luna-voice.nix). Two binds,
|
||||
# not one: the daemon records between the press and the release, so
|
||||
# the release edge is what ends the utterance.
|
||||
(bind "SUPER + V" (dsp.exec "luna-voice ptt-down"))
|
||||
(bindo "SUPER + V" (dsp.exec "luna-voice ptt-up") { release = true; })
|
||||
# toggle luna's hands-free voice mode (services/desktop/luna-voice.nix).
|
||||
# Runs the CLI rather than a quickshell GlobalShortcut so the mic can
|
||||
# still be switched off while the shell is reloading.
|
||||
(bind "SUPER + V" (dsp.exec "luna-voice toggle"))
|
||||
|
||||
(bind "SUPER + B" (dsp.exec "vivaldi"))
|
||||
(bind "SUPER + E" (dsp.exec "cosmic-files"))
|
||||
|
||||
@@ -32,9 +32,14 @@ impl Capture {
|
||||
|
||||
let (r, rec, buf) = (rms.clone(), recording.clone(), buffer.clone());
|
||||
let rate = spawn_stream(move || {
|
||||
let device = cpal::default_host()
|
||||
.default_input_device()
|
||||
.ok_or_else(|| anyhow!("no input device"))?;
|
||||
let device = input_device()?;
|
||||
// Named in the log because "the mic does not work" is usually
|
||||
// "the default source is the wrong device", and nothing else in
|
||||
// the pipeline can tell you that.
|
||||
eprintln!(
|
||||
"luna-voice: input device {:?}",
|
||||
device.name().unwrap_or_else(|_| "<unnamed>".into())
|
||||
);
|
||||
let config = device.default_input_config()?;
|
||||
let rate = config.sample_rate().0;
|
||||
let channels = config.channels() as usize;
|
||||
@@ -163,6 +168,47 @@ impl Playback {
|
||||
}
|
||||
}
|
||||
|
||||
/// The microphone, honouring LUNA_INPUT_DEVICE (a case-insensitive substring
|
||||
/// of the device name) before falling back to the system default.
|
||||
///
|
||||
/// Worth having because the default source is a system-wide setting that has
|
||||
/// nothing to do with which input you actually speak into: a box with an
|
||||
/// audio interface and onboard audio has several, and picking the silent one
|
||||
/// looks exactly like a broken pipeline.
|
||||
fn input_device() -> Result<cpal::Device> {
|
||||
let host = cpal::default_host();
|
||||
if let Ok(want) = std::env::var("LUNA_INPUT_DEVICE") {
|
||||
let want = want.to_lowercase();
|
||||
if let Ok(devices) = host.input_devices() {
|
||||
for device in devices {
|
||||
if device
|
||||
.name()
|
||||
.map(|n| n.to_lowercase().contains(&want))
|
||||
.unwrap_or(false)
|
||||
{
|
||||
return Ok(device);
|
||||
}
|
||||
}
|
||||
}
|
||||
eprintln!("luna-voice: no input device matching {want:?}; using the default");
|
||||
}
|
||||
host.default_input_device()
|
||||
.ok_or_else(|| anyhow!("no input device"))
|
||||
}
|
||||
|
||||
/// Every input cpal can see, for `luna-voice devices`.
|
||||
pub fn list_inputs() -> Result<Vec<String>> {
|
||||
let host = cpal::default_host();
|
||||
let default = host.default_input_device().and_then(|d| d.name().ok());
|
||||
let mut out = Vec::new();
|
||||
for device in host.input_devices()? {
|
||||
let name = device.name().unwrap_or_else(|_| "<unnamed>".into());
|
||||
let marker = if Some(&name) == default.as_ref() { " (default)" } else { "" };
|
||||
out.push(format!("{name}{marker}"));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Builds a cpal stream on a dedicated thread that then parks forever,
|
||||
/// keeping it alive. Returns the device's sample rate, or the build error —
|
||||
/// a stream that fails here must surface at startup, not as silence later.
|
||||
|
||||
@@ -19,6 +19,10 @@ pub struct Config {
|
||||
pub whisper_model: PathBuf,
|
||||
|
||||
pub socket_path: PathBuf,
|
||||
/// Silence that ends an utterance in hands-free mode. Hermes' own default.
|
||||
pub silence_ms: u64,
|
||||
/// Level above the running noise floor that starts one.
|
||||
pub vad_mult: f32,
|
||||
/// Speech during playback louder than noise_floor * this counts as barge-in.
|
||||
pub barge_in_mult: f32,
|
||||
pub barge_in_grace_ms: u64,
|
||||
@@ -63,6 +67,12 @@ impl Config {
|
||||
"LUNA_SOCKET",
|
||||
&format!("{runtime}/luna-voiced.sock"),
|
||||
)),
|
||||
silence_ms: env_or("LUNA_SILENCE_MS", "2000").parse().unwrap_or(2000),
|
||||
// 1.8, not Hermes' 3.0-ish: terra's interface has speech only
|
||||
// ~6dB over its own ambient (0.0045 against 0.0022 measured), and
|
||||
// no multiplier above ~2 can separate those. The real fix is gain
|
||||
// on the interface; this makes it work at the gain it has.
|
||||
vad_mult: env_or("LUNA_VAD_MULT", "1.8").parse().unwrap_or(1.8),
|
||||
barge_in_mult: env_or("LUNA_BARGE_IN_MULT", "3.0").parse().unwrap_or(3.0),
|
||||
barge_in_grace_ms: env_or("LUNA_BARGE_IN_GRACE_MS", "500")
|
||||
.parse()
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use anyhow::Result;
|
||||
use serde::Serialize;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||
use tokio::net::{UnixListener, UnixStream};
|
||||
use tokio::sync::{broadcast, mpsc};
|
||||
@@ -12,6 +13,10 @@ use crate::visemes::Viseme;
|
||||
#[derive(Serialize, Clone, Debug)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum Event {
|
||||
/// Whether voice mode is on at all. Distinct from `state`: the widget is
|
||||
/// only on screen while this is true, and every state inside it is a
|
||||
/// phase of an active conversation.
|
||||
Mode { active: bool },
|
||||
State { value: &'static str },
|
||||
/// Emitted when the first PCM frame reaches the speaker. Viseme offsets
|
||||
/// are relative to THIS, so the two processes never compare clocks.
|
||||
@@ -35,11 +40,19 @@ pub enum Command {
|
||||
/// how the brain -> TTS -> speaker half gets tested without speaking, and
|
||||
/// how you tell a broken mic from a broken pipeline.
|
||||
Say { text: String },
|
||||
/// Turn hands-free voice mode on, off, or (None) to the opposite of what
|
||||
/// it is now — the keybind sends None so one key does both.
|
||||
Mode { on: Option<bool> },
|
||||
}
|
||||
|
||||
pub struct Ipc {
|
||||
pub events: broadcast::Sender<Event>,
|
||||
path: PathBuf,
|
||||
/// Replayed to every client the moment it connects. The shell can be
|
||||
/// restarted at any time — including while voice mode is on — and a
|
||||
/// broadcast-only socket would leave it showing nothing until the next
|
||||
/// event happened to fire, which in an idle conversation is never.
|
||||
sticky: Arc<Mutex<Vec<Event>>>,
|
||||
}
|
||||
|
||||
impl Ipc {
|
||||
@@ -55,15 +68,21 @@ impl Ipc {
|
||||
}
|
||||
let listener = UnixListener::bind(path)?;
|
||||
let (events, _) = broadcast::channel(256);
|
||||
let sticky: Arc<Mutex<Vec<Event>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
|
||||
let tx = events.clone();
|
||||
let accept_sticky = sticky.clone();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
match listener.accept().await {
|
||||
Ok((stream, _)) => {
|
||||
let rx = tx.subscribe();
|
||||
let cmds = commands.clone();
|
||||
tokio::spawn(client(stream, rx, cmds));
|
||||
let snapshot = accept_sticky
|
||||
.lock()
|
||||
.map(|s| s.clone())
|
||||
.unwrap_or_default();
|
||||
tokio::spawn(client(stream, rx, cmds, snapshot));
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("luna-voice: accept failed: {e}");
|
||||
@@ -73,10 +92,19 @@ impl Ipc {
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Ipc { events, path: path.to_path_buf() })
|
||||
Ok(Ipc { events, path: path.to_path_buf(), sticky })
|
||||
}
|
||||
|
||||
pub fn emit(&self, event: Event) {
|
||||
// Only the two events that describe a lasting condition are kept;
|
||||
// replaying a viseme timeline or an old caption to a fresh client
|
||||
// would animate a conversation that already finished.
|
||||
if matches!(event, Event::Mode { .. } | Event::State { .. }) {
|
||||
if let Ok(mut sticky) = self.sticky.lock() {
|
||||
sticky.retain(|e| std::mem::discriminant(e) != std::mem::discriminant(&event));
|
||||
sticky.push(event.clone());
|
||||
}
|
||||
}
|
||||
// An error here only means nobody is connected, which is normal.
|
||||
let _ = self.events.send(event);
|
||||
}
|
||||
@@ -92,10 +120,23 @@ async fn client(
|
||||
stream: UnixStream,
|
||||
mut rx: broadcast::Receiver<Event>,
|
||||
commands: mpsc::Sender<Command>,
|
||||
snapshot: Vec<Event>,
|
||||
) {
|
||||
let (read, mut write) = stream.into_split();
|
||||
let mut lines = BufReader::new(read).lines();
|
||||
|
||||
// Failure here must NOT abandon the connection: the one-shot clients
|
||||
// (`luna-voice on`, `toggle`, …) write a command and close immediately,
|
||||
// so this write hits a closed socket and errors — and returning on that
|
||||
// would drop the command unread, which is exactly what it did once.
|
||||
for event in snapshot {
|
||||
let Ok(mut line) = serde_json::to_string(&event) else { continue };
|
||||
line.push('\n');
|
||||
if write.write_all(line.as_bytes()).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
event = rx.recv() => {
|
||||
@@ -130,6 +171,9 @@ fn parse_command(line: &str) -> Option<Command> {
|
||||
"say" => Some(Command::Say {
|
||||
text: v.get("text")?.as_str()?.to_string(),
|
||||
}),
|
||||
"mode" => Some(Command::Mode {
|
||||
on: v.get("on").and_then(|o| o.as_bool()),
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -167,6 +211,18 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mode_without_a_value_means_toggle() {
|
||||
assert!(matches!(
|
||||
parse_command(r#"{"type":"mode"}"#),
|
||||
Some(Command::Mode { on: None })
|
||||
));
|
||||
assert!(matches!(
|
||||
parse_command(r#"{"type":"mode","on":true}"#),
|
||||
Some(Command::Mode { on: Some(true) })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignores_unknown_and_malformed() {
|
||||
assert!(parse_command(r#"{"type":"nope"}"#).is_none());
|
||||
|
||||
+244
-7
@@ -33,10 +33,22 @@ async fn main() -> Result<()> {
|
||||
Some("ptt-down") => client(r#"{"type":"ptt","down":true}"#).await,
|
||||
Some("ptt-up") => client(r#"{"type":"ptt","down":false}"#).await,
|
||||
Some("cancel") => client(r#"{"type":"cancel"}"#).await,
|
||||
Some("toggle") => client(r#"{"type":"mode"}"#).await,
|
||||
Some("on") => client(r#"{"type":"mode","on":true}"#).await,
|
||||
Some("off") => client(r#"{"type":"mode","on":false}"#).await,
|
||||
// The debugging tool: everything the avatar will eventually see,
|
||||
// printed. `journalctl -fu luna-voiced` shows only what the daemon
|
||||
// chose to log; this shows the actual protocol.
|
||||
Some("watch") => watch().await,
|
||||
// Opens the mic directly, bypassing the daemon: answers "is the
|
||||
// microphone working at all" without any of the rest being involved.
|
||||
Some("level") => level().await,
|
||||
Some("devices") => {
|
||||
for name in audio::list_inputs()? {
|
||||
println!("{name}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Some("say") => {
|
||||
let text: Vec<String> = args.collect();
|
||||
if text.is_empty() {
|
||||
@@ -48,12 +60,29 @@ async fn main() -> Result<()> {
|
||||
}
|
||||
Some(other) => {
|
||||
eprintln!("luna-voice: unknown command {other:?}");
|
||||
eprintln!("usage: luna-voice [daemon|ptt-down|ptt-up|cancel|watch]");
|
||||
eprintln!(
|
||||
"usage: luna-voice [daemon|toggle|on|off|say <text>|watch|level|devices|cancel]"
|
||||
);
|
||||
std::process::exit(2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Level that counts as speech.
|
||||
///
|
||||
/// The lower clamp is the number that matters, and it has to sit under the
|
||||
/// QUIETEST mic worth supporting, not under a typical one: terra's AIR 192|4
|
||||
/// reads ~0.001 idle and only 0.004–0.005 for normal speech, so a 0.006 floor
|
||||
/// — which looked conservative — put the trigger permanently above the user's
|
||||
/// voice. Overshooting downward only costs silent false triggers.
|
||||
fn root_threshold(floor: f32, mult: f32) -> f32 {
|
||||
let min = std::env::var("LUNA_VAD_FLOOR")
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(0.0025);
|
||||
(floor * mult).clamp(min, 0.05)
|
||||
}
|
||||
|
||||
fn socket_path() -> std::path::PathBuf {
|
||||
let runtime = std::env::var("XDG_RUNTIME_DIR").unwrap_or_else(|_| "/run/user/1000".into());
|
||||
std::env::var("LUNA_SOCKET")
|
||||
@@ -75,6 +104,46 @@ async fn watch() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Live microphone meter with the trigger point marked, so a mic that reads
|
||||
/// nothing and a threshold that is set too high look different.
|
||||
async fn level() -> Result<()> {
|
||||
let capture = audio::Capture::new()?;
|
||||
let mult: f32 = std::env::var("LUNA_VAD_MULT")
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(1.8);
|
||||
let mut floor: f32 = capture.level().max(0.003);
|
||||
eprintln!("luna-voice: speak — ctrl-c to stop");
|
||||
loop {
|
||||
let level = capture.level();
|
||||
floor = floor * 0.95 + level * 0.05;
|
||||
let threshold = root_threshold(floor, mult);
|
||||
// Full scale 0.02, not 0.05: a quiet interface tops out around
|
||||
// 0.005 and was drawing a single bar on the wider scale.
|
||||
const FULL: f32 = 0.02;
|
||||
let bars = ((level / FULL).min(1.0) * 40.0) as usize;
|
||||
let mark = ((threshold / FULL).min(1.0) * 40.0) as usize;
|
||||
let meter: String = (0..40)
|
||||
.map(|i| {
|
||||
if i == mark {
|
||||
'|'
|
||||
} else if i < bars {
|
||||
'#'
|
||||
} else {
|
||||
'.'
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
print!(
|
||||
"\r{meter} rms {level:.4} floor {floor:.4} trigger {threshold:.4} {} ",
|
||||
if level > threshold { "SPEECH" } else { " " }
|
||||
);
|
||||
use std::io::Write;
|
||||
let _ = std::io::stdout().flush();
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn client(line: &str) -> Result<()> {
|
||||
ipc::send_command(&socket_path(), line).await
|
||||
}
|
||||
@@ -98,6 +167,9 @@ async fn daemon() -> Result<()> {
|
||||
let playback = Arc::new(audio::Playback::new()?);
|
||||
|
||||
let (cmd_tx, mut cmd_rx) = mpsc::channel::<Command>(16);
|
||||
// Kept so a turn can put back commands that arrived while she was
|
||||
// speaking instead of destroying them.
|
||||
let requeue = cmd_tx.clone();
|
||||
let ipc = Arc::new(ipc::Ipc::bind(&cfg.socket_path, cmd_tx)?);
|
||||
eprintln!("luna-voice: listening on {}", cfg.socket_path.display());
|
||||
|
||||
@@ -133,8 +205,109 @@ async fn daemon() -> Result<()> {
|
||||
}
|
||||
});
|
||||
|
||||
while let Some(cmd) = cmd_rx.recv().await {
|
||||
// Hands-free state. The key toggles `mode`; everything inside it is
|
||||
// driven by the microphone rather than by keys.
|
||||
let mut mode = false;
|
||||
// Running estimate of the room, updated only while not capturing, so a
|
||||
// loud sentence never raises the bar for the next one.
|
||||
let mut floor: f32 = 0.01;
|
||||
let mut capturing = false;
|
||||
let mut last_voice = tokio::time::Instant::now();
|
||||
let mut ticker = tokio::time::interval(Duration::from_millis(50));
|
||||
|
||||
loop {
|
||||
let cmd = tokio::select! {
|
||||
cmd = cmd_rx.recv() => match cmd {
|
||||
Some(cmd) => Some(cmd),
|
||||
None => break,
|
||||
},
|
||||
// Only ticks while voice mode is on. A turn is awaited inline
|
||||
// below, so this cannot fire while she is speaking — which is
|
||||
// also what stops her from hearing herself.
|
||||
_ = ticker.tick(), if mode => None,
|
||||
};
|
||||
|
||||
let Some(cmd) = cmd else {
|
||||
let level = capture.level();
|
||||
// Every tick, not just while she speaks: a meter that only moves
|
||||
// during playback cannot tell "the mic is dead" apart from "the
|
||||
// threshold is too high", which is exactly the question you have
|
||||
// when nothing happens.
|
||||
ipc.emit(Event::Amplitude { rms: level });
|
||||
let threshold = root_threshold(floor, cfg.vad_mult);
|
||||
if !capturing {
|
||||
// Only samples BELOW the trigger update the floor. Feeding
|
||||
// every sample in lets a rising sound drag the floor up with
|
||||
// it, so the threshold outruns the speech that caused it and
|
||||
// nothing ever fires — measured: a tone peaking at 0.029 was
|
||||
// chased by a threshold that reached 0.018 on the way up.
|
||||
if level <= threshold {
|
||||
floor = floor * 0.95 + level * 0.05;
|
||||
}
|
||||
if level > threshold {
|
||||
capture.start();
|
||||
capturing = true;
|
||||
last_voice = tokio::time::Instant::now();
|
||||
ipc.emit(Event::State { value: "listening" });
|
||||
}
|
||||
} else {
|
||||
if level > threshold {
|
||||
last_voice = tokio::time::Instant::now();
|
||||
}
|
||||
if last_voice.elapsed() > Duration::from_millis(cfg.silence_ms) {
|
||||
capturing = false;
|
||||
let samples = capture.stop();
|
||||
ipc.emit(Event::State { value: "thinking" });
|
||||
match stt::transcribe(&cfg.whisper_bin, &cfg.whisper_model, &samples).await {
|
||||
Ok(heard) if !heard.is_empty() => {
|
||||
run_turn(
|
||||
&cfg, &http, &ipc, &capture, &playback, &mut cmd_rx,
|
||||
&requeue, &mut history, heard,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Ok(_) => {
|
||||
// Silent on purpose: in hands-free mode a false
|
||||
// trigger happens whenever a door closes, and a
|
||||
// beep for each one would be maddening.
|
||||
ipc.emit(Event::State { value: "idle" });
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("luna-voice: {e:#}");
|
||||
ipc.emit(Event::Error { message: e.to_string() });
|
||||
notify(&e.to_string());
|
||||
ipc.emit(Event::State { value: "idle" });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
continue;
|
||||
};
|
||||
|
||||
match cmd {
|
||||
Command::Mode { on } => {
|
||||
let want = on.unwrap_or(!mode);
|
||||
if want != mode {
|
||||
mode = want;
|
||||
if mode {
|
||||
// Seeded low: a floor seeded high takes seconds to
|
||||
// decay, and those are the seconds you spend
|
||||
// talking to something that is not listening yet.
|
||||
floor = capture.level().max(0.003);
|
||||
playback.beep(880.0, 70);
|
||||
} else {
|
||||
if capturing {
|
||||
capture.stop();
|
||||
capturing = false;
|
||||
}
|
||||
playback.clear();
|
||||
playback.beep(440.0, 90);
|
||||
}
|
||||
ipc.emit(Event::Mode { active: mode });
|
||||
ipc.emit(Event::State { value: "idle" });
|
||||
eprintln!("luna-voice: voice mode {}", if mode { "on" } else { "off" });
|
||||
}
|
||||
}
|
||||
Command::Ptt { down: true } => {
|
||||
// Anything still playing is cut off: pressing the key while
|
||||
// she talks means you want to talk instead.
|
||||
@@ -171,12 +344,12 @@ async fn daemon() -> Result<()> {
|
||||
continue;
|
||||
}
|
||||
|
||||
run_turn(&cfg, &http, &ipc, &capture, &playback, &mut cmd_rx, &mut history, heard)
|
||||
run_turn(&cfg, &http, &ipc, &capture, &playback, &mut cmd_rx, &requeue, &mut history, heard)
|
||||
.await;
|
||||
}
|
||||
Command::Say { text } => {
|
||||
ipc.emit(Event::State { value: "thinking" });
|
||||
run_turn(&cfg, &http, &ipc, &capture, &playback, &mut cmd_rx, &mut history, text)
|
||||
run_turn(&cfg, &http, &ipc, &capture, &playback, &mut cmd_rx, &requeue, &mut history, text)
|
||||
.await;
|
||||
}
|
||||
Command::Cancel => {
|
||||
@@ -191,6 +364,32 @@ async fn daemon() -> Result<()> {
|
||||
|
||||
/// Runs one turn and reports failure through every channel that has a
|
||||
/// listener: journal, socket, desktop notification.
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::root_threshold;
|
||||
|
||||
#[test]
|
||||
fn threshold_sits_between_terras_room_tone_and_terras_speech() {
|
||||
// Measured on the AIR 192|4: ~0.001 idle, 0.004–0.005 speaking. The
|
||||
// trigger has to land strictly between those two, which is the whole
|
||||
// reason the lower clamp is 0.0025 and not something rounder.
|
||||
let t = root_threshold(0.001, 3.5);
|
||||
assert!(t > 0.001, "would trigger on room tone: {t}");
|
||||
assert!(t < 0.004, "would never trigger on speech: {t}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn threshold_never_climbs_out_of_reach_of_speech() {
|
||||
// A floor seeded from a loud moment must not lock the mic out.
|
||||
assert_eq!(root_threshold(0.5, 3.5), 0.05);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn threshold_tracks_the_room_in_between() {
|
||||
assert!((root_threshold(0.01, 3.5) - 0.035).abs() < 1e-6);
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn run_turn(
|
||||
cfg: &Config,
|
||||
@@ -199,10 +398,11 @@ async fn run_turn(
|
||||
capture: &audio::Capture,
|
||||
playback: &Arc<audio::Playback>,
|
||||
cmd_rx: &mut mpsc::Receiver<Command>,
|
||||
requeue: &mpsc::Sender<Command>,
|
||||
history: &mut Vec<brain::Message>,
|
||||
heard: String,
|
||||
) {
|
||||
if let Err(e) = turn(cfg, http, ipc, capture, playback, cmd_rx, history, heard).await {
|
||||
if let Err(e) = turn(cfg, http, ipc, capture, playback, cmd_rx, requeue, history, heard).await {
|
||||
eprintln!("luna-voice: {e:#}");
|
||||
ipc.emit(Event::Error { message: e.to_string() });
|
||||
notify(&e.to_string());
|
||||
@@ -218,6 +418,7 @@ async fn turn(
|
||||
capture: &audio::Capture,
|
||||
playback: &Arc<audio::Playback>,
|
||||
cmd_rx: &mut mpsc::Receiver<Command>,
|
||||
requeue: &mpsc::Sender<Command>,
|
||||
history: &mut Vec<brain::Message>,
|
||||
heard: String,
|
||||
) -> Result<()> {
|
||||
@@ -225,6 +426,10 @@ async fn turn(
|
||||
ipc.emit(Event::Caption { speaker: "user", text: heard.clone() });
|
||||
history.push(brain::Message::user(heard));
|
||||
|
||||
// Commands that arrived while she was speaking, replayed into the
|
||||
// channel once the turn is over rather than dropped.
|
||||
let mut deferred: Vec<Command> = Vec::new();
|
||||
|
||||
// Noise floor sampled before she starts, so barge-in adapts to the room
|
||||
// rather than to a constant.
|
||||
let floor = capture.level().max(0.005);
|
||||
@@ -320,17 +525,49 @@ async fn turn(
|
||||
));
|
||||
break;
|
||||
}
|
||||
// A cancel arriving mid-playback must be honoured now, not after.
|
||||
if let Ok(Command::Cancel) = cmd_rx.try_recv() {
|
||||
// Commands arriving mid-playback. try_recv POPS, so anything not
|
||||
// handled here is destroyed: an `if let Ok(Cancel)` silently ate
|
||||
// every `say`, `toggle` and `mode off` sent while she was talking,
|
||||
// which presents as a keypress that does nothing.
|
||||
match cmd_rx.try_recv() {
|
||||
Ok(Command::Cancel) => {
|
||||
playback.clear();
|
||||
ipc.emit(Event::SpeechStop { reason: "cancelled" });
|
||||
break;
|
||||
}
|
||||
// Leaving voice mode has to shut her up immediately; the mode
|
||||
// change itself is replayed below.
|
||||
Ok(cmd @ Command::Mode { .. }) => {
|
||||
playback.clear();
|
||||
ipc.emit(Event::SpeechStop { reason: "cancelled" });
|
||||
deferred.push(cmd);
|
||||
break;
|
||||
}
|
||||
Ok(cmd) => deferred.push(cmd),
|
||||
Err(_) => {}
|
||||
}
|
||||
ipc.emit(Event::Amplitude { rms: capture.level() });
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
|
||||
for cmd in deferred {
|
||||
let _ = requeue.try_send(cmd);
|
||||
}
|
||||
|
||||
// A TTS failure must NOT be silent. The captions come from the text
|
||||
// stream, not from the synthesiser, so a dead websocket still paints her
|
||||
// whole reply on screen — "she answered but said nothing" is then
|
||||
// indistinguishable from a working reply with the volume down, which is
|
||||
// exactly how it presented.
|
||||
if speech.is_finished() {
|
||||
if let Ok(Err(e)) = speech.await {
|
||||
eprintln!("luna-voice: tts failed: {e:#}");
|
||||
ipc.emit(Event::Error { message: format!("tts: {e}") });
|
||||
notify(&format!("voice failed: {e}"));
|
||||
}
|
||||
} else {
|
||||
speech.abort();
|
||||
}
|
||||
player.abort();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ pub async fn transcribe(bin: &str, model: &Path, samples: &[f32]) -> Result<Stri
|
||||
|
||||
let dir = std::env::var("XDG_RUNTIME_DIR").unwrap_or_else(|_| "/tmp".into());
|
||||
let wav = Path::new(&dir).join(format!("luna-stt-{}.wav", std::process::id()));
|
||||
audio::write_wav(&wav, samples, audio::STT_RATE)?;
|
||||
audio::write_wav(&wav, &normalize(samples), audio::STT_RATE)?;
|
||||
|
||||
let out = Command::new(bin)
|
||||
.args(["-m"])
|
||||
@@ -40,11 +40,38 @@ pub async fn transcribe(bin: &str, model: &Path, samples: &[f32]) -> Result<Stri
|
||||
Ok(clean(&String::from_utf8_lossy(&out.stdout)))
|
||||
}
|
||||
|
||||
/// Scales a quiet recording up before whisper sees it.
|
||||
///
|
||||
/// Whisper is trained on normalised audio and degrades badly on very quiet
|
||||
/// input, which is precisely what a low-gain interface produces — speech at a
|
||||
/// 0.03 peak is intelligible to a person and mush to the model. The gain is
|
||||
/// capped so a recording of near-silence is not amplified into hiss the model
|
||||
/// will confidently transcribe as words.
|
||||
fn normalize(samples: &[f32]) -> Vec<f32> {
|
||||
const TARGET_PEAK: f32 = 0.6;
|
||||
const MAX_GAIN: f32 = 30.0;
|
||||
|
||||
let peak = samples.iter().fold(0.0f32, |m, s| m.max(s.abs()));
|
||||
if peak <= f32::EPSILON {
|
||||
return samples.to_vec();
|
||||
}
|
||||
let gain = (TARGET_PEAK / peak).min(MAX_GAIN);
|
||||
// Never attenuate: a loud recording is already fine, and turning it down
|
||||
// would only lose resolution in the 16-bit conversion.
|
||||
if gain <= 1.0 {
|
||||
return samples.to_vec();
|
||||
}
|
||||
samples.iter().map(|s| (s * gain).clamp(-1.0, 1.0)).collect()
|
||||
}
|
||||
|
||||
/// True if any 100ms window is loud enough to be a voice. Peak rather than
|
||||
/// overall RMS: a short sentence inside a long silent recording averages down
|
||||
/// to nothing, and that is the common case for a held key.
|
||||
fn has_speech(samples: &[f32]) -> bool {
|
||||
const FLOOR: f32 = 0.015;
|
||||
// Matches the VAD's lower clamp rather than a "typical" mic: a quiet
|
||||
// interface puts normal speech at ~0.004, and gating at 0.015 threw away
|
||||
// every word it captured.
|
||||
const FLOOR: f32 = 0.0025;
|
||||
let window = (audio::STT_RATE as usize) / 10;
|
||||
samples.chunks(window).any(|w| {
|
||||
let sum: f32 = w.iter().map(|s| s * s).sum();
|
||||
@@ -70,14 +97,56 @@ fn clean(raw: &str) -> String {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{clean, has_speech};
|
||||
use super::{clean, has_speech, normalize};
|
||||
use crate::audio::STT_RATE;
|
||||
|
||||
fn peak(v: &[f32]) -> f32 {
|
||||
v.iter().fold(0.0f32, |m, s| m.max(s.abs()))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_lifts_a_quiet_recording() {
|
||||
// 0.004 RMS speech peaks around 0.02, which is 30x under target —
|
||||
// exactly at the gain cap, so it just reaches full scale.
|
||||
let quiet: Vec<f32> = (0..1000).map(|i| 0.02 * ((i % 7) as f32 - 3.0) / 3.0).collect();
|
||||
assert!((peak(&normalize(&quiet)) - 0.6).abs() < 0.01);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_leaves_a_loud_recording_alone() {
|
||||
let loud: Vec<f32> = vec![0.9, -0.8, 0.7];
|
||||
assert_eq!(normalize(&loud), loud);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_caps_the_gain_so_silence_stays_silent() {
|
||||
let hiss: Vec<f32> = vec![0.0001, -0.0001];
|
||||
// 0.6/0.0001 would be 6000x; the cap keeps it at 30x.
|
||||
assert!(peak(&normalize(&hiss)) <= 0.0001 * 30.0 + 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_handles_pure_silence() {
|
||||
assert_eq!(normalize(&[0.0, 0.0]), vec![0.0, 0.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn silence_is_not_speech() {
|
||||
assert!(!has_speech(&vec![0.0f32; STT_RATE as usize]));
|
||||
// Room tone: audible on a meter, not a voice.
|
||||
assert!(!has_speech(&vec![0.004f32; STT_RATE as usize]));
|
||||
// Terra's measured room tone. Speech on the same mic reaches
|
||||
// 0.004–0.005, so the gate sits between the two.
|
||||
assert!(!has_speech(&vec![0.001f32; STT_RATE as usize]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quiet_speech_still_counts() {
|
||||
// 0.004 is what normal speech measures on terra's interface; gating
|
||||
// it out discarded every word the VAD had just captured.
|
||||
let mut s = vec![0.001f32; STT_RATE as usize];
|
||||
for v in s.iter_mut().take(STT_RATE as usize / 5) {
|
||||
*v = 0.004;
|
||||
}
|
||||
assert!(has_speech(&s));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -71,6 +71,34 @@ in
|
||||
description = "File holding the ElevenLabs API key.";
|
||||
};
|
||||
|
||||
# Mic tuning. Defaults are set for terra's AIR 192|4, which is QUIET:
|
||||
# ~0.002 ambient against ~0.0045 for normal speech. Raising the gain on
|
||||
# the interface is worth more than any value here — everything downstream,
|
||||
# whisper included, works better with headroom.
|
||||
inputDevice = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Substring of the input device name; null uses the system default.";
|
||||
};
|
||||
|
||||
vadMult = lib.mkOption {
|
||||
type = lib.types.float;
|
||||
default = 1.8;
|
||||
description = "Speech triggers above noise_floor * this.";
|
||||
};
|
||||
|
||||
vadFloor = lib.mkOption {
|
||||
type = lib.types.float;
|
||||
default = 0.0025;
|
||||
description = "Absolute lower bound on the trigger, whatever the floor says.";
|
||||
};
|
||||
|
||||
silenceMs = lib.mkOption {
|
||||
type = lib.types.int;
|
||||
default = 2000;
|
||||
description = "Silence that ends an utterance.";
|
||||
};
|
||||
|
||||
systemPrompt = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
@@ -100,8 +128,13 @@ in
|
||||
LUNA_ELEVENLABS_KEY_FILE = toString cfg.elevenLabsKeyFile;
|
||||
LUNA_WHISPER_BIN = "${pkgs.whisper-cpp-vulkan}/bin/whisper-cli";
|
||||
LUNA_WHISPER_MODEL = whisperModel;
|
||||
LUNA_VAD_MULT = toString cfg.vadMult;
|
||||
LUNA_VAD_FLOOR = toString cfg.vadFloor;
|
||||
LUNA_SILENCE_MS = toString cfg.silenceMs;
|
||||
} // lib.optionalAttrs (cfg.systemPrompt != null) {
|
||||
LUNA_SYSTEM_PROMPT = cfg.systemPrompt;
|
||||
} // lib.optionalAttrs (cfg.inputDevice != null) {
|
||||
LUNA_INPUT_DEVICE = cfg.inputDevice;
|
||||
};
|
||||
|
||||
serviceConfig = {
|
||||
|
||||
Reference in New Issue
Block a user