From 66760c1dedb93def21fd03e4e2f2fbebd23ab95a Mon Sep 17 00:00:00 2001 From: Erik Simon Date: Fri, 16 Oct 2026 00:01:48 +0200 Subject: [PATCH] feat(terra): SUPER+V mutes luna's mic without cutting her off The key now only opens and closes the mic. Her reply keeps playing, the switch takes effect mid-turn (also while a tool-call reply is still streaming), and a muted mic can't barge in. Muting mid-sentence sends what was said so far, like releasing push-to-talk. The rail panel shows MIC ON / MIC OFF and her own state even while muted. Co-Authored-By: Claude Opus 5.5 --- .../HyprChrome/Widgets/Bar/Luna/LunaPanel.qml | 7 +- .../HyprChrome/Widgets/Bar/Luna/LunaState.qml | 2 +- hosts/terra/home/hyprland.nix | 3 +- pkgs/luna-voice/src/main.rs | 264 ++++++++++++------ 4 files changed, 187 insertions(+), 89 deletions(-) diff --git a/dotfiles/quickshell/HyprChrome/Widgets/Bar/Luna/LunaPanel.qml b/dotfiles/quickshell/HyprChrome/Widgets/Bar/Luna/LunaPanel.qml index 4e1d9a9..023075f 100644 --- a/dotfiles/quickshell/HyprChrome/Widgets/Bar/Luna/LunaPanel.qml +++ b/dotfiles/quickshell/HyprChrome/Widgets/Bar/Luna/LunaPanel.qml @@ -32,7 +32,7 @@ BarPanel { panelId: "LUN" title: "LUNA" - meta: luna.active ? "VOICE" : "IDLE" + meta: luna.active ? "MIC ON" : "MIC OFF" // Sized to content like the tray, and per DENSITY: collapsed it is one // word, so reserving the expanded body's width would leave the rail @@ -61,8 +61,7 @@ BarPanel { readonly property string stateLabel: { if (!luna.connected) return "OFFLINE"; - if (!luna.active) - return "OFF"; + // Her own state first: a muted mic no longer stops her answering. switch (luna.daemonState) { case "listening": return "LISTENING"; @@ -71,7 +70,7 @@ BarPanel { case "speaking": return "SPEAKING"; default: - return "READY"; + return luna.active ? "READY" : "MUTED"; } } diff --git a/dotfiles/quickshell/HyprChrome/Widgets/Bar/Luna/LunaState.qml b/dotfiles/quickshell/HyprChrome/Widgets/Bar/Luna/LunaState.qml index a8b5734..324a416 100644 --- a/dotfiles/quickshell/HyprChrome/Widgets/Bar/Luna/LunaState.qml +++ b/dotfiles/quickshell/HyprChrome/Widgets/Bar/Luna/LunaState.qml @@ -21,7 +21,7 @@ QtObject { // "idle" | "listening" | "thinking" | "speaking" property string state: "idle" - // Voice mode itself — the widget is only on screen while this is true. + // Whether the mic is open. She still answers and speaks while it is not. property bool active: false // Set by the first line, not by the process starting: the daemon replays // its state on connect, so only a live connection produces one. diff --git a/hosts/terra/home/hyprland.nix b/hosts/terra/home/hyprland.nix index d5f6478..b5e884f 100644 --- a/hosts/terra/home/hyprland.nix +++ b/hosts/terra/home/hyprland.nix @@ -195,7 +195,8 @@ in # expand / collapse the hyprchrome bar as a whole (bind "SUPER + A" (dsp.global "quickshell:chrome")) - # toggle luna's hands-free voice mode (services/desktop/luna-voice.nix). + # mute / unmute your mic for luna (services/desktop/luna-voice.nix); + # never cuts off a reply she is already giving. # Runs the CLI rather than a quickshell GlobalShortcut so the mic can # still be switched off while the shell is reloading. (bind "SUPER + V" (dsp.exec "luna-voice toggle")) diff --git a/pkgs/luna-voice/src/main.rs b/pkgs/luna-voice/src/main.rs index 4ac61dc..8765fa1 100644 --- a/pkgs/luna-voice/src/main.rs +++ b/pkgs/luna-voice/src/main.rs @@ -19,6 +19,7 @@ use anyhow::Result; use serde_json; use config::Config; use ipc::{Command, Event}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; @@ -170,6 +171,109 @@ fn notify(body: &str) { .spawn(); } +/// The SUPER+V switch. It only opens and closes the mic: her reply is never +/// cut by it, so muting during an answer means "just listen". +struct Mic { + on: bool, + /// Running estimate of the room, updated only while not capturing, so a + /// loud sentence never raises the bar for the next one. + floor: f32, +} + +impl Mic { + /// Returns whether anything changed. `speaking` skips the beep, which + /// would otherwise land in the middle of her sentence. + fn set( + &mut self, + want: bool, + speaking: bool, + capture: &audio::Capture, + playback: &audio::Playback, + ipc: &ipc::Ipc, + ) -> bool { + if want == self.on { + return false; + } + self.on = want; + if want { + // Seeded low: a floor seeded high takes seconds to decay, and + // those are the seconds you spend talking to something that is + // not listening yet. Not sampled while she talks: that would + // measure her, not the room. + self.floor = if speaking { 0.003 } else { capture.level().max(0.003) }; + } + if !speaking { + if want { + playback.beep(880.0, 70); + } else { + playback.beep(440.0, 90); + } + } + ipc.emit(Event::Mode { active: want }); + eprintln!("luna-voice: mic {}", if want { "on" } else { "off" }); + true + } +} + +/// A command that arrives while she is answering. Returns true for Cancel, +/// the only one that stops her; the mic switch applies at once, and the +/// rest wait for the turn to end. +fn mid_turn( + cmd: Command, + speaking: bool, + mic: &mut Mic, + capture: &audio::Capture, + playback: &audio::Playback, + ipc: &ipc::Ipc, + deferred: &mut Vec, +) -> bool { + match cmd { + Command::Cancel => true, + Command::Mode { on } => { + mic.set(on.unwrap_or(!mic.on), speaking, capture, playback, ipc); + false + } + cmd => { + deferred.push(cmd); + false + } + } +} + +/// End of a hands-free utterance: transcribe it and answer. +#[allow(clippy::too_many_arguments)] +async fn reply_to( + cfg: &Config, + http: &reqwest::Client, + ipc: &Arc, + capture: &audio::Capture, + playback: &Arc, + cmd_rx: &mut mpsc::Receiver, + requeue: &mpsc::Sender, + conv: &mut Conversation, + mic: &mut Mic, + samples: Vec, +) { + ipc.emit(Event::State { value: "thinking" }); + match stt::transcribe(&cfg.whisper_bin, &cfg.whisper_model, &cfg.whisper_language, &samples).await { + Ok(heard) if !heard.is_empty() => { + run_turn(cfg, http, ipc, capture, playback, cmd_rx, requeue, conv, mic, heard).await; + } + Ok(_) => { + // Silent on purpose: in hands-free mode a false trigger happens + // whenever a door closes, and a beep for each one would be + // maddening. + ipc.emit(Event::State { value: "idle" }); + } + Err(e) => { + eprintln!("luna-voice: {e:#}"); + ipc.emit(Event::Error { message: e.to_string() }); + notify(&e.to_string()); + ipc.emit(Event::State { value: "idle" }); + } + } +} + /// Where the loop is. One utterance at a time by construction: the state /// machine is a single task, so there is no way to start a second reply while /// one is in flight. @@ -219,12 +323,9 @@ async fn daemon() -> Result<()> { } }); - // Hands-free state. The key toggles `mode`; everything inside it is - // driven by the microphone rather than by keys. - let mut mode = false; - // Running estimate of the room, updated only while not capturing, so a - // loud sentence never raises the bar for the next one. - let mut floor: f32 = 0.01; + // Hands-free state. The key only switches the mic; everything inside it + // is driven by the microphone rather than by keys. + let mut mic = Mic { on: false, floor: 0.01 }; let mut capturing = false; let mut last_voice = tokio::time::Instant::now(); let mut ticker = tokio::time::interval(Duration::from_millis(50)); @@ -238,7 +339,7 @@ async fn daemon() -> Result<()> { // Only ticks while voice mode is on. A turn is awaited inline // below, so this cannot fire while she is speaking — which is // also what stops her from hearing herself. - _ = ticker.tick(), if mode => None, + _ = ticker.tick(), if mic.on => None, }; let Some(cmd) = cmd else { @@ -248,7 +349,7 @@ async fn daemon() -> Result<()> { // threshold is too high", which is exactly the question you have // when nothing happens. ipc.emit(Event::Amplitude { rms: level }); - let threshold = root_threshold(floor, cfg.vad_mult); + let threshold = root_threshold(mic.floor, cfg.vad_mult); if !capturing { // Only samples BELOW the trigger update the floor. Feeding // every sample in lets a rising sound drag the floor up with @@ -256,7 +357,7 @@ async fn daemon() -> Result<()> { // nothing ever fires — measured: a tone peaking at 0.029 was // chased by a threshold that reached 0.018 on the way up. if level <= threshold { - floor = floor * 0.95 + level * 0.05; + mic.floor = mic.floor * 0.95 + level * 0.05; } if level > threshold { capture.start(); @@ -271,28 +372,11 @@ async fn daemon() -> Result<()> { if last_voice.elapsed() > Duration::from_millis(cfg.silence_ms) { capturing = false; let samples = capture.stop(); - ipc.emit(Event::State { value: "thinking" }); - match stt::transcribe(&cfg.whisper_bin, &cfg.whisper_model, &cfg.whisper_language, &samples).await { - Ok(heard) if !heard.is_empty() => { - run_turn( - &cfg, &http, &ipc, &capture, &playback, &mut cmd_rx, - &requeue, &mut conv, heard, - ) - .await; - } - Ok(_) => { - // Silent on purpose: in hands-free mode a false - // trigger happens whenever a door closes, and a - // beep for each one would be maddening. - ipc.emit(Event::State { value: "idle" }); - } - Err(e) => { - eprintln!("luna-voice: {e:#}"); - ipc.emit(Event::Error { message: e.to_string() }); - notify(&e.to_string()); - ipc.emit(Event::State { value: "idle" }); - } - } + reply_to( + &cfg, &http, &ipc, &capture, &playback, &mut cmd_rx, &requeue, + &mut conv, &mut mic, samples, + ) + .await; } } continue; @@ -300,26 +384,21 @@ async fn daemon() -> Result<()> { match cmd { Command::Mode { on } => { - let want = on.unwrap_or(!mode); - if want != mode { - mode = want; - if mode { - // Seeded low: a floor seeded high takes seconds to - // decay, and those are the seconds you spend - // talking to something that is not listening yet. - floor = capture.level().max(0.003); - playback.beep(880.0, 70); - } else { - if capturing { - capture.stop(); - capturing = false; - } - playback.clear(); - playback.beep(440.0, 90); - } - ipc.emit(Event::Mode { active: mode }); + if !mic.set(on.unwrap_or(!mic.on), false, &capture, &playback, &ipc) { + continue; + } + if capturing { + // Muting mid-sentence sends what was said so far, like + // letting go of a push-to-talk key. + capturing = false; + let samples = capture.stop(); + reply_to( + &cfg, &http, &ipc, &capture, &playback, &mut cmd_rx, &requeue, + &mut conv, &mut mic, samples, + ) + .await; + } else { ipc.emit(Event::State { value: "idle" }); - eprintln!("luna-voice: voice mode {}", if mode { "on" } else { "off" }); } } Command::Ptt { down: true } => { @@ -358,13 +437,19 @@ async fn daemon() -> Result<()> { continue; } - run_turn(&cfg, &http, &ipc, &capture, &playback, &mut cmd_rx, &requeue, &mut conv, heard) - .await; + run_turn( + &cfg, &http, &ipc, &capture, &playback, &mut cmd_rx, &requeue, &mut conv, + &mut mic, heard, + ) + .await; } Command::Say { text } => { ipc.emit(Event::State { value: "thinking" }); - run_turn(&cfg, &http, &ipc, &capture, &playback, &mut cmd_rx, &requeue, &mut conv, text) - .await; + run_turn( + &cfg, &http, &ipc, &capture, &playback, &mut cmd_rx, &requeue, &mut conv, + &mut mic, text, + ) + .await; } Command::Cancel => { playback.clear(); @@ -437,9 +522,10 @@ async fn run_turn( cmd_rx: &mut mpsc::Receiver, requeue: &mpsc::Sender, conv: &mut Conversation, + mic: &mut Mic, heard: String, ) { - if let Err(e) = turn(cfg, http, ipc, capture, playback, cmd_rx, requeue, conv, heard).await { + if let Err(e) = turn(cfg, http, ipc, capture, playback, cmd_rx, requeue, conv, mic, heard).await { eprintln!("luna-voice: {e:#}"); ipc.emit(Event::Error { message: e.to_string() }); notify(&e.to_string()); @@ -457,6 +543,7 @@ async fn turn( cmd_rx: &mut mpsc::Receiver, requeue: &mpsc::Sender, conv: &mut Conversation, + mic: &mut Mic, heard: String, ) -> Result<()> { let history = &mut conv.history; @@ -486,11 +573,16 @@ async fn turn( async move { tts::speak(&tts_cfg, sentence_rx, event_tx).await } }); + // Set once her first audio is queued; from then on the mic switch is + // silent. + let speaking = Arc::new(AtomicBool::new(false)); + // Audio and visemes flow to the speaker/shell on their own task so the // reply stream is never blocked by playback. let player = tokio::spawn({ let ipc = ipc.clone(); let playback = playback.clone(); + let speaking = speaking.clone(); async move { let mut started = false; while let Some(ev) = event_rx.recv().await { @@ -498,6 +590,7 @@ async fn turn( tts::SpeechEvent::Audio(pcm) => { if !started { started = true; + speaking.store(true, Ordering::Relaxed); ipc.emit(Event::SpeechStart); ipc.emit(Event::State { value: "speaking" }); } @@ -552,12 +645,24 @@ async fn turn( ipc.emit(Event::Caption { speaker: "luna", text }); } }; - while let Some(delta) = delta_rx.recv().await { - for sentence in splitter.push(&delta) { - caption(&sentence); - billed += sentence.chars().count(); - if sentence_tx.send(sentence).await.is_err() { - break; + // A reply with tool calls can stream for minutes, so commands are taken + // here too: the mic switch has to answer now, not once she is done. + let mut cancelled = false; + loop { + tokio::select! { + delta = delta_rx.recv() => { + let Some(delta) = delta else { break }; + for sentence in splitter.push(&delta) { + caption(&sentence); + billed += sentence.chars().count(); + if sentence_tx.send(sentence).await.is_err() { + break; + } + } + } + Some(cmd) = cmd_rx.recv() => { + let speaking = speaking.load(Ordering::Relaxed); + cancelled |= mid_turn(cmd, speaking, mic, capture, playback, ipc, &mut deferred); } } } @@ -580,10 +685,24 @@ async fn turn( // finished" is invisible. let grace = tokio::time::Instant::now() + Duration::from_millis(cfg.barge_in_grace_ms); loop { + // try_recv POPS, so anything not handled is destroyed: an `if let + // Ok(Cancel)` once silently ate every `say` and `toggle` sent while + // she was talking, which presents as a keypress that does nothing. + if let Ok(cmd) = cmd_rx.try_recv() { + let speaking = speaking.load(Ordering::Relaxed); + cancelled |= mid_turn(cmd, speaking, mic, capture, playback, ipc, &mut deferred); + } + if cancelled { + playback.clear(); + ipc.emit(Event::SpeechStop { reason: "cancelled" }); + break; + } if speech.is_finished() && playback.pending() == 0 { break; } - if tokio::time::Instant::now() > grace + // A muted mic does not hear you, so it cannot interrupt her either. + if mic.on + && tokio::time::Instant::now() > grace && capture.level() > floor * cfg.barge_in_mult { playback.clear(); @@ -595,27 +714,6 @@ async fn turn( )); break; } - // Commands arriving mid-playback. try_recv POPS, so anything not - // handled here is destroyed: an `if let Ok(Cancel)` silently ate - // every `say`, `toggle` and `mode off` sent while she was talking, - // which presents as a keypress that does nothing. - match cmd_rx.try_recv() { - Ok(Command::Cancel) => { - playback.clear(); - ipc.emit(Event::SpeechStop { reason: "cancelled" }); - break; - } - // Leaving voice mode has to shut her up immediately; the mode - // change itself is replayed below. - Ok(cmd @ Command::Mode { .. }) => { - playback.clear(); - ipc.emit(Event::SpeechStop { reason: "cancelled" }); - deferred.push(cmd); - break; - } - Ok(cmd) => deferred.push(cmd), - Err(_) => {} - } ipc.emit(Event::Amplitude { rms: capture.level() }); tokio::time::sleep(Duration::from_millis(50)).await; }