feat(terra): SUPER+V mutes luna's mic without cutting her off
The key now only opens and closes the mic. Her reply keeps playing, the switch takes effect mid-turn (also while a tool-call reply is still streaming), and a muted mic can't barge in. Muting mid-sentence sends what was said so far, like releasing push-to-talk. The rail panel shows MIC ON / MIC OFF and her own state even while muted. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -32,7 +32,7 @@ BarPanel {
|
||||
|
||||
panelId: "LUN"
|
||||
title: "LUNA"
|
||||
meta: luna.active ? "VOICE" : "IDLE"
|
||||
meta: luna.active ? "MIC ON" : "MIC OFF"
|
||||
|
||||
// Sized to content like the tray, and per DENSITY: collapsed it is one
|
||||
// word, so reserving the expanded body's width would leave the rail
|
||||
@@ -61,8 +61,7 @@ BarPanel {
|
||||
readonly property string stateLabel: {
|
||||
if (!luna.connected)
|
||||
return "OFFLINE";
|
||||
if (!luna.active)
|
||||
return "OFF";
|
||||
// Her own state first: a muted mic no longer stops her answering.
|
||||
switch (luna.daemonState) {
|
||||
case "listening":
|
||||
return "LISTENING";
|
||||
@@ -71,7 +70,7 @@ BarPanel {
|
||||
case "speaking":
|
||||
return "SPEAKING";
|
||||
default:
|
||||
return "READY";
|
||||
return luna.active ? "READY" : "MUTED";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ QtObject {
|
||||
|
||||
// "idle" | "listening" | "thinking" | "speaking"
|
||||
property string state: "idle"
|
||||
// Voice mode itself — the widget is only on screen while this is true.
|
||||
// Whether the mic is open. She still answers and speaks while it is not.
|
||||
property bool active: false
|
||||
// Set by the first line, not by the process starting: the daemon replays
|
||||
// its state on connect, so only a live connection produces one.
|
||||
|
||||
@@ -195,7 +195,8 @@ in
|
||||
# expand / collapse the hyprchrome bar as a whole
|
||||
(bind "SUPER + A" (dsp.global "quickshell:chrome"))
|
||||
|
||||
# toggle luna's hands-free voice mode (services/desktop/luna-voice.nix).
|
||||
# mute / unmute your mic for luna (services/desktop/luna-voice.nix);
|
||||
# never cuts off a reply she is already giving.
|
||||
# Runs the CLI rather than a quickshell GlobalShortcut so the mic can
|
||||
# still be switched off while the shell is reloading.
|
||||
(bind "SUPER + V" (dsp.exec "luna-voice toggle"))
|
||||
|
||||
+181
-83
@@ -19,6 +19,7 @@ use anyhow::Result;
|
||||
use serde_json;
|
||||
use config::Config;
|
||||
use ipc::{Command, Event};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::mpsc;
|
||||
@@ -170,6 +171,109 @@ fn notify(body: &str) {
|
||||
.spawn();
|
||||
}
|
||||
|
||||
/// The SUPER+V switch. It only opens and closes the mic: her reply is never
|
||||
/// cut by it, so muting during an answer means "just listen".
|
||||
struct Mic {
|
||||
on: bool,
|
||||
/// Running estimate of the room, updated only while not capturing, so a
|
||||
/// loud sentence never raises the bar for the next one.
|
||||
floor: f32,
|
||||
}
|
||||
|
||||
impl Mic {
|
||||
/// Returns whether anything changed. `speaking` skips the beep, which
|
||||
/// would otherwise land in the middle of her sentence.
|
||||
fn set(
|
||||
&mut self,
|
||||
want: bool,
|
||||
speaking: bool,
|
||||
capture: &audio::Capture,
|
||||
playback: &audio::Playback,
|
||||
ipc: &ipc::Ipc,
|
||||
) -> bool {
|
||||
if want == self.on {
|
||||
return false;
|
||||
}
|
||||
self.on = want;
|
||||
if want {
|
||||
// Seeded low: a floor seeded high takes seconds to decay, and
|
||||
// those are the seconds you spend talking to something that is
|
||||
// not listening yet. Not sampled while she talks: that would
|
||||
// measure her, not the room.
|
||||
self.floor = if speaking { 0.003 } else { capture.level().max(0.003) };
|
||||
}
|
||||
if !speaking {
|
||||
if want {
|
||||
playback.beep(880.0, 70);
|
||||
} else {
|
||||
playback.beep(440.0, 90);
|
||||
}
|
||||
}
|
||||
ipc.emit(Event::Mode { active: want });
|
||||
eprintln!("luna-voice: mic {}", if want { "on" } else { "off" });
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/// A command that arrives while she is answering. Returns true for Cancel,
|
||||
/// the only one that stops her; the mic switch applies at once, and the
|
||||
/// rest wait for the turn to end.
|
||||
fn mid_turn(
|
||||
cmd: Command,
|
||||
speaking: bool,
|
||||
mic: &mut Mic,
|
||||
capture: &audio::Capture,
|
||||
playback: &audio::Playback,
|
||||
ipc: &ipc::Ipc,
|
||||
deferred: &mut Vec<Command>,
|
||||
) -> bool {
|
||||
match cmd {
|
||||
Command::Cancel => true,
|
||||
Command::Mode { on } => {
|
||||
mic.set(on.unwrap_or(!mic.on), speaking, capture, playback, ipc);
|
||||
false
|
||||
}
|
||||
cmd => {
|
||||
deferred.push(cmd);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// End of a hands-free utterance: transcribe it and answer.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn reply_to(
|
||||
cfg: &Config,
|
||||
http: &reqwest::Client,
|
||||
ipc: &Arc<ipc::Ipc>,
|
||||
capture: &audio::Capture,
|
||||
playback: &Arc<audio::Playback>,
|
||||
cmd_rx: &mut mpsc::Receiver<Command>,
|
||||
requeue: &mpsc::Sender<Command>,
|
||||
conv: &mut Conversation,
|
||||
mic: &mut Mic,
|
||||
samples: Vec<f32>,
|
||||
) {
|
||||
ipc.emit(Event::State { value: "thinking" });
|
||||
match stt::transcribe(&cfg.whisper_bin, &cfg.whisper_model, &cfg.whisper_language, &samples).await {
|
||||
Ok(heard) if !heard.is_empty() => {
|
||||
run_turn(cfg, http, ipc, capture, playback, cmd_rx, requeue, conv, mic, heard).await;
|
||||
}
|
||||
Ok(_) => {
|
||||
// Silent on purpose: in hands-free mode a false trigger happens
|
||||
// whenever a door closes, and a beep for each one would be
|
||||
// maddening.
|
||||
ipc.emit(Event::State { value: "idle" });
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("luna-voice: {e:#}");
|
||||
ipc.emit(Event::Error { message: e.to_string() });
|
||||
notify(&e.to_string());
|
||||
ipc.emit(Event::State { value: "idle" });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Where the loop is. One utterance at a time by construction: the state
|
||||
/// machine is a single task, so there is no way to start a second reply while
|
||||
/// one is in flight.
|
||||
@@ -219,12 +323,9 @@ async fn daemon() -> Result<()> {
|
||||
}
|
||||
});
|
||||
|
||||
// Hands-free state. The key toggles `mode`; everything inside it is
|
||||
// driven by the microphone rather than by keys.
|
||||
let mut mode = false;
|
||||
// Running estimate of the room, updated only while not capturing, so a
|
||||
// loud sentence never raises the bar for the next one.
|
||||
let mut floor: f32 = 0.01;
|
||||
// Hands-free state. The key only switches the mic; everything inside it
|
||||
// is driven by the microphone rather than by keys.
|
||||
let mut mic = Mic { on: false, floor: 0.01 };
|
||||
let mut capturing = false;
|
||||
let mut last_voice = tokio::time::Instant::now();
|
||||
let mut ticker = tokio::time::interval(Duration::from_millis(50));
|
||||
@@ -238,7 +339,7 @@ async fn daemon() -> Result<()> {
|
||||
// Only ticks while voice mode is on. A turn is awaited inline
|
||||
// below, so this cannot fire while she is speaking — which is
|
||||
// also what stops her from hearing herself.
|
||||
_ = ticker.tick(), if mode => None,
|
||||
_ = ticker.tick(), if mic.on => None,
|
||||
};
|
||||
|
||||
let Some(cmd) = cmd else {
|
||||
@@ -248,7 +349,7 @@ async fn daemon() -> Result<()> {
|
||||
// threshold is too high", which is exactly the question you have
|
||||
// when nothing happens.
|
||||
ipc.emit(Event::Amplitude { rms: level });
|
||||
let threshold = root_threshold(floor, cfg.vad_mult);
|
||||
let threshold = root_threshold(mic.floor, cfg.vad_mult);
|
||||
if !capturing {
|
||||
// Only samples BELOW the trigger update the floor. Feeding
|
||||
// every sample in lets a rising sound drag the floor up with
|
||||
@@ -256,7 +357,7 @@ async fn daemon() -> Result<()> {
|
||||
// nothing ever fires — measured: a tone peaking at 0.029 was
|
||||
// chased by a threshold that reached 0.018 on the way up.
|
||||
if level <= threshold {
|
||||
floor = floor * 0.95 + level * 0.05;
|
||||
mic.floor = mic.floor * 0.95 + level * 0.05;
|
||||
}
|
||||
if level > threshold {
|
||||
capture.start();
|
||||
@@ -271,28 +372,11 @@ async fn daemon() -> Result<()> {
|
||||
if last_voice.elapsed() > Duration::from_millis(cfg.silence_ms) {
|
||||
capturing = false;
|
||||
let samples = capture.stop();
|
||||
ipc.emit(Event::State { value: "thinking" });
|
||||
match stt::transcribe(&cfg.whisper_bin, &cfg.whisper_model, &cfg.whisper_language, &samples).await {
|
||||
Ok(heard) if !heard.is_empty() => {
|
||||
run_turn(
|
||||
&cfg, &http, &ipc, &capture, &playback, &mut cmd_rx,
|
||||
&requeue, &mut conv, heard,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Ok(_) => {
|
||||
// Silent on purpose: in hands-free mode a false
|
||||
// trigger happens whenever a door closes, and a
|
||||
// beep for each one would be maddening.
|
||||
ipc.emit(Event::State { value: "idle" });
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("luna-voice: {e:#}");
|
||||
ipc.emit(Event::Error { message: e.to_string() });
|
||||
notify(&e.to_string());
|
||||
ipc.emit(Event::State { value: "idle" });
|
||||
}
|
||||
}
|
||||
reply_to(
|
||||
&cfg, &http, &ipc, &capture, &playback, &mut cmd_rx, &requeue,
|
||||
&mut conv, &mut mic, samples,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
@@ -300,26 +384,21 @@ async fn daemon() -> Result<()> {
|
||||
|
||||
match cmd {
|
||||
Command::Mode { on } => {
|
||||
let want = on.unwrap_or(!mode);
|
||||
if want != mode {
|
||||
mode = want;
|
||||
if mode {
|
||||
// Seeded low: a floor seeded high takes seconds to
|
||||
// decay, and those are the seconds you spend
|
||||
// talking to something that is not listening yet.
|
||||
floor = capture.level().max(0.003);
|
||||
playback.beep(880.0, 70);
|
||||
} else {
|
||||
if capturing {
|
||||
capture.stop();
|
||||
capturing = false;
|
||||
}
|
||||
playback.clear();
|
||||
playback.beep(440.0, 90);
|
||||
}
|
||||
ipc.emit(Event::Mode { active: mode });
|
||||
if !mic.set(on.unwrap_or(!mic.on), false, &capture, &playback, &ipc) {
|
||||
continue;
|
||||
}
|
||||
if capturing {
|
||||
// Muting mid-sentence sends what was said so far, like
|
||||
// letting go of a push-to-talk key.
|
||||
capturing = false;
|
||||
let samples = capture.stop();
|
||||
reply_to(
|
||||
&cfg, &http, &ipc, &capture, &playback, &mut cmd_rx, &requeue,
|
||||
&mut conv, &mut mic, samples,
|
||||
)
|
||||
.await;
|
||||
} else {
|
||||
ipc.emit(Event::State { value: "idle" });
|
||||
eprintln!("luna-voice: voice mode {}", if mode { "on" } else { "off" });
|
||||
}
|
||||
}
|
||||
Command::Ptt { down: true } => {
|
||||
@@ -358,13 +437,19 @@ async fn daemon() -> Result<()> {
|
||||
continue;
|
||||
}
|
||||
|
||||
run_turn(&cfg, &http, &ipc, &capture, &playback, &mut cmd_rx, &requeue, &mut conv, heard)
|
||||
.await;
|
||||
run_turn(
|
||||
&cfg, &http, &ipc, &capture, &playback, &mut cmd_rx, &requeue, &mut conv,
|
||||
&mut mic, heard,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Command::Say { text } => {
|
||||
ipc.emit(Event::State { value: "thinking" });
|
||||
run_turn(&cfg, &http, &ipc, &capture, &playback, &mut cmd_rx, &requeue, &mut conv, text)
|
||||
.await;
|
||||
run_turn(
|
||||
&cfg, &http, &ipc, &capture, &playback, &mut cmd_rx, &requeue, &mut conv,
|
||||
&mut mic, text,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Command::Cancel => {
|
||||
playback.clear();
|
||||
@@ -437,9 +522,10 @@ async fn run_turn(
|
||||
cmd_rx: &mut mpsc::Receiver<Command>,
|
||||
requeue: &mpsc::Sender<Command>,
|
||||
conv: &mut Conversation,
|
||||
mic: &mut Mic,
|
||||
heard: String,
|
||||
) {
|
||||
if let Err(e) = turn(cfg, http, ipc, capture, playback, cmd_rx, requeue, conv, heard).await {
|
||||
if let Err(e) = turn(cfg, http, ipc, capture, playback, cmd_rx, requeue, conv, mic, heard).await {
|
||||
eprintln!("luna-voice: {e:#}");
|
||||
ipc.emit(Event::Error { message: e.to_string() });
|
||||
notify(&e.to_string());
|
||||
@@ -457,6 +543,7 @@ async fn turn(
|
||||
cmd_rx: &mut mpsc::Receiver<Command>,
|
||||
requeue: &mpsc::Sender<Command>,
|
||||
conv: &mut Conversation,
|
||||
mic: &mut Mic,
|
||||
heard: String,
|
||||
) -> Result<()> {
|
||||
let history = &mut conv.history;
|
||||
@@ -486,11 +573,16 @@ async fn turn(
|
||||
async move { tts::speak(&tts_cfg, sentence_rx, event_tx).await }
|
||||
});
|
||||
|
||||
// Set once her first audio is queued; from then on the mic switch is
|
||||
// silent.
|
||||
let speaking = Arc::new(AtomicBool::new(false));
|
||||
|
||||
// Audio and visemes flow to the speaker/shell on their own task so the
|
||||
// reply stream is never blocked by playback.
|
||||
let player = tokio::spawn({
|
||||
let ipc = ipc.clone();
|
||||
let playback = playback.clone();
|
||||
let speaking = speaking.clone();
|
||||
async move {
|
||||
let mut started = false;
|
||||
while let Some(ev) = event_rx.recv().await {
|
||||
@@ -498,6 +590,7 @@ async fn turn(
|
||||
tts::SpeechEvent::Audio(pcm) => {
|
||||
if !started {
|
||||
started = true;
|
||||
speaking.store(true, Ordering::Relaxed);
|
||||
ipc.emit(Event::SpeechStart);
|
||||
ipc.emit(Event::State { value: "speaking" });
|
||||
}
|
||||
@@ -552,12 +645,24 @@ async fn turn(
|
||||
ipc.emit(Event::Caption { speaker: "luna", text });
|
||||
}
|
||||
};
|
||||
while let Some(delta) = delta_rx.recv().await {
|
||||
for sentence in splitter.push(&delta) {
|
||||
caption(&sentence);
|
||||
billed += sentence.chars().count();
|
||||
if sentence_tx.send(sentence).await.is_err() {
|
||||
break;
|
||||
// A reply with tool calls can stream for minutes, so commands are taken
|
||||
// here too: the mic switch has to answer now, not once she is done.
|
||||
let mut cancelled = false;
|
||||
loop {
|
||||
tokio::select! {
|
||||
delta = delta_rx.recv() => {
|
||||
let Some(delta) = delta else { break };
|
||||
for sentence in splitter.push(&delta) {
|
||||
caption(&sentence);
|
||||
billed += sentence.chars().count();
|
||||
if sentence_tx.send(sentence).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(cmd) = cmd_rx.recv() => {
|
||||
let speaking = speaking.load(Ordering::Relaxed);
|
||||
cancelled |= mid_turn(cmd, speaking, mic, capture, playback, ipc, &mut deferred);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -580,10 +685,24 @@ async fn turn(
|
||||
// finished" is invisible.
|
||||
let grace = tokio::time::Instant::now() + Duration::from_millis(cfg.barge_in_grace_ms);
|
||||
loop {
|
||||
// try_recv POPS, so anything not handled is destroyed: an `if let
|
||||
// Ok(Cancel)` once silently ate every `say` and `toggle` sent while
|
||||
// she was talking, which presents as a keypress that does nothing.
|
||||
if let Ok(cmd) = cmd_rx.try_recv() {
|
||||
let speaking = speaking.load(Ordering::Relaxed);
|
||||
cancelled |= mid_turn(cmd, speaking, mic, capture, playback, ipc, &mut deferred);
|
||||
}
|
||||
if cancelled {
|
||||
playback.clear();
|
||||
ipc.emit(Event::SpeechStop { reason: "cancelled" });
|
||||
break;
|
||||
}
|
||||
if speech.is_finished() && playback.pending() == 0 {
|
||||
break;
|
||||
}
|
||||
if tokio::time::Instant::now() > grace
|
||||
// A muted mic does not hear you, so it cannot interrupt her either.
|
||||
if mic.on
|
||||
&& tokio::time::Instant::now() > grace
|
||||
&& capture.level() > floor * cfg.barge_in_mult
|
||||
{
|
||||
playback.clear();
|
||||
@@ -595,27 +714,6 @@ async fn turn(
|
||||
));
|
||||
break;
|
||||
}
|
||||
// Commands arriving mid-playback. try_recv POPS, so anything not
|
||||
// handled here is destroyed: an `if let Ok(Cancel)` silently ate
|
||||
// every `say`, `toggle` and `mode off` sent while she was talking,
|
||||
// which presents as a keypress that does nothing.
|
||||
match cmd_rx.try_recv() {
|
||||
Ok(Command::Cancel) => {
|
||||
playback.clear();
|
||||
ipc.emit(Event::SpeechStop { reason: "cancelled" });
|
||||
break;
|
||||
}
|
||||
// Leaving voice mode has to shut her up immediately; the mode
|
||||
// change itself is replayed below.
|
||||
Ok(cmd @ Command::Mode { .. }) => {
|
||||
playback.clear();
|
||||
ipc.emit(Event::SpeechStop { reason: "cancelled" });
|
||||
deferred.push(cmd);
|
||||
break;
|
||||
}
|
||||
Ok(cmd) => deferred.push(cmd),
|
||||
Err(_) => {}
|
||||
}
|
||||
ipc.emit(Event::Amplitude { rms: capture.level() });
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user