feat(terra): luna v3 voice, channel marker and rail panel

- TTS defaults to eleven_v3_conversational over the Text to Dialogue
  socket: performs audio tags and gets her German right. Flash still
  works via the TTS socket.
- Every spoken message is prefixed with a [voice] marker so luna's
  profile on mars can tell the channel apart.
- LunaIndicator/LunaCall replaced by a LunaPanel in the bar rail, with
  captions; headless test updated to match.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-09-25 00:52:03 +02:00
co-authored by Claude Opus 5.5
parent 386a46c4fe
commit e4ee637ccd
13 changed files with 675 additions and 467 deletions
@@ -4,7 +4,7 @@ import QtQuick.Layouts
import Quickshell
import Quickshell.Wayland
import qs.HyprChrome.Theme
import qs.HyprChrome.Widgets.Bar.Debug
import qs.HyprChrome.Widgets.Bar.Luna
import qs.HyprChrome.Widgets.Bar.Host
import qs.HyprChrome.Widgets.Bar.Vitals
import qs.HyprChrome.Widgets.Bar.Workspaces
@@ -87,8 +87,8 @@ PanelWindow {
//
// The zone still follows the target height, so tiled windows reflow once
// per toggle, at the start, and slide while the panels animate.
readonly property real expandedContent: Math.max(hostPanel.expandedHeight, workspacesPanel.expandedHeight, vitalsPanel.expandedHeight, testPanel.expandedHeight, trayPanel.expandedHeight) + window.margin * 2
readonly property real contentHeight: Math.max(hostPanel.targetHeight, workspacesPanel.targetHeight, vitalsPanel.targetHeight, testPanel.targetHeight, trayPanel.targetHeight) + window.margin * 2
readonly property real expandedContent: Math.max(hostPanel.expandedHeight, workspacesPanel.expandedHeight, vitalsPanel.expandedHeight, lunaPanel.expandedHeight, trayPanel.expandedHeight) + window.margin * 2
readonly property real contentHeight: Math.max(hostPanel.targetHeight, workspacesPanel.targetHeight, vitalsPanel.targetHeight, lunaPanel.targetHeight, trayPanel.targetHeight) + window.margin * 2
implicitHeight: Math.round(window.expandedContent)
@@ -157,8 +157,8 @@ PanelWindow {
// and a trace at each end.
Item { Layout.fillWidth: true }
TestPanel {
id: testPanel
LunaPanel {
id: lunaPanel
expanded: window.expanded
toggleOnClick: false
@@ -0,0 +1,200 @@
pragma ComponentBehavior: Bound
import QtQuick
import qs.HyprChrome.Theme
import qs.HyprChrome.Widgets.Bar.Panels
// luna's voice loop on the rail (services/desktop/luna-voice.nix).
//
// Always present, unlike the daemon it watches: the panel is where you look
// to find out whether she is listening, so it has to be there when the answer
// is "no" — including when luna-voiced is not running at all. SUPER+V toggles
// LISTENING, not this.
//
// Collapsed it is the state and nothing else, because that is the one fact
// worth a glance; expanded it carries the last exchange, which is the thing
// you actually want during the seconds her reply takes to arrive.
BarPanel {
id: luna
// Overridable so the headless render can stage every state without a
// daemon; in the rail they track the socket below.
property string daemonState: state.state
property bool active: state.active
property bool connected: state.connected
property real rms: state.rms
property string userText: state.userText
property string lunaText: state.lunaText
// Instantiated here rather than in the shell: the panel is the only
// reader, and it is a second reader that would make this shell state.
readonly property var state: LunaState {}
panelId: "LUN"
title: "LUNA"
meta: luna.active ? "VOICE" : "IDLE"
// Sized to content like the tray, and per DENSITY: collapsed it is one
// word, so reserving the expanded body's width would leave the rail
// carrying a mostly empty panel for the state it spends most of its time
// in. It floats between two spacers, so the width change costs no other
// panel anything.
implicitWidth: luna.expanded
? Math.max(luna.headerMinWidth, luna.padding * 2 + body.implicitWidth)
: Math.max(luna.headerMinWidth, luna.briefLeft + brief.implicitWidth + luna.padding)
readonly property color stateColor: {
if (!luna.connected)
return Theme.disabled;
switch (luna.daemonState) {
case "listening":
return Theme.accent;
case "thinking":
return Theme.muted;
case "speaking":
return Theme.accentSoft;
default:
return luna.active ? Theme.text : Theme.raised;
}
}
readonly property string stateLabel: {
if (!luna.connected)
return "OFFLINE";
if (!luna.active)
return "OFF";
switch (luna.daemonState) {
case "listening":
return "LISTENING";
case "thinking":
return "THINKING";
case "speaking":
return "SPEAKING";
default:
return "READY";
}
}
summary: Row {
id: brief
spacing: 6
// Breathes only while the mic is open, so "she is listening to me
// right now" is legible without reading the word next to it.
Rectangle {
width: 7
height: 7
radius: 4
anchors.verticalCenter: parent.verticalCenter
color: luna.stateColor
SequentialAnimation on opacity {
running: luna.daemonState === "listening"
loops: Animation.Infinite
alwaysRunToEnd: true
NumberAnimation { to: 0.25; duration: 700; easing.type: Easing.InOutQuad }
NumberAnimation { to: 1.0; duration: 700; easing.type: Easing.InOutQuad }
}
}
Text {
anchors.verticalCenter: parent.verticalCenter
text: luna.stateLabel
color: luna.stateColor
font.family: Theme.displayFont
font.pixelSize: 13
font.bold: true
}
}
Column {
id: body
spacing: 6
// Level meter: live whenever voice mode is on, not only once a
// trigger has fired, so a mic too quiet to reach the threshold still
// shows movement instead of looking broken.
Row {
id: meter
width: 260
height: 18
spacing: 2
readonly property int count: 28
readonly property bool sweeping: luna.daemonState === "thinking"
property real phase: 0
NumberAnimation on phase {
running: meter.sweeping
loops: Animation.Infinite
from: 0
to: 1
duration: 1200
}
Repeater {
model: meter.count
Rectangle {
required property int index
width: (meter.width - (meter.count - 1) * meter.spacing) / meter.count
height: {
if (meter.sweeping) {
const pos = index / (meter.count - 1);
return 2 + 14 * Math.max(0, 1 - Math.abs(pos - meter.phase) * 6);
}
if (!luna.active && luna.daemonState !== "speaking")
return 2;
// sqrt opens up the quiet end, which on this mic is
// where all of the speech actually sits.
const level = Math.min(1, Math.sqrt(luna.rms) * 6.0);
const centre = 1 - Math.abs(index / (meter.count - 1) - 0.5) * 1.6;
return 2 + 16 * level * Math.max(0.15, centre);
}
anchors.bottom: parent.bottom
color: luna.stateColor
opacity: meter.sweeping ? 0.9 : 0.5 + 0.5 * (height / 18)
Behavior on height {
enabled: !meter.sweeping
NumberAnimation { duration: 70; easing.type: Easing.OutQuad }
}
}
}
}
Text {
width: 260
visible: luna.userText !== ""
text: `> ${luna.userText}`
color: Theme.muted
font.family: Theme.microFont
font.pixelSize: 10
wrapMode: Text.WordWrap
maximumLineCount: 2
elide: Text.ElideRight
}
Text {
width: 260
visible: luna.lunaText !== ""
text: luna.lunaText
color: Theme.text
font.family: Theme.microFont
font.pixelSize: 11
wrapMode: Text.WordWrap
maximumLineCount: 4
elide: Text.ElideRight
}
Text {
visible: luna.userText === "" && luna.lunaText === ""
text: luna.connected ? "SUPER+V TO TALK" : "DAEMON NOT RUNNING"
color: Theme.muted
font.family: Theme.microFont
font.pixelSize: 9
font.letterSpacing: 1.4
}
}
}
@@ -9,7 +9,6 @@ import qs.HyprChrome.Widgets.Polkit
import qs.HyprChrome.Widgets.Launcher
import qs.HyprChrome.Widgets.Askpass
import qs.HyprChrome.Widgets.Notifications
import qs.HyprChrome.Widgets.Luna
// The hyprchrome shell: owns everything the rail's surfaces have to agree on,
// and instantiates them.
@@ -217,25 +216,6 @@ Scope {
screen: shell.targetScreen
}
// luna's voice-mode readout. Beside the notification stack rather than
// among the modals: it never takes the keyboard and raises no scrim, so
// it is deliberately NOT part of `modalOpen`. It does read `targetScreen`,
// which is what makes it belong to the shell at all.
LunaState {
id: lunaState
}
LunaCall {
state: lunaState
targetScreen: shell.targetScreen
// Same source the backdrop uses. The rail's exclusive zone does NOT
// push this surface down on its own — measured: the bar occupies
// y 0..147 and the readout still landed at y=8 — so the offset is
// passed explicitly rather than left to the compositor. That makes
// the bar's height shell state by the usual rule: a second reader.
barHeight: bar.contentHeight
}
// Polkit authentication agent. It registers for this logind session on
// creation, so it replaces hyprpolkitagent rather than coexisting with it —
// only one agent may hold a session (see hosts/terra/home/hyprland.nix).
@@ -1,60 +0,0 @@
pragma ComponentBehavior: Bound
import QtQuick
import Quickshell
import Quickshell.Wayland
import qs.HyprChrome.Widgets.Luna
// Voice-mode surface: top-left, directly under the rail.
//
// Thin by design — the visuals are all in LunaIndicator so they stay
// headless-renderable. Anchoring top+left is enough to sit under the bar:
// the rail sets an exclusiveZone of its collapsed height, so the compositor
// places this below it without either surface knowing the other's metrics.
//
// NOT a modal. She never takes the keyboard — the whole point is that you
// keep working while talking — so this does not join the shell's `modalOpen`
// and raises no scrim.
PanelWindow {
id: window
required property var state
required property var targetScreen
// The rail's settled height, from the shell. Not the compositor's job
// here: the bar's exclusive zone leaves this surface at the top of the
// output, so the gap is opened by hand.
required property real barHeight
screen: window.targetScreen
visible: window.state.active
anchors { top: true; left: true }
margins { left: 8; top: window.barHeight + 8 }
implicitWidth: indicator.implicitWidth
implicitHeight: indicator.implicitHeight
color: "transparent"
// Below the rail's own layer: this is a readout, not a surface that
// should ever cover the bar.
WlrLayershell.layer: WlrLayer.Top
// A readout that stole focus would break typing the moment she answered.
WlrLayershell.keyboardFocus: WlrKeyboardFocus.None
// Normal with a zero zone: reserve nothing of our own, but still RESPECT
// the rail's exclusion so the compositor places this under it. Ignore
// would skip other surfaces' zones too, which put it over the bar.
exclusionMode: ExclusionMode.Normal
exclusiveZone: 0
LunaIndicator {
id: indicator
anchors.fill: parent
state: window.state.state
active: window.state.active
connected: window.state.connected
rms: window.state.rms
userText: window.state.userText
lunaText: window.state.lunaText
}
}
@@ -1,229 +0,0 @@
pragma ComponentBehavior: Bound
import QtQuick
import QtQuick.Shapes
import qs.HyprChrome.Theme
// The voice-mode readout: whether luna is listening, thinking or talking.
//
// Item-rooted with no window of its own so tools/quickshell-preview/render.sh
// can shoot it offscreen — the PanelWindow lives in LunaCall.qml. A standalone
// 3D avatar replaces the meter later; the states and their colours are the
// part that has to read correctly first.
Item {
id: root
// Driven by LunaState; defaulted so the headless render and the
// DebugWindow can stage this without a daemon.
property string state: "idle"
property bool active: true
property bool connected: true
property real rms: 0
property string userText: ""
property string lunaText: ""
// Metrics are per-component in this tree; Theme carries colour and type only.
readonly property int pad: 8
readonly property int chamfer: 10
implicitWidth: 260
implicitHeight: content.implicitHeight + root.pad * 2
// Not readonly: a Behavior cannot attach to a readonly property, and the
// colour crossfade is what keeps state changes from snapping.
property color stateColor: {
if (!root.connected)
return Theme.disabled;
switch (root.state) {
case "listening":
return Theme.accent;
case "thinking":
return Theme.muted;
case "speaking":
return Theme.accentSoft;
default:
return Theme.raised;
}
}
readonly property string stateLabel: {
if (!root.connected)
return "OFFLINE";
switch (root.state) {
case "listening":
return "LISTENING";
case "thinking":
return "THINKING";
case "speaking":
return "SPEAKING";
default:
return "READY";
}
}
Behavior on stateColor {
ColorAnimation { duration: 180 }
}
// The card is ONE chamfered path, not a rectangle with a triangle drawn
// over its corner — that leaves the square border showing underneath and
// reads as an artifact rather than a cut.
Shape {
anchors.fill: parent
preferredRendererType: Shape.CurveRenderer
opacity: 0.96
ShapePath {
strokeWidth: 1
strokeColor: root.stateColor
fillColor: Theme.surface
startX: 0
startY: 0
PathLine { x: root.width - root.chamfer; y: 0 }
PathLine { x: root.width; y: root.chamfer }
PathLine { x: root.width; y: root.height }
PathLine { x: 0; y: root.height }
PathLine { x: 0; y: 0 }
}
}
Column {
id: content
anchors.fill: parent
anchors.margins: root.pad
spacing: root.pad
Row {
spacing: root.pad
// The one element that is alive while nothing else moves: a dot
// that breathes in listening and holds steady otherwise, so "the
// mic is open" is legible from across the room.
Rectangle {
id: dot
width: 8
height: 8
radius: 4
anchors.verticalCenter: parent.verticalCenter
color: root.stateColor
SequentialAnimation on opacity {
running: root.state === "listening"
loops: Animation.Infinite
alwaysRunToEnd: true
NumberAnimation { to: 0.25; duration: 700; easing.type: Easing.InOutQuad }
NumberAnimation { to: 1.0; duration: 700; easing.type: Easing.InOutQuad }
}
// An animation that stops mid-fade would leave the dot dim
// and looking half-dead.
onVisibleChanged: if (root.state !== "listening") opacity = 1
}
Text {
anchors.verticalCenter: parent.verticalCenter
text: "LUNA"
color: Theme.text
font.family: Theme.displayFont
font.pixelSize: 12
font.letterSpacing: 2
}
Text {
anchors.verticalCenter: parent.verticalCenter
text: root.stateLabel
color: root.stateColor
font.family: Theme.microFont
font.pixelSize: 10
font.letterSpacing: 1
}
}
// Level meter. Bars react to rms while listening or speaking; during
// thinking they run a sweep instead, because a dead meter and a
// crashed daemon look identical and her replies take seconds.
Row {
id: meter
width: parent.width
height: 22
spacing: 2
readonly property int count: 24
readonly property bool sweeping: root.state === "thinking"
property real phase: 0
NumberAnimation on phase {
running: meter.sweeping
loops: Animation.Infinite
from: 0
to: 1
duration: 1200
}
Repeater {
model: meter.count
Rectangle {
required property int index
width: (meter.width - (meter.count - 1) * meter.spacing) / meter.count
height: {
if (meter.sweeping) {
// A travelling bump rather than a bar graph.
const pos = index / (meter.count - 1);
const d = Math.abs(pos - meter.phase);
return 3 + 16 * Math.max(0, 1 - d * 6);
}
// Live whenever voice mode is on, not only once a
// trigger has fired: "is it hearing me at all" has to
// be answerable BEFORE anything happens, or a mic
// that is too quiet looks exactly like a broken one.
if (!root.active && root.state !== "speaking")
return 3;
// rms is small and non-linear; sqrt opens up the
// quiet end where speech actually sits.
const level = Math.min(1, Math.sqrt(root.rms) * 3.2);
// Centre bars taller, so it reads as a voice rather
// than a progress bar.
const centre = 1 - Math.abs(index / (meter.count - 1) - 0.5) * 1.6;
return 3 + 19 * level * Math.max(0.15, centre);
}
anchors.bottom: parent.bottom
color: root.stateColor
opacity: meter.sweeping ? 0.9 : 0.55 + 0.45 * (height / 22)
Behavior on height {
enabled: !meter.sweeping
NumberAnimation { duration: 70; easing.type: Easing.OutQuad }
}
}
}
}
// Captions: hers leads the audio (the daemon emits each sentence as
// it goes to the synthesiser), which is what makes a 5s reply feel
// like it has started.
Text {
width: parent.width
visible: root.userText !== ""
text: `> ${root.userText}`
color: Theme.muted
font.family: Theme.microFont
font.pixelSize: 10
wrapMode: Text.WordWrap
maximumLineCount: 2
elide: Text.ElideRight
}
Text {
width: parent.width
visible: root.lunaText !== ""
text: root.lunaText
color: Theme.text
font.family: Theme.microFont
font.pixelSize: 11
wrapMode: Text.WordWrap
maximumLineCount: 3
elide: Text.ElideRight
}
}
}
@@ -1,55 +0,0 @@
import QtQuick
import qs.HyprChrome.Widgets.Luna
// Offscreen render of every voice-mode state, plus the disconnected one —
// the states are the whole point of this widget, and a broken colour or a
// collapsed meter is invisible in a screenshot of just one of them.
//
// ./tools/quickshell-preview/render.sh \
// tests/LunaIndicatorHeadless.qml \
// .artifacts/quickshell-preview/luna.png 300 620
Rectangle {
width: 300
height: 620
color: "#1b1d20"
Column {
x: 20
y: 20
width: 260
spacing: 14
LunaIndicator {
width: parent.width
state: "idle"
rms: 0.002
}
LunaIndicator {
width: parent.width
state: "listening"
rms: 0.08
userText: "hey luna, are you there"
}
LunaIndicator {
width: parent.width
state: "thinking"
userText: "what's the time?"
}
LunaIndicator {
width: parent.width
state: "speaking"
rms: 0.14
userText: "what's the time?"
lunaText: "It's 22:25 on Sunday night, chief. Shouldn't you be asleep?"
}
LunaIndicator {
width: parent.width
state: "idle"
connected: false
}
}
}
@@ -0,0 +1,90 @@
import QtQuick
import qs.HyprChrome.Widgets.Bar.Luna
// The rail's luna panel in both densities and every daemon state. Collapsed
// is the left column (state only), expanded the right (messages) — the two
// things the panel has to get right, side by side.
//
// ./tools/quickshell-preview/render.sh \
// tests/LunaPanelHeadless.qml \
// .artifacts/quickshell-preview/luna-panel.png 640 560
Rectangle {
width: 640
height: 560
color: "#0f1012"
Row {
x: 16
y: 16
spacing: 24
Column {
spacing: 12
LunaPanel {
expanded: false
connected: true
active: false
daemonState: "idle"
}
LunaPanel {
expanded: false
connected: true
active: true
daemonState: "listening"
rms: 0.02
}
LunaPanel {
expanded: false
connected: true
active: true
daemonState: "thinking"
}
LunaPanel {
expanded: false
connected: true
active: true
daemonState: "speaking"
rms: 0.05
}
LunaPanel {
expanded: false
connected: false
}
}
Column {
spacing: 12
LunaPanel {
expanded: true
connected: true
active: false
daemonState: "idle"
}
LunaPanel {
expanded: true
connected: true
active: true
daemonState: "listening"
rms: 0.02
userText: "hey luna, what's the time?"
}
LunaPanel {
expanded: true
connected: true
active: true
daemonState: "speaking"
rms: 0.05
userText: "hey luna, what's the time?"
lunaText: "It's 23:40 on Sunday night, chief. Shouldn't you be asleep?"
}
}
}
}
+2 -1
View File
@@ -37,7 +37,8 @@ in
You are being spoken to out loud and your reply will be read aloud by a
speech synthesiser. Answer in one or two sentences unless asked for
detail. No markdown, no lists, no code blocks — they do not survive the
trip to audio.
trip to audio. You may use an audio tag now and then, such as [laughs],
[sighs] or [whispers]; the synthesiser performs them.
'';
};
+12 -1
View File
@@ -10,6 +10,12 @@ pub struct Config {
pub hermes_key: String,
pub model: String,
pub system_prompt: Option<String>,
/// Prefixed to every spoken message so luna can tell this channel apart
/// from Telegram or the dashboard. Her own profile on mars keys off it —
/// which is why it is a per-message marker and not a system prompt: a
/// system prompt is this client's opinion, while the marker survives into
/// whatever history Hermes keeps and still identifies the channel later.
pub voice_marker: Option<String>,
pub eleven_key: String,
pub voice_id: String,
@@ -55,10 +61,15 @@ impl Config {
hermes_key: secret("LUNA_HERMES_KEY_FILE")?,
model: env_or("LUNA_MODEL", "hermes"),
system_prompt: std::env::var("LUNA_SYSTEM_PROMPT").ok(),
// Empty string disables it rather than sending a bare prefix.
voice_marker: match env_or("LUNA_VOICE_MARKER", "[voice]") {
m if m.is_empty() => None,
m => Some(m),
},
eleven_key: secret("LUNA_ELEVENLABS_KEY_FILE")?,
voice_id: env("LUNA_VOICE_ID")?,
tts_model: env_or("LUNA_TTS_MODEL", "eleven_flash_v2_5"),
tts_model: env_or("LUNA_TTS_MODEL", "eleven_v3_conversational"),
whisper_bin: env_or("LUNA_WHISPER_BIN", "whisper-cli"),
whisper_model: PathBuf::from(env("LUNA_WHISPER_MODEL")?),
+48 -10
View File
@@ -83,6 +83,18 @@ fn root_threshold(floor: f32, mult: f32) -> f32 {
(floor * mult).clamp(min, 0.05)
}
/// Tags a spoken line so luna can recognise the channel it came from.
///
/// Prefix rather than suffix: a long utterance can be truncated by whatever
/// history window she is running under, and the front is the end that
/// survives.
fn mark(marker: &Option<String>, text: &str) -> String {
match marker {
Some(m) => format!("{m} {text}"),
None => text.to_string(),
}
}
fn socket_path() -> std::path::PathBuf {
let runtime = std::env::var("XDG_RUNTIME_DIR").unwrap_or_else(|_| "/run/user/1000".into());
std::env::var("LUNA_SOCKET")
@@ -366,7 +378,20 @@ async fn daemon() -> Result<()> {
/// listener: journal, socket, desktop notification.
#[cfg(test)]
mod tests {
use super::root_threshold;
use super::{mark, root_threshold};
#[test]
fn marker_prefixes_spoken_lines() {
assert_eq!(
mark(&Some("[voice]".into()), "what's the time"),
"[voice] what's the time"
);
}
#[test]
fn no_marker_leaves_the_line_alone() {
assert_eq!(mark(&None, "what's the time"), "what's the time");
}
#[test]
fn threshold_sits_between_terras_room_tone_and_terras_speech() {
@@ -423,8 +448,9 @@ async fn turn(
heard: String,
) -> Result<()> {
eprintln!("luna-voice: heard {heard:?}");
// The caption shows what you SAID; the marker is only for her.
ipc.emit(Event::Caption { speaker: "user", text: heard.clone() });
history.push(brain::Message::user(heard));
history.push(brain::Message::user(mark(&cfg.voice_marker, &heard)));
// Commands that arrived while she was speaking, replayed into the
// channel once the turn is over rather than dropped.
@@ -437,12 +463,13 @@ async fn turn(
let (sentence_tx, sentence_rx) = mpsc::channel::<String>(16);
let (event_tx, mut event_rx) = mpsc::channel::<tts::SpeechEvent>(64);
let tts_cfg = tts::TtsConfig {
key: cfg.eleven_key.clone(),
voice_id: cfg.voice_id.clone(),
model: cfg.tts_model.clone(),
};
let keep_tags = tts_cfg.is_v3();
let speech = tokio::spawn({
let tts_cfg = tts::TtsConfig {
key: cfg.eleven_key.clone(),
voice_id: cfg.voice_id.clone(),
model: cfg.tts_model.clone(),
};
async move { tts::speak(&tts_cfg, sentence_rx, event_tx).await }
});
@@ -484,20 +511,31 @@ async fn turn(
async move { brain::stream_reply(&http, &url, &key, &model, &history, delta_tx).await }
});
let mut splitter = tts::SentenceSplitter::new();
let mut splitter = tts::SentenceSplitter::new(keep_tags);
// What ElevenLabs bills: every character sent, audio tags included.
let mut billed = 0usize;
let caption = |sentence: &str| {
let text = tts::strip_tags(sentence);
if !text.is_empty() {
ipc.emit(Event::Caption { speaker: "luna", text });
}
};
while let Some(delta) = delta_rx.recv().await {
for sentence in splitter.push(&delta) {
ipc.emit(Event::Caption { speaker: "luna", text: sentence.clone() });
caption(&sentence);
billed += sentence.chars().count();
if sentence_tx.send(sentence).await.is_err() {
break;
}
}
}
if let Some(tail) = splitter.finish() {
ipc.emit(Event::Caption { speaker: "luna", text: tail.clone() });
caption(&tail);
billed += tail.chars().count();
let _ = sentence_tx.send(tail).await;
}
drop(sentence_tx);
eprintln!("luna-voice: tts {billed} chars on {}", cfg.tts_model);
let full = reply.await??;
eprintln!("luna-voice: said {:?}", full.trim());
+290 -80
View File
@@ -1,8 +1,9 @@
use crate::audio;
use crate::visemes::{self, Viseme};
use anyhow::Result;
use anyhow::{anyhow, Result};
use base64::Engine;
use futures_util::{SinkExt, StreamExt};
use std::time::Duration;
use tokio::sync::mpsc;
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
use tokio_tungstenite::tungstenite::Message as Ws;
@@ -21,6 +22,33 @@ pub struct TtsConfig {
pub model: String,
}
impl TtsConfig {
/// v3 is not served on the TTS websocket at all; it streams only over
/// Text to Dialogue, a different protocol.
pub fn is_v3(&self) -> bool {
self.model.starts_with("eleven_v3")
}
}
/// The two sockets send the same audio frames under different field names.
struct Wire {
alignment: &'static [&'static str],
final_key: &'static str,
}
const TTS_WIRE: Wire = Wire { alignment: &["normalizedAlignment", "alignment"], final_key: "isFinal" };
const DIALOGUE_WIRE: Wire = Wire { alignment: &["alignment"], final_key: "is_final" };
/// Dialogue drops the socket after 20s without a client message, and a tool
/// call on mars can stall a reply longer than that.
const KEEP_ALIVE: Duration = Duration::from_secs(10);
type Socket = tokio_tungstenite::WebSocketStream<
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
>;
type Sink = futures_util::stream::SplitSink<Socket, Ws>;
type Source = futures_util::stream::SplitStream<Socket>;
/// Speaks sentences as they arrive on `sentences`, over one websocket per
/// reply.
///
@@ -28,6 +56,35 @@ pub struct TtsConfig {
/// and no reason to add one. sync_alignment makes the character timings ride
/// along with the audio frames instead of needing a second request.
pub async fn speak(
cfg: &TtsConfig,
sentences: mpsc::Receiver<String>,
events: mpsc::Sender<SpeechEvent>,
) -> Result<()> {
if cfg.is_v3() {
speak_dialogue(cfg, sentences, events).await
} else {
speak_tts(cfg, sentences, events).await
}
}
async fn connect(url: &str, key: &str) -> Result<(Sink, Source)> {
let mut req = url.into_client_request()?;
// Header, not the ?authorization= query param: a key in a URL ends up in
// every proxy and error log that touches it.
req.headers_mut().insert(
"xi-api-key",
tokio_tungstenite::tungstenite::http::HeaderValue::from_str(key)?,
);
let (stream, _) = tokio_tungstenite::connect_async(req).await?;
Ok(stream.split())
}
async fn send_json(sink: &mut Sink, v: serde_json::Value) -> Result<()> {
sink.send(Ws::Text(v.to_string())).await?;
Ok(())
}
async fn speak_tts(
cfg: &TtsConfig,
mut sentences: mpsc::Receiver<String>,
events: mpsc::Sender<SpeechEvent>,
@@ -39,85 +96,151 @@ pub async fn speak(
cfg.model,
audio::TTS_RATE
);
let mut req = url.into_client_request()?;
// Header, not the ?authorization= query param: a key in a URL ends up in
// every proxy and error log that touches it.
req.headers_mut().insert(
"xi-api-key",
tokio_tungstenite::tungstenite::http::HeaderValue::from_str(&cfg.key)?,
);
let (stream, _) = tokio_tungstenite::connect_async(req).await?;
let (mut sink, mut source) = stream.split();
let (mut sink, source) = connect(&url, &cfg.key).await?;
// The protocol requires an opening frame of a single space before any
// real text; it carries the voice settings for the whole connection.
sink.send(Ws::Text(
send_json(
&mut sink,
serde_json::json!({
"text": " ",
"voice_settings": { "stability": 0.4, "similarity_boost": 0.8, "speed": 1.0 },
})
.to_string(),
))
}),
)
.await?;
let reader = tokio::spawn(async move {
// Alignment times may be per-chunk or cumulative depending on the
// model; `emitted_ms` lets us normalise either into one timeline.
let mut emitted_ms: u32 = 0;
while let Some(Ok(msg)) = source.next().await {
let Ws::Text(txt) = msg else { continue };
let Ok(v) = serde_json::from_str::<serde_json::Value>(&txt) else { continue };
let reader = tokio::spawn(read_audio(source, events, &TTS_WIRE));
if let Some(b64) = v.get("audio").and_then(|a| a.as_str()) {
if let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(b64) {
let pcm = audio::pcm16_to_f32(&bytes);
let chunk_ms = (pcm.len() as f64 / audio::TTS_RATE as f64 * 1000.0) as u32;
if let Some(al) = v.get("normalizedAlignment").or_else(|| v.get("alignment")) {
if let Some(t) = alignment_to_timeline(al, emitted_ms) {
let _ = events.send(SpeechEvent::Visemes(t)).await;
}
}
emitted_ms += chunk_ms;
if events.send(SpeechEvent::Audio(pcm)).await.is_err() {
return; // receiver gone: barge-in cancelled this reply
}
}
}
if v.get("isFinal").and_then(|f| f.as_bool()).unwrap_or(false) {
break;
}
let sent: Result<()> = async {
while let Some(text) = sentences.recv().await {
// Every text frame must end with a space or the model holds the
// last word back waiting for more input.
let text = format!("{} ", text.trim());
send_json(&mut sink, serde_json::json!({ "text": text })).await?;
}
let _ = events.send(SpeechEvent::Done).await;
});
while let Some(text) = sentences.recv().await {
// Every text frame must end with a space or the model holds the last
// word back waiting for more input.
let text = format!("{} ", text.trim());
sink.send(Ws::Text(
serde_json::json!({ "text": text }).to_string(),
))
.await?;
// Empty string = end of input. Without it the socket sits until the
// inactivity timeout and the last sentence never gets flushed.
send_json(&mut sink, serde_json::json!({ "text": "" })).await
}
.await;
// Empty string = end of input. Without it the socket sits until the
// inactivity timeout and the last sentence never gets flushed.
sink.send(Ws::Text(serde_json::json!({ "text": "" }).to_string()))
.await?;
let _ = reader.await;
Ok(())
// The server's own error beats the "connection reset" a send hit after it.
reader.await??;
sent
}
fn alignment_to_timeline(al: &serde_json::Value, emitted_ms: u32) -> Option<Vec<Viseme>> {
async fn speak_dialogue(
cfg: &TtsConfig,
mut sentences: mpsc::Receiver<String>,
events: mpsc::Sender<SpeechEvent>,
) -> Result<()> {
let url = format!(
"wss://api.elevenlabs.io/v1/text-to-dialogue/stream-input\
?model_id={}&output_format=pcm_{}&sync_alignment=true",
cfg.model,
audio::TTS_RATE
);
let (mut sink, source) = connect(&url, &cfg.key).await?;
// No voice_settings: v3 reads only stability, and only as one of three
// presets; the default is its "natural" one.
send_json(&mut sink, serde_json::json!({ "voices": [cfg.voice_id] })).await?;
let reader = tokio::spawn(read_audio(source, events, &DIALOGUE_WIRE));
let sent: Result<()> = async {
loop {
match tokio::time::timeout(KEEP_ALIVE, sentences.recv()).await {
Ok(Some(text)) => {
// Inputs are concatenated verbatim, so the space matters
// here too. flush: the server otherwise holds text until
// ~40 chars and 8 words, so a short first sentence would
// wait for the second.
let text = format!("{} ", text.trim());
send_json(
&mut sink,
serde_json::json!({
"inputs": [{ "text": text, "voice_id": cfg.voice_id }],
"flush": true,
}),
)
.await?
}
Ok(None) => break,
Err(_) => send_json(&mut sink, serde_json::json!({ "keep_alive": true })).await?,
}
}
send_json(&mut sink, serde_json::json!({ "close_socket": true })).await
}
.await;
reader.await??;
sent
}
async fn read_audio(
mut source: Source,
events: mpsc::Sender<SpeechEvent>,
wire: &'static Wire,
) -> Result<()> {
// Alignment times may be per-chunk or cumulative depending on the
// model; `emitted_ms` lets us normalise either into one timeline.
let mut emitted_ms: u32 = 0;
// v3 audio tags come back as characters, and one can straddle chunks.
let mut in_tag = false;
let mut result = Ok(());
while let Some(Ok(msg)) = source.next().await {
let Ws::Text(txt) = msg else { continue };
let Ok(v) = serde_json::from_str::<serde_json::Value>(&txt) else { continue };
if let Some(err) = v.get("error").and_then(|e| e.as_str()) {
let detail = v.get("message").and_then(|m| m.as_str()).unwrap_or("");
result = Err(anyhow!("elevenlabs: {err} {detail}"));
break;
}
if let Some(b64) = v.get("audio").and_then(|a| a.as_str()) {
if let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(b64) {
let pcm = audio::pcm16_to_f32(&bytes);
let chunk_ms = (pcm.len() as f64 / audio::TTS_RATE as f64 * 1000.0) as u32;
let al = wire.alignment.iter().find_map(|k| v.get(*k).filter(|a| !a.is_null()));
if let Some(t) = al.and_then(|al| alignment_to_timeline(al, emitted_ms, &mut in_tag)) {
let _ = events.send(SpeechEvent::Visemes(t)).await;
}
emitted_ms += chunk_ms;
if events.send(SpeechEvent::Audio(pcm)).await.is_err() {
return Ok(()); // receiver gone: barge-in cancelled this reply
}
}
}
if v.get(wire.final_key).and_then(|f| f.as_bool()).unwrap_or(false) {
break;
}
}
let _ = events.send(SpeechEvent::Done).await;
result
}
fn alignment_to_timeline(
al: &serde_json::Value,
emitted_ms: u32,
in_tag: &mut bool,
) -> Option<Vec<Viseme>> {
let field = |camel: &str, snake: &str| al.get(camel).or_else(|| al.get(snake));
let chars: Vec<String> = al
.get("chars")?
.as_array()?
.iter()
.map(|c| c.as_str().unwrap_or("").to_string())
.map(|c| c.as_str().unwrap_or(""))
.map(|c| {
// A tag is a laugh or a sigh, not letters: the mouth rests.
let tagged = *in_tag || c == "[";
*in_tag = tagged && c != "]";
if tagged { " ".to_string() } else { c.to_string() }
})
.collect();
let starts: Vec<i64> = num_array(al.get("charStartTimesMs")?)?;
let durs: Vec<i64> = num_array(al.get("charDurationsMs")?)?;
let starts: Vec<i64> = num_array(field("charStartTimesMs", "char_start_times_ms")?)?;
let durs: Vec<i64> = num_array(field("charDurationsMs", "char_durations_ms")?)?;
let starts = absolutize(&starts, emitted_ms);
Some(visemes::timeline(&chars, &starts, &durs))
}
@@ -143,11 +266,13 @@ fn absolutize(starts: &[i64], emitted_ms: u32) -> Vec<i64> {
pub struct SentenceSplitter {
buf: String,
in_think: bool,
/// Whether v3 audio tags survive to the synthesiser or are dropped.
keep_tags: bool,
}
impl SentenceSplitter {
pub fn new() -> Self {
SentenceSplitter { buf: String::new(), in_think: false }
pub fn new(keep_tags: bool) -> Self {
SentenceSplitter { buf: String::new(), in_think: false, keep_tags }
}
/// Feeds a delta, returning any sentences that completed.
@@ -174,24 +299,24 @@ impl SentenceSplitter {
let head: String = self.buf.drain(..i).collect();
self.buf.drain(.."<think>".len());
self.in_think = true;
out.extend(drain_sentences(&mut { head.clone() }, true));
out.extend(drain_sentences(&mut { head.clone() }, true, self.keep_tags));
}
None => break,
}
}
out.extend(drain_sentences(&mut self.buf, false));
out.extend(drain_sentences(&mut self.buf, false, self.keep_tags));
out.into_iter().filter(|s| !s.is_empty()).collect()
}
/// Whatever is left when the reply ends.
pub fn finish(&mut self) -> Option<String> {
let rest = clean_for_speech(&std::mem::take(&mut self.buf));
let rest = clean_for_speech(&std::mem::take(&mut self.buf), self.keep_tags);
(!rest.is_empty()).then_some(rest)
}
}
fn drain_sentences(buf: &mut String, force: bool) -> Vec<String> {
fn drain_sentences(buf: &mut String, force: bool, keep_tags: bool) -> Vec<String> {
let mut out = Vec::new();
// A sentence is only worth sending once it is long enough to synthesise
// well; ElevenLabs' own guidance is ~20 characters minimum.
@@ -203,13 +328,13 @@ fn drain_sentences(buf: &mut String, force: bool) -> Vec<String> {
}
}
let s: String = buf.drain(..end + 1).collect();
let s = clean_for_speech(&s);
let s = clean_for_speech(&s, keep_tags);
if !s.is_empty() {
out.push(s);
}
}
if force && !buf.is_empty() {
let s = clean_for_speech(&std::mem::take(buf));
let s = clean_for_speech(&std::mem::take(buf), keep_tags);
if !s.is_empty() {
out.push(s);
}
@@ -217,16 +342,64 @@ fn drain_sentences(buf: &mut String, force: bool) -> Vec<String> {
out
}
/// Byte index just past a v3 audio tag starting at `i` ("[laughs]",
/// "[clears throat]"), if there is one. Letters and spaces only, so
/// "[1]" or a markdown link stays ordinary noise.
fn tag_end(s: &str, i: usize) -> Option<usize> {
let rest = s[i..].strip_prefix('[')?;
let close = rest.find(']')?;
let body = &rest[..close];
let end = i + 1 + close + 1;
let ok = !body.trim().is_empty()
&& body.len() <= 40
&& body.chars().all(|c| c.is_alphabetic() || c == ' ' || c == '-')
&& !s[end..].starts_with('(');
ok.then_some(end)
}
/// The text as a person reads it: captions show "ha!", not "[laughs]".
pub fn strip_tags(s: &str) -> String {
let mut out = String::with_capacity(s.len());
let mut skip_to = 0;
for (i, c) in s.char_indices() {
if i < skip_to {
continue;
}
match tag_end(s, i) {
Some(end) => {
skip_to = end;
out.push(' ');
}
None => out.push(c),
}
}
out.split_whitespace().collect::<Vec<_>>().join(" ")
}
/// Markdown, emoji and kaomoji are for eyes.
///
/// A whitelist, not a blacklist: she writes things like "(¬‿¬)", whose
/// characters live in ordinary Latin-1 and punctuation blocks rather than in
/// any emoji range, so enumerating what to drop never converges. Letters
/// stays unicode-aware on purpose — German is in scope, "ö" is a letter.
pub fn clean_for_speech(s: &str) -> String {
///
/// Audio tags pass through untouched when `keep_tags`; otherwise they are
/// dropped whole, since a model without them would read "laughs" aloud.
pub fn clean_for_speech(s: &str, keep_tags: bool) -> String {
let mut out = String::with_capacity(s.len());
for c in s.chars() {
if c.is_alphanumeric() || c.is_whitespace() || is_speakable_punctuation(c) {
let mut skip_to = 0;
for (i, c) in s.char_indices() {
if i < skip_to {
continue;
}
if let Some(end) = tag_end(s, i) {
skip_to = end;
out.push(' ');
if keep_tags {
out.push_str(&s[i..end]);
out.push(' ');
}
} else if c.is_alphanumeric() || c.is_whitespace() || is_speakable_punctuation(c) {
out.push(c);
} else if matches!(c, '—' | '–') {
// " — " must not leave " , ": the comma binds to the word before it.
@@ -271,7 +444,7 @@ mod tests {
#[test]
fn splitter_emits_complete_sentences_only() {
let mut s = SentenceSplitter::new();
let mut s = SentenceSplitter::new(false);
assert!(s.push("This is a long enough first sentence").is_empty());
let got = s.push(". And more");
assert_eq!(got, vec!["This is a long enough first sentence."]);
@@ -279,7 +452,7 @@ mod tests {
#[test]
fn splitter_drops_think_blocks() {
let mut s = SentenceSplitter::new();
let mut s = SentenceSplitter::new(false);
s.push("<think>internal monologue that is long</think>");
let got = s.push("Actually speakable sentence here. ");
assert_eq!(got, vec!["Actually speakable sentence here."]);
@@ -287,33 +460,70 @@ mod tests {
#[test]
fn splitter_returns_tail_on_finish() {
let mut s = SentenceSplitter::new();
let mut s = SentenceSplitter::new(false);
s.push("no terminator here");
assert_eq!(s.finish().as_deref(), Some("no terminator here"));
}
#[test]
fn markdown_and_emoji_never_reach_the_voice() {
assert_eq!(clean_for_speech("**bold** `code` 🎉 ok"), "bold code ok");
assert_eq!(clean_for_speech("**bold** `code` 🎉 ok", false), "bold code ok");
}
#[test]
fn kaomoji_and_dashes_are_stripped_without_gluing_words() {
// Her actual first reply over the API.
assert_eq!(
clean_for_speech("Loud and clear, boss — hallo! (¬‿¬)"),
clean_for_speech("Loud and clear, boss — hallo! (¬‿¬)", false),
"Loud and clear, boss, hallo!"
);
}
#[test]
fn em_dash_becomes_a_pause_but_hyphens_do_not() {
assert_eq!(clean_for_speech("I can't confirm — there's nothing"), "I can't confirm, there's nothing");
assert_eq!(clean_for_speech("push-to-talk"), "push to talk");
assert_eq!(clean_for_speech("I can't confirm — there's nothing", false), "I can't confirm, there's nothing");
assert_eq!(clean_for_speech("push-to-talk", false), "push to talk");
}
#[test]
fn keeps_non_ascii_letters() {
assert_eq!(clean_for_speech("schön, größer"), "schön, größer");
assert_eq!(clean_for_speech("schön, größer", false), "schön, größer");
}
#[test]
fn audio_tags_survive_for_v3_only() {
let s = "[laughs]Fine, boss. [clears throat] Na gut.";
assert_eq!(clean_for_speech(s, true), "[laughs] Fine, boss. [clears throat] Na gut.");
assert_eq!(clean_for_speech(s, false), "Fine, boss. Na gut.");
}
#[test]
fn brackets_that_are_not_tags_stay_noise() {
assert_eq!(clean_for_speech("see [1] and [the docs](x)", true), "see 1 and the docs x");
}
#[test]
fn captions_hide_tags() {
assert_eq!(strip_tags("[sighs] Okay, boss."), "Okay, boss.");
}
#[test]
fn tag_characters_rest_the_mouth_across_chunks() {
let mut in_tag = false;
let chunk = |chars: &[&str], in_tag: &mut bool| {
let n = chars.len() as i64;
let al = serde_json::json!({
"chars": chars,
"char_start_times_ms": (0..n).map(|i| i * 100).collect::<Vec<_>>(),
"char_durations_ms": vec![100; chars.len()],
});
alignment_to_timeline(&al, 0, in_tag).unwrap()
};
let t = chunk(&["[", "l", "a"], &mut in_tag);
assert!(t.iter().all(|v| v.shape == visemes::SILENCE));
assert!(in_tag);
let t = chunk(&["u", "]", "a"], &mut in_tag);
assert!(!in_tag);
assert_eq!(t.last().unwrap().shape, "aa");
}
}
+28 -6
View File
@@ -1,11 +1,12 @@
{ config, pkgs, lib, ... }:
# Voice loop for luna: hold a key, talk, hear her answer.
# Voice loop for luna: SUPER+V opens the mic, talk, hear her answer.
#
# mic -> whisper-cpp (local, vulkan) -> Hermes' OpenAI-compatible API on mars
# -> ElevenLabs streaming TTS -> speaker. State, amplitude and the viseme
# timeline are published on a unix socket for the quickshell avatar to read
# (nothing consumes it yet — the widget is the next phase).
# -> ElevenLabs streaming TTS -> speaker. State, amplitude, captions and a
# viseme timeline are published on a unix socket; the rail's LunaPanel reads
# them (dotfiles/quickshell/HyprChrome/Widgets/Bar/Luna), and the viseme
# stream is there for the avatar that replaces its meter later.
#
# A USER service, not a system one: it needs darman's pipewire session and his
# XDG_RUNTIME_DIR for the socket. That is also why the two secrets it reads
@@ -57,8 +58,13 @@ in
ttsModel = lib.mkOption {
type = lib.types.str;
default = "eleven_flash_v2_5";
description = "~75ms model, the one ElevenLabs recommends for live conversation.";
default = "eleven_v3_conversational";
description = ''
eleven_v3* streams over the Text to Dialogue socket and performs audio
tags ([laughs], [sighs]); it also gets her German right, which Flash
reads with an English accent. Anything else uses the TTS socket, e.g.
"eleven_flash_v2_5" — half the price per character, faster, no tags.
'';
};
hermesKeyFile = lib.mkOption {
@@ -99,6 +105,21 @@ in
description = "Silence that ends an utterance.";
};
voiceMarker = lib.mkOption {
type = lib.types.str;
default = "[voice]";
description = ''
Prefixed to every spoken message so luna can tell this channel from
Telegram or the dashboard, and adjust — no kaomoji, no markdown, short
answers. Put the rule in HER profile on mars rather than only in
systemPrompt below: the marker travels with the message, so it still
identifies the channel in whatever history Hermes keeps, and one place
governs her voice manners across every client that sends it.
Empty string disables the marker.
'';
};
systemPrompt = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
@@ -131,6 +152,7 @@ in
LUNA_VAD_MULT = toString cfg.vadMult;
LUNA_VAD_FLOOR = toString cfg.vadFloor;
LUNA_SILENCE_MS = toString cfg.silenceMs;
LUNA_VOICE_MARKER = cfg.voiceMarker;
} // lib.optionalAttrs (cfg.systemPrompt != null) {
LUNA_SYSTEM_PROMPT = cfg.systemPrompt;
} // lib.optionalAttrs (cfg.inputDevice != null) {