From e4ee637ccdc60a95d261f29753028ed66df780ef Mon Sep 17 00:00:00 2001 From: Erik Simon Date: Fri, 25 Sep 2026 00:52:03 +0200 Subject: [PATCH] feat(terra): luna v3 voice, channel marker and rail panel - TTS defaults to eleven_v3_conversational over the Text to Dialogue socket: performs audio tags and gets her German right. Flash still works via the TTS socket. - Every spoken message is prefixed with a [voice] marker so luna's profile on mars can tell the channel apart. - LunaIndicator/LunaCall replaced by a LunaPanel in the bar rail, with captions; headless test updated to match. Co-Authored-By: Claude Opus 5.5 --- .../HyprChrome/Widgets/Bar/HyprChromeBar.qml | 10 +- .../HyprChrome/Widgets/Bar/Luna/LunaPanel.qml | 200 ++++++++++ .../Widgets/{ => Bar}/Luna/LunaState.qml | 0 .../HyprChrome/Widgets/HyprChromeShell.qml | 20 - .../HyprChrome/Widgets/Luna/LunaCall.qml | 60 --- .../HyprChrome/Widgets/Luna/LunaIndicator.qml | 229 ----------- .../tests/LunaIndicatorHeadless.qml | 55 --- .../quickshell/tests/LunaPanelHeadless.qml | 90 +++++ hosts/terra/configuration.nix | 3 +- pkgs/luna-voice/src/config.rs | 13 +- pkgs/luna-voice/src/main.rs | 58 ++- pkgs/luna-voice/src/tts.rs | 370 ++++++++++++++---- services/desktop/luna-voice.nix | 34 +- 13 files changed, 675 insertions(+), 467 deletions(-) create mode 100644 dotfiles/quickshell/HyprChrome/Widgets/Bar/Luna/LunaPanel.qml rename dotfiles/quickshell/HyprChrome/Widgets/{ => Bar}/Luna/LunaState.qml (100%) delete mode 100644 dotfiles/quickshell/HyprChrome/Widgets/Luna/LunaCall.qml delete mode 100644 dotfiles/quickshell/HyprChrome/Widgets/Luna/LunaIndicator.qml delete mode 100644 dotfiles/quickshell/tests/LunaIndicatorHeadless.qml create mode 100644 dotfiles/quickshell/tests/LunaPanelHeadless.qml diff --git a/dotfiles/quickshell/HyprChrome/Widgets/Bar/HyprChromeBar.qml b/dotfiles/quickshell/HyprChrome/Widgets/Bar/HyprChromeBar.qml index f85e001..68edbf2 100644 --- a/dotfiles/quickshell/HyprChrome/Widgets/Bar/HyprChromeBar.qml +++ b/dotfiles/quickshell/HyprChrome/Widgets/Bar/HyprChromeBar.qml @@ -4,7 +4,7 @@ import QtQuick.Layouts import Quickshell import Quickshell.Wayland import qs.HyprChrome.Theme -import qs.HyprChrome.Widgets.Bar.Debug +import qs.HyprChrome.Widgets.Bar.Luna import qs.HyprChrome.Widgets.Bar.Host import qs.HyprChrome.Widgets.Bar.Vitals import qs.HyprChrome.Widgets.Bar.Workspaces @@ -87,8 +87,8 @@ PanelWindow { // // The zone still follows the target height, so tiled windows reflow once // per toggle, at the start, and slide while the panels animate. - readonly property real expandedContent: Math.max(hostPanel.expandedHeight, workspacesPanel.expandedHeight, vitalsPanel.expandedHeight, testPanel.expandedHeight, trayPanel.expandedHeight) + window.margin * 2 - readonly property real contentHeight: Math.max(hostPanel.targetHeight, workspacesPanel.targetHeight, vitalsPanel.targetHeight, testPanel.targetHeight, trayPanel.targetHeight) + window.margin * 2 + readonly property real expandedContent: Math.max(hostPanel.expandedHeight, workspacesPanel.expandedHeight, vitalsPanel.expandedHeight, lunaPanel.expandedHeight, trayPanel.expandedHeight) + window.margin * 2 + readonly property real contentHeight: Math.max(hostPanel.targetHeight, workspacesPanel.targetHeight, vitalsPanel.targetHeight, lunaPanel.targetHeight, trayPanel.targetHeight) + window.margin * 2 implicitHeight: Math.round(window.expandedContent) @@ -157,8 +157,8 @@ PanelWindow { // and a trace at each end. Item { Layout.fillWidth: true } - TestPanel { - id: testPanel + LunaPanel { + id: lunaPanel expanded: window.expanded toggleOnClick: false diff --git a/dotfiles/quickshell/HyprChrome/Widgets/Bar/Luna/LunaPanel.qml b/dotfiles/quickshell/HyprChrome/Widgets/Bar/Luna/LunaPanel.qml new file mode 100644 index 0000000..4e1d9a9 --- /dev/null +++ b/dotfiles/quickshell/HyprChrome/Widgets/Bar/Luna/LunaPanel.qml @@ -0,0 +1,200 @@ +pragma ComponentBehavior: Bound + +import QtQuick +import qs.HyprChrome.Theme +import qs.HyprChrome.Widgets.Bar.Panels + +// luna's voice loop on the rail (services/desktop/luna-voice.nix). +// +// Always present, unlike the daemon it watches: the panel is where you look +// to find out whether she is listening, so it has to be there when the answer +// is "no" — including when luna-voiced is not running at all. SUPER+V toggles +// LISTENING, not this. +// +// Collapsed it is the state and nothing else, because that is the one fact +// worth a glance; expanded it carries the last exchange, which is the thing +// you actually want during the seconds her reply takes to arrive. +BarPanel { + id: luna + + // Overridable so the headless render can stage every state without a + // daemon; in the rail they track the socket below. + property string daemonState: state.state + property bool active: state.active + property bool connected: state.connected + property real rms: state.rms + property string userText: state.userText + property string lunaText: state.lunaText + + // Instantiated here rather than in the shell: the panel is the only + // reader, and it is a second reader that would make this shell state. + readonly property var state: LunaState {} + + panelId: "LUN" + title: "LUNA" + meta: luna.active ? "VOICE" : "IDLE" + + // Sized to content like the tray, and per DENSITY: collapsed it is one + // word, so reserving the expanded body's width would leave the rail + // carrying a mostly empty panel for the state it spends most of its time + // in. It floats between two spacers, so the width change costs no other + // panel anything. + implicitWidth: luna.expanded + ? Math.max(luna.headerMinWidth, luna.padding * 2 + body.implicitWidth) + : Math.max(luna.headerMinWidth, luna.briefLeft + brief.implicitWidth + luna.padding) + + readonly property color stateColor: { + if (!luna.connected) + return Theme.disabled; + switch (luna.daemonState) { + case "listening": + return Theme.accent; + case "thinking": + return Theme.muted; + case "speaking": + return Theme.accentSoft; + default: + return luna.active ? Theme.text : Theme.raised; + } + } + + readonly property string stateLabel: { + if (!luna.connected) + return "OFFLINE"; + if (!luna.active) + return "OFF"; + switch (luna.daemonState) { + case "listening": + return "LISTENING"; + case "thinking": + return "THINKING"; + case "speaking": + return "SPEAKING"; + default: + return "READY"; + } + } + + summary: Row { + id: brief + spacing: 6 + + // Breathes only while the mic is open, so "she is listening to me + // right now" is legible without reading the word next to it. + Rectangle { + width: 7 + height: 7 + radius: 4 + anchors.verticalCenter: parent.verticalCenter + color: luna.stateColor + + SequentialAnimation on opacity { + running: luna.daemonState === "listening" + loops: Animation.Infinite + alwaysRunToEnd: true + NumberAnimation { to: 0.25; duration: 700; easing.type: Easing.InOutQuad } + NumberAnimation { to: 1.0; duration: 700; easing.type: Easing.InOutQuad } + } + } + + Text { + anchors.verticalCenter: parent.verticalCenter + text: luna.stateLabel + color: luna.stateColor + font.family: Theme.displayFont + font.pixelSize: 13 + font.bold: true + } + } + + Column { + id: body + spacing: 6 + + // Level meter: live whenever voice mode is on, not only once a + // trigger has fired, so a mic too quiet to reach the threshold still + // shows movement instead of looking broken. + Row { + id: meter + width: 260 + height: 18 + spacing: 2 + + readonly property int count: 28 + readonly property bool sweeping: luna.daemonState === "thinking" + property real phase: 0 + + NumberAnimation on phase { + running: meter.sweeping + loops: Animation.Infinite + from: 0 + to: 1 + duration: 1200 + } + + Repeater { + model: meter.count + + Rectangle { + required property int index + + width: (meter.width - (meter.count - 1) * meter.spacing) / meter.count + height: { + if (meter.sweeping) { + const pos = index / (meter.count - 1); + return 2 + 14 * Math.max(0, 1 - Math.abs(pos - meter.phase) * 6); + } + if (!luna.active && luna.daemonState !== "speaking") + return 2; + // sqrt opens up the quiet end, which on this mic is + // where all of the speech actually sits. + const level = Math.min(1, Math.sqrt(luna.rms) * 6.0); + const centre = 1 - Math.abs(index / (meter.count - 1) - 0.5) * 1.6; + return 2 + 16 * level * Math.max(0.15, centre); + } + anchors.bottom: parent.bottom + color: luna.stateColor + opacity: meter.sweeping ? 0.9 : 0.5 + 0.5 * (height / 18) + + Behavior on height { + enabled: !meter.sweeping + NumberAnimation { duration: 70; easing.type: Easing.OutQuad } + } + } + } + } + + Text { + width: 260 + visible: luna.userText !== "" + text: `> ${luna.userText}` + color: Theme.muted + font.family: Theme.microFont + font.pixelSize: 10 + wrapMode: Text.WordWrap + maximumLineCount: 2 + elide: Text.ElideRight + } + + Text { + width: 260 + visible: luna.lunaText !== "" + text: luna.lunaText + color: Theme.text + font.family: Theme.microFont + font.pixelSize: 11 + wrapMode: Text.WordWrap + maximumLineCount: 4 + elide: Text.ElideRight + } + + Text { + visible: luna.userText === "" && luna.lunaText === "" + text: luna.connected ? "SUPER+V TO TALK" : "DAEMON NOT RUNNING" + color: Theme.muted + font.family: Theme.microFont + font.pixelSize: 9 + font.letterSpacing: 1.4 + } + } +} diff --git a/dotfiles/quickshell/HyprChrome/Widgets/Luna/LunaState.qml b/dotfiles/quickshell/HyprChrome/Widgets/Bar/Luna/LunaState.qml similarity index 100% rename from dotfiles/quickshell/HyprChrome/Widgets/Luna/LunaState.qml rename to dotfiles/quickshell/HyprChrome/Widgets/Bar/Luna/LunaState.qml diff --git a/dotfiles/quickshell/HyprChrome/Widgets/HyprChromeShell.qml b/dotfiles/quickshell/HyprChrome/Widgets/HyprChromeShell.qml index 93acdc6..0e21110 100644 --- a/dotfiles/quickshell/HyprChrome/Widgets/HyprChromeShell.qml +++ b/dotfiles/quickshell/HyprChrome/Widgets/HyprChromeShell.qml @@ -9,7 +9,6 @@ import qs.HyprChrome.Widgets.Polkit import qs.HyprChrome.Widgets.Launcher import qs.HyprChrome.Widgets.Askpass import qs.HyprChrome.Widgets.Notifications -import qs.HyprChrome.Widgets.Luna // The hyprchrome shell: owns everything the rail's surfaces have to agree on, // and instantiates them. @@ -217,25 +216,6 @@ Scope { screen: shell.targetScreen } - // luna's voice-mode readout. Beside the notification stack rather than - // among the modals: it never takes the keyboard and raises no scrim, so - // it is deliberately NOT part of `modalOpen`. It does read `targetScreen`, - // which is what makes it belong to the shell at all. - LunaState { - id: lunaState - } - - LunaCall { - state: lunaState - targetScreen: shell.targetScreen - // Same source the backdrop uses. The rail's exclusive zone does NOT - // push this surface down on its own — measured: the bar occupies - // y 0..147 and the readout still landed at y=8 — so the offset is - // passed explicitly rather than left to the compositor. That makes - // the bar's height shell state by the usual rule: a second reader. - barHeight: bar.contentHeight - } - // Polkit authentication agent. It registers for this logind session on // creation, so it replaces hyprpolkitagent rather than coexisting with it — // only one agent may hold a session (see hosts/terra/home/hyprland.nix). diff --git a/dotfiles/quickshell/HyprChrome/Widgets/Luna/LunaCall.qml b/dotfiles/quickshell/HyprChrome/Widgets/Luna/LunaCall.qml deleted file mode 100644 index fffb6a4..0000000 --- a/dotfiles/quickshell/HyprChrome/Widgets/Luna/LunaCall.qml +++ /dev/null @@ -1,60 +0,0 @@ -pragma ComponentBehavior: Bound - -import QtQuick -import Quickshell -import Quickshell.Wayland -import qs.HyprChrome.Widgets.Luna - -// Voice-mode surface: top-left, directly under the rail. -// -// Thin by design — the visuals are all in LunaIndicator so they stay -// headless-renderable. Anchoring top+left is enough to sit under the bar: -// the rail sets an exclusiveZone of its collapsed height, so the compositor -// places this below it without either surface knowing the other's metrics. -// -// NOT a modal. She never takes the keyboard — the whole point is that you -// keep working while talking — so this does not join the shell's `modalOpen` -// and raises no scrim. -PanelWindow { - id: window - - required property var state - required property var targetScreen - // The rail's settled height, from the shell. Not the compositor's job - // here: the bar's exclusive zone leaves this surface at the top of the - // output, so the gap is opened by hand. - required property real barHeight - - screen: window.targetScreen - visible: window.state.active - - anchors { top: true; left: true } - margins { left: 8; top: window.barHeight + 8 } - - implicitWidth: indicator.implicitWidth - implicitHeight: indicator.implicitHeight - - color: "transparent" - // Below the rail's own layer: this is a readout, not a surface that - // should ever cover the bar. - WlrLayershell.layer: WlrLayer.Top - // A readout that stole focus would break typing the moment she answered. - WlrLayershell.keyboardFocus: WlrKeyboardFocus.None - // Normal with a zero zone: reserve nothing of our own, but still RESPECT - // the rail's exclusion so the compositor places this under it. Ignore - // would skip other surfaces' zones too, which put it over the bar. - exclusionMode: ExclusionMode.Normal - exclusiveZone: 0 - - LunaIndicator { - id: indicator - anchors.fill: parent - - state: window.state.state - active: window.state.active - connected: window.state.connected - rms: window.state.rms - userText: window.state.userText - lunaText: window.state.lunaText - } -} diff --git a/dotfiles/quickshell/HyprChrome/Widgets/Luna/LunaIndicator.qml b/dotfiles/quickshell/HyprChrome/Widgets/Luna/LunaIndicator.qml deleted file mode 100644 index 78e4872..0000000 --- a/dotfiles/quickshell/HyprChrome/Widgets/Luna/LunaIndicator.qml +++ /dev/null @@ -1,229 +0,0 @@ -pragma ComponentBehavior: Bound - -import QtQuick -import QtQuick.Shapes -import qs.HyprChrome.Theme - -// The voice-mode readout: whether luna is listening, thinking or talking. -// -// Item-rooted with no window of its own so tools/quickshell-preview/render.sh -// can shoot it offscreen — the PanelWindow lives in LunaCall.qml. A standalone -// 3D avatar replaces the meter later; the states and their colours are the -// part that has to read correctly first. -Item { - id: root - - // Driven by LunaState; defaulted so the headless render and the - // DebugWindow can stage this without a daemon. - property string state: "idle" - property bool active: true - property bool connected: true - property real rms: 0 - property string userText: "" - property string lunaText: "" - - // Metrics are per-component in this tree; Theme carries colour and type only. - readonly property int pad: 8 - readonly property int chamfer: 10 - - implicitWidth: 260 - implicitHeight: content.implicitHeight + root.pad * 2 - - // Not readonly: a Behavior cannot attach to a readonly property, and the - // colour crossfade is what keeps state changes from snapping. - property color stateColor: { - if (!root.connected) - return Theme.disabled; - switch (root.state) { - case "listening": - return Theme.accent; - case "thinking": - return Theme.muted; - case "speaking": - return Theme.accentSoft; - default: - return Theme.raised; - } - } - - readonly property string stateLabel: { - if (!root.connected) - return "OFFLINE"; - switch (root.state) { - case "listening": - return "LISTENING"; - case "thinking": - return "THINKING"; - case "speaking": - return "SPEAKING"; - default: - return "READY"; - } - } - - Behavior on stateColor { - ColorAnimation { duration: 180 } - } - - // The card is ONE chamfered path, not a rectangle with a triangle drawn - // over its corner — that leaves the square border showing underneath and - // reads as an artifact rather than a cut. - Shape { - anchors.fill: parent - preferredRendererType: Shape.CurveRenderer - opacity: 0.96 - - ShapePath { - strokeWidth: 1 - strokeColor: root.stateColor - fillColor: Theme.surface - startX: 0 - startY: 0 - PathLine { x: root.width - root.chamfer; y: 0 } - PathLine { x: root.width; y: root.chamfer } - PathLine { x: root.width; y: root.height } - PathLine { x: 0; y: root.height } - PathLine { x: 0; y: 0 } - } - } - - Column { - id: content - anchors.fill: parent - anchors.margins: root.pad - spacing: root.pad - - Row { - spacing: root.pad - - // The one element that is alive while nothing else moves: a dot - // that breathes in listening and holds steady otherwise, so "the - // mic is open" is legible from across the room. - Rectangle { - id: dot - width: 8 - height: 8 - radius: 4 - anchors.verticalCenter: parent.verticalCenter - color: root.stateColor - - SequentialAnimation on opacity { - running: root.state === "listening" - loops: Animation.Infinite - alwaysRunToEnd: true - NumberAnimation { to: 0.25; duration: 700; easing.type: Easing.InOutQuad } - NumberAnimation { to: 1.0; duration: 700; easing.type: Easing.InOutQuad } - } - // An animation that stops mid-fade would leave the dot dim - // and looking half-dead. - onVisibleChanged: if (root.state !== "listening") opacity = 1 - } - - Text { - anchors.verticalCenter: parent.verticalCenter - text: "LUNA" - color: Theme.text - font.family: Theme.displayFont - font.pixelSize: 12 - font.letterSpacing: 2 - } - - Text { - anchors.verticalCenter: parent.verticalCenter - text: root.stateLabel - color: root.stateColor - font.family: Theme.microFont - font.pixelSize: 10 - font.letterSpacing: 1 - } - } - - // Level meter. Bars react to rms while listening or speaking; during - // thinking they run a sweep instead, because a dead meter and a - // crashed daemon look identical and her replies take seconds. - Row { - id: meter - width: parent.width - height: 22 - spacing: 2 - - readonly property int count: 24 - readonly property bool sweeping: root.state === "thinking" - property real phase: 0 - - NumberAnimation on phase { - running: meter.sweeping - loops: Animation.Infinite - from: 0 - to: 1 - duration: 1200 - } - - Repeater { - model: meter.count - - Rectangle { - required property int index - - width: (meter.width - (meter.count - 1) * meter.spacing) / meter.count - height: { - if (meter.sweeping) { - // A travelling bump rather than a bar graph. - const pos = index / (meter.count - 1); - const d = Math.abs(pos - meter.phase); - return 3 + 16 * Math.max(0, 1 - d * 6); - } - // Live whenever voice mode is on, not only once a - // trigger has fired: "is it hearing me at all" has to - // be answerable BEFORE anything happens, or a mic - // that is too quiet looks exactly like a broken one. - if (!root.active && root.state !== "speaking") - return 3; - // rms is small and non-linear; sqrt opens up the - // quiet end where speech actually sits. - const level = Math.min(1, Math.sqrt(root.rms) * 3.2); - // Centre bars taller, so it reads as a voice rather - // than a progress bar. - const centre = 1 - Math.abs(index / (meter.count - 1) - 0.5) * 1.6; - return 3 + 19 * level * Math.max(0.15, centre); - } - anchors.bottom: parent.bottom - color: root.stateColor - opacity: meter.sweeping ? 0.9 : 0.55 + 0.45 * (height / 22) - - Behavior on height { - enabled: !meter.sweeping - NumberAnimation { duration: 70; easing.type: Easing.OutQuad } - } - } - } - } - - // Captions: hers leads the audio (the daemon emits each sentence as - // it goes to the synthesiser), which is what makes a 5s reply feel - // like it has started. - Text { - width: parent.width - visible: root.userText !== "" - text: `> ${root.userText}` - color: Theme.muted - font.family: Theme.microFont - font.pixelSize: 10 - wrapMode: Text.WordWrap - maximumLineCount: 2 - elide: Text.ElideRight - } - - Text { - width: parent.width - visible: root.lunaText !== "" - text: root.lunaText - color: Theme.text - font.family: Theme.microFont - font.pixelSize: 11 - wrapMode: Text.WordWrap - maximumLineCount: 3 - elide: Text.ElideRight - } - } -} diff --git a/dotfiles/quickshell/tests/LunaIndicatorHeadless.qml b/dotfiles/quickshell/tests/LunaIndicatorHeadless.qml deleted file mode 100644 index 37b72f3..0000000 --- a/dotfiles/quickshell/tests/LunaIndicatorHeadless.qml +++ /dev/null @@ -1,55 +0,0 @@ -import QtQuick -import qs.HyprChrome.Widgets.Luna - -// Offscreen render of every voice-mode state, plus the disconnected one — -// the states are the whole point of this widget, and a broken colour or a -// collapsed meter is invisible in a screenshot of just one of them. -// -// ./tools/quickshell-preview/render.sh \ -// tests/LunaIndicatorHeadless.qml \ -// .artifacts/quickshell-preview/luna.png 300 620 -Rectangle { - width: 300 - height: 620 - color: "#1b1d20" - - Column { - x: 20 - y: 20 - width: 260 - spacing: 14 - - LunaIndicator { - width: parent.width - state: "idle" - rms: 0.002 - } - - LunaIndicator { - width: parent.width - state: "listening" - rms: 0.08 - userText: "hey luna, are you there" - } - - LunaIndicator { - width: parent.width - state: "thinking" - userText: "what's the time?" - } - - LunaIndicator { - width: parent.width - state: "speaking" - rms: 0.14 - userText: "what's the time?" - lunaText: "It's 22:25 on Sunday night, chief. Shouldn't you be asleep?" - } - - LunaIndicator { - width: parent.width - state: "idle" - connected: false - } - } -} diff --git a/dotfiles/quickshell/tests/LunaPanelHeadless.qml b/dotfiles/quickshell/tests/LunaPanelHeadless.qml new file mode 100644 index 0000000..4d59bf9 --- /dev/null +++ b/dotfiles/quickshell/tests/LunaPanelHeadless.qml @@ -0,0 +1,90 @@ +import QtQuick +import qs.HyprChrome.Widgets.Bar.Luna + +// The rail's luna panel in both densities and every daemon state. Collapsed +// is the left column (state only), expanded the right (messages) — the two +// things the panel has to get right, side by side. +// +// ./tools/quickshell-preview/render.sh \ +// tests/LunaPanelHeadless.qml \ +// .artifacts/quickshell-preview/luna-panel.png 640 560 +Rectangle { + width: 640 + height: 560 + color: "#0f1012" + + Row { + x: 16 + y: 16 + spacing: 24 + + Column { + spacing: 12 + + LunaPanel { + expanded: false + connected: true + active: false + daemonState: "idle" + } + + LunaPanel { + expanded: false + connected: true + active: true + daemonState: "listening" + rms: 0.02 + } + + LunaPanel { + expanded: false + connected: true + active: true + daemonState: "thinking" + } + + LunaPanel { + expanded: false + connected: true + active: true + daemonState: "speaking" + rms: 0.05 + } + + LunaPanel { + expanded: false + connected: false + } + } + + Column { + spacing: 12 + + LunaPanel { + expanded: true + connected: true + active: false + daemonState: "idle" + } + + LunaPanel { + expanded: true + connected: true + active: true + daemonState: "listening" + rms: 0.02 + userText: "hey luna, what's the time?" + } + + LunaPanel { + expanded: true + connected: true + active: true + daemonState: "speaking" + rms: 0.05 + userText: "hey luna, what's the time?" + lunaText: "It's 23:40 on Sunday night, chief. Shouldn't you be asleep?" + } + } + } +} diff --git a/hosts/terra/configuration.nix b/hosts/terra/configuration.nix index add320a..5be5dc1 100644 --- a/hosts/terra/configuration.nix +++ b/hosts/terra/configuration.nix @@ -37,7 +37,8 @@ in You are being spoken to out loud and your reply will be read aloud by a speech synthesiser. Answer in one or two sentences unless asked for detail. No markdown, no lists, no code blocks — they do not survive the - trip to audio. + trip to audio. You may use an audio tag now and then, such as [laughs], + [sighs] or [whispers]; the synthesiser performs them. ''; }; diff --git a/pkgs/luna-voice/src/config.rs b/pkgs/luna-voice/src/config.rs index 0fc4a00..99923f4 100644 --- a/pkgs/luna-voice/src/config.rs +++ b/pkgs/luna-voice/src/config.rs @@ -10,6 +10,12 @@ pub struct Config { pub hermes_key: String, pub model: String, pub system_prompt: Option, + /// Prefixed to every spoken message so luna can tell this channel apart + /// from Telegram or the dashboard. Her own profile on mars keys off it — + /// which is why it is a per-message marker and not a system prompt: a + /// system prompt is this client's opinion, while the marker survives into + /// whatever history Hermes keeps and still identifies the channel later. + pub voice_marker: Option, pub eleven_key: String, pub voice_id: String, @@ -55,10 +61,15 @@ impl Config { hermes_key: secret("LUNA_HERMES_KEY_FILE")?, model: env_or("LUNA_MODEL", "hermes"), system_prompt: std::env::var("LUNA_SYSTEM_PROMPT").ok(), + // Empty string disables it rather than sending a bare prefix. + voice_marker: match env_or("LUNA_VOICE_MARKER", "[voice]") { + m if m.is_empty() => None, + m => Some(m), + }, eleven_key: secret("LUNA_ELEVENLABS_KEY_FILE")?, voice_id: env("LUNA_VOICE_ID")?, - tts_model: env_or("LUNA_TTS_MODEL", "eleven_flash_v2_5"), + tts_model: env_or("LUNA_TTS_MODEL", "eleven_v3_conversational"), whisper_bin: env_or("LUNA_WHISPER_BIN", "whisper-cli"), whisper_model: PathBuf::from(env("LUNA_WHISPER_MODEL")?), diff --git a/pkgs/luna-voice/src/main.rs b/pkgs/luna-voice/src/main.rs index 9a7307b..c2fb57f 100644 --- a/pkgs/luna-voice/src/main.rs +++ b/pkgs/luna-voice/src/main.rs @@ -83,6 +83,18 @@ fn root_threshold(floor: f32, mult: f32) -> f32 { (floor * mult).clamp(min, 0.05) } +/// Tags a spoken line so luna can recognise the channel it came from. +/// +/// Prefix rather than suffix: a long utterance can be truncated by whatever +/// history window she is running under, and the front is the end that +/// survives. +fn mark(marker: &Option, text: &str) -> String { + match marker { + Some(m) => format!("{m} {text}"), + None => text.to_string(), + } +} + fn socket_path() -> std::path::PathBuf { let runtime = std::env::var("XDG_RUNTIME_DIR").unwrap_or_else(|_| "/run/user/1000".into()); std::env::var("LUNA_SOCKET") @@ -366,7 +378,20 @@ async fn daemon() -> Result<()> { /// listener: journal, socket, desktop notification. #[cfg(test)] mod tests { - use super::root_threshold; + use super::{mark, root_threshold}; + + #[test] + fn marker_prefixes_spoken_lines() { + assert_eq!( + mark(&Some("[voice]".into()), "what's the time"), + "[voice] what's the time" + ); + } + + #[test] + fn no_marker_leaves_the_line_alone() { + assert_eq!(mark(&None, "what's the time"), "what's the time"); + } #[test] fn threshold_sits_between_terras_room_tone_and_terras_speech() { @@ -423,8 +448,9 @@ async fn turn( heard: String, ) -> Result<()> { eprintln!("luna-voice: heard {heard:?}"); + // The caption shows what you SAID; the marker is only for her. ipc.emit(Event::Caption { speaker: "user", text: heard.clone() }); - history.push(brain::Message::user(heard)); + history.push(brain::Message::user(mark(&cfg.voice_marker, &heard))); // Commands that arrived while she was speaking, replayed into the // channel once the turn is over rather than dropped. @@ -437,12 +463,13 @@ async fn turn( let (sentence_tx, sentence_rx) = mpsc::channel::(16); let (event_tx, mut event_rx) = mpsc::channel::(64); + let tts_cfg = tts::TtsConfig { + key: cfg.eleven_key.clone(), + voice_id: cfg.voice_id.clone(), + model: cfg.tts_model.clone(), + }; + let keep_tags = tts_cfg.is_v3(); let speech = tokio::spawn({ - let tts_cfg = tts::TtsConfig { - key: cfg.eleven_key.clone(), - voice_id: cfg.voice_id.clone(), - model: cfg.tts_model.clone(), - }; async move { tts::speak(&tts_cfg, sentence_rx, event_tx).await } }); @@ -484,20 +511,31 @@ async fn turn( async move { brain::stream_reply(&http, &url, &key, &model, &history, delta_tx).await } }); - let mut splitter = tts::SentenceSplitter::new(); + let mut splitter = tts::SentenceSplitter::new(keep_tags); + // What ElevenLabs bills: every character sent, audio tags included. + let mut billed = 0usize; + let caption = |sentence: &str| { + let text = tts::strip_tags(sentence); + if !text.is_empty() { + ipc.emit(Event::Caption { speaker: "luna", text }); + } + }; while let Some(delta) = delta_rx.recv().await { for sentence in splitter.push(&delta) { - ipc.emit(Event::Caption { speaker: "luna", text: sentence.clone() }); + caption(&sentence); + billed += sentence.chars().count(); if sentence_tx.send(sentence).await.is_err() { break; } } } if let Some(tail) = splitter.finish() { - ipc.emit(Event::Caption { speaker: "luna", text: tail.clone() }); + caption(&tail); + billed += tail.chars().count(); let _ = sentence_tx.send(tail).await; } drop(sentence_tx); + eprintln!("luna-voice: tts {billed} chars on {}", cfg.tts_model); let full = reply.await??; eprintln!("luna-voice: said {:?}", full.trim()); diff --git a/pkgs/luna-voice/src/tts.rs b/pkgs/luna-voice/src/tts.rs index cf2cffa..1d48ae5 100644 --- a/pkgs/luna-voice/src/tts.rs +++ b/pkgs/luna-voice/src/tts.rs @@ -1,8 +1,9 @@ use crate::audio; use crate::visemes::{self, Viseme}; -use anyhow::Result; +use anyhow::{anyhow, Result}; use base64::Engine; use futures_util::{SinkExt, StreamExt}; +use std::time::Duration; use tokio::sync::mpsc; use tokio_tungstenite::tungstenite::client::IntoClientRequest; use tokio_tungstenite::tungstenite::Message as Ws; @@ -21,6 +22,33 @@ pub struct TtsConfig { pub model: String, } +impl TtsConfig { + /// v3 is not served on the TTS websocket at all; it streams only over + /// Text to Dialogue, a different protocol. + pub fn is_v3(&self) -> bool { + self.model.starts_with("eleven_v3") + } +} + +/// The two sockets send the same audio frames under different field names. +struct Wire { + alignment: &'static [&'static str], + final_key: &'static str, +} + +const TTS_WIRE: Wire = Wire { alignment: &["normalizedAlignment", "alignment"], final_key: "isFinal" }; +const DIALOGUE_WIRE: Wire = Wire { alignment: &["alignment"], final_key: "is_final" }; + +/// Dialogue drops the socket after 20s without a client message, and a tool +/// call on mars can stall a reply longer than that. +const KEEP_ALIVE: Duration = Duration::from_secs(10); + +type Socket = tokio_tungstenite::WebSocketStream< + tokio_tungstenite::MaybeTlsStream, +>; +type Sink = futures_util::stream::SplitSink; +type Source = futures_util::stream::SplitStream; + /// Speaks sentences as they arrive on `sentences`, over one websocket per /// reply. /// @@ -28,6 +56,35 @@ pub struct TtsConfig { /// and no reason to add one. sync_alignment makes the character timings ride /// along with the audio frames instead of needing a second request. pub async fn speak( + cfg: &TtsConfig, + sentences: mpsc::Receiver, + events: mpsc::Sender, +) -> Result<()> { + if cfg.is_v3() { + speak_dialogue(cfg, sentences, events).await + } else { + speak_tts(cfg, sentences, events).await + } +} + +async fn connect(url: &str, key: &str) -> Result<(Sink, Source)> { + let mut req = url.into_client_request()?; + // Header, not the ?authorization= query param: a key in a URL ends up in + // every proxy and error log that touches it. + req.headers_mut().insert( + "xi-api-key", + tokio_tungstenite::tungstenite::http::HeaderValue::from_str(key)?, + ); + let (stream, _) = tokio_tungstenite::connect_async(req).await?; + Ok(stream.split()) +} + +async fn send_json(sink: &mut Sink, v: serde_json::Value) -> Result<()> { + sink.send(Ws::Text(v.to_string())).await?; + Ok(()) +} + +async fn speak_tts( cfg: &TtsConfig, mut sentences: mpsc::Receiver, events: mpsc::Sender, @@ -39,85 +96,151 @@ pub async fn speak( cfg.model, audio::TTS_RATE ); - let mut req = url.into_client_request()?; - // Header, not the ?authorization= query param: a key in a URL ends up in - // every proxy and error log that touches it. - req.headers_mut().insert( - "xi-api-key", - tokio_tungstenite::tungstenite::http::HeaderValue::from_str(&cfg.key)?, - ); - - let (stream, _) = tokio_tungstenite::connect_async(req).await?; - let (mut sink, mut source) = stream.split(); + let (mut sink, source) = connect(&url, &cfg.key).await?; // The protocol requires an opening frame of a single space before any // real text; it carries the voice settings for the whole connection. - sink.send(Ws::Text( + send_json( + &mut sink, serde_json::json!({ "text": " ", "voice_settings": { "stability": 0.4, "similarity_boost": 0.8, "speed": 1.0 }, - }) - .to_string(), - )) + }), + ) .await?; - let reader = tokio::spawn(async move { - // Alignment times may be per-chunk or cumulative depending on the - // model; `emitted_ms` lets us normalise either into one timeline. - let mut emitted_ms: u32 = 0; - while let Some(Ok(msg)) = source.next().await { - let Ws::Text(txt) = msg else { continue }; - let Ok(v) = serde_json::from_str::(&txt) else { continue }; + let reader = tokio::spawn(read_audio(source, events, &TTS_WIRE)); - if let Some(b64) = v.get("audio").and_then(|a| a.as_str()) { - if let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(b64) { - let pcm = audio::pcm16_to_f32(&bytes); - let chunk_ms = (pcm.len() as f64 / audio::TTS_RATE as f64 * 1000.0) as u32; - if let Some(al) = v.get("normalizedAlignment").or_else(|| v.get("alignment")) { - if let Some(t) = alignment_to_timeline(al, emitted_ms) { - let _ = events.send(SpeechEvent::Visemes(t)).await; - } - } - emitted_ms += chunk_ms; - if events.send(SpeechEvent::Audio(pcm)).await.is_err() { - return; // receiver gone: barge-in cancelled this reply - } - } - } - if v.get("isFinal").and_then(|f| f.as_bool()).unwrap_or(false) { - break; - } + let sent: Result<()> = async { + while let Some(text) = sentences.recv().await { + // Every text frame must end with a space or the model holds the + // last word back waiting for more input. + let text = format!("{} ", text.trim()); + send_json(&mut sink, serde_json::json!({ "text": text })).await?; } - let _ = events.send(SpeechEvent::Done).await; - }); - - while let Some(text) = sentences.recv().await { - // Every text frame must end with a space or the model holds the last - // word back waiting for more input. - let text = format!("{} ", text.trim()); - sink.send(Ws::Text( - serde_json::json!({ "text": text }).to_string(), - )) - .await?; + // Empty string = end of input. Without it the socket sits until the + // inactivity timeout and the last sentence never gets flushed. + send_json(&mut sink, serde_json::json!({ "text": "" })).await } + .await; - // Empty string = end of input. Without it the socket sits until the - // inactivity timeout and the last sentence never gets flushed. - sink.send(Ws::Text(serde_json::json!({ "text": "" }).to_string())) - .await?; - let _ = reader.await; - Ok(()) + // The server's own error beats the "connection reset" a send hit after it. + reader.await??; + sent } -fn alignment_to_timeline(al: &serde_json::Value, emitted_ms: u32) -> Option> { +async fn speak_dialogue( + cfg: &TtsConfig, + mut sentences: mpsc::Receiver, + events: mpsc::Sender, +) -> Result<()> { + let url = format!( + "wss://api.elevenlabs.io/v1/text-to-dialogue/stream-input\ + ?model_id={}&output_format=pcm_{}&sync_alignment=true", + cfg.model, + audio::TTS_RATE + ); + let (mut sink, source) = connect(&url, &cfg.key).await?; + + // No voice_settings: v3 reads only stability, and only as one of three + // presets; the default is its "natural" one. + send_json(&mut sink, serde_json::json!({ "voices": [cfg.voice_id] })).await?; + + let reader = tokio::spawn(read_audio(source, events, &DIALOGUE_WIRE)); + + let sent: Result<()> = async { + loop { + match tokio::time::timeout(KEEP_ALIVE, sentences.recv()).await { + Ok(Some(text)) => { + // Inputs are concatenated verbatim, so the space matters + // here too. flush: the server otherwise holds text until + // ~40 chars and 8 words, so a short first sentence would + // wait for the second. + let text = format!("{} ", text.trim()); + send_json( + &mut sink, + serde_json::json!({ + "inputs": [{ "text": text, "voice_id": cfg.voice_id }], + "flush": true, + }), + ) + .await? + } + Ok(None) => break, + Err(_) => send_json(&mut sink, serde_json::json!({ "keep_alive": true })).await?, + } + } + send_json(&mut sink, serde_json::json!({ "close_socket": true })).await + } + .await; + + reader.await??; + sent +} + +async fn read_audio( + mut source: Source, + events: mpsc::Sender, + wire: &'static Wire, +) -> Result<()> { + // Alignment times may be per-chunk or cumulative depending on the + // model; `emitted_ms` lets us normalise either into one timeline. + let mut emitted_ms: u32 = 0; + // v3 audio tags come back as characters, and one can straddle chunks. + let mut in_tag = false; + let mut result = Ok(()); + while let Some(Ok(msg)) = source.next().await { + let Ws::Text(txt) = msg else { continue }; + let Ok(v) = serde_json::from_str::(&txt) else { continue }; + + if let Some(err) = v.get("error").and_then(|e| e.as_str()) { + let detail = v.get("message").and_then(|m| m.as_str()).unwrap_or(""); + result = Err(anyhow!("elevenlabs: {err} {detail}")); + break; + } + + if let Some(b64) = v.get("audio").and_then(|a| a.as_str()) { + if let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(b64) { + let pcm = audio::pcm16_to_f32(&bytes); + let chunk_ms = (pcm.len() as f64 / audio::TTS_RATE as f64 * 1000.0) as u32; + let al = wire.alignment.iter().find_map(|k| v.get(*k).filter(|a| !a.is_null())); + if let Some(t) = al.and_then(|al| alignment_to_timeline(al, emitted_ms, &mut in_tag)) { + let _ = events.send(SpeechEvent::Visemes(t)).await; + } + emitted_ms += chunk_ms; + if events.send(SpeechEvent::Audio(pcm)).await.is_err() { + return Ok(()); // receiver gone: barge-in cancelled this reply + } + } + } + if v.get(wire.final_key).and_then(|f| f.as_bool()).unwrap_or(false) { + break; + } + } + let _ = events.send(SpeechEvent::Done).await; + result +} + +fn alignment_to_timeline( + al: &serde_json::Value, + emitted_ms: u32, + in_tag: &mut bool, +) -> Option> { + let field = |camel: &str, snake: &str| al.get(camel).or_else(|| al.get(snake)); let chars: Vec = al .get("chars")? .as_array()? .iter() - .map(|c| c.as_str().unwrap_or("").to_string()) + .map(|c| c.as_str().unwrap_or("")) + .map(|c| { + // A tag is a laugh or a sigh, not letters: the mouth rests. + let tagged = *in_tag || c == "["; + *in_tag = tagged && c != "]"; + if tagged { " ".to_string() } else { c.to_string() } + }) .collect(); - let starts: Vec = num_array(al.get("charStartTimesMs")?)?; - let durs: Vec = num_array(al.get("charDurationsMs")?)?; + let starts: Vec = num_array(field("charStartTimesMs", "char_start_times_ms")?)?; + let durs: Vec = num_array(field("charDurationsMs", "char_durations_ms")?)?; let starts = absolutize(&starts, emitted_ms); Some(visemes::timeline(&chars, &starts, &durs)) } @@ -143,11 +266,13 @@ fn absolutize(starts: &[i64], emitted_ms: u32) -> Vec { pub struct SentenceSplitter { buf: String, in_think: bool, + /// Whether v3 audio tags survive to the synthesiser or are dropped. + keep_tags: bool, } impl SentenceSplitter { - pub fn new() -> Self { - SentenceSplitter { buf: String::new(), in_think: false } + pub fn new(keep_tags: bool) -> Self { + SentenceSplitter { buf: String::new(), in_think: false, keep_tags } } /// Feeds a delta, returning any sentences that completed. @@ -174,24 +299,24 @@ impl SentenceSplitter { let head: String = self.buf.drain(..i).collect(); self.buf.drain(.."".len()); self.in_think = true; - out.extend(drain_sentences(&mut { head.clone() }, true)); + out.extend(drain_sentences(&mut { head.clone() }, true, self.keep_tags)); } None => break, } } - out.extend(drain_sentences(&mut self.buf, false)); + out.extend(drain_sentences(&mut self.buf, false, self.keep_tags)); out.into_iter().filter(|s| !s.is_empty()).collect() } /// Whatever is left when the reply ends. pub fn finish(&mut self) -> Option { - let rest = clean_for_speech(&std::mem::take(&mut self.buf)); + let rest = clean_for_speech(&std::mem::take(&mut self.buf), self.keep_tags); (!rest.is_empty()).then_some(rest) } } -fn drain_sentences(buf: &mut String, force: bool) -> Vec { +fn drain_sentences(buf: &mut String, force: bool, keep_tags: bool) -> Vec { let mut out = Vec::new(); // A sentence is only worth sending once it is long enough to synthesise // well; ElevenLabs' own guidance is ~20 characters minimum. @@ -203,13 +328,13 @@ fn drain_sentences(buf: &mut String, force: bool) -> Vec { } } let s: String = buf.drain(..end + 1).collect(); - let s = clean_for_speech(&s); + let s = clean_for_speech(&s, keep_tags); if !s.is_empty() { out.push(s); } } if force && !buf.is_empty() { - let s = clean_for_speech(&std::mem::take(buf)); + let s = clean_for_speech(&std::mem::take(buf), keep_tags); if !s.is_empty() { out.push(s); } @@ -217,16 +342,64 @@ fn drain_sentences(buf: &mut String, force: bool) -> Vec { out } +/// Byte index just past a v3 audio tag starting at `i` ("[laughs]", +/// "[clears throat]"), if there is one. Letters and spaces only, so +/// "[1]" or a markdown link stays ordinary noise. +fn tag_end(s: &str, i: usize) -> Option { + let rest = s[i..].strip_prefix('[')?; + let close = rest.find(']')?; + let body = &rest[..close]; + let end = i + 1 + close + 1; + let ok = !body.trim().is_empty() + && body.len() <= 40 + && body.chars().all(|c| c.is_alphabetic() || c == ' ' || c == '-') + && !s[end..].starts_with('('); + ok.then_some(end) +} + +/// The text as a person reads it: captions show "ha!", not "[laughs]". +pub fn strip_tags(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut skip_to = 0; + for (i, c) in s.char_indices() { + if i < skip_to { + continue; + } + match tag_end(s, i) { + Some(end) => { + skip_to = end; + out.push(' '); + } + None => out.push(c), + } + } + out.split_whitespace().collect::>().join(" ") +} + /// Markdown, emoji and kaomoji are for eyes. /// /// A whitelist, not a blacklist: she writes things like "(¬‿¬)", whose /// characters live in ordinary Latin-1 and punctuation blocks rather than in /// any emoji range, so enumerating what to drop never converges. Letters /// stays unicode-aware on purpose — German is in scope, "ö" is a letter. -pub fn clean_for_speech(s: &str) -> String { +/// +/// Audio tags pass through untouched when `keep_tags`; otherwise they are +/// dropped whole, since a model without them would read "laughs" aloud. +pub fn clean_for_speech(s: &str, keep_tags: bool) -> String { let mut out = String::with_capacity(s.len()); - for c in s.chars() { - if c.is_alphanumeric() || c.is_whitespace() || is_speakable_punctuation(c) { + let mut skip_to = 0; + for (i, c) in s.char_indices() { + if i < skip_to { + continue; + } + if let Some(end) = tag_end(s, i) { + skip_to = end; + out.push(' '); + if keep_tags { + out.push_str(&s[i..end]); + out.push(' '); + } + } else if c.is_alphanumeric() || c.is_whitespace() || is_speakable_punctuation(c) { out.push(c); } else if matches!(c, '—' | '–') { // " — " must not leave " , ": the comma binds to the word before it. @@ -271,7 +444,7 @@ mod tests { #[test] fn splitter_emits_complete_sentences_only() { - let mut s = SentenceSplitter::new(); + let mut s = SentenceSplitter::new(false); assert!(s.push("This is a long enough first sentence").is_empty()); let got = s.push(". And more"); assert_eq!(got, vec!["This is a long enough first sentence."]); @@ -279,7 +452,7 @@ mod tests { #[test] fn splitter_drops_think_blocks() { - let mut s = SentenceSplitter::new(); + let mut s = SentenceSplitter::new(false); s.push("internal monologue that is long"); let got = s.push("Actually speakable sentence here. "); assert_eq!(got, vec!["Actually speakable sentence here."]); @@ -287,33 +460,70 @@ mod tests { #[test] fn splitter_returns_tail_on_finish() { - let mut s = SentenceSplitter::new(); + let mut s = SentenceSplitter::new(false); s.push("no terminator here"); assert_eq!(s.finish().as_deref(), Some("no terminator here")); } #[test] fn markdown_and_emoji_never_reach_the_voice() { - assert_eq!(clean_for_speech("**bold** `code` 🎉 ok"), "bold code ok"); + assert_eq!(clean_for_speech("**bold** `code` 🎉 ok", false), "bold code ok"); } #[test] fn kaomoji_and_dashes_are_stripped_without_gluing_words() { // Her actual first reply over the API. assert_eq!( - clean_for_speech("Loud and clear, boss — hallo! (¬‿¬)"), + clean_for_speech("Loud and clear, boss — hallo! (¬‿¬)", false), "Loud and clear, boss, hallo!" ); } #[test] fn em_dash_becomes_a_pause_but_hyphens_do_not() { - assert_eq!(clean_for_speech("I can't confirm — there's nothing"), "I can't confirm, there's nothing"); - assert_eq!(clean_for_speech("push-to-talk"), "push to talk"); + assert_eq!(clean_for_speech("I can't confirm — there's nothing", false), "I can't confirm, there's nothing"); + assert_eq!(clean_for_speech("push-to-talk", false), "push to talk"); } #[test] fn keeps_non_ascii_letters() { - assert_eq!(clean_for_speech("schön, größer"), "schön, größer"); + assert_eq!(clean_for_speech("schön, größer", false), "schön, größer"); + } + + #[test] + fn audio_tags_survive_for_v3_only() { + let s = "[laughs]Fine, boss. [clears throat] Na gut."; + assert_eq!(clean_for_speech(s, true), "[laughs] Fine, boss. [clears throat] Na gut."); + assert_eq!(clean_for_speech(s, false), "Fine, boss. Na gut."); + } + + #[test] + fn brackets_that_are_not_tags_stay_noise() { + assert_eq!(clean_for_speech("see [1] and [the docs](x)", true), "see 1 and the docs x"); + } + + #[test] + fn captions_hide_tags() { + assert_eq!(strip_tags("[sighs] Okay, boss."), "Okay, boss."); + } + + #[test] + fn tag_characters_rest_the_mouth_across_chunks() { + let mut in_tag = false; + let chunk = |chars: &[&str], in_tag: &mut bool| { + let n = chars.len() as i64; + let al = serde_json::json!({ + "chars": chars, + "char_start_times_ms": (0..n).map(|i| i * 100).collect::>(), + "char_durations_ms": vec![100; chars.len()], + }); + alignment_to_timeline(&al, 0, in_tag).unwrap() + }; + let t = chunk(&["[", "l", "a"], &mut in_tag); + assert!(t.iter().all(|v| v.shape == visemes::SILENCE)); + assert!(in_tag); + let t = chunk(&["u", "]", "a"], &mut in_tag); + assert!(!in_tag); + assert_eq!(t.last().unwrap().shape, "aa"); } } diff --git a/services/desktop/luna-voice.nix b/services/desktop/luna-voice.nix index 9ce12ff..448173e 100644 --- a/services/desktop/luna-voice.nix +++ b/services/desktop/luna-voice.nix @@ -1,11 +1,12 @@ { config, pkgs, lib, ... }: -# Voice loop for luna: hold a key, talk, hear her answer. +# Voice loop for luna: SUPER+V opens the mic, talk, hear her answer. # # mic -> whisper-cpp (local, vulkan) -> Hermes' OpenAI-compatible API on mars -# -> ElevenLabs streaming TTS -> speaker. State, amplitude and the viseme -# timeline are published on a unix socket for the quickshell avatar to read -# (nothing consumes it yet — the widget is the next phase). +# -> ElevenLabs streaming TTS -> speaker. State, amplitude, captions and a +# viseme timeline are published on a unix socket; the rail's LunaPanel reads +# them (dotfiles/quickshell/HyprChrome/Widgets/Bar/Luna), and the viseme +# stream is there for the avatar that replaces its meter later. # # A USER service, not a system one: it needs darman's pipewire session and his # XDG_RUNTIME_DIR for the socket. That is also why the two secrets it reads @@ -57,8 +58,13 @@ in ttsModel = lib.mkOption { type = lib.types.str; - default = "eleven_flash_v2_5"; - description = "~75ms model, the one ElevenLabs recommends for live conversation."; + default = "eleven_v3_conversational"; + description = '' + eleven_v3* streams over the Text to Dialogue socket and performs audio + tags ([laughs], [sighs]); it also gets her German right, which Flash + reads with an English accent. Anything else uses the TTS socket, e.g. + "eleven_flash_v2_5" — half the price per character, faster, no tags. + ''; }; hermesKeyFile = lib.mkOption { @@ -99,6 +105,21 @@ in description = "Silence that ends an utterance."; }; + voiceMarker = lib.mkOption { + type = lib.types.str; + default = "[voice]"; + description = '' + Prefixed to every spoken message so luna can tell this channel from + Telegram or the dashboard, and adjust — no kaomoji, no markdown, short + answers. Put the rule in HER profile on mars rather than only in + systemPrompt below: the marker travels with the message, so it still + identifies the channel in whatever history Hermes keeps, and one place + governs her voice manners across every client that sends it. + + Empty string disables the marker. + ''; + }; + systemPrompt = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; @@ -131,6 +152,7 @@ in LUNA_VAD_MULT = toString cfg.vadMult; LUNA_VAD_FLOOR = toString cfg.vadFloor; LUNA_SILENCE_MS = toString cfg.silenceMs; + LUNA_VOICE_MARKER = cfg.voiceMarker; } // lib.optionalAttrs (cfg.systemPrompt != null) { LUNA_SYSTEM_PROMPT = cfg.systemPrompt; } // lib.optionalAttrs (cfg.inputDevice != null) {