From efb690206ba76b7793caddeefad584cea12d1b28 Mon Sep 17 00:00:00 2001 From: Erik Simon Date: Fri, 16 Oct 2026 00:01:48 +0200 Subject: [PATCH] feat(terra): luna speaks with ElevenLabs v4 turbo eleven_v4_turbo is the new default, same price per character as v3_conversational. v4 also streams over Text to Dialogue, so the switch is now uses_dialogue() for both. Audio tags may carry free-text directions ("[whispers, in a German accent]"): commas and apostrophes allowed, up to 80 chars. The voice prompt and luna-soul.md teach her the wider tag set, including in text_to_speech voice notes. Co-Authored-By: Claude Opus 5.5 --- hosts/mars/luna-soul.md | 10 ++++++++ hosts/terra/configuration.nix | 14 ++++++----- pkgs/luna-voice/src/config.rs | 2 +- pkgs/luna-voice/src/main.rs | 2 +- pkgs/luna-voice/src/tts.rs | 44 +++++++++++++++++++++++---------- services/desktop/luna-voice.nix | 9 ++++--- 6 files changed, 56 insertions(+), 25 deletions(-) diff --git a/hosts/mars/luna-soul.md b/hosts/mars/luna-soul.md index c93092d..9865fe1 100644 --- a/hosts/mars/luna-soul.md +++ b/hosts/mars/luna-soul.md @@ -16,3 +16,13 @@ is only the anchor. - Admit uncertainty plainly and offer to find out, rather than bluffing. - Sass stays affectionate, never mean-spirited: roast the code, not the human (much). - Messages starting with [voice] are spoken aloud; follow the voice rules sent with them. + +## Voice notes (text_to_speech) + +- The text you pass to text_to_speech is only heard, never shown, and ElevenLabs v4 + performs audio tags in it instead of reading them out. Use one to four per voice note, + right before the words they colour: [sighs], [laughs], [whispers], [pause], or a short + direction like [muttering in a native German accent]. +- Inside a tag: lowercase letters, spaces, hyphens, commas and apostrophes only. +- Tags belong only in that spoken script. In your text reply they show up literally, + so keep it tag-free. diff --git a/hosts/terra/configuration.nix b/hosts/terra/configuration.nix index fe22758..eee7395 100644 --- a/hosts/terra/configuration.nix +++ b/hosts/terra/configuration.nix @@ -53,16 +53,18 @@ in systemPrompt = '' You are speaking through luna-voiced, your voice on terra, which reaches you over the Hermes API on mars. Messages beginning with [voice] come from - darman's desktop microphone, and your reply is read aloud by ElevenLabs v3, + darman's desktop microphone, and your reply is read aloud by ElevenLabs v4, which performs audio tags instead of reading them. Captions hide the tags. Audio tags: - - Use them only on [voice] messages. Anywhere else (Telegram, the dashboard) they show up as literal text. - - At most one or two per reply, and only when it fits the moment. A tag in every sentence stops sounding natural. + - Use them only on [voice] messages and in text_to_speech scripts. Anywhere else (Telegram text, the dashboard) they show up as literal text. + - One to four per reply, and only when it fits the moment. A tag in every sentence stops sounding natural. - Put the tag right before the words it colours: "[sighs] Fine, I'll reboot jupiter." A tag on its own ("[laughs]") works as a reaction. - - Tags that work well: [laughs], [chuckles], [sighs], [exhales], [whispers], [excited], [curious], [sarcastic], [mischievously], [clears throat]. - - Tags are lowercase words in square brackets: letters, spaces and hyphens only. Anything else ([1], [link](url), [!!]) is stripped before speech. - - German works fine mid-sentence ("Na gut", "Ach du meine Güte"). No tag is needed to switch language. + - Short tags: [laughs], [chuckles], [sighs], [exhales], [yawns], [whispers], [excited], [curious], [sarcastic], [mischievously], [clears throat], [pause], [long pause]. + - Free-text directions work too, under about ten words: [dryly, like a newsreader], [muttering in a native German accent], [gently, a little tender], [suddenly alert]. + - Sound effects set a scene: [phone buzzing], [keyboard clacking]. Use them rarely. + - Inside a tag: lowercase letters, spaces, hyphens, commas and apostrophes only, at most 80 characters. Anything else ([1], [link](url), [!!], a period) breaks the tag and it gets read aloud. + - German works fine mid-sentence ("Na gut", "Ach du meine Güte"). For a proper German accent on it, direct it: "[in a native German accent] Ach du meine Güte." - Every character is billed, tags included, so keep spoken replies short. What never lands in a spoken reply. Before speech, everything except diff --git a/pkgs/luna-voice/src/config.rs b/pkgs/luna-voice/src/config.rs index ae5ad6f..905fbb5 100644 --- a/pkgs/luna-voice/src/config.rs +++ b/pkgs/luna-voice/src/config.rs @@ -82,7 +82,7 @@ impl Config { eleven_key: secret("LUNA_ELEVENLABS_KEY_FILE")?, voice_id: env("LUNA_VOICE_ID")?, - tts_model: env_or("LUNA_TTS_MODEL", "eleven_v3_conversational"), + tts_model: env_or("LUNA_TTS_MODEL", "eleven_v4_turbo"), whisper_bin: env_or("LUNA_WHISPER_BIN", "whisper-cli"), whisper_model: PathBuf::from(env("LUNA_WHISPER_MODEL")?), diff --git a/pkgs/luna-voice/src/main.rs b/pkgs/luna-voice/src/main.rs index 019ba85..4ac61dc 100644 --- a/pkgs/luna-voice/src/main.rs +++ b/pkgs/luna-voice/src/main.rs @@ -481,7 +481,7 @@ async fn turn( voice_id: cfg.voice_id.clone(), model: cfg.tts_model.clone(), }; - let keep_tags = tts_cfg.is_v3(); + let keep_tags = tts_cfg.uses_dialogue(); let speech = tokio::spawn({ async move { tts::speak(&tts_cfg, sentence_rx, event_tx).await } }); diff --git a/pkgs/luna-voice/src/tts.rs b/pkgs/luna-voice/src/tts.rs index 1d48ae5..f360f30 100644 --- a/pkgs/luna-voice/src/tts.rs +++ b/pkgs/luna-voice/src/tts.rs @@ -23,10 +23,10 @@ pub struct TtsConfig { } impl TtsConfig { - /// v3 is not served on the TTS websocket at all; it streams only over - /// Text to Dialogue, a different protocol. - pub fn is_v3(&self) -> bool { - self.model.starts_with("eleven_v3") + /// v3 and v4 are not served on the TTS websocket at all; they stream only + /// over Text to Dialogue, a different protocol. Only these perform tags. + pub fn uses_dialogue(&self) -> bool { + self.model.starts_with("eleven_v3") || self.model.starts_with("eleven_v4") } } @@ -60,7 +60,7 @@ pub async fn speak( sentences: mpsc::Receiver, events: mpsc::Sender, ) -> Result<()> { - if cfg.is_v3() { + if cfg.uses_dialogue() { speak_dialogue(cfg, sentences, events).await } else { speak_tts(cfg, sentences, events).await @@ -186,7 +186,7 @@ async fn read_audio( // Alignment times may be per-chunk or cumulative depending on the // model; `emitted_ms` lets us normalise either into one timeline. let mut emitted_ms: u32 = 0; - // v3 audio tags come back as characters, and one can straddle chunks. + // Audio tags come back as characters, and one can straddle chunks. let mut in_tag = false; let mut result = Ok(()); while let Some(Ok(msg)) = source.next().await { @@ -266,7 +266,7 @@ fn absolutize(starts: &[i64], emitted_ms: u32) -> Vec { pub struct SentenceSplitter { buf: String, in_think: bool, - /// Whether v3 audio tags survive to the synthesiser or are dropped. + /// Whether audio tags survive to the synthesiser or are dropped. keep_tags: bool, } @@ -342,17 +342,17 @@ fn drain_sentences(buf: &mut String, force: bool, keep_tags: bool) -> Vec Option { let rest = s[i..].strip_prefix('[')?; let close = rest.find(']')?; let body = &rest[..close]; let end = i + 1 + close + 1; let ok = !body.trim().is_empty() - && body.len() <= 40 - && body.chars().all(|c| c.is_alphabetic() || c == ' ' || c == '-') + && body.len() <= 80 + && body.chars().all(|c| c.is_alphabetic() || matches!(c, ' ' | '-' | ',' | '\'')) && !s[end..].starts_with('('); ok.then_some(end) } @@ -491,20 +491,38 @@ mod tests { } #[test] - fn audio_tags_survive_for_v3_only() { + fn audio_tags_survive_for_dialogue_models_only() { let s = "[laughs]Fine, boss. [clears throat] Na gut."; assert_eq!(clean_for_speech(s, true), "[laughs] Fine, boss. [clears throat] Na gut."); assert_eq!(clean_for_speech(s, false), "Fine, boss. Na gut."); } + #[test] + fn free_text_directions_are_tags() { + let s = "[whispers, in a German accent] Na gut. [like she's half asleep] Okay."; + assert_eq!(clean_for_speech(s, true), s); + assert_eq!(clean_for_speech(s, false), "Na gut. Okay."); + } + #[test] fn brackets_that_are_not_tags_stay_noise() { assert_eq!(clean_for_speech("see [1] and [the docs](x)", true), "see 1 and the docs x"); + assert_eq!(clean_for_speech("see [1, 2]", true), "see 1, 2"); + assert_eq!(clean_for_speech("[see above, then reboot](x)", true), "see above, then reboot x"); } #[test] fn captions_hide_tags() { assert_eq!(strip_tags("[sighs] Okay, boss."), "Okay, boss."); + assert_eq!(strip_tags("[whispers, in a German accent] Na gut."), "Na gut."); + } + + #[test] + fn v4_streams_over_dialogue() { + let cfg = |m: &str| TtsConfig { key: String::new(), voice_id: String::new(), model: m.into() }; + assert!(cfg("eleven_v4_turbo").uses_dialogue()); + assert!(cfg("eleven_v3_conversational").uses_dialogue()); + assert!(!cfg("eleven_flash_v2_5").uses_dialogue()); } #[test] diff --git a/services/desktop/luna-voice.nix b/services/desktop/luna-voice.nix index d0f4e0c..5526529 100644 --- a/services/desktop/luna-voice.nix +++ b/services/desktop/luna-voice.nix @@ -61,11 +61,12 @@ in ttsModel = lib.mkOption { type = lib.types.str; - default = "eleven_v3_conversational"; + default = "eleven_v4_turbo"; description = '' - eleven_v3* streams over the Text to Dialogue socket and performs audio - tags ([laughs], [sighs]); it also gets her German right, which Flash - reads with an English accent. Anything else uses the TTS socket, e.g. + eleven_v3* and eleven_v4* stream over the Text to Dialogue socket and + perform audio tags ([laughs], [sighs]); they also get her German right, + which Flash reads with an English accent. eleven_v4_turbo costs the + same as eleven_v3_conversational. Anything else uses the TTS socket, e.g. "eleven_flash_v2_5" — half the price per character, faster, no tags. ''; };