feat(terra): luna v3 voice, channel marker and rail panel
- TTS defaults to eleven_v3_conversational over the Text to Dialogue socket: performs audio tags and gets her German right. Flash still works via the TTS socket. - Every spoken message is prefixed with a [voice] marker so luna's profile on mars can tell the channel apart. - LunaIndicator/LunaCall replaced by a LunaPanel in the bar rail, with captions; headless test updated to match. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -10,6 +10,12 @@ pub struct Config {
|
||||
pub hermes_key: String,
|
||||
pub model: String,
|
||||
pub system_prompt: Option<String>,
|
||||
/// Prefixed to every spoken message so luna can tell this channel apart
|
||||
/// from Telegram or the dashboard. Her own profile on mars keys off it —
|
||||
/// which is why it is a per-message marker and not a system prompt: a
|
||||
/// system prompt is this client's opinion, while the marker survives into
|
||||
/// whatever history Hermes keeps and still identifies the channel later.
|
||||
pub voice_marker: Option<String>,
|
||||
|
||||
pub eleven_key: String,
|
||||
pub voice_id: String,
|
||||
@@ -55,10 +61,15 @@ impl Config {
|
||||
hermes_key: secret("LUNA_HERMES_KEY_FILE")?,
|
||||
model: env_or("LUNA_MODEL", "hermes"),
|
||||
system_prompt: std::env::var("LUNA_SYSTEM_PROMPT").ok(),
|
||||
// Empty string disables it rather than sending a bare prefix.
|
||||
voice_marker: match env_or("LUNA_VOICE_MARKER", "[voice]") {
|
||||
m if m.is_empty() => None,
|
||||
m => Some(m),
|
||||
},
|
||||
|
||||
eleven_key: secret("LUNA_ELEVENLABS_KEY_FILE")?,
|
||||
voice_id: env("LUNA_VOICE_ID")?,
|
||||
tts_model: env_or("LUNA_TTS_MODEL", "eleven_flash_v2_5"),
|
||||
tts_model: env_or("LUNA_TTS_MODEL", "eleven_v3_conversational"),
|
||||
|
||||
whisper_bin: env_or("LUNA_WHISPER_BIN", "whisper-cli"),
|
||||
whisper_model: PathBuf::from(env("LUNA_WHISPER_MODEL")?),
|
||||
|
||||
+48
-10
@@ -83,6 +83,18 @@ fn root_threshold(floor: f32, mult: f32) -> f32 {
|
||||
(floor * mult).clamp(min, 0.05)
|
||||
}
|
||||
|
||||
/// Tags a spoken line so luna can recognise the channel it came from.
|
||||
///
|
||||
/// Prefix rather than suffix: a long utterance can be truncated by whatever
|
||||
/// history window she is running under, and the front is the end that
|
||||
/// survives.
|
||||
fn mark(marker: &Option<String>, text: &str) -> String {
|
||||
match marker {
|
||||
Some(m) => format!("{m} {text}"),
|
||||
None => text.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn socket_path() -> std::path::PathBuf {
|
||||
let runtime = std::env::var("XDG_RUNTIME_DIR").unwrap_or_else(|_| "/run/user/1000".into());
|
||||
std::env::var("LUNA_SOCKET")
|
||||
@@ -366,7 +378,20 @@ async fn daemon() -> Result<()> {
|
||||
/// listener: journal, socket, desktop notification.
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::root_threshold;
|
||||
use super::{mark, root_threshold};
|
||||
|
||||
#[test]
|
||||
fn marker_prefixes_spoken_lines() {
|
||||
assert_eq!(
|
||||
mark(&Some("[voice]".into()), "what's the time"),
|
||||
"[voice] what's the time"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_marker_leaves_the_line_alone() {
|
||||
assert_eq!(mark(&None, "what's the time"), "what's the time");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn threshold_sits_between_terras_room_tone_and_terras_speech() {
|
||||
@@ -423,8 +448,9 @@ async fn turn(
|
||||
heard: String,
|
||||
) -> Result<()> {
|
||||
eprintln!("luna-voice: heard {heard:?}");
|
||||
// The caption shows what you SAID; the marker is only for her.
|
||||
ipc.emit(Event::Caption { speaker: "user", text: heard.clone() });
|
||||
history.push(brain::Message::user(heard));
|
||||
history.push(brain::Message::user(mark(&cfg.voice_marker, &heard)));
|
||||
|
||||
// Commands that arrived while she was speaking, replayed into the
|
||||
// channel once the turn is over rather than dropped.
|
||||
@@ -437,12 +463,13 @@ async fn turn(
|
||||
let (sentence_tx, sentence_rx) = mpsc::channel::<String>(16);
|
||||
let (event_tx, mut event_rx) = mpsc::channel::<tts::SpeechEvent>(64);
|
||||
|
||||
let tts_cfg = tts::TtsConfig {
|
||||
key: cfg.eleven_key.clone(),
|
||||
voice_id: cfg.voice_id.clone(),
|
||||
model: cfg.tts_model.clone(),
|
||||
};
|
||||
let keep_tags = tts_cfg.is_v3();
|
||||
let speech = tokio::spawn({
|
||||
let tts_cfg = tts::TtsConfig {
|
||||
key: cfg.eleven_key.clone(),
|
||||
voice_id: cfg.voice_id.clone(),
|
||||
model: cfg.tts_model.clone(),
|
||||
};
|
||||
async move { tts::speak(&tts_cfg, sentence_rx, event_tx).await }
|
||||
});
|
||||
|
||||
@@ -484,20 +511,31 @@ async fn turn(
|
||||
async move { brain::stream_reply(&http, &url, &key, &model, &history, delta_tx).await }
|
||||
});
|
||||
|
||||
let mut splitter = tts::SentenceSplitter::new();
|
||||
let mut splitter = tts::SentenceSplitter::new(keep_tags);
|
||||
// What ElevenLabs bills: every character sent, audio tags included.
|
||||
let mut billed = 0usize;
|
||||
let caption = |sentence: &str| {
|
||||
let text = tts::strip_tags(sentence);
|
||||
if !text.is_empty() {
|
||||
ipc.emit(Event::Caption { speaker: "luna", text });
|
||||
}
|
||||
};
|
||||
while let Some(delta) = delta_rx.recv().await {
|
||||
for sentence in splitter.push(&delta) {
|
||||
ipc.emit(Event::Caption { speaker: "luna", text: sentence.clone() });
|
||||
caption(&sentence);
|
||||
billed += sentence.chars().count();
|
||||
if sentence_tx.send(sentence).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(tail) = splitter.finish() {
|
||||
ipc.emit(Event::Caption { speaker: "luna", text: tail.clone() });
|
||||
caption(&tail);
|
||||
billed += tail.chars().count();
|
||||
let _ = sentence_tx.send(tail).await;
|
||||
}
|
||||
drop(sentence_tx);
|
||||
eprintln!("luna-voice: tts {billed} chars on {}", cfg.tts_model);
|
||||
|
||||
let full = reply.await??;
|
||||
eprintln!("luna-voice: said {:?}", full.trim());
|
||||
|
||||
+290
-80
@@ -1,8 +1,9 @@
|
||||
use crate::audio;
|
||||
use crate::visemes::{self, Viseme};
|
||||
use anyhow::Result;
|
||||
use anyhow::{anyhow, Result};
|
||||
use base64::Engine;
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
|
||||
use tokio_tungstenite::tungstenite::Message as Ws;
|
||||
@@ -21,6 +22,33 @@ pub struct TtsConfig {
|
||||
pub model: String,
|
||||
}
|
||||
|
||||
impl TtsConfig {
|
||||
/// v3 is not served on the TTS websocket at all; it streams only over
|
||||
/// Text to Dialogue, a different protocol.
|
||||
pub fn is_v3(&self) -> bool {
|
||||
self.model.starts_with("eleven_v3")
|
||||
}
|
||||
}
|
||||
|
||||
/// The two sockets send the same audio frames under different field names.
|
||||
struct Wire {
|
||||
alignment: &'static [&'static str],
|
||||
final_key: &'static str,
|
||||
}
|
||||
|
||||
const TTS_WIRE: Wire = Wire { alignment: &["normalizedAlignment", "alignment"], final_key: "isFinal" };
|
||||
const DIALOGUE_WIRE: Wire = Wire { alignment: &["alignment"], final_key: "is_final" };
|
||||
|
||||
/// Dialogue drops the socket after 20s without a client message, and a tool
|
||||
/// call on mars can stall a reply longer than that.
|
||||
const KEEP_ALIVE: Duration = Duration::from_secs(10);
|
||||
|
||||
type Socket = tokio_tungstenite::WebSocketStream<
|
||||
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
|
||||
>;
|
||||
type Sink = futures_util::stream::SplitSink<Socket, Ws>;
|
||||
type Source = futures_util::stream::SplitStream<Socket>;
|
||||
|
||||
/// Speaks sentences as they arrive on `sentences`, over one websocket per
|
||||
/// reply.
|
||||
///
|
||||
@@ -28,6 +56,35 @@ pub struct TtsConfig {
|
||||
/// and no reason to add one. sync_alignment makes the character timings ride
|
||||
/// along with the audio frames instead of needing a second request.
|
||||
pub async fn speak(
|
||||
cfg: &TtsConfig,
|
||||
sentences: mpsc::Receiver<String>,
|
||||
events: mpsc::Sender<SpeechEvent>,
|
||||
) -> Result<()> {
|
||||
if cfg.is_v3() {
|
||||
speak_dialogue(cfg, sentences, events).await
|
||||
} else {
|
||||
speak_tts(cfg, sentences, events).await
|
||||
}
|
||||
}
|
||||
|
||||
async fn connect(url: &str, key: &str) -> Result<(Sink, Source)> {
|
||||
let mut req = url.into_client_request()?;
|
||||
// Header, not the ?authorization= query param: a key in a URL ends up in
|
||||
// every proxy and error log that touches it.
|
||||
req.headers_mut().insert(
|
||||
"xi-api-key",
|
||||
tokio_tungstenite::tungstenite::http::HeaderValue::from_str(key)?,
|
||||
);
|
||||
let (stream, _) = tokio_tungstenite::connect_async(req).await?;
|
||||
Ok(stream.split())
|
||||
}
|
||||
|
||||
async fn send_json(sink: &mut Sink, v: serde_json::Value) -> Result<()> {
|
||||
sink.send(Ws::Text(v.to_string())).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn speak_tts(
|
||||
cfg: &TtsConfig,
|
||||
mut sentences: mpsc::Receiver<String>,
|
||||
events: mpsc::Sender<SpeechEvent>,
|
||||
@@ -39,85 +96,151 @@ pub async fn speak(
|
||||
cfg.model,
|
||||
audio::TTS_RATE
|
||||
);
|
||||
let mut req = url.into_client_request()?;
|
||||
// Header, not the ?authorization= query param: a key in a URL ends up in
|
||||
// every proxy and error log that touches it.
|
||||
req.headers_mut().insert(
|
||||
"xi-api-key",
|
||||
tokio_tungstenite::tungstenite::http::HeaderValue::from_str(&cfg.key)?,
|
||||
);
|
||||
|
||||
let (stream, _) = tokio_tungstenite::connect_async(req).await?;
|
||||
let (mut sink, mut source) = stream.split();
|
||||
let (mut sink, source) = connect(&url, &cfg.key).await?;
|
||||
|
||||
// The protocol requires an opening frame of a single space before any
|
||||
// real text; it carries the voice settings for the whole connection.
|
||||
sink.send(Ws::Text(
|
||||
send_json(
|
||||
&mut sink,
|
||||
serde_json::json!({
|
||||
"text": " ",
|
||||
"voice_settings": { "stability": 0.4, "similarity_boost": 0.8, "speed": 1.0 },
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let reader = tokio::spawn(async move {
|
||||
// Alignment times may be per-chunk or cumulative depending on the
|
||||
// model; `emitted_ms` lets us normalise either into one timeline.
|
||||
let mut emitted_ms: u32 = 0;
|
||||
while let Some(Ok(msg)) = source.next().await {
|
||||
let Ws::Text(txt) = msg else { continue };
|
||||
let Ok(v) = serde_json::from_str::<serde_json::Value>(&txt) else { continue };
|
||||
let reader = tokio::spawn(read_audio(source, events, &TTS_WIRE));
|
||||
|
||||
if let Some(b64) = v.get("audio").and_then(|a| a.as_str()) {
|
||||
if let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(b64) {
|
||||
let pcm = audio::pcm16_to_f32(&bytes);
|
||||
let chunk_ms = (pcm.len() as f64 / audio::TTS_RATE as f64 * 1000.0) as u32;
|
||||
if let Some(al) = v.get("normalizedAlignment").or_else(|| v.get("alignment")) {
|
||||
if let Some(t) = alignment_to_timeline(al, emitted_ms) {
|
||||
let _ = events.send(SpeechEvent::Visemes(t)).await;
|
||||
}
|
||||
}
|
||||
emitted_ms += chunk_ms;
|
||||
if events.send(SpeechEvent::Audio(pcm)).await.is_err() {
|
||||
return; // receiver gone: barge-in cancelled this reply
|
||||
}
|
||||
}
|
||||
}
|
||||
if v.get("isFinal").and_then(|f| f.as_bool()).unwrap_or(false) {
|
||||
break;
|
||||
}
|
||||
let sent: Result<()> = async {
|
||||
while let Some(text) = sentences.recv().await {
|
||||
// Every text frame must end with a space or the model holds the
|
||||
// last word back waiting for more input.
|
||||
let text = format!("{} ", text.trim());
|
||||
send_json(&mut sink, serde_json::json!({ "text": text })).await?;
|
||||
}
|
||||
let _ = events.send(SpeechEvent::Done).await;
|
||||
});
|
||||
|
||||
while let Some(text) = sentences.recv().await {
|
||||
// Every text frame must end with a space or the model holds the last
|
||||
// word back waiting for more input.
|
||||
let text = format!("{} ", text.trim());
|
||||
sink.send(Ws::Text(
|
||||
serde_json::json!({ "text": text }).to_string(),
|
||||
))
|
||||
.await?;
|
||||
// Empty string = end of input. Without it the socket sits until the
|
||||
// inactivity timeout and the last sentence never gets flushed.
|
||||
send_json(&mut sink, serde_json::json!({ "text": "" })).await
|
||||
}
|
||||
.await;
|
||||
|
||||
// Empty string = end of input. Without it the socket sits until the
|
||||
// inactivity timeout and the last sentence never gets flushed.
|
||||
sink.send(Ws::Text(serde_json::json!({ "text": "" }).to_string()))
|
||||
.await?;
|
||||
let _ = reader.await;
|
||||
Ok(())
|
||||
// The server's own error beats the "connection reset" a send hit after it.
|
||||
reader.await??;
|
||||
sent
|
||||
}
|
||||
|
||||
fn alignment_to_timeline(al: &serde_json::Value, emitted_ms: u32) -> Option<Vec<Viseme>> {
|
||||
async fn speak_dialogue(
|
||||
cfg: &TtsConfig,
|
||||
mut sentences: mpsc::Receiver<String>,
|
||||
events: mpsc::Sender<SpeechEvent>,
|
||||
) -> Result<()> {
|
||||
let url = format!(
|
||||
"wss://api.elevenlabs.io/v1/text-to-dialogue/stream-input\
|
||||
?model_id={}&output_format=pcm_{}&sync_alignment=true",
|
||||
cfg.model,
|
||||
audio::TTS_RATE
|
||||
);
|
||||
let (mut sink, source) = connect(&url, &cfg.key).await?;
|
||||
|
||||
// No voice_settings: v3 reads only stability, and only as one of three
|
||||
// presets; the default is its "natural" one.
|
||||
send_json(&mut sink, serde_json::json!({ "voices": [cfg.voice_id] })).await?;
|
||||
|
||||
let reader = tokio::spawn(read_audio(source, events, &DIALOGUE_WIRE));
|
||||
|
||||
let sent: Result<()> = async {
|
||||
loop {
|
||||
match tokio::time::timeout(KEEP_ALIVE, sentences.recv()).await {
|
||||
Ok(Some(text)) => {
|
||||
// Inputs are concatenated verbatim, so the space matters
|
||||
// here too. flush: the server otherwise holds text until
|
||||
// ~40 chars and 8 words, so a short first sentence would
|
||||
// wait for the second.
|
||||
let text = format!("{} ", text.trim());
|
||||
send_json(
|
||||
&mut sink,
|
||||
serde_json::json!({
|
||||
"inputs": [{ "text": text, "voice_id": cfg.voice_id }],
|
||||
"flush": true,
|
||||
}),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(_) => send_json(&mut sink, serde_json::json!({ "keep_alive": true })).await?,
|
||||
}
|
||||
}
|
||||
send_json(&mut sink, serde_json::json!({ "close_socket": true })).await
|
||||
}
|
||||
.await;
|
||||
|
||||
reader.await??;
|
||||
sent
|
||||
}
|
||||
|
||||
async fn read_audio(
|
||||
mut source: Source,
|
||||
events: mpsc::Sender<SpeechEvent>,
|
||||
wire: &'static Wire,
|
||||
) -> Result<()> {
|
||||
// Alignment times may be per-chunk or cumulative depending on the
|
||||
// model; `emitted_ms` lets us normalise either into one timeline.
|
||||
let mut emitted_ms: u32 = 0;
|
||||
// v3 audio tags come back as characters, and one can straddle chunks.
|
||||
let mut in_tag = false;
|
||||
let mut result = Ok(());
|
||||
while let Some(Ok(msg)) = source.next().await {
|
||||
let Ws::Text(txt) = msg else { continue };
|
||||
let Ok(v) = serde_json::from_str::<serde_json::Value>(&txt) else { continue };
|
||||
|
||||
if let Some(err) = v.get("error").and_then(|e| e.as_str()) {
|
||||
let detail = v.get("message").and_then(|m| m.as_str()).unwrap_or("");
|
||||
result = Err(anyhow!("elevenlabs: {err} {detail}"));
|
||||
break;
|
||||
}
|
||||
|
||||
if let Some(b64) = v.get("audio").and_then(|a| a.as_str()) {
|
||||
if let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(b64) {
|
||||
let pcm = audio::pcm16_to_f32(&bytes);
|
||||
let chunk_ms = (pcm.len() as f64 / audio::TTS_RATE as f64 * 1000.0) as u32;
|
||||
let al = wire.alignment.iter().find_map(|k| v.get(*k).filter(|a| !a.is_null()));
|
||||
if let Some(t) = al.and_then(|al| alignment_to_timeline(al, emitted_ms, &mut in_tag)) {
|
||||
let _ = events.send(SpeechEvent::Visemes(t)).await;
|
||||
}
|
||||
emitted_ms += chunk_ms;
|
||||
if events.send(SpeechEvent::Audio(pcm)).await.is_err() {
|
||||
return Ok(()); // receiver gone: barge-in cancelled this reply
|
||||
}
|
||||
}
|
||||
}
|
||||
if v.get(wire.final_key).and_then(|f| f.as_bool()).unwrap_or(false) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let _ = events.send(SpeechEvent::Done).await;
|
||||
result
|
||||
}
|
||||
|
||||
fn alignment_to_timeline(
|
||||
al: &serde_json::Value,
|
||||
emitted_ms: u32,
|
||||
in_tag: &mut bool,
|
||||
) -> Option<Vec<Viseme>> {
|
||||
let field = |camel: &str, snake: &str| al.get(camel).or_else(|| al.get(snake));
|
||||
let chars: Vec<String> = al
|
||||
.get("chars")?
|
||||
.as_array()?
|
||||
.iter()
|
||||
.map(|c| c.as_str().unwrap_or("").to_string())
|
||||
.map(|c| c.as_str().unwrap_or(""))
|
||||
.map(|c| {
|
||||
// A tag is a laugh or a sigh, not letters: the mouth rests.
|
||||
let tagged = *in_tag || c == "[";
|
||||
*in_tag = tagged && c != "]";
|
||||
if tagged { " ".to_string() } else { c.to_string() }
|
||||
})
|
||||
.collect();
|
||||
let starts: Vec<i64> = num_array(al.get("charStartTimesMs")?)?;
|
||||
let durs: Vec<i64> = num_array(al.get("charDurationsMs")?)?;
|
||||
let starts: Vec<i64> = num_array(field("charStartTimesMs", "char_start_times_ms")?)?;
|
||||
let durs: Vec<i64> = num_array(field("charDurationsMs", "char_durations_ms")?)?;
|
||||
let starts = absolutize(&starts, emitted_ms);
|
||||
Some(visemes::timeline(&chars, &starts, &durs))
|
||||
}
|
||||
@@ -143,11 +266,13 @@ fn absolutize(starts: &[i64], emitted_ms: u32) -> Vec<i64> {
|
||||
pub struct SentenceSplitter {
|
||||
buf: String,
|
||||
in_think: bool,
|
||||
/// Whether v3 audio tags survive to the synthesiser or are dropped.
|
||||
keep_tags: bool,
|
||||
}
|
||||
|
||||
impl SentenceSplitter {
|
||||
pub fn new() -> Self {
|
||||
SentenceSplitter { buf: String::new(), in_think: false }
|
||||
pub fn new(keep_tags: bool) -> Self {
|
||||
SentenceSplitter { buf: String::new(), in_think: false, keep_tags }
|
||||
}
|
||||
|
||||
/// Feeds a delta, returning any sentences that completed.
|
||||
@@ -174,24 +299,24 @@ impl SentenceSplitter {
|
||||
let head: String = self.buf.drain(..i).collect();
|
||||
self.buf.drain(.."<think>".len());
|
||||
self.in_think = true;
|
||||
out.extend(drain_sentences(&mut { head.clone() }, true));
|
||||
out.extend(drain_sentences(&mut { head.clone() }, true, self.keep_tags));
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
|
||||
out.extend(drain_sentences(&mut self.buf, false));
|
||||
out.extend(drain_sentences(&mut self.buf, false, self.keep_tags));
|
||||
out.into_iter().filter(|s| !s.is_empty()).collect()
|
||||
}
|
||||
|
||||
/// Whatever is left when the reply ends.
|
||||
pub fn finish(&mut self) -> Option<String> {
|
||||
let rest = clean_for_speech(&std::mem::take(&mut self.buf));
|
||||
let rest = clean_for_speech(&std::mem::take(&mut self.buf), self.keep_tags);
|
||||
(!rest.is_empty()).then_some(rest)
|
||||
}
|
||||
}
|
||||
|
||||
fn drain_sentences(buf: &mut String, force: bool) -> Vec<String> {
|
||||
fn drain_sentences(buf: &mut String, force: bool, keep_tags: bool) -> Vec<String> {
|
||||
let mut out = Vec::new();
|
||||
// A sentence is only worth sending once it is long enough to synthesise
|
||||
// well; ElevenLabs' own guidance is ~20 characters minimum.
|
||||
@@ -203,13 +328,13 @@ fn drain_sentences(buf: &mut String, force: bool) -> Vec<String> {
|
||||
}
|
||||
}
|
||||
let s: String = buf.drain(..end + 1).collect();
|
||||
let s = clean_for_speech(&s);
|
||||
let s = clean_for_speech(&s, keep_tags);
|
||||
if !s.is_empty() {
|
||||
out.push(s);
|
||||
}
|
||||
}
|
||||
if force && !buf.is_empty() {
|
||||
let s = clean_for_speech(&std::mem::take(buf));
|
||||
let s = clean_for_speech(&std::mem::take(buf), keep_tags);
|
||||
if !s.is_empty() {
|
||||
out.push(s);
|
||||
}
|
||||
@@ -217,16 +342,64 @@ fn drain_sentences(buf: &mut String, force: bool) -> Vec<String> {
|
||||
out
|
||||
}
|
||||
|
||||
/// Byte index just past a v3 audio tag starting at `i` ("[laughs]",
|
||||
/// "[clears throat]"), if there is one. Letters and spaces only, so
|
||||
/// "[1]" or a markdown link stays ordinary noise.
|
||||
fn tag_end(s: &str, i: usize) -> Option<usize> {
|
||||
let rest = s[i..].strip_prefix('[')?;
|
||||
let close = rest.find(']')?;
|
||||
let body = &rest[..close];
|
||||
let end = i + 1 + close + 1;
|
||||
let ok = !body.trim().is_empty()
|
||||
&& body.len() <= 40
|
||||
&& body.chars().all(|c| c.is_alphabetic() || c == ' ' || c == '-')
|
||||
&& !s[end..].starts_with('(');
|
||||
ok.then_some(end)
|
||||
}
|
||||
|
||||
/// The text as a person reads it: captions show "ha!", not "[laughs]".
|
||||
pub fn strip_tags(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
let mut skip_to = 0;
|
||||
for (i, c) in s.char_indices() {
|
||||
if i < skip_to {
|
||||
continue;
|
||||
}
|
||||
match tag_end(s, i) {
|
||||
Some(end) => {
|
||||
skip_to = end;
|
||||
out.push(' ');
|
||||
}
|
||||
None => out.push(c),
|
||||
}
|
||||
}
|
||||
out.split_whitespace().collect::<Vec<_>>().join(" ")
|
||||
}
|
||||
|
||||
/// Markdown, emoji and kaomoji are for eyes.
|
||||
///
|
||||
/// A whitelist, not a blacklist: she writes things like "(¬‿¬)", whose
|
||||
/// characters live in ordinary Latin-1 and punctuation blocks rather than in
|
||||
/// any emoji range, so enumerating what to drop never converges. Letters
|
||||
/// stays unicode-aware on purpose — German is in scope, "ö" is a letter.
|
||||
pub fn clean_for_speech(s: &str) -> String {
|
||||
///
|
||||
/// Audio tags pass through untouched when `keep_tags`; otherwise they are
|
||||
/// dropped whole, since a model without them would read "laughs" aloud.
|
||||
pub fn clean_for_speech(s: &str, keep_tags: bool) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
for c in s.chars() {
|
||||
if c.is_alphanumeric() || c.is_whitespace() || is_speakable_punctuation(c) {
|
||||
let mut skip_to = 0;
|
||||
for (i, c) in s.char_indices() {
|
||||
if i < skip_to {
|
||||
continue;
|
||||
}
|
||||
if let Some(end) = tag_end(s, i) {
|
||||
skip_to = end;
|
||||
out.push(' ');
|
||||
if keep_tags {
|
||||
out.push_str(&s[i..end]);
|
||||
out.push(' ');
|
||||
}
|
||||
} else if c.is_alphanumeric() || c.is_whitespace() || is_speakable_punctuation(c) {
|
||||
out.push(c);
|
||||
} else if matches!(c, '—' | '–') {
|
||||
// " — " must not leave " , ": the comma binds to the word before it.
|
||||
@@ -271,7 +444,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn splitter_emits_complete_sentences_only() {
|
||||
let mut s = SentenceSplitter::new();
|
||||
let mut s = SentenceSplitter::new(false);
|
||||
assert!(s.push("This is a long enough first sentence").is_empty());
|
||||
let got = s.push(". And more");
|
||||
assert_eq!(got, vec!["This is a long enough first sentence."]);
|
||||
@@ -279,7 +452,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn splitter_drops_think_blocks() {
|
||||
let mut s = SentenceSplitter::new();
|
||||
let mut s = SentenceSplitter::new(false);
|
||||
s.push("<think>internal monologue that is long</think>");
|
||||
let got = s.push("Actually speakable sentence here. ");
|
||||
assert_eq!(got, vec!["Actually speakable sentence here."]);
|
||||
@@ -287,33 +460,70 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn splitter_returns_tail_on_finish() {
|
||||
let mut s = SentenceSplitter::new();
|
||||
let mut s = SentenceSplitter::new(false);
|
||||
s.push("no terminator here");
|
||||
assert_eq!(s.finish().as_deref(), Some("no terminator here"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn markdown_and_emoji_never_reach_the_voice() {
|
||||
assert_eq!(clean_for_speech("**bold** `code` 🎉 ok"), "bold code ok");
|
||||
assert_eq!(clean_for_speech("**bold** `code` 🎉 ok", false), "bold code ok");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kaomoji_and_dashes_are_stripped_without_gluing_words() {
|
||||
// Her actual first reply over the API.
|
||||
assert_eq!(
|
||||
clean_for_speech("Loud and clear, boss — hallo! (¬‿¬)"),
|
||||
clean_for_speech("Loud and clear, boss — hallo! (¬‿¬)", false),
|
||||
"Loud and clear, boss, hallo!"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn em_dash_becomes_a_pause_but_hyphens_do_not() {
|
||||
assert_eq!(clean_for_speech("I can't confirm — there's nothing"), "I can't confirm, there's nothing");
|
||||
assert_eq!(clean_for_speech("push-to-talk"), "push to talk");
|
||||
assert_eq!(clean_for_speech("I can't confirm — there's nothing", false), "I can't confirm, there's nothing");
|
||||
assert_eq!(clean_for_speech("push-to-talk", false), "push to talk");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_non_ascii_letters() {
|
||||
assert_eq!(clean_for_speech("schön, größer"), "schön, größer");
|
||||
assert_eq!(clean_for_speech("schön, größer", false), "schön, größer");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_tags_survive_for_v3_only() {
|
||||
let s = "[laughs]Fine, boss. [clears throat] Na gut.";
|
||||
assert_eq!(clean_for_speech(s, true), "[laughs] Fine, boss. [clears throat] Na gut.");
|
||||
assert_eq!(clean_for_speech(s, false), "Fine, boss. Na gut.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn brackets_that_are_not_tags_stay_noise() {
|
||||
assert_eq!(clean_for_speech("see [1] and [the docs](x)", true), "see 1 and the docs x");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn captions_hide_tags() {
|
||||
assert_eq!(strip_tags("[sighs] Okay, boss."), "Okay, boss.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tag_characters_rest_the_mouth_across_chunks() {
|
||||
let mut in_tag = false;
|
||||
let chunk = |chars: &[&str], in_tag: &mut bool| {
|
||||
let n = chars.len() as i64;
|
||||
let al = serde_json::json!({
|
||||
"chars": chars,
|
||||
"char_start_times_ms": (0..n).map(|i| i * 100).collect::<Vec<_>>(),
|
||||
"char_durations_ms": vec![100; chars.len()],
|
||||
});
|
||||
alignment_to_timeline(&al, 0, in_tag).unwrap()
|
||||
};
|
||||
let t = chunk(&["[", "l", "a"], &mut in_tag);
|
||||
assert!(t.iter().all(|v| v.shape == visemes::SILENCE));
|
||||
assert!(in_tag);
|
||||
let t = chunk(&["u", "]", "a"], &mut in_tag);
|
||||
assert!(!in_tag);
|
||||
assert_eq!(t.last().unwrap().shape, "aa");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user