Files
homelab/pkgs/luna-voice/src/visemes.rs
T
darmanandClaude Opus 5 d089743f75 feat(terra): talk to luna with push-to-talk
luna-voiced: mic -> whisper-cpp (local, vulkan) -> Hermes' API on mars ->
ElevenLabs streaming TTS -> speaker, bound to SUPER+V. State, amplitude
and a viseme timeline are published on a unix socket; nothing consumes
that yet, but it is the data source the quickshell avatar will read.

STT stays local on purpose. mars already runs faster-whisper for Telegram
voice notes, but routing the mic through it would put every spoken word
on the network for no latency win.

Shape notes, all of them things the first live runs decided:

  - Viseme offsets are relative to speech_start (emitted when the first
    PCM frame reaches the speaker), never wall clock, so the shell and
    the daemon never have to compare clocks. ElevenLabs sends character
    alignment, not visemes, so the mapping is ours; the glTF morph target
    names are the contract.
  - clean_for_speech is a whitelist. She writes "(¬‿¬)", whose characters
    live in Latin-1 and punctuation blocks rather than any emoji range,
    so enumerating what to drop never converges. Em dashes become commas
    because they are a pause — dropping one ran two clauses together and
    changed what a sentence meant.
  - whisper invents words out of silence ("you", "Thank you."), and a
    held key over a quiet room is long AND empty, so has_speech gates on
    the loudest 100ms window rather than on duration.
  - Beeps on key down/up and a distinct low tone for "heard nothing":
    push-to-talk with no acknowledgement is indistinguishable from broken,
    and a dead mic must not feel like a slow reply.
  - A warmup request at startup: her first reply after the container
    starts costs ~16s against ~5s warm, and the first one is always the
    one you are waiting for.
  - `luna-voice watch` prints the socket protocol and `luna-voice say`
    runs a turn from typed text, which is how a broken mic is told apart
    from a broken pipeline.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-20 22:45:17 +02:00

132 lines
4.2 KiB
Rust

use serde::Serialize;
/// VRM 1.0's standard mouth expressions, which is what the avatar rig exposes
/// (`aa`/`ih`/`ou`/`ee`/`oh`). A rig with the full ARKit 52 can map these onto
/// jawOpen/mouthFunnel/etc — the shell side owns that, not this.
///
/// These names are a CONTRACT with the balsam-generated morph targets: the
/// glTF target names become the QML ids, so renaming a shape key in Blender
/// breaks the binding silently, with no eval error.
pub const SILENCE: &str = "sil";
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct Viseme {
pub shape: &'static str,
/// Milliseconds from speech_start, NOT wall clock — the shell starts its
/// own timer when the first PCM frame hits the speaker, so the two clocks
/// never have to agree.
pub at: u32,
pub ms: u32,
}
/// One character to a mouth shape. English orthography is a lie about
/// phonetics, but at 40ms of animation per character nobody can tell: what
/// reads as "talking" is the OPEN/CLOSE rhythm, not vowel accuracy.
fn shape_for(c: char) -> &'static str {
match c.to_ascii_lowercase() {
'a' => "aa",
'e' => "ee",
'i' | 'y' => "ih",
'o' => "oh",
'u' | 'w' => "ou",
// Bilabials close the mouth; giving them silence is what produces the
// visible "p/b/m" beat.
'p' | 'b' | 'm' => SILENCE,
// Everything else is a partially open consonant. "ih" is the least
// committal shape to land on.
c if c.is_alphabetic() => "ih",
_ => SILENCE,
}
}
/// Builds a playable timeline from ElevenLabs' character alignment.
///
/// The three arrays are parallel; a short or missing one means the frame was
/// truncated, so we take the shortest common prefix rather than panicking on
/// an index.
pub fn timeline(chars: &[String], starts: &[i64], durations: &[i64]) -> Vec<Viseme> {
let n = chars.len().min(starts.len()).min(durations.len());
let mut out: Vec<Viseme> = Vec::with_capacity(n);
for i in 0..n {
let c = match chars[i].chars().next() {
Some(c) => c,
None => continue,
};
let shape = shape_for(c);
let at = starts[i].max(0) as u32;
let ms = durations[i].max(0) as u32;
// Merge runs of the same shape: "ll" is one mouth position held, not
// two identical keyframes the shell would re-trigger.
match out.last_mut() {
Some(last) if last.shape == shape && last.at + last.ms >= at => {
last.ms = (at + ms).saturating_sub(last.at);
}
_ => out.push(Viseme { shape, at, ms }),
}
}
// Sub-frame keys cannot be seen and only cost socket bytes.
out.retain(|v| v.ms >= 20);
out
}
#[cfg(test)]
mod tests {
use super::*;
fn s(v: &[&str]) -> Vec<String> {
v.iter().map(|s| s.to_string()).collect()
}
#[test]
fn maps_vowels_to_distinct_shapes() {
let t = timeline(
&s(&["a", "e", "o"]),
&[0, 100, 200],
&[100, 100, 100],
);
assert_eq!(
t.iter().map(|v| v.shape).collect::<Vec<_>>(),
vec!["aa", "ee", "oh"]
);
}
#[test]
fn merges_repeated_shapes_into_one_held_key() {
let t = timeline(&s(&["l", "l", "l"]), &[0, 50, 100], &[50, 50, 50]);
assert_eq!(t.len(), 1);
assert_eq!(t[0].shape, "ih");
assert_eq!(t[0].at, 0);
assert_eq!(t[0].ms, 150);
}
#[test]
fn bilabials_close_the_mouth() {
let t = timeline(&s(&["a", "m", "a"]), &[0, 100, 200], &[100, 100, 100]);
assert_eq!(t[1].shape, SILENCE);
}
#[test]
fn drops_keys_too_short_to_see() {
let t = timeline(&s(&["a", "e"]), &[0, 5], &[5, 100]);
assert_eq!(t.len(), 1);
assert_eq!(t[0].shape, "ee");
}
#[test]
fn tolerates_truncated_arrays() {
let t = timeline(&s(&["a", "e", "o"]), &[0, 100], &[100]);
assert_eq!(t.len(), 1);
}
#[test]
fn punctuation_and_spaces_are_silence() {
let t = timeline(&s(&[" ", ",", "!"]), &[0, 100, 200], &[100, 100, 100]);
assert_eq!(t.len(), 1);
assert_eq!(t[0].shape, SILENCE);
assert_eq!(t[0].ms, 300);
}
}