use serde::Serialize; /// VRM 1.0's standard mouth expressions, which is what the avatar rig exposes /// (`aa`/`ih`/`ou`/`ee`/`oh`). A rig with the full ARKit 52 can map these onto /// jawOpen/mouthFunnel/etc — the shell side owns that, not this. /// /// These names are a CONTRACT with the balsam-generated morph targets: the /// glTF target names become the QML ids, so renaming a shape key in Blender /// breaks the binding silently, with no eval error. pub const SILENCE: &str = "sil"; #[derive(Debug, Clone, PartialEq, Serialize)] pub struct Viseme { pub shape: &'static str, /// Milliseconds from speech_start, NOT wall clock — the shell starts its /// own timer when the first PCM frame hits the speaker, so the two clocks /// never have to agree. pub at: u32, pub ms: u32, } /// One character to a mouth shape. English orthography is a lie about /// phonetics, but at 40ms of animation per character nobody can tell: what /// reads as "talking" is the OPEN/CLOSE rhythm, not vowel accuracy. fn shape_for(c: char) -> &'static str { match c.to_ascii_lowercase() { 'a' => "aa", 'e' => "ee", 'i' | 'y' => "ih", 'o' => "oh", 'u' | 'w' => "ou", // Bilabials close the mouth; giving them silence is what produces the // visible "p/b/m" beat. 'p' | 'b' | 'm' => SILENCE, // Everything else is a partially open consonant. "ih" is the least // committal shape to land on. c if c.is_alphabetic() => "ih", _ => SILENCE, } } /// Builds a playable timeline from ElevenLabs' character alignment. /// /// The three arrays are parallel; a short or missing one means the frame was /// truncated, so we take the shortest common prefix rather than panicking on /// an index. pub fn timeline(chars: &[String], starts: &[i64], durations: &[i64]) -> Vec { let n = chars.len().min(starts.len()).min(durations.len()); let mut out: Vec = Vec::with_capacity(n); for i in 0..n { let c = match chars[i].chars().next() { Some(c) => c, None => continue, }; let shape = shape_for(c); let at = starts[i].max(0) as u32; let ms = durations[i].max(0) as u32; // Merge runs of the same shape: "ll" is one mouth position held, not // two identical keyframes the shell would re-trigger. match out.last_mut() { Some(last) if last.shape == shape && last.at + last.ms >= at => { last.ms = (at + ms).saturating_sub(last.at); } _ => out.push(Viseme { shape, at, ms }), } } // Sub-frame keys cannot be seen and only cost socket bytes. out.retain(|v| v.ms >= 20); out } #[cfg(test)] mod tests { use super::*; fn s(v: &[&str]) -> Vec { v.iter().map(|s| s.to_string()).collect() } #[test] fn maps_vowels_to_distinct_shapes() { let t = timeline( &s(&["a", "e", "o"]), &[0, 100, 200], &[100, 100, 100], ); assert_eq!( t.iter().map(|v| v.shape).collect::>(), vec!["aa", "ee", "oh"] ); } #[test] fn merges_repeated_shapes_into_one_held_key() { let t = timeline(&s(&["l", "l", "l"]), &[0, 50, 100], &[50, 50, 50]); assert_eq!(t.len(), 1); assert_eq!(t[0].shape, "ih"); assert_eq!(t[0].at, 0); assert_eq!(t[0].ms, 150); } #[test] fn bilabials_close_the_mouth() { let t = timeline(&s(&["a", "m", "a"]), &[0, 100, 200], &[100, 100, 100]); assert_eq!(t[1].shape, SILENCE); } #[test] fn drops_keys_too_short_to_see() { let t = timeline(&s(&["a", "e"]), &[0, 5], &[5, 100]); assert_eq!(t.len(), 1); assert_eq!(t[0].shape, "ee"); } #[test] fn tolerates_truncated_arrays() { let t = timeline(&s(&["a", "e", "o"]), &[0, 100], &[100]); assert_eq!(t.len(), 1); } #[test] fn punctuation_and_spaces_are_silence() { let t = timeline(&s(&[" ", ",", "!"]), &[0, 100, 200], &[100, 100, 100]); assert_eq!(t.len(), 1); assert_eq!(t[0].shape, SILENCE); assert_eq!(t[0].ms, 300); } }