From 6fc56bfefebd80936bc7a5a04f4d79531cabc342 Mon Sep 17 00:00:00 2001 From: Tanish Manku Date: Wed, 18 Nov 2026 02:44:19 -0700 Subject: [PATCH] feat(engine): sound.to_keyframes, sound.lip_sync and sound.analyse: a sound's loudness or speech keys an object on another layer, one undo step (M5) --- crates/audio/src/analysis.rs | 8 +- crates/engine/src/cmds_audio_keys.rs | 755 ++++++++++++++++++++++++++ crates/engine/src/lib.rs | 6 +- crates/engine/src/tests.rs | 9 + crates/engine/src/tests_audio_keys.rs | 438 +++++++++++++++ 5 files changed, 1213 insertions(+), 3 deletions(-) create mode 100644 crates/engine/src/cmds_audio_keys.rs create mode 100644 crates/engine/src/tests_audio_keys.rs diff --git a/crates/audio/src/analysis.rs b/crates/audio/src/analysis.rs index 14b0aecc..6fc03188 100644 --- a/crates/audio/src/analysis.rs +++ b/crates/audio/src/analysis.rs @@ -331,8 +331,10 @@ fn held(mut v: Vec, hold: usize) -> Vec { let Some(&cur) = v.get(i) else { break }; let end = v.iter().skip(i).position(|x| *x != cur).map_or(v.len(), |n| i + n); let short = end - i < hold && cur != Viseme::Closed; - // What a short run becomes: the run before it, or (at the start) the one after. - let join = if short { before.or_else(|| v.get(end).copied()) } else { None }; + // What a short run becomes: the run before it, or the one after at the start and after + // a closure (lips together don't stay together for the sound that follows them). + let open = |x: &Viseme| *x != Viseme::Closed; + let join = if short { before.filter(open).or_else(|| v.get(end).copied().filter(open)) } else { None }; match join { Some(to) if to != cur => { for slot in v.iter_mut().take(end).skip(i) { @@ -616,6 +618,8 @@ mod tests { assert_eq!(held(vec![Open, Open, Wide, Open, Open], 2), vec![Open; 5]); assert_eq!(held(vec![Wide, Open, Open, Open], 2), vec![Open; 4], "the first joins what follows"); assert_eq!(held(vec![Open, Open, Closed, Wide, Wide], 2), vec![Open, Open, Closed, Wide, Wide]); + assert_eq!(held(vec![Open, Open, Closed, Wide, Round, Round], 2), vec![Open, Open, Closed, Round, Round, Round], "after a closure, the sound that follows"); + assert_eq!(held(vec![Closed, Wide, Closed], 2), vec![Closed, Wide, Closed], "between closures it stays"); assert_eq!(held(vec![Open, Wide, Round, Open], 1), vec![Open, Wide, Round, Open]); assert_eq!(held(vec![Open, Open, Open, Wide, Wide, Round], 3), vec![Open; 6]); assert_eq!(held(vec![], 3), vec![]); diff --git a/crates/engine/src/cmds_audio_keys.rs b/crates/engine/src/cmds_audio_keys.rs new file mode 100644 index 00000000..2590bf10 --- /dev/null +++ b/crates/engine/src/cmds_audio_keys.rs @@ -0,0 +1,755 @@ +//! Convert Audio to Keyframes and auto lip-sync (ADR 0114): a sound on the timeline drives an +//! object on another layer. +//! +//! * `sound.to_keyframes` maps the sound's loudness on each frame (gated and smoothed) onto a +//! property of an object: scale, alpha, a position or rotation offset, or a graphic +//! symbol's first frame. It writes keyframes, or a motion tween's property curve. +//! * `sound.lip_sync` sorts each frame of speech into a small set of mouth shapes and writes +//! keyframes that show the matching frame of a graphic symbol of mouth poses. +//! * `sound.analyse` reports what both would work from, without changing anything. +//! +//! The analysis is `anicraft_audio::analysis` (pure, bounded); this module finds the sound and +//! the object, and writes the keys. Each command is one undo step. + +use crate::params::*; +use crate::{CommandSpec, EngineError, MAX_FRAME, Result, Session}; +use anicraft_doc::audio::analysis::{self, Channel, LipOptions, MAX_ANALYSIS_FRAMES, MAX_HOLD, MAX_SMOOTHING, Span, Viseme}; +use anicraft_doc::motion::MAX_KEYS; +use anicraft_doc::sound::frame_start; +use anicraft_doc::{Element, ElementId, ElementKind, FrameSound, Layer, LayerId, LayerKind, LoopMode, Property, Sound, SoundSync, Symbol, SymbolKind, own_value}; +use anicraft_geom::{Affine, Vec2}; +use serde_json::{Value, json}; +use std::sync::Arc; + +/// What the dialogs start from. +pub const DEFAULT_THRESHOLD: f32 = 0.05; +pub const DEFAULT_SMOOTHING: u32 = 1; +/// Most frames between keys. +pub const MAX_STEP: u32 = 240; + +pub fn specs() -> Vec { + let c = |id, label, menu, params, mutates, run| CommandSpec { id, label, menu, shortcut: None, params, mutates, run }; + vec![ + c( + "sound.to_keyframes", + "Convert Audio to Keyframes…", + "Modify/Timeline", + "{property: scale|scale_x|scale_y|alpha|x|y|rotation|first_frame, min, max (the property's value in silence and at the loudest frame: scale % of the object's size, alpha %, x and y pixels from where it is, rotation degrees from how it is turned, first_frame a 0-based frame of the graphic symbol), layer? (the layer that gets the keys; default the active layer), element? (the object on it; default the selected one, or the only one), sound_layer?, sound_frame? (the sound's keyframe; default the sound at the playhead), threshold? (0–0.99 of the loudest frame: quieter is silence; default 0.05), smoothing? (0–24 frames either side; default 1), channel?: both|left|right, step? (a key every n frames, 1–240; default 1), output?: auto|keyframes|motion (motion writes the property's curve in the motion tween on the span; auto picks it when the span has one), from?, to? (frames; default the frames the sound is heard on)} → {keys, from, to, output, property, layer, element}", + true, + to_keyframes as crate::RunFn, + ), + c( + "sound.lip_sync", + "Auto Lip-Sync…", + "Modify/Timeline", + "{layer? (the layer with the mouth; default the active layer), element? (a graphic symbol instance of mouth poses on it; default the selected one, or the only one), mouths?: {rest, open, wide, round, closed} (each a 0-based frame of the symbol or a frame label in it; default by the symbol's frame labels, as the Lip-sync mouth chart template names them), sound_layer?, sound_frame? (the speech; default the sound at the playhead), threshold? (0–0.99 of the loudest frame: quieter is silence; default 0.08), hold? (fewest frames a mouth is held, 1–12; default 2), from?, to? (frames; default the frames the sound is heard on)} → {keys, from, to, layer, element, mouths, counts}", + true, + lip_sync, + ), + c( + "sound.analyse", + "Analyse Sound", + "", + "{what?: levels|visemes (default levels), sound_layer?, sound_frame?, from?, to?, threshold?, smoothing?, channel? (levels), hold? (visemes)} → {sound, name, layer, keyframe, from, to, fps, levels (each frame's loudness, 0–1 of the loudest) and values (gated and smoothed), or visemes (rest|open|wide|round|closed per frame)}", + false, + analyse, + ), + ] +} + +/// A property Convert Audio to Keyframes drives. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum KeyProperty { + /// Both scales together, percent of the object's own size. + Scale, + ScaleX, + ScaleY, + /// Opacity, percent. + Alpha, + /// Pixels from where the object is. + X, + Y, + /// Degrees from how the object is turned. + Rotation, + /// The frame a graphic symbol instance shows (0-based), held as a single frame. + FirstFrame, +} + +impl KeyProperty { + pub const ALL: [KeyProperty; 8] = [KeyProperty::Scale, KeyProperty::ScaleX, KeyProperty::ScaleY, KeyProperty::Alpha, KeyProperty::X, KeyProperty::Y, KeyProperty::Rotation, KeyProperty::FirstFrame]; + + pub fn key(self) -> &'static str { + match self { + KeyProperty::Scale => "scale", + KeyProperty::ScaleX => "scale_x", + KeyProperty::ScaleY => "scale_y", + KeyProperty::Alpha => "alpha", + KeyProperty::X => "x", + KeyProperty::Y => "y", + KeyProperty::Rotation => "rotation", + KeyProperty::FirstFrame => "first_frame", + } + } + + pub fn label(self) -> &'static str { + match self { + KeyProperty::Scale => "Scale", + KeyProperty::ScaleX => "Scale X", + KeyProperty::ScaleY => "Scale Y", + KeyProperty::Alpha => "Alpha", + KeyProperty::X => "X offset", + KeyProperty::Y => "Y offset", + KeyProperty::Rotation => "Rotation", + KeyProperty::FirstFrame => "First frame", + } + } + + pub fn unit(self) -> &'static str { + match self { + KeyProperty::Scale | KeyProperty::ScaleX | KeyProperty::ScaleY | KeyProperty::Alpha => "%", + KeyProperty::X | KeyProperty::Y => "px", + KeyProperty::Rotation => "°", + KeyProperty::FirstFrame => "", + } + } + + pub fn from_key(s: &str) -> Option { + let s = s.trim().to_ascii_lowercase().replace([' ', '-'], "_"); + let s = match s.as_str() { + "opacity" => "alpha", + "scalex" => "scale_x", + "scaley" => "scale_y", + "rotate" => "rotation", + "frame" | "firstframe" => "first_frame", + other => other, + }; + KeyProperty::ALL.into_iter().find(|p| p.key() == s) + } + + /// The values `min` and `max` may take. + pub fn range(self) -> std::ops::RangeInclusive { + match self { + KeyProperty::Scale | KeyProperty::ScaleX | KeyProperty::ScaleY => 1.0..=10_000.0, + KeyProperty::Alpha => 0.0..=100.0, + KeyProperty::X | KeyProperty::Y => -100_000.0..=100_000.0, + KeyProperty::Rotation => -3_600.0..=3_600.0, + KeyProperty::FirstFrame => 0.0..=MAX_FRAME as f32, + } + } + + /// What the dialog offers first: (in silence, at the loudest). + pub fn defaults(self) -> (f32, f32) { + match self { + KeyProperty::Scale | KeyProperty::ScaleX | KeyProperty::ScaleY => (100.0, 150.0), + KeyProperty::Alpha => (20.0, 100.0), + KeyProperty::X => (0.0, 40.0), + KeyProperty::Y => (0.0, -40.0), + KeyProperty::Rotation => (0.0, 20.0), + KeyProperty::FirstFrame => (0.0, 1.0), + } + } + + /// The motion tween tracks it writes (none for a symbol's first frame). + fn tracks(self) -> &'static [Property] { + match self { + KeyProperty::Scale => &[Property::ScaleX, Property::ScaleY], + KeyProperty::ScaleX => &[Property::ScaleX], + KeyProperty::ScaleY => &[Property::ScaleY], + KeyProperty::Alpha => &[Property::Alpha], + KeyProperty::X => &[Property::X], + KeyProperty::Y => &[Property::Y], + KeyProperty::Rotation => &[Property::Rotation], + KeyProperty::FirstFrame => &[], + } + } + + /// A track's value for the property's value `v`, on an object whose own value is `own`. + fn track_value(self, own: f32, v: f32) -> f32 { + match self { + KeyProperty::Scale | KeyProperty::ScaleX | KeyProperty::ScaleY => own * v / 100.0, + KeyProperty::Alpha | KeyProperty::FirstFrame => v, + KeyProperty::X | KeyProperty::Y | KeyProperty::Rotation => own + v, + } + } +} + +/// How the keys are written. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Output { + /// Keyframes on the layer, each holding the object with the value. + Keyframes, + /// Property keys in the motion tween on the span. + Motion, +} + +impl Output { + pub fn key(self) -> &'static str { + match self { + Output::Keyframes => "keyframes", + Output::Motion => "motion", + } + } +} + +/// The sound a command reads: where it is, and the frames it is heard on. +#[derive(Clone, Debug)] +pub struct Source { + pub layer: LayerId, + /// Its keyframe's first frame. + pub keyframe: u32, + pub sound: Arc, + /// The frames analysed, first and last. + pub from: u32, + pub to: u32, + span: Span, +} + +impl Source { + /// Frames analysed. + pub fn frames(&self) -> u32 { + self.to - self.from + 1 + } +} + +fn sounding(k: &anicraft_doc::Keyframe) -> Option<&FrameSound> { + k.sound.as_ref().filter(|f| f.sync != SoundSync::Stop) +} + +/// The keyframe with the sound to read, as `(layer, keyframe start)`: on `sound_layer` when +/// given, else the first layer sounding at the frame (a stream sound before the others), else +/// the timeline's first sound. +fn find_sound(s: &Session, p: &Value) -> Result<(LayerId, u32)> { + let frame = opt_u32(p, "sound_frame")?.unwrap_or(s.frame).min(MAX_FRAME); + let tl = s.timeline(); + // On a layer: the sound whose keyframe holds the frame, else the last one before it, else + // its first. + let on = |l: &Layer| { + let at = l.keyframe_at(frame).filter(|k| sounding(k).is_some()); + let before = || l.keyframes.iter().rev().find(|k| k.start <= frame && sounding(k).is_some()).map(|k| &**k); + let first = || l.keyframes.iter().find(|k| sounding(k).is_some()).map(|k| &**k); + at.or_else(before).or_else(first).map(|k| (l.id, k.start)) + }; + if let Some(id) = opt_u64(p, "sound_layer")?.map(LayerId) { + let l = tl.layer(id).ok_or_else(|| EngineError::State(format!("no layer with id {}", id.0)))?; + return on(l).ok_or_else(|| EngineError::State(format!("layer '{}' has no sound on its keyframes", l.name))); + } + let here = |stream: bool| tl.layers.iter().find_map(|l| l.keyframe_at(frame).filter(|k| sounding(k).is_some_and(|f| !stream || f.sync == SoundSync::Stream)).map(|k| (l.id, k.start))); + here(true) + .or_else(|| here(false)) + .or_else(|| tl.layers.iter().find_map(|l| on(l))) + .ok_or_else(|| EngineError::State("there is no sound on this timeline: import one (File › Import) or put one on a keyframe first".into())) +} + +/// The sound the params name and the frames to read (`from`, `to`; default the frames it is +/// heard on from its keyframe, within the timeline). +pub fn source(s: &Session, p: &Value) -> Result { + let (layer, keyframe) = find_sound(s, p)?; + let tl = s.timeline(); + let k = tl.layer(layer).and_then(|l| l.keyframe_starting_at(keyframe).and_then(|i| l.keyframes.get(i))).ok_or_else(|| EngineError::State("the sound's keyframe went away".into()))?; + let fs = sounding(k).ok_or_else(|| EngineError::State("the keyframe has no sound".into()))?; + let sound = s.doc.sounds.iter().find(|x| x.id == fs.sound).cloned().ok_or_else(|| EngineError::State("the keyframe's sound is no longer in the library".into()))?; + let fps = s.doc.stage.fps; + let rate = sound.rate.max(1); + // The timeline's last frame, at least the sound keyframe's own. + let last = tl.frame_count().max(k.end()).saturating_sub(1).min(MAX_FRAME); + let rest = u64::from(last - keyframe.min(last) + 1); + let heard = fs.heard(sound.frames, frame_start(u64::from(k.duration), fps, rate), frame_start(rest, fps, rate)); + // Movie frames it is heard on (the last one may be partly silent). + let per = (f64::from(rate) / f64::from(if fps.is_finite() && fps > 0.0 { fps } else { 24.0 })).max(1e-9); + let heard_frames = ((heard as f64 / per).ceil() as u64).clamp(1, rest) as u32; + let from = opt_u32(p, "from")?.unwrap_or(keyframe); + let to = opt_u32(p, "to")?.unwrap_or(keyframe + heard_frames - 1); + if from < keyframe { + return Err(EngineError::Param(format!("'from' must not be before the sound's keyframe (frame {keyframe})"))); + } + if to < from { + return Err(EngineError::Param("'to' must not be before 'from'".into())); + } + if to > MAX_FRAME { + return Err(EngineError::Param(format!("'to' must be at most {MAX_FRAME}"))); + } + if (to - from) as usize >= MAX_ANALYSIS_FRAMES { + return Err(EngineError::Param(format!("at most {MAX_ANALYSIS_FRAMES} frames are analysed at once"))); + } + // A loop has no end of its own: it sounds for as long as its span (or the movie) lets it. + let span = Span { fps, offset: from - keyframe, frames: to - from + 1, heard: if fs.looping { u64::MAX } else { heard } }; + Ok(Source { layer, keyframe, sound, from, to, span }) +} + +fn pcm(src: &Source) -> Result> { + src.sound.decoded().map(|d| d.pcm.clone()).ok_or_else(|| EngineError::State(format!("'{}' can't be decoded", src.sound.name))) +} + +/// How the loudness becomes a value. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct LevelSettings { + pub threshold: f32, + pub smoothing: u32, + pub channel: Channel, +} + +impl Default for LevelSettings { + fn default() -> Self { + LevelSettings { threshold: DEFAULT_THRESHOLD, smoothing: DEFAULT_SMOOTHING, channel: Channel::Both } + } +} + +fn threshold_param(p: &Value, default: f32) -> Result { + let t = opt_f32(p, "threshold")?.unwrap_or(default); + if !(0.0..=0.99).contains(&t) { + return Err(EngineError::Param("param 'threshold' must be between 0 and 0.99 (a share of the loudest frame)".into())); + } + Ok(t) +} + +fn level_settings(p: &Value) -> Result { + let smoothing = opt_u32(p, "smoothing")?.unwrap_or(DEFAULT_SMOOTHING); + if smoothing > MAX_SMOOTHING { + return Err(EngineError::Param(format!("param 'smoothing' must be between 0 and {MAX_SMOOTHING} frames"))); + } + let channel = match opt_str(p, "channel")? { + None => Channel::Both, + Some(c) => Channel::from_key(c).ok_or_else(|| EngineError::Param("param 'channel' must be both, left or right".into()))?, + }; + Ok(LevelSettings { threshold: threshold_param(p, DEFAULT_THRESHOLD)?, smoothing, channel }) +} + +fn lip_options(p: &Value) -> Result { + let hold = opt_u32(p, "hold")?.unwrap_or(LipOptions::default().hold); + if !(1..=MAX_HOLD).contains(&hold) { + return Err(EngineError::Param(format!("param 'hold' must be between 1 and {MAX_HOLD} frames"))); + } + Ok(LipOptions { threshold: threshold_param(p, LipOptions::default().threshold)?, hold }) +} + +/// A sound's loudness frame by frame, as Convert Audio to Keyframes reads it. +#[derive(Clone, Debug, PartialEq)] +pub struct Levels { + /// The first frame analysed. + pub from: u32, + /// Each frame's loudness as a share of the loudest (`0..=1`). + pub raw: Vec, + /// The same gated by the threshold and smoothed: what is mapped onto the property. + pub shaped: Vec, +} + +/// The loudness of `src` under `set`. +pub fn levels(src: &Source, set: &LevelSettings) -> Result { + let raw = analysis::normalised(&analysis::levels(&*pcm(src)?, &src.span, set.channel)); + let shaped = analysis::shape(&raw, set.threshold, set.smoothing); + Ok(Levels { from: src.from, raw, shaped }) +} + +/// The mouth shape of each frame of `src`. +pub fn visemes(src: &Source, opts: &LipOptions) -> Result> { + Ok(analysis::visemes(&*pcm(src)?, &src.span, opts)) +} + +/// The frames that get a key when `values` (one per frame) are held between keys: every +/// `step`-th frame whose value differs from the last key's by more than `eps`, the first always. +pub fn held_keys(values: &[f32], step: u32, eps: f32) -> Vec<(u32, f32)> { + let mut out: Vec<(u32, f32)> = Vec::new(); + for (i, &v) in values.iter().enumerate().step_by(step.max(1) as usize) { + if out.last().is_none_or(|(_, last)| (v - last).abs() > eps) { + out.push((i as u32, v)); + } + } + out +} + +/// The frames that get a key when the value travels in straight lines between keys: every +/// `step`-th frame, less the insides of flat stretches (their two ends stay, so a flat stretch +/// stays flat). +pub fn curve_keys(values: &[f32], step: u32, eps: f32) -> Vec<(u32, f32)> { + let sampled: Vec<(u32, f32)> = values.iter().enumerate().step_by(step.max(1) as usize).map(|(i, &v)| (i as u32, v)).collect(); + let mut out: Vec<(u32, f32)> = Vec::new(); + for (i, &(f, v)) in sampled.iter().enumerate() { + let moved = out.last().is_none_or(|(_, last)| (v - last).abs() > eps); + let turns = sampled.get(i + 1).is_none_or(|(_, next)| (next - v).abs() > eps); + if moved || turns { + out.push((f, v)); + } + } + out +} + +/// Differences smaller than this share of the range don't make a key. +const KEY_EPS: f32 = 0.002; + +/// The layer and object that get the keys: `layer` (default the active one) and on it +/// `element`, else the selected object, else the only one on the keyframe at `frame`. +fn target(s: &Session, p: &Value, frame: u32) -> Result<(LayerId, Arc)> { + let lid = opt_u64(p, "layer")?.map_or(s.layer, LayerId); + let layer = s.timeline().layer(lid).ok_or_else(|| EngineError::State(format!("no layer with id {}", lid.0)))?; + match layer.kind { + LayerKind::Folder => return Err(EngineError::State(format!("'{}' is a folder; pick the layer with the object to animate", layer.name))), + LayerKind::Armature => return Err(EngineError::State(format!("'{}' is an armature layer: its poses place its objects, so sound can't key them yet", layer.name))), + LayerKind::Camera => return Err(EngineError::State("the camera can't be driven by sound yet; pick a layer with artwork".into())), + _ => {} + } + // A layer that ends before the frame shows nothing there, but its object is the one meant: + // the keys carry it on over the sound. + let k = layer.keyframe_at(frame).or_else(|| layer.keyframes.iter().rev().find(|k| k.start <= frame).map(|k| &**k)).ok_or_else(|| EngineError::State(format!("layer '{}' is empty at frame {}: put the object to animate there first", layer.name, frame + 1)))?; + if k.elements.is_empty() { + return Err(EngineError::State(format!("layer '{}' has nothing at frame {} to animate", layer.name, frame + 1))); + } + let pick = |id: ElementId| k.elements.iter().find(|e| e.id == id).cloned(); + if let Some(id) = opt_u64(p, "element")?.map(ElementId) { + return pick(id).map(|e| (lid, e)).ok_or_else(|| EngineError::State(format!("no object with id {} on layer '{}' at frame {}", id.0, layer.name, frame + 1))); + } + let selected: Vec> = s.selection.on(lid).into_iter().filter_map(pick).collect(); + match (selected.as_slice(), k.elements.as_slice()) { + ([one], _) | ([], [one]) => Ok((lid, one.clone())), + _ => Err(EngineError::State(format!("layer '{}' has {} objects at frame {}: select the one to animate, or pass 'element'", layer.name, k.elements.len(), frame + 1))), + } +} + +/// Whether the target layer has a motion tween anywhere on `from..=to`. +fn motion_in(layer: &Layer, from: u32, to: u32) -> bool { + layer.keyframes.iter().any(|k| k.motion.is_some() && k.start <= to && k.end() > from) +} + +/// Writes held keys for object `id` on `layer`: a keyframe at each of `keys`' frames (made as +/// Insert Keyframe makes it, holding what the frame showed), then every keyframe on +/// `from..=to` gets the object as `apply` makes it from what it was and the value held there. +/// The frame after `to`, when the layer goes on, keeps what it showed. How many keyframes hold +/// the object changed. +fn write_held(layer: &mut Layer, id: ElementId, from: u32, to: u32, keys: &[(u32, T)], apply: &dyn Fn(&Element, &T) -> Element) -> Result { + if keys.len() > MAX_KEYS { + return Err(EngineError::Param(format!("that is {} keyframes, more than {MAX_KEYS} at once: raise 'step' or the threshold, or shorten the range with 'from' and 'to'", keys.len()))); + } + if motion_in(layer, from, to) { + return Err(EngineError::State(format!("layer '{}' has a motion tween on these frames: keyframes would cut it up. Use the motion tween's curve (output: motion), or remove the tween first", layer.name))); + } + // First every keyframe, while each frame still shows what it showed. + let after = to.checked_add(1).filter(|f| *f < layer.end_frame()); + for f in keys.iter().map(|(f, _)| *f).chain(after) { + if layer.keyframe_index_at(f).is_none_or(|i| layer.keyframes.get(i).is_some_and(|k| k.start != f)) { + crate::cmds_timeline::insert_keyframe_at(layer, f, true)?; + } + } + // Then the values: each keyframe in the range holds the value of the last key at or + // before it (keyframes that were there already included). + let first = layer.keyframes.partition_point(|k| k.start < from); + let count = layer.keyframes.iter().skip(first).take_while(|k| k.start <= to).count(); + let mut changed = 0; + for i in first..first + count { + let Some(k) = layer.keyframe_mut(i) else { continue }; + let Some(value) = keys.partition_point(|(f, _)| *f <= k.start).checked_sub(1).and_then(|j| keys.get(j)).map(|(_, v)| v) else { continue }; + if let Some(slot) = k.elements.iter_mut().find(|e| e.id == id) { + let next = apply(slot, value); + if next != **slot { + *slot = Arc::new(next); + } + changed += 1; + } + } + // The object is there for the whole of the sound: a layer that ended sooner goes on to + // the last frame keyed. + if layer.end_frame() <= to + && let Some(i) = layer.keyframes.len().checked_sub(1) + && let Some(k) = layer.keyframe_mut(i) + { + k.duration = (to - k.start).saturating_add(1); + } + if changed == 0 { + return Err(EngineError::State("the object isn't on the layer's keyframes in that range".into())); + } + Ok(changed) +} + +/// The point of `el`'s own space that scale and rotation keys hold still: its transformation +/// point when it has one, a symbol's registration point, the middle of anything else. +fn anchor(s: &Session, el: &Element) -> Vec2 { + if let Some(p) = el.pivot() { + return p; + } + if matches!(el.kind, ElementKind::Symbol { .. }) { + return Vec2::ZERO; + } + let b = el.local_bounds(&s.doc); + let c = b.center(); + if b.is_empty() || !c.is_finite() { Vec2::ZERO } else { c } +} + +/// `base` with `prop` at value `v` (see [`KeyProperty`]), scaled and turned about `anchor`. +fn with_value(base: &Element, prop: KeyProperty, v: f32, anchor: Vec2) -> Element { + let mut el = base.clone(); + let about_local = |m: Affine| base.transform.mul(&Affine::translate(anchor.x, anchor.y)).mul(&m).mul(&Affine::translate(-anchor.x, -anchor.y)); + let scale = |pct: f32| { + let k = pct / 100.0; + if k.abs() < 1e-4 { 1e-4 } else { k } + }; + let t = match prop { + KeyProperty::Scale => about_local(Affine::scale(scale(v), scale(v))), + KeyProperty::ScaleX => about_local(Affine::scale(scale(v), 1.0)), + KeyProperty::ScaleY => about_local(Affine::scale(1.0, scale(v))), + KeyProperty::X => Affine::translate(v, 0.0).mul(&base.transform), + KeyProperty::Y => Affine::translate(0.0, v).mul(&base.transform), + KeyProperty::Rotation => { + let at = base.transform.apply(anchor); + Affine::translate(at.x, at.y).mul(&Affine::rotate(v.to_radians())).mul(&Affine::translate(-at.x, -at.y)).mul(&base.transform) + } + KeyProperty::Alpha => { + el.color = base.color.with_alpha((v / 100.0).clamp(0.0, 1.0)).sanitized(); + base.transform + } + KeyProperty::FirstFrame => { + show_frame(&mut el, v.round().clamp(0.0, MAX_FRAME as f32) as u32); + base.transform + } + }; + if t.is_finite() { + el.transform = t; + } + el +} + +/// Makes a symbol instance show `frame` of its symbol and hold it. +fn show_frame(el: &mut Element, frame: u32) { + if let ElementKind::Symbol { loop_mode, first_frame, .. } = &mut el.kind { + *loop_mode = LoopMode::SingleFrame; + *first_frame = frame.min(MAX_FRAME); + } +} + +/// The graphic symbol `el` is an instance of. +fn graphic<'a>(s: &'a Session, el: &Element, what: &str) -> Result<&'a Symbol> { + let sym = match &el.kind { + ElementKind::Symbol { symbol, .. } => s.doc.symbol(*symbol), + _ => None, + }; + sym.filter(|x| x.kind == SymbolKind::Graphic).ok_or_else(|| EngineError::State(format!("{what} needs an instance of a graphic symbol (one frame per pose): convert the artwork with Modify › Convert to Symbol, type Graphic"))) +} + +/// The settings of a `sound.to_keyframes` run, read and checked. +struct Plan { + prop: KeyProperty, + min: f32, + max: f32, + step: u32, + output: Option, +} + +fn plan(p: &Value) -> Result { + let all = || KeyProperty::ALL.map(KeyProperty::key).join(", "); + let prop = match opt_str(p, "property")? { + Some(k) => KeyProperty::from_key(k).ok_or_else(|| EngineError::Param(format!("unknown property '{k}' ({})", all())))?, + None => return Err(EngineError::Param(format!("param 'property' is required ({})", all()))), + }; + let r = prop.range(); + let bound = |key: &str| -> Result { + let v = f32_req(p, key)?; + if !r.contains(&v) { + return Err(EngineError::Param(format!("param '{key}' must be between {} and {} for {}", r.start(), r.end(), prop.key()))); + } + Ok(v) + }; + let step = opt_u32(p, "step")?.unwrap_or(1); + if !(1..=MAX_STEP).contains(&step) { + return Err(EngineError::Param(format!("param 'step' must be between 1 and {MAX_STEP} frames"))); + } + let output = match opt_str(p, "output")? { + None | Some("auto") => None, + Some("keyframes") => Some(Output::Keyframes), + Some("motion") => Some(Output::Motion), + Some(other) => return Err(EngineError::Param(format!("unknown output '{other}' (auto, keyframes or motion)"))), + }; + Ok(Plan { prop, min: bound("min")?, max: bound("max")?, step, output }) +} + +/// Convert Audio to Keyframes. +fn to_keyframes(s: &mut Session, p: &Value) -> Result { + let plan = plan(p)?; + let src = source(s, p)?; + let set = level_settings(p)?; + let (lid, el) = target(s, p, src.from)?; + if plan.prop == KeyProperty::FirstFrame { + let frames = graphic(s, &el, "First frame")?.timeline.frame_count().max(1); + if plan.min >= frames as f32 || plan.max >= frames as f32 { + return Err(EngineError::Param(format!("the symbol has {frames} frames: 'min' and 'max' must be between 0 and {}", frames - 1))); + } + } + let lv = levels(&src, &set)?; + let values: Vec = lv.shaped.iter().map(|l| plan.min + (plan.max - plan.min) * l).map(|v| if plan.prop == KeyProperty::FirstFrame { v.round() } else { v }).collect(); + let eps = ((plan.max - plan.min).abs() * KEY_EPS).max(1e-6); + let layer = s.timeline().layer(lid).ok_or_else(|| EngineError::State("the layer went away".into()))?; + let in_motion = layer.keyframe_at(src.from).is_some_and(|k| k.motion.is_some()); + let output = plan.output.unwrap_or(if in_motion && plan.prop != KeyProperty::FirstFrame { Output::Motion } else { Output::Keyframes }); + let (keys, to) = match output { + Output::Keyframes => { + let keys: Vec<(u32, f32)> = held_keys(&values, plan.step, eps).into_iter().map(|(i, v)| (src.from + i, v)).collect(); + let at = anchor(s, &el); + let prop = plan.prop; + let layer = s.layer_mut(Some(lid))?; + write_held(layer, el.id, src.from, src.to, &keys, &|base, v| with_value(base, prop, *v, at))?; + (keys.len(), src.to) + } + Output::Motion => write_curve(s, lid, &el, &plan, &src, &values, eps)?, + }; + Ok(json!({"keys": keys, "from": src.from, "to": to, "output": output.key(), "property": plan.prop.key(), "layer": lid.0, "element": el.id.0, "sound": src.sound.id.0})) +} + +/// Writes `values` (one per frame from `src.from`) as the property's keys in the motion tween +/// on the span at `src.from`, making the tween when the span has none. The range is cut to the +/// span. The keys written (per track) and the last frame they reach. +fn write_curve(s: &mut Session, lid: LayerId, el: &Element, plan: &Plan, src: &Source, values: &[f32], eps: f32) -> Result<(usize, u32)> { + let tracks = plan.prop.tracks(); + if tracks.is_empty() { + return Err(EngineError::Param("a symbol's first frame has no motion tween curve: use output: keyframes".into())); + } + let has = s.timeline().layer(lid).and_then(|l| l.keyframe_at(src.from)).is_some_and(|k| k.motion.is_some()); + if !has { + let alone = s.timeline().layer(lid).and_then(|l| l.keyframe_at(src.from)).is_some_and(|k| k.elements.len() == 1); + if !alone { + return Err(EngineError::State("a motion tween moves one object: put the object on a layer of its own (or Create Motion Tween on it) first, or use output: keyframes".into())); + } + crate::cmds_motion::create(s, &json!({"layer": lid.0, "frame": src.from}))?; + } + let layer = s.layer_mut(Some(lid))?; + let i = layer.keyframe_index_at(src.from).ok_or_else(|| EngineError::State("the span went away".into()))?; + let kf = layer.keyframe_mut(i).ok_or_else(|| EngineError::State("the span went away".into()))?; + let (start, last) = (kf.start, kf.duration.saturating_sub(1)); + let target = kf.motion.as_ref().and_then(|m| m.target_index(&kf.elements)).and_then(|t| kf.elements.get(t)).cloned().ok_or_else(|| EngineError::State("the motion tween's object is no longer on its keyframe".into()))?; + if target.id != el.id { + return Err(EngineError::State("the motion tween on this span moves another object: pick that one, or use output: keyframes on a layer without a motion tween".into())); + } + let m = kf.motion.as_mut().ok_or_else(|| EngineError::State("no motion tween here".into()))?; + if m.path.is_some() && tracks.iter().any(|t| matches!(t, Property::X | Property::Y)) { + return Err(EngineError::State("the motion tween follows a motion path, which places the object: clear the path first, or drive another property".into())); + } + // The range, cut to the span. + let a = src.from - start; + let b = (src.to - start).min(last); + let len = (b - a + 1) as usize; + let keys = curve_keys(values.get(..len).unwrap_or(values), plan.step, eps); + if keys.len() > MAX_KEYS { + return Err(EngineError::Param(format!("that is {} keys, more than {MAX_KEYS} on one curve: raise 'step' or the threshold, or shorten the range with 'from' and 'to'", keys.len()))); + } + for &prop in tracks { + let own = own_value(&target, prop).ok_or_else(|| EngineError::State("the motion tween's object has no such property".into()))?; + let r = prop.range(); + // The curve's keys in the range give way to the new ones; what follows the range + // starts from the object's own value again unless it has keys of its own there. + let later = m.track(prop).is_some_and(|t| t.keys.iter().any(|k| k.frame > b)); + if let Some(t) = m.track_mut(prop) { + t.keys.retain(|k| k.frame < a || k.frame > b); + } + for &(f, v) in &keys { + m.set_key(prop, a + f, plan.prop.track_value(own, v).clamp(*r.start(), *r.end()), own); + } + if !later && b < last { + m.set_key(prop, b + 1, own, own); + } + } + Ok((keys.len(), start + b)) +} + +/// The names a mouth shape's frame goes by: the Lip-sync mouth chart template's labels first +/// (ADR 0065), then other common ones. Compared without case, spaces or punctuation. +fn mouth_names(v: Viseme) -> &'static [&'static str] { + match v { + Viseme::Rest => &["rest", "neutral", "silence", "sil", "idle", "x"], + Viseme::Open => &["ai", "a", "ah", "aa", "open"], + Viseme::Wide => &["e", "ee", "wide", "i", "cdgknrsthyz", "smile", "etc"], + Viseme::Round => &["o", "oh", "round", "u", "oo", "wq", "w"], + Viseme::Closed => &["mbp", "m", "closed", "bmp", "pbm", "b", "p"], + } +} + +fn plain(label: &str) -> String { + label.chars().filter(|c| c.is_alphanumeric()).flat_map(char::to_lowercase).collect() +} + +/// The frame of `sym` labelled `label` (without case, spaces or punctuation), the first when +/// several are. +pub fn labelled_frame(sym: &Symbol, label: &str) -> Option { + let want = plain(label); + if want.is_empty() { + return None; + } + sym.timeline.layers.iter().flat_map(|l| l.keyframes.iter()).filter(|k| plain(&k.label) == want).map(|k| k.start).min() +} + +/// The frame of `sym` each mouth shape shows by default, in [`Viseme::ALL`]'s order: by the +/// symbol's frame labels; a shape without one borrows the nearest shape's frame (lips together +/// and rest stand in for each other, round and wide borrow open's), and a symbol without any +/// of the labels gives its first five frames in order. +pub fn default_mouths(sym: &Symbol) -> [u32; 5] { + let last = sym.timeline.frame_count().max(1) - 1; + let found = Viseme::ALL.map(|v| mouth_names(v).iter().find_map(|n| labelled_frame(sym, n))); + if found.iter().all(Option::is_none) { + return [0, 1, 2, 3, 4].map(|i: u32| i.min(last)); + } + let [rest, open, wide, round, closed] = found; + let rest_f = rest.or(closed).unwrap_or(0); + let open_f = open.or(wide).or(round).unwrap_or(rest_f); + [rest_f, open_f, wide.unwrap_or(open_f), round.unwrap_or(open_f), closed.or(rest).unwrap_or(rest_f)].map(|f| f.min(last)) +} + +/// The frame of `sym` each mouth shape shows: `mouths` over the defaults. +fn mouths_param(sym: &Symbol, p: &Value) -> Result<[u32; 5]> { + let mut out = default_mouths(sym); + let frames = sym.timeline.frame_count().max(1); + let Some(m) = p.get("mouths").filter(|m| !m.is_null()) else { return Ok(out) }; + let map = m.as_object().ok_or_else(|| EngineError::Param("param 'mouths' must be an object {rest, open, wide, round, closed}, each a frame of the symbol (0-based) or a frame label".into()))?; + for (key, v) in map { + let shape = Viseme::from_key(key).ok_or_else(|| EngineError::Param(format!("unknown mouth '{}' (rest, open, wide, round or closed)", key.chars().take(40).collect::())))?; + let frame = match v { + Value::String(label) => labelled_frame(sym, label).ok_or_else(|| EngineError::Param(format!("'{}' has no frame labelled '{}'", sym.name, label.chars().take(60).collect::())))?, + other => other.as_u64().and_then(|f| u32::try_from(f).ok()).ok_or_else(|| EngineError::Param(format!("mouth '{key}' must be a frame of the symbol (0-based) or a frame label")))?, + }; + if frame >= frames { + return Err(EngineError::Param(format!("mouth '{key}': '{}' has {frames} frames (0 to {})", sym.name, frames - 1))); + } + if let Some(slot) = out.get_mut(shape.index()) { + *slot = frame; + } + } + Ok(out) +} + +/// Auto Lip-Sync. +fn lip_sync(s: &mut Session, p: &Value) -> Result { + let src = source(s, p)?; + let opts = lip_options(p)?; + let (lid, el) = target(s, p, src.from)?; + let mouths = mouths_param(graphic(s, &el, "Lip-sync")?, p)?; + let shapes = visemes(&src, &opts)?; + // A key where the mouth changes. + let mut keys: Vec<(u32, Viseme)> = Vec::new(); + let mut counts = [0u32; 5]; + for (i, v) in shapes.iter().enumerate() { + if keys.last().is_none_or(|(_, last)| last != v) { + keys.push((src.from + i as u32, *v)); + if let Some(n) = counts.get_mut(v.index()) { + *n += 1; + } + } + } + let layer = s.layer_mut(Some(lid))?; + write_held(layer, el.id, src.from, src.to, &keys, &|base, v| { + let mut el = base.clone(); + show_frame(&mut el, mouths.get(v.index()).copied().unwrap_or(0)); + el + })?; + let named = |list: [u32; 5]| Value::Object(Viseme::ALL.iter().zip(list).map(|(v, n)| (v.key().to_string(), json!(n))).collect()); + Ok(json!({"keys": keys.len(), "from": src.from, "to": src.to, "layer": lid.0, "element": el.id.0, "sound": src.sound.id.0, "mouths": named(mouths), "counts": named(counts)})) +} + +/// Analyse Sound: what the two commands above work from. +fn analyse(s: &mut Session, p: &Value) -> Result { + let src = source(s, p)?; + let mut out = json!({"sound": src.sound.id.0, "name": src.sound.name, "layer": src.layer.0, "keyframe": src.keyframe, "from": src.from, "to": src.to, "fps": s.doc.stage.fps}); + let round = |v: &f32| json!((f64::from(*v) * 10_000.0).round() / 10_000.0); + match opt_str(p, "what")?.unwrap_or("levels") { + "levels" => { + let lv = levels(&src, &level_settings(p)?)?; + out["levels"] = lv.raw.iter().map(round).collect(); + out["values"] = lv.shaped.iter().map(round).collect(); + } + "visemes" => out["visemes"] = visemes(&src, &lip_options(p)?)?.iter().map(|v| json!(v.key())).collect(), + other => return Err(EngineError::Param(format!("unknown 'what' '{}' (levels or visemes)", other.chars().take(40).collect::()))), + } + Ok(out) +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 3aaf5f1f..3bd21129 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -47,6 +47,7 @@ mod cmds_script; pub mod cmds_plugin; mod cmds_scene; pub mod cmds_sound; +pub mod cmds_audio_keys; pub mod cmds_text; mod cmds_timeline; mod cmds_view; @@ -191,6 +192,7 @@ pub fn commands() -> Vec { v.extend(cmds_shape::specs()); v.extend(cmds_plugin::specs()); v.extend(cmds_sound::specs()); + v.extend(cmds_audio_keys::specs()); v.extend(cmds_actions::specs()); v.extend(cmds_script::specs()); v.extend(cmds_macro::specs()); @@ -233,7 +235,7 @@ pub use cmds_layer::LAYER_COLORS; /// The command namespaces that change a timeline's layers and frames, refused while a group is /// open for editing (ADR 0073): a group is one layer and one frame. A name ending in `.` is a /// namespace, any other one command. -const GROUP_REFUSES: &[&str] = &["timeline.", "layer.", "camera.", "bone.", "motion.", "tween.", "frames.", "sound.place", "script.set", "script.clear"]; +const GROUP_REFUSES: &[&str] = &["timeline.", "layer.", "camera.", "bone.", "motion.", "tween.", "frames.", "sound.place", "sound.to_keyframes", "sound.lip_sync", "script.set", "script.clear"]; /// Last addressable frame (0-based); about 70 minutes at 24 fps. pub const MAX_FRAME: u32 = 100_000; @@ -1397,6 +1399,8 @@ mod plugin_kinds_tests; #[cfg(test)] mod tests_sound; #[cfg(test)] +mod tests_audio_keys; +#[cfg(test)] mod hint_tests; #[cfg(test)] mod tests_actions; diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 3eb65b05..496b2f81 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -643,6 +643,15 @@ fn panic_hunt_every_command() { json!({"sound": SOUND, "frame": 3, "extend": true, "name": "renamed"}), json!({"sound": u64::MAX, "repeat": 1e9}), json!({"sound": "a", "sync": 3}), + // Audio to keyframes and lip-sync (ADR 0114): every property at its limits, ranges + // past the sound, mouths that aren't frames, on whatever layer and object is there. + json!({"property": "scale", "min": 1, "max": 10_000, "threshold": 0, "smoothing": 24, "step": 1, "output": "keyframes", "to": 99_999}), + json!({"property": "rotation", "min": -3600, "max": 3600, "output": "motion", "channel": "left", "step": 240, "layer": 2}), + json!({"property": "first_frame", "min": 0, "max": 100_000, "threshold": 0.99, "from": 0, "to": 0, "mouths": {"rest": u64::MAX, "open": "nope"}}), + json!({"property": "alpha", "min": 100, "max": 0, "sound_layer": u64::MAX, "sound_frame": u64::MAX, "hold": 12, "what": "visemes"}), + json!({"property": "x", "min": -1e5, "max": 1e5, "from": 5, "to": 2, "element": u64::MAX, "mouths": [1, 2], "hold": 0, "what": 7}), + json!({"property": "y", "min": f64::NAN, "max": 1e30, "threshold": -1, "smoothing": 1e9, "channel": "middle", "step": 0, "output": "film", "mouths": {"grin": 1}}), + json!({"property": "scale_x", "min": 100, "max": 150, "sound_layer": 1, "sound_frame": 0, "layer": 1, "from": 0, "to": 100_000, "hold": 2, "mouths": {"closed": 0, "wide": 0}}), // Effects, envelopes and layer volumes (ADR 0039). json!({"effect": "custom", "envelope": [{"at": 1e300, "left": 2, "right": -1}], "volume": 1e30, "muted": "x"}), json!({"effect": "fade_in", "envelope": (0..40).map(|i| json!([i, 0.5, 0.5])).collect::>(), "volume": -0.5}), diff --git a/crates/engine/src/tests_audio_keys.rs b/crates/engine/src/tests_audio_keys.rs new file mode 100644 index 00000000..2fbed88b --- /dev/null +++ b/crates/engine/src/tests_audio_keys.rs @@ -0,0 +1,438 @@ +//! Convert Audio to Keyframes and auto lip-sync end to end (ADR 0114): a synthetic sound on +//! the timeline, the keys it writes on another layer, undo, and what is refused. + +use super::*; +use crate::cmds_audio_keys::{self, KeyProperty, curve_keys, default_mouths, held_keys}; +use anicraft_doc::{ElementKind, LayerId, LoopMode, Property}; +use serde_json::json; +use std::f32::consts::TAU; + +const RATE: u32 = 24_000; +const FPS: f32 = 24.0; + +fn run(s: &mut Session, id: &str, p: Value) -> Value { + match s.execute(id, &p) { + Ok(v) => v, + Err(e) => panic!("{id} {p}: {e}"), + } +} + +fn fails(s: &mut Session, id: &str, p: Value) -> String { + match s.execute(id, &p) { + Ok(v) => panic!("{id} {p} should fail, gave {v}"), + Err(e) => e.to_string(), + } +} + +/// Samples in `frames` movie frames: a thousand a frame, and a 480 Hz tone is twenty whole +/// cycles of one, so every frame of a steady tone is exactly as loud as the next. +fn samples(frames: u32) -> usize { + (f64::from(frames) * f64::from(RATE) / f64::from(FPS)).round() as usize +} + +fn silence(frames: u32) -> Vec { + vec![0.0; samples(frames)] +} + +/// `frames` movie frames of sines: (Hz, amplitude) summed. +fn tone(parts: &[(f32, f32)], frames: u32) -> Vec { + (0..samples(frames)).map(|i| parts.iter().map(|(f, a)| a * (TAU * f * i as f32 / RATE as f32).sin()).sum()).collect() +} + +/// Twelve silent frames, a six-frame burst, six silent frames. +fn burst() -> Vec { + [silence(12), tone(&[(480.0, 0.5)], 6), silence(6)].concat() +} + +/// Imports `sound` to the stage as a stream on a layer of its own; that layer's id. +fn add_sound(s: &mut Session, tag: &str, sound: &[f32]) -> u64 { + let dir = std::env::temp_dir().join(format!("anicraft-audio-keys-{tag}-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let wav = dir.join("voice.wav"); + std::fs::write(&wav, crate::cmds_sound::test_wav(RATE, sound)).unwrap(); + let v = run(s, "file.import", json!({"path": wav.display().to_string()})); + let layer = v["layer"].as_u64().expect("the sound's layer"); + run(s, "sound.set", json!({"layer": layer, "frame": 0, "sync": "stream"})); + let _ = std::fs::remove_dir_all(&dir); + layer +} + +/// A 24 fps document with a 100 px square (a drawing object) at 100, 100 on its first layer, +/// and `sound` as a stream on a layer above. `(session, art layer, square, sound layer)`. +fn stage(tag: &str, sound: &[f32]) -> (Session, u64, u64, u64) { + let mut s = Session::default(); + run(&mut s, "doc.set_stage", json!({"fps": FPS})); + run(&mut s, "draw.rect", json!({"x": 100, "y": 100, "w": 100, "h": 100, "fill": "#3366cc", "stroke": null, "object": true})); + let art = s.layer.0; + let square = s.selection.first().expect("the square is selected").1.0; + let sound = add_sound(&mut s, tag, sound); + (s, art, square, sound) +} + +fn key_starts(s: &Session, layer: u64) -> Vec { + s.timeline().layer(LayerId(layer)).expect("the layer").keyframes.iter().map(|k| k.start).collect() +} + +/// The square's width at `frame`. +fn width_at(s: &mut Session, layer: u64, id: u64, frame: u32) -> f32 { + s.set_frame(frame); + s.element_bounds_on(LayerId(layer), anicraft_doc::ElementId(id)).width() +} + +/// A sine burst gives keys exactly where it sounds: one where it starts, one where it stops. +#[test] +fn a_burst_keys_the_scale_exactly_where_it_sounds() { + let (mut s, art, square, sound) = stage("burst", &burst()); + let before = s.doc.clone(); + let v = run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 200, "threshold": 0.1, "smoothing": 0})); + assert_eq!((v["keys"].as_u64(), v["from"].as_u64(), v["to"].as_u64(), v["output"].as_str()), (Some(3), Some(0), Some(23), Some("keyframes"))); + assert_eq!((v["layer"].as_u64(), v["element"].as_u64(), v["property"].as_str()), (Some(art), Some(square), Some("scale"))); + assert_eq!(key_starts(&s, art), vec![0, 12, 18], "a key where the burst starts and where it ends"); + assert_eq!(key_starts(&s, sound), vec![0], "the sound's layer is untouched"); + for f in 0..24 { + let want = if (12..18).contains(&f) { 200.0 } else { 100.0 }; + assert!((width_at(&mut s, art, square, f) - want).abs() < 0.01, "frame {f}"); + } + // It grows about its middle. + s.set_frame(14); + let b = s.element_bounds_on(LayerId(art), anicraft_doc::ElementId(square)); + assert!((b.center().x - 150.0).abs() < 0.01 && (b.center().y - 150.0).abs() < 0.01, "{b:?}"); + // One undo step. + assert_eq!(s.undo_label(), Some("Convert Audio to Keyframes…")); + run(&mut s, "edit.undo", json!({})); + assert!(s.doc == before, "undo brings the document back"); +} + +#[test] +fn the_same_sound_gives_the_same_keys() { + let voice = [tone(&[(730.0, 0.6), (1090.0, 0.3)], 7), silence(3), tone(&[(300.0, 0.5), (870.0, 0.15)], 9), tone(&[(480.0, 0.2)], 5)].concat(); + let once = |tag: &str| { + let (mut s, art, _, _) = stage(tag, &voice); + let v = run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "y", "min": 0, "max": -60, "smoothing": 2})); + let a = run(&mut s, "sound.analyse", json!({})); + (s.doc.timeline(0).layer(LayerId(art)).cloned(), v, a) + }; + let (a, b) = (once("same-a"), once("same-b")); + assert!(a.0 == b.0 && a.1 == b.1 && a.2 == b.2, "same sound, same keys"); + assert!(a.1["keys"].as_u64().is_some_and(|n| n > 4)); +} + +#[test] +fn every_property_maps_silence_to_min_and_the_loudest_frame_to_max() { + let cases: [(&str, f64, f64); 5] = [("scale_x", 100.0, 300.0), ("scale_y", 50.0, 100.0), ("x", 0.0, 40.0), ("y", -10.0, 30.0), ("rotation", 0.0, 90.0)]; + for (prop, min, max) in cases { + let (mut s, art, square, _) = stage(&format!("prop-{prop}"), &burst()); + run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": prop, "min": min, "max": max, "threshold": 0.1, "smoothing": 0})); + let at = |s: &mut Session, f: u32| { + s.set_frame(f); + s.element_bounds_on(LayerId(art), anicraft_doc::ElementId(square)) + }; + let (quiet, loud) = (at(&mut s, 3), at(&mut s, 14)); + let near = |a: f32, b: f64| (f64::from(a) - b).abs() < 0.01; + match prop { + "scale_x" => assert!(near(quiet.width(), 100.0) && near(loud.width(), 300.0) && near(loud.height(), 100.0), "{prop}: {loud:?}"), + "scale_y" => assert!(near(quiet.height(), 50.0) && near(loud.height(), 100.0) && near(loud.width(), 100.0), "{prop}: {quiet:?}"), + "x" => assert!(near(quiet.min.x, 100.0) && near(loud.min.x, 140.0) && near(loud.min.y, 100.0), "{prop}: {loud:?}"), + "y" => assert!(near(quiet.min.y, 90.0) && near(loud.min.y, 130.0), "{prop}: {quiet:?} {loud:?}"), + // A quarter turn about its middle: the square covers itself. + _ => assert!(near(loud.min.x, 100.0) && near(loud.min.y, 100.0) && near(loud.width(), 100.0), "{prop}: {loud:?}"), + } + } + // Alpha: the colour effect, 20 % in silence and 90 % on the burst. + let (mut s, art, square, _) = stage("prop-alpha", &burst()); + run(&mut s, "sound.to_keyframes", json!({"layer": art, "element": square, "property": "opacity", "min": 20, "max": 90, "threshold": 0.1, "smoothing": 0})); + let alpha = |s: &Session, f: u32| s.timeline().layer(LayerId(art)).and_then(|l| l.keyframe_at(f)).and_then(|k| k.elements.first().map(|e| e.color.alpha())).expect("the square"); + assert!((alpha(&s, 5) - 0.2).abs() < 1e-5 && (alpha(&s, 13) - 0.9).abs() < 1e-5 && (alpha(&s, 20) - 0.2).abs() < 1e-5); +} + +#[test] +fn smoothing_threshold_step_and_range_shape_the_keys() { + // Smoothing ramps into and out of the burst: more keys, the peak still on the burst. + let (mut s, art, square, _) = stage("shape", &burst()); + let v = run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 200, "smoothing": 2})); + assert!(v["keys"].as_u64().is_some_and(|n| n > 6), "{v}"); + assert!(width_at(&mut s, art, square, 9) == 100.0 && width_at(&mut s, art, square, 11) > 100.0 && (width_at(&mut s, art, square, 14) - 200.0).abs() < 0.01); + // A key every four frames: the keys fall on frames 0, 4, 8…, held in between. + run(&mut s, "edit.undo", json!({})); + run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 200, "smoothing": 0, "step": 4})); + assert_eq!(key_starts(&s, art), vec![0, 12, 20]); + // A range inside the sound: only its frames are keyed. + run(&mut s, "edit.undo", json!({})); + let v = run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 200, "smoothing": 0, "from": 14, "to": 20})); + assert_eq!((v["from"].as_u64(), v["to"].as_u64()), (Some(14), Some(20))); + assert_eq!(key_starts(&s, art), vec![0, 14, 18]); + // A threshold above the burst's own level leaves nothing to key but the first frame. + run(&mut s, "edit.undo", json!({})); + let quiet = [silence(6), tone(&[(480.0, 0.05)], 6), tone(&[(480.0, 0.5)], 6), silence(6)].concat(); + let (mut s, art, _, _) = stage("gate", &quiet); + run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 200, "smoothing": 0, "threshold": 0.3})); + assert_eq!(key_starts(&s, art), vec![0, 12, 18], "the quiet part is under the threshold"); +} + +/// Keyframes already on the layer keep their place and take the value; the frame after the +/// sound shows what it showed before. +#[test] +fn keyframes_already_there_take_the_value_and_the_layer_goes_on_as_it_was() { + let loud_to_the_end = [silence(12), tone(&[(480.0, 0.5)], 12)].concat(); + let (mut s, art, square, _) = stage("existing", &loud_to_the_end); + run(&mut s, "timeline.insert_frame", json!({"layer": art, "frame": 0, "count": 39})); + run(&mut s, "timeline.insert_keyframe", json!({"layer": art, "frame": 14})); + run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 200, "threshold": 0.1, "smoothing": 0})); + assert_eq!(key_starts(&s, art), vec![0, 12, 14, 24]); + assert!((width_at(&mut s, art, square, 14) - 200.0).abs() < 0.01, "the keyframe that was there is loud too"); + assert!((width_at(&mut s, art, square, 23) - 200.0).abs() < 0.01); + assert!((width_at(&mut s, art, square, 24) - 100.0).abs() < 0.01 && (width_at(&mut s, art, square, 39) - 100.0).abs() < 0.01, "after the sound, as it was"); +} + +/// On a motion tween the keys go on the property's curve, and the span stays one span. +#[test] +fn a_motion_tween_takes_the_keys_on_its_curve() { + let mut s = Session::default(); + run(&mut s, "doc.set_stage", json!({"fps": FPS})); + run(&mut s, "draw.rect", json!({"x": 100, "y": 100, "w": 100, "h": 100, "fill": "#3366cc", "stroke": null, "object": true})); + run(&mut s, "modify.convert_to_symbol", json!({"name": "Ball"})); + let art = s.layer.0; + run(&mut s, "timeline.insert_frame", json!({"frame": 0, "count": 29})); + run(&mut s, "timeline.create_motion_tween", json!({"frame": 0})); + add_sound(&mut s, "motion", &burst()); + let v = run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "alpha", "min": 10, "max": 100, "threshold": 0.1, "smoothing": 0})); + assert_eq!((v["output"].as_str(), v["keys"].as_u64(), v["to"].as_u64()), (Some("motion"), Some(6), Some(23))); + assert_eq!(key_starts(&s, art), vec![0], "the span is not cut up"); + let layer = s.timeline().layer(LayerId(art)).cloned().expect("the layer"); + let track = |p: Property| layer.keyframes[0].motion.as_ref().and_then(|m| m.track(p)).map(|t| t.keys.iter().map(|k| (k.frame, k.value)).collect::>()); + // Flat stretches keep their two ends, and the frame after the sound returns to the + // object's own alpha. + let near = |got: Option>, want: &[(u32, f32)]| got.is_some_and(|g| g.len() == want.len() && g.iter().zip(want).all(|(a, b)| a.0 == b.0 && (a.1 - b.1).abs() < 0.01)); + assert!(near(track(Property::Alpha), &[(0, 10.0), (11, 10.0), (12, 100.0), (17, 100.0), (18, 10.0), (23, 10.0), (24, 100.0)]), "{:?}", track(Property::Alpha)); + let alpha = |f: u32| anicraft_doc::evaluate_layer_elements(&layer, f).and_then(|(_, e)| e.first().map(|e| e.color.alpha())).expect("the ball"); + assert!((alpha(5) - 0.1).abs() < 1e-4 && (alpha(14) - 1.0).abs() < 1e-4 && (alpha(20) - 0.1).abs() < 1e-4 && (alpha(28) - 1.0).abs() < 1e-4); + // Scale writes both scale curves, as shares of the object's own size; a second run + // replaces the keys of the first. + for _ in 0..2 { + run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 150, "threshold": 0.1, "smoothing": 0, "output": "motion"})); + } + let layer = s.timeline().layer(LayerId(art)).cloned().expect("the layer"); + let track = |p: Property| layer.keyframes[0].motion.as_ref().and_then(|m| m.track(p)).map(|t| t.keys.iter().map(|k| (k.frame, k.value)).collect::>()); + let want = [(0, 100.0), (11, 100.0), (12, 150.0), (17, 150.0), (18, 100.0), (23, 100.0), (24, 100.0)]; + assert!(near(track(Property::ScaleX), &want), "{:?}", track(Property::ScaleX)); + assert!(near(track(Property::ScaleY), &want), "{:?}", track(Property::ScaleY)); + // Keyframes would cut the tween up: refused, saying what to do. + let why = fails(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 150, "output": "keyframes"})); + assert!(why.contains("motion tween"), "{why}"); + // Without a tween, asking for the curve makes one on a lone object. + let (mut s, art, _, _) = stage("motion-new", &burst()); + run(&mut s, "timeline.insert_frame", json!({"layer": art, "frame": 0, "count": 23})); + let v = run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "rotation", "min": 0, "max": 45, "threshold": 0.1, "smoothing": 0, "output": "motion"})); + assert_eq!(v["output"].as_str(), Some("motion")); + assert!(s.timeline().layer(LayerId(art)).is_some_and(|l| l.keyframes.len() == 1 && l.keyframes[0].motion.as_ref().is_some_and(|m| m.track(Property::Rotation).is_some()))); +} + +/// A graphic symbol's first frame follows the loudness: louder, a later frame. +#[test] +fn the_first_frame_of_a_graphic_follows_the_loudness() { + let mut s = Session::default(); + run(&mut s, "file.new_from_template", json!({"template": "lip-sync"})); + let fps = s.doc.stage.fps; + assert_eq!(fps, FPS, "the template's frame rate is the tests'"); + let mouth = s.timeline().layers.iter().find(|l| l.name == "Mouth").map(|l| l.id.0).expect("the Mouth layer"); + add_sound(&mut s, "first", &[silence(6), tone(&[(480.0, 0.25)], 6), tone(&[(480.0, 0.5)], 6), silence(6)].concat()); + run(&mut s, "sound.to_keyframes", json!({"layer": mouth, "property": "first_frame", "min": 0, "max": 4, "smoothing": 0})); + let shown = |s: &Session, f: u32| match s.timeline().layer(LayerId(mouth)).and_then(|l| l.keyframe_at(f)).and_then(|k| k.elements.first().map(|e| e.kind.clone())) { + Some(ElementKind::Symbol { first_frame, loop_mode, .. }) => (first_frame, loop_mode), + other => panic!("{other:?}"), + }; + assert_eq!(shown(&s, 2), (0, LoopMode::SingleFrame)); + assert_eq!(shown(&s, 8), (2, LoopMode::SingleFrame), "half as loud, half way"); + assert_eq!(shown(&s, 14), (4, LoopMode::SingleFrame)); + assert_eq!(shown(&s, 20), (0, LoopMode::SingleFrame)); + // A frame the symbol doesn't have, and an object that isn't a graphic symbol. + let why = fails(&mut s, "sound.to_keyframes", json!({"layer": mouth, "property": "first_frame", "min": 0, "max": 10})); + assert!(why.contains("10 frames"), "{why}"); + let (mut s, art, _, _) = stage("first-shape", &burst()); + let why = fails(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "first_frame", "min": 0, "max": 1})); + assert!(why.contains("graphic symbol"), "{why}"); + let why = fails(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "first_frame", "min": 0, "max": 1, "output": "motion"})); + assert!(why.contains("graphic symbol"), "{why}"); +} + +/// The mouth chart template lip-synced to "ah", a stop, "ee", "oo": each mouth's labelled +/// frame on the frame its sound starts. +#[test] +fn the_mouth_chart_is_lip_synced_to_synthetic_speech() { + let mut s = Session::default(); + run(&mut s, "file.new_from_template", json!({"template": "lip-sync"})); + let mouth = s.timeline().layers.iter().find(|l| l.name == "Mouth").map(|l| l.id.0).expect("the Mouth layer"); + let speech = [silence(12), tone(&[(730.0, 0.6), (1090.0, 0.3)], 6), silence(2), tone(&[(270.0, 0.6), (2290.0, 0.15)], 6), tone(&[(300.0, 0.6), (870.0, 0.2)], 6), silence(12)].concat(); + let sound = add_sound(&mut s, "lips", &speech); + let before = s.doc.clone(); + // The sound's layer is active: the mouth's layer is named, and its one object found. + let v = run(&mut s, "sound.lip_sync", json!({"layer": mouth})); + assert_eq!((v["keys"].as_u64(), v["from"].as_u64(), v["to"].as_u64()), (Some(6), Some(0), Some(43)), "{v}"); + // Rest, AI, E, O and MBP are the chart's frames 0, 1, 2, 3 and 8. + assert_eq!(v["mouths"], json!({"rest": 0, "open": 1, "wide": 2, "round": 3, "closed": 8})); + assert_eq!(v["counts"], json!({"rest": 2, "open": 1, "wide": 1, "round": 1, "closed": 1})); + let layer = s.timeline().layer(LayerId(mouth)).cloned().expect("the layer"); + let keys: Vec<(u32, u32)> = layer + .keyframes + .iter() + .map(|k| match k.elements.first().map(|e| &e.kind) { + Some(ElementKind::Symbol { first_frame, loop_mode: LoopMode::SingleFrame, .. }) => (k.start, *first_frame), + other => panic!("{other:?}"), + }) + .collect(); + assert_eq!(keys, vec![(0, 0), (12, 1), (18, 8), (20, 2), (26, 3), (32, 0)]); + assert_eq!(key_starts(&s, sound), vec![0]); + assert_eq!(s.undo_label(), Some("Auto Lip-Sync…")); + run(&mut s, "edit.undo", json!({})); + assert!(s.doc == before, "one undo step"); + // Mouths by label or by frame, over the defaults; a longer hold drops the short ones. + let v = run(&mut s, "sound.lip_sync", json!({"layer": mouth, "mouths": {"open": "fv", "round": 9, "closed": "L"}, "hold": 7})); + assert_eq!(v["mouths"], json!({"rest": 0, "open": 6, "wide": 2, "round": 9, "closed": 7})); + assert_eq!(v["counts"], json!({"rest": 2, "open": 0, "wide": 0, "round": 1, "closed": 1}), "six-frame mouths join their neighbour; the stop stays"); + // What is refused, and why. + for (p, part) in [ + (json!({"layer": mouth, "mouths": {"open": "nope"}}), "no frame labelled"), + (json!({"layer": mouth, "mouths": {"open": 10}}), "has 10 frames"), + (json!({"layer": mouth, "mouths": {"grin": 1}}), "unknown mouth"), + (json!({"layer": mouth, "mouths": [1, 2]}), "must be an object"), + (json!({"layer": mouth, "hold": 0}), "'hold'"), + (json!({"layer": mouth, "threshold": 1}), "'threshold'"), + (json!({"layer": mouth, "from": 9, "to": 3}), "'to'"), + (json!({"layer": sound}), "nothing at frame"), + (json!({"layer": mouth, "element": 999_999}), "no object with id"), + (json!({"layer": mouth, "sound_layer": mouth}), "has no sound"), + ] { + let why = fails(&mut s, "sound.lip_sync", p.clone()); + assert!(why.contains(part), "{p}: {why}"); + } +} + +#[test] +fn mouths_are_found_by_label_and_fall_back_sensibly() { + let chart = anicraft_doc::templates::TEMPLATES.iter().find(|t| t.id == "lip-sync").map(|t| (t.build)()).expect("the template"); + let sym = chart.library.iter().find(|x| x.name == "Mouth").expect("the Mouth symbol"); + assert_eq!(default_mouths(sym), [0, 1, 2, 3, 8]); + assert_eq!(cmds_audio_keys::labelled_frame(sym, " c-d-g-k-n-r-s-th-y-z "), Some(5)); + assert_eq!(cmds_audio_keys::labelled_frame(sym, ""), None); + // No labels: the first five frames, in order; fewer frames, the last one again. + let mut bare = (**sym).clone(); + for l in &mut bare.timeline.layers { + for k in &mut Arc::make_mut(l).keyframes { + Arc::make_mut(k).label.clear(); + } + } + assert_eq!(default_mouths(&bare), [0, 1, 2, 3, 4]); + for l in &mut bare.timeline.layers { + Arc::make_mut(l).keyframes.truncate(3); + } + assert_eq!(default_mouths(&bare), [0, 1, 2, 2, 2]); + // Only some labelled: the others borrow the nearest mouth. + let mut some = (**sym).clone(); + for l in &mut some.timeline.layers { + for k in &mut Arc::make_mut(l).keyframes { + let k = Arc::make_mut(k); + if !matches!(k.label.as_str(), "AI" | "MBP") { + k.label.clear(); + } + } + } + assert_eq!(default_mouths(&some), [8, 1, 1, 1, 8], "rest borrows lips together; wide and round borrow open"); +} + +#[test] +fn analyse_reports_levels_and_mouths_without_changing_anything() { + let (mut s, _, _, sound) = stage("analyse", &burst()); + let before = s.doc.clone(); + let v = run(&mut s, "sound.analyse", json!({"smoothing": 0, "threshold": 0.1})); + assert_eq!((v["layer"].as_u64(), v["keyframe"].as_u64(), v["from"].as_u64(), v["to"].as_u64(), v["name"].as_str()), (Some(sound), Some(0), Some(0), Some(23), Some("voice.wav"))); + let levels: Vec = v["levels"].as_array().expect("levels").iter().filter_map(Value::as_f64).collect(); + assert_eq!(levels.len(), 24); + assert!(levels.iter().enumerate().all(|(i, l)| if (12..18).contains(&i) { *l > 0.98 } else { *l == 0.0 }), "{levels:?}"); + assert_eq!(v["values"], v["levels"], "nothing under the threshold, no smoothing"); + let v = run(&mut s, "sound.analyse", json!({"what": "visemes", "from": 10, "to": 13})); + assert_eq!(v["visemes"], json!(["rest", "rest", "round", "round"]), "a lone low tone reads as a round mouth"); + assert!(s.doc == before && !s.can_undo() || s.undo_label() != Some("Analyse Sound")); + assert!(fails(&mut s, "sound.analyse", json!({"what": "words"})).contains("levels or visemes")); + // A timeline without a sound says so. + let mut empty = Session::default(); + let why = fails(&mut empty, "sound.to_keyframes", json!({"property": "scale", "min": 100, "max": 150})); + assert!(why.contains("no sound"), "{why}"); +} + +#[test] +fn what_cannot_be_keyed_is_refused_with_a_reason() { + let (mut s, art, _, sound) = stage("refuse", &burst()); + let before = s.doc.clone(); + for (p, part) in [ + (json!({"layer": art}), "'property' is required"), + (json!({"layer": art, "property": "skew", "min": 0, "max": 1}), "unknown property"), + (json!({"layer": art, "property": "scale", "min": 0, "max": 150}), "'min'"), + (json!({"layer": art, "property": "alpha", "min": 0, "max": 101}), "'max'"), + (json!({"layer": art, "property": "scale", "max": 150}), "min"), + (json!({"layer": art, "property": "scale", "min": 100, "max": 150, "step": 0}), "'step'"), + (json!({"layer": art, "property": "scale", "min": 100, "max": 150, "smoothing": 25}), "'smoothing'"), + (json!({"layer": art, "property": "scale", "min": 100, "max": 150, "channel": "centre"}), "'channel'"), + (json!({"layer": art, "property": "scale", "min": 100, "max": 150, "output": "film"}), "unknown output"), + (json!({"layer": sound, "property": "scale", "min": 100, "max": 150}), "nothing at frame"), + (json!({"layer": 999_999, "property": "scale", "min": 100, "max": 150}), "no layer"), + (json!({"layer": art, "property": "scale", "min": 100, "max": 150, "to": 200_000}), "'to'"), + ] { + let why = fails(&mut s, "sound.to_keyframes", p.clone()); + assert!(why.contains(part), "{p}: {why}"); + } + assert!(s.doc == before, "a refused command changes nothing"); + // Two objects on the keyframe: the selected one is keyed; with none selected it asks. + run(&mut s, "layer.select", json!({"layer": art})); + run(&mut s, "draw.rect", json!({"x": 400, "y": 100, "w": 50, "h": 50, "fill": "#cc3366", "stroke": null, "object": true})); + let second = s.selection.first().expect("the second square").1.0; + let v = run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 200, "threshold": 0.1, "smoothing": 0})); + assert_eq!(v["element"].as_u64(), Some(second)); + assert!((width_at(&mut s, art, second, 14) - 100.0).abs() < 0.01); + run(&mut s, "edit.undo", json!({})); + run(&mut s, "edit.deselect", json!({})); + let why = fails(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 200})); + assert!(why.contains("2 objects"), "{why}"); + // Plug-ins may call them (they edit the document, like the other sound commands). + for id in ["sound.to_keyframes", "sound.lip_sync", "sound.analyse"] { + assert!(crate::cmds_plugin::plugin_may_call(id).is_ok(), "{id}"); + } +} + +#[test] +fn keys_are_thinned_without_changing_what_is_seen() { + let v = [0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.5, 0.0, 0.0]; + assert_eq!(held_keys(&v, 1, 0.001), vec![(0, 0.0), (3, 1.0), (6, 0.5), (7, 0.0)]); + assert_eq!(held_keys(&v, 3, 0.001), vec![(0, 0.0), (3, 1.0), (6, 0.5)]); + assert_eq!(curve_keys(&v, 1, 0.001), vec![(0, 0.0), (2, 0.0), (3, 1.0), (5, 1.0), (6, 0.5), (7, 0.0), (8, 0.0)]); + assert_eq!(curve_keys(&[0.4; 6], 1, 0.001), vec![(0, 0.4), (5, 0.4)], "a flat sound: its two ends"); + assert!(held_keys(&[], 1, 0.1).is_empty() && curve_keys(&[], 0, 0.1).is_empty()); + assert_eq!(held_keys(&[1.0, 2.0], 0, 0.0).len(), 2, "a step of zero is one"); + for p in KeyProperty::ALL { + assert_eq!(KeyProperty::from_key(p.key()), Some(p)); + let (min, max) = p.defaults(); + assert!(p.range().contains(&min) && p.range().contains(&max), "{}", p.key()); + } +} + +/// How long the analysis and the keys take (run with `--ignored --nocapture`). +#[test] +#[ignore = "a timing, not a check"] +fn timing_of_a_long_sound() { + // Ten minutes of a voice-like sound at 24 kHz, 24 fps: 14 400 frames. + let n = RATE as usize * 600; + let voice: Vec = (0..n).map(|i| { + let t = i as f32 / RATE as f32; + let syllable = ((t * 4.0).fract() < 0.7) as u8 as f32; + syllable * (0.5 * (TAU * 730.0 * t).sin() + 0.2 * (TAU * (1090.0 + 600.0 * (t * 0.5).sin()) * t).sin()) + }).collect(); + let (mut s, art, _, _) = stage("timing", &voice); + let t = std::time::Instant::now(); + let v = run(&mut s, "sound.analyse", json!({})); + eprintln!("levels of {} frames: {:?}", v["levels"].as_array().map_or(0, Vec::len), t.elapsed()); + let t = std::time::Instant::now(); + let v = run(&mut s, "sound.analyse", json!({"what": "visemes"})); + eprintln!("visemes of {} frames: {:?}", v["visemes"].as_array().map_or(0, Vec::len), t.elapsed()); + let t = std::time::Instant::now(); + let v = run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 150, "step": 2})); + eprintln!("{} keyframes written: {:?}", v["keys"], t.elapsed()); +}