feat(engine): sound.to_keyframes, sound.lip_sync and sound.analyse: a sound's loudness or speech keys an object on another layer, one undo step (M5)

This commit is contained in:
Tanish Manku committed 2026-11-18 04:30:42 -07:00
1 parent dd31814668
commit 6fc56bfefe
5 files changed
+1213 -3

No files matched your search

+6 -2
View File
@@ -331,8 +331,10 @@ fn held(mut v: Vec<Viseme>, hold: usize) -> Vec<Viseme> {
let Some(&cur) = v.get(i) else { break };
let end = v.iter().skip(i).position(|x| *x != cur).map_or(v.len(), |n| i + n);
let short = end - i < hold && cur != Viseme::Closed;
// What a short run becomes: the run before it, or (at the start) the one after.
let join = if short { before.or_else(|| v.get(end).copied()) } else { None };
// What a short run becomes: the run before it, or the one after at the start and after
// a closure (lips together don't stay together for the sound that follows them).
let open = |x: &Viseme| *x != Viseme::Closed;
let join = if short { before.filter(open).or_else(|| v.get(end).copied().filter(open)) } else { None };
match join {
Some(to) if to != cur => {
for slot in v.iter_mut().take(end).skip(i) {
@@ -616,6 +618,8 @@ mod tests {
assert_eq!(held(vec![Open, Open, Wide, Open, Open], 2), vec![Open; 5]);
assert_eq!(held(vec![Wide, Open, Open, Open], 2), vec![Open; 4], "the first joins what follows");
assert_eq!(held(vec![Open, Open, Closed, Wide, Wide], 2), vec![Open, Open, Closed, Wide, Wide]);
assert_eq!(held(vec![Open, Open, Closed, Wide, Round, Round], 2), vec![Open, Open, Closed, Round, Round, Round], "after a closure, the sound that follows");
assert_eq!(held(vec![Closed, Wide, Closed], 2), vec![Closed, Wide, Closed], "between closures it stays");
assert_eq!(held(vec![Open, Wide, Round, Open], 1), vec![Open, Wide, Round, Open]);
assert_eq!(held(vec![Open, Open, Open, Wide, Wide, Round], 3), vec![Open; 6]);
assert_eq!(held(vec![], 3), vec![]);
+755
View File
@@ -0,0 +1,755 @@
//! Convert Audio to Keyframes and auto lip-sync (ADR 0114): a sound on the timeline drives an
//! object on another layer.
//!
//! * `sound.to_keyframes` maps the sound's loudness on each frame (gated and smoothed) onto a
//! property of an object: scale, alpha, a position or rotation offset, or a graphic
//! symbol's first frame. It writes keyframes, or a motion tween's property curve.
//! * `sound.lip_sync` sorts each frame of speech into a small set of mouth shapes and writes
//! keyframes that show the matching frame of a graphic symbol of mouth poses.
//! * `sound.analyse` reports what both would work from, without changing anything.
//!
//! The analysis is `anicraft_audio::analysis` (pure, bounded); this module finds the sound and
//! the object, and writes the keys. Each command is one undo step.
use crate::params::*;
use crate::{CommandSpec, EngineError, MAX_FRAME, Result, Session};
use anicraft_doc::audio::analysis::{self, Channel, LipOptions, MAX_ANALYSIS_FRAMES, MAX_HOLD, MAX_SMOOTHING, Span, Viseme};
use anicraft_doc::motion::MAX_KEYS;
use anicraft_doc::sound::frame_start;
use anicraft_doc::{Element, ElementId, ElementKind, FrameSound, Layer, LayerId, LayerKind, LoopMode, Property, Sound, SoundSync, Symbol, SymbolKind, own_value};
use anicraft_geom::{Affine, Vec2};
use serde_json::{Value, json};
use std::sync::Arc;
/// What the dialogs start from.
pub const DEFAULT_THRESHOLD: f32 = 0.05;
pub const DEFAULT_SMOOTHING: u32 = 1;
/// Most frames between keys.
pub const MAX_STEP: u32 = 240;
pub fn specs() -> Vec<CommandSpec> {
let c = |id, label, menu, params, mutates, run| CommandSpec { id, label, menu, shortcut: None, params, mutates, run };
vec![
c(
"sound.to_keyframes",
"Convert Audio to Keyframes…",
"Modify/Timeline",
"{property: scale|scale_x|scale_y|alpha|x|y|rotation|first_frame, min, max (the property's value in silence and at the loudest frame: scale % of the object's size, alpha %, x and y pixels from where it is, rotation degrees from how it is turned, first_frame a 0-based frame of the graphic symbol), layer? (the layer that gets the keys; default the active layer), element? (the object on it; default the selected one, or the only one), sound_layer?, sound_frame? (the sound's keyframe; default the sound at the playhead), threshold? (0–0.99 of the loudest frame: quieter is silence; default 0.05), smoothing? (0–24 frames either side; default 1), channel?: both|left|right, step? (a key every n frames, 1–240; default 1), output?: auto|keyframes|motion (motion writes the property's curve in the motion tween on the span; auto picks it when the span has one), from?, to? (frames; default the frames the sound is heard on)} → {keys, from, to, output, property, layer, element}",
true,
to_keyframes as crate::RunFn,
),
c(
"sound.lip_sync",
"Auto Lip-Sync…",
"Modify/Timeline",
"{layer? (the layer with the mouth; default the active layer), element? (a graphic symbol instance of mouth poses on it; default the selected one, or the only one), mouths?: {rest, open, wide, round, closed} (each a 0-based frame of the symbol or a frame label in it; default by the symbol's frame labels, as the Lip-sync mouth chart template names them), sound_layer?, sound_frame? (the speech; default the sound at the playhead), threshold? (0–0.99 of the loudest frame: quieter is silence; default 0.08), hold? (fewest frames a mouth is held, 1–12; default 2), from?, to? (frames; default the frames the sound is heard on)} → {keys, from, to, layer, element, mouths, counts}",
true,
lip_sync,
),
c(
"sound.analyse",
"Analyse Sound",
"",
"{what?: levels|visemes (default levels), sound_layer?, sound_frame?, from?, to?, threshold?, smoothing?, channel? (levels), hold? (visemes)} → {sound, name, layer, keyframe, from, to, fps, levels (each frame's loudness, 0–1 of the loudest) and values (gated and smoothed), or visemes (rest|open|wide|round|closed per frame)}",
false,
analyse,
),
]
}
/// A property Convert Audio to Keyframes drives.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum KeyProperty {
/// Both scales together, percent of the object's own size.
Scale,
ScaleX,
ScaleY,
/// Opacity, percent.
Alpha,
/// Pixels from where the object is.
X,
Y,
/// Degrees from how the object is turned.
Rotation,
/// The frame a graphic symbol instance shows (0-based), held as a single frame.
FirstFrame,
}
impl KeyProperty {
pub const ALL: [KeyProperty; 8] = [KeyProperty::Scale, KeyProperty::ScaleX, KeyProperty::ScaleY, KeyProperty::Alpha, KeyProperty::X, KeyProperty::Y, KeyProperty::Rotation, KeyProperty::FirstFrame];
pub fn key(self) -> &'static str {
match self {
KeyProperty::Scale => "scale",
KeyProperty::ScaleX => "scale_x",
KeyProperty::ScaleY => "scale_y",
KeyProperty::Alpha => "alpha",
KeyProperty::X => "x",
KeyProperty::Y => "y",
KeyProperty::Rotation => "rotation",
KeyProperty::FirstFrame => "first_frame",
}
}
pub fn label(self) -> &'static str {
match self {
KeyProperty::Scale => "Scale",
KeyProperty::ScaleX => "Scale X",
KeyProperty::ScaleY => "Scale Y",
KeyProperty::Alpha => "Alpha",
KeyProperty::X => "X offset",
KeyProperty::Y => "Y offset",
KeyProperty::Rotation => "Rotation",
KeyProperty::FirstFrame => "First frame",
}
}
pub fn unit(self) -> &'static str {
match self {
KeyProperty::Scale | KeyProperty::ScaleX | KeyProperty::ScaleY | KeyProperty::Alpha => "%",
KeyProperty::X | KeyProperty::Y => "px",
KeyProperty::Rotation => "°",
KeyProperty::FirstFrame => "",
}
}
pub fn from_key(s: &str) -> Option<KeyProperty> {
let s = s.trim().to_ascii_lowercase().replace([' ', '-'], "_");
let s = match s.as_str() {
"opacity" => "alpha",
"scalex" => "scale_x",
"scaley" => "scale_y",
"rotate" => "rotation",
"frame" | "firstframe" => "first_frame",
other => other,
};
KeyProperty::ALL.into_iter().find(|p| p.key() == s)
}
/// The values `min` and `max` may take.
pub fn range(self) -> std::ops::RangeInclusive<f32> {
match self {
KeyProperty::Scale | KeyProperty::ScaleX | KeyProperty::ScaleY => 1.0..=10_000.0,
KeyProperty::Alpha => 0.0..=100.0,
KeyProperty::X | KeyProperty::Y => -100_000.0..=100_000.0,
KeyProperty::Rotation => -3_600.0..=3_600.0,
KeyProperty::FirstFrame => 0.0..=MAX_FRAME as f32,
}
}
/// What the dialog offers first: (in silence, at the loudest).
pub fn defaults(self) -> (f32, f32) {
match self {
KeyProperty::Scale | KeyProperty::ScaleX | KeyProperty::ScaleY => (100.0, 150.0),
KeyProperty::Alpha => (20.0, 100.0),
KeyProperty::X => (0.0, 40.0),
KeyProperty::Y => (0.0, -40.0),
KeyProperty::Rotation => (0.0, 20.0),
KeyProperty::FirstFrame => (0.0, 1.0),
}
}
/// The motion tween tracks it writes (none for a symbol's first frame).
fn tracks(self) -> &'static [Property] {
match self {
KeyProperty::Scale => &[Property::ScaleX, Property::ScaleY],
KeyProperty::ScaleX => &[Property::ScaleX],
KeyProperty::ScaleY => &[Property::ScaleY],
KeyProperty::Alpha => &[Property::Alpha],
KeyProperty::X => &[Property::X],
KeyProperty::Y => &[Property::Y],
KeyProperty::Rotation => &[Property::Rotation],
KeyProperty::FirstFrame => &[],
}
}
/// A track's value for the property's value `v`, on an object whose own value is `own`.
fn track_value(self, own: f32, v: f32) -> f32 {
match self {
KeyProperty::Scale | KeyProperty::ScaleX | KeyProperty::ScaleY => own * v / 100.0,
KeyProperty::Alpha | KeyProperty::FirstFrame => v,
KeyProperty::X | KeyProperty::Y | KeyProperty::Rotation => own + v,
}
}
}
/// How the keys are written.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Output {
/// Keyframes on the layer, each holding the object with the value.
Keyframes,
/// Property keys in the motion tween on the span.
Motion,
}
impl Output {
pub fn key(self) -> &'static str {
match self {
Output::Keyframes => "keyframes",
Output::Motion => "motion",
}
}
}
/// The sound a command reads: where it is, and the frames it is heard on.
#[derive(Clone, Debug)]
pub struct Source {
pub layer: LayerId,
/// Its keyframe's first frame.
pub keyframe: u32,
pub sound: Arc<Sound>,
/// The frames analysed, first and last.
pub from: u32,
pub to: u32,
span: Span,
}
impl Source {
/// Frames analysed.
pub fn frames(&self) -> u32 {
self.to - self.from + 1
}
}
fn sounding(k: &anicraft_doc::Keyframe) -> Option<&FrameSound> {
k.sound.as_ref().filter(|f| f.sync != SoundSync::Stop)
}
/// The keyframe with the sound to read, as `(layer, keyframe start)`: on `sound_layer` when
/// given, else the first layer sounding at the frame (a stream sound before the others), else
/// the timeline's first sound.
fn find_sound(s: &Session, p: &Value) -> Result<(LayerId, u32)> {
let frame = opt_u32(p, "sound_frame")?.unwrap_or(s.frame).min(MAX_FRAME);
let tl = s.timeline();
// On a layer: the sound whose keyframe holds the frame, else the last one before it, else
// its first.
let on = |l: &Layer| {
let at = l.keyframe_at(frame).filter(|k| sounding(k).is_some());
let before = || l.keyframes.iter().rev().find(|k| k.start <= frame && sounding(k).is_some()).map(|k| &**k);
let first = || l.keyframes.iter().find(|k| sounding(k).is_some()).map(|k| &**k);
at.or_else(before).or_else(first).map(|k| (l.id, k.start))
};
if let Some(id) = opt_u64(p, "sound_layer")?.map(LayerId) {
let l = tl.layer(id).ok_or_else(|| EngineError::State(format!("no layer with id {}", id.0)))?;
return on(l).ok_or_else(|| EngineError::State(format!("layer '{}' has no sound on its keyframes", l.name)));
}
let here = |stream: bool| tl.layers.iter().find_map(|l| l.keyframe_at(frame).filter(|k| sounding(k).is_some_and(|f| !stream || f.sync == SoundSync::Stream)).map(|k| (l.id, k.start)));
here(true)
.or_else(|| here(false))
.or_else(|| tl.layers.iter().find_map(|l| on(l)))
.ok_or_else(|| EngineError::State("there is no sound on this timeline: import one (File › Import) or put one on a keyframe first".into()))
}
/// The sound the params name and the frames to read (`from`, `to`; default the frames it is
/// heard on from its keyframe, within the timeline).
pub fn source(s: &Session, p: &Value) -> Result<Source> {
let (layer, keyframe) = find_sound(s, p)?;
let tl = s.timeline();
let k = tl.layer(layer).and_then(|l| l.keyframe_starting_at(keyframe).and_then(|i| l.keyframes.get(i))).ok_or_else(|| EngineError::State("the sound's keyframe went away".into()))?;
let fs = sounding(k).ok_or_else(|| EngineError::State("the keyframe has no sound".into()))?;
let sound = s.doc.sounds.iter().find(|x| x.id == fs.sound).cloned().ok_or_else(|| EngineError::State("the keyframe's sound is no longer in the library".into()))?;
let fps = s.doc.stage.fps;
let rate = sound.rate.max(1);
// The timeline's last frame, at least the sound keyframe's own.
let last = tl.frame_count().max(k.end()).saturating_sub(1).min(MAX_FRAME);
let rest = u64::from(last - keyframe.min(last) + 1);
let heard = fs.heard(sound.frames, frame_start(u64::from(k.duration), fps, rate), frame_start(rest, fps, rate));
// Movie frames it is heard on (the last one may be partly silent).
let per = (f64::from(rate) / f64::from(if fps.is_finite() && fps > 0.0 { fps } else { 24.0 })).max(1e-9);
let heard_frames = ((heard as f64 / per).ceil() as u64).clamp(1, rest) as u32;
let from = opt_u32(p, "from")?.unwrap_or(keyframe);
let to = opt_u32(p, "to")?.unwrap_or(keyframe + heard_frames - 1);
if from < keyframe {
return Err(EngineError::Param(format!("'from' must not be before the sound's keyframe (frame {keyframe})")));
}
if to < from {
return Err(EngineError::Param("'to' must not be before 'from'".into()));
}
if to > MAX_FRAME {
return Err(EngineError::Param(format!("'to' must be at most {MAX_FRAME}")));
}
if (to - from) as usize >= MAX_ANALYSIS_FRAMES {
return Err(EngineError::Param(format!("at most {MAX_ANALYSIS_FRAMES} frames are analysed at once")));
}
// A loop has no end of its own: it sounds for as long as its span (or the movie) lets it.
let span = Span { fps, offset: from - keyframe, frames: to - from + 1, heard: if fs.looping { u64::MAX } else { heard } };
Ok(Source { layer, keyframe, sound, from, to, span })
}
fn pcm(src: &Source) -> Result<Arc<anicraft_doc::audio::Pcm>> {
src.sound.decoded().map(|d| d.pcm.clone()).ok_or_else(|| EngineError::State(format!("'{}' can't be decoded", src.sound.name)))
}
/// How the loudness becomes a value.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct LevelSettings {
pub threshold: f32,
pub smoothing: u32,
pub channel: Channel,
}
impl Default for LevelSettings {
fn default() -> Self {
LevelSettings { threshold: DEFAULT_THRESHOLD, smoothing: DEFAULT_SMOOTHING, channel: Channel::Both }
}
}
fn threshold_param(p: &Value, default: f32) -> Result<f32> {
let t = opt_f32(p, "threshold")?.unwrap_or(default);
if !(0.0..=0.99).contains(&t) {
return Err(EngineError::Param("param 'threshold' must be between 0 and 0.99 (a share of the loudest frame)".into()));
}
Ok(t)
}
fn level_settings(p: &Value) -> Result<LevelSettings> {
let smoothing = opt_u32(p, "smoothing")?.unwrap_or(DEFAULT_SMOOTHING);
if smoothing > MAX_SMOOTHING {
return Err(EngineError::Param(format!("param 'smoothing' must be between 0 and {MAX_SMOOTHING} frames")));
}
let channel = match opt_str(p, "channel")? {
None => Channel::Both,
Some(c) => Channel::from_key(c).ok_or_else(|| EngineError::Param("param 'channel' must be both, left or right".into()))?,
};
Ok(LevelSettings { threshold: threshold_param(p, DEFAULT_THRESHOLD)?, smoothing, channel })
}
fn lip_options(p: &Value) -> Result<LipOptions> {
let hold = opt_u32(p, "hold")?.unwrap_or(LipOptions::default().hold);
if !(1..=MAX_HOLD).contains(&hold) {
return Err(EngineError::Param(format!("param 'hold' must be between 1 and {MAX_HOLD} frames")));
}
Ok(LipOptions { threshold: threshold_param(p, LipOptions::default().threshold)?, hold })
}
/// A sound's loudness frame by frame, as Convert Audio to Keyframes reads it.
#[derive(Clone, Debug, PartialEq)]
pub struct Levels {
/// The first frame analysed.
pub from: u32,
/// Each frame's loudness as a share of the loudest (`0..=1`).
pub raw: Vec<f32>,
/// The same gated by the threshold and smoothed: what is mapped onto the property.
pub shaped: Vec<f32>,
}
/// The loudness of `src` under `set`.
pub fn levels(src: &Source, set: &LevelSettings) -> Result<Levels> {
let raw = analysis::normalised(&analysis::levels(&*pcm(src)?, &src.span, set.channel));
let shaped = analysis::shape(&raw, set.threshold, set.smoothing);
Ok(Levels { from: src.from, raw, shaped })
}
/// The mouth shape of each frame of `src`.
pub fn visemes(src: &Source, opts: &LipOptions) -> Result<Vec<Viseme>> {
Ok(analysis::visemes(&*pcm(src)?, &src.span, opts))
}
/// The frames that get a key when `values` (one per frame) are held between keys: every
/// `step`-th frame whose value differs from the last key's by more than `eps`, the first always.
pub fn held_keys(values: &[f32], step: u32, eps: f32) -> Vec<(u32, f32)> {
let mut out: Vec<(u32, f32)> = Vec::new();
for (i, &v) in values.iter().enumerate().step_by(step.max(1) as usize) {
if out.last().is_none_or(|(_, last)| (v - last).abs() > eps) {
out.push((i as u32, v));
}
}
out
}
/// The frames that get a key when the value travels in straight lines between keys: every
/// `step`-th frame, less the insides of flat stretches (their two ends stay, so a flat stretch
/// stays flat).
pub fn curve_keys(values: &[f32], step: u32, eps: f32) -> Vec<(u32, f32)> {
let sampled: Vec<(u32, f32)> = values.iter().enumerate().step_by(step.max(1) as usize).map(|(i, &v)| (i as u32, v)).collect();
let mut out: Vec<(u32, f32)> = Vec::new();
for (i, &(f, v)) in sampled.iter().enumerate() {
let moved = out.last().is_none_or(|(_, last)| (v - last).abs() > eps);
let turns = sampled.get(i + 1).is_none_or(|(_, next)| (next - v).abs() > eps);
if moved || turns {
out.push((f, v));
}
}
out
}
/// Differences smaller than this share of the range don't make a key.
const KEY_EPS: f32 = 0.002;
/// The layer and object that get the keys: `layer` (default the active one) and on it
/// `element`, else the selected object, else the only one on the keyframe at `frame`.
fn target(s: &Session, p: &Value, frame: u32) -> Result<(LayerId, Arc<Element>)> {
let lid = opt_u64(p, "layer")?.map_or(s.layer, LayerId);
let layer = s.timeline().layer(lid).ok_or_else(|| EngineError::State(format!("no layer with id {}", lid.0)))?;
match layer.kind {
LayerKind::Folder => return Err(EngineError::State(format!("'{}' is a folder; pick the layer with the object to animate", layer.name))),
LayerKind::Armature => return Err(EngineError::State(format!("'{}' is an armature layer: its poses place its objects, so sound can't key them yet", layer.name))),
LayerKind::Camera => return Err(EngineError::State("the camera can't be driven by sound yet; pick a layer with artwork".into())),
_ => {}
}
// A layer that ends before the frame shows nothing there, but its object is the one meant:
// the keys carry it on over the sound.
let k = layer.keyframe_at(frame).or_else(|| layer.keyframes.iter().rev().find(|k| k.start <= frame).map(|k| &**k)).ok_or_else(|| EngineError::State(format!("layer '{}' is empty at frame {}: put the object to animate there first", layer.name, frame + 1)))?;
if k.elements.is_empty() {
return Err(EngineError::State(format!("layer '{}' has nothing at frame {} to animate", layer.name, frame + 1)));
}
let pick = |id: ElementId| k.elements.iter().find(|e| e.id == id).cloned();
if let Some(id) = opt_u64(p, "element")?.map(ElementId) {
return pick(id).map(|e| (lid, e)).ok_or_else(|| EngineError::State(format!("no object with id {} on layer '{}' at frame {}", id.0, layer.name, frame + 1)));
}
let selected: Vec<Arc<Element>> = s.selection.on(lid).into_iter().filter_map(pick).collect();
match (selected.as_slice(), k.elements.as_slice()) {
([one], _) | ([], [one]) => Ok((lid, one.clone())),
_ => Err(EngineError::State(format!("layer '{}' has {} objects at frame {}: select the one to animate, or pass 'element'", layer.name, k.elements.len(), frame + 1))),
}
}
/// Whether the target layer has a motion tween anywhere on `from..=to`.
fn motion_in(layer: &Layer, from: u32, to: u32) -> bool {
layer.keyframes.iter().any(|k| k.motion.is_some() && k.start <= to && k.end() > from)
}
/// Writes held keys for object `id` on `layer`: a keyframe at each of `keys`' frames (made as
/// Insert Keyframe makes it, holding what the frame showed), then every keyframe on
/// `from..=to` gets the object as `apply` makes it from what it was and the value held there.
/// The frame after `to`, when the layer goes on, keeps what it showed. How many keyframes hold
/// the object changed.
fn write_held<T>(layer: &mut Layer, id: ElementId, from: u32, to: u32, keys: &[(u32, T)], apply: &dyn Fn(&Element, &T) -> Element) -> Result<usize> {
if keys.len() > MAX_KEYS {
return Err(EngineError::Param(format!("that is {} keyframes, more than {MAX_KEYS} at once: raise 'step' or the threshold, or shorten the range with 'from' and 'to'", keys.len())));
}
if motion_in(layer, from, to) {
return Err(EngineError::State(format!("layer '{}' has a motion tween on these frames: keyframes would cut it up. Use the motion tween's curve (output: motion), or remove the tween first", layer.name)));
}
// First every keyframe, while each frame still shows what it showed.
let after = to.checked_add(1).filter(|f| *f < layer.end_frame());
for f in keys.iter().map(|(f, _)| *f).chain(after) {
if layer.keyframe_index_at(f).is_none_or(|i| layer.keyframes.get(i).is_some_and(|k| k.start != f)) {
crate::cmds_timeline::insert_keyframe_at(layer, f, true)?;
}
}
// Then the values: each keyframe in the range holds the value of the last key at or
// before it (keyframes that were there already included).
let first = layer.keyframes.partition_point(|k| k.start < from);
let count = layer.keyframes.iter().skip(first).take_while(|k| k.start <= to).count();
let mut changed = 0;
for i in first..first + count {
let Some(k) = layer.keyframe_mut(i) else { continue };
let Some(value) = keys.partition_point(|(f, _)| *f <= k.start).checked_sub(1).and_then(|j| keys.get(j)).map(|(_, v)| v) else { continue };
if let Some(slot) = k.elements.iter_mut().find(|e| e.id == id) {
let next = apply(slot, value);
if next != **slot {
*slot = Arc::new(next);
}
changed += 1;
}
}
// The object is there for the whole of the sound: a layer that ended sooner goes on to
// the last frame keyed.
if layer.end_frame() <= to
&& let Some(i) = layer.keyframes.len().checked_sub(1)
&& let Some(k) = layer.keyframe_mut(i)
{
k.duration = (to - k.start).saturating_add(1);
}
if changed == 0 {
return Err(EngineError::State("the object isn't on the layer's keyframes in that range".into()));
}
Ok(changed)
}
/// The point of `el`'s own space that scale and rotation keys hold still: its transformation
/// point when it has one, a symbol's registration point, the middle of anything else.
fn anchor(s: &Session, el: &Element) -> Vec2 {
if let Some(p) = el.pivot() {
return p;
}
if matches!(el.kind, ElementKind::Symbol { .. }) {
return Vec2::ZERO;
}
let b = el.local_bounds(&s.doc);
let c = b.center();
if b.is_empty() || !c.is_finite() { Vec2::ZERO } else { c }
}
/// `base` with `prop` at value `v` (see [`KeyProperty`]), scaled and turned about `anchor`.
fn with_value(base: &Element, prop: KeyProperty, v: f32, anchor: Vec2) -> Element {
let mut el = base.clone();
let about_local = |m: Affine| base.transform.mul(&Affine::translate(anchor.x, anchor.y)).mul(&m).mul(&Affine::translate(-anchor.x, -anchor.y));
let scale = |pct: f32| {
let k = pct / 100.0;
if k.abs() < 1e-4 { 1e-4 } else { k }
};
let t = match prop {
KeyProperty::Scale => about_local(Affine::scale(scale(v), scale(v))),
KeyProperty::ScaleX => about_local(Affine::scale(scale(v), 1.0)),
KeyProperty::ScaleY => about_local(Affine::scale(1.0, scale(v))),
KeyProperty::X => Affine::translate(v, 0.0).mul(&base.transform),
KeyProperty::Y => Affine::translate(0.0, v).mul(&base.transform),
KeyProperty::Rotation => {
let at = base.transform.apply(anchor);
Affine::translate(at.x, at.y).mul(&Affine::rotate(v.to_radians())).mul(&Affine::translate(-at.x, -at.y)).mul(&base.transform)
}
KeyProperty::Alpha => {
el.color = base.color.with_alpha((v / 100.0).clamp(0.0, 1.0)).sanitized();
base.transform
}
KeyProperty::FirstFrame => {
show_frame(&mut el, v.round().clamp(0.0, MAX_FRAME as f32) as u32);
base.transform
}
};
if t.is_finite() {
el.transform = t;
}
el
}
/// Makes a symbol instance show `frame` of its symbol and hold it.
fn show_frame(el: &mut Element, frame: u32) {
if let ElementKind::Symbol { loop_mode, first_frame, .. } = &mut el.kind {
*loop_mode = LoopMode::SingleFrame;
*first_frame = frame.min(MAX_FRAME);
}
}
/// The graphic symbol `el` is an instance of.
fn graphic<'a>(s: &'a Session, el: &Element, what: &str) -> Result<&'a Symbol> {
let sym = match &el.kind {
ElementKind::Symbol { symbol, .. } => s.doc.symbol(*symbol),
_ => None,
};
sym.filter(|x| x.kind == SymbolKind::Graphic).ok_or_else(|| EngineError::State(format!("{what} needs an instance of a graphic symbol (one frame per pose): convert the artwork with Modify › Convert to Symbol, type Graphic")))
}
/// The settings of a `sound.to_keyframes` run, read and checked.
struct Plan {
prop: KeyProperty,
min: f32,
max: f32,
step: u32,
output: Option<Output>,
}
fn plan(p: &Value) -> Result<Plan> {
let all = || KeyProperty::ALL.map(KeyProperty::key).join(", ");
let prop = match opt_str(p, "property")? {
Some(k) => KeyProperty::from_key(k).ok_or_else(|| EngineError::Param(format!("unknown property '{k}' ({})", all())))?,
None => return Err(EngineError::Param(format!("param 'property' is required ({})", all()))),
};
let r = prop.range();
let bound = |key: &str| -> Result<f32> {
let v = f32_req(p, key)?;
if !r.contains(&v) {
return Err(EngineError::Param(format!("param '{key}' must be between {} and {} for {}", r.start(), r.end(), prop.key())));
}
Ok(v)
};
let step = opt_u32(p, "step")?.unwrap_or(1);
if !(1..=MAX_STEP).contains(&step) {
return Err(EngineError::Param(format!("param 'step' must be between 1 and {MAX_STEP} frames")));
}
let output = match opt_str(p, "output")? {
None | Some("auto") => None,
Some("keyframes") => Some(Output::Keyframes),
Some("motion") => Some(Output::Motion),
Some(other) => return Err(EngineError::Param(format!("unknown output '{other}' (auto, keyframes or motion)"))),
};
Ok(Plan { prop, min: bound("min")?, max: bound("max")?, step, output })
}
/// Convert Audio to Keyframes.
fn to_keyframes(s: &mut Session, p: &Value) -> Result<Value> {
let plan = plan(p)?;
let src = source(s, p)?;
let set = level_settings(p)?;
let (lid, el) = target(s, p, src.from)?;
if plan.prop == KeyProperty::FirstFrame {
let frames = graphic(s, &el, "First frame")?.timeline.frame_count().max(1);
if plan.min >= frames as f32 || plan.max >= frames as f32 {
return Err(EngineError::Param(format!("the symbol has {frames} frames: 'min' and 'max' must be between 0 and {}", frames - 1)));
}
}
let lv = levels(&src, &set)?;
let values: Vec<f32> = lv.shaped.iter().map(|l| plan.min + (plan.max - plan.min) * l).map(|v| if plan.prop == KeyProperty::FirstFrame { v.round() } else { v }).collect();
let eps = ((plan.max - plan.min).abs() * KEY_EPS).max(1e-6);
let layer = s.timeline().layer(lid).ok_or_else(|| EngineError::State("the layer went away".into()))?;
let in_motion = layer.keyframe_at(src.from).is_some_and(|k| k.motion.is_some());
let output = plan.output.unwrap_or(if in_motion && plan.prop != KeyProperty::FirstFrame { Output::Motion } else { Output::Keyframes });
let (keys, to) = match output {
Output::Keyframes => {
let keys: Vec<(u32, f32)> = held_keys(&values, plan.step, eps).into_iter().map(|(i, v)| (src.from + i, v)).collect();
let at = anchor(s, &el);
let prop = plan.prop;
let layer = s.layer_mut(Some(lid))?;
write_held(layer, el.id, src.from, src.to, &keys, &|base, v| with_value(base, prop, *v, at))?;
(keys.len(), src.to)
}
Output::Motion => write_curve(s, lid, &el, &plan, &src, &values, eps)?,
};
Ok(json!({"keys": keys, "from": src.from, "to": to, "output": output.key(), "property": plan.prop.key(), "layer": lid.0, "element": el.id.0, "sound": src.sound.id.0}))
}
/// Writes `values` (one per frame from `src.from`) as the property's keys in the motion tween
/// on the span at `src.from`, making the tween when the span has none. The range is cut to the
/// span. The keys written (per track) and the last frame they reach.
fn write_curve(s: &mut Session, lid: LayerId, el: &Element, plan: &Plan, src: &Source, values: &[f32], eps: f32) -> Result<(usize, u32)> {
let tracks = plan.prop.tracks();
if tracks.is_empty() {
return Err(EngineError::Param("a symbol's first frame has no motion tween curve: use output: keyframes".into()));
}
let has = s.timeline().layer(lid).and_then(|l| l.keyframe_at(src.from)).is_some_and(|k| k.motion.is_some());
if !has {
let alone = s.timeline().layer(lid).and_then(|l| l.keyframe_at(src.from)).is_some_and(|k| k.elements.len() == 1);
if !alone {
return Err(EngineError::State("a motion tween moves one object: put the object on a layer of its own (or Create Motion Tween on it) first, or use output: keyframes".into()));
}
crate::cmds_motion::create(s, &json!({"layer": lid.0, "frame": src.from}))?;
}
let layer = s.layer_mut(Some(lid))?;
let i = layer.keyframe_index_at(src.from).ok_or_else(|| EngineError::State("the span went away".into()))?;
let kf = layer.keyframe_mut(i).ok_or_else(|| EngineError::State("the span went away".into()))?;
let (start, last) = (kf.start, kf.duration.saturating_sub(1));
let target = kf.motion.as_ref().and_then(|m| m.target_index(&kf.elements)).and_then(|t| kf.elements.get(t)).cloned().ok_or_else(|| EngineError::State("the motion tween's object is no longer on its keyframe".into()))?;
if target.id != el.id {
return Err(EngineError::State("the motion tween on this span moves another object: pick that one, or use output: keyframes on a layer without a motion tween".into()));
}
let m = kf.motion.as_mut().ok_or_else(|| EngineError::State("no motion tween here".into()))?;
if m.path.is_some() && tracks.iter().any(|t| matches!(t, Property::X | Property::Y)) {
return Err(EngineError::State("the motion tween follows a motion path, which places the object: clear the path first, or drive another property".into()));
}
// The range, cut to the span.
let a = src.from - start;
let b = (src.to - start).min(last);
let len = (b - a + 1) as usize;
let keys = curve_keys(values.get(..len).unwrap_or(values), plan.step, eps);
if keys.len() > MAX_KEYS {
return Err(EngineError::Param(format!("that is {} keys, more than {MAX_KEYS} on one curve: raise 'step' or the threshold, or shorten the range with 'from' and 'to'", keys.len())));
}
for &prop in tracks {
let own = own_value(&target, prop).ok_or_else(|| EngineError::State("the motion tween's object has no such property".into()))?;
let r = prop.range();
// The curve's keys in the range give way to the new ones; what follows the range
// starts from the object's own value again unless it has keys of its own there.
let later = m.track(prop).is_some_and(|t| t.keys.iter().any(|k| k.frame > b));
if let Some(t) = m.track_mut(prop) {
t.keys.retain(|k| k.frame < a || k.frame > b);
}
for &(f, v) in &keys {
m.set_key(prop, a + f, plan.prop.track_value(own, v).clamp(*r.start(), *r.end()), own);
}
if !later && b < last {
m.set_key(prop, b + 1, own, own);
}
}
Ok((keys.len(), start + b))
}
/// The names a mouth shape's frame goes by: the Lip-sync mouth chart template's labels first
/// (ADR 0065), then other common ones. Compared without case, spaces or punctuation.
fn mouth_names(v: Viseme) -> &'static [&'static str] {
match v {
Viseme::Rest => &["rest", "neutral", "silence", "sil", "idle", "x"],
Viseme::Open => &["ai", "a", "ah", "aa", "open"],
Viseme::Wide => &["e", "ee", "wide", "i", "cdgknrsthyz", "smile", "etc"],
Viseme::Round => &["o", "oh", "round", "u", "oo", "wq", "w"],
Viseme::Closed => &["mbp", "m", "closed", "bmp", "pbm", "b", "p"],
}
}
fn plain(label: &str) -> String {
label.chars().filter(|c| c.is_alphanumeric()).flat_map(char::to_lowercase).collect()
}
/// The frame of `sym` labelled `label` (without case, spaces or punctuation), the first when
/// several are.
pub fn labelled_frame(sym: &Symbol, label: &str) -> Option<u32> {
let want = plain(label);
if want.is_empty() {
return None;
}
sym.timeline.layers.iter().flat_map(|l| l.keyframes.iter()).filter(|k| plain(&k.label) == want).map(|k| k.start).min()
}
/// The frame of `sym` each mouth shape shows by default, in [`Viseme::ALL`]'s order: by the
/// symbol's frame labels; a shape without one borrows the nearest shape's frame (lips together
/// and rest stand in for each other, round and wide borrow open's), and a symbol without any
/// of the labels gives its first five frames in order.
pub fn default_mouths(sym: &Symbol) -> [u32; 5] {
let last = sym.timeline.frame_count().max(1) - 1;
let found = Viseme::ALL.map(|v| mouth_names(v).iter().find_map(|n| labelled_frame(sym, n)));
if found.iter().all(Option::is_none) {
return [0, 1, 2, 3, 4].map(|i: u32| i.min(last));
}
let [rest, open, wide, round, closed] = found;
let rest_f = rest.or(closed).unwrap_or(0);
let open_f = open.or(wide).or(round).unwrap_or(rest_f);
[rest_f, open_f, wide.unwrap_or(open_f), round.unwrap_or(open_f), closed.or(rest).unwrap_or(rest_f)].map(|f| f.min(last))
}
/// The frame of `sym` each mouth shape shows: `mouths` over the defaults.
fn mouths_param(sym: &Symbol, p: &Value) -> Result<[u32; 5]> {
let mut out = default_mouths(sym);
let frames = sym.timeline.frame_count().max(1);
let Some(m) = p.get("mouths").filter(|m| !m.is_null()) else { return Ok(out) };
let map = m.as_object().ok_or_else(|| EngineError::Param("param 'mouths' must be an object {rest, open, wide, round, closed}, each a frame of the symbol (0-based) or a frame label".into()))?;
for (key, v) in map {
let shape = Viseme::from_key(key).ok_or_else(|| EngineError::Param(format!("unknown mouth '{}' (rest, open, wide, round or closed)", key.chars().take(40).collect::<String>())))?;
let frame = match v {
Value::String(label) => labelled_frame(sym, label).ok_or_else(|| EngineError::Param(format!("'{}' has no frame labelled '{}'", sym.name, label.chars().take(60).collect::<String>())))?,
other => other.as_u64().and_then(|f| u32::try_from(f).ok()).ok_or_else(|| EngineError::Param(format!("mouth '{key}' must be a frame of the symbol (0-based) or a frame label")))?,
};
if frame >= frames {
return Err(EngineError::Param(format!("mouth '{key}': '{}' has {frames} frames (0 to {})", sym.name, frames - 1)));
}
if let Some(slot) = out.get_mut(shape.index()) {
*slot = frame;
}
}
Ok(out)
}
/// Auto Lip-Sync.
fn lip_sync(s: &mut Session, p: &Value) -> Result<Value> {
let src = source(s, p)?;
let opts = lip_options(p)?;
let (lid, el) = target(s, p, src.from)?;
let mouths = mouths_param(graphic(s, &el, "Lip-sync")?, p)?;
let shapes = visemes(&src, &opts)?;
// A key where the mouth changes.
let mut keys: Vec<(u32, Viseme)> = Vec::new();
let mut counts = [0u32; 5];
for (i, v) in shapes.iter().enumerate() {
if keys.last().is_none_or(|(_, last)| last != v) {
keys.push((src.from + i as u32, *v));
if let Some(n) = counts.get_mut(v.index()) {
*n += 1;
}
}
}
let layer = s.layer_mut(Some(lid))?;
write_held(layer, el.id, src.from, src.to, &keys, &|base, v| {
let mut el = base.clone();
show_frame(&mut el, mouths.get(v.index()).copied().unwrap_or(0));
el
})?;
let named = |list: [u32; 5]| Value::Object(Viseme::ALL.iter().zip(list).map(|(v, n)| (v.key().to_string(), json!(n))).collect());
Ok(json!({"keys": keys.len(), "from": src.from, "to": src.to, "layer": lid.0, "element": el.id.0, "sound": src.sound.id.0, "mouths": named(mouths), "counts": named(counts)}))
}
/// Analyse Sound: what the two commands above work from.
fn analyse(s: &mut Session, p: &Value) -> Result<Value> {
let src = source(s, p)?;
let mut out = json!({"sound": src.sound.id.0, "name": src.sound.name, "layer": src.layer.0, "keyframe": src.keyframe, "from": src.from, "to": src.to, "fps": s.doc.stage.fps});
let round = |v: &f32| json!((f64::from(*v) * 10_000.0).round() / 10_000.0);
match opt_str(p, "what")?.unwrap_or("levels") {
"levels" => {
let lv = levels(&src, &level_settings(p)?)?;
out["levels"] = lv.raw.iter().map(round).collect();
out["values"] = lv.shaped.iter().map(round).collect();
}
"visemes" => out["visemes"] = visemes(&src, &lip_options(p)?)?.iter().map(|v| json!(v.key())).collect(),
other => return Err(EngineError::Param(format!("unknown 'what' '{}' (levels or visemes)", other.chars().take(40).collect::<String>()))),
}
Ok(out)
}
+5 -1
View File
@@ -47,6 +47,7 @@ mod cmds_script;
pub mod cmds_plugin;
mod cmds_scene;
pub mod cmds_sound;
pub mod cmds_audio_keys;
pub mod cmds_text;
mod cmds_timeline;
mod cmds_view;
@@ -191,6 +192,7 @@ pub fn commands() -> Vec<CommandSpec> {
v.extend(cmds_shape::specs());
v.extend(cmds_plugin::specs());
v.extend(cmds_sound::specs());
v.extend(cmds_audio_keys::specs());
v.extend(cmds_actions::specs());
v.extend(cmds_script::specs());
v.extend(cmds_macro::specs());
@@ -233,7 +235,7 @@ pub use cmds_layer::LAYER_COLORS;
/// The command namespaces that change a timeline's layers and frames, refused while a group is
/// open for editing (ADR 0073): a group is one layer and one frame. A name ending in `.` is a
/// namespace, any other one command.
const GROUP_REFUSES: &[&str] = &["timeline.", "layer.", "camera.", "bone.", "motion.", "tween.", "frames.", "sound.place", "script.set", "script.clear"];
const GROUP_REFUSES: &[&str] = &["timeline.", "layer.", "camera.", "bone.", "motion.", "tween.", "frames.", "sound.place", "sound.to_keyframes", "sound.lip_sync", "script.set", "script.clear"];
/// Last addressable frame (0-based); about 70 minutes at 24 fps.
pub const MAX_FRAME: u32 = 100_000;
@@ -1397,6 +1399,8 @@ mod plugin_kinds_tests;
#[cfg(test)]
mod tests_sound;
#[cfg(test)]
mod tests_audio_keys;
#[cfg(test)]
mod hint_tests;
#[cfg(test)]
mod tests_actions;
+9
View File
@@ -643,6 +643,15 @@ fn panic_hunt_every_command() {
json!({"sound": SOUND, "frame": 3, "extend": true, "name": "renamed"}),
json!({"sound": u64::MAX, "repeat": 1e9}),
json!({"sound": "a", "sync": 3}),
// Audio to keyframes and lip-sync (ADR 0114): every property at its limits, ranges
// past the sound, mouths that aren't frames, on whatever layer and object is there.
json!({"property": "scale", "min": 1, "max": 10_000, "threshold": 0, "smoothing": 24, "step": 1, "output": "keyframes", "to": 99_999}),
json!({"property": "rotation", "min": -3600, "max": 3600, "output": "motion", "channel": "left", "step": 240, "layer": 2}),
json!({"property": "first_frame", "min": 0, "max": 100_000, "threshold": 0.99, "from": 0, "to": 0, "mouths": {"rest": u64::MAX, "open": "nope"}}),
json!({"property": "alpha", "min": 100, "max": 0, "sound_layer": u64::MAX, "sound_frame": u64::MAX, "hold": 12, "what": "visemes"}),
json!({"property": "x", "min": -1e5, "max": 1e5, "from": 5, "to": 2, "element": u64::MAX, "mouths": [1, 2], "hold": 0, "what": 7}),
json!({"property": "y", "min": f64::NAN, "max": 1e30, "threshold": -1, "smoothing": 1e9, "channel": "middle", "step": 0, "output": "film", "mouths": {"grin": 1}}),
json!({"property": "scale_x", "min": 100, "max": 150, "sound_layer": 1, "sound_frame": 0, "layer": 1, "from": 0, "to": 100_000, "hold": 2, "mouths": {"closed": 0, "wide": 0}}),
// Effects, envelopes and layer volumes (ADR 0039).
json!({"effect": "custom", "envelope": [{"at": 1e300, "left": 2, "right": -1}], "volume": 1e30, "muted": "x"}),
json!({"effect": "fade_in", "envelope": (0..40).map(|i| json!([i, 0.5, 0.5])).collect::<Vec<_>>(), "volume": -0.5}),
+438
View File
@@ -0,0 +1,438 @@
//! Convert Audio to Keyframes and auto lip-sync end to end (ADR 0114): a synthetic sound on
//! the timeline, the keys it writes on another layer, undo, and what is refused.
use super::*;
use crate::cmds_audio_keys::{self, KeyProperty, curve_keys, default_mouths, held_keys};
use anicraft_doc::{ElementKind, LayerId, LoopMode, Property};
use serde_json::json;
use std::f32::consts::TAU;
const RATE: u32 = 24_000;
const FPS: f32 = 24.0;
fn run(s: &mut Session, id: &str, p: Value) -> Value {
match s.execute(id, &p) {
Ok(v) => v,
Err(e) => panic!("{id} {p}: {e}"),
}
}
fn fails(s: &mut Session, id: &str, p: Value) -> String {
match s.execute(id, &p) {
Ok(v) => panic!("{id} {p} should fail, gave {v}"),
Err(e) => e.to_string(),
}
}
/// Samples in `frames` movie frames: a thousand a frame, and a 480 Hz tone is twenty whole
/// cycles of one, so every frame of a steady tone is exactly as loud as the next.
fn samples(frames: u32) -> usize {
(f64::from(frames) * f64::from(RATE) / f64::from(FPS)).round() as usize
}
fn silence(frames: u32) -> Vec<f32> {
vec![0.0; samples(frames)]
}
/// `frames` movie frames of sines: (Hz, amplitude) summed.
fn tone(parts: &[(f32, f32)], frames: u32) -> Vec<f32> {
(0..samples(frames)).map(|i| parts.iter().map(|(f, a)| a * (TAU * f * i as f32 / RATE as f32).sin()).sum()).collect()
}
/// Twelve silent frames, a six-frame burst, six silent frames.
fn burst() -> Vec<f32> {
[silence(12), tone(&[(480.0, 0.5)], 6), silence(6)].concat()
}
/// Imports `sound` to the stage as a stream on a layer of its own; that layer's id.
fn add_sound(s: &mut Session, tag: &str, sound: &[f32]) -> u64 {
let dir = std::env::temp_dir().join(format!("anicraft-audio-keys-{tag}-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let wav = dir.join("voice.wav");
std::fs::write(&wav, crate::cmds_sound::test_wav(RATE, sound)).unwrap();
let v = run(s, "file.import", json!({"path": wav.display().to_string()}));
let layer = v["layer"].as_u64().expect("the sound's layer");
run(s, "sound.set", json!({"layer": layer, "frame": 0, "sync": "stream"}));
let _ = std::fs::remove_dir_all(&dir);
layer
}
/// A 24 fps document with a 100 px square (a drawing object) at 100, 100 on its first layer,
/// and `sound` as a stream on a layer above. `(session, art layer, square, sound layer)`.
fn stage(tag: &str, sound: &[f32]) -> (Session, u64, u64, u64) {
let mut s = Session::default();
run(&mut s, "doc.set_stage", json!({"fps": FPS}));
run(&mut s, "draw.rect", json!({"x": 100, "y": 100, "w": 100, "h": 100, "fill": "#3366cc", "stroke": null, "object": true}));
let art = s.layer.0;
let square = s.selection.first().expect("the square is selected").1.0;
let sound = add_sound(&mut s, tag, sound);
(s, art, square, sound)
}
fn key_starts(s: &Session, layer: u64) -> Vec<u32> {
s.timeline().layer(LayerId(layer)).expect("the layer").keyframes.iter().map(|k| k.start).collect()
}
/// The square's width at `frame`.
fn width_at(s: &mut Session, layer: u64, id: u64, frame: u32) -> f32 {
s.set_frame(frame);
s.element_bounds_on(LayerId(layer), anicraft_doc::ElementId(id)).width()
}
/// A sine burst gives keys exactly where it sounds: one where it starts, one where it stops.
#[test]
fn a_burst_keys_the_scale_exactly_where_it_sounds() {
let (mut s, art, square, sound) = stage("burst", &burst());
let before = s.doc.clone();
let v = run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 200, "threshold": 0.1, "smoothing": 0}));
assert_eq!((v["keys"].as_u64(), v["from"].as_u64(), v["to"].as_u64(), v["output"].as_str()), (Some(3), Some(0), Some(23), Some("keyframes")));
assert_eq!((v["layer"].as_u64(), v["element"].as_u64(), v["property"].as_str()), (Some(art), Some(square), Some("scale")));
assert_eq!(key_starts(&s, art), vec![0, 12, 18], "a key where the burst starts and where it ends");
assert_eq!(key_starts(&s, sound), vec![0], "the sound's layer is untouched");
for f in 0..24 {
let want = if (12..18).contains(&f) { 200.0 } else { 100.0 };
assert!((width_at(&mut s, art, square, f) - want).abs() < 0.01, "frame {f}");
}
// It grows about its middle.
s.set_frame(14);
let b = s.element_bounds_on(LayerId(art), anicraft_doc::ElementId(square));
assert!((b.center().x - 150.0).abs() < 0.01 && (b.center().y - 150.0).abs() < 0.01, "{b:?}");
// One undo step.
assert_eq!(s.undo_label(), Some("Convert Audio to Keyframes…"));
run(&mut s, "edit.undo", json!({}));
assert!(s.doc == before, "undo brings the document back");
}
#[test]
fn the_same_sound_gives_the_same_keys() {
let voice = [tone(&[(730.0, 0.6), (1090.0, 0.3)], 7), silence(3), tone(&[(300.0, 0.5), (870.0, 0.15)], 9), tone(&[(480.0, 0.2)], 5)].concat();
let once = |tag: &str| {
let (mut s, art, _, _) = stage(tag, &voice);
let v = run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "y", "min": 0, "max": -60, "smoothing": 2}));
let a = run(&mut s, "sound.analyse", json!({}));
(s.doc.timeline(0).layer(LayerId(art)).cloned(), v, a)
};
let (a, b) = (once("same-a"), once("same-b"));
assert!(a.0 == b.0 && a.1 == b.1 && a.2 == b.2, "same sound, same keys");
assert!(a.1["keys"].as_u64().is_some_and(|n| n > 4));
}
#[test]
fn every_property_maps_silence_to_min_and_the_loudest_frame_to_max() {
let cases: [(&str, f64, f64); 5] = [("scale_x", 100.0, 300.0), ("scale_y", 50.0, 100.0), ("x", 0.0, 40.0), ("y", -10.0, 30.0), ("rotation", 0.0, 90.0)];
for (prop, min, max) in cases {
let (mut s, art, square, _) = stage(&format!("prop-{prop}"), &burst());
run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": prop, "min": min, "max": max, "threshold": 0.1, "smoothing": 0}));
let at = |s: &mut Session, f: u32| {
s.set_frame(f);
s.element_bounds_on(LayerId(art), anicraft_doc::ElementId(square))
};
let (quiet, loud) = (at(&mut s, 3), at(&mut s, 14));
let near = |a: f32, b: f64| (f64::from(a) - b).abs() < 0.01;
match prop {
"scale_x" => assert!(near(quiet.width(), 100.0) && near(loud.width(), 300.0) && near(loud.height(), 100.0), "{prop}: {loud:?}"),
"scale_y" => assert!(near(quiet.height(), 50.0) && near(loud.height(), 100.0) && near(loud.width(), 100.0), "{prop}: {quiet:?}"),
"x" => assert!(near(quiet.min.x, 100.0) && near(loud.min.x, 140.0) && near(loud.min.y, 100.0), "{prop}: {loud:?}"),
"y" => assert!(near(quiet.min.y, 90.0) && near(loud.min.y, 130.0), "{prop}: {quiet:?} {loud:?}"),
// A quarter turn about its middle: the square covers itself.
_ => assert!(near(loud.min.x, 100.0) && near(loud.min.y, 100.0) && near(loud.width(), 100.0), "{prop}: {loud:?}"),
}
}
// Alpha: the colour effect, 20 % in silence and 90 % on the burst.
let (mut s, art, square, _) = stage("prop-alpha", &burst());
run(&mut s, "sound.to_keyframes", json!({"layer": art, "element": square, "property": "opacity", "min": 20, "max": 90, "threshold": 0.1, "smoothing": 0}));
let alpha = |s: &Session, f: u32| s.timeline().layer(LayerId(art)).and_then(|l| l.keyframe_at(f)).and_then(|k| k.elements.first().map(|e| e.color.alpha())).expect("the square");
assert!((alpha(&s, 5) - 0.2).abs() < 1e-5 && (alpha(&s, 13) - 0.9).abs() < 1e-5 && (alpha(&s, 20) - 0.2).abs() < 1e-5);
}
#[test]
fn smoothing_threshold_step_and_range_shape_the_keys() {
// Smoothing ramps into and out of the burst: more keys, the peak still on the burst.
let (mut s, art, square, _) = stage("shape", &burst());
let v = run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 200, "smoothing": 2}));
assert!(v["keys"].as_u64().is_some_and(|n| n > 6), "{v}");
assert!(width_at(&mut s, art, square, 9) == 100.0 && width_at(&mut s, art, square, 11) > 100.0 && (width_at(&mut s, art, square, 14) - 200.0).abs() < 0.01);
// A key every four frames: the keys fall on frames 0, 4, 8…, held in between.
run(&mut s, "edit.undo", json!({}));
run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 200, "smoothing": 0, "step": 4}));
assert_eq!(key_starts(&s, art), vec![0, 12, 20]);
// A range inside the sound: only its frames are keyed.
run(&mut s, "edit.undo", json!({}));
let v = run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 200, "smoothing": 0, "from": 14, "to": 20}));
assert_eq!((v["from"].as_u64(), v["to"].as_u64()), (Some(14), Some(20)));
assert_eq!(key_starts(&s, art), vec![0, 14, 18]);
// A threshold above the burst's own level leaves nothing to key but the first frame.
run(&mut s, "edit.undo", json!({}));
let quiet = [silence(6), tone(&[(480.0, 0.05)], 6), tone(&[(480.0, 0.5)], 6), silence(6)].concat();
let (mut s, art, _, _) = stage("gate", &quiet);
run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 200, "smoothing": 0, "threshold": 0.3}));
assert_eq!(key_starts(&s, art), vec![0, 12, 18], "the quiet part is under the threshold");
}
/// Keyframes already on the layer keep their place and take the value; the frame after the
/// sound shows what it showed before.
#[test]
fn keyframes_already_there_take_the_value_and_the_layer_goes_on_as_it_was() {
let loud_to_the_end = [silence(12), tone(&[(480.0, 0.5)], 12)].concat();
let (mut s, art, square, _) = stage("existing", &loud_to_the_end);
run(&mut s, "timeline.insert_frame", json!({"layer": art, "frame": 0, "count": 39}));
run(&mut s, "timeline.insert_keyframe", json!({"layer": art, "frame": 14}));
run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 200, "threshold": 0.1, "smoothing": 0}));
assert_eq!(key_starts(&s, art), vec![0, 12, 14, 24]);
assert!((width_at(&mut s, art, square, 14) - 200.0).abs() < 0.01, "the keyframe that was there is loud too");
assert!((width_at(&mut s, art, square, 23) - 200.0).abs() < 0.01);
assert!((width_at(&mut s, art, square, 24) - 100.0).abs() < 0.01 && (width_at(&mut s, art, square, 39) - 100.0).abs() < 0.01, "after the sound, as it was");
}
/// On a motion tween the keys go on the property's curve, and the span stays one span.
#[test]
fn a_motion_tween_takes_the_keys_on_its_curve() {
let mut s = Session::default();
run(&mut s, "doc.set_stage", json!({"fps": FPS}));
run(&mut s, "draw.rect", json!({"x": 100, "y": 100, "w": 100, "h": 100, "fill": "#3366cc", "stroke": null, "object": true}));
run(&mut s, "modify.convert_to_symbol", json!({"name": "Ball"}));
let art = s.layer.0;
run(&mut s, "timeline.insert_frame", json!({"frame": 0, "count": 29}));
run(&mut s, "timeline.create_motion_tween", json!({"frame": 0}));
add_sound(&mut s, "motion", &burst());
let v = run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "alpha", "min": 10, "max": 100, "threshold": 0.1, "smoothing": 0}));
assert_eq!((v["output"].as_str(), v["keys"].as_u64(), v["to"].as_u64()), (Some("motion"), Some(6), Some(23)));
assert_eq!(key_starts(&s, art), vec![0], "the span is not cut up");
let layer = s.timeline().layer(LayerId(art)).cloned().expect("the layer");
let track = |p: Property| layer.keyframes[0].motion.as_ref().and_then(|m| m.track(p)).map(|t| t.keys.iter().map(|k| (k.frame, k.value)).collect::<Vec<_>>());
// Flat stretches keep their two ends, and the frame after the sound returns to the
// object's own alpha.
let near = |got: Option<Vec<(u32, f32)>>, want: &[(u32, f32)]| got.is_some_and(|g| g.len() == want.len() && g.iter().zip(want).all(|(a, b)| a.0 == b.0 && (a.1 - b.1).abs() < 0.01));
assert!(near(track(Property::Alpha), &[(0, 10.0), (11, 10.0), (12, 100.0), (17, 100.0), (18, 10.0), (23, 10.0), (24, 100.0)]), "{:?}", track(Property::Alpha));
let alpha = |f: u32| anicraft_doc::evaluate_layer_elements(&layer, f).and_then(|(_, e)| e.first().map(|e| e.color.alpha())).expect("the ball");
assert!((alpha(5) - 0.1).abs() < 1e-4 && (alpha(14) - 1.0).abs() < 1e-4 && (alpha(20) - 0.1).abs() < 1e-4 && (alpha(28) - 1.0).abs() < 1e-4);
// Scale writes both scale curves, as shares of the object's own size; a second run
// replaces the keys of the first.
for _ in 0..2 {
run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 150, "threshold": 0.1, "smoothing": 0, "output": "motion"}));
}
let layer = s.timeline().layer(LayerId(art)).cloned().expect("the layer");
let track = |p: Property| layer.keyframes[0].motion.as_ref().and_then(|m| m.track(p)).map(|t| t.keys.iter().map(|k| (k.frame, k.value)).collect::<Vec<_>>());
let want = [(0, 100.0), (11, 100.0), (12, 150.0), (17, 150.0), (18, 100.0), (23, 100.0), (24, 100.0)];
assert!(near(track(Property::ScaleX), &want), "{:?}", track(Property::ScaleX));
assert!(near(track(Property::ScaleY), &want), "{:?}", track(Property::ScaleY));
// Keyframes would cut the tween up: refused, saying what to do.
let why = fails(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 150, "output": "keyframes"}));
assert!(why.contains("motion tween"), "{why}");
// Without a tween, asking for the curve makes one on a lone object.
let (mut s, art, _, _) = stage("motion-new", &burst());
run(&mut s, "timeline.insert_frame", json!({"layer": art, "frame": 0, "count": 23}));
let v = run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "rotation", "min": 0, "max": 45, "threshold": 0.1, "smoothing": 0, "output": "motion"}));
assert_eq!(v["output"].as_str(), Some("motion"));
assert!(s.timeline().layer(LayerId(art)).is_some_and(|l| l.keyframes.len() == 1 && l.keyframes[0].motion.as_ref().is_some_and(|m| m.track(Property::Rotation).is_some())));
}
/// A graphic symbol's first frame follows the loudness: louder, a later frame.
#[test]
fn the_first_frame_of_a_graphic_follows_the_loudness() {
let mut s = Session::default();
run(&mut s, "file.new_from_template", json!({"template": "lip-sync"}));
let fps = s.doc.stage.fps;
assert_eq!(fps, FPS, "the template's frame rate is the tests'");
let mouth = s.timeline().layers.iter().find(|l| l.name == "Mouth").map(|l| l.id.0).expect("the Mouth layer");
add_sound(&mut s, "first", &[silence(6), tone(&[(480.0, 0.25)], 6), tone(&[(480.0, 0.5)], 6), silence(6)].concat());
run(&mut s, "sound.to_keyframes", json!({"layer": mouth, "property": "first_frame", "min": 0, "max": 4, "smoothing": 0}));
let shown = |s: &Session, f: u32| match s.timeline().layer(LayerId(mouth)).and_then(|l| l.keyframe_at(f)).and_then(|k| k.elements.first().map(|e| e.kind.clone())) {
Some(ElementKind::Symbol { first_frame, loop_mode, .. }) => (first_frame, loop_mode),
other => panic!("{other:?}"),
};
assert_eq!(shown(&s, 2), (0, LoopMode::SingleFrame));
assert_eq!(shown(&s, 8), (2, LoopMode::SingleFrame), "half as loud, half way");
assert_eq!(shown(&s, 14), (4, LoopMode::SingleFrame));
assert_eq!(shown(&s, 20), (0, LoopMode::SingleFrame));
// A frame the symbol doesn't have, and an object that isn't a graphic symbol.
let why = fails(&mut s, "sound.to_keyframes", json!({"layer": mouth, "property": "first_frame", "min": 0, "max": 10}));
assert!(why.contains("10 frames"), "{why}");
let (mut s, art, _, _) = stage("first-shape", &burst());
let why = fails(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "first_frame", "min": 0, "max": 1}));
assert!(why.contains("graphic symbol"), "{why}");
let why = fails(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "first_frame", "min": 0, "max": 1, "output": "motion"}));
assert!(why.contains("graphic symbol"), "{why}");
}
/// The mouth chart template lip-synced to "ah", a stop, "ee", "oo": each mouth's labelled
/// frame on the frame its sound starts.
#[test]
fn the_mouth_chart_is_lip_synced_to_synthetic_speech() {
let mut s = Session::default();
run(&mut s, "file.new_from_template", json!({"template": "lip-sync"}));
let mouth = s.timeline().layers.iter().find(|l| l.name == "Mouth").map(|l| l.id.0).expect("the Mouth layer");
let speech = [silence(12), tone(&[(730.0, 0.6), (1090.0, 0.3)], 6), silence(2), tone(&[(270.0, 0.6), (2290.0, 0.15)], 6), tone(&[(300.0, 0.6), (870.0, 0.2)], 6), silence(12)].concat();
let sound = add_sound(&mut s, "lips", &speech);
let before = s.doc.clone();
// The sound's layer is active: the mouth's layer is named, and its one object found.
let v = run(&mut s, "sound.lip_sync", json!({"layer": mouth}));
assert_eq!((v["keys"].as_u64(), v["from"].as_u64(), v["to"].as_u64()), (Some(6), Some(0), Some(43)), "{v}");
// Rest, AI, E, O and MBP are the chart's frames 0, 1, 2, 3 and 8.
assert_eq!(v["mouths"], json!({"rest": 0, "open": 1, "wide": 2, "round": 3, "closed": 8}));
assert_eq!(v["counts"], json!({"rest": 2, "open": 1, "wide": 1, "round": 1, "closed": 1}));
let layer = s.timeline().layer(LayerId(mouth)).cloned().expect("the layer");
let keys: Vec<(u32, u32)> = layer
.keyframes
.iter()
.map(|k| match k.elements.first().map(|e| &e.kind) {
Some(ElementKind::Symbol { first_frame, loop_mode: LoopMode::SingleFrame, .. }) => (k.start, *first_frame),
other => panic!("{other:?}"),
})
.collect();
assert_eq!(keys, vec![(0, 0), (12, 1), (18, 8), (20, 2), (26, 3), (32, 0)]);
assert_eq!(key_starts(&s, sound), vec![0]);
assert_eq!(s.undo_label(), Some("Auto Lip-Sync…"));
run(&mut s, "edit.undo", json!({}));
assert!(s.doc == before, "one undo step");
// Mouths by label or by frame, over the defaults; a longer hold drops the short ones.
let v = run(&mut s, "sound.lip_sync", json!({"layer": mouth, "mouths": {"open": "fv", "round": 9, "closed": "L"}, "hold": 7}));
assert_eq!(v["mouths"], json!({"rest": 0, "open": 6, "wide": 2, "round": 9, "closed": 7}));
assert_eq!(v["counts"], json!({"rest": 2, "open": 0, "wide": 0, "round": 1, "closed": 1}), "six-frame mouths join their neighbour; the stop stays");
// What is refused, and why.
for (p, part) in [
(json!({"layer": mouth, "mouths": {"open": "nope"}}), "no frame labelled"),
(json!({"layer": mouth, "mouths": {"open": 10}}), "has 10 frames"),
(json!({"layer": mouth, "mouths": {"grin": 1}}), "unknown mouth"),
(json!({"layer": mouth, "mouths": [1, 2]}), "must be an object"),
(json!({"layer": mouth, "hold": 0}), "'hold'"),
(json!({"layer": mouth, "threshold": 1}), "'threshold'"),
(json!({"layer": mouth, "from": 9, "to": 3}), "'to'"),
(json!({"layer": sound}), "nothing at frame"),
(json!({"layer": mouth, "element": 999_999}), "no object with id"),
(json!({"layer": mouth, "sound_layer": mouth}), "has no sound"),
] {
let why = fails(&mut s, "sound.lip_sync", p.clone());
assert!(why.contains(part), "{p}: {why}");
}
}
#[test]
fn mouths_are_found_by_label_and_fall_back_sensibly() {
let chart = anicraft_doc::templates::TEMPLATES.iter().find(|t| t.id == "lip-sync").map(|t| (t.build)()).expect("the template");
let sym = chart.library.iter().find(|x| x.name == "Mouth").expect("the Mouth symbol");
assert_eq!(default_mouths(sym), [0, 1, 2, 3, 8]);
assert_eq!(cmds_audio_keys::labelled_frame(sym, " c-d-g-k-n-r-s-th-y-z "), Some(5));
assert_eq!(cmds_audio_keys::labelled_frame(sym, ""), None);
// No labels: the first five frames, in order; fewer frames, the last one again.
let mut bare = (**sym).clone();
for l in &mut bare.timeline.layers {
for k in &mut Arc::make_mut(l).keyframes {
Arc::make_mut(k).label.clear();
}
}
assert_eq!(default_mouths(&bare), [0, 1, 2, 3, 4]);
for l in &mut bare.timeline.layers {
Arc::make_mut(l).keyframes.truncate(3);
}
assert_eq!(default_mouths(&bare), [0, 1, 2, 2, 2]);
// Only some labelled: the others borrow the nearest mouth.
let mut some = (**sym).clone();
for l in &mut some.timeline.layers {
for k in &mut Arc::make_mut(l).keyframes {
let k = Arc::make_mut(k);
if !matches!(k.label.as_str(), "AI" | "MBP") {
k.label.clear();
}
}
}
assert_eq!(default_mouths(&some), [8, 1, 1, 1, 8], "rest borrows lips together; wide and round borrow open");
}
#[test]
fn analyse_reports_levels_and_mouths_without_changing_anything() {
let (mut s, _, _, sound) = stage("analyse", &burst());
let before = s.doc.clone();
let v = run(&mut s, "sound.analyse", json!({"smoothing": 0, "threshold": 0.1}));
assert_eq!((v["layer"].as_u64(), v["keyframe"].as_u64(), v["from"].as_u64(), v["to"].as_u64(), v["name"].as_str()), (Some(sound), Some(0), Some(0), Some(23), Some("voice.wav")));
let levels: Vec<f64> = v["levels"].as_array().expect("levels").iter().filter_map(Value::as_f64).collect();
assert_eq!(levels.len(), 24);
assert!(levels.iter().enumerate().all(|(i, l)| if (12..18).contains(&i) { *l > 0.98 } else { *l == 0.0 }), "{levels:?}");
assert_eq!(v["values"], v["levels"], "nothing under the threshold, no smoothing");
let v = run(&mut s, "sound.analyse", json!({"what": "visemes", "from": 10, "to": 13}));
assert_eq!(v["visemes"], json!(["rest", "rest", "round", "round"]), "a lone low tone reads as a round mouth");
assert!(s.doc == before && !s.can_undo() || s.undo_label() != Some("Analyse Sound"));
assert!(fails(&mut s, "sound.analyse", json!({"what": "words"})).contains("levels or visemes"));
// A timeline without a sound says so.
let mut empty = Session::default();
let why = fails(&mut empty, "sound.to_keyframes", json!({"property": "scale", "min": 100, "max": 150}));
assert!(why.contains("no sound"), "{why}");
}
#[test]
fn what_cannot_be_keyed_is_refused_with_a_reason() {
let (mut s, art, _, sound) = stage("refuse", &burst());
let before = s.doc.clone();
for (p, part) in [
(json!({"layer": art}), "'property' is required"),
(json!({"layer": art, "property": "skew", "min": 0, "max": 1}), "unknown property"),
(json!({"layer": art, "property": "scale", "min": 0, "max": 150}), "'min'"),
(json!({"layer": art, "property": "alpha", "min": 0, "max": 101}), "'max'"),
(json!({"layer": art, "property": "scale", "max": 150}), "min"),
(json!({"layer": art, "property": "scale", "min": 100, "max": 150, "step": 0}), "'step'"),
(json!({"layer": art, "property": "scale", "min": 100, "max": 150, "smoothing": 25}), "'smoothing'"),
(json!({"layer": art, "property": "scale", "min": 100, "max": 150, "channel": "centre"}), "'channel'"),
(json!({"layer": art, "property": "scale", "min": 100, "max": 150, "output": "film"}), "unknown output"),
(json!({"layer": sound, "property": "scale", "min": 100, "max": 150}), "nothing at frame"),
(json!({"layer": 999_999, "property": "scale", "min": 100, "max": 150}), "no layer"),
(json!({"layer": art, "property": "scale", "min": 100, "max": 150, "to": 200_000}), "'to'"),
] {
let why = fails(&mut s, "sound.to_keyframes", p.clone());
assert!(why.contains(part), "{p}: {why}");
}
assert!(s.doc == before, "a refused command changes nothing");
// Two objects on the keyframe: the selected one is keyed; with none selected it asks.
run(&mut s, "layer.select", json!({"layer": art}));
run(&mut s, "draw.rect", json!({"x": 400, "y": 100, "w": 50, "h": 50, "fill": "#cc3366", "stroke": null, "object": true}));
let second = s.selection.first().expect("the second square").1.0;
let v = run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 200, "threshold": 0.1, "smoothing": 0}));
assert_eq!(v["element"].as_u64(), Some(second));
assert!((width_at(&mut s, art, second, 14) - 100.0).abs() < 0.01);
run(&mut s, "edit.undo", json!({}));
run(&mut s, "edit.deselect", json!({}));
let why = fails(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 200}));
assert!(why.contains("2 objects"), "{why}");
// Plug-ins may call them (they edit the document, like the other sound commands).
for id in ["sound.to_keyframes", "sound.lip_sync", "sound.analyse"] {
assert!(crate::cmds_plugin::plugin_may_call(id).is_ok(), "{id}");
}
}
#[test]
fn keys_are_thinned_without_changing_what_is_seen() {
let v = [0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.5, 0.0, 0.0];
assert_eq!(held_keys(&v, 1, 0.001), vec![(0, 0.0), (3, 1.0), (6, 0.5), (7, 0.0)]);
assert_eq!(held_keys(&v, 3, 0.001), vec![(0, 0.0), (3, 1.0), (6, 0.5)]);
assert_eq!(curve_keys(&v, 1, 0.001), vec![(0, 0.0), (2, 0.0), (3, 1.0), (5, 1.0), (6, 0.5), (7, 0.0), (8, 0.0)]);
assert_eq!(curve_keys(&[0.4; 6], 1, 0.001), vec![(0, 0.4), (5, 0.4)], "a flat sound: its two ends");
assert!(held_keys(&[], 1, 0.1).is_empty() && curve_keys(&[], 0, 0.1).is_empty());
assert_eq!(held_keys(&[1.0, 2.0], 0, 0.0).len(), 2, "a step of zero is one");
for p in KeyProperty::ALL {
assert_eq!(KeyProperty::from_key(p.key()), Some(p));
let (min, max) = p.defaults();
assert!(p.range().contains(&min) && p.range().contains(&max), "{}", p.key());
}
}
/// How long the analysis and the keys take (run with `--ignored --nocapture`).
#[test]
#[ignore = "a timing, not a check"]
fn timing_of_a_long_sound() {
// Ten minutes of a voice-like sound at 24 kHz, 24 fps: 14 400 frames.
let n = RATE as usize * 600;
let voice: Vec<f32> = (0..n).map(|i| {
let t = i as f32 / RATE as f32;
let syllable = ((t * 4.0).fract() < 0.7) as u8 as f32;
syllable * (0.5 * (TAU * 730.0 * t).sin() + 0.2 * (TAU * (1090.0 + 600.0 * (t * 0.5).sin()) * t).sin())
}).collect();
let (mut s, art, _, _) = stage("timing", &voice);
let t = std::time::Instant::now();
let v = run(&mut s, "sound.analyse", json!({}));
eprintln!("levels of {} frames: {:?}", v["levels"].as_array().map_or(0, Vec::len), t.elapsed());
let t = std::time::Instant::now();
let v = run(&mut s, "sound.analyse", json!({"what": "visemes"}));
eprintln!("visemes of {} frames: {:?}", v["visemes"].as_array().map_or(0, Vec::len), t.elapsed());
let t = std::time::Instant::now();
let v = run(&mut s, "sound.to_keyframes", json!({"layer": art, "property": "scale", "min": 100, "max": 150, "step": 2}));
eprintln!("{} keyframes written: {:?}", v["keys"], t.elapsed());
}