feat(audio): Convert Audio to Keyframes and Auto Lip-Sync note when the sound is not Stream-synced (M8)

This commit is contained in:
Tanish Manku committed 2026-11-18 06:36:26 -07:00
1 parent f90a3c45c1
commit a8b43fd8c8
7 files changed
+65 -7

No files matched your search

+4
View File
@@ -20,6 +20,10 @@
* `anicraft --check-theme <file>` checks a theme without opening a window, and
`anicraft --theme <file>.json` uses one for a single run.
* **Convert Audio to Keyframes and Auto Lip-Sync say when a sound can drift** (ADR 0114
follow-up, M8): with a sound whose sync is Event or Start, the dialog notes that the keys
can drift from what is heard after a jump on the timeline, and that Stream keeps them locked.
* **Convert Audio to Keyframes and Auto Lip-Sync** (ADR 0114, #332, M5).
* **Convert Audio to Keyframes** (Modify › Timeline, or **To Keyframes…** in a sound
keyframe's Properties): the sound's loudness on each frame drives an object on another
@@ -113,3 +113,7 @@ back into the dialog to edit them.
Any sync mode but Stop is accepted, although the task said "from a stream sound". An Event
sound keeps time with the frames only while the movie plays straight through, so the keys can
drift from what is heard after a jump. The dialog doesn't warn about it yet.
Answered in the follow-up (with ADR 0117's PR): both dialogs now show one line under the frames
when the sound's sync is not Stream, saying the keys can drift after a jump on the timeline and
that Stream keeps them locked. They still accept the sound.
+3 -1
View File
@@ -196,7 +196,9 @@ logo that brightens on every hit, a jaw that drops when a character talks.
![Convert Audio to Keyframes: the star's scale follows a beat, previewed on the timeline](images/audio-to-keyframes.png)
1. Put the sound on a layer and the object on another layer. **Stream** sync is best: the
sound then stays on its frames.
sound then stays on its frames. With Event or Start sync the dialog still works, and a line
under **Frames** reminds you that the keys can drift from what is heard after a jump on the
timeline.
2. Choose **Modify › Timeline › Convert Audio to Keyframes…**, or select the sound's keyframe
and press **To Keyframes…** in **Properties**.
3. Pick the **Layer** that gets the keys (and the **Object**, when the layer holds several).
+1 -1
View File
@@ -443,7 +443,7 @@ mod tests {
for tool in [Tool::Bone, Tool::Bind, Tool::Warp, Tool::Rect, Tool::Star, Tool::Text, Tool::Pencil, Tool::Brush, Tool::PaintBrush, Tool::Eraser, Tool::PaintBucket, Tool::InkBottle, Tool::Eyedropper, Tool::Pen] {
found.extend(check(&format!("{tool:?} tool"), |a, _| a.tool = tool));
}
for panel in ["select", "motion", "motionspan", "motionpath", "motionpoints", "motionedit", "motionpins", "motionpinsspan", "rawsub", "camera", "cameraview", "bones", "springbones", "shapebones", "shapeweights", "mixedbones", "warp", "sound", "envelope", "audiokeys", "lipsync", "prefs", "shortcuts", "palette", "about", "files", "licenses", "confirm", "recovery", "frames", "actions", "script", "scriptproblems", "testscript", "onion", "onionmenu", "editmultiple", "library", "librarymenu", "edittransform", "onionsymbol", "layerprops", "layeropacity", "pivot", "frameview", "layercolour", "timelinepolish", "convert", "mixed", "bitmap", "nineslice", "nineslicedit", "find", "explorer", "publish", "publishreport", "publishanim", "start", "drop", "droplibrary", "testmovie", "testmovieinside", "templates", "savetemplate", "droptimeline", "textfields", "testtext", "richtext", "softmask", "pathtext", "gradient", "gradientbucket", "swatches", "swatchpanel", "brushlibrary", "brushcurve", "paintbrush"] {
for panel in ["select", "motion", "motionspan", "motionpath", "motionpoints", "motionedit", "motionpins", "motionpinsspan", "rawsub", "camera", "cameraview", "bones", "springbones", "shapebones", "shapeweights", "mixedbones", "warp", "sound", "envelope", "audiokeys", "audiokeysevent", "lipsync", "prefs", "shortcuts", "palette", "about", "files", "licenses", "confirm", "recovery", "frames", "actions", "script", "scriptproblems", "testscript", "onion", "onionmenu", "editmultiple", "library", "librarymenu", "edittransform", "onionsymbol", "layerprops", "layeropacity", "pivot", "frameview", "layercolour", "timelinepolish", "convert", "mixed", "bitmap", "nineslice", "nineslicedit", "find", "explorer", "publish", "publishreport", "publishanim", "start", "drop", "droplibrary", "testmovie", "testmovieinside", "templates", "savetemplate", "droptimeline", "textfields", "testtext", "richtext", "softmask", "pathtext", "gradient", "gradientbucket", "swatches", "swatchpanel", "brushlibrary", "brushcurve", "paintbrush"] {
found.extend(check(panel, |a, _| a.open_panel(Some(panel))));
}
// Every panel in front where it docks, and floating (ADR 0075); the other workspaces.
+3 -2
View File
@@ -2628,11 +2628,12 @@ impl App {
self.run("timeline.goto", json!({"frame": 14}));
}
}
Some("audiokeys") => {
Some(which @ ("audiokeys" | "audiokeysevent")) => {
// Convert Audio to Keyframes (ADR 0114): the beat as a stream, its dialog open
// on the demo's star, whose scale follows the beat (previewed on the timeline).
// `audiokeysevent`: as an Event sound, with the dialog's note that keys can drift.
if self.import_demo_beat() {
self.run("sound.set", json!({"frame": 0, "sync": "stream"}));
self.run("sound.set", json!({"frame": 0, "sync": if which == "audiokeysevent" { "event" } else { "stream" }}));
let star = self.s.timeline().layers.iter().find(|l| l.name == "Star").map(|l| l.id.0);
crate::audio_keys::open(self, crate::audio_keys::Kind::Keys);
if let Some(id) = star {
+47
View File
@@ -164,6 +164,13 @@ fn sounds(app: &App) -> Vec<((u64, u32), String)> {
out
}
/// Whether the sound the dialog reads is not a Stream sound (ADR 0114): it keeps time with the
/// frames only while the movie plays straight through, so the dialog says so.
fn can_drift(app: &App, sound: (u64, u32)) -> bool {
let key = app.s.timeline().layers.iter().find(|l| l.id.0 == sound.0).and_then(|l| l.keyframes.iter().find(|k| k.start == sound.1));
key.and_then(|k| k.sound.as_ref()).is_some_and(|f| f.sync != anicraft_engine::doc::SoundSync::Stream)
}
/// The sound the dialog reads, as the engine finds it.
fn source(app: &App, sound: (u64, u32)) -> Option<Source> {
keys::source(&app.s, &json!({"sound_layer": sound.0, "sound_frame": sound.1})).ok()
@@ -428,6 +435,7 @@ pub fn show(app: &mut App, ctx: &egui::Context) {
let sound_list = sounds(app);
let symbol = mouth_symbol(app, &d).map(|s| (s.timeline.frame_count().max(1), s.clone()));
let stereo = src.sound.channels == 2;
let drifts = can_drift(app, d.sound);
let mut changed = false;
let (mut ok, mut cancel) = (false, false);
let mut retarget = false;
@@ -466,6 +474,10 @@ pub fn show(app: &mut App, ctx: &egui::Context) {
form_row(ui, FORM_LABEL_DIALOG, tr!("Frames"), |ui| {
ui.label(text(Type::Data, tr!("{from} to {to}", from = src.from + 1, to = src.to + 1)));
});
if drifts {
// The keys are right for the frames; what is heard follows them only with Stream.
theme::form_note(ui, FORM_LABEL_DIALOG, text(Type::Caption, tr!("After a timeline jump the keys can drift from this sound. Stream sync keeps them locked.")));
}
form_row(ui, FORM_LABEL_DIALOG, tr!("Layer"), |ui| {
if let Some(l) = choice(ui, "audio_keys_layer", tr!("Layer that gets the keys"), d.layer, &layer_list, tr!("The layer whose object the sound drives: the keys are written on it")) {
d.layer = l;
@@ -836,6 +848,41 @@ mod tests {
assert!(app.s.doc == before, "one undo step");
}
/// ADR 0114's follow-up: both dialogs say when the sound isn't a Stream sound, and say
/// nothing when it is.
#[test]
fn a_sound_that_is_not_stream_synced_gets_a_note_about_drift() {
let ctx = egui::Context::default();
ctx.enable_accesskit();
let (mut app, _) = app(&ctx, "drift");
let sound = app.s.layer.0;
let noted = |app: &mut App, ctx: &egui::Context| {
let mut out = ctx.run_ui(egui::RawInput::default(), |ui| show(app, ui.ctx()));
out.textures_delta.clear();
out.platform_output.accesskit_update.is_some_and(|t| t.nodes.iter().any(|(_, n)| n.value().is_some_and(|v| v.contains("Stream sync keeps them locked"))))
};
for kind in [Kind::Keys, Kind::Lips] {
app.run("sound.set", json!({"frame": 0, "sync": "stream"}));
open(&mut app, kind);
// Auto Lip-Sync opens only with a graphic symbol to drive; the note is in the part
// of the dialog both share, so Convert Audio to Keyframes covers it here.
let Some(at) = app.audio_keys.as_ref().map(|d| d.sound) else {
assert_eq!(kind, Kind::Lips, "Convert Audio to Keyframes opens on the square");
continue;
};
assert_eq!(at.0, sound);
assert!(!can_drift(&app, at) && !noted(&mut app, &ctx), "{kind:?}: a stream sound stays in time");
cancel(&mut app);
for sync in ["event", "start"] {
app.run("sound.set", json!({"frame": 0, "sync": sync}));
open(&mut app, kind);
assert!(can_drift(&app, at) && noted(&mut app, &ctx), "{kind:?}, {sync}");
cancel(&mut app);
}
}
assert!(!can_drift(&app, (9999, 0)), "no such sound: nothing to say");
}
#[test]
fn cancel_takes_the_preview_back_and_leaves_no_step() {
let ctx = egui::Context::default();
+3 -3
View File
@@ -450,9 +450,9 @@ mod tests {
/// gradient accent bars, and nothing about it is noted as low in contrast.
#[test]
fn the_guides_sample_theme_is_a_good_theme() {
let chapter = include_str!("../docs/guide/preferences-shortcuts-accessibility.md");
let sample = chapter.split("```json
").nth(1).and_then(|s| s.split("```").next()).expect("the chapter has a sample theme");
// (Whatever line endings the checkout gave the chapter.)
let chapter = include_str!("../docs/guide/preferences-shortcuts-accessibility.md").replace("\r\n", "\n");
let sample = chapter.split("```json\n").nth(1).and_then(|s| s.split("```").next()).expect("the chapter has a sample theme");
let t = parse(sample).unwrap_or_else(|e| panic!("{}", e.message()));
assert_eq!((t.name.as_deref(), t.base, t.notes.len()), (Some("Ember"), Palette::Graphite, 0), "{:?}", t.notes);
assert!(t.tokens.icon_accent.a() > 0 && t.tokens.accent_end != t.tokens.accent);