From a8b43fd8c8a91432e4e1edead544b80fc3483f19 Mon Sep 17 00:00:00 2001 From: Tanish Manku Date: Wed, 18 Nov 2026 06:36:26 -0700 Subject: [PATCH] feat(audio): Convert Audio to Keyframes and Auto Lip-Sync note when the sound is not Stream-synced (M8) --- CHANGELOG.md | 4 ++ .../0114-audio-to-keyframes-and-lip-sync.md | 4 ++ docs/guide/images-and-sound.md | 4 +- src/a11y.rs | 2 +- src/app.rs | 5 +- src/audio_keys.rs | 47 +++++++++++++++++++ src/theme_file.rs | 6 +-- 7 files changed, 65 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df379dac..d68ea146 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ * `anicraft --check-theme ` checks a theme without opening a window, and `anicraft --theme .json` uses one for a single run. +* **Convert Audio to Keyframes and Auto Lip-Sync say when a sound can drift** (ADR 0114 + follow-up, M8): with a sound whose sync is Event or Start, the dialog notes that the keys + can drift from what is heard after a jump on the timeline, and that Stream keeps them locked. + * **Convert Audio to Keyframes and Auto Lip-Sync** (ADR 0114, #332, M5). * **Convert Audio to Keyframes** (Modify › Timeline, or **To Keyframes…** in a sound keyframe's Properties): the sound's loudness on each frame drives an object on another diff --git a/docs/adr/0114-audio-to-keyframes-and-lip-sync.md b/docs/adr/0114-audio-to-keyframes-and-lip-sync.md index ffff968c..a34362c2 100644 --- a/docs/adr/0114-audio-to-keyframes-and-lip-sync.md +++ b/docs/adr/0114-audio-to-keyframes-and-lip-sync.md @@ -113,3 +113,7 @@ back into the dialog to edit them. Any sync mode but Stop is accepted, although the task said "from a stream sound". An Event sound keeps time with the frames only while the movie plays straight through, so the keys can drift from what is heard after a jump. The dialog doesn't warn about it yet. + +Answered in the follow-up (with ADR 0117's PR): both dialogs now show one line under the frames +when the sound's sync is not Stream, saying the keys can drift after a jump on the timeline and +that Stream keeps them locked. They still accept the sound. diff --git a/docs/guide/images-and-sound.md b/docs/guide/images-and-sound.md index 8e42d755..73800569 100644 --- a/docs/guide/images-and-sound.md +++ b/docs/guide/images-and-sound.md @@ -196,7 +196,9 @@ logo that brightens on every hit, a jaw that drops when a character talks. ![Convert Audio to Keyframes: the star's scale follows a beat, previewed on the timeline](images/audio-to-keyframes.png) 1. Put the sound on a layer and the object on another layer. **Stream** sync is best: the - sound then stays on its frames. + sound then stays on its frames. With Event or Start sync the dialog still works, and a line + under **Frames** reminds you that the keys can drift from what is heard after a jump on the + timeline. 2. Choose **Modify › Timeline › Convert Audio to Keyframes…**, or select the sound's keyframe and press **To Keyframes…** in **Properties**. 3. Pick the **Layer** that gets the keys (and the **Object**, when the layer holds several). diff --git a/src/a11y.rs b/src/a11y.rs index c3fc957e..d8caabb0 100644 --- a/src/a11y.rs +++ b/src/a11y.rs @@ -443,7 +443,7 @@ mod tests { for tool in [Tool::Bone, Tool::Bind, Tool::Warp, Tool::Rect, Tool::Star, Tool::Text, Tool::Pencil, Tool::Brush, Tool::PaintBrush, Tool::Eraser, Tool::PaintBucket, Tool::InkBottle, Tool::Eyedropper, Tool::Pen] { found.extend(check(&format!("{tool:?} tool"), |a, _| a.tool = tool)); } - for panel in ["select", "motion", "motionspan", "motionpath", "motionpoints", "motionedit", "motionpins", "motionpinsspan", "rawsub", "camera", "cameraview", "bones", "springbones", "shapebones", "shapeweights", "mixedbones", "warp", "sound", "envelope", "audiokeys", "lipsync", "prefs", "shortcuts", "palette", "about", "files", "licenses", "confirm", "recovery", "frames", "actions", "script", "scriptproblems", "testscript", "onion", "onionmenu", "editmultiple", "library", "librarymenu", "edittransform", "onionsymbol", "layerprops", "layeropacity", "pivot", "frameview", "layercolour", "timelinepolish", "convert", "mixed", "bitmap", "nineslice", "nineslicedit", "find", "explorer", "publish", "publishreport", "publishanim", "start", "drop", "droplibrary", "testmovie", "testmovieinside", "templates", "savetemplate", "droptimeline", "textfields", "testtext", "richtext", "softmask", "pathtext", "gradient", "gradientbucket", "swatches", "swatchpanel", "brushlibrary", "brushcurve", "paintbrush"] { + for panel in ["select", "motion", "motionspan", "motionpath", "motionpoints", "motionedit", "motionpins", "motionpinsspan", "rawsub", "camera", "cameraview", "bones", "springbones", "shapebones", "shapeweights", "mixedbones", "warp", "sound", "envelope", "audiokeys", "audiokeysevent", "lipsync", "prefs", "shortcuts", "palette", "about", "files", "licenses", "confirm", "recovery", "frames", "actions", "script", "scriptproblems", "testscript", "onion", "onionmenu", "editmultiple", "library", "librarymenu", "edittransform", "onionsymbol", "layerprops", "layeropacity", "pivot", "frameview", "layercolour", "timelinepolish", "convert", "mixed", "bitmap", "nineslice", "nineslicedit", "find", "explorer", "publish", "publishreport", "publishanim", "start", "drop", "droplibrary", "testmovie", "testmovieinside", "templates", "savetemplate", "droptimeline", "textfields", "testtext", "richtext", "softmask", "pathtext", "gradient", "gradientbucket", "swatches", "swatchpanel", "brushlibrary", "brushcurve", "paintbrush"] { found.extend(check(panel, |a, _| a.open_panel(Some(panel)))); } // Every panel in front where it docks, and floating (ADR 0075); the other workspaces. diff --git a/src/app.rs b/src/app.rs index a5fa2a81..90fbfd13 100644 --- a/src/app.rs +++ b/src/app.rs @@ -2628,11 +2628,12 @@ impl App { self.run("timeline.goto", json!({"frame": 14})); } } - Some("audiokeys") => { + Some(which @ ("audiokeys" | "audiokeysevent")) => { // Convert Audio to Keyframes (ADR 0114): the beat as a stream, its dialog open // on the demo's star, whose scale follows the beat (previewed on the timeline). + // `audiokeysevent`: as an Event sound, with the dialog's note that keys can drift. if self.import_demo_beat() { - self.run("sound.set", json!({"frame": 0, "sync": "stream"})); + self.run("sound.set", json!({"frame": 0, "sync": if which == "audiokeysevent" { "event" } else { "stream" }})); let star = self.s.timeline().layers.iter().find(|l| l.name == "Star").map(|l| l.id.0); crate::audio_keys::open(self, crate::audio_keys::Kind::Keys); if let Some(id) = star { diff --git a/src/audio_keys.rs b/src/audio_keys.rs index 063c9e84..bb25ff74 100644 --- a/src/audio_keys.rs +++ b/src/audio_keys.rs @@ -164,6 +164,13 @@ fn sounds(app: &App) -> Vec<((u64, u32), String)> { out } +/// Whether the sound the dialog reads is not a Stream sound (ADR 0114): it keeps time with the +/// frames only while the movie plays straight through, so the dialog says so. +fn can_drift(app: &App, sound: (u64, u32)) -> bool { + let key = app.s.timeline().layers.iter().find(|l| l.id.0 == sound.0).and_then(|l| l.keyframes.iter().find(|k| k.start == sound.1)); + key.and_then(|k| k.sound.as_ref()).is_some_and(|f| f.sync != anicraft_engine::doc::SoundSync::Stream) +} + /// The sound the dialog reads, as the engine finds it. fn source(app: &App, sound: (u64, u32)) -> Option { keys::source(&app.s, &json!({"sound_layer": sound.0, "sound_frame": sound.1})).ok() @@ -428,6 +435,7 @@ pub fn show(app: &mut App, ctx: &egui::Context) { let sound_list = sounds(app); let symbol = mouth_symbol(app, &d).map(|s| (s.timeline.frame_count().max(1), s.clone())); let stereo = src.sound.channels == 2; + let drifts = can_drift(app, d.sound); let mut changed = false; let (mut ok, mut cancel) = (false, false); let mut retarget = false; @@ -466,6 +474,10 @@ pub fn show(app: &mut App, ctx: &egui::Context) { form_row(ui, FORM_LABEL_DIALOG, tr!("Frames"), |ui| { ui.label(text(Type::Data, tr!("{from} to {to}", from = src.from + 1, to = src.to + 1))); }); + if drifts { + // The keys are right for the frames; what is heard follows them only with Stream. + theme::form_note(ui, FORM_LABEL_DIALOG, text(Type::Caption, tr!("After a timeline jump the keys can drift from this sound. Stream sync keeps them locked."))); + } form_row(ui, FORM_LABEL_DIALOG, tr!("Layer"), |ui| { if let Some(l) = choice(ui, "audio_keys_layer", tr!("Layer that gets the keys"), d.layer, &layer_list, tr!("The layer whose object the sound drives: the keys are written on it")) { d.layer = l; @@ -836,6 +848,41 @@ mod tests { assert!(app.s.doc == before, "one undo step"); } + /// ADR 0114's follow-up: both dialogs say when the sound isn't a Stream sound, and say + /// nothing when it is. + #[test] + fn a_sound_that_is_not_stream_synced_gets_a_note_about_drift() { + let ctx = egui::Context::default(); + ctx.enable_accesskit(); + let (mut app, _) = app(&ctx, "drift"); + let sound = app.s.layer.0; + let noted = |app: &mut App, ctx: &egui::Context| { + let mut out = ctx.run_ui(egui::RawInput::default(), |ui| show(app, ui.ctx())); + out.textures_delta.clear(); + out.platform_output.accesskit_update.is_some_and(|t| t.nodes.iter().any(|(_, n)| n.value().is_some_and(|v| v.contains("Stream sync keeps them locked")))) + }; + for kind in [Kind::Keys, Kind::Lips] { + app.run("sound.set", json!({"frame": 0, "sync": "stream"})); + open(&mut app, kind); + // Auto Lip-Sync opens only with a graphic symbol to drive; the note is in the part + // of the dialog both share, so Convert Audio to Keyframes covers it here. + let Some(at) = app.audio_keys.as_ref().map(|d| d.sound) else { + assert_eq!(kind, Kind::Lips, "Convert Audio to Keyframes opens on the square"); + continue; + }; + assert_eq!(at.0, sound); + assert!(!can_drift(&app, at) && !noted(&mut app, &ctx), "{kind:?}: a stream sound stays in time"); + cancel(&mut app); + for sync in ["event", "start"] { + app.run("sound.set", json!({"frame": 0, "sync": sync})); + open(&mut app, kind); + assert!(can_drift(&app, at) && noted(&mut app, &ctx), "{kind:?}, {sync}"); + cancel(&mut app); + } + } + assert!(!can_drift(&app, (9999, 0)), "no such sound: nothing to say"); + } + #[test] fn cancel_takes_the_preview_back_and_leaves_no_step() { let ctx = egui::Context::default(); diff --git a/src/theme_file.rs b/src/theme_file.rs index a8297299..ed441ace 100644 --- a/src/theme_file.rs +++ b/src/theme_file.rs @@ -450,9 +450,9 @@ mod tests { /// gradient accent bars, and nothing about it is noted as low in contrast. #[test] fn the_guides_sample_theme_is_a_good_theme() { - let chapter = include_str!("../docs/guide/preferences-shortcuts-accessibility.md"); - let sample = chapter.split("```json -").nth(1).and_then(|s| s.split("```").next()).expect("the chapter has a sample theme"); + // (Whatever line endings the checkout gave the chapter.) + let chapter = include_str!("../docs/guide/preferences-shortcuts-accessibility.md").replace("\r\n", "\n"); + let sample = chapter.split("```json\n").nth(1).and_then(|s| s.split("```").next()).expect("the chapter has a sample theme"); let t = parse(sample).unwrap_or_else(|e| panic!("{}", e.message())); assert_eq!((t.name.as_deref(), t.base, t.notes.len()), (Some("Ember"), Palette::Graphite, 0), "{:?}", t.notes); assert!(t.tokens.icon_accent.a() > 0 && t.tokens.accent_end != t.tokens.accent);