From e4de73b5ce5d8b90c0490cd4ff9dec28d6770fc7 Mon Sep 17 00:00:00 2001 From: Tanish Manku Date: Mon, 9 Nov 2026 22:17:53 -0700 Subject: [PATCH] test(engine): golden renders of the XFL corpus with a reusable golden-image helper (M4) 20 frames of the corpus rendered by the CPU oracle at 160x90 and compared with committed PNGs (a pixel differs past 3 per channel; at most 0.2 % may), plus hand-worked colour probes, and every package opened with file.open and round-tripped through .anicraft. UPDATE_GOLDENS=1 rewrites the goldens; a mismatch writes the render and a diff image and names them. Rendering and comparing all 20 frames takes about 20 ms. --- crates/engine/tests/golden/mod.rs | 124 ++++++++++++++++++++++ crates/engine/tests/xfl_corpus.rs | 167 ++++++++++++++++++++++++++++++ 2 files changed, 291 insertions(+) create mode 100644 crates/engine/tests/golden/mod.rs create mode 100644 crates/engine/tests/xfl_corpus.rs diff --git a/crates/engine/tests/golden/mod.rs b/crates/engine/tests/golden/mod.rs new file mode 100644 index 00000000..55505757 --- /dev/null +++ b/crates/engine/tests/golden/mod.rs @@ -0,0 +1,124 @@ +//! Golden image checks (ADR 0035): a render is compared with a committed PNG, pixel by pixel, +//! within a tolerance. +//! +//! * `UPDATE_GOLDENS=1 cargo test …` writes the renders as the new goldens (look at them before +//! committing: a golden is only as right as the eyes that accepted it). +//! * A missing golden fails (CI never makes its own), and says how to make it. +//! * On a mismatch the render and a diff image (the golden faded, with the pixels that differ +//! in red, brighter the more they differ) are written under Cargo's temporary folder for +//! integration tests, and the failure names both paths. + +use std::path::{Path, PathBuf}; + +/// How far a render may stray from its golden: a pixel differs when any channel (RGBA, 0-255, +/// not premultiplied) is off by more than `channel`; the check fails when more than `pixels` +/// (a fraction of the image) differ. The default absorbs floating-point differences between +/// computers on anti-aliased edges, and catches a shape moved by a pixel or a colour changed +/// anywhere it covers. +#[derive(Clone, Copy, Debug)] +pub struct Tolerance { + pub channel: u8, + pub pixels: f32, +} + +pub const DEFAULT: Tolerance = Tolerance { channel: 3, pixels: 0.002 }; + +/// Whether this run writes goldens instead of checking them. +pub fn updating() -> bool { + std::env::var("UPDATE_GOLDENS").is_ok_and(|v| v.trim() == "1") +} + +/// Where failed checks leave their render and diff images. +fn out_dir() -> PathBuf { + Path::new(env!("CARGO_TARGET_TMPDIR")).join("golden-diffs") +} + +fn decode(bytes: &[u8]) -> Result<(u32, u32, Vec), String> { + let mut reader = png::Decoder::new(bytes).read_info().map_err(|e| e.to_string())?; + let mut buf = vec![0; reader.output_buffer_size()]; + let info = reader.next_frame(&mut buf).map_err(|e| e.to_string())?; + if info.color_type != png::ColorType::Rgba || info.bit_depth != png::BitDepth::Eight { + return Err(format!("expected 8-bit RGBA, found {:?} {:?}", info.color_type, info.bit_depth)); + } + buf.truncate(info.buffer_size()); + Ok((info.width, info.height, buf)) +} + +fn write(path: &Path, width: u32, height: u32, rgba: &[u8]) { + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir).unwrap(); + } + std::fs::write(path, anicraft_render::encode_png(width, height, rgba).unwrap()).unwrap(); +} + +/// Checks `rgba` (`width` × `height`, straight alpha) against the golden at `golden`, or writes +/// it there when updating. `name` names the leftovers of a failure. +pub fn check(golden: &Path, name: &str, width: u32, height: u32, rgba: &[u8], tol: Tolerance) -> Result<(), String> { + if updating() { + assert_eq!(rgba.len(), (width * height * 4) as usize); + write(golden, width, height, rgba); + return Ok(()); + } + compare(golden, name, width, height, rgba, tol) +} + +/// The comparison [`check`] makes when it isn't updating. +fn compare(golden: &Path, name: &str, width: u32, height: u32, rgba: &[u8], tol: Tolerance) -> Result<(), String> { + assert_eq!(rgba.len(), (width * height * 4) as usize); + let keep = |why: String| -> String { + let actual = out_dir().join(format!("{name}.actual.png")); + write(&actual, width, height, rgba); + format!("{name}: {why}\n golden: {}\n render: {}", golden.display(), actual.display()) + }; + let bytes = std::fs::read(golden).map_err(|e| keep(format!("no golden ({e}); run with UPDATE_GOLDENS=1 to make it, then look at it")))?; + let (gw, gh, want) = decode(&bytes).map_err(|e| keep(format!("unreadable golden: {e}")))?; + if (gw, gh) != (width, height) { + return Err(keep(format!("the golden is {gw}×{gh}, the render {width}×{height}"))); + } + let mut bad = 0usize; + let mut worst = 0u8; + let mut diff = Vec::with_capacity(rgba.len()); + for (a, b) in rgba.chunks_exact(4).zip(want.chunks_exact(4)) { + let d = a.iter().zip(b).map(|(x, y)| x.abs_diff(*y)).max().unwrap_or(0); + worst = worst.max(d); + if d > tol.channel { + bad += 1; + diff.extend_from_slice(&[128u8.saturating_add(d / 2), 0, 0, 255]); + } else { + // The golden, faded towards white, for context. + let fade = |c: u8| 192 + c / 4; + diff.extend_from_slice(&[fade(b[0]), fade(b[1]), fade(b[2]), 255]); + } + } + let allowed = (tol.pixels * (width * height) as f32).floor() as usize; + if bad > allowed { + let diff_path = out_dir().join(format!("{name}.diff.png")); + write(&diff_path, width, height, &diff); + return Err(format!("{}\n diff: {}", keep(format!("{bad} pixels differ (at most {allowed} may), by up to {worst}")), diff_path.display())); + } + Ok(()) +} + +#[test] +fn the_check_passes_within_the_tolerance_and_fails_past_it() { + let dir = out_dir().join("self-test"); + let golden = dir.join("g.png"); + let (w, h) = (20, 10); + let base: Vec = (0..w * h).flat_map(|i| [i as u8, 100, 200, 255]).collect(); + write(&golden, w, h, &base); + assert!(compare(&golden, "self-test-same", w, h, &base, DEFAULT).is_ok()); + // Small differences everywhere are within the channel tolerance. + let near: Vec = base.iter().map(|c| c.saturating_sub(2)).collect(); + assert!(compare(&golden, "self-test-near", w, h, &near, DEFAULT).is_ok()); + // A whole row changed is not. + let mut off = base.clone(); + for px in off.chunks_exact_mut(4).take(w as usize) { + px[1] = 0; + } + let err = compare(&golden, "self-test-off", w, h, &off, DEFAULT).unwrap_err(); + assert!(err.contains("20 pixels differ") && err.contains("self-test-off.diff.png"), "{err}"); + assert!(out_dir().join("self-test-off.diff.png").exists() && out_dir().join("self-test-off.actual.png").exists()); + // A missing golden fails and says how to make one. + let err = compare(&dir.join("none.png"), "self-test-missing", w, h, &base, DEFAULT).unwrap_err(); + assert!(err.contains("UPDATE_GOLDENS=1"), "{err}"); +} diff --git a/crates/engine/tests/xfl_corpus.rs b/crates/engine/tests/xfl_corpus.rs new file mode 100644 index 00000000..8e142475 --- /dev/null +++ b/crates/engine/tests/xfl_corpus.rs @@ -0,0 +1,167 @@ +//! Golden renders of the self-authored XFL corpus (`crates/xfl/tests/corpus/`, ADR 0035). +//! +//! Each package is imported (without fonts, so text is reported rather than drawn in whatever +//! fonts this computer has, and the renders match everywhere), chosen frames are rendered by +//! the CPU oracle at half size (160 × 90), and each is compared with the golden PNG in the +//! package's `golden/` folder. Probes then check, by hand-worked colours, what the golden +//! should show. `UPDATE_GOLDENS=1` rewrites the goldens (see `golden/mod.rs`). +//! +//! The report and structure of each package, and its hostile variants, are checked in the xfl +//! crate (`crates/xfl/tests/corpus.rs`). + +mod golden; + +use anicraft_engine::Session; +use anicraft_engine::doc::Document; +use anicraft_engine::text::Fonts; +use serde_json::json; +use std::path::{Path, PathBuf}; + +fn corpus() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../xfl/tests/corpus") +} + +/// The frames rendered for each package (frames count across scenes, from 0). +const FRAMES: &[(&str, &[u32])] = &[ + ("bitmaps", &[0]), + ("filters", &[0]), + ("gradients", &[0]), + ("guides", &[0, 6, 12]), + ("masks", &[0]), + ("primitives", &[0]), + ("scenes", &[0, 3, 7, 8, 10]), + ("shapes", &[0]), + ("symbols", &[0, 5]), + ("text", &[0]), + ("tweens", &[0, 6, 12]), +]; + +/// Render scale: the corpus stages are 320 × 180. +const SCALE: f32 = 0.5; + +/// What a frame must show at a stage point: (package, frame, x, y, colour). +const PROBES: &[(&str, u32, f32, f32, u32)] = &[ + // Both sides of a shared edge, the hole, the hex-coordinate triangle. + ("shapes", 0, 45.0, 50.0, 0xD9452B), + ("shapes", 0, 80.0, 30.0, 0x2B6CD9), + ("shapes", 0, 95.0, 50.0, 0xF4F1EA), + ("shapes", 0, 270.0, 60.0, 0xF29E1F), + ("primitives", 0, 40.0, 50.0, 0xF2CC8F), + ("primitives", 0, 210.0, 40.0, 0xE07A5F), + ("primitives", 0, 6.0, 170.0, 0xFFFFFF), + // The checker's transparent corner, then a teal and a cream square, four times as large. + ("bitmaps", 0, 14.0, 14.0, 0xFFFFFF), + ("bitmaps", 0, 34.0, 18.0, 0x2A9D8F), + ("bitmaps", 0, 34.0, 34.0, 0xE9C46A), + // Loop modes: frame 0, then frame 5. + ("symbols", 0, 30.0, 40.0, 0xE63946), + ("symbols", 0, 80.0, 40.0, 0x2A9D8F), + ("symbols", 0, 130.0, 40.0, 0x457B9D), + ("symbols", 0, 180.0, 40.0, 0xE9C46A), + ("symbols", 5, 30.0, 40.0, 0x2A9D8F), + ("symbols", 5, 80.0, 40.0, 0xE9C46A), + ("symbols", 5, 130.0, 40.0, 0x457B9D), + // The button's Up frame. + ("symbols", 0, 82.0, 130.0, 0x2A9D8F), + // Brightness +60% and -50% of #6D597A. + ("symbols", 0, 230.0, 130.0, 0xC5BDCA), + ("symbols", 0, 260.0, 130.0, 0x372D3D), + ("tweens", 0, 20.0, 25.0, 0xF4A261), + ("tweens", 12, 300.0, 25.0, 0xF4A261), + // Masked stripes: inside the spot, and outside it. + ("masks", 0, 60.0, 22.0, 0x264653), + ("masks", 0, 60.0, 27.0, 0xFFFFFF), + ("masks", 0, 10.0, 22.0, 0xFFFFFF), + ("masks", 0, 145.0, 10.0, 0xFFFFFF), + ("masks", 0, 270.0, 70.0, 0xCB997E), + // Guide artwork never draws. + ("guides", 0, 5.0, 5.0, 0xFFFFFF), + ("guides", 0, 160.0, 60.0, 0xFFFFFF), + ("scenes", 0, 160.0, 165.0, 0xFCA311), + ("scenes", 7, 160.0, 90.0, 0xE5E5E5), + ("scenes", 8, 160.0, 90.0, 0xFCA311), + ("scenes", 10, 160.0, 90.0, 0x14213D), + ("text", 0, 300.0, 80.0, 0xE9EDC9), +]; + +fn import(case: &str) -> Document { + let dir = corpus().join(case); + let read = |p: &str| -> Result>, anicraft_xfl::XflError> { + match std::fs::read(dir.join(p)) { + Ok(b) => Ok(Some(b)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e.into()), + } + }; + anicraft_xfl::import_with_fonts(&read, &Fonts::from_faces(Vec::new())).unwrap_or_else(|e| panic!("{case}: {e}")).0 +} + +fn render(doc: &Document, frame: u32) -> (u32, u32, Vec) { + let pm = anicraft_render::render_frame(doc, frame, SCALE).unwrap(); + (pm.width, pm.height, pm.to_rgba8()) +} + +#[test] +fn every_package_has_golden_frames() { + let mut cases: Vec = std::fs::read_dir(corpus()).unwrap().flatten().filter(|e| e.path().is_dir()).map(|e| e.file_name().to_string_lossy().into_owned()).collect(); + cases.sort(); + assert_eq!(cases, FRAMES.iter().map(|(c, _)| *c).collect::>(), "give the new package its frames here and its report in crates/xfl/tests/corpus.rs"); +} + +#[test] +fn corpus_frames_match_their_goldens() { + let mut failures = Vec::new(); + let mut checked = 0; + let start = std::time::Instant::now(); + for (case, frames) in FRAMES { + let doc = import(case); + for &f in *frames { + assert!(f < doc.frame_count(), "{case}: frame {f} of {}", doc.frame_count()); + let (w, h, rgba) = render(&doc, f); + assert_eq!((w, h), (160, 90), "{case}"); + let golden = corpus().join(case).join("golden").join(format!("frame-{f:03}.png")); + if let Err(e) = golden::check(&golden, &format!("{case}-frame-{f:03}"), w, h, &rgba, golden::DEFAULT) { + failures.push(e); + } + checked += 1; + } + } + eprintln!("{checked} corpus frames rendered and compared in {:?}", start.elapsed()); + assert!(failures.is_empty(), "{} of {checked} frames differ from their goldens:\n{}", failures.len(), failures.join("\n")); +} + +#[test] +fn corpus_frames_show_what_the_files_describe() { + let mut wrong = Vec::new(); + for &(case, frame, x, y, want) in PROBES { + let (w, _, rgba) = render(&import(case), frame); + let (px, py) = ((x * SCALE) as u32, (y * SCALE) as u32); + let i = ((py * w + px) * 4) as usize; + let got = &rgba[i..i + 3]; + let want = [(want >> 16) as u8, (want >> 8) as u8, want as u8]; + if got.iter().zip(want).any(|(g, w)| g.abs_diff(w) > 6) { + wrong.push(format!("{case} frame {frame} at ({x}, {y}): #{:02X}{:02X}{:02X}, want #{:02X}{:02X}{:02X}", got[0], got[1], got[2], want[0], want[1], want[2])); + } + } + assert!(wrong.is_empty(), "{}", wrong.join("\n")); +} + +/// Opening a package is one command, like opening any file, and what it imports saves and +/// opens again as the same document. +#[test] +fn every_package_opens_with_file_open_and_round_trips_through_the_file_format() { + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("xfl-corpus-round-trip"); + std::fs::create_dir_all(&dir).unwrap(); + for (case, _) in FRAMES { + let mut s = Session::default(); + let info = s.execute("file.open", &json!({"path": corpus().join(case).display().to_string()})).unwrap_or_else(|e| panic!("{case}: {e}")); + assert!(info["imported"].is_string(), "{case}: {info}"); + assert!(s.path.is_none(), "an imported document is untitled"); + let path = dir.join(format!("{case}.anicraft")); + // A rerun saves over the last run's file, which asks first (ADR 0004). + s.execute("file.save", &json!({"path": path.display().to_string(), "confirm": true})).unwrap(); + let mut t = Session::default(); + t.execute("file.open", &json!({"path": path.display().to_string()})).unwrap(); + assert!(s.doc == t.doc, "{case}: the saved document opens differently"); + } +}