test(engine): golden renders of the XFL corpus with a reusable golden-image helper (M4)

20 frames of the corpus rendered by the CPU oracle at 160x90 and compared
with committed PNGs (a pixel differs past 3 per channel; at most 0.2 %
may), plus hand-worked colour probes, and every package opened with
file.open and round-tripped through .anicraft. UPDATE_GOLDENS=1 rewrites
the goldens; a mismatch writes the render and a diff image and names
them. Rendering and comparing all 20 frames takes about 20 ms.
This commit is contained in:
Tanish Manku committed 2026-11-09 22:17:53 -07:00
1 parent cc85a76be1
commit e4de73b5ce
2 files changed
+291

No files matched your search

+124
View File
@@ -0,0 +1,124 @@
//! Golden image checks (ADR 0035): a render is compared with a committed PNG, pixel by pixel,
//! within a tolerance.
//!
//! * `UPDATE_GOLDENS=1 cargo test …` writes the renders as the new goldens (look at them before
//! committing: a golden is only as right as the eyes that accepted it).
//! * A missing golden fails (CI never makes its own), and says how to make it.
//! * On a mismatch the render and a diff image (the golden faded, with the pixels that differ
//! in red, brighter the more they differ) are written under Cargo's temporary folder for
//! integration tests, and the failure names both paths.
use std::path::{Path, PathBuf};
/// How far a render may stray from its golden: a pixel differs when any channel (RGBA, 0-255,
/// not premultiplied) is off by more than `channel`; the check fails when more than `pixels`
/// (a fraction of the image) differ. The default absorbs floating-point differences between
/// computers on anti-aliased edges, and catches a shape moved by a pixel or a colour changed
/// anywhere it covers.
#[derive(Clone, Copy, Debug)]
pub struct Tolerance {
pub channel: u8,
pub pixels: f32,
}
pub const DEFAULT: Tolerance = Tolerance { channel: 3, pixels: 0.002 };
/// Whether this run writes goldens instead of checking them.
pub fn updating() -> bool {
std::env::var("UPDATE_GOLDENS").is_ok_and(|v| v.trim() == "1")
}
/// Where failed checks leave their render and diff images.
fn out_dir() -> PathBuf {
Path::new(env!("CARGO_TARGET_TMPDIR")).join("golden-diffs")
}
fn decode(bytes: &[u8]) -> Result<(u32, u32, Vec<u8>), String> {
let mut reader = png::Decoder::new(bytes).read_info().map_err(|e| e.to_string())?;
let mut buf = vec![0; reader.output_buffer_size()];
let info = reader.next_frame(&mut buf).map_err(|e| e.to_string())?;
if info.color_type != png::ColorType::Rgba || info.bit_depth != png::BitDepth::Eight {
return Err(format!("expected 8-bit RGBA, found {:?} {:?}", info.color_type, info.bit_depth));
}
buf.truncate(info.buffer_size());
Ok((info.width, info.height, buf))
}
fn write(path: &Path, width: u32, height: u32, rgba: &[u8]) {
if let Some(dir) = path.parent() {
std::fs::create_dir_all(dir).unwrap();
}
std::fs::write(path, anicraft_render::encode_png(width, height, rgba).unwrap()).unwrap();
}
/// Checks `rgba` (`width` × `height`, straight alpha) against the golden at `golden`, or writes
/// it there when updating. `name` names the leftovers of a failure.
pub fn check(golden: &Path, name: &str, width: u32, height: u32, rgba: &[u8], tol: Tolerance) -> Result<(), String> {
if updating() {
assert_eq!(rgba.len(), (width * height * 4) as usize);
write(golden, width, height, rgba);
return Ok(());
}
compare(golden, name, width, height, rgba, tol)
}
/// The comparison [`check`] makes when it isn't updating.
fn compare(golden: &Path, name: &str, width: u32, height: u32, rgba: &[u8], tol: Tolerance) -> Result<(), String> {
assert_eq!(rgba.len(), (width * height * 4) as usize);
let keep = |why: String| -> String {
let actual = out_dir().join(format!("{name}.actual.png"));
write(&actual, width, height, rgba);
format!("{name}: {why}\n golden: {}\n render: {}", golden.display(), actual.display())
};
let bytes = std::fs::read(golden).map_err(|e| keep(format!("no golden ({e}); run with UPDATE_GOLDENS=1 to make it, then look at it")))?;
let (gw, gh, want) = decode(&bytes).map_err(|e| keep(format!("unreadable golden: {e}")))?;
if (gw, gh) != (width, height) {
return Err(keep(format!("the golden is {gw}×{gh}, the render {width}×{height}")));
}
let mut bad = 0usize;
let mut worst = 0u8;
let mut diff = Vec::with_capacity(rgba.len());
for (a, b) in rgba.chunks_exact(4).zip(want.chunks_exact(4)) {
let d = a.iter().zip(b).map(|(x, y)| x.abs_diff(*y)).max().unwrap_or(0);
worst = worst.max(d);
if d > tol.channel {
bad += 1;
diff.extend_from_slice(&[128u8.saturating_add(d / 2), 0, 0, 255]);
} else {
// The golden, faded towards white, for context.
let fade = |c: u8| 192 + c / 4;
diff.extend_from_slice(&[fade(b[0]), fade(b[1]), fade(b[2]), 255]);
}
}
let allowed = (tol.pixels * (width * height) as f32).floor() as usize;
if bad > allowed {
let diff_path = out_dir().join(format!("{name}.diff.png"));
write(&diff_path, width, height, &diff);
return Err(format!("{}\n diff: {}", keep(format!("{bad} pixels differ (at most {allowed} may), by up to {worst}")), diff_path.display()));
}
Ok(())
}
#[test]
fn the_check_passes_within_the_tolerance_and_fails_past_it() {
let dir = out_dir().join("self-test");
let golden = dir.join("g.png");
let (w, h) = (20, 10);
let base: Vec<u8> = (0..w * h).flat_map(|i| [i as u8, 100, 200, 255]).collect();
write(&golden, w, h, &base);
assert!(compare(&golden, "self-test-same", w, h, &base, DEFAULT).is_ok());
// Small differences everywhere are within the channel tolerance.
let near: Vec<u8> = base.iter().map(|c| c.saturating_sub(2)).collect();
assert!(compare(&golden, "self-test-near", w, h, &near, DEFAULT).is_ok());
// A whole row changed is not.
let mut off = base.clone();
for px in off.chunks_exact_mut(4).take(w as usize) {
px[1] = 0;
}
let err = compare(&golden, "self-test-off", w, h, &off, DEFAULT).unwrap_err();
assert!(err.contains("20 pixels differ") && err.contains("self-test-off.diff.png"), "{err}");
assert!(out_dir().join("self-test-off.diff.png").exists() && out_dir().join("self-test-off.actual.png").exists());
// A missing golden fails and says how to make one.
let err = compare(&dir.join("none.png"), "self-test-missing", w, h, &base, DEFAULT).unwrap_err();
assert!(err.contains("UPDATE_GOLDENS=1"), "{err}");
}
+167
View File
@@ -0,0 +1,167 @@
//! Golden renders of the self-authored XFL corpus (`crates/xfl/tests/corpus/`, ADR 0035).
//!
//! Each package is imported (without fonts, so text is reported rather than drawn in whatever
//! fonts this computer has, and the renders match everywhere), chosen frames are rendered by
//! the CPU oracle at half size (160 × 90), and each is compared with the golden PNG in the
//! package's `golden/` folder. Probes then check, by hand-worked colours, what the golden
//! should show. `UPDATE_GOLDENS=1` rewrites the goldens (see `golden/mod.rs`).
//!
//! The report and structure of each package, and its hostile variants, are checked in the xfl
//! crate (`crates/xfl/tests/corpus.rs`).
mod golden;
use anicraft_engine::Session;
use anicraft_engine::doc::Document;
use anicraft_engine::text::Fonts;
use serde_json::json;
use std::path::{Path, PathBuf};
fn corpus() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR")).join("../xfl/tests/corpus")
}
/// The frames rendered for each package (frames count across scenes, from 0).
const FRAMES: &[(&str, &[u32])] = &[
("bitmaps", &[0]),
("filters", &[0]),
("gradients", &[0]),
("guides", &[0, 6, 12]),
("masks", &[0]),
("primitives", &[0]),
("scenes", &[0, 3, 7, 8, 10]),
("shapes", &[0]),
("symbols", &[0, 5]),
("text", &[0]),
("tweens", &[0, 6, 12]),
];
/// Render scale: the corpus stages are 320 × 180.
const SCALE: f32 = 0.5;
/// What a frame must show at a stage point: (package, frame, x, y, colour).
const PROBES: &[(&str, u32, f32, f32, u32)] = &[
// Both sides of a shared edge, the hole, the hex-coordinate triangle.
("shapes", 0, 45.0, 50.0, 0xD9452B),
("shapes", 0, 80.0, 30.0, 0x2B6CD9),
("shapes", 0, 95.0, 50.0, 0xF4F1EA),
("shapes", 0, 270.0, 60.0, 0xF29E1F),
("primitives", 0, 40.0, 50.0, 0xF2CC8F),
("primitives", 0, 210.0, 40.0, 0xE07A5F),
("primitives", 0, 6.0, 170.0, 0xFFFFFF),
// The checker's transparent corner, then a teal and a cream square, four times as large.
("bitmaps", 0, 14.0, 14.0, 0xFFFFFF),
("bitmaps", 0, 34.0, 18.0, 0x2A9D8F),
("bitmaps", 0, 34.0, 34.0, 0xE9C46A),
// Loop modes: frame 0, then frame 5.
("symbols", 0, 30.0, 40.0, 0xE63946),
("symbols", 0, 80.0, 40.0, 0x2A9D8F),
("symbols", 0, 130.0, 40.0, 0x457B9D),
("symbols", 0, 180.0, 40.0, 0xE9C46A),
("symbols", 5, 30.0, 40.0, 0x2A9D8F),
("symbols", 5, 80.0, 40.0, 0xE9C46A),
("symbols", 5, 130.0, 40.0, 0x457B9D),
// The button's Up frame.
("symbols", 0, 82.0, 130.0, 0x2A9D8F),
// Brightness +60% and -50% of #6D597A.
("symbols", 0, 230.0, 130.0, 0xC5BDCA),
("symbols", 0, 260.0, 130.0, 0x372D3D),
("tweens", 0, 20.0, 25.0, 0xF4A261),
("tweens", 12, 300.0, 25.0, 0xF4A261),
// Masked stripes: inside the spot, and outside it.
("masks", 0, 60.0, 22.0, 0x264653),
("masks", 0, 60.0, 27.0, 0xFFFFFF),
("masks", 0, 10.0, 22.0, 0xFFFFFF),
("masks", 0, 145.0, 10.0, 0xFFFFFF),
("masks", 0, 270.0, 70.0, 0xCB997E),
// Guide artwork never draws.
("guides", 0, 5.0, 5.0, 0xFFFFFF),
("guides", 0, 160.0, 60.0, 0xFFFFFF),
("scenes", 0, 160.0, 165.0, 0xFCA311),
("scenes", 7, 160.0, 90.0, 0xE5E5E5),
("scenes", 8, 160.0, 90.0, 0xFCA311),
("scenes", 10, 160.0, 90.0, 0x14213D),
("text", 0, 300.0, 80.0, 0xE9EDC9),
];
fn import(case: &str) -> Document {
let dir = corpus().join(case);
let read = |p: &str| -> Result<Option<Vec<u8>>, anicraft_xfl::XflError> {
match std::fs::read(dir.join(p)) {
Ok(b) => Ok(Some(b)),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(e) => Err(e.into()),
}
};
anicraft_xfl::import_with_fonts(&read, &Fonts::from_faces(Vec::new())).unwrap_or_else(|e| panic!("{case}: {e}")).0
}
fn render(doc: &Document, frame: u32) -> (u32, u32, Vec<u8>) {
let pm = anicraft_render::render_frame(doc, frame, SCALE).unwrap();
(pm.width, pm.height, pm.to_rgba8())
}
#[test]
fn every_package_has_golden_frames() {
let mut cases: Vec<String> = std::fs::read_dir(corpus()).unwrap().flatten().filter(|e| e.path().is_dir()).map(|e| e.file_name().to_string_lossy().into_owned()).collect();
cases.sort();
assert_eq!(cases, FRAMES.iter().map(|(c, _)| *c).collect::<Vec<_>>(), "give the new package its frames here and its report in crates/xfl/tests/corpus.rs");
}
#[test]
fn corpus_frames_match_their_goldens() {
let mut failures = Vec::new();
let mut checked = 0;
let start = std::time::Instant::now();
for (case, frames) in FRAMES {
let doc = import(case);
for &f in *frames {
assert!(f < doc.frame_count(), "{case}: frame {f} of {}", doc.frame_count());
let (w, h, rgba) = render(&doc, f);
assert_eq!((w, h), (160, 90), "{case}");
let golden = corpus().join(case).join("golden").join(format!("frame-{f:03}.png"));
if let Err(e) = golden::check(&golden, &format!("{case}-frame-{f:03}"), w, h, &rgba, golden::DEFAULT) {
failures.push(e);
}
checked += 1;
}
}
eprintln!("{checked} corpus frames rendered and compared in {:?}", start.elapsed());
assert!(failures.is_empty(), "{} of {checked} frames differ from their goldens:\n{}", failures.len(), failures.join("\n"));
}
#[test]
fn corpus_frames_show_what_the_files_describe() {
let mut wrong = Vec::new();
for &(case, frame, x, y, want) in PROBES {
let (w, _, rgba) = render(&import(case), frame);
let (px, py) = ((x * SCALE) as u32, (y * SCALE) as u32);
let i = ((py * w + px) * 4) as usize;
let got = &rgba[i..i + 3];
let want = [(want >> 16) as u8, (want >> 8) as u8, want as u8];
if got.iter().zip(want).any(|(g, w)| g.abs_diff(w) > 6) {
wrong.push(format!("{case} frame {frame} at ({x}, {y}): #{:02X}{:02X}{:02X}, want #{:02X}{:02X}{:02X}", got[0], got[1], got[2], want[0], want[1], want[2]));
}
}
assert!(wrong.is_empty(), "{}", wrong.join("\n"));
}
/// Opening a package is one command, like opening any file, and what it imports saves and
/// opens again as the same document.
#[test]
fn every_package_opens_with_file_open_and_round_trips_through_the_file_format() {
let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("xfl-corpus-round-trip");
std::fs::create_dir_all(&dir).unwrap();
for (case, _) in FRAMES {
let mut s = Session::default();
let info = s.execute("file.open", &json!({"path": corpus().join(case).display().to_string()})).unwrap_or_else(|e| panic!("{case}: {e}"));
assert!(info["imported"].is_string(), "{case}: {info}");
assert!(s.path.is_none(), "an imported document is untitled");
let path = dir.join(format!("{case}.anicraft"));
// A rerun saves over the last run's file, which asks first (ADR 0004).
s.execute("file.save", &json!({"path": path.display().to_string(), "confirm": true})).unwrap();
let mut t = Session::default();
t.execute("file.open", &json!({"path": path.display().to_string()})).unwrap();
assert!(s.doc == t.doc, "{case}: the saved document opens differently");
}
}