mirror of
https://github.com/storytold/AniCraft.git
synced 2026-11-21 18:12:28 +01:00
test(engine): golden renders of the XFL corpus with a reusable golden-image helper (M4)
20 frames of the corpus rendered by the CPU oracle at 160x90 and compared with committed PNGs (a pixel differs past 3 per channel; at most 0.2 % may), plus hand-worked colour probes, and every package opened with file.open and round-tripped through .anicraft. UPDATE_GOLDENS=1 rewrites the goldens; a mismatch writes the render and a diff image and names them. Rendering and comparing all 20 frames takes about 20 ms.
This commit is contained in:
1 parent
cc85a76be1
commit
e4de73b5ce
2 files changed
+291
No files matched your search
@@ -0,0 +1,124 @@
|
||||
//! Golden image checks (ADR 0035): a render is compared with a committed PNG, pixel by pixel,
|
||||
//! within a tolerance.
|
||||
//!
|
||||
//! * `UPDATE_GOLDENS=1 cargo test …` writes the renders as the new goldens (look at them before
|
||||
//! committing: a golden is only as right as the eyes that accepted it).
|
||||
//! * A missing golden fails (CI never makes its own), and says how to make it.
|
||||
//! * On a mismatch the render and a diff image (the golden faded, with the pixels that differ
|
||||
//! in red, brighter the more they differ) are written under Cargo's temporary folder for
|
||||
//! integration tests, and the failure names both paths.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// How far a render may stray from its golden: a pixel differs when any channel (RGBA, 0-255,
|
||||
/// not premultiplied) is off by more than `channel`; the check fails when more than `pixels`
|
||||
/// (a fraction of the image) differ. The default absorbs floating-point differences between
|
||||
/// computers on anti-aliased edges, and catches a shape moved by a pixel or a colour changed
|
||||
/// anywhere it covers.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Tolerance {
|
||||
pub channel: u8,
|
||||
pub pixels: f32,
|
||||
}
|
||||
|
||||
pub const DEFAULT: Tolerance = Tolerance { channel: 3, pixels: 0.002 };
|
||||
|
||||
/// Whether this run writes goldens instead of checking them.
|
||||
pub fn updating() -> bool {
|
||||
std::env::var("UPDATE_GOLDENS").is_ok_and(|v| v.trim() == "1")
|
||||
}
|
||||
|
||||
/// Where failed checks leave their render and diff images.
|
||||
fn out_dir() -> PathBuf {
|
||||
Path::new(env!("CARGO_TARGET_TMPDIR")).join("golden-diffs")
|
||||
}
|
||||
|
||||
fn decode(bytes: &[u8]) -> Result<(u32, u32, Vec<u8>), String> {
|
||||
let mut reader = png::Decoder::new(bytes).read_info().map_err(|e| e.to_string())?;
|
||||
let mut buf = vec![0; reader.output_buffer_size()];
|
||||
let info = reader.next_frame(&mut buf).map_err(|e| e.to_string())?;
|
||||
if info.color_type != png::ColorType::Rgba || info.bit_depth != png::BitDepth::Eight {
|
||||
return Err(format!("expected 8-bit RGBA, found {:?} {:?}", info.color_type, info.bit_depth));
|
||||
}
|
||||
buf.truncate(info.buffer_size());
|
||||
Ok((info.width, info.height, buf))
|
||||
}
|
||||
|
||||
fn write(path: &Path, width: u32, height: u32, rgba: &[u8]) {
|
||||
if let Some(dir) = path.parent() {
|
||||
std::fs::create_dir_all(dir).unwrap();
|
||||
}
|
||||
std::fs::write(path, anicraft_render::encode_png(width, height, rgba).unwrap()).unwrap();
|
||||
}
|
||||
|
||||
/// Checks `rgba` (`width` × `height`, straight alpha) against the golden at `golden`, or writes
|
||||
/// it there when updating. `name` names the leftovers of a failure.
|
||||
pub fn check(golden: &Path, name: &str, width: u32, height: u32, rgba: &[u8], tol: Tolerance) -> Result<(), String> {
|
||||
if updating() {
|
||||
assert_eq!(rgba.len(), (width * height * 4) as usize);
|
||||
write(golden, width, height, rgba);
|
||||
return Ok(());
|
||||
}
|
||||
compare(golden, name, width, height, rgba, tol)
|
||||
}
|
||||
|
||||
/// The comparison [`check`] makes when it isn't updating.
|
||||
fn compare(golden: &Path, name: &str, width: u32, height: u32, rgba: &[u8], tol: Tolerance) -> Result<(), String> {
|
||||
assert_eq!(rgba.len(), (width * height * 4) as usize);
|
||||
let keep = |why: String| -> String {
|
||||
let actual = out_dir().join(format!("{name}.actual.png"));
|
||||
write(&actual, width, height, rgba);
|
||||
format!("{name}: {why}\n golden: {}\n render: {}", golden.display(), actual.display())
|
||||
};
|
||||
let bytes = std::fs::read(golden).map_err(|e| keep(format!("no golden ({e}); run with UPDATE_GOLDENS=1 to make it, then look at it")))?;
|
||||
let (gw, gh, want) = decode(&bytes).map_err(|e| keep(format!("unreadable golden: {e}")))?;
|
||||
if (gw, gh) != (width, height) {
|
||||
return Err(keep(format!("the golden is {gw}×{gh}, the render {width}×{height}")));
|
||||
}
|
||||
let mut bad = 0usize;
|
||||
let mut worst = 0u8;
|
||||
let mut diff = Vec::with_capacity(rgba.len());
|
||||
for (a, b) in rgba.chunks_exact(4).zip(want.chunks_exact(4)) {
|
||||
let d = a.iter().zip(b).map(|(x, y)| x.abs_diff(*y)).max().unwrap_or(0);
|
||||
worst = worst.max(d);
|
||||
if d > tol.channel {
|
||||
bad += 1;
|
||||
diff.extend_from_slice(&[128u8.saturating_add(d / 2), 0, 0, 255]);
|
||||
} else {
|
||||
// The golden, faded towards white, for context.
|
||||
let fade = |c: u8| 192 + c / 4;
|
||||
diff.extend_from_slice(&[fade(b[0]), fade(b[1]), fade(b[2]), 255]);
|
||||
}
|
||||
}
|
||||
let allowed = (tol.pixels * (width * height) as f32).floor() as usize;
|
||||
if bad > allowed {
|
||||
let diff_path = out_dir().join(format!("{name}.diff.png"));
|
||||
write(&diff_path, width, height, &diff);
|
||||
return Err(format!("{}\n diff: {}", keep(format!("{bad} pixels differ (at most {allowed} may), by up to {worst}")), diff_path.display()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_check_passes_within_the_tolerance_and_fails_past_it() {
|
||||
let dir = out_dir().join("self-test");
|
||||
let golden = dir.join("g.png");
|
||||
let (w, h) = (20, 10);
|
||||
let base: Vec<u8> = (0..w * h).flat_map(|i| [i as u8, 100, 200, 255]).collect();
|
||||
write(&golden, w, h, &base);
|
||||
assert!(compare(&golden, "self-test-same", w, h, &base, DEFAULT).is_ok());
|
||||
// Small differences everywhere are within the channel tolerance.
|
||||
let near: Vec<u8> = base.iter().map(|c| c.saturating_sub(2)).collect();
|
||||
assert!(compare(&golden, "self-test-near", w, h, &near, DEFAULT).is_ok());
|
||||
// A whole row changed is not.
|
||||
let mut off = base.clone();
|
||||
for px in off.chunks_exact_mut(4).take(w as usize) {
|
||||
px[1] = 0;
|
||||
}
|
||||
let err = compare(&golden, "self-test-off", w, h, &off, DEFAULT).unwrap_err();
|
||||
assert!(err.contains("20 pixels differ") && err.contains("self-test-off.diff.png"), "{err}");
|
||||
assert!(out_dir().join("self-test-off.diff.png").exists() && out_dir().join("self-test-off.actual.png").exists());
|
||||
// A missing golden fails and says how to make one.
|
||||
let err = compare(&dir.join("none.png"), "self-test-missing", w, h, &base, DEFAULT).unwrap_err();
|
||||
assert!(err.contains("UPDATE_GOLDENS=1"), "{err}");
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
//! Golden renders of the self-authored XFL corpus (`crates/xfl/tests/corpus/`, ADR 0035).
|
||||
//!
|
||||
//! Each package is imported (without fonts, so text is reported rather than drawn in whatever
|
||||
//! fonts this computer has, and the renders match everywhere), chosen frames are rendered by
|
||||
//! the CPU oracle at half size (160 × 90), and each is compared with the golden PNG in the
|
||||
//! package's `golden/` folder. Probes then check, by hand-worked colours, what the golden
|
||||
//! should show. `UPDATE_GOLDENS=1` rewrites the goldens (see `golden/mod.rs`).
|
||||
//!
|
||||
//! The report and structure of each package, and its hostile variants, are checked in the xfl
|
||||
//! crate (`crates/xfl/tests/corpus.rs`).
|
||||
|
||||
mod golden;
|
||||
|
||||
use anicraft_engine::Session;
|
||||
use anicraft_engine::doc::Document;
|
||||
use anicraft_engine::text::Fonts;
|
||||
use serde_json::json;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
fn corpus() -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR")).join("../xfl/tests/corpus")
|
||||
}
|
||||
|
||||
/// The frames rendered for each package (frames count across scenes, from 0).
|
||||
const FRAMES: &[(&str, &[u32])] = &[
|
||||
("bitmaps", &[0]),
|
||||
("filters", &[0]),
|
||||
("gradients", &[0]),
|
||||
("guides", &[0, 6, 12]),
|
||||
("masks", &[0]),
|
||||
("primitives", &[0]),
|
||||
("scenes", &[0, 3, 7, 8, 10]),
|
||||
("shapes", &[0]),
|
||||
("symbols", &[0, 5]),
|
||||
("text", &[0]),
|
||||
("tweens", &[0, 6, 12]),
|
||||
];
|
||||
|
||||
/// Render scale: the corpus stages are 320 × 180.
|
||||
const SCALE: f32 = 0.5;
|
||||
|
||||
/// What a frame must show at a stage point: (package, frame, x, y, colour).
|
||||
const PROBES: &[(&str, u32, f32, f32, u32)] = &[
|
||||
// Both sides of a shared edge, the hole, the hex-coordinate triangle.
|
||||
("shapes", 0, 45.0, 50.0, 0xD9452B),
|
||||
("shapes", 0, 80.0, 30.0, 0x2B6CD9),
|
||||
("shapes", 0, 95.0, 50.0, 0xF4F1EA),
|
||||
("shapes", 0, 270.0, 60.0, 0xF29E1F),
|
||||
("primitives", 0, 40.0, 50.0, 0xF2CC8F),
|
||||
("primitives", 0, 210.0, 40.0, 0xE07A5F),
|
||||
("primitives", 0, 6.0, 170.0, 0xFFFFFF),
|
||||
// The checker's transparent corner, then a teal and a cream square, four times as large.
|
||||
("bitmaps", 0, 14.0, 14.0, 0xFFFFFF),
|
||||
("bitmaps", 0, 34.0, 18.0, 0x2A9D8F),
|
||||
("bitmaps", 0, 34.0, 34.0, 0xE9C46A),
|
||||
// Loop modes: frame 0, then frame 5.
|
||||
("symbols", 0, 30.0, 40.0, 0xE63946),
|
||||
("symbols", 0, 80.0, 40.0, 0x2A9D8F),
|
||||
("symbols", 0, 130.0, 40.0, 0x457B9D),
|
||||
("symbols", 0, 180.0, 40.0, 0xE9C46A),
|
||||
("symbols", 5, 30.0, 40.0, 0x2A9D8F),
|
||||
("symbols", 5, 80.0, 40.0, 0xE9C46A),
|
||||
("symbols", 5, 130.0, 40.0, 0x457B9D),
|
||||
// The button's Up frame.
|
||||
("symbols", 0, 82.0, 130.0, 0x2A9D8F),
|
||||
// Brightness +60% and -50% of #6D597A.
|
||||
("symbols", 0, 230.0, 130.0, 0xC5BDCA),
|
||||
("symbols", 0, 260.0, 130.0, 0x372D3D),
|
||||
("tweens", 0, 20.0, 25.0, 0xF4A261),
|
||||
("tweens", 12, 300.0, 25.0, 0xF4A261),
|
||||
// Masked stripes: inside the spot, and outside it.
|
||||
("masks", 0, 60.0, 22.0, 0x264653),
|
||||
("masks", 0, 60.0, 27.0, 0xFFFFFF),
|
||||
("masks", 0, 10.0, 22.0, 0xFFFFFF),
|
||||
("masks", 0, 145.0, 10.0, 0xFFFFFF),
|
||||
("masks", 0, 270.0, 70.0, 0xCB997E),
|
||||
// Guide artwork never draws.
|
||||
("guides", 0, 5.0, 5.0, 0xFFFFFF),
|
||||
("guides", 0, 160.0, 60.0, 0xFFFFFF),
|
||||
("scenes", 0, 160.0, 165.0, 0xFCA311),
|
||||
("scenes", 7, 160.0, 90.0, 0xE5E5E5),
|
||||
("scenes", 8, 160.0, 90.0, 0xFCA311),
|
||||
("scenes", 10, 160.0, 90.0, 0x14213D),
|
||||
("text", 0, 300.0, 80.0, 0xE9EDC9),
|
||||
];
|
||||
|
||||
fn import(case: &str) -> Document {
|
||||
let dir = corpus().join(case);
|
||||
let read = |p: &str| -> Result<Option<Vec<u8>>, anicraft_xfl::XflError> {
|
||||
match std::fs::read(dir.join(p)) {
|
||||
Ok(b) => Ok(Some(b)),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
};
|
||||
anicraft_xfl::import_with_fonts(&read, &Fonts::from_faces(Vec::new())).unwrap_or_else(|e| panic!("{case}: {e}")).0
|
||||
}
|
||||
|
||||
fn render(doc: &Document, frame: u32) -> (u32, u32, Vec<u8>) {
|
||||
let pm = anicraft_render::render_frame(doc, frame, SCALE).unwrap();
|
||||
(pm.width, pm.height, pm.to_rgba8())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_package_has_golden_frames() {
|
||||
let mut cases: Vec<String> = std::fs::read_dir(corpus()).unwrap().flatten().filter(|e| e.path().is_dir()).map(|e| e.file_name().to_string_lossy().into_owned()).collect();
|
||||
cases.sort();
|
||||
assert_eq!(cases, FRAMES.iter().map(|(c, _)| *c).collect::<Vec<_>>(), "give the new package its frames here and its report in crates/xfl/tests/corpus.rs");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corpus_frames_match_their_goldens() {
|
||||
let mut failures = Vec::new();
|
||||
let mut checked = 0;
|
||||
let start = std::time::Instant::now();
|
||||
for (case, frames) in FRAMES {
|
||||
let doc = import(case);
|
||||
for &f in *frames {
|
||||
assert!(f < doc.frame_count(), "{case}: frame {f} of {}", doc.frame_count());
|
||||
let (w, h, rgba) = render(&doc, f);
|
||||
assert_eq!((w, h), (160, 90), "{case}");
|
||||
let golden = corpus().join(case).join("golden").join(format!("frame-{f:03}.png"));
|
||||
if let Err(e) = golden::check(&golden, &format!("{case}-frame-{f:03}"), w, h, &rgba, golden::DEFAULT) {
|
||||
failures.push(e);
|
||||
}
|
||||
checked += 1;
|
||||
}
|
||||
}
|
||||
eprintln!("{checked} corpus frames rendered and compared in {:?}", start.elapsed());
|
||||
assert!(failures.is_empty(), "{} of {checked} frames differ from their goldens:\n{}", failures.len(), failures.join("\n"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corpus_frames_show_what_the_files_describe() {
|
||||
let mut wrong = Vec::new();
|
||||
for &(case, frame, x, y, want) in PROBES {
|
||||
let (w, _, rgba) = render(&import(case), frame);
|
||||
let (px, py) = ((x * SCALE) as u32, (y * SCALE) as u32);
|
||||
let i = ((py * w + px) * 4) as usize;
|
||||
let got = &rgba[i..i + 3];
|
||||
let want = [(want >> 16) as u8, (want >> 8) as u8, want as u8];
|
||||
if got.iter().zip(want).any(|(g, w)| g.abs_diff(w) > 6) {
|
||||
wrong.push(format!("{case} frame {frame} at ({x}, {y}): #{:02X}{:02X}{:02X}, want #{:02X}{:02X}{:02X}", got[0], got[1], got[2], want[0], want[1], want[2]));
|
||||
}
|
||||
}
|
||||
assert!(wrong.is_empty(), "{}", wrong.join("\n"));
|
||||
}
|
||||
|
||||
/// Opening a package is one command, like opening any file, and what it imports saves and
|
||||
/// opens again as the same document.
|
||||
#[test]
|
||||
fn every_package_opens_with_file_open_and_round_trips_through_the_file_format() {
|
||||
let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("xfl-corpus-round-trip");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
for (case, _) in FRAMES {
|
||||
let mut s = Session::default();
|
||||
let info = s.execute("file.open", &json!({"path": corpus().join(case).display().to_string()})).unwrap_or_else(|e| panic!("{case}: {e}"));
|
||||
assert!(info["imported"].is_string(), "{case}: {info}");
|
||||
assert!(s.path.is_none(), "an imported document is untitled");
|
||||
let path = dir.join(format!("{case}.anicraft"));
|
||||
// A rerun saves over the last run's file, which asks first (ADR 0004).
|
||||
s.execute("file.save", &json!({"path": path.display().to_string(), "confirm": true})).unwrap();
|
||||
let mut t = Session::default();
|
||||
t.execute("file.open", &json!({"path": path.display().to_string()})).unwrap();
|
||||
assert!(s.doc == t.doc, "{case}: the saved document opens differently");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user