mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-20 17:41:41 +01:00
craft-media-testkit: run hostile and with_deadline on craft-testkit (#1462)
This commit is contained in:
1 parent
e2583a4a58
commit
c91e32eb67
16 files changed
+6332
-117
No files matched your search
Generated
+7
@@ -2298,6 +2298,9 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "craft-media-testkit"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"craft-testkit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-menu"
|
||||
@@ -2978,6 +2981,10 @@ dependencies = [
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-testkit"
|
||||
version = "0.1.0"
|
||||
|
||||
[[package]]
|
||||
name = "craft-text"
|
||||
version = "0.1.0"
|
||||
|
||||
@@ -164,6 +164,7 @@ craft-stencilcraft-format = { path = "crates/stencilcraft-format" }
|
||||
craft-stencilcraft-document = { path = "crates/stencilcraft-document" }
|
||||
craft-svg-paths = { path = "crates/svg-paths" }
|
||||
craft-stream-codecs = { path = "crates/stream-codecs" }
|
||||
craft-testkit = { path = "crates/testkit" }
|
||||
craft-text = { path = "crates/text" }
|
||||
craft-tcraft = { path = "crates/tcraft" }
|
||||
craft-texture = { path = "crates/texture" }
|
||||
|
||||
@@ -367,6 +367,13 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
|
||||
check for drawing a tree with resvg. Apps take usvg from here, not as their own dependency.
|
||||
- [`craft-svg-paths`](crates/svg-paths/src/lib.rs): the former name of `craft-svg`'s `paths`,
|
||||
`color` and `write`; re-exports them for existing users.
|
||||
- [`craft-testkit`](crates/testkit/README.md): test support for the apps' own tests. Its `guards` module
|
||||
reports source patterns that broke platforms before: muda's predefined Quit item, WGSL float literals
|
||||
out of range, and `std::time::Instant` in code the web build runs. Its `deadline` module fails a test
|
||||
whose body runs past a limit, or whose condition does not hold in time; a variable the caller names
|
||||
replaces each limit. Its `hostile` module runs a parser on every truncation and on seeded damaged
|
||||
copies of a valid input, and fails the test with the case that panicked or did not finish. Not
|
||||
published.
|
||||
- [`craft-text`](crates/text/README.md): text processing below the composer. Its `hyphen` module finds
|
||||
hyphenation points, with a bundled US English dictionary and patterns and optional hypher patterns
|
||||
for 47 more languages (one cargo feature each).
|
||||
|
||||
@@ -8,5 +8,8 @@ rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
publish = false
|
||||
|
||||
[dependencies]
|
||||
craft-testkit = { workspace = true }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -1,89 +1,26 @@
|
||||
//! Deterministic hostile-input hammer: truncations, bit flips and extreme-value overwrites of a
|
||||
//! valid seed, each run under `catch_unwind` with a time budget. No dependencies, no `unsafe`.
|
||||
//! Deterministic hostile-input hammer, from [`craft_testkit::hostile`]: [`hammer`] runs a parser on
|
||||
//! a valid seed, every truncation of it and seeded damage of it, one case after another on a worker
|
||||
//! thread with a time limit for each case. [`Rng`], [`mutations`] and the other names here are
|
||||
//! `craft_testkit`'s.
|
||||
|
||||
use std::fmt::Write as _;
|
||||
use std::panic::{AssertUnwindSafe, catch_unwind};
|
||||
use std::time::{Duration, Instant};
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub use craft_testkit::hostile::{HammerConfig, HammerError};
|
||||
pub use craft_testkit::hostile::{MAX_CASES, Mutation, Mutations, Rng, mutations};
|
||||
|
||||
/// xorshift64* PRNG.
|
||||
pub struct Rng(pub u64);
|
||||
/// The variable that multiplies [`hammer`]'s iterations.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
const MULTIPLIER_VARIABLE: &str = "CRAFT_MEDIA_HOSTILE_MULT";
|
||||
|
||||
impl Rng {
|
||||
/// The next value.
|
||||
pub fn next_u64(&mut self) -> u64 {
|
||||
self.0 ^= self.0 >> 12;
|
||||
self.0 ^= self.0 << 25;
|
||||
self.0 ^= self.0 >> 27;
|
||||
self.0.wrapping_mul(0x2545_F491_4F6C_DD1D)
|
||||
}
|
||||
/// A value below `n` (0 for `n` = 0).
|
||||
pub fn below(&mut self, n: usize) -> usize {
|
||||
(self.next_u64() % n.max(1) as u64) as usize
|
||||
}
|
||||
}
|
||||
|
||||
const EXTREMES: [u64; 8] = [0, 1, 0x7F, 0x80, 0xFF, 0xFFFF, 0x8000_0000, u64::MAX];
|
||||
|
||||
fn run_one(name: &str, what: &str, data: &[u8], run: &dyn Fn(&[u8])) {
|
||||
let t = Instant::now();
|
||||
let r = catch_unwind(AssertUnwindSafe(|| run(data)));
|
||||
let el = t.elapsed();
|
||||
assert!(r.is_ok(), "{name}: PANIC on {what} (len {})", data.len());
|
||||
assert!(el < Duration::from_secs(5), "{name}: HANG/slow ({el:?}) on {what} (len {})", data.len());
|
||||
}
|
||||
|
||||
/// Run `run` over every truncation of `seed` (stepped beyond 4096 lengths), then `iters` random
|
||||
/// mutations (bit flips, random bytes, 1/2/4/8-byte extreme-value overwrites in both
|
||||
/// endiannesses, deletions, truncation). Panics, naming the mutation, on the first panic or on
|
||||
/// a run slower than 5 s.
|
||||
pub fn hammer(name: &str, seed: &[u8], iters: usize, run: impl Fn(&[u8])) {
|
||||
run_one(name, "the unmodified seed", seed, &run);
|
||||
let step = (seed.len() / 4096).max(1);
|
||||
for n in (0..seed.len()).step_by(step) {
|
||||
run_one(name, &format!("truncation to {n}"), &seed[..n], &run);
|
||||
}
|
||||
let mut rng = Rng((0x5EED_F00D ^ seed.len() as u64) | 1);
|
||||
// CRAFT_MEDIA_HOSTILE_MULT=10 runs ten times the iterations (soak runs).
|
||||
let mult = std::env::var("CRAFT_MEDIA_HOSTILE_MULT").ok().and_then(|v| v.parse::<usize>().ok()).unwrap_or(1);
|
||||
for _ in 0..iters.saturating_mul(mult) {
|
||||
let mut d = seed.to_vec();
|
||||
let mut what = String::new();
|
||||
for _ in 0..1 + rng.below(4) {
|
||||
if d.is_empty() {
|
||||
break;
|
||||
}
|
||||
let pos = rng.below(d.len());
|
||||
match rng.below(7) {
|
||||
0 => {
|
||||
let b = rng.below(8);
|
||||
d[pos] ^= 1 << b;
|
||||
let _ = write!(what, "flip[{pos}].{b} ");
|
||||
}
|
||||
1 => {
|
||||
d[pos] = rng.next_u64() as u8;
|
||||
let _ = write!(what, "set[{pos}]={:#x} ", d[pos]);
|
||||
}
|
||||
2..=4 => {
|
||||
let w = [1usize, 2, 4, 8][rng.below(4)];
|
||||
let v = EXTREMES[rng.below(EXTREMES.len())];
|
||||
let be = rng.below(2) == 0;
|
||||
let bytes = if be { v.to_be_bytes()[8 - w..].to_vec() } else { v.to_le_bytes()[..w].to_vec() };
|
||||
let end = (pos + w).min(d.len());
|
||||
d[pos..end].copy_from_slice(&bytes[..end - pos]);
|
||||
let _ = write!(what, "put{w}{}[{pos}]={v:#x} ", if be { "be" } else { "le" });
|
||||
}
|
||||
5 => {
|
||||
let n = 1 + rng.below(16);
|
||||
let end = (pos + n).min(d.len());
|
||||
d.drain(pos..end);
|
||||
let _ = write!(what, "del[{pos}..{end}] ");
|
||||
}
|
||||
_ => {
|
||||
d.truncate(pos);
|
||||
let _ = write!(what, "trunc{pos} ");
|
||||
}
|
||||
}
|
||||
}
|
||||
run_one(name, &what, &d, &run);
|
||||
}
|
||||
/// Run `run` over `seed`, every truncation of it, then `iters` random mutations (bit flips, random
|
||||
/// bytes, 1/2/4/8-byte extreme-value overwrites in both endiannesses, deletions, truncation). The
|
||||
/// cases run on a worker thread with a limit of 5 s each, which `CRAFT_TEST_TIMEOUT_SECS` replaces.
|
||||
/// Panics on the first case that panics or does not finish, naming `name`, the case, its length and
|
||||
/// the panic's message.
|
||||
///
|
||||
/// `CRAFT_MEDIA_HOSTILE_MULT=10` runs ten times the iterations (soak runs). A value that is not a
|
||||
/// whole number above zero panics, naming the variable, before `run` is called.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
#[track_caller]
|
||||
pub fn hammer(name: &str, seed: &[u8], iters: usize, run: impl Fn(&[u8]) + Send + Sync + 'static) {
|
||||
craft_testkit::hostile::hammer(&HammerConfig::new(name, iters).with_multiplier_variable(MULTIPLIER_VARIABLE), seed, run);
|
||||
}
|
||||
@@ -11,8 +11,9 @@
|
||||
//! Fixtures are generated into `$CRAFT_MEDIA_FIXTURES_DIR` or `<workspace>/target/fixtures`, one
|
||||
//! directory per crate or format, and never committed.
|
||||
//!
|
||||
//! [`hostile::hammer`] runs a parser over truncations and mutations of a valid input and fails on
|
||||
//! a panic or a slow run; `CRAFT_MEDIA_HOSTILE_MULT=10` runs ten times the iterations.
|
||||
//! [`hostile::hammer`] runs a parser over every truncation and mutations of a valid input and fails
|
||||
//! on a panic or a case that does not finish in time; `CRAFT_MEDIA_HOSTILE_MULT=10` runs ten times
|
||||
//! the iterations. It and [`with_deadline`] call `craft_testkit`.
|
||||
//!
|
||||
//! Failing loudly is what a test helper is for: a wrong `CRAFT_MEDIA_*` setting or an unwritable
|
||||
//! fixture directory panics, which fails the test that asked.
|
||||
@@ -216,35 +217,19 @@ pub fn set_progress(label: &str) {
|
||||
eprintln!("PROGRESS: {label}");
|
||||
}
|
||||
|
||||
/// Run closure with an execution deadline, terminating with a panic if the limit is exceeded.
|
||||
#[allow(clippy::panic)]
|
||||
/// Run closure with an execution deadline of `secs` seconds, which `CRAFT_TEST_TIMEOUT_SECS`
|
||||
/// replaces, through [`craft_testkit::with_deadline`]. Panics when the closure does not finish in
|
||||
/// time, when `secs` is zero, or when the variable is not a whole number of seconds above zero.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
#[track_caller]
|
||||
pub fn with_deadline<F: FnOnce() + Send + 'static>(secs: u64, f: F) {
|
||||
let timeout_secs = std::env::var("CRAFT_TEST_TIMEOUT_SECS").ok().and_then(|s| s.parse().ok()).unwrap_or(secs);
|
||||
with_timeout(timeout_secs, f);
|
||||
with_limit(craft_testkit::DeadlineLimit::new(std::time::Duration::from_secs(secs)), f);
|
||||
}
|
||||
|
||||
#[allow(clippy::panic)]
|
||||
fn with_timeout<F: FnOnce() + Send + 'static>(timeout_secs: u64, f: F) {
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let handle = std::thread::spawn(move || {
|
||||
f();
|
||||
let _ = tx.send(());
|
||||
});
|
||||
match rx.recv_timeout(std::time::Duration::from_secs(timeout_secs)) {
|
||||
Ok(()) => {
|
||||
if let Err(payload) = handle.join() {
|
||||
std::panic::resume_unwind(payload);
|
||||
}
|
||||
}
|
||||
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
|
||||
panic!("test exceeded execution deadline of {timeout_secs} seconds");
|
||||
}
|
||||
Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {
|
||||
if let Err(payload) = handle.join() {
|
||||
std::panic::resume_unwind(payload);
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
#[track_caller]
|
||||
fn with_limit<F: FnOnce() + Send + 'static>(limit: craft_testkit::DeadlineLimit<'_>, f: F) {
|
||||
craft_testkit::with_deadline(limit, "the test body", f);
|
||||
}
|
||||
|
||||
/// Return fixture directory for a crate/category, creating it if needed.
|
||||
@@ -339,10 +324,31 @@ Encoders:
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "test exceeded execution deadline")]
|
||||
#[should_panic(expected = "the test body did not finish within 1s")]
|
||||
fn with_deadline_times_out() {
|
||||
with_timeout(1, || {
|
||||
std::thread::sleep(std::time::Duration::from_secs(3));
|
||||
let (_keep, rx) = std::sync::mpsc::channel::<()>();
|
||||
let limit = craft_testkit::DeadlineLimit::new(std::time::Duration::from_secs(1)).with_variable("CRAFT_MEDIA_TESTKIT_UNSET_IN_TESTS");
|
||||
with_limit(limit, move || {
|
||||
let _ = rx.recv();
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "the default limit is zero")]
|
||||
fn with_deadline_of_zero_fails() {
|
||||
with_deadline(0, || {});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_deadline_names_a_bad_timeout_variable() {
|
||||
let out = Command::new(std::env::current_exe().unwrap())
|
||||
.args(["--exact", "tests::with_deadline_completes", "--nocapture", "--test-threads=1"])
|
||||
.env("CRAFT_TEST_TIMEOUT_SECS", "soon")
|
||||
.output()
|
||||
.unwrap();
|
||||
let printed = format!("{}{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr));
|
||||
assert!(printed.contains("running 1 test"), "the child did not run the test:\n{printed}");
|
||||
assert!(!out.status.success(), "the child passed:\n{printed}");
|
||||
assert!(printed.contains(r#"CRAFT_TEST_TIMEOUT_SECS="soon" is not a whole number of seconds"#), "{printed}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
//! `hostile::hammer` as the media crates' tests call it.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::process::Command;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use craft_media_testkit::hostile::hammer;
|
||||
|
||||
/// Set only in the child processes of `run_child`: the value names what `child_hammer` does.
|
||||
const CHILD_VARIABLE: &str = "CRAFT_MEDIA_TESTKIT_HOSTILE_CHILD";
|
||||
|
||||
/// Overrides the two-minute limit on each child process of `run_child`. The limit does not read
|
||||
/// `CRAFT_TEST_TIMEOUT_SECS`, which `a_hang_fails_at_the_deadline` sets for the child.
|
||||
const WATCHDOG_VARIABLE: &str = "CRAFT_MEDIA_TESTKIT_CHILD_WATCHDOG_SECS";
|
||||
|
||||
/// Runs `child_hammer` in a child process with `vars` set, and returns what it printed. The child
|
||||
/// must fail. A child that never returns fails the test after two minutes.
|
||||
fn run_child(vars: &'static [(&'static str, &'static str)]) -> String {
|
||||
let limit = craft_testkit::DeadlineLimit::new(Duration::from_secs(120)).with_variable(WATCHDOG_VARIABLE);
|
||||
craft_testkit::with_deadline(limit, "the child test", move || {
|
||||
let out = Command::new(std::env::current_exe().unwrap())
|
||||
.args(["--exact", "child_hammer", "--nocapture", "--test-threads=1"])
|
||||
.envs(vars.iter().copied())
|
||||
.output()
|
||||
.unwrap();
|
||||
let printed = format!("{}{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr));
|
||||
assert!(printed.contains("running 1 test"), "the child did not run the test:\n{printed}");
|
||||
assert!(!out.status.success(), "the child passed:\n{printed}");
|
||||
printed
|
||||
})
|
||||
}
|
||||
|
||||
/// Runs in the child processes of `run_child`. Without `CHILD_VARIABLE` it hammers a small seed and
|
||||
/// passes; with `hang`, its parser never returns.
|
||||
#[test]
|
||||
fn child_hammer() {
|
||||
if std::env::var_os(CHILD_VARIABLE).is_some_and(|v| v == "hang") {
|
||||
hammer("hang", &[1, 2, 3], 0, |_| {
|
||||
loop {
|
||||
std::thread::park();
|
||||
}
|
||||
});
|
||||
} else {
|
||||
hammer("small", &[1, 2, 3], 10, |_| {});
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_hang_fails_at_the_deadline() {
|
||||
let printed = run_child(&[(CHILD_VARIABLE, "hang"), ("CRAFT_TEST_TIMEOUT_SECS", "1")]);
|
||||
assert!(printed.contains("hang: the unmodified seed (len 3) did not finish within 1s, set by CRAFT_TEST_TIMEOUT_SECS"), "{printed}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bad_multiplier_names_the_variable() {
|
||||
let printed = run_child(&[("CRAFT_MEDIA_HOSTILE_MULT", "ten")]);
|
||||
assert!(printed.contains(r#"small: CRAFT_MEDIA_HOSTILE_MULT="ten" is not a whole number written in digits"#), "{printed}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_panic_keeps_its_message() {
|
||||
let payload = std::panic::catch_unwind(|| hammer("demo", &[9, 8, 7, 6], 0, |d| assert!(d.len() != 2, "boom at {}", d.len()))).unwrap_err();
|
||||
let message = payload.downcast_ref::<String>().cloned().unwrap_or_default();
|
||||
assert!(message.starts_with("demo: truncation to 2 (len 2): the parser panicked: "), "{message}");
|
||||
assert!(message.contains("boom at 2"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_truncation_of_a_large_seed_runs() {
|
||||
let seed: Vec<u8> = (0..8_192u32).map(|i| i as u8).collect();
|
||||
let lengths = Arc::new(Mutex::new(BTreeSet::new()));
|
||||
let record = Arc::clone(&lengths);
|
||||
hammer("large", &seed, 0, move |d| {
|
||||
record.lock().unwrap().insert(d.len());
|
||||
});
|
||||
// The seed and its 8,192 truncations.
|
||||
assert_eq!(lengths.lock().unwrap().len(), 8_193);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_child_timeout_variable_does_not_set_the_watchdog() {
|
||||
// `a_hang_fails_at_the_deadline` sets `CRAFT_TEST_TIMEOUT_SECS` for its child. Run it with a value
|
||||
// that `with_deadline` rejects; it passes only if `run_child`'s watchdog reads another variable.
|
||||
let out = Command::new(std::env::current_exe().unwrap())
|
||||
.args(["--exact", "a_hang_fails_at_the_deadline", "--nocapture", "--test-threads=1"])
|
||||
.env("CRAFT_TEST_TIMEOUT_SECS", "not a number")
|
||||
.output()
|
||||
.unwrap();
|
||||
let printed = format!("{}{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr));
|
||||
assert!(printed.contains("running 1 test"), "the test did not run:\n{printed}");
|
||||
assert!(out.status.success(), "{printed}");
|
||||
}
|
||||
@@ -89,7 +89,8 @@ fn elementary_stream_frame_parsers_never_panic() {
|
||||
ac3.extend([1u8; 60]);
|
||||
let seeds = [adts_frame(3), mp2_frame(1), ac3, mpeg2_frame(1, 0, false, 64), vec![0x56, 0xE0, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80]];
|
||||
for (i, s) in seeds.iter().enumerate() {
|
||||
craft_media_testkit::hostile::hammer(&format!("es seed {i}"), s, 40_000, |d| {
|
||||
let codecs = codecs.clone();
|
||||
craft_media_testkit::hostile::hammer(&format!("es seed {i}"), s, 40_000, move |d| {
|
||||
for c in &codecs {
|
||||
let _ = frame_bytes(c, d);
|
||||
let _ = frame_info(c, d);
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
[package]
|
||||
name = "craft-testkit"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
description = "Source-level platform regression guards and verification utilities for the Craft application suite"
|
||||
publish = false
|
||||
|
||||
[dependencies]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,268 @@
|
||||
# craft-testkit
|
||||
|
||||
Source-level platform regression guards and verification utilities for the Craft application suite.
|
||||
|
||||
## Overview
|
||||
|
||||
`craft-testkit` provides lightweight, non-panicking source guards that prevent applications in the Craft ecosystem from repeating platform defects that were previously diagnosed and resolved in peer applications:
|
||||
|
||||
1. **`predefined_quit`**: Flags direct calls to `muda`'s `PredefinedMenuItem::quit` outside comments, strings and character literals, also with whitespace or a line break around `::`, in the qualified form `<muda::PredefinedMenuItem>::quit`, and through a `use muda::PredefinedMenuItem as ...` rename. A type of the same name reached through another crate or module, such as `other::PredefinedMenuItem::quit`, is not flagged. On macOS, this predefined menu item routes directly to `NSApplication terminate:`, completely bypassing application dirty document checks and unsaved changes save prompts. Applications should instead wire Quit through `craft-quit` lifecycle interceptors.
|
||||
2. **`wgsl_float_literals` / `wgsl_float_literals_in`**: Scans WGSL shader sources and flags decimal and hexadecimal float literals whose value, rounded to `f64`, exceeds `f32::MAX` (such as `3.40282347e38` or `0x1p+500f`), or 65504, the largest finite `f16`, for a literal with an `h` suffix. A literal that exceeds the maximum by less than `f64` precision rounds to the maximum and is not reported. Integer literals are not checked. Some native compilers round such literals down, but browser WebGPU compilers reject them with pipeline compilation errors, forcing applications into CPU fallback rendering. `f32::MAX` can be written as the hex float `0x1.fffffep+127f`, which equals it, or as the decimal `3.4028234663852886e38`, which rounds to it.
|
||||
3. **`std_instant`**: Flags `std::time::Instant` outside comments and strings: the full path, `<m>::Instant` where a `use` binds `m` to `std::time`, and each `use` that imports `std::time::Instant` or `std::time::*`, under any name. The parts of a path may be split by whitespace, comments or line breaks. Other crates' `Instant` (such as `web_time::Instant` or `tokio::time::Instant`) is not flagged, and neither is an item whose `cfg` attribute is false on `wasm32-unknown-unknown`; code before that attribute or after its item on the same line is still checked. A `mod name;` inside such an item also skips the module's file (`name.rs` or `name/mod.rs`, or the file a `#[path]` attribute names) and the files of the modules declared in it, when they are among the files passed in and no declaration outside such items also reaches them. On that target, `std::time::Instant::now()` panics at runtime with `"time not implemented on this platform"`. Code targeting the web must use `web_time::Instant`.
|
||||
4. **`rust_sources`**: Recursively discovers a crate's `.rs` source files while excluding integration tests (`tests/`), test modules (`tests.rs`, `tests_*.rs`, `*_tests.rs`, `*_test.rs`), build artifacts (`target/`, any folder holding a `CACHEDIR.TAG` file that starts with the line `Signature: 8a477f597d28d172789f06886806bc55`, `.git/` and other folders whose name starts with `.`) and symbolic links, which it does not follow. A file over 8 MiB, more than 256 MiB in all or more than 50,000 files returns an error; `rust_sources_with_limits` takes other limits. It is not built for `wasm32`.
|
||||
|
||||
## Epistemic Scope: Syntactic Guards vs Runtime Validation
|
||||
|
||||
Source-level guards provide deterministic syntactic evidence about codebase structure:
|
||||
|
||||
### What They Establish
|
||||
- Checked source files do not contain forbidden symbol invocations, known out-of-range literal representations, or platform-hazardous imports.
|
||||
- Reusable, deterministic checks that run in milliseconds as regular Cargo tests without external runners.
|
||||
|
||||
### What They Cannot Establish
|
||||
- **Runtime execution safety**: Syntactic checks do not verify whether code branches actually execute at runtime.
|
||||
- **Semantic type resolution**: The scanners follow `use` renames within one file only. They do not resolve paths through other modules or re-exports, a rename of `std` itself, or expand macros.
|
||||
- **Complex target predicates**: `std_instant` evaluates `cfg` predicates (`all`, `any`, `not`, `unix`, `windows` and the `target_*` keys) for `wasm32-unknown-unknown`, on outer attributes for the item that follows and on inner attributes (`#![cfg(...)]`) for the rest of the module, and on `mod name;` declarations for the module's file and its descendants. It compares file paths as text, so the files must come from one `rust_sources` call or be named the same way. A predicate that depends on a feature or on `test` is not known from the source, so its item is checked. Applications should scope `std_instant` tests to the specific crates that compile into their web distribution.
|
||||
- **Driver and host OS behavior**: Syntactic checks do not verify WebGPU driver compatibility or host window manager event dispatch.
|
||||
|
||||
Source guards serve as fast, zero-dependency regression gates in application test suites, complementing platform integration tests and CI workflows.
|
||||
|
||||
## Consumer Adoption Examples
|
||||
|
||||
### 1. PhotoCraft: Adopt `wgsl_float_literals` and `predefined_quit`
|
||||
|
||||
In `photocraft/crates/gpu/src/lib.rs`, replace the in-tree `decimal_float_literals` helper with `craft_testkit::wgsl_float_literals`:
|
||||
|
||||
```rust
|
||||
#[test]
|
||||
fn shader_float_literals_fit_f32() {
|
||||
let bad = craft_testkit::wgsl_float_literals(SHADER);
|
||||
assert!(
|
||||
bad.is_empty(),
|
||||
"float literals outside f32 finite range reject on WebGPU (Tint):\n{}",
|
||||
bad.iter().map(|f| f.to_string()).collect::<Vec<_>>().join("\n")
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
In `photocraft/apps/photocraft/tests/native_menu_tests.rs`:
|
||||
|
||||
```rust
|
||||
#[test]
|
||||
fn no_unintercepted_predefined_quit_in_menu() {
|
||||
let sources = craft_testkit::rust_sources("src").expect("scan sources");
|
||||
let findings = craft_testkit::predefined_quit(&sources);
|
||||
assert!(
|
||||
findings.is_empty(),
|
||||
"predefined muda Quit bypasses dirty document save prompts:\n{}",
|
||||
findings.iter().map(|f| f.to_string()).collect::<Vec<_>>().join("\n")
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
### 2. FilmCraft: Adopt `std_instant` and `wgsl_float_literals`
|
||||
|
||||
In `filmcraft/crates/engine/src/wasm_clock_tests.rs`, replace in-tree textual search with `craft_testkit::std_instant`:
|
||||
|
||||
```rust
|
||||
#[test]
|
||||
fn no_std_instant_in_code_the_web_build_runs() {
|
||||
let sources = craft_testkit::rust_sources(Path::new(env!("CARGO_MANIFEST_DIR")).join("src"))
|
||||
.expect("scan engine sources");
|
||||
let findings = craft_testkit::std_instant(&sources);
|
||||
assert!(
|
||||
findings.is_empty(),
|
||||
"std::time::Instant panics on wasm32; use web_time::Instant:\n{}",
|
||||
findings.iter().map(|f| f.to_string()).collect::<Vec<_>>().join("\n")
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
## deadline
|
||||
|
||||
The `deadline` module is not built for `wasm32`.
|
||||
|
||||
`with_deadline(limit, label, f)` runs `f` on its own thread and returns its value. When `f` does not
|
||||
finish within the limit, the test fails with a message that names `label`, the limit and the variable
|
||||
that sets it. A panic in `f` is raised again in the test with its original payload. The limit counts
|
||||
from the call, including the time spent starting the thread. When `f` returns or panics after the
|
||||
limit has passed, the test fails with the label and the limit, and with the panic's message if it has
|
||||
one.
|
||||
|
||||
`wait_until(limit, what, cond)` calls `cond` on the test's thread until it returns `true`. It calls
|
||||
`cond` at once, then after pauses that start at 1 ms and double up to 50 ms, and does not call it
|
||||
again once the limit has passed. When the limit passes first, or `cond` returns `true` only after the
|
||||
limit has passed, the test fails with a message that names `what`.
|
||||
|
||||
```rust
|
||||
use std::time::Duration;
|
||||
use craft_testkit::deadline::{DeadlineLimit, wait_until, with_deadline};
|
||||
|
||||
// The default of 30 s; CRAFT_TEST_TIMEOUT_SECS replaces it.
|
||||
let pixels = with_deadline(Duration::from_secs(30), "decode seed 7", move || decode(&bytes));
|
||||
|
||||
// The app's own variable replaces the default instead.
|
||||
let limit = DeadlineLimit::new(Duration::from_secs(30)).with_variable("FILMCRAFT_HOSTILE_TIMEOUT_SECS");
|
||||
with_deadline(limit, "hostile input 12", move || parse(&input));
|
||||
|
||||
wait_until(Duration::from_secs(10), "the export to finish", || job.is_done());
|
||||
```
|
||||
|
||||
### The limit variable
|
||||
|
||||
A `Duration` reads `CRAFT_TEST_TIMEOUT_SECS`. `DeadlineLimit::with_variable` names another variable.
|
||||
The variable is read on every call; neither function sets or caches it.
|
||||
|
||||
| Value | Result |
|
||||
|---|---|
|
||||
| Not set | The default limit applies. |
|
||||
| Decimal digits above zero, such as `30` or `007` | That many seconds replace the default. |
|
||||
| Empty, signed (`+5`, `-5`), fractional (`1.5`), with spaces or units (`5s`), zero, or larger than a `u64` | The test fails with the variable's name and value, before `f` runs or `cond` is called. |
|
||||
| Not Unicode | The test fails with the variable's name, before `f` runs or `cond` is called. |
|
||||
|
||||
A zero default, a variable name that is empty or holds `=` or a NUL character, and a limit that ends past
|
||||
the latest time the platform's clock can represent also fail the test before any work starts.
|
||||
|
||||
FilmCraft's `filmcraft-testkit` uses the default when the variable holds something other than a
|
||||
number or holds zero. This module fails the test instead. `craft-media-testkit`'s `with_deadline`
|
||||
calls this module's.
|
||||
|
||||
### Limits
|
||||
|
||||
- A body that runs past its limit fails the test, but its thread is left running until the test
|
||||
process exits. A body that never returns can keep the test binary alive after its tests finish.
|
||||
- `wait_until` checks its limit only between calls of `cond`. A `cond` that blocks is not interrupted,
|
||||
so it must return promptly.
|
||||
- Neither function returns elapsed time. Tests of this crate use channel handshakes, and the only time
|
||||
bounds in them are the limits under test.
|
||||
|
||||
## hostile
|
||||
|
||||
`mutations(seed, rounds)` yields `rounds` damaged copies of a valid seed. Each round makes one to four
|
||||
edits: a bit flip, a random byte, a 1, 2, 4 or 8-byte overwrite with an extreme value
|
||||
(0, 1, 0x7F, 0x80, 0xFF, 0xFFFF, 0x8000_0000 or `u64::MAX`) in either byte order, a deletion of up to
|
||||
16 bytes, or a truncation. Each copy carries a label that lists its edits, such as
|
||||
`flip[3].5 del[7..9]`. The xorshift64* generator behind it is seeded from the seed's length, so the
|
||||
same seed length and round count give the same edits on every run and every platform. An empty seed
|
||||
yields no copies. `mutations` and `Rng` build for `wasm32`.
|
||||
|
||||
`hammer(&config, seed, parse)` runs `parse` on the unmodified seed, on every truncation of it
|
||||
(lengths 0 to one less than its length), and on `mutations(seed, rounds)`. The cases run one after
|
||||
another on one worker thread, and the test's thread fails the test when a case runs past its limit.
|
||||
A case that finishes after its limit also fails the test. The test fails on the first case that
|
||||
panics or does not finish, with a message that names the config's label, the case
|
||||
(`the unmodified seed`, `truncation to N` or the edits), the input's length and the panic's message.
|
||||
`parse` must be `Fn(&[u8]) + Send + Sync + 'static`, since a case that does not finish keeps running
|
||||
on the worker thread after the test fails. `hammer` is not built for `wasm32`.
|
||||
|
||||
```rust
|
||||
use std::time::Duration;
|
||||
use craft_testkit::DeadlineLimit;
|
||||
use craft_testkit::hostile::{HammerConfig, hammer};
|
||||
|
||||
// 2,000 rounds times CRAFT_HOSTILE_MULT; 5 s per case, or CRAFT_TEST_TIMEOUT_SECS.
|
||||
hammer(&HammerConfig::new("png seed 3", 2_000), &seed, |data| {
|
||||
let _ = decode(data);
|
||||
});
|
||||
|
||||
// The app's own variables instead.
|
||||
let config = HammerConfig::new("mxf primer", 5_000)
|
||||
.with_multiplier_variable("CRAFT_MEDIA_HOSTILE_MULT")
|
||||
.with_timeout(DeadlineLimit::new(Duration::from_secs(5)).with_variable("FILMCRAFT_HOSTILE_TIMEOUT_SECS"));
|
||||
hammer(&config, &primer, |data| {
|
||||
let _ = parse_primer(data);
|
||||
});
|
||||
```
|
||||
|
||||
### The rounds
|
||||
|
||||
The rounds are the config's iterations times the multiplier variable, `CRAFT_HOSTILE_MULT` unless
|
||||
`with_multiplier_variable` names another. The limit for each case is 5 s, replaced by
|
||||
`CRAFT_TEST_TIMEOUT_SECS` or the variable that `with_timeout` names, as in `deadline`. Both variables
|
||||
are read once, before any case runs. `HammerConfig::rounds` returns the rounds or the error that `hammer` would fail with.
|
||||
|
||||
| Multiplier value | Result |
|
||||
|---|---|
|
||||
| Not set | The multiplier is 1. |
|
||||
| Decimal digits above zero, such as `10` | The rounds are multiplied by that number. |
|
||||
| Empty, signed, fractional, with spaces, zero, or larger than a `usize` | The test fails with the variable's name and value before `parse` is called. |
|
||||
| Not Unicode | The test fails with the variable's name before `parse` is called. |
|
||||
|
||||
A product of the iterations and the multiplier that does not fit in a `usize`, a call that would run
|
||||
more than `MAX_CASES` (10,000,000) cases in all, and a limit variable that `deadline` would reject
|
||||
also fail the test before `parse` is called.
|
||||
|
||||
Changes from `craft-media-testkit`'s `hammer`:
|
||||
|
||||
- A case that does not finish fails the test after its limit with the case's label. Before, a case
|
||||
was timed after it returned and failed only when it took longer than 5 s.
|
||||
- A panic's message is part of the failure.
|
||||
- Every truncation runs. Before, a seed of 8,192 bytes or more ran every `len / 4096`th truncation.
|
||||
- The caller names the multiplier variable. Before, it was always `CRAFT_MEDIA_HOSTILE_MULT`, and a
|
||||
value that was not a number counted as 1.
|
||||
- An empty seed runs once. Before, it ran once more for each round.
|
||||
- A mutation's label joins its edits with single spaces. Before, every edit ended with a space, so
|
||||
each label ended with one.
|
||||
- `parse` is owned and runs on a worker thread. A closure that borrows from the test needs its data
|
||||
moved or shared with an `Arc`.
|
||||
|
||||
## Origin
|
||||
|
||||
### guards
|
||||
|
||||
| Module | Source | Path | Commit |
|
||||
|---|---|---|---|
|
||||
| `guards` | storytold/craft-libs#163 (zachshallbetter), with the rules of PhotoCraft's shader float test and FilmCraft's web clock test | `crates/testkit/src/guards.rs` | `e92825e427` |
|
||||
|
||||
### deadline
|
||||
|
||||
| Code | Repository | Path | SHA |
|
||||
|---|---|---|---|
|
||||
| `with_deadline`, its panic and timeout tests (`a_slow_body_passes`, `a_panic_in_the_body_is_re_raised`, `a_hang_fails_and_names_the_input`) | storytold/filmcraft | `crates/testkit/src/deadline.rs` | `cff7e144` |
|
||||
| `with_deadline` under `CRAFT_TEST_TIMEOUT_SECS`, tests `with_deadline_completes` and `with_deadline_times_out` | storytold/craft-libs | `crates/media-testkit/src/lib.rs` | `75a654f4` |
|
||||
| `within`, a deadline that returns the body's value | storytold/lightcraft | `crates/engine/src/demo/tests.rs` | `487833ad` |
|
||||
| `within`, a deadline that returns the body's value | storytold/gridcraft | `crates/math/src/lib.rs` | `c2aba7a3` |
|
||||
| `wait_until` | storytold/photocraft | `crates/ui-egui/src/canvas/paint_thread_tests.rs` | `0d3b1e85` |
|
||||
| `wait_until` | storytold/lightcraft | `crates/engine/src/availability.rs` | `7ebbb17a` |
|
||||
| `wait_until` | storytold/effectcraft | `crates/engine/src/footage/footage_reload.rs` | `53576d22` |
|
||||
| `wait_until` | storytold/filmcraft | `crates/ui-egui/tests/it/export_ui.rs` | `cff7e144` |
|
||||
|
||||
### hostile
|
||||
|
||||
| Code | Repository | Path | SHA |
|
||||
|---|---|---|---|
|
||||
| `Rng`, the edits of `mutations` and `hammer` | storytold/craft-libs | `crates/media-testkit/src/hostile.rs` | `75a654f4` |
|
||||
|
||||
## Migration
|
||||
|
||||
### guards
|
||||
|
||||
| Consumer | Guard | Replaces | State |
|
||||
|---|---|---|---|
|
||||
| PhotoCraft | `wgsl_float_literals` | `crates/gpu/src/tests.rs` scanner | Not started |
|
||||
| FilmCraft | `std_instant` | `crates/engine/src/wasm_clock_tests.rs` search | Not started |
|
||||
|
||||
### deadline
|
||||
|
||||
| Consumer | Code it replaces | State |
|
||||
|---|---|---|
|
||||
| FilmCraft | `filmcraft-testkit`'s `with_deadline` (`FILMCRAFT_HOSTILE_TIMEOUT_SECS`) and the export UI tests' `wait_until` | Pending |
|
||||
| LightCraft | `within` in the demo tests and `wait_until` in the availability tests | Pending |
|
||||
| GridCraft | `within` in the tests of its `crates/math` | Pending |
|
||||
| PhotoCraft | `wait_until` in the paint thread tests | Pending |
|
||||
| EffectCraft | `wait_until` in the footage reload tests | Pending |
|
||||
| `craft-media-testkit` | `with_deadline` (`CRAFT_TEST_TIMEOUT_SECS`) | Done |
|
||||
|
||||
An adoption keeps the variable name the app uses today.
|
||||
|
||||
### hostile
|
||||
|
||||
| Consumer | Code it replaces | State |
|
||||
|---|---|---|
|
||||
| `craft-media-testkit` | `hostile::hammer` (`CRAFT_MEDIA_HOSTILE_MULT`), called by the ISO BMFF, AVI, Matroska, MXF and MPEG TS hostile tests | Done |
|
||||
| `craft-cfb` | its `hostile_input_never_panics` test, through `craft-media-testkit` | Done |
|
||||
| CADCraft | the mutation loop of `hostile_files_never_panic` in `crates/fonts/src/shx/tests.rs` | Pending |
|
||||
| TextureCraft | the `hammer` of `malformed_meshes_are_errors` in `crates/formats/src/mesh_import/tests.rs` | Pending |
|
||||
|
||||
CADCraft's and TextureCraft's loops make other edits than `mutations`. Their adoptions say which
|
||||
cases change, and keep the test names.
|
||||
@@ -0,0 +1,788 @@
|
||||
//! Limits on how long a test waits.
|
||||
//!
|
||||
//! [`with_deadline`] runs a test body on its own thread and returns its value. When the body does not
|
||||
//! finish within the limit, the test fails with the call's label, the limit and the variable that sets
|
||||
//! it. A panic in the body is raised again in the test with its original payload.
|
||||
//!
|
||||
//! [`wait_until`] calls a condition on the test's thread until it returns `true`, pausing between
|
||||
//! calls, and fails the test with its description when the limit passes first.
|
||||
//!
|
||||
//! A limit is a default [`Duration`] and the name of a variable that replaces it. A plain `Duration`
|
||||
//! reads [`DEFAULT_VARIABLE`]; [`DeadlineLimit::with_variable`] names another one. The variable holds
|
||||
//! whole seconds as decimal digits and is read on every call. A value that is empty, signed,
|
||||
//! fractional, not Unicode, zero, or larger than a `u64` fails the test that reads it, as does a zero
|
||||
//! default and a limit past the latest time the platform's clock can represent.
|
||||
//!
|
||||
//! Neither function can stop code that does not return. A body that runs past its limit fails the
|
||||
//! test, and its thread is left running until the test process exits. A condition that blocks is
|
||||
//! not interrupted: [`wait_until`] checks its limit only between calls, so a condition must return
|
||||
//! promptly.
|
||||
|
||||
use std::any::Any;
|
||||
use std::ffi::OsString;
|
||||
use std::fmt;
|
||||
use std::io;
|
||||
use std::panic::{AssertUnwindSafe, catch_unwind, resume_unwind};
|
||||
use std::sync::mpsc::{self, RecvTimeoutError, Sender};
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// The variable a limit reads unless [`DeadlineLimit::with_variable`] names another.
|
||||
pub const DEFAULT_VARIABLE: &str = "CRAFT_TEST_TIMEOUT_SECS";
|
||||
|
||||
/// The first pause of [`wait_until`] between two calls of its condition.
|
||||
const FIRST_PAUSE: Duration = Duration::from_millis(1);
|
||||
|
||||
/// The longest pause of [`wait_until`]: each pause doubles up to this.
|
||||
const MAX_PAUSE: Duration = Duration::from_millis(50);
|
||||
|
||||
/// The name of the thread [`with_deadline`] runs a body on.
|
||||
const WORKER_NAME: &str = "craft-testkit deadline";
|
||||
|
||||
/// A limit for [`with_deadline`] and [`wait_until`]: a default duration and the name of the
|
||||
/// variable that replaces it.
|
||||
///
|
||||
/// ```
|
||||
/// use std::time::Duration;
|
||||
/// use craft_testkit::deadline::DeadlineLimit;
|
||||
///
|
||||
/// // Reads CRAFT_TEST_TIMEOUT_SECS.
|
||||
/// let limit: DeadlineLimit = Duration::from_secs(30).into();
|
||||
/// assert_eq!(limit.variable(), "CRAFT_TEST_TIMEOUT_SECS");
|
||||
///
|
||||
/// // Reads the app's own variable instead.
|
||||
/// let limit = DeadlineLimit::new(Duration::from_secs(30)).with_variable("FILMCRAFT_HOSTILE_TIMEOUT_SECS");
|
||||
/// assert_eq!(limit.default_limit(), Duration::from_secs(30));
|
||||
/// ```
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct DeadlineLimit<'a> {
|
||||
default: Duration,
|
||||
variable: &'a str,
|
||||
}
|
||||
|
||||
impl DeadlineLimit<'static> {
|
||||
/// A limit of `default` that [`DEFAULT_VARIABLE`] replaces.
|
||||
pub fn new(default: Duration) -> Self {
|
||||
Self { default, variable: DEFAULT_VARIABLE }
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> DeadlineLimit<'a> {
|
||||
/// The same default, replaced by the variable `name` instead.
|
||||
pub fn with_variable<'b>(self, name: &'b str) -> DeadlineLimit<'b> {
|
||||
DeadlineLimit { default: self.default, variable: name }
|
||||
}
|
||||
|
||||
/// The limit that applies when the variable is not set.
|
||||
pub fn default_limit(&self) -> Duration {
|
||||
self.default
|
||||
}
|
||||
|
||||
/// The name of the variable that replaces the default.
|
||||
pub fn variable(&self) -> &'a str {
|
||||
self.variable
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Duration> for DeadlineLimit<'static> {
|
||||
fn from(default: Duration) -> Self {
|
||||
Self::new(default)
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs `f` on its own thread and returns its value, failing the calling test when `f` does not
|
||||
/// finish within `limit`.
|
||||
///
|
||||
/// The failure names `label`, the limit and the variable that sets it. A panic in `f` is raised
|
||||
/// again here with its original payload. The limit is read and checked before `f` starts, and a
|
||||
/// limit that is not valid fails the test without running `f`. The limit counts from the call, so
|
||||
/// the time spent starting the thread counts against it. When `f` returns or panics after the limit
|
||||
/// has passed, the test fails with the label and the limit, and with the panic's message if it has
|
||||
/// one. When the limit passes, the thread running `f` is left running.
|
||||
///
|
||||
/// ```
|
||||
/// use std::time::Duration;
|
||||
/// use craft_testkit::deadline::with_deadline;
|
||||
///
|
||||
/// let sum = with_deadline(Duration::from_secs(30), "sum of 1 to 10", || (1..=10).sum::<u32>());
|
||||
/// assert_eq!(sum, 55);
|
||||
/// ```
|
||||
// Failing the calling test is what this function is for.
|
||||
#[allow(clippy::panic)]
|
||||
#[track_caller]
|
||||
pub fn with_deadline<'a, T: Send + 'static>(limit: impl Into<DeadlineLimit<'a>>, label: &str, f: impl FnOnce() -> T + Send + 'static) -> T {
|
||||
let start = Instant::now();
|
||||
match run_with_deadline(limit.into(), label, f, read_variable, spawn_worker, Arc::new(move || start.elapsed())) {
|
||||
Ok(value) => value,
|
||||
Err(Failure::Panic(payload)) => resume_unwind(payload),
|
||||
Err(Failure::Message(message)) => panic!("{message}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Calls `cond` on the calling thread until it returns `true`, failing the calling test when the
|
||||
/// limit passes first.
|
||||
///
|
||||
/// `cond` is called at once, and then after pauses that grow from 1 ms to 50 ms, but not again once
|
||||
/// the limit has passed. The failure names `what`, the limit and the variable that sets it. A `true`
|
||||
/// that `cond` returns after the limit has passed also fails the test. A panic in `cond` reaches the
|
||||
/// caller unchanged. `cond` runs on the calling thread and is not interrupted, so it must return
|
||||
/// promptly.
|
||||
///
|
||||
/// ```
|
||||
/// use std::sync::atomic::{AtomicBool, Ordering};
|
||||
/// use std::time::Duration;
|
||||
/// use craft_testkit::deadline::wait_until;
|
||||
///
|
||||
/// let ready = AtomicBool::new(false);
|
||||
/// std::thread::scope(|s| {
|
||||
/// s.spawn(|| ready.store(true, Ordering::SeqCst));
|
||||
/// wait_until(Duration::from_secs(30), "the worker to start", || ready.load(Ordering::SeqCst));
|
||||
/// });
|
||||
/// ```
|
||||
// Failing the calling test is what this function is for.
|
||||
#[allow(clippy::panic)]
|
||||
#[track_caller]
|
||||
pub fn wait_until<'a>(limit: impl Into<DeadlineLimit<'a>>, what: &str, mut cond: impl FnMut() -> bool) {
|
||||
let limit = limit.into();
|
||||
let start = Instant::now();
|
||||
let resolved = match resolve(&limit, read_variable, start) {
|
||||
Ok(resolved) => resolved,
|
||||
Err(error) => panic!("waiting for {what}: {error}"),
|
||||
};
|
||||
let mut elapsed = || start.elapsed();
|
||||
if let Err(message) = poll_until(&resolved, what, &mut cond, &mut elapsed, &mut thread::sleep) {
|
||||
panic!("{message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// A limit after reading its variable.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct Resolved<'a> {
|
||||
pub(crate) limit: Duration,
|
||||
variable: &'a str,
|
||||
from_variable: bool,
|
||||
}
|
||||
|
||||
impl fmt::Display for Resolved<'_> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
if self.from_variable {
|
||||
write!(f, "{:?}, set by {}", self.limit, self.variable)
|
||||
} else {
|
||||
write!(f, "{:?}, the default; {} replaces it in whole seconds", self.limit, self.variable)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Why a limit cannot be used.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum LimitError<'a> {
|
||||
/// The variable name is empty or holds `=` or a NUL character.
|
||||
BadName { variable: &'a str },
|
||||
/// The variable's value is not Unicode.
|
||||
NotUnicode { variable: &'a str },
|
||||
/// The variable's value is not a whole number of seconds above zero.
|
||||
BadValue { variable: &'a str, value: String, reason: &'static str },
|
||||
/// The default limit is zero.
|
||||
ZeroDefault { variable: &'a str },
|
||||
/// The limit ends past the latest time the clock can represent.
|
||||
TooLarge { resolved: Resolved<'a> },
|
||||
}
|
||||
|
||||
impl fmt::Display for LimitError<'_> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::BadName { variable } => write!(f, "{variable:?} cannot name a variable: it is empty or holds `=` or a NUL character"),
|
||||
Self::NotUnicode { variable } => write!(f, "{variable} is not valid Unicode; set it to a whole number of seconds above zero"),
|
||||
Self::BadValue { variable, value, reason } => {
|
||||
write!(f, "{variable}={value:?} {reason}; set it to a whole number of seconds above zero")
|
||||
}
|
||||
Self::ZeroDefault { variable } => write!(f, "the default limit is zero; pass a default above zero ({variable} replaces it)"),
|
||||
Self::TooLarge { resolved } => write!(f, "the limit of {resolved} ends past the latest time this platform's clock can represent"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads and checks `limit`: the variable's value when `lookup` finds one, otherwise the default.
|
||||
/// `now` is the time the limit starts from.
|
||||
pub(crate) fn resolve<'a>(
|
||||
limit: &DeadlineLimit<'a>,
|
||||
lookup: impl FnOnce(&str) -> Option<OsString>,
|
||||
now: Instant,
|
||||
) -> Result<Resolved<'a>, LimitError<'a>> {
|
||||
let variable = limit.variable;
|
||||
if variable.is_empty() || variable.contains(['=', '\0']) {
|
||||
return Err(LimitError::BadName { variable });
|
||||
}
|
||||
if limit.default.is_zero() {
|
||||
return Err(LimitError::ZeroDefault { variable });
|
||||
}
|
||||
let resolved = match lookup(variable) {
|
||||
None => Resolved { limit: limit.default, variable, from_variable: false },
|
||||
Some(raw) => {
|
||||
let value = raw.into_string().map_err(|_| LimitError::NotUnicode { variable })?;
|
||||
let secs = parse_secs(&value).map_err(|reason| LimitError::BadValue { variable, value: value.clone(), reason })?;
|
||||
Resolved { limit: Duration::from_secs(secs), variable, from_variable: true }
|
||||
}
|
||||
};
|
||||
if now.checked_add(resolved.limit).is_none() {
|
||||
return Err(LimitError::TooLarge { resolved });
|
||||
}
|
||||
Ok(resolved)
|
||||
}
|
||||
|
||||
/// Parses a whole number of seconds above zero, written as decimal digits only.
|
||||
fn parse_secs(value: &str) -> Result<u64, &'static str> {
|
||||
if value.is_empty() {
|
||||
return Err("is empty");
|
||||
}
|
||||
if !value.bytes().all(|b| b.is_ascii_digit()) {
|
||||
return Err("is not a whole number of seconds written in digits");
|
||||
}
|
||||
match value.parse::<u64>() {
|
||||
Ok(0) => Err("is zero"),
|
||||
Ok(secs) => Ok(secs),
|
||||
Err(_) => Err("is more seconds than a u64 holds"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads the variable `name` from the process environment.
|
||||
fn read_variable(name: &str) -> Option<OsString> {
|
||||
std::env::var_os(name)
|
||||
}
|
||||
|
||||
/// A body boxed for the worker thread.
|
||||
type Job = Box<dyn FnOnce() + Send + 'static>;
|
||||
|
||||
/// Starts `job` on a named thread and detaches it. The thread sends on `done` after `job` returns.
|
||||
fn spawn_worker(job: Job, done: Sender<()>) -> io::Result<()> {
|
||||
thread::Builder::new()
|
||||
.name(WORKER_NAME.to_owned())
|
||||
.spawn(move || {
|
||||
job();
|
||||
let _ = done.send(());
|
||||
})
|
||||
.map(drop)
|
||||
}
|
||||
|
||||
/// The body's result and the clock reading when it finished, filled in by the worker.
|
||||
type Slot<T> = Arc<Mutex<Option<(thread::Result<T>, Duration)>>>;
|
||||
|
||||
/// How a [`with_deadline`] call failed.
|
||||
enum Failure {
|
||||
/// The body panicked with this payload.
|
||||
Panic(Box<dyn Any + Send>),
|
||||
/// The call failed for this reason.
|
||||
Message(String),
|
||||
}
|
||||
|
||||
/// A clock that returns the time since a [`with_deadline`] call started.
|
||||
type Clock = Arc<dyn Fn() -> Duration + Send + Sync>;
|
||||
|
||||
/// The text of a panic payload that holds a message.
|
||||
fn panic_text(payload: &(dyn Any + Send)) -> Option<&str> {
|
||||
payload.downcast_ref::<String>().map(String::as_str).or_else(|| payload.downcast_ref::<&str>().copied())
|
||||
}
|
||||
|
||||
/// [`with_deadline`] with the variable lookup, the thread start and the clock passed in. Starting
|
||||
/// the worker counts against the limit. The worker stores the body's result with the clock reading
|
||||
/// when it finished, under a lock, and then sends on the channel `spawn` receives. When the wait
|
||||
/// times out, the stored result is still used if it is there, so a body that finished in time passes
|
||||
/// even when its thread sends late.
|
||||
fn run_with_deadline<T: Send + 'static>(
|
||||
limit: DeadlineLimit<'_>,
|
||||
label: &str,
|
||||
f: impl FnOnce() -> T + Send + 'static,
|
||||
lookup: impl FnOnce(&str) -> Option<OsString>,
|
||||
spawn: impl FnOnce(Job, Sender<()>) -> io::Result<()>,
|
||||
clock: Clock,
|
||||
) -> Result<T, Failure> {
|
||||
let resolved = resolve(&limit, lookup, Instant::now()).map_err(|error| Failure::Message(format!("{label}: {error}")))?;
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let slot: Slot<T> = Arc::new(Mutex::new(None));
|
||||
let worker_slot = Arc::clone(&slot);
|
||||
let worker_clock = Arc::clone(&clock);
|
||||
let job: Job = Box::new(move || {
|
||||
let result = catch_unwind(AssertUnwindSafe(f));
|
||||
// The clock is read under the lock: once the caller finds the slot empty after its limit,
|
||||
// any later reading is past the limit too.
|
||||
let mut stored = worker_slot.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
*stored = Some((result, worker_clock()));
|
||||
});
|
||||
spawn(job, tx).map_err(|error| Failure::Message(format!("{label}: could not start the worker thread: {error}")))?;
|
||||
let waited = rx.recv_timeout(resolved.limit.saturating_sub(clock()));
|
||||
let stored = slot.lock().unwrap_or_else(PoisonError::into_inner).take();
|
||||
match (stored, waited) {
|
||||
(Some((result, finished)), _) if finished > resolved.limit => {
|
||||
let panicked = match &result {
|
||||
Ok(_) => String::new(),
|
||||
Err(payload) => format!(" (it panicked: {})", panic_text(payload.as_ref()).unwrap_or("a payload that is not a message")),
|
||||
};
|
||||
Err(Failure::Message(format!("{label} finished only after its limit of {resolved}{panicked}")))
|
||||
}
|
||||
(Some((Ok(value), _)), _) => Ok(value),
|
||||
(Some((Err(payload), _)), _) => Err(Failure::Panic(payload)),
|
||||
(None, Err(RecvTimeoutError::Timeout)) => {
|
||||
Err(Failure::Message(format!("{label} did not finish within {resolved}; its thread is left running")))
|
||||
}
|
||||
(None, _) => Err(Failure::Message(format!("{label}: the worker thread stopped without a result"))),
|
||||
}
|
||||
}
|
||||
|
||||
/// The loop of [`wait_until`], with the clock and the pause passed in. `elapsed` returns the time
|
||||
/// since the wait started.
|
||||
fn poll_until(
|
||||
resolved: &Resolved<'_>,
|
||||
what: &str,
|
||||
cond: &mut dyn FnMut() -> bool,
|
||||
elapsed: &mut dyn FnMut() -> Duration,
|
||||
pause: &mut dyn FnMut(Duration),
|
||||
) -> Result<(), String> {
|
||||
let mut next_pause = FIRST_PAUSE;
|
||||
let mut first = true;
|
||||
loop {
|
||||
// After the first call, the condition is called only while the limit has not passed.
|
||||
if !first && elapsed() >= resolved.limit {
|
||||
return Err(format!("timed out waiting for {what} within {resolved}"));
|
||||
}
|
||||
first = false;
|
||||
let held = cond();
|
||||
let spent = elapsed();
|
||||
if held && spent <= resolved.limit {
|
||||
return Ok(());
|
||||
}
|
||||
if held {
|
||||
return Err(format!("timed out waiting for {what} within {resolved} (it held only after the limit had passed)"));
|
||||
}
|
||||
if spent < resolved.limit {
|
||||
pause(next_pause.min(resolved.limit - spent));
|
||||
}
|
||||
next_pause = next_pause.saturating_mul(2).min(MAX_PAUSE);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::ffi::OsStr;
|
||||
use std::process::Command;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
|
||||
/// A variable no test sets, so the environment cannot lengthen a test's short limit.
|
||||
const UNSET: &str = "CRAFT_TESTKIT_DEADLINE_UNSET_IN_TESTS";
|
||||
|
||||
/// The variable `run_child` sets for the child process.
|
||||
const CHILD_VARIABLE: &str = "CRAFT_TESTKIT_DEADLINE_CHILD_SECS";
|
||||
|
||||
fn short() -> DeadlineLimit<'static> {
|
||||
DeadlineLimit::new(Duration::from_millis(50)).with_variable(UNSET)
|
||||
}
|
||||
|
||||
fn long() -> DeadlineLimit<'static> {
|
||||
DeadlineLimit::new(Duration::from_secs(600)).with_variable(UNSET)
|
||||
}
|
||||
|
||||
fn message(payload: Box<dyn Any + Send>) -> String {
|
||||
match payload.downcast::<String>() {
|
||||
Ok(text) => *text,
|
||||
Err(payload) => match payload.downcast::<&str>() {
|
||||
Ok(text) => (*text).to_owned(),
|
||||
Err(_) => panic!("the panic payload is not a message"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn failure_message<T>(result: Result<T, Failure>) -> String {
|
||||
match result {
|
||||
Ok(_) => panic!("the call did not fail"),
|
||||
Err(Failure::Panic(payload)) => message(payload),
|
||||
Err(Failure::Message(text)) => text,
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs `child_reads_its_variable` in a child process with `CHILD_VARIABLE` set to `value`, and
|
||||
/// returns what the child printed. The child must fail.
|
||||
fn run_child(value: &OsStr) -> String {
|
||||
let out = Command::new(std::env::current_exe().unwrap())
|
||||
.args(["--exact", "deadline::tests::child_reads_its_variable", "--nocapture", "--test-threads=1"])
|
||||
.env(CHILD_VARIABLE, value)
|
||||
.output()
|
||||
.unwrap();
|
||||
let printed = format!("{}{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr));
|
||||
assert!(!out.status.success(), "the child passed:\n{printed}");
|
||||
assert!(printed.contains("running 1 test"), "the child did not run the test:\n{printed}");
|
||||
printed
|
||||
}
|
||||
|
||||
/// Runs in the child processes of `run_child`. Without the variable it takes its default and
|
||||
/// passes; with it, the body blocks until the limit passes.
|
||||
#[test]
|
||||
fn child_reads_its_variable() {
|
||||
let blocks = std::env::var_os(CHILD_VARIABLE).is_some();
|
||||
let (_keep, rx) = mpsc::channel::<()>();
|
||||
with_deadline(DeadlineLimit::new(Duration::from_secs(2)).with_variable(CHILD_VARIABLE), "child body", move || {
|
||||
if blocks {
|
||||
let _ = rx.recv();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_slow_body_passes() {
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let sender = thread::spawn(move || tx.send(()).unwrap());
|
||||
with_deadline(long(), "slow body", move || rx.recv().unwrap());
|
||||
sender.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "boom")]
|
||||
fn a_panic_in_the_body_is_re_raised() {
|
||||
with_deadline(long(), "panicking body", || panic!("boom"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "seed 7 byte 3 did not finish within 50ms")]
|
||||
fn a_hang_fails_and_names_the_input() {
|
||||
let (gate, gate_rx) = mpsc::channel::<()>();
|
||||
let (done_tx, done) = mpsc::channel();
|
||||
let result = catch_unwind(AssertUnwindSafe(|| {
|
||||
with_deadline(short(), "seed 7 byte 3", move || {
|
||||
let _ = gate_rx.recv();
|
||||
done_tx.send(()).unwrap();
|
||||
})
|
||||
}));
|
||||
drop(gate);
|
||||
done.recv().unwrap();
|
||||
if let Err(payload) = result {
|
||||
resume_unwind(payload);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_deadline_completes() {
|
||||
with_deadline(Duration::from_secs(5), "fast body", || {
|
||||
// fast operation
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "did not finish within")]
|
||||
fn with_deadline_times_out() {
|
||||
let (gate, gate_rx) = mpsc::channel::<()>();
|
||||
let result = catch_unwind(AssertUnwindSafe(|| with_deadline(short(), "blocked body", move || gate_rx.recv().unwrap_or_default())));
|
||||
drop(gate);
|
||||
if let Err(payload) = result {
|
||||
resume_unwind(payload);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn returns_worker_value() {
|
||||
assert_eq!(with_deadline(long(), "value", || vec![1u8, 2, 3]), vec![1, 2, 3]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_panic_keeps_original_payload() {
|
||||
#[derive(Debug, PartialEq)]
|
||||
struct Marker(u32);
|
||||
let payload = catch_unwind(|| with_deadline(long(), "typed panic", || std::panic::panic_any(Marker(7)))).unwrap_err();
|
||||
assert_eq!(payload.downcast_ref::<Marker>(), Some(&Marker(7)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn timeout_names_this_call_only() {
|
||||
let call = |label: &'static str| {
|
||||
thread::spawn(move || {
|
||||
let (gate, gate_rx) = mpsc::channel::<()>();
|
||||
let result = catch_unwind(AssertUnwindSafe(|| with_deadline(short(), label, move || gate_rx.recv().unwrap_or_default())));
|
||||
drop(gate);
|
||||
message(result.unwrap_err())
|
||||
})
|
||||
};
|
||||
let first = call("first call");
|
||||
let second = call("second call");
|
||||
let first = first.join().unwrap();
|
||||
let second = second.join().unwrap();
|
||||
assert!(first.contains("first call") && !first.contains("second call"), "{first}");
|
||||
assert!(second.contains("second call") && !second.contains("first call"), "{second}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_calls_keep_limits_and_labels_separate() {
|
||||
let (release, release_rx) = mpsc::channel();
|
||||
let long_call = thread::spawn(move || {
|
||||
with_deadline(long(), "long call", move || {
|
||||
release_rx.recv().unwrap();
|
||||
7
|
||||
})
|
||||
});
|
||||
let (gate, gate_rx) = mpsc::channel::<()>();
|
||||
let short_call = catch_unwind(AssertUnwindSafe(|| with_deadline(short(), "short call", move || gate_rx.recv().unwrap_or_default())));
|
||||
let text = message(short_call.unwrap_err());
|
||||
assert!(text.contains("short call") && text.contains("50ms") && !text.contains("long call"), "{text}");
|
||||
release.send(()).unwrap();
|
||||
assert_eq!(long_call.join().unwrap(), 7);
|
||||
drop(gate);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spawn_failure_reports_cause() {
|
||||
let ran = AtomicBool::new(false);
|
||||
let spawn = |_job: Job, _done: Sender<()>| {
|
||||
ran.store(true, Ordering::SeqCst);
|
||||
Err(io::Error::other("no threads left"))
|
||||
};
|
||||
let result = run_with_deadline(long(), "no thread", || (), |_| None, spawn, Arc::new(|| Duration::ZERO));
|
||||
let text = failure_message(result);
|
||||
assert!(ran.load(Ordering::SeqCst));
|
||||
assert!(text.contains("no thread: could not start the worker thread: no threads left"), "{text}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_override_fails() {
|
||||
let printed = run_child(OsStr::new("5s"));
|
||||
assert!(printed.contains("CRAFT_TESTKIT_DEADLINE_CHILD_SECS=\"5s\" is not a whole number of seconds"), "{printed}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_override_fails() {
|
||||
let printed = run_child(OsStr::new("0"));
|
||||
assert!(printed.contains("CRAFT_TESTKIT_DEADLINE_CHILD_SECS=\"0\" is zero"), "{printed}");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn non_unicode_override_fails() {
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
let printed = run_child(OsStr::from_bytes(b"3\xff"));
|
||||
assert!(printed.contains("CRAFT_TESTKIT_DEADLINE_CHILD_SECS is not valid Unicode"), "{printed}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn valid_override_replaces_the_default() {
|
||||
let printed = run_child(OsStr::new("1"));
|
||||
assert!(printed.contains("child body did not finish within 1s, set by CRAFT_TESTKIT_DEADLINE_CHILD_SECS"), "{printed}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "the default limit is zero")]
|
||||
fn zero_default_fails() {
|
||||
with_deadline(DeadlineLimit::new(Duration::ZERO).with_variable(UNSET), "zero", || ());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn huge_limit_fails_cleanly() {
|
||||
let huge = DeadlineLimit::new(Duration::MAX).with_variable(UNSET);
|
||||
let ran = AtomicBool::new(false);
|
||||
let spawn = |_job: Job, _done: Sender<()>| {
|
||||
ran.store(true, Ordering::SeqCst);
|
||||
Ok(())
|
||||
};
|
||||
let text = failure_message(run_with_deadline(huge, "huge", || (), |_| None, spawn, Arc::new(|| Duration::ZERO)));
|
||||
assert!(text.contains("ends past the latest time"), "{text}");
|
||||
assert!(!ran.load(Ordering::SeqCst), "the body started");
|
||||
let text = message(catch_unwind(|| wait_until(huge, "a huge limit", || true)).unwrap_err());
|
||||
assert!(text.contains("waiting for a huge limit: the limit of") && text.contains("ends past the latest time"), "{text}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_values_are_rejected() {
|
||||
let limit = DeadlineLimit::new(Duration::from_secs(5)).with_variable("X");
|
||||
let now = Instant::now();
|
||||
for value in ["", "+5", "-5", "1.5", " 5", "5 ", "abc", "5s", "0", "000", "18446744073709551616"] {
|
||||
let result = resolve(&limit, |_| Some(OsString::from(value)), now);
|
||||
assert!(matches!(result, Err(LimitError::BadValue { .. })), "{value:?} gave {result:?}");
|
||||
}
|
||||
let resolved = resolve(&limit, |_| Some(OsString::from("007")), now).unwrap();
|
||||
assert_eq!(resolved, Resolved { limit: Duration::from_secs(7), variable: "X", from_variable: true });
|
||||
let resolved = resolve(&limit, |_| None, now).unwrap();
|
||||
assert_eq!(resolved, Resolved { limit: Duration::from_secs(5), variable: "X", from_variable: false });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bad_variable_names_are_rejected_before_reading() {
|
||||
for name in ["", "A=B", "A\0B"] {
|
||||
let limit = DeadlineLimit::new(Duration::from_secs(5)).with_variable(name);
|
||||
let result = resolve(&limit, |_| panic!("the variable was read"), Instant::now());
|
||||
assert_eq!(result, Err(LimitError::BadName { variable: name }));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_duration_reads_the_default_variable() {
|
||||
let limit: DeadlineLimit = Duration::from_secs(3).into();
|
||||
assert_eq!(limit.variable(), DEFAULT_VARIABLE);
|
||||
assert_eq!(limit.default_limit(), Duration::from_secs(3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wait_until_returns_on_condition() {
|
||||
let ready = &AtomicBool::new(false);
|
||||
let (asked, asked_rx) = mpsc::channel();
|
||||
let (stored, stored_rx) = mpsc::channel();
|
||||
thread::scope(|s| {
|
||||
s.spawn(move || {
|
||||
asked_rx.recv().unwrap();
|
||||
ready.store(true, Ordering::SeqCst);
|
||||
stored.send(()).unwrap();
|
||||
});
|
||||
let mut calls = 0;
|
||||
wait_until(long(), "the flag", || {
|
||||
calls += 1;
|
||||
if calls == 1 {
|
||||
// The flag is set before this call returns, and it still answers false, so the
|
||||
// condition is called a second time whatever order the threads run in.
|
||||
asked.send(()).unwrap();
|
||||
stored_rx.recv().unwrap();
|
||||
return false;
|
||||
}
|
||||
ready.load(Ordering::SeqCst)
|
||||
});
|
||||
assert_eq!(calls, 2);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "timed out waiting for the export to finish within 50ms")]
|
||||
fn wait_until_times_out_with_label() {
|
||||
wait_until(short(), "the export to finish", || false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "condition boom")]
|
||||
fn wait_until_propagates_condition_panic() {
|
||||
wait_until(long(), "a panicking condition", || panic!("condition boom"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wait_until_keeps_a_typed_condition_payload() {
|
||||
#[derive(Debug, PartialEq)]
|
||||
struct Marker(u32);
|
||||
let payload = catch_unwind(|| wait_until(long(), "a typed panic", || std::panic::panic_any(Marker(9)))).unwrap_err();
|
||||
assert_eq!(payload.downcast_ref::<Marker>(), Some(&Marker(9)));
|
||||
}
|
||||
|
||||
/// A default limit for the loop tests, which pass their own clock.
|
||||
fn fake_resolved(limit: Duration) -> Resolved<'static> {
|
||||
Resolved { limit, variable: UNSET, from_variable: false }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wait_until_rejects_a_true_after_the_limit() {
|
||||
let resolved = fake_resolved(Duration::from_millis(10));
|
||||
let mut now = Duration::ZERO;
|
||||
let mut cond = || true;
|
||||
// The condition took 11 ms to return.
|
||||
let mut elapsed = || {
|
||||
now += Duration::from_millis(11);
|
||||
now
|
||||
};
|
||||
let result = poll_until(&resolved, "a late flag", &mut cond, &mut elapsed, &mut |_| panic!("paused"));
|
||||
let text = result.unwrap_err();
|
||||
assert!(text.contains("timed out waiting for a late flag") && text.contains("held only after the limit"), "{text}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wait_until_pauses_grow_and_stop_at_the_limit() {
|
||||
let resolved = fake_resolved(Duration::from_millis(100));
|
||||
let now = std::cell::Cell::new(Duration::ZERO);
|
||||
let mut pauses = Vec::new();
|
||||
let mut calls = 0;
|
||||
let mut cond = || {
|
||||
calls += 1;
|
||||
false
|
||||
};
|
||||
let mut elapsed = || now.get();
|
||||
let mut pause = |d: Duration| {
|
||||
pauses.push(d);
|
||||
now.set(now.get() + d);
|
||||
};
|
||||
let result = poll_until(&resolved, "never", &mut cond, &mut elapsed, &mut pause);
|
||||
assert!(result.is_err());
|
||||
let ms: Vec<u128> = pauses.iter().map(Duration::as_millis).collect();
|
||||
// 1 + 2 + 4 + 8 + 16 + 32 = 63, then the 37 ms left, after which the limit is reached.
|
||||
assert_eq!(ms, [1, 2, 4, 8, 16, 32, 37]);
|
||||
// One call before each pause, and none after the last one.
|
||||
assert_eq!(calls, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wait_until_stops_calling_after_an_overshooting_pause() {
|
||||
let resolved = fake_resolved(Duration::from_millis(100));
|
||||
let now = std::cell::Cell::new(Duration::from_millis(99));
|
||||
let mut calls = 0;
|
||||
let mut cond = || {
|
||||
calls += 1;
|
||||
false
|
||||
};
|
||||
let mut elapsed = || now.get();
|
||||
// The thread wakes 50 ms late from the 1 ms pause.
|
||||
let mut pause = |d: Duration| now.set(now.get() + d + Duration::from_millis(50));
|
||||
let result = poll_until(&resolved, "late wake", &mut cond, &mut elapsed, &mut pause);
|
||||
assert!(result.unwrap_err().contains("timed out waiting for late wake"));
|
||||
assert_eq!(calls, 1);
|
||||
}
|
||||
|
||||
/// Runs `run_with_deadline` with a 5 s limit on a fake clock. The worker runs on the calling
|
||||
/// thread: the clock reads `start_secs` while the body runs and `received_secs` when the result
|
||||
/// is received. When `sends` is false, the worker stores its result but does not send, as if its
|
||||
/// thread stopped running between the two.
|
||||
fn on_fake_clock<T: Send + 'static>(
|
||||
f: impl FnOnce() -> T + Send + 'static,
|
||||
start_secs: u64,
|
||||
received_secs: u64,
|
||||
sends: bool,
|
||||
) -> Result<T, Failure> {
|
||||
let now = Arc::new(AtomicU64::new(0));
|
||||
let clock_now = Arc::clone(&now);
|
||||
let clock: Clock = Arc::new(move || Duration::from_secs(clock_now.load(Ordering::SeqCst)));
|
||||
let unsent = std::cell::RefCell::new(None);
|
||||
let spawn = |job: Job, done: Sender<()>| {
|
||||
now.store(start_secs, Ordering::SeqCst);
|
||||
job();
|
||||
now.store(received_secs, Ordering::SeqCst);
|
||||
if sends {
|
||||
let _ = done.send(());
|
||||
} else {
|
||||
// Kept alive so that the wait times out instead of seeing a closed channel.
|
||||
*unsent.borrow_mut() = Some(done);
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
let limit = DeadlineLimit::new(Duration::from_secs(5)).with_variable(UNSET);
|
||||
run_with_deadline(limit, "fake clock", f, |_| None, spawn, clock)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_value_returned_after_the_limit_fails() {
|
||||
let text = failure_message(on_fake_clock(|| 1, 6, 6, true));
|
||||
assert!(text.contains("fake clock finished only after its limit of 5s"), "{text}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_value_finished_in_time_but_received_late_passes() {
|
||||
assert!(matches!(on_fake_clock(|| 1, 1, 6, true), Ok(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_value_finished_in_time_but_sent_late_passes() {
|
||||
assert!(matches!(on_fake_clock(|| 1, 1, 6, false), Ok(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_panic_finished_in_time_but_sent_late_keeps_its_payload() {
|
||||
let payload = match on_fake_clock(|| -> u8 { panic!("early boom") }, 1, 6, false) {
|
||||
Err(Failure::Panic(payload)) => payload,
|
||||
Ok(_) => panic!("the call did not fail"),
|
||||
Err(Failure::Message(text)) => panic!("the payload was replaced: {text}"),
|
||||
};
|
||||
assert_eq!(message(payload), "early boom");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_panic_after_the_limit_names_the_limit_and_the_panic() {
|
||||
let text = failure_message(on_fake_clock(|| -> u8 { panic!("late boom") }, 6, 6, true));
|
||||
assert!(text.contains("fake clock finished only after its limit of 5s") && text.ends_with("(it panicked: late boom)"), "{text}");
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,817 @@
|
||||
//! Hostile inputs made from a valid seed, for tests that a parser fails cleanly on damaged data.
|
||||
//!
|
||||
//! [`mutations`] yields a fixed sequence of damaged copies of a seed: each round applies one to
|
||||
//! four edits (a bit flip, a random byte, a 1, 2, 4 or 8-byte overwrite with an extreme value in
|
||||
//! either byte order, a deletion of up to 16 bytes, or a truncation) and labels the result with
|
||||
//! the edits it made. The sequence depends only on the seed's length and the number of rounds, so
|
||||
//! a failing case can be found again from its label. It builds for every target, `wasm32`
|
||||
//! included.
|
||||
//!
|
||||
//! [`hammer`] runs a parser on the unmodified seed, on every truncation of it and on the
|
||||
//! mutations, one case after another on a worker thread, with a time limit for each case. It fails
|
||||
//! the test, naming the case and the input length, when the parser panics or does not finish in
|
||||
//! time. The number of rounds is a count the caller gives, multiplied by a variable the caller
|
||||
//! names. [`hammer`] and [`HammerConfig`] are not built for `wasm32`.
|
||||
|
||||
use std::fmt::Write as _;
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
use crate::deadline::{DeadlineLimit, Resolved, resolve};
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
use std::any::Any;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
use std::ffi::OsString;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
use std::fmt;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
use std::panic::{AssertUnwindSafe, catch_unwind};
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
use std::sync::{Arc, Condvar, Mutex, MutexGuard, PoisonError};
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
use std::thread;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// The name of the thread [`hammer`] runs its cases on.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
const WORKER_NAME: &str = "craft-testkit hostile";
|
||||
|
||||
/// The variable that multiplies a [`HammerConfig`]'s rounds unless
|
||||
/// [`HammerConfig::with_multiplier_variable`] names another.
|
||||
pub const DEFAULT_MULTIPLIER_VARIABLE: &str = "CRAFT_HOSTILE_MULT";
|
||||
|
||||
/// The default limit for one case of [`hammer`]. The limit's variable, `CRAFT_TEST_TIMEOUT_SECS`
|
||||
/// unless [`HammerConfig::with_timeout`] names another, replaces it.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
/// The most cases one [`hammer`] call runs: the unmodified seed, its truncations and the rounds
|
||||
/// together. A call that would run more fails before it runs any.
|
||||
pub const MAX_CASES: usize = 10_000_000;
|
||||
|
||||
/// The values the overwrite edits write, in 1, 2, 4 or 8 bytes.
|
||||
const EXTREMES: [u64; 8] = [0, 1, 0x7F, 0x80, 0xFF, 0xFFFF, 0x8000_0000, u64::MAX];
|
||||
|
||||
/// The xorshift64* generator behind [`mutations`].
|
||||
///
|
||||
/// ```
|
||||
/// use craft_testkit::hostile::Rng;
|
||||
///
|
||||
/// let mut rng = Rng(1);
|
||||
/// assert_eq!(rng.next_u64(), 0x47E4_CE4B_896C_DD1D);
|
||||
/// ```
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Rng(pub u64);
|
||||
|
||||
impl Rng {
|
||||
/// The next value.
|
||||
pub fn next_u64(&mut self) -> u64 {
|
||||
self.0 ^= self.0 >> 12;
|
||||
self.0 ^= self.0 << 25;
|
||||
self.0 ^= self.0 >> 27;
|
||||
self.0.wrapping_mul(0x2545_F491_4F6C_DD1D)
|
||||
}
|
||||
|
||||
/// A value below `n`, or 0 when `n` is 0.
|
||||
pub fn below(&mut self, n: usize) -> usize {
|
||||
(self.next_u64() % n.max(1) as u64) as usize
|
||||
}
|
||||
}
|
||||
|
||||
/// One damaged copy of a seed, from [`mutations`].
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Mutation {
|
||||
/// The damaged bytes.
|
||||
pub bytes: Vec<u8>,
|
||||
/// The edits made, in order, separated by spaces, such as `flip[3].5 del[7..9]`.
|
||||
pub label: String,
|
||||
}
|
||||
|
||||
/// `rounds` damaged copies of `seed`, in a sequence fixed by the seed's length and `rounds`.
|
||||
///
|
||||
/// An empty seed yields no copies, since there is no byte to edit.
|
||||
///
|
||||
/// ```
|
||||
/// use craft_testkit::hostile::mutations;
|
||||
///
|
||||
/// let all: Vec<_> = mutations(&[0, 1, 2, 3], 2).collect();
|
||||
/// assert_eq!(all[1].label, "trunc2");
|
||||
/// assert_eq!(all[1].bytes, [0, 1]);
|
||||
/// ```
|
||||
pub fn mutations(seed: &[u8], rounds: usize) -> Mutations<'_> {
|
||||
Mutations { seed, rng: Rng((0x5EED_F00D ^ seed.len() as u64) | 1), remaining: if seed.is_empty() { 0 } else { rounds } }
|
||||
}
|
||||
|
||||
/// The iterator [`mutations`] returns.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Mutations<'a> {
|
||||
seed: &'a [u8],
|
||||
rng: Rng,
|
||||
remaining: usize,
|
||||
}
|
||||
|
||||
impl Iterator for Mutations<'_> {
|
||||
type Item = Mutation;
|
||||
|
||||
fn next(&mut self) -> Option<Mutation> {
|
||||
if self.remaining == 0 {
|
||||
return None;
|
||||
}
|
||||
self.remaining -= 1;
|
||||
let rng = &mut self.rng;
|
||||
let mut d = self.seed.to_vec();
|
||||
let mut label = String::new();
|
||||
for _ in 0..1 + rng.below(4) {
|
||||
if d.is_empty() {
|
||||
break;
|
||||
}
|
||||
if !label.is_empty() {
|
||||
label.push(' ');
|
||||
}
|
||||
let pos = rng.below(d.len());
|
||||
match rng.below(7) {
|
||||
0 => {
|
||||
let b = rng.below(8);
|
||||
if let Some(byte) = d.get_mut(pos) {
|
||||
*byte ^= 1 << b;
|
||||
}
|
||||
let _ = write!(label, "flip[{pos}].{b}");
|
||||
}
|
||||
1 => {
|
||||
let value = rng.next_u64() as u8;
|
||||
if let Some(byte) = d.get_mut(pos) {
|
||||
*byte = value;
|
||||
}
|
||||
let _ = write!(label, "set[{pos}]={value:#x}");
|
||||
}
|
||||
2..=4 => {
|
||||
let w = [1usize, 2, 4, 8][rng.below(4)];
|
||||
let v = EXTREMES[rng.below(EXTREMES.len())];
|
||||
let be = rng.below(2) == 0;
|
||||
let all = if be { v.to_be_bytes() } else { v.to_le_bytes() };
|
||||
let bytes = if be { &all[8 - w..] } else { &all[..w] };
|
||||
let end = (pos + w).min(d.len());
|
||||
if let (Some(target), Some(source)) = (d.get_mut(pos..end), bytes.get(..end - pos)) {
|
||||
target.copy_from_slice(source);
|
||||
}
|
||||
let _ = write!(label, "put{w}{}[{pos}]={v:#x}", if be { "be" } else { "le" });
|
||||
}
|
||||
5 => {
|
||||
let n = 1 + rng.below(16);
|
||||
let end = (pos + n).min(d.len());
|
||||
d.drain(pos..end);
|
||||
let _ = write!(label, "del[{pos}..{end}]");
|
||||
}
|
||||
_ => {
|
||||
d.truncate(pos);
|
||||
let _ = write!(label, "trunc{pos}");
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(Mutation { bytes: d, label })
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
(self.remaining, Some(self.remaining))
|
||||
}
|
||||
}
|
||||
|
||||
impl ExactSizeIterator for Mutations<'_> {}
|
||||
|
||||
/// The settings of one [`hammer`] call: a label for its failures, the number of rounds, the
|
||||
/// variable that multiplies them, and the limit for each case.
|
||||
///
|
||||
/// ```
|
||||
/// use std::time::Duration;
|
||||
/// use craft_testkit::hostile::HammerConfig;
|
||||
///
|
||||
/// // 2,000 rounds, times CRAFT_HOSTILE_MULT when it is set; 5 s per case, or CRAFT_TEST_TIMEOUT_SECS.
|
||||
/// let config = HammerConfig::new("png seed 3", 2_000);
|
||||
///
|
||||
/// // The app's own variables instead.
|
||||
/// let config = HammerConfig::new("mxf primer", 5_000)
|
||||
/// .with_multiplier_variable("CRAFT_MEDIA_HOSTILE_MULT")
|
||||
/// .with_timeout(craft_testkit::DeadlineLimit::new(Duration::from_secs(5)).with_variable("FILMCRAFT_HOSTILE_TIMEOUT_SECS"));
|
||||
/// assert_eq!(config.multiplier_variable(), "CRAFT_MEDIA_HOSTILE_MULT");
|
||||
/// ```
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct HammerConfig<'a> {
|
||||
label: &'a str,
|
||||
iterations: usize,
|
||||
multiplier_variable: &'a str,
|
||||
timeout: DeadlineLimit<'a>,
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
impl<'a> HammerConfig<'a> {
|
||||
/// `iterations` rounds of mutations under `label`, multiplied by [`DEFAULT_MULTIPLIER_VARIABLE`],
|
||||
/// with a limit of [`DEFAULT_TIMEOUT`] for each case.
|
||||
pub fn new(label: &'a str, iterations: usize) -> Self {
|
||||
Self { label, iterations, multiplier_variable: DEFAULT_MULTIPLIER_VARIABLE, timeout: DeadlineLimit::new(DEFAULT_TIMEOUT) }
|
||||
}
|
||||
|
||||
/// The same settings, with the rounds multiplied by the variable `name` instead.
|
||||
pub fn with_multiplier_variable(self, name: &'a str) -> Self {
|
||||
Self { multiplier_variable: name, ..self }
|
||||
}
|
||||
|
||||
/// The same settings, with `limit` for each case.
|
||||
pub fn with_timeout(self, limit: impl Into<DeadlineLimit<'a>>) -> Self {
|
||||
Self { timeout: limit.into(), ..self }
|
||||
}
|
||||
|
||||
/// The label that failures start with.
|
||||
pub fn label(&self) -> &'a str {
|
||||
self.label
|
||||
}
|
||||
|
||||
/// The rounds before the multiplier.
|
||||
pub fn iterations(&self) -> usize {
|
||||
self.iterations
|
||||
}
|
||||
|
||||
/// The name of the variable that multiplies the rounds.
|
||||
pub fn multiplier_variable(&self) -> &'a str {
|
||||
self.multiplier_variable
|
||||
}
|
||||
|
||||
/// The limit for each case.
|
||||
pub fn timeout(&self) -> DeadlineLimit<'a> {
|
||||
self.timeout
|
||||
}
|
||||
|
||||
/// The rounds after reading the multiplier variable, or why they cannot be used.
|
||||
///
|
||||
/// When the variable is not set, the multiplier is 1. A value must be a whole number above
|
||||
/// zero written in decimal digits.
|
||||
pub fn rounds(&self) -> Result<usize, HammerError> {
|
||||
self.rounds_with(read_variable)
|
||||
}
|
||||
|
||||
/// [`Self::rounds`] with the variable lookup passed in.
|
||||
fn rounds_with(&self, lookup: impl FnOnce(&str) -> Option<OsString>) -> Result<usize, HammerError> {
|
||||
let variable = self.multiplier_variable.to_owned();
|
||||
if variable.is_empty() || variable.contains(['=', '\0']) {
|
||||
return Err(HammerError::BadName { variable });
|
||||
}
|
||||
let multiplier = match lookup(&variable) {
|
||||
None => 1,
|
||||
Some(raw) => {
|
||||
let value = raw.into_string().map_err(|_| HammerError::NotUnicode { variable: variable.clone() })?;
|
||||
parse_multiplier(&value).map_err(|reason| HammerError::BadValue { variable: variable.clone(), value, reason })?
|
||||
}
|
||||
};
|
||||
self.iterations.checked_mul(multiplier).ok_or(HammerError::Overflow { iterations: self.iterations, variable, multiplier })
|
||||
}
|
||||
}
|
||||
|
||||
/// Why a [`hammer`] call cannot run.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum HammerError {
|
||||
/// The multiplier variable's name is empty or holds `=` or a NUL character.
|
||||
BadName {
|
||||
/// The name.
|
||||
variable: String,
|
||||
},
|
||||
/// The multiplier variable's value is not Unicode.
|
||||
NotUnicode {
|
||||
/// The variable's name.
|
||||
variable: String,
|
||||
},
|
||||
/// The multiplier variable's value is not a whole number above zero.
|
||||
BadValue {
|
||||
/// The variable's name.
|
||||
variable: String,
|
||||
/// The value.
|
||||
value: String,
|
||||
/// What is wrong with the value.
|
||||
reason: &'static str,
|
||||
},
|
||||
/// The rounds times the multiplier is more than a `usize` holds.
|
||||
Overflow {
|
||||
/// The rounds before the multiplier.
|
||||
iterations: usize,
|
||||
/// The variable's name.
|
||||
variable: String,
|
||||
/// The multiplier.
|
||||
multiplier: usize,
|
||||
},
|
||||
/// The call would run more than [`MAX_CASES`] cases.
|
||||
TooManyCases {
|
||||
/// The cases the call would run.
|
||||
cases: u128,
|
||||
},
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
impl fmt::Display for HammerError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::BadName { variable } => write!(f, "{variable:?} cannot name a variable: it is empty or holds `=` or a NUL character"),
|
||||
Self::NotUnicode { variable } => write!(f, "{variable} is not valid Unicode; set it to a whole number above zero"),
|
||||
Self::BadValue { variable, value, reason } => write!(f, "{variable}={value:?} {reason}; set it to a whole number above zero"),
|
||||
Self::Overflow { iterations, variable, multiplier } => {
|
||||
write!(f, "{iterations} rounds times {variable}={multiplier} is more than a usize holds")
|
||||
}
|
||||
Self::TooManyCases { cases } => write!(f, "{cases} cases is more than the limit of {MAX_CASES}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
impl std::error::Error for HammerError {}
|
||||
|
||||
/// Parses a whole number above zero, written as decimal digits only.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn parse_multiplier(value: &str) -> Result<usize, &'static str> {
|
||||
if value.is_empty() {
|
||||
return Err("is empty");
|
||||
}
|
||||
if !value.bytes().all(|b| b.is_ascii_digit()) {
|
||||
return Err("is not a whole number written in digits");
|
||||
}
|
||||
match value.parse::<usize>() {
|
||||
Ok(0) => Err("is zero"),
|
||||
Ok(n) => Ok(n),
|
||||
Err(_) => Err("is more than a usize holds"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads the variable `name` from the process environment.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn read_variable(name: &str) -> Option<OsString> {
|
||||
std::env::var_os(name)
|
||||
}
|
||||
|
||||
/// Runs `parse` on `seed`, on every truncation of it (lengths 0 to one less than its length), and
|
||||
/// on [`mutations`] of it for the config's rounds, failing the calling test on the first case
|
||||
/// that panics or does not finish within the config's limit.
|
||||
///
|
||||
/// The cases run one after another on one worker thread, while the calling thread fails the test
|
||||
/// when a case runs past the config's limit. A case that finishes after the limit also fails it. A
|
||||
/// failure starts with the config's label and names the case (`the unmodified seed`,
|
||||
/// `truncation to N` or the mutation's edits), the input's length and the panic's message. A case
|
||||
/// that does not finish fails the test while the worker thread is left running, so `parse` must be
|
||||
/// owned and able to move to another thread. The multiplier and limit variables are read before
|
||||
/// any case runs; a value that cannot be used, or a call that would run more than [`MAX_CASES`]
|
||||
/// cases, fails the test before `parse` is called.
|
||||
///
|
||||
/// ```
|
||||
/// use craft_testkit::hostile::{HammerConfig, hammer};
|
||||
///
|
||||
/// hammer(&HammerConfig::new("length prefix", 200), &[3, b'a', b'b', b'c'], |data| {
|
||||
/// if let Some((&n, rest)) = data.split_first() {
|
||||
/// let _ = rest.get(..usize::from(n));
|
||||
/// }
|
||||
/// });
|
||||
/// ```
|
||||
// Failing the calling test is what this function is for.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
#[allow(clippy::panic)]
|
||||
#[track_caller]
|
||||
pub fn hammer<F>(config: &HammerConfig<'_>, seed: &[u8], parse: F)
|
||||
where
|
||||
F: Fn(&[u8]) + Send + Sync + 'static,
|
||||
{
|
||||
if let Err(message) = hammer_with(config, seed, parse, read_variable) {
|
||||
panic!("{message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// [`hammer`] with the variable lookup passed in, returning the failure's message.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn hammer_with<F>(config: &HammerConfig<'_>, seed: &[u8], parse: F, lookup: impl Fn(&str) -> Option<OsString>) -> Result<(), String>
|
||||
where
|
||||
F: Fn(&[u8]) + Send + Sync + 'static,
|
||||
{
|
||||
let label = config.label;
|
||||
let rounds = config.rounds_with(&lookup).map_err(|error| format!("{label}: {error}"))?;
|
||||
let rounds = if seed.is_empty() { 0 } else { rounds };
|
||||
let cases = 1 + seed.len() as u128 + rounds as u128;
|
||||
if cases > MAX_CASES as u128 {
|
||||
return Err(format!("{label}: {}", HammerError::TooManyCases { cases }));
|
||||
}
|
||||
let start = Instant::now();
|
||||
let limit = resolve(&config.timeout, &lookup, start).map(Limit::from).map_err(|error| format!("{label}: {error}"))?;
|
||||
let clock: Clock = Arc::new(move || start.elapsed());
|
||||
run_cases(label.to_owned(), limit, Arc::from(seed), rounds, parse, clock)
|
||||
}
|
||||
|
||||
/// A clock that returns the time since a [`hammer`] call started.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
type Clock = Arc<dyn Fn() -> Duration + Send + Sync>;
|
||||
|
||||
/// The limit for each case, and how failures describe it.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
#[derive(Clone, Debug)]
|
||||
struct Limit {
|
||||
limit: Duration,
|
||||
text: String,
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
impl From<Resolved<'_>> for Limit {
|
||||
fn from(resolved: Resolved<'_>) -> Self {
|
||||
Self { limit: resolved.limit, text: resolved.to_string() }
|
||||
}
|
||||
}
|
||||
|
||||
/// The case the worker is running, shared with the calling thread that watches it.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
struct Progress {
|
||||
/// The case's description: `the unmodified seed`, `truncation to N` or the mutation's edits.
|
||||
case: String,
|
||||
/// The length of the case's input.
|
||||
len: usize,
|
||||
/// The clock reading when the case started.
|
||||
started: Duration,
|
||||
/// Whether the case is still running.
|
||||
running: bool,
|
||||
/// Set when the worker stops: `Ok` after the last case, or the failure's message.
|
||||
outcome: Option<Result<(), String>>,
|
||||
}
|
||||
|
||||
/// [`Progress`] under a lock, with a signal for when the worker stops.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
struct Watch {
|
||||
state: Mutex<Progress>,
|
||||
stopped: Condvar,
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
impl Watch {
|
||||
fn lock(&self) -> MutexGuard<'_, Progress> {
|
||||
self.state.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
}
|
||||
|
||||
/// Hands the worker's outcome to the calling thread when the worker stops, including when it
|
||||
/// unwinds outside a case.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
struct Report {
|
||||
watch: Arc<Watch>,
|
||||
outcome: Result<(), String>,
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
impl Drop for Report {
|
||||
fn drop(&mut self) {
|
||||
let mut state = self.watch.lock();
|
||||
state.outcome = Some(std::mem::replace(&mut self.outcome, Ok(())));
|
||||
self.watch.stopped.notify_all();
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts the worker that runs every case, then waits for it, failing when a case runs past the
|
||||
/// limit.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn run_cases<F>(label: String, limit: Limit, seed: Arc<[u8]>, rounds: usize, parse: F, clock: Clock) -> Result<(), String>
|
||||
where
|
||||
F: Fn(&[u8]) + Send + Sync + 'static,
|
||||
{
|
||||
// No case counts as running until the worker starts it, so a worker that starts late is not
|
||||
// reported as a case past its limit.
|
||||
let first = Progress { case: "the unmodified seed".to_owned(), len: seed.len(), started: clock(), running: false, outcome: None };
|
||||
let watch = Arc::new(Watch { state: Mutex::new(first), stopped: Condvar::new() });
|
||||
let (worker_watch, worker_clock, worker_label, worker_limit) = (Arc::clone(&watch), Arc::clone(&clock), label.clone(), limit.clone());
|
||||
thread::Builder::new()
|
||||
.name(WORKER_NAME.to_owned())
|
||||
.spawn(move || {
|
||||
let mut report = Report { watch: worker_watch, outcome: Err(format!("{worker_label}: the worker thread stopped without a result")) };
|
||||
report.outcome = run_all(&report.watch, &worker_clock, &worker_label, &worker_limit, &seed, rounds, &parse);
|
||||
})
|
||||
.map_err(|error| format!("{label}: could not start the worker thread: {error}"))?;
|
||||
let mut state = watch.lock();
|
||||
loop {
|
||||
if let Some(outcome) = state.outcome.take() {
|
||||
return outcome;
|
||||
}
|
||||
let deadline = state.started.saturating_add(limit.limit);
|
||||
let now = clock();
|
||||
if state.running && now >= deadline {
|
||||
return Err(format!("{label}: {} (len {}) did not finish within {}; its thread is left running", state.case, state.len, limit.text));
|
||||
}
|
||||
let wait = if state.running { deadline.saturating_sub(now) } else { limit.limit };
|
||||
state = watch.stopped.wait_timeout(state, wait).unwrap_or_else(PoisonError::into_inner).0;
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs the unmodified seed, its truncations and its mutations in order, stopping at the first
|
||||
/// failure.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn run_all<F: Fn(&[u8])>(watch: &Watch, clock: &Clock, label: &str, limit: &Limit, seed: &[u8], rounds: usize, parse: &F) -> Result<(), String> {
|
||||
for n in std::iter::once(seed.len()).chain(0..seed.len()) {
|
||||
let case = if n == seed.len() { "the unmodified seed".to_owned() } else { format!("truncation to {n}") };
|
||||
run_one(watch, clock, label, limit, case, seed.get(..n).unwrap_or_default(), parse)?;
|
||||
}
|
||||
for mutation in mutations(seed, rounds) {
|
||||
run_one(watch, clock, label, limit, mutation.label, &mutation.bytes, parse)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Runs one case, recording it in `watch` while it runs, and returns the failure's message when it
|
||||
/// panics or finishes after the limit.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn run_one<F: Fn(&[u8])>(watch: &Watch, clock: &Clock, label: &str, limit: &Limit, case: String, data: &[u8], parse: &F) -> Result<(), String> {
|
||||
let started = clock();
|
||||
{
|
||||
let mut state = watch.lock();
|
||||
(state.case, state.len, state.started, state.running) = (case, data.len(), started, true);
|
||||
}
|
||||
let outcome = catch_unwind(AssertUnwindSafe(|| parse(data)));
|
||||
let spent = clock().saturating_sub(started);
|
||||
// The payload is dropped before the lock is taken and while the case still counts as running, so
|
||||
// the watchdog still reports a payload whose drop does not return.
|
||||
let panicked = outcome.err().map(|payload| panic_text(payload.as_ref()));
|
||||
let mut state = watch.lock();
|
||||
state.running = false;
|
||||
let name = || format!("{label}: {} (len {})", state.case, state.len);
|
||||
if spent > limit.limit {
|
||||
let panicked = panicked.map_or_else(String::new, |text| format!(" (it panicked: {text})"));
|
||||
return Err(format!("{} finished only after its limit of {}{panicked}", name(), limit.text));
|
||||
}
|
||||
panicked.map_or(Ok(()), |text| Err(format!("{}: the parser panicked: {text}", name())))
|
||||
}
|
||||
|
||||
/// The text of a panic payload.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn panic_text(payload: &(dyn Any + Send)) -> String {
|
||||
payload
|
||||
.downcast_ref::<String>()
|
||||
.cloned()
|
||||
.or_else(|| payload.downcast_ref::<&str>().map(|text| (*text).to_owned()))
|
||||
.unwrap_or_else(|| "a payload that is not a message".to_owned())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::mpsc;
|
||||
|
||||
/// A variable no test sets, so the environment cannot change a test's rounds or limits.
|
||||
const UNSET: &str = "CRAFT_TESTKIT_HOSTILE_UNSET_IN_TESTS";
|
||||
|
||||
fn config(label: &'static str, iterations: usize) -> HammerConfig<'static> {
|
||||
HammerConfig::new(label, iterations)
|
||||
.with_multiplier_variable(UNSET)
|
||||
.with_timeout(DeadlineLimit::new(Duration::from_secs(600)).with_variable(UNSET))
|
||||
}
|
||||
|
||||
fn unset(_: &str) -> Option<OsString> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Runs `hammer_with` with no variables set and returns the inputs `parse` saw, in order.
|
||||
fn inputs(config: &HammerConfig<'_>, seed: &[u8]) -> Vec<Vec<u8>> {
|
||||
let seen = Arc::new(Mutex::new(Vec::new()));
|
||||
let record = Arc::clone(&seen);
|
||||
hammer_with(config, seed, move |d| record.lock().unwrap().push(d.to_vec()), unset).unwrap();
|
||||
seen.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reports_truncation_panic_with_case_and_payload() {
|
||||
let result = hammer_with(&config("demo seed 2", 0), &[9, 8, 7, 6], |d| assert!(d.len() != 2, "boom at {}", d.len()), unset);
|
||||
let message = result.unwrap_err();
|
||||
assert!(message.starts_with("demo seed 2: truncation to 2 (len 2): the parser panicked: "), "{message}");
|
||||
assert!(message.contains("boom at 2"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_panic_without_a_message_is_reported() {
|
||||
let message = hammer_with(&config("demo", 0), &[1], |_| std::panic::panic_any(7u8), unset).unwrap_err();
|
||||
assert_eq!(message, "demo: the unmodified seed (len 1): the parser panicked: a payload that is not a message");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deadline_names_stuck_case() {
|
||||
let (release, wait) = mpsc::channel::<()>();
|
||||
let wait = Mutex::new(wait);
|
||||
let short = config("stuck demo", 0).with_timeout(DeadlineLimit::new(Duration::from_millis(50)).with_variable(UNSET));
|
||||
let result = hammer_with(
|
||||
&short,
|
||||
&[1, 2, 3],
|
||||
move |d| {
|
||||
if d.len() == 3 {
|
||||
let _ = wait.lock().unwrap().recv();
|
||||
}
|
||||
},
|
||||
unset,
|
||||
);
|
||||
release.send(()).unwrap();
|
||||
let message = result.unwrap_err();
|
||||
assert!(message.contains("stuck demo: the unmodified seed (len 3) did not finish within 50ms"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_case_that_finishes_after_the_limit_fails() {
|
||||
let readings = Arc::new(Mutex::new(vec![Duration::from_secs(6), Duration::ZERO]));
|
||||
let clock: Clock = Arc::new(move || readings.lock().unwrap().pop().unwrap_or(Duration::MAX));
|
||||
let watch = Watch {
|
||||
state: Mutex::new(Progress { case: String::new(), len: 0, started: Duration::ZERO, running: true, outcome: None }),
|
||||
stopped: Condvar::new(),
|
||||
};
|
||||
let limit = Limit { limit: Duration::from_secs(5), text: "5s".to_owned() };
|
||||
let message = run_one(&watch, &clock, "late", &limit, "the unmodified seed".to_owned(), &[1], &|_: &[u8]| {}).unwrap_err();
|
||||
assert_eq!(message, "late: the unmodified seed (len 1) finished only after its limit of 5s");
|
||||
assert!(!watch.lock().running);
|
||||
}
|
||||
|
||||
/// A panic payload that records, when it is dropped, whether the progress lock was free and
|
||||
/// whether the case still counted as running.
|
||||
struct DropProbe {
|
||||
watch: Arc<Watch>,
|
||||
seen: Arc<Mutex<Option<Option<bool>>>>,
|
||||
}
|
||||
|
||||
impl Drop for DropProbe {
|
||||
fn drop(&mut self) {
|
||||
let running = self.watch.state.try_lock().ok().map(|state| state.running);
|
||||
*self.seen.lock().unwrap() = Some(running);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_panic_payload_is_dropped_outside_the_lock_while_the_case_runs() {
|
||||
let watch = Arc::new(Watch {
|
||||
state: Mutex::new(Progress { case: String::new(), len: 0, started: Duration::ZERO, running: false, outcome: None }),
|
||||
stopped: Condvar::new(),
|
||||
});
|
||||
let seen = Arc::new(Mutex::new(None));
|
||||
let clock: Clock = Arc::new(|| Duration::ZERO);
|
||||
let limit = Limit { limit: Duration::from_secs(5), text: "5s".to_owned() };
|
||||
let probe = (Arc::clone(&watch), Arc::clone(&seen));
|
||||
let parse = move |_: &[u8]| std::panic::panic_any(DropProbe { watch: Arc::clone(&probe.0), seen: Arc::clone(&probe.1) });
|
||||
let message = run_one(&watch, &clock, "probe", &limit, "the unmodified seed".to_owned(), &[1], &parse).unwrap_err();
|
||||
assert_eq!(message, "probe: the unmodified seed (len 1): the parser panicked: a payload that is not a message");
|
||||
assert_eq!(*seen.lock().unwrap(), Some(Some(true)), "the payload was dropped with the lock held or the case stopped");
|
||||
assert!(!watch.lock().running);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_worker_that_starts_late_is_not_a_case_past_its_limit() {
|
||||
// The worker's first clock reading waits until the calling thread has checked the limit once,
|
||||
// and that check reads an hour after the start; every other reading is zero.
|
||||
let (checked, worker_waits) = mpsc::channel::<()>();
|
||||
let (checked, worker_waits) = (Mutex::new(Some(checked)), Mutex::new(worker_waits));
|
||||
let caller_reads = Mutex::new(0usize);
|
||||
let clock: Clock = Arc::new(move || {
|
||||
if thread::current().name() == Some(WORKER_NAME) {
|
||||
let _ = worker_waits.lock().unwrap().recv();
|
||||
return Duration::ZERO;
|
||||
}
|
||||
let mut reads = caller_reads.lock().unwrap();
|
||||
*reads += 1;
|
||||
if *reads == 2 {
|
||||
drop(checked.lock().unwrap().take());
|
||||
return Duration::from_secs(3600);
|
||||
}
|
||||
Duration::ZERO
|
||||
});
|
||||
let limit = Limit { limit: Duration::from_secs(5), text: "5s".to_owned() };
|
||||
assert_eq!(run_cases("late start".to_owned(), limit, Arc::from(&[1u8][..]), 0, |_: &[u8]| {}, clock), Ok(()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_case_runs_on_one_worker_thread() {
|
||||
let threads = Arc::new(Mutex::new(std::collections::BTreeSet::new()));
|
||||
let record = Arc::clone(&threads);
|
||||
let parse = move |_: &[u8]| {
|
||||
record.lock().unwrap().insert(format!("{:?}", thread::current().id()));
|
||||
};
|
||||
hammer_with(&config("one thread", 50), &[1, 2, 3, 4], parse, unset).unwrap();
|
||||
let threads = threads.lock().unwrap();
|
||||
assert_eq!(threads.len(), 1, "{threads:?}");
|
||||
assert!(!threads.contains(&format!("{:?}", thread::current().id())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_limit_variable_fails_before_any_parse() {
|
||||
let called = Arc::new(Mutex::new(false));
|
||||
let flag = Arc::clone(&called);
|
||||
let config = config("bad limit", 3).with_timeout(DeadlineLimit::new(Duration::from_secs(5)).with_variable("MY_LIMIT"));
|
||||
let lookup = |name: &str| (name == "MY_LIMIT").then(|| OsString::from("soon"));
|
||||
let message = hammer_with(&config, &[1, 2], move |_| *flag.lock().unwrap() = true, lookup).unwrap_err();
|
||||
assert_eq!(
|
||||
message,
|
||||
r#"bad limit: MY_LIMIT="soon" is not a whole number of seconds written in digits; set it to a whole number of seconds above zero"#
|
||||
);
|
||||
assert!(!*called.lock().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mutation_sequence_matches_source() {
|
||||
let all: Vec<_> = mutations(&[0, 1, 2, 3], 4).collect();
|
||||
let bytes: Vec<&[u8]> = all.iter().map(|m| m.bytes.as_slice()).collect();
|
||||
let labels: Vec<&str> = all.iter().map(|m| m.label.as_str()).collect();
|
||||
assert_eq!(bytes, [&[0u8, 1, 2, 0][..], &[0, 1][..], &[0, 1, 0, 0][..], &[0, 0, 0, 1][..]]);
|
||||
assert_eq!(labels, ["put1be[3]=0x0", "trunc2", "put4le[1]=0x1", "put4be[0]=0x1"]);
|
||||
let mut expected: Vec<Vec<u8>> = vec![vec![0, 1, 2, 3], vec![], vec![0], vec![0, 1], vec![0, 1, 2]];
|
||||
expected.extend(all.into_iter().map(|m| m.bytes));
|
||||
assert_eq!(inputs(&config("sequence", 4), &[0, 1, 2, 3]), expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mutation_labels_join_edits_with_spaces() {
|
||||
let seed: Vec<u8> = (0..64).collect();
|
||||
let joined = mutations(&seed, 200).find(|m| m.label.contains(' ')).unwrap();
|
||||
assert!(!joined.label.starts_with(' ') && !joined.label.ends_with(' ') && !joined.label.contains(" "), "{:?}", joined.label);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uses_caller_multiplier_variable() {
|
||||
let config = config("mult", 7).with_multiplier_variable("CRAFT_MEDIA_HOSTILE_MULT");
|
||||
let lookup = |name: &str| match name {
|
||||
"CRAFT_MEDIA_HOSTILE_MULT" => Some(OsString::from("3")),
|
||||
DEFAULT_MULTIPLIER_VARIABLE => Some(OsString::from("100")),
|
||||
_ => None,
|
||||
};
|
||||
assert_eq!(config.rounds_with(lookup), Ok(21));
|
||||
let calls = Arc::new(Mutex::new(0usize));
|
||||
let count = Arc::clone(&calls);
|
||||
hammer_with(&config, &[5, 6], move |_| *count.lock().unwrap() += 1, lookup).unwrap();
|
||||
// The seed, two truncations and 21 rounds.
|
||||
assert_eq!(*calls.lock().unwrap(), 24);
|
||||
assert_eq!(HammerConfig::new("default", 7).multiplier_variable(), "CRAFT_HOSTILE_MULT");
|
||||
assert_eq!(config.rounds_with(unset), Ok(7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_multiplier_names_variable() {
|
||||
for (value, reason) in [
|
||||
("0", "is zero"),
|
||||
("", "is empty"),
|
||||
("ten", "is not a whole number written in digits"),
|
||||
("-2", "is not a whole number written in digits"),
|
||||
("1.5", "is not a whole number written in digits"),
|
||||
("99999999999999999999999999", "is more than a usize holds"),
|
||||
] {
|
||||
let called = Arc::new(Mutex::new(false));
|
||||
let flag = Arc::clone(&called);
|
||||
let message = hammer_with(
|
||||
&config("bad mult", 3).with_multiplier_variable("MY_MULT"),
|
||||
&[1, 2],
|
||||
move |_| *flag.lock().unwrap() = true,
|
||||
|_| Some(OsString::from(value)),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert_eq!(message, format!("bad mult: MY_MULT={value:?} {reason}; set it to a whole number above zero"));
|
||||
assert!(!*called.lock().unwrap(), "parse ran before the bad value {value:?} was reported");
|
||||
}
|
||||
for name in ["", "A=B", "A\0B"] {
|
||||
assert_eq!(config("x", 1).with_multiplier_variable(name).rounds_with(unset), Err(HammerError::BadName { variable: name.to_owned() }));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_seed_runs_once() {
|
||||
assert_eq!(inputs(&config("empty", 1_000), &[]), [Vec::<u8>::new()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn large_seed_checks_every_prefix() {
|
||||
let seed: Vec<u8> = (0..10_000u32).map(|i| i as u8).collect();
|
||||
let mut lengths = BTreeMap::new();
|
||||
for input in inputs(&config("large", 0), &seed) {
|
||||
assert_eq!(input, seed[..input.len()]);
|
||||
*lengths.entry(input.len()).or_insert(0usize) += 1;
|
||||
}
|
||||
assert_eq!(lengths.len(), 10_001);
|
||||
assert!(lengths.values().all(|&n| n == 1), "a length ran more than once");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn iteration_product_overflow_is_error() {
|
||||
let config = config("overflow", usize::MAX / 2 + 1);
|
||||
let error = config.rounds_with(|_| Some(OsString::from("2"))).unwrap_err();
|
||||
assert_eq!(error, HammerError::Overflow { iterations: usize::MAX / 2 + 1, variable: UNSET.to_owned(), multiplier: 2 });
|
||||
let message = hammer_with(&config, &[1], |_| {}, |_| Some(OsString::from("2"))).unwrap_err();
|
||||
assert!(message.starts_with("overflow: ") && message.ends_with("is more than a usize holds"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn too_many_cases_fail_before_any_parse() {
|
||||
let called = Arc::new(Mutex::new(false));
|
||||
let flag = Arc::clone(&called);
|
||||
// The seed and its two truncations make three cases more.
|
||||
let message = hammer_with(&config("budget", MAX_CASES - 2), &[1, 2], move |_| *flag.lock().unwrap() = true, unset).unwrap_err();
|
||||
assert_eq!(message, format!("budget: {} cases is more than the limit of {MAX_CASES}", MAX_CASES + 1));
|
||||
assert!(!*called.lock().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mutations_are_deterministic_and_count_rounds() {
|
||||
let seed = b"a seed of some length";
|
||||
let first: Vec<_> = mutations(seed, 50).collect();
|
||||
assert_eq!(first.len(), 50);
|
||||
assert_eq!(first, mutations(seed, 50).collect::<Vec<_>>());
|
||||
assert_eq!(mutations(seed, 50).len(), 50);
|
||||
assert_eq!(mutations(&[], 50).count(), 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
//! Source-level platform regression guards and verification utilities for the Craft application suite.
|
||||
//!
|
||||
//! This crate provides non-panicking source analysis guards that detect known platform hazards
|
||||
//! and lifecycle regressions across Craft applications:
|
||||
//!
|
||||
//! * [`guards::predefined_quit`]: flags direct calls to `muda`'s predefined Quit item that bypass
|
||||
//! dirty document save prompts and window teardown on macOS.
|
||||
//! * [`guards::wgsl_float_literals`]: flags WGSL float literals above the finite range of `f32`
|
||||
//! (or of `f16` for an `h` suffix), which browser WebGPU compilers reject.
|
||||
//! * [`guards::std_instant`]: flags uses of `std::time::Instant` in code paths targeting the web,
|
||||
//! where `std::time::Instant::now()` panics on `wasm32`.
|
||||
//! * [`guards::rust_sources`]: directory crawler collecting `.rs` sources while excluding tests, with
|
||||
//! size limits ([`guards::rust_sources_with_limits`]). It is not built for `wasm32`.
|
||||
//!
|
||||
//! On targets other than `wasm32`, [`deadline::with_deadline`] runs a test body on its own thread
|
||||
//! and fails the test when the body does not finish within its limit, and [`deadline::wait_until`]
|
||||
//! polls a condition until it holds and fails the test when the limit passes first. Both read an
|
||||
//! optional limit variable that the caller names, `CRAFT_TEST_TIMEOUT_SECS` by default.
|
||||
//!
|
||||
//! [`hostile::mutations`] yields a fixed sequence of damaged copies of a valid seed, on every
|
||||
//! target. On targets other than `wasm32`, [`hostile::hammer`] runs a parser on the seed, every
|
||||
//! truncation of it and those copies, one after another on a worker thread with a time limit for
|
||||
//! each, and fails the test with the case that panicked or did not finish.
|
||||
//!
|
||||
//! ### Epistemic Scope: What Source Guards Establish vs What They Cannot
|
||||
//!
|
||||
//! Source-level guards provide deterministic syntactic evidence about codebase structure.
|
||||
//! They establish that checked source files do not contain forbidden symbol names, known invalid
|
||||
//! literal patterns, or unsafe platform invocations.
|
||||
//!
|
||||
//! They do **not** establish runtime correctness, protocol concurrency safety, or platform behavior
|
||||
//! under foreign host environments. They are lightweight syntactic regression gates designed to complement,
|
||||
//! rather than replace, platform integration tests and CI execution.
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub mod deadline;
|
||||
pub mod guards;
|
||||
pub mod hostile;
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub use deadline::{DeadlineLimit, wait_until, with_deadline};
|
||||
pub use guards::{AsFileSource, Finding, predefined_quit, std_instant, wgsl_float_literals, wgsl_float_literals_in};
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub use guards::{SourceLimits, rust_sources, rust_sources_with_limits};
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,99 @@
|
||||
#![cfg(not(target_arch = "wasm32"))]
|
||||
|
||||
use craft_testkit::{SourceLimits, rust_sources_with_limits};
|
||||
use std::io::ErrorKind;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// A scratch folder that this test made and removes on drop, also when the test panics.
|
||||
struct ScratchDir(PathBuf);
|
||||
|
||||
impl ScratchDir {
|
||||
fn new(tag: &str) -> Self {
|
||||
let base = std::env::temp_dir();
|
||||
for n in 0..1000u32 {
|
||||
let path = base.join(format!("craft_testkit_limits_{tag}_{}_{n}", std::process::id()));
|
||||
match std::fs::create_dir(&path) {
|
||||
Ok(()) => return Self(path),
|
||||
Err(e) if e.kind() == ErrorKind::AlreadyExists => continue,
|
||||
Err(e) => panic!("create scratch folder {}: {e}", path.display()),
|
||||
}
|
||||
}
|
||||
panic!("no free scratch folder name for {tag}");
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
&self.0
|
||||
}
|
||||
|
||||
fn write(&self, relative: &str, content: &str) {
|
||||
let path = self.0.join(relative);
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent).expect("create parent folder");
|
||||
}
|
||||
std::fs::write(path, content).expect("write file");
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ScratchDir {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_dir_all(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
fn three_files(tag: &str) -> ScratchDir {
|
||||
let dir = ScratchDir::new(tag);
|
||||
dir.write("src/a.rs", "0123456789");
|
||||
dir.write("src/b.rs", "0123456789");
|
||||
dir.write("src/c.rs", "0123456789");
|
||||
dir
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn defaults_are_documented_values() {
|
||||
let limits = SourceLimits::default();
|
||||
assert_eq!(limits.max_file_bytes, 8 << 20);
|
||||
assert_eq!(limits.max_total_bytes, 256 << 20);
|
||||
assert_eq!(limits.max_files, 50_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn files_within_the_limits_are_read() {
|
||||
let dir = three_files("within");
|
||||
let limits = SourceLimits { max_file_bytes: 10, max_total_bytes: 30, max_files: 3 };
|
||||
let sources = rust_sources_with_limits(dir.path(), limits).expect("within limits");
|
||||
assert_eq!(sources.len(), 3);
|
||||
assert!(sources.iter().all(|(_, content)| content == "0123456789"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_over_the_size_limit_is_an_error() {
|
||||
let dir = three_files("file");
|
||||
let limits = SourceLimits { max_file_bytes: 9, ..SourceLimits::default() };
|
||||
let err = rust_sources_with_limits(dir.path(), limits).expect_err("file over the limit");
|
||||
assert_eq!(err.kind(), ErrorKind::FileTooLarge);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn files_over_the_total_limit_are_an_error() {
|
||||
let dir = three_files("total");
|
||||
let limits = SourceLimits { max_total_bytes: 29, ..SourceLimits::default() };
|
||||
let err = rust_sources_with_limits(dir.path(), limits).expect_err("total over the limit");
|
||||
assert_eq!(err.kind(), ErrorKind::QuotaExceeded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn more_files_than_the_count_limit_is_an_error() {
|
||||
let dir = three_files("count");
|
||||
let limits = SourceLimits { max_files: 2, ..SourceLimits::default() };
|
||||
let err = rust_sources_with_limits(dir.path(), limits).expect_err("too many files");
|
||||
assert_eq!(err.kind(), ErrorKind::QuotaExceeded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn files_the_filter_skips_do_not_count_against_the_limits() {
|
||||
let dir = three_files("skipped");
|
||||
dir.write("src/tests.rs", &"x".repeat(100));
|
||||
dir.write("tests/big.rs", &"x".repeat(100));
|
||||
let limits = SourceLimits { max_file_bytes: 10, max_total_bytes: 30, max_files: 3 };
|
||||
assert_eq!(rust_sources_with_limits(dir.path(), limits).expect("skipped files are not read").len(), 3);
|
||||
}
|
||||
Reference in new issue
Block a user