From 4cd8be548c794c7248e9c9e955eae4e26daa32c9 Mon Sep 17 00:00:00 2001 From: Andy Leiserson Date: Mon, 26 May 2025 00:28:50 -0700 Subject: [PATCH] Add an xtask to run the CTS (#7719) --- Cargo.toml | 2 +- README.md | 26 ++++++++------ xtask/src/cts.rs | 87 +++++++++++++++++++++++++++++++++++++++++++++++ xtask/src/main.rs | 5 +++ 4 files changed, 108 insertions(+), 12 deletions(-) create mode 100644 xtask/src/cts.rs diff --git a/Cargo.toml b/Cargo.toml index 89ea85680..28b65cae0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -190,7 +190,7 @@ unicode-ident = "1.0.5" walkdir = "2.3" winit = { version = "0.29", features = ["android-native-activity"] } which = "7" -xshell = "0.2" +xshell = "0.2.2" # Metal dependencies metal = "0.32" diff --git a/README.md b/README.md index fdcf878dc..f5d58547b 100644 --- a/README.md +++ b/README.md @@ -216,26 +216,30 @@ WebGPU includes a Conformance Test Suite to validate that implementations are wo To have GitHub run the CTS against a pull request, you can add the `PR: run CTS` label to the PR. -To run the CTS locally, first, you need to check it out: +To run the CTS locally, run: ``` -# In the root of your wgpu tree: -git clone https://github.com/gpuweb/cts.git -cd cts -# works in bash and powershell -git checkout $(cat ../cts_runner/revision.txt) +cargo xtask cts ``` -To run a given set of tests: +This will clone the CTS into the `cts` directory, check out the +[pinned version](./cts_runner/revision.txt), and run the +[default list of enabled tests](./cts_runner/test.lst). + +You can also specify a test selector on the command line: ``` -# Must be inside the `cts` folder we just checked out, else this will fail -cargo run --manifest-path ../Cargo.toml -p cts_runner --bin cts_runner -- ./tools/run_deno --verbose "" +cargo xtask cts 'webgpu:api,operation,command_buffer,basic:*' ``` -To find the full list of tests, go to the [online cts viewer](https://gpuweb.github.io/cts/standalone/?runnow=0&worker=0&debug=0&q=webgpu:*). +Or supply your own test list in a file: -The list of currently enabled CTS tests can be found [here](./cts_runner/test.lst). +``` +cargo xtask cts -f your_tests.lst +``` + +To find the full list of tests, go to the +[web version of the CTS](https://gpuweb.github.io/cts/standalone/?runnow=0&worker=0&debug=0&q=webgpu:*). ## Tracking the WebGPU and WGSL draft specifications diff --git a/xtask/src/cts.rs b/xtask/src/cts.rs new file mode 100644 index 000000000..9d76620e4 --- /dev/null +++ b/xtask/src/cts.rs @@ -0,0 +1,87 @@ +use anyhow::Context; +use pico_args::Arguments; +use std::ffi::OsString; +use xshell::Shell; + +/// Path within the repository where the CTS will be checked out. +const CTS_CHECKOUT_PATH: &str = "cts"; + +/// Path within the repository to a file containing the git revision of the CTS to check out. +const CTS_REVISION_PATH: &str = "cts_runner/revision.txt"; + +/// URL of the CTS git repository. +const CTS_GIT_URL: &str = "https://github.com/gpuweb/cts.git"; + +/// Path to default CTS test list. +const CTS_DEFAULT_TEST_LIST: &str = "cts_runner/test.lst"; + +pub fn run_cts(shell: Shell, mut args: Arguments) -> anyhow::Result<()> { + let mut list_files = Vec::::new(); + + while let Some(file) = args.opt_value_from_str("-f")? { + list_files.push(file); + } + + let mut tests = args.finish(); + + if tests.is_empty() && list_files.is_empty() { + log::info!("Reading default test list from {CTS_DEFAULT_TEST_LIST}"); + list_files.push(OsString::from(CTS_DEFAULT_TEST_LIST)); + } + + for file in list_files { + tests.extend(shell.read_file(file)?.lines().filter_map(|line| { + let trimmed = line.trim(); + let is_comment = trimmed.starts_with("//") || trimmed.starts_with("#"); + (!is_comment).then(|| OsString::from(trimmed)) + })) + } + + let cts_revision = shell + .read_file(CTS_REVISION_PATH) + .context(format!( + "Failed to read CTS git SHA from {CTS_REVISION_PATH}" + ))? + .trim() + .to_string(); + + if !shell.path_exists(CTS_CHECKOUT_PATH) { + log::info!("Cloning CTS"); + shell + .cmd("git") + .args(["clone", CTS_GIT_URL, CTS_CHECKOUT_PATH]) + .quiet() + .run() + .context("Failed to clone CTS")?; + } + + shell.change_dir(CTS_CHECKOUT_PATH); + + log::info!("Checking out CTS"); + shell + .cmd("git") + .args(["checkout", "--quiet", &cts_revision]) + .quiet() + .run() + .context("Failed to check out CTS")?; + + log::info!("Running CTS"); + for test in &tests { + shell + .cmd("cargo") + .args(["run"]) + .args(["--manifest-path", "../Cargo.toml"]) + .args(["-p", "cts_runner"]) + .args(["--bin", "cts_runner"]) + .args(["--", "./tools/run_deno", "--verbose"]) + .args([test]) + .run() + .context("CTS failed")?; + } + + if tests.len() > 1 { + log::info!("Summary reflects only tests from the last selector, not the entire run."); + } + + Ok(()) +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index d1c12f762..02b2fab2c 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -6,6 +6,7 @@ use std::process::ExitCode; use anyhow::Context; use pico_args::Arguments; +mod cts; mod run_wasm; mod test; mod util; @@ -15,6 +16,9 @@ const HELP: &str = "\ Usage: xtask Commands: + cts [ | -f ]... + Check out, build, and run CTS tests + run-wasm Build and run web examples @@ -76,6 +80,7 @@ fn main() -> anyhow::Result { shell.change_dir(String::from(env!("CARGO_MANIFEST_DIR")) + "/.."); match subcommand.as_deref() { + Some("cts") => cts::run_cts(shell, args)?, Some("run-wasm") => run_wasm::run_wasm(shell, args)?, Some("test") => test::run_tests(shell, args)?, Some("vendor-web-sys") => vendor_web_sys::run_vendor_web_sys(shell, args)?,