test(cts): Invoke cts_runner directly instead of via a cargo build (#8960)

Co-authored-by: Erich Gubler <erichdongubler@gmail.com>
This commit is contained in:
Andy LeisersonandErich Gubler authored and GitHub committed 2026-02-04 11:41:39 -07:00
1 parent 3b55cf9b8c
commit 26a4f6679d
6 files changed
+125 -36

No files matched your search

+5 -2
View File
@@ -8,6 +8,9 @@ inputs:
# Corresponds to https://github.com/gfx-rs/ci-build/releases
ci-binary-build:
default: "build26"
target-dir:
description: "The directory into which to install the Mesa libraries."
default: "target/llvm-cov-target/debug"
runs:
using: "composite"
steps:
@@ -53,8 +56,8 @@ runs:
curl.exe -L --retry 5 https://github.com/pal1000/mesa-dist-win/releases/download/$MESA_VERSION/mesa3d-$MESA_VERSION-release-msvc.7z -o mesa.7z
7z.exe e mesa.7z -omesa x64/{opengl32.dll,libgallium_wgl.dll,libglapi.dll,vulkan_lvp.dll,lvp_icd.x86_64.json}
cp -v mesa/* target/llvm-cov-target/debug/
cp -v mesa/* target/llvm-cov-target/debug/deps
cp -v mesa/* ${{ inputs.target-dir }}/
cp -v mesa/* ${{ inputs.target-dir }}/deps
# We need to use cygpath to convert PWD to a windows path as we're using bash.
echo "VK_DRIVER_FILES=`cygpath --windows $PWD/mesa/lvp_icd.x86_64.json`" >> "$GITHUB_ENV"
+23 -6
View File
@@ -50,22 +50,33 @@ jobs:
os: windows-2022
target: x86_64-pc-windows-msvc
backend: dx12
shell: bash
# Mac
# Need to use `zsh` for `source <(cmd)` because it doesn't work in
# ancient MacOS bash. `zsh` is a non-standard shell in GitHub
# actions, so must be specified as a command string.
- platform: Mac aarch64
os: macos-14
target: x86_64-apple-darwin
backend: metal
shell: zsh {0}
# Linux
- platform: Linux x86_64
os: ubuntu-24.04
target: x86_64-unknown-linux-gnu
backend: vulkan
shell: bash
name: ${{ matrix.suite }} ${{ matrix.platform }}
runs-on: ${{ matrix.os }}
defaults:
run:
# Substitution of ${{ matrix.shell }} appears not to work in per-step configuration
shell: ${{ matrix.shell }}
steps:
- name: checkout repo
uses: actions/checkout@v6
@@ -102,23 +113,29 @@ jobs:
if: matrix.os == 'windows-2022'
uses: ./.github/actions/install-dxc
# Note: `target-dir` is intentionally different from other jobs.
# See note in xtask about `llvm-cov show-env`.
- name: (Windows) Install WARP
if: matrix.os == 'windows-2022'
uses: ./.github/actions/install-warp
with:
target-dir: "target/llvm-cov-target/debug"
target-dir: "target/debug"
- name: (Linux) Install Mesa
if: matrix.os == 'ubuntu-24.04'
uses: ./.github/actions/install-mesa
with:
target-dir: "target/debug"
# Mixing --llvm-cov with deno (rusty_v8) has linking problems.
# Explicitly set the backend to avoid EGL messages in output,
# because these tests check stdout.
# Some of these tests check stdout, so we explicitly set the backend
# to avoid getting EGL messages in the output.
- name: Test cts_runner
if: matrix.suite == 'other'
shell: bash
run: DENO_WEBGPU_BACKEND=${{ matrix.backend }} cargo --locked test -p cts_runner
run: |
export LLVM_PROFILE_FILE=${{ github.workspace }}/target/wgpu-%p-%m.profraw
export DENO_WEBGPU_BACKEND=${{ matrix.backend }}
cargo --locked llvm-cov --no-cfg-coverage --no-report test -p cts_runner
- name: Run CTS
shell: bash
@@ -126,11 +143,11 @@ jobs:
- name: Generate coverage report
id: coverage
shell: bash
continue-on-error: true
run: |
set -e
source <(cargo llvm-cov --no-cfg-coverage show-env --sh)
cargo --locked llvm-cov report --lcov --output-path lcov.info
- name: Upload coverage report to Codecov
Generated
+1
View File
@@ -5014,6 +5014,7 @@ dependencies = [
"log",
"pico-args",
"regex-lite",
"serde_json",
"xshell",
]
+1
View File
@@ -18,4 +18,5 @@ env_logger.workspace = true
regex-lite.workspace = true
log.workspace = true
pico-args.workspace = true
serde_json.workspace = true
xshell.workspace = true
+79 -28
View File
@@ -37,7 +37,7 @@ use regex_lite::{Regex, RegexBuilder};
use std::{ffi::OsString, sync::LazyLock};
use xshell::Shell;
use crate::util::git_version_at_least;
use crate::util::{git_version_at_least, parse_binary_from_cargo_json};
/// Path within the repository where the CTS will be checked out.
const CTS_CHECKOUT_PATH: &str = "cts";
@@ -215,12 +215,12 @@ pub fn run_cts(
}
let wgpu_cargo_toml = std::path::absolute(shell.current_dir().join("Cargo.toml"))
.context("Failed to get path to Cargo.toml")?;
.context("Failed to get path to `Cargo.toml`")?;
let cts_revision = shell
.read_file(CTS_REVISION_PATH)
.context(format!(
"Failed to read CTS git SHA from {CTS_REVISION_PATH}"
"Failed to read CTS git SHA from `{CTS_REVISION_PATH}`"
))?
.trim()
.to_string();
@@ -311,27 +311,83 @@ pub fn run_cts(
log::info!("Skipping CTS checkout because --skip-checkout was specified");
}
let run_flags = if llvm_cov {
&["llvm-cov", "--no-cfg-coverage", "--no-report", "run"][..]
} else {
&["run"][..]
};
let mut cargo_opts: Vec<OsString> = vec![
"--manifest-path".into(),
wgpu_cargo_toml.into(),
"-p".into(),
"cts_runner".into(),
"--bin".into(),
"cts_runner".into(),
];
if release {
cargo_opts.push("--release".into());
}
if let Some(passthrough_args) = passthrough_args {
let env_vars = if llvm_cov {
// Typically coverage runs are done via cargo with `cargo llvm-cov run`. Running the
// coverage-instrumented binary directly requires setting some environment variables. See
// <https://github.com/taiki-e/cargo-llvm-cov/blob/main/README.md#get-coverage-of-external-tests>
//
// Unlike regular `llvm-cov run`, which builds artifacts in `target/llvm-cov-target`,
// `llvm-cov show-env` uses the regular target directory for artifacts. Because of this,
// the CTS job configures the `install-mesa` and `install-warp` actions with the regular
// target directory, not the `llvm-cov-target` directory.
let env = shell
.cmd("cargo")
.args(&["llvm-cov", "--no-cfg-coverage", "show-env"])
.read()
.context("Failed to get `llvm-cov` environment variables")?
.lines()
.filter_map(|line| {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
None
} else {
line.split_once('=')
}
})
.map(|(key, value)| {
let value = value.trim_matches('"').trim_matches('\'');
(key.to_string(), value.to_string())
})
.collect::<Vec<_>>()
.into_iter();
// Avoid conflicts between coverage and non-coverage build artifacts.
// This is recommended by the `cargo-llvm-cov` docs.
shell
.cmd("cargo")
.args(run_flags)
.args(["--manifest-path".as_ref(), wgpu_cargo_toml.as_os_str()])
.args(["-p", "cts_runner"])
.args(["--bin", "cts_runner"])
.args(release.then_some("--release"))
.arg("--")
.args(enable_external_texture.then_some("--enable-external-texture"))
.args(["./tools/run_deno", "--verbose"])
.args(&passthrough_args)
.run()?;
.envs(env.clone())
.args(["llvm-cov", "clean", "--workspace"])
.run()
.context("Failed to run `llvm-cov clean`")?;
return Ok(());
env
} else {
vec![].into_iter()
};
let build_output = shell
.cmd("cargo")
.envs(env_vars.clone())
.args(["build", "--message-format", "json-render-diagnostics"])
.args(&cargo_opts)
.read()
.context("Failed to build `cts_runner`")?;
let bin = parse_binary_from_cargo_json(&build_output)
.context("Failed to identify executable from cargo build output")?;
let cts_bin = &["./tools/run_deno", "--verbose"];
if let Some(passthrough_args) = passthrough_args {
return Ok(shell
.cmd(bin)
.envs(env_vars)
.args(cts_bin)
.args(enable_external_texture.then_some("--enable-external-texture"))
.args(&passthrough_args)
.run()?);
}
log::info!("Running CTS");
@@ -352,15 +408,10 @@ pub fn run_cts(
}
let cmd = shell
.cmd("cargo")
.args(run_flags)
.args(["--manifest-path".as_ref(), wgpu_cargo_toml.as_os_str()])
.args(["-p", "cts_runner"])
.args(["--bin", "cts_runner"])
.args(release.then_some("--release"))
.arg("--")
.cmd(&bin)
.envs(env_vars.clone())
.args(enable_external_texture.then_some("--enable-external-texture"))
.args(["./tools/run_deno", "--verbose"])
.args(cts_bin)
.args([&test.selector]);
match output_filter {
+16
View File
@@ -2,6 +2,7 @@ use std::{ffi::OsString, io, process::Command};
use anyhow::Context;
use pico_args::Arguments;
use serde_json::{json, Value};
use xshell::Shell;
pub(crate) struct Program {
@@ -128,6 +129,21 @@ fn parse_git_version_output(output: &str) -> anyhow::Result<GitVersion> {
Ok(parsed)
}
pub(crate) fn parse_binary_from_cargo_json(jsonl: &str) -> Option<String> {
jsonl
.lines()
.filter_map(|line| serde_json::from_str::<Value>(line).ok())
.filter(|json| {
json.get("reason") == Some(&json!("compiler-artifact"))
&& json.get("target").and_then(|obj| obj.get("kind")) == Some(&json!(["bin"]))
})
.find_map(|json| {
json.get("executable")
.and_then(Value::as_str)
.map(ToOwned::to_owned)
})
}
#[test]
fn test_git_version_parsing() {
macro_rules! test_ok {