diff --git a/.github/actions/install-mesa/action.yml b/.github/actions/install-mesa/action.yml index 4a61c948c..7994b0726 100644 --- a/.github/actions/install-mesa/action.yml +++ b/.github/actions/install-mesa/action.yml @@ -8,6 +8,9 @@ inputs: # Corresponds to https://github.com/gfx-rs/ci-build/releases ci-binary-build: default: "build26" + target-dir: + description: "The directory into which to install the Mesa libraries." + default: "target/llvm-cov-target/debug" runs: using: "composite" steps: @@ -53,8 +56,8 @@ runs: curl.exe -L --retry 5 https://github.com/pal1000/mesa-dist-win/releases/download/$MESA_VERSION/mesa3d-$MESA_VERSION-release-msvc.7z -o mesa.7z 7z.exe e mesa.7z -omesa x64/{opengl32.dll,libgallium_wgl.dll,libglapi.dll,vulkan_lvp.dll,lvp_icd.x86_64.json} - cp -v mesa/* target/llvm-cov-target/debug/ - cp -v mesa/* target/llvm-cov-target/debug/deps + cp -v mesa/* ${{ inputs.target-dir }}/ + cp -v mesa/* ${{ inputs.target-dir }}/deps # We need to use cygpath to convert PWD to a windows path as we're using bash. echo "VK_DRIVER_FILES=`cygpath --windows $PWD/mesa/lvp_icd.x86_64.json`" >> "$GITHUB_ENV" diff --git a/.github/workflows/cts.yml b/.github/workflows/cts.yml index 9fad1867b..03c244518 100644 --- a/.github/workflows/cts.yml +++ b/.github/workflows/cts.yml @@ -50,22 +50,33 @@ jobs: os: windows-2022 target: x86_64-pc-windows-msvc backend: dx12 + shell: bash # Mac + # Need to use `zsh` for `source <(cmd)` because it doesn't work in + # ancient MacOS bash. `zsh` is a non-standard shell in GitHub + # actions, so must be specified as a command string. - platform: Mac aarch64 os: macos-14 target: x86_64-apple-darwin backend: metal + shell: zsh {0} # Linux - platform: Linux x86_64 os: ubuntu-24.04 target: x86_64-unknown-linux-gnu backend: vulkan + shell: bash name: ${{ matrix.suite }} ${{ matrix.platform }} runs-on: ${{ matrix.os }} + defaults: + run: + # Substitution of ${{ matrix.shell }} appears not to work in per-step configuration + shell: ${{ matrix.shell }} + steps: - name: checkout repo uses: actions/checkout@v6 @@ -102,23 +113,29 @@ jobs: if: matrix.os == 'windows-2022' uses: ./.github/actions/install-dxc + # Note: `target-dir` is intentionally different from other jobs. + # See note in xtask about `llvm-cov show-env`. - name: (Windows) Install WARP if: matrix.os == 'windows-2022' uses: ./.github/actions/install-warp with: - target-dir: "target/llvm-cov-target/debug" + target-dir: "target/debug" - name: (Linux) Install Mesa if: matrix.os == 'ubuntu-24.04' uses: ./.github/actions/install-mesa + with: + target-dir: "target/debug" - # Mixing --llvm-cov with deno (rusty_v8) has linking problems. - # Explicitly set the backend to avoid EGL messages in output, - # because these tests check stdout. + # Some of these tests check stdout, so we explicitly set the backend + # to avoid getting EGL messages in the output. - name: Test cts_runner if: matrix.suite == 'other' shell: bash - run: DENO_WEBGPU_BACKEND=${{ matrix.backend }} cargo --locked test -p cts_runner + run: | + export LLVM_PROFILE_FILE=${{ github.workspace }}/target/wgpu-%p-%m.profraw + export DENO_WEBGPU_BACKEND=${{ matrix.backend }} + cargo --locked llvm-cov --no-cfg-coverage --no-report test -p cts_runner - name: Run CTS shell: bash @@ -126,11 +143,11 @@ jobs: - name: Generate coverage report id: coverage - shell: bash continue-on-error: true run: | set -e + source <(cargo llvm-cov --no-cfg-coverage show-env --sh) cargo --locked llvm-cov report --lcov --output-path lcov.info - name: Upload coverage report to Codecov diff --git a/Cargo.lock b/Cargo.lock index 8882ccdaf..689d9b547 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5014,6 +5014,7 @@ dependencies = [ "log", "pico-args", "regex-lite", + "serde_json", "xshell", ] diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 330949337..c167e26ca 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -18,4 +18,5 @@ env_logger.workspace = true regex-lite.workspace = true log.workspace = true pico-args.workspace = true +serde_json.workspace = true xshell.workspace = true diff --git a/xtask/src/cts.rs b/xtask/src/cts.rs index eb7c04de7..f24e78239 100644 --- a/xtask/src/cts.rs +++ b/xtask/src/cts.rs @@ -37,7 +37,7 @@ use regex_lite::{Regex, RegexBuilder}; use std::{ffi::OsString, sync::LazyLock}; use xshell::Shell; -use crate::util::git_version_at_least; +use crate::util::{git_version_at_least, parse_binary_from_cargo_json}; /// Path within the repository where the CTS will be checked out. const CTS_CHECKOUT_PATH: &str = "cts"; @@ -215,12 +215,12 @@ pub fn run_cts( } let wgpu_cargo_toml = std::path::absolute(shell.current_dir().join("Cargo.toml")) - .context("Failed to get path to Cargo.toml")?; + .context("Failed to get path to `Cargo.toml`")?; let cts_revision = shell .read_file(CTS_REVISION_PATH) .context(format!( - "Failed to read CTS git SHA from {CTS_REVISION_PATH}" + "Failed to read CTS git SHA from `{CTS_REVISION_PATH}`" ))? .trim() .to_string(); @@ -311,27 +311,83 @@ pub fn run_cts( log::info!("Skipping CTS checkout because --skip-checkout was specified"); } - let run_flags = if llvm_cov { - &["llvm-cov", "--no-cfg-coverage", "--no-report", "run"][..] - } else { - &["run"][..] - }; + let mut cargo_opts: Vec = vec![ + "--manifest-path".into(), + wgpu_cargo_toml.into(), + "-p".into(), + "cts_runner".into(), + "--bin".into(), + "cts_runner".into(), + ]; + if release { + cargo_opts.push("--release".into()); + } - if let Some(passthrough_args) = passthrough_args { + let env_vars = if llvm_cov { + // Typically coverage runs are done via cargo with `cargo llvm-cov run`. Running the + // coverage-instrumented binary directly requires setting some environment variables. See + // + // + // Unlike regular `llvm-cov run`, which builds artifacts in `target/llvm-cov-target`, + // `llvm-cov show-env` uses the regular target directory for artifacts. Because of this, + // the CTS job configures the `install-mesa` and `install-warp` actions with the regular + // target directory, not the `llvm-cov-target` directory. + let env = shell + .cmd("cargo") + .args(&["llvm-cov", "--no-cfg-coverage", "show-env"]) + .read() + .context("Failed to get `llvm-cov` environment variables")? + .lines() + .filter_map(|line| { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + None + } else { + line.split_once('=') + } + }) + .map(|(key, value)| { + let value = value.trim_matches('"').trim_matches('\''); + (key.to_string(), value.to_string()) + }) + .collect::>() + .into_iter(); + + // Avoid conflicts between coverage and non-coverage build artifacts. + // This is recommended by the `cargo-llvm-cov` docs. shell .cmd("cargo") - .args(run_flags) - .args(["--manifest-path".as_ref(), wgpu_cargo_toml.as_os_str()]) - .args(["-p", "cts_runner"]) - .args(["--bin", "cts_runner"]) - .args(release.then_some("--release")) - .arg("--") - .args(enable_external_texture.then_some("--enable-external-texture")) - .args(["./tools/run_deno", "--verbose"]) - .args(&passthrough_args) - .run()?; + .envs(env.clone()) + .args(["llvm-cov", "clean", "--workspace"]) + .run() + .context("Failed to run `llvm-cov clean`")?; - return Ok(()); + env + } else { + vec![].into_iter() + }; + + let build_output = shell + .cmd("cargo") + .envs(env_vars.clone()) + .args(["build", "--message-format", "json-render-diagnostics"]) + .args(&cargo_opts) + .read() + .context("Failed to build `cts_runner`")?; + + let bin = parse_binary_from_cargo_json(&build_output) + .context("Failed to identify executable from cargo build output")?; + + let cts_bin = &["./tools/run_deno", "--verbose"]; + + if let Some(passthrough_args) = passthrough_args { + return Ok(shell + .cmd(bin) + .envs(env_vars) + .args(cts_bin) + .args(enable_external_texture.then_some("--enable-external-texture")) + .args(&passthrough_args) + .run()?); } log::info!("Running CTS"); @@ -352,15 +408,10 @@ pub fn run_cts( } let cmd = shell - .cmd("cargo") - .args(run_flags) - .args(["--manifest-path".as_ref(), wgpu_cargo_toml.as_os_str()]) - .args(["-p", "cts_runner"]) - .args(["--bin", "cts_runner"]) - .args(release.then_some("--release")) - .arg("--") + .cmd(&bin) + .envs(env_vars.clone()) .args(enable_external_texture.then_some("--enable-external-texture")) - .args(["./tools/run_deno", "--verbose"]) + .args(cts_bin) .args([&test.selector]); match output_filter { diff --git a/xtask/src/util.rs b/xtask/src/util.rs index aaece549a..7e50cd942 100644 --- a/xtask/src/util.rs +++ b/xtask/src/util.rs @@ -2,6 +2,7 @@ use std::{ffi::OsString, io, process::Command}; use anyhow::Context; use pico_args::Arguments; +use serde_json::{json, Value}; use xshell::Shell; pub(crate) struct Program { @@ -128,6 +129,21 @@ fn parse_git_version_output(output: &str) -> anyhow::Result { Ok(parsed) } +pub(crate) fn parse_binary_from_cargo_json(jsonl: &str) -> Option { + jsonl + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter(|json| { + json.get("reason") == Some(&json!("compiler-artifact")) + && json.get("target").and_then(|obj| obj.get("kind")) == Some(&json!(["bin"])) + }) + .find_map(|json| { + json.get("executable") + .and_then(Value::as_str) + .map(ToOwned::to_owned) + }) +} + #[test] fn test_git_version_parsing() { macro_rules! test_ok {