Update to the latest CTS; run it on all platforms and on all PRs (gfx-rs#7752)

Author: andyleiserson

.github/workflows/ci.yml

@@ -29,6 +29,7 @@ env:
   DXC_RELEASE: "v1.8.2502"
   DXC_FILENAME: "dxc_2025_02_20.zip"
 
+  # These Mesa version definitions are duplicated in the CTS job.
   # Sourced from https://archive.mesa3d.org/. Bumping this requires
   # updating the mesa build in https://github.com/gfx-rs/ci-build and creating a new release.
   MESA_VERSION: "24.3.4"
@@ -626,6 +627,7 @@ jobs:
           sudo apt-get update
           sudo apt install -y vulkan-sdk
 
+      # This step is duplicated in the CTS job.
       - name: (Linux) Install Mesa
         if: matrix.os == 'ubuntu-24.04'
         shell: bash

.github/workflows/cts.yml

@@ -13,11 +13,15 @@ env:
   RUST_BACKTRACE: full
   MSRV: "1.84"
 
+  # These Mesa version definitions are duplicated in the CI job.
+  # Sourced from https://archive.mesa3d.org/. Bumping this requires
+  # updating the mesa build in https://github.com/gfx-rs/ci-build and creating a new release.
+  MESA_VERSION: "24.3.4"
+  # Corresponds to https://github.com/gfx-rs/ci-build/releases
+  CI_BINARY_BUILD: "build20"
+
 jobs:
   cts:
-    # For pull requests, only run if we add the "PR: run CTS" label
-    if: "github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'PR: run CTS')"
-
     strategy:
       fail-fast: false
       matrix:
@@ -26,28 +30,23 @@ jobs:
           - name: Windows x86_64
             os: windows-2022
             target: x86_64-pc-windows-msvc
-            backends: dx12
+
+          # Mac
+          - name: Mac aarch64
+            os: macos-14
+            target: x86_64-apple-darwin
 
           # Linux
-          #- name: Linux x86_64
-          #  os: ubuntu-20.04
-          #  target: x86_64-unknown-linux-gnu
-          #  backends: vulkan # gl
+          - name: Linux x86_64
+            os: ubuntu-24.04
+            target: x86_64-unknown-linux-gnu
 
     name: CTS ${{ matrix.name }}
     runs-on: ${{ matrix.os }}
 
     steps:
       - name: checkout repo
         uses: actions/checkout@v4
-        with:
-          path: wgpu
-
-      - name: checkout cts
-        run: |
-          git clone https://github.com/gpuweb/cts.git
-          cd cts
-          git checkout $(cat ../wgpu/cts_runner/revision.txt)
 
       - name: Install Repo MSRV toolchain
         run: |
@@ -58,31 +57,47 @@ jobs:
       - name: caching
         uses: Swatinem/rust-cache@v2
         with:
-          key: cts-b # suffix for cache busting
-          workspaces: wgpu
+          prefix-key: v1-rust # Increment version for cache busting
+          cache-directories: cts
 
       # We enable line numbers for panics, but that's it
       - name: disable debug
         shell: bash
         run: |
-          mkdir -p wgpu/.cargo
-          echo """[profile.dev]
-          debug = 1" > wgpu/.cargo/config.toml
+          mkdir -p .cargo
+          cat <<EOF >> .cargo/config.toml
+          [profile.dev]
+          debug = 1
+          EOF
 
-      - name: build CTS runner
+      # This step is duplicated in the CI job.
+      - name: (Linux) Install Mesa
+        if: matrix.os == 'ubuntu-24.04'
+        shell: bash
         run: |
-          cargo build --manifest-path wgpu/cts_runner/Cargo.toml
+          set -e
+
+          curl -L --retry 5 https://github.com/gfx-rs/ci-build/releases/download/$CI_BINARY_BUILD/mesa-$MESA_VERSION-linux-x86_64.tar.xz -o mesa.tar.xz
+          mkdir mesa
+          tar xpf mesa.tar.xz -C mesa
+
+          # The ICD provided by the mesa build is hardcoded to the build environment.
+          #
+          # We write out our own ICD file to point to the mesa vulkan
+          cat <<- EOF > icd.json
+          {
+            "ICD": {
+                "api_version": "1.1.255",
+                "library_path": "$PWD/mesa/lib/x86_64-linux-gnu/libvulkan_lvp.so"
+            },
+            "file_format_version": "1.0.0"
+          }
+          EOF
+
+          echo "VK_DRIVER_FILES=$PWD/icd.json" >> "$GITHUB_ENV"
+          echo "LD_LIBRARY_PATH=$PWD/mesa/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH" >> "$GITHUB_ENV"
+          echo "LIBGL_DRIVERS_PATH=$PWD/mesa/lib/x86_64-linux-gnu/dri" >> "$GITHUB_ENV"
 
       - name: run CTS
         shell: bash
-        run: |
-          cd cts;
-          for backend in ${{ matrix.backends }}; do
-            echo "======= CTS TESTS $backend ======";
-            grep -v '^//' ../wgpu/cts_runner/test.lst | while IFS=$' \t\r\n' read test; do
-              echo "=== Running $test ===";
-              DENO_WEBGPU_BACKEND=$backend cargo run --manifest-path ../wgpu/cts_runner/Cargo.toml --frozen -- ./tools/run_deno --verbose "$test";
-            done
-          done
-          echo;
-          echo "Note: Summary reflects only the last test suite, not the entire run."
+        run: cargo xtask cts

cts_runner/revision.txt

@@ -1 +1 @@
-049916a6191725aad5a5692e5a22df47d45c88fa
+ccd84d9943abc01b0421c48080280e9e940336b4

cts_runner/test.lst

@@ -16,5 +16,7 @@ webgpu:api,operation,rendering,color_target_state:blending,formats:*
 webgpu:api,operation,rendering,color_target_state:blend_constant,setting:*
 webgpu:api,operation,rendering,depth:*
 webgpu:api,operation,rendering,draw:*
-webgpu:api,operation,uncapturederror:*
+webgpu:api,operation,uncapturederror:iff_uncaptured:*
+//FAIL: webgpu:api,operation,uncapturederror:onuncapturederror_order_wrt_addEventListener
+// There are also two unimplemented SKIPs in uncapturederror not enumerated here.
 webgpu:shader,execution,flow_control,return:*

deno_webgpu/webidl.rs

@@ -487,7 +487,7 @@ pub fn feature_names_to_features(names: Vec<GPUFeatureName>) -> wgpu_types::Feat
       GPUFeatureName::Multiview => Features::MULTIVIEW,
       GPUFeatureName::VertexAttribute64Bit => Features::VERTEX_ATTRIBUTE_64BIT,
       GPUFeatureName::ShaderF64 => Features::SHADER_F64,
-      GPUFeatureName::ShaderI16 => Features::SHADER_F16,
+      GPUFeatureName::ShaderI16 => Features::SHADER_I16,
       GPUFeatureName::ShaderPrimitiveIndex => Features::SHADER_PRIMITIVE_INDEX,
       GPUFeatureName::ShaderEarlyDepthTest => Features::SHADER_EARLY_DEPTH_TEST,
     };

xtask/src/cts.rs

@@ -38,7 +38,8 @@ pub fn run_cts(shell: Shell, mut args: Arguments) -> anyhow::Result<()> {
         }))
     }
 
-    let wgpu_cargo_toml = shell.current_dir().join("Cargo.toml").canonicalize()?;
+    let wgpu_cargo_toml = std::path::absolute(shell.current_dir().join("Cargo.toml"))
+        .context("Failed to get path to Cargo.toml")?;
 
     let cts_revision = shell
         .read_file(CTS_REVISION_PATH)
@@ -59,9 +60,32 @@ pub fn run_cts(shell: Shell, mut args: Arguments) -> anyhow::Result<()> {
             .quiet()
             .run()
             .context("Failed to clone CTS")?;
-    }
 
-    shell.change_dir(CTS_CHECKOUT_PATH);
+        shell.change_dir(CTS_CHECKOUT_PATH);
+    } else if !skip_checkout {
+        shell.change_dir(CTS_CHECKOUT_PATH);
+
+        // If we don't have the CTS commit we want, try to fetch it.
+        if shell
+            .cmd("git")
+            .args(["cat-file", "commit", &cts_revision])
+            .quiet()
+            .ignore_stdout()
+            .ignore_stderr()
+            .run()
+            .is_err()
+        {
+            log::info!("Fetching CTS");
+            shell
+                .cmd("git")
+                .args(["fetch", "--quiet"])
+                .quiet()
+                .run()
+                .context("Failed to fetch CTS")?;
+        }
+    } else {
+        shell.change_dir(CTS_CHECKOUT_PATH);
+    }
 
     if !skip_checkout {
         log::info!("Checking out CTS");
@@ -77,6 +101,7 @@ pub fn run_cts(shell: Shell, mut args: Arguments) -> anyhow::Result<()> {
 
     log::info!("Running CTS");
     for test in &tests {
+        log::info!("Running {}", test.to_string_lossy());
         shell
             .cmd("cargo")
             .args(["run"])