cts.yml

name: CTS

on:
  push:
    branches-ignore: [
        # Renovate branches are always PRs, so they will be covered
        # by the pull_request event.
        "renovate/**",
        # Branches with the `gh-readonly-queue` prefix are used by the
        # merge queue, so they are already covered by the `merge_group` event.
        "gh-readonly-queue/**",
      ]
  pull_request:
  merge_group:

env:
  CARGO_INCREMENTAL: false
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: full
  MSRV: "1.93"

# Every time a PR is pushed to, cancel any previous jobs. This
# makes us behave nicer to github and get faster turnaround times
# on PRs that are pushed to multiple times in rapid succession.
concurrency:
  group: ${{github.workflow}}-${{github.ref}}
  cancel-in-progress: ${{github.event_name == 'pull_request'}}

jobs:
  cts:
    strategy:
      fail-fast: false
      matrix:
        suite: ["API Operation", "API Validation", "Other"]
        platform: ["Windows x86_64", "Mac aarch64", "Linux x86_64"]
        include:
          # When an `include` item matches existing matrix entries, the additional items are
          # added only to the matching entries. The two lines above define the matrix, and
          # the items below attach the rest of the configuration values that apply to suites
          # and platforms.
          - suite: API Operation
            filter: "^webgpu:api,operation,"
          - suite: API Validation
            filter: "^webgpu:api,validation,"
          - suite: Other
            filter: "!^webgpu:api,operation,|^webgpu:api,validation,"

          # Windows
          - platform: Windows x86_64
            os: windows-2022
            target: x86_64-pc-windows-msvc
            backend: dx12

          # Mac
          - platform: Mac aarch64
            os: macos-14
            target: x86_64-apple-darwin
            backend: metal

          # Linux
          - platform: Linux x86_64
            os: ubuntu-24.04
            target: x86_64-unknown-linux-gnu
            backend: vulkan

    name: ${{ matrix.suite }} ${{ matrix.platform }}
    runs-on: ${{ matrix.os }}

    steps:
      - name: checkout repo
        uses: actions/checkout@v6

      - name: Install Repo MSRV toolchain
        run: |
          rustup toolchain install ${{ env.MSRV }} --no-self-update --profile=minimal --target ${{ matrix.target }} --component llvm-tools
          rustup override set ${{ env.MSRV }}
          cargo -V

      - name: Install `cargo-llvm-cov`
        uses: taiki-e/install-action@v2
        with:
          tool: cargo-llvm-cov

      - name: caching
        uses: Swatinem/rust-cache@v2
        with:
          # The version number can be incremented for cache busting.
          prefix-key: v2-rust-${{ hashFiles('cts_runner/revision.txt') }}
          cache-directories: cts

      # We enable line numbers for panics, but that's it
      - name: disable debug
        shell: bash
        run: |
          mkdir -p .cargo
          cat <<EOF >> .cargo/config.toml
          [profile.dev]
          debug = "line-tables-only"
          EOF

      - name: (Windows) Install DXC
        if: matrix.os == 'windows-2022'
        uses: ./.github/actions/install-dxc

      - name: (Windows) Install WARP
        if: matrix.os == 'windows-2022'
        uses: ./.github/actions/install-warp
        with:
          target-dir: "target/llvm-cov-target/debug"

      - name: (Linux) Install Mesa
        if: matrix.os == 'ubuntu-24.04'
        uses: ./.github/actions/install-mesa

      # Mixing --llvm-cov with deno (rusty_v8) has linking problems.
      # Explicitly set the backend to avoid EGL messages in output,
      # because these tests check stdout.
      - name: Test cts_runner
        if: matrix.suite == 'other'
        shell: bash
        run: DENO_WEBGPU_BACKEND=${{ matrix.backend }} cargo --locked test -p cts_runner

      - name: Run CTS
        shell: bash
        run: cargo --locked xtask cts --llvm-cov --backend ${{ matrix.backend }} --filter '${{matrix.filter }}'

      - name: Generate coverage report
        id: coverage
        shell: bash
        continue-on-error: true
        run: |
          set -e

          cargo --locked llvm-cov report --lcov --output-path lcov.info

      - name: Upload coverage report to Codecov
        uses: codecov/codecov-action@v5
        if: steps.coverage.outcome == 'success'
        with:
          files: lcov.info
          token: ${{ secrets.CODECOV_TOKEN }}