Skip to content

ci: run smoke tests across a python x torch version matrix #32

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
guru-desh wants to merge 2 commits into
base: main
Choose a base branch
from
Open

ci: run smoke tests across a python x torch version matrix #32

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4dfc5f0
ci: run smoke tests across a python x torch version matrix
guru-desh Jul 10, 2026
6eac37a
fix: allow digits in AVAILABLE_GROUPS regex for torch_2_* group names
guru-desh Jul 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions .github/workflows/ci.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,23 @@ jobs:
run: make check

test-smoke:
name: Linux / make test-smoke
name: Linux / make test-smoke / torch=${{ matrix.torch_version }}
runs-on: ubuntu-latest
timeout-minutes: 60
env:
INSTALL_PRECOMMIT: "false"
strategy:
fail-fast: false
matrix:
include:
- torch_group: torch_2_8
torch_version: "2.8"
- torch_group: torch_2_9
torch_version: "2.9"
- torch_group: torch_2_10
torch_version: "2.10"
- torch_group: torch_2_11
torch_version: "2.11"
steps:
- name: Check out repository
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
Expand All @@ -51,7 +63,7 @@ jobs:
with:
enable-cache: false
- name: Run `make test-smoke`
run: make test-smoke
run: make test-smoke TORCH_GROUP=${{ matrix.torch_group }}

test-tutorials:
name: Linux / make test-tutorials
Expand Down
14 changes: 10 additions & 4 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,11 @@ VENV_HIGHEST_TORCH ?= .venv-highest-torch
VENV_LOWEST_TORCH ?= .venv-lowest-torch
VENV_TUTORIAL ?= .venv-tutorial

# Torch dependency group (pyproject.toml [dependency-groups]) that
# `test-smoke` installs; override to smoke test against a different pinned
# torch minor version, e.g. `make test-smoke TORCH_GROUP=torch_2_8`.
TORCH_GROUP ?= torch_2_11

# Documentation directory. Defaults to $(MAKEFILE_DIR)docs so the same recipe
# works in both contexts:
#
Expand Down Expand Up @@ -176,7 +181,7 @@ env: _maybe_patch_pyproject

# Set up development environment with latest supported PyTorch version
env-highest-torch: _maybe_patch_pyproject
@$(SETUP_ENV) --venv $(VENV_HIGHEST_TORCH) --python-version $(PYTHON_VERSION) --with-highest_tested_torch
@$(SETUP_ENV) --venv $(VENV_HIGHEST_TORCH) --python-version $(PYTHON_VERSION) --with-torch_2_11
@$(call write_active_venv,$(VENV_HIGHEST_TORCH))

# Set up environment for running tutorials (quantization notebook)
Expand Down Expand Up @@ -235,16 +240,17 @@ test-slow:
@$(MAKE) test PYTEST_ARGS="--marker slow"

# Run smoke tests only (pass PYTEST_ARGS for custom flags, e.g., make test-smoke PYTEST_ARGS="--junitxml=results.xml").
# Pass TORCH_GROUP to smoke test against a specific torch version (default: torch_2_11).
test-smoke:
@$(call use_env,VENV) && \
echo "Running smoke tests..." && \
uv run --no-sync --active nox -f $(MAKEFILE_DIR)ci/nox/noxfile.py -s smoke_tests -- $(PYTEST_ARGS) && \
SMOKE_TEST_TORCH_GROUP=$(TORCH_GROUP) uv run --no-sync --active nox -f $(MAKEFILE_DIR)ci/nox/noxfile.py -s smoke_tests -- $(PYTEST_ARGS) && \
echo "All smoke tests passed!"

# Run tests on lowest supported PyTorch version (pass PYTEST_ARGS for custom flags)
test-lowest-pytorch:
@echo "Running tests on lowest PyTorch version supported..."
@$(call use_env,VENV_LOWEST_TORCH,--with-lowest_tested_torch) && \
@$(call use_env,VENV_LOWEST_TORCH,--with-torch_2_8) && \
echo "Testing with lowest supported PyTorch versions" && \
uv run --no-sync --active python $(SCRIPTS)/make/log_versions.py && \
$(RUN_TESTS) $(PYTEST_ARGS) && \
Expand All @@ -253,7 +259,7 @@ test-lowest-pytorch:
# Run tests on highest supported PyTorch version (pass PYTEST_ARGS for custom flags)
test-highest-pytorch:
@echo "Running tests on highest PyTorch version supported..."
@$(call use_env,VENV_HIGHEST_TORCH,--with-highest_tested_torch) && \
@$(call use_env,VENV_HIGHEST_TORCH,--with-torch_2_11) && \
echo "Testing with latest supported PyTorch versions" && \
uv run --no-sync --active python $(SCRIPTS)/make/log_versions.py && \
$(RUN_TESTS) $(PYTEST_ARGS) && \
Expand Down
11 changes: 10 additions & 1 deletion ci/nox/noxfile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,17 @@
options.default_venv_backend = "uv"
options.error_on_missing_interpreters = True

# Pins the torch minor version smoke tests install, one of the torch_2_*
# dependency groups in pyproject.toml. Set by the CI matrix (ci.yaml) so each
# job tests a different torch version; defaults to the newest supported.
SMOKE_TEST_TORCH_GROUP = os.environ.get("SMOKE_TEST_TORCH_GROUP", "torch_2_11")

@session(python=get_supported_python_versions(), uv_extras=["coreai"], uv_groups=["test"])

@session(
python=get_supported_python_versions(),
uv_extras=["coreai"],
uv_groups=["test", SMOKE_TEST_TORCH_GROUP],
)
def smoke_tests(session: Session) -> None:
"""Smoke test the package build and coreai_opt imports and basic functionality.

Expand Down
45 changes: 28 additions & 17 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ dependencies = [
# based on other package versions. We can either 1) add a stricter check to require the newer torchao
# version for everyone, or 2) add a runtime check in src/coreai_opt/init.py to ensure torchao
# version >= 0.15.0 for torch version >= 2.9.0. Opting option 1.
# These torch versions must be in bounds of lowest_tested_torch and highest_tested_torch
# These torch versions must be in bounds of torch_2_8, torch_2_9, torch_2_10, and torch_2_11
"torch>=2.8.0,<=2.11.0",
"torchao>=0.15.0,<=0.17.0",
"tqdm>=4.65",
Expand Down Expand Up @@ -96,20 +96,6 @@ docs = [
# single source of truth; this group self-references it to avoid duplicating the
# package list.
coreai = [ "coreai-opt[coreai]" ]
# Used in CI to force latest mimimum supported torch version
# These torch versions must be in bounds of torch versions listed in project dependencies
highest_tested_torch = [
"torch==2.11.0",
"torchao==0.17.0",
"torchvision==0.26.0",
]
# Used in CI to force lowest mimimum supported torch version
# These torch versions must be in bounds of torch versions listed in project dependencies
lowest_tested_torch = [
"torch==2.8.0",
"torchao==0.15.0",
"torchvision==0.23.0",
]
pre-commit = [
"bashate>=2.1.1",
"darker>=3.0.0",
Expand All @@ -133,6 +119,29 @@ pre-commit = [
"tomli-w>=1.0.0",
]
tamm-export = []
# Used in CI/smoke tests to pin specific torch minor versions for the
# cartesian python x torch smoke test matrix.
# These torch versions must be in bounds of torch versions listed in project dependencies
torch_2_10 = [
"torch==2.10.0",
"torchao==0.16.0",
"torchvision==0.25.0",
]
torch_2_11 = [
"torch==2.11.0",
"torchao==0.17.0",
"torchvision==0.26.0",
]
torch_2_8 = [
"torch==2.8.0",
"torchao==0.15.0",
"torchvision==0.23.0",
]
torch_2_9 = [
"torch==2.9.1",
"torchao==0.15.0",
"torchvision==0.24.1",
]
# Since torch and torchao are project dependencies, we need to include torchvision in dev
# This allows standard `make test` to find torchvision
torchvision = [
Expand Down Expand Up @@ -178,8 +187,10 @@ exclude-newer-package = { coreai-core = false, coreai-torch = false }
# Declare conflicting groups so uv does not error
conflicts = [
[
{ group = "highest_tested_torch" },
{ group = "lowest_tested_torch" },
{ group = "torch_2_8" },
{ group = "torch_2_9" },
{ group = "torch_2_10" },
{ group = "torch_2_11" },
],
]
[tool.uv.sources]
Expand Down
14 changes: 7 additions & 7 deletions scripts/make/setup_env.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ fi
# - in_section && /^\[/: If we're in the section AND hit another section header (line starting with [)
# * exit: Stop processing (we've left the dependency-groups section)
# - in_section {print}: If we're in the section, print the line
# 2. grep -E '^[a-z_-]+ = \[': Filter to lines that define groups
# - ^[a-z_-]+: Group name at start of line (lowercase letters, hyphens, underscores)
# 2. grep -E '^[a-z0-9_-]+ = \[': Filter to lines that define groups
# - ^[a-z0-9_-]+: Group name at start of line (lowercase letters, digits, hyphens, underscores)
# - = \[: Followed by space, equals sign, space, opening bracket
# 3. cut -d' ' -f1: Extract just the group name
# - -d' ': Use space as delimiter
Expand All @@ -75,7 +75,7 @@ AVAILABLE_GROUPS=$(
in_section && /^\[/ { exit }
in_section { print }
' "$PYPROJECT_TOML" |
grep -E '^[a-z_-]+ = \[' |
grep -E '^[a-z0-9_-]+ = \[' |
cut -d' ' -f1 |
tr '\n' ' '
)
Expand Down Expand Up @@ -109,7 +109,7 @@ ENSURE_MODE=false

# Groups excluded from --all-groups due to mutual conflicts in pyproject.toml.
# tamm-export is omitted because it's opt-in only (never in default-groups or --all-groups).
CONFLICTING_GROUPS=("highest_tested_torch" "lowest_tested_torch")
CONFLICTING_GROUPS=("torch_2_8" "torch_2_9" "torch_2_10" "torch_2_11")

show_help() {
echo "Usage: $0 [OPTIONS]"
Expand All @@ -135,8 +135,8 @@ show_help() {
echo " $0 --python-version 3.11 # Setup with dev group only"
echo " $0 --python-version 3.11 --with-docs # Setup with dev and docs groups"
echo " $0 --python-version 3.11 --all-groups # Setup with all non-conflicting groups"
echo " $0 --python-version 3.11 --all-groups --with-highest_tested_torch # Setup with all groups and highest torch"
echo " $0 --python-version 3.11 --all-groups --with-lowest_tested_torch # Setup with all groups and lowest torch"
echo " $0 --python-version 3.11 --all-groups --with-torch_2_11 # Setup with all groups and torch 2.11"
echo " $0 --python-version 3.11 --all-groups --with-torch_2_8 # Setup with all groups and torch 2.8"
echo " $0 --python-version 3.11 --venv .venv-exp # Setup with custom venv name"
echo " $0 --python-version 3.11 --with-docs --venv .venv-exp # Setup with docs group and custom venv name"
echo " $0 --python-version 3.12 # Setup with Python 3.12"
Expand Down Expand Up @@ -267,7 +267,7 @@ if [[ "$ENSURE_MODE" == "true" ]] && [ -f "$VENV/bin/python" ]; then
for GROUP in "${EXTRA_GROUPS[@]}"; do
case "$GROUP" in
docs) IMPORT_STMTS+="; import sphinx" ;;
highest_tested_torch | lowest_tested_torch)
torch_2_8 | torch_2_9 | torch_2_10 | torch_2_11)
IMPORT_STMTS+="; import torchao"
EXPECTED_TORCH="$(group_torch_pin "$GROUP")"
# These groups always pin torch, so an empty result means the
Expand Down