From d7c4b23a70daaa16413f9c491e1e6940e89de58f Mon Sep 17 00:00:00 2001 From: Guru Deshpande <66385690+guru-desh@users.noreply.github.com> Date: Thu, 9 Jul 2026 22:02:58 -0700 Subject: [PATCH 1/8] ci: run smoke tests across a python x torch version matrix Smoke tests previously ran on one implicit torch version (whatever pyproject.toml's unpinned bound resolved to) across all supported python versions. Add torch_2_8/2_9/2_10/2_11 dependency groups (build-matched torchao/torchvision per pytorch/ao issue 2919) and a matrix.torch_group axis on the test-smoke CI job so each of the 4 torch minor versions is smoke tested independently, still covering all 3 python versions per job via the existing nox session. lowest_tested_torch/highest_tested_torch are renamed to torch_2_8/ torch_2_11 for a consistent naming scheme; the linux-tests/macos-tests full-suite jobs and their Makefile targets are unchanged. --- .github/workflows/ci.yaml | 16 ++++++++++++-- Makefile | 14 ++++++++---- ci/nox/noxfile.py | 11 +++++++++- pyproject.toml | 45 ++++++++++++++++++++++++--------------- scripts/make/setup_env.sh | 8 +++---- 5 files changed, 66 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 92ff6f1..a0c6542 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -38,11 +38,23 @@ jobs: run: make check test-smoke: - name: Linux / make test-smoke + name: Linux / make test-smoke / torch=${{ matrix.torch_version }} runs-on: ubuntu-latest timeout-minutes: 60 env: INSTALL_PRECOMMIT: "false" + strategy: + fail-fast: false + matrix: + include: + - torch_group: torch_2_8 + torch_version: "2.8" + - torch_group: torch_2_9 + torch_version: "2.9" + - torch_group: torch_2_10 + torch_version: "2.10" + - torch_group: torch_2_11 + torch_version: "2.11" steps: - name: Check out repository uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -51,7 +63,7 @@ jobs: with: enable-cache: false - name: Run `make test-smoke` - run: make test-smoke + run: make test-smoke TORCH_GROUP=${{ matrix.torch_group }} test-tutorials: name: Linux / make test-tutorials diff --git a/Makefile b/Makefile index 421b733..6b16bc4 100644 --- a/Makefile +++ b/Makefile @@ -53,6 +53,11 @@ VENV_HIGHEST_TORCH ?= .venv-highest-torch VENV_LOWEST_TORCH ?= .venv-lowest-torch VENV_TUTORIAL ?= .venv-tutorial +# Torch dependency group (pyproject.toml [dependency-groups]) that +# `test-smoke` installs; override to smoke test against a different pinned +# torch minor version, e.g. `make test-smoke TORCH_GROUP=torch_2_8`. +TORCH_GROUP ?= torch_2_11 + # Documentation directory. Defaults to $(MAKEFILE_DIR)docs so the same recipe # works in both contexts: # @@ -176,7 +181,7 @@ env: _maybe_patch_pyproject # Set up development environment with latest supported PyTorch version env-highest-torch: _maybe_patch_pyproject - @$(SETUP_ENV) --venv $(VENV_HIGHEST_TORCH) --python-version $(PYTHON_VERSION) --with-highest_tested_torch + @$(SETUP_ENV) --venv $(VENV_HIGHEST_TORCH) --python-version $(PYTHON_VERSION) --with-torch_2_11 @$(call write_active_venv,$(VENV_HIGHEST_TORCH)) # Set up environment for running tutorials (quantization notebook) @@ -235,16 +240,17 @@ test-slow: @$(MAKE) test PYTEST_ARGS="--marker slow" # Run smoke tests only (pass PYTEST_ARGS for custom flags, e.g., make test-smoke PYTEST_ARGS="--junitxml=results.xml"). +# Pass TORCH_GROUP to smoke test against a specific torch version (default: torch_2_11). test-smoke: @$(call use_env,VENV) && \ echo "Running smoke tests..." && \ - uv run --no-sync --active nox -f $(MAKEFILE_DIR)ci/nox/noxfile.py -s smoke_tests -- $(PYTEST_ARGS) && \ + SMOKE_TEST_TORCH_GROUP=$(TORCH_GROUP) uv run --no-sync --active nox -f $(MAKEFILE_DIR)ci/nox/noxfile.py -s smoke_tests -- $(PYTEST_ARGS) && \ echo "All smoke tests passed!" # Run tests on lowest supported PyTorch version (pass PYTEST_ARGS for custom flags) test-lowest-pytorch: @echo "Running tests on lowest PyTorch version supported..." - @$(call use_env,VENV_LOWEST_TORCH,--with-lowest_tested_torch) && \ + @$(call use_env,VENV_LOWEST_TORCH,--with-torch_2_8) && \ echo "Testing with lowest supported PyTorch versions" && \ uv run --no-sync --active python $(SCRIPTS)/make/log_versions.py && \ $(RUN_TESTS) $(PYTEST_ARGS) && \ @@ -253,7 +259,7 @@ test-lowest-pytorch: # Run tests on highest supported PyTorch version (pass PYTEST_ARGS for custom flags) test-highest-pytorch: @echo "Running tests on highest PyTorch version supported..." - @$(call use_env,VENV_HIGHEST_TORCH,--with-highest_tested_torch) && \ + @$(call use_env,VENV_HIGHEST_TORCH,--with-torch_2_11) && \ echo "Testing with latest supported PyTorch versions" && \ uv run --no-sync --active python $(SCRIPTS)/make/log_versions.py && \ $(RUN_TESTS) $(PYTEST_ARGS) && \ diff --git a/ci/nox/noxfile.py b/ci/nox/noxfile.py index bfa3270..6bb65e5 100644 --- a/ci/nox/noxfile.py +++ b/ci/nox/noxfile.py @@ -34,8 +34,17 @@ options.default_venv_backend = "uv" options.error_on_missing_interpreters = True +# Pins the torch minor version smoke tests install, one of the torch_2_* +# dependency groups in pyproject.toml. Set by the CI matrix (ci.yaml) so each +# job tests a different torch version; defaults to the newest supported. +SMOKE_TEST_TORCH_GROUP = os.environ.get("SMOKE_TEST_TORCH_GROUP", "torch_2_11") -@session(python=get_supported_python_versions(), uv_extras=["coreai"], uv_groups=["test"]) + +@session( + python=get_supported_python_versions(), + uv_extras=["coreai"], + uv_groups=["test", SMOKE_TEST_TORCH_GROUP], +) def smoke_tests(session: Session) -> None: """Smoke test the package build and coreai_opt imports and basic functionality. diff --git a/pyproject.toml b/pyproject.toml index 374a210..e952bd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ dependencies = [ # based on other package versions. We can either 1) add a stricter check to require the newer torchao # version for everyone, or 2) add a runtime check in src/coreai_opt/init.py to ensure torchao # version >= 0.15.0 for torch version >= 2.9.0. Opting option 1. - # These torch versions must be in bounds of lowest_tested_torch and highest_tested_torch + # These torch versions must be in bounds of torch_2_8, torch_2_9, torch_2_10, and torch_2_11 "torch>=2.8.0,<=2.11.0", "torchao>=0.15.0,<=0.17.0", "tqdm>=4.65", @@ -107,20 +107,6 @@ docs = [ # This applies to the `coreml` group as well. coreai = [ "coreai-opt[coreai]" ] coreml = [ "coreai-opt[coreml]" ] -# Used in CI to force latest mimimum supported torch version -# These torch versions must be in bounds of torch versions listed in project dependencies -highest_tested_torch = [ - "torch==2.11.0", - "torchao==0.17.0", - "torchvision==0.26.0", -] -# Used in CI to force lowest mimimum supported torch version -# These torch versions must be in bounds of torch versions listed in project dependencies -lowest_tested_torch = [ - "torch==2.8.0", - "torchao==0.15.0", - "torchvision==0.23.0", -] pre-commit = [ "bashate>=2.1.1", "darker>=3.0.0", @@ -144,6 +130,29 @@ pre-commit = [ "tomli-w>=1.0.0", ] tamm-export = [] +# Used in CI/smoke tests to pin specific torch minor versions for the +# cartesian python x torch smoke test matrix. +# These torch versions must be in bounds of torch versions listed in project dependencies +torch_2_10 = [ + "torch==2.10.0", + "torchao==0.16.0", + "torchvision==0.25.0", +] +torch_2_11 = [ + "torch==2.11.0", + "torchao==0.17.0", + "torchvision==0.26.0", +] +torch_2_8 = [ + "torch==2.8.0", + "torchao==0.15.0", + "torchvision==0.23.0", +] +torch_2_9 = [ + "torch==2.9.1", + "torchao==0.15.0", + "torchvision==0.24.1", +] # Since torch and torchao are project dependencies, we need to include torchvision in dev # This allows standard `make test` to find torchvision torchvision = [ @@ -190,8 +199,10 @@ exclude-newer-package = { coreai-core = false, coreai-torch = false } # Declare conflicting groups so uv does not error conflicts = [ [ - { group = "highest_tested_torch" }, - { group = "lowest_tested_torch" }, + { group = "torch_2_8" }, + { group = "torch_2_9" }, + { group = "torch_2_10" }, + { group = "torch_2_11" }, ], ] [tool.uv.sources] diff --git a/scripts/make/setup_env.sh b/scripts/make/setup_env.sh index 36b123b..7369e04 100755 --- a/scripts/make/setup_env.sh +++ b/scripts/make/setup_env.sh @@ -109,7 +109,7 @@ ENSURE_MODE=false # Groups excluded from --all-groups due to mutual conflicts in pyproject.toml. # tamm-export is omitted because it's opt-in only (never in default-groups or --all-groups). -CONFLICTING_GROUPS=("highest_tested_torch" "lowest_tested_torch") +CONFLICTING_GROUPS=("torch_2_8" "torch_2_9" "torch_2_10" "torch_2_11") show_help() { echo "Usage: $0 [OPTIONS]" @@ -135,8 +135,8 @@ show_help() { echo " $0 --python-version 3.11 # Setup with dev group only" echo " $0 --python-version 3.11 --with-docs # Setup with dev and docs groups" echo " $0 --python-version 3.11 --all-groups # Setup with all non-conflicting groups" - echo " $0 --python-version 3.11 --all-groups --with-highest_tested_torch # Setup with all groups and highest torch" - echo " $0 --python-version 3.11 --all-groups --with-lowest_tested_torch # Setup with all groups and lowest torch" + echo " $0 --python-version 3.11 --all-groups --with-torch_2_11 # Setup with all groups and torch 2.11" + echo " $0 --python-version 3.11 --all-groups --with-torch_2_8 # Setup with all groups and torch 2.8" echo " $0 --python-version 3.11 --venv .venv-exp # Setup with custom venv name" echo " $0 --python-version 3.11 --with-docs --venv .venv-exp # Setup with docs group and custom venv name" echo " $0 --python-version 3.12 # Setup with Python 3.12" @@ -267,7 +267,7 @@ if [[ "$ENSURE_MODE" == "true" ]] && [ -f "$VENV/bin/python" ]; then for GROUP in "${EXTRA_GROUPS[@]}"; do case "$GROUP" in docs) IMPORT_STMTS+="; import sphinx" ;; - highest_tested_torch | lowest_tested_torch) + torch_2_8 | torch_2_9 | torch_2_10 | torch_2_11) IMPORT_STMTS+="; import torchao" EXPECTED_TORCH="$(group_torch_pin "$GROUP")" # These groups always pin torch, so an empty result means the From a5b4ab924d212a2787469c3194aed9986e5e9ae7 Mon Sep 17 00:00:00 2001 From: Guru Deshpande <66385690+guru-desh@users.noreply.github.com> Date: Fri, 10 Jul 2026 01:17:20 -0700 Subject: [PATCH 2/8] fix: allow digits in AVAILABLE_GROUPS regex for torch_2_* group names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit setup_env.sh's group discovery used grep -E '^[a-z_-]+ = \[', which excludes digits and made torch_2_8/2_9/2_10/2_11 invisible to validate_groups — breaking test-highest-pytorch/test-lowest-pytorch with 'Unknown dependency group', caught by the fork CI run. --- scripts/make/setup_env.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/make/setup_env.sh b/scripts/make/setup_env.sh index 7369e04..2e74067 100755 --- a/scripts/make/setup_env.sh +++ b/scripts/make/setup_env.sh @@ -59,8 +59,8 @@ fi # - in_section && /^\[/: If we're in the section AND hit another section header (line starting with [) # * exit: Stop processing (we've left the dependency-groups section) # - in_section {print}: If we're in the section, print the line -# 2. grep -E '^[a-z_-]+ = \[': Filter to lines that define groups -# - ^[a-z_-]+: Group name at start of line (lowercase letters, hyphens, underscores) +# 2. grep -E '^[a-z0-9_-]+ = \[': Filter to lines that define groups +# - ^[a-z0-9_-]+: Group name at start of line (lowercase letters, digits, hyphens, underscores) # - = \[: Followed by space, equals sign, space, opening bracket # 3. cut -d' ' -f1: Extract just the group name # - -d' ': Use space as delimiter @@ -75,7 +75,7 @@ AVAILABLE_GROUPS=$( in_section && /^\[/ { exit } in_section { print } ' "$PYPROJECT_TOML" | - grep -E '^[a-z_-]+ = \[' | + grep -E '^[a-z0-9_-]+ = \[' | cut -d' ' -f1 | tr '\n' ' ' ) From 71f712a0c664f54a0ca1e8d1b7f4dab9f06a3c14 Mon Sep 17 00:00:00 2001 From: Guru Deshpande <66385690+guru-desh@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:06:15 -0700 Subject: [PATCH 3/8] feat: make TORCH_GROUP the single source of truth for torch pinning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit setup_env.sh now reads TORCH_GROUP directly and folds it into every sync it runs, so make env/test/test-fast/docs/etc. all pin torch instead of floating on whatever pyproject.toml's plain bound resolves to. test-highest-pytorch/test-lowest-pytorch/env-highest-torch lock their group via an inline `export TORCH_GROUP=... &&` at the top of the recipe (rather than a hardcoded --with-torch_2_X flag), so their name stays a promise regardless of a command-line override — verified that GNU Make's own `override` directive achieves the same lock but trips up the mbake Makefile formatter's duplicate-target detector, which doesn't recognize that syntax. --with-torch_2_X stays as a guarded alternate path for direct, non-Make invocations of setup_env.sh; a guard now errors clearly if it disagrees with an explicitly-set TORCH_GROUP instead of surfacing a raw uv conflicting-groups resolver error. --- Makefile | 36 +++++++++++-------- scripts/make/setup_env.sh | 74 ++++++++++++++++++++++++++++++--------- 2 files changed, 80 insertions(+), 30 deletions(-) diff --git a/Makefile b/Makefile index 6b16bc4..cc15188 100644 --- a/Makefile +++ b/Makefile @@ -53,10 +53,15 @@ VENV_HIGHEST_TORCH ?= .venv-highest-torch VENV_LOWEST_TORCH ?= .venv-lowest-torch VENV_TUTORIAL ?= .venv-tutorial -# Torch dependency group (pyproject.toml [dependency-groups]) that -# `test-smoke` installs; override to smoke test against a different pinned -# torch minor version, e.g. `make test-smoke TORCH_GROUP=torch_2_8`. +# Torch dependency group (pyproject.toml [dependency-groups]) that every +# environment-building target (env, test, test-smoke, docs, ...) pins to. +# Single source of truth for which torch/torchao/torchvision triple +# setup_env.sh installs; override e.g. `make test-fast TORCH_GROUP=torch_2_8`. +# test-highest-pytorch/test-lowest-pytorch/env-highest-torch lock this to a +# specific group via `export TORCH_GROUP=... &&` at the top of their recipe, +# so their name stays a promise regardless of a command-line value. TORCH_GROUP ?= torch_2_11 +export TORCH_GROUP # Documentation directory. Defaults to $(MAKEFILE_DIR)docs so the same recipe # works in both contexts: @@ -181,7 +186,8 @@ env: _maybe_patch_pyproject # Set up development environment with latest supported PyTorch version env-highest-torch: _maybe_patch_pyproject - @$(SETUP_ENV) --venv $(VENV_HIGHEST_TORCH) --python-version $(PYTHON_VERSION) --with-torch_2_11 + @export TORCH_GROUP=torch_2_11 && \ + $(SETUP_ENV) --venv $(VENV_HIGHEST_TORCH) --python-version $(PYTHON_VERSION) @$(call write_active_venv,$(VENV_HIGHEST_TORCH)) # Set up environment for running tutorials (quantization notebook) @@ -250,20 +256,22 @@ test-smoke: # Run tests on lowest supported PyTorch version (pass PYTEST_ARGS for custom flags) test-lowest-pytorch: @echo "Running tests on lowest PyTorch version supported..." - @$(call use_env,VENV_LOWEST_TORCH,--with-torch_2_8) && \ - echo "Testing with lowest supported PyTorch versions" && \ - uv run --no-sync --active python $(SCRIPTS)/make/log_versions.py && \ - $(RUN_TESTS) $(PYTEST_ARGS) && \ - echo "All tests passed!" + @export TORCH_GROUP=torch_2_8 && \ + $(call use_env,VENV_LOWEST_TORCH) && \ + echo "Testing with lowest supported PyTorch versions" && \ + uv run --no-sync --active python $(SCRIPTS)/make/log_versions.py && \ + $(RUN_TESTS) $(PYTEST_ARGS) && \ + echo "All tests passed!" # Run tests on highest supported PyTorch version (pass PYTEST_ARGS for custom flags) test-highest-pytorch: @echo "Running tests on highest PyTorch version supported..." - @$(call use_env,VENV_HIGHEST_TORCH,--with-torch_2_11) && \ - echo "Testing with latest supported PyTorch versions" && \ - uv run --no-sync --active python $(SCRIPTS)/make/log_versions.py && \ - $(RUN_TESTS) $(PYTEST_ARGS) && \ - echo "All tests passed!" + @export TORCH_GROUP=torch_2_11 && \ + $(call use_env,VENV_HIGHEST_TORCH) && \ + echo "Testing with latest supported PyTorch versions" && \ + uv run --no-sync --active python $(SCRIPTS)/make/log_versions.py && \ + $(RUN_TESTS) $(PYTEST_ARGS) && \ + echo "All tests passed!" # Run tutorial notebook tests test-tutorials: diff --git a/scripts/make/setup_env.sh b/scripts/make/setup_env.sh index 2e74067..10f7ebf 100755 --- a/scripts/make/setup_env.sh +++ b/scripts/make/setup_env.sh @@ -127,6 +127,7 @@ show_help() { echo "" echo "Environment variables:" echo " VENV Virtual environment name, overrides --venv (default: .venv)" + echo " TORCH_GROUP Torch dependency group to pin; source of truth for torch pinning" echo "" echo "Available dependency groups: $AVAILABLE_GROUPS" echo "Conflicting groups (excluded from --all-groups): ${CONFLICTING_GROUPS[*]}" @@ -135,8 +136,8 @@ show_help() { echo " $0 --python-version 3.11 # Setup with dev group only" echo " $0 --python-version 3.11 --with-docs # Setup with dev and docs groups" echo " $0 --python-version 3.11 --all-groups # Setup with all non-conflicting groups" - echo " $0 --python-version 3.11 --all-groups --with-torch_2_11 # Setup with all groups and torch 2.11" - echo " $0 --python-version 3.11 --all-groups --with-torch_2_8 # Setup with all groups and torch 2.8" + echo " TORCH_GROUP=torch_2_11 $0 --python-version 3.11 --all-groups # Setup with all groups and torch 2.11" + echo " TORCH_GROUP=torch_2_8 $0 --python-version 3.11 --all-groups # Setup with all groups and torch 2.8" echo " $0 --python-version 3.11 --venv .venv-exp # Setup with custom venv name" echo " $0 --python-version 3.11 --with-docs --venv .venv-exp # Setup with docs group and custom venv name" echo " $0 --python-version 3.12 # Setup with Python 3.12" @@ -222,6 +223,7 @@ validate_groups() { # Validate dependency group names early [[ ${#EXTRA_GROUPS[@]} -gt 0 ]] && validate_groups "${EXTRA_GROUPS[@]}" [[ ${#EXCLUDE_GROUPS[@]} -gt 0 ]] && validate_groups "${EXCLUDE_GROUPS[@]}" +[[ -n "${TORCH_GROUP:-}" ]] && validate_groups "$TORCH_GROUP" # Check for conflicts between --with- and --without- if [[ ${#EXTRA_GROUPS[@]} -gt 0 && ${#EXCLUDE_GROUPS[@]} -gt 0 ]]; then @@ -233,6 +235,21 @@ if [[ ${#EXTRA_GROUPS[@]} -gt 0 && ${#EXCLUDE_GROUPS[@]} -gt 0 ]]; then done fi +# TORCH_GROUP (env var) is the single source of truth for torch pinning; a +# --with- flag that disagrees with it is almost certainly a +# mistake (e.g. TORCH_GROUP left set from a prior `make` invocation) rather +# than an intentional double-specification, so fail loudly instead of +# leaving it to a raw `uv` conflicting-groups resolver error. +if [[ -n "${TORCH_GROUP:-}" ]]; then + for GROUP in "${EXTRA_GROUPS[@]:-}"; do + if [[ " ${CONFLICTING_GROUPS[*]} " == *" ${GROUP} "* && "$GROUP" != "$TORCH_GROUP" ]]; then + echo "Error: --with-$GROUP conflicts with TORCH_GROUP=$TORCH_GROUP." >&2 + echo "TORCH_GROUP is the source of truth; unset it or set it to '$GROUP' instead." >&2 + exit 1 + fi + done +fi + # Validate venv name contains only allowed characters if [[ ! "$VENV" =~ ^[a-zA-Z0-9._/-]+$ ]]; then echo "Error: Virtual environment name contains invalid characters" @@ -267,24 +284,35 @@ if [[ "$ENSURE_MODE" == "true" ]] && [ -f "$VENV/bin/python" ]; then for GROUP in "${EXTRA_GROUPS[@]}"; do case "$GROUP" in docs) IMPORT_STMTS+="; import sphinx" ;; - torch_2_8 | torch_2_9 | torch_2_10 | torch_2_11) - IMPORT_STMTS+="; import torchao" - EXPECTED_TORCH="$(group_torch_pin "$GROUP")" - # These groups always pin torch, so an empty result means the - # pyproject parse regressed — fail loudly instead of silently - # skipping the version check (which would reintroduce the bug). - if [[ -z "$EXPECTED_TORCH" ]]; then - echo "Error: could not parse a torch pin for group '$GROUP' in $PYPROJECT_TOML" >&2 - exit 1 - fi - IMPORT_STMTS+="; import torch; assert torch.__version__.split('+')[0] == '$EXPECTED_TORCH'" - ;; rio) IMPORT_STMTS+="; import turi_lightning" ;; tamm-export) IMPORT_STMTS+="; import tamm_export" ;; esac done fi + # TORCH_GROUP is the single source of truth for torch pinning; fall back + # to a --with-torch_2_* flag for direct, non-Make invocations. Checked + # unconditionally rather than only when present in EXTRA_GROUPS, since + # TORCH_GROUP drives the sync regardless of what's in EXTRA_GROUPS. + EXPECTED_GROUP="${TORCH_GROUP:-}" + if [[ -z "$EXPECTED_GROUP" ]]; then + for GROUP in "${EXTRA_GROUPS[@]:-}"; do + [[ " ${CONFLICTING_GROUPS[*]} " == *" ${GROUP} "* ]] && EXPECTED_GROUP="$GROUP" + done + fi + if [[ -n "$EXPECTED_GROUP" ]]; then + IMPORT_STMTS+="; import torchao" + EXPECTED_TORCH="$(group_torch_pin "$EXPECTED_GROUP")" + # These groups always pin torch, so an empty result means the + # pyproject parse regressed — fail loudly instead of silently + # skipping the version check (which would reintroduce the bug). + if [[ -z "$EXPECTED_TORCH" ]]; then + echo "Error: could not parse a torch pin for '$EXPECTED_GROUP' in $PYPROJECT_TOML" >&2 + exit 1 + fi + IMPORT_STMTS+="; import torch; assert torch.__version__.split('+')[0] == '$EXPECTED_TORCH'" + fi + if "$VENV/bin/python" -c "$IMPORT_STMTS" 2>/dev/null; then exit 0 fi @@ -324,9 +352,14 @@ echo "[2/3] Installing dependencies..." SYNC_CMD=(uv sync --active) if [[ "$ALL_GROUPS" == "true" ]]; then SYNC_CMD+=(--all-groups) - # Exclude conflicting groups unless explicitly requested via --with-* + # Exclude conflicting torch groups other than the one TORCH_GROUP or + # --with- selects, so --all-groups doesn't trip the mutual + # -conflicts declaration in pyproject.toml. for GROUP in "${CONFLICTING_GROUPS[@]}"; do - if [[ ! " ${EXTRA_GROUPS[*]:-} " == *" ${GROUP} "* ]]; then + GROUP_REQUESTED=false + [[ "$GROUP" == "${TORCH_GROUP:-}" ]] && GROUP_REQUESTED=true + [[ " ${EXTRA_GROUPS[*]:-} " == *" ${GROUP} "* ]] && GROUP_REQUESTED=true + if [[ "$GROUP_REQUESTED" == "false" ]]; then SYNC_CMD+=(--no-group "$GROUP") fi done @@ -335,6 +368,15 @@ elif [[ ${#EXTRA_GROUPS[@]} -gt 0 ]]; then SYNC_CMD+=(--group "$GROUP") done fi +# TORCH_GROUP is the single source of truth for torch pinning: fold it in +# unless --all-groups already swept it up or a --with- above already +# requested it explicitly (the earlier guard ensures no disagreement). +TORCH_GROUP_ALREADY_SYNCED=false +[[ "$ALL_GROUPS" == "true" ]] && TORCH_GROUP_ALREADY_SYNCED=true +[[ " ${EXTRA_GROUPS[*]:-} " == *" ${TORCH_GROUP:-} "* ]] && TORCH_GROUP_ALREADY_SYNCED=true +if [[ -n "${TORCH_GROUP:-}" && "$TORCH_GROUP_ALREADY_SYNCED" == "false" ]]; then + SYNC_CMD+=(--group "$TORCH_GROUP") +fi # Apply explicit group exclusions (e.g., --without-coreai) if [[ ${#EXCLUDE_GROUPS[@]} -gt 0 ]]; then for GROUP in "${EXCLUDE_GROUPS[@]}"; do From 939853f12b6f80f9e38bbc83507d12bbd6e644a0 Mon Sep 17 00:00:00 2001 From: Guru Deshpande <66385690+guru-desh@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:09:26 -0700 Subject: [PATCH 4/8] refactor: read TORCH_GROUP directly in smoke tests, drop SMOKE_TEST_TORCH_GROUP Now that the Makefile exports TORCH_GROUP unconditionally, the smoke-test-specific SMOKE_TEST_TORCH_GROUP env var was redundant indirection. ci/nox/noxfile.py reads TORCH_GROUP directly, and `make test-smoke` no longer needs to translate the Makefile variable into a differently-named env var for the nox subprocess. --- Makefile | 2 +- ci/nox/noxfile.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index cc15188..adc0af9 100644 --- a/Makefile +++ b/Makefile @@ -250,7 +250,7 @@ test-slow: test-smoke: @$(call use_env,VENV) && \ echo "Running smoke tests..." && \ - SMOKE_TEST_TORCH_GROUP=$(TORCH_GROUP) uv run --no-sync --active nox -f $(MAKEFILE_DIR)ci/nox/noxfile.py -s smoke_tests -- $(PYTEST_ARGS) && \ + uv run --no-sync --active nox -f $(MAKEFILE_DIR)ci/nox/noxfile.py -s smoke_tests -- $(PYTEST_ARGS) && \ echo "All smoke tests passed!" # Run tests on lowest supported PyTorch version (pass PYTEST_ARGS for custom flags) diff --git a/ci/nox/noxfile.py b/ci/nox/noxfile.py index 6bb65e5..b37f6df 100644 --- a/ci/nox/noxfile.py +++ b/ci/nox/noxfile.py @@ -35,15 +35,16 @@ options.error_on_missing_interpreters = True # Pins the torch minor version smoke tests install, one of the torch_2_* -# dependency groups in pyproject.toml. Set by the CI matrix (ci.yaml) so each -# job tests a different torch version; defaults to the newest supported. -SMOKE_TEST_TORCH_GROUP = os.environ.get("SMOKE_TEST_TORCH_GROUP", "torch_2_11") +# dependency groups in pyproject.toml — TORCH_GROUP is the project-wide +# source of truth (see Makefile), exported by `make test-smoke` and set per +# job by the CI matrix (ci.yaml); defaults to the newest supported. +TORCH_GROUP = os.environ.get("TORCH_GROUP", "torch_2_11") @session( python=get_supported_python_versions(), uv_extras=["coreai"], - uv_groups=["test", SMOKE_TEST_TORCH_GROUP], + uv_groups=["test", TORCH_GROUP], ) def smoke_tests(session: Session) -> None: """Smoke test the package build and coreai_opt imports and basic functionality. From b11305ba18b757fbe90b7b8f0045579efe3f3d53 Mon Sep 17 00:00:00 2001 From: Guru Deshpande <66385690+guru-desh@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:42:45 -0700 Subject: [PATCH 5/8] refactor(setup_env): drop bespoke TORCH_GROUP conflict guard The custom bash check that detected TORCH_GROUP disagreeing with an explicit --with-torch_2_X flag duplicated logic uv already provides: `uv sync --group torch_2_11 --group torch_2_8` fails on its own with a clear "Groups ... are incompatible with the conflicts: ..." error and non-zero exit. Simplifies the --all-groups exclusion loop and the plain --group append accordingly, removing the GROUP_REQUESTED/ TORCH_GROUP_ALREADY_SYNCED bookkeeping that only existed to feed it. --- scripts/make/setup_env.sh | 39 +++++++++++---------------------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/scripts/make/setup_env.sh b/scripts/make/setup_env.sh index 10f7ebf..77ebeba 100755 --- a/scripts/make/setup_env.sh +++ b/scripts/make/setup_env.sh @@ -235,21 +235,6 @@ if [[ ${#EXTRA_GROUPS[@]} -gt 0 && ${#EXCLUDE_GROUPS[@]} -gt 0 ]]; then done fi -# TORCH_GROUP (env var) is the single source of truth for torch pinning; a -# --with- flag that disagrees with it is almost certainly a -# mistake (e.g. TORCH_GROUP left set from a prior `make` invocation) rather -# than an intentional double-specification, so fail loudly instead of -# leaving it to a raw `uv` conflicting-groups resolver error. -if [[ -n "${TORCH_GROUP:-}" ]]; then - for GROUP in "${EXTRA_GROUPS[@]:-}"; do - if [[ " ${CONFLICTING_GROUPS[*]} " == *" ${GROUP} "* && "$GROUP" != "$TORCH_GROUP" ]]; then - echo "Error: --with-$GROUP conflicts with TORCH_GROUP=$TORCH_GROUP." >&2 - echo "TORCH_GROUP is the source of truth; unset it or set it to '$GROUP' instead." >&2 - exit 1 - fi - done -fi - # Validate venv name contains only allowed characters if [[ ! "$VENV" =~ ^[a-zA-Z0-9._/-]+$ ]]; then echo "Error: Virtual environment name contains invalid characters" @@ -353,13 +338,12 @@ SYNC_CMD=(uv sync --active) if [[ "$ALL_GROUPS" == "true" ]]; then SYNC_CMD+=(--all-groups) # Exclude conflicting torch groups other than the one TORCH_GROUP or - # --with- selects, so --all-groups doesn't trip the mutual - # -conflicts declaration in pyproject.toml. + # --with- selects, so --all-groups doesn't try to sync every + # torch_2_* group at once. If both TORCH_GROUP and --with- + # name different groups, neither gets excluded here and `uv sync` itself + # rejects the combination via its own conflicting-groups resolution. for GROUP in "${CONFLICTING_GROUPS[@]}"; do - GROUP_REQUESTED=false - [[ "$GROUP" == "${TORCH_GROUP:-}" ]] && GROUP_REQUESTED=true - [[ " ${EXTRA_GROUPS[*]:-} " == *" ${GROUP} "* ]] && GROUP_REQUESTED=true - if [[ "$GROUP_REQUESTED" == "false" ]]; then + if [[ "$GROUP" != "${TORCH_GROUP:-}" ]] && [[ ! " ${EXTRA_GROUPS[*]:-} " == *" ${GROUP} "* ]]; then SYNC_CMD+=(--no-group "$GROUP") fi done @@ -368,13 +352,12 @@ elif [[ ${#EXTRA_GROUPS[@]} -gt 0 ]]; then SYNC_CMD+=(--group "$GROUP") done fi -# TORCH_GROUP is the single source of truth for torch pinning: fold it in -# unless --all-groups already swept it up or a --with- above already -# requested it explicitly (the earlier guard ensures no disagreement). -TORCH_GROUP_ALREADY_SYNCED=false -[[ "$ALL_GROUPS" == "true" ]] && TORCH_GROUP_ALREADY_SYNCED=true -[[ " ${EXTRA_GROUPS[*]:-} " == *" ${TORCH_GROUP:-} "* ]] && TORCH_GROUP_ALREADY_SYNCED=true -if [[ -n "${TORCH_GROUP:-}" && "$TORCH_GROUP_ALREADY_SYNCED" == "false" ]]; then +# TORCH_GROUP is the single source of truth for torch pinning: append it +# unconditionally (already covered under --all-groups by not being +# excluded above). A disagreeing --with- ends up as a +# second, different --group flag here, which `uv sync` itself rejects via +# its conflicting-groups resolution rather than a bespoke check. +if [[ -n "${TORCH_GROUP:-}" && "$ALL_GROUPS" != "true" ]]; then SYNC_CMD+=(--group "$TORCH_GROUP") fi # Apply explicit group exclusions (e.g., --without-coreai) From 5d6f41a08f25175e8239d59f622a6af9d4c2044a Mon Sep 17 00:00:00 2001 From: Guru Deshpande <66385690+guru-desh@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:43:19 -0700 Subject: [PATCH 6/8] refactor(nox): drop unused default for TORCH_GROUP The Makefile always exports TORCH_GROUP before invoking nox, so the Python-side default was dead code for every make test-smoke run; kept only for a bare, non-Make nox invocation which isn't a supported entrypoint. --- ci/nox/noxfile.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ci/nox/noxfile.py b/ci/nox/noxfile.py index b37f6df..8e52ba8 100644 --- a/ci/nox/noxfile.py +++ b/ci/nox/noxfile.py @@ -34,11 +34,7 @@ options.default_venv_backend = "uv" options.error_on_missing_interpreters = True -# Pins the torch minor version smoke tests install, one of the torch_2_* -# dependency groups in pyproject.toml — TORCH_GROUP is the project-wide -# source of truth (see Makefile), exported by `make test-smoke` and set per -# job by the CI matrix (ci.yaml); defaults to the newest supported. -TORCH_GROUP = os.environ.get("TORCH_GROUP", "torch_2_11") +TORCH_GROUP = os.environ.get("TORCH_GROUP") @session( From ea8356a81840199079b570f371fb9594d6d9cb5d Mon Sep 17 00:00:00 2001 From: Guru Deshpande <66385690+guru-desh@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:44:15 -0700 Subject: [PATCH 7/8] docs(Makefile): trim TORCH_GROUP comment The expanded rationale duplicated what's now self-evident from reading the three lock-target recipes directly. --- Makefile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Makefile b/Makefile index adc0af9..4de8979 100644 --- a/Makefile +++ b/Makefile @@ -55,11 +55,6 @@ VENV_TUTORIAL ?= .venv-tutorial # Torch dependency group (pyproject.toml [dependency-groups]) that every # environment-building target (env, test, test-smoke, docs, ...) pins to. -# Single source of truth for which torch/torchao/torchvision triple -# setup_env.sh installs; override e.g. `make test-fast TORCH_GROUP=torch_2_8`. -# test-highest-pytorch/test-lowest-pytorch/env-highest-torch lock this to a -# specific group via `export TORCH_GROUP=... &&` at the top of their recipe, -# so their name stays a promise regardless of a command-line value. TORCH_GROUP ?= torch_2_11 export TORCH_GROUP From 147cf24a9972f19fea40417dba8f76ba805b42ce Mon Sep 17 00:00:00 2001 From: Guru Deshpande <66385690+guru-desh@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:45:29 -0700 Subject: [PATCH 8/8] fix(Makefile): narrow TORCH_GROUP override to the setup_env.sh call only `export TORCH_GROUP=torch_2_X && ...` set the variable for the whole recipe's shell, which happened to be harmless since nothing else in these three recipes reads TORCH_GROUP, but the intent was always to scope the override to just the one setup_env.sh invocation that needs it. Using `TORCH_GROUP=torch_2_X ` as an inline prefix does that precisely: it applies only to the first command in the `&&` chain, reverting to the Makefile's own default/command-line value for everything after. --- Makefile | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 4de8979..d1e7b38 100644 --- a/Makefile +++ b/Makefile @@ -181,8 +181,7 @@ env: _maybe_patch_pyproject # Set up development environment with latest supported PyTorch version env-highest-torch: _maybe_patch_pyproject - @export TORCH_GROUP=torch_2_11 && \ - $(SETUP_ENV) --venv $(VENV_HIGHEST_TORCH) --python-version $(PYTHON_VERSION) + @TORCH_GROUP=torch_2_11 $(SETUP_ENV) --venv $(VENV_HIGHEST_TORCH) --python-version $(PYTHON_VERSION) @$(call write_active_venv,$(VENV_HIGHEST_TORCH)) # Set up environment for running tutorials (quantization notebook) @@ -251,8 +250,7 @@ test-smoke: # Run tests on lowest supported PyTorch version (pass PYTEST_ARGS for custom flags) test-lowest-pytorch: @echo "Running tests on lowest PyTorch version supported..." - @export TORCH_GROUP=torch_2_8 && \ - $(call use_env,VENV_LOWEST_TORCH) && \ + @TORCH_GROUP=torch_2_8 $(call use_env,VENV_LOWEST_TORCH) && \ echo "Testing with lowest supported PyTorch versions" && \ uv run --no-sync --active python $(SCRIPTS)/make/log_versions.py && \ $(RUN_TESTS) $(PYTEST_ARGS) && \ @@ -261,8 +259,7 @@ test-lowest-pytorch: # Run tests on highest supported PyTorch version (pass PYTEST_ARGS for custom flags) test-highest-pytorch: @echo "Running tests on highest PyTorch version supported..." - @export TORCH_GROUP=torch_2_11 && \ - $(call use_env,VENV_HIGHEST_TORCH) && \ + @TORCH_GROUP=torch_2_11 $(call use_env,VENV_HIGHEST_TORCH) && \ echo "Testing with latest supported PyTorch versions" && \ uv run --no-sync --active python $(SCRIPTS)/make/log_versions.py && \ $(RUN_TESTS) $(PYTEST_ARGS) && \