ci: targeted PR tests, runtime summary, and config-consistency check

.github/workflows/ci.yaml

@@ -50,6 +50,8 @@ jobs:
         run: pip install pre-commit
       - name: Run pre-commit (all hooks, all files)
         run: pre-commit run --all-files --show-diff-on-failure
+      - name: Check version/commit pin consistency
+        run: python tools/ci/check_version_consistency.py
 
   # Flag PRs that introduce known-vulnerable dependencies. Warn-only during
   # burn-in; drop warn-only to turn it into a hard gate once the team is ready.
@@ -237,6 +239,7 @@ jobs:
       - run: echo "Begin AITER + Primus-Turbo Install."
       - name: Install AITER
         run: |
+          : > "$RUNNER_TEMP/runtime.tsv"  # reset the CI runtime log for this job
           echo "✅ [Uninstall old aiter] started at: $(date)"
           pip3 uninstall aiter amd-aiter -y || true
           rm -rf /tmp/aiter || true
@@ -253,6 +256,7 @@ jobs:
           elapsed=$((end_time - start_time))
           echo "✅ [Build aiter] ended at: $(date)"
           echo "⏱️ [Build aiter] Total elapsed time: ${elapsed} seconds"
+          echo -e "Build aiter\t${elapsed}" >> "$RUNNER_TEMP/runtime.tsv"
       - name: Install Primus-Turbo
         run: |
           rm -rf /tmp/Primus-Turbo || true
@@ -268,13 +272,15 @@ jobs:
           elapsed=$((end_time - start_time))
           echo "✅ [Pip install requirements] ended at: $(date)"
           echo "⏱️ [Pip install requirements] Total elapsed time: ${elapsed} seconds"
+          echo -e "primus-turbo: pip install requirements\t${elapsed}" >> "$RUNNER_TEMP/runtime.tsv"
           start_time=$(date +%s)
           echo "✅ [build primus-turbo] started at: $(date)"
           pip3 install --no-build-isolation -e . -v
           end_time=$(date +%s)
           elapsed=$((end_time - start_time))
           echo "✅ [build primus-turbo] ended at: $(date)"
           echo "⏱️ [build primus-turbo] Total elapsed time: ${elapsed} seconds"
+          echo -e "primus-turbo: build/install\t${elapsed}" >> "$RUNNER_TEMP/runtime.tsv"
       - run: echo "🎉 Begin Primus Unit Test."
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
         with:
@@ -331,7 +337,21 @@ jobs:
           # Note HSA_NO_SCRATCH_RECLAIM=1 must be set to avoid RCCL perf hit (TAS-8N Node), rocm ver:70125424
           export HSA_NO_SCRATCH_RECLAIM=1
           mkdir -p "${GITHUB_WORKSPACE}/test-reports"
-          pytest --maxfail=1 -s ./tests/unit_tests/ \
+          # Component-aware selection on PRs; full suite on push/release/dispatch.
+          # Fail-safe: any failure to compute the diff falls back to the full suite.
+          TARGETS="./tests/unit_tests/"
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            base_sha="${{ github.event.pull_request.base.sha }}"
+            git fetch --no-tags --depth=200 origin "${{ github.base_ref }}" 2>/dev/null || true
+            changed="$(git diff --name-only "${base_sha}" HEAD 2>/dev/null || true)"
+            if [[ -n "${changed}" ]]; then
+              sel="$(printf '%s\n' "${changed}" | python tools/ci/select_tests.py)"
+              [[ -n "${sel}" ]] && TARGETS="${sel}"
+            fi
+          fi
+          echo "Selected unit-test targets: ${TARGETS}"
+          # shellcheck disable=SC2086 # intentional word-splitting of multiple paths
+          pytest --maxfail=1 -s ${TARGETS} \
             --cov=primus --cov-report=term-missing:skip-covered \
             --junitxml="${GITHUB_WORKSPACE}/test-reports/core-unit.xml" \
             --deselect=tests/unit_tests/megatron/cco/test_tp_overlap.py::TPOverlapTestCase::test_fp8_te_linear \
@@ -361,10 +381,31 @@ jobs:
           printf '[run]\nparallel = true\nsource = primus\nsigterm = true\n' > "$GITHUB_WORKSPACE/.coveragerc_e2e"
           echo "COVERAGE_PROCESS_START=$GITHUB_WORKSPACE/.coveragerc_e2e" >> "$GITHUB_ENV"
           echo "COVERAGE_FILE=$GITHUB_WORKSPACE/.coverage_e2e" >> "$GITHUB_ENV"
+      - name: Decide E2E scope (torch)
+        run: |
+          # Fail-safe: default to running all E2E (push/release/dispatch, or any
+          # diff failure). On PRs, narrow to the suites the changed files affect.
+          M=1; T=1
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            base="${{ github.event.pull_request.base.sha }}"
+            git fetch --no-tags --depth=200 origin "${{ github.base_ref }}" 2>/dev/null || true
+            changed="$(git diff --name-only "${base}" HEAD 2>/dev/null || true)"
+            if [[ -n "${changed}" ]]; then
+              e2e="$(printf '%s\n' "${changed}" | python tools/ci/select_tests.py --e2e)"
+              echo "Selected torch E2E scope: ${e2e:-<none>}"
+              if [[ "${e2e}" != "all" ]]; then
+                echo "${e2e}" | grep -qw megatron   || M=0
+                echo "${e2e}" | grep -qw torchtitan || T=0
+              fi
+            fi
+          fi
+          echo "RUN_MEGATRON_E2E=${M}" >> "$GITHUB_ENV"
+          echo "RUN_TORCHTITAN_E2E=${T}" >> "$GITHUB_ENV"
       - name: Run Primus Model Tests -- Megatron-LM
         env:
           HF_TOKEN: ${{secrets.HF_TOKEN}}
         run: |
+          if [[ "${RUN_MEGATRON_E2E:-1}" != "1" ]]; then echo "Skipping Megatron-LM E2E: no relevant changes."; exit 0; fi
           echo "Set UT_LOG_PATH: ${{ env.UT_LOG_PATH }}"
           rm -rf "${{ env.UT_LOG_PATH }}"
           mkdir -p "${{ env.UT_LOG_PATH }}"
@@ -377,6 +418,7 @@ jobs:
         env:
           HF_TOKEN: ${{secrets.HF_TOKEN}}
         run: |
+          if [[ "${RUN_TORCHTITAN_E2E:-1}" != "1" ]]; then echo "Skipping TorchTitan E2E: no relevant changes."; exit 0; fi
           echo "Set UT_LOG_PATH: ${{ env.UT_LOG_PATH }}"
           rm -rf "${{ env.UT_LOG_PATH }}"
           mkdir -p "${{ env.UT_LOG_PATH }}"
@@ -416,6 +458,10 @@ jobs:
         if: always()
         run: |
           python tools/ci/junit_summary.py --title torch test-reports/*.xml >> "$GITHUB_STEP_SUMMARY" || true
+      - name: Write runtime summary
+        if: always()
+        run: |
+          python tools/ci/runtime_summary.py --title torch "$RUNNER_TEMP/runtime.tsv" >> "$GITHUB_STEP_SUMMARY" || true
       - name: Build torch coverage json (unit + E2E)
         if: always()
         continue-on-error: true
@@ -480,6 +526,7 @@ jobs:
           echo "Primus-Turbo dir: /tmp/Primus-Turbo"
           git config --global --add safe.directory /tmp/Primus-Turbo
           cd /tmp/Primus-Turbo
+          : > "$RUNNER_TEMP/runtime.tsv"  # reset the CI runtime log for this job
           start_time=$(date +%s)
           echo "✅ [Pip install requirements] started at: $(date)"
           mkdir -p  ${PRIMUS_WORKDIR}/primus-cache
@@ -488,6 +535,7 @@ jobs:
           elapsed=$((end_time - start_time))
           echo "✅ [Pip install requirements] ended at: $(date)"
           echo "⏱️ [Pip install requirements] Total elapsed time: ${elapsed} seconds"
+          echo -e "pip install/upgrade\t${elapsed}" >> "$RUNNER_TEMP/runtime.tsv"
           start_time=$(date +%s)
           echo "✅ [build primus-turbo] started at: $(date)"
           end_time=$(date +%s)
@@ -541,10 +589,29 @@ jobs:
           printf '[run]\nparallel = true\nsource = primus\nsigterm = true\n' > "$GITHUB_WORKSPACE/.coveragerc_e2e"
           echo "COVERAGE_PROCESS_START=$GITHUB_WORKSPACE/.coveragerc_e2e" >> "$GITHUB_ENV"
           echo "COVERAGE_FILE=$GITHUB_WORKSPACE/.coverage_e2e" >> "$GITHUB_ENV"
+      - name: Decide E2E scope (jax)
+        run: |
+          # Fail-safe: default to running MaxText E2E; on PRs skip it when no
+          # relevant (maxtext / shared) changes are detected.
+          X=1
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            base="${{ github.event.pull_request.base.sha }}"
+            git fetch --no-tags --depth=200 origin "${{ github.base_ref }}" 2>/dev/null || true
+            changed="$(git diff --name-only "${base}" HEAD 2>/dev/null || true)"
+            if [[ -n "${changed}" ]]; then
+              e2e="$(printf '%s\n' "${changed}" | python tools/ci/select_tests.py --e2e)"
+              echo "Selected jax E2E scope: ${e2e:-<none>}"
+              if [[ "${e2e}" != "all" ]]; then
+                echo "${e2e}" | grep -qw maxtext || X=0
+              fi
+            fi
+          fi
+          echo "RUN_MAXTEXT_E2E=${X}" >> "$GITHUB_ENV"
       - name: Run Unit Tests
         env:
           HF_TOKEN: ${{secrets.HF_TOKEN}}
         run: |
+          if [[ "${RUN_MAXTEXT_E2E:-1}" != "1" ]]; then echo "Skipping MaxText E2E: no relevant changes."; exit 0; fi
           echo "Set UT_LOG_PATH: ${{ env.UT_LOG_PATH }}"
           rm -rf "${{ env.UT_LOG_PATH }}"
           mkdir -p "${{ env.UT_LOG_PATH }}"
@@ -567,6 +634,10 @@ jobs:
         if: always()
         run: |
           python tools/ci/junit_summary.py --title jax test-reports/*.xml >> "$GITHUB_STEP_SUMMARY" || true
+      - name: Write runtime summary
+        if: always()
+        run: |
+          python tools/ci/runtime_summary.py --title jax "$RUNNER_TEMP/runtime.tsv" >> "$GITHUB_STEP_SUMMARY" || true
       - name: Build jax coverage json (MaxText E2E)
         if: always()
         continue-on-error: true

tests/unit_tests/ci/test_select_tests.py

@@ -0,0 +1,104 @@
+###############################################################################
+# Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved.
+#
+# See LICENSE for license information.
+###############################################################################
+
+"""Unit tests for tools/ci/select_tests.py (classify-based PR test selection)."""
+
+import importlib.util
+from pathlib import Path
+
+_ROOT = Path(__file__).resolve().parents[3]
+_SPEC = importlib.util.spec_from_file_location("select_tests", _ROOT / "tools/ci/select_tests.py")
+_MOD = importlib.util.module_from_spec(_SPEC)
+_SPEC.loader.exec_module(_MOD)
+select = _MOD.select_targets
+
+FULL = ["tests/unit_tests/"]
+
+
+# --- unit-test selection ---------------------------------------------------
+def test_empty_runs_full():
+    assert select([]) == FULL
+
+
+def test_global_change_runs_full():
+    assert select([".github/workflows/ci.yaml"]) == FULL
+    assert select(["tools/ci/select_tests.py"]) == FULL
+    assert select(["runner/helpers/x.sh"]) == FULL
+    assert select(["requirements.txt"]) == FULL
+    assert select(["primus/core/launcher/initialize.py"]) == FULL
+
+
+def test_non_py_under_primus_runs_full():
+    # configs / fixtures can't be localized to a unit dir -> fail-safe.
+    assert select(["primus/configs/x.yaml"]) == FULL
+
+
+def test_backend_maps_to_its_unit_dir():
+    out = select(["primus/backends/megatron/training/global_vars.py"])
+    assert "tests/unit_tests/backends/megatron/" in out
+    assert "tests/unit_tests/megatron/" in out  # megatron's extra GPU-operator tests
+
+
+def test_backend_without_unit_dir_runs_full():
+    # transformer_engine has no tests/unit_tests/backends/transformer_engine/.
+    assert select(["primus/backends/transformer_engine/x.py"]) == FULL
+
+
+def test_component_maps_to_isomorphic_dir():
+    assert select(["primus/core/projection/engine.py"]) == ["tests/unit_tests/core/projection/"]
+    assert select(["primus/agents/a.py"]) == ["tests/unit_tests/agents/"]
+
+
+def test_changed_unit_test_runs_its_dir():
+    assert select(["tests/unit_tests/agents/test_tools.py"]) == ["tests/unit_tests/agents/"]
+
+
+def test_docs_only_runs_full():
+    assert select(["README.md", "docs/guide.md"]) == FULL
+
+
+# --- E2E selection (pass a fixed suite set for determinism) ----------------
+SUITES = {"megatron", "torchtitan", "maxtext"}
+
+
+def e2e(files):
+    return _MOD.select_e2e(files, SUITES)
+
+
+def test_e2e_empty_runs_all():
+    assert set(e2e([])) == SUITES
+
+
+def test_e2e_global_runs_all():
+    assert set(e2e([".github/workflows/ci.yaml"])) == SUITES
+    assert set(e2e(["runner/helpers/x.sh"])) == SUITES
+
+
+def test_e2e_backend_with_trainer_runs_its_suite():
+    assert e2e(["primus/backends/megatron/x.py"]) == ["megatron"]
+    assert e2e(["examples/maxtext/configs/x.yaml"]) == ["maxtext"]
+    assert e2e(["tests/trainer/test_torchtitan_trainer.py"]) == ["torchtitan"]
+
+
+def test_e2e_backend_without_trainer_runs_all():
+    # No trainer suite (bridge / hummingbirdxt / transformer_engine) -> fail-safe all.
+    assert set(e2e(["primus/backends/megatron_bridge/x.py"])) == SUITES
+    assert set(e2e(["primus/backends/hummingbirdxt/x.py"])) == SUITES
+    assert set(e2e(["primus/backends/transformer_engine/x.py"])) == SUITES
+
+
+def test_e2e_component_change_runs_all():
+    assert set(e2e(["primus/core/trainer/base.py"])) == SUITES
+
+
+def test_e2e_docs_only_runs_none():
+    assert e2e(["README.md"]) == []
+
+
+def test_e2e_added_trainer_is_auto_picked_up():
+    # Adding a bridge trainer makes a bridge change run it -- no code change here.
+    suites = SUITES | {"megatron_bridge"}
+    assert _MOD.select_e2e(["primus/backends/megatron_bridge/x.py"], suites) == ["megatron_bridge"]