[8.19][ML] Run QA and PyTorch suites concurrently in a single triggered build (#3131) (#3134)

Manual 8.19 backport of #3131. The auto-backport conflicts because 8.19
lacks the "Upload ES inference tests x86_64" step that sits adjacent to
the QA-trigger block on main, so the hunk context cannot anchor.

Consolidates the separate run_qa_tests / run_pytorch_tests upload steps
into a single trigger with a comma-separated QAF_TESTS_TO_RUN, so the
appex-qa generator fans QA (ml_cpp_pr) and PyTorch suites out into
concurrent parallel jobs in one downstream build (avoiding the
skip_queued_branch_builds de-dup that dropped one of two separate
triggers). run_qa_tests.yml.sh now derives its step label and commit
status context (QA / PyTorch / QA + PyTorch) from the requested suites.

Adds "import os" to pipeline.json.py: 8.19 did not previously import it
and the consolidated block reads QAF_TESTS_TO_RUN via os.environ.

Co-authored-by: Cursor <[email protected]>

Author: edsavage

.buildkite/pipeline.json.py

@@ -17,6 +17,7 @@
 #
 
 import json
+import os
 
 from ml_pipeline import (
     step,
@@ -53,13 +54,31 @@ def main():
             if not config.skip_version_bump_pr_ci:
                 pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests x86_64 runner pipeline",
                                                                    ".buildkite/pipelines/run_es_tests_x86_64.yml.sh"))
-            # We only use linux x86_64 builds for QA tests.
-            if config.run_qa_tests:
-                pipeline_steps.append(pipeline_steps.generate_step("Upload QA tests runner pipeline",
+            # We only use linux x86_64 builds for QA tests. When both the QA
+            # (ml_cpp_pr) and PyTorch suites are requested we trigger a SINGLE
+            # downstream build with a comma-separated QAF_TESTS_TO_RUN so the
+            # appex-qa generator fans them out into concurrent parallel jobs.
+            # A single trigger avoids the same-branch build de-duplication
+            # (skip_queued_branch_builds) on the appex pipeline that otherwise
+            # skipped one of two separately-triggered builds.
+            if config.run_qa_tests or config.run_pytorch_tests:
+                qa_suites = []
+                if config.run_qa_tests:
+                    # QAF_TESTS_TO_RUN may itself be a comma-separated override
+                    # (e.g. a subset of QA markers), so split it into individual
+                    # suites rather than appending the whole string verbatim.
+                    override = os.environ.get("QAF_TESTS_TO_RUN")
+                    if override:
+                        qa_suites.extend(s.strip() for s in override.split(",") if s.strip())
+                    else:
+                        qa_suites.append("ml_cpp_pr")
+                if config.run_pytorch_tests:
+                    qa_suites.append("pytorch_tests")
+                # De-duplicate while preserving order so an override that already
+                # includes pytorch_tests does not produce a duplicate suite.
+                env["QAF_TESTS_TO_RUN"] = ",".join(dict.fromkeys(qa_suites))
+                pipeline_steps.append(pipeline_steps.generate_step("Upload QA/PyTorch tests runner pipeline",
                                                                    ".buildkite/pipelines/run_qa_tests.yml.sh"))
-            if config.run_pytorch_tests:
-                pipeline_steps.append(pipeline_steps.generate_step("Upload QA PyTorch tests runner pipeline",
-                                                                   ".buildkite/pipelines/run_pytorch_tests.yml.sh"))
         if config.build_aarch64 and not config.skip_version_bump_pr_ci:
             pipeline_steps.append(pipeline_steps.generate_step("Upload ES tests aarch64 runner pipeline",
                                                                ".buildkite/pipelines/run_es_tests_aarch64.yml.sh"))

.buildkite/pipelines/run_qa_tests.yml.sh

@@ -16,16 +16,34 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 # shellcheck source=/dev/null
 source "${SCRIPT_DIR}/derive_qa_stack_env.sh"
 
+# Derive a human-readable descriptor for the trigger step's label and GitHub
+# commit-status context from the requested suites. This single script now
+# handles QA-only, PyTorch-only and combined runs, so a fixed "QA Tests" label
+# would mislabel the other two. Substring matches (rather than exact tokens)
+# keep this correct for marker expressions such as "ml_cpp_pr and not slow".
+QAF_SUITES="${QAF_TESTS_TO_RUN:-ml_cpp_pr}"
+_has_qa=false
+_has_pytorch=false
+case "${QAF_SUITES}" in *ml_cpp_pr*) _has_qa=true ;; esac
+case "${QAF_SUITES}" in *pytorch_tests*) _has_pytorch=true ;; esac
+if [ "${_has_qa}" = true ] && [ "${_has_pytorch}" = true ]; then
+    QA_TESTS_DESC="QA + PyTorch"
+elif [ "${_has_pytorch}" = true ]; then
+    QA_TESTS_DESC="PyTorch"
+else
+    QA_TESTS_DESC="QA"
+fi
+
 cat <<EOL
 steps:
-  - label: "Trigger Appex QA Tests :test_tube:"
+  - label: "Trigger Appex ${QA_TESTS_DESC} Tests :test_tube:"
     command:
-      - echo 'Trigger QA Tests'
+      - echo 'Trigger ${QA_TESTS_DESC} Tests'
       - 'buildkite-agent artifact download "build/*" . --step build_test_linux-x86_64-RelWithDebInfo'
     depends_on: "build_test_linux-x86_64-RelWithDebInfo"
     notify:
       -  github_commit_status:
-           context: "Trigger Appex QA Tests"
+           context: "Trigger Appex ${QA_TESTS_DESC} Tests"
   - wait
   - trigger: appex-qa-stateful-custom-ml-cpp-build-testing
     async: false