Reframe CI: per-extension isolated validation

.ci/coexistence_groups.yaml

@@ -23,9 +23,6 @@ groups:
       - SemanticExtraSpecialProperties
       - SemanticScribunto
       - SemanticWatchlist
-      - SemanticBreadcrumbLinks
-      - SemanticFormsSelect
-      - SemanticTasks
       - Mermaid
       - Scribunto  # required by SemanticScribunto
 

.ci/run_isolated_tests.py

@@ -132,19 +132,59 @@ def run_db_update(mw_dir: str) -> bool:
         return False
 
 
-def run_phpunit(mw_dir: str, test_dir: str, result_file: str, timeout: int = 300) -> dict:
+def run_phpunit(
+    mw_dir: str,
+    test_dir: str,
+    result_file: str,
+    exclude_files: list[str] | None = None,
+    timeout: int = 300,
+) -> dict:
     """
     Run PHPUnit for an extension's test directory.
 
+    Args:
+        exclude_files: Optional list of test file paths (relative to extension
+                       root) to exclude.  Since PHPUnit 9 has no --exclude-file
+                       flag, we enumerate the test files and omit the excluded
+                       ones, passing the remaining files individually.
+
     Returns a dict with status and details.
     """
     phpunit_runner = os.path.join(mw_dir, "tests", "phpunit", "phpunit.php")
     if not os.path.isfile(phpunit_runner):
         return {"status": "error", "message": "No PHPUnit runner found"}
 
     try:
+        cmd = ["php", phpunit_runner, "--log-junit", result_file]
+
+        if exclude_files:
+            # Resolve excluded paths to absolute paths for comparison.
+            # test_dir is <mw_dir>/extensions/<Name>/tests/phpunit
+            # exclude paths are relative to <mw_dir>/extensions/<Name>/
+            ext_base = os.path.dirname(os.path.dirname(test_dir))
+            excluded_abs = set()
+            for ef in exclude_files:
+                abs_path = os.path.realpath(os.path.join(ext_base, ef))
+                excluded_abs.add(abs_path)
+
+            # Enumerate all test PHP files and filter out excluded ones.
+            test_files = []
+            for root, _dirs, files in os.walk(test_dir):
+                for f in sorted(files):
+                    if f.endswith(".php"):
+                        fpath = os.path.realpath(os.path.join(root, f))
+                        if fpath not in excluded_abs:
+                            test_files.append(fpath)
+
+            if not test_files:
+                return {"status": "no_tests", "message": "All test files excluded"}
+
+            cmd.extend(test_files)
+        else:
+            cmd.append(test_dir)
+
         proc = subprocess.run(
-            ["php", phpunit_runner, "--log-junit", result_file, test_dir],
+            cmd,
             cwd=mw_dir,
             capture_output=True,
             text=True,
@@ -265,9 +305,13 @@ def main():
             continue
 
         # ── Phase 4: Run PHPUnit tests ───────────────────────────────
-        print(f"  Running PHPUnit tests...")
+        exclude_files = entry.get("exclude_tests", [])
+        if exclude_files:
+            print(f"  Running PHPUnit tests (excluding {len(exclude_files)} file(s))...")
+        else:
+            print(f"  Running PHPUnit tests...")
         result_file = os.path.join(results_dir, f"{name}.xml")
-        test_result = run_phpunit(mw_dir, test_dir, result_file)
+        test_result = run_phpunit(mw_dir, test_dir, result_file, exclude_files=exclude_files)
 
         status = test_result["status"]
         results["details"][name] = test_result

.ci/skip_list.yaml

@@ -108,8 +108,20 @@ upstream_test_compat:
     reason: Integration test suite errors / dependency on SMW test classes
   - name: SemanticScribunto
     reason: Integration test suite errors / dependency on SMW test classes
+
+# ── Extensions with partial test exclusions ──────────────────────────────────
+# These extensions pass the vast majority of their test suite. Only a few
+# specific test files fail due to MW 1.43 / PHP 8.3 incompatibilities.
+# The listed test files are excluded; the rest of the suite runs normally
+# and counts as blocking validation.
+
+partial_test_compat:
   - name: Cargo
-    reason: Fails 1 test inside its own suite under PHP 8.3/MW 1.43
+    reason: CargoFeedFormatTest expects old heading HTML; MW 1.43 changed heading markup
+    exclude_tests:
+      - tests/phpunit/integration/formats/CargoFeedFormatTest.php
   - name: PageForms
-    reason: Fails assertions and has errors due to environment differences under MW 1.43
-
+    reason: PFHelperFormActionTest references removed MediaWiki\Skin\SkinTemplate class; PFFormLinkerTest has environment-dependent assertion
+    exclude_tests:
+      - tests/phpunit/integration/includes/PFFormLinkerTest.php
+      - tests/phpunit/integration/includes/PFHelperFormActionTest.php

.github/workflows/ci.yml

@@ -639,7 +639,12 @@ jobs:
               print(f"   All {len(installed)} extensions coexist without fatal errors.")
           else:
               print(f"❌ Co-existence group '{group_name}' has load errors!")
-              print(verify.stderr[-500:] if verify.stderr else "Unknown error")
+              if verify.stderr:
+                  print(verify.stderr[-500:])
+              elif verify.stdout:
+                  print(verify.stdout[-500:])
+              else:
+                  print(f"Exit code: {verify.returncode}, no output captured")
               sys.exit(1)
           COEXIST_TEST
 

scripts/parse_yaml.py

@@ -293,28 +293,34 @@ def load_skip_list(ci_dir: str) -> dict:
       {
         "external_services": {"ExtName", ...},
         "upstream_test_compat": {"ExtName", ...},
+        "partial_test_compat": {"ExtName": ["path/to/excluded_test.php", ...], ...},
       }
     """
     skip_path = os.path.join(ci_dir, "skip_list.yaml")
     if not os.path.exists(skip_path):
-        return {"external_services": set(), "upstream_test_compat": set()}
+        return {"external_services": set(), "upstream_test_compat": set(),
+                "partial_test_compat": {}}
 
     with open(skip_path) as f:
         data = yaml.safe_load(f)
 
     if not data:
-        return {"external_services": set(), "upstream_test_compat": set()}
+        return {"external_services": set(), "upstream_test_compat": set(),
+                "partial_test_compat": {}}
 
     result = {
         "external_services": set(),
         "upstream_test_compat": set(),
+        "partial_test_compat": {},
     }
 
     # Handle the new categorized format.
     for item in data.get("external_services", []):
         result["external_services"].add(item["name"])
     for item in data.get("upstream_test_compat", []):
         result["upstream_test_compat"].add(item["name"])
+    for item in data.get("partial_test_compat", []):
+        result["partial_test_compat"][item["name"]] = item.get("exclude_tests", [])
 
     # Backward compat: also handle the old flat "skip:" format.
     for item in data.get("skip", []):
@@ -390,6 +396,7 @@ def build_manifest(yaml_path: str, skip_data: dict | None = None) -> dict:
     if skip_data:
         external = skip_data.get("external_services", set())
         upstream = skip_data.get("upstream_test_compat", set())
+        partial = skip_data.get("partial_test_compat", {})
 
         for e in all_entries:
             if e["name"] in external:
@@ -401,6 +408,10 @@ def build_manifest(yaml_path: str, skip_data: dict | None = None) -> dict:
                 e["skip_tests"] = True
                 e["skip_reason"] = "Upstream test compat issue (non-blocking)"
                 e["skip_category"] = "upstream_test_compat"
+            elif e["name"] in partial:
+                e["skip"] = False
+                e["exclude_tests"] = partial[e["name"]]
+                e["skip_category"] = "partial_test_compat"
 
         # Transitive skipping: if a required extension is fully skipped
         # (external_services), skip dependents too.