fix: count a release as tested when only skipped tests remain

Tests filtered by an unmet assumption - an absent optional dependency, a
documented gap - are reported as skipped rather than failed. Requiring zero
skips meant a suite that passed everything it actually ran still disqualified
its release. Errors and failures still disqualify a release.

When no release qualified, the task returned without writing the badge and
without reporting anything, so the caller only found out when a later step
tripped over the missing file. Assert instead, and point at the reports.

Co-Authored-By: Claude Opus 5 <[email protected]>

Author: ehennestad

code/+matbox/+tasks/createTestedWithBadgeforToolbox.m

@@ -45,10 +45,12 @@ function createTestedWithBadgeforToolbox(versionNumber, projectRootDirectory)
         % Read the test results file
         testResults = readstruct(currentFile);
 
-        % If no tests failed, errors, or were skipped, then add it to the list
+        % A release counts as tested when nothing errored or failed. Skipped
+        % tests are deliberately filtered by test assumptions (an optional
+        % dependency that is absent, a documented gap), so they say nothing
+        % about whether the toolbox works on that release.
         if sum([testResults.testsuite.errorsAttribute]) == 0 ...
-           && sum([testResults.testsuite.failuresAttribute]) == 0 ...
-           && sum([testResults.testsuite.skippedAttribute]) == 0
+           && sum([testResults.testsuite.failuresAttribute]) == 0
             if releasesTestedWith ~= ""
                 % Insert the separator between released after the first one
                 releasesTestedWith = releasesTestedWith + " | ";
@@ -58,22 +60,28 @@ function createTestedWithBadgeforToolbox(versionNumber, projectRootDirectory)
             releasesFailed = releasesFailed + 1;
         end
     end
-    if releasesTestedWith ~= ""
-        switch releasesFailed
-            case 0
-                badgecolor = "green";
-            case 1
-                badgecolor = "orange";
-            case 2
-                badgecolor = "yellow";
-            otherwise
-                badgecolor = "red";
-        end
 
-        outputDirectory = fullfile(projectRootDirectory, '.github', 'badges', versionNumber);
-        matbox.utility.writeBadgeJSONFile("tested with", releasesTestedWith, badgecolor,...
-            "OutputFolder", outputDirectory)
+    % Fail loudly rather than leaving the caller with a missing badge file.
+    assert( releasesTestedWith ~= "", ...
+        'MATBOX:BadgeCreation:NoReleasePassed', ...
+        ['No release passed its tests, so no "tested with" badge was ', ...
+         'created. Inspect the test reports under docs/reports and fix the ', ...
+         'failing tests.\n'] )
+
+    switch releasesFailed
+        case 0
+            badgecolor = "green";
+        case 1
+            badgecolor = "orange";
+        case 2
+            badgecolor = "yellow";
+        otherwise
+            badgecolor = "red";
     end
+
+    outputDirectory = fullfile(projectRootDirectory, '.github', 'badges', versionNumber);
+    matbox.utility.writeBadgeJSONFile("tested with", releasesTestedWith, badgecolor,...
+        "OutputFolder", outputDirectory)
 end
 
 function result = getReleaseNamesFromFolderPaths(folderPaths)

tools/tests/+matboxtools/+unittest/TasksTest.m

@@ -122,5 +122,82 @@ function testPackageToolboxShadowedRootFile(testCase)
             testCase.verifyEqual(fileread(packagedLicenseFile), shadowText)
             testCase.verifyTrue(isfile(fullfile(pwd, 'code', 'LICENSE')))
         end
+
+        function testTestedWithBadgeCountsReleaseWithSkippedTests(testCase)
+            % Tests filtered by an unmet assumption are reported as skipped
+            % rather than failed, so the release is still tested with.
+            writeTestResultsReport(pwd, "R2024a", "Skipped", 4);
+            writeTestResultsReport(pwd, "R2024b");
+
+            matbox.tasks.createTestedWithBadgeforToolbox("v1.2.3", pwd);
+
+            badgeInfo = readTestedWithBadge(testCase, pwd, "v1.2.3");
+            testCase.verifyEqual(string(badgeInfo.message), "R2024a | R2024b")
+            testCase.verifyEqual(string(badgeInfo.color), "green")
+        end
+
+        function testTestedWithBadgeExcludesFailingRelease(testCase)
+            % Errors and failures still disqualify a release, and the badge
+            % turns orange once a single release is left out.
+            writeTestResultsReport(pwd, "R2024a", "Failures", 1);
+            writeTestResultsReport(pwd, "R2024b", "Skipped", 2);
+
+            matbox.tasks.createTestedWithBadgeforToolbox("v1.2.3", pwd);
+
+            badgeInfo = readTestedWithBadge(testCase, pwd, "v1.2.3");
+            testCase.verifyEqual(string(badgeInfo.message), "R2024b")
+            testCase.verifyEqual(string(badgeInfo.color), "orange")
+        end
+
+        function testTestedWithBadgeErrorsWhenNoReleasePassed(testCase)
+            % Writing no badge at all would surface much later as a missing
+            % file, so the task has to report the problem itself.
+            writeTestResultsReport(pwd, "R2024a", "Errors", 1);
+            writeTestResultsReport(pwd, "R2024b", "Failures", 1);
+
+            testCase.verifyError( ...
+                @() matbox.tasks.createTestedWithBadgeforToolbox("v1.2.3", pwd), ...
+                "MATBOX:BadgeCreation:NoReleasePassed")
+        end
     end
 end
+
+function writeTestResultsReport(projectRootDirectory, releaseName, options)
+% writeTestResultsReport - Write a minimal JUnit-style report for one release
+%
+%   The report holds a single test suite whose error, failure and skip counts
+%   are given by the optional arguments. It mirrors the layout that the release
+%   workflow produces by downloading one report artifact per MATLAB release.
+
+    arguments
+        projectRootDirectory (1,1) string
+        releaseName (1,1) string
+        options.Errors (1,1) double = 0
+        options.Failures (1,1) double = 0
+        options.Skipped (1,1) double = 0
+    end
+
+    reportFolder = fullfile(projectRootDirectory, "docs", "reports", "reports-" + releaseName);
+    if ~isfolder(reportFolder)
+        mkdir(reportFolder)
+    end
+
+    reportXml = sprintf([...
+        '<?xml version="1.0" encoding="UTF-8" standalone="no" ?>\n' ...
+        '<testsuites>\n' ...
+        '  <testsuite errors="%d" failures="%d" name="ExampleTest" skipped="%d" tests="%d" time="1.0"/>\n' ...
+        '</testsuites>\n'], ...
+        options.Errors, options.Failures, options.Skipped, ...
+        options.Errors + options.Failures + options.Skipped + 1);
+
+    matbox.utility.filewrite(fullfile(reportFolder, "test-results.xml"), reportXml);
+end
+
+function badgeInfo = readTestedWithBadge(testCase, projectRootDirectory, versionNumber)
+% readTestedWithBadge - Read back the badge written for a given version
+
+    badgeFile = fullfile(projectRootDirectory, ".github", "badges", versionNumber, "tested_with.json");
+    testCase.assertTrue(isfile(badgeFile), ...
+        "Expected 'tested with' badge file was not created.")
+    badgeInfo = jsondecode(fileread(badgeFile));
+end