Add pack review smoke clean option

Author: helloJamest

.codex-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "skillbench",
-  "version": "0.5.25",
+  "version": "0.5.26",
   "description": "Evaluate, compare, trace, and evolve Codex skills with eval cases, judge feedback, harness matrices, GEPA-style optimization, and optional Comet tracking.",
   "author": {
     "name": "helloJamest",

README.md

@@ -335,10 +335,11 @@ python -m skillbench pack-review-smoke `
   examples\eval_packs\generic-skill-release.json `
   --review-dir .skillbench\pack-review-smoke `
   --bundle-output .skillbench\pack-review-bundle `
+  --clean `
   --json
 ```
 
-This writes validation JSON, comparison JSON/Markdown, `pack_review_ci_result.json`, JUnit, SARIF, and a bundled dashboard for local inspection.
+This writes validation JSON, comparison JSON/Markdown, `pack_review_ci_result.json`, JUnit, SARIF, and a bundled dashboard for local inspection. Add `--clean` when repeated local runs should remove stale review and bundle artifacts before rebuilding.
 
 Build one uploadable report bundle for CI artifacts or release evidence:
 

ROADMAP.md

@@ -59,4 +59,5 @@ SkillBench is moving from a working MVP toward a trustworthy, reproducible Agent
 - v0.5.23 shipped eval pack review report bundles and uploaded bundle artifacts from the pack checklist workflow.
 - v0.5.24 shipped release-quality docs for consuming eval pack review bundles from CI.
 - v0.5.25 shipped `skillbench pack-review-smoke` as a compact local command for producing eval pack review evidence bundles.
-- Next: add optional pack review bundle cleanup/retention controls for repeated local runs.
+- v0.5.26 shipped `pack-review-smoke --clean` for repeated local runs without stale artifacts.
+- Next: add richer pack review smoke summaries with top gate failures and artifact hints.

VERSION

@@ -1 +1 @@
-0.5.25
+0.5.26

docs/eval-pack-review-bundles.md

@@ -68,10 +68,11 @@ To reproduce the CI evidence locally:
 ```bash
 skillbench pack-review-smoke examples/eval_packs/generic-skill-smoke.json examples/eval_packs/generic-skill-release.json \
   --review-dir .skillbench/pack-review-smoke \
-  --bundle-output .skillbench/pack-review-bundle
+  --bundle-output .skillbench/pack-review-bundle \
+  --clean
 ```
 
-The command above is equivalent to the expanded workflow below:
+Use `--clean` for repeated local runs so stale validation, comparison, JUnit, SARIF, or dashboard files do not survive from earlier smoke checks. The command above is equivalent to the expanded workflow below:
 
 ```bash
 mkdir -p .skillbench/pack-checklists

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "skillbench"
-version = "0.5.25"
+version = "0.5.26"
 description = "Evaluate, compare, trace, and evolve Codex skills with eval cases, judge feedback, harness matrices, GEPA-style loops, and optional Comet tracking."
 requires-python = ">=3.10"
 authors = [{ name = "SkillBench Contributors" }]

runtime/skillbench/__init__.py

@@ -1,3 +1,3 @@
 """SkillBench runtime package."""
 
-__version__ = "0.5.25"
+__version__ = "0.5.26"

runtime/skillbench/cli.py

@@ -2,6 +2,7 @@
 
 import argparse
 import json
+import shutil
 import sys
 from pathlib import Path
 
@@ -206,6 +207,7 @@ def build_parser() -> argparse.ArgumentParser:
     pack_smoke_parser.add_argument("right", help="Candidate or release eval pack JSON.")
     pack_smoke_parser.add_argument("--review-dir", default=".skillbench/pack-review-smoke", help="Directory for intermediate validation, comparison, JUnit, SARIF, and CI artifacts.")
     pack_smoke_parser.add_argument("--bundle-output", default=".skillbench/pack-review-bundle", help="Directory for the generated report bundle.")
+    pack_smoke_parser.add_argument("--clean", action="store_true", help="Remove the review and bundle output directories before writing new smoke artifacts.")
     pack_smoke_parser.add_argument("--json", action="store_true", help="Print machine-readable smoke result JSON.")
 
     bundle_parser = sub.add_parser("bundle", help="Build a publishable report bundle with dashboard, PR comment, CI artifacts, and raw artifact manifests.")
@@ -686,6 +688,11 @@ def _case_selection_kwargs(args: argparse.Namespace) -> dict:
 
 def _run_pack_review_smoke(args: argparse.Namespace) -> dict:
     review_dir = Path(args.review_dir)
+    bundle_output = Path(args.bundle_output)
+    if args.clean:
+        _clean_output_dir(review_dir)
+        if bundle_output.resolve() != review_dir.resolve():
+            _clean_output_dir(bundle_output)
     review_dir.mkdir(parents=True, exist_ok=True)
 
     left = Path(args.left)
@@ -736,11 +743,12 @@ def _run_pack_review_smoke(args: argparse.Namespace) -> dict:
     pack_review["artifacts"]["sarif_json"] = str(sarif_path)
     write_json(pack_review_path, pack_review)
 
-    bundle_manifest = build_report_bundle(review_dir, args.bundle_output)
+    bundle_manifest = build_report_bundle(review_dir, bundle_output)
     return {
         "passed": bool(pack_review.get("passed")),
+        "cleaned": bool(args.clean),
         "review_dir": str(review_dir),
-        "bundle_output": str(Path(args.bundle_output)),
+        "bundle_output": str(bundle_output),
         "left_validation": str(left_validation_path),
         "right_validation": str(right_validation_path),
         "comparison_json": str(comparison_json_path),
@@ -753,6 +761,17 @@ def _run_pack_review_smoke(args: argparse.Namespace) -> dict:
     }
 
 
+def _clean_output_dir(path: Path) -> None:
+    target = path.resolve()
+    if not target.exists():
+        return
+    if not target.is_dir():
+        raise ValueError(f"Cannot clean non-directory path: {path}")
+    if target.parent == target:
+        raise ValueError(f"Refusing to clean filesystem root: {path}")
+    shutil.rmtree(target)
+
+
 def _apply_agent_args(config: SkillBenchConfig, args: argparse.Namespace) -> None:
     if getattr(args, "agent_runner", None):
         config.agent_runner = args.agent_runner

skills/skillbench/SKILL.md

@@ -18,7 +18,7 @@ Use the SkillBench runtime to evaluate and evolve Codex skills.
    - `pack-checklist`: render a Markdown authoring checklist for reviewing an eval pack before CI.
    - `pack-compare`: compare eval pack versions for case and coverage changes, with optional removed-coverage CI gates.
    - `pack-review-artifacts`: build pack review CI result JSON, JUnit, and SARIF from eval pack review artifacts.
-   - `pack-review-smoke`: run local pack review smoke from two eval packs and build the same evidence bundle used in CI.
+   - `pack-review-smoke`: run local pack review smoke from two eval packs and build the same evidence bundle used in CI; add `--clean` for repeated local runs.
    - `list-cases`: inspect case IDs, tags, modes, and dimensions before filtering.
    - `eval`: score one candidate against an eval set.
    - `lift`: compare with-skill and without-skill runs to measure skill utility.

tests/test_skillbench_core.py

@@ -2247,6 +2247,37 @@ def test_pack_review_smoke_cli_builds_review_bundle(tmp_path, capsys):
     assert (bundle_dir / "skillbench-comment.md").exists()
 
 
+def test_pack_review_smoke_cli_clean_removes_stale_outputs(tmp_path, capsys):
+    review_dir = tmp_path / "pack-review-smoke"
+    bundle_dir = tmp_path / "pack-review-bundle"
+    review_dir.mkdir()
+    bundle_dir.mkdir()
+    (review_dir / "stale.validation.json").write_text("{}", encoding="utf-8")
+    (bundle_dir / "stale.txt").write_text("old", encoding="utf-8")
+
+    exit_code = skillbench_main(
+        [
+            "pack-review-smoke",
+            str(GENERIC_SKILL_SMOKE_PACK),
+            str(EVAL_PACKS_DIR / "generic-skill-release.json"),
+            "--review-dir",
+            str(review_dir),
+            "--bundle-output",
+            str(bundle_dir),
+            "--clean",
+            "--json",
+        ]
+    )
+
+    data = json.loads(capsys.readouterr().out)
+    assert exit_code == 0
+    assert data["cleaned"] is True
+    assert not (review_dir / "stale.validation.json").exists()
+    assert not (bundle_dir / "stale.txt").exists()
+    assert (review_dir / "pack_review_ci_result.json").exists()
+    assert (bundle_dir / "bundle_manifest.json").exists()
+
+
 def test_report_bundle_reads_utf8_bom_external_ci_result_json(tmp_path, capsys):
     exit_code = skillbench_main(
         [