diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index df12724..2a869d5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,7 +39,10 @@ jobs: run: | set -eu out="rs/target/${{ matrix.target }}/release" - "$out/bean-check" --help >/dev/null + # all five runtime binaries must be present and runnable + for b in bean-check bean-verify bean-run bean-hook bean-lessons; do + "$out/$b" --help >/dev/null + done "$out/bean-check" --dir test/fixtures/converged --no-state >/dev/null # exit 0 ready if "$out/bean-check" --dir test/fixtures/open-risk --no-state; then echo "expected blocked exit"; exit 1 @@ -51,7 +54,7 @@ jobs: dir="bean-${GITHUB_REF_NAME}-${{ matrix.target }}" out="rs/target/${{ matrix.target }}/release" mkdir -p "$dir" - cp "$out"/bean-check "$out"/bean-verify "$out"/bean-run "$out"/bean-hook "$dir"/ + cp "$out"/bean-check "$out"/bean-verify "$out"/bean-run "$out"/bean-hook "$out"/bean-lessons "$dir"/ cp README.md LICENSE "$dir"/ 2>/dev/null || true tar -czf "$dir.tar.gz" "$dir" shasum -a 256 "$dir.tar.gz" > "$dir.tar.gz.sha256" diff --git a/CHANGELOG.md b/CHANGELOG.md index 67295c3..c410265 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## Unreleased +- **`bean-lessons` — trace analyzer (read-only, deterministic).** Reads the `.bean/runs/*.json` + trace corpus and writes a ranked lessons-candidates report to `.bean/lessons.json` (+ optional + `--markdown`). Five candidate kinds: `recurring_residual`, `high_pivot`, `budget_exceeded`, + `blocker_code_frequency`, `verifier_failure`. No LLM, no network. Read-only with respect to + claims, prompts, skills, and memory — it **proposes, never applies** (this is NOT cross-task + learning). Exit codes: `0` candidates found, `2` none/empty corpus (report still written), `3` + invalid corpus or write failure (fail closed). Schema: `schemas/lessons.schema.json`; docs: + `skills/bean/references/lessons.md`; fixtures: `test/fixtures/traces/`; built + shipped by + `install.sh` and the release tarballs. + - **Trace artifact v0 (emit-only).** Every `bean-run` now writes a stable post-run record to `.bean/runs/.json`: `schema_version`, `run_id`, `goal`, `started_at`/`ended_at`, `status`, `certificate`, `rounds`, `pivot_count`, `blockers_opened`/`blockers_closed`, diff --git a/README.md b/README.md index 48cd25b..61fedb7 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ git clone https://github.com/grainulation/bean.git cd bean && ./install.sh ``` -This builds the runtime (`bean-check`, `bean-verify`, `bean-run`, `bean-hook`), installs the `/bean` skill, and registers the native Stop hook for Claude Code and Codex. +This builds the runtime (`bean-check`, `bean-verify`, `bean-run`, `bean-hook`, `bean-lessons`), installs the `/bean` skill, and registers the native Stop hook for Claude Code and Codex. **Prebuilt binaries** (no Rust): download the tarball for your platform from the [latest release](https://github.com/grainulation/bean/releases) and put the binaries on your `PATH`. @@ -79,14 +79,15 @@ bean also triggers on "do this thoroughly", "be systematic", or when a task obje | **5. Revise beliefs** | Supersede claims that new evidence overturns; resolve conflicts. | | **6. Converged?** | No unresolved conflicts + evidence bar (or oracle) met + a dry round → deliver. | -## The runtime (four binaries) +## The runtime (five binaries) A single self-contained binary per tool — no install dependency. (The JS `bean-check` is kept as the conformance reference; the Rust runtime must match it byte-for-byte, including certificates.) - **`bean-check`** — the compiler/gate. Reads `.bean/claims.json` (+ optional `run.json`) and exits nonzero until the loop converges: conflicts, undischarged risks, below-bar load-bearing claims, dry-round, budget — plus the 2.0 oracle gate. Emits a deterministic certificate. - **`bean-verify`** — the only path that runs an oracle: a declared command (argv, no shell, claim JSON on stdin); records a scrubbed verdict `bean-check` adjudicates. -- **`bean-run`** — the driver: injects the compiler signal into the agent's prompt each round, records what it emits, enforces forward progress. Model-agnostic (`--agent "claude -p"` / `"codex exec -"`). +- **`bean-run`** — the driver: injects the compiler signal into the agent's prompt each round, records what it emits, enforces forward progress, and writes a per-run trace artifact (`.bean/runs/.json`). Model-agnostic (`--agent "claude -p"` / `"codex exec -"`). - **`bean-hook`** — the native Stop hook for Claude Code and Codex: blocks the agent from finishing until the loop converges; inert when a project has no `.bean/` ledger. +- **`bean-lessons`** — read-only trace analyzer: reads `.bean/runs/*.json` and writes a ranked lessons-candidates report (`.bean/lessons.json`). Deterministic; proposes, never applies. The 2.0 oracle gate is opt-in via `run.json` → `verification.mode`: `compat` (default, == 1.x), `advisory` (warn), `strict` (require a passing oracle or a named residual). See [`skills/bean/references/oracle-gate.md`](skills/bean/references/oracle-gate.md). diff --git a/install.sh b/install.sh index 5a285eb..acf801f 100755 --- a/install.sh +++ b/install.sh @@ -3,7 +3,7 @@ # bean installer — builds the Rust runtime and wires it into Claude Code (and Codex). # # What you get after this: -# - the bean runtime binaries (bean-check, bean-verify, bean-run, bean-hook) built into ./bin +# - the bean runtime binaries (bean-check, bean-verify, bean-run, bean-hook, bean-lessons) built into ./bin # - the /bean skill installed # - a native Stop hook registered so bean COUPLES to execution: an agent can't finish a # bean-tracked task (one with a .bean/ ledger) until the loop converges. Inert otherwise. @@ -28,10 +28,10 @@ build_runtime() { echo " building the Rust runtime (release)..." cargo build --release --quiet --manifest-path "$REPO_DIR/rs/Cargo.toml" mkdir -p "$REPO_DIR/bin" - for b in bean-check bean-verify bean-run bean-hook; do + for b in bean-check bean-verify bean-run bean-hook bean-lessons; do cp "$REPO_DIR/rs/target/release/$b" "$REPO_DIR/bin/$b" done - echo " binaries -> $REPO_DIR/bin (bean-check, bean-verify, bean-run, bean-hook)" + echo " binaries -> $REPO_DIR/bin (bean-check, bean-verify, bean-run, bean-hook, bean-lessons)" } install_claude() { diff --git a/rs/src/bin/bean-lessons.rs b/rs/src/bin/bean-lessons.rs new file mode 100644 index 0000000..3756a22 --- /dev/null +++ b/rs/src/bin/bean-lessons.rs @@ -0,0 +1,379 @@ +// bean-lessons — the first consumer of the trace artifact (Rust, read-only). +// +// Reads .bean/runs/*.json (trace/v0) and emits a ranked LESSONS-CANDIDATES report to +// .bean/lessons.json (+ optional .bean/lessons.md). It proposes; it never applies. NO mutation +// of claims, prompts, skills, or memory. Deterministic — no LLM, no network. +// +// bean-lessons --dir [--markdown] +// +// This is NOT cross-task learning. It surfaces patterns for a human (or a later, separate step) +// to act on. See skills/bean/references/lessons.md and schemas/lessons.schema.json. +// +// Exit: 0 = report written with >=1 candidate; 2 = no runs / no candidates above threshold +// (still writes a report); 3 = invalid trace corpus or write failure. + +use serde_json::{json, Value}; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::exit; +use std::time::{SystemTime, UNIX_EPOCH}; + +fn die(code: i32, msg: &str) -> ! { + eprintln!("bean-lessons: {msg}"); + exit(code); +} + +fn s<'a>(v: &'a Value, k: &str) -> &'a str { + v.get(k).and_then(|x| x.as_str()).unwrap_or("") +} + +fn norm(reason: &str) -> String { + reason + .split_whitespace() + .collect::>() + .join(" ") + .to_lowercase() +} + +// The trace/v0 required keys. bean-lessons is downstream of a strict trace schema; a partial or +// hand-copied corpus must FAIL CLOSED rather than be silently summarized into misleading +// candidates (missing run_id/status/arrays would otherwise default to empty). +const TRACE_REQUIRED: &[&str] = &[ + "schema_version", + "run_id", + "goal", + "started_at", + "ended_at", + "status", + "certificate", + "rounds", + "pivot_count", + "blockers_opened", + "blockers_closed", + "blocker_codes", + "verifier_verdicts", + "residuals", + "artifacts_changed", +]; + +fn validate_trace(v: &Value) -> Result<(), String> { + for k in TRACE_REQUIRED { + if v.get(*k).is_none() { + return Err(format!("missing required field `{k}`")); + } + } + if v.get("run_id") + .and_then(|x| x.as_str()) + .filter(|s| !s.is_empty()) + .is_none() + { + return Err("`run_id` must be a non-empty string".into()); + } + if v.get("status") + .and_then(|x| x.as_str()) + .filter(|s| !s.is_empty()) + .is_none() + { + return Err("`status` must be a non-empty string".into()); + } + if v.get("pivot_count").and_then(|x| x.as_u64()).is_none() { + return Err("`pivot_count` must be a non-negative integer".into()); + } + for arr in ["blocker_codes", "verifier_verdicts", "residuals"] { + if !v.get(arr).map(|x| x.is_array()).unwrap_or(false) { + return Err(format!("`{arr}` must be an array")); + } + } + Ok(()) +} + +// A candidate accumulates the distinct runs it covers; count == number of distinct runs. +struct Cand { + kind: &'static str, + signal: String, + // run_id -> status, kept ordered + deduped by run_id + evidence: BTreeMap, +} +impl Cand { + fn add(&mut self, run_id: &str, status: &str) { + self.evidence + .entry(run_id.to_string()) + .or_insert_with(|| status.to_string()); + } + fn count(&self) -> usize { + self.evidence.len() + } +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let mut dir = ".".to_string(); + let mut markdown = false; + let mut i = 0; + while i < args.len() { + match args[i].as_str() { + "--dir" => { + i += 1; + dir = args + .get(i) + .cloned() + .unwrap_or_else(|| die(3, "--dir requires a path")); + } + "--markdown" => markdown = true, + "-h" | "--help" => { + println!("bean-lessons --dir [--markdown]"); + exit(0); + } + other => die(3, &format!("unknown argument: {other}")), + } + i += 1; + } + + let bean_dir = Path::new(&dir).join(".bean"); + let runs_dir = bean_dir.join("runs"); + + // ---- read the trace corpus (fail closed on any malformed/foreign trace) ---- + let mut runs: Vec = vec![]; + if runs_dir.is_dir() { + let mut paths: Vec = std::fs::read_dir(&runs_dir) + .unwrap_or_else(|e| die(3, &format!("cannot read {}: {e}", runs_dir.display()))) + .flatten() + .map(|e| e.path()) + .filter(|p| p.extension().map(|x| x == "json").unwrap_or(false)) + .collect(); + paths.sort(); + for p in paths { + let txt = std::fs::read_to_string(&p) + .unwrap_or_else(|e| die(3, &format!("cannot read {}: {e}", p.display()))); + let v: Value = serde_json::from_str(&txt) + .unwrap_or_else(|_| die(3, &format!("invalid trace JSON: {}", p.display()))); + if s(&v, "schema_version") != "trace/v0" { + die(3, &format!("not a trace/v0 artifact: {}", p.display())); + } + if let Err(why) = validate_trace(&v) { + die(3, &format!("invalid trace {}: {why}", p.display())); + } + runs.push(v); + } + } + let source_run_count = runs.len(); + + // ---- derive candidates (deterministic) ---- + let mut cands: Vec = vec![]; + + // 1. recurring_residual: distinct runs per normalized residual reason (threshold >= 2). + // signal = the original reason from the smallest run_id in the group. + let mut residual_groups: BTreeMap< + String, + (BTreeMap, BTreeMap), + > = BTreeMap::new(); // norm -> (run_id->status, run_id->original_reason) + for r in &runs { + let rid = s(r, "run_id"); + let st = s(r, "status"); + if let Some(arr) = r.get("residuals").and_then(|v| v.as_array()) { + for res in arr { + let reason = s(res, "reason"); + if reason.is_empty() { + continue; + } + let g = residual_groups.entry(norm(reason)).or_default(); + g.0.entry(rid.to_string()).or_insert_with(|| st.to_string()); + g.1.entry(rid.to_string()) + .or_insert_with(|| reason.to_string()); + } + } + } + for (_n, (ev, originals)) in residual_groups { + if ev.len() >= 2 { + // smallest run_id's original reason (BTreeMap iterates sorted by key) + let signal = originals.values().next().cloned().unwrap_or_default(); + cands.push(Cand { + kind: "recurring_residual", + signal, + evidence: ev, + }); + } + } + + // 2. high_pivot: runs with pivot_count >= 2. + let mut high = Cand { + kind: "high_pivot", + signal: "pivot_count >= 2".into(), + evidence: BTreeMap::new(), + }; + for r in &runs { + if r.get("pivot_count").and_then(|v| v.as_u64()).unwrap_or(0) >= 2 { + high.add(s(r, "run_id"), s(r, "status")); + } + } + if high.count() >= 1 { + cands.push(high); + } + + // 3. budget_exceeded: runs that hit the round ceiling. + let mut budget = Cand { + kind: "budget_exceeded", + signal: "budget-exceeded".into(), + evidence: BTreeMap::new(), + }; + for r in &runs { + if s(r, "status") == "budget-exceeded" { + budget.add(s(r, "run_id"), s(r, "status")); + } + } + if budget.count() >= 1 { + cands.push(budget); + } + + // 4. blocker_code_frequency: distinct runs per blocker code (no threshold; ranked by count). + let mut by_code: BTreeMap> = BTreeMap::new(); + for r in &runs { + if let Some(arr) = r.get("blocker_codes").and_then(|v| v.as_array()) { + for c in arr { + if let Some(code) = c.as_str() { + by_code + .entry(code.to_string()) + .or_default() + .entry(s(r, "run_id").to_string()) + .or_insert_with(|| s(r, "status").to_string()); + } + } + } + } + for (code, ev) in by_code { + cands.push(Cand { + kind: "blocker_code_frequency", + signal: code, + evidence: ev, + }); + } + + // 5. verifier_failure: distinct runs per verifier with a non-pass embedded verdict. + let mut by_verifier: BTreeMap> = BTreeMap::new(); + for r in &runs { + if let Some(arr) = r.get("verifier_verdicts").and_then(|v| v.as_array()) { + for vd in arr { + let verdict = s(vd, "verdict"); + if verdict.is_empty() || verdict == "pass" { + continue; + } + let who = { + let v = s(vd, "verifier"); + if v.is_empty() { + s(vd, "claim") + } else { + v + } + }; + by_verifier + .entry(format!("{who} failed")) + .or_default() + .entry(s(r, "run_id").to_string()) + .or_insert_with(|| s(r, "status").to_string()); + } + } + } + for (signal, ev) in by_verifier { + cands.push(Cand { + kind: "verifier_failure", + signal, + evidence: ev, + }); + } + + // ---- rank: count desc, kind asc, signal asc ---- + cands.sort_by(|a, b| { + b.count() + .cmp(&a.count()) + .then(a.kind.cmp(b.kind)) + .then(a.signal.cmp(&b.signal)) + }); + let candidates: Vec = cands + .iter() + .enumerate() + .map(|(idx, c)| { + let evidence: Vec = c + .evidence + .iter() + .map(|(rid, st)| json!({ "run_id": rid, "status": st })) + .collect(); + json!({ + "kind": c.kind, + "rank": idx + 1, + "count": c.count(), + "signal": c.signal, + "evidence": evidence, + }) + }) + .collect(); + + let generated_at = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0); + let report = json!({ + "schema_version": "bean.lessons.v0", + "generated_at": generated_at, + "source_run_count": source_run_count, + "candidates": candidates, + }); + + // ---- write (read-only w.r.t. everything except the report files); fail closed ---- + if let Err(e) = std::fs::create_dir_all(&bean_dir).and_then(|_| { + std::fs::write( + bean_dir.join("lessons.json"), + serde_json::to_string_pretty(&report).unwrap() + "\n", + ) + }) { + die( + 3, + &format!("could not write lessons.json: {e} — failing closed"), + ); + } + if markdown { + if let Err(e) = std::fs::write(bean_dir.join("lessons.md"), render_md(&report)) { + die( + 3, + &format!("could not write lessons.md: {e} — failing closed"), + ); + } + } + + eprintln!( + "bean-lessons: {} candidate(s) from {} run(s) -> {}", + candidates.len(), + source_run_count, + bean_dir.join("lessons.json").display() + ); + // 0 = something to report; 2 = nothing (no runs or no candidates above threshold). + exit(if candidates.is_empty() { 2 } else { 0 }); +} + +fn render_md(report: &Value) -> String { + let mut out = String::from("# bean lessons (candidates)\n\n"); + out.push_str(&format!( + "Source runs: {}\n\n", + report + .get("source_run_count") + .and_then(|v| v.as_u64()) + .unwrap_or(0) + )); + let cands = report.get("candidates").and_then(|v| v.as_array()); + match cands { + Some(a) if !a.is_empty() => { + out.push_str("| rank | kind | count | signal |\n|---|---|---|---|\n"); + for c in a { + out.push_str(&format!( + "| {} | {} | {} | {} |\n", + c.get("rank").and_then(|v| v.as_u64()).unwrap_or(0), + s(c, "kind"), + c.get("count").and_then(|v| v.as_u64()).unwrap_or(0), + s(c, "signal").replace('|', "\\|"), + )); + } + } + _ => out.push_str("_No candidates above threshold._\n"), + } + out.push_str("\n_Candidates are observations for a human to triage — nothing is applied._\n"); + out +} diff --git a/schemas/lessons.schema.json b/schemas/lessons.schema.json new file mode 100644 index 0000000..2271791 --- /dev/null +++ b/schemas/lessons.schema.json @@ -0,0 +1,74 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/grainulation/bean/schemas/lessons.schema.json", + "title": "bean lessons-candidates report (v0)", + "description": "Read-only report emitted by bean-lessons to .bean/lessons.json. It ranks candidate lessons derived deterministically from the trace corpus (.bean/runs/*.json). It PROPOSES; it never applies. No mutation of claims, prompts, skills, or memory. NOT cross-task learning.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "generated_at", + "source_run_count", + "candidates" + ], + "properties": { + "schema_version": { "const": "bean.lessons.v0" }, + "generated_at": { + "type": "integer", + "description": "Report generation time as an INTEGER, milliseconds since the Unix epoch (not ISO-8601). Metadata only — excluded from determinism comparisons (the candidates are deterministic for a given corpus)." + }, + "source_run_count": { + "type": "integer", + "minimum": 0, + "description": "Number of trace/v0 runs analyzed." + }, + "candidates": { + "type": "array", + "description": "Ranked lesson candidates. Sorted by count desc, then kind asc, then signal asc.", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "rank", "count", "signal", "evidence"], + "properties": { + "kind": { + "type": "string", + "enum": [ + "recurring_residual", + "high_pivot", + "budget_exceeded", + "blocker_code_frequency", + "verifier_failure" + ] + }, + "rank": { + "type": "integer", + "minimum": 1, + "description": "1-based global rank." + }, + "count": { + "type": "integer", + "minimum": 1, + "description": "Number of distinct runs the candidate covers." + }, + "signal": { + "type": "string", + "description": "The thing observed: a residual reason, a blocker code, ' failed', etc." + }, + "evidence": { + "type": "array", + "description": "The runs supporting this candidate, sorted by run_id.", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["run_id", "status"], + "properties": { + "run_id": { "type": "string" }, + "status": { "type": "string" } + } + } + } + } + } + } + } +} diff --git a/skills/bean/references/lessons.md b/skills/bean/references/lessons.md new file mode 100644 index 0000000..74a91a0 --- /dev/null +++ b/skills/bean/references/lessons.md @@ -0,0 +1,69 @@ +# Reference: bean-lessons (trace analyzer v0) + +`bean-lessons` is the first consumer of the [trace artifact](trace.md). It reads the accumulated +`.bean/runs/*.json` corpus and writes a ranked **lessons-candidates** report. It proposes; it +never applies. + +> **Scope (read this).** Read-only with respect to claims, prompts, skills, and memory. +> Deterministic — no LLM, no network. This is NOT cross-task learning: it surfaces patterns for a +> human (or a later, separate step) to triage. The moment anything here edits a skill/prompt/ +> memory, it has left this tool. + +## Use + +``` +bean-lessons --dir [--markdown] +``` + +- Reads `/.bean/runs/*.json` (must be `schema_version: "trace/v0"`). +- Writes `/.bean/lessons.json` (always on success), and `lessons.md` with `--markdown`. +- Exit codes: `0` report written with ≥1 candidate; `2` no runs / no candidates above threshold + (still writes a report with `candidates: []`); `3` invalid trace corpus or write failure + (fail closed — never emit a partial report as success). + +## Candidate kinds + thresholds + +| kind | from trace | threshold | signal | +| ------------------------ | -------------------------------------- | ----------------- | ------------------------ | +| `recurring_residual` | `residuals[].reason` (normalized) | ≥ 2 runs | representative reason | +| `high_pivot` | `pivot_count` | `pivot_count ≥ 2` | `"pivot_count >= 2"` | +| `budget_exceeded` | `status == "budget-exceeded"` | ≥ 1 run | `"budget-exceeded"` | +| `blocker_code_frequency` | `blocker_codes[]` | none (ranked) | the blocker code (`E_*`) | +| `verifier_failure` | `verifier_verdicts[]` (verdict ≠ pass) | ≥ 1 run | `" failed"` | + +`count` is always the number of **distinct runs** the candidate covers. Residual reasons are +grouped by a normalized key (trim, collapse whitespace, lowercase); the `signal` is the original +reason text from the smallest `run_id` in the group. + +## Determinism + +Candidates are sorted by `count` desc → `kind` asc → `signal` asc; `rank` is the 1-based index. +`evidence` within a candidate is sorted by `run_id`. The only non-deterministic field is +`generated_at` (a timestamp) — exclude it when comparing two reports for stability. + +Schema: [`schemas/lessons.schema.json`](../../../schemas/lessons.schema.json) +(`schema_version: "bean.lessons.v0"`, `additionalProperties: false`). + +## Output shape + +```json +{ + "schema_version": "bean.lessons.v0", + "generated_at": 1782017126557, + "source_run_count": 18, + "candidates": [ + { + "kind": "blocker_code_frequency", + "rank": 1, + "count": 9, + "signal": "E_UNVERIFIED_LOADBEARING", + "evidence": [{ "run_id": "run-...", "status": "stuck" }] + } + ] +} +``` + +## What it is not + +It does not edit prompts, skills, memory, or claims. Turning these candidates into harness +changes is the cross-task learning step — a separate, later decision, deliberately out of scope. diff --git a/test/conformance.mjs b/test/conformance.mjs index 69254e5..c08e516 100644 --- a/test/conformance.mjs +++ b/test/conformance.mjs @@ -11,6 +11,7 @@ // // Once this is green, Rust bean-check has earned the right to gate its own development // (the self-hosting ratchet). +import assert from "node:assert/strict"; import fs from "node:fs"; import path from "node:path"; import { spawnSync } from "node:child_process"; @@ -49,6 +50,8 @@ const shape = (r) => ({ const fixtures = fs .readdirSync(FIX, { withFileTypes: true }) .filter((e) => e.isDirectory()) + // `traces/` holds trace/v0 sample files for bean-lessons, not bean-check claim fixtures. + .filter((e) => e.name !== "traces") .map((e) => e.name) .sort(); @@ -566,7 +569,111 @@ console.log( `${gpass}/${GATE_CASES.length + 1} oracle-gate behavior checks pass`, ); -const total = pass + tpass + dpass + gpass; -const totalN = fixtures.length + SCENARIOS.length + 3 + GATE_CASES.length + 1; -console.log(`\n${total}/${totalN} conformance + driver + gate checks pass`); +// ---- bean-lessons (trace analyzer, read-only) ---- +const LESSONS = path.join(root, "rs", "target", "release", "bean-lessons"); +const FIXTURE_TRACES = path.join(root, "test", "fixtures", "traces"); +/** @param {string[]} files */ +const lessonsDir = (files) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bean-lessons-")); + fs.mkdirSync(path.join(dir, ".bean", "runs"), { recursive: true }); + for (const [name, body] of files) + fs.writeFileSync(path.join(dir, ".bean", "runs", name), body); + return dir; +}; +const fixtureCorpus = fs + .readdirSync(FIXTURE_TRACES) + .filter((f) => f.endsWith(".json")) + .map((f) => [f, fs.readFileSync(path.join(FIXTURE_TRACES, f), "utf8")]); +let lpass = 0; +const lcheck = (name, fn) => { + try { + fn(); + lpass++; + console.log(` ok ${name}`); + } catch (e) { + fails.push(name); + console.log(` DIFF ${name}: ${e.message}`); + } +}; +const runLessons = (dir) => { + const r = spawnSync(LESSONS, ["--dir", dir], { encoding: "utf8" }); + const p = path.join(dir, ".bean", "lessons.json"); + return { + exit: r.status, + report: fs.existsSync(p) ? JSON.parse(fs.readFileSync(p, "utf8")) : null, + }; +}; +lcheck("lessons: ranks all 5 kinds from the fixture corpus", () => { + const dir = lessonsDir(fixtureCorpus); + const { exit, report } = runLessons(dir); + assert.equal(exit, 0, `exit ${exit}`); + assert.equal(report.schema_version, "bean.lessons.v0"); + assert.equal(report.source_run_count, 4); + const top = report.candidates[0]; + assert.equal(top.kind, "blocker_code_frequency"); + assert.equal(top.signal, "E_OPEN_RISK"); + assert.equal(top.count, 3); + assert.equal(top.rank, 1); + const kinds = new Set(report.candidates.map((c) => c.kind)); + for (const k of [ + "recurring_residual", + "high_pivot", + "budget_exceeded", + "blocker_code_frequency", + "verifier_failure", + ]) + assert.ok(kinds.has(k), `missing kind ${k}`); + const rr = report.candidates.find((c) => c.kind === "recurring_residual"); + assert.equal(rr.count, 2); + assert.equal(rr.signal, "no write access to staging"); +}); +lcheck("lessons: deterministic (modulo generated_at)", () => { + const dir = lessonsDir(fixtureCorpus); + const a = runLessons(dir).report; + const b = runLessons(dir).report; + delete a.generated_at; + delete b.generated_at; + assert.equal(JSON.stringify(a), JSON.stringify(b)); +}); +lcheck("lessons: empty corpus -> exit 2, report written", () => { + const dir = lessonsDir([]); + const { exit, report } = runLessons(dir); + assert.equal(exit, 2, `exit ${exit}`); + assert.equal(report.source_run_count, 0); + assert.deepEqual(report.candidates, []); +}); +lcheck("lessons: malformed trace -> exit 3 (fail closed)", () => { + const dir = lessonsDir([["bad.json", "{ not json"]]); + const { exit } = runLessons(dir); + assert.equal(exit, 3, `exit ${exit}`); +}); +lcheck("lessons: foreign schema_version -> exit 3", () => { + const dir = lessonsDir([ + ["x.json", JSON.stringify({ schema_version: "trace/v9" })], + ]); + const { exit } = runLessons(dir); + assert.equal(exit, 3, `exit ${exit}`); +}); +lcheck( + "lessons: trace/v0 missing required fields -> exit 3 (fail closed)", + () => { + // correct schema_version but a partial/copied trace must NOT be silently summarized. + const dir = lessonsDir([ + [ + "partial.json", + JSON.stringify({ schema_version: "trace/v0", run_id: "x" }), + ], + ]); + const { exit } = runLessons(dir); + assert.equal(exit, 3, `exit ${exit}`); + }, +); +console.log(`${lpass}/6 bean-lessons checks pass`); + +const total = pass + tpass + dpass + gpass + lpass; +const totalN = + fixtures.length + SCENARIOS.length + 3 + GATE_CASES.length + 1 + 6; +console.log( + `\n${total}/${totalN} conformance + driver + gate + lessons checks pass`, +); process.exit(fails.length ? 1 : 0); diff --git a/test/fixtures/traces/run-0001.json b/test/fixtures/traces/run-0001.json new file mode 100644 index 0000000..b60dc6e --- /dev/null +++ b/test/fixtures/traces/run-0001.json @@ -0,0 +1,18 @@ +{ + "schema_version": "trace/v0", + "run_id": "run-0001", + "goal": "g1", + "started_at": 1, + "ended_at": 2, + "status": "ready", + "certificate": "c1", + "rounds": 1, + "pivot_count": 0, + "blockers_opened": 1, + "blockers_closed": 1, + "blocker_codes": ["E_OPEN_RISK"], + "verifier_verdicts": [], + "residuals": [{ "id": "r1", "reason": "no write access to staging" }], + "artifacts_changed": [], + "metadata": {} +} diff --git a/test/fixtures/traces/run-0002.json b/test/fixtures/traces/run-0002.json new file mode 100644 index 0000000..f19fd77 --- /dev/null +++ b/test/fixtures/traces/run-0002.json @@ -0,0 +1,18 @@ +{ + "schema_version": "trace/v0", + "run_id": "run-0002", + "goal": "g2", + "started_at": 1, + "ended_at": 2, + "status": "converged-with-residuals", + "certificate": "c2", + "rounds": 4, + "pivot_count": 3, + "blockers_opened": 2, + "blockers_closed": 1, + "blocker_codes": ["E_CONFLICT", "E_OPEN_RISK"], + "verifier_verdicts": [], + "residuals": [{ "id": "r2", "reason": "No write access to staging" }], + "artifacts_changed": [], + "metadata": {} +} diff --git a/test/fixtures/traces/run-0003.json b/test/fixtures/traces/run-0003.json new file mode 100644 index 0000000..5fcb9a1 --- /dev/null +++ b/test/fixtures/traces/run-0003.json @@ -0,0 +1,20 @@ +{ + "schema_version": "trace/v0", + "run_id": "run-0003", + "goal": "g3", + "started_at": 1, + "ended_at": 2, + "status": "budget-exceeded", + "certificate": "c3", + "rounds": 5, + "pivot_count": 2, + "blockers_opened": 1, + "blockers_closed": 0, + "blocker_codes": ["E_CONFLICT"], + "verifier_verdicts": [ + { "verifier": "unit", "verdict": "fail", "claim": "c1" } + ], + "residuals": [], + "artifacts_changed": [], + "metadata": {} +} diff --git a/test/fixtures/traces/run-0004.json b/test/fixtures/traces/run-0004.json new file mode 100644 index 0000000..da21518 --- /dev/null +++ b/test/fixtures/traces/run-0004.json @@ -0,0 +1,20 @@ +{ + "schema_version": "trace/v0", + "run_id": "run-0004", + "goal": "g4", + "started_at": 1, + "ended_at": 2, + "status": "stuck", + "certificate": "c4", + "rounds": 5, + "pivot_count": 0, + "blockers_opened": 1, + "blockers_closed": 0, + "blocker_codes": ["E_OPEN_RISK"], + "verifier_verdicts": [ + { "verifier": "unit", "verdict": "pass", "claim": "c2" } + ], + "residuals": [], + "artifacts_changed": [], + "metadata": {} +}