feat(cockpit): loop-health panel — last tick, cadence, verdict history, stall detection (#85)

.claude/scripts/cockpit.sh

@@ -9,6 +9,12 @@
 # no persistent server, no watch daemon (re-run this script, or wrap it in
 # `watch -n 30 bash .claude/scripts/cockpit.sh`).
 #
+# Issue #85 adds a "Loop health" panel, sourced from loop-tick.sh's tick
+# record log (see loop-tick.sh's write_tick_record): the last tick's verdict,
+# the current cadence (FAST/WATCH/IDLE), the full verdict history (newest
+# first), and a STALLED banner if no tick has landed in over 2x the cadence's
+# expected interval (FAST=60s -> 120s, WATCH=300s -> 600s, IDLE=900s -> 1800s).
+#
 # Usage:
 #   cockpit.sh [--fixtures <dir>] [output-path]
 #   cockpit.sh --parse-blocking
@@ -22,8 +28,9 @@
 # like `gh issue|pr list --json ...` output) instead of calling gh at all.
 # This is the offline seam cockpit.test.sh uses — no live gh/network in tests.
 # In this mode, the live-progress panel also reads <dir>/events.jsonl (if
-# present; missing = "no active workers") instead of the real event log, so
-# tests never touch .claude/state/.
+# present; missing = "no active workers") instead of the real event log, and
+# the loop-health panel likewise reads <dir>/loop-ticks.jsonl (if present;
+# missing = "loop not armed"), so tests never touch .claude/state/.
 #
 # Degrades gracefully: if a bot-gh.sh call fails (no network / no gh auth),
 # that section renders an "unavailable (gh/network)" placeholder instead of
@@ -212,6 +219,20 @@ else
 fi
 if [ -f "$events_file" ]; then cp "$events_file" "$tmpdir/events.jsonl"; else : >"$tmpdir/events.jsonl"; fi
 
+# ---- loop tick records (issue #85, "Loop health" panel) --------------------
+# Same offline seam as the events.jsonl block above: fixtures mode reads
+# <dir>/loop-ticks.jsonl (if present); otherwise honors CLAUDE_TICKS_FILE for
+# parity with loop-tick.sh's own override, defaulting to the same gitignored
+# .claude/state/loop-ticks.jsonl. A missing/empty log just means the loop has
+# never ticked (or isn't armed yet) — rendered as a placeholder below, never
+# an error.
+if [ -n "$fixtures" ]; then
+  ticks_file="$fixtures/loop-ticks.jsonl"
+else
+  ticks_file="${CLAUDE_TICKS_FILE:-$root/.claude/state/loop-ticks.jsonl}"
+fi
+if [ -f "$ticks_file" ]; then cp "$ticks_file" "$tmpdir/loop-ticks.jsonl"; else : >"$tmpdir/loop-ticks.jsonl"; fi
+
 # ---- active worktrees -----------------------------------------------------------
 node -e '
   const fs = require("fs");
@@ -233,6 +254,8 @@ COCKPIT_OUT="$out" \
 COCKPIT_ISSUES_UNAVAILABLE="$issues_unavailable" \
 COCKPIT_PRS_UNAVAILABLE="$prs_unavailable" \
 COCKPIT_GATES_REF="$gates_ref" \
+COCKPIT_NOW="${COCKPIT_NOW:-}" \
+COCKPIT_VERDICT_HISTORY_N="${COCKPIT_VERDICT_HISTORY_N:-10}" \
 node - <<'NODE_RENDER'
 const fs = require("fs");
 const path = require("path");
@@ -271,6 +294,26 @@ function readEvents() {
 }
 const events = readEvents();
 
+// Loop tick records (issue #85): JSONL, one object per line, appended by
+// loop-tick.sh's write_tick_record — schema {ts, verdict, cadence, action,
+// issue, pr}. Same tolerate-and-skip contract as readEvents() above: a
+// blank/malformed line must never crash the whole render.
+function readTicks() {
+  let text = "";
+  try { text = fs.readFileSync(path.join(tmpdir, "loop-ticks.jsonl"), "utf8"); } catch (e) { return []; }
+  const ticks = [];
+  for (const line of text.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try {
+      const obj = JSON.parse(trimmed);
+      if (obj && typeof obj === "object" && !Array.isArray(obj)) ticks.push(obj);
+    } catch (e) { /* skip malformed line */ }
+  }
+  return ticks;
+}
+const ticks = readTicks();
+
 function esc(s) {
   return String(s == null ? "" : s)
     .replace(/&/g, "&amp;")
@@ -353,6 +396,61 @@ function renderLiveProgress() {
   return html;
 }
 
+// ---- Loop health section (issue #85) ---------------------------------------
+// Sourced from loop-tick.sh's tick record log (loop-ticks.jsonl, one line per
+// firing, file order == append order == chronological). No records at all
+// (missing file, or a file with zero valid lines) means the loop has never
+// ticked in this environment -- rendered as "loop not armed", never a crash.
+// Otherwise: the last tick's ts/verdict, the current cadence, a STALLED
+// banner when now - lastTick exceeds 2x the cadence's expected interval, and
+// the last N verdict lines, newest-first (N is bounded, NOT the full
+// potentially ~2000-row retained log -- see COCKPIT_VERDICT_HISTORY_N below).
+const CADENCE_INTERVAL_SECONDS = { FAST: 60, WATCH: 300, IDLE: 900 };
+const nowMs = process.env.COCKPIT_NOW ? Date.parse(process.env.COCKPIT_NOW) : Date.now();
+// Verdict-history table depth: "the last N verdict lines, newest first"
+// (issue #85). Overridable for testability, consistent with the
+// COCKPIT_NOW/CLAUDE_TICKS_FILE override style used elsewhere in this file.
+// Falls back to 10 if unset/non-numeric/non-positive.
+const VERDICT_HISTORY_N = (() => {
+  const n = parseInt(process.env.COCKPIT_VERDICT_HISTORY_N, 10);
+  return Number.isFinite(n) && n > 0 ? n : 10;
+})();
+function renderLoopHealth() {
+  let html = `<section id="loop-health"><h2>Loop health</h2>`;
+  if (ticks.length === 0) {
+    html += `<p class="muted">loop not armed</p></section>`;
+    return html;
+  }
+  const last = ticks[ticks.length - 1]; // file order = append order -> last line = most recent tick
+  const cadence = last.cadence != null ? String(last.cadence) : "";
+  const intervalSec = CADENCE_INTERVAL_SECONDS[cadence];
+
+  html += `<p>Last tick: <code>${esc(last.ts)}</code> &middot; verdict <code>${esc(last.verdict)}</code></p>`;
+  html += `<p>Cadence: <span class="badge muted">${esc(cadence || "(unknown)")}</span>`;
+  if (intervalSec) html += ` <span class="muted">(every ${intervalSec}s)</span>`;
+  html += `</p>`;
+
+  const lastMs = Date.parse(last.ts);
+  let stalled = false;
+  if (intervalSec && Number.isFinite(lastMs) && Number.isFinite(nowMs)) {
+    stalled = nowMs - lastMs > intervalSec * 2 * 1000;
+  }
+  if (stalled) {
+    html += `<p class="unavailable">STALLED — no tick in over ${intervalSec * 2}s (cadence ${esc(cadence)})</p>`;
+  }
+
+  html += `<table class="routing"><thead><tr><th>Time</th><th>Verdict</th><th>Cadence</th></tr></thead><tbody>`;
+  const historyStop = Math.max(0, ticks.length - VERDICT_HISTORY_N);
+  for (let i = ticks.length - 1; i >= historyStop; i--) {
+    const t = ticks[i];
+    html += `<tr><td>${esc(t.ts)}</td><td><code>${esc(t.verdict)}</code></td><td>${esc(t.cadence)}</td></tr>`;
+  }
+  html += `</tbody></table>`;
+
+  html += `</section>`;
+  return html;
+}
+
 // ---- Issues section: group by module label, parse blocking graph per issue ----
 function renderIssues() {
   if (issuesUnavailable) {
@@ -479,7 +577,7 @@ function renderWorktrees() {
   return html;
 }
 
-const generatedAt = new Date().toISOString();
+const generatedAt = Number.isFinite(nowMs) ? new Date(nowMs).toISOString() : new Date().toISOString();
 // Dark-theme stable marker (issue #69): the `data-theme="dark"` attribute
 // below is the CONTRACT a test/consumer can grep for to confirm the default
 // theme. The tiny <script> right after it restores a saved light-theme
@@ -548,8 +646,9 @@ const html = `<!doctype html>
 </head>
 <body>
 <h1>Cockpit <button id="theme-toggle" type="button">Toggle theme</button></h1>
-<p class="meta">Generated ${esc(generatedAt)} &middot; read-only Phase 1 snapshot (issue #51) + Phase 2 live progress (issue #52) + Phase 3a serve/theme/filter (issue #69) &middot; re-run <code>cockpit.sh</code> to refresh (or run <code>cockpit-serve.sh</code> for live auto-update)</p>
+<p class="meta">Generated ${esc(generatedAt)} &middot; read-only Phase 1 snapshot (issue #51) + Phase 2 live progress (issue #52) + Phase 3a serve/theme/filter (issue #69) + loop health panel (issue #85) &middot; re-run <code>cockpit.sh</code> to refresh (or run <code>cockpit-serve.sh</code> for live auto-update)</p>
 ${renderLiveProgress()}
+${renderLoopHealth()}
 ${renderIssues()}
 ${renderPRs()}
 ${renderRouting()}

.claude/scripts/cockpit.test.sh

@@ -1,20 +1,24 @@
 #!/usr/bin/env bash
 # cockpit.test.sh — offline smoke test for cockpit.sh (issue #51, extended for
-# Phase 2 live progress in issue #52, and Phase 3a serve/theme/filter in
-# issue #69).
+# Phase 2 live progress in issue #52, Phase 3a serve/theme/filter in issue #69,
+# and the "Loop health" panel in issue #85).
 #
-# Runs the generator against controlled FIXTURE issue/PR/events JSON (never
-# live gh/network, and never the real event log — see cockpit.sh's
-# --fixtures mode), then asserts the produced HTML contains every required
-# section (issues-by-module with blocking relationships, PRs with review/CI
-# badges, a routing table with a real `model:` value, a worktrees section,
-# a live-progress panel deduped to each worker's latest phase, the default
-# dark-theme marker) and that the blocking-relationship parser
-# (`cockpit.sh --parse-blocking`) produces the expected edges for a known
-# fixture body. Also exercises the "gh/network unavailable" degrade path via
-# COCKPIT_GH_BIN, entirely offline (no real gh call, no .env), and a serve-mode
-# smoke case (cockpit-serve.sh) against the SAME fixtures, over 127.0.0.1 only
-# — no real network/gh either way.
+# Runs the generator against controlled FIXTURE issue/PR/events/loop-ticks
+# JSON (never live gh/network, and never the real event/tick logs — see
+# cockpit.sh's --fixtures mode), then asserts the produced HTML contains every
+# required section (issues-by-module with blocking relationships, PRs with
+# review/CI badges, a routing table with a real `model:` value, a worktrees
+# section, a live-progress panel deduped to each worker's latest phase, the
+# default dark-theme marker, and a loop-health panel showing the last tick +
+# cadence + a verdict history capped to the last COCKPIT_VERDICT_HISTORY_N
+# ticks (newest first) + a STALLED banner once a tick is overdue) and that
+# the blocking-relationship parser (`cockpit.sh
+# --parse-blocking`) produces the expected edges for a known fixture body.
+# Also exercises the "gh/network unavailable" / "no log at all" degrade paths
+# via COCKPIT_GH_BIN / missing CLAUDE_EVENTS_FILE / CLAUDE_TICKS_FILE, entirely
+# offline (no real gh call, no .env), and a serve-mode smoke case
+# (cockpit-serve.sh) against the SAME fixtures, over 127.0.0.1 only — no real
+# network/gh either way.
 #
 # Exit 0 on success, non-zero if any assertion fails. Runnable bare:
 #   bash .claude/scripts/cockpit.test.sh
@@ -121,9 +125,20 @@ cat > "$work/fixtures/events.jsonl" <<'EOF'
 {"ts":"2026-01-01T00:02:00Z","role":"reviewer","model":"opus","task":"52b","phase":"reviewing","lens":"correctness","detail":""}
 {"ts":"2026-01-01T00:03:00Z","role":"<script>xss()</script>\"","model":"sonnet","task":"52c","phase":"scoped","lens":"","detail":""}
 EOF
+# Loop tick fixture (issue #85, "Loop health" panel): two ticks, file order =
+# chronological, so the LAST line (FAST/advance) is the most recent tick and
+# must render as "Last tick" while BOTH rows appear in the verdict history,
+# newest first.
+cat > "$work/fixtures/loop-ticks.jsonl" <<'EOF'
+{"ts":"2026-01-01T00:00:00Z","verdict":"action=none","cadence":"IDLE","action":"none","issue":"","pr":""}
+{"ts":"2026-01-01T00:15:00Z","verdict":"action=advance issue=7","cadence":"FAST","action":"advance","issue":"7","pr":""}
+EOF
 
 html="$work/cockpit.html"
-bash "$cockpit" --fixtures "$work/fixtures" "$html" >"$work/stdout.log" 2>"$work/stderr.log"
+# COCKPIT_NOW pins "now" to 90s after the last tick above -- inside FAST's
+# 120s stall threshold, so this run must NOT show the STALLED banner (that
+# path is exercised separately in section 2b below).
+COCKPIT_NOW="2026-01-01T00:16:30Z" bash "$cockpit" --fixtures "$work/fixtures" "$html" >"$work/stdout.log" 2>"$work/stderr.log"
 rc=$?
 check "generator exits 0 on fixture run" [ "$rc" -eq 0 ]
 check "generator prints the output path" grep -qF "$html" "$work/stdout.log"
@@ -208,6 +223,87 @@ check "dark theme marker present by default" grep -qF 'data-theme="dark"' "$html
 check "theme-toggle button present" grep -q 'id="theme-toggle"' "$html"
 check "issue rows carry data-module for the client-side filter" grep -q 'data-module="module:harness"' "$html"
 
+# Loop health panel (issue #85): last tick, cadence, verdict history newest
+# first, and NO stall banner (COCKPIT_NOW above is only 90s past the last
+# tick, inside FAST's 120s threshold).
+check "loop health section present" grep -q '<section id="loop-health"' "$html"
+check "last tick's ts and verdict are rendered" grep -qF '<code>2026-01-01T00:15:00Z</code> &middot; verdict <code>action=advance issue=7</code>' "$html"
+check "current cadence (FAST) is rendered" grep -qF '<span class="badge muted">FAST</span>' "$html"
+check "no STALLED banner when the last tick is within the cadence threshold" bash -c '! grep -q "STALLED" "$1"' _ "$html"
+check "verdict history renders BOTH ticks, newest first" node -e '
+  const fs = require("fs");
+  const html = fs.readFileSync(process.argv[1], "utf8");
+  const m = html.match(/<section id="loop-health">[\s\S]*?<\/section>/);
+  if (!m) throw new Error("loop-health section not found");
+  const rows = [...m[0].matchAll(/<tr><td>([^<]*)<\/td><td><code>([^<]*)<\/code><\/td>/g)].map((r) => r[2]);
+  const want = ["action=advance issue=7", "action=none"];
+  if (JSON.stringify(rows) !== JSON.stringify(want)) {
+    throw new Error("got " + JSON.stringify(rows) + " want " + JSON.stringify(want));
+  }
+' "$html"
+
+# ---------------------------------------------------------------------------
+# 2b. Loop health STALLED banner: a last tick far older than 2x its cadence's
+#     expected interval must render the STALLED banner. Reuses the SAME
+#     fixtures dir (issues/prs/events unrelated) but pins COCKPIT_NOW well
+#     past the FAST tick's 120s threshold.
+# ---------------------------------------------------------------------------
+html_stalled="$work/cockpit-stalled.html"
+COCKPIT_NOW="2026-01-01T01:00:00Z" bash "$cockpit" --fixtures "$work/fixtures" "$html_stalled" >/dev/null 2>"$work/stderr-stalled.log"
+check "STALLED banner renders once the last tick exceeds 2x its cadence interval" grep -qF 'STALLED — no tick in over 120s (cadence FAST)' "$html_stalled"
+
+# ---------------------------------------------------------------------------
+# 2c. Verdict-history cap (review fix for issue #85): the panel must show only
+#     the last N verdict lines, newest first -- NOT every retained tick (the
+#     ticks file itself may hold up to LOOP_TICKS_MAX_LINES/2000 rows). Uses a
+#     dedicated fixtures dir with 5 DISTINGUISHABLE ticks (unique issue= per
+#     line, mirroring the loop-tick.test.sh rotation fix) and
+#     COCKPIT_VERDICT_HISTORY_N=3 so the cap is exercised deterministically
+#     without needing a huge fixture.
+# ---------------------------------------------------------------------------
+mkdir -p "$work/fixtures-history"
+echo "[]" >"$work/fixtures-history/issues.json"
+echo "[]" >"$work/fixtures-history/prs.json"
+: >"$work/fixtures-history/events.jsonl"
+cat > "$work/fixtures-history/loop-ticks.jsonl" <<'EOF'
+{"ts":"2026-01-01T00:00:00Z","verdict":"action=advance issue=1","cadence":"FAST","action":"advance","issue":"1","pr":""}
+{"ts":"2026-01-01T00:01:00Z","verdict":"action=advance issue=2","cadence":"FAST","action":"advance","issue":"2","pr":""}
+{"ts":"2026-01-01T00:02:00Z","verdict":"action=advance issue=3","cadence":"FAST","action":"advance","issue":"3","pr":""}
+{"ts":"2026-01-01T00:03:00Z","verdict":"action=advance issue=4","cadence":"FAST","action":"advance","issue":"4","pr":""}
+{"ts":"2026-01-01T00:04:00Z","verdict":"action=advance issue=5","cadence":"FAST","action":"advance","issue":"5","pr":""}
+EOF
+html_history="$work/cockpit-history.html"
+COCKPIT_NOW="2026-01-01T00:04:30Z" COCKPIT_VERDICT_HISTORY_N=3 bash "$cockpit" --fixtures "$work/fixtures-history" "$html_history" >/dev/null 2>"$work/stderr-history.log"
+check "verdict-history cap: last tick is still the most recent (issue=5)" grep -qF '<code>2026-01-01T00:04:00Z</code> &middot; verdict <code>action=advance issue=5</code>' "$html_history"
+check "verdict-history cap: table renders exactly COCKPIT_VERDICT_HISTORY_N=3 rows, newest first" node -e '
+  const fs = require("fs");
+  const html = fs.readFileSync(process.argv[1], "utf8");
+  const m = html.match(/<section id="loop-health">[\s\S]*?<\/section>/);
+  if (!m) throw new Error("loop-health section not found");
+  const rows = [...m[0].matchAll(/<tr><td>([^<]*)<\/td><td><code>([^<]*)<\/code><\/td>/g)].map((r) => r[2]);
+  const want = ["action=advance issue=5", "action=advance issue=4", "action=advance issue=3"];
+  if (JSON.stringify(rows) !== JSON.stringify(want)) {
+    throw new Error("got " + JSON.stringify(rows) + " want " + JSON.stringify(want));
+  }
+' "$html_history"
+
+# Default (COCKPIT_VERDICT_HISTORY_N unset) with only 5 ticks retained must
+# still render all 5 -- the default cap (10) must not truncate BELOW what's
+# actually there.
+html_history_default="$work/cockpit-history-default.html"
+COCKPIT_NOW="2026-01-01T00:04:30Z" bash "$cockpit" --fixtures "$work/fixtures-history" "$html_history_default" >/dev/null 2>"$work/stderr-history-default.log"
+check "verdict-history default cap (10) does not truncate a shorter (5-tick) history" node -e '
+  const fs = require("fs");
+  const html = fs.readFileSync(process.argv[1], "utf8");
+  const m = html.match(/<section id="loop-health">[\s\S]*?<\/section>/);
+  if (!m) throw new Error("loop-health section not found");
+  const rows = [...m[0].matchAll(/<tr><td>([^<]*)<\/td><td><code>([^<]*)<\/code><\/td>/g)].map((r) => r[2]);
+  const want = ["action=advance issue=5", "action=advance issue=4", "action=advance issue=3", "action=advance issue=2", "action=advance issue=1"];
+  if (JSON.stringify(rows) !== JSON.stringify(want)) {
+    throw new Error("got " + JSON.stringify(rows) + " want " + JSON.stringify(want));
+  }
+' "$html_history_default"
+
 # ---------------------------------------------------------------------------
 # 3. GATES_FILE override is honored (self-host adapter), still with fixtures
 #    (no gh/network either way).
@@ -227,16 +323,19 @@ exit 1
 EOF
 chmod +x "$fake_gh"
 html_unavail="$work/cockpit-unavail.html"
-# CLAUDE_EVENTS_FILE points at a guaranteed-missing path so this run is fully
-# offline/deterministic (never touches the real, gitignored event log) and
-# doubles as the "no events file at all" -> "no active workers" assertion.
-COCKPIT_GH_BIN="$fake_gh" CLAUDE_EVENTS_FILE="$work/no-such-events.jsonl" bash "$cockpit" "$html_unavail" >/dev/null 2>"$work/stderr-unavail.log"
+# CLAUDE_EVENTS_FILE/CLAUDE_TICKS_FILE point at guaranteed-missing paths so
+# this run is fully offline/deterministic (never touches the real,
+# gitignored logs) and doubles as the "no log at all" degrade assertions for
+# both the live-progress panel ("no active workers") and the loop-health
+# panel ("loop not armed", issue #85) -- neither must crash the render.
+COCKPIT_GH_BIN="$fake_gh" CLAUDE_EVENTS_FILE="$work/no-such-events.jsonl" CLAUDE_TICKS_FILE="$work/no-such-ticks.jsonl" bash "$cockpit" "$html_unavail" >/dev/null 2>"$work/stderr-unavail.log"
 rc_unavail=$?
 check "generator still exits 0 when gh is unavailable" [ "$rc_unavail" -eq 0 ]
 check "issues section shows unavailable placeholder" grep -q '<section id="issues"><h2>Open issues</h2><p class="unavailable">unavailable (gh/network)</p>' "$html_unavail"
 check "PRs section shows unavailable placeholder" grep -q '<section id="prs"><h2>Open PRs</h2><p class="unavailable">unavailable (gh/network)</p>' "$html_unavail"
 check "routing/worktrees sections still render (no crash) despite gh failure" bash -c 'grep -q "routing" "$1" && grep -q "worktrees" "$1"' _ "$html_unavail"
 check "missing events file renders 'no active workers' placeholder" grep -q '<section id="live"><h2>Live worker progress</h2><p class="muted">no active workers</p>' "$html_unavail"
+check "missing loop-ticks log renders 'loop not armed' placeholder, no crash" grep -qF '<section id="loop-health"><h2>Loop health</h2><p class="muted">loop not armed</p></section>' "$html_unavail"
 
 # ---------------------------------------------------------------------------
 # 5. Serve mode (cockpit-serve.sh, issue #69): dashboard over HTTP + SSE live