diff --git a/.agents/skills/afk/SKILL.md b/.agents/skills/afk/SKILL.md index aba6e3fb00..91e6fb52c6 100644 --- a/.agents/skills/afk/SKILL.md +++ b/.agents/skills/afk/SKILL.md @@ -60,7 +60,7 @@ No `/back` is needed. The first genuine message is the return signal: - A message **without** the current operational prefix or a legacy bare marker, and **not** starting with `/afk` -> the captain is back. Run `bin/fm-afk-return.sh` before acting on the message that brought the captain back. - That script owns correct-ordered daemon shutdown, durable wake draining, escalation and wedge evidence, and the return-catch-up gate. + That script owns correct-ordered daemon shutdown, durable wake draining, escalation, wedge, and staleness auto-close reclaim evidence, and the return-catch-up gate. If it reports a firstmate-actionable `blocked:` event, remediate it immediately through the normal lifecycle, or explicitly reclassify it with a durable reason and close its decision key with `resolved [key=...]`, then run `bin/fm-afk-return.sh check`. Once the daemon stops, resume full per-wake responsiveness through the emitted primary-harness supervision protocol while blocker handling proceeds, so the gate never creates a blind wait. Do not answer a Bearings request or perform any other ordinary captain work until the check exits successfully. diff --git a/AGENTS.md b/AGENTS.md index 15c035f5d5..beea72410e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -89,6 +89,7 @@ state/ volatile runtime signals; gitignored .status appended by crewmates: ": " wake-event lines, not current-state truth .turn-ended touched by turn-end hooks .grok-turnend-token firstmate-owned grok hook registry token for the task; removed by teardown + .staleness-unfiled written by fm-teardown.sh's --staleness-autoclose reclaim only when filing the triage bead failed, before .meta is removed; records the preserved worktree, branch, and project so the location is never lost .kimi-turnend-token firstmate-owned Kimi hook registry token for the task; removed by teardown .parlay-listen-pid background `parlay listen` pid from a best-effort Parlay chat-panel enrollment at spawn (bin/fm-spawn.sh header); killed and removed by teardown .meta written by fm-spawn: window=, endpoint_task_id=, worktree=, project=, harness=, model=, effort=, kind=, mode=, yolo=, tasktmp=; kind=secondmate also records home= and projects=; a non-default runtime backend records further backend-specific fields (docs/configuration.md "Runtime backend"; bin/fm-backend.sh, section 8); optional label= is recorded only when --label was passed at spawn; optional beads_id= is recorded only when --beads was passed at spawn, driving the claim-first/close-last bead lifecycle (bin/fm-brief-hooks.d/beads.sh, bin/fm-teardown.sh header, bin/fm-ledger.sh) under section 7's bead-linked dispatch; fm-pr-check, including through fm-pr-merge, records one canonical pr= and the forge's pr_head= when available (GitHub pull requests and GitLab merge requests; docs/gitlab-merge-watch.md); fm-x-link appends x_request=, x_request_ts=, x_followups=, and optional x_platform=/x_reply_max_chars= for an X-mode-originated task (section 14) @@ -113,7 +114,9 @@ state/ volatile runtime signals; gitignored .afk durable away-mode flag; present = sub-supervisor may inject escalations (set by /afk, cleared on user return) .watch.lock .wake-queue.lock watcher singleton and queue serialization locks .claude-autoarm.lock .claude-autoarm-epoch .turnend-claude-blocks Claude Stop auto-arm single-flight, epoch, and guard-budget records; never touch - .hash-* .count-* .stale-* .stale-since-* .paused-* .wedge-escalations-* .seen-* .hb-surfaced-* .last-* .heartbeat-streak watcher internals; never touch + .hash-* .count-* .stale-* .stale-since-* .paused-* .wedge-escalations-* .seen-* .hb-surfaced-* .last-* .heartbeat-streak .staleness-fails-* .staleness-next-* .staleness-working-* watcher internals; never touch + .staleness-autoclose.log append-only log of the watcher's idle>2h staleness auto-close reclaim attempts against a ship task (bin/fm-teardown.sh --staleness-autoclose); never relied on, safe to delete + .staleness-autoclose-afk.log durable evidence of staleness auto-close reclaims made while away; surfaced by /afk return as catch-up evidence, then cleared .watch-triage.log watcher's absorbed-wake debug log (size-capped); never relied on, safe to delete .last-watcher-beat watcher liveness beacon, touched every poll (including while absorbing benign wakes); guard scripts read it .subsuper-* .supervise-daemon.* sub-supervisor internals; never touch @@ -330,6 +333,10 @@ A teardown refusal for uncommitted or unlanded work is a stop-and-investigate re Never force teardown without explicit discard authority. After successful teardown, record completion, retain only the configured recent Done history, and re-evaluate queued work whose blockers and time gates have cleared. +The watcher also reclaims a ship task's live process on its own once its window has sat idle past the configured staleness threshold (default two hours) and the crew is not provably working or parked at a captain-relevant gate. +Landed work gets the ordinary full teardown above; unlanded work only has its runtime endpoint reclaimed and a triage record filed, leaving the worktree, branch, and every uncommitted change untouched for later deliberate triage. +Treat a task whose endpoint has gone quiet with no captain-facing wake as a possible automatic reclaim rather than a crash; the filed staleness bead (or, if filing failed, `state/.staleness-unfiled`), the preserved worktree and branch, and `state/.staleness-autoclose.log` still hold the evidence. + A secondmate is persistent and an empty queue is healthy. Retire one only on an explicit captain or main-firstmate decision, after loading `secondmate-provisioning`; its home must contain no work under way, and forced discard still requires explicit captain authority. diff --git a/bin/fm-afk-return.sh b/bin/fm-afk-return.sh index 316479852f..810b4c23ed 100755 --- a/bin/fm-afk-return.sh +++ b/bin/fm-afk-return.sh @@ -16,10 +16,12 @@ # # The durable state/.afk-return-catchup file is written BEFORE daemon shutdown, # so a crash between stopping, draining, and blocker handling fails closed. It -# retains the drained wake, buffered-escalation, and wedge-marker evidence until -# every live open blocker is closed and `check` succeeds. Repeated begin/check -# calls are idempotent. `guard` never mutates state and is suitable for ordinary -# read entrypoints such as fm-bearings-snapshot.sh. +# retains the drained wake, buffered-escalation, wedge-marker, and +# staleness-autoclose-reclaim evidence (bin/fm-watch.sh's idle>2h backstop, +# which also runs while away) until every live open blocker is closed and +# `check` succeeds. Repeated begin/check calls are idempotent. `guard` never +# mutates state and is suitable for ordinary read entrypoints such as +# fm-bearings-snapshot.sh. set -u SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -124,7 +126,8 @@ clear_delivery_artifacts() { rm -f \ "$STATE/.subsuper-escalations" \ "$STATE/.subsuper-escalations.since" \ - "$STATE/.subsuper-inject-wedged" + "$STATE/.subsuper-inject-wedged" \ + "$STATE/.staleness-autoclose-afk.log" } return_guard() { @@ -141,7 +144,7 @@ return_guard() { } return_reconcile() { - local evidence blockers drained wedge escalations lifecycle_ok=1 + local evidence blockers drained wedge escalations staleness lifecycle_ok=1 evidence=$(mktemp "$STATE/.afk-return-evidence.XXXXXX") || return 1 blockers=$(mktemp "$STATE/.afk-return-blockers.XXXXXX") || { rm -f "$evidence"; return 1; } preserve_evidence "$evidence" @@ -168,6 +171,10 @@ return_reconcile() { escalations=$(cat "$STATE/.subsuper-escalations" 2>/dev/null || true) append_evidence escalation "$escalations" "$evidence" fi + if [ -s "$STATE/.staleness-autoclose-afk.log" ]; then + staleness=$(cat "$STATE/.staleness-autoclose-afk.log" 2>/dev/null || true) + append_evidence staleness "$staleness" "$evidence" + fi scan_open_blockers > "$blockers" if [ "$lifecycle_ok" -ne 1 ] || [ -s "$blockers" ]; then diff --git a/bin/fm-staleness-file.sh b/bin/fm-staleness-file.sh new file mode 100755 index 0000000000..cd2b862704 --- /dev/null +++ b/bin/fm-staleness-file.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# File a triage bead into the `staleness` federated store for a task whose +# expensive live process was just reclaimed by bin/fm-watch.sh's idle>2h +# backstop because its worktree still holds unlanded work. Called only from +# bin/fm-teardown.sh's staleness_chat_only_teardown, after work_is_landed has +# already returned false for the task's branch. +# Fail-open by design, matching bin/fm-bead-stamp.sh: a missing `staleness` CLI +# or a create call it rejects warns on stderr and exits 0 so a triage-filing +# problem never blocks or fails the reclaim that is already under way. +# Usage: fm-staleness-file.sh +set -u + +ID=${1-} +PURPOSE=${2-} +WORKTREE=${3-} +BRANCH=${4-} +PROJECT=${5-} +HARNESS=${6-unknown} +IDLE_SINCE=${7-} +SUMMARY=${8-} + +if [ -z "$ID" ] || [ -z "$WORKTREE" ]; then + echo "warning: fm-staleness-file.sh needs at least a task id and worktree, skipping" >&2 + exit 0 +fi + +if ! command -v staleness >/dev/null 2>&1; then + echo "warning: staleness CLI not found on PATH, could not file $ID for triage" >&2 + exit 0 +fi + +idle_human=$IDLE_SINCE +case "$IDLE_SINCE" in + '') idle_human=unknown ;; + *[!0-9]*) ;; + *) idle_human=$(date -r "$IDLE_SINCE" 2>/dev/null || date -d "@$IDLE_SINCE" 2>/dev/null || echo "$IDLE_SINCE") ;; +esac + +body=$(printf 'task: %s\npurpose: %s\nworktree: %s\nbranch: %s\nproject: %s\nharness: %s\nidle since: %s\n\n%s\n' \ + "$ID" "${PURPOSE:-unknown}" "$WORKTREE" "${BRANCH:-unknown}" "${PROJECT:-unknown}" "$HARNESS" \ + "$idle_human" "${SUMMARY:-}") + +bead_id=$(staleness create "Stale task $ID: ${BRANCH:-unknown branch}" \ + --description "$body" --labels staleness-autoclose --silent) || { + echo "warning: could not file staleness bead for $ID (see stderr above)" >&2 + exit 0 +} +echo "filed staleness bead $bead_id for $ID" diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index 38bb35196e..caad50c0bd 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -63,10 +63,18 @@ # like fm-bead-stamp.sh: a missing task CLI or a close the CLI rejects warns on # stderr and never blocks or fails an already-confirmed teardown. bin/fm-ledger.sh # is the safety net for a bead that was claimed but never reaches this path. -# Usage: fm-teardown.sh [--force] +# Usage: fm-teardown.sh [--force | --staleness-autoclose []] # --force skips ordinary-task dirty and landed-work checks, skips scout report # checks, and discards secondmate child work for kind=secondmate. Only use it # when the captain has explicitly said to discard the work. +# --staleness-autoclose is bin/fm-watch.sh's idle>2h backstop for kind=ship +# tasks only: work_is_landed decides the branch. Landed falls through to the +# ordinary teardown above unchanged. Unlanded reclaims only the runtime +# endpoint and firstmate's own tracking state (never the worktree, its +# branch, or any uncommitted change) and files a bin/fm-staleness-file.sh bead +# recording the worktree, branch, harness, and idle-since timestamp for later +# triage - deletion happens only during that deliberate triage, still under +# the never-discard-unlanded-work guard. # # Transient / stale worktree git lock recovery (teardown-lock-race): a crew process # killed mid-git-operation can leave a .git/worktrees//index.lock (or, for a @@ -125,6 +133,18 @@ if [ "$#" -lt 1 ] || ! fm_task_id_path_safe "$1"; then fi ID=$1 FORCE=${2:-} +# --staleness-autoclose (bin/fm-watch.sh's idle>2h backstop): resolve landed vs +# unlanded via work_is_landed and either fall through to the ordinary teardown +# below (landed - FORCE reset to "" so it behaves exactly like a plain call) or +# take the staleness_chat_only_teardown branch (unlanded - kill only the runtime +# endpoint, file a staleness-store bead, and never touch the worktree). The +# optional third argument is the idle-since epoch, recorded on the filed bead. +STALENESS_AUTOCLOSE=0 +STALENESS_IDLE_SINCE=${3:-} +if [ "$FORCE" = "--staleness-autoclose" ]; then + STALENESS_AUTOCLOSE=1 + FORCE="" +fi # Fail closed before any fleet mutation: a no-mistakes gate agent must never tear # down a worktree (see bin/fm-gate-refuse-lib.sh). fm_refuse_if_gate_agent @@ -159,6 +179,10 @@ KIND=$(grep '^kind=' "$META" | cut -d= -f2- || true) [ -n "$KIND" ] || KIND=ship MODE=$(grep '^mode=' "$META" | cut -d= -f2- || true) [ -n "$MODE" ] || MODE=no-mistakes +if [ "$STALENESS_AUTOCLOSE" = 1 ] && [ "$KIND" != ship ]; then + echo "error: staleness auto-close only supports kind=ship tasks (task $ID is kind=$KIND)" >&2 + exit 1 +fi BEADS_ID=$(fm_meta_get "$META" beads_id) PUBLIC_FOLLOWUP_HOME=$FM_HOME PUBLIC_FOLLOWUP_STATE=$STATE @@ -417,6 +441,53 @@ remove_pr_poll_artifacts() { fi } +# Best-effort uncommitted/unpushed summary for a staleness-store bead. Read-only +# (git status/log against $1); never writes into the worktree. +staleness_worktree_summary() { + local wt=$1 dirty unpushed + dirty=$(git -C "$wt" status --porcelain 2>/dev/null | head -20) + unpushed=$(git -C "$wt" log --oneline HEAD --not --remotes -- 2>/dev/null | head -20) + printf 'uncommitted changes:\n%s\nunpushed commits:\n%s' "${dirty:-}" "${unpushed:-}" +} + +# --staleness-autoclose, unlanded branch: reclaim the expensive live process for +# a task whose worktree has not landed (work_is_landed already returned false), +# file it to the staleness triage store, and leave the worktree, its branch, and +# every uncommitted change on disk exactly as-is - deletion happens only later, +# during deliberate triage. Mirrors the fail-open external-CLI pattern used by +# close_linked_bead(): a missing/failing staleness CLI never blocks reclaim. +staleness_chat_only_teardown() { + local harness purpose summary bead_out + fm_backend_kill "$BACKEND" "$T" "$(meta_value "$META" zellij_tab_id)" "fm-$ID" 2>/dev/null || true + remove_grok_turnend_auth "$STATE" "$ID" + remove_kimi_turnend_auth "$STATE" "$ID" + deregister_parlay_agent "$STATE" "$ID" + fm_backend_clear_transition "$BACKEND" "$STATE" "$T" || true + remove_pr_poll_artifacts "$STATE" "$ID" || true + retire_busy_state "$STATE" "$ID" "$BUSY_GEN" || true + harness=$(meta_value "$META" harness) + purpose="" + if [ -f "$DATA/$ID/brief.md" ]; then + purpose=$(awk '/^# Task/{f=1;next} f && NF {print; exit}' "$DATA/$ID/brief.md" 2>/dev/null || true) + fi + summary=$(staleness_worktree_summary "$WT") + bead_out=$("$SCRIPT_DIR/fm-staleness-file.sh" "$ID" "$purpose" "$WT" "$STALENESS_BRANCH" "$PROJ" \ + "${harness:-unknown}" "$STALENESS_IDLE_SINCE" "$summary" 2>&1) || true + [ -z "$bead_out" ] || printf '%s\n' "$bead_out" + # fm-staleness-file.sh is fail-open (always exits 0), so a failed filing is + # only visible by the absence of its success line. $ID.meta is about to be + # removed below, so on failure write a durable fallback record first - + # otherwise a preserved unlanded worktree loses its only location pointer. + if ! printf '%s\n' "$bead_out" | grep -q '^filed staleness bead '; then + printf 'task: %s\npurpose: %s\nworktree: %s\nbranch: %s\nproject: %s\nharness: %s\nidle since: %s\n' \ + "$ID" "${purpose:-unknown}" "$WT" "${STALENESS_BRANCH:-unknown}" "${PROJ:-unknown}" \ + "${harness:-unknown}" "$STALENESS_IDLE_SINCE" > "$STATE/$ID.staleness-unfiled" + fi + rm -f "$STATE/$ID.status" "$STATE/$ID.turn-ended" "$STATE/$ID.meta" \ + "$STATE/$ID.grok-turnend-token" "$STATE/$ID.kimi-turnend-token" + echo "staleness auto-close $ID: chat reclaimed, worktree $WT preserved for triage" +} + # Resolve the PR number for a worktree branch via gh-axi. Echoes the number on a # single match and returns 0; returns non-zero on no match or any lookup failure, # so the caller treats it as "no PR found" (fail-safe). @@ -1203,6 +1274,23 @@ remove_secondmate_registry_entry() { validate_pr_poll_cleanup "$STATE" "$ID" || exit 1 +if [ "$STALENESS_AUTOCLOSE" = 1 ]; then + if [ ! -d "$WT" ]; then + # Nothing to preserve or file a triage bead about - report the actual + # state instead of claiming a worktree that is already gone. + echo "staleness auto-close $ID: worktree $WT is already gone, skipping triage filing" + rm -f "$STATE/$ID.status" "$STATE/$ID.turn-ended" "$STATE/$ID.meta" \ + "$STATE/$ID.grok-turnend-token" "$STATE/$ID.kimi-turnend-token" + exit 0 + fi + STALENESS_BRANCH=$(git -C "$WT" rev-parse --abbrev-ref HEAD 2>/dev/null || echo HEAD) + if ! work_is_landed "$STALENESS_BRANCH"; then + staleness_chat_only_teardown + exit 0 + fi + # Landed: fall through to the ordinary full teardown below (FORCE is ""). +fi + if [ "$KIND" = secondmate ]; then [ -n "$HOME_PATH" ] || HOME_PATH=$WT validate_firstmate_home_for_removal "$HOME_PATH" "secondmate home" "$ID" >/dev/null || exit 1 diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 8cabd0c713..8b0f93d7ce 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -134,6 +134,30 @@ IDLE_DISCOVERY_INTERVAL=${FM_IDLE_DISCOVERY_INTERVAL:-60} # seconds between idl # daemon owns triage, so this watcher reverts to one-shot (enqueue + exit on every # wake) and never double-triages - and never runs the costly provably-working read. STALE_ESCALATE_SECS=${FM_STALE_ESCALATE_SECS:-240} # idle secs before a provably-working stale escalates as a possible wedge +# Idle>2h staleness auto-close backstop (captain design, 2026-07-31): once an +# ordinary ship task's pane content (.hash-, see staleness_autoclose_reclaim) +# has sat unchanged this long AND the crew is not provably working (see +# crew_is_provably_working - a validating crew legitimately sits on a static +# pane for its whole run), the prompt-cache advantage of keeping its live +# process around is already gone (Anthropic cache TTL is 5m/1h), so the watcher +# reclaims it via bin/fm-teardown.sh --staleness-autoclose rather than leaving +# costly compute idle indefinitely. Also runs while away (state/.afk exists) - +# reclaiming wasted idle compute matters most while nobody is watching it - and +# bin/fm-teardown.sh's own landed-check still guarantees unlanded work is only +# ever chat-reclaimed (worktree, branch, and uncommitted changes preserved), +# never force-discarded, regardless of afk state. +STALENESS_AUTOCLOSE_SECS=${FM_STALENESS_AUTOCLOSE_SECS:-7200} +# A reclaim call that keeps failing (a broken FM_TEARDOWN_BIN, a wedged +# git/teardown dependency) must not retry silently forever: bounded attempts +# with doubling backoff, then give up until the pane's hash next changes and +# let the window fall through to ordinary stale surfacing/escalation below so +# a stuck reclaim notifies instead of looping invisibly. +STALENESS_AUTOCLOSE_MAX_RETRIES=${FM_STALENESS_AUTOCLOSE_MAX_RETRIES:-5} +STALENESS_AUTOCLOSE_RETRY_BASE_SECS=${FM_STALENESS_AUTOCLOSE_RETRY_BASE_SECS:-300} +STALENESS_AUTOCLOSE_RETRY_MAX_SECS=${FM_STALENESS_AUTOCLOSE_RETRY_MAX_SECS:-3600} +# FM_TEARDOWN_BIN lets tests stub the reclaim call, matching the +# FM_CREW_STATE_BIN seam in bin/fm-classify-lib.sh. +FM_TEARDOWN_BIN="${FM_TEARDOWN_BIN:-$SCRIPT_DIR/fm-teardown.sh}" # A busy pane is unconditional proof of liveness with no built-in duration bound, # so a hung foreground call can remain hidden even while its rendered busy # footer changes every poll. BUSY_TURN_MAX_SECS bounds how long any busy pane @@ -323,6 +347,72 @@ wedge_timer_check() { # 2h staleness auto-close (see STALENESS_AUTOCLOSE_SECS above): reclaim an +# ordinary ship task's expensive live process once its pane has shown zero +# change for that long, regardless of wedge/pause classification. Delegates the +# landed-vs-unlanded decision entirely to bin/fm-teardown.sh's +# --staleness-autoclose mode, which never deletes a worktree that has not +# landed (git-verified) and instead files it to the staleness triage store. +# Silent by design (like idle-task-discovery below): this is a deterministic, +# reversible-for-unlanded-work reclaim, not a captain-actionable wake - except +# while away (afk_present), when a successful reclaim also appends one line to +# STALE_AUTOCLOSE_AFK_LOG so bin/fm-afk-return.sh can surface it as durable +# catch-up evidence the next time the captain returns; nothing reads that log +# while away, so it never wakes anything. Returns 1 on a failed reclaim so the +# caller can drive the bounded retry/backoff below. +STALE_AUTOCLOSE_AFK_LOG="$STATE/.staleness-autoclose-afk.log" +staleness_autoclose_reclaim() { # + local win=$1 task=$2 hf=$3 idle_since idle_age + idle_since=$(stat_mtime "$hf") + idle_age=$(age_of "$hf") + if "$FM_TEARDOWN_BIN" "$task" --staleness-autoclose "${idle_since:-}" \ + >>"$STATE/.staleness-autoclose.log" 2>&1; then + triage_log "staleness auto-close reclaimed $win (task $task, idle ${idle_age}s)" + if afk_present; then + printf 'reclaimed %s (task %s, idle %ss)\n' "$win" "$task" "$idle_age" >> "$STALE_AUTOCLOSE_AFK_LOG" + fi + return 0 + fi + triage_log "staleness auto-close FAILED for $win (task $task); leaving window as-is for the next poll" + return 1 +} + +# staleness_autoclose_should_retry: 0 (attempt now) unless this key's reclaim +# has either exhausted STALENESS_AUTOCLOSE_MAX_RETRIES consecutive failures +# (permanently, until the pane's hash next changes and resets the counters via +# staleness_autoclose_clear_retries) or is still inside the doubling backoff +# window set by the last failure. +staleness_autoclose_should_retry() { # + local key=$1 fails next now + fails=$(cat "$STATE/.staleness-fails-$key" 2>/dev/null || echo 0) + case "$fails" in ''|*[!0-9]*) fails=0 ;; esac + [ "$fails" -lt "$STALENESS_AUTOCLOSE_MAX_RETRIES" ] || return 1 + next=$(cat "$STATE/.staleness-next-$key" 2>/dev/null || echo 0) + case "$next" in ''|*[!0-9]*) next=0 ;; esac + now=$(date +%s) + [ "$now" -ge "$next" ] +} + +# staleness_autoclose_record_failure: bumps this key's failure count and sets +# its next-retry time base*2^(fails-1) seconds out (capped), then echoes the +# new failure count so the caller can tell whether the retry budget is spent. +staleness_autoclose_record_failure() { # + local key=$1 fails backoff shift_by + fails=$(( $(cat "$STATE/.staleness-fails-$key" 2>/dev/null || echo 0) + 1 )) + case "$fails" in ''|*[!0-9]*) fails=1 ;; esac + echo "$fails" > "$STATE/.staleness-fails-$key" + shift_by=$(( fails - 1 )) + [ "$shift_by" -lt 16 ] || shift_by=16 + backoff=$(( STALENESS_AUTOCLOSE_RETRY_BASE_SECS * (1 << shift_by) )) + [ "$backoff" -le "$STALENESS_AUTOCLOSE_RETRY_MAX_SECS" ] || backoff=$STALENESS_AUTOCLOSE_RETRY_MAX_SECS + echo "$(( $(date +%s) + backoff ))" > "$STATE/.staleness-next-$key" + printf '%s' "$fails" +} + +staleness_autoclose_clear_retries() { # + rm -f "$STATE/.staleness-fails-$1" "$STATE/.staleness-next-$1" "$STATE/.staleness-working-$1" +} + # busy_turn_over_age: 0 iff 's latest completed-turn marker is at least # BUSY_TURN_MAX_SECS old. Ages the per-task turn-ended marker, the harness-neutral # signal every verified harness's turn-end hook touches; before any turn has @@ -972,6 +1062,7 @@ EOF ssf="$STATE/.stale-since-$key" ewf="$STATE/.wedge-escalations-$key" pf="$STATE/.paused-$key" # flag: this key's stale is using the bounded pause cadence + pwf="$STATE/.staleness-working-$key" # cache: hash last found provably-working by auto-close prev=$(cat "$hf" 2>/dev/null || true) # Busy match: a backend's native semantic state when available (herdr), else # the last 6 non-blank lines only (the TUI footer area, where every verified @@ -983,6 +1074,49 @@ EOF n=$(( $(cat "$cf" 2>/dev/null || echo 0) + 1 )) echo "$n" > "$cf" if [ "$n" -ge 2 ] && [ "$busy_now" -ne 0 ]; then + # Idle>2h staleness auto-close backstop, ahead of ordinary + # classification below: an ordinary ship task, not declared paused or + # captain-held (that wait is deliberate and may legitimately run much + # longer than two hours), not parked at a captain-relevant gate (a + # needs-decision/blocked/done/failed status means the crew is waiting + # on the captain, not idling wastefully - the stale_is_terminal path + # below exists to surface that, not have it silently reclaimed out + # from under a pending decision), not provably working (a no-mistakes + # validation legitimately sits on a static pane for its whole run per + # AGENTS.md's sparse status-reporting contract - the same predicate + # the terminal-stale path below trusts to avoid the 2026-07 herdr + # false-surface incident), and not already out of retry budget for + # this stale hash. Runs during afk too - reclaiming wasted idle + # compute matters most while away - with a successful reclaim logged + # for the returning captain by staleness_autoclose_reclaim itself. See + # staleness_autoclose_reclaim above. crew_is_provably_working is only + # invoked on the first poll of a given stale hash (cached in $pwf), + # matching the first-sighting-only contract every other caller in this + # file follows - not re-run on every ~15s poll for the whole time a + # provably-working task sits past the threshold. + if [ "$kind" = ship ] \ + && ! status_is_paused_or_captain_held "$last" \ + && ! status_is_captain_relevant "$last" \ + && [ "$(age_of "$hf")" -ge "$STALENESS_AUTOCLOSE_SECS" ] \ + && staleness_autoclose_should_retry "$key"; then + if [ "$(cat "$pwf" 2>/dev/null || true)" != "$h" ]; then + if crew_is_provably_working "$task"; then + printf '%s' "$h" > "$pwf" + else + rm -f "$pwf" + fi + fi + if [ "$(cat "$pwf" 2>/dev/null || true)" != "$h" ]; then + if staleness_autoclose_reclaim "$w" "$task" "$hf"; then + staleness_autoclose_clear_retries "$key" + continue + fi + if [ "$(staleness_autoclose_record_failure "$key")" -lt "$STALENESS_AUTOCLOSE_MAX_RETRIES" ]; then + continue + fi + triage_log "staleness auto-close exhausted retries for $w (task $task); falling through to ordinary stale surfacing" + fi + fi # The pane is idle/stale at hash $h. Triage decides whether this wakes # firstmate. Detection itself is unchanged from above. if [ "$kind" = secondmate ]; then @@ -1097,6 +1231,7 @@ EOF else printf '%s' "$h" > "$hf" echo 0 > "$cf" + staleness_autoclose_clear_retries "$key" if [ "$busy_now" -eq 0 ] && busy_turn_over_age "$task"; then wedge_timer_check "$w" "$ssf" "busy (no completed turn)" "$ewf" else diff --git a/docs/architecture.md b/docs/architecture.md index 5c66c6fe07..f4052ef152 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -12,6 +12,8 @@ A zero-token bash watcher (`bin/fm-watch.sh`) sleeps on the fleet, classifies de Actionable wakes include captain-relevant status signals, no-verb signals whose crew is not provably working, authenticated check output such as PR polling (merge or automated-reviewer submissions) or an X-mode mention, stale panes whose crew is not provably working whether their status log looks terminal or non-terminal, provably-working stale panes that persist past `FM_STALE_ESCALATE_SECS`, declared external waits that remain paused past `FM_PAUSE_RESURFACE_SECS`, and heartbeat backstop hits. Repeated provably-working stale escalations on the same unchanged pane add an escalation count to the wake reason and, at `FM_WEDGE_DEMAND_INSPECT_COUNT`, a `demand-deep-inspection` marker. A busy pane is otherwise exempt from staleness, but only until its latest `state/.turn-ended` marker reaches `FM_BUSY_TURN_MAX_SECS`, or its `state/.meta` spawn record reaches that age before any turn completes; past that bound it is routed through the same wedge escalation, with the identical reason, escalation count, and `demand-deep-inspection` marker, for inspection only - never an automatic interrupt, signal, or restart. + +Ahead of that ordinary stale classification, the watcher also runs a silent idle>2h staleness auto-close backstop for ordinary ship tasks: once a task's pane hash has sat unchanged for `FM_STALENESS_AUTOCLOSE_SECS`, it is not declared paused/captain-held or parked at a captain-relevant gate, and `crew_is_provably_working` (checked once per stale hash) says no, the watcher reclaims its live process via `bin/fm-teardown.sh --staleness-autoclose` rather than surfacing a wake. `fm-teardown.sh` alone decides landed versus unlanded from the branch: landed work gets the ordinary full teardown, and unlanded work only has its runtime endpoint reclaimed and a triage bead filed into the `staleness` federated store (`bin/fm-staleness-file.sh`), leaving the worktree, branch, and every uncommitted change untouched for later deliberate triage. A failed reclaim retries with doubling backoff up to `FM_STALENESS_AUTOCLOSE_MAX_RETRIES` times before falling through to ordinary stale surfacing for that pane hash. This backstop also runs while away; a successful reclaim during that window appends one line to `state/.staleness-autoclose-afk.log`, which `bin/fm-afk-return.sh` surfaces as durable catch-up evidence on return. Those actionable wakes are written to a durable local queue (`state/.wake-queue`) before detector state advances, so a missed process exit can be recovered by draining the queue. When a canonical validated PR poll returns `merged` or `bot-review`, the watcher appends that durable notification before publishing a private receipt bound to the poll's registration, bytes, file identities, metadata, provider, URL, and task ID. `merged` is terminal and triggers retirement; `bot-review` (GitHub only) signals a new automated-reviewer submission and keeps the poll armed for further reviews and eventual merge. diff --git a/docs/configuration.md b/docs/configuration.md index 903112b97f..d7c6abc2b5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -448,6 +448,7 @@ FM_CODEX_WATCH_CHECKPOINT=180 # seconds per foreground watcher checkpoint in C FM_CREW_STATE_NM_TIMEOUT=10 # seconds allowed per no-mistakes query inside fm-crew-state.sh FM_CREW_STATE_RUNS_LIMIT=200 # recent no-mistakes run rows scanned when axi status cannot be attributed to the current code FM_CREW_STATE_BIN=bin/fm-crew-state.sh # test override for the current-state reader used by working/paused watcher triage +FM_TEARDOWN_BIN=bin/fm-teardown.sh # test override for the reclaim call staleness auto-close makes against a stale ship task FM_NM_LIVENESS_TIMEOUT=15 # seconds allowed per no-mistakes query inside fm-no-mistakes-liveness.sh and fm-nm-run-is-live.sh FM_NM_LIVENESS_STALE=300 # seconds; a run is considered hung if its active step has not had activity for longer than this threshold FMX_PAIRING_TOKEN= # X mode pairing token; .env opt-in authorizes replies and eligible lifecycle actions @@ -481,6 +482,10 @@ FM_IDLE_DISCOVERY_INTERVAL=60 # seconds between idle-task-discovery attempts (w FM_CAPTAIN_RE='done:|needs-decision:|blocked:|failed:|PR ready|checks green|ready in branch|merged' # captain-relevant status regex; nonterminal progress verbs remain excluded even when their prose matches FM_CLASSIFY_PAUSED_VERB=paused # leading status verb for a declared external wait; excluded from FM_CAPTAIN_RE and distinct from blocked FM_STALE_ESCALATE_SECS=240 # idle seconds before a provably-working stale pane escalates; stale panes whose crew is not provably working surface immediately unless they declare the pause verb +FM_STALENESS_AUTOCLOSE_SECS=7200 # idle seconds before the watcher reclaims an ordinary ship task's live process via `fm-teardown.sh --staleness-autoclose`, ahead of ordinary stale surfacing; skipped for paused/captain-held or captain-relevant-gate status and for a pane crew_is_provably_working still finds working +FM_STALENESS_AUTOCLOSE_MAX_RETRIES=5 # consecutive failed reclaim attempts allowed for one stale pane hash before the watcher gives up until the hash next changes +FM_STALENESS_AUTOCLOSE_RETRY_BASE_SECS=300 # seconds before the first retry after a failed reclaim; doubles per additional failure +FM_STALENESS_AUTOCLOSE_RETRY_MAX_SECS=3600 # cap on the doubling reclaim-retry backoff FM_BUSY_TURN_MAX_SECS=3600 # maximum age of a busy pane's latest state/.turn-ended marker, or its state/.meta spawn record before any turn completes, before the same wedge escalation used for a provably-working non-busy stale takes over; inspection-only, never an automatic interrupt or restart FM_PAUSE_RESURFACE_SECS=3600 # seconds before an idle declared external wait re-surfaces for a recheck in the watcher or away-mode daemon FM_WEDGE_DEMAND_INSPECT_COUNT=3 # consecutive provably-working stale escalations on the same unchanged pane before demand-deep-inspection is added diff --git a/docs/scripts.md b/docs/scripts.md index 5b85ad204a..965808d2f0 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -93,6 +93,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-pr-merge.sh` | Record PR metadata, then merge a task's canonical full GitHub URL | | `fm-promote.sh` | Promote a scout task in place to a protected ship task | | `fm-teardown.sh` | Fail-closed teardown: return landed ship worktrees, require completed scout deliverables, retire secondmate homes | +| `fm-staleness-file.sh` | File a triage bead into the `staleness` federated store for a ship task reclaimed unlanded by staleness auto-close | | `fm-harness.sh` | Detect the running harness and resolve crew or secondmate harness, model, and effort | | `fm-lock.sh` | Per-home firstmate session lock | | `fm-x-lib.sh` | Shared X-mode config, relay, and reply-threading helpers | diff --git a/tests/fm-teardown.test.sh b/tests/fm-teardown.test.sh index e3d647b74d..a27e8b7bb8 100755 --- a/tests/fm-teardown.test.sh +++ b/tests/fm-teardown.test.sh @@ -185,6 +185,20 @@ SH chmod +x "$case_dir/fakebin/task" } +# Mock `staleness` (the staleness-store beads CLI that bin/fm-staleness-file.sh +# invokes): logs every invocation to $case_dir/staleness-calls.log and echoes a +# bead id, like a real `create --silent` would. Args: case_dir +add_staleness_mock() { + local case_dir=$1 + cat > "$case_dir/fakebin/staleness" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$FM_TEST_STALENESS_CALLS_LOG" +[ "${1:-}" = create ] && printf 'stale-bead-1\n' +exit 0 +SH + chmod +x "$case_dir/fakebin/staleness" +} + # Add a fork bare repo and register it as a remote on the project, then push # the worktree's task branch to it and fetch into the project so the worktree # sees the remote-tracking ref. Args: case_dir @@ -1505,6 +1519,68 @@ test_teardown_kills_pid_even_when_parlay_absent() { pass "a clean teardown still kills the recorded parlay-listen pid even when parlay is absent from PATH" } +# --- staleness auto-close (bin/fm-watch.sh idle>2h backstop) --------------- + +test_staleness_autoclose_landed_falls_through_to_full_teardown() { + local case_dir rc wt_head + case_dir=$(make_case staleness-landed) + write_meta "$case_dir" local-only ship + wt_commit "$case_dir" "merged work" + # Fast-forward local main to HEAD so work_is_landed sees it as landed, the + # same fixture test_local_only_merged_to_local_main_allows uses. + wt_head=$(git -C "$case_dir/wt" rev-parse HEAD) + git -C "$case_dir/project" update-ref refs/heads/main "$wt_head" + + set +e + run_teardown "$case_dir" --staleness-autoclose 1700000000 \ + > "$case_dir/stdout" 2> "$case_dir/stderr" + rc=$? + set -e + + expect_code 0 "$rc" "staleness-landed: teardown should succeed on landed work" + grep -q "teardown task-x1 complete" "$case_dir/stdout" \ + || fail "staleness-landed: did not fall through to the ordinary full teardown: $(cat "$case_dir/stdout")" + ! grep -q "chat reclaimed" "$case_dir/stdout" \ + || fail "staleness-landed: took the chat-only path despite landed work" + assert_absent "$case_dir/state/task-x1.meta" "staleness-landed: task metadata should be removed" + pass "staleness auto-close on already-landed work falls through to an ordinary full teardown" +} + +test_staleness_autoclose_unlanded_chat_only_preserves_worktree_and_files_bead() { + local case_dir rc + case_dir=$(make_case staleness-unlanded) + write_meta "$case_dir" local-only ship + add_staleness_mock "$case_dir" + # A real file change (not an empty commit) so its content genuinely differs + # from main - content_in_default must not mistake it for already-landed. + wt_commit_file "$case_dir" work.txt "unpushed work in progress" + # No fork, no push to origin, not merged into main: never-discard-unlanded-work + # applies, so this must reclaim the process only, not the worktree. + + set +e + FM_TEST_STALENESS_CALLS_LOG="$case_dir/staleness-calls.log" \ + run_teardown "$case_dir" --staleness-autoclose 1700000000 \ + > "$case_dir/stdout" 2> "$case_dir/stderr" + rc=$? + set -e + + expect_code 0 "$rc" "staleness-unlanded: chat-only teardown should succeed" + grep -q "chat reclaimed" "$case_dir/stdout" \ + || fail "staleness-unlanded: did not take the chat-only path: $(cat "$case_dir/stdout")" + [ -d "$case_dir/wt" ] || fail "staleness-unlanded: worktree directory was deleted" + [ -f "$case_dir/wt/work.txt" ] \ + || fail "staleness-unlanded: the unlanded file no longer exists in the worktree" + git -C "$case_dir/wt" log --oneline -1 2>/dev/null | grep -q "add work.txt" \ + || fail "staleness-unlanded: the unlanded commit no longer exists in the worktree" + assert_absent "$case_dir/state/task-x1.meta" "staleness-unlanded: task metadata should still be retired" + assert_absent "$case_dir/state/task-x1.status" "staleness-unlanded: task status should still be retired" + grep -q '^create ' "$case_dir/staleness-calls.log" 2>/dev/null \ + || fail "staleness-unlanded: no bead filed in the staleness store: $(cat "$case_dir/staleness-calls.log" 2>/dev/null)" + grep -q "fm/task-x1" "$case_dir/staleness-calls.log" 2>/dev/null \ + || fail "staleness-unlanded: filed bead did not record the branch: $(cat "$case_dir/staleness-calls.log" 2>/dev/null)" + pass "staleness auto-close on unlanded work reclaims the process only, preserves the worktree, and files a triage bead" +} + test_beads_linked_task_closes_bead_on_landed_teardown() { local case_dir rc case_dir=$(make_case beads-close-on-land) @@ -1604,3 +1680,5 @@ test_teardown_kills_pid_even_when_parlay_absent test_beads_linked_task_closes_bead_on_landed_teardown test_beads_linked_task_does_not_close_bead_on_force_teardown test_beads_linked_task_does_not_close_bead_on_refused_teardown +test_staleness_autoclose_landed_falls_through_to_full_teardown +test_staleness_autoclose_unlanded_chat_only_preserves_worktree_and_files_bead diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index c6d7928bb7..ddeddf4eb6 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -36,7 +36,8 @@ watch_bg() { # [extra env assignments...] local state=$1 fakebin=$2 out=$3 shift 3 PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ - FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$@" "$WATCH" > "$out" & + FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \ + env "$@" "$WATCH" > "$out" & } # Wait up to 0.1s ticks while stays alive; 0 if still alive, 1 if it died. @@ -352,6 +353,7 @@ test_provably_working_signal_absorbed() { [ -s "$state/.seen-task_status" ] || fail "provably-working signal did not advance its .seen-* suppressor" [ -e "$state/.last-watcher-beat" ] || fail "watcher beacon was not touched while absorbing" reap "$pid" + unset FM_FAKE_CREW_STATE pass "a no-verb signal whose crew is provably working is absorbed (no exit, no queue, suppressor advanced, beacon present)" } @@ -370,6 +372,7 @@ test_turn_ended_provably_working_absorbed() { [ ! -s "$out" ] || fail "provably-working turn-end printed a wake reason: $(cat "$out")" [ ! -s "$state/.wake-queue" ] || fail "provably-working turn-end enqueued a durable wake record" reap "$pid" + unset FM_FAKE_CREW_STATE pass "a bare turn-end whose crew is provably working (busy pane) is absorbed" } @@ -392,6 +395,7 @@ test_turn_ended_not_working_surfaced() { grep -F "signal: $state/task.turn-ended" "$out" >/dev/null || fail "watcher did not print the surfaced turn-end signal" FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the surfaced turn-end failed" grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/task.turn-ended" >/dev/null || fail "surfaced turn-end was not queued" + unset FM_FAKE_CREW_STATE pass "a bare turn-end whose crew is not provably working is surfaced (the swallowed-finish fix)" } @@ -412,6 +416,7 @@ test_working_note_not_working_surfaced() { FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the surfaced working: note failed" grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$status_file" >/dev/null || fail "surfaced working: note was not queued" [ -s "$state/.seen-task_status" ] || fail "surfaced working: note did not advance its .seen-* suppressor" + unset FM_FAKE_CREW_STATE pass "a no-verb working: note whose crew is idle with no running pipeline is surfaced" } @@ -566,6 +571,7 @@ test_nonterminal_stale_provably_working_absorbed_then_escalated() { [ ! -e "$state/.stale-since-$key" ] || fail "stale-since timer was not cleared after escalation" FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the wedge escalation failed" grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null || fail "wedge escalation was not queued" + unset FM_FAKE_CREW_STATE pass "provably-working non-terminal stale is absorbed on first sight, then wedge-escalated past the threshold" } @@ -605,6 +611,7 @@ test_nonterminal_stale_not_working_surfaced() { [ ! -e "$state/.stale-since-$key" ] || fail "stale-since timer should not be set when surfacing immediately" FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the immediate stale failed" grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null || fail "immediate stale wake was not queued" + unset FM_FAKE_CREW_STATE pass "a not-provably-working non-terminal stale is surfaced immediately (never left to wait out the timer)" } @@ -674,6 +681,7 @@ test_nonterminal_stale_paused_absorbed_then_resurfaced() { [ ! -e "$state/.stale-since-$key" ] || fail "a paused re-surface must not use the wedge timer" FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the paused re-surface failed" grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null || fail "paused re-surface was not queued" + unset FM_FAKE_CREW_STATE pass "a declared pause is absorbed on first sight, then re-surfaced as a recheck past the threshold, never wedge-escalated" } @@ -1125,6 +1133,244 @@ test_busy_pane_below_turn_age_bound_is_absorbed() { pass "a busy worker below the turn-age bound remains working with no escalation" } +# --- idle>2h staleness auto-close backstop (bin/fm-watch.sh) ---------------- +# Captain design, 2026-07-31: a ship task's pane idle (unchanged .hash-) +# past STALENESS_AUTOCLOSE_SECS is reclaimed via bin/fm-teardown.sh +# --staleness-autoclose, regardless of wedge/pause classification. FM_TEARDOWN_BIN +# (a test seam mirroring FM_CREW_STATE_BIN in bin/fm-classify-lib.sh) stubs the +# real teardown call so this test asserts the TRIGGER only - landed-vs-unlanded +# and worktree preservation are covered against the real script in +# tests/fm-teardown.test.sh. +add_fake_teardown() { # + local fakebin=$1 + cat > "$fakebin/fake-teardown" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$FM_TEST_TEARDOWN_CALLS_LOG" +exit 0 +SH + chmod +x "$fakebin/fake-teardown" +} + +test_staleness_autoclose_fires_once_idle_past_threshold() { + local dir state fakebin out capture_file window key pane_hash sig pid + dir=$(make_case staleness-autoclose-fires); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt" + window="test:fm-stale-reclaim" + add_fake_teardown "$fakebin" + printf 'idle, nothing changing' > "$capture_file" + printf 'window=%s\nkind=ship\n' "$window" > "$state/stale-reclaim.meta" + printf 'working: idle waiting\n' > "$state/stale-reclaim.status" + sig=$(seen_sig "$state/stale-reclaim.status"); printf '%s' "$sig" > "$state/.seen-stale-reclaim_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + pane_hash=$(hash_text "idle, nothing changing") + printf '%s' "$pane_hash" > "$state/.hash-$key" + # Past the (deliberately low, for the test) auto-close threshold. + set_mtime "$(( $(date +%s) - 10000 ))" "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_TEST_TEARDOWN_CALLS_LOG="$dir/teardown-calls.log" \ + watch_bg "$state" "$fakebin" "$out" \ + FM_TEARDOWN_BIN="$fakebin/fake-teardown" FM_STALENESS_AUTOCLOSE_SECS=5 + pid=$! + if ! wait_live "$pid" 30; then + reap "$pid"; fail "watcher exited on a silent staleness auto-close reclaim: $(cat "$out")" + fi + reap "$pid" + grep -qE '^stale-reclaim --staleness-autoclose [0-9]+$' "$dir/teardown-calls.log" 2>/dev/null \ + || fail "staleness auto-close did not invoke teardown with the expected args: $(cat "$dir/teardown-calls.log" 2>/dev/null)" + grep -qF "staleness auto-close reclaimed $window" "$state/.watch-triage.log" 2>/dev/null \ + || fail "staleness auto-close reclaim was not recorded in the triage log" + pass "a ship task idle past the auto-close threshold is reclaimed via bin/fm-teardown.sh --staleness-autoclose" +} + +test_staleness_autoclose_does_not_fire_below_threshold() { + local dir state fakebin out capture_file window key pane_hash sig pid + dir=$(make_case staleness-autoclose-below-threshold); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt" + window="test:fm-stale-fresh" + add_fake_teardown "$fakebin" + printf 'idle, nothing changing' > "$capture_file" + printf 'window=%s\nkind=ship\n' "$window" > "$state/stale-fresh.meta" + printf 'working: idle waiting\n' > "$state/stale-fresh.status" + sig=$(seen_sig "$state/stale-fresh.status"); printf '%s' "$sig" > "$state/.seen-stale-fresh_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + pane_hash=$(hash_text "idle, nothing changing") + printf '%s' "$pane_hash" > "$state/.hash-$key" + # Well under the auto-close threshold: idle just started. + set_mtime "$(( $(date +%s) - 2 ))" "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + export FM_FAKE_CREW_STATE='state: working · source: run-step · validating (running)' + + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_TEST_TEARDOWN_CALLS_LOG="$dir/teardown-calls.log" \ + watch_bg "$state" "$fakebin" "$out" \ + FM_TEARDOWN_BIN="$fakebin/fake-teardown" FM_STALENESS_AUTOCLOSE_SECS=7200 FM_STALE_ESCALATE_SECS=999 + pid=$! + if ! wait_live "$pid" 30; then + reap "$pid"; fail "watcher exited on an ordinary provably-working stale pane: $(cat "$out")" + fi + reap "$pid" + unset FM_FAKE_CREW_STATE + [ ! -s "$dir/teardown-calls.log" ] \ + || fail "staleness auto-close fired before the idle threshold was reached: $(cat "$dir/teardown-calls.log")" + pass "a ship task idle below the auto-close threshold is left to ordinary stale classification" +} + +test_staleness_autoclose_does_not_fire_while_provably_working() { + local dir state fakebin out capture_file window key pane_hash sig pid + dir=$(make_case staleness-autoclose-provably-working); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt" + window="test:fm-stale-validating" + add_fake_teardown "$fakebin" + printf 'idle, nothing changing' > "$capture_file" + printf 'window=%s\nkind=ship\n' "$window" > "$state/stale-validating.meta" + printf 'working: idle waiting\n' > "$state/stale-validating.status" + sig=$(seen_sig "$state/stale-validating.status"); printf '%s' "$sig" > "$state/.seen-stale-validating_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + pane_hash=$(hash_text "idle, nothing changing") + printf '%s' "$pane_hash" > "$state/.hash-$key" + # Past the auto-close threshold, exactly like the firing case above. + set_mtime "$(( $(date +%s) - 10000 ))" "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + export FM_FAKE_CREW_STATE='state: working · source: run-step · validating (running)' + + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_TEST_TEARDOWN_CALLS_LOG="$dir/teardown-calls.log" \ + watch_bg "$state" "$fakebin" "$out" \ + FM_TEARDOWN_BIN="$fakebin/fake-teardown" FM_STALENESS_AUTOCLOSE_SECS=5 FM_STALE_ESCALATE_SECS=999 + pid=$! + if ! wait_live "$pid" 30; then + reap "$pid"; fail "watcher exited while a provably-working task was correctly spared from auto-close: $(cat "$out")" + fi + reap "$pid" + unset FM_FAKE_CREW_STATE + [ ! -s "$dir/teardown-calls.log" ] \ + || fail "staleness auto-close reclaimed a provably-working task: $(cat "$dir/teardown-calls.log")" + pass "a ship task past the auto-close threshold but provably working (e.g. mid no-mistakes validation) is spared" +} + +# Captain design, 2026-08-01: a task parked at a captain-relevant gate +# (needs-decision/blocked) is waiting on the captain, not idling wastefully - +# auto-close must skip it and fall through to ordinary stale surfacing so the +# pending decision is surfaced, never silently reclaimed out from under it. +test_staleness_autoclose_does_not_fire_at_needs_decision_gate() { + local dir state fakebin out capture_file window key pane_hash sig pid + dir=$(make_case staleness-autoclose-needs-decision); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt" + window="test:fm-stale-needs-decision" + add_fake_teardown "$fakebin" + printf 'idle, nothing changing' > "$capture_file" + printf 'window=%s\nkind=ship\n' "$window" > "$state/stale-needs-decision.meta" + printf 'working: implementing\nneeds-decision: pick A or B\n' > "$state/stale-needs-decision.status" + sig=$(seen_sig "$state/stale-needs-decision.status"); printf '%s' "$sig" > "$state/.seen-stale-needs-decision_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + pane_hash=$(hash_text "idle, nothing changing") + printf '%s' "$pane_hash" > "$state/.hash-$key" + # Past the auto-close threshold, exactly like the firing case above. + set_mtime "$(( $(date +%s) - 10000 ))" "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_TEST_TEARDOWN_CALLS_LOG="$dir/teardown-calls.log" \ + watch_bg "$state" "$fakebin" "$out" \ + FM_TEARDOWN_BIN="$fakebin/fake-teardown" FM_STALENESS_AUTOCLOSE_SECS=5 FM_STALE_ESCALATE_SECS=999 + pid=$! + wait_for_exit "$pid" 40 || { reap "$pid"; fail "watcher did not surface a task parked at a needs-decision gate: $(cat "$out")"; } + grep -Fx "stale: $window" "$out" >/dev/null \ + || fail "a task parked at needs-decision was not surfaced via ordinary stale classification: $(cat "$out")" + [ ! -s "$dir/teardown-calls.log" ] \ + || fail "staleness auto-close reclaimed a task parked at a needs-decision gate: $(cat "$dir/teardown-calls.log")" + pass "a ship task past the auto-close threshold but parked at a needs-decision gate is spared and surfaced" +} + +# Captain design, 2026-08-01: reclaiming wasted idle compute matters MOST while +# nobody is watching it, so the idle>2h backstop also runs during away mode +# (state/.afk present) instead of being skipped - the reclaim is still gated by +# crew_is_provably_working and bin/fm-teardown.sh's own landed-check, and a +# successful reclaim leaves a durable line in +# state/.staleness-autoclose-afk.log for bin/fm-afk-return.sh to surface to the +# returning captain. +test_staleness_autoclose_fires_during_afk_and_logs_evidence() { + local dir state fakebin out capture_file window key pane_hash sig pid + dir=$(make_case staleness-autoclose-afk); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt" + window="test:fm-stale-afk-reclaim" + add_fake_teardown "$fakebin" + : > "$state/.afk" + printf 'idle, nothing changing' > "$capture_file" + printf 'window=%s\nkind=ship\n' "$window" > "$state/stale-afk-reclaim.meta" + printf 'working: idle waiting\n' > "$state/stale-afk-reclaim.status" + sig=$(seen_sig "$state/stale-afk-reclaim.status"); printf '%s' "$sig" > "$state/.seen-stale-afk-reclaim_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + pane_hash=$(hash_text "idle, nothing changing") + printf '%s' "$pane_hash" > "$state/.hash-$key" + set_mtime "$(( $(date +%s) - 10000 ))" "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_TEST_TEARDOWN_CALLS_LOG="$dir/teardown-calls.log" \ + watch_bg "$state" "$fakebin" "$out" \ + FM_TEARDOWN_BIN="$fakebin/fake-teardown" FM_STALENESS_AUTOCLOSE_SECS=5 + pid=$! + if ! wait_live "$pid" 30; then + reap "$pid"; fail "watcher exited on a silent staleness auto-close reclaim during afk: $(cat "$out")" + fi + reap "$pid" + grep -qE '^stale-afk-reclaim --staleness-autoclose [0-9]+$' "$dir/teardown-calls.log" 2>/dev/null \ + || fail "staleness auto-close did not reclaim during afk: $(cat "$dir/teardown-calls.log" 2>/dev/null)" + grep -qF "reclaimed $window" "$state/.staleness-autoclose-afk.log" 2>/dev/null \ + || fail "afk staleness reclaim left no durable evidence for the returning captain: $(cat "$state/.staleness-autoclose-afk.log" 2>/dev/null)" + pass "the idle>2h auto-close backstop also reclaims during afk and logs durable evidence for the returning captain" +} + +# Captain design, 2026-08-01: a persistently-failing reclaim must not retry +# forever - bounded attempts with backoff, then fall through to ordinary +# stale surfacing so a stuck reclaim notifies instead of looping invisibly. +test_staleness_autoclose_exhausts_retries_then_surfaces() { + local dir state fakebin out drain_out capture_file window key pane_hash sig pid + dir=$(make_case staleness-autoclose-retry-exhausted); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt" + window="test:fm-stale-retry-exhausted" + cat > "$fakebin/fake-teardown-failing" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$FM_TEST_TEARDOWN_CALLS_LOG" +exit 1 +SH + chmod +x "$fakebin/fake-teardown-failing" + printf 'idle, nothing changing' > "$capture_file" + printf 'window=%s\nkind=ship\n' "$window" > "$state/stale-retry-exhausted.meta" + # Non-terminal status so the fallthrough after exhausted retries takes the + # immediate non-terminal-stale surface path. + printf 'working: implementing\n' > "$state/stale-retry-exhausted.status" + sig=$(seen_sig "$state/stale-retry-exhausted.status"); printf '%s' "$sig" > "$state/.seen-stale-retry-exhausted_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + pane_hash=$(hash_text "idle, nothing changing") + printf '%s' "$pane_hash" > "$state/.hash-$key" + set_mtime "$(( $(date +%s) - 10000 ))" "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + export FM_FAKE_CREW_STATE='state: unknown · source: none · no current-state source available' + + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_TEST_TEARDOWN_CALLS_LOG="$dir/teardown-calls.log" \ + FM_TEARDOWN_BIN="$fakebin/fake-teardown-failing" FM_STALENESS_AUTOCLOSE_SECS=5 \ + FM_STALENESS_AUTOCLOSE_MAX_RETRIES=2 FM_STALENESS_AUTOCLOSE_RETRY_BASE_SECS=0 \ + FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 \ + "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || fail "watcher did not surface a stuck reclaim once its retry budget was spent" + unset FM_FAKE_CREW_STATE + [ "$(wc -l < "$dir/teardown-calls.log" 2>/dev/null || echo 0)" -eq 2 ] \ + || fail "reclaim retried a different number of times than the configured budget: $(cat "$dir/teardown-calls.log" 2>/dev/null)" + grep -Fx "stale: $window" "$out" >/dev/null \ + || fail "a reclaim that exhausted its retry budget did not fall through to ordinary stale surfacing: $(cat "$out")" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the exhausted-retry surface failed" + grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null \ + || fail "exhausted-retry stale wake was not queued" + pass "a persistently-failing reclaim retries a bounded number of times, then falls through to ordinary stale surfacing" +} + test_busy_pane_stable_hash_escalates_past_turn_age_bound() { local dir state fakebin out capture_file window key pane_hash sig pid dir=$(make_case busy-stable-hash-turn-age); state="$dir/state"; fakebin="$dir/fakebin" @@ -1419,6 +1665,7 @@ SH [ "$lines" -le 2000 ] || { reap "$pid"; fail "triage log was not capped when wc emitted a spaced byte count (lines=$lines)"; } [ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "benign signal enqueued a wake while testing log capping"; } reap "$pid" + unset FM_FAKE_CREW_STATE pass "triage log capping handles wc byte counts with leading spaces" } @@ -1490,6 +1737,7 @@ test_beacon_stays_fresh_while_absorbing() { [ "$(( now - m2 ))" -lt 10 ] || { reap "$pid"; fail "beacon went stale while absorbing (age $(( now - m2 ))s)"; } [ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "absorbing benign signals enqueued a wake"; } reap "$pid" + unset FM_FAKE_CREW_STATE pass "the liveness beacon stays fresh while the watcher absorbs benign wakes (fm-guard never false-alarms)" } @@ -1513,6 +1761,7 @@ test_afk_present_reverts_watcher_to_one_shot() { FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the afk-mode signal failed" grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$status_file" >/dev/null \ || fail "afk-mode benign signal was not queued for the daemon to classify" + unset FM_FAKE_CREW_STATE pass "with .afk present the watcher reverts to one-shot so the daemon owns triage (no double-triage)" } @@ -1592,3 +1841,9 @@ test_heartbeat_backstop_surfaces_unsurfaced_status test_beacon_stays_fresh_while_absorbing test_afk_present_reverts_watcher_to_one_shot test_afk_paused_changed_pane_hands_off_plain_stale +test_staleness_autoclose_fires_once_idle_past_threshold +test_staleness_autoclose_does_not_fire_below_threshold +test_staleness_autoclose_does_not_fire_while_provably_working +test_staleness_autoclose_does_not_fire_at_needs_decision_gate +test_staleness_autoclose_fires_during_afk_and_logs_evidence +test_staleness_autoclose_exhausts_retries_then_surfaces