From 67462069148c7c711702887e0c2797f9ae1d9c65 Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Sun, 26 Jul 2026 17:47:28 -0400 Subject: [PATCH 01/28] Automate terminal crew reaping --- AGENTS.md | 15 +- bin/fm-auto-reap.sh | 441 +++++++++++++++++++++ bin/fm-merge-local.sh | 4 + bin/fm-spawn.sh | 107 ++++- bin/fm-teardown.sh | 68 +--- bin/fm-treehouse-lib.sh | 105 +++++ bin/fm-watch.sh | 42 +- docs/architecture.md | 8 + docs/scripts.md | 6 +- tests/fixtures/herdr-lab-wrapper.sh | 32 ++ tests/fixtures/treehouse-return-fixture.sh | 45 +++ tests/fm-auto-reap-herdr-e2e.test.sh | 161 ++++++++ tests/fm-auto-reap.test.sh | 354 +++++++++++++++++ 13 files changed, 1305 insertions(+), 83 deletions(-) create mode 100755 bin/fm-auto-reap.sh create mode 100755 bin/fm-treehouse-lib.sh create mode 100755 tests/fixtures/herdr-lab-wrapper.sh create mode 100755 tests/fixtures/treehouse-return-fixture.sh create mode 100755 tests/fm-auto-reap-herdr-e2e.test.sh create mode 100755 tests/fm-auto-reap.test.sh diff --git a/AGENTS.md b/AGENTS.md index 0b6959e1e7..a5b203a314 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -478,7 +478,12 @@ A ship task's path from `done` to landed on `main` is set by the project's `mode - **no-mistakes** - the stages below as written: no-mistakes validation pipeline -> PR -> captain merge. - **direct-PR** - no pipeline. The crewmate pushes and opens the PR itself (its brief says so) and reports `done: PR `. Skip the Validate step and go straight to PR ready (run `fm-pr-check`, relay the PR). Teardown uses the normal landed-work check. -- **local-only** - no remote, no PR. The crewmate stops at `done: ready in branch fm/`. Review the diff with `bin/fm-review-diff.sh `, relay a one-paragraph summary to the captain, and on approval run `bin/fm-merge-local.sh ` to fast-forward local `main` (it refuses anything but a clean fast-forward - if it does, have the crewmate rebase). No `fm-pr-check`. Then teardown, whose safety check requires the branch already merged into local `main`, OR the work pushed to any remote (a fork counts - relevant for upstream-contribution PRs on a local-only-registered project). +- **local-only** - no remote, no PR. + The crewmate stops at `done: ready in branch fm/`. + Review the diff with `bin/fm-review-diff.sh `, relay a one-paragraph summary to the captain, and on approval run `bin/fm-merge-local.sh ` to fast-forward local `main`; it refuses anything but a clean fast-forward, and on success immediately invokes fail-closed auto-reaping. + If either the merge or automatic teardown refuses, inspect the retained state rather than weakening the safety check. + No `fm-pr-check`. + Teardown requires the branch already merged into local `main`, OR the work pushed to any remote; a fork counts, which is relevant for upstream-contribution PRs on a local-only-registered project. When reviewing any crewmate branch diff, use `bin/fm-review-diff.sh ` rather than `git diff ...branch` directly. Pooled clones keep their local default refs frozen at clone time and can lag `origin`; the helper always compares against the authoritative base. @@ -540,6 +545,11 @@ The helper defaults to `--squash`, accepts explicit merge-method flags such as ` bin/fm-teardown.sh ``` +The watcher normally invokes `bin/fm-auto-reap.sh` as soon as a terminal `done` task's recorded PR is provably merged. +The approved local-only merge helper and completed-scout signal take the same automatic path. +Auto-reaping cancels only an exactly attributed no-mistakes run, never guesses cross-branch process ownership, and then calls this ordinary teardown without `--force`. +Persistent secondmates are excluded, and X-mode-linked tasks wait for their required final follow-up. +An automatic refusal is an actionable wake and retains its metadata, worktree, and acquisition authority; after resolving the reported cause, retry with the ordinary command above. The script refuses if the worktree holds uncommitted changes or committed work that has not landed; treat a refusal as a stop-and-investigate, not an obstacle. Teardown validates that the recorded project and worktree are exact roots with the expected repository registration, quiesces every ordinary task endpoint, and then runs the final non-destructive safety checks before any Treehouse return. For a task whose metadata carries `report_required=1`, teardown also publishes the validated completion report before releasing the account lease or removing the worktree. @@ -566,7 +576,8 @@ A scout task follows Intake, Spawn, and Supervise exactly as above - scaffold th - There is no Validate or PR-ready stage. When the crewmate's status says `done`, read `data//report.md`. - Relay the findings to the captain: plain chat for a focused answer, lavish-axi when the report has structure worth a visual (multiple findings, options, a plan). -- Tear down immediately - no merge gate. For a post-cutover scout, `bin/fm-teardown.sh` requires the report's completion sections and publishes it before removing the declared scratch worktree; a missing or incomplete report refuses teardown because the findings are the work product. +- The watcher automatically tears down on the terminal `done` signal - no merge gate. + For a post-cutover scout, ordinary teardown still requires the report's completion sections and publishes it before removing the declared scratch worktree; a missing or incomplete report refuses auto-reaping because the findings are the work product. - Record it in Done with the report path instead of a PR link using `tasks-axi done` when the default tasks-axi backend is active and compatible, otherwise hand-edit `data/backlog.md` and keep Done to the 10 most recent, then re-evaluate the queue and dispatch only queued work whose blockers are gone and whose time/date gate, if any, has arrived. When the captain invokes `/reports` or asks to browse, open, search, or summarize completed work, load the `reports` skill. diff --git a/bin/fm-auto-reap.sh b/bin/fm-auto-reap.sh new file mode 100755 index 0000000000..3a15f01011 --- /dev/null +++ b/bin/fm-auto-reap.sh @@ -0,0 +1,441 @@ +#!/usr/bin/env bash +# Automatically reap terminal crewmate resources through fm-teardown.sh. +# +# `task ` validates the terminal event, +# reaps an exactly-attributed no-mistakes run when necessary, and delegates all +# endpoint, cleanliness, landing, report, and Treehouse-return proofs to ordinary +# teardown without --force. +# +# `maintenance` recovers pre-metadata Treehouse acquisitions left by a crashed +# spawn. A record is eligible only after an age threshold and exact PID/start-time +# death proof. Recovery installs fail-closed cleanup metadata, then invokes the +# same ordinary teardown proof. Every refusal stays on disk and is printed. +# Usage: fm-auto-reap.sh task +# fm-auto-reap.sh maintenance +set -eu + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" +FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" +STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" +AUTO_REAP_STALE_SECS=${FM_AUTO_REAP_STALE_SECS:-300} +AUTO_REAP_COMMAND_TIMEOUT=${FM_AUTO_REAP_COMMAND_TIMEOUT:-20} + +# shellcheck source=bin/fm-gate-refuse-lib.sh +. "$SCRIPT_DIR/fm-gate-refuse-lib.sh" +fm_refuse_if_gate_agent +# shellcheck source=bin/fm-account-routing-lib.sh +. "$SCRIPT_DIR/fm-account-routing-lib.sh" +# shellcheck source=bin/fm-checkout-lock-lib.sh +. "$SCRIPT_DIR/fm-checkout-lock-lib.sh" +# shellcheck source=bin/fm-treehouse-lib.sh +. "$SCRIPT_DIR/fm-treehouse-lib.sh" +# shellcheck source=bin/fm-process-tree-lib.sh +. "$SCRIPT_DIR/fm-process-tree-lib.sh" + +case "$AUTO_REAP_STALE_SECS:$AUTO_REAP_COMMAND_TIMEOUT" in + *[!0-9:]*|0:*|*:0) + echo "error: auto-reap age and command timeouts must be positive integers" >&2 + exit 2 + ;; +esac +[ -d "$STATE" ] && [ ! -L "$STATE" ] || { + echo "error: auto-reap state must be a real directory: $STATE" >&2 + exit 1 +} + +auto_reap_tool() { # + local variable=$1 fallback=$2 value + eval "value=\${$variable:-}" + if [ -n "$value" ]; then + [ "${FM_AUTO_REAP_TEST_HOOKS:-}" = firstmate-auto-reap-tests-v1 ] || { + echo "error: $variable requires the explicit auto-reap test hook" >&2 + return 1 + } + [ -x "$value" ] || return 1 + printf '%s\n' "$value" + return 0 + fi + command -v "$fallback" +} + +meta_value() { # + local meta=$1 key=$2 + sed -n "s/^${key}=//p" "$meta" | tail -1 +} + +single_meta_value() { # + local file=$1 key=$2 values count + values=$(sed -n "s/^${key}=//p" "$file") + count=$(printf '%s\n' "$values" | awk 'NF { n++ } END { print n + 0 }') + [ "$count" -eq 1 ] || return 1 + printf '%s\n' "$values" +} + +status_last_verb() { # + local last + last=$(grep -v '^[[:space:]]*$' "$STATE/$1.status" 2>/dev/null | tail -1) + last=${last%%:*} + last=${last%%\[key=*} + printf '%s' "$last" | tr -d '[:space:]' +} + +refuse() { + printf 'auto-reap refused %s: %s\n' "${AUTO_REAP_ID:-maintenance}" "$*" >&2 + return 1 +} + +log_result() { + local line=$1 log="$STATE/.auto-reap.log" size tmp + [ ! -L "$log" ] && { [ ! -e "$log" ] || [ -f "$log" ]; } || return 0 + printf '[%s] %s\n' "$(date '+%Y-%m-%dT%H:%M:%S%z')" "$line" >> "$log" 2>/dev/null || return 0 + size=$(wc -c < "$log" 2>/dev/null | tr -d '[:space:]') + case "$size" in ''|*[!0-9]*) return 0 ;; esac + if [ "$size" -ge 262144 ]; then + tmp=$(mktemp "$STATE/.auto-reap-log.XXXXXX") || return 0 + tail -n 2000 "$log" > "$tmp" 2>/dev/null && mv "$tmp" "$log" 2>/dev/null || true + rm -f "$tmp" 2>/dev/null || true + fi +} + +run_capture() { # [args...] + local output_variable=$1 + shift + if fm_run_bounded_capture --combine-stderr "$output_variable" "$AUTO_REAP_COMMAND_TIMEOUT" "$@"; then + RUN_CAPTURE_STATUS=0 + else + RUN_CAPTURE_STATUS=$? + fi + fm_process_tree_cleanup_verified || { + echo "error: bounded auto-reap command cleanup could not be verified" >&2 + return 1 + } + return 0 +} + +parse_pr_url() { # + local url=$1 rest + case "$url" in + https://github.com/*/*/pull/[0-9]*) ;; + *) return 1 ;; + esac + rest=${url#https://github.com/} + PR_OWNER=${rest%%/*} + rest=${rest#*/} + PR_REPO=${rest%%/*} + rest=${rest#*/pull/} + PR_NUMBER=${rest%%[/?#]*} + case "$PR_OWNER:$PR_REPO:$PR_NUMBER" in + *[!A-Za-z0-9._:/-]*|::*|*::*|*:) return 1 ;; + esac + case "$PR_NUMBER" in ''|*[!0-9]*) return 1 ;; esac +} + +pr_is_merged() { # + local meta=$1 pr gh captured + pr=$(meta_value "$meta" pr) + parse_pr_url "$pr" || { + refuse "recorded PR URL is missing or invalid" + return 1 + } + gh=$(auto_reap_tool FM_AUTO_REAP_GH_AXI_BIN gh-axi) || { + refuse "gh-axi is unavailable" + return 1 + } + run_capture captured "$gh" pr view "$PR_NUMBER" -R "$PR_OWNER/$PR_REPO" || return 1 + [ "$RUN_CAPTURE_STATUS" -eq 0 ] || { + refuse "gh-axi could not verify PR $PR_NUMBER as merged" + return 1 + } + [ "$(printf '%s\n' "$captured" | grep -Ec '^[[:space:]]*state:[[:space:]]*merged[[:space:]]*$')" -eq 1 ] || { + refuse "PR $PR_NUMBER is not provably merged" + return 1 + } +} + +reap_no_mistakes_run() { # + local meta=$1 worktree branch nm captured run_branch run_id run_status outcome runs_output row status rest row_branch + [ "$(meta_value "$meta" mode)" = no-mistakes ] || return 0 + worktree=$(meta_value "$meta" worktree) + branch=$(git -C "$worktree" symbolic-ref --quiet --short HEAD 2>/dev/null || true) + [ "$branch" = "fm/$AUTO_REAP_ID" ] || { + refuse "no-mistakes run attribution requires exact branch fm/$AUTO_REAP_ID" + return 1 + } + nm=$(auto_reap_tool FM_AUTO_REAP_NO_MISTAKES_BIN no-mistakes) || { + refuse "no-mistakes is unavailable for exact run reaping" + return 1 + } + if run_capture captured env -C "$worktree" "$nm" axi status; then :; else return 1; fi + [ "$RUN_CAPTURE_STATUS" -eq 0 ] || { + refuse "no-mistakes status could not be inspected safely" + return 1 + } + run_branch=$(printf '%s\n' "$captured" | sed -n 's/^[[:space:]]*branch:[[:space:]]*//p' | head -1) + run_branch=${run_branch#\"} + run_branch=${run_branch%\"} + if [ "$run_branch" = "$branch" ]; then + run_id=$(printf '%s\n' "$captured" | sed -n 's/^[[:space:]]*id:[[:space:]]*//p' | head -1) + run_id=${run_id#\"} + run_id=${run_id%\"} + run_status=$(printf '%s\n' "$captured" | sed -n 's/^[[:space:]]*status:[[:space:]]*//p' | head -1) + run_status=${run_status#\"} + run_status=${run_status%\"} + outcome=$(printf '%s\n' "$captured" | sed -n 's/^[[:space:]]*outcome:[[:space:]]*//p' | head -1) + outcome=${outcome#\"} + outcome=${outcome%\"} + [ -n "$run_id" ] || { + refuse "matching no-mistakes run has no exact run ID" + return 1 + } + if [ -z "$outcome" ] && [ "$run_status" != completed ] \ + && [ "$run_status" != cancelled ] && [ "$run_status" != failed ]; then + if run_capture captured "$nm" axi abort --run "$run_id"; then :; else return 1; fi + [ "$RUN_CAPTURE_STATUS" -eq 0 ] || { + refuse "failed to cancel exact no-mistakes run $run_id" + return 1 + } + log_result "cancelled no-mistakes run $run_id for $AUTO_REAP_ID" + fi + return 0 + fi + if run_capture runs_output env -C "$worktree" "$nm" runs --limit 200; then :; else return 1; fi + [ "$RUN_CAPTURE_STATUS" -eq 0 ] || { + refuse "cross-branch no-mistakes attribution could not be inspected" + return 1 + } + while IFS= read -r row; do + row=$(printf '%s' "$row" | sed 's/^[[:space:]]*//') + [ -n "$row" ] || continue + status=${row%%[[:space:]]*} + rest=${row#"$status"} + rest=$(printf '%s' "$rest" | sed 's/^[[:space:]]*//') + row_branch=${rest%%[[:space:]]*} + [ "$row_branch" = "$branch" ] || continue + case "$status" in + running) + refuse "branch still has an active no-mistakes run but its exact run ID is unavailable" + return 1 + ;; + esac + return 0 + done < + local task=$1 teardown output + teardown=$(auto_reap_tool FM_AUTO_REAP_TEARDOWN_BIN fm-teardown.sh 2>/dev/null || true) + [ -n "$teardown" ] || teardown="$SCRIPT_DIR/fm-teardown.sh" + if output=$("$teardown" "$task" 2>&1); then + [ -z "$output" ] || printf '%s\n' "$output" + log_result "auto-reaped $task" + printf 'auto-reaped %s\n' "$task" + return 0 + fi + [ -z "$output" ] || printf '%s\n' "$output" >&2 + refuse "ordinary teardown refused; retained endpoint/worktree metadata" +} + +reap_task() { # + local id=$1 trigger=$2 meta kind mode + AUTO_REAP_ID=$id + fm_account_valid_id "$id" || refuse "invalid task id" + meta="$STATE/$id.meta" + [ -f "$meta" ] && [ ! -L "$meta" ] || refuse "task metadata is unavailable" + kind=$(meta_value "$meta" kind) + [ -n "$kind" ] || kind=ship + mode=$(meta_value "$meta" mode) + [ -z "$(meta_value "$meta" x_request)" ] || refuse "X-linked tasks require their final follow-up before teardown" + [ "$kind" != secondmate ] || refuse "persistent secondmates are never auto-reaped" + [ "$(status_last_verb "$id")" = "done" ] || refuse "last task status is not terminal done" + case "$trigger:$kind:$mode" in + pr-merged:ship:local-only) refuse "local-only tasks cannot use a PR-merged trigger" ;; + pr-merged:ship:*) + [ "${AUTO_REAP_PR_VERIFIED:-0}" = 1 ] || pr_is_merged "$meta" || return 1 + ;; + scout-done:scout:*) ;; + local-merged:ship:local-only) ;; + *) refuse "trigger $trigger does not match kind=$kind mode=${mode:-no-mistakes}" ;; + esac + reap_no_mistakes_run "$meta" || return 1 + run_teardown "$id" +} + +path_age() { + local path=$1 mtime + if [ "$(uname)" = Darwin ]; then + mtime=$(stat -f %m "$path" 2>/dev/null) || return 1 + else + mtime=$(stat -c %Y "$path" 2>/dev/null) || return 1 + fi + printf '%s\n' "$(( $(date +%s) - mtime ))" +} + +recover_acquisition() { # + local record=$1 id project holder recorded_worktree worktree snapshot owner_state lock tmp find_status + [ -f "$record" ] && [ ! -L "$record" ] || return 0 + [ "$(path_age "$record")" -ge "$AUTO_REAP_STALE_SECS" ] || return 0 + AUTO_REAP_ID=${record##*/.worktree-acquire-} + AUTO_REAP_ID=${AUTO_REAP_ID%.pending} + if fm_account_lock_owner_state "$record"; then owner_state=0; else owner_state=$?; fi + case "$owner_state" in + 0) return 0 ;; + 1) ;; + 2) + refuse "stale acquisition owner liveness is indeterminate" + return 0 + ;; + *) + refuse "stale acquisition owner record is malformed" + return 0 + ;; + esac + id=$(single_meta_value "$record" id) || { + refuse "stale acquisition record has invalid task identity" + return 0 + } + if [ "$id" != "$AUTO_REAP_ID" ] || ! fm_account_valid_id "$id"; then + refuse "stale acquisition filename does not match its task identity" + return 0 + fi + project=$(single_meta_value "$record" project) || { + refuse "stale acquisition record has invalid project" + return 0 + } + holder=$(single_meta_value "$record" holder) || { + refuse "stale acquisition record has invalid lease holder" + return 0 + } + [ "$holder" = "firstmate-$id" ] || { + refuse "stale acquisition lease holder does not match task" + return 0 + } + snapshot=$(cat "$record") || return 0 + lock=$(fm_account_lifecycle_lock_acquire "$STATE" "$id") || { + refuse "could not serialize stale acquisition recovery" + return 0 + } + if [ "$(cat "$record" 2>/dev/null || true)" != "$snapshot" ]; then + fm_account_lifecycle_lock_release "$lock" >/dev/null 2>&1 || true + refuse "stale acquisition record changed during recovery" + return 0 + fi + if [ -e "$STATE/$id.meta" ] || [ -L "$STATE/$id.meta" ]; then + fm_account_lifecycle_lock_release "$lock" >/dev/null 2>&1 || true + refuse "task metadata exists; retained stale acquisition for operator reconciliation" + return 0 + fi + recorded_worktree=$(sed -n 's/^worktree=//p' "$record" | tail -1) + if [ -n "$recorded_worktree" ]; then + worktree=$(fm_checkout_trusted_dir "$recorded_worktree" 2>/dev/null || true) + [ -n "$worktree" ] && fm_treehouse_require_task_lease "$worktree" "$holder" >/dev/null 2>&1 || worktree= + else + worktree= + fi + if [ -z "$worktree" ]; then + if worktree=$(fm_treehouse_find_task_lease "$project" "$holder" 2>/dev/null); then + find_status=0 + else + find_status=$? + fi + if [ "$find_status" -eq 2 ]; then + rm -f "$record" + fm_account_lifecycle_lock_release "$lock" >/dev/null 2>&1 || true + log_result "cleared owner-dead acquisition $id after proving it owns no Treehouse lease" + printf 'auto-reap cleared %s: owner is dead and no Treehouse lease exists\n' "$id" + return 0 + fi + fi + [ -n "$worktree" ] || { + fm_account_lifecycle_lock_release "$lock" >/dev/null 2>&1 || true + refuse "exact stranded Treehouse lease is not uniquely provable" + return 0 + } + tmp=$(mktemp "$STATE/.$id.meta.auto-reap.XXXXXX") || { + fm_account_lifecycle_lock_release "$lock" >/dev/null 2>&1 || true + return 0 + } + { + printf 'window=\n' + printf 'worktree=%s\n' "$worktree" + printf 'project=%s\n' "$project" + printf 'harness=unknown\nkind=ship\n' + printf 'mode=%s\n' "$(single_meta_value "$record" mode 2>/dev/null || printf no-mistakes)" + printf 'yolo=%s\n' "$(single_meta_value "$record" yolo 2>/dev/null || printf off)" + printf 'tasktmp=\nmodel=default\neffort=default\n' + printf 'generation_id=%s\n' "$(single_meta_value "$record" generation_id 2>/dev/null || printf orphan)" + printf 'direct_spawn_endpoint=not-created\n' + printf 'direct_spawn_cleanup=pending\nrollback_pending=1\n' + } > "$tmp" + mv "$tmp" "$STATE/$id.meta" || { + rm -f "$tmp" + fm_account_lifecycle_lock_release "$lock" >/dev/null 2>&1 || true + return 0 + } + fm_account_lifecycle_lock_release "$lock" >/dev/null 2>&1 || { + refuse "failed to release recovery lifecycle lock" + return 0 + } + if run_teardown "$id"; then + rm -f "$record" + fi +} + +maintenance() { + local record meta id kind mode probe_status + for record in "$STATE"/.worktree-acquire-*.pending; do + [ -e "$record" ] || continue + recover_acquisition "$record" + done + # Backstop terminal events even when a task-specific check or turn-end signal + # was missed while no watcher was running. An ordinary open PR is expected and + # stays silent; once GitHub proves merged, every subsequent refusal is surfaced. + for meta in "$STATE"/*.meta; do + [ -f "$meta" ] && [ ! -L "$meta" ] || continue + id=${meta##*/} + id=${id%.meta} + fm_account_valid_id "$id" || continue + [ "$(status_last_verb "$id")" = "done" ] || continue + [ -z "$(meta_value "$meta" x_request)" ] || continue + kind=$(meta_value "$meta" kind) + [ -n "$kind" ] || kind=ship + mode=$(meta_value "$meta" mode) + case "$kind:$mode" in + scout:*) + reap_task "$id" scout-done + ;; + ship:local-only) + # The approved merge action calls auto-reap synchronously. Do not probe + # an unapproved local branch as if it were a terminal merge. + ;; + ship:*) + [ -n "$(meta_value "$meta" pr)" ] || continue + AUTO_REAP_ID=$id + if pr_is_merged "$meta" >/dev/null 2>&1; then + probe_status=0 + else + probe_status=$? + fi + [ "$probe_status" -eq 0 ] || continue + AUTO_REAP_PR_VERIFIED=1 reap_task "$id" pr-merged + ;; + esac + done +} + +case "${1:-}" in + task) + [ "$#" -eq 3 ] || { echo "usage: fm-auto-reap.sh task " >&2; exit 2; } + reap_task "$2" "$3" + ;; + maintenance) + [ "$#" -eq 1 ] || { echo "usage: fm-auto-reap.sh maintenance" >&2; exit 2; } + maintenance + ;; + *) + echo "usage: fm-auto-reap.sh task | maintenance" >&2 + exit 2 + ;; +esac diff --git a/bin/fm-merge-local.sh b/bin/fm-merge-local.sh index 8936f2ae28..e1b8543406 100755 --- a/bin/fm-merge-local.sh +++ b/bin/fm-merge-local.sh @@ -71,3 +71,7 @@ before=$(git -C "$PROJ" rev-parse --short "$DEFAULT") git -C "$PROJ" merge --ff-only "$BRANCH" >/dev/null after=$(git -C "$PROJ" rev-parse --short "$DEFAULT") echo "merged $BRANCH into local $DEFAULT ($before -> $after) in $PROJ" +if ! "$SCRIPT_DIR/fm-auto-reap.sh" task "$ID" local-merged; then + echo "error: local merge landed, but automatic teardown refused and retained task state for inspection" >&2 + exit 1 +fi diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index e66d231827..111eb4a065 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -975,6 +975,8 @@ T= WORKTREE_CREATED=0 WORKTREE_RETAIN_ON_ABORT=0 WORKTREE_EXPECTED_TIP= +WORKTREE_ACQUIRE_RECORD= +WORKTREE_ACQUIRE_OWNER_START= META_INSTALLED=0 META_BACKUP= EXISTING_ARTIFACT_BACKUP= @@ -1425,6 +1427,66 @@ cleanup_continuation_launch_transport() { CONTINUATION_PROMPT_CONTENT_ID= } +create_worktree_acquisition_record() { + local record tmp start + start=$(fm_account_process_start_time "$$") || { + echo "error: cannot record Treehouse acquisition owner for $ID" >&2 + return 1 + } + record="$STATE/.worktree-acquire-$ID.pending" + [ ! -e "$record" ] && [ ! -L "$record" ] || { + echo "error: stale or concurrent Treehouse acquisition record exists for $ID; let auto-reap reconcile it before retrying" >&2 + return 1 + } + tmp=$(mktemp "$STATE/.worktree-acquire-$ID.XXXXXX") || return 1 + { + printf '%s\n%s\n' "$$" "$start" + printf 'id=%s\n' "$ID" + printf 'project=%s\n' "$PROJ_ABS" + printf 'holder=firstmate-%s\n' "$ID" + printf 'kind=%s\nmode=%s\nyolo=%s\n' "$KIND" "$MODE" "$YOLO" + printf 'generation_id=%s\nworktree=\n' "$SPAWN_GENERATION_ID" + } > "$tmp" || { + rm -f "$tmp" + return 1 + } + if ! ln "$tmp" "$record" 2>/dev/null; then + rm -f "$tmp" + echo "error: could not claim Treehouse acquisition record for $ID" >&2 + return 1 + fi + rm -f "$tmp" + WORKTREE_ACQUIRE_RECORD=$record + WORKTREE_ACQUIRE_OWNER_START=$start +} + +record_acquired_worktree() { + local tmp + [ -n "$WORKTREE_ACQUIRE_RECORD" ] || return 1 + [ "$(sed -n '1p' "$WORKTREE_ACQUIRE_RECORD" 2>/dev/null)" = "$$" ] \ + && [ "$(sed -n '2p' "$WORKTREE_ACQUIRE_RECORD" 2>/dev/null)" = "$WORKTREE_ACQUIRE_OWNER_START" ] || { + echo "error: Treehouse acquisition ownership changed for $ID" >&2 + return 1 + } + tmp=$(mktemp "$STATE/.worktree-acquire-$ID.update.XXXXXX") || return 1 + if ! sed '/^worktree=/d' "$WORKTREE_ACQUIRE_RECORD" > "$tmp" \ + || ! printf 'worktree=%s\n' "$WT" >> "$tmp" \ + || ! mv "$tmp" "$WORKTREE_ACQUIRE_RECORD"; then + rm -f "$tmp" + return 1 + fi +} + +clear_worktree_acquisition_record() { + [ -n "${WORKTREE_ACQUIRE_RECORD:-}" ] || return 0 + if [ "$(sed -n '1p' "$WORKTREE_ACQUIRE_RECORD" 2>/dev/null)" = "$$" ] \ + && [ "$(sed -n '2p' "$WORKTREE_ACQUIRE_RECORD" 2>/dev/null)" = "$WORKTREE_ACQUIRE_OWNER_START" ]; then + rm -f "$WORKTREE_ACQUIRE_RECORD" + fi + WORKTREE_ACQUIRE_RECORD= + WORKTREE_ACQUIRE_OWNER_START= +} + spawn_return_created_worktree() { local return_output return_status [ "$WORKTREE_CREATED" = 1 ] || return 0 @@ -1790,6 +1852,14 @@ spawn_abort_cleanup() { fi fi [ -z "$rollback_lock" ] || fm_account_meta_lock_release "$rollback_lock" >/dev/null 2>&1 || true + if [ -n "${WORKTREE_ACQUIRE_RECORD:-}" ]; then + if [ -f "$STATE/${ID:-unknown}.meta" ] \ + || { [ "$WORKTREE_CREATED" = 1 ] && [ "$worktree_clean" = 1 ]; }; then + clear_worktree_acquisition_record + else + echo "warning: retained Treehouse acquisition record for ${ID:-unknown}; auto-reap will reconcile any lease after owner death" >&2 + fi + fi [ -z "$META_BACKUP" ] || [ -f "$META_BACKUP" ] || META_BACKUP= [ -z "$EXISTING_ARTIFACT_BACKUP" ] || [ -d "$EXISTING_ARTIFACT_BACKUP" ] || EXISTING_ARTIFACT_BACKUP= [ "${LIFECYCLE_LOCK_OWNED:-0}" != 1 ] || [ -z "${LIFECYCLE_LOCK:-}" ] || fm_account_lifecycle_lock_release "$LIFECYCLE_LOCK" >/dev/null 2>&1 || true @@ -2634,6 +2704,23 @@ else fi fi +# Per-project delivery mode + yolo flag (bin/fm-project-mode.sh; AGENTS.md +# project management and task lifecycle). Resolve it before Treehouse +# acquisition so the crash-recovery record carries exact teardown authority +# even if spawn dies before endpoint creation or metadata installation. +if [ "$DIRECT_ACCOUNT_RECOVERY" = 1 ]; then + MODE=$RECORDED_MODE + YOLO=$RECORDED_YOLO +elif [ "$KIND" = secondmate ]; then + MODE=secondmate + YOLO=off +else + PROJ_NAME=$(basename "$PROJ_ABS") + read -r MODE YOLO </dev/null) @@ -2843,6 +2930,7 @@ if [ "$KIND" != secondmate ] && [ "$BACKEND" != orca ] && [ "$RECOVERY_ACCOUNT" echo "error: refusing Treehouse acquisition because pool safety could not be inspected for $PROJ_ABS" >&2 exit 1 } + create_worktree_acquisition_record || exit 1 acquire_status=0 WT=$("$SCRIPT_DIR/fm-checkout-refresh.sh" acquire-worktree "$PROJ_ABS" "firstmate-$ID") || acquire_status=$? if [ "$acquire_status" -ne 0 ]; then @@ -2859,6 +2947,7 @@ if [ "$KIND" != secondmate ] && [ "$BACKEND" != orca ] && [ "$RECOVERY_ACCOUNT" } WORKTREE_CREATED=1 WORKTREE_RETAIN_ON_ABORT=1 + record_acquired_worktree || exit 1 validate_spawn_worktree "treehouse get --lease" "$PROJ_ABS" freshness_status=0 "$SCRIPT_DIR/fm-checkout-refresh.sh" verify-worktree "$WT" "$PROJ_ABS" || freshness_status=$? @@ -3103,23 +3192,6 @@ EOF esac fi -# Per-project delivery mode + yolo flag (bin/fm-project-mode.sh; AGENTS.md project management and task lifecycle). -# Recorded in meta so fm-teardown's safety check and the validate/merge stages can -# branch on them. Mode governs ship tasks; a scout's deliverable is a report, not a -# merge, so scout teardown ignores mode. -if [ "$DIRECT_ACCOUNT_RECOVERY" = 1 ]; then - MODE=$RECORDED_MODE - YOLO=$RECORDED_YOLO -elif [ "$KIND" = secondmate ]; then - MODE=secondmate - YOLO=off -else - PROJ_NAME=$(basename "$PROJ_ABS") - read -r MODE YOLO <&2; exit 1; } mv "$META_TMP" "$STATE/$ID.meta" META_INSTALLED=1 +clear_worktree_acquisition_record [ -z "$META_WRITE_LOCK" ] || fm_account_meta_lock_release "$META_WRITE_LOCK" META_WRITE_LOCK= [ "$BACKEND" = orca ] && ORCA_ABORT_CLEANUP=0 diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index ea03eda10b..9c50cf00c2 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -114,6 +114,8 @@ CHECKOUT_LOCK_ROOT=$(fm_checkout_lock_root "$CHECKOUT_STATE_BASE") fm_refuse_if_gate_agent # shellcheck source=bin/fm-account-routing-lib.sh . "$SCRIPT_DIR/fm-account-routing-lib.sh" +# shellcheck source=bin/fm-treehouse-lib.sh +. "$SCRIPT_DIR/fm-treehouse-lib.sh" FM_LOCK_LOG_PREFIX=teardown "$FM_ROOT/bin/fm-guard.sh" || true TEARDOWN_UPSTREAM_TIMEOUT=${FM_CHECKOUT_REFRESH_PROBE_TIMEOUT:-15} @@ -978,62 +980,6 @@ exact_git_worktree_root() { printf '%s\n' "$canonical" } -treehouse_state_for_worktree() { - local worktree=$1 slot pool state - slot=$(canonical_existing_dir "$(dirname "$worktree")") || return 1 - pool=$(canonical_existing_dir "$(dirname "$slot")") || return 1 - state="$pool/treehouse-state.json" - [ -f "$state" ] && [ ! -L "$state" ] || return 1 - printf '%s\n' "$state" -} - -require_treehouse_task_lease() { - local worktree=$1 expected_holder=$2 state - state=$(treehouse_state_for_worktree "$worktree") || { - echo "error: cannot resolve authoritative Treehouse state for $worktree" >&2 - return 1 - } - python3 - "$state" "$worktree" "$expected_holder" <<'PY' -import json -import os -import sys - -state_path, expected_path, expected_holder = sys.argv[1:] -try: - with open(state_path, encoding="utf-8") as stream: - state = json.load(stream) - worktrees = state["worktrees"] - if not isinstance(worktrees, list): - raise TypeError("worktrees must be an array") - matches = [] - for entry in worktrees: - if not isinstance(entry, dict): - continue - path = entry.get("path") - if not isinstance(path, str) or not path: - continue - if os.path.realpath(path) == expected_path: - matches.append(entry) - if len(matches) != 1: - raise ValueError("expected exactly one matching worktree entry") - entry = matches[0] - if entry.get("leased") is not True: - raise ValueError("worktree is not durably leased") - if entry.get("lease_holder") != expected_holder: - raise ValueError( - f"lease holder is {entry.get('lease_holder')!r}, expected {expected_holder!r}" - ) - if entry.get("destroying") is True: - raise ValueError("worktree is already being destroyed") -except (OSError, ValueError, TypeError, KeyError, json.JSONDecodeError) as error: - print( - f"error: Treehouse ownership for {expected_path} is unprovable: {error}", - file=sys.stderr, - ) - raise SystemExit(1) -PY -} - require_treehouse_return_authority() { local worktree=$1 project=$2 worktree_root project_root worktree_common project_common worktree_root=$(exact_git_worktree_root "$worktree") || return 1 @@ -1048,7 +994,7 @@ require_treehouse_return_authority() { echo "error: Treehouse return target $worktree_root is not registered to $project_root" >&2 return 1 } - require_treehouse_task_lease "$worktree_root" "$3" + fm_treehouse_require_task_lease "$worktree_root" "$3" } validate_teardown_target_identity() { @@ -1083,7 +1029,7 @@ validate_teardown_target_identity() { echo "error: teardown worktree is not registered to the recorded project: $worktree_root" >&2 return 1 } - require_treehouse_task_lease "$worktree_root" "firstmate-$ID" + fm_treehouse_require_task_lease "$worktree_root" "firstmate-$ID" } retry_wait_secs_is_valid() { @@ -3432,7 +3378,7 @@ validate_firstmate_home_for_removal() { "secondmate top-level source repository" "$abs_home_path" || return 1 fi if [ -n "$expected_id" ] && firstmate_home_has_treehouse_slot "$abs_home_path" "$expected_source"; then - require_treehouse_task_lease "$abs_home_path" "$expected_id" || return 1 + fm_treehouse_require_task_lease "$abs_home_path" "$expected_id" || return 1 fi validate_secondmate_home_landed_state "$abs_home_path" "$expected_source" || return 1 if [ -n "$expected_id" ]; then @@ -3567,7 +3513,7 @@ validate_firstmate_home_children_removal() { elif [ -n "$child_wt" ] && [ -d "$child_wt" ]; then child_proj=$(meta_value "$child_meta" project) validate_child_worktree_for_removal "$child_wt" "$child_proj" >/dev/null || return 1 - require_treehouse_task_lease "$(canonical_existing_dir "$child_wt")" "firstmate-$child_id" || return 1 + fm_treehouse_require_task_lease "$(canonical_existing_dir "$child_wt")" "firstmate-$child_id" || return 1 validate_child_worktree_landed_state "$child_meta" "$child_id" "$child_wt" "$child_proj" || return 1 else echo "error: retained child metadata for $child_id because its Treehouse worktree is missing or uninspectable" >&2 @@ -3659,7 +3605,7 @@ cleanup_firstmate_home_children() { return 1 fi validate_child_worktree_for_removal "$child_wt" "$child_proj" >/dev/null || return 1 - require_treehouse_task_lease "$(canonical_existing_dir "$child_wt")" "firstmate-$child_id" || return 1 + fm_treehouse_require_task_lease "$(canonical_existing_dir "$child_wt")" "firstmate-$child_id" || return 1 fi if managed_account_meta "$child_meta"; then child_endpoint_home=$(fm_backend_endpoint_home "$child_backend" "$child_kind" "$home" "$child_home") diff --git a/bin/fm-treehouse-lib.sh b/bin/fm-treehouse-lib.sh new file mode 100755 index 0000000000..71dca3330c --- /dev/null +++ b/bin/fm-treehouse-lib.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Shared Treehouse lease authority helpers. +# +# Consumers may prove that one exact worktree is durably leased to one exact +# holder, or resolve the unique leased worktree for an exact project and holder. +# The latter is intentionally narrow: it walks only Git's registered worktrees +# for the trusted project and refuses zero or multiple matches. + +fm_treehouse_state_for_worktree() { # + local worktree=$1 slot pool state + worktree=$(fm_checkout_trusted_dir "$worktree") || return 1 + slot=$(fm_checkout_trusted_dir "$(dirname "$worktree")") || return 1 + pool=$(fm_checkout_trusted_dir "$(dirname "$slot")") || return 1 + state="$pool/treehouse-state.json" + [ -f "$state" ] && [ ! -L "$state" ] || return 1 + printf '%s\n' "$state" +} + +fm_treehouse_require_task_lease() { # + local worktree=$1 expected_holder=$2 state + worktree=$(fm_checkout_trusted_dir "$worktree") || { + echo "error: Treehouse worktree is unavailable or redirected: $1" >&2 + return 1 + } + state=$(fm_treehouse_state_for_worktree "$worktree") || { + echo "error: cannot resolve authoritative Treehouse state for $worktree" >&2 + return 1 + } + python3 - "$state" "$worktree" "$expected_holder" <<'PY' +import json +import os +import sys + +state_path, expected_path, expected_holder = sys.argv[1:] +try: + with open(state_path, encoding="utf-8") as stream: + state = json.load(stream) + worktrees = state["worktrees"] + if not isinstance(worktrees, list): + raise TypeError("worktrees must be an array") + matches = [] + for entry in worktrees: + if not isinstance(entry, dict): + continue + path = entry.get("path") + if not isinstance(path, str) or not path: + continue + if os.path.realpath(path) == expected_path: + matches.append(entry) + if len(matches) != 1: + raise ValueError("expected exactly one matching worktree entry") + entry = matches[0] + if entry.get("leased") is not True: + raise ValueError("worktree is not durably leased") + if entry.get("lease_holder") != expected_holder: + raise ValueError( + f"lease holder is {entry.get('lease_holder')!r}, expected {expected_holder!r}" + ) + if entry.get("destroying") is True: + raise ValueError("worktree is already being destroyed") +except (OSError, ValueError, TypeError, KeyError, json.JSONDecodeError) as error: + print( + f"error: Treehouse ownership for {expected_path} is unprovable: {error}", + file=sys.stderr, + ) + raise SystemExit(1) +PY +} + +fm_treehouse_find_task_lease() { # + local project=$1 expected_holder=$2 listed line candidate common project_common + local matches=0 match= + project=$(fm_checkout_trusted_dir "$project") || return 1 + [ "$(git -C "$project" rev-parse --show-toplevel 2>/dev/null)" = "$project" ] || return 1 + project_common=$(fm_checkout_git_common_dir "$project") || return 1 + listed=$(git -C "$project" -c core.quotePath=false worktree list --porcelain 2>/dev/null) || return 1 + while IFS= read -r line; do + case "$line" in + worktree\ *) + candidate=$(fm_checkout_trusted_dir "${line#worktree }" 2>/dev/null || true) + [ -n "$candidate" ] || continue + [ "$candidate" != "$project" ] || continue + common=$(fm_checkout_git_common_dir "$candidate" 2>/dev/null || true) + [ "$common" = "$project_common" ] || continue + if fm_treehouse_require_task_lease "$candidate" "$expected_holder" >/dev/null 2>&1; then + matches=$((matches + 1)) + match=$candidate + fi + ;; + esac + done <&2 + return 2 + ;; + *) + echo "error: multiple Treehouse leases exist for $expected_holder in $project" >&2 + return 3 + ;; + esac +} diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 9b4495edb6..1b41bb4cc8 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -37,6 +37,10 @@ # FM_PERMISSION_STALL_ESCALATE_SECS as a possible macOS # permission/system-dialog block. Unless afk is active. # check: