diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 652e768c25..87a130647d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,9 +27,10 @@ jobs: tests: name: Behavior tests runs-on: ubuntu-latest - # The pre-cutover suite reached 25m35s, and the exhaustive Bridge crash - # matrix adds bounded work. Keep enough margin without masking a real hang. - timeout-minutes: 45 + # The serial suite now includes the bounded direct-account recovery matrix + # as well as the exhaustive Bridge crash matrix. Keep enough margin for both + # without masking a real hang. + timeout-minutes: 60 steps: - uses: actions/checkout@v6 with: diff --git a/AGENTS.md b/AGENTS.md index 0b6959e1e7..23ecac170a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -108,6 +108,7 @@ state/ volatile runtime signals; gitignored .meta written by fm-spawn: window=, worktree=, project=, harness=, model=, effort=, kind=, mode=, yolo=, tasktmp=, generation_id=, report_required= Direct ship and scout launches also own account_home=, worktree_git_dir=, worktree_git_dir_identity=, and exactly one authoritative final-state field: worktree_git_ref= for an attached branch or worktree_git_head= for an intentional detached HEAD. Their metadata may temporarily carry worktree_git_setup_ref= and worktree_git_setup_head= while the brief's required `fm/` branch transition is pending; recovery accepts only that exact setup state or the authoritative `fm/` ref and removes the setup fields after adoption. Direct recovery validates the canonical worktree path, exact physical Git-dir identity, and authoritative final state before account preparation and again immediately before endpoint creation; any drift fails closed without launching. + Automatic account-scoped capacity rescue appends capacity_rescue_attempts=, repeated capacity_rescue_exhausted_account= entries, per-attempt audit/result fields, and an optional capacity_rescue_stopped= terminal reason; Codex model capacity writes none of them, and docs/account-capacity-rescue.md owns the full routing, transaction, and retry boundary. If endpoint removal after a failed new direct spawn cannot be confirmed, direct_spawn_cleanup=pending and rollback_pending=1 retain the endpoint and worktree identity for explicit teardown. A failed new direct spawn that never created an endpoint records direct_spawn_endpoint=not-created and an empty window= so teardown skips endpoint quiescence without skipping worktree safety. Secondmate Agent Fleet routing and legacy managed recovery own account_pool=, account_profile=, account_task=, account_attempt=, and provider_session_id= (docs/configuration.md "Agent Fleet account routing"). @@ -585,6 +586,8 @@ The emitted block is the only per-harness operating recipe in the session contex Do not substitute another harness's command shape for it. **Always-on wake triage (absorb only when provably working).** `bin/fm-watch.sh` classifies every wake in bash and absorbs the benign majority without waking you: crewmates with positive working evidence (an actively-running no-mistakes step for their branch, or a busy pane read via `bin/fm-crew-state.sh`) unless the separate permission-stall no-progress threshold has expired, a declared `paused:` external wait until its bounded recheck cadence, and no-change heartbeats. +Before ordinary stale classification, it recognizes empirically verified provider-capacity chrome only on an idle endpoint: account-scoped Claude exhaustion enters guarded direct-account recovery, while Codex model capacity surfaces once without endpoint removal, account rotation, or rescue metadata. +Account rescue is lifecycle-serialized, attempt-capped, and durably stopped with one blocked wake when no unused eligible account remains; `docs/account-capacity-rescue.md` owns the signatures, routing, audit fields, and failure contract. It never absorbs a crewmate that stopped without that evidence - whatever its stale status log claims - and only an actionable wake is queued durably and ends the supervision wait, so you resume the emitted protocol exactly once per actionable event. A `paused:` status is a deliberate external wait, not `blocked:`; its initial signal still surfaces once, and a forgotten pause re-surfaces for a recheck once per window. Repeated unchanged wedge or permission-stall escalations eventually add `demand-deep-inspection` to the wake reason so they are not mistaken for another routine validation wait. diff --git a/bin/backends/herdr.sh b/bin/backends/herdr.sh index 238669a7e8..693f8fb250 100644 --- a/bin/backends/herdr.sh +++ b/bin/backends/herdr.sh @@ -2654,20 +2654,20 @@ fm_backend_herdr_expected_label_matches() { # [expected-label] # # Reading a pane or sending keys to it does not care how the server was # launched - the pane already exists and the server is up. It only needs the -# server to be running and adapter-owned (this HOME's own certificate names the -# live pid), which fm_backend_herdr_server_adapter_owned proves without the -# closed-shell launch certification. This is what keeps peek/steer working when -# FirstMate itself runs INSIDE the herdr session it manages (HERDR_ENV=1): that -# server was not launched through the crewmate adapter's own closed-shell path, so -# closed_shell_environment_ready is false and the full ensure would try to -# restart+recertify - impossible while the session is occupied by live crewmates and -# FirstMate itself. The SPAWN path deliberately keeps the strict ensure so a new -# crewmate still launches only in a certified closed-shell server. +# server to be running. During the legacy certificate lifecycle it must also be +# adapter-owned (this HOME's certificate names the live pid), which +# fm_backend_herdr_server_adapter_owned proves without requiring the current +# release's closed-shell launch certification. Native-agent production no longer +# requires that legacy server certificate; pane isolation is applied directly by +# `agent start --env`, so read/steer follows server ensure's native cutover and +# accepts a reachable server. Certificate-enforcing tests retain the stricter +# ownership proof. fm_backend_herdr_server_reachable_for_readsteer() { # local session=$1 running running=$(fm_backend_herdr_cli "$session" status --json 2>/dev/null \ | fm_backend_herdr_control_jq -r '.server.running // false' 2>/dev/null) [ "$running" = true ] || return 1 + fm_backend_herdr_server_certificate_required || return 0 fm_backend_herdr_server_adapter_owned "$session" } @@ -3013,9 +3013,10 @@ fm_backend_herdr_send_text_submit() { # # tmux-kill-window's `|| true` contract). Verified: closing a tab's only pane # closes the tab too, so a separate tab close is unnecessary. fm_backend_herdr_kill() { # [backend-id] [expected-label] - if ! fm_backend_herdr_target_ready "$1" "${3:-}"; then - [ -z "${3:-}" ] && return 0 - return 1 + if [ -n "${3:-}" ]; then + fm_backend_herdr_target_ready "$1" "$3" || return 1 + else + fm_backend_herdr_parse_target "$1" || return 0 fi fm_backend_herdr_cli "$FM_BACKEND_HERDR_SESSION" pane close "$FM_BACKEND_HERDR_PANE" >/dev/null 2>&1 || true } diff --git a/bin/backends/orca.sh b/bin/backends/orca.sh index 7f6bc1b1ee..4f7766ad0f 100644 --- a/bin/backends/orca.sh +++ b/bin/backends/orca.sh @@ -58,7 +58,7 @@ fm_backend_orca_authority_capabilities_check() { return 1 } -fm_backend_orca_json_get() { # ; fields: worktree-id worktree-path terminal-handle terminal-title worktree-terminal-handle repo-id +fm_backend_orca_json_get() { # ; fields: worktree-id worktree-path worktree-name terminal-handle terminal-title worktree-terminal-handle repo-id # Terminal handles are accepted only from verified terminal result shapes: # result.terminal or a root terminal object with .handle. Undocumented # result.id and result.worktree.terminal shapes are ignored until a real Orca @@ -88,6 +88,7 @@ function handle(obj) { let v = ""; if (field === "worktree-id") v = wt.id || wt.worktreeId || r.worktreeId || ""; if (field === "worktree-path") v = wt.path || (wt.git && wt.git.path) || r.path || ""; +if (field === "worktree-name") v = scalar(wt.name) || scalar(wt.title) || scalar(r.worktreeName) || ""; if (field === "terminal-handle") v = handle(explicitTerm || r) || ""; if (field === "terminal-title") v = scalar((explicitTerm || r).title) || scalar((explicitTerm || r).name) || ""; if (field === "worktree-terminal-handle") v = handle(explicitTerm) || ""; @@ -141,7 +142,7 @@ fm_backend_orca_repo_ensure() { # } fm_backend_orca_worktree_create() { # - local project=$1 name=$2 repo_id out wt_id wt_path terminal status proof + local project=$1 name=$2 repo_id out wt_id wt_path wt_name terminal status proof repo_id=$(fm_backend_orca_repo_ensure "$project") || return 1 if out=$(orca worktree create --repo "id:$repo_id" --name "$name" --no-parent --setup skip --json); then status=0 @@ -151,10 +152,11 @@ fm_backend_orca_worktree_create() { # wt_id=$(printf '%s' "$out" | fm_backend_orca_json_get worktree-id 2>/dev/null || true) terminal=$(printf '%s' "$out" | fm_backend_orca_json_get worktree-terminal-handle 2>/dev/null || true) wt_path=$(printf '%s' "$out" | fm_backend_orca_json_get worktree-path 2>/dev/null || true) + wt_name=$(printf '%s' "$out" | fm_backend_orca_json_get worktree-name 2>/dev/null || true) proof=unproven [ -z "$terminal" ] || proof=recorded - printf '%s\t%s\t%s\t%s\t%s' "$wt_id" "$wt_path" "$terminal" "$proof" "$repo_id" - if [ "$status" -ne 0 ] || [ -z "$wt_id" ] || [ -z "$wt_path" ]; then + printf '%s\t%s\t%s\t%s\t%s\t%s' "$wt_id" "$wt_path" "$terminal" "$proof" "$repo_id" "$wt_name" + if [ "$status" -ne 0 ] || [ -z "$wt_id" ] || [ -z "$wt_path" ] || [ -z "$wt_name" ]; then echo "error: orca worktree create returned incomplete or unsuccessful authority for $name" >&2 return 2 fi diff --git a/bin/fm-account-directory.sh b/bin/fm-account-directory.sh index 3ba66b5462..209fce2b60 100755 --- a/bin/fm-account-directory.sh +++ b/bin/fm-account-directory.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash # Select and prepare direct Claude or Codex account-directory launches. # Usage: -# fm-account-directory.sh select +# fm-account-directory.sh select [excluded-account-home...] # fm-account-directory.sh install-herdr-hook -# fm-account-directory.sh prepare +# fm-account-directory.sh prepare [excluded-account-home...] # # This header is the single owner of the direct account-directory contract. # FM_ACCOUNT_DIRECTORY_CUTOVER: direct-observe-passwd-home-v2 @@ -12,13 +12,15 @@ # Codex selection removes that account's quota-axi window cache immediately # before every read, sets CODEX_HOME plus the account-isolated XDG_CACHE_HOME, # accepts only a fresh result with at least one numeric five_hour or weekly -# window, and picks the account with the highest minimum remaining percentage. +# window, skips excluded and zero-capacity accounts, and picks the remaining +# account with the highest minimum remaining percentage. # A Codex account with no such freshly readable window is skipped as unhealthy. # Claude quota is not currently distinguishable per config directory because # quota-axi cannot non-interactively resolve Claude's config-dir-specific macOS # Keychain credential. # Claude therefore never treats a missing usage window as account failure and -# selects the first real account directory in stable bytewise sort order. +# selects the first non-excluded real account directory in stable bytewise sort +# order. # Selection prints only the chosen absolute account home on stdout and logs # health, fallback, and choice diagnostics on stderr. # prepare selects the account and idempotently runs Herdr's own integration @@ -203,8 +205,18 @@ valid_account_home() { # esac } -first_account_home() { # +account_home_is_excluded() { # [excluded-account-home...] + local candidate=$1 excluded + shift + for excluded in "$@"; do + [ "$candidate" != "$excluded" ] || return 0 + done + return 1 +} + +first_account_home() { # [excluded-account-home...] local vendor=$1 root vendor_dir candidate + shift root=$(account_root) || return 1 vendor_dir=$root/$vendor [ -d "$vendor_dir" ] && [ ! -L "$vendor_dir" ] || { @@ -215,10 +227,14 @@ first_account_home() { # export LC_ALL for candidate in "$vendor_dir"/*; do valid_account_home "$vendor_dir" "$candidate" || continue + if account_home_is_excluded "$candidate" "$@"; then + log "$vendor account $candidate skipped: exhausted by this task" + continue + fi printf '%s\n' "$candidate" return 0 done - echo "error: no account directories found for $vendor under $vendor_dir" >&2 + echo "CAPACITY_UNAVAILABLE: no unused $vendor account directories remain under $vendor_dir" >&2 return 1 } @@ -279,7 +295,7 @@ $1 EOF } -select_codex() { +select_codex() { # [excluded-account-home...] local root vendor_dir quota_bin candidate usage score local best_home='' best_score='' root=$(account_root) || return 1 @@ -297,12 +313,20 @@ select_codex() { export LC_ALL for candidate in "$vendor_dir"/*; do valid_account_home "$vendor_dir" "$candidate" || continue + if account_home_is_excluded "$candidate" "$@"; then + log "codex account $candidate skipped: exhausted by this task" + continue + fi usage=$(fresh_codex_usage_json "$candidate" "$quota_bin") || usage= score=$(codex_score "$usage") || score= if [ -z "$score" ]; then log "codex account $candidate skipped: no freshly readable usage window" continue fi + if ! awk -v candidate_score="$score" 'BEGIN { exit !(candidate_score > 0) }'; then + log "codex account $candidate skipped: fresh remaining score=$score has no capacity" + continue + fi log "codex account $candidate fresh remaining score=$score" if [ -z "$best_home" ] || awk -v candidate_score="$score" -v current_score="$best_score" \ 'BEGIN { exit !(candidate_score > current_score) }'; then @@ -311,26 +335,28 @@ select_codex() { fi done [ -n "$best_home" ] || { - echo "error: no healthy Codex account has a freshly readable usage window" >&2 + echo "CAPACITY_UNAVAILABLE: no unused Codex account has a freshly readable positive usage window" >&2 return 1 } log "selected codex account $best_home with fresh remaining score=$best_score" printf '%s\n' "$best_home" } -select_claude() { +select_claude() { # [excluded-account-home...] local selected - selected=$(first_account_home claude) || return 1 + selected=$(first_account_home claude "$@") || return 1 log "CLAUDE USAGE UNREADABLE: quota-axi cannot non-interactively resolve Claude's config-dir-specific macOS Keychain credential today; selecting the first account directory by stable sort: $selected" printf '%s\n' "$selected" } -select_account() { # - case "$1" in - codex) select_codex ;; - claude) select_claude ;; +select_account() { # [excluded-account-home...] + local vendor=$1 + shift + case "$vendor" in + codex) select_codex "$@" ;; + claude) select_claude "$@" ;; *) - echo "error: direct account-directory selection supports only claude or codex, not '$1'" >&2 + echo "error: direct account-directory selection supports only claude or codex, not '$vendor'" >&2 return 1 ;; esac @@ -378,17 +404,21 @@ case "${1:-}" in exit 0 ;; select) - [ "$#" -eq 2 ] || { usage; exit 2; } - select_account "$2" + [ "$#" -ge 2 ] || { usage; exit 2; } + vendor=$2 + shift 2 + select_account "$vendor" "$@" ;; install-herdr-hook) [ "$#" -eq 3 ] || { usage; exit 2; } install_herdr_hook "$2" "$3" ;; prepare) - [ "$#" -eq 2 ] || { usage; exit 2; } - selected_home=$(select_account "$2") || exit 1 - install_herdr_hook "$2" "$selected_home" || exit 1 + [ "$#" -ge 2 ] || { usage; exit 2; } + vendor=$2 + shift 2 + selected_home=$(select_account "$vendor" "$@") || exit 1 + install_herdr_hook "$vendor" "$selected_home" || exit 1 printf '%s\n' "$selected_home" ;; *) diff --git a/bin/fm-checkout-refresh.sh b/bin/fm-checkout-refresh.sh index 91af403fb6..acfc40e7f4 100755 --- a/bin/fm-checkout-refresh.sh +++ b/bin/fm-checkout-refresh.sh @@ -1869,7 +1869,7 @@ record_alert() { record_reinspection_failure() { local checkout=$1 key alert output - key=$(checkout_key "$checkout") || { + key=$(fm_checkout_hash_value "$checkout" 24) || { printf '%s: skipped: covered checkout lock identity cannot be resolved\n' "$checkout" return 1 } diff --git a/bin/fm-classify-lib.sh b/bin/fm-classify-lib.sh index b3465c4a04..1b0412f892 100755 --- a/bin/fm-classify-lib.sh +++ b/bin/fm-classify-lib.sh @@ -64,6 +64,59 @@ FM_PAUSE_RESURFACE_SECS_DEFAULT=3600 # this is the one owner of the verb literal, overridable via FM_CLASSIFY_RESOLVE_VERB. FM_CLASSIFY_RESOLVE_VERB_DEFAULT='resolved' +# Classify a provider-owned capacity failure from the bounded pane capture of a +# known harness. Prints a stable failure token and returns 0 on a verified shape; +# prints nothing and returns 1 otherwise. +# +# These match complete TUI chrome, not capacity-like prose. Codex requires its +# standalone warning glyph and exact line. Claude requires either its tool-result +# glyph plus exact session-limit sentence or the complete usage-limit choice +# dialog. This prevents a task discussing "usage limits", quoting one sentence, +# or printing ordinary API rate-limit output from being mistaken for an exhausted +# harness account. Empirical captures and versions are recorded in +# docs/account-capacity-rescue.md. +provider_capacity_failure_kind() { # + local harness=$1 tail40=$2 prompt_tail + prompt_tail=$(printf '%s\n' "$tail40" | tail -18) + case "$harness" in + codex) + if printf '%s\n' "$prompt_tail" \ + | grep -Eq '^[[:space:]]*⚠[[:space:]]+Selected model is at capacity\. Please try a different model\.[[:space:]]*$'; then + printf 'codex-model-capacity' + return 0 + fi + ;; + claude) + if printf '%s\n' "$prompt_tail" \ + | grep -Eq "^[[:space:]]*⎿.*You've hit your session limit · resets .+[[:space:]]*$"; then + printf 'claude-session-limit' + return 0 + fi + if printf '%s\n' "$prompt_tail" | grep -Fq 'What do you want to do?' \ + && printf '%s\n' "$prompt_tail" | grep -Eq '1\. Stop and wait for limit to reset[[:space:]]*$' \ + && printf '%s\n' "$prompt_tail" | grep -Eq '2\. Switch to usage credits[[:space:]]*$' \ + && printf '%s\n' "$prompt_tail" | grep -Eq '3\. Switch to Team plan[[:space:]]*$' \ + && printf '%s\n' "$prompt_tail" | grep -Eq 'Enter to confirm.*Esc to cancel'; then + printf 'claude-usage-limit-dialog' + return 0 + fi + ;; + esac + return 1 +} + +provider_credential_failure_kind() { # + local harness=$1 tail40=$2 prompt_tail + [ "$harness" = claude ] || return 1 + prompt_tail=$(printf '%s\n' "$tail40" | tail -18) + if printf '%s\n' "$prompt_tail" \ + | grep -Eq '^[[:space:]]*Not logged in - Please run /login[[:space:]]*$'; then + printf 'claude-not-logged-in' + return 0 + fi + return 1 +} + # Return the last non-blank line of a status file (empty if missing/blank). last_status_line() { local f=$1 diff --git a/bin/fm-home-seed.sh b/bin/fm-home-seed.sh index eac1c90072..8dbe270f8a 100755 --- a/bin/fm-home-seed.sh +++ b/bin/fm-home-seed.sh @@ -929,6 +929,33 @@ seed_home() { SEED_CHARTER_EXISTED=0 SEED_MARKER_EXISTED=0 trap seed_exit_cleanup EXIT + if [ ! -f "$SEED_PARENT_BRIEF" ]; then + [ -n "${FM_SECONDMATE_CHARTER:-}" ] || { + echo "error: no filled secondmate charter brief at $SEED_PARENT_BRIEF; set FM_SECONDMATE_CHARTER or scaffold one and replace {TASK}" >&2 + return 1 + } + [ -d "$DATA/$id" ] || SEED_PARENT_BRIEF_DIR_CREATED=1 + if [ "$no_projects" -eq 1 ]; then + "$FM_ROOT/bin/fm-brief.sh" "$id" --secondmate --no-projects + else + "$FM_ROOT/bin/fm-brief.sh" "$id" --secondmate "$@" + fi + SEED_PARENT_BRIEF_CREATED=1 + fi + if grep -F '{TASK}' "$SEED_PARENT_BRIEF" >/dev/null 2>&1; then + echo "error: secondmate charter brief at $SEED_PARENT_BRIEF still contains {TASK}; fill it before seeding" >&2 + return 1 + fi + charter_summary=$(registry_summary_for_brief "$SEED_PARENT_BRIEF") + [ -n "$charter_summary" ] || { + echo "error: secondmate charter brief at $SEED_PARENT_BRIEF has an empty Charter section; fill it before seeding" >&2 + return 1 + } + charter_scope=$(registry_scope_for_brief "$SEED_PARENT_BRIEF") + [ -n "$charter_scope" ] || { + echo "error: secondmate charter brief at $SEED_PARENT_BRIEF has an empty Routing scope section; fill it before seeding" >&2 + return 1 + } SEED_PARENT_HOME_LIFECYCLE_LOCK=$(fm_secondmate_home_lifecycle_lock_acquire "$CHECKOUT_LOCK_ROOT" "$FM_HOME") || return 1 fm_checkout_trusted_dir "$FM_HOME" >/dev/null || { echo "error: active firstmate home was removed or redirected while seed waited for lifecycle ownership" >&2 @@ -946,6 +973,7 @@ seed_home() { home=$(acquire_treehouse_home "$id") SEED_HOME="$home" SEED_HOME_RETAINED=1 + refuse_active_home_path "$home" || return 1 SEED_HOME_LIFECYCLE_LOCK=$(fm_secondmate_home_lifecycle_lock_acquire "$CHECKOUT_LOCK_ROOT" "$home") || return 1 freshness_status=0 "$SCRIPT_DIR/fm-checkout-refresh.sh" verify-worktree "$home" "$FM_ROOT" || freshness_status=$? @@ -965,6 +993,10 @@ seed_home() { SEED_HOME="$requested_abs" [ -e "$requested_abs" ] || SEED_HOME_CREATED=1 home=$(ensure_home "$id" "$requested_abs") + if [ "$no_projects" -eq 1 ]; then + refuse_populated_projectless_home "$home" || return 1 + refuse_projectful_projectless_charter "$id" "$SEED_PARENT_BRIEF" || return 1 + fi "$SCRIPT_DIR/fm-checkout-refresh.sh" preflight "$home" || { echo "error: refusing explicit secondmate home whose default branch cannot be refreshed safely" >&2 return 1 @@ -1000,34 +1032,6 @@ seed_home() { fi SEED_HOME_BACKED_UP=1 - if [ ! -f "$SEED_PARENT_BRIEF" ]; then - [ -n "${FM_SECONDMATE_CHARTER:-}" ] || { - echo "error: no filled secondmate charter brief at $SEED_PARENT_BRIEF; set FM_SECONDMATE_CHARTER or scaffold one and replace {TASK}" >&2 - return 1 - } - [ -d "$DATA/$id" ] || SEED_PARENT_BRIEF_DIR_CREATED=1 - if [ "$no_projects" -eq 1 ]; then - "$FM_ROOT/bin/fm-brief.sh" "$id" --secondmate --no-projects - else - "$FM_ROOT/bin/fm-brief.sh" "$id" --secondmate "$@" - fi - SEED_PARENT_BRIEF_CREATED=1 - fi - if grep -F '{TASK}' "$SEED_PARENT_BRIEF" >/dev/null 2>&1; then - echo "error: secondmate charter brief at $SEED_PARENT_BRIEF still contains {TASK}; fill it before seeding" >&2 - return 1 - fi - charter_summary=$(registry_summary_for_brief "$SEED_PARENT_BRIEF") - [ -n "$charter_summary" ] || { - echo "error: secondmate charter brief at $SEED_PARENT_BRIEF has an empty Charter section; fill it before seeding" >&2 - return 1 - } - charter_scope=$(registry_scope_for_brief "$SEED_PARENT_BRIEF") - [ -n "$charter_scope" ] || { - echo "error: secondmate charter brief at $SEED_PARENT_BRIEF has an empty Routing scope section; fill it before seeding" >&2 - return 1 - } - for project in "$@"; do project_dst=$(validate_project_destination "$home" "$project") || return 1 [ -e "$project_dst" ] || printf '%s\n' "$project_dst" >> "$SEED_CREATED_PROJECTS_FILE" diff --git a/bin/fm-report-contract-lib.sh b/bin/fm-report-contract-lib.sh index 5b46fe9265..7d49ce31c9 100644 --- a/bin/fm-report-contract-lib.sh +++ b/bin/fm-report-contract-lib.sh @@ -5,10 +5,12 @@ fm_completion_report_contract() { # local data=$1 task=$2 + # The literal markdown examples intentionally use backticks without shell expansion. + # shellcheck disable=SC2016 printf '%s\n' \ '# Completion report' \ "Before the final \`done:\` status, write \`$data/$task/completion.md\` with these six sections, each as a LEVEL-TWO markdown heading spelled exactly: \`## Summary\`, \`## What changed\`, \`## Verification\`, \`## Visual evidence\`, \`## Artifacts\`, \`## Follow-ups\`." \ - "Publication rejects the report if any of those headings is missing, spelled differently, or written at another level - a level-one \`# Summary\` fails. Each section needs substantive prose directly under it; if you use sub-headings inside a section, make them \`###\` so they nest rather than ending the section." \ + 'Publication rejects the report if any of those headings is missing, spelled differently, or written at another level - a level-one `# Summary` fails. Each section needs substantive prose directly under it; if you use sub-headings inside a section, make them `###` so they nest rather than ending the section.' \ 'When a section genuinely does not apply, say so in a sentence under the heading rather than omitting the heading.' \ 'Make it stand alone for the captain: explain the outcome, name important files or links, record the validation performed, and call out remaining risk or decisions.' \ "Put screenshots, diagrams, or other visual artifacts under \`$data/$task/visuals/\` and reference them from the report when they materially help review." \ diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index e66d231827..d0484efd66 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -76,6 +76,10 @@ # project, worktree, harness, backend, model, effort, mode, yolo, and report # requirements from metadata, selects a fresh account directory, and creates # only a replacement endpoint in the recorded worktree. +# The watcher sets FM_CAPACITY_RESCUE_RECOVERY=account-exhaustion-v1 for an +# automatic exhaustion handoff. That scoped mode excludes the recorded account +# plus every capacity_rescue_exhausted_account= extension in task metadata, so +# a rescue can never bounce straight back onto a known-exhausted account. # A --secondmate spawn also propagates the primary's declared inheritable config # into the secondmate home's config/, so the secondmate's OWN crewmates, # dispatch profiles, and backlog backend inherit the primary's settings @@ -986,6 +990,7 @@ ACCOUNT_NATIVE_LAUNCH_GO= ACCOUNT_NATIVE_LAUNCH_DIR= DIRECT_ACCOUNT_ROUTING=0 DIRECT_ACCOUNT_HOME= +DIRECT_ACCOUNT_EXCLUDES=() # Environment delivered natively by `herdr agent start --env KEY=VALUE` (one # repeated flag per entry). Empty for every other backend, which has no native # env channel and keeps using command-scoped shell prefixes instead. @@ -1042,9 +1047,10 @@ parse_orca_worktree_result() { ORCA_TERMINAL_PROOF=${rest%%$'\t'*} [ "$rest" != "$ORCA_TERMINAL_PROOF" ] || return 1 rest=${rest#*$'\t'} - ORCA_REPO_ID=$rest - case "$ORCA_REPO_ID" in *$'\t'*) return 1 ;; esac - ORCA_PROVIDER_TASK= + ORCA_REPO_ID=${rest%%$'\t'*} + [ "$rest" != "$ORCA_REPO_ID" ] || return 1 + ORCA_PROVIDER_TASK=${rest#*$'\t'} + case "$ORCA_PROVIDER_TASK" in *$'\t'*) return 1 ;; esac } persist_orca_cleanup_quarantine() { @@ -1426,21 +1432,33 @@ cleanup_continuation_launch_transport() { } spawn_return_created_worktree() { - local return_output return_status + local return_output return_status current_tip [ "$WORKTREE_CREATED" = 1 ] || return 0 [ "${BACKEND:-tmux}" != orca ] || return 0 [ -n "${WT:-}" ] && [ -d "$WT" ] || return 0 + current_tip=$(git -C "$WT" rev-parse HEAD 2>/dev/null) || current_tip= + if [ -n "$WORKTREE_EXPECTED_TIP" ] && [ -n "$current_tip" ] \ + && [ "$current_tip" != "$WORKTREE_EXPECTED_TIP" ]; then + echo "error: acquired worktree changed from expected detached tip $WORKTREE_EXPECTED_TIP to $current_tip; retain it for manual recovery" >&2 + echo "warning: retained acquired worktree $WT because repository identity and its expected detached tip could not be re-proven" >&2 + return 1 + fi if [ "$WORKTREE_RETAIN_ON_ABORT" = 1 ]; then - echo "warning: retained unsafe acquired worktree $WT for manual recovery" >&2 + echo "warning: retained unsafe acquired worktree $WT because repository identity or its expected detached tip could not be re-proven; inspect it manually" >&2 return 1 fi - if [ -z "$WORKTREE_EXPECTED_TIP" ] \ - || ! "$SCRIPT_DIR/fm-checkout-refresh.sh" verify-returnable "$WT" "$PROJ_ABS" "$WORKTREE_EXPECTED_TIP"; then + if [ -n "$WORKTREE_EXPECTED_TIP" ] && [ "$current_tip" != "$WORKTREE_EXPECTED_TIP" ]; then + echo "error: acquired worktree changed from expected detached tip $WORKTREE_EXPECTED_TIP to $current_tip; retain it for manual recovery" >&2 echo "warning: retained acquired worktree $WT because repository identity and its expected detached tip could not be re-proven" >&2 return 1 fi rm -f "$WT/.claude/settings.local.json" "$WT/.opencode/plugins/fm-turn-end.js" "$WT/.fm-grok-turnend" - if ! "$SCRIPT_DIR/fm-checkout-refresh.sh" verify-returnable "$WT" "$PROJ_ABS" "$WORKTREE_EXPECTED_TIP"; then + if [ -z "$WORKTREE_EXPECTED_TIP" ] \ + || ! "$SCRIPT_DIR/fm-checkout-refresh.sh" verify-returnable "$WT" "${PROJ_ABS_REAL:-$PROJ_ABS}" "$WORKTREE_EXPECTED_TIP"; then + echo "warning: retained acquired worktree $WT because repository identity and its expected detached tip could not be re-proven" >&2 + return 1 + fi + if ! "$SCRIPT_DIR/fm-checkout-refresh.sh" verify-returnable "$WT" "${PROJ_ABS_REAL:-$PROJ_ABS}" "$WORKTREE_EXPECTED_TIP"; then echo "warning: retained acquired worktree $WT because post-cleanup repository safety could not be re-proven" >&2 return 1 fi @@ -1981,6 +1999,13 @@ if [ "$RECOVERY_ACCOUNT" = 1 ]; then exit 1 } [ -n "$RECORDED_ACCOUNT_HOME" ] || { echo "error: direct account recovery metadata has no account_home for $ID" >&2; exit 1; } + if [ "${FM_CAPACITY_RESCUE_RECOVERY:-}" = account-exhaustion-v1 ]; then + while IFS= read -r exhausted_account; do + [ -n "$exhausted_account" ] || continue + DIRECT_ACCOUNT_EXCLUDES+=("$exhausted_account") + done < <(sed -n 's/^capacity_rescue_exhausted_account=//p' "$RESUME_META") + DIRECT_ACCOUNT_EXCLUDES+=("$RECORDED_ACCOUNT_HOME") + fi [ -z "$(fm_meta_get "$RESUME_META" account_profile)" ] || { echo "error: direct account recovery cannot replace legacy account_profile metadata for $ID" >&2; exit 1; } [ -z "$(fm_meta_get "$RESUME_META" account_rollback_cleanup)" ] || { echo "error: direct account recovery cannot bypass pending legacy rollback cleanup for $ID" >&2; exit 1; } [ -n "$RECORDED_PROJECT" ] || { echo "error: direct account recovery metadata has no project for $ID" >&2; exit 1; } @@ -2701,7 +2726,7 @@ if [ "$KIND" = secondmate ]; then fi if [ "$ACCOUNT_EFFECTIVE_MODE" = enforce ]; then if ! secondmate_home_supports_account_routing "$PROJ_ABS"; then - echo "error: refusing account-routed secondmate launch for $PROJ_ABS: the home lacks Agent Fleet routing support. Fast-forward or otherwise reconcile the home to this Firstmate revision, run bin/fm-config-push.sh, and retry." >&2 + echo "error: refusing account-routed secondmate launch for $ID at $PROJ_ABS: the home lacks Agent Fleet routing support. Fast-forward or otherwise reconcile the home to this Firstmate revision, run bin/fm-config-push.sh, and retry." >&2 exit 1 fi elif ! secondmate_home_supports_account_routing "$PROJ_ABS"; then @@ -2751,7 +2776,7 @@ real_path_or_raw() { # # and resets the selected clean pool worktree from that remote-tracking ref. # The post-acquisition verification below is the fail-closed freshness proof. if [ "$KIND" != secondmate ] && [ "$BACKEND" != orca ] && [ "$RECOVERY_ACCOUNT" != 1 ]; then - if CHECKOUT_PREFLIGHT_OUT=$("$SCRIPT_DIR/fm-checkout-refresh.sh" preflight "$PROJ_ABS" 2>&1); then + if CHECKOUT_PREFLIGHT_OUT=$("$SCRIPT_DIR/fm-checkout-refresh.sh" preflight "$PROJ_ABS_REAL" 2>&1); then CHECKOUT_PREFLIGHT_STATUS=0 else CHECKOUT_PREFLIGHT_STATUS=$? @@ -2786,7 +2811,11 @@ validate_spawn_worktree() { # exit 1 fi fm_checkout_validate_git_metadata "$wt_real" >/dev/null || { - echo "error: $source returned redirected or unprovable Git metadata at $wt_real" >&2 + if [ "$source" = "recorded direct account recovery" ]; then + echo "error: recorded direct account recovery worktree '$WT' does not belong to recorded project '$PROJ_ABS' because its Git metadata is redirected or unprovable; refusing endpoint creation" >&2 + else + echo "error: $source returned redirected or unprovable Git metadata at $wt_real" >&2 + fi exit 1 } fm_checkout_validate_git_metadata "$proj_real" >/dev/null || { @@ -2839,12 +2868,12 @@ if [ "$DIRECT_ACCOUNT_RECOVERY" = 1 ]; then fi if [ "$KIND" != secondmate ] && [ "$BACKEND" != orca ] && [ "$RECOVERY_ACCOUNT" != 1 ]; then - "$SCRIPT_DIR/fm-checkout-refresh.sh" pool-preflight "$PROJ_ABS" || { + "$SCRIPT_DIR/fm-checkout-refresh.sh" pool-preflight "$PROJ_ABS_REAL" || { echo "error: refusing Treehouse acquisition because pool safety could not be inspected for $PROJ_ABS" >&2 exit 1 } acquire_status=0 - WT=$("$SCRIPT_DIR/fm-checkout-refresh.sh" acquire-worktree "$PROJ_ABS" "firstmate-$ID") || acquire_status=$? + WT=$("$SCRIPT_DIR/fm-checkout-refresh.sh" acquire-worktree "$PROJ_ABS_REAL" "firstmate-$ID") || acquire_status=$? if [ "$acquire_status" -ne 0 ]; then if [ "$acquire_status" -eq 124 ]; then echo "error: refusing to spawn $ID after the bounded Treehouse acquisition timed out" >&2 @@ -2861,7 +2890,7 @@ if [ "$KIND" != secondmate ] && [ "$BACKEND" != orca ] && [ "$RECOVERY_ACCOUNT" WORKTREE_RETAIN_ON_ABORT=1 validate_spawn_worktree "treehouse get --lease" "$PROJ_ABS" freshness_status=0 - "$SCRIPT_DIR/fm-checkout-refresh.sh" verify-worktree "$WT" "$PROJ_ABS" || freshness_status=$? + "$SCRIPT_DIR/fm-checkout-refresh.sh" verify-worktree "$WT" "$PROJ_ABS_REAL" || freshness_status=$? if [ "$freshness_status" -ne 0 ]; then echo "error: refusing to launch fm-$ID from a leased worktree whose repository identity, cleanliness, or default-tip freshness could not be proved" >&2 exit 1 @@ -2884,7 +2913,11 @@ if [ "$DIRECT_ACCOUNT_RECOVERY" = 1 ]; then fi if [ "$DIRECT_ACCOUNT_PREPARE_DEFERRED" = 1 ]; then - DIRECT_ACCOUNT_HOME=$("$SCRIPT_DIR/fm-account-directory.sh" prepare "$HARNESS") || exit 1 + if [ "${FM_CAPACITY_RESCUE_RECOVERY:-}" = account-exhaustion-v1 ]; then + DIRECT_ACCOUNT_HOME=$("$SCRIPT_DIR/fm-account-directory.sh" prepare "$HARNESS" "${DIRECT_ACCOUNT_EXCLUDES[@]}") || exit 1 + else + DIRECT_ACCOUNT_HOME=$("$SCRIPT_DIR/fm-account-directory.sh" prepare "$HARNESS") || exit 1 + fi echo "fm-spawn: selected direct $HARNESS account home $DIRECT_ACCOUNT_HOME" >&2 DIRECT_ACCOUNT_PREPARE_DEFERRED=0 fi diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index ea03eda10b..c086b28654 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -302,7 +302,7 @@ fi managed_endpoint_is_gone() { # [probe-home] [recorded-scoped-target] local backend=$1 target=$2 expected=$3 probe_home=${4:-} recorded_scoped_target=${5:-} - local attempt state agent_state last=unknown + local attempt state last=unknown [ -n "$target" ] || return 2 for attempt in 1 2 3 4 5 6 7 8 9 10; do if [ -n "$probe_home" ]; then @@ -312,17 +312,7 @@ managed_endpoint_is_gone() { # [probe-home] fi case "$state" in absent) return 0 ;; - present) - if [ -n "$probe_home" ]; then - agent_state=$(unset FM_ROOT_OVERRIDE; FM_HOME="$probe_home" FM_ROOT="$probe_home" fm_backend_agent_alive "$backend" "$target" "$expected" "$recorded_scoped_target" 2>/dev/null) - else - agent_state=$(fm_backend_agent_alive "$backend" "$target" "$expected" "$recorded_scoped_target" 2>/dev/null) - fi - case "$agent_state" in - dead|alive) last=present ;; - *) last=unknown ;; - esac - ;; + present) return 1 ;; unknown) last=unknown ;; *) last=unknown ;; esac @@ -1052,9 +1042,30 @@ require_treehouse_return_authority() { } validate_teardown_target_identity() { - local project_root worktree_root project_common worktree_common + local project_root worktree_root project_common worktree_common project_git [ "$KIND" != secondmate ] || return 0 require_safe_task_metadata || return 1 + if [ "$BACKEND" != orca ] && [ ! -d "$WT" ]; then + project_root=$(canonical_existing_dir "$PROJ") || { + echo "error: teardown project metadata is not an exact inspectable repository root: ${PROJ:-}" >&2 + return 1 + } + [ "$(git -C "$project_root" rev-parse --show-toplevel 2>/dev/null)" = "$project_root" ] || { + echo "error: teardown project metadata is not an exact inspectable repository root: ${PROJ:-}" >&2 + return 1 + } + project_git=$(git -C "$project_root" rev-parse --absolute-git-dir 2>/dev/null) || return 1 + [ "$project_git" = "$project_root/.git" ] && [ -d "$project_git" ] && [ ! -L "$project_git" ] || { + echo "error: teardown project metadata is not an exact inspectable repository root: ${PROJ:-}" >&2 + return 1 + } + worktree_registered_for_project "$project_root" "$WT" || { + echo "error: absent teardown worktree is not registered to the recorded project: ${WT:-}" >&2 + return 1 + } + require_treehouse_task_lease "$WT" "firstmate-$ID" + return $? + fi project_root=$(exact_git_worktree_root "$PROJ") || { echo "error: teardown project metadata is not an exact inspectable repository root: ${PROJ:-}" >&2 return 1 @@ -1849,6 +1860,7 @@ if not stat.S_ISDIR(os.lstat(base).st_mode): print(base) PY ) || return 1 + [ -e "$base/fm-$ID" ] || [ -L "$base/fm-$ID" ] || return 0 removal_tree_operation "$base/fm-$ID" "task temp root" remove } @@ -2330,7 +2342,7 @@ def confined(path): except ValueError: return False -def retain(path, expected_directory): +def retain(path, expected_directory, keep_open=False): metadata = os.lstat(path) if stat.S_ISLNK(metadata.st_mode): raise OSError(f"redirected object storage entry: {path}") @@ -2363,49 +2375,58 @@ def retain(path, expected_directory): if confined(os.path.realpath(path)): os.close(descriptor) raise OSError(f"object storage depends on retiring home: {path}") - held.append((path, descriptor, expected)) - return descriptor, opened + held.append((path, expected)) + if keep_open: + return descriptor, opened + os.close(descriptor) + return None, opened def inspect(objects): objects = os.path.normpath(objects) - directory, metadata = retain(objects, True) - identity = (metadata.st_dev, metadata.st_ino) - if identity in visited: - return - visited.add(identity) - object_device = metadata.st_dev - for name in sorted(os.listdir(directory)): - path = os.path.join(objects, name) - item = os.stat(name, dir_fd=directory, follow_symlinks=False) - if item.st_dev != object_device: - raise OSError(f"object storage crosses a filesystem boundary: {path}") - if stat.S_ISLNK(item.st_mode): - raise OSError(f"redirected object storage entry: {path}") - if stat.S_ISDIR(item.st_mode): - inspect(path) - elif stat.S_ISREG(item.st_mode): - retain(path, False) - else: - raise OSError(f"unsafe object storage entry: {path}") - alternates = os.path.join(objects, "info", "alternates") - http_alternates = os.path.join(objects, "info", "http-alternates") - if os.path.lexists(http_alternates): - raise OSError(f"HTTP alternates are not durable proof: {http_alternates}") - if not os.path.lexists(alternates): - return - alternate_fd, _ = retain(alternates, False) - with os.fdopen(os.dup(alternate_fd), "r", encoding="utf-8") as stream: - entries = [line.rstrip("\n") for line in stream] - if not entries or any(not entry or "\x00" in entry for entry in entries): - raise OSError(f"malformed alternates file: {alternates}") - for entry in entries: - if entry.startswith('"') or entry.endswith('"'): - raise OSError(f"quoted alternates are ambiguous: {alternates}") - candidate = entry if os.path.isabs(entry) else os.path.join(objects, entry) - inspect(os.path.realpath(candidate)) + directory, metadata = retain(objects, True, True) + try: + identity = (metadata.st_dev, metadata.st_ino) + if identity in visited: + return + visited.add(identity) + object_device = metadata.st_dev + for name in sorted(os.listdir(directory)): + path = os.path.join(objects, name) + item = os.stat(name, dir_fd=directory, follow_symlinks=False) + if item.st_dev != object_device: + raise OSError(f"object storage crosses a filesystem boundary: {path}") + if stat.S_ISLNK(item.st_mode): + raise OSError(f"redirected object storage entry: {path}") + if stat.S_ISDIR(item.st_mode): + inspect(path) + elif stat.S_ISREG(item.st_mode): + retain(path, False) + else: + raise OSError(f"unsafe object storage entry: {path}") + alternates = os.path.join(objects, "info", "alternates") + http_alternates = os.path.join(objects, "info", "http-alternates") + if os.path.lexists(http_alternates): + raise OSError(f"HTTP alternates are not durable proof: {http_alternates}") + if not os.path.lexists(alternates): + return + alternate_fd, _ = retain(alternates, False, True) + try: + with os.fdopen(os.dup(alternate_fd), "r", encoding="utf-8") as stream: + entries = [line.rstrip("\n") for line in stream] + finally: + os.close(alternate_fd) + if not entries or any(not entry or "\x00" in entry for entry in entries): + raise OSError(f"malformed alternates file: {alternates}") + for entry in entries: + if entry.startswith('"') or entry.endswith('"'): + raise OSError(f"quoted alternates are ambiguous: {alternates}") + candidate = entry if os.path.isabs(entry) else os.path.join(objects, entry) + inspect(os.path.realpath(candidate)) + finally: + os.close(directory) def verify_retained(): - for path, descriptor, expected in held: + for path, expected in held: metadata = os.lstat(path) current = ( metadata.st_dev, @@ -2413,14 +2434,34 @@ def verify_retained(): stat.S_IFMT(metadata.st_mode), metadata.st_size, ) - opened = os.fstat(descriptor) - retained = ( - opened.st_dev, - opened.st_ino, - stat.S_IFMT(opened.st_mode), - opened.st_size, - ) - if current != expected or retained != expected or stat.S_ISLNK(metadata.st_mode): + if ( + current != expected + or stat.S_ISLNK(metadata.st_mode) + or confined(os.path.realpath(path)) + ): + raise OSError(f"object storage identity changed during graph proof: {path}") + flags = os.O_RDONLY + if expected[2] == stat.S_IFDIR: + flags |= os.O_DIRECTORY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(path, flags) + try: + opened = os.fstat(descriptor) + retained = ( + opened.st_dev, + opened.st_ino, + stat.S_IFMT(opened.st_mode), + opened.st_size, + ) + finally: + os.close(descriptor) + except OSError as error: + raise OSError( + f"object storage identity changed during graph proof: {path}" + ) from error + if retained != expected: raise OSError(f"object storage identity changed during graph proof: {path}") def run(arguments, input_data=None): @@ -2503,9 +2544,6 @@ try: except (OSError, UnicodeError, subprocess.SubprocessError) as error: print(f"REFUSED: {label} complete object graph is unavailable: {error}", file=sys.stderr) raise SystemExit(1) -finally: - for _, descriptor, _ in reversed(held): - os.close(descriptor) PY } diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 9b4495edb6..8d950205bc 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -36,6 +36,11 @@ # status, or turn-end progress surfaces after # FM_PERMISSION_STALL_ESCALATE_SECS as a possible macOS # permission/system-dialog block. Unless afk is active. +# A verified Claude/Codex account-capacity pane bypasses +# stale timing: the watcher removes the exhausted +# endpoint and hands the same brief/worktree to a fresh +# direct account automatically. Attempts and exhausted +# accounts are recorded in meta and bounded per task. # check: