From ee8f11700966ea39f94dece3df240d1e55de258e Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Sat, 25 Jul 2026 01:19:18 -0400 Subject: [PATCH 001/168] rescue exhausted crew accounts --- AGENTS.md | 3 + bin/fm-account-directory.sh | 70 +++-- bin/fm-classify-lib.sh | 41 +++ bin/fm-spawn.sh | 14 +- bin/fm-watch.sh | 436 +++++++++++++++++++++++++++++ docs/account-capacity-rescue.md | 100 +++++++ docs/architecture.md | 3 + docs/configuration.md | 5 + docs/scripts.md | 6 +- tests/fm-account-directory.test.sh | 96 ++++++- tests/fm-watch-triage.test.sh | 265 ++++++++++++++++++ 11 files changed, 1014 insertions(+), 25 deletions(-) create mode 100644 docs/account-capacity-rescue.md diff --git a/AGENTS.md b/AGENTS.md index 9f11c95bc5..a74c877dde 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -108,6 +108,7 @@ state/ volatile runtime signals; gitignored .meta written by fm-spawn: window=, worktree=, project=, harness=, model=, effort=, kind=, mode=, yolo=, tasktmp=, generation_id=, report_required= Direct ship and scout launches also own account_home=, worktree_git_dir=, worktree_git_dir_identity=, and exactly one authoritative final-state field: worktree_git_ref= for an attached branch or worktree_git_head= for an intentional detached HEAD. Their metadata may temporarily carry worktree_git_setup_ref= and worktree_git_setup_head= while the brief's required `fm/` branch transition is pending; recovery accepts only that exact setup state or the authoritative `fm/` ref and removes the setup fields after adoption. Direct recovery validates the canonical worktree path, exact physical Git-dir identity, and authoritative final state before account preparation and again immediately before endpoint creation; any drift fails closed without launching. + Automatic capacity rescue appends capacity_rescue_attempts=, repeated capacity_rescue_exhausted_account= entries, per-attempt audit/result fields, and an optional capacity_rescue_stopped= terminal reason; docs/account-capacity-rescue.md owns the full transaction and retry boundary. If endpoint removal after a failed new direct spawn cannot be confirmed, direct_spawn_cleanup=pending and rollback_pending=1 retain the endpoint and worktree identity for explicit teardown. A failed new direct spawn that never created an endpoint records direct_spawn_endpoint=not-created and an empty window= so teardown skips endpoint quiescence without skipping worktree safety. Secondmate Agent Fleet routing and legacy managed recovery own account_pool=, account_profile=, account_task=, account_attempt=, and provider_session_id= (docs/configuration.md "Agent Fleet account routing"). @@ -585,6 +586,8 @@ The emitted block is the only per-harness operating recipe in the session contex Do not substitute another harness's command shape for it. **Always-on wake triage (absorb only when provably working).** `bin/fm-watch.sh` classifies every wake in bash and absorbs the benign majority without waking you: crewmates with positive working evidence (an actively-running no-mistakes step for their branch, or a busy pane read via `bin/fm-crew-state.sh`) unless the separate permission-stall no-progress threshold has expired, a declared `paused:` external wait until its bounded recheck cadence, and no-change heartbeats. +Before ordinary stale classification, it recognizes empirically verified Claude and Codex capacity chrome and automatically hands a direct-account ship or scout task to a different eligible account through the existing guarded recovery path. +That rescue is lifecycle-serialized, attempt-capped, and durably stopped with one blocked wake when no unused eligible account remains; `docs/account-capacity-rescue.md` owns the signatures, audit fields, and failure contract. It never absorbs a crewmate that stopped without that evidence - whatever its stale status log claims - and only an actionable wake is queued durably and ends the supervision wait, so you resume the emitted protocol exactly once per actionable event. A `paused:` status is a deliberate external wait, not `blocked:`; its initial signal still surfaces once, and a forgotten pause re-surfaces for a recheck once per window. Repeated unchanged wedge or permission-stall escalations eventually add `demand-deep-inspection` to the wake reason so they are not mistaken for another routine validation wait. diff --git a/bin/fm-account-directory.sh b/bin/fm-account-directory.sh index 3ba66b5462..209fce2b60 100755 --- a/bin/fm-account-directory.sh +++ b/bin/fm-account-directory.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash # Select and prepare direct Claude or Codex account-directory launches. # Usage: -# fm-account-directory.sh select +# fm-account-directory.sh select [excluded-account-home...] # fm-account-directory.sh install-herdr-hook -# fm-account-directory.sh prepare +# fm-account-directory.sh prepare [excluded-account-home...] # # This header is the single owner of the direct account-directory contract. # FM_ACCOUNT_DIRECTORY_CUTOVER: direct-observe-passwd-home-v2 @@ -12,13 +12,15 @@ # Codex selection removes that account's quota-axi window cache immediately # before every read, sets CODEX_HOME plus the account-isolated XDG_CACHE_HOME, # accepts only a fresh result with at least one numeric five_hour or weekly -# window, and picks the account with the highest minimum remaining percentage. +# window, skips excluded and zero-capacity accounts, and picks the remaining +# account with the highest minimum remaining percentage. # A Codex account with no such freshly readable window is skipped as unhealthy. # Claude quota is not currently distinguishable per config directory because # quota-axi cannot non-interactively resolve Claude's config-dir-specific macOS # Keychain credential. # Claude therefore never treats a missing usage window as account failure and -# selects the first real account directory in stable bytewise sort order. +# selects the first non-excluded real account directory in stable bytewise sort +# order. # Selection prints only the chosen absolute account home on stdout and logs # health, fallback, and choice diagnostics on stderr. # prepare selects the account and idempotently runs Herdr's own integration @@ -203,8 +205,18 @@ valid_account_home() { # esac } -first_account_home() { # +account_home_is_excluded() { # [excluded-account-home...] + local candidate=$1 excluded + shift + for excluded in "$@"; do + [ "$candidate" != "$excluded" ] || return 0 + done + return 1 +} + +first_account_home() { # [excluded-account-home...] local vendor=$1 root vendor_dir candidate + shift root=$(account_root) || return 1 vendor_dir=$root/$vendor [ -d "$vendor_dir" ] && [ ! -L "$vendor_dir" ] || { @@ -215,10 +227,14 @@ first_account_home() { # export LC_ALL for candidate in "$vendor_dir"/*; do valid_account_home "$vendor_dir" "$candidate" || continue + if account_home_is_excluded "$candidate" "$@"; then + log "$vendor account $candidate skipped: exhausted by this task" + continue + fi printf '%s\n' "$candidate" return 0 done - echo "error: no account directories found for $vendor under $vendor_dir" >&2 + echo "CAPACITY_UNAVAILABLE: no unused $vendor account directories remain under $vendor_dir" >&2 return 1 } @@ -279,7 +295,7 @@ $1 EOF } -select_codex() { +select_codex() { # [excluded-account-home...] local root vendor_dir quota_bin candidate usage score local best_home='' best_score='' root=$(account_root) || return 1 @@ -297,12 +313,20 @@ select_codex() { export LC_ALL for candidate in "$vendor_dir"/*; do valid_account_home "$vendor_dir" "$candidate" || continue + if account_home_is_excluded "$candidate" "$@"; then + log "codex account $candidate skipped: exhausted by this task" + continue + fi usage=$(fresh_codex_usage_json "$candidate" "$quota_bin") || usage= score=$(codex_score "$usage") || score= if [ -z "$score" ]; then log "codex account $candidate skipped: no freshly readable usage window" continue fi + if ! awk -v candidate_score="$score" 'BEGIN { exit !(candidate_score > 0) }'; then + log "codex account $candidate skipped: fresh remaining score=$score has no capacity" + continue + fi log "codex account $candidate fresh remaining score=$score" if [ -z "$best_home" ] || awk -v candidate_score="$score" -v current_score="$best_score" \ 'BEGIN { exit !(candidate_score > current_score) }'; then @@ -311,26 +335,28 @@ select_codex() { fi done [ -n "$best_home" ] || { - echo "error: no healthy Codex account has a freshly readable usage window" >&2 + echo "CAPACITY_UNAVAILABLE: no unused Codex account has a freshly readable positive usage window" >&2 return 1 } log "selected codex account $best_home with fresh remaining score=$best_score" printf '%s\n' "$best_home" } -select_claude() { +select_claude() { # [excluded-account-home...] local selected - selected=$(first_account_home claude) || return 1 + selected=$(first_account_home claude "$@") || return 1 log "CLAUDE USAGE UNREADABLE: quota-axi cannot non-interactively resolve Claude's config-dir-specific macOS Keychain credential today; selecting the first account directory by stable sort: $selected" printf '%s\n' "$selected" } -select_account() { # - case "$1" in - codex) select_codex ;; - claude) select_claude ;; +select_account() { # [excluded-account-home...] + local vendor=$1 + shift + case "$vendor" in + codex) select_codex "$@" ;; + claude) select_claude "$@" ;; *) - echo "error: direct account-directory selection supports only claude or codex, not '$1'" >&2 + echo "error: direct account-directory selection supports only claude or codex, not '$vendor'" >&2 return 1 ;; esac @@ -378,17 +404,21 @@ case "${1:-}" in exit 0 ;; select) - [ "$#" -eq 2 ] || { usage; exit 2; } - select_account "$2" + [ "$#" -ge 2 ] || { usage; exit 2; } + vendor=$2 + shift 2 + select_account "$vendor" "$@" ;; install-herdr-hook) [ "$#" -eq 3 ] || { usage; exit 2; } install_herdr_hook "$2" "$3" ;; prepare) - [ "$#" -eq 2 ] || { usage; exit 2; } - selected_home=$(select_account "$2") || exit 1 - install_herdr_hook "$2" "$selected_home" || exit 1 + [ "$#" -ge 2 ] || { usage; exit 2; } + vendor=$2 + shift 2 + selected_home=$(select_account "$vendor" "$@") || exit 1 + install_herdr_hook "$vendor" "$selected_home" || exit 1 printf '%s\n' "$selected_home" ;; *) diff --git a/bin/fm-classify-lib.sh b/bin/fm-classify-lib.sh index 641db7d7be..b436413f99 100755 --- a/bin/fm-classify-lib.sh +++ b/bin/fm-classify-lib.sh @@ -64,6 +64,47 @@ FM_PAUSE_RESURFACE_SECS_DEFAULT=3600 # this is the one owner of the verb literal, overridable via FM_CLASSIFY_RESOLVE_VERB. FM_CLASSIFY_RESOLVE_VERB_DEFAULT='resolved' +# Classify a provider-owned capacity failure from the bounded pane capture of a +# known harness. Prints a stable failure token and returns 0 on a verified shape; +# prints nothing and returns 1 otherwise. +# +# These match complete TUI chrome, not capacity-like prose. Codex requires its +# standalone warning glyph and exact line. Claude requires either its tool-result +# glyph plus exact session-limit sentence or the complete usage-limit choice +# dialog. This prevents a task discussing "usage limits", quoting one sentence, +# or printing ordinary API rate-limit output from being mistaken for an exhausted +# harness account. Empirical captures and versions are recorded in +# docs/account-capacity-rescue.md. +provider_capacity_failure_kind() { # + local harness=$1 tail40=$2 prompt_tail + prompt_tail=$(printf '%s\n' "$tail40" | tail -18) + case "$harness" in + codex) + if printf '%s\n' "$prompt_tail" \ + | grep -Eq '^[[:space:]]*⚠[[:space:]]+Selected model is at capacity\. Please try a different model\.[[:space:]]*$'; then + printf 'codex-model-capacity' + return 0 + fi + ;; + claude) + if printf '%s\n' "$prompt_tail" \ + | grep -Eq "^[[:space:]]*⎿.*You've hit your session limit · resets .+[[:space:]]*$"; then + printf 'claude-session-limit' + return 0 + fi + if printf '%s\n' "$prompt_tail" | grep -Fq 'What do you want to do?' \ + && printf '%s\n' "$prompt_tail" | grep -Eq '1\. Stop and wait for limit to reset[[:space:]]*$' \ + && printf '%s\n' "$prompt_tail" | grep -Eq '2\. Switch to usage credits[[:space:]]*$' \ + && printf '%s\n' "$prompt_tail" | grep -Eq '3\. Switch to Team plan[[:space:]]*$' \ + && printf '%s\n' "$prompt_tail" | grep -Eq 'Enter to confirm.*Esc to cancel'; then + printf 'claude-usage-limit-dialog' + return 0 + fi + ;; + esac + return 1 +} + # Return the last non-blank line of a status file (empty if missing/blank). last_status_line() { local f=$1 diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index e66d231827..ee773937a4 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -76,6 +76,10 @@ # project, worktree, harness, backend, model, effort, mode, yolo, and report # requirements from metadata, selects a fresh account directory, and creates # only a replacement endpoint in the recorded worktree. +# The watcher sets FM_CAPACITY_RESCUE_RECOVERY=account-exhaustion-v1 for an +# automatic exhaustion handoff. That scoped mode excludes the recorded account +# plus every capacity_rescue_exhausted_account= extension in task metadata, so +# a rescue can never bounce straight back onto a known-exhausted account. # A --secondmate spawn also propagates the primary's declared inheritable config # into the secondmate home's config/, so the secondmate's OWN crewmates, # dispatch profiles, and backlog backend inherit the primary's settings @@ -986,6 +990,7 @@ ACCOUNT_NATIVE_LAUNCH_GO= ACCOUNT_NATIVE_LAUNCH_DIR= DIRECT_ACCOUNT_ROUTING=0 DIRECT_ACCOUNT_HOME= +DIRECT_ACCOUNT_EXCLUDES=() # Environment delivered natively by `herdr agent start --env KEY=VALUE` (one # repeated flag per entry). Empty for every other backend, which has no native # env channel and keeps using command-scoped shell prefixes instead. @@ -1981,6 +1986,13 @@ if [ "$RECOVERY_ACCOUNT" = 1 ]; then exit 1 } [ -n "$RECORDED_ACCOUNT_HOME" ] || { echo "error: direct account recovery metadata has no account_home for $ID" >&2; exit 1; } + if [ "${FM_CAPACITY_RESCUE_RECOVERY:-}" = account-exhaustion-v1 ]; then + while IFS= read -r exhausted_account; do + [ -n "$exhausted_account" ] || continue + DIRECT_ACCOUNT_EXCLUDES+=("$exhausted_account") + done < <(sed -n 's/^capacity_rescue_exhausted_account=//p' "$RESUME_META") + DIRECT_ACCOUNT_EXCLUDES+=("$RECORDED_ACCOUNT_HOME") + fi [ -z "$(fm_meta_get "$RESUME_META" account_profile)" ] || { echo "error: direct account recovery cannot replace legacy account_profile metadata for $ID" >&2; exit 1; } [ -z "$(fm_meta_get "$RESUME_META" account_rollback_cleanup)" ] || { echo "error: direct account recovery cannot bypass pending legacy rollback cleanup for $ID" >&2; exit 1; } [ -n "$RECORDED_PROJECT" ] || { echo "error: direct account recovery metadata has no project for $ID" >&2; exit 1; } @@ -2884,7 +2896,7 @@ if [ "$DIRECT_ACCOUNT_RECOVERY" = 1 ]; then fi if [ "$DIRECT_ACCOUNT_PREPARE_DEFERRED" = 1 ]; then - DIRECT_ACCOUNT_HOME=$("$SCRIPT_DIR/fm-account-directory.sh" prepare "$HARNESS") || exit 1 + DIRECT_ACCOUNT_HOME=$("$SCRIPT_DIR/fm-account-directory.sh" prepare "$HARNESS" "${DIRECT_ACCOUNT_EXCLUDES[@]}") || exit 1 echo "fm-spawn: selected direct $HARNESS account home $DIRECT_ACCOUNT_HOME" >&2 DIRECT_ACCOUNT_PREPARE_DEFERRED=0 fi diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 9b4495edb6..333dc2a5fb 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -36,6 +36,11 @@ # status, or turn-end progress surfaces after # FM_PERMISSION_STALL_ESCALATE_SECS as a possible macOS # permission/system-dialog block. Unless afk is active. +# A verified Claude/Codex account-capacity pane bypasses +# stale timing: the watcher removes the exhausted +# endpoint and hands the same brief/worktree to a fresh +# direct account automatically. Attempts and exhausted +# accounts are recorded in meta and bounded per task. # check: