Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 16 additions & 13 deletions bin/backends/herdr.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2653,16 +2653,11 @@ fm_backend_herdr_expected_label_matches() { # <target> [expected-label]
# launch-grade fm_backend_herdr_server_ensure used to spawn new crewmates.
#
# Reading a pane or sending keys to it does not care how the server was
# launched - the pane already exists and the server is up. It only needs the
# server to be running and adapter-owned (this HOME's own certificate names the
# live pid), which fm_backend_herdr_server_adapter_owned proves without the
# closed-shell launch certification. This is what keeps peek/steer working when
# FirstMate itself runs INSIDE the herdr session it manages (HERDR_ENV=1): that
# server was not launched through the crewmate adapter's own closed-shell path, so
# closed_shell_environment_ready is false and the full ensure would try to
# restart+recertify - impossible while the session is occupied by live crewmates and
# FirstMate itself. The SPAWN path deliberately keeps the strict ensure so a new
# crewmate still launches only in a certified closed-shell server.
# launched - the pane already exists and the server is up. Native-agent
# production servers therefore need only report running, matching
# fm_backend_herdr_server_ensure after the native cutover. The explicit legacy
# certificate lab retains its adapter-ownership proof so that retired lifecycle
# remains testable without constraining ordinary read/steer.
fm_backend_herdr_server_reachable_for_readsteer() { # <session>
local session=$1 running
if fm_backend_herdr_test_hooks_enabled \
Expand All @@ -2672,6 +2667,9 @@ fm_backend_herdr_server_reachable_for_readsteer() { # <session>
running=$(fm_backend_herdr_cli "$session" status --json 2>/dev/null \
| fm_backend_herdr_control_jq -r '.server.running // false' 2>/dev/null)
[ "$running" = true ] || return 1
if ! fm_backend_herdr_server_certificate_required; then
return 0
fi
fm_backend_herdr_server_adapter_owned "$session"
}

Expand Down Expand Up @@ -3269,9 +3267,14 @@ fm_backend_herdr_events_capable() { # <session>
case "$protocol" in ''|*[!0-9]*) return 1 ;; esac
[ "$protocol" -ge "$FM_BACKEND_HERDR_MIN_EVENTS_PROTOCOL" ] || return 1
schema=$(fm_backend_herdr_scrubbed_exec "$herdr_bin" api schema --json 2>/dev/null) || return 1
printf '%s' "$schema" | fm_backend_herdr_control_grep -Fq 'events.subscribe' || return 1
printf '%s' "$schema" | fm_backend_herdr_control_grep -Fq 'pane.agent_status_changed' || return 1
return 0
# Parse once and consume the complete schema. Early-exit grep -q pipelines
# close their input as soon as a match is found, which makes the upstream
# printf hit EPIPE and turns a real capability into a false negative whenever
# the caller inherits pipefail.
printf '%s' "$schema" | fm_backend_herdr_control_jq -e '
any(.. | scalars; . == "events.subscribe")
and any(.. | scalars; . == "pane.agent_status_changed")
' >/dev/null 2>&1
}

# fm_backend_herdr_normalize_event: THE single normalize point (report section 5
Expand Down
2 changes: 1 addition & 1 deletion bin/fm-spawn.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2334,7 +2334,7 @@ if [ "$ACCOUNT_EFFECTIVE_MODE" = enforce ] && [ "$BACKEND" = orca ]; then
fi
if [ "$ACCOUNT_EFFECTIVE_MODE" = enforce ] && ! fm_account_test_lab_enabled \
&& [ "$BACKEND" != herdr ]; then
echo "error: enforced Agent Fleet routing requires backend=herdr with a process-bound closed-shell certificate; backend=$BACKEND cannot prove that its pane shell was sanitized before startup" >&2
echo "error: enforced Agent Fleet routing requires backend=herdr so native agent startup can inject the selected account environment; backend=$BACKEND cannot provide that isolation, so retry with --backend herdr" >&2
exit 1
fi
if [ "$ACCOUNT_EFFECTIVE_MODE" != off ] && [ "$RESUME_ACCOUNT" != 1 ]; then
Expand Down
83 changes: 81 additions & 2 deletions docs/herdr-backend.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ Herdr is [an agent-native terminal multiplexer](https://herdr.dev) with a socket
Originally verified against herdr 0.7.1, protocol 14, on macOS aarch64; the latest dated evidence below uses herdr 0.7.3, protocol 16.
Current real-herdr verification uses isolated `HERDR_SESSION` names plus the guarded teardown helper in `tests/herdr-test-safety.sh`.
A 2026-07-02 cleanup bug proved that `HERDR_SESSION` alone is not a safe way to target destructive session cleanup; see "Session targeting: the `--session` flag, not `HERDR_SESSION` alone" below.
All real-herdr verification in this document uses isolated sessions and guarded cleanup; the captain's default herdr session and live tmux fleet were never intended targets.
Real-herdr lifecycle verification in this document uses isolated sessions and guarded cleanup.
The 2026-07-29 read/steer incident was instead verified without lifecycle mutation against the fixing task's own pane in the already-running default session.

## Setup

Expand Down Expand Up @@ -248,9 +249,87 @@ Under the explicit legacy certified-lifecycle lab path, a full current-release p
For an older adapter-owned certificate, the adapter queries the explicitly named session's workspace, tab, and pane collections.
An exact-empty session is stopped only after two complete empty proofs plus repeated process-ownership proof, then detached-launched and recertified before the lock is released.
An occupied session - one reporting any workspace, tab, or pane - is not restarted; routing continues in that same running session and may create the workspace for the native-agent spawn path.
Unreadable or indeterminate state still refuses before workspace mutation, and manual or uncertified ownership still fails closed.
Unreadable or indeterminate state still refuses before workspace mutation, and the explicit legacy lab still rejects manual or uncertified ownership.
This convergence path never uses ambient `herdr server stop`, never deletes session state (including `default`), and never moves Herdr ownership into a captain launcher.

## Incident (2026-07-29): native cutover left read and steer behind the retired certificate gate

Every `fm-send.sh` steer to a live Herdr crewmate failed even though Herdr reported the exact recorded pane ids as live and direct `herdr agent send`, `herdr pane send-text`, `herdr pane send-keys`, and `herdr pane read` calls succeeded.
The same stale precondition made `fm-peek.sh` return zero pane lines.
The earlier peek report was therefore reproducible, not stale.

The regression was reproduced against Herdr 0.7.3, protocol 16, and the fixing codex task's own live pane `default:w6:pFZ`.
No Herdr session, workspace, tab, pane, or agent lifecycle command was run.
A task-local `FM_HOME` fixture recorded the live pane and label so the public selector paths were exercised without reading or mutating another firstmate home's files.

Pre-fix `fm-send.sh` evidence:

```text
$ FM_HOME=/Users/dongkeun/.treehouse/firstmate-21f368/6/firstmate/.task-home \
FM_SEND_SETTLE=0 bin/fm-send.sh fm-herdr-send-fix-h7 \
'[pre-fix live verification probe: do not alter task direction]'
error: text not sent to default:w6:pFZ (herdr send failed; tried meta=/Users/dongkeun/.treehouse/firstmate-21f368/6/firstmate/.task-home/state/fm-herdr-send-fix-h7.meta; backend=from-meta)
exit 1

$ herdr agent get w6:pFZ
{"result":{"agent":{"agent":"codex","agent_status":"working","name":"fm-fm-herdr-send-fix-h7","pane_id":"w6:pFZ",...}}}
```

Pre-fix `fm-peek.sh` returned exit 1 and zero stdout lines against the same recorded target.
Tracing stopped in `fm_backend_herdr_server_adapter_owned`: the default server had no closed-shell certificate, so neither path reached its Herdr read or mutation call.

The root cause was an incomplete native-agent cutover.
Commit `cc0c56c` correctly made `fm_backend_herdr_server_ensure` reuse any running production server because `agent start --env` owns new-agent environment isolation, but `fm_backend_herdr_server_reachable_for_readsteer` still required the retired server certificate added before that cutover.
Production read and steer now require the exact requested session to report running, matching spawn, while the explicitly enabled legacy certificate lab retains the old ownership proof.

The separate `printf: write error: Broken pipe` lead was real but not the read/steer root cause.
`fm_backend_herdr_events_capable` piped the approximately 220 KB schema string into two early-exit `grep -q` calls.
With inherited `pipefail`, the first match closed the consumer before `printf` finished and made a capable Herdr build return false.
The probe now parses the complete schema once with `jq`, eliminating the early close and the diagnostic.

The focused regression fails against the pre-fix adapter:

```text
$ FM_TEST_FOCUSED=readsteer-native-cutover tests/fm-backend-herdr.test.sh
not ok - native read/steer should accept a running server without the retired closed-shell certificate: expected exit 0, got 1
```

The same regression passes after the fix:

```text
$ FM_TEST_FOCUSED=readsteer-native-cutover tests/fm-backend-herdr.test.sh
ok - fm_backend_herdr_server_reachable_for_readsteer: native-agent servers need only be running
ok - fm_backend_herdr_server_reachable_for_readsteer: the explicit legacy lab retains certificate enforcement
ok - fm_backend_herdr_events_capable: consumes the full schema without a broken pipe
```

The repaired public paths were then exercised against that same live pane:

```text
$ FM_HOME=/Users/dongkeun/.treehouse/firstmate-21f368/6/firstmate/.task-home \
bin/fm-peek.sh fm-herdr-send-fix-h7 12
exit 0
stdout lines: 11

$ FM_HOME=/Users/dongkeun/.treehouse/firstmate-21f368/6/firstmate/.task-home \
FM_SEND_SETTLE=0 bin/fm-send.sh fm-herdr-send-fix-h7 \
'LIVE_E2E_H7: continue the current herdr send-fix task without changing scope; preserve this token in the transcript.'
exit 0

$ FM_HOME=/Users/dongkeun/.treehouse/firstmate-21f368/6/firstmate/.task-home \
bin/fm-peek.sh fm-herdr-send-fix-h7 24 \
> /Users/dongkeun/.treehouse/firstmate-21f368/6/firstmate/.task-home/peek-after-send.out
$ grep -F LIVE_E2E_H7 /Users/dongkeun/.treehouse/firstmate-21f368/6/firstmate/.task-home/peek-after-send.out
↳ LIVE_E2E_H7: continue the current herdr send-fix task without changing scope; preserve this token in the

$ FM_HOME=/Users/dongkeun/.treehouse/firstmate-21f368/6/firstmate/.task-home \
bash -o pipefail -c '. bin/backends/herdr.sh; fm_backend_herdr_events_capable default; printf "events_rc=%s\n" "$?"'
events_rc=0
stderr bytes: 0
```

The existing submission regressions still cover first-Enter autocomplete swallowing, Enter-only retry, native `working` or `blocked` confirmation, and preservation of unsubmitted text on exhausted retries.

## Incident (2026-07-13): the ASCII request separator erased the secondmate marker

A routed request reached a Pi/Herdr secondmate without the visible `[fm-from-firstmate]` label, so the secondmate correctly treated it as direct captain conversation and returned nothing to the parent status path.
Expand Down
18 changes: 15 additions & 3 deletions tests/fm-afk-inject-herdr-e2e.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,16 @@ wait_daemon_started() {
fail "$label did not record backend=herdr after 6s: $new_log"
}

wait_for_digest() {
local i=0
while [ "$i" -lt 120 ]; do
grep -q 'Supervisor escalate' "$LOG_FILE" && return 0
sleep 0.5
i=$((i + 1))
done
return 1
}

start_daemon() {
local log_start=0
[ ! -f "$STATE_DIR/.supervise-daemon.log" ] || log_start=$(wc -l < "$STATE_DIR/.supervise-daemon.log")
Expand Down Expand Up @@ -337,7 +347,7 @@ test_scenario_a() {
fm_backend_herdr_send_key "$SUPERVISOR_TARGET" Enter
sleep 0.5

sleep 8
wait_for_digest || true

grep -q 'human draft text' "$LOG_FILE" \
|| fail "Scenario A: human text not in log after submit"
Expand Down Expand Up @@ -378,7 +388,8 @@ test_scenario_b() {

echo "done: PR https://example.test/pr/200" > "$STATE_DIR/fake-c1.status"

sleep 10
wait_for_digest || true
sleep 2

local digest_count
digest_count=$(grep -c 'Supervisor escalate' "$LOG_FILE" || true)
Expand Down Expand Up @@ -414,7 +425,8 @@ test_scenario_c() {
start_daemon

echo "done: PR https://example.test/pr/300" > "$STATE_DIR/fake-c1.status"
sleep 8
wait_for_digest || true
sleep 2

local digest_count
digest_count=$(grep -c 'Supervisor escalate' "$LOG_FILE" || true)
Expand Down
100 changes: 100 additions & 0 deletions tests/fm-backend-herdr.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,54 @@ test_version_check_refuses_missing_herdr() {
pass "fm_backend_herdr_version_check: refuses loudly when herdr is not installed"
}

test_readsteer_native_server_does_not_require_legacy_certificate() {
local dir lock_root log resp fb status
dir="$TMP_ROOT/readsteer-native-server"
lock_root="$dir/locks"
log="$dir/log"
resp="$dir/responses"
mkdir -p "$lock_root" "$resp"
chmod 700 "$lock_root"
: > "$log"
fb=$(make_herdr_fakebin "$dir")

# shellcheck disable=SC2016
env -u FM_TEST_HERDR_REQUIRE_CERT_LIFECYCLE \
PATH="$fb:$PATH" FM_HERDR_LOG="$log" FM_HERDR_RESPONSES="$resp" \
FM_BACKEND_HERDR_SERVER_LOCK_ROOT="$lock_root" \
bash -c '. "$0/bin/backends/herdr.sh"; fm_backend_herdr_server_reachable_for_readsteer native' "$ROOT"
status=$?

expect_code 0 "$status" "native read/steer should accept a running server without the retired closed-shell certificate"
assert_contains "$(cat "$log")" $'\x1f''status'$'\x1f''--json'$'\x1f''--session'$'\x1f''native' \
"native read/steer did not check the exact session's running state"
[ -z "$(find "$lock_root" -mindepth 1 -print -quit)" ] \
|| fail "native read/steer consulted or created retired certificate artifacts"
pass "fm_backend_herdr_server_reachable_for_readsteer: native-agent servers need only be running"
}

test_readsteer_legacy_lab_still_requires_certificate() {
local dir lock_root log resp fb status
dir="$TMP_ROOT/readsteer-legacy-certificate"
lock_root="$dir/locks"
log="$dir/log"
resp="$dir/responses"
mkdir -p "$lock_root" "$resp"
chmod 700 "$lock_root"
: > "$log"
fb=$(make_herdr_fakebin "$dir")

# shellcheck disable=SC2016
PATH="$fb:$PATH" FM_HERDR_LOG="$log" FM_HERDR_RESPONSES="$resp" \
FM_BACKEND_HERDR_SERVER_LOCK_ROOT="$lock_root" \
FM_TEST_HERDR_REQUIRE_CERT_LIFECYCLE=firstmate-herdr-tests-v1 \
bash -c '. "$0/bin/backends/herdr.sh"; fm_backend_herdr_server_reachable_for_readsteer legacy' "$ROOT"
status=$?

[ "$status" -ne 0 ] || fail "the explicit legacy lifecycle lab accepted an uncertified server"
pass "fm_backend_herdr_server_reachable_for_readsteer: the explicit legacy lab retains certificate enforcement"
}

test_herdr_binary_revalidates_leaf_and_physical_ancestry() {
local dir safe bin release unsafe hardlink out status
dir="$TMP_ROOT/herdr-physical-pin"
Expand Down Expand Up @@ -3883,6 +3931,48 @@ SH
printf '%s\n' "$fb"
}

make_herdr_schemafake() { # <dir> -> echoes fakebin dir
local dir=$1 fb="$1/fakebin"
mkdir -p "$fb"
cat > "$fb/herdr" <<'SH'
#!/usr/bin/env bash
set -u
case "${1:-} ${2:-}" in
"status --json")
printf '{"client":{"version":"0.7.3","protocol":16},"server":{"running":true}}\n'
;;
"api schema")
printf '{"method":"events.subscribe","event":"pane.agent_status_changed","padding":"'
awk 'BEGIN { for (i = 0; i < 20000; i++) printf "0123456789abcdef" }'
printf '"}\n'
;;
*)
exit 1
;;
esac
SH
chmod +x "$fb/herdr"
printf '%s\n' "$fb"
}

test_events_capable_consumes_schema_without_broken_pipe() {
local dir fb err status
dir="$TMP_ROOT/events-schema-pipe"
mkdir -p "$dir"
fb=$(make_herdr_schemafake "$dir")
err="$dir/stderr"

# shellcheck disable=SC2016
PATH="$fb:$PATH" FM_BACKEND_HERDR_EVENT_READER=/bin/true \
bash -o pipefail -c '. "$0/bin/backends/herdr.sh"; fm_backend_herdr_events_capable native' "$ROOT" \
2>"$err"
status=$?

expect_code 0 "$status" "events capability should recognize both required schema values with pipefail enabled"
assert_not_contains "$(cat "$err")" "Broken pipe" "events capability emitted a broken-pipe diagnostic"
pass "fm_backend_herdr_events_capable: consumes the full schema without a broken pipe"
}

# make_fake_reader: a stand-in for bin/backends/herdr-eventwait.py. It ignores
# the socket, streams the TAB-separated lines in $FM_FAKE_READER_LINES to stdout
# (one projected event per line: pane_id\tworkspace_id\tagent_status\tagent),
Expand Down Expand Up @@ -4143,6 +4233,13 @@ test_wait_transition_clean_timeout_returns_1() {
# shellcheck source=bin/fm-backend.sh
. "$ROOT/bin/fm-backend.sh"

if [ "${FM_TEST_FOCUSED:-}" = readsteer-native-cutover ]; then
test_readsteer_native_server_does_not_require_legacy_certificate
test_readsteer_legacy_lab_still_requires_certificate
test_events_capable_consumes_schema_without_broken_pipe
exit 0
fi

if [ "${FM_TEST_FOCUSED:-}" = review-round-25 ]; then
test_target_state_distinguishes_absent_from_malformed_panes
test_target_state_refuses_missing_recorded_pane_with_replacement
Expand Down Expand Up @@ -4221,6 +4318,8 @@ fi
test_version_check_accepts_current_protocol
test_version_check_refuses_old_protocol
test_version_check_refuses_missing_herdr
test_readsteer_native_server_does_not_require_legacy_certificate
test_readsteer_legacy_lab_still_requires_certificate
test_server_test_hooks_are_inert_without_explicit_opt_in
test_herdr_binary_revalidates_leaf_and_physical_ancestry
test_server_launch_scrubs_hostile_perl_and_control_environment
Expand Down Expand Up @@ -4324,6 +4423,7 @@ test_dispatch_routes_herdr_backend
test_dispatch_busy_state_unknown_for_tmux
test_dispatch_composer_state_routes_by_backend
test_scripts_route_explicit_target_through_meta_backend
test_events_capable_consumes_schema_without_broken_pipe
test_normalize_event_leaves_from_empty
test_escalation_marker_keys_like_watcher
test_apply_transition_blocked_requires_commit_to_dedupe
Expand Down
3 changes: 2 additions & 1 deletion tests/fm-secondmate-harness.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,8 @@ SH
make_fixture_clone() {
local destination=$1
git clone -q --no-checkout "$ROOT" "$destination"
git -C "$destination" checkout -q -b main "$(git -C "$ROOT" rev-parse HEAD)"
# Push checkouts may already clone a local main; reset the disposable fixture either way.
git -C "$destination" checkout -q -B main "$(git -C "$ROOT" rev-parse HEAD)"
git -C "$destination" update-ref refs/remotes/origin/main HEAD
git -C "$destination" remote set-head origin main
}
Expand Down
6 changes: 6 additions & 0 deletions tests/fm-secondmate-sync.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -405,6 +405,12 @@ case "\$cmd \$sub" in
printf '{"error":{"code":"agent_not_found","message":"gone"}}\n' >&2
fi
;;
"agent send")
if [ "\$arg" = "${stale#*:}" ]; then
exit 1
fi
exit 0
;;
"pane send-text"|"pane run"|"pane send-keys")
if [ "\$arg" = "${stale#*:}" ]; then
exit 1
Expand Down
Loading