From 849c8dad723dd0a780d475db2d905b38891afeba Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Wed, 29 Jul 2026 02:21:21 -0400 Subject: [PATCH 1/4] fix herdr live steering --- bin/backends/herdr.sh | 29 +++++----- docs/herdr-backend.md | 83 ++++++++++++++++++++++++++- tests/fm-backend-herdr.test.sh | 100 +++++++++++++++++++++++++++++++++ 3 files changed, 197 insertions(+), 15 deletions(-) diff --git a/bin/backends/herdr.sh b/bin/backends/herdr.sh index 4045912885..f9264e7e59 100644 --- a/bin/backends/herdr.sh +++ b/bin/backends/herdr.sh @@ -2653,16 +2653,11 @@ fm_backend_herdr_expected_label_matches() { # [expected-label] # launch-grade fm_backend_herdr_server_ensure used to spawn new crewmates. # # Reading a pane or sending keys to it does not care how the server was -# launched - the pane already exists and the server is up. It only needs the -# server to be running and adapter-owned (this HOME's own certificate names the -# live pid), which fm_backend_herdr_server_adapter_owned proves without the -# closed-shell launch certification. This is what keeps peek/steer working when -# FirstMate itself runs INSIDE the herdr session it manages (HERDR_ENV=1): that -# server was not launched through the crewmate adapter's own closed-shell path, so -# closed_shell_environment_ready is false and the full ensure would try to -# restart+recertify - impossible while the session is occupied by live crewmates and -# FirstMate itself. The SPAWN path deliberately keeps the strict ensure so a new -# crewmate still launches only in a certified closed-shell server. +# launched - the pane already exists and the server is up. Native-agent +# production servers therefore need only report running, matching +# fm_backend_herdr_server_ensure after the native cutover. The explicit legacy +# certificate lab retains its adapter-ownership proof so that retired lifecycle +# remains testable without constraining ordinary read/steer. fm_backend_herdr_server_reachable_for_readsteer() { # local session=$1 running if fm_backend_herdr_test_hooks_enabled \ @@ -2672,6 +2667,9 @@ fm_backend_herdr_server_reachable_for_readsteer() { # running=$(fm_backend_herdr_cli "$session" status --json 2>/dev/null \ | fm_backend_herdr_control_jq -r '.server.running // false' 2>/dev/null) [ "$running" = true ] || return 1 + if ! fm_backend_herdr_server_certificate_required; then + return 0 + fi fm_backend_herdr_server_adapter_owned "$session" } @@ -3269,9 +3267,14 @@ fm_backend_herdr_events_capable() { # case "$protocol" in ''|*[!0-9]*) return 1 ;; esac [ "$protocol" -ge "$FM_BACKEND_HERDR_MIN_EVENTS_PROTOCOL" ] || return 1 schema=$(fm_backend_herdr_scrubbed_exec "$herdr_bin" api schema --json 2>/dev/null) || return 1 - printf '%s' "$schema" | fm_backend_herdr_control_grep -Fq 'events.subscribe' || return 1 - printf '%s' "$schema" | fm_backend_herdr_control_grep -Fq 'pane.agent_status_changed' || return 1 - return 0 + # Parse once and consume the complete schema. Early-exit grep -q pipelines + # close their input as soon as a match is found, which makes the upstream + # printf hit EPIPE and turns a real capability into a false negative whenever + # the caller inherits pipefail. + printf '%s' "$schema" | fm_backend_herdr_control_jq -e ' + any(.. | scalars; . == "events.subscribe") + and any(.. | scalars; . == "pane.agent_status_changed") + ' >/dev/null 2>&1 } # fm_backend_herdr_normalize_event: THE single normalize point (report section 5 diff --git a/docs/herdr-backend.md b/docs/herdr-backend.md index d86f77dc76..0c4e7b767e 100644 --- a/docs/herdr-backend.md +++ b/docs/herdr-backend.md @@ -7,7 +7,8 @@ Herdr is [an agent-native terminal multiplexer](https://herdr.dev) with a socket Originally verified against herdr 0.7.1, protocol 14, on macOS aarch64; the latest dated evidence below uses herdr 0.7.3, protocol 16. Current real-herdr verification uses isolated `HERDR_SESSION` names plus the guarded teardown helper in `tests/herdr-test-safety.sh`. A 2026-07-02 cleanup bug proved that `HERDR_SESSION` alone is not a safe way to target destructive session cleanup; see "Session targeting: the `--session` flag, not `HERDR_SESSION` alone" below. -All real-herdr verification in this document uses isolated sessions and guarded cleanup; the captain's default herdr session and live tmux fleet were never intended targets. +Real-herdr lifecycle verification in this document uses isolated sessions and guarded cleanup. +The 2026-07-29 read/steer incident was instead verified without lifecycle mutation against the fixing task's own pane in the already-running default session. ## Setup @@ -248,9 +249,87 @@ Under the explicit legacy certified-lifecycle lab path, a full current-release p For an older adapter-owned certificate, the adapter queries the explicitly named session's workspace, tab, and pane collections. An exact-empty session is stopped only after two complete empty proofs plus repeated process-ownership proof, then detached-launched and recertified before the lock is released. An occupied session - one reporting any workspace, tab, or pane - is not restarted; routing continues in that same running session and may create the workspace for the native-agent spawn path. -Unreadable or indeterminate state still refuses before workspace mutation, and manual or uncertified ownership still fails closed. +Unreadable or indeterminate state still refuses before workspace mutation, and the explicit legacy lab still rejects manual or uncertified ownership. This convergence path never uses ambient `herdr server stop`, never deletes session state (including `default`), and never moves Herdr ownership into a captain launcher. +## Incident (2026-07-29): native cutover left read and steer behind the retired certificate gate + +Every `fm-send.sh` steer to a live Herdr crewmate failed even though Herdr reported the exact recorded pane ids as live and direct `herdr agent send`, `herdr pane send-text`, `herdr pane send-keys`, and `herdr pane read` calls succeeded. +The same stale precondition made `fm-peek.sh` return zero pane lines. +The earlier peek report was therefore reproducible, not stale. + +The regression was reproduced against Herdr 0.7.3, protocol 16, and the fixing codex task's own live pane `default:w6:pFZ`. +No Herdr session, workspace, tab, pane, or agent lifecycle command was run. +A task-local `FM_HOME` fixture recorded the live pane and label so the public selector paths were exercised without reading or mutating another firstmate home's files. + +Pre-fix `fm-send.sh` evidence: + +```text +$ FM_HOME=/Users/dongkeun/.treehouse/firstmate-21f368/6/firstmate/.task-home \ + FM_SEND_SETTLE=0 bin/fm-send.sh fm-herdr-send-fix-h7 \ + '[pre-fix live verification probe: do not alter task direction]' +error: text not sent to default:w6:pFZ (herdr send failed; tried meta=/Users/dongkeun/.treehouse/firstmate-21f368/6/firstmate/.task-home/state/fm-herdr-send-fix-h7.meta; backend=from-meta) +exit 1 + +$ herdr agent get w6:pFZ +{"result":{"agent":{"agent":"codex","agent_status":"working","name":"fm-fm-herdr-send-fix-h7","pane_id":"w6:pFZ",...}}} +``` + +Pre-fix `fm-peek.sh` returned exit 1 and zero stdout lines against the same recorded target. +Tracing stopped in `fm_backend_herdr_server_adapter_owned`: the default server had no closed-shell certificate, so neither path reached its Herdr read or mutation call. + +The root cause was an incomplete native-agent cutover. +Commit `cc0c56c` correctly made `fm_backend_herdr_server_ensure` reuse any running production server because `agent start --env` owns new-agent environment isolation, but `fm_backend_herdr_server_reachable_for_readsteer` still required the retired server certificate added before that cutover. +Production read and steer now require the exact requested session to report running, matching spawn, while the explicitly enabled legacy certificate lab retains the old ownership proof. + +The separate `printf: write error: Broken pipe` lead was real but not the read/steer root cause. +`fm_backend_herdr_events_capable` piped the approximately 220 KB schema string into two early-exit `grep -q` calls. +With inherited `pipefail`, the first match closed the consumer before `printf` finished and made a capable Herdr build return false. +The probe now parses the complete schema once with `jq`, eliminating the early close and the diagnostic. + +The focused regression fails against the pre-fix adapter: + +```text +$ FM_TEST_FOCUSED=readsteer-native-cutover tests/fm-backend-herdr.test.sh +not ok - native read/steer should accept a running server without the retired closed-shell certificate: expected exit 0, got 1 +``` + +The same regression passes after the fix: + +```text +$ FM_TEST_FOCUSED=readsteer-native-cutover tests/fm-backend-herdr.test.sh +ok - fm_backend_herdr_server_reachable_for_readsteer: native-agent servers need only be running +ok - fm_backend_herdr_server_reachable_for_readsteer: the explicit legacy lab retains certificate enforcement +ok - fm_backend_herdr_events_capable: consumes the full schema without a broken pipe +``` + +The repaired public paths were then exercised against that same live pane: + +```text +$ FM_HOME=/Users/dongkeun/.treehouse/firstmate-21f368/6/firstmate/.task-home \ + bin/fm-peek.sh fm-herdr-send-fix-h7 12 +exit 0 +stdout lines: 11 + +$ FM_HOME=/Users/dongkeun/.treehouse/firstmate-21f368/6/firstmate/.task-home \ + FM_SEND_SETTLE=0 bin/fm-send.sh fm-herdr-send-fix-h7 \ + 'LIVE_E2E_H7: continue the current herdr send-fix task without changing scope; preserve this token in the transcript.' +exit 0 + +$ FM_HOME=/Users/dongkeun/.treehouse/firstmate-21f368/6/firstmate/.task-home \ + bin/fm-peek.sh fm-herdr-send-fix-h7 24 \ + > /Users/dongkeun/.treehouse/firstmate-21f368/6/firstmate/.task-home/peek-after-send.out +$ grep -F LIVE_E2E_H7 /Users/dongkeun/.treehouse/firstmate-21f368/6/firstmate/.task-home/peek-after-send.out + ↳ LIVE_E2E_H7: continue the current herdr send-fix task without changing scope; preserve this token in the + +$ FM_HOME=/Users/dongkeun/.treehouse/firstmate-21f368/6/firstmate/.task-home \ + bash -o pipefail -c '. bin/backends/herdr.sh; fm_backend_herdr_events_capable default; printf "events_rc=%s\n" "$?"' +events_rc=0 +stderr bytes: 0 +``` + +The existing submission regressions still cover first-Enter autocomplete swallowing, Enter-only retry, native `working` or `blocked` confirmation, and preservation of unsubmitted text on exhausted retries. + ## Incident (2026-07-13): the ASCII request separator erased the secondmate marker A routed request reached a Pi/Herdr secondmate without the visible `[fm-from-firstmate]` label, so the secondmate correctly treated it as direct captain conversation and returned nothing to the parent status path. diff --git a/tests/fm-backend-herdr.test.sh b/tests/fm-backend-herdr.test.sh index 084e8b3dc7..66ecbe7230 100755 --- a/tests/fm-backend-herdr.test.sh +++ b/tests/fm-backend-herdr.test.sh @@ -242,6 +242,54 @@ test_version_check_refuses_missing_herdr() { pass "fm_backend_herdr_version_check: refuses loudly when herdr is not installed" } +test_readsteer_native_server_does_not_require_legacy_certificate() { + local dir lock_root log resp fb status + dir="$TMP_ROOT/readsteer-native-server" + lock_root="$dir/locks" + log="$dir/log" + resp="$dir/responses" + mkdir -p "$lock_root" "$resp" + chmod 700 "$lock_root" + : > "$log" + fb=$(make_herdr_fakebin "$dir") + + # shellcheck disable=SC2016 + env -u FM_TEST_HERDR_REQUIRE_CERT_LIFECYCLE \ + PATH="$fb:$PATH" FM_HERDR_LOG="$log" FM_HERDR_RESPONSES="$resp" \ + FM_BACKEND_HERDR_SERVER_LOCK_ROOT="$lock_root" \ + bash -c '. "$0/bin/backends/herdr.sh"; fm_backend_herdr_server_reachable_for_readsteer native' "$ROOT" + status=$? + + expect_code 0 "$status" "native read/steer should accept a running server without the retired closed-shell certificate" + assert_contains "$(cat "$log")" $'\x1f''status'$'\x1f''--json'$'\x1f''--session'$'\x1f''native' \ + "native read/steer did not check the exact session's running state" + [ -z "$(find "$lock_root" -mindepth 1 -print -quit)" ] \ + || fail "native read/steer consulted or created retired certificate artifacts" + pass "fm_backend_herdr_server_reachable_for_readsteer: native-agent servers need only be running" +} + +test_readsteer_legacy_lab_still_requires_certificate() { + local dir lock_root log resp fb status + dir="$TMP_ROOT/readsteer-legacy-certificate" + lock_root="$dir/locks" + log="$dir/log" + resp="$dir/responses" + mkdir -p "$lock_root" "$resp" + chmod 700 "$lock_root" + : > "$log" + fb=$(make_herdr_fakebin "$dir") + + # shellcheck disable=SC2016 + PATH="$fb:$PATH" FM_HERDR_LOG="$log" FM_HERDR_RESPONSES="$resp" \ + FM_BACKEND_HERDR_SERVER_LOCK_ROOT="$lock_root" \ + FM_TEST_HERDR_REQUIRE_CERT_LIFECYCLE=firstmate-herdr-tests-v1 \ + bash -c '. "$0/bin/backends/herdr.sh"; fm_backend_herdr_server_reachable_for_readsteer legacy' "$ROOT" + status=$? + + [ "$status" -ne 0 ] || fail "the explicit legacy lifecycle lab accepted an uncertified server" + pass "fm_backend_herdr_server_reachable_for_readsteer: the explicit legacy lab retains certificate enforcement" +} + test_herdr_binary_revalidates_leaf_and_physical_ancestry() { local dir safe bin release unsafe hardlink out status dir="$TMP_ROOT/herdr-physical-pin" @@ -3883,6 +3931,48 @@ SH printf '%s\n' "$fb" } +make_herdr_schemafake() { # -> echoes fakebin dir + local dir=$1 fb="$1/fakebin" + mkdir -p "$fb" + cat > "$fb/herdr" <<'SH' +#!/usr/bin/env bash +set -u +case "${1:-} ${2:-}" in + "status --json") + printf '{"client":{"version":"0.7.3","protocol":16},"server":{"running":true}}\n' + ;; + "api schema") + printf '{"method":"events.subscribe","event":"pane.agent_status_changed","padding":"' + awk 'BEGIN { for (i = 0; i < 20000; i++) printf "0123456789abcdef" }' + printf '"}\n' + ;; + *) + exit 1 + ;; +esac +SH + chmod +x "$fb/herdr" + printf '%s\n' "$fb" +} + +test_events_capable_consumes_schema_without_broken_pipe() { + local dir fb err status + dir="$TMP_ROOT/events-schema-pipe" + mkdir -p "$dir" + fb=$(make_herdr_schemafake "$dir") + err="$dir/stderr" + + # shellcheck disable=SC2016 + PATH="$fb:$PATH" FM_BACKEND_HERDR_EVENT_READER=/bin/true \ + bash -o pipefail -c '. "$0/bin/backends/herdr.sh"; fm_backend_herdr_events_capable native' "$ROOT" \ + 2>"$err" + status=$? + + expect_code 0 "$status" "events capability should recognize both required schema values with pipefail enabled" + assert_not_contains "$(cat "$err")" "Broken pipe" "events capability emitted a broken-pipe diagnostic" + pass "fm_backend_herdr_events_capable: consumes the full schema without a broken pipe" +} + # make_fake_reader: a stand-in for bin/backends/herdr-eventwait.py. It ignores # the socket, streams the TAB-separated lines in $FM_FAKE_READER_LINES to stdout # (one projected event per line: pane_id\tworkspace_id\tagent_status\tagent), @@ -4143,6 +4233,13 @@ test_wait_transition_clean_timeout_returns_1() { # shellcheck source=bin/fm-backend.sh . "$ROOT/bin/fm-backend.sh" +if [ "${FM_TEST_FOCUSED:-}" = readsteer-native-cutover ]; then + test_readsteer_native_server_does_not_require_legacy_certificate + test_readsteer_legacy_lab_still_requires_certificate + test_events_capable_consumes_schema_without_broken_pipe + exit 0 +fi + if [ "${FM_TEST_FOCUSED:-}" = review-round-25 ]; then test_target_state_distinguishes_absent_from_malformed_panes test_target_state_refuses_missing_recorded_pane_with_replacement @@ -4221,6 +4318,8 @@ fi test_version_check_accepts_current_protocol test_version_check_refuses_old_protocol test_version_check_refuses_missing_herdr +test_readsteer_native_server_does_not_require_legacy_certificate +test_readsteer_legacy_lab_still_requires_certificate test_server_test_hooks_are_inert_without_explicit_opt_in test_herdr_binary_revalidates_leaf_and_physical_ancestry test_server_launch_scrubs_hostile_perl_and_control_environment @@ -4324,6 +4423,7 @@ test_dispatch_routes_herdr_backend test_dispatch_busy_state_unknown_for_tmux test_dispatch_composer_state_routes_by_backend test_scripts_route_explicit_target_through_meta_backend +test_events_capable_consumes_schema_without_broken_pipe test_normalize_event_leaves_from_empty test_escalation_marker_keys_like_watcher test_apply_transition_blocked_requires_commit_to_dedupe From 31d4c87ed4a4d9e464d9c45875f17ea5991690be Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Wed, 29 Jul 2026 05:51:54 -0400 Subject: [PATCH 2/4] no-mistakes(test): Stabilize Herdr send regression tests --- tests/fm-afk-inject-herdr-e2e.test.sh | 18 +++++++++++++++--- tests/fm-secondmate-sync.test.sh | 6 ++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/fm-afk-inject-herdr-e2e.test.sh b/tests/fm-afk-inject-herdr-e2e.test.sh index 34ee3e6152..91d2a63b5d 100755 --- a/tests/fm-afk-inject-herdr-e2e.test.sh +++ b/tests/fm-afk-inject-herdr-e2e.test.sh @@ -242,6 +242,16 @@ wait_daemon_started() { fail "$label did not record backend=herdr after 6s: $new_log" } +wait_for_digest() { + local i=0 + while [ "$i" -lt 120 ]; do + grep -q 'Supervisor escalate' "$LOG_FILE" && return 0 + sleep 0.5 + i=$((i + 1)) + done + return 1 +} + start_daemon() { local log_start=0 [ ! -f "$STATE_DIR/.supervise-daemon.log" ] || log_start=$(wc -l < "$STATE_DIR/.supervise-daemon.log") @@ -337,7 +347,7 @@ test_scenario_a() { fm_backend_herdr_send_key "$SUPERVISOR_TARGET" Enter sleep 0.5 - sleep 8 + wait_for_digest || true grep -q 'human draft text' "$LOG_FILE" \ || fail "Scenario A: human text not in log after submit" @@ -378,7 +388,8 @@ test_scenario_b() { echo "done: PR https://example.test/pr/200" > "$STATE_DIR/fake-c1.status" - sleep 10 + wait_for_digest || true + sleep 2 local digest_count digest_count=$(grep -c 'Supervisor escalate' "$LOG_FILE" || true) @@ -414,7 +425,8 @@ test_scenario_c() { start_daemon echo "done: PR https://example.test/pr/300" > "$STATE_DIR/fake-c1.status" - sleep 8 + wait_for_digest || true + sleep 2 local digest_count digest_count=$(grep -c 'Supervisor escalate' "$LOG_FILE" || true) diff --git a/tests/fm-secondmate-sync.test.sh b/tests/fm-secondmate-sync.test.sh index 232f038f88..0f0f972f02 100755 --- a/tests/fm-secondmate-sync.test.sh +++ b/tests/fm-secondmate-sync.test.sh @@ -405,6 +405,12 @@ case "\$cmd \$sub" in printf '{"error":{"code":"agent_not_found","message":"gone"}}\n' >&2 fi ;; + "agent send") + if [ "\$arg" = "${stale#*:}" ]; then + exit 1 + fi + exit 0 + ;; "pane send-text"|"pane run"|"pane send-keys") if [ "\$arg" = "${stale#*:}" ]; then exit 1 From 1456cbb284661d4532103216eb663e14eb9bc8c1 Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Wed, 29 Jul 2026 10:17:41 -0400 Subject: [PATCH 3/4] no-mistakes(document): Correct Herdr enforced-routing error guidance --- bin/fm-spawn.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index cec185d8bb..6bda6fc2ae 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -2334,7 +2334,7 @@ if [ "$ACCOUNT_EFFECTIVE_MODE" = enforce ] && [ "$BACKEND" = orca ]; then fi if [ "$ACCOUNT_EFFECTIVE_MODE" = enforce ] && ! fm_account_test_lab_enabled \ && [ "$BACKEND" != herdr ]; then - echo "error: enforced Agent Fleet routing requires backend=herdr with a process-bound closed-shell certificate; backend=$BACKEND cannot prove that its pane shell was sanitized before startup" >&2 + echo "error: enforced Agent Fleet routing requires backend=herdr so native agent startup can inject the selected account environment; backend=$BACKEND cannot provide that isolation, so retry with --backend herdr" >&2 exit 1 fi if [ "$ACCOUNT_EFFECTIVE_MODE" != off ] && [ "$RESUME_ACCOUNT" != 1 ]; then From 857a75f7de29bb5a1679a2e7106bd2fc18c0c6d0 Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Wed, 29 Jul 2026 11:12:49 -0400 Subject: [PATCH 4/4] no-mistakes: apply CI fixes --- tests/fm-secondmate-harness.test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/fm-secondmate-harness.test.sh b/tests/fm-secondmate-harness.test.sh index 1dea94fa22..86f79fc682 100755 --- a/tests/fm-secondmate-harness.test.sh +++ b/tests/fm-secondmate-harness.test.sh @@ -254,7 +254,8 @@ SH make_fixture_clone() { local destination=$1 git clone -q --no-checkout "$ROOT" "$destination" - git -C "$destination" checkout -q -b main "$(git -C "$ROOT" rev-parse HEAD)" + # Push checkouts may already clone a local main; reset the disposable fixture either way. + git -C "$destination" checkout -q -B main "$(git -C "$ROOT" rev-parse HEAD)" git -C "$destination" update-ref refs/remotes/origin/main HEAD git -C "$destination" remote set-head origin main }