From 097714aaa0601b1058fea0f921f44188ed5ea386 Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Fri, 31 Jul 2026 15:19:11 -0400 Subject: [PATCH 01/10] ci: shard behavior tests by measured duration --- .github/workflows/ci.yml | 67 ++++++- bin/fm-behavior-shards.sh | 295 ++++++++++++++++++++++++++++++ docs/ci-behavior-shards.md | 71 +++++++ tests/behavior-test-durations.tsv | 84 +++++++++ tests/fm-behavior-shards.test.sh | 231 +++++++++++++++++++++++ 5 files changed, 739 insertions(+), 9 deletions(-) create mode 100755 bin/fm-behavior-shards.sh create mode 100644 docs/ci-behavior-shards.md create mode 100644 tests/behavior-test-durations.tsv create mode 100755 tests/fm-behavior-shards.test.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 875050cc0b..5d7dcfd9ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,12 +24,23 @@ jobs: # re-spell the shellcheck command here; keep CI and the pre-push gate on it. - run: bin/fm-lint.sh - tests: - name: Behavior tests + behavior-test-plan: + name: Behavior test shard plan + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Prove complete, disjoint shard coverage + run: bin/fm-behavior-shards.sh --check 6 + + behavior-tests: + name: Behavior tests (shard ${{ matrix.shard }}/6) + needs: behavior-test-plan runs-on: ubuntu-latest - # The pre-cutover suite reached 25m35s, and the exhaustive Bridge crash - # matrix adds bounded work. Keep enough margin without masking a real hang. - timeout-minutes: 90 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4, 5, 6] steps: - uses: actions/checkout@v6 with: @@ -55,11 +66,49 @@ jobs: set -eu npm install -g tasks-axi tasks-axi --version - - run: | + - name: Prepare private shard state + run: | + set -eu + shard_root="$RUNNER_TEMP/fm-behavior-${{ matrix.shard }}" + install -d -m 700 "$shard_root/tmp" "$shard_root/tmux" + { + echo "TMPDIR=$shard_root/tmp" + echo "TMP=$shard_root/tmp" + echo "TEMP=$shard_root/tmp" + echo "TMUX_TMPDIR=$shard_root/tmux" + } >> "$GITHUB_ENV" + - name: Run behavior shard ${{ matrix.shard }}/6 + run: | set -eu - for test_script in tests/*.test.sh; do - "$test_script" - done + manifest="$RUNNER_TEMP/executed-${{ matrix.shard }}.tsv" + bin/fm-behavior-shards.sh --run "${{ matrix.shard }}" 6 "$manifest" + - name: Upload executed-test manifest + if: always() + uses: actions/upload-artifact@v4 + with: + name: behavior-shard-${{ matrix.shard }} + path: ${{ runner.temp }}/executed-${{ matrix.shard }}.tsv + if-no-files-found: warn + overwrite: true + + behavior-tests-complete: + # Preserve the historical required-check name while proving the union of + # what the six runners actually executed, not only the planned inventory. + name: Behavior tests + needs: behavior-tests + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Download executed-test manifests + uses: actions/download-artifact@v4 + continue-on-error: true + with: + pattern: behavior-shard-* + path: ${{ runner.temp }}/behavior-manifests + merge-multiple: true + - name: Verify complete execution union + run: bin/fm-behavior-shards.sh --verify 6 "$RUNNER_TEMP/behavior-manifests" agent-fleet: name: Agent Fleet package diff --git a/bin/fm-behavior-shards.sh b/bin/fm-behavior-shards.sh new file mode 100755 index 0000000000..51875bf20a --- /dev/null +++ b/bin/fm-behavior-shards.sh @@ -0,0 +1,295 @@ +#!/usr/bin/env bash +# fm-behavior-shards.sh - duration-balanced behavior-test planning, execution, +# timing refresh, and executed-manifest completeness verification. +# +# Usage: +# bin/fm-behavior-shards.sh --check +# bin/fm-behavior-shards.sh --plan +# bin/fm-behavior-shards.sh --run +# bin/fm-behavior-shards.sh --verify +# bin/fm-behavior-shards.sh --record +# +# The checked-in duration file defaults to tests/behavior-test-durations.tsv. +# Override it with FM_BEHAVIOR_DURATIONS_FILE for fixture tests only. +# Planning uses deterministic longest-processing-time assignment: descending +# duration, path as the stable secondary key, and lowest shard number for load +# ties. Each shard runs its assigned files serially. CI supplies one isolated +# runner, TMPDIR, and TMUX_TMPDIR per shard. +set -eu + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P) +DURATIONS=${FM_BEHAVIOR_DURATIONS_FILE:-$ROOT/tests/behavior-test-durations.tsv} +TAB=$(printf '\t') +WORK=$(mktemp -d "${TMPDIR:-/tmp}/fm-behavior-shards.XXXXXX") || exit 1 +trap 'rm -rf "$WORK"' EXIT + +usage() { + sed -n '2,18s/^# \{0,1\}//p' "$0" +} + +die() { + printf 'fm-behavior-shards: %s\n' "$*" >&2 + exit 2 +} + +validate_positive_integer() { + local value=$1 label=$2 + case "$value" in + ''|*[!0-9]*) die "$label must be a positive integer (got '$value')" ;; + esac + [ "$value" -gt 0 ] || die "$label must be greater than zero" +} + +inventory() { + find "$ROOT/tests" -maxdepth 1 -type f -name '*.test.sh' -print \ + | sed "s#^$ROOT/##" \ + | LC_ALL=C sort +} + +normalize_durations() { + local normalized=$1 + [ -f "$DURATIONS" ] || die "duration file not found: $DURATIONS" + if ! awk -F '\t' ' + /^[[:space:]]*#/ || /^[[:space:]]*$/ { next } + NF != 2 { + printf "duration row %d must be: tests/.test.sh\n", NR > "/dev/stderr" + bad = 1 + next + } + $1 !~ /^[1-9][0-9]*$/ { + printf "duration row %d has invalid milliseconds: %s\n", NR, $1 > "/dev/stderr" + bad = 1 + next + } + $2 !~ /^tests\/[A-Za-z0-9._-]+\.test\.sh$/ { + printf "duration row %d has invalid test path: %s\n", NR, $2 > "/dev/stderr" + bad = 1 + next + } + { print $1 "\t" $2 } + END { exit bad ? 1 : 0 } + ' "$DURATIONS" > "$normalized"; then + die "invalid duration file: $DURATIONS" + fi + [ -s "$normalized" ] || die "duration file has no test rows: $DURATIONS" +} + +validate_inventory() { + local normalized=$1 inventory_file=$2 paths_file=$3 duplicates + inventory > "$inventory_file" + cut -f2 "$normalized" | LC_ALL=C sort > "$paths_file" + duplicates=$(cut -f2 "$normalized" | LC_ALL=C sort | uniq -d) + if [ -n "$duplicates" ]; then + printf 'fm-behavior-shards: duplicate duration paths:\n%s\n' "$duplicates" >&2 + return 1 + fi + if ! diff -u "$inventory_file" "$paths_file" >&2; then + printf 'fm-behavior-shards: duration inventory does not exactly match tests/*.test.sh\n' >&2 + return 1 + fi +} + +make_plan() { + local shard_count=$1 destination=$2 normalized inventory_file paths_file + validate_positive_integer "$shard_count" "shard count" + normalized="$WORK/durations.tsv" + inventory_file="$WORK/inventory.txt" + paths_file="$WORK/duration-paths.txt" + normalize_durations "$normalized" + validate_inventory "$normalized" "$inventory_file" "$paths_file" \ + || die "coverage guard failed" + + LC_ALL=C sort -t "$TAB" -k1,1nr -k2,2 "$normalized" \ + | awk -F '\t' -v OFS='\t' -v shard_count="$shard_count" ' + BEGIN { + for (i = 1; i <= shard_count; i++) load[i] = 0 + } + { + target = 1 + for (i = 2; i <= shard_count; i++) { + if (load[i] < load[target]) target = i + } + print target, $1, $2 + load[target] += $1 + } + ' > "$destination" + + [ "$(wc -l < "$destination" | tr -d ' ')" -eq "$(wc -l < "$inventory_file" | tr -d ' ')" ] \ + || die "planner did not assign every test exactly once" +} + +print_plan_summary() { + local plan=$1 shard_count=$2 + awk -F '\t' -v shard_count="$shard_count" ' + { count[$1]++; load[$1] += $2 } + END { + for (i = 1; i <= shard_count; i++) { + printf "FM_BEHAVIOR_SHARD shard=%d tests=%d estimated_ms=%d\n", i, count[i] + 0, load[i] + 0 + } + } + ' "$plan" +} + +now_ms() { + local value + value=$(date +%s%3N 2>/dev/null || true) + case "$value" in + *[!0-9]*|'') printf '%s000\n' "$(date +%s)" ;; + *) printf '%s\n' "$value" ;; + esac +} + +run_test() { + local path=$1 begin_ms end_ms duration rc had_errexit=0 + begin_ms=$(now_ms) + printf 'FM_BEHAVIOR_TEST_BEGIN %s\n' "$path" + if [ "${GITHUB_ACTIONS:-}" = true ]; then + printf '::group::%s\n' "$path" + fi + case $- in *e*) had_errexit=1 ;; esac + set +e + "$ROOT/$path" + rc=$? + if [ "$had_errexit" -eq 1 ]; then + set -e + fi + if [ "${GITHUB_ACTIONS:-}" = true ]; then + printf '::endgroup::\n' + fi + end_ms=$(now_ms) + duration=$((end_ms - begin_ms)) + [ "$duration" -gt 0 ] || duration=1 + printf 'FM_BEHAVIOR_TEST_END %s exit=%s duration_ms=%s\n' "$path" "$rc" "$duration" + RUN_TEST_RC=$rc + RUN_TEST_DURATION=$duration +} + +run_shard() { + local shard=$1 shard_count=$2 manifest=$3 plan path plan_shard _estimate + local selected=0 failures=0 + validate_positive_integer "$shard" "shard" + validate_positive_integer "$shard_count" "shard count" + [ "$shard" -le "$shard_count" ] || die "shard $shard exceeds shard count $shard_count" + plan="$WORK/plan.tsv" + make_plan "$shard_count" "$plan" + mkdir -p "$(dirname "$manifest")" + : > "$manifest" + cd "$ROOT" || die "cannot enter repo root: $ROOT" + while IFS="$TAB" read -r plan_shard _estimate path; do + [ "$plan_shard" = "$shard" ] || continue + selected=$((selected + 1)) + run_test "$path" + printf '%s\t%s\t%s\t%s\n' "$shard" "$path" "$RUN_TEST_RC" "$RUN_TEST_DURATION" >> "$manifest" + [ "$RUN_TEST_RC" -eq 0 ] || failures=$((failures + 1)) + done < "$plan" + [ "$selected" -gt 0 ] || die "shard $shard has no assigned tests" + printf 'FM_BEHAVIOR_SHARD_RESULT shard=%s/%s tests=%s failed=%s manifest=%s\n' \ + "$shard" "$shard_count" "$selected" "$failures" "$manifest" + [ "$failures" -eq 0 ] +} + +record_durations() { + local destination=$1 staging path failed=0 + [ -n "$destination" ] || die "--record requires an output path" + staging="$WORK/recorded.tsv" + : > "$staging" + cd "$ROOT" || die "cannot enter repo root: $ROOT" + while IFS= read -r path; do + run_test "$path" + printf '%s\t%s\n' "$RUN_TEST_DURATION" "$path" >> "$staging" + [ "$RUN_TEST_RC" -eq 0 ] || failed=$((failed + 1)) + done < <(inventory) + if [ "$failed" -ne 0 ]; then + printf 'fm-behavior-shards: refusing to publish timings because %s tests failed\n' "$failed" >&2 + return 1 + fi + mkdir -p "$(dirname "$destination")" + { + printf '# Behavior test durations in milliseconds.\n' + printf '# Refresh with: bin/fm-behavior-shards.sh --record tests/behavior-test-durations.tsv\n' + printf '# Recorded UTC: %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" + LC_ALL=C sort -t "$TAB" -k2,2 "$staging" + } > "$destination.tmp.$$" + mv "$destination.tmp.$$" "$destination" + printf 'FM_BEHAVIOR_DURATIONS recorded=%s tests=%s\n' "$destination" "$(wc -l < "$staging" | tr -d ' ')" +} + +verify_manifests() { + local shard_count=$1 manifest_dir=$2 plan all observed expected file file_count shard + validate_positive_integer "$shard_count" "shard count" + [ -d "$manifest_dir" ] || die "manifest directory not found: $manifest_dir" + plan="$WORK/plan.tsv" + all="$WORK/executed.tsv" + observed="$WORK/observed.tsv" + expected="$WORK/expected.tsv" + make_plan "$shard_count" "$plan" + : > "$all" + + file_count=$(find "$manifest_dir" -maxdepth 1 -type f -name 'executed-*.tsv' | wc -l | tr -d ' ') + [ "$file_count" -eq "$shard_count" ] \ + || die "expected $shard_count executed manifests, found $file_count in $manifest_dir" + shard=1 + while [ "$shard" -le "$shard_count" ]; do + file="$manifest_dir/executed-$shard.tsv" + [ -s "$file" ] || die "missing or empty executed manifest for shard $shard: $file" + if ! awk -F '\t' -v expected_shard="$shard" ' + NF != 4 { printf "invalid manifest row %d in shard %d\n", NR, expected_shard > "/dev/stderr"; bad = 1; next } + $1 != expected_shard { printf "manifest row %d claims shard %s, expected %d\n", NR, $1, expected_shard > "/dev/stderr"; bad = 1 } + $2 !~ /^tests\/[A-Za-z0-9._-]+\.test\.sh$/ { printf "invalid executed path at row %d: %s\n", NR, $2 > "/dev/stderr"; bad = 1 } + $3 !~ /^[0-9]+$/ { printf "invalid exit code at row %d: %s\n", NR, $3 > "/dev/stderr"; bad = 1 } + $4 !~ /^[0-9]+$/ { printf "invalid duration at row %d: %s\n", NR, $4 > "/dev/stderr"; bad = 1 } + END { exit bad ? 1 : 0 } + ' "$file"; then + die "invalid executed manifest: $file" + fi + cat "$file" >> "$all" + shard=$((shard + 1)) + done + + awk -F '\t' -v OFS='\t' '{ print $1, $3 }' "$plan" | LC_ALL=C sort -t "$TAB" -k1,1n -k2,2 > "$expected" + awk -F '\t' -v OFS='\t' '{ print $1, $2 }' "$all" | LC_ALL=C sort -t "$TAB" -k1,1n -k2,2 > "$observed" + if ! diff -u "$expected" "$observed" >&2; then + die "executed shard union differs from the complete planned test inventory" + fi + if ! awk -F '\t' ' + $3 != 0 { printf "failed test: shard=%s path=%s exit=%s duration_ms=%s\n", $1, $2, $3, $4 > "/dev/stderr"; bad = 1 } + END { exit bad ? 1 : 0 } + ' "$all"; then + die "one or more behavior tests failed" + fi + printf 'FM_BEHAVIOR_COMPLETENESS ok tests=%s shards=%s\n' "$(wc -l < "$all" | tr -d ' ')" "$shard_count" +} + +case "${1:-}" in + --check) + [ "$#" -eq 2 ] || die "--check requires " + plan="$WORK/plan.tsv" + make_plan "$2" "$plan" + printf 'FM_BEHAVIOR_PLAN ok tests=%s shards=%s\n' "$(wc -l < "$plan" | tr -d ' ')" "$2" + print_plan_summary "$plan" "$2" + ;; + --plan) + [ "$#" -eq 2 ] || die "--plan requires " + make_plan "$2" "$WORK/plan.tsv" + cat "$WORK/plan.tsv" + ;; + --run) + [ "$#" -eq 4 ] || die "--run requires " + run_shard "$2" "$3" "$4" + ;; + --verify) + [ "$#" -eq 3 ] || die "--verify requires " + verify_manifests "$2" "$3" + ;; + --record) + [ "$#" -eq 2 ] || die "--record requires " + record_durations "$2" + ;; + -h|--help) + usage + ;; + *) + usage >&2 + exit 2 + ;; +esac diff --git a/docs/ci-behavior-shards.md b/docs/ci-behavior-shards.md new file mode 100644 index 0000000000..d7283fb031 --- /dev/null +++ b/docs/ci-behavior-shards.md @@ -0,0 +1,71 @@ +# Behavior test sharding + +`bin/fm-behavior-shards.sh` is the single owner of behavior-test shard planning, execution manifests, timing refresh, and completeness verification. + +## Baseline + +The 2026-07-31 `main` behavior job ran all 80 scripts serially from 16:41:30 UTC to 17:38:53 UTC, or 57 minutes 23 seconds. + +The source run is [GitHub Actions run 30648119449](https://github.com/ruby-dlee/firstmate/actions/runs/30648119449). + +`tests/behavior-test-durations.tsv` records the per-script measurements derived from that run. + +The runner contract test added with the sharding implementation is included with a conservative two-second initial estimate, so the completeness guard covers it immediately. + +## Assignment + +The planner uses deterministic longest-processing-time assignment. + +It sorts by descending recorded duration, uses the test path as the stable secondary key, assigns the next test to the currently lightest shard, and breaks load ties by the lowest shard number. + +The six-shard checked-in plan has these estimated serial loads. + +| Shard | Tests | Estimated load | +|---:|---:|---:| +| 1 | 1 | 673400 ms (11m13s) | +| 2 | 1 | 620746 ms (10m21s) | +| 3 | 1 | 563637 ms (9m24s) | +| 4 | 22 | 528938 ms (8m49s) | +| 5 | 28 | 528929 ms (8m49s) | +| 6 | 28 | 528933 ms (8m49s) | + +The expected healthy critical path is therefore about 11 minutes rather than 57 minutes. + +The largest single test file, `tests/fm-account-routing.test.sh`, sets the remaining floor. + +## Coverage guard + +`bin/fm-behavior-shards.sh --check 6` fails when the duration inventory has a missing path, duplicate path, malformed duration, or any difference from the complete `tests/*.test.sh` inventory. + +Every matrix runner writes an executed manifest while continuing through all assigned scripts and preserving each exit code. + +The final `Behavior tests` job downloads all six manifests and runs `bin/fm-behavior-shards.sh --verify 6 `. + +Verification fails for a missing shard, missing test, duplicate test, wrong shard assignment, malformed row, or recorded test failure. + +Shard artifacts are replaceable across workflow attempts, so rerunning failed jobs refreshes the failed shard while retaining successful shard evidence from the same workflow run. + +The historical required-check name stays intact on this final union guard. + +## Isolation and failure output + +GitHub Actions gives every matrix leg a separate virtual machine, so shards do not share processes, ports, locks, paths, or terminal servers. + +The workflow also assigns each shard private mode-0700 `TMPDIR` and `TMUX_TMPDIR` roots under that runner's `RUNNER_TEMP`. + +Scripts remain serial within a shard, so no existing test needed weaker assertions, a mock conversion, a skip, a retry, or an added sleep. + +Each matrix job is named `Behavior tests (shard N/6)`, and the runner emits explicit begin and end markers containing the test path and exit code. + +The 15-minute per-shard timeout leaves bounded margin above the measured 11m13s slowest shard while replacing the prior 90-minute blanket. + +## Refreshing timings + +Run the complete suite serially in the target environment and atomically replace the duration data only if every test passes. + +```sh +bin/fm-behavior-shards.sh --record tests/behavior-test-durations.tsv +bin/fm-behavior-shards.sh --check 6 +``` + +Review the new `--check` load summary before committing refreshed timings. diff --git a/tests/behavior-test-durations.tsv b/tests/behavior-test-durations.tsv new file mode 100644 index 0000000000..0de22a8e8c --- /dev/null +++ b/tests/behavior-test-durations.tsv @@ -0,0 +1,84 @@ +# Behavior test durations in milliseconds. +# Refresh with: bin/fm-behavior-shards.sh --record tests/behavior-test-durations.tsv +# Recorded from https://github.com/ruby-dlee/firstmate/actions/runs/30648119449 on 2026-07-31. +55000 tests/bridge-cutover-python.test.sh +71100 tests/fm-account-directory.test.sh +673400 tests/fm-account-routing.test.sh +33600 tests/fm-afk-inject-e2e.test.sh +20 tests/fm-afk-inject-herdr-e2e.test.sh +16500 tests/fm-afk-launch.test.sh +6900 tests/fm-afk-reap-wake-e2e.test.sh +26565 tests/fm-arm-pretool-check.test.sh +5950 tests/fm-autocompact.test.sh +20 tests/fm-backend-autodetect-smoke.test.sh +20 tests/fm-backend-cmux-smoke.test.sh +3400 tests/fm-backend-cmux.test.sh +20 tests/fm-backend-herdr-eventwait-smoke.test.sh +20 tests/fm-backend-herdr-prune-safety-e2e.test.sh +20 tests/fm-backend-herdr-respawn-idem-e2e.test.sh +20 tests/fm-backend-herdr-smoke.test.sh +20 tests/fm-backend-herdr-workspace-per-home-e2e.test.sh +148500 tests/fm-backend-herdr.test.sh +12746 tests/fm-backend-orca.test.sh +2573 tests/fm-backend-tmux-smoke.test.sh +20 tests/fm-backend-zellij-smoke.test.sh +3820 tests/fm-backend-zellij.test.sh +25568 tests/fm-backend.test.sh +2000 tests/fm-behavior-shards.test.sh +5222 tests/fm-backlog-handoff.test.sh +17908 tests/fm-bearings-snapshot.test.sh +9402 tests/fm-bootstrap.test.sh +1875 tests/fm-brief.test.sh +13892 tests/fm-cd-pretool-check.test.sh +475500 tests/fm-checkout-refresh.test.sh +336 tests/fm-checkout-return-boundary.test.sh +263 tests/fm-composer-ghost.test.sh +30 tests/fm-composer-lib.test.sh +10055 tests/fm-crew-state.test.sh +13991 tests/fm-daemon.test.sh +685 tests/fm-dispatch-select.test.sh +156 tests/fm-ensure-agents-md.test.sh +3009 tests/fm-fleet-snapshot-view.test.sh +50266 tests/fm-fleet-sync.test.sh +8940 tests/fm-gate-refuse.test.sh +25 tests/fm-gotmp.test.sh +3459 tests/fm-grok-harness.test.sh +6995 tests/fm-herdr-lab.test.sh +98 tests/fm-lint.test.sh +187 tests/fm-lock.test.sh +1686 tests/fm-macos-permissions.test.sh +10 tests/fm-pi-primary-live-e2e.test.sh +80 tests/fm-pi-primary-types.test.sh +3170 tests/fm-pi-watch-extension.test.sh +1574 tests/fm-pr-merge.test.sh +802 tests/fm-prompt-exec.test.sh +620746 tests/fm-report-stack.test.sh +495 tests/fm-review-diff.test.sh +26463 tests/fm-secondmate-harness.test.sh +37513 tests/fm-secondmate-lifecycle-e2e.test.sh +6623 tests/fm-secondmate-liveness.test.sh +193035 tests/fm-secondmate-safety.test.sh +5767 tests/fm-secondmate-sync.test.sh +1337 tests/fm-send-popup-settle.test.sh +14 tests/fm-send-secondmate-marker-herdr-e2e.test.sh +1481 tests/fm-send-secondmate-marker.test.sh +223 tests/fm-send-settle.test.sh +18168 tests/fm-send-strict.test.sh +6911 tests/fm-session-start.test.sh +2094 tests/fm-spawn-batch.test.sh +42452 tests/fm-spawn-dispatch-profile.test.sh +42 tests/fm-stow-contract.test.sh +500 tests/fm-supervision-events.test.sh +95 tests/fm-supervision-instructions.test.sh +10408 tests/fm-tangle-guard.test.sh +563637 tests/fm-teardown.test.sh +600 tests/fm-transition-lib.test.sh +1849 tests/fm-turnend-guard.test.sh +1511 tests/fm-update.test.sh +4597 tests/fm-wake-daemon-lifecycle-e2e.test.sh +36178 tests/fm-wake-queue.test.sh +4510 tests/fm-watch-checkpoint.test.sh +95645 tests/fm-watch-triage.test.sh +20435 tests/fm-watcher-lock.test.sh +23744 tests/fm-x-mode.test.sh +92 tests/operating-fundamentals.test.sh diff --git a/tests/fm-behavior-shards.test.sh b/tests/fm-behavior-shards.test.sh new file mode 100755 index 0000000000..be751b883a --- /dev/null +++ b/tests/fm-behavior-shards.test.sh @@ -0,0 +1,231 @@ +#!/usr/bin/env bash +# Behavior tests for deterministic duration-balanced CI sharding and the +# post-execution completeness guard. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +SHARDER="$ROOT/bin/fm-behavior-shards.sh" +DURATIONS="$ROOT/tests/behavior-test-durations.tsv" +CI="$ROOT/.github/workflows/ci.yml" +SHARD_COUNT=6 + +test_checked_in_plan_is_complete_balanced_and_deterministic() { + local tmp plan_a plan_b inventory planned out + tmp=$(fm_test_tmproot fm-behavior-plan) + plan_a="$tmp/plan-a.tsv" + plan_b="$tmp/plan-b.tsv" + inventory="$tmp/inventory.txt" + planned="$tmp/planned.txt" + + out=$("$SHARDER" --check "$SHARD_COUNT") \ + || fail "checked-in behavior shard plan failed its coverage guard" + assert_contains "$out" "FM_BEHAVIOR_PLAN ok tests=81 shards=6" \ + "coverage guard did not report the complete 81-test inventory" + "$SHARDER" --plan "$SHARD_COUNT" > "$plan_a" + "$SHARDER" --plan "$SHARD_COUNT" > "$plan_b" + cmp -s "$plan_a" "$plan_b" || fail "same durations produced different shard plans" + + find "$ROOT/tests" -maxdepth 1 -type f -name '*.test.sh' -print \ + | sed "s#^$ROOT/##" | LC_ALL=C sort > "$inventory" + cut -f3 "$plan_a" | LC_ALL=C sort > "$planned" + cmp -s "$inventory" "$planned" \ + || fail "planned shard union did not equal tests/*.test.sh" + [ "$(cut -f3 "$plan_a" | LC_ALL=C sort | uniq -d | wc -l | tr -d ' ')" -eq 0 ] \ + || fail "a test was assigned to more than one shard" + [ "$(awk -F '\t' '{ load[$1] += $2 } END { max=0; for (s in load) if (load[s] > max) max=load[s]; print max }' "$plan_a")" -le 675000 ] \ + || fail "duration-balanced plan exceeds the measured single-test floor" + pass "checked-in LPT plan is deterministic, complete, disjoint, and duration-balanced" +} + +test_plan_refuses_missing_and_duplicate_duration_entries() { + local tmp missing duplicate out rc + tmp=$(fm_test_tmproot fm-behavior-plan-invalid) + missing="$tmp/missing.tsv" + duplicate="$tmp/duplicate.tsv" + grep -vF $'\ttests/fm-x-mode.test.sh' "$DURATIONS" > "$missing" + cp "$DURATIONS" "$duplicate" + grep -F $'\ttests/fm-x-mode.test.sh' "$DURATIONS" >> "$duplicate" + + rc=0 + out=$(FM_BEHAVIOR_DURATIONS_FILE="$missing" "$SHARDER" --check "$SHARD_COUNT" 2>&1) || rc=$? + [ "$rc" -ne 0 ] || fail "coverage guard accepted a missing duration entry" + assert_contains "$out" "duration inventory does not exactly match" \ + "missing-entry refusal did not name the inventory mismatch" + + rc=0 + out=$(FM_BEHAVIOR_DURATIONS_FILE="$duplicate" "$SHARDER" --check "$SHARD_COUNT" 2>&1) || rc=$? + [ "$rc" -ne 0 ] || fail "coverage guard accepted a duplicate duration entry" + assert_contains "$out" "duplicate duration paths" \ + "duplicate-entry refusal did not name the duplicate" + pass "coverage guard rejects missing and duplicate duration entries" +} + +make_runner_fixture() { + local root=$1 + mkdir -p "$root/bin" "$root/tests" + cp "$SHARDER" "$root/bin/fm-behavior-shards.sh" + chmod +x "$root/bin/fm-behavior-shards.sh" + cat > "$root/tests/one.test.sh" <<'SH' +#!/usr/bin/env bash +printf 'one\n' >> "${FM_FIXTURE_EXECUTED:?}" +printf 'ok - fixture one\n' +SH + cat > "$root/tests/two.test.sh" <<'SH' +#!/usr/bin/env bash +printf 'two\n' >> "${FM_FIXTURE_EXECUTED:?}" +printf 'ok - fixture two\n' +SH + chmod +x "$root/tests/one.test.sh" "$root/tests/two.test.sh" + cat > "$root/tests/behavior-test-durations.tsv" <<'EOF_DURATIONS' +10 tests/one.test.sh +20 tests/two.test.sh +EOF_DURATIONS +} + +test_runner_executes_every_assigned_test_and_records_failures() { + local tmp fixture manifest executed out rc + tmp=$(fm_test_tmproot fm-behavior-run) + fixture="$tmp/fixture" + manifest="$tmp/executed-1.tsv" + executed="$tmp/executed.log" + make_runner_fixture "$fixture" + : > "$executed" + + FM_FIXTURE_EXECUTED="$executed" "$fixture/bin/fm-behavior-shards.sh" \ + --run 1 1 "$manifest" > "$tmp/success.out" \ + || fail "fixture shard failed despite two passing tests" + [ "$(wc -l < "$manifest" | tr -d ' ')" -eq 2 ] \ + || fail "successful shard manifest did not record both tests" + [ "$(wc -l < "$executed" | tr -d ' ')" -eq 2 ] \ + || fail "successful shard did not execute both assigned tests" + + cat > "$fixture/tests/two.test.sh" <<'SH' +#!/usr/bin/env bash +printf 'two\n' >> "${FM_FIXTURE_EXECUTED:?}" +printf 'not ok - fixture two\n' >&2 +exit 7 +SH + chmod +x "$fixture/tests/two.test.sh" + : > "$executed" + rc=0 + out=$(FM_FIXTURE_EXECUTED="$executed" "$fixture/bin/fm-behavior-shards.sh" \ + --run 1 1 "$manifest" 2>&1) || rc=$? + [ "$rc" -ne 0 ] || fail "shard runner hid a failing test" + [ "$(wc -l < "$manifest" | tr -d ' ')" -eq 2 ] \ + || fail "failing shard stopped before recording every assigned test" + [ "$(awk -F '\t' '$3 == 7 { count++ } END { print count + 0 }' "$manifest")" -eq 1 ] \ + || fail "failing test exit code was not preserved in the manifest" + assert_contains "$out" "FM_BEHAVIOR_TEST_BEGIN tests/two.test.sh" \ + "failure output did not name the failing test" + assert_contains "$out" "FM_BEHAVIOR_SHARD_RESULT shard=1/1 tests=2 failed=1" \ + "failure output did not identify the failing shard" + pass "shard runner names tests, runs the full assignment, and preserves failures" +} + +test_record_refreshes_complete_fixture_timings() { + local tmp fixture recorded executed + tmp=$(fm_test_tmproot fm-behavior-record) + fixture="$tmp/fixture" + recorded="$tmp/refreshed.tsv" + executed="$tmp/executed.log" + make_runner_fixture "$fixture" + : > "$executed" + FM_FIXTURE_EXECUTED="$executed" "$fixture/bin/fm-behavior-shards.sh" \ + --record "$recorded" > "$tmp/record.out" \ + || fail "timing refresh failed for passing fixture tests" + grep -Fq $'\ttests/one.test.sh' "$recorded" \ + || fail "timing refresh omitted fixture one" + grep -Fq $'\ttests/two.test.sh' "$recorded" \ + || fail "timing refresh omitted fixture two" + [ "$(grep -c $'^[1-9][0-9]*\ttests/' "$recorded")" -eq 2 ] \ + || fail "timing refresh did not emit positive milliseconds for both tests" + pass "timing data is checked in and refreshable through the real runner" +} + +write_complete_manifests() { + local plan=$1 dir=$2 shard ms path + mkdir -p "$dir" + while IFS=$'\t' read -r shard ms path; do + printf '%s\t%s\t0\t%s\n' "$shard" "$path" "$ms" >> "$dir/executed-$shard.tsv" + done < "$plan" +} + +test_post_run_guard_requires_the_exact_executed_union() { + local tmp plan good missing duplicate failed out rc first_file first_row + tmp=$(fm_test_tmproot fm-behavior-verify) + plan="$tmp/plan.tsv" + good="$tmp/good" + missing="$tmp/missing" + duplicate="$tmp/duplicate" + failed="$tmp/failed" + "$SHARDER" --plan "$SHARD_COUNT" > "$plan" + write_complete_manifests "$plan" "$good" + out=$("$SHARDER" --verify "$SHARD_COUNT" "$good") \ + || fail "post-run guard rejected the exact complete manifest union" + assert_contains "$out" "FM_BEHAVIOR_COMPLETENESS ok tests=81 shards=6" \ + "post-run guard did not report complete execution" + + cp -R "$good" "$missing" + first_file=$(find "$missing" -type f -name 'executed-*.tsv' -print \ + | while IFS= read -r file; do printf '%s\t%s\n' "$(wc -l < "$file" | tr -d ' ')" "$file"; done \ + | LC_ALL=C sort -t $'\t' -k1,1nr -k2,2 | head -n 1 | cut -f2-) + sed '$d' "$first_file" > "$first_file.tmp" + mv "$first_file.tmp" "$first_file" + rc=0 + out=$("$SHARDER" --verify "$SHARD_COUNT" "$missing" 2>&1) || rc=$? + [ "$rc" -ne 0 ] || fail "post-run guard accepted a missing execution row" + assert_contains "$out" "executed shard union differs" \ + "missing-execution refusal did not name the union mismatch" + + cp -R "$good" "$duplicate" + first_file=$(find "$duplicate" -type f -name 'executed-*.tsv' | LC_ALL=C sort | head -n 1) + first_row=$(head -n 1 "$first_file") + printf '%s\n' "$first_row" >> "$first_file" + rc=0 + "$SHARDER" --verify "$SHARD_COUNT" "$duplicate" >/dev/null 2>&1 || rc=$? + [ "$rc" -ne 0 ] || fail "post-run guard accepted a duplicate execution row" + + cp -R "$good" "$failed" + first_file=$(find "$failed" -type f -name 'executed-*.tsv' | LC_ALL=C sort | head -n 1) + awk -F '\t' -v OFS='\t' 'NR == 1 { $3 = 9 } { print }' "$first_file" > "$first_file.tmp" + mv "$first_file.tmp" "$first_file" + rc=0 + out=$("$SHARDER" --verify "$SHARD_COUNT" "$failed" 2>&1) || rc=$? + [ "$rc" -ne 0 ] || fail "post-run guard hid a recorded test failure" + assert_contains "$out" "failed test: shard=" \ + "recorded failure did not name its shard and test" + pass "post-run guard rejects missing, duplicate, and failed executions" +} + +test_ci_wires_matrix_isolation_timeout_and_union_verification() { + # shellcheck disable=SC2016 # The GitHub expression is a literal YAML needle. + assert_grep 'name: Behavior tests (shard ${{ matrix.shard }}/6)' "$CI" \ + "CI does not expose the failing shard in the job name" + assert_grep 'shard: [1, 2, 3, 4, 5, 6]' "$CI" \ + "CI does not launch every deterministic shard" + [ "$(grep -Fc 'timeout-minutes: 15' "$CI")" -eq 1 ] \ + || fail "behavior matrix must have one 15-minute per-shard timeout" + # shellcheck disable=SC2016 # Workflow shell variables must remain literal. + assert_grep 'echo "TMPDIR=$shard_root/tmp"' "$CI" \ + "CI does not give each shard a private temp root" + # shellcheck disable=SC2016 # Workflow shell variables must remain literal. + assert_grep 'echo "TMUX_TMPDIR=$shard_root/tmux"' "$CI" \ + "CI does not give each shard a private tmux socket root" + # shellcheck disable=SC2016 # Workflow shell variables must remain literal. + assert_grep 'bin/fm-behavior-shards.sh --verify 6 "$RUNNER_TEMP/behavior-manifests"' "$CI" \ + "CI does not verify the union of executed manifests" + assert_grep 'overwrite: true' "$CI" \ + "CI cannot refresh a failed shard manifest during a failed-jobs rerun" + assert_no_grep 'for test_script in tests/*.test.sh' "$CI" \ + "CI retained the 57-minute serial behavior loop" + pass "CI wires six named isolated shards, a tight timeout, and executed-union verification" +} + +test_checked_in_plan_is_complete_balanced_and_deterministic +test_plan_refuses_missing_and_duplicate_duration_entries +test_runner_executes_every_assigned_test_and_records_failures +test_record_refreshes_complete_fixture_timings +test_post_run_guard_requires_the_exact_executed_union +test_ci_wires_matrix_isolation_timeout_and_union_verification From b1d7caed0ad14a85f3288b62e48f2baaf31d4f67 Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Fri, 31 Jul 2026 21:35:09 -0400 Subject: [PATCH 02/10] test: cover lavish in behavior shards --- tests/behavior-test-durations.tsv | 1 + tests/fm-behavior-shards.test.sh | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/behavior-test-durations.tsv b/tests/behavior-test-durations.tsv index 0de22a8e8c..780312a790 100644 --- a/tests/behavior-test-durations.tsv +++ b/tests/behavior-test-durations.tsv @@ -81,4 +81,5 @@ 95645 tests/fm-watch-triage.test.sh 20435 tests/fm-watcher-lock.test.sh 23744 tests/fm-x-mode.test.sh +6200 tests/lavish.test.sh 92 tests/operating-fundamentals.test.sh diff --git a/tests/fm-behavior-shards.test.sh b/tests/fm-behavior-shards.test.sh index be751b883a..0ecd595a1e 100755 --- a/tests/fm-behavior-shards.test.sh +++ b/tests/fm-behavior-shards.test.sh @@ -21,8 +21,8 @@ test_checked_in_plan_is_complete_balanced_and_deterministic() { out=$("$SHARDER" --check "$SHARD_COUNT") \ || fail "checked-in behavior shard plan failed its coverage guard" - assert_contains "$out" "FM_BEHAVIOR_PLAN ok tests=81 shards=6" \ - "coverage guard did not report the complete 81-test inventory" + assert_contains "$out" "FM_BEHAVIOR_PLAN ok tests=82 shards=6" \ + "coverage guard did not report the complete 82-test inventory" "$SHARDER" --plan "$SHARD_COUNT" > "$plan_a" "$SHARDER" --plan "$SHARD_COUNT" > "$plan_b" cmp -s "$plan_a" "$plan_b" || fail "same durations produced different shard plans" @@ -164,7 +164,7 @@ test_post_run_guard_requires_the_exact_executed_union() { write_complete_manifests "$plan" "$good" out=$("$SHARDER" --verify "$SHARD_COUNT" "$good") \ || fail "post-run guard rejected the exact complete manifest union" - assert_contains "$out" "FM_BEHAVIOR_COMPLETENESS ok tests=81 shards=6" \ + assert_contains "$out" "FM_BEHAVIOR_COMPLETENESS ok tests=82 shards=6" \ "post-run guard did not report complete execution" cp -R "$good" "$missing" From cf364e8d9e9420489a600aee2df588be325b0c59 Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Fri, 31 Jul 2026 21:57:09 -0400 Subject: [PATCH 03/10] no-mistakes(review): Rebalance shards and document teardown root cause --- .github/workflows/ci.yml | 12 +- docs/ci-behavior-shards.md | 46 ++-- tests/behavior-test-durations.tsv | 6 +- tests/fm-account-routing-a.test.sh | 3 + tests/fm-account-routing-b.test.sh | 3 + ...ng.test.sh => fm-account-routing-suite.sh} | 9 + tests/fm-behavior-shards.test.sh | 16 +- tests/fm-report-stack-a.test.sh | 3 + tests/fm-report-stack-b.test.sh | 3 + ...stack.test.sh => fm-report-stack-suite.sh} | 197 ++++++++++-------- 10 files changed, 175 insertions(+), 123 deletions(-) create mode 100755 tests/fm-account-routing-a.test.sh create mode 100755 tests/fm-account-routing-b.test.sh rename tests/{fm-account-routing.test.sh => fm-account-routing-suite.sh} (99%) mode change 100755 => 100644 create mode 100755 tests/fm-report-stack-a.test.sh create mode 100755 tests/fm-report-stack-b.test.sh rename tests/{fm-report-stack.test.sh => fm-report-stack-suite.sh} (96%) mode change 100755 => 100644 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d7dcfd9ec..ab8813e473 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,17 +30,17 @@ jobs: steps: - uses: actions/checkout@v6 - name: Prove complete, disjoint shard coverage - run: bin/fm-behavior-shards.sh --check 6 + run: bin/fm-behavior-shards.sh --check 8 behavior-tests: - name: Behavior tests (shard ${{ matrix.shard }}/6) + name: Behavior tests (shard ${{ matrix.shard }}/8) needs: behavior-test-plan runs-on: ubuntu-latest timeout-minutes: 15 strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6] + shard: [1, 2, 3, 4, 5, 6, 7, 8] steps: - uses: actions/checkout@v6 with: @@ -77,11 +77,11 @@ jobs: echo "TEMP=$shard_root/tmp" echo "TMUX_TMPDIR=$shard_root/tmux" } >> "$GITHUB_ENV" - - name: Run behavior shard ${{ matrix.shard }}/6 + - name: Run behavior shard ${{ matrix.shard }}/8 run: | set -eu manifest="$RUNNER_TEMP/executed-${{ matrix.shard }}.tsv" - bin/fm-behavior-shards.sh --run "${{ matrix.shard }}" 6 "$manifest" + bin/fm-behavior-shards.sh --run "${{ matrix.shard }}" 8 "$manifest" - name: Upload executed-test manifest if: always() uses: actions/upload-artifact@v4 @@ -108,7 +108,7 @@ jobs: path: ${{ runner.temp }}/behavior-manifests merge-multiple: true - name: Verify complete execution union - run: bin/fm-behavior-shards.sh --verify 6 "$RUNNER_TEMP/behavior-manifests" + run: bin/fm-behavior-shards.sh --verify 8 "$RUNNER_TEMP/behavior-manifests" agent-fleet: name: Agent Fleet package diff --git a/docs/ci-behavior-shards.md b/docs/ci-behavior-shards.md index d7283fb031..b040706bf8 100644 --- a/docs/ci-behavior-shards.md +++ b/docs/ci-behavior-shards.md @@ -10,7 +10,7 @@ The source run is [GitHub Actions run 30648119449](https://github.com/ruby-dlee/ `tests/behavior-test-durations.tsv` records the per-script measurements derived from that run. -The runner contract test added with the sharding implementation is included with a conservative two-second initial estimate, so the completeness guard covers it immediately. +The runner contract test added with the sharding implementation and the newly landed `tests/lavish.test.sh` are included, so the completeness guard covers both immediately. ## Assignment @@ -18,28 +18,32 @@ The planner uses deterministic longest-processing-time assignment. It sorts by descending recorded duration, uses the test path as the stable secondary key, assigns the next test to the currently lightest shard, and breaks load ties by the lowest shard number. -The six-shard checked-in plan has these estimated serial loads. +The eight-shard checked-in plan has these estimated serial loads. | Shard | Tests | Estimated load | |---:|---:|---:| -| 1 | 1 | 673400 ms (11m13s) | -| 2 | 1 | 620746 ms (10m21s) | -| 3 | 1 | 563637 ms (9m24s) | -| 4 | 22 | 528938 ms (8m49s) | -| 5 | 28 | 528929 ms (8m49s) | -| 6 | 28 | 528933 ms (8m49s) | +| 1 | 1 | 563637 ms (9m24s) | +| 2 | 1 | 475500 ms (7m56s) | +| 3 | 13 | 401927 ms (6m42s) | +| 4 | 13 | 401942 ms (6m42s) | +| 5 | 13 | 401927 ms (6m42s) | +| 6 | 10 | 401979 ms (6m42s) | +| 7 | 16 | 401935 ms (6m42s) | +| 8 | 17 | 401936 ms (6m42s) | -The expected healthy critical path is therefore about 11 minutes rather than 57 minutes. +The expected healthy behavior-execution critical path is 9 minutes 24 seconds rather than 57 minutes 23 seconds. -The largest single test file, `tests/fm-account-routing.test.sh`, sets the remaining floor. +The account-routing and report-stack suites keep their original test functions and assertions in shared suite files, while two runner wrappers partition each call list deterministically between isolated runners. + +The largest remaining indivisible test file, `tests/fm-teardown.test.sh`, sets the 9-minute-24-second floor. ## Coverage guard -`bin/fm-behavior-shards.sh --check 6` fails when the duration inventory has a missing path, duplicate path, malformed duration, or any difference from the complete `tests/*.test.sh` inventory. +`bin/fm-behavior-shards.sh --check 8` fails when the duration inventory has a missing path, duplicate path, malformed duration, or any difference from the complete `tests/*.test.sh` inventory. Every matrix runner writes an executed manifest while continuing through all assigned scripts and preserving each exit code. -The final `Behavior tests` job downloads all six manifests and runs `bin/fm-behavior-shards.sh --verify 6 `. +The final `Behavior tests` job downloads all eight manifests and runs `bin/fm-behavior-shards.sh --verify 8 `. Verification fails for a missing shard, missing test, duplicate test, wrong shard assignment, malformed row, or recorded test failure. @@ -55,9 +59,21 @@ The workflow also assigns each shard private mode-0700 `TMPDIR` and `TMUX_TMPDIR Scripts remain serial within a shard, so no existing test needed weaker assertions, a mock conversion, a skip, a retry, or an added sleep. -Each matrix job is named `Behavior tests (shard N/6)`, and the runner emits explicit begin and end markers containing the test path and exit code. +Each matrix job is named `Behavior tests (shard N/8)`, and the runner emits explicit begin and end markers containing the test path and exit code. + +The 15-minute per-shard timeout leaves bounded margin above the 9m24s slowest planned shard while replacing the prior 90-minute blanket. + +## Teardown child-endpoint investigation + +GitHub Actions run 30649486198 failed at 17:52:54 UTC after `test_forced_secondmate_child_uses_child_home_for_endpoint_verification` correctly retained the Agent Fleet lease, child metadata, and child worktree but its final message assertion expected the endpoint state to be `unknown`. + +The Zellij fixture returns a live session, pane, and tab from the child firstmate home, so endpoint discovery deterministically classifies it as `present` before destructive cleanup and emits `managed endpoint ... is still alive`. + +The failure was therefore an assertion-ordering mismatch introduced when the fixture became capable of proving presence, not a delayed endpoint shutdown. + +Commit `d7db2dcb010cc48f4ca6d34b4386d934e6c9cdde` fixes the assertion to match the proven state while retaining the refusal and all three containment checks. -The 15-minute per-shard timeout leaves bounded margin above the measured 11m13s slowest shard while replacing the prior 90-minute blanket. +GitHub Actions run 30657355610 executed the corrected sequence at 19:51:41 UTC and passed the full behavior job without a retry or added sleep. ## Refreshing timings @@ -65,7 +81,7 @@ Run the complete suite serially in the target environment and atomically replace ```sh bin/fm-behavior-shards.sh --record tests/behavior-test-durations.tsv -bin/fm-behavior-shards.sh --check 6 +bin/fm-behavior-shards.sh --check 8 ``` Review the new `--check` load summary before committing refreshed timings. diff --git a/tests/behavior-test-durations.tsv b/tests/behavior-test-durations.tsv index 780312a790..60df56f456 100644 --- a/tests/behavior-test-durations.tsv +++ b/tests/behavior-test-durations.tsv @@ -3,7 +3,8 @@ # Recorded from https://github.com/ruby-dlee/firstmate/actions/runs/30648119449 on 2026-07-31. 55000 tests/bridge-cutover-python.test.sh 71100 tests/fm-account-directory.test.sh -673400 tests/fm-account-routing.test.sh +336700 tests/fm-account-routing-a.test.sh +336700 tests/fm-account-routing-b.test.sh 33600 tests/fm-afk-inject-e2e.test.sh 20 tests/fm-afk-inject-herdr-e2e.test.sh 16500 tests/fm-afk-launch.test.sh @@ -52,7 +53,8 @@ 3170 tests/fm-pi-watch-extension.test.sh 1574 tests/fm-pr-merge.test.sh 802 tests/fm-prompt-exec.test.sh -620746 tests/fm-report-stack.test.sh +310373 tests/fm-report-stack-a.test.sh +310373 tests/fm-report-stack-b.test.sh 495 tests/fm-review-diff.test.sh 26463 tests/fm-secondmate-harness.test.sh 37513 tests/fm-secondmate-lifecycle-e2e.test.sh diff --git a/tests/fm-account-routing-a.test.sh b/tests/fm-account-routing-a.test.sh new file mode 100755 index 0000000000..f3b95c8a1b --- /dev/null +++ b/tests/fm-account-routing-a.test.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -u +FM_TEST_PART_INDEX=1 FM_TEST_PART_TOTAL=2 . "$(dirname "${BASH_SOURCE[0]}")/fm-account-routing-suite.sh" diff --git a/tests/fm-account-routing-b.test.sh b/tests/fm-account-routing-b.test.sh new file mode 100755 index 0000000000..d03951790b --- /dev/null +++ b/tests/fm-account-routing-b.test.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -u +FM_TEST_PART_INDEX=2 FM_TEST_PART_TOTAL=2 . "$(dirname "${BASH_SOURCE[0]}")/fm-account-routing-suite.sh" diff --git a/tests/fm-account-routing.test.sh b/tests/fm-account-routing-suite.sh old mode 100755 new mode 100644 similarity index 99% rename from tests/fm-account-routing.test.sh rename to tests/fm-account-routing-suite.sh index 4dcbb8eb66..de63e19601 --- a/tests/fm-account-routing.test.sh +++ b/tests/fm-account-routing-suite.sh @@ -5939,11 +5939,20 @@ run_isolated_test() { # a condition context suppresses errexit for the whole test body, so an # in-test `set -e` would be silently ignored. local status + FM_TEST_PART_SEQUENCE=$((FM_TEST_PART_SEQUENCE + 1)) + if [ "$FM_TEST_PART_TOTAL" -gt 1 ] \ + && [ $(((FM_TEST_PART_SEQUENCE - 1) % FM_TEST_PART_TOTAL + 1)) -ne "$FM_TEST_PART_INDEX" ]; then + return 0 + fi ( "$@" ) status=$? [ "$status" -eq 0 ] || exit "$status" } +FM_TEST_PART_INDEX=${FM_TEST_PART_INDEX:-1} +FM_TEST_PART_TOTAL=${FM_TEST_PART_TOTAL:-1} +FM_TEST_PART_SEQUENCE=0 + if [ "${FM_TEST_FOCUSED:-}" = stale-reclaim-generation ]; then run_isolated_test test_stale_reclaim_guard_is_owned_before_lock_removal exit 0 diff --git a/tests/fm-behavior-shards.test.sh b/tests/fm-behavior-shards.test.sh index 0ecd595a1e..b54c07d2a3 100755 --- a/tests/fm-behavior-shards.test.sh +++ b/tests/fm-behavior-shards.test.sh @@ -9,7 +9,7 @@ set -u SHARDER="$ROOT/bin/fm-behavior-shards.sh" DURATIONS="$ROOT/tests/behavior-test-durations.tsv" CI="$ROOT/.github/workflows/ci.yml" -SHARD_COUNT=6 +SHARD_COUNT=8 test_checked_in_plan_is_complete_balanced_and_deterministic() { local tmp plan_a plan_b inventory planned out @@ -21,8 +21,8 @@ test_checked_in_plan_is_complete_balanced_and_deterministic() { out=$("$SHARDER" --check "$SHARD_COUNT") \ || fail "checked-in behavior shard plan failed its coverage guard" - assert_contains "$out" "FM_BEHAVIOR_PLAN ok tests=82 shards=6" \ - "coverage guard did not report the complete 82-test inventory" + assert_contains "$out" "FM_BEHAVIOR_PLAN ok tests=84 shards=8" \ + "coverage guard did not report the complete 84-test inventory" "$SHARDER" --plan "$SHARD_COUNT" > "$plan_a" "$SHARDER" --plan "$SHARD_COUNT" > "$plan_b" cmp -s "$plan_a" "$plan_b" || fail "same durations produced different shard plans" @@ -164,7 +164,7 @@ test_post_run_guard_requires_the_exact_executed_union() { write_complete_manifests "$plan" "$good" out=$("$SHARDER" --verify "$SHARD_COUNT" "$good") \ || fail "post-run guard rejected the exact complete manifest union" - assert_contains "$out" "FM_BEHAVIOR_COMPLETENESS ok tests=82 shards=6" \ + assert_contains "$out" "FM_BEHAVIOR_COMPLETENESS ok tests=84 shards=8" \ "post-run guard did not report complete execution" cp -R "$good" "$missing" @@ -201,9 +201,9 @@ test_post_run_guard_requires_the_exact_executed_union() { test_ci_wires_matrix_isolation_timeout_and_union_verification() { # shellcheck disable=SC2016 # The GitHub expression is a literal YAML needle. - assert_grep 'name: Behavior tests (shard ${{ matrix.shard }}/6)' "$CI" \ + assert_grep 'name: Behavior tests (shard ${{ matrix.shard }}/8)' "$CI" \ "CI does not expose the failing shard in the job name" - assert_grep 'shard: [1, 2, 3, 4, 5, 6]' "$CI" \ + assert_grep 'shard: [1, 2, 3, 4, 5, 6, 7, 8]' "$CI" \ "CI does not launch every deterministic shard" [ "$(grep -Fc 'timeout-minutes: 15' "$CI")" -eq 1 ] \ || fail "behavior matrix must have one 15-minute per-shard timeout" @@ -214,13 +214,13 @@ test_ci_wires_matrix_isolation_timeout_and_union_verification() { assert_grep 'echo "TMUX_TMPDIR=$shard_root/tmux"' "$CI" \ "CI does not give each shard a private tmux socket root" # shellcheck disable=SC2016 # Workflow shell variables must remain literal. - assert_grep 'bin/fm-behavior-shards.sh --verify 6 "$RUNNER_TEMP/behavior-manifests"' "$CI" \ + assert_grep 'bin/fm-behavior-shards.sh --verify 8 "$RUNNER_TEMP/behavior-manifests"' "$CI" \ "CI does not verify the union of executed manifests" assert_grep 'overwrite: true' "$CI" \ "CI cannot refresh a failed shard manifest during a failed-jobs rerun" assert_no_grep 'for test_script in tests/*.test.sh' "$CI" \ "CI retained the 57-minute serial behavior loop" - pass "CI wires six named isolated shards, a tight timeout, and executed-union verification" + pass "CI wires eight named isolated shards, a tight timeout, and executed-union verification" } test_checked_in_plan_is_complete_balanced_and_deterministic diff --git a/tests/fm-report-stack-a.test.sh b/tests/fm-report-stack-a.test.sh new file mode 100755 index 0000000000..00d58627c1 --- /dev/null +++ b/tests/fm-report-stack-a.test.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -u +FM_TEST_PART_INDEX=1 FM_TEST_PART_TOTAL=2 . "$(dirname "${BASH_SOURCE[0]}")/fm-report-stack-suite.sh" diff --git a/tests/fm-report-stack-b.test.sh b/tests/fm-report-stack-b.test.sh new file mode 100755 index 0000000000..b750b715f4 --- /dev/null +++ b/tests/fm-report-stack-b.test.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -u +FM_TEST_PART_INDEX=2 FM_TEST_PART_TOTAL=2 . "$(dirname "${BASH_SOURCE[0]}")/fm-report-stack-suite.sh" diff --git a/tests/fm-report-stack.test.sh b/tests/fm-report-stack-suite.sh old mode 100755 new mode 100644 similarity index 96% rename from tests/fm-report-stack.test.sh rename to tests/fm-report-stack-suite.sh index 0692309ea5..fb5a3d5288 --- a/tests/fm-report-stack.test.sh +++ b/tests/fm-report-stack-suite.sh @@ -4084,95 +4084,108 @@ if [ "${FM_TEST_FOCUSED:-}" = retention-handoff-state-leak ]; then exit 0 fi -test_publish_ship_with_visual -test_report_artifacts_remain_verbatim_across_key_shaped_content -test_report_links_reject_credentials_and_encode_visual_paths -test_pr_url_strips_query_and_fragment -test_revision_fields_distinguish_pr_head_from_worktree_head -test_republish_new_generation_refreshes_completion_time -test_same_generation_republish_preserves_revision_without_worktree -test_generation_identity_falls_back_only_when_both_sides_are_legacy -test_text_sources_are_stored_verbatim_and_completion_is_bounded -test_metadata_is_bounded_before_reading -test_report_temps_are_exclusive_and_randomized -test_visual_inventory_is_count_and_depth_bounded -test_required_source_fails_closed -test_required_sections_fail_actionably -test_required_sections_reject_empty_bodies -test_required_sections_reject_container_only_markers -test_fenced_required_section_bodies_use_scoped_content -test_nested_short_fences_do_not_satisfy_required_sections -test_raw_html_does_not_satisfy_required_sections -test_container_scoped_fences_do_not_close_from_top_level -test_indented_pseudo_closers_do_not_end_fences -test_required_headings_follow_commonmark_atx_rules -test_invalid_backtick_info_string_does_not_open_fence -test_summary_extraction_uses_validated_markdown_structure -test_list_container_fences_hide_report_headings_and_summaries -test_list_lazy_continuations_do_not_satisfy_required_sections -test_underindented_list_headings_exit_lazy_continuation -test_nested_list_parent_scope_hides_required_headings -test_blockquote_list_scope_requires_quote_markers -test_container_scopes_preserve_commonmark_blank_and_exit_rules -test_large_non_utf8_text_artifacts_are_stored_verbatim -test_large_visual_inventory_does_not_share_text_buffer_headroom -test_scout_and_legacy_sources -test_stale_lock_rejects_reused_pid -test_stale_lock_reclaim_is_serialized -test_install_guard_release_failure_cleans_owned_lock -test_post_install_guard_owner_death_is_recovered -test_reclaim_guard_fences_the_stale_generation_gap -test_abandoned_reclaim_guard_is_recovered -test_abandoned_reclaim_marker_is_recovered -test_abandoned_reclaim_directory_is_recovered -test_publish_lock_directory_symlink_fails_closed -test_lock_control_files_are_bounded_and_nonfollowing -test_namespace_cutover_waiter_pins_entries_after_lock_acquisition -test_post_rename_lock_setup_failure_releases_owned_lock -test_publication_lock_release_failures_are_observable -test_previous_generation_is_recovered_for_readers -test_replacement_transaction_recovery_restores_entry_and_index -test_first_publication_transaction_recovery_removes_unindexed_entry -test_pre_rename_transaction_recovery_keeps_previous_generation -test_transaction_recovery_fails_when_every_generation_is_lost -test_aged_transactionless_staging_is_reclaimed -test_completed_reports_prune_after_minimum_age -test_retention_binds_manifests_to_entry_directories -test_watcher_periodically_owns_idle_report_retention -test_retention_batches_make_interruption_safe_progress -test_persistent_retention_owner_prunes_without_tasks_or_watcher -test_retention_generations_survive_install_interruptions -test_retention_error_publication_is_atomic_and_nonfollowing -test_legacy_cutover_preserves_fresh_reports_and_retires_expired_raw_paths -test_retention_owner_advances_pending_legacy_migration -test_manifest_cohort_must_match_completion_time -test_manifest_cohort_deadline_cannot_precede_expiry -test_manifest_validation_is_cohort_width_independent -test_retention_cohort_never_precedes_exact_expiry -test_retention_cohort_and_sweep_share_drift_budget -test_retention_cutoff_is_authoritative_before_cleanup -test_retention_cohort_tombstone_is_noreplace_owned -test_retention_cohort_source_swap_restores_replacement -test_retention_handoff_persists_and_retries_old_owner_fencing -test_retention_prepointer_recovery_fences_candidate -test_retention_candidate_is_fenced_before_bootstrap -test_retention_cleanup_is_file_granular -test_retention_fresh_handoff_is_cohort_bounded_and_continuous -test_report_entry_manifest_reads_stay_on_pinned_generation -test_report_publication_restores_swapped_staging_generation -test_owned_tree_cleanup_quarantines_before_deletion -test_interrupted_owned_tree_cleanup_enters_retention_recovery -test_retention_activation_requires_launched_nonce_without_owner_gap -test_retention_accepts_runatload_heartbeat_after_prebootstrap_baseline -test_retention_pointer_failure_retires_only_candidate -test_retention_pointer_failure_retains_unfenced_candidate -test_nested_list_parent_scope_hides_required_headings -test_report_destination_roots_remain_pinned_during_ancestor_swap -test_report_publication_gate_uses_framed_fifo -test_index_failure_restores_previous_generation -test_readers_wait_for_publication_lock -test_visual_symlink_fails_closed_and_cleans_staging -test_visual_copy_is_descriptor_bounded -test_visual_containment_precedes_ancestor_swap -test_source_symlinks_fail_closed -test_ambiguous_task_ids_require_report_ids +run_partitioned_test() { + FM_TEST_PART_SEQUENCE=$((FM_TEST_PART_SEQUENCE + 1)) + if [ "$FM_TEST_PART_TOTAL" -gt 1 ] \ + && [ $(((FM_TEST_PART_SEQUENCE - 1) % FM_TEST_PART_TOTAL + 1)) -ne "$FM_TEST_PART_INDEX" ]; then + return 0 + fi + "$@" +} + +FM_TEST_PART_INDEX=${FM_TEST_PART_INDEX:-1} +FM_TEST_PART_TOTAL=${FM_TEST_PART_TOTAL:-1} +FM_TEST_PART_SEQUENCE=0 + +run_partitioned_test test_publish_ship_with_visual +run_partitioned_test test_report_artifacts_remain_verbatim_across_key_shaped_content +run_partitioned_test test_report_links_reject_credentials_and_encode_visual_paths +run_partitioned_test test_pr_url_strips_query_and_fragment +run_partitioned_test test_revision_fields_distinguish_pr_head_from_worktree_head +run_partitioned_test test_republish_new_generation_refreshes_completion_time +run_partitioned_test test_same_generation_republish_preserves_revision_without_worktree +run_partitioned_test test_generation_identity_falls_back_only_when_both_sides_are_legacy +run_partitioned_test test_text_sources_are_stored_verbatim_and_completion_is_bounded +run_partitioned_test test_metadata_is_bounded_before_reading +run_partitioned_test test_report_temps_are_exclusive_and_randomized +run_partitioned_test test_visual_inventory_is_count_and_depth_bounded +run_partitioned_test test_required_source_fails_closed +run_partitioned_test test_required_sections_fail_actionably +run_partitioned_test test_required_sections_reject_empty_bodies +run_partitioned_test test_required_sections_reject_container_only_markers +run_partitioned_test test_fenced_required_section_bodies_use_scoped_content +run_partitioned_test test_nested_short_fences_do_not_satisfy_required_sections +run_partitioned_test test_raw_html_does_not_satisfy_required_sections +run_partitioned_test test_container_scoped_fences_do_not_close_from_top_level +run_partitioned_test test_indented_pseudo_closers_do_not_end_fences +run_partitioned_test test_required_headings_follow_commonmark_atx_rules +run_partitioned_test test_invalid_backtick_info_string_does_not_open_fence +run_partitioned_test test_summary_extraction_uses_validated_markdown_structure +run_partitioned_test test_list_container_fences_hide_report_headings_and_summaries +run_partitioned_test test_list_lazy_continuations_do_not_satisfy_required_sections +run_partitioned_test test_underindented_list_headings_exit_lazy_continuation +run_partitioned_test test_nested_list_parent_scope_hides_required_headings +run_partitioned_test test_blockquote_list_scope_requires_quote_markers +run_partitioned_test test_container_scopes_preserve_commonmark_blank_and_exit_rules +run_partitioned_test test_large_non_utf8_text_artifacts_are_stored_verbatim +run_partitioned_test test_large_visual_inventory_does_not_share_text_buffer_headroom +run_partitioned_test test_scout_and_legacy_sources +run_partitioned_test test_stale_lock_rejects_reused_pid +run_partitioned_test test_stale_lock_reclaim_is_serialized +run_partitioned_test test_install_guard_release_failure_cleans_owned_lock +run_partitioned_test test_post_install_guard_owner_death_is_recovered +run_partitioned_test test_reclaim_guard_fences_the_stale_generation_gap +run_partitioned_test test_abandoned_reclaim_guard_is_recovered +run_partitioned_test test_abandoned_reclaim_marker_is_recovered +run_partitioned_test test_abandoned_reclaim_directory_is_recovered +run_partitioned_test test_publish_lock_directory_symlink_fails_closed +run_partitioned_test test_lock_control_files_are_bounded_and_nonfollowing +run_partitioned_test test_namespace_cutover_waiter_pins_entries_after_lock_acquisition +run_partitioned_test test_post_rename_lock_setup_failure_releases_owned_lock +run_partitioned_test test_publication_lock_release_failures_are_observable +run_partitioned_test test_previous_generation_is_recovered_for_readers +run_partitioned_test test_replacement_transaction_recovery_restores_entry_and_index +run_partitioned_test test_first_publication_transaction_recovery_removes_unindexed_entry +run_partitioned_test test_pre_rename_transaction_recovery_keeps_previous_generation +run_partitioned_test test_transaction_recovery_fails_when_every_generation_is_lost +run_partitioned_test test_aged_transactionless_staging_is_reclaimed +run_partitioned_test test_completed_reports_prune_after_minimum_age +run_partitioned_test test_retention_binds_manifests_to_entry_directories +run_partitioned_test test_watcher_periodically_owns_idle_report_retention +run_partitioned_test test_retention_batches_make_interruption_safe_progress +run_partitioned_test test_persistent_retention_owner_prunes_without_tasks_or_watcher +run_partitioned_test test_retention_generations_survive_install_interruptions +run_partitioned_test test_retention_error_publication_is_atomic_and_nonfollowing +run_partitioned_test test_legacy_cutover_preserves_fresh_reports_and_retires_expired_raw_paths +run_partitioned_test test_retention_owner_advances_pending_legacy_migration +run_partitioned_test test_manifest_cohort_must_match_completion_time +run_partitioned_test test_manifest_cohort_deadline_cannot_precede_expiry +run_partitioned_test test_manifest_validation_is_cohort_width_independent +run_partitioned_test test_retention_cohort_never_precedes_exact_expiry +run_partitioned_test test_retention_cohort_and_sweep_share_drift_budget +run_partitioned_test test_retention_cutoff_is_authoritative_before_cleanup +run_partitioned_test test_retention_cohort_tombstone_is_noreplace_owned +run_partitioned_test test_retention_cohort_source_swap_restores_replacement +run_partitioned_test test_retention_handoff_persists_and_retries_old_owner_fencing +run_partitioned_test test_retention_prepointer_recovery_fences_candidate +run_partitioned_test test_retention_candidate_is_fenced_before_bootstrap +run_partitioned_test test_retention_cleanup_is_file_granular +run_partitioned_test test_retention_fresh_handoff_is_cohort_bounded_and_continuous +run_partitioned_test test_report_entry_manifest_reads_stay_on_pinned_generation +run_partitioned_test test_report_publication_restores_swapped_staging_generation +run_partitioned_test test_owned_tree_cleanup_quarantines_before_deletion +run_partitioned_test test_interrupted_owned_tree_cleanup_enters_retention_recovery +run_partitioned_test test_retention_activation_requires_launched_nonce_without_owner_gap +run_partitioned_test test_retention_accepts_runatload_heartbeat_after_prebootstrap_baseline +run_partitioned_test test_retention_pointer_failure_retires_only_candidate +run_partitioned_test test_retention_pointer_failure_retains_unfenced_candidate +run_partitioned_test test_nested_list_parent_scope_hides_required_headings +run_partitioned_test test_report_destination_roots_remain_pinned_during_ancestor_swap +run_partitioned_test test_report_publication_gate_uses_framed_fifo +run_partitioned_test test_index_failure_restores_previous_generation +run_partitioned_test test_readers_wait_for_publication_lock +run_partitioned_test test_visual_symlink_fails_closed_and_cleans_staging +run_partitioned_test test_visual_copy_is_descriptor_bounded +run_partitioned_test test_visual_containment_precedes_ancestor_swap +run_partitioned_test test_source_symlinks_fail_closed +run_partitioned_test test_ambiguous_task_ids_require_report_ids From 4dcafc2cbc0d81c993e2314a6ad83e4aa1d9e008 Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Fri, 31 Jul 2026 22:59:30 -0400 Subject: [PATCH 04/10] test: keep report retention setup with dependent shard --- tests/fm-report-stack-suite.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/fm-report-stack-suite.sh b/tests/fm-report-stack-suite.sh index fb5a3d5288..f6aa4c6912 100644 --- a/tests/fm-report-stack-suite.sh +++ b/tests/fm-report-stack-suite.sh @@ -4093,6 +4093,18 @@ run_partitioned_test() { "$@" } +run_partitioned_group() { + local assigned_part group_size=$# test_function + assigned_part=$((FM_TEST_PART_SEQUENCE % FM_TEST_PART_TOTAL + 1)) + FM_TEST_PART_SEQUENCE=$((FM_TEST_PART_SEQUENCE + group_size)) + if [ "$FM_TEST_PART_TOTAL" -gt 1 ] && [ "$assigned_part" -ne "$FM_TEST_PART_INDEX" ]; then + return 0 + fi + for test_function in "$@"; do + "$test_function" + done +} + FM_TEST_PART_INDEX=${FM_TEST_PART_INDEX:-1} FM_TEST_PART_TOTAL=${FM_TEST_PART_TOTAL:-1} FM_TEST_PART_SEQUENCE=0 @@ -4153,8 +4165,9 @@ run_partitioned_test test_completed_reports_prune_after_minimum_age run_partitioned_test test_retention_binds_manifests_to_entry_directories run_partitioned_test test_watcher_periodically_owns_idle_report_retention run_partitioned_test test_retention_batches_make_interruption_safe_progress -run_partitioned_test test_persistent_retention_owner_prunes_without_tasks_or_watcher -run_partitioned_test test_retention_generations_survive_install_interruptions +run_partitioned_group \ + test_persistent_retention_owner_prunes_without_tasks_or_watcher \ + test_retention_generations_survive_install_interruptions run_partitioned_test test_retention_error_publication_is_atomic_and_nonfollowing run_partitioned_test test_legacy_cutover_preserves_fresh_reports_and_retires_expired_raw_paths run_partitioned_test test_retention_owner_advances_pending_legacy_migration From 7057251c3f63e0cd7d0018dbf69d0d6a8819516a Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Fri, 31 Jul 2026 23:14:16 -0400 Subject: [PATCH 05/10] docs: record measured behavior shard timing --- docs/ci-behavior-shards.md | 43 ++++++++++++++++++++++++------- tests/behavior-test-durations.tsv | 11 ++++---- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/docs/ci-behavior-shards.md b/docs/ci-behavior-shards.md index b040706bf8..049fa8e223 100644 --- a/docs/ci-behavior-shards.md +++ b/docs/ci-behavior-shards.md @@ -24,19 +24,44 @@ The eight-shard checked-in plan has these estimated serial loads. |---:|---:|---:| | 1 | 1 | 563637 ms (9m24s) | | 2 | 1 | 475500 ms (7m56s) | -| 3 | 13 | 401927 ms (6m42s) | -| 4 | 13 | 401942 ms (6m42s) | -| 5 | 13 | 401927 ms (6m42s) | -| 6 | 10 | 401979 ms (6m42s) | -| 7 | 16 | 401935 ms (6m42s) | -| 8 | 17 | 401936 ms (6m42s) | - -The expected healthy behavior-execution critical path is 9 minutes 24 seconds rather than 57 minutes 23 seconds. +| 3 | 1 | 423936 ms (7m04s) | +| 4 | 15 | 391516 ms (6m32s) | +| 5 | 17 | 391524 ms (6m32s) | +| 6 | 17 | 391519 ms (6m32s) | +| 7 | 17 | 391519 ms (6m32s) | +| 8 | 15 | 391517 ms (6m32s) | The account-routing and report-stack suites keep their original test functions and assertions in shared suite files, while two runner wrappers partition each call list deterministically between isolated runners. +The report-stack partition keeps the persistent-retention-owner setup in the same wrapper as its generation-interruption dependent. + The largest remaining indivisible test file, `tests/fm-teardown.test.sh`, sets the 9-minute-24-second floor. +## Measured GitHub Actions result + +[GitHub Actions run 30681093133](https://github.com/ruby-dlee/firstmate/actions/runs/30681093133) ran all eight shards and the executed-union guard from commit `4dcafc2cbc0d81c993e2314a6ad83e4aa1d9e008`. + +The behavior check took 9 minutes 57 seconds from workflow creation at 02:59:39 UTC through final guard completion at 03:09:36 UTC. + +The slowest matrix job was shard 1 at 9 minutes 25 seconds, and its behavior step took 9 minutes 13 seconds. + +The serial baseline was 57 minutes 23 seconds, so the measured behavior check wall-clock fell by 47 minutes 26 seconds while retaining every test file and the final completeness proof. + +| Shard | Job wall-clock | Behavior step | Result | +|---:|---:|---:|---:| +| 1 | 9m25s | 9m13s | pass | +| 2 | 8m28s | 8m19s | pass | +| 3 | 8m32s | 8m20s | pass | +| 4 | 6m59s | 6m48s | pass | +| 5 | 6m10s | 6m03s | pass | +| 6 | 6m18s | 6m11s | pass | +| 7 | 5m51s | 5m40s | pass | +| 8 | 8m43s | 8m31s | pass | + +The same run measured the split wrappers at 423936 ms for account-routing A, 335548 ms for account-routing B, 261383 ms for report-stack A, and 243164 ms for report-stack B. + +Those values replaced the provisional half-suite estimates in `tests/behavior-test-durations.tsv` and produced the checked-in assignment table above. + ## Coverage guard `bin/fm-behavior-shards.sh --check 8` fails when the duration inventory has a missing path, duplicate path, malformed duration, or any difference from the complete `tests/*.test.sh` inventory. @@ -61,7 +86,7 @@ Scripts remain serial within a shard, so no existing test needed weaker assertio Each matrix job is named `Behavior tests (shard N/8)`, and the runner emits explicit begin and end markers containing the test path and exit code. -The 15-minute per-shard timeout leaves bounded margin above the 9m24s slowest planned shard while replacing the prior 90-minute blanket. +The 15-minute per-shard timeout leaves bounded margin above the 9m25s measured slowest job while replacing the prior 90-minute blanket. ## Teardown child-endpoint investigation diff --git a/tests/behavior-test-durations.tsv b/tests/behavior-test-durations.tsv index 60df56f456..3e3fc6f619 100644 --- a/tests/behavior-test-durations.tsv +++ b/tests/behavior-test-durations.tsv @@ -1,10 +1,11 @@ # Behavior test durations in milliseconds. # Refresh with: bin/fm-behavior-shards.sh --record tests/behavior-test-durations.tsv -# Recorded from https://github.com/ruby-dlee/firstmate/actions/runs/30648119449 on 2026-07-31. +# Baseline durations recorded from https://github.com/ruby-dlee/firstmate/actions/runs/30648119449 on 2026-07-31. +# Split-wrapper durations refreshed from https://github.com/ruby-dlee/firstmate/actions/runs/30681093133 on 2026-08-01 UTC. 55000 tests/bridge-cutover-python.test.sh 71100 tests/fm-account-directory.test.sh -336700 tests/fm-account-routing-a.test.sh -336700 tests/fm-account-routing-b.test.sh +423936 tests/fm-account-routing-a.test.sh +335548 tests/fm-account-routing-b.test.sh 33600 tests/fm-afk-inject-e2e.test.sh 20 tests/fm-afk-inject-herdr-e2e.test.sh 16500 tests/fm-afk-launch.test.sh @@ -53,8 +54,8 @@ 3170 tests/fm-pi-watch-extension.test.sh 1574 tests/fm-pr-merge.test.sh 802 tests/fm-prompt-exec.test.sh -310373 tests/fm-report-stack-a.test.sh -310373 tests/fm-report-stack-b.test.sh +261383 tests/fm-report-stack-a.test.sh +243164 tests/fm-report-stack-b.test.sh 495 tests/fm-review-diff.test.sh 26463 tests/fm-secondmate-harness.test.sh 37513 tests/fm-secondmate-lifecycle-e2e.test.sh From f7d16e22a94cb1a83ea6df8e8219d5ecadfc0d32 Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Sat, 1 Aug 2026 10:39:06 -0400 Subject: [PATCH 06/10] ci: split teardown behavior suite across shards --- tests/behavior-test-durations.tsv | 4 +- tests/fm-behavior-shards.test.sh | 53 +++- tests/fm-teardown-a.test.sh | 3 + tests/fm-teardown-b.test.sh | 3 + ...-teardown.test.sh => fm-teardown-suite.sh} | 250 ++++++++++-------- 5 files changed, 202 insertions(+), 111 deletions(-) create mode 100755 tests/fm-teardown-a.test.sh create mode 100755 tests/fm-teardown-b.test.sh rename tests/{fm-teardown.test.sh => fm-teardown-suite.sh} (96%) mode change 100755 => 100644 diff --git a/tests/behavior-test-durations.tsv b/tests/behavior-test-durations.tsv index 3e3fc6f619..416c232344 100644 --- a/tests/behavior-test-durations.tsv +++ b/tests/behavior-test-durations.tsv @@ -2,6 +2,7 @@ # Refresh with: bin/fm-behavior-shards.sh --record tests/behavior-test-durations.tsv # Baseline durations recorded from https://github.com/ruby-dlee/firstmate/actions/runs/30648119449 on 2026-07-31. # Split-wrapper durations refreshed from https://github.com/ruby-dlee/firstmate/actions/runs/30681093133 on 2026-08-01 UTC. +# Teardown wrapper starting weights split the 642448 ms whole-suite measurement from https://github.com/ruby-dlee/firstmate/actions/runs/30681622585. 55000 tests/bridge-cutover-python.test.sh 71100 tests/fm-account-directory.test.sh 423936 tests/fm-account-routing-a.test.sh @@ -74,7 +75,8 @@ 500 tests/fm-supervision-events.test.sh 95 tests/fm-supervision-instructions.test.sh 10408 tests/fm-tangle-guard.test.sh -563637 tests/fm-teardown.test.sh +321224 tests/fm-teardown-a.test.sh +321224 tests/fm-teardown-b.test.sh 600 tests/fm-transition-lib.test.sh 1849 tests/fm-turnend-guard.test.sh 1511 tests/fm-update.test.sh diff --git a/tests/fm-behavior-shards.test.sh b/tests/fm-behavior-shards.test.sh index b54c07d2a3..a97c4b29c6 100755 --- a/tests/fm-behavior-shards.test.sh +++ b/tests/fm-behavior-shards.test.sh @@ -9,6 +9,7 @@ set -u SHARDER="$ROOT/bin/fm-behavior-shards.sh" DURATIONS="$ROOT/tests/behavior-test-durations.tsv" CI="$ROOT/.github/workflows/ci.yml" +TEARDOWN_SUITE="$ROOT/tests/fm-teardown-suite.sh" SHARD_COUNT=8 test_checked_in_plan_is_complete_balanced_and_deterministic() { @@ -21,8 +22,8 @@ test_checked_in_plan_is_complete_balanced_and_deterministic() { out=$("$SHARDER" --check "$SHARD_COUNT") \ || fail "checked-in behavior shard plan failed its coverage guard" - assert_contains "$out" "FM_BEHAVIOR_PLAN ok tests=84 shards=8" \ - "coverage guard did not report the complete 84-test inventory" + assert_contains "$out" "FM_BEHAVIOR_PLAN ok tests=85 shards=8" \ + "coverage guard did not report the complete 85-test inventory" "$SHARDER" --plan "$SHARD_COUNT" > "$plan_a" "$SHARDER" --plan "$SHARD_COUNT" > "$plan_b" cmp -s "$plan_a" "$plan_b" || fail "same durations produced different shard plans" @@ -164,7 +165,7 @@ test_post_run_guard_requires_the_exact_executed_union() { write_complete_manifests "$plan" "$good" out=$("$SHARDER" --verify "$SHARD_COUNT" "$good") \ || fail "post-run guard rejected the exact complete manifest union" - assert_contains "$out" "FM_BEHAVIOR_COMPLETENESS ok tests=84 shards=8" \ + assert_contains "$out" "FM_BEHAVIOR_COMPLETENESS ok tests=85 shards=8" \ "post-run guard did not report complete execution" cp -R "$good" "$missing" @@ -223,9 +224,55 @@ test_ci_wires_matrix_isolation_timeout_and_union_verification() { pass "CI wires eight named isolated shards, a tight timeout, and executed-union verification" } +test_teardown_partition_preserves_every_full_suite_case() { + local tmp definitions listed focused expected_focused wrapper_a wrapper_b + tmp=$(fm_test_tmproot fm-teardown-partition) + definitions="$tmp/definitions.txt" + listed="$tmp/listed.txt" + focused="$tmp/focused.txt" + expected_focused="$tmp/expected-focused.txt" + wrapper_a="$ROOT/tests/fm-teardown-a.test.sh" + wrapper_b="$ROOT/tests/fm-teardown-b.test.sh" + + sed -n 's/^\(test_[A-Za-z0-9_]*\)() {.*/\1/p' "$TEARDOWN_SUITE" \ + | LC_ALL=C sort > "$definitions" + awk ' + /^TEARDOWN_FULL_SUITE_CASES=\($/ { in_cases = 1; next } + in_cases && /^\)$/ { exit } + in_cases && $1 ~ /^test_[A-Za-z0-9_]+$/ { print $1 } + ' "$TEARDOWN_SUITE" > "$listed" + [ "$(wc -l < "$listed" | tr -d ' ')" -eq 106 ] \ + || fail "teardown partition does not retain all 106 normal-run cases" + [ "$(LC_ALL=C sort "$listed" | uniq -d | wc -l | tr -d ' ')" -eq 0 ] \ + || fail "teardown partition lists a normal-run case more than once" + comm -13 "$definitions" <(LC_ALL=C sort "$listed") > "$tmp/undefined.txt" + [ ! -s "$tmp/undefined.txt" ] \ + || fail "teardown partition lists an undefined test function" + comm -23 "$definitions" <(LC_ALL=C sort "$listed") > "$focused" + printf '%s\n' \ + test_bounded_runner_preserves_command_status_125 \ + test_pr_check_backfills_legacy_generation_and_records_state \ + test_pr_check_backfills_legacy_generation_before_race_check \ + | LC_ALL=C sort > "$expected_focused" + cmp -s "$expected_focused" "$focused" \ + || fail "teardown partition dropped or absorbed a focused-only case" + [ "$(awk 'NR % 2 == 1 { count++ } END { print count + 0 }' "$listed")" -eq 53 ] \ + || fail "teardown partition A does not own exactly 53 cases" + [ "$(awk 'NR % 2 == 0 { count++ } END { print count + 0 }' "$listed")" -eq 53 ] \ + || fail "teardown partition B does not own exactly 53 cases" + assert_grep 'FM_TEST_PART_INDEX=1 FM_TEST_PART_TOTAL=2' "$wrapper_a" \ + "teardown wrapper A does not select partition 1/2" + assert_grep 'FM_TEST_PART_INDEX=2 FM_TEST_PART_TOTAL=2' "$wrapper_b" \ + "teardown wrapper B does not select partition 2/2" + [ ! -e "$ROOT/tests/fm-teardown.test.sh" ] \ + || fail "the unsplit teardown test remains in the behavior inventory" + pass "teardown wrappers preserve all 106 normal cases and three focused-only cases" +} + test_checked_in_plan_is_complete_balanced_and_deterministic test_plan_refuses_missing_and_duplicate_duration_entries test_runner_executes_every_assigned_test_and_records_failures test_record_refreshes_complete_fixture_timings test_post_run_guard_requires_the_exact_executed_union test_ci_wires_matrix_isolation_timeout_and_union_verification +test_teardown_partition_preserves_every_full_suite_case diff --git a/tests/fm-teardown-a.test.sh b/tests/fm-teardown-a.test.sh new file mode 100755 index 0000000000..f9a7672126 --- /dev/null +++ b/tests/fm-teardown-a.test.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -u +FM_TEST_PART_INDEX=1 FM_TEST_PART_TOTAL=2 . "$(dirname "${BASH_SOURCE[0]}")/fm-teardown-suite.sh" diff --git a/tests/fm-teardown-b.test.sh b/tests/fm-teardown-b.test.sh new file mode 100755 index 0000000000..5ff9f16019 --- /dev/null +++ b/tests/fm-teardown-b.test.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -u +FM_TEST_PART_INDEX=2 FM_TEST_PART_TOTAL=2 . "$(dirname "${BASH_SOURCE[0]}")/fm-teardown-suite.sh" diff --git a/tests/fm-teardown.test.sh b/tests/fm-teardown-suite.sh old mode 100755 new mode 100644 similarity index 96% rename from tests/fm-teardown.test.sh rename to tests/fm-teardown-suite.sh index fee6e45c8d..92e3127146 --- a/tests/fm-teardown.test.sh +++ b/tests/fm-teardown-suite.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tests for bin/fm-teardown.sh's landed-work safety and stale-lock recovery. +# Shared tests for bin/fm-teardown.sh's landed-work safety and stale-lock recovery. # # The check refuses to tear down a worktree whose work has not LANDED, because # treehouse return hard-resets the worktree. "Landed" means reachable from a remote @@ -5092,109 +5092,145 @@ if [ "${FM_TEST_FOCUSED:-}" = preserve-scratch ]; then exit 0 fi -test_local_only_fork_remote_allows -test_teardown_prompts_tasks_axi_done_when_compatible -test_teardown_manual_backend_prompts_hand_edit_even_when_tasks_axi_present -test_local_only_truly_unpushed_refuses -test_local_only_merged_to_local_main_allows -test_no_mistakes_origin_remote_allows -test_no_mistakes_truly_unpushed_refuses -test_local_only_force_retains_unpushed -test_managed_force_teardown_retains_unlanded_lease_and_session -test_managed_teardown_retains_lease_when_endpoint_state_is_unknown -test_managed_release_failure_preserves_unrecycled_worktree_for_retry -test_managed_teardown_locks_generation_before_endpoint_cleanup -test_managed_child_teardown_locks_generation_before_snapshot -test_forced_secondmate_child_uses_child_home_for_endpoint_verification -test_forced_secondmate_quiesces_parent_before_child_cleanup -test_forced_secondmate_retains_child_on_treehouse_failure -test_forced_secondmate_retains_unverified_process_group -test_forced_secondmate_retains_child_when_treehouse_unavailable -test_forced_secondmate_retains_child_on_checkout_lock_contention -test_herdr_teardown_clears_escalation_marker -test_required_report_blocks_then_publishes_before_cleanup -test_required_report_restores_rollback_generation_before_publish -test_required_report_revalidates_after_quiescence -test_legacy_teardown_revalidates_after_quiescence -test_teardown_rejects_nested_metadata_roots_before_quiescence -test_teardown_rejects_drifted_treehouse_task_lease -test_teardown_rechecks_treehouse_lease_after_locked_safety -test_secondmate_rejects_drifted_home_repository_identity -test_normal_secondmate_retires_proven_detached_head -test_forced_secondmate_retains_untracked_skill_draft -test_forced_secondmate_retains_unique_detached_head -test_forced_secondmate_retains_stash -test_forced_secondmate_retains_unlanded_child_work -test_forced_secondmate_retains_unquiesced_unmanaged_child -test_secondmate_registry_duplicate_home_blocks_removal -test_secondmate_retirement_retains_idle_registered_child -test_secondmate_retirement_retains_unlanded_project_clone -test_secondmate_project_tags_do_not_prove_landing -test_secondmate_project_origin_authority_survives_home_removal -test_secondmate_retirement_recurses_into_ignored_nested_repositories -test_secondmate_retirement_rejects_linked_worktree_graphs -test_secondmate_retirement_accounts_for_directory_symlinks -test_secondmate_retirement_rejects_loopback_and_stale_tracking_authority -test_secondmate_retirement_rejects_mount_boundaries -test_secondmate_retirement_rejects_effective_ssh_redirects -test_secondmate_retirement_rejects_incomplete_surviving_authority -test_secondmate_retirement_validates_top_level_source_storage -test_secondmate_retirement_rejects_local_network_aliases -test_secondmate_retirement_rejects_in_home_remote_object_storage -test_secondmate_retirement_rejects_source_common_dir_in_home -test_teardown_removal_roots_fail_closed -test_treehouse_return_stays_bound_to_validated_root -test_teardown_distinguishes_dead_and_live_harness_processes -test_secondmate_retirement_retains_reflog_and_rewritten_history -test_secondmate_retirement_rejects_http_proxy_and_object_redirects -test_secondmate_network_fetches_pin_validated_addresses -test_surviving_object_storage_is_bound_through_graph_proof -test_secondmate_retirement_serializes_child_spawn -test_nested_secondmate_cleanup_requires_child_home_lock -test_secondmate_registry_updates_are_locked_and_literal -test_teardown_retains_untracked_claude_skill_draft -test_teardown_refuses_unsafe_tasktmp_metadata -test_teardown_removes_safe_tasktmp_and_accepts_absence -test_teardown_rejects_malformed_report_requirement -test_secondmate_state_enumeration_fails_closed -test_secondmate_missing_treehouse_child_is_retained -test_secondmate_registry_home_drift_blocks_removal -test_retained_direct_spawn_requires_confirmed_endpoint_quiescence -test_never_created_direct_spawn_endpoint_is_not_quiesced -test_squash_merged_branch_deleted_allows -test_squash_merged_pr_allows_when_head_ancestor_of_pr_head -test_no_pr_recorded_discovers_merged_pr_by_branch_allows -test_squash_merged_pr_allows_replayed_unpushed_patch -test_merged_pr_with_later_local_commit_refuses -test_pr_check_does_not_refresh_stale_pr_head -test_pr_check_records_remote_head_when_local_lags -test_pr_check_serializes_with_account_session_updates -test_pr_check_rejects_reused_task_generation -test_content_in_default_fallback_allows -test_content_fallback_refreshes_stale_origin_ref -test_content_fallback_uses_live_default -test_content_fallback_reprobes_live_default_after_fetch -test_content_fallback_honors_shared_checkout_lock -test_locked_return_reuses_checkout_lock_for_landing_recheck -test_treehouse_return_timeout_reaps_children_before_unlock -test_dirty_worktree_refuses -test_nonignored_untracked_work_refuses_without_preservation -test_already_returned_worktree_finishes_bookkeeping -test_already_returned_worktree_refuses_preservation_without_mutation -test_watchman_cookies_do_not_block_teardown -test_ignored_worktree_content_is_summarized_without_blocking -test_preserve_scratch_captures_then_reclaims_dirty_worktree -test_preserve_scratch_never_cleans_unlanded_commits -test_preserve_scratch_refuses_tracked_drift_before_cleanup -test_preserve_scratch_refuses_index_drift_during_tracked_verification -test_gh_error_and_content_absent_refuses -test_stale_index_lock_cleared_and_teardown_succeeds -test_live_index_lock_is_never_removed_and_teardown_refuses -test_lsof_error_never_clears_index_lock -test_stale_index_lock_cleanup_rechecks_dirty_worktree -test_non_linked_index_lock_path_is_checked_from_worktree -test_index_lock_mtime_read_failure_refuses -test_transient_index_lock_clears_after_first_attempt_and_retry_succeeds -test_persistent_index_lock_exhausts_retries_and_refuses_loudly -test_empty_retry_wait_uses_default_without_aborting -test_fractional_legacy_retry_wait_refuses_without_arithmetic_error +run_partitioned_test() { + local assigned_part test_function=$1 + assigned_part=$((FM_TEST_PART_SEQUENCE % FM_TEST_PART_TOTAL + 1)) + FM_TEST_PART_SEQUENCE=$((FM_TEST_PART_SEQUENCE + 1)) + if [ "$assigned_part" -ne "$FM_TEST_PART_INDEX" ]; then + return 0 + fi + FM_TEARDOWN_PART_CASES=$((FM_TEARDOWN_PART_CASES + 1)) + printf 'FM_TEARDOWN_CASE_BEGIN part=%s/%s case=%s\n' \ + "$FM_TEST_PART_INDEX" "$FM_TEST_PART_TOTAL" "$test_function" + "$test_function" + printf 'FM_TEARDOWN_CASE_END part=%s/%s case=%s exit=0\n' \ + "$FM_TEST_PART_INDEX" "$FM_TEST_PART_TOTAL" "$test_function" +} + +FM_TEST_PART_INDEX=${FM_TEST_PART_INDEX:-1} +FM_TEST_PART_TOTAL=${FM_TEST_PART_TOTAL:-1} +case "$FM_TEST_PART_INDEX:$FM_TEST_PART_TOTAL" in + *[!0-9:]* | 0:* | *:0) + fail "teardown partition requires positive numeric part index and total" + ;; +esac +[ "$FM_TEST_PART_INDEX" -le "$FM_TEST_PART_TOTAL" ] \ + || fail "teardown partition index exceeds total" +FM_TEST_PART_SEQUENCE=0 +FM_TEARDOWN_PART_CASES=0 + +TEARDOWN_FULL_SUITE_CASES=( + test_local_only_fork_remote_allows + test_teardown_prompts_tasks_axi_done_when_compatible + test_teardown_manual_backend_prompts_hand_edit_even_when_tasks_axi_present + test_local_only_truly_unpushed_refuses + test_local_only_merged_to_local_main_allows + test_no_mistakes_origin_remote_allows + test_no_mistakes_truly_unpushed_refuses + test_local_only_force_retains_unpushed + test_managed_force_teardown_retains_unlanded_lease_and_session + test_managed_teardown_retains_lease_when_endpoint_state_is_unknown + test_managed_release_failure_preserves_unrecycled_worktree_for_retry + test_managed_teardown_locks_generation_before_endpoint_cleanup + test_managed_child_teardown_locks_generation_before_snapshot + test_forced_secondmate_child_uses_child_home_for_endpoint_verification + test_forced_secondmate_quiesces_parent_before_child_cleanup + test_forced_secondmate_retains_child_on_treehouse_failure + test_forced_secondmate_retains_unverified_process_group + test_forced_secondmate_retains_child_when_treehouse_unavailable + test_forced_secondmate_retains_child_on_checkout_lock_contention + test_herdr_teardown_clears_escalation_marker + test_required_report_blocks_then_publishes_before_cleanup + test_required_report_restores_rollback_generation_before_publish + test_required_report_revalidates_after_quiescence + test_legacy_teardown_revalidates_after_quiescence + test_teardown_rejects_nested_metadata_roots_before_quiescence + test_teardown_rejects_drifted_treehouse_task_lease + test_teardown_rechecks_treehouse_lease_after_locked_safety + test_secondmate_rejects_drifted_home_repository_identity + test_normal_secondmate_retires_proven_detached_head + test_forced_secondmate_retains_untracked_skill_draft + test_forced_secondmate_retains_unique_detached_head + test_forced_secondmate_retains_stash + test_forced_secondmate_retains_unlanded_child_work + test_forced_secondmate_retains_unquiesced_unmanaged_child + test_secondmate_registry_duplicate_home_blocks_removal + test_secondmate_retirement_retains_idle_registered_child + test_secondmate_retirement_retains_unlanded_project_clone + test_secondmate_project_tags_do_not_prove_landing + test_secondmate_project_origin_authority_survives_home_removal + test_secondmate_retirement_recurses_into_ignored_nested_repositories + test_secondmate_retirement_rejects_linked_worktree_graphs + test_secondmate_retirement_accounts_for_directory_symlinks + test_secondmate_retirement_rejects_loopback_and_stale_tracking_authority + test_secondmate_retirement_rejects_mount_boundaries + test_secondmate_retirement_rejects_effective_ssh_redirects + test_secondmate_retirement_rejects_incomplete_surviving_authority + test_secondmate_retirement_validates_top_level_source_storage + test_secondmate_retirement_rejects_local_network_aliases + test_secondmate_retirement_rejects_in_home_remote_object_storage + test_secondmate_retirement_rejects_source_common_dir_in_home + test_teardown_removal_roots_fail_closed + test_treehouse_return_stays_bound_to_validated_root + test_teardown_distinguishes_dead_and_live_harness_processes + test_secondmate_retirement_retains_reflog_and_rewritten_history + test_secondmate_retirement_rejects_http_proxy_and_object_redirects + test_secondmate_network_fetches_pin_validated_addresses + test_surviving_object_storage_is_bound_through_graph_proof + test_secondmate_retirement_serializes_child_spawn + test_nested_secondmate_cleanup_requires_child_home_lock + test_secondmate_registry_updates_are_locked_and_literal + test_teardown_retains_untracked_claude_skill_draft + test_teardown_refuses_unsafe_tasktmp_metadata + test_teardown_removes_safe_tasktmp_and_accepts_absence + test_teardown_rejects_malformed_report_requirement + test_secondmate_state_enumeration_fails_closed + test_secondmate_missing_treehouse_child_is_retained + test_secondmate_registry_home_drift_blocks_removal + test_retained_direct_spawn_requires_confirmed_endpoint_quiescence + test_never_created_direct_spawn_endpoint_is_not_quiesced + test_squash_merged_branch_deleted_allows + test_squash_merged_pr_allows_when_head_ancestor_of_pr_head + test_no_pr_recorded_discovers_merged_pr_by_branch_allows + test_squash_merged_pr_allows_replayed_unpushed_patch + test_merged_pr_with_later_local_commit_refuses + test_pr_check_does_not_refresh_stale_pr_head + test_pr_check_records_remote_head_when_local_lags + test_pr_check_serializes_with_account_session_updates + test_pr_check_rejects_reused_task_generation + test_content_in_default_fallback_allows + test_content_fallback_refreshes_stale_origin_ref + test_content_fallback_uses_live_default + test_content_fallback_reprobes_live_default_after_fetch + test_content_fallback_honors_shared_checkout_lock + test_locked_return_reuses_checkout_lock_for_landing_recheck + test_treehouse_return_timeout_reaps_children_before_unlock + test_dirty_worktree_refuses + test_nonignored_untracked_work_refuses_without_preservation + test_already_returned_worktree_finishes_bookkeeping + test_already_returned_worktree_refuses_preservation_without_mutation + test_watchman_cookies_do_not_block_teardown + test_ignored_worktree_content_is_summarized_without_blocking + test_preserve_scratch_captures_then_reclaims_dirty_worktree + test_preserve_scratch_never_cleans_unlanded_commits + test_preserve_scratch_refuses_tracked_drift_before_cleanup + test_preserve_scratch_refuses_index_drift_during_tracked_verification + test_gh_error_and_content_absent_refuses + test_stale_index_lock_cleared_and_teardown_succeeds + test_live_index_lock_is_never_removed_and_teardown_refuses + test_lsof_error_never_clears_index_lock + test_stale_index_lock_cleanup_rechecks_dirty_worktree + test_non_linked_index_lock_path_is_checked_from_worktree + test_index_lock_mtime_read_failure_refuses + test_transient_index_lock_clears_after_first_attempt_and_retry_succeeds + test_persistent_index_lock_exhausts_retries_and_refuses_loudly + test_empty_retry_wait_uses_default_without_aborting + test_fractional_legacy_retry_wait_refuses_without_arithmetic_error +) + +for teardown_test_function in "${TEARDOWN_FULL_SUITE_CASES[@]}"; do + run_partitioned_test "$teardown_test_function" +done +printf 'FM_TEARDOWN_PART_RESULT part=%s/%s cases=%s total=%s\n' \ + "$FM_TEST_PART_INDEX" "$FM_TEST_PART_TOTAL" "$FM_TEARDOWN_PART_CASES" \ + "${#TEARDOWN_FULL_SUITE_CASES[@]}" From 47ded157c566a5e93d86418cd540f3f4a8e7a33f Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Sat, 1 Aug 2026 18:31:35 -0400 Subject: [PATCH 07/10] ci: refresh behavior shard inventory --- .github/workflows/ci.yml | 2 +- tests/behavior-test-durations.tsv | 2 ++ tests/fm-account-routing-a.test.sh | 1 + tests/fm-account-routing-b.test.sh | 1 + tests/fm-behavior-shards.test.sh | 16 ++++++++-------- tests/fm-report-stack-a.test.sh | 1 + tests/fm-report-stack-b.test.sh | 1 + tests/fm-teardown-a.test.sh | 1 + tests/fm-teardown-b.test.sh | 1 + 9 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab8813e473..1a761c6bf2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: behavior-tests-complete: # Preserve the historical required-check name while proving the union of - # what the six runners actually executed, not only the planned inventory. + # what the eight runners actually executed, not only the planned inventory. name: Behavior tests needs: behavior-tests if: always() diff --git a/tests/behavior-test-durations.tsv b/tests/behavior-test-durations.tsv index 416c232344..f41d28b99c 100644 --- a/tests/behavior-test-durations.tsv +++ b/tests/behavior-test-durations.tsv @@ -39,6 +39,7 @@ 30 tests/fm-composer-lib.test.sh 10055 tests/fm-crew-state.test.sh 13991 tests/fm-daemon.test.sh +340 tests/fm-decision-pretool-check.test.sh 685 tests/fm-dispatch-select.test.sh 156 tests/fm-ensure-agents-md.test.sh 3009 tests/fm-fleet-snapshot-view.test.sh @@ -69,6 +70,7 @@ 223 tests/fm-send-settle.test.sh 18168 tests/fm-send-strict.test.sh 6911 tests/fm-session-start.test.sh +28460 tests/fm-spawn-backlog.test.sh 2094 tests/fm-spawn-batch.test.sh 42452 tests/fm-spawn-dispatch-profile.test.sh 42 tests/fm-stow-contract.test.sh diff --git a/tests/fm-account-routing-a.test.sh b/tests/fm-account-routing-a.test.sh index f3b95c8a1b..02889fd321 100755 --- a/tests/fm-account-routing-a.test.sh +++ b/tests/fm-account-routing-a.test.sh @@ -1,3 +1,4 @@ #!/usr/bin/env bash set -u +# shellcheck source=tests/fm-account-routing-suite.sh disable=SC1091 FM_TEST_PART_INDEX=1 FM_TEST_PART_TOTAL=2 . "$(dirname "${BASH_SOURCE[0]}")/fm-account-routing-suite.sh" diff --git a/tests/fm-account-routing-b.test.sh b/tests/fm-account-routing-b.test.sh index d03951790b..ee32a936dc 100755 --- a/tests/fm-account-routing-b.test.sh +++ b/tests/fm-account-routing-b.test.sh @@ -1,3 +1,4 @@ #!/usr/bin/env bash set -u +# shellcheck source=tests/fm-account-routing-suite.sh disable=SC1091 FM_TEST_PART_INDEX=2 FM_TEST_PART_TOTAL=2 . "$(dirname "${BASH_SOURCE[0]}")/fm-account-routing-suite.sh" diff --git a/tests/fm-behavior-shards.test.sh b/tests/fm-behavior-shards.test.sh index a97c4b29c6..5c413cc409 100755 --- a/tests/fm-behavior-shards.test.sh +++ b/tests/fm-behavior-shards.test.sh @@ -22,8 +22,8 @@ test_checked_in_plan_is_complete_balanced_and_deterministic() { out=$("$SHARDER" --check "$SHARD_COUNT") \ || fail "checked-in behavior shard plan failed its coverage guard" - assert_contains "$out" "FM_BEHAVIOR_PLAN ok tests=85 shards=8" \ - "coverage guard did not report the complete 85-test inventory" + assert_contains "$out" "FM_BEHAVIOR_PLAN ok tests=87 shards=8" \ + "coverage guard did not report the complete 87-test inventory" "$SHARDER" --plan "$SHARD_COUNT" > "$plan_a" "$SHARDER" --plan "$SHARD_COUNT" > "$plan_b" cmp -s "$plan_a" "$plan_b" || fail "same durations produced different shard plans" @@ -165,7 +165,7 @@ test_post_run_guard_requires_the_exact_executed_union() { write_complete_manifests "$plan" "$good" out=$("$SHARDER" --verify "$SHARD_COUNT" "$good") \ || fail "post-run guard rejected the exact complete manifest union" - assert_contains "$out" "FM_BEHAVIOR_COMPLETENESS ok tests=85 shards=8" \ + assert_contains "$out" "FM_BEHAVIOR_COMPLETENESS ok tests=87 shards=8" \ "post-run guard did not report complete execution" cp -R "$good" "$missing" @@ -241,8 +241,8 @@ test_teardown_partition_preserves_every_full_suite_case() { in_cases && /^\)$/ { exit } in_cases && $1 ~ /^test_[A-Za-z0-9_]+$/ { print $1 } ' "$TEARDOWN_SUITE" > "$listed" - [ "$(wc -l < "$listed" | tr -d ' ')" -eq 106 ] \ - || fail "teardown partition does not retain all 106 normal-run cases" + [ "$(wc -l < "$listed" | tr -d ' ')" -eq 107 ] \ + || fail "teardown partition does not retain all 107 normal-run cases" [ "$(LC_ALL=C sort "$listed" | uniq -d | wc -l | tr -d ' ')" -eq 0 ] \ || fail "teardown partition lists a normal-run case more than once" comm -13 "$definitions" <(LC_ALL=C sort "$listed") > "$tmp/undefined.txt" @@ -256,8 +256,8 @@ test_teardown_partition_preserves_every_full_suite_case() { | LC_ALL=C sort > "$expected_focused" cmp -s "$expected_focused" "$focused" \ || fail "teardown partition dropped or absorbed a focused-only case" - [ "$(awk 'NR % 2 == 1 { count++ } END { print count + 0 }' "$listed")" -eq 53 ] \ - || fail "teardown partition A does not own exactly 53 cases" + [ "$(awk 'NR % 2 == 1 { count++ } END { print count + 0 }' "$listed")" -eq 54 ] \ + || fail "teardown partition A does not own exactly 54 cases" [ "$(awk 'NR % 2 == 0 { count++ } END { print count + 0 }' "$listed")" -eq 53 ] \ || fail "teardown partition B does not own exactly 53 cases" assert_grep 'FM_TEST_PART_INDEX=1 FM_TEST_PART_TOTAL=2' "$wrapper_a" \ @@ -266,7 +266,7 @@ test_teardown_partition_preserves_every_full_suite_case() { "teardown wrapper B does not select partition 2/2" [ ! -e "$ROOT/tests/fm-teardown.test.sh" ] \ || fail "the unsplit teardown test remains in the behavior inventory" - pass "teardown wrappers preserve all 106 normal cases and three focused-only cases" + pass "teardown wrappers preserve all 107 normal cases and three focused-only cases" } test_checked_in_plan_is_complete_balanced_and_deterministic diff --git a/tests/fm-report-stack-a.test.sh b/tests/fm-report-stack-a.test.sh index 00d58627c1..4864abca4d 100755 --- a/tests/fm-report-stack-a.test.sh +++ b/tests/fm-report-stack-a.test.sh @@ -1,3 +1,4 @@ #!/usr/bin/env bash set -u +# shellcheck source=tests/fm-report-stack-suite.sh disable=SC1091 FM_TEST_PART_INDEX=1 FM_TEST_PART_TOTAL=2 . "$(dirname "${BASH_SOURCE[0]}")/fm-report-stack-suite.sh" diff --git a/tests/fm-report-stack-b.test.sh b/tests/fm-report-stack-b.test.sh index b750b715f4..5b5b5c8f87 100755 --- a/tests/fm-report-stack-b.test.sh +++ b/tests/fm-report-stack-b.test.sh @@ -1,3 +1,4 @@ #!/usr/bin/env bash set -u +# shellcheck source=tests/fm-report-stack-suite.sh disable=SC1091 FM_TEST_PART_INDEX=2 FM_TEST_PART_TOTAL=2 . "$(dirname "${BASH_SOURCE[0]}")/fm-report-stack-suite.sh" diff --git a/tests/fm-teardown-a.test.sh b/tests/fm-teardown-a.test.sh index f9a7672126..1d343286e0 100755 --- a/tests/fm-teardown-a.test.sh +++ b/tests/fm-teardown-a.test.sh @@ -1,3 +1,4 @@ #!/usr/bin/env bash set -u +# shellcheck source=tests/fm-teardown-suite.sh disable=SC1091 FM_TEST_PART_INDEX=1 FM_TEST_PART_TOTAL=2 . "$(dirname "${BASH_SOURCE[0]}")/fm-teardown-suite.sh" diff --git a/tests/fm-teardown-b.test.sh b/tests/fm-teardown-b.test.sh index 5ff9f16019..653021fbbe 100755 --- a/tests/fm-teardown-b.test.sh +++ b/tests/fm-teardown-b.test.sh @@ -1,3 +1,4 @@ #!/usr/bin/env bash set -u +# shellcheck source=tests/fm-teardown-suite.sh disable=SC1091 FM_TEST_PART_INDEX=2 FM_TEST_PART_TOTAL=2 . "$(dirname "${BASH_SOURCE[0]}")/fm-teardown-suite.sh" From 622eb712e228cf813b1dce45a472778dc769dd56 Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Sat, 1 Aug 2026 19:18:29 -0400 Subject: [PATCH 08/10] no-mistakes(review): Refresh current behavior shard plan documentation --- docs/ci-behavior-shards.md | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/docs/ci-behavior-shards.md b/docs/ci-behavior-shards.md index 049fa8e223..9e73119020 100644 --- a/docs/ci-behavior-shards.md +++ b/docs/ci-behavior-shards.md @@ -10,7 +10,7 @@ The source run is [GitHub Actions run 30648119449](https://github.com/ruby-dlee/ `tests/behavior-test-durations.tsv` records the per-script measurements derived from that run. -The runner contract test added with the sharding implementation and the newly landed `tests/lavish.test.sh` are included, so the completeness guard covers both immediately. +The current inventory contains 87 behavior-test scripts, including the runner contract test and every test added on `main` since the baseline run. ## Assignment @@ -22,20 +22,20 @@ The eight-shard checked-in plan has these estimated serial loads. | Shard | Tests | Estimated load | |---:|---:|---:| -| 1 | 1 | 563637 ms (9m24s) | -| 2 | 1 | 475500 ms (7m56s) | -| 3 | 1 | 423936 ms (7m04s) | -| 4 | 15 | 391516 ms (6m32s) | -| 5 | 17 | 391524 ms (6m32s) | -| 6 | 17 | 391519 ms (6m32s) | -| 7 | 17 | 391519 ms (6m32s) | -| 8 | 15 | 391517 ms (6m32s) | +| 1 | 1 | 475500 ms (7m56s) | +| 2 | 11 | 436053 ms (7m16s) | +| 3 | 15 | 436051 ms (7m16s) | +| 4 | 13 | 436048 ms (7m16s) | +| 5 | 12 | 436056 ms (7m16s) | +| 6 | 10 | 436296 ms (7m16s) | +| 7 | 10 | 436222 ms (7m16s) | +| 8 | 15 | 436053 ms (7m16s) | The account-routing and report-stack suites keep their original test functions and assertions in shared suite files, while two runner wrappers partition each call list deterministically between isolated runners. The report-stack partition keeps the persistent-retention-owner setup in the same wrapper as its generation-interruption dependent. -The largest remaining indivisible test file, `tests/fm-teardown.test.sh`, sets the 9-minute-24-second floor. +The largest remaining indivisible test file, `tests/fm-checkout-refresh.test.sh`, sets the 7-minute-56-second floor. ## Measured GitHub Actions result @@ -60,7 +60,9 @@ The serial baseline was 57 minutes 23 seconds, so the measured behavior check wa The same run measured the split wrappers at 423936 ms for account-routing A, 335548 ms for account-routing B, 261383 ms for report-stack A, and 243164 ms for report-stack B. -Those values replaced the provisional half-suite estimates in `tests/behavior-test-durations.tsv` and produced the checked-in assignment table above. +Those values replaced the provisional half-suite estimates in `tests/behavior-test-durations.tsv` and produced the assignment measured in that run. + +The checked-in plan above also reflects the later teardown-suite split and tests added on `main`. ## Coverage guard From 7263f68bc79af5b417d9a51ed034a059f3dd0b86 Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Sat, 1 Aug 2026 19:19:51 -0400 Subject: [PATCH 09/10] no-mistakes(document): Refresh behavior sharding documentation --- CONTRIBUTING.md | 4 ++-- docs/configuration.md | 2 +- tests/fm-fleet-sync.test.sh | 2 +- tests/fm-gotmp.test.sh | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3833b7ed9d..a3b6b44ac4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,8 +79,8 @@ tmp=$(mktemp -d) && printf 'done: smoke\n' > "$tmp/smoke.status" && FM_STATE_OVE Agent Fleet is independently packaged under `tools/agent-fleet` and requires Python 3.11 or newer plus `uv`. Run the complete locked verification in [`tools/agent-fleet/RELEASING.md`](tools/agent-fleet/RELEASING.md) before pushing; that document also owns versioning, tagging, and clean-install verification. -Discover tests by listing `tests/*.test.sh`: each is a self-contained bash script named `.test.sh`, and its header comment describes what it covers, so run one directly to focus on a subject. -When triaging a red `Behavior tests` job, remember that CI runs that loop under `set -eu` while the local loop above does not: CI stops at the first failing script, so its log proves nothing about the suites that sort after it, and a red job is a lower bound on the failure count rather than the whole list. +Discover behavior-test entrypoints by listing `tests/*.test.sh` and run one directly to focus on a subject; partition wrappers source their matching `tests/*-suite.sh` implementation. +When triaging a red behavior shard, use its begin and end markers to identify each failing script: the shard continues through its complete assignment and records every exit code before the final `Behavior tests` job verifies the executed union. Reproduce with the exclusion-aware local loop to see every safe failure at once before concluding which ones are real. Reproduce in a checkout whose `origin` is the repository's https URL, as CI's own checkout is: the secondmate network-authority fixtures assert that the product pins the resolved address of the origin host, and a checkout whose `origin` is a local filesystem path has no host to pin, so those cases refuse for a reason that exists only locally. Run the suites from a checkout sitting on its default branch, not from a task-branch worktree - the worktree-tangle guard fires and several secondmate suites require the default branch, which produces more failures that are pure local artifacts. diff --git a/docs/configuration.md b/docs/configuration.md index 8d4f1ff52c..503f41e2c6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -94,7 +94,7 @@ See [`wedge-alarm.md`](wedge-alarm.md) for the channel reference and macOS verif The tracked `.no-mistakes.yaml` keeps test evidence outside the repo and defines `commands.test` so no-mistakes runs firstmate's bash behavior suite directly. That evidence policy is specific to the firstmate repo: target projects may legitimately commit `.no-mistakes/evidence/` from their own no-mistakes pipeline, but firstmate keeps `.no-mistakes/` local and CI rejects tracked entries under that path. That command requires `tmux` on `PATH`, prints `tmux -V`, runs every `tests/*.test.sh` with `bash`, and fails if any script exits non-zero. -It intentionally mirrors the behavior-test baseline in [`.github/workflows/ci.yml`](../.github/workflows/ci.yml) instead of delegating the test step to an agent. +It intentionally runs the complete behavior-test inventory serially instead of reproducing the duration-balanced CI sharding owned by [`bin/fm-behavior-shards.sh`](../bin/fm-behavior-shards.sh) or delegating the test step to an agent. ## Captain preferences (data/captain.md) diff --git a/tests/fm-fleet-sync.test.sh b/tests/fm-fleet-sync.test.sh index 1c388b49d1..7c7031c38c 100755 --- a/tests/fm-fleet-sync.test.sh +++ b/tests/fm-fleet-sync.test.sh @@ -139,7 +139,7 @@ build_packed_prunable() { plant_packed_refs_lock() { : > "$1/.git/packed-refs.lock"; } -# lsof shims mirror tests/fm-teardown.test.sh: no-holder (provably free), a live +# lsof shims mirror tests/fm-teardown-suite.sh: no-holder (provably free), a live # holder, and an lsof error. Written into a per-home fakebin/ prepended to PATH. lsof_no_holder() { cat > "$1/lsof" <<'SH' diff --git a/tests/fm-gotmp.test.sh b/tests/fm-gotmp.test.sh index cd2e62421e..8c4cc4fe1d 100755 --- a/tests/fm-gotmp.test.sh +++ b/tests/fm-gotmp.test.sh @@ -7,7 +7,7 @@ # # The fm-spawn side is verified both structurally (the source has the contract lines) # and behaviorally (the mkdir + meta-write pattern it uses). -# Teardown cleanup is covered by tests/fm-teardown.test.sh's full lifecycle fixture. +# Teardown cleanup is covered by tests/fm-teardown-suite.sh's full lifecycle fixture. set -u ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" From cff6249bbb21e6e76c8eb76973c27b9b3a359cbb Mon Sep 17 00:00:00 2001 From: Dongkeun Lee Date: Sat, 1 Aug 2026 20:01:52 -0400 Subject: [PATCH 10/10] ci: rerun checks after alias cleanup