|
| 1 | +#!/usr/bin/env bash |
| 2 | +# fm-behavior-shards.sh - duration-balanced behavior-test planning, execution, |
| 3 | +# timing refresh, and executed-manifest completeness verification. |
| 4 | +# |
| 5 | +# Usage: |
| 6 | +# bin/fm-behavior-shards.sh --check <shard-count> |
| 7 | +# bin/fm-behavior-shards.sh --plan <shard-count> |
| 8 | +# bin/fm-behavior-shards.sh --run <shard> <shard-count> <manifest.tsv> |
| 9 | +# bin/fm-behavior-shards.sh --verify <shard-count> <manifest-dir> |
| 10 | +# bin/fm-behavior-shards.sh --record <durations.tsv> |
| 11 | +# |
| 12 | +# The checked-in duration file defaults to tests/behavior-test-durations.tsv. |
| 13 | +# Override it with FM_BEHAVIOR_DURATIONS_FILE for fixture tests only. |
| 14 | +# Planning uses deterministic longest-processing-time assignment: descending |
| 15 | +# duration, path as the stable secondary key, and lowest shard number for load |
| 16 | +# ties. Each shard runs its assigned files serially. CI supplies one isolated |
| 17 | +# runner, TMPDIR, and TMUX_TMPDIR per shard. |
| 18 | +set -eu |
| 19 | + |
| 20 | +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P) |
| 21 | +DURATIONS=${FM_BEHAVIOR_DURATIONS_FILE:-$ROOT/tests/behavior-test-durations.tsv} |
| 22 | +TAB=$(printf '\t') |
| 23 | +WORK=$(mktemp -d "${TMPDIR:-/tmp}/fm-behavior-shards.XXXXXX") || exit 1 |
| 24 | +trap 'rm -rf "$WORK"' EXIT |
| 25 | + |
| 26 | +usage() { |
| 27 | + sed -n '2,18s/^# \{0,1\}//p' "$0" |
| 28 | +} |
| 29 | + |
| 30 | +die() { |
| 31 | + printf 'fm-behavior-shards: %s\n' "$*" >&2 |
| 32 | + exit 2 |
| 33 | +} |
| 34 | + |
| 35 | +validate_positive_integer() { |
| 36 | + local value=$1 label=$2 |
| 37 | + case "$value" in |
| 38 | + ''|*[!0-9]*) die "$label must be a positive integer (got '$value')" ;; |
| 39 | + esac |
| 40 | + [ "$value" -gt 0 ] || die "$label must be greater than zero" |
| 41 | +} |
| 42 | + |
| 43 | +inventory() { |
| 44 | + find "$ROOT/tests" -maxdepth 1 -type f -name '*.test.sh' -print \ |
| 45 | + | sed "s#^$ROOT/##" \ |
| 46 | + | LC_ALL=C sort |
| 47 | +} |
| 48 | + |
| 49 | +normalize_durations() { |
| 50 | + local normalized=$1 |
| 51 | + [ -f "$DURATIONS" ] || die "duration file not found: $DURATIONS" |
| 52 | + if ! awk -F '\t' ' |
| 53 | + /^[[:space:]]*#/ || /^[[:space:]]*$/ { next } |
| 54 | + NF != 2 { |
| 55 | + printf "duration row %d must be: <positive-ms><TAB>tests/<name>.test.sh\n", NR > "/dev/stderr" |
| 56 | + bad = 1 |
| 57 | + next |
| 58 | + } |
| 59 | + $1 !~ /^[1-9][0-9]*$/ { |
| 60 | + printf "duration row %d has invalid milliseconds: %s\n", NR, $1 > "/dev/stderr" |
| 61 | + bad = 1 |
| 62 | + next |
| 63 | + } |
| 64 | + $2 !~ /^tests\/[A-Za-z0-9._-]+\.test\.sh$/ { |
| 65 | + printf "duration row %d has invalid test path: %s\n", NR, $2 > "/dev/stderr" |
| 66 | + bad = 1 |
| 67 | + next |
| 68 | + } |
| 69 | + { print $1 "\t" $2 } |
| 70 | + END { exit bad ? 1 : 0 } |
| 71 | + ' "$DURATIONS" > "$normalized"; then |
| 72 | + die "invalid duration file: $DURATIONS" |
| 73 | + fi |
| 74 | + [ -s "$normalized" ] || die "duration file has no test rows: $DURATIONS" |
| 75 | +} |
| 76 | + |
| 77 | +validate_inventory() { |
| 78 | + local normalized=$1 inventory_file=$2 paths_file=$3 duplicates |
| 79 | + inventory > "$inventory_file" |
| 80 | + cut -f2 "$normalized" | LC_ALL=C sort > "$paths_file" |
| 81 | + duplicates=$(cut -f2 "$normalized" | LC_ALL=C sort | uniq -d) |
| 82 | + if [ -n "$duplicates" ]; then |
| 83 | + printf 'fm-behavior-shards: duplicate duration paths:\n%s\n' "$duplicates" >&2 |
| 84 | + return 1 |
| 85 | + fi |
| 86 | + if ! diff -u "$inventory_file" "$paths_file" >&2; then |
| 87 | + printf 'fm-behavior-shards: duration inventory does not exactly match tests/*.test.sh\n' >&2 |
| 88 | + return 1 |
| 89 | + fi |
| 90 | +} |
| 91 | + |
| 92 | +make_plan() { |
| 93 | + local shard_count=$1 destination=$2 normalized inventory_file paths_file |
| 94 | + validate_positive_integer "$shard_count" "shard count" |
| 95 | + normalized="$WORK/durations.tsv" |
| 96 | + inventory_file="$WORK/inventory.txt" |
| 97 | + paths_file="$WORK/duration-paths.txt" |
| 98 | + normalize_durations "$normalized" |
| 99 | + validate_inventory "$normalized" "$inventory_file" "$paths_file" \ |
| 100 | + || die "coverage guard failed" |
| 101 | + |
| 102 | + LC_ALL=C sort -t "$TAB" -k1,1nr -k2,2 "$normalized" \ |
| 103 | + | awk -F '\t' -v OFS='\t' -v shard_count="$shard_count" ' |
| 104 | + BEGIN { |
| 105 | + for (i = 1; i <= shard_count; i++) load[i] = 0 |
| 106 | + } |
| 107 | + { |
| 108 | + target = 1 |
| 109 | + for (i = 2; i <= shard_count; i++) { |
| 110 | + if (load[i] < load[target]) target = i |
| 111 | + } |
| 112 | + print target, $1, $2 |
| 113 | + load[target] += $1 |
| 114 | + } |
| 115 | + ' > "$destination" |
| 116 | + |
| 117 | + [ "$(wc -l < "$destination" | tr -d ' ')" -eq "$(wc -l < "$inventory_file" | tr -d ' ')" ] \ |
| 118 | + || die "planner did not assign every test exactly once" |
| 119 | +} |
| 120 | + |
| 121 | +print_plan_summary() { |
| 122 | + local plan=$1 shard_count=$2 |
| 123 | + awk -F '\t' -v shard_count="$shard_count" ' |
| 124 | + { count[$1]++; load[$1] += $2 } |
| 125 | + END { |
| 126 | + for (i = 1; i <= shard_count; i++) { |
| 127 | + printf "FM_BEHAVIOR_SHARD shard=%d tests=%d estimated_ms=%d\n", i, count[i] + 0, load[i] + 0 |
| 128 | + } |
| 129 | + } |
| 130 | + ' "$plan" |
| 131 | +} |
| 132 | + |
| 133 | +now_ms() { |
| 134 | + local value |
| 135 | + value=$(date +%s%3N 2>/dev/null || true) |
| 136 | + case "$value" in |
| 137 | + *[!0-9]*|'') printf '%s000\n' "$(date +%s)" ;; |
| 138 | + *) printf '%s\n' "$value" ;; |
| 139 | + esac |
| 140 | +} |
| 141 | + |
| 142 | +run_test() { |
| 143 | + local path=$1 begin_ms end_ms duration rc had_errexit=0 |
| 144 | + begin_ms=$(now_ms) |
| 145 | + printf 'FM_BEHAVIOR_TEST_BEGIN %s\n' "$path" |
| 146 | + if [ "${GITHUB_ACTIONS:-}" = true ]; then |
| 147 | + printf '::group::%s\n' "$path" |
| 148 | + fi |
| 149 | + case $- in *e*) had_errexit=1 ;; esac |
| 150 | + set +e |
| 151 | + "$ROOT/$path" |
| 152 | + rc=$? |
| 153 | + if [ "$had_errexit" -eq 1 ]; then |
| 154 | + set -e |
| 155 | + fi |
| 156 | + if [ "${GITHUB_ACTIONS:-}" = true ]; then |
| 157 | + printf '::endgroup::\n' |
| 158 | + fi |
| 159 | + end_ms=$(now_ms) |
| 160 | + duration=$((end_ms - begin_ms)) |
| 161 | + [ "$duration" -gt 0 ] || duration=1 |
| 162 | + printf 'FM_BEHAVIOR_TEST_END %s exit=%s duration_ms=%s\n' "$path" "$rc" "$duration" |
| 163 | + RUN_TEST_RC=$rc |
| 164 | + RUN_TEST_DURATION=$duration |
| 165 | +} |
| 166 | + |
| 167 | +run_shard() { |
| 168 | + local shard=$1 shard_count=$2 manifest=$3 plan path plan_shard _estimate |
| 169 | + local selected=0 failures=0 |
| 170 | + validate_positive_integer "$shard" "shard" |
| 171 | + validate_positive_integer "$shard_count" "shard count" |
| 172 | + [ "$shard" -le "$shard_count" ] || die "shard $shard exceeds shard count $shard_count" |
| 173 | + plan="$WORK/plan.tsv" |
| 174 | + make_plan "$shard_count" "$plan" |
| 175 | + mkdir -p "$(dirname "$manifest")" |
| 176 | + : > "$manifest" |
| 177 | + cd "$ROOT" || die "cannot enter repo root: $ROOT" |
| 178 | + while IFS="$TAB" read -r plan_shard _estimate path; do |
| 179 | + [ "$plan_shard" = "$shard" ] || continue |
| 180 | + selected=$((selected + 1)) |
| 181 | + run_test "$path" |
| 182 | + printf '%s\t%s\t%s\t%s\n' "$shard" "$path" "$RUN_TEST_RC" "$RUN_TEST_DURATION" >> "$manifest" |
| 183 | + [ "$RUN_TEST_RC" -eq 0 ] || failures=$((failures + 1)) |
| 184 | + done < "$plan" |
| 185 | + [ "$selected" -gt 0 ] || die "shard $shard has no assigned tests" |
| 186 | + printf 'FM_BEHAVIOR_SHARD_RESULT shard=%s/%s tests=%s failed=%s manifest=%s\n' \ |
| 187 | + "$shard" "$shard_count" "$selected" "$failures" "$manifest" |
| 188 | + [ "$failures" -eq 0 ] |
| 189 | +} |
| 190 | + |
| 191 | +record_durations() { |
| 192 | + local destination=$1 staging path failed=0 |
| 193 | + [ -n "$destination" ] || die "--record requires an output path" |
| 194 | + staging="$WORK/recorded.tsv" |
| 195 | + : > "$staging" |
| 196 | + cd "$ROOT" || die "cannot enter repo root: $ROOT" |
| 197 | + while IFS= read -r path; do |
| 198 | + run_test "$path" |
| 199 | + printf '%s\t%s\n' "$RUN_TEST_DURATION" "$path" >> "$staging" |
| 200 | + [ "$RUN_TEST_RC" -eq 0 ] || failed=$((failed + 1)) |
| 201 | + done < <(inventory) |
| 202 | + if [ "$failed" -ne 0 ]; then |
| 203 | + printf 'fm-behavior-shards: refusing to publish timings because %s tests failed\n' "$failed" >&2 |
| 204 | + return 1 |
| 205 | + fi |
| 206 | + mkdir -p "$(dirname "$destination")" |
| 207 | + { |
| 208 | + printf '# Behavior test durations in milliseconds.\n' |
| 209 | + printf '# Refresh with: bin/fm-behavior-shards.sh --record tests/behavior-test-durations.tsv\n' |
| 210 | + printf '# Recorded UTC: %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" |
| 211 | + LC_ALL=C sort -t "$TAB" -k2,2 "$staging" |
| 212 | + } > "$destination.tmp.$$" |
| 213 | + mv "$destination.tmp.$$" "$destination" |
| 214 | + printf 'FM_BEHAVIOR_DURATIONS recorded=%s tests=%s\n' "$destination" "$(wc -l < "$staging" | tr -d ' ')" |
| 215 | +} |
| 216 | + |
| 217 | +verify_manifests() { |
| 218 | + local shard_count=$1 manifest_dir=$2 plan all observed expected file file_count shard |
| 219 | + validate_positive_integer "$shard_count" "shard count" |
| 220 | + [ -d "$manifest_dir" ] || die "manifest directory not found: $manifest_dir" |
| 221 | + plan="$WORK/plan.tsv" |
| 222 | + all="$WORK/executed.tsv" |
| 223 | + observed="$WORK/observed.tsv" |
| 224 | + expected="$WORK/expected.tsv" |
| 225 | + make_plan "$shard_count" "$plan" |
| 226 | + : > "$all" |
| 227 | + |
| 228 | + file_count=$(find "$manifest_dir" -maxdepth 1 -type f -name 'executed-*.tsv' | wc -l | tr -d ' ') |
| 229 | + [ "$file_count" -eq "$shard_count" ] \ |
| 230 | + || die "expected $shard_count executed manifests, found $file_count in $manifest_dir" |
| 231 | + shard=1 |
| 232 | + while [ "$shard" -le "$shard_count" ]; do |
| 233 | + file="$manifest_dir/executed-$shard.tsv" |
| 234 | + [ -s "$file" ] || die "missing or empty executed manifest for shard $shard: $file" |
| 235 | + if ! awk -F '\t' -v expected_shard="$shard" ' |
| 236 | + NF != 4 { printf "invalid manifest row %d in shard %d\n", NR, expected_shard > "/dev/stderr"; bad = 1; next } |
| 237 | + $1 != expected_shard { printf "manifest row %d claims shard %s, expected %d\n", NR, $1, expected_shard > "/dev/stderr"; bad = 1 } |
| 238 | + $2 !~ /^tests\/[A-Za-z0-9._-]+\.test\.sh$/ { printf "invalid executed path at row %d: %s\n", NR, $2 > "/dev/stderr"; bad = 1 } |
| 239 | + $3 !~ /^[0-9]+$/ { printf "invalid exit code at row %d: %s\n", NR, $3 > "/dev/stderr"; bad = 1 } |
| 240 | + $4 !~ /^[0-9]+$/ { printf "invalid duration at row %d: %s\n", NR, $4 > "/dev/stderr"; bad = 1 } |
| 241 | + END { exit bad ? 1 : 0 } |
| 242 | + ' "$file"; then |
| 243 | + die "invalid executed manifest: $file" |
| 244 | + fi |
| 245 | + cat "$file" >> "$all" |
| 246 | + shard=$((shard + 1)) |
| 247 | + done |
| 248 | + |
| 249 | + awk -F '\t' -v OFS='\t' '{ print $1, $3 }' "$plan" | LC_ALL=C sort -t "$TAB" -k1,1n -k2,2 > "$expected" |
| 250 | + awk -F '\t' -v OFS='\t' '{ print $1, $2 }' "$all" | LC_ALL=C sort -t "$TAB" -k1,1n -k2,2 > "$observed" |
| 251 | + if ! diff -u "$expected" "$observed" >&2; then |
| 252 | + die "executed shard union differs from the complete planned test inventory" |
| 253 | + fi |
| 254 | + if ! awk -F '\t' ' |
| 255 | + $3 != 0 { printf "failed test: shard=%s path=%s exit=%s duration_ms=%s\n", $1, $2, $3, $4 > "/dev/stderr"; bad = 1 } |
| 256 | + END { exit bad ? 1 : 0 } |
| 257 | + ' "$all"; then |
| 258 | + die "one or more behavior tests failed" |
| 259 | + fi |
| 260 | + printf 'FM_BEHAVIOR_COMPLETENESS ok tests=%s shards=%s\n' "$(wc -l < "$all" | tr -d ' ')" "$shard_count" |
| 261 | +} |
| 262 | + |
| 263 | +case "${1:-}" in |
| 264 | + --check) |
| 265 | + [ "$#" -eq 2 ] || die "--check requires <shard-count>" |
| 266 | + plan="$WORK/plan.tsv" |
| 267 | + make_plan "$2" "$plan" |
| 268 | + printf 'FM_BEHAVIOR_PLAN ok tests=%s shards=%s\n' "$(wc -l < "$plan" | tr -d ' ')" "$2" |
| 269 | + print_plan_summary "$plan" "$2" |
| 270 | + ;; |
| 271 | + --plan) |
| 272 | + [ "$#" -eq 2 ] || die "--plan requires <shard-count>" |
| 273 | + make_plan "$2" "$WORK/plan.tsv" |
| 274 | + cat "$WORK/plan.tsv" |
| 275 | + ;; |
| 276 | + --run) |
| 277 | + [ "$#" -eq 4 ] || die "--run requires <shard> <shard-count> <manifest.tsv>" |
| 278 | + run_shard "$2" "$3" "$4" |
| 279 | + ;; |
| 280 | + --verify) |
| 281 | + [ "$#" -eq 3 ] || die "--verify requires <shard-count> <manifest-dir>" |
| 282 | + verify_manifests "$2" "$3" |
| 283 | + ;; |
| 284 | + --record) |
| 285 | + [ "$#" -eq 2 ] || die "--record requires <durations.tsv>" |
| 286 | + record_durations "$2" |
| 287 | + ;; |
| 288 | + -h|--help) |
| 289 | + usage |
| 290 | + ;; |
| 291 | + *) |
| 292 | + usage >&2 |
| 293 | + exit 2 |
| 294 | + ;; |
| 295 | +esac |
0 commit comments