From b9da5666ba113360c5698801e666a5ebe440b629 Mon Sep 17 00:00:00 2001 From: Trillium Smith Date: Fri, 31 Jul 2026 09:49:23 -0700 Subject: [PATCH 1/8] Add fm-no-mistakes-liveness.sh: check no-mistakes run liveness and task ownership --- bin/fm-no-mistakes-liveness.sh | 464 +++++++++++++++++++++++++++++++++ 1 file changed, 464 insertions(+) create mode 100755 bin/fm-no-mistakes-liveness.sh diff --git a/bin/fm-no-mistakes-liveness.sh b/bin/fm-no-mistakes-liveness.sh new file mode 100755 index 0000000000..e1474ea967 --- /dev/null +++ b/bin/fm-no-mistakes-liveness.sh @@ -0,0 +1,464 @@ +#!/usr/bin/env bash +# fm-no-mistakes-liveness.sh - check liveness of no-mistakes runs and match them to tasks. +# +# Purpose: answer two related questions about shared no-mistakes daemon runs: +# 1. Which entries in `no-mistakes runs` showing `running` are genuinely live and +# in-progress, and which task/home (if any) owns each one? +# 2. During ordinary supervision of an in-flight no-mistakes run, detect when the +# run itself has silently hung (not just waiting on CI). +# +# This is a check/tool that reports liveness status, not an auto-remediator. +# It does not restart the daemon, kill a run, or interrupt a crewmate. +# +# Usage: +# fm-no-mistakes-liveness.sh [] Check a specific task's run liveness +# fm-no-mistakes-liveness.sh --all List liveness of all "running" runs +# fm-no-mistakes-liveness.sh --help Show this help +# +# Exit codes: +# 0 - success (run is live, or all runs checked) +# 1 - run is not live/hung +# 2 - usage error or precondition failed +# +# Output format (one line per run when checking multiple): +# run: · branch: · status: · owned-by: · last-activity: s +# +set -u + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" +FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" +STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" + +# shellcheck source=bin/fm-backend.sh +. "$SCRIPT_DIR/fm-backend.sh" + +# Timeouts for CLI calls +NM_TIMEOUT=${FM_NM_LIVENESS_TIMEOUT:-15} +case "$NM_TIMEOUT" in ''|*[!0-9]*) NM_TIMEOUT=15 ;; esac + +# Freshness threshold: a run is considered hung if its last activity is older than this +STALE_THRESHOLD=${FM_NM_LIVENESS_STALE:-300} +case "$STALE_THRESHOLD" in ''|*[!0-9]*) STALE_THRESHOLD=300 ;; esac + +# --- utilities --------------------------------------------------------------- + +trim() { + local s=${1:-} + s="${s#"${s%%[![:space:]]*}"}" + s="${s%"${s##*[![:space:]]}"}" + printf '%s' "$s" +} + +strip_quotes() { + local s + s=$(trim "${1:-}") + case "$s" in + \"*\") s=${s#\"}; s=${s%\"} ;; + esac + trim "$s" +} + +die() { + printf 'error: %s\n' "$@" >&2 + exit 2 +} + +usage() { + sed -n '2,11p' "$0" | sed 's/^# //' + exit 2 +} + +# Bounded no-mistakes call; stdout only, never fails the script. +HAVE_TIMEOUT=none +if command -v timeout >/dev/null 2>&1; then HAVE_TIMEOUT=timeout +elif command -v gtimeout >/dev/null 2>&1; then HAVE_TIMEOUT=gtimeout +elif command -v perl >/dev/null 2>&1; then HAVE_TIMEOUT=perl +fi +nm_run() { # + case "$HAVE_TIMEOUT" in + timeout) timeout "$NM_TIMEOUT" no-mistakes "$@" 2>/dev/null || true ;; + gtimeout) gtimeout "$NM_TIMEOUT" no-mistakes "$@" 2>/dev/null || true ;; + perl) perl -e 'my $t = shift; my $pid = fork; die "fork failed" unless defined $pid; if (!$pid) { setpgrp(0, 0); exec @ARGV } local $SIG{ALRM} = sub { kill "TERM", -$pid; select undef, undef, undef, 0.2; kill "KILL", -$pid; exit 124 }; alarm $t; waitpid $pid, 0; exit($? >> 8)' "$NM_TIMEOUT" no-mistakes "$@" 2>/dev/null || true ;; + *) true ;; + esac +} + +# Extract TOON field value +nm_field() { # + local output=$1 key=$2 + printf '%s\n' "$output" | sed -n "s/^[[:space:]]*$key:[[:space:]]*\(.*\)/\1/p" | head -1 +} + +# Get current time in seconds since epoch +get_timestamp() { + if command -v gdate >/dev/null 2>&1; then + gdate +%s + else + date +%s + fi +} + +# Parse timestamp from log line (common formats: "2026-07-31 09:22:15", "2026-07-31 09:22") +parse_log_timestamp() { + local line=$1 + # Try to extract ISO-like timestamp from log line + if [[ $line =~ ([0-9]{4})-([0-9]{2})-([0-9]{2})\ ([0-9]{2}):([0-9]{2}) ]]; then + local year="${BASH_REMATCH[1]}" + local month="${BASH_REMATCH[2]}" + local day="${BASH_REMATCH[3]}" + local hour="${BASH_REMATCH[4]}" + local min="${BASH_REMATCH[5]}" + + # Convert to epoch seconds using date + if command -v gdate >/dev/null 2>&1; then + gdate -d "$year-$month-$day $hour:$min" +%s 2>/dev/null || echo 0 + else + date -j -f "%Y-%m-%d %H:%M" "$year-$month-$day $hour:$min" +%s 2>/dev/null || echo 0 + fi + else + echo 0 + fi +} + +# Check if a run is owned by a task by matching branch names +# Also checks secondmate homes and project homes +find_task_for_branch() { # + local branch=$1 task_meta task_id + local homes_to_check=("$STATE") + + # Also check the current directory's git root state (for project tasks) + local git_root + if git_root=$(git rev-parse --show-toplevel 2>/dev/null); then + [ -d "$git_root/state" ] && [ "$git_root/state" != "$STATE" ] && homes_to_check+=("$git_root/state") + fi + + # Also check secondmate homes if secondmates.md exists + if [ -f "$FM_HOME/data/secondmates.md" ] 2>/dev/null; then + while IFS= read -r line; do + [ -n "$line" ] || continue + case "$line" in '#'*|'- '*) continue ;; esac + # Parse secondmate entry: "- [id](path) — description" + # Simple extraction: look for [...] and (...) patterns + if [[ "$line" =~ \[([^\]]+)\]\(([^\)]+)\) ]]; then + local sm_path="${BASH_REMATCH[2]}" + [ -d "$sm_path/state" ] && homes_to_check+=("$sm_path/state") + fi + done < "$FM_HOME/data/secondmates.md" + fi + + # Check all meta files in all homes + for home_state in "${homes_to_check[@]}"; do + [ -d "$home_state" ] || continue + for task_meta in "$home_state"/*.meta; do + [ -f "$task_meta" ] || continue + local worktree + worktree=$(grep "^worktree=" "$task_meta" | tail -1 | cut -d= -f2-) + [ -n "$worktree" ] || continue + [ -d "$worktree" ] || continue + + local task_branch + task_branch=$(git -C "$worktree" symbolic-ref --quiet --short HEAD 2>/dev/null || true) + if [ "$task_branch" = "$branch" ]; then + # Found a match - return the task id (basename of meta file without .meta) + # Qualify with home label if from a secondmate + task_id=$(basename "$task_meta" .meta) + if [ "$home_state" != "$STATE" ]; then + task_id="$(basename "$(dirname "$home_state")")/$task_id" + fi + printf '%s' "$task_id" + return 0 + fi + done + done + + printf 'unknown' +} + +# --- run liveness check ------------------------------------------------------ + +# Extract step information: name,status,findings,duration_ms from the steps table +# Returns: name|status|duration (e.g., "review|fix_review|430107") +get_step_info() { # + local output=$1 step_name=$2 line + line=$(printf '%s\n' "$output" | grep -E "^[[:space:]]*$step_name," | head -1 | sed 's/^ *//') + if [ -n "$line" ]; then + local step status rest duration + step="${line%%,*}" + rest="${line#*,}" + status="$(strip_quotes "$(trim "${rest%%,*}")")" + rest="${rest#*,}" + # Skip findings count (third field) + rest="${rest#*,}" + duration="$(trim "${rest#*,}")" + printf '%s|%s|%s' "$step" "$status" "$duration" + fi +} + +# Extract the currently-active step (running, fixing, or if awaiting_agent, the gate step) +# Returns step name +get_active_step() { # + local output=$1 line step + + # First check if there's a running or fixing step + line=$(printf '%s\n' "$output" | grep -E '^[[:space:]]*[^,]+,[[:space:]]*"?(running|fixing)"?[[:space:]]*,' | head -1 | sed 's/^ *//') + if [ -n "$line" ]; then + step="${line%%,*}" + printf '%s' "$step" + return 0 + fi + + # If parked at gate, find the gate step + if printf '%s\n' "$output" | grep -q '^[[:space:]]*gate:[[:space:]]*$'; then + step=$(printf '%s\n' "$output" | sed -n '/^[[:space:]]*gate:[[:space:]]*$/,/^[^[:space:]][^:]*:/s/^[[:space:]]*step:[[:space:]]*\(.*\)/\1/p' | head -1) + step=$(strip_quotes "$(trim "$step")") + [ -n "$step" ] && printf '%s' "$step" && return 0 + fi + + # Fallback to step from status field + step=$(printf '%s\n' "$output" | sed -n 's/^[[:space:]]*status:[[:space:]]*\(.*\)/\1/p' | head -1) + step=$(strip_quotes "$(trim "$step")") + [ -n "$step" ] && printf '%s' "$step" && return 0 + + return 1 +} + +# Check a single run's liveness by reading step activity +# Returns elapsed seconds (> 0) and exit code 0 for live, 1 for hung +check_run_liveness() { # + local run_id=$1 + local run_out status outcome awaiting_msg elapsed + + run_out=$(nm_run axi status --run "$run_id") + [ -n "$run_out" ] || return 1 + + # Parse run info + status=$(strip_quotes "$(nm_field "$run_out" status)") + outcome=$(strip_quotes "$(nm_field "$run_out" outcome)") + + # Terminal outcomes are not hung + if [ -n "$outcome" ]; then + printf '0' + return 0 + fi + + # Check if awaiting_agent (parked at gate waiting for input) + awaiting_msg=$(printf '%s\n' "$run_out" | grep -E '^[[:space:]]*awaiting_agent:' | head -1) + if [ -n "$awaiting_msg" ]; then + # Deliberately parked at gate - not hung, not stale. Report 0 elapsed. + printf '0' + return 0 + fi + + # Get the active step + local active_step step_info step_status duration + active_step=$(get_active_step "$run_out") || return 0 + + # Get step info (name|status|duration_ms) + step_info=$(get_step_info "$run_out" "$active_step") + if [ -z "$step_info" ]; then + # Could not get step info - assume live + printf '0' + return 0 + fi + + step_status="${step_info#*|}" + step_status="${step_status%|*}" + duration="${step_info##*|}" + + # Ensure duration is numeric + case "$duration" in ''|*[!0-9]*) + # Check log for timestamps instead + local log_tail last_update + log_tail=$(nm_run axi logs --step "$active_step" --run "$run_id" 2>/dev/null || true) + if [ -z "$log_tail" ]; then + printf '0' + return 0 + fi + + # Get the date from the last non-empty line in the log + last_update=$(printf '%s\n' "$log_tail" | grep -v '^[[:space:]]*$' | tail -1) + if [ -z "$last_update" ]; then + printf '0' + return 0 + fi + + # Try to extract timestamp - if we can't, assume live + local ts now + ts=$(parse_log_timestamp "$last_update") + if [ "$ts" -eq 0 ]; then + printf '0' + return 0 + fi + now=$(get_timestamp) + elapsed=$((now - ts)) + ;; + *) + # Convert milliseconds to seconds + elapsed=$((duration / 1000)) + ;; + esac + + # Step is only hung if it's running/fixing and has been inactive for > threshold + case "$step_status" in + running|fixing) + if [ "$elapsed" -gt "$STALE_THRESHOLD" ]; then + printf '%d' "$elapsed" + return 1 # Hung + fi + ;; + pending|completed) + # Not actively running + elapsed=0 + ;; + esac + + printf '%d' "$elapsed" + return 0 # Live +} + +# List and check all running runs +check_all_runs() { + local runs_out run_row status branch run_id task elapsed liveness + runs_out=$(nm_run runs --limit 50) || die "Failed to list runs" + [ -n "$runs_out" ] || { printf 'no runs found\n'; return 0; } + + local total=0 live=0 hung=0 + while IFS= read -r run_row; do + run_row=$(trim "$run_row") + [ -n "$run_row" ] || continue + + status=${run_row%% *} + [ "$status" = "running" ] || continue + + total=$((total + 1)) + + # Parse: status branch sha date [pr-url] + # We only need branch for task lookup + local rest + rest=${run_row#* } + rest=$(trim "$rest") + branch=${rest%% *} + + # Get the run id by looking for axi status on a worktree with this branch + run_id="unknown" + # Try to find a task with this branch and get its run ID + task=$(find_task_for_branch "$branch") + if [ "$task" != "unknown" ]; then + # Found a task - get its run ID from axi status + local task_id="${task##*/}" + local meta_file="$STATE/$task_id.meta" + [ -f "$meta_file" ] || meta_file="/Users/trilliumsmith/code/firstmate/state/$task_id.meta" + if [ -f "$meta_file" ]; then + local worktree + worktree=$(grep "^worktree=" "$meta_file" | tail -1 | cut -d= -f2-) + if [ -n "$worktree" ] && [ -d "$worktree" ]; then + local axi_out + axi_out=$(cd "$worktree" && nm_run axi status 2>/dev/null || true) + if [ -n "$axi_out" ]; then + run_id=$(strip_quotes "$(nm_field "$axi_out" id)") + fi + fi + fi + fi + + # Check liveness only if we got a valid run ID + if [ "$run_id" != "unknown" ]; then + if elapsed=$(check_run_liveness "$run_id" 2>/dev/null); then + liveness="live" + live=$((live + 1)) + else + liveness="hung" + hung=$((hung + 1)) + fi + else + # Could not get run ID - report as unknown liveness + elapsed="unknown" + liveness="unknown" + fi + + printf 'run: %s · branch: %s · status: %s · owned-by: %s · last-activity: %ss\n' \ + "$run_id" "$branch" "$liveness" "$task" "$elapsed" + done <<< "$runs_out" + + printf '\nsummary: %d total, %d live, %d hung\n' "$total" "$live" "$hung" + [ "$hung" -eq 0 ] && return 0 || return 1 +} + +# Check a specific task's run liveness +check_task_run() { # + local task_id=$1 meta_file worktree branch task_branch git_root + + meta_file="$STATE/$task_id.meta" + if [ ! -f "$meta_file" ]; then + # Try the current git root's state directory + git_root=$(git rev-parse --show-toplevel 2>/dev/null || true) + [ -n "$git_root" ] && meta_file="$git_root/state/$task_id.meta" + fi + [ -f "$meta_file" ] || die "no metadata for $task_id" + + worktree=$(grep "^worktree=" "$meta_file" | tail -1 | cut -d= -f2-) + [ -n "$worktree" ] || die "no worktree in metadata for $task_id" + [ -d "$worktree" ] || die "worktree gone for $task_id" + + task_branch=$(git -C "$worktree" symbolic-ref --quiet --short HEAD 2>/dev/null || true) + [ -n "$task_branch" ] || die "detached HEAD in $task_id (no run to check)" + + # Find the run for this branch + local runs_out run_row status branch + runs_out=$(nm_run runs --limit 200) || die "Failed to list runs" + [ -n "$runs_out" ] || die "no runs found" + + while IFS= read -r run_row; do + run_row=$(trim "$run_row") + [ -n "$run_row" ] || continue + + status=${run_row%% *} + [ "$status" = "running" ] || continue + + local rest + rest=${run_row#* } + rest=$(trim "$rest") + branch=${rest%% *} + + if [ "$branch" = "$task_branch" ]; then + # Found a run for this task's branch + # Get full run details via axi status + local axi_out run_id + axi_out=$(nm_run axi status 2>/dev/null || true) + run_id=$(strip_quotes "$(nm_field "$axi_out" id)") + + [ -n "$run_id" ] || die "Could not get run ID" + + # Check liveness + local elapsed + if elapsed=$(check_run_liveness "$run_id" 2>/dev/null); then + printf 'task: %s · branch: %s · status: live · last-activity: %ss\n' \ + "$task_id" "$branch" "$elapsed" + return 0 + else + printf 'task: %s · branch: %s · status: hung · last-activity: %ss\n' \ + "$task_id" "$branch" "$elapsed" + return 1 + fi + fi + done <<< "$runs_out" + + die "no running no-mistakes run for $task_id's branch ($task_branch)" +} + +# --- main -------------------------------------------------------------------- + +case "${1:-}" in + --help|-h) usage ;; + --all) check_all_runs ;; + --*) die "unknown option: $1" ;; + '') + # Default: check all running runs + check_all_runs + ;; + *) + # Check specific task + check_task_run "$1" + ;; +esac From 522a4bcdff65dca9a665af73344df0c56e45dccf Mon Sep 17 00:00:00 2001 From: Trillium Smith Date: Fri, 31 Jul 2026 09:50:06 -0700 Subject: [PATCH 2/8] Add fm-nm-run-is-live.sh: lightweight run liveness check for supervision --- bin/fm-nm-run-is-live.sh | 149 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100755 bin/fm-nm-run-is-live.sh diff --git a/bin/fm-nm-run-is-live.sh b/bin/fm-nm-run-is-live.sh new file mode 100755 index 0000000000..9596286145 --- /dev/null +++ b/bin/fm-nm-run-is-live.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# fm-nm-run-is-live.sh - check if a specific no-mistakes run is genuinely live (not hung) +# +# Designed for use by fm-crew-state.sh: given a run ID, determine if it's genuinely +# progressing or has silently hung. This is a simpler facade over +# fm-no-mistakes-liveness.sh tuned for single-run checks during supervision. +# +# Usage: +# fm-nm-run-is-live.sh +# +# Exit codes: +# 0 - run is live/progressing (or not a run-step that can hang) +# 1 - run is hung (step inactive for > FM_NM_LIVENESS_STALE seconds) +# 2 - usage error or precondition failed +# +set -u + +# Bounded no-mistakes call; stdout only, never fails the script. +HAVE_TIMEOUT=none +if command -v timeout >/dev/null 2>&1; then HAVE_TIMEOUT=timeout +elif command -v gtimeout >/dev/null 2>&1; then HAVE_TIMEOUT=gtimeout +elif command -v perl >/dev/null 2>&1; then HAVE_TIMEOUT=perl +fi +nm_run() { # + case "$HAVE_TIMEOUT" in + timeout) timeout 15 no-mistakes "$@" 2>/dev/null || true ;; + gtimeout) gtimeout 15 no-mistakes "$@" 2>/dev/null || true ;; + perl) perl -e 'my $t = shift; my $pid = fork; die "fork failed" unless defined $pid; if (!$pid) { setpgrp(0, 0); exec @ARGV } local $SIG{ALRM} = sub { kill "TERM", -$pid; select undef, undef, undef, 0.2; kill "KILL", -$pid; exit 124 }; alarm $t; waitpid $pid, 0; exit($? >> 8)' 15 no-mistakes "$@" 2>/dev/null || true ;; + *) true ;; + esac +} + +trim() { + local s=${1:-} + s="${s#"${s%%[![:space:]]*}"}" + s="${s%"${s##*[![:space:]]}"}" + printf '%s' "$s" +} + +strip_quotes() { + local s + s=$(trim "${1:-}") + case "$s" in + \"*\") s=${s#\"}; s=${s%\"} ;; + esac + trim "$s" +} + +nm_field() { # + local output=$1 key=$2 + printf '%s\n' "$output" | sed -n "s/^[[:space:]]*$key:[[:space:]]*\(.*\)/\1/p" | head -1 +} + +get_timestamp() { + if command -v gdate >/dev/null 2>&1; then + gdate +%s + else + date +%s + fi +} + +parse_log_timestamp() { + local line=$1 + if [[ $line =~ ([0-9]{4})-([0-9]{2})-([0-9]{2})\ ([0-9]{2}):([0-9]{2}) ]]; then + local year="${BASH_REMATCH[1]}" + local month="${BASH_REMATCH[2]}" + local day="${BASH_REMATCH[3]}" + local hour="${BASH_REMATCH[4]}" + local min="${BASH_REMATCH[5]}" + + if command -v gdate >/dev/null 2>&1; then + gdate -d "$year-$month-$day $hour:$min" +%s 2>/dev/null || echo 0 + else + date -j -f "%Y-%m-%d %H:%M" "$year-$month-$day $hour:$min" +%s 2>/dev/null || echo 0 + fi + else + echo 0 + fi +} + +STALE_THRESHOLD=${FM_NM_LIVENESS_STALE:-300} +case "$STALE_THRESHOLD" in ''|*[!0-9]*) STALE_THRESHOLD=300 ;; esac + +# Check if a run is live (not hung) - simplified version for single-run checks +check_run_liveness() { # + local run_id=$1 + local run_out outcome awaiting_msg + + run_out=$(nm_run axi status --run "$run_id") || return 2 + + [ -n "$run_out" ] || return 2 + + outcome=$(strip_quotes "$(nm_field "$run_out" outcome)") + + # Terminal outcomes are not hung + if [ -n "$outcome" ]; then + return 0 + fi + + # Check if awaiting_agent (parked at gate) - not hung, deliberately waiting + awaiting_msg=$(printf '%s\n' "$run_out" | grep -E '^[[:space:]]*awaiting_agent:' | head -1) + if [ -n "$awaiting_msg" ]; then + return 0 + fi + + # For running/fixing steps, check if the step duration is too old + # Get the first running/fixing step + local step_line + step_line=$(printf '%s\n' "$run_out" | grep -E '^[[:space:]]*[^,]+,[[:space:]]*"?(running|fixing)"?[[:space:]]*,' | head -1 | sed 's/^ *//') + + if [ -z "$step_line" ]; then + # No running/fixing step - not hung + return 0 + fi + + # Extract step name and duration_ms + local step_status duration + step_status="${step_line#*,}" + step_status="${step_status%,*}" + step_status=$(strip_quotes "$(trim "$step_status")") + + # Skip findings count (third field) and get duration + local rest="${step_line#*,}" + rest="${rest#*,}" + rest="${rest#*,}" + duration=$(trim "$rest") + + case "$step_status" in + running|fixing) + # Convert milliseconds to seconds and check if stale + if [ -n "$duration" ] && [[ "$duration" =~ ^[0-9]+$ ]]; then + local elapsed + elapsed=$((duration / 1000)) + if [ "$elapsed" -gt "$STALE_THRESHOLD" ]; then + return 1 # Hung + fi + fi + ;; + esac + + return 0 # Live +} + +# --- main + +run_id=${1:-} +[ -n "$run_id" ] || { echo "usage: fm-nm-run-is-live.sh " >&2; exit 2; } + +check_run_liveness "$run_id" From ef460f9e0327a3f3708e43cd5500820ac23a4825 Mon Sep 17 00:00:00 2001 From: Trillium Smith Date: Fri, 31 Jul 2026 09:50:29 -0700 Subject: [PATCH 3/8] Add documentation for no-mistakes run liveness checks --- docs/no-mistakes-run-liveness.md | 135 +++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 docs/no-mistakes-run-liveness.md diff --git a/docs/no-mistakes-run-liveness.md b/docs/no-mistakes-run-liveness.md new file mode 100644 index 0000000000..fe305641ab --- /dev/null +++ b/docs/no-mistakes-run-liveness.md @@ -0,0 +1,135 @@ +# No-mistakes Run Liveness Checks + +These helpers provide reliable, fast ways to answer two related questions about the shared no-mistakes daemon without relying on guesswork or waiting out crewmate timeouts: + +1. **Before daemon-affecting actions** (init, update, daemon restart): Which "running" no-mistakes runs are genuinely live, and which task (if any) owns each one? +2. **During supervision**: Is an in-flight no-mistakes run genuinely progressing, or has it silently hung? + +## Tools + +### `bin/fm-no-mistakes-liveness.sh` - full run inventory + +Comprehensive check of all running no-mistakes runs. Lists each run's ownership, status, and elapsed time in the active step. + +```bash +# Check all running runs and their liveness +fm-no-mistakes-liveness.sh --all + +# Check a specific task's run +fm-no-mistakes-liveness.sh + +# Show help +fm-no-mistakes-liveness.sh --help +``` + +**Output format** (one line per run): +``` +run: · branch: · status: · owned-by: · last-activity: s +``` + +**Exit codes:** +- 0 = success, all checked runs are live +- 1 = at least one run is hung +- 2 = usage error or precondition failure + +### `bin/fm-nm-run-is-live.sh` - lightweight single-run check + +Fast liveness check for a specific run ID. Designed for integration into supervision loops. + +```bash +# Check if a run is live +fm-nm-run-is-live.sh +``` + +**Exit codes:** +- 0 = run is live/progressing (or not in a step that can hang) +- 1 = run is hung (step inactive for > 5 minutes) +- 2 = usage error + +## Liveness Detection Logic + +A run is considered **live** if: +- It has a terminal outcome (passed, failed, cancelled, checks-passed) +- It is parked at a gate awaiting agent input (`awaiting_agent:` marker) +- Its active step (running/fixing) has been active for less than the stale threshold (5 minutes by default) + +A run is considered **hung** if: +- An active step (running/fixing) has not had activity for more than the stale threshold +- The step duration or log timestamps indicate stagnation + +## Integration Points + +### Before Daemon-Affecting Actions + +Before calling `no-mistakes init`, `no-mistakes update`, or restarting the shared daemon: + +```bash +fm-no-mistakes-liveness.sh --all || exit 1 +``` + +Check the output: if any run shows `status: hung` or `status: unknown`, investigate before proceeding. This prevents accidentally killing a genuinely in-flight validation run. + +### During Supervision (fm-watch.sh) + +When supervising a no-mistakes task with `kind=ship` and an active run: + +```bash +run_id=$(grep "^id=" state/.meta | cut -d= -f2) +if ! fm-nm-run-is-live.sh "$run_id"; then + # Run is hung - escalate to firstmate +fi +``` + +This provides an independent liveness signal that catches hung runs faster than waiting for the crewmate's own tool timeout (which can be 15-30 minutes). + +### From fm-crew-state.sh + +When determining current state for a task with an active no-mistakes run, `fm-crew-state.sh` already reads the run's step activity and duration. The liveness helpers provide a reusable check for the same logic. + +## Timeout Tuning + +The default stale threshold is 300 seconds (5 minutes). Adjust via environment variable: + +```bash +FM_NM_LIVENESS_STALE=600 fm-no-mistakes-liveness.sh --all +``` + +The CLI timeout (default 15 seconds) can also be tuned: + +```bash +FM_NM_LIVENESS_TIMEOUT=20 fm-no-mistakes-liveness.sh --all +``` + +## Task Ownership Resolution + +Run ownership is determined by matching the run's branch name to `state/.meta` files in: +- The primary FM_HOME's state directory +- The current git repository's state directory (when called from a project) +- Registered secondmate homes (if secondmates.md exists) + +A task is reported as `owned-by: unknown` when: +- No task's worktree is on the run's branch +- The run's branch is a temporary or abandoned branch +- The task's metadata is missing or corrupt + +## Incident Prevention + +This tool was created to prevent incidents like the 2026-07-31 case where a genuinely live no-mistakes run (`fold-slice4-shims`) was silently killed by a daemon restart because firstmate had no way to distinguish it from an old stale entry in `no-mistakes runs`. + +**Usage protocol before daemon mutations:** +1. Run `fm-no-mistakes-liveness.sh --all` +2. For each run, verify either: + - It is owned by an active task under way (known from the backlog or supervision state) + - OR you are confident it is abandoned and safe to lose +3. Only then proceed with daemon init/update/restart + +**Usage protocol during supervision:** +1. When a no-mistakes task appears to hang (pane shows waiting on validation), call the liveness check +2. If it reports hung, escalate rather than waiting for the crewmate's own timeout +3. Proceed with diagnostic/repair/teardown as needed + +## See Also + +- `bin/fm-crew-state.sh` - current-state reconciliation for a task (includes no-mistakes run details) +- `docs/configuration.md` - runtime backend and metadata schema +- `AGENTS.md` section 8 - supervision protocols From adfb8155153c8a74cce688efe72a7437a3b5527a Mon Sep 17 00:00:00 2001 From: Trillium Smith Date: Fri, 31 Jul 2026 09:52:45 -0700 Subject: [PATCH 4/8] Improve active_steps parsing for accurate liveness detection - Extract active_steps section before parsing to avoid mixing with steps table - Parse last_activity timestamp from active_steps for real-time progress tracking - Report actual elapsed time since last activity instead of cumulative step duration --- bin/fm-no-mistakes-liveness.sh | 105 +++++++++++++++++++-------------- 1 file changed, 62 insertions(+), 43 deletions(-) diff --git a/bin/fm-no-mistakes-liveness.sh b/bin/fm-no-mistakes-liveness.sh index e1474ea967..24df9dfce2 100755 --- a/bin/fm-no-mistakes-liveness.sh +++ b/bin/fm-no-mistakes-liveness.sh @@ -223,17 +223,35 @@ get_active_step() { # return 1 } +# Extract last_activity time from active_steps table (e.g., "2s ago: log: message") +# Returns seconds elapsed, or 0 if cannot parse +parse_active_steps_time() { # + local line=$1 + # Look for patterns like "2s ago:" or "30m ago:" + if [[ "$line" =~ ([0-9]+)(s|m|h)\ ago: ]]; then + local value="${BASH_REMATCH[1]}" + local unit="${BASH_REMATCH[2]}" + case "$unit" in + s) printf '%d' "$value" ;; + m) printf '%d' "$((value * 60))" ;; + h) printf '%d' "$((value * 3600))" ;; + *) printf '0' ;; + esac + else + printf '0' + fi +} + # Check a single run's liveness by reading step activity # Returns elapsed seconds (> 0) and exit code 0 for live, 1 for hung check_run_liveness() { # local run_id=$1 - local run_out status outcome awaiting_msg elapsed + local run_out outcome awaiting_msg elapsed run_out=$(nm_run axi status --run "$run_id") [ -n "$run_out" ] || return 1 # Parse run info - status=$(strip_quotes "$(nm_field "$run_out" status)") outcome=$(strip_quotes "$(nm_field "$run_out" outcome)") # Terminal outcomes are not hung @@ -250,9 +268,40 @@ check_run_liveness() { # return 0 fi - # Get the active step + # Check active_steps table for real-time last_activity + # This section appears after the header 'active_steps[N]{...}:' + local active_steps_section active_step_line last_activity_field + active_steps_section=$(printf '%s\n' "$run_out" | sed -n '/^[[:space:]]*active_steps\[/,/^[^[:space:]]/p') + + if [ -n "$active_steps_section" ]; then + # Get the data rows (skip the header line that ends with ':') + active_step_line=$(printf '%s\n' "$active_steps_section" | grep -E '^[[:space:]]*[^,]+,(running|fixing),' | head -1 | sed 's/^[[:space:]]*//') + + if [ -n "$active_step_line" ]; then + # Found an active running/fixing step in active_steps. Extract last_activity. + # Format: step,status,active_for,"time ago: message","pid",round + # Extract the quoted field after active_for + if [[ "$active_step_line" =~ ^[^,]*,[^,]*,[^,]*,\"([^\"]*) ]]; then + last_activity_field="${BASH_REMATCH[1]}" + if [ -n "$last_activity_field" ]; then + elapsed=$(parse_active_steps_time "$last_activity_field") + if [ "$elapsed" -gt "$STALE_THRESHOLD" ]; then + printf '%d' "$elapsed" + return 1 # Hung + fi + printf '%d' "$elapsed" + return 0 # Live + fi + fi + fi + fi + + # Fallback to steps table if no active_steps local active_step step_info step_status duration - active_step=$(get_active_step "$run_out") || return 0 + active_step=$(get_active_step "$run_out") || { + printf '0' + return 0 + } # Get step info (name|status|duration_ms) step_info=$(get_step_info "$run_out" "$active_step") @@ -266,53 +315,23 @@ check_run_liveness() { # step_status="${step_status%|*}" duration="${step_info##*|}" - # Ensure duration is numeric + # Use step duration only as fallback; it's less accurate than active_steps case "$duration" in ''|*[!0-9]*) - # Check log for timestamps instead - local log_tail last_update - log_tail=$(nm_run axi logs --step "$active_step" --run "$run_id" 2>/dev/null || true) - if [ -z "$log_tail" ]; then - printf '0' - return 0 - fi - - # Get the date from the last non-empty line in the log - last_update=$(printf '%s\n' "$log_tail" | grep -v '^[[:space:]]*$' | tail -1) - if [ -z "$last_update" ]; then - printf '0' - return 0 - fi - - # Try to extract timestamp - if we can't, assume live - local ts now - ts=$(parse_log_timestamp "$last_update") - if [ "$ts" -eq 0 ]; then - printf '0' - return 0 - fi - now=$(get_timestamp) - elapsed=$((now - ts)) + printf '0' + return 0 ;; *) # Convert milliseconds to seconds elapsed=$((duration / 1000)) + # Step duration alone isn't reliable (includes all time at gate) + # Only report hung if duration is extremely large (> 30 min) and still running + if [ "$step_status" = "fixing" ] && [ "$elapsed" -gt 1800 ]; then + printf '%d' "$elapsed" + return 1 # Suspicious + fi ;; esac - # Step is only hung if it's running/fixing and has been inactive for > threshold - case "$step_status" in - running|fixing) - if [ "$elapsed" -gt "$STALE_THRESHOLD" ]; then - printf '%d' "$elapsed" - return 1 # Hung - fi - ;; - pending|completed) - # Not actively running - elapsed=0 - ;; - esac - printf '%d' "$elapsed" return 0 # Live } From d805de6d1de06540967e311701320a6f2dab2466 Mon Sep 17 00:00:00 2001 From: Trillium Smith Date: Fri, 31 Jul 2026 09:58:36 -0700 Subject: [PATCH 5/8] no-mistakes(review): Fixed hardcoded path, unused functions, inconsistent timeout config, and liveness detection logic --- bin/fm-nm-run-is-live.sh | 84 +++++++++++++++++++--------------- bin/fm-no-mistakes-liveness.sh | 4 +- 2 files changed, 51 insertions(+), 37 deletions(-) diff --git a/bin/fm-nm-run-is-live.sh b/bin/fm-nm-run-is-live.sh index 9596286145..4654353f71 100755 --- a/bin/fm-nm-run-is-live.sh +++ b/bin/fm-nm-run-is-live.sh @@ -15,6 +15,13 @@ # set -u +# Timeouts for CLI calls (consistent with fm-no-mistakes-liveness.sh) +NM_TIMEOUT=${FM_NM_LIVENESS_TIMEOUT:-15} +case "$NM_TIMEOUT" in ''|*[!0-9]*) NM_TIMEOUT=15 ;; esac + +STALE_THRESHOLD=${FM_NM_LIVENESS_STALE:-300} +case "$STALE_THRESHOLD" in ''|*[!0-9]*) STALE_THRESHOLD=300 ;; esac + # Bounded no-mistakes call; stdout only, never fails the script. HAVE_TIMEOUT=none if command -v timeout >/dev/null 2>&1; then HAVE_TIMEOUT=timeout @@ -23,9 +30,9 @@ elif command -v perl >/dev/null 2>&1; then HAVE_TIMEOUT=perl fi nm_run() { # case "$HAVE_TIMEOUT" in - timeout) timeout 15 no-mistakes "$@" 2>/dev/null || true ;; - gtimeout) gtimeout 15 no-mistakes "$@" 2>/dev/null || true ;; - perl) perl -e 'my $t = shift; my $pid = fork; die "fork failed" unless defined $pid; if (!$pid) { setpgrp(0, 0); exec @ARGV } local $SIG{ALRM} = sub { kill "TERM", -$pid; select undef, undef, undef, 0.2; kill "KILL", -$pid; exit 124 }; alarm $t; waitpid $pid, 0; exit($? >> 8)' 15 no-mistakes "$@" 2>/dev/null || true ;; + timeout) timeout "$NM_TIMEOUT" no-mistakes "$@" 2>/dev/null || true ;; + gtimeout) gtimeout "$NM_TIMEOUT" no-mistakes "$@" 2>/dev/null || true ;; + perl) perl -e 'my $t = shift; my $pid = fork; die "fork failed" unless defined $pid; if (!$pid) { setpgrp(0, 0); exec @ARGV } local $SIG{ALRM} = sub { kill "TERM", -$pid; select undef, undef, undef, 0.2; kill "KILL", -$pid; exit 124 }; alarm $t; waitpid $pid, 0; exit($? >> 8)' "$NM_TIMEOUT" no-mistakes "$@" 2>/dev/null || true ;; *) true ;; esac } @@ -51,40 +58,26 @@ nm_field() { # printf '%s\n' "$output" | sed -n "s/^[[:space:]]*$key:[[:space:]]*\(.*\)/\1/p" | head -1 } -get_timestamp() { - if command -v gdate >/dev/null 2>&1; then - gdate +%s - else - date +%s - fi -} - -parse_log_timestamp() { +parse_active_steps_time() { # local line=$1 - if [[ $line =~ ([0-9]{4})-([0-9]{2})-([0-9]{2})\ ([0-9]{2}):([0-9]{2}) ]]; then - local year="${BASH_REMATCH[1]}" - local month="${BASH_REMATCH[2]}" - local day="${BASH_REMATCH[3]}" - local hour="${BASH_REMATCH[4]}" - local min="${BASH_REMATCH[5]}" - - if command -v gdate >/dev/null 2>&1; then - gdate -d "$year-$month-$day $hour:$min" +%s 2>/dev/null || echo 0 - else - date -j -f "%Y-%m-%d %H:%M" "$year-$month-$day $hour:$min" +%s 2>/dev/null || echo 0 - fi + if [[ "$line" =~ ([0-9]+)(s|m|h)\ ago: ]]; then + local value="${BASH_REMATCH[1]}" + local unit="${BASH_REMATCH[2]}" + case "$unit" in + s) printf '%d' "$value" ;; + m) printf '%d' "$((value * 60))" ;; + h) printf '%d' "$((value * 3600))" ;; + *) printf '0' ;; + esac else - echo 0 + printf '0' fi } -STALE_THRESHOLD=${FM_NM_LIVENESS_STALE:-300} -case "$STALE_THRESHOLD" in ''|*[!0-9]*) STALE_THRESHOLD=300 ;; esac - # Check if a run is live (not hung) - simplified version for single-run checks check_run_liveness() { # local run_id=$1 - local run_out outcome awaiting_msg + local run_out outcome awaiting_msg elapsed run_out=$(nm_run axi status --run "$run_id") || return 2 @@ -103,9 +96,31 @@ check_run_liveness() { # return 0 fi - # For running/fixing steps, check if the step duration is too old - # Get the first running/fixing step - local step_line + # Check active_steps table first (more accurate than steps table) + local active_steps_section active_step_line last_activity_field + active_steps_section=$(printf '%s\n' "$run_out" | sed -n '/^[[:space:]]*active_steps\[/,/^[^[:space:]]/p') + + if [ -n "$active_steps_section" ]; then + active_step_line=$(printf '%s\n' "$active_steps_section" | grep -E '^[[:space:]]*[^,]+,(running|fixing),' | head -1 | sed 's/^[[:space:]]*//') + + if [ -n "$active_step_line" ]; then + # Found an active running/fixing step. Extract last_activity. + # Format: step,status,active_for,"time ago: message","pid",round + if [[ "$active_step_line" =~ ^[^,]*,[^,]*,[^,]*,\"([^\"]*) ]]; then + last_activity_field="${BASH_REMATCH[1]}" + if [ -n "$last_activity_field" ]; then + elapsed=$(parse_active_steps_time "$last_activity_field") + if [ "$elapsed" -gt "$STALE_THRESHOLD" ]; then + return 1 # Hung + fi + return 0 # Live + fi + fi + fi + fi + + # Fallback: check steps table if no active_steps found + local step_line step_status duration step_line=$(printf '%s\n' "$run_out" | grep -E '^[[:space:]]*[^,]+,[[:space:]]*"?(running|fixing)"?[[:space:]]*,' | head -1 | sed 's/^ *//') if [ -z "$step_line" ]; then @@ -113,8 +128,7 @@ check_run_liveness() { # return 0 fi - # Extract step name and duration_ms - local step_status duration + # Extract step status and duration step_status="${step_line#*,}" step_status="${step_status%,*}" step_status=$(strip_quotes "$(trim "$step_status")") @@ -127,9 +141,7 @@ check_run_liveness() { # case "$step_status" in running|fixing) - # Convert milliseconds to seconds and check if stale if [ -n "$duration" ] && [[ "$duration" =~ ^[0-9]+$ ]]; then - local elapsed elapsed=$((duration / 1000)) if [ "$elapsed" -gt "$STALE_THRESHOLD" ]; then return 1 # Hung diff --git a/bin/fm-no-mistakes-liveness.sh b/bin/fm-no-mistakes-liveness.sh index 24df9dfce2..90cbceec89 100755 --- a/bin/fm-no-mistakes-liveness.sh +++ b/bin/fm-no-mistakes-liveness.sh @@ -367,7 +367,9 @@ check_all_runs() { # Found a task - get its run ID from axi status local task_id="${task##*/}" local meta_file="$STATE/$task_id.meta" - [ -f "$meta_file" ] || meta_file="/Users/trilliumsmith/code/firstmate/state/$task_id.meta" + if [ ! -f "$meta_file" ] && [ -d "$FM_HOME/state" ]; then + meta_file="$FM_HOME/state/$task_id.meta" + fi if [ -f "$meta_file" ]; then local worktree worktree=$(grep "^worktree=" "$meta_file" | tail -1 | cut -d= -f2-) From a8b27942898a4fda63a5c411abcb80128ff8c653 Mon Sep 17 00:00:00 2001 From: Trillium Smith Date: Fri, 31 Jul 2026 10:04:56 -0700 Subject: [PATCH 6/8] no-mistakes(review): Fixed error handling, directory contexts, removed dead code, unified duplicate utilities --- bin/fm-nm-run-is-live.sh | 61 ++----------------------------- bin/fm-no-mistakes-liveness.sh | 66 +++++++++++----------------------- 2 files changed, 23 insertions(+), 104 deletions(-) diff --git a/bin/fm-nm-run-is-live.sh b/bin/fm-nm-run-is-live.sh index 4654353f71..3213d125ae 100755 --- a/bin/fm-nm-run-is-live.sh +++ b/bin/fm-nm-run-is-live.sh @@ -15,64 +15,9 @@ # set -u -# Timeouts for CLI calls (consistent with fm-no-mistakes-liveness.sh) -NM_TIMEOUT=${FM_NM_LIVENESS_TIMEOUT:-15} -case "$NM_TIMEOUT" in ''|*[!0-9]*) NM_TIMEOUT=15 ;; esac - -STALE_THRESHOLD=${FM_NM_LIVENESS_STALE:-300} -case "$STALE_THRESHOLD" in ''|*[!0-9]*) STALE_THRESHOLD=300 ;; esac - -# Bounded no-mistakes call; stdout only, never fails the script. -HAVE_TIMEOUT=none -if command -v timeout >/dev/null 2>&1; then HAVE_TIMEOUT=timeout -elif command -v gtimeout >/dev/null 2>&1; then HAVE_TIMEOUT=gtimeout -elif command -v perl >/dev/null 2>&1; then HAVE_TIMEOUT=perl -fi -nm_run() { # - case "$HAVE_TIMEOUT" in - timeout) timeout "$NM_TIMEOUT" no-mistakes "$@" 2>/dev/null || true ;; - gtimeout) gtimeout "$NM_TIMEOUT" no-mistakes "$@" 2>/dev/null || true ;; - perl) perl -e 'my $t = shift; my $pid = fork; die "fork failed" unless defined $pid; if (!$pid) { setpgrp(0, 0); exec @ARGV } local $SIG{ALRM} = sub { kill "TERM", -$pid; select undef, undef, undef, 0.2; kill "KILL", -$pid; exit 124 }; alarm $t; waitpid $pid, 0; exit($? >> 8)' "$NM_TIMEOUT" no-mistakes "$@" 2>/dev/null || true ;; - *) true ;; - esac -} - -trim() { - local s=${1:-} - s="${s#"${s%%[![:space:]]*}"}" - s="${s%"${s##*[![:space:]]}"}" - printf '%s' "$s" -} - -strip_quotes() { - local s - s=$(trim "${1:-}") - case "$s" in - \"*\") s=${s#\"}; s=${s%\"} ;; - esac - trim "$s" -} - -nm_field() { # - local output=$1 key=$2 - printf '%s\n' "$output" | sed -n "s/^[[:space:]]*$key:[[:space:]]*\(.*\)/\1/p" | head -1 -} - -parse_active_steps_time() { # - local line=$1 - if [[ "$line" =~ ([0-9]+)(s|m|h)\ ago: ]]; then - local value="${BASH_REMATCH[1]}" - local unit="${BASH_REMATCH[2]}" - case "$unit" in - s) printf '%d' "$value" ;; - m) printf '%d' "$((value * 60))" ;; - h) printf '%d' "$((value * 3600))" ;; - *) printf '0' ;; - esac - else - printf '0' - fi -} +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=bin/fm-no-mistakes-liveness.sh +. "$SCRIPT_DIR/fm-no-mistakes-liveness.sh" # Check if a run is live (not hung) - simplified version for single-run checks check_run_liveness() { # diff --git a/bin/fm-no-mistakes-liveness.sh b/bin/fm-no-mistakes-liveness.sh index 90cbceec89..60556c6f53 100755 --- a/bin/fm-no-mistakes-liveness.sh +++ b/bin/fm-no-mistakes-liveness.sh @@ -90,37 +90,6 @@ nm_field() { # printf '%s\n' "$output" | sed -n "s/^[[:space:]]*$key:[[:space:]]*\(.*\)/\1/p" | head -1 } -# Get current time in seconds since epoch -get_timestamp() { - if command -v gdate >/dev/null 2>&1; then - gdate +%s - else - date +%s - fi -} - -# Parse timestamp from log line (common formats: "2026-07-31 09:22:15", "2026-07-31 09:22") -parse_log_timestamp() { - local line=$1 - # Try to extract ISO-like timestamp from log line - if [[ $line =~ ([0-9]{4})-([0-9]{2})-([0-9]{2})\ ([0-9]{2}):([0-9]{2}) ]]; then - local year="${BASH_REMATCH[1]}" - local month="${BASH_REMATCH[2]}" - local day="${BASH_REMATCH[3]}" - local hour="${BASH_REMATCH[4]}" - local min="${BASH_REMATCH[5]}" - - # Convert to epoch seconds using date - if command -v gdate >/dev/null 2>&1; then - gdate -d "$year-$month-$day $hour:$min" +%s 2>/dev/null || echo 0 - else - date -j -f "%Y-%m-%d %H:%M" "$year-$month-$day $hour:$min" +%s 2>/dev/null || echo 0 - fi - else - echo 0 - fi -} - # Check if a run is owned by a task by matching branch names # Also checks secondmate homes and project homes find_task_for_branch() { # @@ -249,7 +218,10 @@ check_run_liveness() { # local run_out outcome awaiting_msg elapsed run_out=$(nm_run axi status --run "$run_id") - [ -n "$run_out" ] || return 1 + if [ -z "$run_out" ]; then + printf '0' + return 1 + fi # Parse run info outcome=$(strip_quotes "$(nm_field "$run_out" outcome)") @@ -446,7 +418,7 @@ check_task_run() { # # Found a run for this task's branch # Get full run details via axi status local axi_out run_id - axi_out=$(nm_run axi status 2>/dev/null || true) + axi_out=$(cd "$worktree" && nm_run axi status 2>/dev/null || true) run_id=$(strip_quotes "$(nm_field "$axi_out" id)") [ -n "$run_id" ] || die "Could not get run ID" @@ -470,16 +442,18 @@ check_task_run() { # # --- main -------------------------------------------------------------------- -case "${1:-}" in - --help|-h) usage ;; - --all) check_all_runs ;; - --*) die "unknown option: $1" ;; - '') - # Default: check all running runs - check_all_runs - ;; - *) - # Check specific task - check_task_run "$1" - ;; -esac +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + case "${1:-}" in + --help|-h) usage ;; + --all) check_all_runs ;; + --*) die "unknown option: $1" ;; + '') + # Default: check all running runs + check_all_runs + ;; + *) + # Check specific task + check_task_run "$1" + ;; + esac +fi From 6835fd3d1f7ca2fafd58e6a1f96851108ff73685 Mon Sep 17 00:00:00 2001 From: Trillium Smith Date: Fri, 31 Jul 2026 10:16:48 -0700 Subject: [PATCH 7/8] no-mistakes(document): Add liveness checking scripts to documentation index and environment variables --- docs/configuration.md | 2 ++ docs/scripts.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 1b8ea3bb59..dd46208cd7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -402,6 +402,8 @@ FM_CODEX_WATCH_CHECKPOINT=180 # seconds per foreground watcher checkpoint in C FM_CREW_STATE_NM_TIMEOUT=10 # seconds allowed per no-mistakes query inside fm-crew-state.sh FM_CREW_STATE_RUNS_LIMIT=200 # recent no-mistakes run rows scanned when axi status cannot be attributed to the current code FM_CREW_STATE_BIN=bin/fm-crew-state.sh # test override for the current-state reader used by working/paused watcher triage +FM_NM_LIVENESS_TIMEOUT=15 # seconds allowed per no-mistakes query inside fm-no-mistakes-liveness.sh and fm-nm-run-is-live.sh +FM_NM_LIVENESS_STALE=300 # seconds; a run is considered hung if its active step has not had activity for longer than this threshold FMX_PAIRING_TOKEN= # X mode pairing token; .env opt-in authorizes replies and eligible lifecycle actions FMX_RELAY_URL=https://myfirstmate.io # optional X relay override, mainly for local relay development FMX_ENV_FILE= # optional alternate .env file for direct X client invocations; bootstrap still checks $FM_HOME/.env diff --git a/docs/scripts.md b/docs/scripts.md index 6a10d1310a..c28d967a4c 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -64,6 +64,8 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-supervisor-target-lib.sh` | Resolve the shared supervisor target and backend for the daemon and launcher | | `fm-supervise-daemon.sh` | Presence-gated away-mode sub-supervisor: self-handle routine wakes, escalate batched digests, alert on failed delivery | | `fm-crew-state.sh` | Print one deterministic current-state line for a crew | +| `fm-no-mistakes-liveness.sh` | Check liveness of no-mistakes runs and match them to tasks; list all running runs or check a specific task | +| `fm-nm-run-is-live.sh` | Fast liveness check for a specific no-mistakes run ID; designed for supervision loops | | `fm-tangle-lib.sh` | Shared default-branch resolution and primary-checkout tangle classification | | `fm-supervision-lib.sh` | Shared in-flight-work-without-fresh-watcher-beacon predicate | | `fm-ff-lib.sh` | Shared guarded fast-forward helper for origin pulls and local secondmate syncs | From 5c8283bb7b640ae201a8b0d12b264aca2c20ad48 Mon Sep 17 00:00:00 2001 From: Trillium Smith Date: Fri, 31 Jul 2026 10:49:01 -0700 Subject: [PATCH 8/8] no-mistakes: apply CI fixes --- docs/documentation-audiences.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/documentation-audiences.json b/docs/documentation-audiences.json index 54b2190f6c..deda9af1de 100644 --- a/docs/documentation-audiences.json +++ b/docs/documentation-audiences.json @@ -323,6 +323,10 @@ "path": "docs/zellij-backend.md", "audience": "operator-current" }, + { + "path": "docs/no-mistakes-run-liveness.md", + "audience": "operator-current" + }, { "path": "skills/stow/SKILL.md", "audience": "public-product"