From 363982ec9b39d73bc51b7bb0f1fda016a1e69e30 Mon Sep 17 00:00:00 2001
From: Brad Harris <brad@studio.localdomain>
Date: Wed, 11 Mar 2026 08:07:39 -0600
Subject: [PATCH] Add periodic agent status reconciliation loop

Adds a 30-second interval that detects agents stuck in running/stopping/creating
status when their tmux session no longer exists, corrects them to stopped, and
publishes SSE events so the UI updates immediately.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 src/agents/manager.ts | 26 +++++++++++++++++++++++---
 src/server.ts         | 34 ++++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+), 3 deletions(-)

diff --git a/src/agents/manager.ts b/src/agents/manager.ts
index 82b2d1cc..71a600da 100644
--- a/src/agents/manager.ts
+++ b/src/agents/manager.ts
@@ -292,14 +292,34 @@ export class AgentManager {
   }
 
   async reconcileAgents(): Promise<void> {
+    await this.reconcileAgentStatuses();
+  }
+
+  async reconcileAgentStatuses(): Promise<AgentRecord[]> {
     const result = await this.pool.query(
-      "SELECT id, tmux_session AS \"tmuxSession\" FROM agents WHERE status IN ('running', 'creating', 'unknown')"
+      "SELECT id, tmux_session AS \"tmuxSession\", status FROM agents WHERE status IN ('running', 'stopping', 'creating')"
     );
 
-    for (const row of result.rows as Array<{ id: string; tmuxSession: string | null }>) {
+    const reconciled: AgentRecord[] = [];
+
+    for (const row of result.rows as Array<{ id: string; tmuxSession: string | null; status: string }>) {
       const exists = row.tmuxSession ? await this.tmuxHasSession(row.tmuxSession) : false;
-      await this.setAgentStatus(row.id, exists ? "running" : "stopped", null, row.tmuxSession ?? undefined);
+
+      if (!exists) {
+        await this.setAgentStatus(row.id, "stopped", null, row.tmuxSession ?? undefined);
+        await this.setSystemLatestEvent(row.id, {
+          type: "idle",
+          message: "Session ended unexpectedly.",
+          metadata: { source: "system" }
+        });
+        const agent = await this.getAgent(row.id);
+        if (agent) {
+          reconciled.push(agent);
+        }
+      }
     }
+
+    return reconciled;
   }
 
   private async startTmuxSession(
diff --git a/src/server.ts b/src/server.ts
index 1c393a44..4ed182e1 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -151,6 +151,9 @@ const gitRefreshCounters = {
 };
 let gitContextRefreshTimer: NodeJS.Timeout | null = null;
 
+const AGENT_STATUS_RECONCILE_INTERVAL_MS = 30_000;
+let agentStatusReconcileTimer: NodeJS.Timeout | null = null;
+
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 const webDistDir = path.resolve(__dirname, "../web/dist");
@@ -1234,6 +1237,7 @@ async function start() {
   const agents = await agentManager.listAgents();
   queueGitContextRefresh(agents.map((agent) => agent.id));
   startGitContextRefreshLoop();
+  startAgentStatusReconcileLoop();
   await registerRoutes();
 
   const protocol = config.tls ? "https" : "http";
@@ -1396,6 +1400,35 @@ function stopGitContextRefreshLoop(): void {
   gitContextRefreshTimer = null;
 }
 
+function startAgentStatusReconcileLoop(): void {
+  if (agentStatusReconcileTimer) {
+    return;
+  }
+  agentStatusReconcileTimer = setInterval(() => {
+    void runAgentStatusReconciliation();
+  }, AGENT_STATUS_RECONCILE_INTERVAL_MS);
+}
+
+function stopAgentStatusReconcileLoop(): void {
+  if (!agentStatusReconcileTimer) {
+    return;
+  }
+  clearInterval(agentStatusReconcileTimer);
+  agentStatusReconcileTimer = null;
+}
+
+async function runAgentStatusReconciliation(): Promise<void> {
+  try {
+    const reconciled = await agentManager.reconcileAgentStatuses();
+    for (const agent of reconciled) {
+      console.log(`[reconcile] Agent ${agent.id} (${agent.name}) status corrected to stopped`);
+      uiEventBroker.publish({ type: "agent.upsert", agent: withStreamFlag(agent) });
+    }
+  } catch (error) {
+    app.log.warn({ err: error }, "Agent status reconciliation failed.");
+  }
+}
+
 async function refreshAllAgentGitContexts(): Promise<void> {
   try {
     const agents = await agentManager.listAgents();
@@ -1791,6 +1824,7 @@ async function shutdown(code: number): Promise<void> {
 
   streamManager.stopAll();
   stopGitContextRefreshLoop();
+  stopAgentStatusReconcileLoop();
   await pool.end().catch(() => null);
   await app.close().catch(() => null);
   process.exit(code);