feat: auto-resume polish + spawn-gate follow-ups — persistent crash trace, circuit breaker, cancel, Retry-After (katas znhn, bccd)

crates/freshell-codex/src/launch_plan.rs

@@ -52,6 +52,10 @@ pub const CODEX_REMOTE_NON_LOOPBACK_MESSAGE: &str =
 /// launches. Council fence: S4's wiring is FLAG-GATED, default OFF — legacy's proxy path
 /// exists to feed durability binding (S5), so the launch mechanism ships dark until S5's
 /// consumers land; S5 + the flag-default flip land together.
+///
+/// D-C-REVISIT(FRESHELL_CODEX_MANAGED_LAUNCH): flipping this default ON must
+/// revisit the ~226s REST permit-hold (grep for D-C-REVISIT; decision in
+/// docs/plans/2026-07-27-rest-spawn-gate.md §D-C).
 pub const FRESHELL_CODEX_MANAGED_LAUNCH_ENV: &str = "FRESHELL_CODEX_MANAGED_LAUNCH";
 
 /// Whether the managed-launch flag value enables the S4 wiring. Only the exact string

crates/freshell-freshagent/src/lib.rs

@@ -1587,6 +1587,20 @@ async fn send_keys(
         // COLD-START + create the durable session. `create_session` runs `ensure_started`
         // (spawn serve → bounded health wait — the DEV-0001 fix, NO warm-proxy) then
         // `POST /session`. Success here IS the cold-start-clean fingerprint.
+        //
+        // bccd item 4 (council enn3 D-D evaluate-and-decide) — DELIBERATELY
+        // UNGATED. Decision reversed at plan validation: gating this
+        // cold-start would hold a spawn permit for a worst-case ~50-70s
+        // (health_timeout 20_000ms serve.rs:303 + request_timeout 30_000ms
+        // serve.rs:308,546 under the permit; the serialized `running`-mutex
+        // queue adds ~20s per failing holder) vs the 10s gate waits at
+        // every other door — and k cold first-sends queued on the singleton
+        // mutex would hold k permits, starving ALL spawn doors. The
+        // double-mutex single-flight already bounds actual sidecar forks
+        // to AT MOST ONE server-wide: the gate would add starvation
+        // without reducing fork concurrency. Moving the acquire inside
+        // the single-flight would invert lock order (cycle hazard).
+        // Decision record: docs/plans/2026-07-29-znhn-bccd-followups.md §D-7.
         let created = match manager
             .create_session(None, None, pane.cwd.as_deref())
             .await

crates/freshell-freshagent/src/opencode_ws.rs

@@ -557,6 +557,12 @@ impl FreshOpencodeState {
             // Already materialized: THE continuity fix — reuse it, no new session.
             real_id
         } else {
+            // Deliberately ungated (bccd item 4, council D-D evaluate-and-decide;
+            // same decision as the REST send-keys cold arm in lib.rs): the
+            // single-flighted singleton manager bounds sidecar forks to AT MOST
+            // ONE server-wide, and gating would starve the spawn budget on
+            // ~50-70s worst-case cold-start holds (see the lib.rs comment for
+            // the arithmetic). Revisit if the sidecar ever grows fork fan-out.
             let created = match manager.create_session(None, None, cwd.as_deref()).await {
                 Ok(created) => created,
                 Err(err) => {

crates/freshell-freshagent/src/spawn_gate.rs

@@ -186,6 +186,22 @@ impl SpawnGate {
         }
     }
 
+    /// Acquire a spawn permit with no caller-side cancellation (kata znhn
+    /// item 4). Two doors have no connection whose death should cancel the
+    /// wait — the REST door and the auto-resume respawn door. They used to
+    /// mint never-fired watch channels at every call site; that wart belongs
+    /// to the gate. The never-fired sender now lives HERE, held across the
+    /// acquire, so `Cancelled` is unreachable by construction (kata bccd
+    /// item 3: no caller-side sender exists to drop). The timeout still
+    /// bounds the wait.
+    pub async fn acquire_uncancellable(
+        &self,
+        timeout: Duration,
+    ) -> Result<OwnedSemaphorePermit, SpawnGateError> {
+        let (_cancel_tx, mut cancel_rx) = tokio::sync::watch::channel(false);
+        self.acquire(timeout, &mut cancel_rx).await
+    }
+
     pub fn queued_total(&self) -> u64 {
         self.queued_total.load(Ordering::Relaxed)
     }
@@ -213,6 +229,61 @@ mod tests {
         watch::channel(false)
     }
 
+    #[tokio::test]
+    async fn acquire_uncancellable_waits_for_a_permit_and_never_cancels() {
+        let gate = Arc::new(SpawnGate::new(1, 64));
+        let (_tx, mut rx) = cancel_pair();
+        let held = gate
+            .acquire(Duration::from_secs(1), &mut rx)
+            .await
+            .expect("holder");
+        let g2 = Arc::clone(&gate);
+        let waiter =
+            tokio::spawn(async move { g2.acquire_uncancellable(Duration::from_secs(5)).await });
+        // Deterministic queue barrier (established idiom in this module).
+        for _ in 0..200 {
+            if gate.queued_total() == 1 {
+                break;
+            }
+            tokio::time::sleep(Duration::from_millis(5)).await;
+        }
+        assert_eq!(gate.queued_total(), 1, "uncancellable waiter must queue");
+        drop(held);
+        assert!(
+            waiter.await.unwrap().is_ok(),
+            "waiter acquires after release"
+        );
+        assert_eq!(gate.cancellations(), 0, "Cancelled must be unreachable");
+    }
+
+    #[tokio::test]
+    async fn acquire_uncancellable_times_out_as_timeout_not_cancelled() {
+        let gate = SpawnGate::new(1, 64);
+        let (_tx, mut rx) = cancel_pair();
+        let _held = gate
+            .acquire(Duration::from_secs(1), &mut rx)
+            .await
+            .expect("holder");
+        let err = gate
+            .acquire_uncancellable(Duration::from_millis(50))
+            .await
+            .unwrap_err();
+        assert_eq!(err, SpawnGateError::Timeout);
+        assert_eq!(gate.timeouts(), 1);
+        assert_eq!(gate.cancellations(), 0);
+    }
+
+    #[tokio::test]
+    async fn acquire_uncancellable_rejects_queue_full_loudly() {
+        let gate = SpawnGate::new(0, 0);
+        let err = gate
+            .acquire_uncancellable(Duration::from_millis(50))
+            .await
+            .unwrap_err();
+        assert_eq!(err, SpawnGateError::QueueFull);
+        assert_eq!(gate.queue_rejections(), 1);
+    }
+
     #[tokio::test]
     async fn bounds_concurrency_to_n_and_all_complete() {
         // Spawn N+K creates, assert max in-flight == N, all complete.

crates/freshell-freshagent/src/terminal_tabs.rs

@@ -34,7 +34,8 @@ use std::collections::HashSet;
 
 use axum::extract::{Path, Query, State};
 use axum::http::{HeaderMap, StatusCode};
-use axum::response::Response;
+use axum::response::{IntoResponse, Response};
+use axum::Json;
 use serde_json::{json, Value};
 use uuid::Uuid;
 
@@ -580,28 +581,42 @@ fn codex_launch_error_response(
 
 /// REST mapping of a spawn-gate rejection (WS analogue:
 /// `spawn_gate_error_parts` in freshell-ws/src/terminal.rs).
-/// QueueFull -> 429: the caller should back off and retry.
-/// Timeout   -> 503: spawn capacity unavailable right now.
-/// The retry guidance lives in the MESSAGE because the MCP bridge
-/// (server/mcp/freshell-tool.ts) surfaces only the message text, not the
-/// HTTP status. Body key is `code`+`message` (never `error`) so the MCP
-/// http-client's `data.error || data.message` precedence keeps showing the
-/// human message.
-fn spawn_gate_error_response(err: crate::spawn_gate::SpawnGateError) -> Response {
+/// QueueFull -> 429 with Retry-After (bccd item 1): header for HTTP
+/// convention + `retryAfterMs` body field (house convention, session-lease
+/// SESSION_RESERVED). The retry guidance ALSO stays in the MESSAGE because
+/// the MCP bridge (server/mcp/freshell-tool.ts) surfaces only message text.
+/// Timeout -> 503: spawn capacity unavailable right now.
+/// Body key is `code`+`message` (never `error`).
+fn spawn_gate_error_response(
+    err: crate::spawn_gate::SpawnGateError,
+    retry_after: std::time::Duration,
+) -> Response {
     match err {
-        crate::spawn_gate::SpawnGateError::QueueFull => crate::fail_json_code(
-            StatusCode::TOO_MANY_REQUESTS,
-            "SPAWN_QUEUE_FULL",
-            "Too many concurrent terminal spawns; retry shortly".to_string(),
-        ),
+        crate::spawn_gate::SpawnGateError::QueueFull => {
+            let secs = retry_after.as_secs().max(1);
+            (
+                StatusCode::TOO_MANY_REQUESTS,
+                [(
+                    axum::http::header::RETRY_AFTER,
+                    axum::http::HeaderValue::from(secs),
+                )],
+                Json(json!({
+                    "status": "error",
+                    "code": "SPAWN_QUEUE_FULL",
+                    "message": "Too many concurrent terminal spawns; retry shortly",
+                    "retryAfterMs": retry_after.as_millis() as u64,
+                })),
+            )
+                .into_response()
+        }
         crate::spawn_gate::SpawnGateError::Timeout => crate::fail_json_code(
             StatusCode::SERVICE_UNAVAILABLE,
             "SPAWN_TIMEOUT",
             "Timed out waiting for a terminal spawn slot".to_string(),
         ),
-        // Unreachable on the REST door: the handler holds its cancel
-        // sender (never fired, never dropped) across the whole acquire.
-        // Mapped like Timeout so an impossible arm still fails safe.
+        // Unreachable since acquire_uncancellable (znhn item 4): no cancel
+        // sender exists on this door at all. Mapped like Timeout so an
+        // impossible arm still fails safe.
         crate::spawn_gate::SpawnGateError::Cancelled => crate::fail_json_code(
             StatusCode::SERVICE_UNAVAILABLE,
             "SPAWN_TIMEOUT",
@@ -1049,17 +1064,14 @@ pub(crate) async fn spawn_terminal_pane(
     // server wiring keep legacy behavior.
     let spawn_permit = match state.spawn_gate() {
         Some(rest_gate) => {
-            // The gate's acquire is cancellable via a watch channel (the WS
-            // door wires its per-connection cancel signal). REST has no such
-            // signal: hold a live, never-fired sender for the whole acquire
-            // (dropping it early would read as "connection gone" =>
-            // Cancelled). If the HTTP request itself is dropped while
-            // QUEUED, axum drops this future and the gate's queue-slot
-            // guard reclaims the slot — nothing has been spawned yet.
-            let (_cancel_tx, mut cancel_rx) = tokio::sync::watch::channel(false);
+            // Uncancellable acquire (kata znhn item 4): REST has no
+            // connection whose death should cancel the wait — the gate owns
+            // that semantics now. If the HTTP request is dropped while
+            // QUEUED, axum drops this future and the gate's queue-slot guard
+            // reclaims the slot — nothing has been spawned yet.
             match rest_gate
                 .gate
-                .acquire(rest_gate.timeout, &mut cancel_rx)
+                .acquire_uncancellable(rest_gate.timeout)
                 .await
             {
                 Ok(permit) => Some(permit),
@@ -1074,7 +1086,7 @@ pub(crate) async fn spawn_terminal_pane(
                         let _ =
                             freshell_sessions::amplifier_stub::gc_stub_if_unused(&stub.session_dir);
                     }
-                    return Err(spawn_gate_error_response(err));
+                    return Err(spawn_gate_error_response(err, rest_gate.timeout));
                 }
             }
         }
@@ -1268,6 +1280,14 @@ async fn settle_gated_create(inputs: GatedSettleInputs) -> Result<TerminalSpawnR
             .and_then(Value::as_str)
             .map(str::to_string);
 
+        // D-C-REVISIT(FRESHELL_CODEX_MANAGED_LAUNCH): this plan runs UNDER
+        // the held spawn permit — plan_create_with_retry can hold it ~226s
+        // worst case (5 × SIDECAR_START_BUDGET 45s + 1s backoff) vs the 10s
+        // permit wait. Accepted while the flag defaults OFF. If the default
+        // ever flips ON, the permit-hold duration MUST be revisited (likely
+        // a separate sidecar budget covering both doors). Decision record:
+        // docs/plans/2026-07-27-rest-spawn-gate.md §D-C.
+        //
         // DEV-0006 S4 inc.2 (FLAG-GATED, default OFF — council fence): with
         // `FRESHELL_CODEX_MANAGED_LAUNCH=1`, plan the managed app-server launch through
         // the SAME `CodexTerminalLaunchManager` the WS path uses (`router.ts:160-195`
@@ -3920,14 +3940,34 @@ mod tests {
             Arc::new(crate::spawn_gate::SpawnGate::new(0, 0)),
             std::time::Duration::from_secs(5),
         );
-        let (status, body) = post(app(state), "/api/tabs", shell_create_body(), true).await;
+        // Raw oneshot (not the `post` helper): this test also asserts the
+        // Retry-After HEADER, which the (status, body) helper discards.
+        let req = Request::builder()
+            .method("POST")
+            .uri("/api/tabs")
+            .header("content-type", "application/json")
+            .header("x-auth-token", "tok")
+            .body(Body::from(shell_create_body().to_string()))
+            .unwrap();
+        let response = app(state).oneshot(req).await.unwrap();
+        let status = response.status();
+        assert_eq!(
+            response
+                .headers()
+                .get(axum::http::header::RETRY_AFTER)
+                .map(|v| v.to_str().unwrap().to_string()),
+            Some("5".to_string()),
+            "429 must carry a machine-readable Retry-After (bccd item 1)"
+        );
+        let body = body_json(response).await;
         assert_eq!(status, StatusCode::TOO_MANY_REQUESTS, "{body}");
         assert_eq!(body["status"], json!("error"));
         assert_eq!(body["code"], json!("SPAWN_QUEUE_FULL"));
         assert_eq!(
             body["message"],
             json!("Too many concurrent terminal spawns; retry shortly")
         );
+        assert_eq!(body["retryAfterMs"], 5_000);
     }
 
     #[tokio::test]
@@ -3972,38 +4012,58 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread")]
     async fn fifteen_plus_rest_create_burst_is_bounded_and_all_complete() {
-        // Concurrency-1 gate: at most ONE request may hold the spawn permit
-        // at a time, so a 16-burst must serialize through the gate — the
-        // queued_total counter proves the burst actually queued (bounded
-        // in-flight) instead of spawning in parallel, and every request
-        // still completes (FIFO drain, nothing dropped).
+        // Deterministic pin (kata bccd item 2, council enn3): pre-holding the
+        // single permit forces EVERY burst request through the queue —
+        // queued_total() reaches exactly 16 (the fast path cannot fire while
+        // the budget is held), and ZERO requests may complete while the
+        // budget is exhausted. That pins max-in-flight <= budget without the
+        // probabilistic `queued_total >= 8` lower bound (the fast path skips
+        // the counter). Mirrors the re-acquire precedent at
+        // `abort_burst_rest_creates_stay_gated...`.
         let state = state_with_registry();
         let registry = state.terminal_registry.clone().unwrap();
         let gate = Arc::new(crate::spawn_gate::SpawnGate::new(1, 64));
         state.set_spawn_gate(Arc::clone(&gate), std::time::Duration::from_secs(30));
         let router = app(state);
 
+        let held = gate
+            .acquire_uncancellable(std::time::Duration::from_secs(1))
+            .await
+            .expect("test pre-hold of the single permit");
+
         let mut handles = Vec::new();
         for _ in 0..16 {
             let r = router.clone();
             handles.push(tokio::spawn(async move {
                 post(r, "/api/tabs", shell_create_body(), true).await
             }));
         }
+
+        // Every request must queue behind the held permit — exact, not
+        // probabilistic.
+        for _ in 0..600 {
+            if gate.queued_total() == 16 {
+                break;
+            }
+            tokio::time::sleep(std::time::Duration::from_millis(5)).await;
+        }
+        assert_eq!(
+            gate.queued_total(),
+            16,
+            "all 16 burst requests must queue while the permit is held"
+        );
+        assert!(
+            handles.iter().all(|h| !h.is_finished()),
+            "no request may complete while the budget is fully held (max-in-flight <= budget)"
+        );
+
+        drop(held);
         let mut terminal_ids = Vec::new();
         for h in handles {
             let (status, body) = h.await.expect("request task");
             assert_eq!(status, StatusCode::OK, "{body}");
             terminal_ids.push(body["data"]["terminalId"].as_str().unwrap().to_string());
         }
-        // With 16 near-simultaneous arrivals and 1 permit, the overwhelming
-        // majority must have queued. (The fast path skips the counter when
-        // the queue is momentarily empty, hence >= 8, not == 15.)
-        assert!(
-            gate.queued_total() >= 8,
-            "burst did not queue through the gate: queued_total={}",
-            gate.queued_total()
-        );
         assert_eq!(gate.queue_rejections(), 0, "no loud rejections expected");
         assert_eq!(gate.timeouts(), 0, "no permit-wait timeouts expected");
 

crates/freshell-protocol/src/client_messages.rs

@@ -28,6 +28,8 @@ pub enum ClientMessage {
     TerminalCodexCandidatePersisted(TerminalCodexCandidatePersisted),
     #[serde(rename = "terminal.attach")]
     TerminalAttach(TerminalAttach),
+    #[serde(rename = "terminal.autoResumeCancel")]
+    TerminalAutoResumeCancel(TerminalAutoResumeCancel),
     #[serde(rename = "terminal.detach")]
     TerminalDetach(TerminalDetach),
     #[serde(rename = "terminal.input")]
@@ -81,7 +83,7 @@ pub enum ClientMessage {
 
 /// The exact `type` discriminants of every client→server message, in the frozen
 /// inventory's order. This is the T0 conformance checklist.
-pub const CLIENT_MESSAGE_TYPES: [&str; 29] = [
+pub const CLIENT_MESSAGE_TYPES: [&str; 30] = [
     "amplifier.activity.list",
     "claude.activity.list",
     "client.diagnostic",
@@ -103,6 +105,7 @@ pub const CLIENT_MESSAGE_TYPES: [&str; 29] = [
     "pane.reconcile.request",
     "ping",
     "terminal.attach",
+    "terminal.autoResumeCancel",
     "terminal.codex.candidate.persisted",
     "terminal.create",
     "terminal.detach",
@@ -245,6 +248,15 @@ pub struct TerminalCodexCandidatePersisted {
     pub terminal_id: String,
 }
 
+/// znhn item 2: the user opts out of an in-flight auto-resume ("stop
+/// trying, leave it dead"). Carries the OLD (crashed) terminal id — the
+/// same id the recovering `terminal.status` frame was broadcast with.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct TerminalAutoResumeCancel {
+    pub terminal_id: String,
+}
+
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct TerminalAttach {