fix(boot): answer health probes during slow boots (VACUUM / integrity check)

.changeset/boot-health-shim.md

@@ -0,0 +1,13 @@
+---
+"ftw": patch
+---
+
+Answer health probes during slow boots.
+
+The API port is now bound before the state DB opens, serving
+`/api/health` with 200 `{"status":"starting"}` (and 503 elsewhere) until
+the real mux takes over on the same listener. Previously a boot that ran
+a one-time VACUUM or a full integrity check on a multi-GB database left
+the port unbound for up to tens of minutes, so the Docker healthcheck
+failed and the self-update sidecar judged the deploy failed and rolled
+back in the middle of the compaction.

go/cmd/ftw/boothealth.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"net/http"
+	"sync/atomic"
+)
+
+// swappableHandler lets the API port be bound before slow boot work (state
+// integrity check, one-time VACUUM) and atomically hand the same listener
+// over to the real mux once wiring completes. See main.go for why: an
+// unbound port during a 25-minute compaction makes the Docker healthcheck
+// fail and the self-update sidecar roll the deploy back mid-VACUUM.
+type swappableHandler struct {
+	// atomic.Pointer, not atomic.Value: the boot handler and the wired mux
+	// are different concrete types, and Value panics on that.
+	h atomic.Pointer[http.Handler]
+}
+
+func newSwappableHandler(initial http.Handler) *swappableHandler {
+	s := &swappableHandler{}
+	s.h.Store(&initial)
+	return s
+}
+
+func (s *swappableHandler) Swap(h http.Handler) { s.h.Store(&h) }
+
+func (s *swappableHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	(*s.h.Load()).ServeHTTP(w, r)
+}
+
+// bootPhaseHandler answers health probes 200 while the process initializes,
+// so a legitimately slow boot is distinguishable from a dead one. Everything
+// else gets 503 + Retry-After so clients and the UI know to come back.
+func bootPhaseHandler() http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/api/health", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte(`{"status":"starting","phase":"initializing state"}`))
+	})
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		w.Header().Set("Retry-After", "10")
+		w.WriteHeader(http.StatusServiceUnavailable)
+		_, _ = w.Write([]byte(`{"error":"starting","phase":"initializing state"}`))
+	})
+	return mux
+}

go/cmd/ftw/main.go

@@ -155,6 +155,27 @@ func main() {
 	if abs, err := filepath.Abs(statePath); err == nil {
 		statePath = abs
 	}
+
+	// Bind the API port BEFORE the potentially slow state open. A boot that
+	// runs a one-time VACUUM or a full integrity check on a multi-GB DB can
+	// take 25+ minutes, and an unbound port for that long makes the Docker
+	// healthcheck fail and the self-update sidecar judge the deploy failed —
+	// observed 2026-07-16 as an auto-rollback in the middle of a VACUUM.
+	// Until the real mux is wired, /api/health answers 200 "starting" and
+	// everything else 503.
+	apiHandler := newSwappableHandler(bootPhaseHandler())
+	httpSrv := &http.Server{
+		Addr:              fmt.Sprintf(":%d", cfg.API.Port),
+		Handler:           apiHandler,
+		ReadHeaderTimeout: 10 * time.Second,
+	}
+	go func() {
+		slog.Info("HTTP API listening (boot phase)", "addr", httpSrv.Addr)
+		if err := httpSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			slog.Error("http server", "err", err)
+		}
+	}()
+
 	st, err := state.Open(statePath)
 	if err != nil {
 		slog.Error("open state", "err", err)
@@ -1876,17 +1897,10 @@ func main() {
 			"upstream", u.String(),
 			"read_only", readOnly)
 	}
-	httpSrv := &http.Server{
-		Addr:              fmt.Sprintf(":%d", cfg.API.Port),
-		Handler:           handler,
-		ReadHeaderTimeout: 10 * time.Second,
-	}
-	go func() {
-		slog.Info("HTTP API listening", "addr", httpSrv.Addr)
-		if err := httpSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
-			slog.Error("http server", "err", err)
-		}
-	}()
+	// Swap the boot-phase handler for the fully wired mux — the listener
+	// bound at startup stays; no port gap for healthcheck probes.
+	apiHandler.Swap(handler)
+	slog.Info("HTTP API ready", "addr", httpSrv.Addr)
 
 	// Belt-and-suspenders integrity scan, off the startup hot path: a clean
 	// restart skips the blocking boot check (so a multi-GB DB starts in seconds),