diff --git a/.changeset/boot-health-shim.md b/.changeset/boot-health-shim.md new file mode 100644 index 00000000..9169936b --- /dev/null +++ b/.changeset/boot-health-shim.md @@ -0,0 +1,13 @@ +--- +"ftw": patch +--- + +Answer health probes during slow boots. + +The API port is now bound before the state DB opens, serving +`/api/health` with 200 `{"status":"starting"}` (and 503 elsewhere) until +the real mux takes over on the same listener. Previously a boot that ran +a one-time VACUUM or a full integrity check on a multi-GB database left +the port unbound for up to tens of minutes, so the Docker healthcheck +failed and the self-update sidecar judged the deploy failed and rolled +back in the middle of the compaction. diff --git a/go/cmd/ftw/boothealth.go b/go/cmd/ftw/boothealth.go new file mode 100644 index 00000000..946c60d3 --- /dev/null +++ b/go/cmd/ftw/boothealth.go @@ -0,0 +1,48 @@ +package main + +import ( + "net/http" + "sync/atomic" +) + +// swappableHandler lets the API port be bound before slow boot work (state +// integrity check, one-time VACUUM) and atomically hand the same listener +// over to the real mux once wiring completes. See main.go for why: an +// unbound port during a 25-minute compaction makes the Docker healthcheck +// fail and the self-update sidecar roll the deploy back mid-VACUUM. +type swappableHandler struct { + // atomic.Pointer, not atomic.Value: the boot handler and the wired mux + // are different concrete types, and Value panics on that. + h atomic.Pointer[http.Handler] +} + +func newSwappableHandler(initial http.Handler) *swappableHandler { + s := &swappableHandler{} + s.h.Store(&initial) + return s +} + +func (s *swappableHandler) Swap(h http.Handler) { s.h.Store(&h) } + +func (s *swappableHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + (*s.h.Load()).ServeHTTP(w, r) +} + +// bootPhaseHandler answers health probes 200 while the process initializes, +// so a legitimately slow boot is distinguishable from a dead one. Everything +// else gets 503 + Retry-After so clients and the UI know to come back. +func bootPhaseHandler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/api/health", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"status":"starting","phase":"initializing state"}`)) + }) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Retry-After", "10") + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"error":"starting","phase":"initializing state"}`)) + }) + return mux +} diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 11aa9b7d..dda39999 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -155,6 +155,27 @@ func main() { if abs, err := filepath.Abs(statePath); err == nil { statePath = abs } + + // Bind the API port BEFORE the potentially slow state open. A boot that + // runs a one-time VACUUM or a full integrity check on a multi-GB DB can + // take 25+ minutes, and an unbound port for that long makes the Docker + // healthcheck fail and the self-update sidecar judge the deploy failed — + // observed 2026-07-16 as an auto-rollback in the middle of a VACUUM. + // Until the real mux is wired, /api/health answers 200 "starting" and + // everything else 503. + apiHandler := newSwappableHandler(bootPhaseHandler()) + httpSrv := &http.Server{ + Addr: fmt.Sprintf(":%d", cfg.API.Port), + Handler: apiHandler, + ReadHeaderTimeout: 10 * time.Second, + } + go func() { + slog.Info("HTTP API listening (boot phase)", "addr", httpSrv.Addr) + if err := httpSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + slog.Error("http server", "err", err) + } + }() + st, err := state.Open(statePath) if err != nil { slog.Error("open state", "err", err) @@ -1876,17 +1897,10 @@ func main() { "upstream", u.String(), "read_only", readOnly) } - httpSrv := &http.Server{ - Addr: fmt.Sprintf(":%d", cfg.API.Port), - Handler: handler, - ReadHeaderTimeout: 10 * time.Second, - } - go func() { - slog.Info("HTTP API listening", "addr", httpSrv.Addr) - if err := httpSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed { - slog.Error("http server", "err", err) - } - }() + // Swap the boot-phase handler for the fully wired mux — the listener + // bound at startup stays; no port gap for healthcheck probes. + apiHandler.Swap(handler) + slog.Info("HTTP API ready", "addr", httpSrv.Addr) // Belt-and-suspenders integrity scan, off the startup hot path: a clean // restart skips the blocking boot check (so a multi-GB DB starts in seconds),