From 7e074fc1180d815812b3c7d6cf644ed22404bdcf Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Thu, 16 Jul 2026 22:07:04 +0200 Subject: [PATCH 1/2] fix(boot): bind API port and answer health probes before slow state open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A boot running the one-time VACUUM (25 min measured on a 2.3 GB DB on SD) or a full integrity check left :8080 unbound the whole time; the Docker healthcheck failed and ftw-updater judged the v0.130.3 deploy failed and auto-rolled back mid-VACUUM. The port now binds before state.Open with a boot-phase handler (200 on /api/health with status=starting, 503 elsewhere) and the fully wired mux is swapped onto the same listener when ready — no port gap. Co-Authored-By: Claude Fable 5 --- .changeset/boot-health-shim.md | 13 ++++++++++ go/cmd/ftw/boothealth.go | 46 ++++++++++++++++++++++++++++++++++ go/cmd/ftw/main.go | 36 ++++++++++++++++++-------- 3 files changed, 84 insertions(+), 11 deletions(-) create mode 100644 .changeset/boot-health-shim.md create mode 100644 go/cmd/ftw/boothealth.go diff --git a/.changeset/boot-health-shim.md b/.changeset/boot-health-shim.md new file mode 100644 index 00000000..9169936b --- /dev/null +++ b/.changeset/boot-health-shim.md @@ -0,0 +1,13 @@ +--- +"ftw": patch +--- + +Answer health probes during slow boots. + +The API port is now bound before the state DB opens, serving +`/api/health` with 200 `{"status":"starting"}` (and 503 elsewhere) until +the real mux takes over on the same listener. Previously a boot that ran +a one-time VACUUM or a full integrity check on a multi-GB database left +the port unbound for up to tens of minutes, so the Docker healthcheck +failed and the self-update sidecar judged the deploy failed and rolled +back in the middle of the compaction. diff --git a/go/cmd/ftw/boothealth.go b/go/cmd/ftw/boothealth.go new file mode 100644 index 00000000..9bac364d --- /dev/null +++ b/go/cmd/ftw/boothealth.go @@ -0,0 +1,46 @@ +package main + +import ( + "net/http" + "sync/atomic" +) + +// swappableHandler lets the API port be bound before slow boot work (state +// integrity check, one-time VACUUM) and atomically hand the same listener +// over to the real mux once wiring completes. See main.go for why: an +// unbound port during a 25-minute compaction makes the Docker healthcheck +// fail and the self-update sidecar roll the deploy back mid-VACUUM. +type swappableHandler struct { + h atomic.Value // http.Handler +} + +func newSwappableHandler(initial http.Handler) *swappableHandler { + s := &swappableHandler{} + s.h.Store(initial) + return s +} + +func (s *swappableHandler) Swap(h http.Handler) { s.h.Store(h) } + +func (s *swappableHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.h.Load().(http.Handler).ServeHTTP(w, r) +} + +// bootPhaseHandler answers health probes 200 while the process initializes, +// so a legitimately slow boot is distinguishable from a dead one. Everything +// else gets 503 + Retry-After so clients and the UI know to come back. +func bootPhaseHandler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/api/health", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"status":"starting","phase":"initializing state"}`)) + }) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Retry-After", "10") + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"error":"starting","phase":"initializing state"}`)) + }) + return mux +} diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 11aa9b7d..dda39999 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -155,6 +155,27 @@ func main() { if abs, err := filepath.Abs(statePath); err == nil { statePath = abs } + + // Bind the API port BEFORE the potentially slow state open. A boot that + // runs a one-time VACUUM or a full integrity check on a multi-GB DB can + // take 25+ minutes, and an unbound port for that long makes the Docker + // healthcheck fail and the self-update sidecar judge the deploy failed — + // observed 2026-07-16 as an auto-rollback in the middle of a VACUUM. + // Until the real mux is wired, /api/health answers 200 "starting" and + // everything else 503. + apiHandler := newSwappableHandler(bootPhaseHandler()) + httpSrv := &http.Server{ + Addr: fmt.Sprintf(":%d", cfg.API.Port), + Handler: apiHandler, + ReadHeaderTimeout: 10 * time.Second, + } + go func() { + slog.Info("HTTP API listening (boot phase)", "addr", httpSrv.Addr) + if err := httpSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + slog.Error("http server", "err", err) + } + }() + st, err := state.Open(statePath) if err != nil { slog.Error("open state", "err", err) @@ -1876,17 +1897,10 @@ func main() { "upstream", u.String(), "read_only", readOnly) } - httpSrv := &http.Server{ - Addr: fmt.Sprintf(":%d", cfg.API.Port), - Handler: handler, - ReadHeaderTimeout: 10 * time.Second, - } - go func() { - slog.Info("HTTP API listening", "addr", httpSrv.Addr) - if err := httpSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed { - slog.Error("http server", "err", err) - } - }() + // Swap the boot-phase handler for the fully wired mux — the listener + // bound at startup stays; no port gap for healthcheck probes. + apiHandler.Swap(handler) + slog.Info("HTTP API ready", "addr", httpSrv.Addr) // Belt-and-suspenders integrity scan, off the startup hot path: a clean // restart skips the blocking boot check (so a multi-GB DB starts in seconds), From 13000591e347208ec77f99b0b45e5c9f177d9a32 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Thu, 16 Jul 2026 22:11:30 +0200 Subject: [PATCH 2/2] fix(boot): atomic.Pointer for the handler swap (Value panics across types) Co-Authored-By: Claude Fable 5 --- go/cmd/ftw/boothealth.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/go/cmd/ftw/boothealth.go b/go/cmd/ftw/boothealth.go index 9bac364d..946c60d3 100644 --- a/go/cmd/ftw/boothealth.go +++ b/go/cmd/ftw/boothealth.go @@ -11,19 +11,21 @@ import ( // unbound port during a 25-minute compaction makes the Docker healthcheck // fail and the self-update sidecar roll the deploy back mid-VACUUM. type swappableHandler struct { - h atomic.Value // http.Handler + // atomic.Pointer, not atomic.Value: the boot handler and the wired mux + // are different concrete types, and Value panics on that. + h atomic.Pointer[http.Handler] } func newSwappableHandler(initial http.Handler) *swappableHandler { s := &swappableHandler{} - s.h.Store(initial) + s.h.Store(&initial) return s } -func (s *swappableHandler) Swap(h http.Handler) { s.h.Store(h) } +func (s *swappableHandler) Swap(h http.Handler) { s.h.Store(&h) } func (s *swappableHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - s.h.Load().(http.Handler).ServeHTTP(w, r) + (*s.h.Load()).ServeHTTP(w, r) } // bootPhaseHandler answers health probes 200 while the process initializes,