fix: make /ready database check reflect real RPC pool health

src/__tests__/health-dependency-checks.test.ts

@@ -3,7 +3,7 @@ import express from "express";
 
 jest.mock("../lib/stellar", () => ({
   rpcPool: {
-    getMetrics: jest.fn(() => ({ active: 1, idle: 2, total: 3, waitingQueue: 0 })),
+    getMetrics: jest.fn(() => ({ active: 1, idle: 2, total: 3, healthy: 3, waitingQueue: 0 })),
     shutdown: jest.fn(),
   },
   rpcBreaker: {
@@ -38,10 +38,24 @@ import * as satellite from "../lib/satellite-sources";
 describe("health endpoint dependency checks (#277)", () => {
   beforeEach(() => {
     jest.clearAllMocks();
-    (stellar.rpcPool.getMetrics as jest.Mock).mockReturnValue({ active: 1, idle: 2, total: 3, waitingQueue: 0 });
-    (stellar.rpcBreaker.getMetrics as jest.Mock).mockReturnValue({ state: "CLOSED", failures: 0, successes: 10 });
+    (stellar.rpcPool.getMetrics as jest.Mock).mockReturnValue({
+      active: 1,
+      idle: 2,
+      total: 3,
+      healthy: 3,
+      waitingQueue: 0,
+    });
+    (stellar.rpcBreaker.getMetrics as jest.Mock).mockReturnValue({
+      state: "CLOSED",
+      failures: 0,
+      successes: 10,
+    });
     (stellar.rpcBreaker.getState as jest.Mock).mockReturnValue("CLOSED");
-    (stellar.getRpcStatus as jest.Mock).mockReturnValue({ consecutiveFailures: 0, outageDurationMs: 0, lastSuccessAgoMs: 100 });
+    (stellar.getRpcStatus as jest.Mock).mockReturnValue({
+      consecutiveFailures: 0,
+      outageDurationMs: 0,
+      lastSuccessAgoMs: 100,
+    });
     (satellite.getOutageState as jest.Mock).mockReturnValue({ consecutiveFailures: 0 });
   });
 
@@ -94,7 +108,11 @@ describe("health endpoint dependency checks (#277)", () => {
     });
 
     it("reflects OPEN circuit breaker state", async () => {
-      (stellar.rpcBreaker.getMetrics as jest.Mock).mockReturnValue({ state: "OPEN", failures: 10, successes: 0 });
+      (stellar.rpcBreaker.getMetrics as jest.Mock).mockReturnValue({
+        state: "OPEN",
+        failures: 10,
+        successes: 0,
+      });
 
       const health = await getHealth();
       expect(health.circuit_breaker).toMatchObject({ state: "OPEN" });
@@ -109,6 +127,19 @@ describe("health endpoint dependency checks (#277)", () => {
       expect(readiness.checks.rpc_circuit).toBe(false);
     });
 
+    it("returns not_ready when the RPC connection pool has no healthy connections", () => {
+      (stellar.rpcPool.getMetrics as jest.Mock).mockReturnValue({
+        active: 0,
+        idle: 0,
+        total: 2,
+        healthy: 0,
+        waitingQueue: 0,
+      });
+      const readiness = getReadiness();
+      expect(readiness.status).toBe("not_ready");
+      expect(readiness.checks.database).toBe(false);
+    });
+
     it("returns not_ready when satellite has consecutive failures", () => {
       (satellite.getOutageState as jest.Mock).mockReturnValue({ consecutiveFailures: 5 });
       const readiness = getReadiness();
@@ -120,7 +151,11 @@ describe("health endpoint dependency checks (#277)", () => {
       (stellar.rpcBreaker.getState as jest.Mock).mockReturnValue("CLOSED");
       (satellite.getOutageState as jest.Mock).mockReturnValue({ consecutiveFailures: 0 });
       const readiness = getReadiness();
-      expect(readiness.checks).toMatchObject({ database: true, satellite: true, rpc_circuit: true });
+      expect(readiness.checks).toMatchObject({
+        database: true,
+        satellite: true,
+        rpc_circuit: true,
+      });
     });
   });
 

src/lib/db-pool.ts

@@ -14,6 +14,7 @@ export interface PoolMetrics {
   total: number;
   idle: number;
   active: number;
+  healthy: number;
   pendingAcquires: number;
   healthCheckErrors: number;
   totalAcquired: number;
@@ -45,6 +46,7 @@ export class RpcConnectionPool {
     total: 0,
     idle: 0,
     active: 0,
+    healthy: 0,
     pendingAcquires: 0,
     healthCheckErrors: 0,
     totalAcquired: 0,
@@ -80,7 +82,7 @@ export class RpcConnectionPool {
       return Promise.reject(new Error("Pool is shutting down"));
     }
 
-    const idle = this.connections.find(c => !c.inUse && c.healthy);
+    const idle = this.connections.find((c) => !c.inUse && c.healthy);
     if (idle) {
       return Promise.resolve(this.checkout(idle));
     }
@@ -92,7 +94,7 @@ export class RpcConnectionPool {
     return new Promise<PooledConnection>((resolve, reject) => {
       this.metrics.pendingAcquires++;
       const timer = setTimeout(() => {
-        const idx = this.waitQueue.findIndex(e => e.timer === timer);
+        const idx = this.waitQueue.findIndex((e) => e.timer === timer);
         if (idx !== -1) this.waitQueue.splice(idx, 1);
         this.metrics.pendingAcquires--;
         reject(new Error(`Pool acquire timed out after ${this.config.acquireTimeoutMs}ms`));
@@ -139,24 +141,25 @@ export class RpcConnectionPool {
     return {
       ...this.metrics,
       total: this.connections.length,
-      active: this.connections.filter(c => c.inUse).length,
-      idle: this.connections.filter(c => !c.inUse && c.healthy).length,
+      active: this.connections.filter((c) => c.inUse).length,
+      idle: this.connections.filter((c) => !c.inUse && c.healthy).length,
+      healthy: this.connections.filter((c) => c.healthy).length,
     };
   }
 
   private scheduleHealthChecks(): void {
     this.healthTimer = setInterval(async () => {
-      const idle = this.connections.filter(c => !c.inUse);
+      const idle = this.connections.filter((c) => !c.inUse);
       await Promise.allSettled(
-        idle.map(async conn => {
+        idle.map(async (conn) => {
           try {
             await conn.client.getLatestLedger();
             conn.healthy = true;
           } catch {
             conn.healthy = false;
             this.metrics.healthCheckErrors++;
           }
-        })
+        }),
       );
     }, this.config.healthCheckIntervalMs);
     // Don't keep the process alive just for health checks
@@ -180,13 +183,14 @@ export class RpcConnectionPool {
 
     // Drain active connections (max 10 s)
     const deadline = Date.now() + 10_000;
-    while (this.connections.some(c => c.inUse) && Date.now() < deadline) {
-      await new Promise(r => setTimeout(r, 50));
+    while (this.connections.some((c) => c.inUse) && Date.now() < deadline) {
+      await new Promise((r) => setTimeout(r, 50));
     }
 
     this.connections.length = 0;
     this.metrics.total = 0;
     this.metrics.active = 0;
     this.metrics.idle = 0;
+    this.metrics.healthy = 0;
   }
 }

src/lib/health.ts

@@ -66,7 +66,12 @@ export interface ReadinessReport {
 
 export function getReadiness(): ReadinessReport {
   const dbMetrics = rpcPool.getMetrics();
-  const dbReady = dbMetrics.active >= 0;
+  // The pool is ready only if it currently holds at least one connection that
+  // passed its last health check. `active`/`idle`/`total` are all non-negative
+  // counts that stay populated even when every connection is failing, so they
+  // cannot express an unhealthy pool; `healthy` drops to 0 when the RPC
+  // endpoint is unreachable and the periodic health check marks connections bad.
+  const dbReady = dbMetrics.healthy > 0;
   const outage = getOutageState();
   const satelliteReady = outage.consecutiveFailures < 3;
   const rpcReady = rpcBreaker.getState() !== "OPEN";