From 45cc49a08bde9a4e95ea47c611b017447d76f034 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 21 Jun 2026 17:44:55 +0000 Subject: [PATCH 1/8] =?UTF-8?q?feat(http):=20M11=20=E2=80=94=20generic=20H?= =?UTF-8?q?TTP=20adapter=20for=20the=20module=20runtime?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Makes the module runtime (ADR-0019) runnable over REST, not just in tests: - moduleController: runCommand (POST /modules/:module/:command), query (GET .../query/:name, with opt-in ?consistency=strong via readBarrier), and getState. Reuses the book controller's NotLeaderError -> forward-or-421 leader forwarding verbatim; default reads are local. ApplyResult { status, data, message } is relayed honestly (401/404/413/500 pass through). - Signing over HTTP: the server never holds a private key. The client signs the logical payload and sends x-signature; the server relays it onto command.sig, building the command from x-actor/x-request-id (the same values signed over), so a genuine client verifies on apply and a forged signature is rejected 401. - moduleApp/moduleServer: a module analog of createApp/server.ts, reusing the app-agnostic /raft, /audit, /health, /ready, /metrics routes; demo modules counter/notes/accounts. - ModuleNode type added to moduleStateMachine.ts (mirrors BookNode). Additive: no src/consensus or moduleHost changes; the prior 160 tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01NneoGGKxBp5ZExL7D7i1Lu --- src/controllers/moduleController.ts | 125 ++++++++++++++++++++ src/moduleApp.ts | 81 +++++++++++++ src/moduleServer.ts | 64 +++++++++++ src/routes/moduleRoutes.ts | 26 +++++ src/runtime/moduleStateMachine.ts | 10 ++ tests/moduleApi.test.ts | 172 ++++++++++++++++++++++++++++ 6 files changed, 478 insertions(+) create mode 100644 src/controllers/moduleController.ts create mode 100644 src/moduleApp.ts create mode 100644 src/moduleServer.ts create mode 100644 src/routes/moduleRoutes.ts create mode 100644 tests/moduleApi.test.ts diff --git a/src/controllers/moduleController.ts b/src/controllers/moduleController.ts new file mode 100644 index 0000000..2b3363a --- /dev/null +++ b/src/controllers/moduleController.ts @@ -0,0 +1,125 @@ +import { Request, Response } from 'express'; +import { NotLeaderError } from '../consensus/raftNode'; +import { CommandMeta } from '../consensus/types'; +import { getContext } from '../platform/requestContext'; +import { forwardToLeader, isForwarded } from '../platform/forward'; +import { buildModuleCommand } from '../runtime/command'; +import { ModuleNode } from '../runtime/moduleStateMachine'; + +/** A linearizable read is requested via `?consistency=strong` or `X-Consistency: strong`. */ +function wantsStrongRead(req: Request): boolean { + return req.query.consistency === 'strong' || req.header('x-consistency') === 'strong'; +} + +/** + * Generic HTTP adapter between REST and a node whose application is the module + * runtime ({@link ModuleNode}), the runtime analog of `bookController.ts`. Writes + * (module commands) are proposed to the Raft log via the leader; a follower + * transparently forwards them (falling back to 421 if the leader is + * unknown/unreachable). Reads (queries / raw state) are served from the local + * replica by default (eventually consistent); a client can opt into a + * **linearizable** query with `?consistency=strong`, which goes through the + * leader's ReadIndex barrier before serving. + * + * SIGNING OVER HTTP (ADR-0019 pillar 7): the server NEVER holds an actor's + * private key. The client signs the LOGICAL command + * (`{ module, command, input, actor, requestId }`) with its own key and sends the + * base64 signature in the `x-signature` header. This adapter only RELAYS that + * signature onto the command (`sig`); the leader resolves the seed (which is + * deliberately outside the signed payload) and every replica verifies the + * signature on the deterministic apply path. The adapter does no signing itself. + */ +export function createModuleController(node: ModuleNode) { + // Forward to the leader (or reply 421) when this node isn't the leader. + const onNotLeader = async (req: Request, res: Response, err: NotLeaderError): Promise => { + const leaderUrl = node.getLeaderUrl(); + if (leaderUrl && !isForwarded(req)) { + const ok = await forwardToLeader(req, res, leaderUrl); + if (ok) return; + } + res.status(421).json({ message: 'Not the leader — retry against the leader', leader: err.leaderId }); + }; + + // Run `serve` only after the leader's linearizable read barrier resolves. + const strongRead = async (req: Request, res: Response, serve: () => void): Promise => { + try { + await node.readBarrier(); + serve(); + } catch (err) { + if (err instanceof NotLeaderError) return onNotLeader(req, res, err); + console.error(err); + res.status(503).json({ message: 'Read barrier failed', error: (err as Error).message }); + } + }; + + return { + /** + * `POST /modules/:module/:command` — propose a module command. The body is + * the command `input`. The deterministic seed is resolved on the leader by + * `buildModuleCommand`; `actor`/`requestId` come from the request context. + * An `x-signature` header (a client-produced signature over the logical + * command) is relayed onto the command's `sig` for apply-path verification. + */ + runCommand: async (req: Request, res: Response): Promise => { + const ctx = getContext(); + if (!ctx) { + res.status(500).json({ message: 'Missing request context' }); + return; + } + const command = buildModuleCommand(req.params.module, req.params.command, req.body, { + actor: ctx.actor, + requestId: ctx.requestId, + }); + // Relay (never produce) the client's signature. The private key stays + // with the client; the server only carries the signature to the apply + // path, where every replica verifies it against the actor's public key. + const signature = req.header('x-signature'); + if (signature) command.sig = signature; + + const meta: CommandMeta = { + requestId: ctx.requestId, + actor: ctx.actor, + timestamp: new Date().toISOString(), + }; + try { + const result = await node.submit(command, meta); + res.status(result.status).json(result.data ?? { message: result.message }); + } catch (err) { + if (err instanceof NotLeaderError) return onNotLeader(req, res, err); + console.error(err); + res.status(500).json({ message: 'Server error' }); + } + }, + + /** + * `GET /modules/:module/query/:name` — run a read-only query against the + * local replica. The whole `req.query` object is passed as the query args. + * `?consistency=strong` (or `X-Consistency: strong`) routes through the + * leader's ReadIndex barrier first, forwarding to the leader on a follower. + */ + query: async (req: Request, res: Response): Promise => { + const serve = () => { + try { + const value = node.app.host.query(req.params.module, req.params.name, req.query); + res.json(value); + } catch (err) { + // Unknown module/query is a client error, not a server crash. + res.status(404).json({ message: (err as Error).message }); + } + }; + if (wantsStrongRead(req)) return strongRead(req, res, serve); + serve(); + }, + + /** + * `GET /modules/:module/state` — the raw current state of a module from the + * local replica (eventually consistent). A convenience read for inspection. + */ + getState: (req: Request, res: Response): void => { + const state = node.app.host.getState(req.params.module); + res.json(state ?? null); + }, + }; +} + +export type ModuleController = ReturnType; diff --git a/src/moduleApp.ts b/src/moduleApp.ts new file mode 100644 index 0000000..4ec045f --- /dev/null +++ b/src/moduleApp.ts @@ -0,0 +1,81 @@ +import express, { Application, NextFunction, Request, Response } from 'express'; +import cors from 'cors'; +import { ModuleNode } from './runtime/moduleStateMachine'; +import { Logger } from './platform/logger'; +import { MetricsRegistry } from './platform/metrics'; +import { requestContextMiddleware, getContext } from './platform/requestContext'; +import moduleRoutes from './routes/moduleRoutes'; +import raftRoutes from './routes/raftRoutes'; +import auditRoutes from './routes/auditRoutes'; + +export interface ModuleAppDeps { + logger?: Logger; + metrics?: MetricsRegistry; +} + +/** + * Build an Express app bound to a node whose application is the module runtime + * ({@link ModuleNode}) — the runtime analog of `createApp` in `app.ts`. It wires + * the SAME platform middleware (request context, structured access logs, HTTP + * metrics) and mounts the generic module surface under `/modules`. The + * `/raft`, `/audit`, `/health`, `/ready`, `/metrics` routes are application- + * agnostic (generic over any `RaftNode`), so they are reused verbatim. + */ +export function createModuleApp(node: ModuleNode, deps: ModuleAppDeps = {}): Application { + const app: Application = express(); + const { logger, metrics } = deps; + + app.use(express.json()); + app.use(cors()); + app.use(requestContextMiddleware); + + // Access logging + HTTP metrics, recorded once the response is sent. + app.use((req: Request, res: Response, next: NextFunction) => { + const start = Date.now(); + res.on('finish', () => { + const durationMs = Date.now() - start; + // Use the matched route template (e.g. /modules/:module/:command), not the + // raw path, so per-module requests don't explode metric label cardinality. + const route = req.route ? `${req.baseUrl}${req.route.path}` : req.baseUrl || 'unmatched'; + metrics?.httpRequests.inc({ method: req.method, route, status: res.statusCode }); + metrics?.httpDuration.observe(durationMs, { method: req.method, route }); + logger?.info('http_request', { + method: req.method, + path: req.originalUrl, + status: res.statusCode, + durationMs, + }); + }); + next(); + }); + + app.use('/modules', moduleRoutes(node)); + app.use('/raft', raftRoutes(node)); + app.use('/audit', auditRoutes(node)); + + // Liveness: process is up. + app.get('/health', (_req, res) => res.json({ status: 'ok', node: node.status() })); + + // Readiness: the cluster has a leader this node recognises (safe to route writes). + app.get('/ready', (_req, res) => { + const ready = node.getLeaderId() !== null; + res.status(ready ? 200 : 503).json({ ready, leader: node.getLeaderId() }); + }); + + // Prometheus scrape endpoint. + app.get('/metrics', (_req, res) => { + if (!metrics) { + res.status(404).json({ message: 'metrics disabled' }); + return; + } + res.type('text/plain').send(metrics.expose()); + }); + + // Centralized error handler — structured, with the request id for tracing. + app.use((err: Error, _req: Request, res: Response, _next: NextFunction) => { + logger?.error('unhandled_error', { error: err.message, requestId: getContext()?.requestId }); + res.status(500).json({ message: 'Server error' }); + }); + + return app; +} diff --git a/src/moduleServer.ts b/src/moduleServer.ts new file mode 100644 index 0000000..2cc534e --- /dev/null +++ b/src/moduleServer.ts @@ -0,0 +1,64 @@ +import dotenv from 'dotenv'; +import { createModuleApp } from './moduleApp'; +import { RaftNode } from './consensus/raftNode'; +import { HttpTransport } from './consensus/transport'; +import { FileStorage } from './consensus/storage'; +import { getPort, loadRaftConfig } from './consensus/config'; +import { createLogger } from './platform/logger'; +import { metrics } from './platform/metrics'; +import { ModuleStateMachine, ModuleNode } from './runtime/moduleStateMachine'; +import { AnyModuleDefinition } from './runtime/moduleHost'; +import { counter } from './runtime/modules/counter'; +import { notes } from './runtime/modules/notes'; +import { accounts } from './runtime/modules/accounts'; + +dotenv.config(); + +const logger = createLogger(); +const config = loadRaftConfig(); +const storage = new FileStorage(config.id, process.env.DATA_DIR || './data'); + +// Plug the module runtime into the consensus core (the analog of `server.ts` +// wiring `BookStateMachine`). Register a demo module set; because modules are +// registered identically on every node BEFORE start, MODULE commands apply +// deterministically and the cluster converges. +const stateMachine = new ModuleStateMachine(); +// A `Reducer` is INVARIANT in its state type `S` (it both consumes and +// produces `S`), so a strongly-typed `ModuleDefinition` is not +// assignable to the host's erased `ModuleDefinition` slot — even though +// the host treats state as opaque. Erase the state type at this registration +// boundary; the host only ever round-trips it through the reducers, so this is +// sound. (`accounts` is keyed and needs no erasure.) +const demoModules: AnyModuleDefinition[] = [ + counter as AnyModuleDefinition, + notes as AnyModuleDefinition, + accounts, +]; +stateMachine.host.registerModules(demoModules); + +const node: ModuleNode = new RaftNode( + { ...config, stateMachine, logger, metrics, storage }, + new HttpTransport(), +); + +// Refresh Raft gauges whenever /metrics is scraped. +metrics.registerCollector(() => node.collectMetrics()); + +const app = createModuleApp(node, { logger, metrics }); +const PORT = getPort(); + +const server = app.listen(PORT, () => { + logger.info('module node started', { node: config.id, port: PORT, peers: config.peers.length }); + node.start(); +}); + +const shutdown = () => { + logger.info('shutting down', { node: config.id }); + node.stop(); + server.close(() => process.exit(0)); +}; +process.on('SIGINT', shutdown); +process.on('SIGTERM', shutdown); + +export { app, node }; +export default app; diff --git a/src/routes/moduleRoutes.ts b/src/routes/moduleRoutes.ts new file mode 100644 index 0000000..d30e289 --- /dev/null +++ b/src/routes/moduleRoutes.ts @@ -0,0 +1,26 @@ +import express from 'express'; +import { ModuleNode } from '../runtime/moduleStateMachine'; +import { createModuleController } from '../controllers/moduleController'; + +/** + * Generic HTTP routes for a node running the module runtime (ADR-0019), the + * runtime analog of `bookRoutes`. Mounted under `/modules`. + * + * Route ordering is deliberate: the static read subpaths (`.../query/:name`, + * `.../state`) are registered BEFORE the catch-all write path (`.../:command`) so + * a literal `query`/`state` segment can never be mis-bound as a command name. + * (They are also GET vs POST, but ordering keeps the intent unambiguous.) + */ +export default function moduleRoutes(node: ModuleNode) { + const router = express.Router(); + const c = createModuleController(node); + + // @route GET /modules/:module/query/:name @desc Run a read-only query (local; ?consistency=strong for linearizable) + router.get('/:module/query/:name', c.query); + // @route GET /modules/:module/state @desc Raw current state of a module (local read) + router.get('/:module/state', c.getState); + // @route POST /modules/:module/:command @desc Propose a module command (body = input) + router.post('/:module/:command', c.runCommand); + + return router; +} diff --git a/src/runtime/moduleStateMachine.ts b/src/runtime/moduleStateMachine.ts index 4c10335..42a2636 100644 --- a/src/runtime/moduleStateMachine.ts +++ b/src/runtime/moduleStateMachine.ts @@ -1,3 +1,4 @@ +import { RaftNode } from '../consensus/raftNode'; import { StateMachine } from '../consensus/stateMachine'; import { ApplyResult } from '../consensus/types'; import { ModuleHost } from './moduleHost'; @@ -57,3 +58,12 @@ export class ModuleStateMachine implements StateMachine); } } + +/** + * A Raft node whose application is the {@link ModuleStateMachine} runtime — the + * concrete node type the generic module HTTP adapter wires (the runtime analog of + * `BookNode` in `models/bookStateMachine.ts`). `node.app` is the + * `ModuleStateMachine`, so `node.app.host` is the live {@link ModuleHost} the + * controller reads queries/state from. + */ +export type ModuleNode = RaftNode; diff --git a/tests/moduleApi.test.ts b/tests/moduleApi.test.ts new file mode 100644 index 0000000..099642a --- /dev/null +++ b/tests/moduleApi.test.ts @@ -0,0 +1,172 @@ +import request from 'supertest'; +import { Application } from 'express'; +import { createModuleApp } from '../src/moduleApp'; +import { RaftNode } from '../src/consensus/raftNode'; +import { LocalTransport, RpcHandler } from '../src/consensus/transport'; +import { ModuleStateMachine, ModuleNode } from '../src/runtime/moduleStateMachine'; +import { counter } from '../src/runtime/modules/counter'; +import { notes } from '../src/runtime/modules/notes'; +import { accounts } from '../src/runtime/modules/accounts'; +import { generateActorKeypair, signCommand } from '../src/runtime/signing'; +import { waitFor } from './helpers'; + +const TEST_TIMERS = { electionMinMs: 50, electionMaxMs: 100, heartbeatMs: 20 }; + +/** + * Build a 1-node module-runtime cluster with the demo modules registered. The + * node elects itself leader, so writes commit immediately. `setup` runs against + * the fresh `ModuleStateMachine.host` before `start()` (e.g. to register actor + * keys), so signature verification is configured identically on every replica. + */ +function buildSingleNode(setup?: (sm: ModuleStateMachine) => void): ModuleNode { + const registry = new Map(); + const transport = new LocalTransport(registry, 1); + const sm = new ModuleStateMachine(); + sm.host.register(counter); + sm.host.register(notes); + sm.host.register(accounts); + setup?.(sm); + const node: ModuleNode = new RaftNode({ id: 'node1', peers: [], stateMachine: sm, ...TEST_TIMERS }, transport); + registry.set(node.id, node); + return node; +} + +describe('Module API (single-node cluster)', () => { + describe('unsigned commands and queries', () => { + let node: ModuleNode; + let app: Application; + + beforeAll(async () => { + node = buildSingleNode(); + node.start(); + app = createModuleApp(node); + await waitFor(() => node.isLeader()); + }); + + afterAll(() => { + node.stop(); + }); + + it('increments the counter and reads the value back via a query', async () => { + const res = await request(app).post('/modules/counter/increment').send({ by: 5 }); + expect(res.status).toBe(200); + + const value = await request(app).get('/modules/counter/query/value'); + expect(value.status).toBe(200); + expect(value.body).toBe(5); + }); + + it('creates a note (id/createdAt minted from ctx) and lists it', async () => { + const created = await request(app).post('/modules/notes/create').send({ text: 'hi' }); + expect(created.status).toBe(200); + expect(created.body.id).toBeDefined(); + expect(created.body.createdAt).toBeDefined(); + expect(created.body.text).toBe('hi'); + + const list = await request(app).get('/modules/notes/query/list'); + expect(list.status).toBe(200); + expect(list.body).toEqual([created.body]); + }); + + it('exposes raw module state', async () => { + const state = await request(app).get('/modules/counter/state'); + expect(state.status).toBe(200); + expect(state.body).toEqual({ value: 5 }); + }); + + it('serves a linearizable (strong) query through the read barrier', async () => { + const value = await request(app).get('/modules/counter/query/value?consistency=strong'); + expect(value.status).toBe(200); + expect(value.body).toBe(5); + }); + + it('returns 404 for an unknown module without crashing', async () => { + const res = await request(app).post('/modules/ghost/poke').send({}); + expect(res.status).toBe(404); + }); + + it('returns 404 for an unknown query', async () => { + const res = await request(app).get('/modules/counter/query/nope'); + expect(res.status).toBe(404); + }); + }); + + describe('signed commands (ADR-0019 pillar 7) end-to-end over HTTP', () => { + let node: ModuleNode; + let app: Application; + const actor = 'alice'; + const keys = generateActorKeypair(); + + beforeAll(async () => { + // Authorize alice's PUBLIC key on the host before start; the registry is + // now configured, so every command must carry a valid signature. + node = buildSingleNode((sm) => sm.host.registerActorKey(actor, keys.publicKey)); + node.start(); + app = createModuleApp(node); + await waitFor(() => node.isLeader()); + }); + + afterAll(() => { + node.stop(); + }); + + it('accepts a command signed by the actor and relayed via x-signature', async () => { + const requestId = 'req-signed-1'; + const input = { by: 3 }; + // The CLIENT signs the logical payload with its PRIVATE key (the server + // never holds it); the seed is excluded (the leader adds it later). + const sig = signCommand(keys.privateKey, { + module: 'counter', + command: 'increment', + input, + actor, + requestId, + }); + + const res = await request(app) + .post('/modules/counter/increment') + .set('x-actor', actor) + .set('x-request-id', requestId) + .set('x-signature', sig) + .send(input); + expect(res.status).toBe(200); + + const value = await request(app).get('/modules/counter/query/value'); + expect(value.body).toBe(3); + }); + + it('rejects a command signed by the wrong key with 401', async () => { + const requestId = 'req-forged-1'; + const input = { by: 99 }; + // A DIFFERENT keypair (not the one registered for alice) forges the sig. + const forged = generateActorKeypair(); + const sig = signCommand(forged.privateKey, { + module: 'counter', + command: 'increment', + input, + actor, + requestId, + }); + + const res = await request(app) + .post('/modules/counter/increment') + .set('x-actor', actor) + .set('x-request-id', requestId) + .set('x-signature', sig) + .send(input); + expect(res.status).toBe(401); + + // State unchanged — the forged command never reached its reducer. + const value = await request(app).get('/modules/counter/query/value'); + expect(value.body).toBe(3); + }); + + it('rejects a command with no signature when a registry is configured', async () => { + const res = await request(app) + .post('/modules/counter/increment') + .set('x-actor', actor) + .send({ by: 1 }); + expect(res.status).toBe(401); + }); + }); +}); From f2b44f67f831042a74c96be6eb2ee4134c9b0347 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 21 Jun 2026 17:57:21 +0000 Subject: [PATCH 2/8] =?UTF-8?q?feat(runtime):=20M12=20=E2=80=94=20drive=20?= =?UTF-8?q?the=20effect=20outbox=20post-commit=20on=20the=20live=20leader?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the committed-intent effect loop in a running cluster (M2 only ran in tests). Additive — no src/consensus changes, no RaftNode commit hook. - Effect result as an app command: ModuleAppCommand becomes a union of ModuleInvokeCommand (type 'MODULE') and ModuleEffectResultCommand (type 'MODULE_EFFECT_RESULT', carrying the EffectResultEntry). ModuleStateMachine.apply routes the latter to host.applyEffectResult. - EffectDriver: a leadership-aware poller that, on the LEADER only, drains host.pendingEffects(), runs handlers via an EffectExecutor, and submits the result back through the log (buildEffectResultCommand, stable requestId=effect:). Re-entrancy guarded; interval unref'd; NotLeaderError swallowed so a lost-leadership drain leaves the effect pending for the new leader. Exactly-once committed state (outbox done-guard + idempotent applyEffectResult) with at-least-once handlers. - moduleServer wires an EffectDriver with a demo http handler. Review fix: the internal onResult dispatch (e.g. settle) now bypasses signature verification via a shared dispatchAndEnqueue seam — without it, effects on a signed cluster would 401 and stall. verifySignature for actor commands is unchanged. Adds signed-host effect tests proving the loop completes. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01NneoGGKxBp5ZExL7D7i1Lu --- src/index.ts | 5 + src/moduleServer.ts | 20 ++++ src/runtime/command.ts | 29 ++++- src/runtime/effectDriver.ts | 104 ++++++++++++++++++ src/runtime/moduleHost.ts | 34 +++++- src/runtime/moduleStateMachine.ts | 14 ++- src/runtime/types.ts | 35 ++++++- tests/runtime/effectDriver.test.ts | 163 +++++++++++++++++++++++++++++ tests/runtime/effects.test.ts | 97 +++++++++++++++++ 9 files changed, 492 insertions(+), 9 deletions(-) create mode 100644 src/runtime/effectDriver.ts create mode 100644 tests/runtime/effectDriver.test.ts diff --git a/src/index.ts b/src/index.ts index 87461a0..cce487c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -54,6 +54,9 @@ export type { KeyedModuleDefinition, KeyedReducer } from './runtime/keyedModule' export { buildModuleCommand, buildSignedModuleCommand } from './runtime/command'; export { resolveSeed, createContext } from './runtime/context'; export { EffectExecutor } from './runtime/effectExecutor'; +// The live, in-cluster counterpart of `EffectExecutor` (M12): the leadership-aware +// driver that drains the outbox post-commit and rides results back through the log. +export { EffectDriver } from './runtime/effectDriver'; export { defineProjection } from './runtime/projection'; export { ProjectionHost } from './runtime/projectionHost'; export { MemoryStateStore, StoreView } from './runtime/stateStore'; @@ -64,6 +67,8 @@ export { runSaga } from './runtime/saga'; export type { ModuleDefinition, ModuleAppCommand, + ModuleInvokeCommand, + ModuleEffectResultCommand, ModuleCommand, ModuleApplyResult, Reducer, diff --git a/src/moduleServer.ts b/src/moduleServer.ts index 2cc534e..fe7e5eb 100644 --- a/src/moduleServer.ts +++ b/src/moduleServer.ts @@ -11,6 +11,9 @@ import { AnyModuleDefinition } from './runtime/moduleHost'; import { counter } from './runtime/modules/counter'; import { notes } from './runtime/modules/notes'; import { accounts } from './runtime/modules/accounts'; +import { payments } from './runtime/modules/payments'; +import { EffectDriver } from './runtime/effectDriver'; +import { EffectHandler } from './runtime/types'; dotenv.config(); @@ -33,6 +36,7 @@ const demoModules: AnyModuleDefinition[] = [ counter as AnyModuleDefinition, notes as AnyModuleDefinition, accounts, + payments as AnyModuleDefinition, ]; stateMachine.host.registerModules(demoModules); @@ -47,13 +51,29 @@ metrics.registerCollector(() => node.collectMetrics()); const app = createModuleApp(node, { logger, metrics }); const PORT = getPort(); +// The committed-intent effect loop (M12). The driver polls the leader's outbox +// post-commit and runs each pending effect's handler at the edge, feeding the +// result back through the log. The `http` handler is the demo effect for the +// `payments` module: it resolves a deterministic-SHAPED outcome (no real +// network) so the `settle` follow-up can flip the order to paid on every node. +const effectHandlers: Record = { + http: async (intent) => { + const { orderId } = (intent.payload ?? {}) as { orderId: string }; + return { orderId, ok: true }; + }, +}; +const effectDriver = new EffectDriver(node, effectHandlers); + const server = app.listen(PORT, () => { logger.info('module node started', { node: config.id, port: PORT, peers: config.peers.length }); node.start(); + // Start after the node so the driver only ever acts once a leader is known. + effectDriver.start(); }); const shutdown = () => { logger.info('shutting down', { node: config.id }); + effectDriver.stop(); node.stop(); server.close(() => process.exit(0)); }; diff --git a/src/runtime/command.ts b/src/runtime/command.ts index 29fa343..7838c6e 100644 --- a/src/runtime/command.ts +++ b/src/runtime/command.ts @@ -1,6 +1,6 @@ import { resolveSeed } from './context'; import { signCommand } from './signing'; -import { ModuleAppCommand } from './types'; +import { EffectResultEntry, ModuleEffectResultCommand, ModuleInvokeCommand } from './types'; /** * LEADER-SIDE builder for a module application command (ADR-0019). This is where @@ -24,7 +24,7 @@ export function buildModuleCommand( command: string, input: unknown, meta: { actor: string; requestId: string }, -): ModuleAppCommand { +): ModuleInvokeCommand { return { type: 'MODULE', module, @@ -54,7 +54,7 @@ export function buildSignedModuleCommand( command: string, input: unknown, opts: { actor: string; requestId: string; privateKeyPem: string }, -): ModuleAppCommand { +): ModuleInvokeCommand { const sig = signCommand(opts.privateKeyPem, { module, command, @@ -73,3 +73,26 @@ export function buildSignedModuleCommand( sig, }; } + +/** + * Build the committed effect-result command (M12). The leader's `EffectDriver` + * calls this after a handler resolves a pending effect, so the resolved + * {@link EffectResultEntry} replicates and routes to `ModuleHost.applyEffectResult` + * on every node. + * + * The actor is the system identity `'system'` (no human originated this — the + * runtime did) and the `requestId` is STABLE, derived from the entry's + * idempotency key (`effect:${entry.idempotencyKey}`). That stable id means the + * SUBSTRATE's dedup cache treats a re-submitted result for the same effect as a + * replay (returns the cached apply result without re-routing), layering on top of + * `applyEffectResult` itself being idempotent. The entry already carries its own + * leader/edge-resolved `seed`; this does NOT re-resolve one. + */ +export function buildEffectResultCommand(entry: EffectResultEntry): ModuleEffectResultCommand { + return { + type: 'MODULE_EFFECT_RESULT', + entry, + actor: 'system', + requestId: `effect:${entry.idempotencyKey}`, + }; +} diff --git a/src/runtime/effectDriver.ts b/src/runtime/effectDriver.ts new file mode 100644 index 0000000..3429416 --- /dev/null +++ b/src/runtime/effectDriver.ts @@ -0,0 +1,104 @@ +import { CommandMeta } from '../consensus/types'; +import { NotLeaderError } from '../consensus/raftNode'; +import { buildEffectResultCommand } from './command'; +import { EffectExecutor } from './effectExecutor'; +import { ModuleNode } from './moduleStateMachine'; +import { EffectHandler } from './types'; + +/** Default tick interval (ms) for the leader's outbox drain loop. */ +const DEFAULT_INTERVAL_MS = 25; + +/** + * The LIVE, leadership-aware driver of the committed-intent effect loop (M12). + * It is the running-cluster counterpart of what `tests/runtime/effects.test.ts` + * exercises by hand: on the LEADER only, it drains the `ModuleHost` outbox after + * commits, runs each pending effect's handler at the edge (the ONE place I/O is + * allowed), and feeds the resolved {@link EffectResultEntry} back through the + * replicated log (`MODULE_EFFECT_RESULT`) so every node applies it. + * + * EXACTLY-ONCE STATE, AT-LEAST-ONCE HANDLERS. Handler execution is at-least-once + * across leader changes and restarts: if leadership is lost mid-drain (or a node + * crashes after the side effect but before the result commits), the effect stays + * `pending` and the new leader's driver retries it. The COMMITTED STATE effect is + * nonetheless exactly-once because three guards collaborate: + * - leader-only ticks: followers never run handlers, so only one node acts; + * - the outbox dedups enqueue on `idempotencyKey` and `applyEffectResult` is + * idempotent on it (a redelivered/replayed result never re-dispatches); + * - the result command's stable `requestId` (`effect:`) lets the substrate + * dedup cache treat a re-submitted result as a replay. + * Handlers should therefore be idempotent where the external system allows, but + * the replicated state stays correct even if a handler runs twice. + * + * NO consensus-core changes: this is an additive, post-commit poller. It does + * NOT register a commit hook on the node — it polls `pendingEffects()` on a + * timer, which is sufficient because the outbox is durable replicated state. + */ +export class EffectDriver { + private readonly executor: EffectExecutor; + private readonly intervalMs: number; + private timer: ReturnType | undefined; + /** Re-entrancy guard: a tick whose drain is still running skips the next. */ + private draining = false; + + constructor( + private readonly node: ModuleNode, + handlers: Record, + opts: { intervalMs?: number } = {}, + ) { + this.intervalMs = opts.intervalMs ?? DEFAULT_INTERVAL_MS; + // The executor's `submit` rides the resolved result back through the log. + // A `NotLeaderError` (leadership lost between the leader check and submit) + // is swallowed: leave the effect pending so the new leader retries. The + // outbox dedup + idempotent `applyEffectResult` keep state exactly-once. + this.executor = new EffectExecutor(handlers, async (entry) => { + const command = buildEffectResultCommand(entry); + const meta: CommandMeta = { + requestId: command.requestId, + actor: command.actor, + timestamp: new Date().toISOString(), + }; + try { + await this.node.submit(command, meta); + } catch (err) { + if (err instanceof NotLeaderError) return; + throw err; + } + }); + } + + /** + * Start the drain loop. Each tick: if this node is NOT the leader, do nothing + * (followers never execute effects). Otherwise, if a drain is not already in + * flight and there are pending effects, drain them. The `draining` flag makes + * overlapping ticks a no-op so a slow handler is not double-fired by the timer. + */ + start(): void { + if (this.timer) return; + this.timer = setInterval(() => { + void this.tick(); + }, this.intervalMs); + // Don't let the loop keep the process alive on its own. + if (typeof this.timer.unref === 'function') this.timer.unref(); + } + + private async tick(): Promise { + if (this.draining) return; + if (!this.node.isLeader()) return; + const pending = this.node.app.host.pendingEffects(); + if (pending.length === 0) return; + this.draining = true; + try { + await this.executor.drain(pending); + } finally { + this.draining = false; + } + } + + /** Stop the drain loop and clear the timer (no leaks). */ + stop(): void { + if (this.timer) { + clearInterval(this.timer); + this.timer = undefined; + } + } +} diff --git a/src/runtime/moduleHost.ts b/src/runtime/moduleHost.ts index c75809d..c063441 100644 --- a/src/runtime/moduleHost.ts +++ b/src/runtime/moduleHost.ts @@ -242,6 +242,24 @@ export class ModuleHost { return denied; } + return this.dispatchAndEnqueue(cmd, meta); + } + + /** + * Run a command through the reducer dispatch core and record any effects it + * emits into the outbox. This is the SHARED post-verification apply path: + * `apply()` calls it AFTER signature verification (caller commands), and + * `applyEffectResult()` calls it DIRECTLY for the system-internal `onResult` + * follow-up — which carries no actor signature and must bypass verification + * (it is a runtime-internal consequence of an already-verified command, not a + * fresh actor command). Everything else (reducer execution, mutation-safe + * clone, resource-bound 413 check, audit leaf, determinism) is identical for + * both paths because both run the same `dispatch` + the same enqueue here. + */ + private dispatchAndEnqueue( + cmd: ModuleCommand, + meta: { actor: string; requestId: string }, + ): ModuleApplyResult { const result = this.dispatch(cmd, meta); if (result.status === 200) { // Record each emitted effect into the outbox as `pending`, but only if @@ -609,9 +627,19 @@ export class ModuleHost { } // Feed the edge-resolved result to the consuming reducer. We go through - // `apply` (not bare `dispatch`) so any effects the onResult reducer emits - // are themselves enqueued into the outbox. - return this.apply( + // `dispatchAndEnqueue` (dispatch + outbox enqueue) and NOT the public + // `apply`, so this system-internal follow-up SKIPS actor-signature + // verification: the synthesized `onResult` command (e.g. `payments.settle`) + // carries no `sig`, and on a host with a `KeyRegistry` configured `apply` + // would reject it 401 — silently stalling the effect loop. The originating + // command (e.g. `charge`) was already actor-verified; this `settle` is a + // runtime-internal, system-trusted consequence, so it must not require an + // actor signature. All the OTHER apply-path behavior is preserved: the + // reducer runs, mutation-safety clone, the resource-bound 413 check, the + // audit leaf, and — via `dispatchAndEnqueue` — outbox recording of any + // effects this reducer itself emits. Determinism is unchanged: this is a + // pure refactor of WHO verifies, not of the dispatch path. + return this.dispatchAndEnqueue( { module: onResult.module, command: onResult.command, diff --git a/src/runtime/moduleStateMachine.ts b/src/runtime/moduleStateMachine.ts index 42a2636..9ff5b56 100644 --- a/src/runtime/moduleStateMachine.ts +++ b/src/runtime/moduleStateMachine.ts @@ -35,6 +35,18 @@ export class ModuleStateMachine implements StateMachine { + // Route on the application `type` discriminator. A caller invoke + // (`'MODULE'`) runs the reducer; a committed effect result + // (`'MODULE_EFFECT_RESULT'`, M12) routes to `applyEffectResult` so the + // edge-resolved outcome folds back into state identically on every node. + if (command.type === 'MODULE_EFFECT_RESULT') { + const result = this.host.applyEffectResult(command.entry, { + actor: command.actor, + requestId: command.requestId, + }); + return { status: result.status, data: result.result, message: result.message }; + } + const result = this.host.apply( { module: command.module, @@ -46,7 +58,7 @@ export class ModuleStateMachine implements StateMachine { + fn.calls += 1; + const { orderId } = intent.payload as { orderId: string }; + return { orderId, ok: true }; + }) as EffectHandler & { calls: number }; + fn.calls = 0; + return fn; +} + +/** Submit a charge on the current leader and assert it was accepted. */ +async function charge(leader: ModuleNode, orderId: string, amount: number, requestId: string): Promise { + const meta: CommandMeta = { requestId, actor: 'alice', timestamp: new Date().toISOString() }; + const res = await leader.submit(buildModuleCommand('payments', 'charge', { orderId, amount }, meta), meta); + expect(res.status).toBe(200); +} + +/** The order status as seen by a node's local host. */ +function orderStatus(node: ModuleNode, orderId: string): string | undefined { + const order = node.app.host.query('payments', 'order', { orderId }) as { status: string } | undefined; + return order?.status; +} + +describe('EffectDriver: live committed-intent effect loop over Raft', () => { + let nodes: ModuleNode[]; + let drivers: EffectDriver[]; + + afterEach(() => { + drivers.forEach((d) => d.stop()); + nodes.forEach((n) => n.stop()); + }); + + it('leader runs the handler exactly once and settle converges to paid on every node', async () => { + nodes = buildModuleCluster(3, [payments]); + const handler = okHandler(); + // A driver on EVERY node, but only the leader should ever run the handler. + drivers = nodes.map((n) => new EffectDriver(n, { http: handler }, { intervalMs: 10 })); + nodes.forEach((n) => n.start()); + drivers.forEach((d) => d.start()); + + await waitFor(() => leaders(nodes).length === 1); + const leader = leaders(nodes)[0]; + + await charge(leader, 'o1', 100, 'req-charge-1'); + + // The leader's driver drains, the handler runs, the result rides the log, + // and `settle` flips the order to paid on every node. + await waitFor(() => nodes.every((n) => orderStatus(n, 'o1') === 'paid')); + + // Exactly-once execution: a SINGLE handler invocation across the cluster + // (only the leader ran it, even though all 3 nodes have a driver). + expect(handler.calls).toBe(1); + + // Convergence: identical module state + outbox on every node. + const states = new Set(nodes.map((n) => JSON.stringify(n.app.host.getState('payments')))); + expect(states.size).toBe(1); + const outboxes = new Set(nodes.map((n) => JSON.stringify(n.app.host.getOutbox()))); + expect(outboxes.size).toBe(1); + // The single outbox entry is done. + const leaderOutbox = leader.app.host.getOutbox(); + expect(leaderOutbox).toHaveLength(1); + expect(leaderOutbox[0].status).toBe('done'); + }); + + it('followers do not run the handler (only the leader acts)', async () => { + nodes = buildModuleCluster(3, [payments]); + const handler = okHandler(); + drivers = nodes.map((n) => new EffectDriver(n, { http: handler }, { intervalMs: 10 })); + nodes.forEach((n) => n.start()); + drivers.forEach((d) => d.start()); + + await waitFor(() => leaders(nodes).length === 1); + const leader = leaders(nodes)[0]; + + await charge(leader, 'o1', 100, 'req-charge-1'); + await waitFor(() => nodes.every((n) => orderStatus(n, 'o1') === 'paid')); + + // Even with a driver on all 3 nodes, the handler ran exactly once: the two + // followers' drivers no-op every tick because `isLeader()` is false. + expect(handler.calls).toBe(1); + }); + + it('a failing handler leaves the effect pending; a later tick completes it', async () => { + nodes = buildModuleCluster(3, [payments]); + + // A handler that fails until `succeed` is flipped, then reports paid. + let attempts = 0; + let succeed = false; + const handler: EffectHandler = async (intent) => { + attempts += 1; + if (!succeed) throw new Error('gateway down'); + const { orderId } = intent.payload as { orderId: string }; + return { orderId, ok: true }; + }; + drivers = nodes.map((n) => new EffectDriver(n, { http: handler }, { intervalMs: 10 })); + nodes.forEach((n) => n.start()); + drivers.forEach((d) => d.start()); + + await waitFor(() => leaders(nodes).length === 1); + const leader = leaders(nodes)[0]; + + await charge(leader, 'o1', 100, 'req-charge-1'); + + // While failing: the handler is attempted (≥1) but the effect stays pending + // and no node settles the order. + await waitFor(() => attempts >= 1); + expect(leader.app.host.getOutbox()[0].status).toBe('pending'); + expect(orderStatus(leader, 'o1')).toBe('pending'); + + // Make the handler succeed; a later tick drains and completes the effect. + succeed = true; + await waitFor(() => nodes.every((n) => orderStatus(n, 'o1') === 'paid')); + + // Convergence holds; outbox entry is done on every node (single settle). + const outboxes = new Set(nodes.map((n) => JSON.stringify(n.app.host.getOutbox()))); + expect(outboxes.size).toBe(1); + expect(leader.app.host.getOutbox()).toHaveLength(1); + expect(leader.app.host.getOutbox()[0].status).toBe('done'); + }); + + it('idempotency: the committed result applies once and the cluster converges (no double settle)', async () => { + nodes = buildModuleCluster(3, [payments]); + const handler = okHandler(); + drivers = nodes.map((n) => new EffectDriver(n, { http: handler }, { intervalMs: 10 })); + nodes.forEach((n) => n.start()); + drivers.forEach((d) => d.start()); + + await waitFor(() => leaders(nodes).length === 1); + const leader = leaders(nodes)[0]; + + await charge(leader, 'o1', 100, 'req-charge-1'); + await waitFor(() => nodes.every((n) => orderStatus(n, 'o1') === 'paid')); + + // Let several more driver ticks fire: the now-`done` outbox entry is no + // longer pending, so nothing re-drains and the handler is not re-invoked. + await new Promise((r) => setTimeout(r, 60)); + expect(handler.calls).toBe(1); + + // Exactly one outbox entry (done), and all nodes are byte-identical: + // structurally proves no double-apply of the effect result. + const leaderOutbox = leader.app.host.getOutbox(); + expect(leaderOutbox).toHaveLength(1); + expect(leaderOutbox[0].status).toBe('done'); + const snapshots = new Set(nodes.map((n) => JSON.stringify(n.app.host.snapshot()))); + expect(snapshots.size).toBe(1); + }); +}); diff --git a/tests/runtime/effects.test.ts b/tests/runtime/effects.test.ts index 0d94aeb..2dab88a 100644 --- a/tests/runtime/effects.test.ts +++ b/tests/runtime/effects.test.ts @@ -2,6 +2,7 @@ import { EffectExecutor } from '../../src/runtime/effectExecutor'; import { ModuleHost } from '../../src/runtime/moduleHost'; import { defineModule } from '../../src/runtime/defineModule'; import { payments } from '../../src/runtime/modules/payments'; +import { generateActorKeypair, KeyRegistry, signCommand } from '../../src/runtime/signing'; import { EffectHandler, EffectResultEntry, ModuleCommand, Seed } from '../../src/runtime/types'; const META = { actor: 'tester', requestId: 'req-1' }; @@ -292,6 +293,102 @@ describe('committed-intent effects: reserved module names', () => { }); }); +describe('committed-intent effects: signed host (M12 fix)', () => { + // With a KeyRegistry configured, a caller `charge` MUST carry a valid actor + // signature. The effect's `settle` follow-up (onResult) is a runtime-internal, + // system-trusted consequence and carries NO signature — it must bypass actor + // verification, or the effect loop silently never completes (settle 401s and + // the order is stuck `pending`). This is the regression M12's fix addresses. + const SIGNED_META = { actor: 'alice', requestId: 'req-charge-1' }; + + function signedChargeHost(): { host: ModuleHost; signedCharge: ModuleCommand } { + const host = new ModuleHost(); + host.register(payments); + const { publicKey, privateKey } = generateActorKeypair(); + const registry = new KeyRegistry(); + registry.registerActor('alice', publicKey); + host.setKeyRegistry(registry); + + const input = { orderId: 'o1', amount: 100 }; + const sig = signCommand(privateKey, { + module: 'payments', + command: 'charge', + input, + actor: SIGNED_META.actor, + requestId: SIGNED_META.requestId, + }); + const signedCharge: ModuleCommand = { + module: 'payments', + command: 'charge', + input, + seed: seed('k1'), + sig, + }; + return { host, signedCharge }; + } + + it('a SIGNED charge succeeds and its unsigned settle (onResult) still completes the loop', async () => { + const { host, signedCharge } = signedChargeHost(); + + // Signed actor command is accepted and enqueues the effect. + const res = host.apply(signedCharge, SIGNED_META); + expect(res.status).toBe(200); + expect(host.pendingEffects()).toHaveLength(1); + + // Drain at the edge; the resolved result rides back as an EffectResultEntry. + const handler = okHandler(); + const submitted: EffectResultEntry[] = []; + const exec = new EffectExecutor({ http: handler }, (e) => submitted.push(e)); + await exec.drain(host.pendingEffects()); + expect(submitted).toHaveLength(1); + + // applyEffectResult dispatches the UNSIGNED `settle` (onResult). Despite the + // KeyRegistry, it is NOT rejected 401 — the loop completes. + const applyRes = host.applyEffectResult(submitted[0], SIGNED_META); + expect(applyRes.status).toBe(200); + + const order = host.query('payments', 'order', { orderId: 'o1' }) as { status: string }; + expect(order.status).toBe('paid'); + expect(host.getOutbox()[0].status).toBe('done'); + expect(host.pendingEffects()).toHaveLength(0); + }); + + it('on the same signed host, an UNSIGNED actor charge is still rejected 401', () => { + const { host } = signedChargeHost(); + const unsigned: ModuleCommand = { + module: 'payments', + command: 'charge', + input: { orderId: 'o2', amount: 200 }, + seed: seed('k2'), + }; + const res = host.apply(unsigned, { actor: 'alice', requestId: 'req-unsigned' }); + expect(res.status).toBe(401); + // No reducer ran: no effect enqueued, no order recorded. + expect(host.pendingEffects()).toHaveLength(0); + expect(host.query('payments', 'order', { orderId: 'o2' })).toBeUndefined(); + }); + + it('on the same signed host, a FORGED actor charge (mismatched signer) is rejected 401', () => { + const { host } = signedChargeHost(); + // Sign with a DIFFERENT key than alice's registered one (a leader forging + // `actor: alice`). The signature cannot verify against alice's public key. + const { privateKey: attackerKey } = generateActorKeypair(); + const input = { orderId: 'o3', amount: 300 }; + const sig = signCommand(attackerKey, { + module: 'payments', + command: 'charge', + input, + actor: 'alice', + requestId: 'req-forged', + }); + const forged: ModuleCommand = { module: 'payments', command: 'charge', input, seed: seed('k3'), sig }; + const res = host.apply(forged, { actor: 'alice', requestId: 'req-forged' }); + expect(res.status).toBe(401); + expect(host.pendingEffects()).toHaveLength(0); + expect(host.query('payments', 'order', { orderId: 'o3' })).toBeUndefined(); + }); +}); + describe('committed-intent effects: convergence', () => { it('two hosts applying the same charge + result entry reach identical snapshots', async () => { const host1 = freshHost(); From 7103ca640d64bf199d2e2a4fca378fb23781b793 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 21 Jun 2026 18:05:26 +0000 Subject: [PATCH 3/8] =?UTF-8?q?refactor(consensus):=20M13=20=E2=80=94=20ex?= =?UTF-8?q?tract=20the=20Consensus=20interface=20above=20the=20log?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Defines Consensus (src/consensus/consensus.ts) — the commit-ordered-log contract the application/HTTP/runtime layers actually use (submit, readBarrier, changeMembership, leadership/status, lifecycle, app + stateMachine accessors). RaftNode now `implements Consensus` (a 2-line change: import + clause; no algorithm/behavior change). The app-facing consumers — book/module controllers, audit routes, and the effect driver — depend on the interface, not RaftNode; ShardRouter's structural node type is already a subset of it. This is the ordering seam ADR-0021 names as the BFT-swap prerequisite. The Raft RPC surface (RpcHandler) stays protocol-specific (raftRoutes on RaftNode), and RaftNode is still constructed directly by the servers/tests (wiring, not the contract). ADR-0021 updated to record that both the StateMachine seam (below the log) and the Consensus seam (above it) are now extracted. Behavior-preserving: all 176 tests pass, tsc clean, cast-free. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01NneoGGKxBp5ZExL7D7i1Lu --- docs/adr/0021-pluggable-bft-consensus-seam.md | 35 +++++-- src/consensus/consensus.ts | 99 +++++++++++++++++++ src/consensus/raftNode.ts | 3 +- src/controllers/bookController.ts | 9 +- src/controllers/moduleController.ts | 13 ++- src/index.ts | 3 + src/routes/auditRoutes.ts | 11 ++- src/routes/bookRoutes.ts | 6 +- src/routes/moduleRoutes.ts | 6 +- src/routes/raftRoutes.ts | 10 +- src/runtime/effectDriver.ts | 10 +- src/runtime/shardRouter.ts | 4 + 12 files changed, 182 insertions(+), 27 deletions(-) create mode 100644 src/consensus/consensus.ts diff --git a/docs/adr/0021-pluggable-bft-consensus-seam.md b/docs/adr/0021-pluggable-bft-consensus-seam.md index 976892a..9e0d75e 100644 --- a/docs/adr/0021-pluggable-bft-consensus-seam.md +++ b/docs/adr/0021-pluggable-bft-consensus-seam.md @@ -41,11 +41,15 @@ pluggability point. **Do not implement BFT in the prototype.** Instead: -1. **Name the real seam.** Define (in documentation, and as a refactoring target) - a `Consensus` interface capturing exactly what the runtime needs: `submit`, - `readBarrier`, membership, and a leadership/status view — the subset of - `RaftNode` that `ReplicatedStateMachine` and the controllers actually use. A - BFT engine would implement *that*, not the Raft `Transport`. +1. **Name the real seam.** Define a `Consensus` interface + (`src/consensus/consensus.ts`, extracted in M13) capturing exactly what the + runtime needs: `submit`, `readBarrier`, membership, and a leadership/status + view — the subset of `RaftNode` that `ReplicatedStateMachine` and the + controllers actually use. `RaftNode implements Consensus`, and the + application-facing consumers (the book/module controllers, the audit routes, + the effect driver, and the structural `ShardRouter` node) now depend on the + interface, not the concrete node. A BFT engine would implement *that*, not the + Raft `Transport`. 2. **Reuse M7 as a building block.** BFT requires signed, non-repudiable messages; the ed25519 signing + `KeyRegistry` from M7 is the cryptographic primitive a BFT layer would build on (signing *consensus votes*, not just client commands). @@ -74,11 +78,22 @@ pluggability point. - The system remains single-trust-domain (CFT) — unsuitable for adversarial, multi-organization deployments, exactly as ADR-0018 concluded. -- The application boundary *below* the log — the `StateMachine` interface — is now - extracted (ADR-0017), so applications already plug in cleanly. But the ordering - boundary *above* the log (a `Consensus` interface a BFT engine would implement) - is still not extracted: `RaftNode` is referenced directly (e.g. `ShardRouter`, - controllers), so a real BFT swap would first require that second refactor. +- The application boundary *below* the log — the `StateMachine` interface + (ADR-0017) — and the ordering boundary *above* the log — the `Consensus` + interface (extracted in M13, `src/consensus/consensus.ts`) — are now BOTH + extracted, so applications and the HTTP/runtime layers plug in cleanly and a BFT + engine could implement `Consensus` without those consumers changing. Two gaps + remain, both narrow and deliberate: + - **Transport/RPC surface.** `Consensus` is the commit-ordered-log contract, not + the wire protocol. The Raft RPCs (`RpcHandler` in `transport.ts`: + `handleRequestVote`/`handleAppendEntries`/`handleInstallSnapshot`) are + Raft-shaped and are NOT part of `Consensus`; a BFT engine needs a different, + multi-phase, signed message surface, so `raftRoutes.ts` stays typed to the + concrete `RaftNode`. That surface is replaced differently (not 1:1) by BFT. + - **Construction/wiring.** `RaftNode` is still instantiated directly by + `server.ts`/`moduleServer.ts`/tests. That is wiring (choosing the engine), not + the contract — swapping engines means changing those few construction sites, + not the controllers or routes. ## Alternatives considered diff --git a/src/consensus/consensus.ts b/src/consensus/consensus.ts new file mode 100644 index 0000000..b40b99e --- /dev/null +++ b/src/consensus/consensus.ts @@ -0,0 +1,99 @@ +import { ReplicatedStateMachine } from './replicatedStateMachine'; +import { StateMachine } from './stateMachine'; +import { AppCommand, ApplyResult, CommandMeta, PeerInfo } from './types'; + +/** + * The ordering/commit seam ABOVE the log (ADR-0021). + * + * This is exactly the application-facing contract the rest of the system depends + * on from a consensus node: propose a command and learn the committed, ordered, + * applied result (`submit`); read linearizably (`readBarrier`); change cluster + * membership; and inspect leadership/membership status. It is the + * "commit-ordered-log contract" — everything in `src/runtime/`, the HTTP + * controllers, and the audit routes depend only on *this*, never on Raft + * internals. + * + * Naming this seam is the prerequisite ADR-0021 identifies for a future + * Byzantine-Fault-Tolerant (BFT) engine: such an engine would implement + * `Consensus` and the controllers/audit routes would not change. Crucially, the + * Raft RPCs (`RpcHandler` in `transport.ts` — `handleRequestVote` / + * `handleAppendEntries` / `handleInstallSnapshot`) are the SEPARATE, + * protocol-specific surface: a BFT protocol uses an entirely different + * multi-phase, signed message exchange, so those RPCs are intentionally NOT part + * of this interface. The `Transport`/`RpcHandler` boundary is replaced + * *differently* (and not 1:1) by a BFT engine; this `Consensus` boundary is the + * one the application rides unchanged. + * + * @typeParam C - the application's command union (each a string-`type` object). + * @typeParam T - the payload type carried back in {@link ApplyResult}. + * @typeParam A - the concrete application state machine type, so consumers can + * reach domain reads via {@link Consensus.app} (e.g. `node.app.host`). + */ +export interface Consensus { + /** The application state machine plugged into this node (for domain reads). */ + readonly app: A; + + /** The replicated state machine wrapper (audit, idempotency, snapshots, size). */ + get stateMachine(): ReplicatedStateMachine; + + /** + * Propose a command. Resolves once the entry is committed and applied, and + * rejects with a `NotLeaderError` if this node is not the leader. + */ + submit(command: C, meta?: CommandMeta): Promise>; + + /** + * Linearizable read barrier (Raft §6.4, "ReadIndex"): resolving it guarantees + * a subsequent local read reflects every write committed before the barrier + * was requested. Rejects with a `NotLeaderError` if this node is not (or + * ceases to be) the leader, so the caller can forward the read like a write. + */ + readBarrier(): Promise; + + /** + * Add or remove a single voting member (one change at a time). The returned + * promise resolves once the change commits; leader-only. + */ + changeMembership(change: { add?: PeerInfo; remove?: string }, meta?: CommandMeta): Promise>; + + /** Whether this node currently believes itself to be the leader. */ + isLeader(): boolean; + + /** The current leader's id, or null if unknown. */ + getLeaderId(): string | null; + + /** URL of the current leader (for write forwarding), or null if unknown/self. */ + getLeaderUrl(): string | null; + + /** A status view of this node (role, term, indices, membership). */ + status(): { + id: string; + role: string; + term: number; + leaderId: string | null; + lastLogIndex: number; + logEntries: number; + snapshotIndex: number; + commitIndex: number; + lastApplied: number; + stateSize: number; + dedupCacheSize: number; + members: string[]; + }; + + /** The current cluster configuration (voting members, including self). */ + getMembers(): PeerInfo[]; + + /** Begin participating in consensus (restores durable state, arms timers). */ + start(): void; + + /** Stop participating in consensus (clears timers, rejects in-flight work). */ + stop(): void; +} + +/** + * Convenience alias for a {@link Consensus} whose application is a concrete + * {@link StateMachine}. Mirrors how `RaftNode` is parameterized, so a + * consumer can write `ConsensusOf`. + */ +export type ConsensusOf> = Consensus; diff --git a/src/consensus/raftNode.ts b/src/consensus/raftNode.ts index 069e98f..d1734e5 100644 --- a/src/consensus/raftNode.ts +++ b/src/consensus/raftNode.ts @@ -1,3 +1,4 @@ +import { Consensus } from './consensus'; import { ReplicatedStateMachine, RsmSnapshot } from './replicatedStateMachine'; import { StateMachine } from './stateMachine'; import { MemoryStorage, RaftStorage, PersistentState } from './storage'; @@ -91,7 +92,7 @@ export class RaftNode< C extends AppCommand = AppCommand, T = unknown, SM extends StateMachine = StateMachine, -> implements RpcHandler { +> implements Consensus, RpcHandler { readonly id: string; private readonly selfPeer: PeerInfo; private readonly transport: Transport; diff --git a/src/controllers/bookController.ts b/src/controllers/bookController.ts index ec0ae7a..2511483 100644 --- a/src/controllers/bookController.ts +++ b/src/controllers/bookController.ts @@ -1,9 +1,11 @@ import { Request, Response } from 'express'; +import { Consensus } from '../consensus/consensus'; import { NotLeaderError } from '../consensus/raftNode'; import { CommandMeta } from '../consensus/types'; import { getContext } from '../platform/requestContext'; import { forwardToLeader, isForwarded } from '../platform/forward'; import { + Book, BookCommand, buildAddCommand, buildBorrowCommand, @@ -11,7 +13,10 @@ import { buildReturnCommand, buildUpdateCommand, } from '../models/book'; -import { BookNode } from '../models/bookStateMachine'; +import { BookStateMachine } from '../models/bookStateMachine'; + +/** The consensus seam this controller depends on, specialized to the book app. */ +type BookConsensus = Consensus; /** A linearizable read is requested via `?consistency=strong` or `X-Consistency: strong`. */ function wantsStrongRead(req: Request): boolean { @@ -26,7 +31,7 @@ function wantsStrongRead(req: Request): boolean { * via the leader; a follower transparently forwards writes (and strong reads) * to the leader (falling back to 421 if the leader is unknown/unreachable). */ -export function createBookController(node: BookNode) { +export function createBookController(node: BookConsensus) { // Forward to the leader (or reply 421) when this node isn't the leader. const onNotLeader = async (req: Request, res: Response, err: NotLeaderError): Promise => { const leaderUrl = node.getLeaderUrl(); diff --git a/src/controllers/moduleController.ts b/src/controllers/moduleController.ts index 2b3363a..f05ea0d 100644 --- a/src/controllers/moduleController.ts +++ b/src/controllers/moduleController.ts @@ -1,10 +1,19 @@ import { Request, Response } from 'express'; +import { Consensus } from '../consensus/consensus'; import { NotLeaderError } from '../consensus/raftNode'; import { CommandMeta } from '../consensus/types'; import { getContext } from '../platform/requestContext'; import { forwardToLeader, isForwarded } from '../platform/forward'; import { buildModuleCommand } from '../runtime/command'; -import { ModuleNode } from '../runtime/moduleStateMachine'; +import { ModuleStateMachine } from '../runtime/moduleStateMachine'; +import { ModuleAppCommand } from '../runtime/types'; + +/** + * The consensus seam this controller depends on, specialized to the module + * runtime: `node.app` is the {@link ModuleStateMachine}, so `node.app.host` + * resolves to the live `ModuleHost` the controller reads queries/state from. + */ +type ModuleConsensus = Consensus; /** A linearizable read is requested via `?consistency=strong` or `X-Consistency: strong`. */ function wantsStrongRead(req: Request): boolean { @@ -29,7 +38,7 @@ function wantsStrongRead(req: Request): boolean { * deliberately outside the signed payload) and every replica verifies the * signature on the deterministic apply path. The adapter does no signing itself. */ -export function createModuleController(node: ModuleNode) { +export function createModuleController(node: ModuleConsensus) { // Forward to the leader (or reply 421) when this node isn't the leader. const onNotLeader = async (req: Request, res: Response, err: NotLeaderError): Promise => { const leaderUrl = node.getLeaderUrl(); diff --git a/src/index.ts b/src/index.ts index cce487c..67c99a9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,6 +8,9 @@ // Consensus core export { RaftNode, NotLeaderError, MembershipError } from './consensus/raftNode'; export type { RaftConfig } from './consensus/raftNode'; +// The ordering/commit seam above the log (ADR-0021): the application-facing +// contract `RaftNode` implements and a future BFT engine would implement instead. +export type { Consensus, ConsensusOf } from './consensus/consensus'; export type { StateMachine } from './consensus/stateMachine'; export { ReplicatedStateMachine, DEFAULT_DEDUP_LIMIT } from './consensus/replicatedStateMachine'; export type { RsmSnapshot } from './consensus/replicatedStateMachine'; diff --git a/src/routes/auditRoutes.ts b/src/routes/auditRoutes.ts index c372d7c..11c85c0 100644 --- a/src/routes/auditRoutes.ts +++ b/src/routes/auditRoutes.ts @@ -1,14 +1,17 @@ import express from 'express'; -import { RaftNode } from '../consensus/raftNode'; -import { StateMachine } from '../consensus/stateMachine'; +import { Consensus } from '../consensus/consensus'; import { AppCommand } from '../consensus/types'; /** * The replicated, hash-chained audit log — a built-in, tamper-evident history * of every committed state change, available on every node. Application-agnostic. + * + * Depends only on the {@link Consensus} seam (`node.stateMachine`), not on Raft: + * the audit trail is a property of the committed-ordered log, so it works + * unchanged under any engine that implements `Consensus` (ADR-0021). */ -export default function auditRoutes>( - node: RaftNode, +export default function auditRoutes( + node: Consensus, ) { const router = express.Router(); diff --git a/src/routes/bookRoutes.ts b/src/routes/bookRoutes.ts index e75eb69..b5060ae 100644 --- a/src/routes/bookRoutes.ts +++ b/src/routes/bookRoutes.ts @@ -1,8 +1,10 @@ import express from 'express'; -import { BookNode } from '../models/bookStateMachine'; +import { Consensus } from '../consensus/consensus'; +import { Book, BookCommand } from '../models/book'; +import { BookStateMachine } from '../models/bookStateMachine'; import { createBookController } from '../controllers/bookController'; -export default function bookRoutes(node: BookNode) { +export default function bookRoutes(node: Consensus) { const router = express.Router(); const c = createBookController(node); diff --git a/src/routes/moduleRoutes.ts b/src/routes/moduleRoutes.ts index d30e289..c6f3f94 100644 --- a/src/routes/moduleRoutes.ts +++ b/src/routes/moduleRoutes.ts @@ -1,5 +1,7 @@ import express from 'express'; -import { ModuleNode } from '../runtime/moduleStateMachine'; +import { Consensus } from '../consensus/consensus'; +import { ModuleStateMachine } from '../runtime/moduleStateMachine'; +import { ModuleAppCommand } from '../runtime/types'; import { createModuleController } from '../controllers/moduleController'; /** @@ -11,7 +13,7 @@ import { createModuleController } from '../controllers/moduleController'; * a literal `query`/`state` segment can never be mis-bound as a command name. * (They are also GET vs POST, but ordering keeps the intent unambiguous.) */ -export default function moduleRoutes(node: ModuleNode) { +export default function moduleRoutes(node: Consensus) { const router = express.Router(); const c = createModuleController(node); diff --git a/src/routes/raftRoutes.ts b/src/routes/raftRoutes.ts index 2c09efd..d6c26ab 100644 --- a/src/routes/raftRoutes.ts +++ b/src/routes/raftRoutes.ts @@ -5,7 +5,15 @@ import { AppCommand, CommandMeta } from '../consensus/types'; import { getContext } from '../platform/requestContext'; import { forwardToLeader, isForwarded } from '../platform/forward'; -/** Internal cluster endpoints: peer RPCs, membership admin, and a status view. */ +/** + * Internal cluster endpoints: peer RPCs, membership admin, and a status view. + * + * This is the Raft-SPECIFIC adapter: it exposes the protocol RPC endpoints + * (`handleRequestVote`/`handleAppendEntries`/`handleInstallSnapshot`, the + * `RpcHandler` surface) which are Raft-shaped and would be replaced *differently* + * by a BFT engine. It is therefore intentionally typed to the concrete + * {@link RaftNode}, NOT the engine-agnostic {@link Consensus} seam (ADR-0021). + */ export default function raftRoutes>( node: RaftNode, ) { diff --git a/src/runtime/effectDriver.ts b/src/runtime/effectDriver.ts index 3429416..af5044a 100644 --- a/src/runtime/effectDriver.ts +++ b/src/runtime/effectDriver.ts @@ -1,9 +1,10 @@ import { CommandMeta } from '../consensus/types'; import { NotLeaderError } from '../consensus/raftNode'; +import { ConsensusOf } from '../consensus/consensus'; import { buildEffectResultCommand } from './command'; import { EffectExecutor } from './effectExecutor'; -import { ModuleNode } from './moduleStateMachine'; -import { EffectHandler } from './types'; +import { ModuleStateMachine } from './moduleStateMachine'; +import { EffectHandler, ModuleAppCommand } from './types'; /** Default tick interval (ms) for the leader's outbox drain loop. */ const DEFAULT_INTERVAL_MS = 25; @@ -41,7 +42,10 @@ export class EffectDriver { private draining = false; constructor( - private readonly node: ModuleNode, + // Depends on the `Consensus` seam (ADR-0021/M13), not the concrete node: + // it only uses `submit`/`isLeader`/`app`, so any engine implementing + // `Consensus` drives effects unchanged. + private readonly node: ConsensusOf, handlers: Record, opts: { intervalMs?: number } = {}, ) { diff --git a/src/runtime/shardRouter.ts b/src/runtime/shardRouter.ts index 5c5cf27..483eada 100644 --- a/src/runtime/shardRouter.ts +++ b/src/runtime/shardRouter.ts @@ -5,6 +5,10 @@ import { AppCommand, ApplyResult, CommandMeta } from '../consensus/types'; * The minimal node surface the router needs: ask who leads, and submit to it. * Structural so any `RaftNode` (e.g. a {@link ModuleNode}) fits without the * router depending on the concrete node/state-machine types. + * + * Note: this `{ isLeader, submit }` shape is a strict SUBSET of the `Consensus` + * seam (ADR-0021) — the router was already decoupled from the concrete node via + * structural typing, so any `Consensus` satisfies it unchanged. */ export interface ShardNode { isLeader(): boolean; From 4b8d3d5427b421a28b0405e8c139b520643181ea Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 21 Jun 2026 18:11:37 +0000 Subject: [PATCH 4/8] =?UTF-8?q?test(runtime):=20M14=20=E2=80=94=20module?= =?UTF-8?q?=20crash-consistency=20over=20FileStorage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds tests/runtime/moduleCrashConsistency.test.ts, the module analog of the book crashConsistency suite: a durable single-node cluster (RaftNode + LocalTransport + real FileStorage over a temp dir + ModuleStateMachine) is restarted by rebuilding a fresh node on the same data dir, proving module state survives: - log-replay restart (no snapshot): whole-state + keyed module state, the leader-resolved note id/createdAt (deterministic MODULE replay), and the Merkle audit root all round-trip; - snapshot + restart: state reconstructed from the snapshot (keyed StateStore dump + __audit leaves round-trip); - snapshot-then-tail: restore from snapshot + replay the post-boundary tail, exact arithmetic, no double-apply or loss. No bugs found — module state, the keyed store, and the audit root all survive a real durable restart; no source changes needed. The reconcileLog "snapshot landed, log didn't" path is application-agnostic (book suite covers it; modules ride the same RSM snapshot). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01NneoGGKxBp5ZExL7D7i1Lu --- tests/runtime/moduleCrashConsistency.test.ts | 245 +++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100644 tests/runtime/moduleCrashConsistency.test.ts diff --git a/tests/runtime/moduleCrashConsistency.test.ts b/tests/runtime/moduleCrashConsistency.test.ts new file mode 100644 index 0000000..6831d3c --- /dev/null +++ b/tests/runtime/moduleCrashConsistency.test.ts @@ -0,0 +1,245 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import { RaftNode } from '../../src/consensus/raftNode'; +import { LocalTransport, RpcHandler } from '../../src/consensus/transport'; +import { FileStorage } from '../../src/consensus/storage'; +import { CommandMeta, PeerInfo } from '../../src/consensus/types'; +import { buildModuleCommand } from '../../src/runtime/command'; +import { ModuleStateMachine, ModuleNode } from '../../src/runtime/moduleStateMachine'; +import { counter } from '../../src/runtime/modules/counter'; +import { notes } from '../../src/runtime/modules/notes'; +import { accounts } from '../../src/runtime/modules/accounts'; +import { waitFor } from '../helpers'; + +/** + * Milestone 14: prove the module runtime (ADR-0019) survives a REAL durable + * restart through {@link FileStorage} — the module analog of the book + * `crashConsistency` suite. We wire a single-node cluster directly (a single node + * is its own majority, so commits land immediately) with `LocalTransport` + + * `FileStorage` over a temp data dir + a {@link ModuleStateMachine}, exactly as + * `crashConsistency.test.ts` wires a durable book node. + * + * "Restart" = stop the node, then construct a FRESH `RaftNode` with the SAME + * `nodeId`, the SAME data dir, and a FRESH `ModuleStateMachine` re-registered with + * the SAME modules, and `start()` it (which restores from disk via the snapshot + * and/or the persisted log). We assert that whole-state module data, keyed-store + * records, the leader-resolved seed values, AND the Merkle audit root all + * round-trip across the restart via three durable paths: log replay (no snapshot), + * snapshot+restore, and snapshot-then-tail compaction. + */ + +const TIMERS = { electionMinMs: 50, electionMaxMs: 100, heartbeatMs: 20 }; +const MODULES = [counter, notes, accounts]; + +/** + * Build a durable single-node module cluster: `LocalTransport` (no sockets) + + * `FileStorage` over `dataDir` + a `ModuleStateMachine` registered with the demo + * modules. The node is its own sole voting member, so it elects itself and + * commits writes immediately. A fresh `ModuleHost`/state machine is created each + * call, so calling this twice against the same `dataDir` models a process restart. + */ +/** Nodes created in a test, stopped in teardown so no timer outlives the temp dir. */ +let openNodes: ModuleNode[] = []; + +function makeModuleNode(dataDir: string, snapshotThreshold: number): ModuleNode { + const registry = new Map(); + const peers: PeerInfo[] = []; // single-node cluster: only self + const sm = new ModuleStateMachine(); + sm.host.registerModules(MODULES); + const node = new RaftNode( + { id: 'n1', peers, stateMachine: sm, ...TIMERS, snapshotThreshold, storage: new FileStorage('n1', dataDir) }, + new LocalTransport(registry), + ); + registry.set('n1', node); + openNodes.push(node); + return node; +} + +/** Submit a module command and await its commit; `requestId` doubles as the idempotency key. */ +function submit(node: ModuleNode, module: string, command: string, input: unknown, requestId: string) { + const meta: CommandMeta = { requestId, actor: 'tester', timestamp: 't' }; + return node.submit(buildModuleCommand(module, command, input, meta), meta); +} + +/** + * Snapshot the observable runtime state we expect to survive a restart: every + * whole-state blob, every keyed store dump, the outbox, and the Merkle audit root. + * Compared by JSON equality before vs after the restart. + */ +function captureState(node: ModuleNode) { + const host = node.app.host; + return { + counter: host.query('counter', 'value'), + notes: JSON.stringify(host.getState('notes')), + accounts: JSON.stringify(host.getStore('accounts')!.snapshot()), + auditRoot: host.auditRoot(), + auditSize: host.auditSize(), + }; +} + +describe('Module runtime crash consistency over FileStorage (M14)', () => { + let dir: string; + + beforeEach(() => { + openNodes = []; + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mod-crash-')); + }); + + afterEach(() => { + // Stop every node first (clears its election/heartbeat timers) so nothing + // tries to persist into a directory we are about to delete. Idempotent: + // `stop()` on an already-stopped node is a no-op. + openNodes.forEach((n) => n.stop()); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('replays the durable log on restart with NO snapshot (state + keyed store + audit root survive)', async () => { + // High threshold so the log never compacts: restore is pure log replay. + const node = makeModuleNode(dir, 10_000); + node.start(); + await waitFor(() => node.isLeader()); + + await submit(node, 'counter', 'increment', { by: 7 }, 'inc-1'); + await submit(node, 'notes', 'create', { text: 'first note' }, 'note-1'); + await submit(node, 'notes', 'create', { text: 'second note' }, 'note-2'); + await submit(node, 'accounts', 'open', { id: 'acct-a' }, 'open-a'); + await submit(node, 'accounts', 'deposit', { id: 'acct-a', amount: 250 }, 'dep-a'); + + // No snapshot was taken (the whole history lives in the durable log). + expect(node.status().snapshotIndex).toBe(0); + const before = captureState(node); + node.stop(); + + // Restart: fresh host, same dir + modules. start() replays the log. + const restarted = makeModuleNode(dir, 10_000); + restarted.start(); + // Still no snapshot — proof we exercised the pure log-replay path. + expect(restarted.status().snapshotIndex).toBe(0); + await waitFor(() => restarted.isLeader()); + + const after = captureState(restarted); + // Whole-state module value replays deterministically. + expect(after.counter).toBe(7); + // The notes (including the leader-resolved id/createdAt baked into the + // committed seed) replay byte-identically — proving deterministic replay. + expect(after.notes).toBe(before.notes); + const noteList = restarted.app.host.query('notes', 'list') as Array<{ id: string; createdAt: string }>; + expect(noteList).toHaveLength(2); + // The seed-resolved id/createdAt survived (replayed from the committed seed, + // not regenerated — they are present and exactly the pre-restart values). + expect(noteList[0].id).toBeTruthy(); + expect(noteList[0].createdAt).toBeTruthy(); + // Keyed accounts store + balance survive the replay. + expect(restarted.app.host.query('accounts', 'balance', { id: 'acct-a' })).toBe(250); + expect(after.accounts).toBe(before.accounts); + // The Merkle audit root — derived purely from the applied command stream — + // is identical, proving the audit history round-trips through replay. + expect(after.auditRoot).toBe(before.auditRoot); + expect(after.auditSize).toBe(before.auditSize); + restarted.stop(); + }); + + it('restores from a SNAPSHOT on restart (compacted log; keyed store dump round-trips)', async () => { + // Low threshold so a snapshot IS taken; restore comes from the snapshot. + const node = makeModuleNode(dir, 4); + node.start(); + await waitFor(() => node.isLeader()); + + await submit(node, 'counter', 'increment', { by: 3 }, 'inc-1'); + await submit(node, 'notes', 'create', { text: 'snap note' }, 'note-1'); + await submit(node, 'accounts', 'open', { id: 'acct-x' }, 'open-x'); + await submit(node, 'accounts', 'deposit', { id: 'acct-x', amount: 100 }, 'dep-x'); + await submit(node, 'accounts', 'open', { id: 'acct-y' }, 'open-y'); + await submit(node, 'accounts', 'deposit', { id: 'acct-y', amount: 40 }, 'dep-y'); + await submit(node, 'counter', 'increment', { by: 9 }, 'inc-2'); + + // A snapshot landed and the log compacted past index 0. + await waitFor(() => node.status().snapshotIndex > 0); + const before = captureState(node); + node.stop(); + + const restarted = makeModuleNode(dir, 4); + restarted.start(); + // State machine is reconstructed FROM THE SNAPSHOT immediately on start(), + // before any re-election: the snapshot boundary is preserved. + expect(restarted.status().snapshotIndex).toBeGreaterThan(0); + const after = captureState(restarted); + + expect(after.counter).toBe(12); + expect(restarted.app.host.query('accounts', 'balance', { id: 'acct-x' })).toBe(100); + expect(restarted.app.host.query('accounts', 'balance', { id: 'acct-y' })).toBe(40); + // The keyed StateStore dump (sorted [key,value][]) round-trips through the snapshot. + expect(after.accounts).toBe(before.accounts); + expect(after.notes).toBe(before.notes); + // The Merkle audit root reconstructs from the snapshot's __audit leaves. + expect(after.auditRoot).toBe(before.auditRoot); + expect(after.auditSize).toBe(before.auditSize); + + // The node recovers to full liveness and converges the same state after election. + await waitFor(() => restarted.isLeader()); + expect(restarted.app.host.query('counter', 'value')).toBe(12); + restarted.stop(); + }); + + it('restores from snapshot THEN replays the post-snapshot log tail (no double-apply, no loss)', async () => { + const node = makeModuleNode(dir, 4); + node.start(); + await waitFor(() => node.isLeader()); + + // Enough commands to trigger a snapshot. + await submit(node, 'counter', 'increment', { by: 1 }, 't-1'); + await submit(node, 'accounts', 'open', { id: 'acct-1' }, 't-2'); + await submit(node, 'accounts', 'deposit', { id: 'acct-1', amount: 10 }, 't-3'); + await submit(node, 'counter', 'increment', { by: 1 }, 't-4'); + await submit(node, 'counter', 'increment', { by: 1 }, 't-5'); + await waitFor(() => node.status().snapshotIndex > 0); + + // MORE commands AFTER the snapshot point: these live only in the log tail. + await submit(node, 'counter', 'increment', { by: 1 }, 't-6'); + await submit(node, 'accounts', 'deposit', { id: 'acct-1', amount: 5 }, 't-7'); + await submit(node, 'notes', 'create', { text: 'post-snapshot note' }, 't-8'); + + // The DURABLE snapshot may have advanced as the tail kept compacting; read + // the actual boundary now and confirm a genuine post-snapshot tail exists + // (lastLogIndex past the boundary), so restart exercises replay-on-top. + const snapBoundary = node.status().snapshotIndex; + expect(snapBoundary).toBeGreaterThan(0); + expect(node.status().lastLogIndex).toBeGreaterThan(snapBoundary); + + const before = captureState(node); + node.stop(); + + const restarted = makeModuleNode(dir, 4); + restarted.start(); + // Immediately after start() (before re-election/replay) the boundary equals + // the durable snapshot: state is reconstructed FROM the snapshot first. + expect(restarted.status().snapshotIndex).toBe(snapBoundary); + // The post-snapshot tail is then replayed on top. Final state = snapshot + // state + replayed tail, with no double-apply and no loss. + await waitFor(() => restarted.isLeader()); + const after = captureState(restarted); + + // counter incremented once per 't-1','t-4','t-5','t-6' = 4 (t-2,3,7,8 are non-counter). + expect(after.counter).toBe(4); + // acct-1: deposited 10 (pre-snapshot) + 5 (tail) = 15, NOT double-applied. + expect(restarted.app.host.query('accounts', 'balance', { id: 'acct-1' })).toBe(15); + // The post-snapshot note (tail-only) is present exactly once. + const noteList = restarted.app.host.query('notes', 'list') as unknown[]; + expect(noteList).toHaveLength(1); + // Whole capture (state + keyed store + audit root) matches the pre-restart value. + expect(after.notes).toBe(before.notes); + expect(after.accounts).toBe(before.accounts); + expect(after.auditRoot).toBe(before.auditRoot); + expect(after.auditSize).toBe(before.auditSize); + restarted.stop(); + }); + + // The "snapshot landed but the compacted log didn't" reconcile case is covered + // engine-side by the book `crashConsistency` suite (`reconcileLog`): that path + // lives entirely in the consensus core and is application-agnostic. Module + // nodes ride the SAME RSM snapshot/restore + the SAME `reconcileLog`, so the + // module-specific risk (whole-state + keyed-store + Merkle-audit serialization) + // is what the three tests above prove; we do not re-poke storage internals here. +}); From 3bf38a4eb9a09b091148a06a32407a4184ef5246 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 21 Jun 2026 18:22:06 +0000 Subject: [PATCH 5/8] =?UTF-8?q?feat(observability):=20M15=20=E2=80=94=20ru?= =?UTF-8?q?ntime=20metrics=20for=20the=20module=20runtime?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exposes module-runtime signals over the existing Prometheus /metrics endpoint (per-node, like the Raft gauges), additive and host-metrics-free: - module_commands_total{module,command,status} + module_command_duration_ms, incremented in the ModuleStateMachine adapter (pure observability on the apply path — never touches replicated state/snapshot/audit/outbox; optional, no-op without a registry). - Scrape-time collector: module_outbox_pending/done, module_audit_size, module_registered (read from host accessors; the one host addition is a pure moduleCount() read). - effect_runs_total{kind,outcome} from the EffectExecutor; shard_has_leader{shard} + shard_count from ShardRouter.collectMetrics. - moduleServer wires metrics into the state machine, driver, and a collector. Review fix: outcome is attributed from a try/catch around the handler call ALONE (success XOR failure, counted once); a later submit failure is control flow and no longer double-counts as a handler failure. Adds a unit test proving both the failure path and the success-with-failing-submit case. Label cardinality is bounded (module/command from registered defs, status a small set, kind a handler key) — no requestId/actor/input becomes a label. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01NneoGGKxBp5ZExL7D7i1Lu --- src/moduleServer.ts | 7 +- src/platform/metrics.ts | 19 +++ src/runtime/effectDriver.ts | 5 +- src/runtime/effectExecutor.ts | 33 +++- src/runtime/moduleHost.ts | 9 + src/runtime/moduleStateMachine.ts | 78 +++++++-- src/runtime/shardRouter.ts | 14 ++ tests/runtime/metrics.test.ts | 265 ++++++++++++++++++++++++++++++ 8 files changed, 405 insertions(+), 25 deletions(-) create mode 100644 tests/runtime/metrics.test.ts diff --git a/src/moduleServer.ts b/src/moduleServer.ts index fe7e5eb..d76dd19 100644 --- a/src/moduleServer.ts +++ b/src/moduleServer.ts @@ -25,7 +25,7 @@ const storage = new FileStorage(config.id, process.env.DATA_DIR || './data'); // wiring `BookStateMachine`). Register a demo module set; because modules are // registered identically on every node BEFORE start, MODULE commands apply // deterministically and the cluster converges. -const stateMachine = new ModuleStateMachine(); +const stateMachine = new ModuleStateMachine(undefined, metrics); // A `Reducer` is INVARIANT in its state type `S` (it both consumes and // produces `S`), so a strongly-typed `ModuleDefinition` is not // assignable to the host's erased `ModuleDefinition` slot — even though @@ -45,8 +45,9 @@ const node: ModuleNode = new RaftNode( new HttpTransport(), ); -// Refresh Raft gauges whenever /metrics is scraped. +// Refresh Raft + module-runtime gauges whenever /metrics is scraped (M15). metrics.registerCollector(() => node.collectMetrics()); +metrics.registerCollector(() => stateMachine.collectMetrics(metrics)); const app = createModuleApp(node, { logger, metrics }); const PORT = getPort(); @@ -62,7 +63,7 @@ const effectHandlers: Record = { return { orderId, ok: true }; }, }; -const effectDriver = new EffectDriver(node, effectHandlers); +const effectDriver = new EffectDriver(node, effectHandlers, { metrics }); const server = app.listen(PORT, () => { logger.info('module node started', { node: config.id, port: PORT, peers: config.peers.length }); diff --git a/src/platform/metrics.ts b/src/platform/metrics.ts index 238b174..a849e24 100644 --- a/src/platform/metrics.ts +++ b/src/platform/metrics.ts @@ -117,12 +117,31 @@ export class MetricsRegistry { readonly raftClusterSize = new Gauge('raft_cluster_size', 'Voting members in the current cluster configuration'); readonly stateMachineEntries = new Gauge('state_machine_entries', 'Entries currently in the application state machine'); + // --- Module runtime observability (Milestone 15, ADR-0019). These mirror the + // Raft gauges above but for the module runtime: command throughput, outbox/audit + // depth, effect execution, and shard leadership. Per-node, like the Raft series. + readonly moduleCommands = new Counter('module_commands_total', 'Module commands applied, by module/command/status'); + readonly moduleCommandDuration = new Histogram( + 'module_command_duration_ms', + 'Module command host-apply duration in milliseconds', + [0.5, 1, 5, 10, 25, 50, 100, 250, 500, 1000], + ); + readonly moduleOutboxPending = new Gauge('module_outbox_pending', 'Outbox effect intents still pending execution'); + readonly moduleOutboxDone = new Gauge('module_outbox_done', 'Outbox effect intents already executed (done)'); + readonly moduleAuditSize = new Gauge('module_audit_size', 'Audited module commands (Merkle audit leaf count)'); + readonly moduleRegistered = new Gauge('module_registered', 'Modules registered in the runtime host'); + readonly effectRuns = new Counter('effect_runs_total', 'Effect handler invocations, by kind and outcome'); + readonly shardHasLeader = new Gauge('shard_has_leader', '1 if the shard currently has a known leader, else 0'); + readonly shardCount = new Gauge('shard_count', 'Number of shards in the router'); + private collectors: Collector[] = []; private metrics = [ this.httpRequests, this.httpDuration, this.raftElections, this.raftReadBarriers, this.raftTerm, this.raftIsLeader, this.raftCommitIndex, this.raftLastApplied, this.raftLogLength, this.raftSnapshotIndex, this.raftReplicationLag, this.raftDedupCacheSize, this.raftClusterSize, this.stateMachineEntries, + this.moduleCommands, this.moduleCommandDuration, this.moduleOutboxPending, this.moduleOutboxDone, + this.moduleAuditSize, this.moduleRegistered, this.effectRuns, this.shardHasLeader, this.shardCount, ]; registerCollector(c: Collector): void { diff --git a/src/runtime/effectDriver.ts b/src/runtime/effectDriver.ts index af5044a..5ddcdc2 100644 --- a/src/runtime/effectDriver.ts +++ b/src/runtime/effectDriver.ts @@ -1,6 +1,7 @@ import { CommandMeta } from '../consensus/types'; import { NotLeaderError } from '../consensus/raftNode'; import { ConsensusOf } from '../consensus/consensus'; +import { MetricsRegistry } from '../platform/metrics'; import { buildEffectResultCommand } from './command'; import { EffectExecutor } from './effectExecutor'; import { ModuleStateMachine } from './moduleStateMachine'; @@ -47,7 +48,7 @@ export class EffectDriver { // `Consensus` drives effects unchanged. private readonly node: ConsensusOf, handlers: Record, - opts: { intervalMs?: number } = {}, + opts: { intervalMs?: number; metrics?: MetricsRegistry } = {}, ) { this.intervalMs = opts.intervalMs ?? DEFAULT_INTERVAL_MS; // The executor's `submit` rides the resolved result back through the log. @@ -67,7 +68,7 @@ export class EffectDriver { if (err instanceof NotLeaderError) return; throw err; } - }); + }, opts.metrics); } /** diff --git a/src/runtime/effectExecutor.ts b/src/runtime/effectExecutor.ts index 75311e3..95413fd 100644 --- a/src/runtime/effectExecutor.ts +++ b/src/runtime/effectExecutor.ts @@ -1,3 +1,4 @@ +import { MetricsRegistry } from '../platform/metrics'; import { resolveSeed } from './context'; import { EffectHandler, EffectIntent, EffectResultEntry } from './types'; @@ -28,6 +29,10 @@ export class EffectExecutor { constructor( private readonly handlers: Record, private readonly submit: (entry: EffectResultEntry) => void | Promise, + // Optional metrics sink (Milestone 15). When set, each handler invocation + // increments `effect_runs_total{kind,outcome}`. Pure edge-side observability + // (the executor is already the non-deterministic edge), no-op when undefined. + private readonly metrics?: MetricsRegistry, ) {} /** @@ -59,18 +64,34 @@ export class EffectExecutor { this.inFlight.add(key); try { - const result = await handler(intent); + // Attribute the handler OUTCOME from a try/catch around the handler call + // ALONE — counted exactly once (success XOR failure). A later `submit` + // failure is control flow, not a handler failure, so it must not also + // increment `failure` (which would double-count this invocation). + let result: unknown; + try { + result = await handler(intent); + } catch { + // Handler failed (e.g. network error): leave the effect pending so + // the next drain retries. Submitting nothing keeps the outbox entry + // `pending`, preserving at-least-once handler semantics. + this.metrics?.effectRuns.inc({ kind: intent.kind, outcome: 'failure' }); + return; + } + this.metrics?.effectRuns.inc({ kind: intent.kind, outcome: 'success' }); // The seed is resolved once on the edge by the executor, then committed // verbatim so every replica applies the same value (convergence comes // from committing the value, not from where it was generated). That // deterministic seed keeps any consuming `onResult` reducer's `ctx` // identical on every replica. const entry: EffectResultEntry = { idempotencyKey: key, result, seed: resolveSeed() }; - await this.submit(entry); - } catch { - // Handler failed (e.g. network error): swallow and leave the effect - // pending so the next drain retries. Submitting nothing keeps the - // outbox entry `pending`, preserving at-least-once handler semantics. + try { + await this.submit(entry); + } catch { + // Submit failed (lost leadership / transport): no result committed, + // so the outbox entry stays `pending` and a later drain retries. The + // handler already succeeded — do NOT re-attribute this as a failure. + } } finally { this.inFlight.delete(key); } diff --git a/src/runtime/moduleHost.ts b/src/runtime/moduleHost.ts index c063441..08a0b7d 100644 --- a/src/runtime/moduleHost.ts +++ b/src/runtime/moduleHost.ts @@ -784,6 +784,15 @@ export class ModuleHost { return this.audit.size(); } + /** + * Number of registered modules. A pure read (no state/audit/outbox touched), + * used only by the runtime metrics collector (Milestone 15) to report the + * `module_registered` gauge — observability, never on the apply/convergence path. + */ + moduleCount(): number { + return this.modules.size; + } + /** * The code-version hash recorded for a module (or `undefined` if the module * is not registered). The same value every audited leaf from that module diff --git a/src/runtime/moduleStateMachine.ts b/src/runtime/moduleStateMachine.ts index 9ff5b56..788b683 100644 --- a/src/runtime/moduleStateMachine.ts +++ b/src/runtime/moduleStateMachine.ts @@ -1,6 +1,7 @@ import { RaftNode } from '../consensus/raftNode'; import { StateMachine } from '../consensus/stateMachine'; import { ApplyResult } from '../consensus/types'; +import { MetricsRegistry } from '../platform/metrics'; import { ModuleHost } from './moduleHost'; import { ModuleAppCommand } from './types'; @@ -29,9 +30,20 @@ import { ModuleAppCommand } from './types'; export class ModuleStateMachine implements StateMachine { /** The wrapped runtime — use it to register modules/keys and to read state. */ readonly host: ModuleHost; + /** + * Optional metrics sink (Milestone 15). When set, the adapter records command + * throughput/latency on the apply path and exposes a scrape-time collector for + * the outbox/audit/module gauges. PURE OBSERVABILITY: a counter incremented on + * `apply` is identical on every replica (the same committed command stream), + * touches NO replicated state/snapshot/audit, and is fully optional — an + * undefined registry is a no-op, so `buildModuleCluster` (no metrics) is + * unaffected and determinism/convergence are untouched. + */ + private readonly metrics?: MetricsRegistry; - constructor(host: ModuleHost = new ModuleHost()) { + constructor(host: ModuleHost = new ModuleHost(), metrics?: MetricsRegistry) { this.host = host; + this.metrics = metrics; } apply(command: ModuleAppCommand): ApplyResult { @@ -39,29 +51,67 @@ export class ModuleStateMachine implements StateMachine sm.collectMetrics(m))`. + */ + collectMetrics(metrics: MetricsRegistry): void { + let pending = 0; + let done = 0; + for (const entry of this.host.getOutbox()) { + if (entry.status === 'done') done += 1; + else pending += 1; + } + // Single-series gauges (no label dimension), so — unlike raft's per-peer + // labelled gauges — they need no `reset()`: each scrape overwrites the one + // series with the current count. + metrics.moduleOutboxPending.set(pending); + metrics.moduleOutboxDone.set(done); + metrics.moduleAuditSize.set(this.host.auditSize()); + metrics.moduleRegistered.set(this.host.moduleCount()); + } + snapshot(): unknown { return this.host.snapshot(); } diff --git a/src/runtime/shardRouter.ts b/src/runtime/shardRouter.ts index 483eada..646402e 100644 --- a/src/runtime/shardRouter.ts +++ b/src/runtime/shardRouter.ts @@ -1,5 +1,6 @@ import { createHash } from 'crypto'; import { AppCommand, ApplyResult, CommandMeta } from '../consensus/types'; +import { MetricsRegistry } from '../platform/metrics'; /** * The minimal node surface the router needs: ask who leads, and submit to it. @@ -114,4 +115,17 @@ export class ShardRouter { } return leader.submit(command, meta); } + + /** + * Push scrape-time shard gauges into `metrics` (Milestone 15) — the sharding + * analog of `RaftNode.collectMetrics()`. Sets `shard_count` and, per shard, + * `shard_has_leader{shard}` = 1 when a leader is currently known, else 0. Read + * only (derived live from each group's leadership, the router caches nothing). + */ + collectMetrics(metrics: MetricsRegistry): void { + metrics.shardCount.set(this.shards.length); + for (let i = 0; i < this.shards.length; i++) { + metrics.shardHasLeader.set(this.leaderOf(i) ? 1 : 0, { shard: i }); + } + } } diff --git a/tests/runtime/metrics.test.ts b/tests/runtime/metrics.test.ts new file mode 100644 index 0000000..c9c9023 --- /dev/null +++ b/tests/runtime/metrics.test.ts @@ -0,0 +1,265 @@ +import { RaftNode } from '../../src/consensus/raftNode'; +import { LocalTransport, RpcHandler } from '../../src/consensus/transport'; +import { CommandMeta, PeerInfo } from '../../src/consensus/types'; +import { MetricsRegistry } from '../../src/platform/metrics'; +import { buildModuleCommand } from '../../src/runtime/command'; +import { EffectDriver } from '../../src/runtime/effectDriver'; +import { EffectExecutor } from '../../src/runtime/effectExecutor'; +import { ModuleHost } from '../../src/runtime/moduleHost'; +import { ModuleStateMachine, ModuleNode } from '../../src/runtime/moduleStateMachine'; +import { counter } from '../../src/runtime/modules/counter'; +import { payments } from '../../src/runtime/modules/payments'; +import { ShardRouter } from '../../src/runtime/shardRouter'; +import { EffectHandler, ModuleAppCommand } from '../../src/runtime/types'; +import { AnyModuleDefinition } from '../../src/runtime/moduleHost'; +import { buildModuleCluster, leaders, waitFor } from '../helpers'; + +/** + * Milestone 15: runtime observability over the existing Prometheus registry. These + * tests assert the module-runtime series surface by parsing `metrics.expose()` text + * (the same style as `tests/platform.test.ts`), exercise the adapter/driver/scrape + * collectors, and confirm a metrics-less runtime is a pure no-op. + */ + +const TEST_TIMERS = { electionMinMs: 50, electionMaxMs: 100, heartbeatMs: 20 }; + +/** A single-node module cluster whose ModuleStateMachine has `metrics` wired. */ +function buildMeteredNode( + modules: AnyModuleDefinition[], + metrics: MetricsRegistry, +): ModuleNode { + const registry = new Map(); + const transport = new LocalTransport(registry, 1); + const peers: PeerInfo[] = []; + const sm = new ModuleStateMachine(new ModuleHost(), metrics); + sm.host.registerModules(modules); + const node: ModuleNode = new RaftNode( + { id: 'node1', peers, stateMachine: sm, ...TEST_TIMERS }, + transport, + ); + registry.set(node.id, node); + metrics.registerCollector(() => sm.collectMetrics(metrics)); + return node; +} + +const meta = (requestId: string, actor = 'tester'): CommandMeta => ({ + requestId, + actor, + timestamp: new Date().toISOString(), +}); + +/** Parse a single Prometheus sample line's value by exact `name{labels}` prefix. */ +function sampleValue(text: string, prefix: string): number | undefined { + const line = text.split('\n').find((l) => l.startsWith(prefix)); + if (!line) return undefined; + return Number(line.slice(line.lastIndexOf(' ') + 1)); +} + +describe('Module runtime metrics (Milestone 15)', () => { + describe('command throughput by status (apply path)', () => { + let node: ModuleNode; + let metrics: MetricsRegistry; + + beforeEach(async () => { + metrics = new MetricsRegistry(); + node = buildMeteredNode([counter], metrics); + node.start(); + await waitFor(() => node.isLeader()); + }); + + afterEach(() => node.stop()); + + it('counts 200s by module/command and a non-200 under its status label', async () => { + await node.submit(buildModuleCommand('counter', 'increment', { by: 1 }, meta('r1')), meta('r1')); + await node.submit(buildModuleCommand('counter', 'increment', { by: 2 }, meta('r2')), meta('r2')); + // An unknown command resolves with status 404 from the host (not a throw). + await node.submit(buildModuleCommand('counter', 'nope', {}, meta('r3')), meta('r3')); + + const text = metrics.expose(); + expect(text).toContain('module_commands_total'); + expect( + sampleValue(text, 'module_commands_total{command="increment",module="counter",status="200"}'), + ).toBe(2); + expect( + sampleValue(text, 'module_commands_total{command="nope",module="counter",status="404"}'), + ).toBe(1); + // Latency histogram present for the increment command. + expect(text).toContain('module_command_duration_ms_count{command="increment",module="counter"}'); + }); + }); + + describe('scrape-time outbox/audit gauges', () => { + let nodes: ModuleNode[]; + let drivers: EffectDriver[]; + let metrics: MetricsRegistry; + + afterEach(() => { + drivers.forEach((d) => d.stop()); + nodes.forEach((n) => n.stop()); + }); + + it('reflects outbox pending/done and audit size through the collector', async () => { + metrics = new MetricsRegistry(); + // A 3-node cluster whose LEADER's state machine has metrics wired (the + // collector reads the leader's host outbox/audit at scrape time). + const registry = new Map(); + const transport = new LocalTransport(registry, 1); + const ids = ['node1', 'node2', 'node3']; + const machines = new Map(); + nodes = ids.map((id) => { + const peers: PeerInfo[] = ids.filter((p) => p !== id).map((p) => ({ id: p, url: `local://${p}` })); + const sm = new ModuleStateMachine(new ModuleHost(), metrics); + sm.host.registerModules([payments]); + machines.set(id, sm); + const n: ModuleNode = new RaftNode({ id, peers, stateMachine: sm, ...TEST_TIMERS }, transport); + registry.set(id, n); + return n; + }); + const handler: EffectHandler = async (intent) => { + const { orderId } = intent.payload as { orderId: string }; + return { orderId, ok: true }; + }; + drivers = nodes.map((n) => new EffectDriver(n, { http: handler }, { intervalMs: 10, metrics })); + nodes.forEach((n) => n.start()); + + await waitFor(() => leaders(nodes).length === 1); + const leader = leaders(nodes)[0]; + // The scrape collector must read the leader's host, so register it now. + metrics.registerCollector(() => machines.get(leader.id)!.collectMetrics(metrics)); + + const m = meta('charge-1', 'alice'); + const res = await leader.submit(buildModuleCommand('payments', 'charge', { orderId: 'o1', amount: 100 }, m), m); + expect(res.status).toBe(200); + + // After the charge (before the driver settles) the effect is pending. + let text = metrics.expose(); + expect(sampleValue(text, 'module_outbox_pending ')).toBeGreaterThanOrEqual(1); + expect(sampleValue(text, 'module_audit_size ')).toBe(leader.app.host.auditSize()); + expect(sampleValue(text, 'module_registered ')).toBe(1); + + // Start the drivers; the leader drains the effect and `settle` commits. + drivers.forEach((d) => d.start()); + await waitFor(() => leader.app.host.getOutbox()[0]?.status === 'done'); + + text = metrics.expose(); + expect(sampleValue(text, 'module_outbox_done ')).toBeGreaterThanOrEqual(1); + expect(sampleValue(text, 'module_audit_size ')).toBe(leader.app.host.auditSize()); + }); + + it('effect_runs_total{outcome="success"} increments when a handler runs', async () => { + metrics = new MetricsRegistry(); + nodes = buildModuleCluster(3, [payments]); + const handler: EffectHandler = async (intent) => { + const { orderId } = intent.payload as { orderId: string }; + return { orderId, ok: true }; + }; + drivers = nodes.map((n) => new EffectDriver(n, { http: handler }, { intervalMs: 10, metrics })); + nodes.forEach((n) => n.start()); + drivers.forEach((d) => d.start()); + + await waitFor(() => leaders(nodes).length === 1); + const leader = leaders(nodes)[0]; + + const m = meta('charge-1', 'alice'); + await leader.submit(buildModuleCommand('payments', 'charge', { orderId: 'o1', amount: 100 }, m), m); + await waitFor(() => leader.app.host.getOutbox()[0]?.status === 'done'); + + const text = metrics.expose(); + expect(sampleValue(text, 'effect_runs_total{kind="http",outcome="success"}')).toBe(1); + }); + }); + + describe('ShardRouter.collectMetrics', () => { + let shardA: ModuleNode[]; + let shardB: ModuleNode[]; + + afterEach(() => { + [...shardA, ...shardB].forEach((n) => n.stop()); + }); + + it('sets shard_count and shard_has_leader for a 2-shard setup', async () => { + const metrics = new MetricsRegistry(); + shardA = buildModuleCluster(3, [counter]); + shardB = buildModuleCluster(3, [counter]); + [...shardA, ...shardB].forEach((n) => n.start()); + const router = new ShardRouter([{ nodes: shardA }, { nodes: shardB }]); + await waitFor(() => leaders(shardA).length === 1 && leaders(shardB).length === 1); + + router.collectMetrics(metrics); + const text = metrics.expose(); + expect(sampleValue(text, 'shard_count ')).toBe(2); + expect(sampleValue(text, 'shard_has_leader{shard="0"}')).toBe(1); + expect(sampleValue(text, 'shard_has_leader{shard="1"}')).toBe(1); + }); + }); + + describe('metrics-less runtime is a no-op', () => { + it('a ModuleStateMachine with no registry applies commands without throwing', async () => { + const nodes = buildModuleCluster(1, [counter]); + nodes.forEach((n) => n.start()); + try { + await waitFor(() => nodes[0].isLeader()); + const m = meta('r1'); + const res = await nodes[0].submit(buildModuleCommand('counter', 'increment', { by: 5 }, m), m); + expect(res.status).toBe(200); + expect(nodes[0].app.host.query('counter', 'value')).toBe(5); + // No metrics wired and no collectMetrics call: nothing to assert beyond + // the apply path behaving exactly as before. + } finally { + nodes.forEach((n) => n.stop()); + } + }); + + it('attributes handler outcome exactly once: failure on throw, success even if submit fails', async () => { + const intent = { kind: 'http', idempotencyKey: 'k1', payload: {} }; + + // A throwing handler → exactly one `failure`, no `success`. + const failMetrics = new MetricsRegistry(); + const failExec = new EffectExecutor( + { http: async () => { throw new Error('network down'); } }, + async () => {}, + failMetrics, + ); + await failExec.drain([intent]); + const failText = failMetrics.expose(); + expect(sampleValue(failText, 'effect_runs_total{kind="http",outcome="failure"}')).toBe(1); + expect(sampleValue(failText, 'effect_runs_total{kind="http",outcome="success"}')).toBeUndefined(); + + // Handler succeeds but `submit` throws (e.g. lost leadership) → exactly + // one `success`, NO `failure` (submit failure must not double-count). + const okMetrics = new MetricsRegistry(); + const okExec = new EffectExecutor( + { http: async () => ({ ok: true }) }, + async () => { throw new Error('submit boom'); }, + okMetrics, + ); + await okExec.drain([intent]); + const okText = okMetrics.expose(); + expect(sampleValue(okText, 'effect_runs_total{kind="http",outcome="success"}')).toBe(1); + expect(sampleValue(okText, 'effect_runs_total{kind="http",outcome="failure"}')).toBeUndefined(); + }); + + it('an EffectExecutor/driver with no metrics runs handlers normally', async () => { + const nodes = buildModuleCluster(3, [payments]); + const handler: EffectHandler = async (intent) => { + const { orderId } = intent.payload as { orderId: string }; + return { orderId, ok: true }; + }; + // No `metrics` in opts: the driver's executor never touches a registry. + const drivers = nodes.map((n) => new EffectDriver(n, { http: handler }, { intervalMs: 10 })); + nodes.forEach((n) => n.start()); + drivers.forEach((d) => d.start()); + try { + await waitFor(() => leaders(nodes).length === 1); + const leader = leaders(nodes)[0]; + const m = meta('charge-1', 'alice'); + await leader.submit(buildModuleCommand('payments', 'charge', { orderId: 'o1', amount: 1 }, m), m); + await waitFor(() => leader.app.host.getOutbox()[0]?.status === 'done'); + expect(leader.app.host.getOutbox()[0].status).toBe('done'); + } finally { + drivers.forEach((d) => d.stop()); + nodes.forEach((n) => n.stop()); + } + }); + }); +}); From c9f54227c3cc1a356d6385cc0419a95b56934052 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 21 Jun 2026 18:39:48 +0000 Subject: [PATCH 6/8] =?UTF-8?q?feat(consensus):=20M16=20=E2=80=94=20chunke?= =?UTF-8?q?d=20InstallSnapshot=20(Raft=20fig.13)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the single-RPC snapshot transfer (a documented POC limitation) with bounded-size chunked streaming, so a far-behind follower is caught up without a single oversized RPC. - Wire (types.ts): InstallSnapshotArgs.data is now a string slice plus offset/done; identity fields ride every chunk. - Leader (sendSnapshot): serializes the durable snapshot's data once, streams snapshotChunkBytes-sized slices sequentially (awaiting each reply), aborts on lost reply / lost leadership / term change, steps down on a higher-term reply, and sets matchIndex/nextIndex only on the final done ack. A snapshot that fits in one chunk sends exactly one done chunk (single-RPC parity). - Follower (handleInstallSnapshot): a per-node reassembly buffer keyed by snapshot identity; contiguous-extend only, else drop-and-ack-without-install (fail closed → leader retries from 0); JSON.parse failure also fails closed. The install logic (restore, indexing, fig.13 tail, members, persist-before-log) is byte-for-byte unchanged — chunking is pure transport framing. - Concurrency: a snapshotInFlight guard stops a heartbeat launching a second overlapping stream to the same peer (now that a stream spans many awaits); cleared in a finally on every exit + on becomeLeader. - Config: SNAPSHOT_CHUNK_BYTES (default 64 KiB). Review confirmed install semantics/indexing unchanged, reassembly fail-closed (no corrupt-install sequence), and the in-flight guard leak-free. NIT docs applied (code-unit offset wording; CFT buffer-bound note). All 188 tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01NneoGGKxBp5ZExL7D7i1Lu --- src/consensus/config.ts | 2 + src/consensus/raftNode.ts | 149 ++++++++++++++++++++++++++++----- src/consensus/types.ts | 16 +++- tests/crashConsistency.test.ts | 4 +- tests/snapshot.test.ts | 90 +++++++++++++++++++- 5 files changed, 236 insertions(+), 25 deletions(-) diff --git a/src/consensus/config.ts b/src/consensus/config.ts index ef96328..b1284e2 100644 --- a/src/consensus/config.ts +++ b/src/consensus/config.ts @@ -14,6 +14,7 @@ export interface NodeEnvOptions { electionMaxMs?: number; heartbeatMs?: number; snapshotThreshold?: number; + snapshotChunkBytes?: number; dedupLimit?: number; } @@ -50,6 +51,7 @@ export function loadRaftConfig(env: NodeJS.ProcessEnv = process.env): NodeEnvOpt electionMaxMs: env.ELECTION_MAX_MS ? Number(env.ELECTION_MAX_MS) : undefined, heartbeatMs: env.HEARTBEAT_MS ? Number(env.HEARTBEAT_MS) : undefined, snapshotThreshold: env.SNAPSHOT_THRESHOLD ? Number(env.SNAPSHOT_THRESHOLD) : undefined, + snapshotChunkBytes: env.SNAPSHOT_CHUNK_BYTES ? Number(env.SNAPSHOT_CHUNK_BYTES) : undefined, dedupLimit: env.DEDUP_LIMIT ? Number(env.DEDUP_LIMIT) : undefined, }; } diff --git a/src/consensus/raftNode.ts b/src/consensus/raftNode.ts index d1734e5..15b3c62 100644 --- a/src/consensus/raftNode.ts +++ b/src/consensus/raftNode.ts @@ -42,6 +42,8 @@ export interface RaftConfig< heartbeatMs?: number; /** Take a snapshot once the in-memory log holds this many entries. */ snapshotThreshold?: number; + /** Max size (in characters) of a single InstallSnapshot chunk on the wire. */ + snapshotChunkBytes?: number; /** Cap on the idempotency dedup cache (remembered requestIds). FIFO eviction. */ dedupLimit?: number; /** Optional observability/durability collaborators (tests omit them). */ @@ -131,6 +133,10 @@ export class RaftNode< // Leader-only volatile state. private nextIndex = new Map(); private matchIndex = new Map(); + // Peers we are currently streaming a (multi-chunk) snapshot to. Prevents a + // heartbeat from starting a second, interleaving stream to the same peer — + // concurrent offset===0 chunks would reset each other's reassembly buffer. + private snapshotInFlight = new Set(); // Timers. private electionTimer: NodeJS.Timeout | null = null; @@ -146,8 +152,20 @@ export class RaftNode< private readonly electionMaxMs: number; private readonly heartbeatMs: number; private readonly snapshotThreshold: number; + private readonly snapshotChunkBytes: number; private running = false; + // Per-node reassembly buffer for an in-flight chunked InstallSnapshot stream. + // Keyed by snapshot identity (lastIncludedIndex+term); reset on a fresh + // offset===0 chunk, a gap, or a mismatched identity (fail closed — the leader + // retries from offset 0). Bounded by the size of one snapshot. + private snapshotBuffer: { + lastIncludedIndex: number; + lastIncludedTerm: number; + nextOffset: number; + data: string; + } | null = null; + constructor(config: RaftConfig, transport: Transport) { this.id = config.id; this.selfPeer = { id: config.id, url: config.selfUrl ?? `local://${config.id}` }; @@ -159,6 +177,7 @@ export class RaftNode< this.electionMaxMs = config.electionMaxMs ?? 300; this.heartbeatMs = config.heartbeatMs ?? 50; this.snapshotThreshold = config.snapshotThreshold ?? 1000; + this.snapshotChunkBytes = config.snapshotChunkBytes ?? 64 * 1024; this.app = config.stateMachine; this.rsm = new ReplicatedStateMachine(this.app, config.dedupLimit); // Bootstrap configuration: the peers from env plus this node itself. @@ -570,6 +589,7 @@ export class RaftNode< const nextIdx = this.lastLogIndex() + 1; this.nextIndex.clear(); this.matchIndex.clear(); + this.snapshotInFlight.clear(); for (const peer of this.otherMembers()) { this.nextIndex.set(peer.id, nextIdx); this.matchIndex.set(peer.id, 0); @@ -669,6 +689,9 @@ export class RaftNode< // The entry the follower needs has been compacted away — ship a snapshot. if (nextIdx <= this.lastIncludedIndex) { + // Skip if a snapshot stream to this peer is already in progress, so a + // heartbeat can't start an interleaving second stream from offset 0. + if (this.snapshotInFlight.has(peer.id)) return true; return this.sendSnapshot(peer); } @@ -730,26 +753,63 @@ export class RaftNode< // snapshot exists (nextIndex <= lastIncludedIndex >= 1). const durable = this.storage.loadSnapshot(); const snapIndex = durable?.lastIncludedIndex ?? this.lastIncludedIndex; - const args: InstallSnapshotArgs = { - term, - leaderId: this.id, + const lastIncludedTerm = durable?.lastIncludedTerm ?? this.lastIncludedTerm; + const members = durable?.members ?? this.configAt(snapIndex); + const data = durable?.data ?? this.rsm.snapshot(); + + // Serialize the snapshot's data ONCE, then stream byte/character slices. + // Reassembled on the follower this yields a byte-identical string, so + // JSON.parse recovers exactly the durable snapshot object. + const serialized = JSON.stringify(data); + const chunkSize = this.snapshotChunkBytes; + this.logger?.info('sending snapshot', { + peer: peer.id, lastIncludedIndex: snapIndex, - lastIncludedTerm: durable?.lastIncludedTerm ?? this.lastIncludedTerm, - members: durable?.members ?? this.configAt(snapIndex), - data: durable?.data ?? this.rsm.snapshot(), - }; - this.logger?.info('sending snapshot', { peer: peer.id, lastIncludedIndex: snapIndex }); + bytes: serialized.length, + }); - const reply = await this.transport.sendInstallSnapshot(peer, args); - if (!reply || this.role !== 'leader' || this.currentTerm !== term) return false; - if (reply.term > this.currentTerm) { - this.becomeFollower(reply.term); - return false; + // Send chunks SEQUENTIALLY, awaiting each reply before the next so the + // follower reassembles in order. A snapshot whose serialized data fits in + // one chunk sends exactly one `done` chunk (parity with the single RPC). + // `snapshotInFlight` prevents a heartbeat from launching a second, + // interleaving stream to the same peer while this one is mid-flight. + this.snapshotInFlight.add(peer.id); + try { + let offset = 0; + do { + const slice = serialized.slice(offset, offset + chunkSize); + const done = offset + slice.length >= serialized.length; + const args: InstallSnapshotArgs = { + term, + leaderId: this.id, + lastIncludedIndex: snapIndex, + lastIncludedTerm, + members, + offset, + data: slice, + done, + }; + + const reply = await this.transport.sendInstallSnapshot(peer, args); + // Abort if the reply is lost, we lost leadership, or the term moved + // on mid-stream — leader (or its successor) restarts from offset 0. + if (!reply || this.role !== 'leader' || this.currentTerm !== term) return false; + if (reply.term > this.currentTerm) { + this.becomeFollower(reply.term); + return false; + } + + if (done) { + this.matchIndex.set(peer.id, snapIndex); + this.nextIndex.set(peer.id, snapIndex + 1); + this.advanceCommitIndex(); + return true; + } + offset += slice.length; + } while (true); + } finally { + this.snapshotInFlight.delete(peer.id); } - this.matchIndex.set(peer.id, snapIndex); - this.nextIndex.set(peer.id, snapIndex + 1); - this.advanceCommitIndex(); - return true; } handleAppendEntries(args: AppendEntriesArgs): AppendEntriesReply { @@ -829,10 +889,61 @@ export class RaftNode< // Ignore a snapshot we've already covered (don't roll back our own state). if (args.lastIncludedIndex <= this.lastIncludedIndex || args.lastIncludedIndex <= this.commitIndex) { + this.snapshotBuffer = null; + return { term: this.currentTerm }; + } + + // --- Chunk reassembly (Raft figure 13) --- + // A new offset===0 chunk starts (or restarts) a buffer, superseding any + // partial stream. Subsequent chunks must contiguously extend the buffer for + // the SAME snapshot identity; any gap, mismatched identity, or out-of-order + // offset is rejected — we drop the buffer and reply success without + // installing, so the leader retries cleanly from offset 0 (fail closed: + // never install a corrupt, partially-reassembled snapshot). + // The buffer is bounded by the size of one legitimate snapshot: under the + // CFT (non-Byzantine) trust model the leader is honest, so it always sends + // `done` and never streams unbounded chunks. A Byzantine leader is out of + // scope (ADR-0021). + if (args.offset === 0) { + this.snapshotBuffer = { + lastIncludedIndex: args.lastIncludedIndex, + lastIncludedTerm: args.lastIncludedTerm, + nextOffset: 0, + data: '', + }; + } + const buf = this.snapshotBuffer; + if ( + !buf || + buf.lastIncludedIndex !== args.lastIncludedIndex || + buf.lastIncludedTerm !== args.lastIncludedTerm || + buf.nextOffset !== args.offset + ) { + // Out-of-order / mismatched chunk: discard the partial buffer. + this.snapshotBuffer = null; + return { term: this.currentTerm }; + } + buf.data += args.data; + buf.nextOffset += args.data.length; + + // A non-final chunk is acked without installing. + if (!args.done) { + return { term: this.currentTerm }; + } + + // Final chunk: reassembly complete. Parse the full string back into the + // snapshot data object. A corrupt reassembly (which JSON.parse rejects) + // resets the buffer and acks without installing, so the leader retries. + let snapshotData: RsmSnapshot; + try { + snapshotData = JSON.parse(buf.data) as RsmSnapshot; + } catch { + this.snapshotBuffer = null; return { term: this.currentTerm }; } + this.snapshotBuffer = null; - this.rsm.restore(args.data as RsmSnapshot); + this.rsm.restore(snapshotData); // Raft figure 13, step 6: if we already have the entry at the snapshot's // last-included index with a matching term, the snapshot is just a prefix @@ -859,7 +970,7 @@ export class RaftNode< lastIncludedIndex: this.lastIncludedIndex, lastIncludedTerm: this.lastIncludedTerm, members: args.members, - data: args.data, + data: snapshotData, }); this.persist(); this.logger?.info('installed snapshot', { lastIncludedIndex: this.lastIncludedIndex, keptTail: tail.length }); diff --git a/src/consensus/types.ts b/src/consensus/types.ts index b9c3611..c3e9b87 100644 --- a/src/consensus/types.ts +++ b/src/consensus/types.ts @@ -126,7 +126,13 @@ export interface AppendEntriesReply { conflictIndex?: number; } -/** Sent by a leader to a follower that has fallen behind the leader's log snapshot. */ +/** + * Sent by a leader to a follower that has fallen behind the leader's log + * snapshot. Snapshots are streamed in bounded-size chunks (Raft figure 13): the + * leader serializes the snapshot's `data` to a JSON string once and ships + * successive slices. `lastIncludedIndex`/`lastIncludedTerm`/`members` ride every + * chunk (the follower uses them when the final chunk arrives). + */ export interface InstallSnapshotArgs { term: number; leaderId: string; @@ -134,8 +140,12 @@ export interface InstallSnapshotArgs { lastIncludedTerm: number; /** Cluster configuration as of the snapshot point (membership survives compaction). */ members: PeerInfo[]; - /** Serialized state-machine snapshot (opaque to the transport). */ - data: unknown; + /** Code-unit (UTF-16) offset of this chunk within the serialized snapshot string. */ + offset: number; + /** A slice of the JSON-serialized state-machine snapshot at `offset`. */ + data: string; + /** True for the final chunk: the follower reassembles and installs on `done`. */ + done: boolean; } export interface InstallSnapshotReply { diff --git a/tests/crashConsistency.test.ts b/tests/crashConsistency.test.ts index 069276d..c057dc5 100644 --- a/tests/crashConsistency.test.ts +++ b/tests/crashConsistency.test.ts @@ -91,7 +91,9 @@ describe('InstallSnapshot follower safety', () => { lastIncludedIndex: 0, // <= our boundary/commit: stale lastIncludedTerm: 0, members: [{ id: 'n1', url: 'local://n1' }, { id: 'n2', url: 'local://n2' }], - data: { books: [], audit: [], seen: [], lastHash: '0'.repeat(64) }, + offset: 0, + data: JSON.stringify({ books: [], audit: [], seen: [], lastHash: '0'.repeat(64) }), + done: true, } as InstallSnapshotArgs); expect(reply.term).toBe(before.term); diff --git a/tests/snapshot.test.ts b/tests/snapshot.test.ts index e5e7cc5..7c2c1b5 100644 --- a/tests/snapshot.test.ts +++ b/tests/snapshot.test.ts @@ -1,7 +1,7 @@ import { RaftNode } from '../src/consensus/raftNode'; import { LocalTransport, RpcHandler } from '../src/consensus/transport'; import { MemoryStorage, RaftStorage } from '../src/consensus/storage'; -import { PeerInfo } from '../src/consensus/types'; +import { InstallSnapshotArgs, PeerInfo } from '../src/consensus/types'; import { buildAddCommand } from '../src/models/book'; import { BookNode, BookStateMachine } from '../src/models/bookStateMachine'; import { waitFor } from './helpers'; @@ -12,7 +12,7 @@ function makeNode( id: string, peerIds: string[], registry: Map, - opts: { snapshotThreshold?: number; storage?: RaftStorage } = {}, + opts: { snapshotThreshold?: number; storage?: RaftStorage; snapshotChunkBytes?: number } = {}, ): BookNode { const peers: PeerInfo[] = peerIds.filter((p) => p !== id).map((p) => ({ id: p, url: `local://${p}` })); return new RaftNode({ id, peers, stateMachine: new BookStateMachine(), ...TIMERS, ...opts }, new LocalTransport(registry)); @@ -69,6 +69,92 @@ describe('Log compaction (snapshotting)', () => { [n1, n2, n3].forEach((n) => n.stop()); }); + it('catches a lagging follower up via a MULTI-CHUNK snapshot', async () => { + const registry = new Map(); + const ids = ['n1', 'n2', 'n3']; + // Tiny chunk size forces the snapshot to span many InstallSnapshot RPCs. + const chunked = { snapshotThreshold: 4, snapshotChunkBytes: 64 }; + const n1 = makeNode('n1', ids, registry, chunked); + const n2 = makeNode('n2', ids, registry, chunked); + const n3 = makeNode('n3', ids, registry, chunked); + + registry.set('n1', n1); + registry.set('n2', n2); + n1.start(); + n2.start(); + await waitFor(() => [n1, n2].some((n) => n.isLeader())); + const leader = [n1, n2].find((n) => n.isLeader())!; + + for (let i = 0; i < 12; i++) await add(leader, `multi-${i}`); + expect(leader.status().snapshotIndex).toBeGreaterThan(0); + + // n3 comes online far behind; it must reassemble a multi-chunk snapshot. + registry.set('n3', n3); + n3.start(); + + await waitFor(() => n3.stateMachine.size() === 12, 4000); + expect(n3.status().snapshotIndex).toBeGreaterThan(0); + // Converges on commit index and boundary with the leader. + await waitFor(() => n3.status().commitIndex === leader.status().commitIndex, 4000); + expect(n3.status().snapshotIndex).toBe(leader.status().snapshotIndex); + [n1, n2, n3].forEach((n) => n.stop()); + }); + + it('installs cleanly on retry after an interrupted (offset===0-restarted) stream', async () => { + // A transport that drops the leader's snapshot chunks the FIRST time it + // reaches the follower (simulating an interruption mid-stream), then lets + // every subsequent attempt through. The leader retries from offset 0, and + // the follower must discard its stale partial buffer and install cleanly. + let dropFirstStream = true; + let sawPartial = false; + class InterruptingTransport extends LocalTransport { + async sendInstallSnapshot(peer: PeerInfo, args: InstallSnapshotArgs) { + if (dropFirstStream && peer.id === 'n3') { + // Let the first chunk reach the follower (seeding a partial + // buffer), then drop the rest of this stream and disable the + // drop so the next full attempt succeeds. + if (args.offset === 0 && !args.done) { + sawPartial = true; + await super.sendInstallSnapshot(peer, args); + } + dropFirstStream = false; + return null; // "lost" reply — leader aborts and retries + } + return super.sendInstallSnapshot(peer, args); + } + } + + const registry = new Map(); + const ids = ['n1', 'n2', 'n3']; + const chunked = { snapshotThreshold: 4, snapshotChunkBytes: 64 }; + const transport = new InterruptingTransport(registry); + const peersOf = (id: string): PeerInfo[] => + ids.filter((p) => p !== id).map((p) => ({ id: p, url: `local://${p}` })); + const n1 = new RaftNode({ id: 'n1', peers: peersOf('n1'), stateMachine: new BookStateMachine(), ...TIMERS, ...chunked }, transport); + const n2 = new RaftNode({ id: 'n2', peers: peersOf('n2'), stateMachine: new BookStateMachine(), ...TIMERS, ...chunked }, transport); + const n3 = new RaftNode({ id: 'n3', peers: peersOf('n3'), stateMachine: new BookStateMachine(), ...TIMERS, ...chunked }, transport); + + registry.set('n1', n1); + registry.set('n2', n2); + n1.start(); + n2.start(); + await waitFor(() => [n1, n2].some((n) => n.isLeader())); + const leader = [n1, n2].find((n) => n.isLeader())!; + + for (let i = 0; i < 12; i++) await add(leader, `retry-${i}`); + await waitFor(() => leader.status().snapshotIndex > 0); + + // n3 comes online; the first snapshot stream is interrupted, the retry + // installs cleanly with no corrupt partial state. + registry.set('n3', n3); + n3.start(); + + await waitFor(() => n3.stateMachine.size() === 12, 6000); + expect(sawPartial).toBe(true); // we really did interrupt a partial stream + expect(n3.status().snapshotIndex).toBe(leader.status().snapshotIndex); + [n1, n2, n3].forEach((n) => n.stop()); + }); + it('restores from a snapshot after restart', async () => { const storage = new MemoryStorage(); const registry = new Map(); From b535c65190a81469ecdb2bbbe2bc87118ea3d5f9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 21 Jun 2026 18:49:27 +0000 Subject: [PATCH 7/8] =?UTF-8?q?feat(consensus):=20M17=20=E2=80=94=20follow?= =?UTF-8?q?er=20read=20offloading=20via=20ReadIndex=20(Raft=20=C2=A76.4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Linearizable reads can now be served LOCALLY by a follower instead of being forwarded whole to the leader (a documented POC limitation). Additive — the leader readBarrier()/confirmLeadership() paths are byte-for-byte unchanged. - New lightweight ReadIndex RPC (types/transport/raftRoutes): the follower asks the leader to confirm leadership and return a safe read index. - handleReadIndex (leader half exposed as RPC): captures readIndex = commitIndex BEFORE running the existing confirmLeadership heartbeat-quorum round; returns success only if still leader at the same term. Steps down + refuses if the requester's term is higher. - readBarrierLocal (Consensus seam): leader → existing readBarrier; follower → ReadIndex RPC then waitForApplied(readIndex), then serve locally. Fail-closed on every uncertainty (no/unknown leader, null RPC, success:false, higher term, apply timeout, role churn) — never serves stale. - Controllers' strong-read path calls readBarrierLocal and serves locally, keeping the forward/421 fallback. Adds raft_follower_reads_total. Review confirmed: no path serves a non-linearizable read, every uncertainty is fail-closed, the leader barrier is unchanged, and confirmLeadership from within the RPC handler has no re-entrancy/deadlock under LocalTransport. Review NITs applied (requester-term step-down now used; clarifying comments). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01NneoGGKxBp5ZExL7D7i1Lu --- src/consensus/consensus.ts | 11 +++ src/consensus/raftNode.ts | 80 ++++++++++++++++++++ src/consensus/transport.ts | 20 ++++- src/consensus/types.ts | 25 +++++++ src/controllers/bookController.ts | 7 +- src/controllers/moduleController.ts | 7 +- src/platform/metrics.ts | 3 +- src/routes/raftRoutes.ts | 7 ++ tests/crashConsistency.test.ts | 5 ++ tests/followerReads.test.ts | 112 ++++++++++++++++++++++++++++ 10 files changed, 271 insertions(+), 6 deletions(-) create mode 100644 tests/followerReads.test.ts diff --git a/src/consensus/consensus.ts b/src/consensus/consensus.ts index b40b99e..823696e 100644 --- a/src/consensus/consensus.ts +++ b/src/consensus/consensus.ts @@ -50,6 +50,17 @@ export interface Consensus { */ readBarrier(): Promise; + /** + * Linearizable read barrier that can be satisfied on ANY node (Raft §6.4 + * follower read offloading). On the leader it is exactly {@link readBarrier}; + * on a follower it obtains a confirmed ReadIndex from the leader and waits + * until this node has applied through it, so the caller may then serve the + * read from THIS node's local state. Rejects with `NotLeaderError` if no + * confirmed read index can be obtained (no leader / RPC fails / leader can't + * confirm a quorum), so the caller can forward the read like a write. + */ + readBarrierLocal(): Promise; + /** * Add or remove a single voting member (one change at a time). The returned * promise resolves once the change commits; leader-only. diff --git a/src/consensus/raftNode.ts b/src/consensus/raftNode.ts index 15b3c62..5a8de41 100644 --- a/src/consensus/raftNode.ts +++ b/src/consensus/raftNode.ts @@ -17,6 +17,8 @@ import { isConfigCommand, LogEntry, PeerInfo, + ReadIndexArgs, + ReadIndexReply, RequestVoteArgs, RequestVoteReply, Role, @@ -498,6 +500,84 @@ export class RaftNode< await this.waitForApplied(readIndex); } + /** + * Linearizable read barrier that can be satisfied on ANY node, so a strong + * read arriving at a FOLLOWER is served LOCALLY rather than forwarded whole + * to the leader (Raft §6.4 ReadIndex, follower read offloading). + * + * - On the leader, this is exactly {@link readBarrier} (unchanged path). + * - On a follower, it obtains a confirmed `readIndex` from the leader via the + * {@link ReadIndexArgs} RPC, then waits until this node has APPLIED through + * that index. Only then may the caller serve from this follower's local + * state, guaranteeing the read reflects every write committed before the + * request. + * + * Fails closed: any uncertainty (no known leader, RPC lost, the leader can't + * confirm a quorum or reports a higher term, or the local apply times out) + * throws {@link NotLeaderError} (or a barrier-timeout error), so the caller + * forwards/421s instead of serving a possibly-stale value. A candidate (no + * known leader) likewise throws. + */ + async readBarrierLocal(): Promise { + if (this.role === 'leader') return this.readBarrier(); + + // Follower (or candidate): we need a confirmed read index from the leader. + const leaderId = this.leaderId; + if (!leaderId || leaderId === this.id) throw new NotLeaderError(leaderId); + const leader = this.members.get(leaderId); + if (!leader) throw new NotLeaderError(leaderId); + + const term = this.currentTerm; + const reply = await this.transport.sendReadIndex(leader, { term }); + // No reply / leader couldn't confirm / a newer term exists — fail closed. + if (!reply || !reply.success || reply.readIndex === undefined) { + if (reply && reply.term > this.currentTerm) this.becomeFollower(reply.term); + throw new NotLeaderError(this.leaderId); + } + if (reply.term > this.currentTerm) { + // The leader has moved to a newer term: our view is stale, don't serve. + this.becomeFollower(reply.term); + throw new NotLeaderError(this.leaderId); + } + // A `reply.term < currentTerm` with success can't be unsafe: handleReadIndex + // steps down (success:false) when the requester's term is higher than its + // own, so a success here means the leader confirmed a quorum at a term >= + // ours — its `readIndex` is a valid lower bound on writes committed before + // this read. No special handling needed for the strict-less-than case. + + this.metrics?.raftFollowerReads.inc({ node: this.id }); + // Block until THIS node has applied through the confirmed read index, so + // the subsequent local read can't observe state older than the barrier. + await this.waitForApplied(reply.readIndex); + } + + /** + * Leader half of the ReadIndex protocol, exposed as an RPC for follower read + * offloading. If we are not the leader, fail closed (`success: false`) so the + * follower forwards. Otherwise capture `readIndex = commitIndex` and run the + * EXISTING heartbeat-quorum {@link confirmLeadership} round; only on a + * confirmed quorum (and still leader at the same term) do we return the index. + * This is precisely the leader half of {@link readBarrier} — the leader's own + * `readBarrier` semantics are unchanged. + */ + async handleReadIndex(args: ReadIndexArgs): Promise { + // A requester at a higher term means a newer term exists somewhere: step + // down (we cannot be a legitimate leader) and refuse to vouch for an index. + if (args.term > this.currentTerm) { + this.becomeFollower(args.term); + return { term: this.currentTerm, success: false }; + } + if (this.role !== 'leader') return { term: this.currentTerm, success: false }; + const readIndex = this.commitIndex; + const term = this.currentTerm; + const confirmed = await this.confirmLeadership(); + if (!confirmed || this.role !== 'leader' || this.currentTerm !== term) { + return { term: this.currentTerm, success: false }; + } + this.metrics?.raftReadBarriers.inc({ node: this.id }); + return { term: this.currentTerm, success: true, readIndex }; + } + /** * Exchange one round of AppendEntries with the peers and report whether a * majority still acknowledge our leadership for the current term. A reply diff --git a/src/consensus/transport.ts b/src/consensus/transport.ts index 7b389e4..01960f8 100644 --- a/src/consensus/transport.ts +++ b/src/consensus/transport.ts @@ -6,6 +6,8 @@ import { InstallSnapshotArgs, InstallSnapshotReply, PeerInfo, + ReadIndexArgs, + ReadIndexReply, RequestVoteArgs, RequestVoteReply, } from './types'; @@ -19,6 +21,7 @@ export interface Transport { sendRequestVote(peer: PeerInfo, args: RequestVoteArgs): Promise; sendAppendEntries(peer: PeerInfo, args: AppendEntriesArgs): Promise; sendInstallSnapshot(peer: PeerInfo, args: InstallSnapshotArgs): Promise; + sendReadIndex(peer: PeerInfo, args: ReadIndexArgs): Promise; } /** What a node must expose so transports can deliver RPCs to it. */ @@ -26,6 +29,7 @@ export interface RpcHandler { handleRequestVote(args: RequestVoteArgs): RequestVoteReply; handleAppendEntries(args: AppendEntriesArgs): AppendEntriesReply; handleInstallSnapshot(args: InstallSnapshotArgs): InstallSnapshotReply; + handleReadIndex(args: ReadIndexArgs): ReadIndexReply | Promise; } /** Minimal JSON POST over Node's http module (no external deps). */ @@ -98,6 +102,16 @@ export class HttpTransport implements Transport { return null; } } + + async sendReadIndex(peer: PeerInfo, args: ReadIndexArgs): Promise { + try { + // The leader runs a heartbeat-confirmation round before replying, so + // allow more time than a single heartbeat RPC. + return await postJson(`${peer.url}/raft/read-index`, args, this.rpcTimeoutMs * 5); + } catch { + return null; + } + } } /** @@ -111,7 +125,7 @@ export class LocalTransport implements Transport { private readonly latencyMs = 1, ) {} - private async deliver(peerId: string, fn: (h: RpcHandler) => T): Promise { + private async deliver(peerId: string, fn: (h: RpcHandler) => T | Promise): Promise { const handler = this.registry.get(peerId); if (!handler) return null; // peer is "down" await new Promise((r) => setTimeout(r, this.latencyMs)); @@ -129,4 +143,8 @@ export class LocalTransport implements Transport { sendInstallSnapshot(peer: PeerInfo, args: InstallSnapshotArgs): Promise { return this.deliver(peer.id, (h) => h.handleInstallSnapshot(args)); } + + sendReadIndex(peer: PeerInfo, args: ReadIndexArgs): Promise { + return this.deliver(peer.id, (h) => h.handleReadIndex(args)); + } } diff --git a/src/consensus/types.ts b/src/consensus/types.ts index c3e9b87..0892843 100644 --- a/src/consensus/types.ts +++ b/src/consensus/types.ts @@ -152,6 +152,31 @@ export interface InstallSnapshotReply { term: number; } +/** + * Lightweight ReadIndex RPC (Raft §6.4): a follower asks the leader to confirm + * its leadership and return a safe read index, so the follower can serve a + * linearizable read LOCALLY (offloading read serving) without forwarding the + * whole read to the leader. The leader half is exactly the leader's + * `readBarrier` (capture commitIndex, confirm a heartbeat quorum) exposed as an + * RPC — see `RaftNode.handleReadIndex` / `RaftNode.readBarrierLocal`. + */ +export interface ReadIndexArgs { + /** + * The requester's current term. If it exceeds the responder's term, a newer + * term exists and the responder steps down and refuses (success: false) rather + * than vouch for a read index it may no longer be entitled to serve. + */ + term: number; +} + +export interface ReadIndexReply { + term: number; + /** True only if the responder is the leader AND confirmed a quorum this round. */ + success: boolean; + /** The confirmed read index (the leader's commitIndex), present on success. */ + readIndex?: number; +} + /** A point-in-time snapshot that replaces the log up to lastIncludedIndex. */ export interface Snapshot { lastIncludedIndex: number; diff --git a/src/controllers/bookController.ts b/src/controllers/bookController.ts index 2511483..9504a36 100644 --- a/src/controllers/bookController.ts +++ b/src/controllers/bookController.ts @@ -58,10 +58,13 @@ export function createBookController(node: BookConsensus) { } }; - // Run `serve` only after the leader's linearizable read barrier resolves. + // Run `serve` only after the linearizable read barrier resolves. On a + // follower this obtains a ReadIndex from the leader and applies through it, + // then serves LOCALLY (no forwarding); only if no confirmed read index can be + // obtained does it fall back to forwarding/421 (fail closed — never stale). const strongRead = async (req: Request, res: Response, serve: () => void): Promise => { try { - await node.readBarrier(); + await node.readBarrierLocal(); serve(); } catch (err) { if (err instanceof NotLeaderError) return onNotLeader(req, res, err); diff --git a/src/controllers/moduleController.ts b/src/controllers/moduleController.ts index f05ea0d..f8ae718 100644 --- a/src/controllers/moduleController.ts +++ b/src/controllers/moduleController.ts @@ -49,10 +49,13 @@ export function createModuleController(node: ModuleConsensus) { res.status(421).json({ message: 'Not the leader — retry against the leader', leader: err.leaderId }); }; - // Run `serve` only after the leader's linearizable read barrier resolves. + // Run `serve` only after the linearizable read barrier resolves. On a + // follower this obtains a ReadIndex from the leader and applies through it, + // then serves LOCALLY (no forwarding); only if no confirmed read index can be + // obtained does it fall back to forwarding/421 (fail closed — never stale). const strongRead = async (req: Request, res: Response, serve: () => void): Promise => { try { - await node.readBarrier(); + await node.readBarrierLocal(); serve(); } catch (err) { if (err instanceof NotLeaderError) return onNotLeader(req, res, err); diff --git a/src/platform/metrics.ts b/src/platform/metrics.ts index a849e24..7c14c4c 100644 --- a/src/platform/metrics.ts +++ b/src/platform/metrics.ts @@ -106,6 +106,7 @@ export class MetricsRegistry { ); readonly raftElections = new Counter('raft_elections_total', 'Elections this node has started'); readonly raftReadBarriers = new Counter('raft_read_barriers_total', 'Linearizable read barriers served as leader'); + readonly raftFollowerReads = new Counter('raft_follower_reads_total', 'Linearizable reads served locally on a follower via a ReadIndex from the leader'); readonly raftTerm = new Gauge('raft_term', 'Current Raft term'); readonly raftIsLeader = new Gauge('raft_is_leader', '1 if this node is the leader'); readonly raftCommitIndex = new Gauge('raft_commit_index', 'Highest committed log index'); @@ -136,7 +137,7 @@ export class MetricsRegistry { private collectors: Collector[] = []; private metrics = [ - this.httpRequests, this.httpDuration, this.raftElections, this.raftReadBarriers, this.raftTerm, + this.httpRequests, this.httpDuration, this.raftElections, this.raftReadBarriers, this.raftFollowerReads, this.raftTerm, this.raftIsLeader, this.raftCommitIndex, this.raftLastApplied, this.raftLogLength, this.raftSnapshotIndex, this.raftReplicationLag, this.raftDedupCacheSize, this.raftClusterSize, this.stateMachineEntries, diff --git a/src/routes/raftRoutes.ts b/src/routes/raftRoutes.ts index d6c26ab..cfb068e 100644 --- a/src/routes/raftRoutes.ts +++ b/src/routes/raftRoutes.ts @@ -31,6 +31,13 @@ export default function raftRoutes { + res.json(await node.handleReadIndex(req.body)); + }); + router.get('/status', (_req, res) => { res.json(node.status()); }); diff --git a/tests/crashConsistency.test.ts b/tests/crashConsistency.test.ts index c057dc5..03c8ec2 100644 --- a/tests/crashConsistency.test.ts +++ b/tests/crashConsistency.test.ts @@ -5,6 +5,8 @@ import { InstallSnapshotArgs, InstallSnapshotReply, PeerInfo, + ReadIndexArgs, + ReadIndexReply, RequestVoteArgs, RequestVoteReply, AppendEntriesArgs, @@ -121,6 +123,9 @@ class CapturingTransport implements Transport { this.sent.push(a); return this.deliver(p.id, (h) => h.handleInstallSnapshot(a)); } + sendReadIndex(p: PeerInfo, a: ReadIndexArgs): Promise { + return this.deliver(p.id, (h) => Promise.resolve(h.handleReadIndex(a))); + } } describe('sendSnapshot ships the durable boundary', () => { diff --git a/tests/followerReads.test.ts b/tests/followerReads.test.ts new file mode 100644 index 0000000..909618c --- /dev/null +++ b/tests/followerReads.test.ts @@ -0,0 +1,112 @@ +import { RaftNode, NotLeaderError } from '../src/consensus/raftNode'; +import { LocalTransport, RpcHandler } from '../src/consensus/transport'; +import { PeerInfo } from '../src/consensus/types'; +import { buildAddCommand } from '../src/models/book'; +import { BookNode, BookStateMachine } from '../src/models/bookStateMachine'; +import { waitFor } from './helpers'; + +const TIMERS = { electionMinMs: 50, electionMaxMs: 100, heartbeatMs: 20 }; + +/** A cluster whose registry the test controls, so peers can be "partitioned". */ +function cluster(size: number) { + const registry = new Map(); + const transport = new LocalTransport(registry, 1); + const ids = Array.from({ length: size }, (_, i) => `node${i + 1}`); + const nodes = ids.map((id) => { + const peers: PeerInfo[] = ids.filter((p) => p !== id).map((p) => ({ id: p, url: `local://${p}` })); + return new RaftNode({ id, peers, stateMachine: new BookStateMachine(), ...TIMERS }, transport); + }); + nodes.forEach((n) => registry.set(n.id, n)); + return { nodes, registry }; +} + +const leaderOf = (nodes: BookNode[]) => nodes.find((n) => n.isLeader())!; +const followersOf = (nodes: BookNode[]) => nodes.filter((n) => !n.isLeader()); + +describe('Follower read offloading (ReadIndex on a follower)', () => { + let nodes: BookNode[]; + let registry: Map; + + beforeEach(async () => { + ({ nodes, registry } = cluster(3)); + nodes.forEach((n) => n.start()); + await waitFor(() => nodes.filter((n) => n.isLeader()).length === 1); + }); + + afterEach(() => nodes.forEach((n) => n.stop())); + + it('serves the latest committed write LOCALLY from a follower (no forwarding)', async () => { + const leader = leaderOf(nodes); + const { data } = await leader.submit( + buildAddCommand({ title: 'F', author: 'O', publisher: 'S', isbn: 'fr-1', copies: 1 }), + ); + + // The write replicates to followers asynchronously; the follower barrier + // is responsible for waiting until this follower has applied through it. + const follower = followersOf(nodes)[0]; + expect(follower.isLeader()).toBe(false); + + await expect(follower.readBarrierLocal()).resolves.toBeUndefined(); + + // Served from THIS follower's own local state — it was never the leader, + // yet it returns the fresh value, proving it obtained a confirmed + // ReadIndex and applied through it. + expect(follower.isLeader()).toBe(false); + expect(follower.app.get(data!.id)).toBeDefined(); + }); + + it('leader path still resolves locally (readBarrierLocal delegates to readBarrier)', async () => { + const leader = leaderOf(nodes); + const { data } = await leader.submit( + buildAddCommand({ title: 'L', author: 'O', publisher: 'S', isbn: 'fr-2', copies: 1 }), + ); + await expect(leader.readBarrierLocal()).resolves.toBeUndefined(); + expect(leader.app.get(data!.id)).toBeDefined(); + }); + + it('a briefly-behind follower waits until applied before serving (no stale read)', async () => { + const leader = leaderOf(nodes); + const follower = followersOf(nodes)[0]; + + // Cut this follower off so it falls behind, commit a write through the + // remaining majority (leader + the other follower), then reconnect. + registry.delete(follower.id); + const { data } = await leader.submit( + buildAddCommand({ title: 'B', author: 'O', publisher: 'S', isbn: 'fr-3', copies: 1 }), + ); + // Confirm the cluster committed it without this follower. + await waitFor(() => leader.app.get(data!.id) !== undefined); + expect(follower.app.get(data!.id)).toBeUndefined(); // still behind + + // Reconnect and issue the strong read at the stale follower. The barrier + // must block until the follower has caught up through the read index. + registry.set(follower.id, follower); + await expect(follower.readBarrierLocal()).resolves.toBeUndefined(); + expect(follower.app.get(data!.id)).toBeDefined(); + }); + + it('fails closed when the leader is unreachable (caller would forward/421)', async () => { + const leader = leaderOf(nodes); + const follower = followersOf(nodes)[0]; + + // Remove the leader from the registry: the follower still thinks `leader` + // is leader, but the ReadIndex RPC now finds no peer (returns null). + registry.delete(leader.id); + + // Fail closed: throw rather than serve a possibly-stale local value. + await expect(follower.readBarrierLocal()).rejects.toBeInstanceOf(NotLeaderError); + }); + + it('fails closed on a node with no known leader', async () => { + // A freshly-built, never-started node has leaderId === null: it is not the + // leader and knows of none, so a strong read must throw (the controller + // would 421), never serve from empty/stale local state. + const { nodes: fresh } = cluster(3); + const lone = fresh[0]; + try { + await expect(lone.readBarrierLocal()).rejects.toBeInstanceOf(NotLeaderError); + } finally { + fresh.forEach((n) => n.stop()); + } + }); +}); From 6521a87993d47e951d9cc9a7b23707d3607ba83d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 21 Jun 2026 19:35:41 +0000 Subject: [PATCH 8/8] =?UTF-8?q?feat(consensus):=20M18=20=E2=80=94=20dynami?= =?UTF-8?q?c=20membership=20via=20joint=20consensus=20(Raft=20=C2=A76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the single-server membership changes of ADR-0015 with full joint consensus (Raft §6 / dissertation §4.3), recorded as ADR-0022 (supersedes ADR-0015). A change C-old→C-new now transitions through a joint configuration C-old,new in which EVERY quorum decision — election tally, commit advance, and leadership confirmation — requires a dual majority: a majority of BOTH C-old and C-new separately. Safety is now enforced by the quorum math rather than assumed by a one-node restriction, so even an arbitrary (non-overlapping) change is safe. Core changes (src/consensus/raftNode.ts, types.ts): - One dual-majority predicate `inMajority(ids)` gates election, commit, and leadership confirmation; replication targets the voting UNION C-old ∪ C-new. - Configuration stays derived from the log, adopted on append; `configOld`/ `configNew` carry the two decision sets, `members` the union. - Two-phase `changeMembership`: append joint → await commit → append final → await commit. A leader not in C-new steps down once the final config commits. - A new leader that INHERITS an unfinished joint config completes the transition (`inheritedJoint` → finalize in `applyCommitted`), so a mid-transition crash can't wedge the cluster in joint consensus (Raft §4.3). - Joint configs survive compaction/transfer: `oldMembers` is threaded through `Snapshot` and `InstallSnapshotArgs` so a snapshotting/catching-up node keeps enforcing dual majority instead of collapsing to a simple config. - Guard against starting a second change while a transition is in flight (uncommitted CONFIG or still-joint), and reject an empty target configuration. Tests (tests/membership.test.ts): dual-majority partition test (a C-old-only majority can neither elect nor commit), an arbitrary non-overlapping change, and new-leader finalization of an inherited joint transition. The dual-majority test is made deterministic via asymmetric election timers (n1 the sole initiator), so it no longer flaked under parallel-worker CPU contention. Docs: ADR-0022, ADR-0015 marked superseded, README/CLAUDE membership + limitations updated. Full suite green (196 tests). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01NneoGGKxBp5ZExL7D7i1Lu --- CLAUDE.md | 16 +- README.md | 28 +- docs/adr/0015-dynamic-cluster-membership.md | 2 +- docs/adr/0022-joint-consensus-membership.md | 98 +++++++ docs/adr/README.md | 3 +- src/consensus/raftNode.ts | 293 ++++++++++++++++---- src/consensus/types.ts | 31 ++- tests/membership.test.ts | 262 ++++++++++++++++- 8 files changed, 643 insertions(+), 90 deletions(-) create mode 100644 docs/adr/0022-joint-consensus-membership.md diff --git a/CLAUDE.md b/CLAUDE.md index 042b3e7..d19e9c8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -67,10 +67,18 @@ NODE_ID=node1 PORT=3001 PEERS="node2@http://localhost:3002,node3@http://localhos eventually consistent by default. `?consistency=strong` reads go through the leader's ReadIndex barrier (`node.readBarrier()`) and must never be served from a node that can't confirm leadership — fail closed (`NotLeaderError`) instead. -- **Membership: one change at a time (ADR-0015).** A `CONFIG` entry takes effect - on *append*, not commit; never apply two overlapping changes (the code refuses - a second while one is uncommitted). Membership must be derived from the log so - every replica agrees — don't track it out of band. +- **Membership: joint consensus, one transition at a time (ADR-0022, supersedes + ADR-0015).** A change C-old→C-new goes through a **joint** `CONFIG` (`members`= + C-new, `oldMembers`=C-old) then a **final** `CONFIG` (`members`=C-new). A `CONFIG` + entry takes effect on *append*, not commit. During the joint phase EVERY quorum + decision (election, commit, leadership confirmation) needs a **dual majority** — + a majority of BOTH C-old and C-new separately; route every decision through + `inMajority()`, never a single count. Replicate to the voting **union** + (`members` / `otherMembers()`). Never start a second change while one is in + flight (the code refuses while a `CONFIG` is uncommitted *or* the config is still + joint). Membership must be derived from the log so every replica agrees — don't + track it out of band. A new leader that inherits an uncompleted joint config must + finish the transition (`inheritedJoint` → finalize in `applyCommitted`). ## Conventions diff --git a/README.md b/README.md index 4a8a24d..1ff83f6 100644 --- a/README.md +++ b/README.md @@ -152,11 +152,15 @@ the consensus core, so any app built on it inherits them for free. even if a forward times out and returns `421`. The dedup cache is **bounded** (`DEDUP_LIMIT`, default 10,000) with deterministic FIFO eviction, so it never grows without limit. -- **Dynamic membership** — add or remove nodes at runtime (one at a time, Raft - §4.1) via `POST`/`DELETE /raft/members`, with no cluster restart. The new - configuration replicates as a log entry; a joining node catches up by normal - replication or an InstallSnapshot, and a leader removed from the cluster steps - down. A removed/partitioned node can't disrupt the cluster (leader stickiness). +- **Dynamic membership** — add or remove nodes at runtime via **joint consensus** + (Raft §6 / ADR-0022) through `POST`/`DELETE /raft/members`, with no cluster + restart. A change transitions C-old→C-new through a joint configuration in which + every decision needs a majority of **both** sets (so even an arbitrary, + non-overlapping change is safe); the configuration replicates as a log entry, a + joining node catches up by normal replication or an InstallSnapshot, a leader + removed from the cluster steps down, and a new leader finishes a transition its + predecessor left unfinished. A removed/partitioned node can't disrupt the cluster + (leader stickiness). - **Leader forwarding** + `/ready` for load-balancer health checks. **Audit** — the replicated log *is* the audit trail. Each committed change is @@ -248,13 +252,15 @@ Tests use `LocalTransport`, so they need no sockets and no database. ## Limitations (it's a POC) -- Membership changes are one node at a time (Raft §4.1, no joint consensus) and a - joining node has no separate non-voting catch-up phase, so adding a far-behind - node while another is down can briefly stall commits. +- Membership changes still apply one transition at a time, and a joining node has + no separate non-voting catch-up phase, so adding a far-behind node while another + is down can briefly stall commits. (Changes now use joint consensus — Raft §6 / + ADR-0022 — so the *single-server* restriction is gone; arbitrary changes are + safe.) - Reads default to the local replica (eventually consistent); linearizable reads - are available opt-in via `?consistency=strong` (leader-routed, no follower - read offloading or leases). + are available opt-in via `?consistency=strong`. These can be served by the leader + or offloaded to a follower via a ReadIndex RPC (Raft §6.4), but there are no + time-based leader leases (every barrier confirms a fresh heartbeat quorum). - The audit log grows unbounded (kept in full inside snapshots by design). The idempotency cache is bounded (`DEDUP_LIMIT`, FIFO), so a retry older than the window re-applies rather than being deduped. -- Snapshots are sent in a single RPC (no chunking) — fine for a POC-sized state. diff --git a/docs/adr/0015-dynamic-cluster-membership.md b/docs/adr/0015-dynamic-cluster-membership.md index 52052dd..c7661d5 100644 --- a/docs/adr/0015-dynamic-cluster-membership.md +++ b/docs/adr/0015-dynamic-cluster-membership.md @@ -1,6 +1,6 @@ # 0015. Dynamic cluster membership via single-server changes -- Status: Accepted +- Status: Superseded by [ADR-0022](./0022-joint-consensus-membership.md) - Date: 2026-06-19 ## Context diff --git a/docs/adr/0022-joint-consensus-membership.md b/docs/adr/0022-joint-consensus-membership.md new file mode 100644 index 0000000..e16e3ce --- /dev/null +++ b/docs/adr/0022-joint-consensus-membership.md @@ -0,0 +1,98 @@ +# 0022. Dynamic cluster membership via joint consensus + +- Status: Accepted +- Date: 2026-06-21 +- Supersedes: [ADR-0015](./0015-dynamic-cluster-membership.md) + +## Context + +ADR-0015 implemented runtime membership changes with the Raft dissertation's +**single-server change** approach (§4.1): add or remove exactly one voting member +at a time, relying on the fact that consecutive single-server configurations +always overlap in a majority, so two configurations can never independently elect +a leader or commit conflicting entries. That kept the implementation small, but it +buys safety by *constraint*, not by mechanism: + +- It can only ever express **one-node** changes. Replacing a node is two changes + (add then remove); re-sharding or swapping several nodes is a slow sequence. +- The overlap argument is *assumed* by the single-node restriction rather than + *enforced* by the vote/commit math — nothing in the quorum code would stop an + arbitrary (non-overlapping) change from splitting the log if one were ever + appended. The safety property lived in a comment, not in the predicate. +- Diehl/Ongaro later showed even single-server changes have a subtle corner case + around the initial configuration; the robust, fully-general answer Raft actually + specifies is joint consensus. + +For a project whose whole subject is the consensus layer, the membership change +should be safe by *construction* and able to express arbitrary reconfigurations. + +## Decision + +Implement membership changes using Raft's **joint consensus** (§6 / dissertation +§4.3). A change from configuration `C-old` to `C-new` transitions through a +**joint configuration `C-old,new`** in which every quorum decision requires a +majority of **both** `C-old` and `C-new` *separately*. Two phases, each adopted +on append (not on commit) and each awaited to commit: + +1. Append a **joint** `CONFIG` entry (`members: C-new`, `oldMembers: C-old`); + replicate and await its commit under dual majority. +2. Append a **final** simple `CONFIG` entry (`members: C-new`); replicate and + await its commit. + +`changeMembership` runs both phases and resolves when the final config commits. + +The dual-majority rule is the single gate for **every** quorum decision, so the +safety property is now enforced by the math rather than assumed by a restriction: + +- **One predicate, `inMajority(ids)`** — for a simple config it is the ordinary + `|ids ∩ C-new| ≥ ⌊|C-new|/2⌋ + 1`; for a joint config it additionally requires + `|ids ∩ C-old| ≥ ⌊|C-old|/2⌋ + 1`. Election tally, commit advance, and + leadership confirmation all route through it. During the joint phase no decision + can be carried by a majority of only one configuration — making even an + **arbitrary** change (one whose `C-old` and `C-new` do not overlap in a + majority) safe. +- **Replication is driven by the voting UNION** `C-old ∪ C-new`: the leader + replicates to, and confirms leadership against, every node in either + configuration; the dual-majority predicate then decides. (`members` is the + union; `configOld`/`configNew` are the two decision sets.) +- **Configuration remains derived from the log** (latest `CONFIG` over the + snapshot base) and is adopted the instant an entry is **appended**, exactly as + before — so every replica agrees on the shape without out-of-band tracking. +- **Joint configs survive compaction.** Snapshots and `InstallSnapshot` carry an + optional `oldMembers` alongside `members`, so a node that snapshots, restarts, + or catches up while a transition is in flight reconstructs the joint config and + keeps enforcing dual majority rather than silently collapsing to a simple one. +- **One transition at a time** is still enforced (`hasUncommittedConfig` refuses a + second change while one is uncommitted), and a leader **not in `C-new`** steps + down once the final config commits (leader self-removal). +- If leadership is lost between the two phases, the leader does not append the + final config from a stale term; a crash leaves the joint `CONFIG` in the log, + where the next leader observes it and can drive the transition forward. + +The HTTP surface and `raft_cluster_size` metric are unchanged from ADR-0015. + +## Consequences + +- Membership safety is enforced by the quorum math, not by a one-node restriction: + the dual-majority predicate makes it impossible for `C-old` and `C-new` to elect + leaders or commit entries independently during a transition, for *any* change. +- The API still exposes single add/remove operations (the common case), but the + underlying mechanism now generalizes to arbitrary changes — a follow-up can + expose a batch reconfiguration with no further consensus work. +- More moving parts than single-server changes: two voting sets, a union for + replication, and `oldMembers` threaded through `CONFIG`, `Snapshot`, and + `InstallSnapshot`. This is the price of being correct by construction and is + covered by tests (including a partition test proving a `C-old`-only majority can + neither elect nor commit during the joint phase). +- Only one transition may be in flight at a time, as before. A new voting member + still joins immediately (no non-voting learner phase) — the same documented + catch-up caveat as ADR-0015 applies and remains a future refinement. + +## Alternatives considered + +- **Keep single-server changes (ADR-0015)** — simpler, but limited to one-node + changes and safe only by constraint; the corner cases and the inability to + express arbitrary reconfigurations motivated this supersession. +- **Non-voting learner phase before promotion** — strictly better availability + during catch-up; orthogonal to joint vs. single-server and still deferred as an + optimisation on top of this decision. diff --git a/docs/adr/README.md b/docs/adr/README.md index 54b2e98..2938f7a 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -27,13 +27,14 @@ supersedes the old one rather than editing history. | [0012](./0012-platform-layer.md) | A reusable platform layer for cross-cutting concerns | Accepted | | [0013](./0013-minimal-dependencies.md) | Minimal external dependencies (implement primitives in-house) | Accepted | | [0014](./0014-opt-in-linearizable-reads.md) | Opt-in linearizable reads via ReadIndex | Accepted | -| [0015](./0015-dynamic-cluster-membership.md) | Dynamic cluster membership via single-server changes | Accepted | +| [0015](./0015-dynamic-cluster-membership.md) | Dynamic cluster membership via single-server changes | Superseded by ADR-0022 | | [0016](./0016-crash-consistent-snapshot-persistence.md) | Crash-consistent snapshot/log persistence and snapshot transfer | Accepted | | [0017](./0017-generic-pluggable-state-machine.md) | Generic pluggable state machine (framework core) | Accepted | | [0018](./0018-application-logic-as-smart-contracts.md) | Deploying application logic as smart contracts on this framework | Proposed | | [0019](./0019-native-backend-blockchain-benefits.md) | Bringing blockchain benefits to backend development natively | Proposed | | [0020](./0020-multi-raft-sharding.md) | Multi-Raft sharding for write scaling | Proposed | | [0021](./0021-pluggable-bft-consensus-seam.md) | Pluggable BFT consensus — the seam, and why it is not built | Proposed | +| [0022](./0022-joint-consensus-membership.md) | Dynamic cluster membership via joint consensus | Accepted | ## Template diff --git a/src/consensus/raftNode.ts b/src/consensus/raftNode.ts index 5a8de41..5d648af 100644 --- a/src/consensus/raftNode.ts +++ b/src/consensus/raftNode.ts @@ -70,6 +70,25 @@ export class MembershipError extends Error { } } +/** + * A configuration as derived from a CONFIG log entry (or a snapshot base). + * `members` is C-new (the target voting set). When `oldMembers` is present the + * configuration is **joint** (C-old,new) and every decision needs a majority of + * BOTH sets separately (Raft §6 / ADR-0022). + */ +interface ConfigState { + members: PeerInfo[]; + oldMembers?: PeerInfo[]; +} + +/** True iff `acks` contains a strict majority (floor(n/2)+1) of `config`. */ +function majorityOf(acks: Set, config: Set): boolean { + if (config.size === 0) return true; // an empty config imposes no constraint + let count = 0; + for (const id of config) if (acks.has(id)) count += 1; + return count >= Math.floor(config.size / 2) + 1; +} + interface PendingProposal { term: number; resolve: (result: ApplyResult) => void; @@ -116,13 +135,31 @@ export class RaftNode< private lastIncludedIndex = 0; private lastIncludedTerm = 0; - // Cluster membership (Raft dissertation §4). The configuration lives in the - // log and takes effect the moment an entry is appended (not when committed). - // `members` is the current voting set (including self), derived from the log; - // `baseConfig` is the configuration as of the snapshot boundary, the base the - // log's CONFIG entries are layered on top of. + // Cluster membership (Raft §6, joint consensus; ADR-0022). The configuration + // lives in the log and takes effect the moment an entry is appended (not when + // committed). `members` is the current voting UNION (including self), derived + // from the log — during a joint transition that's C-old ∪ C-new, otherwise + // just C-new; it is the set of replication targets. The majority PREDICATE, + // however, is `inMajority`, which during a joint transition requires a majority + // of C-old AND of C-new separately (the two sets below). `baseConfig` is the + // configuration as of the snapshot boundary — the base the log's CONFIG entries + // are layered on top of; it may itself be joint if the snapshot boundary fell + // inside a transition. private members = new Map(); - private baseConfig: PeerInfo[]; + /** During a joint transition: the two voting sets whose majorities are BOTH required. */ + private configOld: Set | null = null; + private configNew = new Set(); + private baseConfig: ConfigState; + /** + * Set when this node becomes leader while the adopted config is already joint — + * i.e. it INHERITED an in-progress transition a previous leader appended but + * never finished. Such a leader must complete the transition (Raft §4.3) by + * appending the final C-new once the joint config commits (see `applyCommitted`), + * or the cluster stays wedged in joint consensus. A leader that ORIGINATES a + * change (via `changeMembership`) finalizes it itself, so this stays false for + * those — the two finalization paths are mutually exclusive. + */ + private inheritedJoint = false; /** Wall-clock of the last AppendEntries/InstallSnapshot from a current leader. */ private lastLeaderContact = 0; @@ -182,40 +219,79 @@ export class RaftNode< this.snapshotChunkBytes = config.snapshotChunkBytes ?? 64 * 1024; this.app = config.stateMachine; this.rsm = new ReplicatedStateMachine(this.app, config.dedupLimit); - // Bootstrap configuration: the peers from env plus this node itself. - this.baseConfig = [this.selfPeer, ...config.peers]; + // Bootstrap configuration: the peers from env plus this node itself (a + // simple, non-joint config). + this.baseConfig = { members: [this.selfPeer, ...config.peers] }; this.recomputeMembers(); } // ---- cluster membership ---- - /** Other voting members (the current configuration minus this node). */ + /** + * Other voting members (the current voting UNION minus this node) — the set + * of replication targets. During a joint transition this is C-old ∪ C-new, so + * the leader replicates to (and confirms leadership against) every node in + * either configuration; the dual-majority predicate then decides separately. + */ private otherMembers(): PeerInfo[] { return [...this.members.values()].filter((p) => p.id !== this.id); } - /** Votes/acks needed for a decision: a strict majority of the current config. */ - private quorum(): number { - return Math.floor(this.members.size / 2) + 1; + /** + * Dual-majority predicate (Raft §6 / ADR-0022) — the single gate for EVERY + * quorum decision (election tally, commit advance, leadership confirmation): + * + * - Simple config: `|ids ∩ C-new| >= maj(C-new)`. + * - Joint config: `|ids ∩ C-old| >= maj(C-old)` AND `|ids ∩ C-new| >= maj(C-new)`. + * + * During the joint phase this means no decision can be carried by a majority + * of only ONE configuration — the property that makes arbitrary membership + * changes safe (it is impossible for C-old and C-new to independently elect + * leaders or commit conflicting entries). + */ + private inMajority(ids: Iterable): boolean { + const set = ids instanceof Set ? ids : new Set(ids); + if (this.configOld && !majorityOf(set, this.configOld)) return false; + return majorityOf(set, this.configNew); + } + + /** True iff the current configuration is the joint (transitional) C-old,new. */ + private isJoint(): boolean { + return this.configOld !== null; } /** Configuration effective at `absIndex`: baseConfig + CONFIG entries up to it. */ - private configAt(absIndex: number): PeerInfo[] { + private configAt(absIndex: number): ConfigState { let cfg = this.baseConfig; const upto = this.pos(Math.min(absIndex, this.lastLogIndex())); for (let p = 1; p <= upto; p++) { const cmd = this.log[p]?.command; - if (cmd && isConfigCommand(cmd)) cfg = cmd.members; + if (cmd && isConfigCommand(cmd)) cfg = { members: cmd.members, oldMembers: cmd.oldMembers }; } return cfg; } - /** Recompute `members` from the latest configuration in the log. */ + /** + * Recompute the derived configuration from the latest CONFIG entry in the log + * (over the snapshot base): the voting union `members` (replication targets) + * plus the two sets `configOld`/`configNew` the dual-majority predicate uses. + */ private recomputeMembers(): void { - const cfg = this.configAt(this.lastLogIndex()); - const next = new Map(); - for (const p of cfg) next.set(p.id, p); - this.members = next; + this.adoptConfig(this.configAt(this.lastLogIndex())); + } + + /** Adopt a (possibly joint) configuration as the current derived membership. */ + private adoptConfig(cfg: ConfigState): void { + const union = new Map(); + for (const p of cfg.members) union.set(p.id, p); + this.configNew = new Set(cfg.members.map((p) => p.id)); + if (cfg.oldMembers) { + for (const p of cfg.oldMembers) union.set(p.id, p); + this.configOld = new Set(cfg.oldMembers.map((p) => p.id)); + } else { + this.configOld = null; + } + this.members = union; } /** True if the log holds a not-yet-committed CONFIG entry (a change in flight). */ @@ -279,7 +355,8 @@ export class RaftNode< this.lastApplied = snap.lastIncludedIndex; this.log = [{ term: snap.lastIncludedTerm, command: { type: 'NOOP' } }]; // The config compacted into the snapshot becomes the new base config. - if (snap.members) this.baseConfig = snap.members; + // Preserve a joint config if the snapshot boundary fell inside one. + if (snap.members) this.baseConfig = { members: snap.members, oldMembers: snap.oldMembers }; this.logger?.info('restored snapshot', { lastIncludedIndex: this.lastIncludedIndex }); } @@ -423,51 +500,94 @@ export class RaftNode< } /** - * Add or remove a single voting member (Raft dissertation §4.1). Changing one - * server at a time keeps the old and new majorities overlapping, so no split - * decision is possible without a joint-consensus phase. The new configuration - * is appended as a CONFIG entry and adopted immediately; the returned promise - * resolves once that entry commits. Leader-only. + * Add or remove a voting member via **joint consensus** (Raft §6 / ADR-0022, + * superseding the single-server changes of ADR-0015). A change from C-old to + * C-new transitions through a joint configuration C-old,new in which every + * decision needs a majority of BOTH configurations separately — which makes + * even an arbitrary change (one that does NOT overlap C-old in a majority) + * safe. Two phases, each adopted on append and awaited to commit: + * + * 1. Append a JOINT CONFIG (`members: C-new, oldMembers: C-old`); await its + * commit (dual majority). + * 2. Append a FINAL CONFIG (`members: C-new`); await its commit. + * + * The returned promise resolves when the FINAL config commits. A leader not in + * C-new steps down once the final config commits. Leader-only. */ - changeMembership(change: { add?: PeerInfo; remove?: string }, meta?: CommandMeta): Promise> { + async changeMembership(change: { add?: PeerInfo; remove?: string }, meta?: CommandMeta): Promise> { if (this.role !== 'leader') throw new NotLeaderError(this.leaderId); - // One change at a time: refuse while a previous change is still uncommitted. - if (this.hasUncommittedConfig()) { - return Promise.reject(new MembershipError('A membership change is already in progress')); + // One transition at a time: refuse while a previous change is still in + // flight. That means both an uncommitted CONFIG entry AND a still-joint + // configuration (a committed joint awaiting its final C-new — e.g. one this + // leader inherited and is finalizing): starting a new change from either + // state would overlap two transitions and break the dual-majority argument. + if (this.hasUncommittedConfig() || this.isJoint()) { + throw new MembershipError('A membership change is already in progress'); } + const oldMembers = [...this.members.values()]; const next = new Map(this.members); if (change.add) { if (next.has(change.add.id)) { - return Promise.reject(new MembershipError(`${change.add.id} is already a member`)); + throw new MembershipError(`${change.add.id} is already a member`); } next.set(change.add.id, change.add); } else if (change.remove) { if (!next.has(change.remove)) { - return Promise.reject(new MembershipError(`${change.remove} is not a member`)); + throw new MembershipError(`${change.remove} is not a member`); } if (next.size === 1) { - return Promise.reject(new MembershipError('Cannot remove the last member')); + throw new MembershipError('Cannot remove the last member'); } next.delete(change.remove); } else { - return Promise.reject(new MembershipError('Specify add or remove')); + throw new MembershipError('Specify add or remove'); + } + const newMembers = [...next.values()]; + + // Phase 1: joint config C-old,new. Adopt on append, replicate, await commit + // under dual majority. (A leader crash here leaves the joint CONFIG in the + // log; the next leader adopts it on election and completes the transition — + // see `inheritedJoint` / the finalization in `applyCommitted`.) + const term = this.currentTerm; + await this.submitConfigEntry({ type: 'CONFIG', members: newMembers, oldMembers }, meta); + + // We may have lost leadership (or moved terms) while awaiting the joint + // commit — don't append the final config from a stale leadership. + if (this.role !== 'leader' || this.currentTerm !== term) { + throw new NotLeaderError(this.leaderId); } - return this.submitConfig([...next.values()], meta); + // Phase 2: final simple config C-new. Adopt on append, replicate, await + // commit; the leader-self-removal step-down (if we're not in C-new) fires + // from applyCommitted once this commits. + return this.submitConfigEntry({ type: 'CONFIG', members: newMembers }, meta); } - /** Append a CONFIG entry, adopt it immediately, and replicate it. */ - private submitConfig(members: PeerInfo[], meta?: CommandMeta): Promise> { - const entry: LogEntry = { term: this.currentTerm, command: { type: 'CONFIG', members }, meta }; + /** Append a CONFIG entry (joint or final), adopt it immediately, replicate it. */ + private submitConfigEntry( + command: { type: 'CONFIG'; members: PeerInfo[]; oldMembers?: PeerInfo[] }, + meta?: CommandMeta, + ): Promise> { + // A configuration's target set C-new must never be empty: an empty config + // makes its majority vacuously satisfiable (see `majorityOf`), which would + // silently drop the dual-majority requirement and let a single side decide. + if (command.members.length === 0) { + return Promise.reject(new MembershipError('a configuration must have at least one member')); + } + const entry: LogEntry = { term: this.currentTerm, command, meta }; this.log.push(entry); - // Adopt the new configuration the instant it is in the log (Raft §4.1). + // Adopt the new configuration the instant it is in the log (Raft §6). this.recomputeMembers(); this.persist(); const index = this.lastLogIndex(); this.matchIndex.set(this.id, index); this.nextIndex.set(this.id, index + 1); - this.logger?.info('membership change proposed', { index, members: members.map((m) => m.id) }); + this.logger?.info('membership change proposed', { + index, + members: command.members.map((m) => m.id), + joint: command.oldMembers !== undefined, + }); const promise = new Promise>((resolve, reject) => { this.pending.set(index, { term: entry.term, resolve, reject }); @@ -585,13 +705,20 @@ export class RaftNode< * reports a log mismatch — the follower still recognises us as leader. */ private async confirmLeadership(): Promise { - const majority = this.quorum(); - if (majority <= 1) return true; // single-node cluster is its own majority + // Single-node config (no peers in any voting set) is its own majority. + if (this.otherMembers().length === 0) return this.inMajority([this.id]); const term = this.currentTerm; - const acks = await Promise.all(this.otherMembers().map((peer) => this.replicateTo(peer))); + const peers = this.otherMembers(); + const acks = await Promise.all(peers.map((peer) => this.replicateTo(peer))); if (this.role !== 'leader' || this.currentTerm !== term) return false; - const confirmed = 1 + acks.filter(Boolean).length; // +1 for self - return confirmed >= majority; + // Tally the set of ids that acknowledged our leadership (plus self), then + // require a DUAL majority during a joint transition (Raft §6 / ADR-0022): + // a single-config majority must never confirm a read on its own. + const confirmed = new Set([this.id]); + peers.forEach((peer, i) => { + if (acks[i]) confirmed.add(peer.id); + }); + return this.inMajority(confirmed); } private waitForApplied(index: number): Promise { @@ -675,6 +802,12 @@ export class RaftNode< this.matchIndex.set(peer.id, 0); } this.logger?.info('became LEADER', { term: this.currentTerm }); + // If we inherited an in-progress joint configuration (a previous leader + // appended the joint C-old,new but crashed before installing the final + // C-new), remember to complete that transition once it commits (Raft §4.3; + // see `applyCommitted`). A leader that originates a change is NOT joint at + // election, so this is false for those. + this.inheritedJoint = this.isJoint(); // Commit a no-op for this term so prior entries can be safely committed. this.log.push({ term: this.currentTerm, command: { type: 'NOOP' } }); this.persist(); @@ -693,9 +826,11 @@ export class RaftNode< lastLogTerm: this.termAt(this.lastLogIndex())!, }; - let votes = 1; // vote for self - const majority = this.quorum(); - if (votes >= majority) { + // Tally granting voter ids (vote for self) and decide via the dual-majority + // predicate, so during a joint transition a candidate that can only reach a + // majority of ONE configuration can never win (Raft §6 / ADR-0022). + const votes = new Set([this.id]); + if (this.inMajority(votes)) { this.becomeLeader(); return; } @@ -709,8 +844,8 @@ export class RaftNode< return; } if (reply.voteGranted) { - votes += 1; - if (votes >= majority && this.role === 'candidate') { + votes.add(peer.id); + if (this.role === 'candidate' && this.inMajority(votes)) { this.becomeLeader(); } } @@ -834,7 +969,16 @@ export class RaftNode< const durable = this.storage.loadSnapshot(); const snapIndex = durable?.lastIncludedIndex ?? this.lastIncludedIndex; const lastIncludedTerm = durable?.lastIncludedTerm ?? this.lastIncludedTerm; - const members = durable?.members ?? this.configAt(snapIndex); + // The config at the snapshot point — may be joint; carry both sets so the + // follower reconstructs the joint config and keeps enforcing dual majority. + // Prefer the durable snapshot's config; fall back to the live config at the + // snapshot point rather than an empty set (an empty config a follower adopted + // as its base would be constraint-free — see `majorityOf`). + const cfg: ConfigState = durable && durable.members + ? { members: durable.members, oldMembers: durable.oldMembers } + : this.configAt(snapIndex); + const members = cfg.members; + const oldMembers = cfg.oldMembers; const data = durable?.data ?? this.rsm.snapshot(); // Serialize the snapshot's data ONCE, then stream byte/character slices. @@ -865,6 +1009,7 @@ export class RaftNode< lastIncludedIndex: snapIndex, lastIncludedTerm, members, + oldMembers, offset, data: slice, done, @@ -1041,8 +1186,9 @@ export class RaftNode< // commitIndex/lastApplied only move forward (guarded above against rollback). this.commitIndex = Math.max(this.commitIndex, args.lastIncludedIndex); this.lastApplied = Math.max(this.lastApplied, args.lastIncludedIndex); - // Adopt the configuration carried with the snapshot as the new base config. - if (args.members) this.baseConfig = args.members; + // Adopt the configuration carried with the snapshot as the new base config, + // preserving a joint config if the snapshot boundary fell inside one. + if (args.members) this.baseConfig = { members: args.members, oldMembers: args.oldMembers }; this.recomputeMembers(); // Persist snapshot before the compacted log (same crash-ordering as takeSnapshot). @@ -1050,6 +1196,7 @@ export class RaftNode< lastIncludedIndex: this.lastIncludedIndex, lastIncludedTerm: this.lastIncludedTerm, members: args.members, + oldMembers: args.oldMembers, data: snapshotData, }); this.persist(); @@ -1062,12 +1209,15 @@ export class RaftNode< for (let n = this.lastLogIndex(); n > this.commitIndex; n--) { // Raft safety: only commit entries from the current term by counting. if (this.termAt(n) !== this.currentTerm) continue; - // Count only current voting members (a removed node mustn't count). - let count = 0; + // The set of voting members (incl. self) that have replicated through n. + // An index is committed iff that set forms a DUAL majority during a joint + // transition (Raft §6 / ADR-0022) — i.e. it never commits on a single + // configuration's majority alone. + const acked = new Set(); for (const id of this.members.keys()) { - if ((this.matchIndex.get(id) ?? 0) >= n) count += 1; + if ((this.matchIndex.get(id) ?? 0) >= n) acked.add(id); } - if (count >= this.quorum()) { + if (this.inMajority(acked)) { this.commitIndex = n; this.applyCommitted(); break; @@ -1090,6 +1240,21 @@ export class RaftNode< } } this.resolveReadWaiters(); + // Complete an INHERITED joint transition (Raft §4.3): a previous leader + // appended the joint C-old,new but crashed before installing the final + // C-new. Once we (the new leader) have COMMITTED that joint config (it is + // joint and no CONFIG entry is still uncommitted), append the final C-new so + // the cluster doesn't stay wedged in joint consensus. `changeMembership` + // finalizes the transitions it originates itself, so `inheritedJoint` gates + // this to inherited ones only and is cleared so it fires at most once. + if (this.role === 'leader' && this.inheritedJoint && this.isJoint() && !this.hasUncommittedConfig()) { + this.inheritedJoint = false; + const cNew = [...this.configNew] + .map((id) => this.members.get(id)) + .filter((p): p is PeerInfo => p !== undefined); + this.logger?.info('completing inherited joint transition', { members: cNew.map((m) => m.id) }); + void this.submitConfigEntry({ type: 'CONFIG', members: cNew }).catch(() => undefined); + } // If a committed config removed us, step down: we are no longer a member. if (this.role === 'leader' && !this.members.has(this.id)) { this.logger?.info('stepping down: removed from cluster configuration'); @@ -1112,8 +1277,10 @@ export class RaftNode< const snapTerm = this.termAt(snapIndex)!; const data = this.rsm.snapshot(); // Capture the configuration at the snapshot point before compacting — the - // CONFIG entries that produced it are about to be discarded. - const members = this.configAt(snapIndex); + // CONFIG entries that produced it are about to be discarded. This may be a + // JOINT config (the snapshot boundary fell inside a transition): preserve + // both sets so the compacted base still enforces dual majority (ADR-0022). + const cfg = this.configAt(snapIndex); // Keep the sentinel + everything after the snapshot point. const tail = this.log.slice(this.pos(snapIndex) + 1); @@ -1122,12 +1289,18 @@ export class RaftNode< // compacted log. That way the durable log base can only ever lag the // snapshot (reconciled on restart), never lead it (which would lose the // state the discarded entries produced). - this.storage.saveSnapshot({ lastIncludedIndex: snapIndex, lastIncludedTerm: snapTerm, members, data }); + this.storage.saveSnapshot({ + lastIncludedIndex: snapIndex, + lastIncludedTerm: snapTerm, + members: cfg.members, + oldMembers: cfg.oldMembers, + data, + }); this.log = [{ term: snapTerm, command: { type: 'NOOP' } }, ...tail]; this.lastIncludedIndex = snapIndex; this.lastIncludedTerm = snapTerm; - this.baseConfig = members; + this.baseConfig = cfg; this.persist(); this.logger?.info('took snapshot', { lastIncludedIndex: snapIndex, remainingEntries: this.log.length - 1 }); } diff --git a/src/consensus/types.ts b/src/consensus/types.ts index 0892843..cf78814 100644 --- a/src/consensus/types.ts +++ b/src/consensus/types.ts @@ -28,14 +28,19 @@ export interface AppCommand { * * - `NOOP` — internal Raft bookkeeping (e.g. the no-op a new leader commits to * make prior-term entries safely committable). Never audited. - * - `CONFIG` — a cluster membership change (Raft dissertation §4). Consumed by - * the Raft node, which adopts `members` as its configuration the + * - `CONFIG` — a cluster membership change (Raft §6, joint consensus; ADR-0022). + * Consumed by the Raft node, which adopts the configuration the * moment the entry is appended (not when committed). A no-op for the - * application state machine. `members` is the full new voting set. + * application state machine. `members` is the new voting set (C-new). + * When `oldMembers` is present the entry is the **joint** config + * C-old,new — a transitional configuration in which EVERY decision + * (election, commit, leadership confirmation) needs a majority of + * BOTH `oldMembers` (C-old) AND `members` (C-new) separately. When + * `oldMembers` is absent the entry is a **final** simple config. */ export type Command = | { type: 'NOOP' } - | { type: 'CONFIG'; members: PeerInfo[] } + | { type: 'CONFIG'; members: PeerInfo[]; oldMembers?: PeerInfo[] } | C; /** A single entry in the replicated log. */ @@ -140,6 +145,13 @@ export interface InstallSnapshotArgs { lastIncludedTerm: number; /** Cluster configuration as of the snapshot point (membership survives compaction). */ members: PeerInfo[]; + /** + * Present iff the snapshot boundary fell inside a joint-consensus transition + * (ADR-0022): the C-old set of the joint config (`members` is then C-new), so + * the receiving follower reconstructs the joint config and keeps enforcing + * dual majority rather than silently collapsing to a simple config. + */ + oldMembers?: PeerInfo[]; /** Code-unit (UTF-16) offset of this chunk within the serialized snapshot string. */ offset: number; /** A slice of the JSON-serialized state-machine snapshot at `offset`. */ @@ -183,6 +195,13 @@ export interface Snapshot { lastIncludedTerm: number; /** Cluster configuration as of the snapshot point (the compacted log's base config). */ members?: PeerInfo[]; + /** + * Present iff the snapshot boundary fell inside a joint-consensus transition + * (ADR-0022): the C-old set of the joint config. `members` is then C-new. + * Preserving it ensures a node restoring/catching-up from this snapshot still + * enforces dual majority and doesn't lose an in-progress joint config. + */ + oldMembers?: PeerInfo[]; data: unknown; } @@ -193,11 +212,11 @@ export interface PeerInfo { } /** True for the framework's reserved control commands (never application commands). */ -export function isControlCommand(command: { type: string }): command is { type: 'NOOP' } | { type: 'CONFIG'; members: PeerInfo[] } { +export function isControlCommand(command: { type: string }): command is { type: 'NOOP' } | { type: 'CONFIG'; members: PeerInfo[]; oldMembers?: PeerInfo[] } { return command.type === 'NOOP' || command.type === 'CONFIG'; } /** Narrow a log command to a CONFIG membership entry. */ -export function isConfigCommand(command: { type: string }): command is { type: 'CONFIG'; members: PeerInfo[] } { +export function isConfigCommand(command: { type: string }): command is { type: 'CONFIG'; members: PeerInfo[]; oldMembers?: PeerInfo[] } { return command.type === 'CONFIG'; } diff --git a/tests/membership.test.ts b/tests/membership.test.ts index 271e910..ad9d774 100644 --- a/tests/membership.test.ts +++ b/tests/membership.test.ts @@ -1,10 +1,73 @@ import { RaftNode, MembershipError } from '../src/consensus/raftNode'; -import { LocalTransport, RpcHandler } from '../src/consensus/transport'; -import { PeerInfo } from '../src/consensus/types'; +import { + LocalTransport, + RpcHandler, + Transport, +} from '../src/consensus/transport'; +import { + AppendEntriesArgs, + InstallSnapshotArgs, + PeerInfo, + ReadIndexArgs, + RequestVoteArgs, +} from '../src/consensus/types'; import { buildAddCommand } from '../src/models/book'; import { BookNode, BookStateMachine } from '../src/models/bookStateMachine'; import { waitFor } from './helpers'; +/** + * A LocalTransport whose deliveries can be partitioned at runtime. `blocked` holds + * unordered "a|b" pairs; an RPC between two endpoints in a blocked pair (either + * direction) resolves to null, simulating a network partition. Used to prove the + * dual-majority property: a candidate that can only reach ONE config's majority + * during a joint transition must not win. + */ +class PartitionableTransport implements Transport { + private readonly inner: LocalTransport; + readonly blocked = new Set(); + + constructor(private readonly registry: Map) { + this.inner = new LocalTransport(registry, 1); + } + + static key(a: string, b: string): string { + return [a, b].sort().join('|'); + } + + /** Block all traffic between every node in `groupA` and every node in `groupB`. */ + partition(groupA: string[], groupB: string[]): void { + for (const a of groupA) for (const b of groupB) this.blocked.add(PartitionableTransport.key(a, b)); + } + + private reachable(from: string, to: string): boolean { + return !this.blocked.has(PartitionableTransport.key(from, to)); + } + + sendRequestVote(peer: PeerInfo, args: RequestVoteArgs) { + if (!this.reachable(args.candidateId, peer.id)) return Promise.resolve(null); + return this.inner.sendRequestVote(peer, args); + } + sendAppendEntries(peer: PeerInfo, args: AppendEntriesArgs) { + if (!this.reachable(args.leaderId, peer.id)) return Promise.resolve(null); + return this.inner.sendAppendEntries(peer, args); + } + sendInstallSnapshot(peer: PeerInfo, args: InstallSnapshotArgs) { + if (!this.reachable(args.leaderId, peer.id)) return Promise.resolve(null); + return this.inner.sendInstallSnapshot(peer, args); + } + sendReadIndex(peer: PeerInfo, _args: ReadIndexArgs) { + // ReadIndex carries no source id; the partition tests don't exercise it. + return this.inner.sendReadIndex(peer, _args); + } +} + +/** Append a raw CONFIG entry (joint or final) via the node's internal two-phase helper. */ +type ConfigSubmitter = { + submitConfigEntry(command: { type: 'CONFIG'; members: PeerInfo[]; oldMembers?: PeerInfo[] }): Promise; + recomputeMembers(): void; +}; +const asInternal = (node: BookNode) => node as unknown as ConfigSubmitter; + const TIMERS = { electionMinMs: 50, electionMaxMs: 100, heartbeatMs: 20 }; // Several tests here run multiple membership changes + elections back-to-back, @@ -12,9 +75,12 @@ const TIMERS = { electionMinMs: 50, electionMaxMs: 100, heartbeatMs: 20 }; // the real guards — they throw a clear "condition not met" if anything genuinely // stalls. Their SUM, however, can exceed Jest's 5s default test timeout when // workers run in parallel under CPU contention (the suite passes in isolation), -// which is the only reason this test flaked. Give the suite ample headroom so a -// slow-but-correct run is never mistaken for a hang. -jest.setTimeout(15000); +// which is the only reason these tests flaked. The joint-consensus tests below +// run 5–6 node clusters with forced elections + partitions, so they get extra +// headroom (30s) so a slow-but-correct run under parallel load is never mistaken +// for a hang. The SAFETY assertions in those tests are negative (assert nothing +// happened) and so are load-insensitive regardless. +jest.setTimeout(30000); /** * A cluster whose registry + node set the test controls, so nodes can be added @@ -43,7 +109,7 @@ const leaderOf = (nodes: Iterable) => [...nodes].find((n) => n.isLeade const addBook = (isbn: string) => buildAddCommand({ title: isbn, author: 'A', publisher: 'P', isbn, copies: 1 }); -describe('Dynamic membership (single-server changes)', () => { +describe('Dynamic membership (joint consensus, ADR-0022)', () => { it('adds a new node, which catches up and joins the quorum', async () => { const c = makeCluster(['n1', 'n2', 'n3']); c.nodes.forEach((n) => n.start()); @@ -113,7 +179,7 @@ describe('Dynamic membership (single-server changes)', () => { // two remaining nodes elect a new one among themselves. await waitFor(() => !leader.isLeader()); const survivors = [...c.nodes.values()].filter((n) => n !== leader); - await waitFor(() => survivors.filter((n) => n.isLeader()).length === 1, 3000); + await waitFor(() => survivors.filter((n) => n.isLeader()).length === 1, 6000); expect(leader.status().members).not.toContain(leader.id); c.nodes.forEach((n) => n.stop()); @@ -132,4 +198,186 @@ describe('Dynamic membership (single-server changes)', () => { c.nodes.forEach((n) => n.stop()); }); + + it('commits an arbitrary change whose C-old and C-new do NOT overlap in a majority', async () => { + // Replace two of three nodes: C-old = {n1,n2,n3}, C-new = {n1,n4,n5}. + // The two configs share only {n1} — a majority of NEITHER. A single-server + // change could never express this; only joint consensus (dual majority) + // commits it safely. Drive the two phases directly through the node. + const c = makeCluster(['n1', 'n2', 'n3']); + c.nodes.forEach((n) => n.start()); + await waitFor(() => [...c.nodes.values()].filter((n) => n.isLeader()).length === 1); + const leader = leaderOf(c.nodes.values()); + const peer = (p: string): PeerInfo => ({ id: p, url: `local://${p}` }); + + // Bring up the two new members n4/n5, knowing the existing peers. + const extra = ['n4', 'n5'].map((id) => { + const node = new RaftNode( + { id, peers: ['n1', 'n2', 'n3'].map(peer), selfUrl: `local://${id}`, stateMachine: new BookStateMachine(), ...TIMERS }, + c.transport, + ); + c.registry.set(id, node); + node.start(); + return node; + }); + const nodes = new Map(c.nodes); + nodes.set('n4', extra[0]); + nodes.set('n5', extra[1]); + + await leader.submit(addBook('pre')); + + // Keep the current leader in C-new (so it survives the transition), and + // replace the other two C-old nodes with n4/n5. C-old = {n1,n2,n3}, C-new = + // {,n4,n5} share only {} — a majority of NEITHER config. + const cOldIds = ['n1', 'n2', 'n3']; + const cNewIds = [leader.id, 'n4', 'n5']; + const cOld = cOldIds.map(peer); + const cNew = cNewIds.map(peer); + + // Phase 1: joint C-old,new. Commits only on a majority of BOTH configs. + await asInternal(leader).submitConfigEntry({ type: 'CONFIG', members: cNew, oldMembers: cOld }); + // Phase 2: final C-new. + await asInternal(leader).submitConfigEntry({ type: 'CONFIG', members: cNew }); + + // The cluster converges to C-new on the surviving leader and the new members. + await waitFor(() => nodes.get('n4')!.status().members.length === 3, 6000); + await waitFor(() => nodes.get('n4')!.app.get(leader.app.getAll()[0].id) !== undefined, 6000); + + // A new write commits through C-new (which now excludes two old nodes). + const w = await leader.submit(addBook('post')); + expect(w.status).toBe(201); + await waitFor(() => nodes.get('n5')!.stateMachine.size() === 2, 6000); + + nodes.forEach((n) => n.stop()); + }); + + it('enforces dual majority during the joint phase: a C-old-only majority can neither elect nor commit', async () => { + // Two DISJOINT configs: C-old = {n1,n2,n3}, C-new = {n4,n5,n6}. A majority of + // one is never a majority of the other, so dual majority is impossible to + // satisfy from a single side. We pin every node into the joint config, then + // sever C-old from C-new and watch a C-old-only group: it can reach a C-old + // majority but ZERO of C-new, so it must neither win an election nor commit. + const registry = new Map(); + const transport = new PartitionableTransport(registry); + const all = ['n1', 'n2', 'n3', 'n4', 'n5', 'n6']; + const cOldIds = ['n1', 'n2', 'n3']; + const cNewIds = ['n4', 'n5', 'n6']; + const peer = (p: string): PeerInfo => ({ id: p, url: `local://${p}` }); + // DETERMINISTIC LEADERSHIP via ASYMMETRIC election timers: n1 alone gets a + // short timeout; n2..n6 get timeouts so long they never spontaneously + // campaign. Only n1 ever *initiates* an election, so n1 is the only node + // that can win one — making n1 the joint leader regardless of CPU load. + // + // This replaces an earlier symmetric-timer setup that flaked under + // parallel-worker contention: forcing *only* n1 to `becomeCandidate` does + // NOT guarantee n1 wins — once the prior leader steps down, no heartbeats + // flow, every node's election timer fires, and under CPU starvation some + // OTHER node (e.g. n4) wins the storm, so `n1.isLeader()` never holds. With + // n1 the sole initiator that race is gone. The dual-majority SAFETY + // assertions below FORCE n2/n3 to campaign explicitly (so their long + // timeouts don't matter there) and assert they still cannot win. + const FAST = { electionMinMs: 50, electionMaxMs: 100, heartbeatMs: 20 }; + const SLOW = { electionMinMs: 2000, electionMaxMs: 4000, heartbeatMs: 20 }; + const nodes = new Map(); + for (const id of all) { + const node = new RaftNode( + { id, peers: all.filter((p) => p !== id).map(peer), selfUrl: `local://${id}`, stateMachine: new BookStateMachine(), ...(id === 'n1' ? FAST : SLOW) }, + transport, + ); + nodes.set(id, node); + registry.set(id, node); + } + all.forEach((id) => nodes.get(id)!.start()); + // n1 is the only node that campaigns, so it is deterministically the leader. + const n1 = nodes.get('n1')!; + await waitFor(() => n1.isLeader(), 6000); + + // Pin EVERY node into the joint config C-old,new = ({n1,n2,n3},{n4,n5,n6}) and + // leave it joint (no final config follows). + await asInternal(n1).submitConfigEntry({ + type: 'CONFIG', + members: cNewIds.map(peer), + oldMembers: cOldIds.map(peer), + }); + await waitFor(() => all.every((id) => nodes.get(id)!.status().members.length === 6), 6000); + + // n1 (a C-old node) is the joint leader: with the cluster still fully + // connected it satisfies BOTH majorities and keeps its leadership. + expect(n1.isLeader()).toBe(true); + // Commit one write through the intact joint cluster as a baseline. + await n1.submit(addBook('joint-pre')); + const sizeBefore = n1.stateMachine.size(); + + // Now sever C-old from C-new entirely. n1 (a C-old node) can still reach a + // C-old majority {n1,n2,n3} but ZERO of C-new {n4,n5,n6}. + transport.partition(cOldIds, cNewIds); + + // SAFETY (commits): n1 is still leader and accepts the proposal into its log, + // but committing it needs a C-new majority it can no longer reach — so it + // must NOT commit (its state machine never grows) within the partial window. + const pending = n1.submit(addBook('joint-stalled')).catch(() => undefined); + await new Promise((r) => setTimeout(r, 500)); + expect(n1.stateMachine.size()).toBe(sizeBefore); + + // SAFETY (elections): force the isolated C-old group to campaign. A C-old + // majority grants votes among themselves, but winning the JOINT config also + // needs a C-new majority, which is unreachable — so no C-old node wins. + cOldIds.forEach((id) => (nodes.get(id)! as unknown as { becomeCandidate(): void }).becomeCandidate()); + await new Promise((r) => setTimeout(r, 500)); + expect(cOldIds.filter((id) => nodes.get(id)!.isLeader())).toEqual([]); + + nodes.forEach((n) => n.stop()); + await pending; + }); + + it('a new leader completes a joint transition its crashed predecessor left unfinished', async () => { + // Raft §4.3: a leader that appends the joint C-old,new but crashes before + // installing the final C-new must not wedge the cluster — the next leader + // adopts the joint config on election and finishes the transition. We drive + // ONLY the joint phase directly, crash the leader, and assert a survivor + // converges the cluster to the simple C-new on its own. + const c = makeCluster(['n1', 'n2', 'n3']); + c.nodes.forEach((n) => n.start()); + await waitFor(() => [...c.nodes.values()].filter((n) => n.isLeader()).length === 1); + const leader = leaderOf(c.nodes.values()); + const peer = (p: string): PeerInfo => ({ id: p, url: `local://${p}` }); + + // A fresh node n4 that joins as part of C-new. + const n4 = new RaftNode( + { id: 'n4', peers: ['n1', 'n2', 'n3'].map(peer), selfUrl: 'local://n4', stateMachine: new BookStateMachine(), ...TIMERS }, + c.transport, + ); + c.registry.set('n4', n4); + n4.start(); + + // Joint transition C-old = {n1,n2,n3} → C-new = the two non-leader originals + // plus n4 (the leader removes itself). Append ONLY the joint config; the + // joint UNION is 4 nodes, so a survivor reaching members.length === 4 proves + // the joint config replicated before we crash the leader. + const keep = [...c.nodes.values()].filter((n) => n !== leader).map((n) => n.id); + const cNew = [...keep, 'n4']; + const survivors = [...keep.map((id) => c.nodes.get(id)!), n4]; + await asInternal(leader).submitConfigEntry({ + type: 'CONFIG', + members: cNew.map(peer), + oldMembers: ['n1', 'n2', 'n3'].map(peer), + }); + await waitFor(() => survivors.some((n) => n.status().members.length === 4), 6000); + + // The originating leader "crashes" before installing the final config. + leader.stop(); + + // A survivor wins (dual majority among the reachable {keep ∪ n4}) and, seeing + // it inherited a joint config, completes the transition to the simple C-new. + await waitFor(() => survivors.some((n) => n.isLeader()), 6000); + await waitFor(() => survivors.every((n) => n.status().members.length === 3), 6000); + survivors.forEach((n) => expect(n.status().members.slice().sort()).toEqual(cNew.slice().sort())); + + // The finalized simple-majority cluster still commits writes. + const newLeader = survivors.find((n) => n.isLeader())!; + const w = await newLeader.submit(addBook('post-finalize')); + expect(w.status).toBe(201); + + survivors.forEach((n) => n.stop()); + }); });