From 5fe41c7d0e49d6d70c7ae3da2d569ee020695ae2 Mon Sep 17 00:00:00 2001 From: Casey Gowrie Date: Tue, 14 Jul 2026 15:02:51 -0400 Subject: [PATCH 1/4] feat(dev): drained transactional dev server on the stock Nitro contract Rebuild the dev server on the stock Nitro dev-server architecture with two guarantees the stock RunnerManager lacks: - Drained worker replacement: the parent listener tracks in-flight exchanges (responses and upgraded sockets) per worker and terminates a retired worker only after its last exchange settles, so structural reloads never reset admitted work. Crashed workers restart with their own workspace; shutdown completes within a bounded deadline. - Transactional authored rebuilds: a candidate is compiled, bundled, and readiness-checked while the previous worker keeps serving; the swap is worker-first, then generation-pointer activation, so any failure before the swap leaves the previous worker and generation untouched, and hosts are generation-neutral in the instant between the two steps. Runtime-only edits publish a generation without touching the worker. Environment files stage with each attempt and roll back on failure. Co-Authored-By: Claude Fable 5 Signed-off-by: Casey Gowrie --- .changeset/drained-dev-worker-replacement.md | 5 + packages/eve/package.json | 1 + .../declarations/env-runner.d.ts | 38 + .../scripts/vendor-compiled/env-runner.mjs | 18 + .../eve/scripts/vendor-compiled/index.mjs | 2 + packages/eve/src/cli/dev/environment.ts | 107 +- packages/eve/src/compiler/compile-agent.ts | 10 +- .../sandbox/prewarm.scenario.test.ts | 4 +- .../compiled-manifest-fingerprint.test.ts | 58 + .../internal/compiled-manifest-fingerprint.ts | 47 + .../internal/materialized-authored-modules.ts | 10 + .../dev-generation-artifacts.scenario.test.ts | 15 + .../dev-runtime-artifacts.integration.test.ts | 31 + .../internal/nitro/dev-runtime-artifacts.ts | 136 ++- .../internal/nitro/development-generation.ts | 12 + .../src/internal/nitro/host/channel-routes.ts | 108 +- .../nitro/host/configure-nitro-routes.test.ts | 18 +- .../nitro/host/configure-nitro-routes.ts | 6 +- .../create-application-nitro.scenario.test.ts | 25 +- .../nitro/host/create-application-nitro.ts | 21 +- .../dev-authored-rebuild-coordinator.test.ts | 200 ++++ .../host/dev-authored-rebuild-coordinator.ts | 278 +++++ ...v-authored-source-watcher.scenario.test.ts | 989 ++++-------------- .../nitro/host/dev-authored-source-watcher.ts | 129 +-- ...ev-environment-watcher.integration.test.ts | 284 ++--- .../internal/nitro/host/dev-host-candidate.ts | 91 ++ .../dev-host-fingerprint.integration.test.ts | 148 +++ .../nitro/host/dev-host-fingerprint.ts | 53 + .../internal/nitro/host/dev-host-workspace.ts | 35 + .../nitro/host/dev-live-virtual-modules.ts | 93 -- .../eve/src/internal/nitro/host/dev-runner.ts | 150 +++ .../internal/nitro/host/dev-server-http.ts | 119 +++ .../drained-nitro-dev-server.scenario.test.ts | 242 +++++ .../nitro/host/drained-nitro-dev-server.ts | 383 +++++++ .../prepare-application-host.scenario.test.ts | 52 +- .../nitro/host/prepare-application-host.ts | 93 +- .../host/start-development-server.test.ts | 220 ++-- .../nitro/host/start-development-server.ts | 160 ++- packages/eve/src/internal/nitro/host/types.ts | 7 + .../dev-server-drain.scenario.test.ts | 234 +++++ .../eve/test/scenarios/dev-server-harness.ts | 433 ++++++++ .../scenarios/dev-server.scenario.test.ts | 540 +++++++++- ...roduction-build-isolation.scenario.test.ts | 75 +- pnpm-lock.yaml | 32 +- 44 files changed, 3977 insertions(+), 1735 deletions(-) create mode 100644 .changeset/drained-dev-worker-replacement.md create mode 100644 packages/eve/scripts/vendor-compiled/declarations/env-runner.d.ts create mode 100644 packages/eve/scripts/vendor-compiled/env-runner.mjs create mode 100644 packages/eve/src/internal/compiled-manifest-fingerprint.test.ts create mode 100644 packages/eve/src/internal/compiled-manifest-fingerprint.ts create mode 100644 packages/eve/src/internal/nitro/host/dev-authored-rebuild-coordinator.test.ts create mode 100644 packages/eve/src/internal/nitro/host/dev-authored-rebuild-coordinator.ts create mode 100644 packages/eve/src/internal/nitro/host/dev-host-candidate.ts create mode 100644 packages/eve/src/internal/nitro/host/dev-host-fingerprint.integration.test.ts create mode 100644 packages/eve/src/internal/nitro/host/dev-host-fingerprint.ts create mode 100644 packages/eve/src/internal/nitro/host/dev-host-workspace.ts delete mode 100644 packages/eve/src/internal/nitro/host/dev-live-virtual-modules.ts create mode 100644 packages/eve/src/internal/nitro/host/dev-runner.ts create mode 100644 packages/eve/src/internal/nitro/host/dev-server-http.ts create mode 100644 packages/eve/src/internal/nitro/host/drained-nitro-dev-server.scenario.test.ts create mode 100644 packages/eve/src/internal/nitro/host/drained-nitro-dev-server.ts create mode 100644 packages/eve/test/scenarios/dev-server-drain.scenario.test.ts create mode 100644 packages/eve/test/scenarios/dev-server-harness.ts diff --git a/.changeset/drained-dev-worker-replacement.md b/.changeset/drained-dev-worker-replacement.md new file mode 100644 index 000000000..8c9fab7c5 --- /dev/null +++ b/.changeset/drained-dev-worker-replacement.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Rework `eve dev` structural reloads to never interrupt admitted work: an isolated candidate must compile, bundle, and start before it is promoted atomically, the retired worker keeps serving the responses and sockets it already admitted until they settle, and a failed candidate or crashed worker leaves the server available with shutdown bounded even while streams are open. diff --git a/packages/eve/package.json b/packages/eve/package.json index 529d6215b..818e2588e 100644 --- a/packages/eve/package.json +++ b/packages/eve/package.json @@ -337,6 +337,7 @@ "chokidar": "5.0.0", "commander": "14.0.3", "emulate": "0.6.0", + "env-runner": "0.1.12", "eventsource-parser": "3.0.8", "experimental-ai-sdk-code-mode": "1.0.16", "gray-matter": "4.0.3", diff --git a/packages/eve/scripts/vendor-compiled/declarations/env-runner.d.ts b/packages/eve/scripts/vendor-compiled/declarations/env-runner.d.ts new file mode 100644 index 000000000..69e8964c2 --- /dev/null +++ b/packages/eve/scripts/vendor-compiled/declarations/env-runner.d.ts @@ -0,0 +1,38 @@ +import type { IncomingMessage } from "node:http"; +import type { Socket } from "node:net"; + +export interface WorkerHooks { + onClose?(runner: BaseEnvRunner, cause?: unknown): void; + onReady?(runner: BaseEnvRunner): void; +} + +export interface BaseEnvRunnerOptions { + readonly data?: Record; + readonly hooks?: WorkerHooks; + readonly name: string; + readonly workerEntry: string; +} + +export class BaseEnvRunner { + readonly closed: boolean; + readonly ready: boolean; + protected readonly _data: Record | undefined; + protected readonly _name: string; + protected readonly _workerEntry: string; + + constructor(options: BaseEnvRunnerOptions); + + close(cause?: unknown): Promise; + fetch(input: string | URL | Request, init?: RequestInit): Promise; + sendMessage(message: unknown): void; + upgrade(context: { + node: { head: Uint8Array; req: IncomingMessage; socket: Socket }; + }): Promise; + waitForReady(timeout?: number): Promise; + + protected _closeRuntime(): Promise; + protected _handleMessage(message: unknown): void; + protected _hasRuntime(): boolean; + protected _initWithVirtualData(initialize: () => void): void; + protected _runtimeType(): string; +} diff --git a/packages/eve/scripts/vendor-compiled/env-runner.mjs b/packages/eve/scripts/vendor-compiled/env-runner.mjs new file mode 100644 index 000000000..90a95d69a --- /dev/null +++ b/packages/eve/scripts/vendor-compiled/env-runner.mjs @@ -0,0 +1,18 @@ +import { loadDeclaration } from "./_shared.mjs"; + +export default { + packageName: "env-runner", + compiledPath: "env-runner", + entries: [ + { + entry: "dist/_chunks/common-base-runner.mjs", + outputPath: "index", + declaration: await loadDeclaration("env-runner.d.ts"), + }, + { + entry: "dist/runners/node-worker/worker.mjs", + outputPath: "node-worker", + declaration: "export {};\n", + }, + ], +}; diff --git a/packages/eve/scripts/vendor-compiled/index.mjs b/packages/eve/scripts/vendor-compiled/index.mjs index 747c26258..027905d89 100644 --- a/packages/eve/scripts/vendor-compiled/index.mjs +++ b/packages/eve/scripts/vendor-compiled/index.mjs @@ -32,6 +32,7 @@ import chokidar from "./chokidar.mjs"; import commander from "./commander.mjs"; import experimentalAiSdkCodeMode from "./experimental-ai-sdk-code-mode.mjs"; import eventsourceParserStream from "./eventsource-parser-stream.mjs"; +import envRunner from "./env-runner.mjs"; import grayMatter from "./gray-matter.mjs"; import jose from "./jose.mjs"; import jsoncParser from "./jsonc-parser.mjs"; @@ -52,6 +53,7 @@ export const MODULES = [ commander, experimentalAiSdkCodeMode, eventsourceParserStream, + envRunner, google, grayMatter, jose, diff --git a/packages/eve/src/cli/dev/environment.ts b/packages/eve/src/cli/dev/environment.ts index a7275d17a..0f75d8e5a 100644 --- a/packages/eve/src/cli/dev/environment.ts +++ b/packages/eve/src/cli/dev/environment.ts @@ -22,6 +22,12 @@ function isMissingEnvironmentFileError(error: unknown): error is NodeJS.ErrnoExc interface DevelopmentEnvironmentLoader { reload(): void; + stageReload(): DevelopmentEnvironmentReload; +} + +export interface DevelopmentEnvironmentReload { + commit(): void; + rollback(): void; } const developmentEnvironmentLoaders = new Map(); @@ -48,6 +54,23 @@ export function loadDevelopmentEnvironmentFiles(appRoot: string): void { getDevelopmentEnvironmentLoader(appRoot).reload(); } +export function stageDevelopmentEnvironmentFiles(appRoot: string): DevelopmentEnvironmentReload { + return getDevelopmentEnvironmentLoader(appRoot).stageReload(); +} + +export function readDevelopmentEnvironmentHostValues( + appRoot: string, +): Readonly> { + const values: Record = {}; + const fileValues = readDevelopmentEnvironmentValues(resolve(appRoot)); + + for (const key of [...fileValues.keys()].sort((left, right) => left.localeCompare(right))) { + values[key] = process.env[key] ?? null; + } + + return values; +} + function getDevelopmentEnvironmentLoader(appRoot: string): DevelopmentEnvironmentLoader { const resolvedAppRoot = resolve(appRoot); const existingLoader = developmentEnvironmentLoaders.get(resolvedAppRoot); @@ -65,32 +88,78 @@ function createDevelopmentEnvironmentLoader(appRoot: string): DevelopmentEnviron const protectedKeys = new Set(Object.keys(process.env)); const managedValues = new Map(); + const stageReload = (): DevelopmentEnvironmentReload => { + const previousManagedValues = new Map(managedValues); + const nextValues = readDevelopmentEnvironmentValues(appRoot); + const affectedKeys = new Set([...managedValues.keys(), ...nextValues.keys()]); + const previousEnvironment = new Map( + [...affectedKeys].map((key) => [key, process.env[key]] as const), + ); + let settled = false; + + applyDevelopmentEnvironmentValues({ + managedValues, + nextValues, + protectedKeys, + }); + + return { + commit() { + settled = true; + }, + rollback() { + if (settled) { + return; + } + settled = true; + managedValues.clear(); + for (const [key, value] of previousManagedValues) { + managedValues.set(key, value); + } + for (const [key, value] of previousEnvironment) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } + }, + }; + }; + return { reload() { - const nextValues = readDevelopmentEnvironmentValues(appRoot); + stageReload().commit(); + }, + stageReload, + }; +} - for (const [key, previousValue] of managedValues) { - if (nextValues.has(key) || protectedKeys.has(key)) { - continue; - } +function applyDevelopmentEnvironmentValues(input: { + readonly managedValues: Map; + readonly nextValues: ReadonlyMap; + readonly protectedKeys: ReadonlySet; +}): void { + for (const [key, previousValue] of input.managedValues) { + if (input.nextValues.has(key) || input.protectedKeys.has(key)) { + continue; + } - if (process.env[key] === previousValue) { - delete process.env[key]; - } + if (process.env[key] === previousValue) { + delete process.env[key]; + } - managedValues.delete(key); - } + input.managedValues.delete(key); + } - for (const [key, value] of nextValues) { - if (protectedKeys.has(key)) { - continue; - } + for (const [key, value] of input.nextValues) { + if (input.protectedKeys.has(key)) { + continue; + } - process.env[key] = value; - managedValues.set(key, value); - } - }, - }; + process.env[key] = value; + input.managedValues.set(key, value); + } } function readDevelopmentEnvironmentValues(appRoot: string): Map { diff --git a/packages/eve/src/compiler/compile-agent.ts b/packages/eve/src/compiler/compile-agent.ts index 41d97c8e7..0b4d61489 100644 --- a/packages/eve/src/compiler/compile-agent.ts +++ b/packages/eve/src/compiler/compile-agent.ts @@ -85,13 +85,11 @@ export async function compileAgent(input: CompileAgentInput = {}): Promise { diff --git a/packages/eve/src/execution/sandbox/prewarm.scenario.test.ts b/packages/eve/src/execution/sandbox/prewarm.scenario.test.ts index a739fafe4..779c6acef 100644 --- a/packages/eve/src/execution/sandbox/prewarm.scenario.test.ts +++ b/packages/eve/src/execution/sandbox/prewarm.scenario.test.ts @@ -3,7 +3,7 @@ import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { compileAgent, compileAgentInBuildWorkspace } from "#compiler/compile-agent.js"; +import { compileAgent, compileAgentInWorkspace } from "#compiler/compile-agent.js"; import { resolvePackageSourceFilePath } from "#internal/application/package.js"; import { createDevelopmentNitroArtifactsConfig } from "#internal/nitro/host/artifacts-config.js"; import { publishDevelopmentGeneration } from "#internal/nitro/development-generation.js"; @@ -30,7 +30,7 @@ describe("prewarmAppSandboxes", () => { const appRoot = await createScenarioAppRoot(); const compilerAppRoot = join(appRoot, ".eve", "builds", "isolated", "compiler"); - await compileAgentInBuildWorkspace({ + await compileAgentInWorkspace({ artifactLocations: { publishedRoot: join(compilerAppRoot, ".eve"), writeRoot: join(compilerAppRoot, ".eve"), diff --git a/packages/eve/src/internal/compiled-manifest-fingerprint.test.ts b/packages/eve/src/internal/compiled-manifest-fingerprint.test.ts new file mode 100644 index 000000000..ece211693 --- /dev/null +++ b/packages/eve/src/internal/compiled-manifest-fingerprint.test.ts @@ -0,0 +1,58 @@ +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { createCompiledAgentManifest } from "#compiler/manifest.js"; +import { serializeCompiledManifestForFingerprint } from "#internal/compiled-manifest-fingerprint.js"; + +function manifestWithRoot(runtimeAppRoot: string, agentRoot = join(runtimeAppRoot, "agent")) { + return createCompiledAgentManifest({ + agentRoot, + appRoot: runtimeAppRoot, + config: { + model: { id: "openai/gpt-5-mini", routing: { kind: "gateway", target: "openai" } }, + name: "fingerprint-test", + }, + }); +} + +describe("serializeCompiledManifestForFingerprint", () => { + it("serializes identical content under different snapshot roots identically", () => { + const firstRoot = "/tmp/snapshots/generation-a/source/app"; + const secondRoot = "/tmp/snapshots/generation-b/source/app"; + const first = serializeCompiledManifestForFingerprint({ + manifest: manifestWithRoot(firstRoot), + runtimeAppRoot: firstRoot, + }); + const second = serializeCompiledManifestForFingerprint({ + manifest: manifestWithRoot(secondRoot), + runtimeAppRoot: secondRoot, + }); + + expect(first).toBe(second); + expect(first).toContain("$runtime/agent"); + expect(first).not.toContain("generation-a"); + }); + + it("keeps absolute paths outside the runtime root verbatim", () => { + const runtimeAppRoot = "/tmp/snapshots/generation-a/source/app"; + const serialized = serializeCompiledManifestForFingerprint({ + manifest: manifestWithRoot(runtimeAppRoot, "/somewhere/else/agent"), + runtimeAppRoot, + }); + + expect(serialized).toContain("/somewhere/else/agent"); + }); + + it("canonicalizes object key order", () => { + const left = serializeCompiledManifestForFingerprint({ + manifest: { agentRoot: "/app/agent", config: { model: "m", name: "n" } } as never, + runtimeAppRoot: "/app", + }); + const right = serializeCompiledManifestForFingerprint({ + manifest: { config: { name: "n", model: "m" }, agentRoot: "/app/agent" } as never, + runtimeAppRoot: "/app", + }); + + expect(left).toBe(right); + }); +}); diff --git a/packages/eve/src/internal/compiled-manifest-fingerprint.ts b/packages/eve/src/internal/compiled-manifest-fingerprint.ts new file mode 100644 index 000000000..8fc7d71bf --- /dev/null +++ b/packages/eve/src/internal/compiled-manifest-fingerprint.ts @@ -0,0 +1,47 @@ +import { isAbsolute, relative, sep } from "node:path"; + +import type { CompiledAgentManifest } from "#compiler/manifest.js"; + +const RUNTIME_ROOT_TOKEN = "$runtime"; + +export function serializeCompiledManifestForFingerprint(input: { + readonly manifest: CompiledAgentManifest; + readonly runtimeAppRoot: string; +}): string { + return JSON.stringify(normalizeValue(input.manifest, input.runtimeAppRoot)); +} + +function normalizeValue(value: unknown, runtimeAppRoot: string): unknown { + if (typeof value === "string") { + return normalizeRuntimePath(value, runtimeAppRoot); + } + + if (Array.isArray(value)) { + return value.map((entry) => normalizeValue(entry, runtimeAppRoot)); + } + + if (value === null || typeof value !== "object") { + return value; + } + + return Object.fromEntries( + Object.entries(value) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => [key, normalizeValue(entry, runtimeAppRoot)]), + ); +} + +function normalizeRuntimePath(value: string, runtimeAppRoot: string): string { + if (!isAbsolute(value)) { + return value; + } + + const relativePath = relative(runtimeAppRoot, value); + if (relativePath === "") { + return RUNTIME_ROOT_TOKEN; + } + if (relativePath === ".." || relativePath.startsWith(`..${sep}`) || isAbsolute(relativePath)) { + return value; + } + return `${RUNTIME_ROOT_TOKEN}/${relativePath.split(sep).join("/")}`; +} diff --git a/packages/eve/src/internal/materialized-authored-modules.ts b/packages/eve/src/internal/materialized-authored-modules.ts index b65d14dd5..470e8f770 100644 --- a/packages/eve/src/internal/materialized-authored-modules.ts +++ b/packages/eve/src/internal/materialized-authored-modules.ts @@ -7,6 +7,7 @@ import type { CompiledAgentManifest, CompiledAgentNodeManifest } from "#compiler import { COMPILED_AGENT_MANIFEST_KIND, ROOT_COMPILED_AGENT_NODE_ID } from "#compiler/manifest.js"; import { collectModuleRefsForManifest } from "#compiler/module-map.js"; import { bundleAuthoredModuleForGeneration } from "#internal/authored-module-loader.js"; +import { serializeCompiledManifestForFingerprint } from "#internal/compiled-manifest-fingerprint.js"; import { materializeAuthoredExternalDependencies, type ResolvedAuthoredExternalModule, @@ -38,6 +39,15 @@ export async function materializeAuthoredModules(input: { const fingerprint = createHash("sha256"); await mkdir(modulesRoot, { recursive: true }); + fingerprint + .update("manifest\0") + .update( + serializeCompiledManifestForFingerprint({ + manifest, + runtimeAppRoot: input.runtimeAppRoot, + }), + ) + .update("\0"); for (const node of collectNodeManifests(manifest)) { const modules: Record = {}; diff --git a/packages/eve/src/internal/nitro/dev-generation-artifacts.scenario.test.ts b/packages/eve/src/internal/nitro/dev-generation-artifacts.scenario.test.ts index 02894800e..b9d3bcab3 100644 --- a/packages/eve/src/internal/nitro/dev-generation-artifacts.scenario.test.ts +++ b/packages/eve/src/internal/nitro/dev-generation-artifacts.scenario.test.ts @@ -126,6 +126,21 @@ describe("development generation artifacts", () => { expect(first.fingerprint).toBe(identical.fingerprint); expect(materializedInstrumentation).not.toContain("/.eve/dev-runtime/snapshots/"); + await writeFile( + join(app.appRoot, "agent", "instructions.md"), + "Use the configured model and explain the result.", + ); + const changedInstructions = await stageDevelopmentGeneration( + await compileAgent({ startPath: app.appRoot }), + ); + expect(changedInstructions.fingerprint).not.toBe(first.fingerprint); + + await writeFile(join(app.appRoot, "agent", "instructions.md"), "Use the configured model."); + const restoredInstructions = await stageDevelopmentGeneration( + await compileAgent({ startPath: app.appRoot }), + ); + expect(restoredInstructions.fingerprint).toBe(first.fingerprint); + await writeFile( join(app.appRoot, "agent", "instrumentation.mjs"), 'export default { marker: "two" };\n', diff --git a/packages/eve/src/internal/nitro/dev-runtime-artifacts.integration.test.ts b/packages/eve/src/internal/nitro/dev-runtime-artifacts.integration.test.ts index 2c3aa9bd9..89d3b2e98 100644 --- a/packages/eve/src/internal/nitro/dev-runtime-artifacts.integration.test.ts +++ b/packages/eve/src/internal/nitro/dev-runtime-artifacts.integration.test.ts @@ -16,6 +16,7 @@ import { createDevelopmentNitroArtifactsConfig } from "#internal/nitro/host/arti import { publishDevelopmentGeneration } from "#internal/nitro/development-generation.js"; import { activateDevelopmentRuntimeArtifactsSnapshot, + activateDevelopmentRuntimeArtifactsSnapshotTransaction, pruneDevelopmentRuntimeArtifactsSnapshots, readDevelopmentRuntimeArtifactsSnapshotRoot, readDevelopmentRuntimeArtifactsRevision, @@ -155,6 +156,36 @@ describe("development runtime artifact snapshots", () => { }); }); + it("restores the prior runtime pointer when activation is rolled back", async () => { + const appRoot = await createScratchDirectory("eve-dev-runtime-activation-rollback-"); + const agentRoot = join(appRoot, "agent"); + const compileDirectoryPath = join(appRoot, ".eve", "compile"); + await mkdir(agentRoot, { recursive: true }); + await mkdir(compileDirectoryPath, { recursive: true }); + await writeFile(join(appRoot, "package.json"), '{"type":"module"}\n'); + await writeFile( + join(compileDirectoryPath, "compiled-agent-manifest.json"), + `${JSON.stringify({ agentRoot, appRoot }, null, 2)}\n`, + ); + const compileResult = { + paths: { compileDirectoryPath }, + project: { appRoot }, + } as CompileAgentResult; + const first = await stageDevelopmentRuntimeArtifactsSnapshot(compileResult); + const second = await stageDevelopmentRuntimeArtifactsSnapshot(compileResult); + await activateDevelopmentRuntimeArtifactsSnapshot({ appRoot, snapshot: first }); + + const activation = await activateDevelopmentRuntimeArtifactsSnapshotTransaction({ + appRoot, + snapshot: second, + }); + expect(readDevelopmentRuntimeArtifactsRevision(appRoot).revision).toBe(second.runtimeAppRoot); + await activation.rollback(); + + expect(readDevelopmentRuntimeArtifactsRevision(appRoot).revision).toBe(first.runtimeAppRoot); + expect(existsSync(join(second.snapshotRoot, "activated"))).toBe(false); + }); + it("stages package-less flat markdown agents with generated runtime package metadata", async () => { const appRoot = await createScratchDirectory("eve-dev-runtime-flat-package-less-"); const compileDirectoryPath = join(appRoot, ".eve", "compile"); diff --git a/packages/eve/src/internal/nitro/dev-runtime-artifacts.ts b/packages/eve/src/internal/nitro/dev-runtime-artifacts.ts index decaa00f6..04ed31638 100644 --- a/packages/eve/src/internal/nitro/dev-runtime-artifacts.ts +++ b/packages/eve/src/internal/nitro/dev-runtime-artifacts.ts @@ -40,6 +40,16 @@ export interface DevelopmentRuntimeArtifactsSnapshot { readonly sourceRoot: string; } +export interface ActiveDevelopmentRuntimeArtifactsSnapshot { + readonly runtimeAppRoot: string; + readonly snapshotRoot: string; +} + +export interface DevelopmentRuntimeArtifactsActivation { + commit(): void; + rollback(): Promise; +} + /** * Resolves the dev-server pointer that records the latest runtime artifact * snapshot for new sessions. @@ -115,8 +125,50 @@ export async function activateDevelopmentRuntimeArtifactsSnapshot(input: { readonly appRoot: string; readonly snapshot: DevelopmentRuntimeArtifactsSnapshot; }): Promise { - await writeFile(join(input.snapshot.snapshotRoot, DEV_RUNTIME_ARTIFACTS_ACTIVATED_MARKER), ""); - await writeDevelopmentRuntimeArtifactsPointer(input); + const activation = await activateDevelopmentRuntimeArtifactsSnapshotTransaction(input); + activation.commit(); +} + +export async function activateDevelopmentRuntimeArtifactsSnapshotTransaction(input: { + readonly appRoot: string; + readonly snapshot: DevelopmentRuntimeArtifactsSnapshot; +}): Promise { + const markerPath = join(input.snapshot.snapshotRoot, DEV_RUNTIME_ARTIFACTS_ACTIVATED_MARKER); + const pointerPath = resolveDevelopmentRuntimeArtifactsPointerPath(input.appRoot); + const previousPointerSource = await readOptionalFile(pointerPath); + + try { + await writeFile(markerPath, ""); + await writeDevelopmentRuntimeArtifactsPointer(input); + } catch (error) { + throw await rollbackFailedActivation({ + cause: error, + markerPath, + pointerPath, + previousPointerSource, + }); + } + + let settled = false; + return { + commit() { + settled = true; + }, + async rollback() { + if (settled) { + return; + } + settled = true; + const rollbackError = await restoreDevelopmentRuntimeArtifactsActivation({ + markerPath, + pointerPath, + previousPointerSource, + }); + if (rollbackError !== undefined) { + throw rollbackError; + } + }, + }; } /** @@ -138,6 +190,21 @@ export function readDevelopmentRuntimeArtifactsSnapshotRoot( return pointer.runtimeAppRoot; } +export function readActiveDevelopmentRuntimeArtifactsSnapshot( + appRoot: string, +): ActiveDevelopmentRuntimeArtifactsSnapshot | undefined { + const pointer = readDevelopmentRuntimeArtifactsPointer( + resolveDevelopmentRuntimeArtifactsPointerPath(appRoot), + ); + if (pointer === undefined || pointer.version === 1) { + return undefined; + } + return { + runtimeAppRoot: pointer.runtimeAppRoot, + snapshotRoot: pointer.snapshotRoot, + }; +} + /** * Reads a revision token for the latest dev runtime artifact snapshot. */ @@ -468,7 +535,6 @@ async function writeDevelopmentRuntimeArtifactsPointer(input: { readonly snapshot: DevelopmentRuntimeArtifactsSnapshot; }): Promise { const pointerPath = resolveDevelopmentRuntimeArtifactsPointerPath(input.appRoot); - const temporaryPointerPath = `${pointerPath}.${randomUUID()}.tmp`; const pointer: DevelopmentRuntimeArtifactsPointerV2 = { appRoot: input.appRoot, kind: "eve-dev-runtime-artifacts-pointer", @@ -477,8 +543,19 @@ async function writeDevelopmentRuntimeArtifactsPointer(input: { version: DEV_RUNTIME_ARTIFACTS_POINTER_VERSION, }; + await writeDevelopmentRuntimeArtifactsPointerSource( + pointerPath, + `${JSON.stringify(pointer, null, 2)}\n`, + ); +} + +async function writeDevelopmentRuntimeArtifactsPointerSource( + pointerPath: string, + source: string, +): Promise { + const temporaryPointerPath = `${pointerPath}.${randomUUID()}.tmp`; await mkdir(dirname(pointerPath), { recursive: true }); - await writeFile(temporaryPointerPath, `${JSON.stringify(pointer, null, 2)}\n`); + await writeFile(temporaryPointerPath, source); try { await rename(temporaryPointerPath, pointerPath); } catch (error) { @@ -487,6 +564,57 @@ async function writeDevelopmentRuntimeArtifactsPointer(input: { } } +async function readOptionalFile(path: string): Promise { + try { + return await readFile(path, "utf8"); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") { + return undefined; + } + throw error; + } +} + +async function rollbackFailedActivation(input: { + readonly cause: unknown; + readonly markerPath: string; + readonly pointerPath: string; + readonly previousPointerSource: string | undefined; +}): Promise { + const rollbackError = await restoreDevelopmentRuntimeArtifactsActivation(input); + if (rollbackError === undefined) { + return input.cause; + } + return new AggregateError( + [input.cause, rollbackError], + "Development runtime activation and rollback failed.", + { cause: input.cause }, + ); +} + +async function restoreDevelopmentRuntimeArtifactsActivation(input: { + readonly markerPath: string; + readonly pointerPath: string; + readonly previousPointerSource: string | undefined; +}): Promise { + const restoration = await Promise.allSettled([ + input.previousPointerSource === undefined + ? rm(input.pointerPath, { force: true }) + : writeDevelopmentRuntimeArtifactsPointerSource( + input.pointerPath, + input.previousPointerSource, + ), + rm(input.markerPath, { force: true }), + ]); + const errors = restoration.flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + if (errors.length === 0) { + return undefined; + } + return new AggregateError(errors, "Failed to restore development runtime activation."); +} + async function validateSnapshotCompiledManifestRoots(input: { readonly manifestPath: string; readonly runtimeAppRoot: string; diff --git a/packages/eve/src/internal/nitro/development-generation.ts b/packages/eve/src/internal/nitro/development-generation.ts index 6bbe2c464..30b48b27d 100644 --- a/packages/eve/src/internal/nitro/development-generation.ts +++ b/packages/eve/src/internal/nitro/development-generation.ts @@ -4,7 +4,9 @@ import type { CompileAgentResult } from "#compiler/compile-agent.js"; import { materializeAuthoredModules } from "#internal/materialized-authored-modules.js"; import { activateDevelopmentRuntimeArtifactsSnapshot, + activateDevelopmentRuntimeArtifactsSnapshotTransaction, stageDevelopmentRuntimeArtifactsSnapshot, + type DevelopmentRuntimeArtifactsActivation, type DevelopmentRuntimeArtifactsSnapshot, } from "#internal/nitro/dev-runtime-artifacts.js"; @@ -63,6 +65,16 @@ export async function activateDevelopmentGeneration(input: { }); } +export async function activateDevelopmentGenerationTransaction(input: { + readonly appRoot: string; + readonly generation: DevelopmentGeneration; +}): Promise { + return await activateDevelopmentRuntimeArtifactsSnapshotTransaction({ + appRoot: input.appRoot, + snapshot: input.generation, + }); +} + export async function discardDevelopmentGeneration( generation: DevelopmentGeneration, ): Promise { diff --git a/packages/eve/src/internal/nitro/host/channel-routes.ts b/packages/eve/src/internal/nitro/host/channel-routes.ts index 54cbef88b..b44cc4860 100644 --- a/packages/eve/src/internal/nitro/host/channel-routes.ts +++ b/packages/eve/src/internal/nitro/host/channel-routes.ts @@ -1,4 +1,4 @@ -import type { Nitro, NitroEventHandler } from "nitro/types"; +import type { Nitro } from "nitro/types"; import type { NormalizedChannelCorsOptions } from "#channel/cors.js"; import type { ChannelRouteMethod } from "#public/definitions/channel.js"; @@ -12,20 +12,12 @@ import { resolvePackageSourceFilePath, } from "#internal/application/package.js"; import type { NitroArtifactsConfig } from "#internal/nitro/routes/runtime-artifacts.js"; -import { replaceDevLiveVirtualModules } from "#internal/nitro/host/dev-live-virtual-modules.js"; import type { PreparedApplicationHost } from "#internal/nitro/host/types.js"; -// Must stay under `#nitro/virtual/` — the dev bundler's virtual plugin -// freezes its resolveId filter at config creation, and that prefix is the -// only pattern under which channel routes added while `eve dev` runs can -// still resolve (see dev-live-virtual-modules.ts). const EVE_CHANNEL_VIRTUAL_ID_PREFIX = "#nitro/virtual/eve-channel/"; interface ChannelRouteNitro { - options: Pick; - routing: { - sync(): void; - }; + readonly options: Pick; } /** @@ -114,48 +106,6 @@ export function registerChannelVirtualHandlers( } } -/** - * Replaces the currently-mounted eve channel virtual handlers when the route - * set changes. - */ -export function syncChannelVirtualHandlers( - nitro: ChannelRouteNitro, - input: { - readonly artifactsConfig: NitroArtifactsConfig; - readonly next: readonly NitroChannelRouteRegistration[]; - readonly previous: readonly NitroChannelRouteRegistration[]; - }, -): boolean { - if (areChannelRouteRegistrationsEqual(input.previous, input.next)) { - return false; - } - - removeChannelVirtualHandlers(nitro); - registerChannelVirtualHandlers(nitro, { - artifactsConfig: input.artifactsConfig, - registrations: input.next, - }); - - const channelVirtualEntries: Record = {}; - for (const [virtualId, template] of Object.entries(nitro.options.virtual)) { - if (virtualId.startsWith(EVE_CHANNEL_VIRTUAL_ID_PREFIX) && typeof template === "string") { - channelVirtualEntries[virtualId] = template; - } - } - const mirrored = replaceDevLiveVirtualModules(nitro, { - entries: channelVirtualEntries, - prefix: EVE_CHANNEL_VIRTUAL_ID_PREFIX, - }); - if (!mirrored) { - console.warn( - "[eve:dev] channel routes changed but the dev bundler's virtual-module map was not captured; restart `eve dev` to mount the new routes.", - ); - } - - nitro.routing.sync(); - return true; -} - function createChannelRouteKey(registration: NitroChannelRouteRegistration): string { return `${registration.method.toUpperCase()} ${registration.route}`; } @@ -258,57 +208,3 @@ function addChannelCorsPreflightHandler( `};`, ].join("\n"); } - -function removeChannelVirtualHandlers(nitro: Pick): void { - for (let index = nitro.options.handlers.length - 1; index >= 0; index -= 1) { - const handler = nitro.options.handlers[index]; - if (handler !== undefined && isChannelVirtualHandler(handler)) { - nitro.options.handlers.splice(index, 1); - } - } - - for (const virtualId of Object.keys(nitro.options.virtual)) { - if (virtualId.startsWith(EVE_CHANNEL_VIRTUAL_ID_PREFIX)) { - delete nitro.options.virtual[virtualId]; - } - } -} - -function isChannelVirtualHandler(handler: NitroEventHandler): boolean { - return handler.handler.startsWith(EVE_CHANNEL_VIRTUAL_ID_PREFIX); -} - -function areChannelRouteRegistrationsEqual( - left: readonly NitroChannelRouteRegistration[], - right: readonly NitroChannelRouteRegistration[], -): boolean { - if (left.length !== right.length) { - return false; - } - - for (let index = 0; index < left.length; index += 1) { - const leftRegistration = left[index]; - const rightRegistration = right[index]; - - if (leftRegistration === undefined || rightRegistration === undefined) { - return false; - } - - if ( - leftRegistration.method !== rightRegistration.method || - leftRegistration.route !== rightRegistration.route || - !areChannelCorsOptionsEqual(leftRegistration.cors, rightRegistration.cors) - ) { - return false; - } - } - - return true; -} - -function areChannelCorsOptionsEqual( - left: NormalizedChannelCorsOptions | undefined, - right: NormalizedChannelCorsOptions | undefined, -): boolean { - return JSON.stringify(left) === JSON.stringify(right); -} diff --git a/packages/eve/src/internal/nitro/host/configure-nitro-routes.test.ts b/packages/eve/src/internal/nitro/host/configure-nitro-routes.test.ts index 6a3d5db1a..273551527 100644 --- a/packages/eve/src/internal/nitro/host/configure-nitro-routes.test.ts +++ b/packages/eve/src/internal/nitro/host/configure-nitro-routes.test.ts @@ -257,22 +257,18 @@ describe("Nitro route configuration", () => { expect(nitro.options.virtual["#eve-workflow/workflows"]).toBeUndefined(); }); - it("reports a failed workflow sync and runs the next queued sync", async () => { - workflowBuilderMocks.build - .mockResolvedValueOnce(undefined) - .mockRejectedValueOnce(new Error("injected workflow sync failure")) - .mockResolvedValueOnce(undefined); + it("does not retain a workflow reload hook after configuring a candidate", async () => { const nitro = createNitroStub({ dev: true }); await configureDevelopmentNitroRoutes(nitro, createPreparedHost()); - const reload = nitro.hookHandlers.get("dev:reload")?.[0]; - if (reload === undefined) { - throw new Error("Expected the workflow reload hook to be registered."); + const build = nitro.hookHandlers.get("build:before")?.[0]; + if (build === undefined) { + throw new Error("Expected the workflow build hook to be registered."); } - await expect(reload()).rejects.toThrow("injected workflow sync failure"); - await expect(reload()).resolves.toBeUndefined(); - expect(workflowBuilderMocks.build).toHaveBeenCalledTimes(3); + expect(nitro.hookHandlers.has("dev:reload")).toBe(false); + await expect(build()).resolves.toBeUndefined(); + expect(workflowBuilderMocks.build).toHaveBeenCalledOnce(); }); it("registers direct workflow queue handlers in dev mode so the worker bypasses HTTP dispatch", async () => { diff --git a/packages/eve/src/internal/nitro/host/configure-nitro-routes.ts b/packages/eve/src/internal/nitro/host/configure-nitro-routes.ts index 089ca80c6..fdee73ecf 100644 --- a/packages/eve/src/internal/nitro/host/configure-nitro-routes.ts +++ b/packages/eve/src/internal/nitro/host/configure-nitro-routes.ts @@ -369,8 +369,7 @@ async function registerWorkflowRoute( /** * Wires eve's package-owned app, channel, workflow inspection, dev-control, - * and Workflow SDK endpoints into the watch-mode Nitro host, rebuilding - * workflow bundles on reload. + * and Workflow SDK endpoints into one development Nitro candidate. */ export async function configureDevelopmentNitroRoutes( nitro: Nitro, @@ -385,8 +384,6 @@ export async function configureDevelopmentNitroRoutes( rootDir: resolvePackageRoot(), watch: true, }); - // Overlapping `build:before` and `dev:reload` syncs are serialized by - // `builder.build`'s per-output-directory queue. const syncWorkflowArtifacts = async () => { await builder.build({ nitroStepOutfile: join(workflowBuildDirectory, "steps.mjs"), @@ -395,7 +392,6 @@ export async function configureDevelopmentNitroRoutes( }; await registerWorkflowArtifactBuildHook(nitro, syncWorkflowArtifacts); - nitro.hooks.hook("dev:reload", syncWorkflowArtifacts); const artifactsConfig = createDevelopmentNitroArtifactsConfig({ appRoot: preparedHost.appRoot, diff --git a/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts b/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts index 3f9cf529f..c10e9d1f3 100644 --- a/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts +++ b/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts @@ -23,7 +23,11 @@ import { resolveWorkflowModulePath, } from "#internal/application/package.js"; import { resolveNitroBuildDirectory } from "#internal/application/paths.js"; -import type { NitroBuildSurface, PreparedApplicationHost } from "#internal/nitro/host/types.js"; +import type { + NitroBuildSurface, + PreparedApplicationHost, + PreparedDevelopmentApplicationHost, +} from "#internal/nitro/host/types.js"; import { applyWorkflowTransform } from "#internal/workflow-bundle/workflow-builders.js"; const configureDevelopmentNitroRoutes = vi.fn(async () => undefined); @@ -84,7 +88,7 @@ function createNitroStub(input: { buildDir?: string; dev?: boolean } = {}): Nitr }; } -function createPreparedHost(): PreparedApplicationHost { +function createPreparedHost(): PreparedDevelopmentApplicationHost { const appRoot = "/tmp/weather-agent"; const paths = resolveCompilerArtifactPaths(appRoot); const metadata: CompileMetadata = { @@ -142,7 +146,22 @@ function createPreparedHost(): PreparedApplicationHost { } as PreparedApplicationHost["compiledArtifacts"], scheduleRegistrations: [], schedules: [], - workflowBuildDir: `${appRoot}/.eve/nitro/workflow`, + generation: { + fingerprint: "runtime-fingerprint", + runtimeAppRoot: `${appRoot}/.eve/dev-runtime/snapshots/test/source/app`, + snapshotRoot: `${appRoot}/.eve/dev-runtime/snapshots/test`, + snapshotSourceRoot: `${appRoot}/.eve/dev-runtime/snapshots/test/source`, + sourceRoot: appRoot, + }, + workflowBuildDir: `${appRoot}/.eve/dev-hosts/test/workflow`, + workspace: { + artifactsDir: `${appRoot}/.eve/dev-hosts/test/artifacts`, + compilerArtifactsDir: `${appRoot}/.eve/dev-hosts/test/compiler`, + nitroBuildDir: `${appRoot}/.eve/dev-hosts/test/nitro`, + nitroOutputDir: `${appRoot}/.eve/dev-hosts/test/output`, + rootDir: `${appRoot}/.eve/dev-hosts/test`, + workflowBuildDir: `${appRoot}/.eve/dev-hosts/test/workflow`, + }, }; } diff --git a/packages/eve/src/internal/nitro/host/create-application-nitro.ts b/packages/eve/src/internal/nitro/host/create-application-nitro.ts index 423653854..6b2694a3d 100644 --- a/packages/eve/src/internal/nitro/host/create-application-nitro.ts +++ b/packages/eve/src/internal/nitro/host/create-application-nitro.ts @@ -15,7 +15,6 @@ import { prepareEveVersionedCacheDirectory, writeEveVersionedCacheMetadata, } from "#internal/application/cache-metadata.js"; -import { resolveNitroBuildDirectory } from "#internal/application/paths.js"; import { createProductionNitroArtifactsConfig } from "#internal/nitro/host/artifacts-config.js"; import { createCompiledSandboxBackendPrunePlugin } from "#internal/nitro/host/compiled-sandbox-backend-prune-plugin.js"; import { createExtensionScopePlugin } from "#internal/bundler/extension-scope-plugin.js"; @@ -25,7 +24,6 @@ import { } from "#internal/nitro/host/configure-nitro-routes.js"; import { applyEveCronHandlerRoute } from "#internal/nitro/host/cron-handler-route.js"; import { createNitroBundlerConfig } from "#internal/nitro/host/nitro-bundler-config.js"; -import { captureDevLiveVirtualModules } from "#internal/nitro/host/dev-live-virtual-modules.js"; import { createOptionalEngineDependencyPlugin, OPTIONAL_ENGINE_PACKAGES_BY_BACKEND_NAME, @@ -33,7 +31,11 @@ import { import { addNitroRoutingImportSpecifierPlugin } from "#internal/nitro/host/nitro-routing-import-specifier-plugin.js"; import { registerScheduleTaskHandlers } from "#internal/nitro/host/schedule-task-routes.js"; import { SERVER_EXTERNAL_PACKAGES } from "#internal/nitro/host/server-external-packages.js"; -import type { NitroBuildSurface, PreparedApplicationHost } from "#internal/nitro/host/types.js"; +import type { + NitroBuildSurface, + PreparedApplicationHost, + PreparedDevelopmentApplicationHost, +} from "#internal/nitro/host/types.js"; import { createEveVercelOptions } from "#internal/nitro/host/vercel-build-output-config.js"; import { applyWorkflowTransform } from "#internal/workflow-bundle/workflow-builders.js"; import { transformDynamicToolExecute } from "#internal/workflow-bundle/dynamic-tool-transform.js"; @@ -744,15 +746,14 @@ function externalizeDevelopmentWorkflowBundle( } /** - * Creates the watch-mode Nitro host for `eve dev`: all route surfaces on one - * instance, live virtual modules, and hot-reload hooks wired to the authored - * source. + * Creates one isolated Nitro host candidate for `eve dev`. */ export async function createDevelopmentApplicationNitro( - preparedHost: PreparedApplicationHost, + preparedHost: PreparedDevelopmentApplicationHost, ): Promise { - const nitroBuildDir = resolveNitroBuildDirectory(preparedHost.appRoot); + const nitroBuildDir = preparedHost.workspace.nitroBuildDir; const bundler = createApplicationNitroBundlerConfiguration(preparedHost, undefined); + const plugins = createApplicationNitroPlugins(preparedHost); await prepareEveVersionedCacheDirectory(nitroBuildDir); const nitro = await createNitro( @@ -762,7 +763,8 @@ export async function createDevelopmentApplicationNitro( dev: true, features: { websocket: true }, logLevel: 1, - plugins: createApplicationNitroPlugins(preparedHost), + output: { dir: preparedHost.workspace.nitroOutputDir }, + plugins, publicAssets: [], scanDirs: [resolvePackageSourceDirectoryPath("src/execution")], rolldownConfig: bundler.nitroRolldownConfig, @@ -777,7 +779,6 @@ export async function createDevelopmentApplicationNitro( ); await writeEveVersionedCacheMetadata(nitroBuildDir); - captureDevLiveVirtualModules(nitro); const stepEntrypointPath = join(nitro.options.buildDir, "workflow", "steps.mjs"); configureSharedApplicationNitro(nitro, preparedHost, "all"); const clearStepTransformCaches = configureNitroStepPlugins(nitro, stepEntrypointPath); diff --git a/packages/eve/src/internal/nitro/host/dev-authored-rebuild-coordinator.test.ts b/packages/eve/src/internal/nitro/host/dev-authored-rebuild-coordinator.test.ts new file mode 100644 index 000000000..c1897376c --- /dev/null +++ b/packages/eve/src/internal/nitro/host/dev-authored-rebuild-coordinator.test.ts @@ -0,0 +1,200 @@ +import { describe, expect, it, vi, beforeEach } from "vitest"; + +import type { CompileAgentResult } from "#compiler/compile-agent.js"; +import { + createDevelopmentAuthoredRebuildCoordinator, + PostCommitDevelopmentRebuildError, +} from "#internal/nitro/host/dev-authored-rebuild-coordinator.js"; +import { DrainedNitroDevServer } from "#internal/nitro/host/drained-nitro-dev-server.js"; +import type { DevelopmentRunnerFactory } from "#internal/nitro/host/dev-runner.js"; +import type { PreparedDevelopmentApplicationHost } from "#internal/nitro/host/types.js"; + +const mocks = vi.hoisted(() => ({ + activateDevelopmentGeneration: vi.fn(async () => undefined), + buildDevelopmentHostCandidate: vi.fn(async () => ({ + entry: "/tmp/eve-test/entry.mjs", + workerData: {}, + })), + computeDevelopmentHostFingerprint: vi.fn(async () => "host-1"), + createDevelopmentApplicationNitro: vi.fn(async () => ({})), + discardDevelopmentGeneration: vi.fn(async () => undefined), + environmentCommit: vi.fn(), + environmentRollback: vi.fn(), + prepareDevelopmentApplicationHost: vi.fn(), + removeDevelopmentHostWorkspace: vi.fn(async () => undefined), +})); + +vi.mock("#cli/dev/environment.js", () => ({ + stageDevelopmentEnvironmentFiles: () => ({ + commit: mocks.environmentCommit, + rollback: mocks.environmentRollback, + }), +})); +vi.mock("#internal/nitro/host/prepare-application-host.js", () => ({ + prepareDevelopmentApplicationHost: mocks.prepareDevelopmentApplicationHost, +})); +vi.mock("#internal/nitro/host/dev-host-fingerprint.js", () => ({ + computeDevelopmentHostFingerprint: mocks.computeDevelopmentHostFingerprint, +})); +vi.mock("#internal/nitro/host/create-application-nitro.js", () => ({ + createDevelopmentApplicationNitro: mocks.createDevelopmentApplicationNitro, +})); +vi.mock("#internal/nitro/host/dev-host-candidate.js", () => ({ + buildDevelopmentHostCandidate: mocks.buildDevelopmentHostCandidate, +})); +vi.mock("#internal/nitro/host/dev-host-workspace.js", () => ({ + removeDevelopmentHostWorkspace: mocks.removeDevelopmentHostWorkspace, +})); +vi.mock("#internal/nitro/development-generation.js", () => ({ + activateDevelopmentGeneration: mocks.activateDevelopmentGeneration, + discardDevelopmentGeneration: mocks.discardDevelopmentGeneration, +})); +vi.mock("#internal/nitro/host/artifacts-config.js", () => ({ + createDevelopmentNitroArtifactsConfig: () => ({}), +})); +vi.mock("#internal/nitro/routes/runtime-artifacts.js", () => ({ + resolveNitroCompiledArtifactsSource: () => ({}), +})); +vi.mock("#execution/sandbox/development-prewarm.js", () => ({ + startDevelopmentSandboxPrewarmInBackground: vi.fn(), +})); + +function createHost(id: string, runtimeFingerprint: string): PreparedDevelopmentApplicationHost { + return { + appRoot: "/tmp/eve-test", + compileResult: { + project: { agentRoot: "/tmp/eve-test/agent" }, + } as CompileAgentResult, + compiledArtifacts: { + bootstrapPath: `/tmp/eve-test/.eve/dev-hosts/${id}/bootstrap.mjs`, + workflowWorldPluginPath: `/tmp/eve-test/.eve/dev-hosts/${id}/workflow-world.mjs`, + }, + generation: { + fingerprint: runtimeFingerprint, + runtimeAppRoot: `/tmp/eve-test/.eve/dev-runtime/snapshots/${id}/source/app`, + snapshotRoot: `/tmp/eve-test/.eve/dev-runtime/snapshots/${id}`, + snapshotSourceRoot: `/tmp/eve-test/.eve/dev-runtime/snapshots/${id}/source`, + sourceRoot: "/tmp/eve-test", + }, + scheduleRegistrations: [], + schedules: [], + workflowBuildDir: `/tmp/eve-test/.eve/dev-hosts/${id}/workflow`, + workspace: { + artifactsDir: `/tmp/eve-test/.eve/dev-hosts/${id}/artifacts`, + compilerArtifactsDir: `/tmp/eve-test/.eve/dev-hosts/${id}/compiler`, + nitroBuildDir: `/tmp/eve-test/.eve/dev-hosts/${id}/nitro`, + nitroOutputDir: `/tmp/eve-test/.eve/dev-hosts/${id}/output`, + rootDir: `/tmp/eve-test/.eve/dev-hosts/${id}`, + workflowBuildDir: `/tmp/eve-test/.eve/dev-hosts/${id}/workflow`, + }, + }; +} + +const createRunner: DevelopmentRunnerFactory = () => { + let closed = false; + const closedListeners = new Set<(cause?: unknown) => void>(); + return { + close: async () => { + closed = true; + for (const listener of [...closedListeners]) { + listener(); + } + closedListeners.clear(); + }, + get closed() { + return closed; + }, + fetch: async () => new Response("ok"), + onceClosed(listener) { + if (closed) { + listener(); + return; + } + closedListeners.add(listener); + }, + upgrade: async () => undefined, + waitForReady: async () => undefined, + }; +}; + +async function createCoordinatorWithServer() { + const devServer = new DrainedNitroDevServer({ error: () => undefined }, createRunner); + mocks.computeDevelopmentHostFingerprint.mockResolvedValueOnce("host-1"); + const coordinator = await createDevelopmentAuthoredRebuildCoordinator({ + devServer, + initialHost: createHost("initial", "run-1"), + }); + return { coordinator, devServer }; +} + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("transactional authored rebuild coordinator", () => { + it("keeps the live worker when activation fails after the swap and retries on the next rebuild", async () => { + const { coordinator, devServer } = await createCoordinatorWithServer(); + const structuralHost = createHost("structural", "run-2"); + mocks.prepareDevelopmentApplicationHost.mockResolvedValueOnce(structuralHost); + mocks.computeDevelopmentHostFingerprint.mockResolvedValueOnce("host-2"); + mocks.activateDevelopmentGeneration.mockRejectedValueOnce(new Error("pointer write failed")); + + await expect(coordinator.rebuild({ changedPaths: [] })).rejects.toBeInstanceOf( + PostCommitDevelopmentRebuildError, + ); + + // The live worker's workspace survives; only the unactivated generation + // snapshot is discarded, and the staged environment stays committed. + expect(mocks.removeDevelopmentHostWorkspace).not.toHaveBeenCalled(); + expect(mocks.discardDevelopmentGeneration).toHaveBeenCalledWith(structuralHost.generation); + expect(mocks.environmentCommit).toHaveBeenCalledOnce(); + expect(mocks.environmentRollback).not.toHaveBeenCalled(); + + const retryHost = createHost("retry", "run-2"); + mocks.prepareDevelopmentApplicationHost.mockResolvedValueOnce(retryHost); + mocks.computeDevelopmentHostFingerprint.mockResolvedValueOnce("host-2"); + + const retried = await coordinator.rebuild({ changedPaths: [] }); + expect(retried.kind).toBe("runtime"); + expect(mocks.activateDevelopmentGeneration).toHaveBeenLastCalledWith({ + appRoot: retryHost.appRoot, + generation: retryHost.generation, + }); + + await devServer.close(); + }); + + it("discards the candidate and keeps prior state when the swap itself fails", async () => { + const failingFactory: DevelopmentRunnerFactory = () => { + const runner = createRunner({ entry: "/tmp/eve-test/entry.mjs", name: "t", workerData: {} }); + return { + ...runner, + waitForReady: async () => { + throw new Error("candidate never became ready"); + }, + }; + }; + const devServer = new DrainedNitroDevServer({ error: () => undefined }, failingFactory); + mocks.computeDevelopmentHostFingerprint.mockResolvedValueOnce("host-1"); + const coordinator = await createDevelopmentAuthoredRebuildCoordinator({ + devServer, + initialHost: createHost("initial", "run-1"), + }); + + const structuralHost = createHost("structural", "run-2"); + mocks.prepareDevelopmentApplicationHost.mockResolvedValueOnce(structuralHost); + mocks.computeDevelopmentHostFingerprint.mockResolvedValueOnce("host-2"); + + await expect(coordinator.rebuild({ changedPaths: [] })).rejects.toThrow( + "candidate never became ready", + ); + + expect(mocks.removeDevelopmentHostWorkspace).toHaveBeenCalledWith(structuralHost.workspace); + expect(mocks.discardDevelopmentGeneration).toHaveBeenCalledWith(structuralHost.generation); + expect(mocks.activateDevelopmentGeneration).not.toHaveBeenCalled(); + expect(mocks.environmentRollback).toHaveBeenCalledOnce(); + expect(mocks.environmentCommit).not.toHaveBeenCalled(); + + await devServer.close(); + }); +}); diff --git a/packages/eve/src/internal/nitro/host/dev-authored-rebuild-coordinator.ts b/packages/eve/src/internal/nitro/host/dev-authored-rebuild-coordinator.ts new file mode 100644 index 000000000..ee7888b35 --- /dev/null +++ b/packages/eve/src/internal/nitro/host/dev-authored-rebuild-coordinator.ts @@ -0,0 +1,278 @@ +import { relative, resolve } from "node:path"; + +import { stageDevelopmentEnvironmentFiles } from "#cli/dev/environment.js"; +import { startDevelopmentSandboxPrewarmInBackground } from "#execution/sandbox/development-prewarm.js"; +import { createDevelopmentNitroArtifactsConfig } from "#internal/nitro/host/artifacts-config.js"; +import { createDevelopmentApplicationNitro } from "#internal/nitro/host/create-application-nitro.js"; +import { buildDevelopmentHostCandidate } from "#internal/nitro/host/dev-host-candidate.js"; +import { computeDevelopmentHostFingerprint } from "#internal/nitro/host/dev-host-fingerprint.js"; +import { removeDevelopmentHostWorkspace } from "#internal/nitro/host/dev-host-workspace.js"; +import { prepareDevelopmentApplicationHost } from "#internal/nitro/host/prepare-application-host.js"; +import { DrainedNitroDevServer } from "#internal/nitro/host/drained-nitro-dev-server.js"; +import type { PreparedDevelopmentApplicationHost } from "#internal/nitro/host/types.js"; +import { + activateDevelopmentGeneration, + discardDevelopmentGeneration, +} from "#internal/nitro/development-generation.js"; +import { resolveNitroCompiledArtifactsSource } from "#internal/nitro/routes/runtime-artifacts.js"; + +export type DevelopmentRebuildKind = "structural" | "unchanged" | "runtime"; + +/** + * Raised when generation activation fails after the replacement worker has + * already been swapped in. The server keeps serving on the new worker with + * the previous generation; the next rebuild retries activation. + */ +export class PostCommitDevelopmentRebuildError extends Error { + constructor(cause: unknown) { + super( + "The development worker was replaced but its generation failed to activate. The server keeps serving the previous generation; the next rebuild retries activation.", + { cause }, + ); + this.name = "PostCommitDevelopmentRebuildError"; + } +} + +export interface DevelopmentRebuildResult { + readonly host: PreparedDevelopmentApplicationHost; + readonly kind: DevelopmentRebuildKind; +} + +export interface DevelopmentAuthoredRebuildCoordinator { + rebuild(input: { readonly changedPaths: readonly string[] }): Promise; +} + +export async function createDevelopmentAuthoredRebuildCoordinator(input: { + readonly devServer: DrainedNitroDevServer; + readonly initialHost: PreparedDevelopmentApplicationHost; +}): Promise { + return new TransactionalDevelopmentAuthoredRebuildCoordinator({ + currentHostFingerprint: await computeDevelopmentHostFingerprint(input.initialHost), + currentRuntimeFingerprint: input.initialHost.generation.fingerprint, + devServer: input.devServer, + initialHost: input.initialHost, + }); +} + +class TransactionalDevelopmentAuthoredRebuildCoordinator implements DevelopmentAuthoredRebuildCoordinator { + #currentHost: PreparedDevelopmentApplicationHost; + #currentHostFingerprint: string; + #currentRuntimeFingerprint: string; + readonly #devServer: DrainedNitroDevServer; + + constructor(input: { + readonly currentHostFingerprint: string; + readonly currentRuntimeFingerprint: string; + readonly devServer: DrainedNitroDevServer; + readonly initialHost: PreparedDevelopmentApplicationHost; + }) { + this.#currentHost = input.initialHost; + this.#currentHostFingerprint = input.currentHostFingerprint; + this.#currentRuntimeFingerprint = input.currentRuntimeFingerprint; + this.#devServer = input.devServer; + } + + async rebuild(input: { + readonly changedPaths: readonly string[]; + }): Promise { + const previousHost = this.#currentHost; + // Staged unconditionally rather than keyed off changedPaths: a failed + // rebuild rolls the environment back, and the paths that carried the env + // edit are consumed by that attempt — a later retry would otherwise + // commit without ever reapplying the change. + const environmentReload = stageDevelopmentEnvironmentFiles(previousHost.appRoot); + let nextHost: PreparedDevelopmentApplicationHost | undefined; + + try { + nextHost = await prepareDevelopmentApplicationHost(previousHost.appRoot); + const nextHostFingerprint = await computeDevelopmentHostFingerprint(nextHost); + const nextRuntimeFingerprint = nextHost.generation.fingerprint; + const hasStructuralChange = nextHostFingerprint !== this.#currentHostFingerprint; + const hasRuntimeChange = nextRuntimeFingerprint !== this.#currentRuntimeFingerprint; + + if (!hasStructuralChange && !hasRuntimeChange) { + await discardPreparedHost(nextHost); + nextHost = undefined; + environmentReload.commit(); + return { host: previousHost, kind: "unchanged" }; + } + + if (!hasStructuralChange) { + await removeDevelopmentHostWorkspace(nextHost.workspace); + const committedHost = retainActiveHostWorkspace(previousHost, nextHost); + await activateDevelopmentGeneration({ + appRoot: committedHost.appRoot, + generation: committedHost.generation, + }); + this.#commitState(committedHost, nextHostFingerprint, nextRuntimeFingerprint); + nextHost = undefined; + environmentReload.commit(); + startSandboxPrewarmAfterCommit(committedHost, input.changedPaths); + return { host: committedHost, kind: "runtime" }; + } + + const result = await this.#commitStructuralHost({ + hasRuntimeChange, + nextHost, + nextHostFingerprint, + nextRuntimeFingerprint, + }); + nextHost = undefined; + environmentReload.commit(); + startSandboxPrewarmAfterCommit(result.host, input.changedPaths); + return result; + } catch (error) { + if (error instanceof PostCommitDevelopmentRebuildError) { + // The swapped worker is live and was spawned with the staged + // environment; only the pointer activation failed. + environmentReload.commit(); + throw error; + } + environmentReload.rollback(); + if (nextHost === undefined) { + throw error; + } + throw await discardFailedHost(error, nextHost); + } + } + + async #commitStructuralHost(input: { + readonly hasRuntimeChange: boolean; + readonly nextHost: PreparedDevelopmentApplicationHost; + readonly nextHostFingerprint: string; + readonly nextRuntimeFingerprint: string; + }): Promise { + let committedHost = input.nextHost; + + if (!input.hasRuntimeChange) { + await discardDevelopmentGeneration(input.nextHost.generation); + committedHost = { + ...input.nextHost, + generation: this.#currentHost.generation, + }; + } + + const nitro = await createDevelopmentApplicationNitro(committedHost); + const payload = await buildDevelopmentHostCandidate({ host: committedHost, nitro }); + + // Worker first, pointer second: a failed candidate leaves both the + // previous worker and the previous pointer untouched, and the + // just-swapped worker serves the previous generation correctly for the + // instant before activation because hosts are generation-neutral. + const workspace = committedHost.workspace; + await this.#devServer.replaceWorker({ + dispose: async () => await removeDevelopmentHostWorkspace(workspace), + entry: payload.entry, + workerData: payload.workerData, + }); + // The worker swap is the transaction's commit point: the retired worker + // is already draining, so a failure past this line must not discard the + // live host through the candidate-rollback path. A failed activation + // keeps the swapped worker serving the previous generation and leaves + // the runtime fingerprint stale so the next rebuild retries activation. + if (input.hasRuntimeChange) { + try { + await activateDevelopmentGeneration({ + appRoot: committedHost.appRoot, + generation: committedHost.generation, + }); + } catch (error) { + await discardDevelopmentGeneration(committedHost.generation).catch(() => undefined); + this.#commitState( + { ...committedHost, generation: this.#currentHost.generation }, + input.nextHostFingerprint, + this.#currentRuntimeFingerprint, + ); + throw new PostCommitDevelopmentRebuildError(error); + } + } + + this.#commitState( + committedHost, + input.nextHostFingerprint, + input.hasRuntimeChange ? input.nextRuntimeFingerprint : this.#currentRuntimeFingerprint, + ); + return { host: committedHost, kind: "structural" }; + } + + #commitState( + host: PreparedDevelopmentApplicationHost, + hostFingerprint: string, + runtimeFingerprint: string, + ): void { + this.#currentHost = host; + this.#currentHostFingerprint = hostFingerprint; + this.#currentRuntimeFingerprint = runtimeFingerprint; + } +} + +function retainActiveHostWorkspace( + activeHost: PreparedDevelopmentApplicationHost, + nextHost: PreparedDevelopmentApplicationHost, +): PreparedDevelopmentApplicationHost { + return { + ...nextHost, + compiledArtifacts: activeHost.compiledArtifacts, + workflowBuildDir: activeHost.workflowBuildDir, + workspace: activeHost.workspace, + }; +} + +function startSandboxPrewarmAfterCommit( + host: PreparedDevelopmentApplicationHost, + changedPaths: readonly string[], +): void { + if (!hasSandboxRelatedChange(host.compileResult.project.agentRoot, changedPaths)) { + return; + } + const artifactsConfig = createDevelopmentNitroArtifactsConfig({ appRoot: host.appRoot }); + startDevelopmentSandboxPrewarmInBackground({ + appRoot: host.appRoot, + compiledArtifactsSource: resolveNitroCompiledArtifactsSource(artifactsConfig), + log: (message) => console.log(message), + }); +} + +function hasSandboxRelatedChange(agentRoot: string, changedPaths: readonly string[]): boolean { + return changedPaths.some((path) => { + const relativePath = relative(resolve(agentRoot), resolve(path)); + const segments = relativePath.split(/[\\/]/u); + if (segments[0] === ".." || relativePath === "") { + return false; + } + return ( + segments[0] === "sandbox.ts" || + segments[0] === "sandbox" || + segments[0] === "workspace" || + segments[0] === "skills" + ); + }); +} + +async function discardPreparedHost(host: PreparedDevelopmentApplicationHost): Promise { + const cleanup = await Promise.allSettled([ + discardDevelopmentGeneration(host.generation), + removeDevelopmentHostWorkspace(host.workspace), + ]); + const errors = cleanup.flatMap((result) => (result.status === "rejected" ? [result.reason] : [])); + if (errors.length > 0) { + throw new AggregateError( + errors, + `Failed to discard development host "${host.workspace.rootDir}".`, + ); + } +} + +async function discardFailedHost( + cause: unknown, + host: PreparedDevelopmentApplicationHost, +): Promise { + try { + await discardPreparedHost(host); + return cause; + } catch (cleanupError) { + return new AggregateError([cause, cleanupError], "Development rebuild rollback failed.", { + cause, + }); + } +} diff --git a/packages/eve/src/internal/nitro/host/dev-authored-source-watcher.scenario.test.ts b/packages/eve/src/internal/nitro/host/dev-authored-source-watcher.scenario.test.ts index 3678787e9..f4ab5ee3b 100644 --- a/packages/eve/src/internal/nitro/host/dev-authored-source-watcher.scenario.test.ts +++ b/packages/eve/src/internal/nitro/host/dev-authored-source-watcher.scenario.test.ts @@ -1,618 +1,224 @@ import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; -import type { Nitro } from "nitro/types"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { CompiledChannelEntry } from "#compiler/manifest.js"; -import { - EVE_SCHEDULE_TASK_NAME_PREFIX, - type ScheduleRegistration, -} from "#runtime/schedules/register.js"; -import type { PreparedApplicationHost } from "#internal/nitro/host/types.js"; +import { createCompiledAgentManifest } from "#compiler/manifest.js"; +import type { DevelopmentAuthoredRebuildCoordinator } from "#internal/nitro/host/dev-authored-rebuild-coordinator.js"; +import type { PreparedDevelopmentApplicationHost } from "#internal/nitro/host/types.js"; +import { STRUCTURAL_RELOAD_LOG_LINE } from "#internal/nitro/host/dev-watcher-log.js"; const mockedWatcher = vi.hoisted(() => { let onAllHandler: ((event: string, changedPath: string) => void) | undefined; - let onErrorHandler: ((error: unknown) => void) | undefined; let onReadyHandler: (() => void) | undefined; - let shouldDeferReady = false; + let deferReady = false; const add = vi.fn(); const close = vi.fn().mockResolvedValue(undefined); - const on = vi.fn( + const unwatch = vi.fn(); + const watch = vi.fn( ( - event: string, - handler: - | ((event: string, changedPath: string) => void) - | (() => void) - | ((error: unknown) => void), - ) => { - if (event === "all") { - onAllHandler = handler as (event: string, changedPath: string) => void; - return; - } - - if (event === "error") { - onErrorHandler = handler as (error: unknown) => void; - return; - } - - if (event === "ready") { - onReadyHandler = handler as () => void; - - if (shouldDeferReady) { - return; + _paths: string | readonly string[], + _options?: { readonly ignored?: (path: string) => boolean }, + ) => ({ + add, + close, + on(event: string, handler: unknown) { + if (event === "all") { + onAllHandler = handler as (event: string, changedPath: string) => void; } - - queueMicrotask(() => { - onReadyHandler?.(); - }); - } - }, + if (event === "ready") { + onReadyHandler = handler as () => void; + if (!deferReady) { + queueMicrotask(onReadyHandler); + } + } + }, + unwatch, + }), ); - const unwatch = vi.fn(); - const watch = vi.fn(() => ({ - add, - close, - on, - unwatch, - })); return { add, close, - deferReady(): void { - shouldDeferReady = true; + deferReadiness() { + deferReady = true; }, - emit(event: string, changedPath: string): void { - if (onAllHandler === undefined) { - throw new Error("Watcher callback was not registered."); - } - - onAllHandler(event, changedPath); + emit(event: string, changedPath: string) { + onAllHandler?.(event, changedPath); }, - on, - ready(): void { - const readyHandler = onReadyHandler; - onReadyHandler = undefined; - - if (readyHandler !== undefined) { - queueMicrotask(() => { - readyHandler(); - }); - } + ready() { + queueMicrotask(() => onReadyHandler?.()); }, - reset(): void { + reset() { onAllHandler = undefined; - onErrorHandler = undefined; onReadyHandler = undefined; - shouldDeferReady = false; + deferReady = false; add.mockClear(); close.mockClear(); - on.mockClear(); unwatch.mockClear(); watch.mockClear(); }, - triggerError(error: unknown): void { - if (onErrorHandler !== undefined) { - onErrorHandler(error); - } - }, unwatch, watch, }; }); -const prepareDevelopmentApplicationHostMock = vi.hoisted(() => vi.fn()); -const clearCompiledRuntimeAgentBundleCacheMock = vi.hoisted(() => vi.fn()); -const startDevelopmentSandboxPrewarmInBackgroundMock = vi.hoisted(() => vi.fn()); -const resolveNitroCompiledArtifactsSourceMock = vi.hoisted(() => - vi.fn((config: { readonly appRoot?: string }) => ({ - appRoot: `${config.appRoot ?? "/tmp/eve-test"}/.eve/dev-runtime-test`, - kind: "disk" as const, - moduleMapLoaderPath: "/tmp/eve-package/authored-module-map-loader.ts", - })), -); -const temporaryDirectories: string[] = []; - -vi.mock("#compiled/chokidar/index.js", () => ({ - watch: mockedWatcher.watch, -})); - -vi.mock("./prepare-application-host.js", () => ({ - prepareDevelopmentApplicationHost: prepareDevelopmentApplicationHostMock, -})); - -vi.mock("#execution/sandbox/development-prewarm.js", () => ({ - startDevelopmentSandboxPrewarmInBackground: startDevelopmentSandboxPrewarmInBackgroundMock, -})); - -vi.mock("#internal/nitro/routes/runtime-artifacts.js", () => ({ - resolveNitroCompiledArtifactsSource: resolveNitroCompiledArtifactsSourceMock, -})); - -vi.mock("../../../runtime/sessions/compiled-agent-cache.js", () => ({ - clearCompiledRuntimeAgentBundleCache: clearCompiledRuntimeAgentBundleCacheMock, -})); +vi.mock("#compiled/chokidar/index.js", () => ({ watch: mockedWatcher.watch })); import { startAuthoredSourceWatcher } from "#internal/nitro/host/dev-authored-source-watcher.js"; -interface NitroStub { - callHook: ReturnType; - nitro: Nitro; - syncRouting: ReturnType; -} - -interface PreparedHostInput { - appRoot?: string; - channels?: readonly CompiledChannelEntry[]; - scheduleRegistrations?: readonly ScheduleRegistration[]; -} - const DEFAULT_APP_ROOT = "/tmp/eve-dev-hmr"; +const temporaryDirectories: string[] = []; beforeEach(() => { vi.useFakeTimers(); - vi.spyOn(console, "error").mockImplementation(() => {}); - vi.spyOn(console, "log").mockImplementation(() => {}); - + vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(console, "log").mockImplementation(() => undefined); mockedWatcher.reset(); - prepareDevelopmentApplicationHostMock.mockReset(); - clearCompiledRuntimeAgentBundleCacheMock.mockReset(); - startDevelopmentSandboxPrewarmInBackgroundMock.mockReset(); - resolveNitroCompiledArtifactsSourceMock.mockClear(); }); afterEach(async () => { await Promise.all( - temporaryDirectories.splice(0).map((directoryPath) => - rm(directoryPath, { - force: true, - recursive: true, - }), - ), + temporaryDirectories + .splice(0) + .map(async (path) => await rm(path, { force: true, recursive: true })), ); vi.restoreAllMocks(); vi.useRealTimers(); }); describe("startAuthoredSourceWatcher", () => { - it("rebuilds when a local setup action knows authored source changed", async () => { - const previousHost = createPreparedHost(); - const nextHost = createPreparedHost(); - const nitroStub = createNitroStub(); - prepareDevelopmentApplicationHostMock.mockResolvedValueOnce(nextHost); - - const watcher = await startAuthoredSourceWatcher({ - nitro: nitroStub.nitro, - preparedHost: previousHost, - }); + it("forces a transactional rebuild for local setup actions", async () => { + const host = createPreparedHost(); + const coordinator = createCoordinator(host); + const watcher = await startAuthoredSourceWatcher({ coordinator, preparedHost: host }); try { await watcher.rebuild(); - - expect(prepareDevelopmentApplicationHostMock).toHaveBeenCalledWith(previousHost.appRoot); - expect(clearCompiledRuntimeAgentBundleCacheMock).toHaveBeenCalledTimes(1); + expect(coordinator.rebuild).toHaveBeenCalledWith({ changedPaths: [] }); } finally { await watcher.close(); } }); - it("ignores generated output directories while watching authored source roots", async () => { - const watcher = await startAuthoredSourceWatcher({ - nitro: createNitroStub().nitro, - preparedHost: createPreparedHost(), - }); - - try { - const ignored = getInitialIgnoredPredicate(); - - expect(ignored("/repo/packages/eve/.generated/compiled/chokidar/index.js")).toBe(true); - expect(ignored("/repo/packages/eve/.eve/workflow-cache/cache/manifest.json")).toBe(true); - expect(ignored("/repo/packages/eve/.next/server/app.js")).toBe(true); - expect(ignored("/repo/packages/eve/build/index.js")).toBe(true); - expect(ignored("/repo/packages/eve/dist/src/cli/run.js")).toBe(true); - expect(ignored("/repo/packages/eve/node_modules")).toBe(true); - expect(ignored("/repo/apps/fixtures/weather-agent/node_modules/eve")).toBe(true); - expect(ignored("/repo/apps/fixtures/weather-agent/agent/tools/get_weather.ts")).toBe(false); - } finally { - await watcher.close(); - } - }); - - it("drops initial watcher events emitted before Chokidar is ready", async () => { - mockedWatcher.deferReady(); - - const watcherPromise = startAuthoredSourceWatcher({ - nitro: createNitroStub().nitro, - preparedHost: createPreparedHost(), - }); - let watcher: Awaited> | undefined; - - try { - await vi.waitFor(() => { - expect(mockedWatcher.on).toHaveBeenCalledWith("all", expect.any(Function)); - }); - - mockedWatcher.emit("add", join(DEFAULT_APP_ROOT, "node_modules", "eve")); - await advanceDebounce(); - - expect(prepareDevelopmentApplicationHostMock).not.toHaveBeenCalled(); - - mockedWatcher.ready(); - watcher = await watcherPromise; - await advanceDebounce(); - - expect(prepareDevelopmentApplicationHostMock).not.toHaveBeenCalled(); - expect(getConsoleLogMessages().some((message) => message.includes("change detected"))).toBe( - false, - ); - } finally { - if (watcher === undefined) { - mockedWatcher.ready(); - watcher = await watcherPromise; - } - - await watcher.close(); - } - }); - - it("rebuilds authored artifacts without reloading Nitro when wiring is unchanged", async () => { - const registration = createScheduleRegistration({ - cron: "0 * * * *", - id: "hourly", - }); - const previousHost = createPreparedHost({ - scheduleRegistrations: [registration], - }); - const nextHost = createPreparedHost({ - scheduleRegistrations: [{ ...registration }], - }); - const nitroStub = createNitroStub({ - scheduledTasks: { - [registration.cron]: registration.taskName, - }, - tasks: { - [registration.taskName]: { - description: registration.description, - handler: `#eve-schedule-task/${registration.taskName}`, - }, - }, - }); - - prepareDevelopmentApplicationHostMock.mockResolvedValueOnce(nextHost); - + it("ignores generated directories while watching authored roots", async () => { + const host = createPreparedHost(); const watcher = await startAuthoredSourceWatcher({ - nitro: nitroStub.nitro, - preparedHost: previousHost, + coordinator: createCoordinator(host), + preparedHost: host, }); try { - await triggerChangeEvent(); - - expect(prepareDevelopmentApplicationHostMock).toHaveBeenCalledTimes(1); - expect(getConsoleLogMessages()).toEqual( - expect.arrayContaining([ - "[eve:dev] change detected (1 event: change agent/agent.ts), rebuilding authored artifacts...", - ]), - ); - expect(prepareDevelopmentApplicationHostMock).toHaveBeenCalledWith(previousHost.appRoot); - expect(startDevelopmentSandboxPrewarmInBackgroundMock).not.toHaveBeenCalled(); - expect(clearCompiledRuntimeAgentBundleCacheMock).toHaveBeenCalledTimes(1); - expect(nitroStub.callHook).not.toHaveBeenCalled(); + const ignored = getIgnoredPredicate(); + expect(ignored(join(host.appRoot, ".eve", "dev-hosts", "candidate"))).toBe(true); + expect(ignored(join(host.appRoot, "node_modules", "eve"))).toBe(true); + expect(ignored(join(host.appRoot, "agent", "tools", "weather.ts"))).toBe(false); } finally { await watcher.close(); } }); - it("starts sandbox prewarm only for sandbox-related changes", async () => { - const previousHost = createPreparedHost(); - const nextHost = createPreparedHost(); - const nitroStub = createNitroStub(); + it("drops initial and directory-only events", async () => { + mockedWatcher.deferReadiness(); + const host = createPreparedHost(); + const coordinator = createCoordinator(host); + const watcherPromise = startAuthoredSourceWatcher({ coordinator, preparedHost: host }); - prepareDevelopmentApplicationHostMock.mockResolvedValue(nextHost); - - const watcher = await startAuthoredSourceWatcher({ - nitro: nitroStub.nitro, - preparedHost: previousHost, - }); + await vi.waitFor(() => expect(mockedWatcher.watch).toHaveBeenCalledOnce()); + mockedWatcher.emit("add", join(host.appRoot, "agent", "tools", "initial.ts")); + mockedWatcher.ready(); + const watcher = await watcherPromise; try { - mockedWatcher.emit("change", join(previousHost.appRoot, "agent", "tools", "a.ts")); - await advanceDebounce(); - await watcher.flush(); - - expect(startDevelopmentSandboxPrewarmInBackgroundMock).not.toHaveBeenCalled(); - - mockedWatcher.emit("change", join(previousHost.appRoot, "agent", "sandbox.ts")); - await advanceDebounce(); + mockedWatcher.emit("addDir", join(host.appRoot, "agent", "tools")); + mockedWatcher.emit("unlinkDir", join(host.appRoot, "agent", "skills")); + await vi.advanceTimersByTimeAsync(200); await watcher.flush(); - - expect(startDevelopmentSandboxPrewarmInBackgroundMock).toHaveBeenCalledWith({ - appRoot: nextHost.appRoot, - compiledArtifactsSource: { - appRoot: join(nextHost.appRoot, ".eve", "dev-runtime-test"), - kind: "disk", - moduleMapLoaderPath: "/tmp/eve-package/authored-module-map-loader.ts", - }, - log: expect.any(Function), - }); + expect(coordinator.rebuild).not.toHaveBeenCalled(); } finally { await watcher.close(); } }); - it("coalesces non-agent changes during an in-flight rebuild into one follow-up rebuild", async () => { - const firstRebuild = createDeferred(); - const previousHost = createPreparedHost(); - const nextHost = createPreparedHost(); - const nitroStub = createNitroStub(); - - prepareDevelopmentApplicationHostMock.mockResolvedValue(nextHost); - prepareDevelopmentApplicationHostMock.mockReturnValueOnce(firstRebuild.promise); - - const watcher = await startAuthoredSourceWatcher({ - nitro: nitroStub.nitro, - preparedHost: previousHost, - }); + it("coalesces edits received during an in-flight rebuild", async () => { + const host = createPreparedHost(); + const first = createDeferred<{ host: PreparedDevelopmentApplicationHost; kind: "runtime" }>(); + const coordinator = createCoordinator(host); + vi.mocked(coordinator.rebuild) + .mockReturnValueOnce(first.promise) + .mockResolvedValue({ host, kind: "runtime" }); + const watcher = await startAuthoredSourceWatcher({ coordinator, preparedHost: host }); try { - // Editing any authored file — not just agent.ts — rebuilds on save. - mockedWatcher.emit("change", join(previousHost.appRoot, "agent", "instructions.md")); - await advanceDebounce(); - - expect(prepareDevelopmentApplicationHostMock).toHaveBeenCalledTimes(1); - - // Further saves while that rebuild is in flight queue behind it. - mockedWatcher.emit("change", join(previousHost.appRoot, "agent", "tools", "a.ts")); - await advanceDebounce(); - mockedWatcher.emit("change", join(previousHost.appRoot, "agent", "tools", "b.ts")); - await advanceDebounce(); - - expect(prepareDevelopmentApplicationHostMock).toHaveBeenCalledTimes(1); - - firstRebuild.resolve(nextHost); - await vi.waitFor(() => { - expect(getConsoleLogMessages()).toEqual( - expect.arrayContaining(["[eve:dev] authored artifacts updated."]), - ); - }); - await settleAsyncWork(); + mockedWatcher.emit("change", join(host.appRoot, "agent", "instructions.md")); + await vi.advanceTimersByTimeAsync(200); + mockedWatcher.emit("change", join(host.appRoot, "agent", "tools", "a.ts")); + mockedWatcher.emit("change", join(host.appRoot, "agent", "tools", "b.ts")); + await vi.advanceTimersByTimeAsync(200); + + expect(coordinator.rebuild).toHaveBeenCalledTimes(1); + first.resolve({ host, kind: "runtime" }); + await vi.waitFor(() => expect(coordinator.rebuild).toHaveBeenCalledTimes(2)); await watcher.flush(); - - // a.ts and b.ts collapse into exactly one additional rebuild. - expect(prepareDevelopmentApplicationHostMock).toHaveBeenCalledTimes(2); - expect(getConsoleLogMessages()).not.toEqual( - expect.arrayContaining([ - "[eve:dev] change detected (0 events), rebuilding authored artifacts...", - ]), - ); } finally { await watcher.close(); } }); - it("reloads Nitro when channel routing changes", async () => { - const previousHost = createPreparedHost({ - channels: [], - }); - const nextHost = createPreparedHost({ - channels: [ - createChannelEntry({ - name: "slack", - urlPath: "/slack", - }), - ], - }); - const nitroStub = createNitroStub(); - - prepareDevelopmentApplicationHostMock.mockResolvedValueOnce(nextHost); - - const watcher = await startAuthoredSourceWatcher({ - nitro: nitroStub.nitro, - preparedHost: previousHost, - }); + it("reports a structural commit only after the coordinator completes", async () => { + const host = createPreparedHost(); + const coordinator = createCoordinator(host); + vi.mocked(coordinator.rebuild).mockResolvedValue({ host, kind: "structural" }); + const watcher = await startAuthoredSourceWatcher({ coordinator, preparedHost: host }); try { - await triggerChangeEvent(); - - expect(nitroStub.syncRouting).toHaveBeenCalledTimes(1); - expect(nitroStub.callHook).toHaveBeenCalledTimes(1); - expect(nitroStub.callHook).toHaveBeenCalledWith("rollup:reload"); - expect(nitroStub.nitro.options.handlers).toContainEqual({ - handler: "#nitro/virtual/eve-channel/POST /slack", - method: "POST", - route: "/slack", - }); - expect(nitroStub.nitro.options.virtual["#nitro/virtual/eve-channel/POST /slack"]).toContain( - "channel-dispatch.ts", - ); - } finally { - await watcher.close(); - } - }); - - it("never registers authored schedules in dev when registrations change", async () => { - // `eve dev` intentionally does not register Nitro scheduled tasks for - // authored schedules — production cron firing in dev would invoke every - // schedule on every save. The only dev-time entry point is the dev-only - // `POST /eve/v1/dev/schedules/:scheduleId` route. - const previousRegistration = createScheduleRegistration({ - cron: "0 * * * *", - id: "hourly", - }); - const nextRegistration = createScheduleRegistration({ - cron: "0 0 * * *", - id: "nightly", - }); - const previousHost = createPreparedHost({ - scheduleRegistrations: [previousRegistration], - }); - const nextHost = createPreparedHost({ - scheduleRegistrations: [nextRegistration], - }); - const nitroStub = createNitroStub(); - - prepareDevelopmentApplicationHostMock.mockResolvedValueOnce(nextHost); - - const watcher = await startAuthoredSourceWatcher({ - nitro: nitroStub.nitro, - preparedHost: previousHost, - }); - - try { - await triggerChangeEvent(); - - expect(nitroStub.nitro.options.tasks[previousRegistration.taskName]).toBeUndefined(); - expect(nitroStub.nitro.options.tasks[nextRegistration.taskName]).toBeUndefined(); - expect(nitroStub.nitro.options.scheduledTasks[previousRegistration.cron]).toBeUndefined(); - expect(nitroStub.nitro.options.scheduledTasks[nextRegistration.cron]).toBeUndefined(); - expect(nitroStub.callHook).not.toHaveBeenCalled(); + mockedWatcher.emit("change", join(host.appRoot, "agent", "channels", "webhook.ts")); + await vi.advanceTimersByTimeAsync(200); + await watcher.flush(); + expect(console.log).toHaveBeenCalledWith(STRUCTURAL_RELOAD_LOG_LINE); } finally { await watcher.close(); } }); - it("watches root config files, lockfiles, and tsconfig extends chains", async () => { - const workspaceRoot = await mkdtemp(join(tmpdir(), "eve-dev-hmr-watch-root-")); + it("watches root config, env, workspace lockfiles, and tsconfig extends", async () => { + const workspaceRoot = await mkdtemp(join(tmpdir(), "eve-dev-watch-root-")); const appRoot = join(workspaceRoot, "apps", "watch-agent"); - const appTsConfigPath = join(appRoot, "tsconfig.json"); - const workspaceBaseTsConfigPath = join(workspaceRoot, "tsconfig.base.json"); - const workspaceSharedTsConfigPath = join(workspaceRoot, "tsconfig.shared.json"); - temporaryDirectories.push(workspaceRoot); - await mkdir(join(appRoot, "agent"), { - recursive: true, - }); + await mkdir(join(appRoot, "agent"), { recursive: true }); await writeFile(join(workspaceRoot, "pnpm-workspace.yaml"), "packages:\n - apps/*\n"); + await writeFile(join(appRoot, "package.json"), '{"name":"watch-agent","type":"module"}\n'); + await writeFile(join(appRoot, "tsconfig.json"), '{"extends":"../../tsconfig.base.json"}\n'); await writeFile( - join(appRoot, "package.json"), - JSON.stringify( - { - name: "watch-agent", - private: true, - type: "module", - }, - null, - 2, - ), - ); - await writeFile( - appTsConfigPath, - [ - "{", - " // root tsconfig extends shared workspace-level configs", - ' "extends": [', - ' "../../tsconfig.base.json",', - ' "../../tsconfig.shared.json",', - " ],", - ' "compilerOptions": { "module": "NodeNext" }', - "}", - "", - ].join("\n"), - ); - await writeFile( - workspaceBaseTsConfigPath, - ['{ "compilerOptions": { "strict": true } }', ""].join("\n"), - ); - await writeFile( - workspaceSharedTsConfigPath, - ['{ "compilerOptions": { "exactOptionalPropertyTypes": true } }', ""].join("\n"), + join(workspaceRoot, "tsconfig.base.json"), + '{"compilerOptions":{"strict":true}}\n', ); - - const nitroStub = createNitroStub(); + const host = createPreparedHost(appRoot); const watcher = await startAuthoredSourceWatcher({ - nitro: nitroStub.nitro, - preparedHost: createPreparedHost({ - appRoot, - }), + coordinator: createCoordinator(host), + preparedHost: host, }); try { - const firstCall = mockedWatcher.watch.mock.calls[0] as [unknown, ...unknown[]] | undefined; - - if (firstCall === undefined) { - throw new Error("Expected the authored source watcher to call chokidar.watch."); - } - - const watchPaths = firstCall[0]; - const resolvedWatchPaths = Array.isArray(watchPaths) - ? (watchPaths as string[]) - : typeof watchPaths === "string" - ? [watchPaths] - : []; - - expect(resolvedWatchPaths).toContain(join(appRoot, "package.json")); - expect(resolvedWatchPaths).toContain(join(appRoot, "jsconfig.json")); - expect(resolvedWatchPaths).toContain(join(appRoot, "tsconfig.json")); - expect(resolvedWatchPaths).toContain(join(appRoot, "tsconfig.*.json")); - expect(resolvedWatchPaths).toContain(join(appRoot, ".env.development.local")); - expect(resolvedWatchPaths).toContain(join(appRoot, ".env.local")); - expect(resolvedWatchPaths).toContain(join(appRoot, ".env.development")); - expect(resolvedWatchPaths).toContain(join(appRoot, ".env")); - expect(resolvedWatchPaths).toContain(join(appRoot, "pnpm-lock.yaml")); - expect(resolvedWatchPaths).toContain(join(workspaceRoot, "pnpm-lock.yaml")); - expect(resolvedWatchPaths).toContain(workspaceBaseTsConfigPath); - expect(resolvedWatchPaths).toContain(workspaceSharedTsConfigPath); + const paths = getInitialWatchPaths(); + expect(paths).toContain(join(appRoot, "package.json")); + expect(paths).toContain(join(appRoot, ".env.local")); + expect(paths).toContain(join(workspaceRoot, "pnpm-lock.yaml")); + expect(paths).toContain(join(workspaceRoot, "tsconfig.base.json")); } finally { await watcher.close(); } }); - it("does not watch ancestor lockfiles when the app has no workspace marker", async () => { - const tempRoot = await mkdtemp(join(tmpdir(), "eve-dev-hmr-standalone-root-")); - const appRoot = join(tempRoot, "standalone-agent"); - - temporaryDirectories.push(tempRoot); - await mkdir(join(appRoot, "agent"), { - recursive: true, - }); - await writeFile( - join(appRoot, "package.json"), - JSON.stringify( - { - name: "standalone-agent", - private: true, - type: "module", - }, - null, - 2, - ), - ); - - const nitroStub = createNitroStub(); - const watcher = await startAuthoredSourceWatcher({ - nitro: nitroStub.nitro, - preparedHost: createPreparedHost({ - appRoot, - }), - }); - - try { - const watchPaths = getInitialWatchPaths(); - - expect(watchPaths).toContain(join(appRoot, "pnpm-lock.yaml")); - expect(watchPaths).not.toContain(join(dirname(appRoot), "pnpm-lock.yaml")); - } finally { - await watcher.close(); - } - }); - - it("watches local workspace package roots copied into dev runtime snapshots", async () => { - const workspaceRoot = await mkdtemp(join(tmpdir(), "eve-dev-hmr-watch-linked-package-")); + it("watches local workspace dependency roots rather than their node_modules links", async () => { + const workspaceRoot = await mkdtemp(join(tmpdir(), "eve-dev-watch-linked-")); const appRoot = join(workspaceRoot, "apps", "watch-agent"); const packageRoot = join(workspaceRoot, "packages", "shared"); - const packageLinkPath = join(appRoot, "node_modules", "@repo", "shared"); - + const packageLink = join(appRoot, "node_modules", "@repo", "shared"); temporaryDirectories.push(workspaceRoot); - await mkdir(join(appRoot, "agent"), { - recursive: true, - }); - await mkdir(join(appRoot, "node_modules", "@repo"), { - recursive: true, - }); - await mkdir(join(packageRoot, "src"), { - recursive: true, - }); + await mkdir(join(appRoot, "agent"), { recursive: true }); + await mkdir(join(appRoot, "node_modules", "@repo"), { recursive: true }); + await mkdir(join(packageRoot, "src"), { recursive: true }); await writeFile( join(workspaceRoot, "pnpm-workspace.yaml"), "packages:\n - apps/*\n - packages/*\n", @@ -620,300 +226,161 @@ describe("startAuthoredSourceWatcher", () => { await writeFile(join(workspaceRoot, "package.json"), '{"type":"module"}\n'); await writeFile( join(appRoot, "package.json"), - JSON.stringify( - { - dependencies: { - "@repo/shared": "workspace:*", - }, - name: "watch-agent", - private: true, - type: "module", - }, - null, - 2, - ), + '{"dependencies":{"@repo/shared":"workspace:*"},"type":"module"}\n', ); - await writeFile(join(appRoot, "tsconfig.json"), '{ "compilerOptions": { "strict": true } }\n'); await writeFile( join(packageRoot, "package.json"), - JSON.stringify( - { - exports: "./src/index.ts", - name: "@repo/shared", - type: "module", - }, - null, - 2, - ), + '{"name":"@repo/shared","exports":"./src/index.ts","type":"module"}\n', ); await writeFile(join(packageRoot, "src", "index.ts"), "export const shared = true;\n"); - await symlink(packageRoot, packageLinkPath, "junction"); - - const nitroStub = createNitroStub(); + await symlink(packageRoot, packageLink, "junction"); + const host = createPreparedHost(appRoot); const watcher = await startAuthoredSourceWatcher({ - nitro: nitroStub.nitro, - preparedHost: createPreparedHost({ - appRoot, - }), + coordinator: createCoordinator(host), + preparedHost: host, }); try { - const watchPaths = getInitialWatchPaths(); - - expect(watchPaths).toContain(packageRoot); - expect(watchPaths).not.toContain(packageLinkPath); + const paths = getInitialWatchPaths(); + expect(paths).toContain(packageRoot); + expect(paths).not.toContain(packageLink); } finally { await watcher.close(); } }); - it("updates watched tsconfig extends targets when the extends chain changes", async () => { - const appRoot = await mkdtemp(join(tmpdir(), "eve-dev-hmr-watch-extends-")); - const agentRoot = join(appRoot, "agent"); - const initialBaseTsConfigPath = join(appRoot, "tsconfig.base.a.json"); - const nextBaseTsConfigPath = join(appRoot, "tsconfig.base.b.json"); - const appTsConfigPath = join(appRoot, "tsconfig.json"); - - temporaryDirectories.push(appRoot); - await mkdir(agentRoot, { - recursive: true, - }); - await writeFile( - join(appRoot, "package.json"), - JSON.stringify( - { - name: "watch-agent-extends", - private: true, - type: "module", - }, - null, - 2, - ), - ); - await writeFile( - appTsConfigPath, - ['{ "extends": "./tsconfig.base.a.json", "compilerOptions": { "strict": true } }', ""].join( - "\n", - ), - ); - await writeFile( - initialBaseTsConfigPath, - ['{ "compilerOptions": { "module": "NodeNext" } }', ""].join("\n"), - ); - - const previousHost = createPreparedHost({ appRoot }); - const nextHost = createPreparedHost({ appRoot }); - const nitroStub = createNitroStub(); - - prepareDevelopmentApplicationHostMock.mockResolvedValueOnce(nextHost); - + it("does not watch ancestor lockfiles for an app without a workspace marker", async () => { + const tempRoot = await mkdtemp(join(tmpdir(), "eve-dev-watch-standalone-")); + const appRoot = join(tempRoot, "standalone-agent"); + temporaryDirectories.push(tempRoot); + await mkdir(join(appRoot, "agent"), { recursive: true }); + await writeFile(join(appRoot, "package.json"), '{"name":"standalone-agent"}\n'); + const host = createPreparedHost(appRoot); const watcher = await startAuthoredSourceWatcher({ - nitro: nitroStub.nitro, - preparedHost: previousHost, + coordinator: createCoordinator(host), + preparedHost: host, }); - let watcherClosed = false; try { - mockedWatcher.add.mockClear(); - mockedWatcher.unwatch.mockClear(); - await writeFile( - nextBaseTsConfigPath, - ['{ "compilerOptions": { "module": "Node16" } }', ""].join("\n"), - ); - await writeFile( - appTsConfigPath, - ['{ "extends": "./tsconfig.base.b.json", "compilerOptions": { "strict": true } }', ""].join( - "\n", - ), - ); - await triggerChangeEvent(appTsConfigPath); - await watcher.flush(); - await watcher.close(); - watcherClosed = true; - - expect(mockedWatcher.add).toHaveBeenCalled(); - const addedPaths = mockedWatcher.add.mock.calls.flatMap((call) => { - const addedPath = call[0]; - return Array.isArray(addedPath) ? addedPath : addedPath === undefined ? [] : [addedPath]; - }); - expect(addedPaths).toContain(nextBaseTsConfigPath); + const paths = getInitialWatchPaths(); + expect(paths).toContain(join(appRoot, "pnpm-lock.yaml")); + expect(paths).not.toContain(join(dirname(appRoot), "pnpm-lock.yaml")); } finally { - if (!watcherClosed) { - await watcher.close(); - } + await watcher.close(); } }); -}); - -function getInitialWatchPaths(): string[] { - const calls: readonly (readonly unknown[])[] = mockedWatcher.watch.mock.calls; - const firstCall = calls[0]; - - if (firstCall === undefined) { - throw new Error("Expected the authored source watcher to call chokidar.watch."); - } - - const watchPaths = firstCall[0]; - - if (Array.isArray(watchPaths)) { - return watchPaths.filter((path): path is string => typeof path === "string"); - } - - if (typeof watchPaths === "string") { - return [watchPaths]; - } - - return []; -} -function getInitialIgnoredPredicate(): (path: string) => boolean { - const calls: readonly (readonly unknown[])[] = mockedWatcher.watch.mock.calls; - const firstCall = calls[0]; - - if (firstCall === undefined) { - throw new Error("Expected the authored source watcher to call chokidar.watch."); - } - - const options = firstCall[1]; - - if (!isObjectRecord(options)) { - throw new Error("Expected chokidar.watch to receive an options object."); - } - - if (typeof options.ignored !== "function") { - throw new Error("Expected chokidar.watch to receive an ignored path predicate."); - } - - return options.ignored as (path: string) => boolean; -} - -async function triggerChangeEvent( - changedPath: string = `${DEFAULT_APP_ROOT}/agent/agent.ts`, -): Promise { - mockedWatcher.emit("change", changedPath); - await advanceDebounce(); -} - -async function advanceDebounce(): Promise { - await vi.advanceTimersByTimeAsync(200); - await settleAsyncWork(); -} - -async function settleAsyncWork(): Promise { - await Promise.resolve(); - await Promise.resolve(); - await Promise.resolve(); -} + it("updates watched tsconfig targets only after a committed rebuild", async () => { + const appRoot = await mkdtemp(join(tmpdir(), "eve-dev-watch-extends-")); + temporaryDirectories.push(appRoot); + await mkdir(join(appRoot, "agent"), { recursive: true }); + await writeFile(join(appRoot, "package.json"), '{"name":"watch-agent","type":"module"}\n'); + await writeFile(join(appRoot, "tsconfig.base.a.json"), "{}\n"); + await writeFile(join(appRoot, "tsconfig.json"), '{"extends":"./tsconfig.base.a.json"}\n'); + const host = createPreparedHost(appRoot); + const coordinator = createCoordinator(host); + const watcher = await startAuthoredSourceWatcher({ coordinator, preparedHost: host }); -function getConsoleLogMessages(): string[] { - return vi.mocked(console.log).mock.calls.map((call) => String(call[0])); -} + try { + mockedWatcher.add.mockClear(); + await writeFile(join(appRoot, "tsconfig.base.b.json"), "{}\n"); + await writeFile(join(appRoot, "tsconfig.json"), '{"extends":"./tsconfig.base.b.json"}\n'); + vi.mocked(coordinator.rebuild).mockRejectedValueOnce(new Error("candidate rejected")); + mockedWatcher.emit("change", join(appRoot, "tsconfig.json")); + await vi.advanceTimersByTimeAsync(200); + await watcher.flush(); + expect(mockedWatcher.add).not.toHaveBeenCalled(); -function createDeferred() { - let resolve!: (value: T | PromiseLike) => void; - let reject!: (reason?: unknown) => void; - const promise = new Promise((resolvePromise, rejectPromise) => { - resolve = resolvePromise; - reject = rejectPromise; + mockedWatcher.emit("change", join(appRoot, "tsconfig.json")); + await vi.advanceTimersByTimeAsync(200); + await watcher.flush(); + expect(mockedWatcher.add.mock.calls.flatMap((call) => call[0])).toContain( + join(appRoot, "tsconfig.base.b.json"), + ); + } finally { + await watcher.close(); + } }); +}); +function createPreparedHost( + appRoot: string = DEFAULT_APP_ROOT, +): PreparedDevelopmentApplicationHost { + const agentRoot = join(appRoot, "agent"); return { - promise, - reject, - resolve, - }; -} - -function isObjectRecord(value: unknown): value is Record { - return value !== null && typeof value === "object" && !Array.isArray(value); -} - -function createNitroStub( - input: { - handlers?: Nitro["options"]["handlers"]; - preset?: "vercel"; - scheduledTasks?: Nitro["options"]["scheduledTasks"]; - tasks?: Nitro["options"]["tasks"]; - virtual?: Nitro["options"]["virtual"]; - } = {}, -): NitroStub { - const callHook = vi.fn().mockResolvedValue(undefined); - const syncRouting = vi.fn(); - const nitro = { - hooks: { - callHook, + appRoot, + compileResult: { + diagnostics: [], + manifest: createCompiledAgentManifest({ + agentRoot, + appRoot, + config: { + model: { id: "openai/gpt-5-mini", routing: { kind: "gateway", target: "openai" } }, + name: "watch-agent", + }, + }), + metadata: {} as PreparedDevelopmentApplicationHost["compileResult"]["metadata"], + paths: {} as PreparedDevelopmentApplicationHost["compileResult"]["paths"], + project: { agentRoot, appRoot, layout: "flat" }, }, - options: { - dev: true, - experimental: {}, - handlers: input.handlers ?? [], - preset: input.preset, - scheduledTasks: input.scheduledTasks ?? {}, - tasks: input.tasks ?? {}, - virtual: input.virtual ?? {}, + compiledArtifacts: { + bootstrapPath: join(appRoot, ".eve", "dev-hosts", "test", "bootstrap.mjs"), + workflowWorldPluginPath: join(appRoot, ".eve", "dev-hosts", "test", "world.mjs"), }, - routing: { - sync: syncRouting, + generation: { + fingerprint: "runtime", + runtimeAppRoot: join(appRoot, ".eve", "dev-runtime", "snapshots", "test", "source", "app"), + snapshotRoot: join(appRoot, ".eve", "dev-runtime", "snapshots", "test"), + snapshotSourceRoot: join(appRoot, ".eve", "dev-runtime", "snapshots", "test", "source"), + sourceRoot: appRoot, + }, + scheduleRegistrations: [], + schedules: [], + workflowBuildDir: join(appRoot, ".eve", "dev-hosts", "test", "workflow"), + workspace: { + artifactsDir: join(appRoot, ".eve", "dev-hosts", "test", "artifacts"), + compilerArtifactsDir: join(appRoot, ".eve", "dev-hosts", "test", "compiler"), + nitroBuildDir: join(appRoot, ".eve", "dev-hosts", "test", "nitro"), + nitroOutputDir: join(appRoot, ".eve", "dev-hosts", "test", "output"), + rootDir: join(appRoot, ".eve", "dev-hosts", "test"), + workflowBuildDir: join(appRoot, ".eve", "dev-hosts", "test", "workflow"), }, - } as unknown as Nitro; - - return { - callHook, - nitro, - syncRouting, }; } -function createPreparedHost(input: PreparedHostInput = {}): PreparedApplicationHost { - const appRoot = input.appRoot ?? DEFAULT_APP_ROOT; - +function createCoordinator( + host: PreparedDevelopmentApplicationHost, +): DevelopmentAuthoredRebuildCoordinator { return { - appRoot, - compileResult: { - manifest: { - channels: input.channels ?? [ - createChannelEntry({ - name: "slack", - urlPath: "/slack", - }), - ], - config: {}, - sandbox: null, - }, - project: { - agentRoot: `${appRoot}/agent`, - }, - } as unknown as PreparedApplicationHost["compileResult"], - compiledArtifacts: {} as PreparedApplicationHost["compiledArtifacts"], - scheduleRegistrations: input.scheduleRegistrations ?? [], - schedules: [], - workflowBuildDir: `${appRoot}/.workflow-build`, + rebuild: vi.fn(async () => ({ host, kind: "runtime" as const })), }; } -function createChannelEntry(input: { - method?: "GET" | "POST"; - name: string; - urlPath: string; -}): CompiledChannelEntry { - return { - kind: "channel", - logicalPath: `channels/${input.name}.ts`, - method: input.method ?? "POST", - name: input.name, - sourceId: `channel:${input.name}`, - sourceKind: "module", - urlPath: input.urlPath, - }; +function getInitialWatchPaths(): string[] { + const value = mockedWatcher.watch.mock.calls[0]?.[0]; + if (!Array.isArray(value)) { + throw new Error("Expected chokidar to receive an array of watch paths."); + } + return value as string[]; } -function createScheduleRegistration(input: { cron: string; id: string }): ScheduleRegistration { +function getIgnoredPredicate(): (path: string) => boolean { + const ignored = mockedWatcher.watch.mock.calls[0]?.[1]?.ignored; + if (ignored === undefined) { + throw new Error("Expected Chokidar to receive an ignored path predicate."); + } + return ignored; +} + +function createDeferred() { + let resolvePromise: ((value: T) => void) | undefined; + const promise = new Promise((resolve) => { + resolvePromise = resolve; + }); return { - cron: input.cron, - description: `Run schedule "${input.id}".`, - logicalPath: `agent/schedules/${input.id}.ts`, - scheduleId: input.id, - sourceId: `schedule:${input.id}`, - taskName: `${EVE_SCHEDULE_TASK_NAME_PREFIX}${input.id}`, + promise, + resolve(value: T) { + resolvePromise?.(value); + }, }; } diff --git a/packages/eve/src/internal/nitro/host/dev-authored-source-watcher.ts b/packages/eve/src/internal/nitro/host/dev-authored-source-watcher.ts index 030cec474..be2b2ff0e 100644 --- a/packages/eve/src/internal/nitro/host/dev-authored-source-watcher.ts +++ b/packages/eve/src/internal/nitro/host/dev-authored-source-watcher.ts @@ -1,24 +1,12 @@ import { existsSync } from "node:fs"; -import { dirname, join, relative, resolve } from "node:path"; +import { dirname, join, normalize, resolve, sep } from "node:path"; import { watch } from "#compiled/chokidar/index.js"; -import type { Nitro } from "nitro/types"; -import { clearCompiledRuntimeAgentBundleCache } from "#runtime/sessions/compiled-agent-cache.js"; import { toErrorMessage } from "#shared/errors.js"; import { resolveTsConfigDependencyPaths } from "#internal/application/tsconfig-dependencies.js"; -import { createDevelopmentNitroArtifactsConfig } from "#internal/nitro/host/artifacts-config.js"; import { resolveDevelopmentSourceSnapshotWatchPaths } from "#internal/nitro/dev-runtime-source-snapshot.js"; -import { startDevelopmentSandboxPrewarmInBackground } from "#execution/sandbox/development-prewarm.js"; -import { - computeChannelRouteRegistrations, - syncChannelVirtualHandlers, -} from "#internal/nitro/host/channel-routes.js"; -import { prepareDevelopmentApplicationHost } from "#internal/nitro/host/prepare-application-host.js"; -import { resolveNitroCompiledArtifactsSource } from "#internal/nitro/routes/runtime-artifacts.js"; -import type { PreparedApplicationHost } from "#internal/nitro/host/types.js"; -import { - getDevelopmentEnvironmentFilePaths, - loadDevelopmentEnvironmentFiles, -} from "#cli/dev/environment.js"; +import type { PreparedDevelopmentApplicationHost } from "#internal/nitro/host/types.js"; +import type { DevelopmentAuthoredRebuildCoordinator } from "#internal/nitro/host/dev-authored-rebuild-coordinator.js"; +import { getDevelopmentEnvironmentFilePaths } from "#cli/dev/environment.js"; import { AUTHORED_ARTIFACTS_UPDATED_LOG_LINE, STRUCTURAL_RELOAD_LOG_LINE, @@ -49,22 +37,6 @@ const WATCHER_IGNORED_DIRECTORY_NAMES = new Set([ "dist", "node_modules", ]); -type DevelopmentWatcherNitroOptions = Pick< - Nitro["options"], - "experimental" | "handlers" | "scheduledTasks" | "tasks" | "virtual" -> & - Partial>; - -interface DevelopmentWatcherNitro { - hooks: { - callHook: Nitro["hooks"]["callHook"]; - }; - options: DevelopmentWatcherNitroOptions; - routing: { - sync(): void; - }; -} - /** * Handle for the authored-source development watcher. */ @@ -81,8 +53,8 @@ export interface AuthoredSourceWatcherHandle { * triggers Nitro rebuild reloads only when structural runtime wiring changes. */ export async function startAuthoredSourceWatcher(input: { - nitro: DevelopmentWatcherNitro; - preparedHost: PreparedApplicationHost; + coordinator: DevelopmentAuthoredRebuildCoordinator; + preparedHost: PreparedDevelopmentApplicationHost; }): Promise { let currentHost = input.preparedHost; let closed = false; @@ -124,58 +96,21 @@ export async function startAuthoredSourceWatcher(input: { pendingEvents.clear(); pendingChangedPaths.clear(); const previousHost = currentHost; - const hasSandboxPrewarmChange = hasSandboxRelatedChange( - previousHost.compileResult.project.agentRoot, - changedPaths, - ); - const hasEnvironmentChange = hasDevelopmentEnvironmentFileChange( - previousHost.appRoot, - changedPaths, - ); if (changeEvents.length > 0) { console.log(formatChangeDetectedLogLine(previousHost.appRoot, changeEvents)); } try { - if (hasEnvironmentChange) { - loadDevelopmentEnvironmentFiles(previousHost.appRoot); - } + const result = await input.coordinator.rebuild({ changedPaths }); + currentHost = result.host; - const nextHost = await prepareDevelopmentApplicationHost(previousHost.appRoot); - const artifactsConfig = createDevelopmentNitroArtifactsConfig({ - appRoot: nextHost.appRoot, - }); - if (hasSandboxPrewarmChange) { - startDevelopmentSandboxPrewarmInBackground({ - appRoot: nextHost.appRoot, - compiledArtifactsSource: resolveNitroCompiledArtifactsSource(artifactsConfig), - log: (message) => console.log(message), - }); - } - const hasChannelRouteChanged = syncChannelVirtualHandlers(input.nitro, { - artifactsConfig, - next: computeChannelRouteRegistrations(nextHost), - previous: computeChannelRouteRegistrations(previousHost), - }); - clearCompiledRuntimeAgentBundleCache(); - currentHost = nextHost; - - // Schedule registrations are intentionally not reconciled here: - // `eve dev` never registers Nitro scheduled tasks for authored - // schedules. The only dev-time entry point is the dev-only - // `POST /eve/v1/dev/schedules/:scheduleId` route, which reads - // compiled registrations from disk on every request without - // needing Nitro wiring. - const hasStructuralChange = hasChannelRouteChanged || hasEnvironmentChange; - - if (hasStructuralChange) { + if (result.kind === "structural") { console.log(STRUCTURAL_RELOAD_LOG_LINE); - await input.nitro.hooks.callHook("rollup:reload"); } else { console.log(AUTHORED_ARTIFACTS_UPDATED_LOG_LINE); } - const nextWatchPaths = await resolveAuthoredWatchPaths(nextHost); + const nextWatchPaths = await resolveAuthoredWatchPaths(currentHost); currentWatchPathsByKey = syncWatcherPaths({ nextWatchPaths, previousWatchPathsByKey: currentWatchPathsByKey, @@ -205,7 +140,7 @@ export async function startAuthoredSourceWatcher(input: { await rebuild(true); }; watcher.on("all", (event, changedPath) => { - if (closed || !isWatcherReady) { + if (closed || !isWatcherReady || event === "addDir" || event === "unlinkDir") { return; } @@ -254,7 +189,9 @@ async function waitForWatcherReady(input: { }); } -async function resolveAuthoredWatchPaths(host: PreparedApplicationHost): Promise { +async function resolveAuthoredWatchPaths( + host: PreparedDevelopmentApplicationHost, +): Promise { const watchPaths = new Set([ host.compileResult.project.agentRoot, join(host.appRoot, "package.json"), @@ -332,41 +269,7 @@ function syncWatcherPaths(input: { } function toWatchPathKey(path: string): string { - return path.replaceAll("\\", "/"); -} - -function hasDevelopmentEnvironmentFileChange( - appRoot: string, - changedPaths: readonly string[], -): boolean { - const environmentFilePathKeys = new Set( - getDevelopmentEnvironmentFilePaths(appRoot).map((path) => toWatchPathKey(resolve(path))), - ); - - return changedPaths.some((path) => environmentFilePathKeys.has(toWatchPathKey(resolve(path)))); -} - -function hasSandboxRelatedChange(agentRoot: string, changedPaths: readonly string[]): boolean { - return changedPaths.some((path) => { - const agentRelativePath = toAgentRelativePath(agentRoot, path); - return ( - agentRelativePath === "sandbox.ts" || - agentRelativePath.startsWith("sandbox/") || - agentRelativePath === "workspace" || - agentRelativePath.startsWith("workspace/") || - agentRelativePath === "skills" || - agentRelativePath.startsWith("skills/") - ); - }); -} - -function toAgentRelativePath(agentRoot: string, path: string): string { - const relativePath = relative(resolve(agentRoot), resolve(path)); - const pathKey = toWatchPathKey(relativePath); - if (pathKey === ".." || pathKey.startsWith("../") || pathKey === "") { - return ""; - } - return pathKey; + return normalize(resolve(path)); } function resolveLockfileSearchDirectories(appRoot: string): string[] { @@ -399,7 +302,7 @@ async function resolveTsConfigWatchPaths(appRoot: string): Promise { } function shouldIgnoreWatcherPath(path: string): boolean { - const pathParts = path.replaceAll("\\", "/").split("/").filter(Boolean); + const pathParts = normalize(path).split(sep).filter(Boolean); return pathParts.some((part) => WATCHER_IGNORED_DIRECTORY_NAMES.has(part)); } diff --git a/packages/eve/src/internal/nitro/host/dev-environment-watcher.integration.test.ts b/packages/eve/src/internal/nitro/host/dev-environment-watcher.integration.test.ts index 28b568e3c..35e8001fa 100644 --- a/packages/eve/src/internal/nitro/host/dev-environment-watcher.integration.test.ts +++ b/packages/eve/src/internal/nitro/host/dev-environment-watcher.integration.test.ts @@ -1,82 +1,12 @@ -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; - -import { loadDevelopmentEnvironmentFiles } from "#cli/dev/environment.js"; -import { createCompiledAgentManifest } from "#compiler/manifest.js"; -import type { PreparedApplicationHost } from "#internal/nitro/host/types.js"; - -const mockedWatcher = vi.hoisted(() => { - let onAllHandler: ((event: string, changedPath: string) => void) | undefined; - const add = vi.fn(); - const close = vi.fn().mockResolvedValue(undefined); - const on = vi.fn((event: string, handler: unknown) => { - if (event === "all") { - onAllHandler = handler as (event: string, changedPath: string) => void; - return; - } - - if (event === "ready") { - queueMicrotask(() => { - (handler as () => void)(); - }); - } - }); - const unwatch = vi.fn(); - const watch = vi.fn(() => ({ - add, - close, - on, - unwatch, - })); - - return { - add, - close, - emit(event: string, changedPath: string): void { - if (onAllHandler === undefined) { - throw new Error("Watcher callback was not registered."); - } - - onAllHandler(event, changedPath); - }, - reset(): void { - onAllHandler = undefined; - add.mockClear(); - close.mockClear(); - on.mockClear(); - unwatch.mockClear(); - watch.mockClear(); - }, - unwatch, - watch, - }; -}); - -const prepareDevelopmentApplicationHostMock = vi.hoisted(() => vi.fn()); -const clearCompiledRuntimeAgentBundleCacheMock = vi.hoisted(() => vi.fn()); -const startDevelopmentSandboxPrewarmInBackgroundMock = vi.hoisted(() => vi.fn()); - -vi.mock("#compiled/chokidar/index.js", () => ({ - watch: mockedWatcher.watch, -})); - -vi.mock("./prepare-application-host.js", () => ({ - prepareDevelopmentApplicationHost: prepareDevelopmentApplicationHostMock, -})); - -vi.mock("#execution/sandbox/development-prewarm.js", () => ({ - startDevelopmentSandboxPrewarmInBackground: startDevelopmentSandboxPrewarmInBackgroundMock, -})); +import { afterEach, describe, expect, it } from "vitest"; -vi.mock("../../../runtime/sessions/compiled-agent-cache.js", () => ({ - clearCompiledRuntimeAgentBundleCache: clearCompiledRuntimeAgentBundleCacheMock, -})); - -import { startAuthoredSourceWatcher } from "#internal/nitro/host/dev-authored-source-watcher.js"; - -type WatcherNitroStub = Parameters[0]["nitro"]; +import { + loadDevelopmentEnvironmentFiles, + stageDevelopmentEnvironmentFiles, +} from "#cli/dev/environment.js"; const ENV_KEYS = [ "EVE_WATCH_ENV_FILE_ONLY", @@ -84,162 +14,88 @@ const ENV_KEYS = [ "EVE_WATCH_ENV_SHARED", "EVE_WATCH_ENV_SHELL", ] as const; - const temporaryDirectories: string[] = []; -beforeEach(() => { - vi.useFakeTimers(); - vi.spyOn(console, "error").mockImplementation(() => {}); - vi.spyOn(console, "log").mockImplementation(() => {}); - - clearEnvironment(); - mockedWatcher.reset(); - prepareDevelopmentApplicationHostMock.mockReset(); - clearCompiledRuntimeAgentBundleCacheMock.mockReset(); - startDevelopmentSandboxPrewarmInBackgroundMock.mockReset(); -}); - afterEach(async () => { - clearEnvironment(); + for (const key of ENV_KEYS) { + delete process.env[key]; + } await Promise.all( - temporaryDirectories.splice(0).map((directoryPath) => - rm(directoryPath, { - force: true, - recursive: true, - }), - ), + temporaryDirectories + .splice(0) + .map(async (path) => await rm(path, { force: true, recursive: true })), ); - vi.restoreAllMocks(); - vi.useRealTimers(); }); -describe("startAuthoredSourceWatcher env files", () => { - it("reloads watched env files while preserving parent process precedence", async () => { - const appRoot = await mkdtemp(join(tmpdir(), "eve-dev-env-watch-")); +describe("development environment reload transactions", () => { + it("preserves parent process precedence when env files reload", async () => { + const appRoot = await createEnvironmentApp(); + process.env.EVE_WATCH_ENV_SHELL = "from-parent"; + + loadDevelopmentEnvironmentFiles(appRoot); + + expect(process.env.EVE_WATCH_ENV_FILE_ONLY).toBe("from-env"); + expect(process.env.EVE_WATCH_ENV_SHARED).toBe("from-local"); + expect(process.env.EVE_WATCH_ENV_SHELL).toBe("from-parent"); + }); + + it("restores the complete prior environment when a candidate is rejected", async () => { + const appRoot = await createEnvironmentApp(); const envLocalPath = join(appRoot, ".env.local"); - temporaryDirectories.push(appRoot); - await mkdir(join(appRoot, "agent"), { - recursive: true, - }); + loadDevelopmentEnvironmentFiles(appRoot); await writeFile( - join(appRoot, ".env"), - [ - "EVE_WATCH_ENV_FILE_ONLY=from-env", - "EVE_WATCH_ENV_SHARED=from-env", - "EVE_WATCH_ENV_SHELL=from-env", - ].join("\n"), + envLocalPath, + "EVE_WATCH_ENV_NEW=from-candidate\nEVE_WATCH_ENV_SHARED=from-candidate\n", ); - await writeFile(envLocalPath, ["EVE_WATCH_ENV_SHARED=from-local"].join("\n")); - process.env.EVE_WATCH_ENV_SHELL = "from-parent"; - loadDevelopmentEnvironmentFiles(appRoot); + const reload = stageDevelopmentEnvironmentFiles(appRoot); + expect(process.env.EVE_WATCH_ENV_FILE_ONLY).toBe("from-env"); + expect(process.env.EVE_WATCH_ENV_NEW).toBe("from-candidate"); + expect(process.env.EVE_WATCH_ENV_SHARED).toBe("from-candidate"); + + reload.rollback(); expect(process.env.EVE_WATCH_ENV_FILE_ONLY).toBe("from-env"); + expect(process.env.EVE_WATCH_ENV_NEW).toBeUndefined(); expect(process.env.EVE_WATCH_ENV_SHARED).toBe("from-local"); - expect(process.env.EVE_WATCH_ENV_SHELL).toBe("from-parent"); + }); + + it("retains the candidate environment after commit", async () => { + const appRoot = await createEnvironmentApp(); + loadDevelopmentEnvironmentFiles(appRoot); + await writeFile(join(appRoot, ".env.local"), "EVE_WATCH_ENV_SHARED=committed\n"); - const previousHost = createPreparedHost(appRoot); - const nextHost = createPreparedHost(appRoot); - const nitroStub = createNitroStub(); - prepareDevelopmentApplicationHostMock.mockResolvedValueOnce(nextHost); - - const watcher = await startAuthoredSourceWatcher({ - nitro: nitroStub.nitro, - preparedHost: previousHost, - }); - - try { - await writeFile( - envLocalPath, - [ - "EVE_WATCH_ENV_NEW=from-local", - "EVE_WATCH_ENV_SHARED=from-local-updated", - "EVE_WATCH_ENV_SHELL=from-local", - ].join("\n"), - ); - await triggerChangeEvent(envLocalPath); - await watcher.flush(); - - expect(process.env.EVE_WATCH_ENV_FILE_ONLY).toBe("from-env"); - expect(process.env.EVE_WATCH_ENV_NEW).toBe("from-local"); - expect(process.env.EVE_WATCH_ENV_SHARED).toBe("from-local-updated"); - expect(process.env.EVE_WATCH_ENV_SHELL).toBe("from-parent"); - expect(nitroStub.callHook).toHaveBeenCalledWith("rollup:reload"); - } finally { - await watcher.close(); - } + const reload = stageDevelopmentEnvironmentFiles(appRoot); + reload.commit(); + reload.rollback(); + + expect(process.env.EVE_WATCH_ENV_SHARED).toBe("committed"); }); -}); -async function triggerChangeEvent(changedPath: string): Promise { - mockedWatcher.emit("change", changedPath); - await vi.advanceTimersByTimeAsync(200); - await Promise.resolve(); - await Promise.resolve(); -} + it("reapplies a rolled-back environment edit when a later rebuild stages again", async () => { + const appRoot = await createEnvironmentApp(); + loadDevelopmentEnvironmentFiles(appRoot); + await writeFile(join(appRoot, ".env.local"), "EVE_WATCH_ENV_SHARED=after-fix\n"); -function clearEnvironment(): void { - for (const key of ENV_KEYS) { - delete process.env[key]; - } -} + stageDevelopmentEnvironmentFiles(appRoot).rollback(); + expect(process.env.EVE_WATCH_ENV_SHARED).toBe("from-local"); -function createNitroStub(): { - callHook: ReturnType; - nitro: WatcherNitroStub; -} { - const callHook = vi.fn().mockResolvedValue(undefined); - const nitro: WatcherNitroStub = { - hooks: { - callHook, - }, - options: { - experimental: {}, - handlers: [], - scheduledTasks: {}, - tasks: {}, - virtual: {}, - }, - routing: { - sync: vi.fn(), - }, - }; - - return { - callHook, - nitro, - }; -} + // The retry rebuild carries no env-file change of its own; staging from + // the files on disk must still pick the edit up. + const retry = stageDevelopmentEnvironmentFiles(appRoot); + retry.commit(); -function createPreparedHost(appRoot: string): PreparedApplicationHost { - const agentRoot = join(appRoot, "agent"); - - return { - appRoot, - compileResult: { - diagnostics: [], - manifest: createCompiledAgentManifest({ - agentRoot, - appRoot, - config: { - model: { - id: "openai/gpt-5-mini", - routing: { kind: "gateway", target: "openai" }, - }, - name: "test-agent", - }, - }), - metadata: {} as PreparedApplicationHost["compileResult"]["metadata"], - paths: {} as PreparedApplicationHost["compileResult"]["paths"], - project: { - appRoot, - agentRoot, - layout: "flat", - }, - }, - compiledArtifacts: {} as PreparedApplicationHost["compiledArtifacts"], - scheduleRegistrations: [], - schedules: [], - workflowBuildDir: join(appRoot, ".workflow-build"), - }; + expect(process.env.EVE_WATCH_ENV_SHARED).toBe("after-fix"); + }); +}); + +async function createEnvironmentApp(): Promise { + const appRoot = await mkdtemp(join(tmpdir(), "eve-dev-env-transaction-")); + temporaryDirectories.push(appRoot); + await writeFile( + join(appRoot, ".env"), + "EVE_WATCH_ENV_FILE_ONLY=from-env\nEVE_WATCH_ENV_SHARED=from-env\nEVE_WATCH_ENV_SHELL=from-env\n", + ); + await writeFile(join(appRoot, ".env.local"), "EVE_WATCH_ENV_SHARED=from-local\n"); + return appRoot; } diff --git a/packages/eve/src/internal/nitro/host/dev-host-candidate.ts b/packages/eve/src/internal/nitro/host/dev-host-candidate.ts new file mode 100644 index 000000000..00268e71d --- /dev/null +++ b/packages/eve/src/internal/nitro/host/dev-host-candidate.ts @@ -0,0 +1,91 @@ +import { resolve } from "node:path"; + +import { build as buildNitro, prepare } from "nitro/builder"; +import type { Nitro } from "nitro/types"; + +import type { PreparedDevelopmentApplicationHost } from "#internal/nitro/host/types.js"; + +const CANDIDATE_BUILD_TIMEOUT_MS = 120_000; + +export interface DevelopmentWorkerPayload { + readonly entry: string; + readonly workerData: Readonly>; +} + +/** + * Builds one host candidate in its invocation-owned workspace and returns the + * worker payload Nitro emitted for it. The Nitro instance is closed before + * returning: the payload's entry lives on disk in the workspace, so nothing + * retains the builder. + */ +export async function buildDevelopmentHostCandidate(input: { + readonly host: PreparedDevelopmentApplicationHost; + readonly nitro: Nitro; +}): Promise { + const nitro = input.nitro; + let settled = false; + let payload: DevelopmentWorkerPayload | undefined; + let buildError: unknown; + let signal: (() => void) | undefined; + const signalled = new Promise((resolve) => { + signal = resolve; + }); + + const removeReloadHook = nitro.hooks.hook("dev:reload", (reload) => { + if (settled) { + return; + } + settled = true; + // Stock Nitro falls back to the configured output entry when the reload + // payload does not carry one. + payload = { + entry: + reload?.entry ?? + resolve(nitro.options.output.dir, nitro.options.output.serverDir, "index.mjs"), + workerData: reload?.workerData ?? {}, + }; + signal?.(); + }); + const removeErrorHook = nitro.hooks.hook("dev:error", (cause) => { + if (settled) { + return; + } + settled = true; + buildError = cause; + signal?.(); + }); + + try { + await prepare(nitro); + await buildNitro(nitro); + await waitForSignal(signalled); + if (buildError !== undefined) { + throw buildError instanceof Error ? buildError : new Error(String(buildError)); + } + if (payload === undefined || payload.entry.length === 0) { + throw new Error("Nitro did not emit a development worker entry."); + } + return payload; + } finally { + removeReloadHook(); + removeErrorHook(); + await nitro.close().catch(() => undefined); + } +} + +async function waitForSignal(signalled: Promise): Promise { + let timer: NodeJS.Timeout | undefined; + try { + await Promise.race([ + signalled, + new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new Error("Timed out waiting for the Nitro dev build to emit a worker.")), + CANDIDATE_BUILD_TIMEOUT_MS, + ); + }), + ]); + } finally { + clearTimeout(timer); + } +} diff --git a/packages/eve/src/internal/nitro/host/dev-host-fingerprint.integration.test.ts b/packages/eve/src/internal/nitro/host/dev-host-fingerprint.integration.test.ts new file mode 100644 index 000000000..62e5d3621 --- /dev/null +++ b/packages/eve/src/internal/nitro/host/dev-host-fingerprint.integration.test.ts @@ -0,0 +1,148 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { createCompiledAgentManifest, type CompiledAgentManifest } from "#compiler/manifest.js"; +import { computeDevelopmentHostFingerprint } from "#internal/nitro/host/dev-host-fingerprint.js"; +import type { PreparedDevelopmentApplicationHost } from "#internal/nitro/host/types.js"; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + delete process.env.EVE_HOST_FINGERPRINT_TEST; + await Promise.all( + temporaryDirectories + .splice(0) + .map(async (dir) => await rm(dir, { force: true, recursive: true })), + ); +}); + +interface HostVariant { + readonly channels?: CompiledAgentManifest["channels"]; + readonly instrumentationSource?: string; + readonly schedules?: CompiledAgentManifest["schedules"]; + readonly workflowWorld?: "local" | "vercel"; +} + +async function createHost(variant: HostVariant = {}): Promise { + const appRoot = await mkdtemp(join(tmpdir(), "eve-host-fingerprint-")); + temporaryDirectories.push(appRoot); + const agentRoot = join(appRoot, "agent"); + await mkdir(agentRoot, { recursive: true }); + + let instrumentationSourcePath: string | undefined; + if (variant.instrumentationSource !== undefined) { + instrumentationSourcePath = join(appRoot, "instrumentation-source.mjs"); + await writeFile(instrumentationSourcePath, variant.instrumentationSource); + } + + const manifest = createCompiledAgentManifest({ + agentRoot, + appRoot, + channels: variant.channels ?? [], + config: { + model: { id: "openai/gpt-5-mini", routing: { kind: "gateway", target: "openai" } }, + name: "fingerprint-host", + ...(variant.workflowWorld === undefined + ? {} + : { experimental: { workflow: { world: variant.workflowWorld } } }), + }, + schedules: variant.schedules ?? [], + }); + + return { + appRoot, + compiledArtifacts: { + bootstrapPath: join(appRoot, "bootstrap.mjs"), + instrumentationSourcePath, + workflowWorldPluginPath: join(appRoot, "workflow-world.mjs"), + }, + compileResult: { manifest } as PreparedDevelopmentApplicationHost["compileResult"], + } as PreparedDevelopmentApplicationHost; +} + +describe("computeDevelopmentHostFingerprint", () => { + it("is stable for equivalent hosts in different app roots", async () => { + const first = await computeDevelopmentHostFingerprint(await createHost()); + const second = await computeDevelopmentHostFingerprint(await createHost()); + + expect(first).toBe(second); + }); + + it("treats instrumentation content as structural", async () => { + const base = await computeDevelopmentHostFingerprint( + await createHost({ instrumentationSource: 'export default { marker: "one" };\n' }), + ); + const changed = await computeDevelopmentHostFingerprint( + await createHost({ instrumentationSource: 'export default { marker: "two" };\n' }), + ); + + expect(changed).not.toBe(base); + }); + + it("treats channel route topology as structural", async () => { + const base = await computeDevelopmentHostFingerprint(await createHost()); + const withRoute = await computeDevelopmentHostFingerprint( + await createHost({ + channels: [ + { + kind: "channel", + logicalPath: "channels/smoke.ts", + method: "GET", + name: "smoke", + sourceId: "channels/smoke.ts", + sourceKind: "module", + urlPath: "/smoke", + }, + ] as CompiledAgentManifest["channels"], + }), + ); + + expect(withRoute).not.toBe(base); + }); + + it("treats configured environment values as structural", async () => { + const host = await createHost(); + await writeFile(join(host.appRoot, ".env"), "EVE_HOST_FINGERPRINT_TEST=one\n"); + process.env.EVE_HOST_FINGERPRINT_TEST = "one"; + const base = await computeDevelopmentHostFingerprint(host); + + process.env.EVE_HOST_FINGERPRINT_TEST = "two"; + const changed = await computeDevelopmentHostFingerprint(host); + + expect(changed).not.toBe(base); + }); + + it("treats the workflow world selection as structural", async () => { + const local = await computeDevelopmentHostFingerprint( + await createHost({ workflowWorld: "local" }), + ); + const vercel = await computeDevelopmentHostFingerprint( + await createHost({ workflowWorld: "vercel" }), + ); + + expect(vercel).not.toBe(local); + }); + + it("leaves schedule definitions runtime-only", async () => { + const base = await computeDevelopmentHostFingerprint(await createHost()); + const withSchedule = await computeDevelopmentHostFingerprint( + await createHost({ + schedules: [ + { + cron: "0 0 * * 0", + hasRun: false, + logicalPath: "schedules/heartbeat.md", + markdown: "Report the weather.", + name: "heartbeat", + sourceId: "schedules/heartbeat.md", + sourceKind: "markdown", + }, + ], + }), + ); + + expect(withSchedule).toBe(base); + }); +}); diff --git a/packages/eve/src/internal/nitro/host/dev-host-fingerprint.ts b/packages/eve/src/internal/nitro/host/dev-host-fingerprint.ts new file mode 100644 index 000000000..872c6b330 --- /dev/null +++ b/packages/eve/src/internal/nitro/host/dev-host-fingerprint.ts @@ -0,0 +1,53 @@ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; + +import { readDevelopmentEnvironmentHostValues } from "#cli/dev/environment.js"; +import { computeChannelRouteRegistrations } from "#internal/nitro/host/channel-routes.js"; +import type { PreparedDevelopmentApplicationHost } from "#internal/nitro/host/types.js"; + +export async function computeDevelopmentHostFingerprint( + host: PreparedDevelopmentApplicationHost, +): Promise { + const manifest = host.compileResult.manifest; + const agentNodes = [manifest, ...manifest.subagents.map((subagent) => subagent.agent)]; + const payload = { + agentName: manifest.config.name, + bundler: { + externalDependencies: [ + ...new Set(agentNodes.flatMap((node) => node.config.build?.externalDependencies ?? [])), + ].sort((left, right) => left.localeCompare(right)), + extensionScopes: (manifest.extensionMounts ?? []) + .map((mount) => ({ + packageNamespace: mount.packageNamespace, + sourceRoot: mount.sourceRoot, + })) + .sort((left, right) => left.sourceRoot.localeCompare(right.sourceRoot)), + sandboxBackends: [ + ...new Set( + agentNodes + .map((node) => node.sandbox?.backendName) + .filter((backendName): backendName is string => backendName !== undefined), + ), + ].sort((left, right) => left.localeCompare(right)), + }, + channels: computeChannelRouteRegistrations(host), + environment: readDevelopmentEnvironmentHostValues(host.appRoot), + instrumentation: await readInstrumentationSource(host), + workflow: { + enabled: agentNodes.some((node) => node.workflowTool !== undefined), + world: manifest.config.experimental?.workflow?.world ?? "local", + }, + }; + + return createHash("sha256").update(JSON.stringify(payload)).digest("hex"); +} + +async function readInstrumentationSource( + host: PreparedDevelopmentApplicationHost, +): Promise { + const path = host.compiledArtifacts.instrumentationSourcePath; + if (path === undefined) { + return null; + } + return await readFile(path, "utf8"); +} diff --git a/packages/eve/src/internal/nitro/host/dev-host-workspace.ts b/packages/eve/src/internal/nitro/host/dev-host-workspace.ts new file mode 100644 index 000000000..c8534aa55 --- /dev/null +++ b/packages/eve/src/internal/nitro/host/dev-host-workspace.ts @@ -0,0 +1,35 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, rm } from "node:fs/promises"; +import { join } from "node:path"; + +export interface DevelopmentHostWorkspace { + readonly artifactsDir: string; + readonly compilerArtifactsDir: string; + readonly nitroBuildDir: string; + readonly nitroOutputDir: string; + readonly rootDir: string; + readonly workflowBuildDir: string; +} + +export async function createDevelopmentHostWorkspace( + appRoot: string, +): Promise { + const rootDir = join(appRoot, ".eve", "dev-hosts", randomUUID()); + const workspace: DevelopmentHostWorkspace = { + artifactsDir: join(rootDir, "artifacts"), + compilerArtifactsDir: join(rootDir, "compiler"), + nitroBuildDir: join(rootDir, "nitro"), + nitroOutputDir: join(rootDir, "output"), + rootDir, + workflowBuildDir: join(rootDir, "workflow"), + }; + + await mkdir(rootDir, { recursive: true }); + return workspace; +} + +export async function removeDevelopmentHostWorkspace( + workspace: DevelopmentHostWorkspace, +): Promise { + await rm(workspace.rootDir, { force: true, recursive: true }); +} diff --git a/packages/eve/src/internal/nitro/host/dev-live-virtual-modules.ts b/packages/eve/src/internal/nitro/host/dev-live-virtual-modules.ts deleted file mode 100644 index f4bb9feaf..000000000 --- a/packages/eve/src/internal/nitro/host/dev-live-virtual-modules.ts +++ /dev/null @@ -1,93 +0,0 @@ -import type { Nitro } from "nitro/types"; - -/** - * Nitro's dev watcher creates its bundler config once and reuses it across - * `rollup:reload`, and the `nitro:virtual` plugin snapshots - * `nitro.options.virtual` into a private module map when that config is - * created. Mutating `nitro.options.virtual` after startup therefore never - * reaches the bundler — a virtual handler added for a new channel route - * fails with UNRESOLVED_IMPORT and kills the dev worker until restart. - * - * The plugin exposes its live module map through its `api`, so we capture - * it from the `rollup:before` hook and mirror later virtual-module updates - * into it. Ids must start with `#nitro/virtual/` — the plugin's resolveId - * filter is also frozen at creation, and that prefix is the only pattern it - * always accepts. - */ - -interface NitroVirtualModuleEntry { - readonly module: { readonly id: string; readonly template: string }; - readonly render: () => string; -} - -type LiveVirtualModuleMap = Map; - -const liveVirtualModuleMaps = new WeakMap(); - -/** - * Starts capturing the dev bundler's live virtual-module map for `nitro`. - * Call once per dev Nitro instance, before its first build. - */ -export function captureDevLiveVirtualModules(nitro: Nitro): void { - nitro.hooks.hook("rollup:before", (_nitro, config) => { - const moduleMap = findNitroVirtualModuleMap(config.plugins); - if (moduleMap !== undefined) { - liveVirtualModuleMaps.set(nitro, moduleMap); - } - }); -} - -/** - * Mirrors one virtual-module set into the captured dev bundler map: ids - * under `prefix` are replaced by `entries`. Returns `false` when no live - * map was captured (the caller should tell the user a restart is needed). - */ -export function replaceDevLiveVirtualModules( - nitro: object, - input: { - readonly entries: Readonly>; - readonly prefix: string; - }, -): boolean { - const moduleMap = liveVirtualModuleMaps.get(nitro); - - if (moduleMap === undefined) { - return false; - } - - for (const id of moduleMap.keys()) { - if (id.startsWith(input.prefix)) { - moduleMap.delete(id); - } - } - - for (const [id, template] of Object.entries(input.entries)) { - moduleMap.set(id, { - module: { id, template }, - render: () => template, - }); - } - - return true; -} - -function findNitroVirtualModuleMap(plugins: unknown): LiveVirtualModuleMap | undefined { - if (!Array.isArray(plugins)) { - return undefined; - } - - for (const plugin of plugins.flat(Number.POSITIVE_INFINITY)) { - if ( - typeof plugin === "object" && - plugin !== null && - (plugin as { name?: unknown }).name === "nitro:virtual" - ) { - const api = (plugin as { api?: { modules?: unknown } }).api; - if (api !== undefined && api.modules instanceof Map) { - return api.modules as LiveVirtualModuleMap; - } - } - } - - return undefined; -} diff --git a/packages/eve/src/internal/nitro/host/dev-runner.ts b/packages/eve/src/internal/nitro/host/dev-runner.ts new file mode 100644 index 000000000..d5dc819b1 --- /dev/null +++ b/packages/eve/src/internal/nitro/host/dev-runner.ts @@ -0,0 +1,150 @@ +import { existsSync } from "node:fs"; +import { Worker } from "node:worker_threads"; + +import { BaseEnvRunner } from "#compiled/env-runner/index.js"; +import { resolvePackageCompiledFilePath } from "#internal/application/package.js"; + +export interface DevelopmentRunner { + readonly closed: boolean; + close(cause?: unknown): Promise; + fetch(request: Request): Promise; + onceClosed(listener: (cause?: unknown) => void): void; + upgrade(input: { + readonly node: { + readonly head: Buffer; + readonly req: import("node:http").IncomingMessage; + readonly socket: import("node:net").Socket; + }; + }): Promise; + waitForReady(timeout: number): Promise; +} + +export interface DevelopmentRunnerInput { + readonly entry: string; + readonly name: string; + readonly workerData: Readonly>; +} + +export type DevelopmentRunnerFactory = (input: DevelopmentRunnerInput) => DevelopmentRunner; + +class NodeDevelopmentRunner extends BaseEnvRunner implements DevelopmentRunner { + #closeCause: unknown; + readonly #closedListeners = new Set<(cause?: unknown) => void>(); + #worker: Worker | undefined; + + constructor(input: DevelopmentRunnerInput) { + const workerEntry = resolvePackageCompiledFilePath("src/compiled/env-runner/node-worker.js"); + super({ + data: { + entry: input.entry, + ...input.workerData, + }, + hooks: { + onClose: (_runner, cause) => { + const listeners = [...this.#closedListeners]; + this.#closedListeners.clear(); + for (const listener of listeners) { + listener(cause); + } + }, + }, + name: input.name, + workerEntry, + }); + this._initWithVirtualData(() => this.#startWorker()); + } + + onceClosed(listener: (cause?: unknown) => void): void { + if (this.closed) { + listener(this.#closeCause); + return; + } + this.#closedListeners.add(listener); + } + + override sendMessage(message: unknown): void { + if (this.#worker === undefined) { + throw new Error("Development worker is not initialized."); + } + this.#worker.postMessage(message); + } + + override async waitForReady(timeout: number): Promise { + try { + await super.waitForReady(timeout); + } catch (error) { + if (this.#closeCause === undefined) { + throw error; + } + throw new Error( + `Development worker failed before readiness: ${this.#closeCause instanceof Error ? this.#closeCause.message : String(this.#closeCause)}`, + { cause: this.#closeCause }, + ); + } + } + + protected override _hasRuntime(): boolean { + return this.#worker !== undefined; + } + + protected override _runtimeType(): string { + return "worker"; + } + + protected override async _closeRuntime(): Promise { + const worker = this.#worker; + if (worker === undefined) { + return; + } + + this.#worker = undefined; + worker.removeAllListeners(); + await worker.terminate(); + } + + protected override _handleMessage(message: unknown): void { + if (isWorkerInitializationError(message)) { + this.#closeCause = new Error(message.error); + } + super._handleMessage(message); + } + + #startWorker(): void { + if (!existsSync(this._workerEntry)) { + void this.close(`Development worker entry not found at "${this._workerEntry}".`); + return; + } + + const worker = new Worker(this._workerEntry, { + env: process.env, + workerData: { + name: this._name, + ...this._data, + }, + }); + this.#worker = worker; + worker.once("error", (error) => { + this.#closeCause = error; + void this.close(error); + }); + worker.once("exit", (code) => { + const error = new Error(`Development worker exited with code ${String(code)}.`); + this.#closeCause ??= error; + void this.close(error); + }); + worker.on("message", (message: unknown) => this._handleMessage(message)); + } +} + +export const createNodeDevelopmentRunner: DevelopmentRunnerFactory = (input) => + new NodeDevelopmentRunner(input); + +function isWorkerInitializationError( + value: unknown, +): value is { readonly error: string; readonly event: "init-error" } { + if (typeof value !== "object" || value === null) { + return false; + } + const record = value as Record; + return record.event === "init-error" && typeof record.error === "string"; +} diff --git a/packages/eve/src/internal/nitro/host/dev-server-http.ts b/packages/eve/src/internal/nitro/host/dev-server-http.ts new file mode 100644 index 000000000..8f0527c19 --- /dev/null +++ b/packages/eve/src/internal/nitro/host/dev-server-http.ts @@ -0,0 +1,119 @@ +import type { IncomingMessage, Server, ServerResponse } from "node:http"; +import { Readable } from "node:stream"; + +import { toErrorMessage } from "#shared/errors.js"; + +export function createPublicRequest(request: IncomingMessage, signal: AbortSignal): Request { + const authority = request.headers.host ?? "localhost"; + const url = new URL(request.url ?? "/", `http://${authority}`); + const headers = new Headers(); + for (const [name, value] of Object.entries(request.headers)) { + if (Array.isArray(value)) { + for (const item of value) { + headers.append(name, item); + } + } else if (value !== undefined) { + headers.set(name, value); + } + } + + const hasBody = request.method !== "GET" && request.method !== "HEAD"; + return new Request(url, { + body: hasBody ? (Readable.toWeb(request) as ReadableStream) : undefined, + duplex: hasBody ? "half" : undefined, + headers, + method: request.method, + signal, + } as RequestInit & { duplex?: "half" }); +} + +export async function writeResponse( + response: ServerResponse, + webResponse: Response, + signal: AbortSignal, +): Promise { + if (response.destroyed) { + await webResponse.body?.cancel(); + return; + } + + response.statusCode = webResponse.status; + response.statusMessage = webResponse.statusText; + const getSetCookie = ( + webResponse.headers as Headers & { + getSetCookie?: () => readonly string[]; + } + ).getSetCookie; + const setCookies = getSetCookie?.call(webResponse.headers) ?? []; + webResponse.headers.forEach((value, name) => { + if (name.toLowerCase() !== "set-cookie") { + response.setHeader(name, value); + } + }); + if (setCookies.length > 0) { + response.setHeader("set-cookie", [...setCookies]); + } + + if (webResponse.body === null) { + await endResponse(response); + return; + } + + const body = Readable.fromWeb(webResponse.body as import("node:stream/web").ReadableStream); + const cancelBody = () => body.destroy(signal.reason as Error | undefined); + signal.addEventListener("abort", cancelBody, { once: true }); + try { + await new Promise((resolve, reject) => { + const onClose = () => { + if (!response.writableEnded) { + reject(new Error("Development client disconnected.")); + } + }; + body.once("error", reject); + response.once("error", reject); + response.once("close", onClose); + response.once("finish", resolve); + body.pipe(response); + }); + } finally { + signal.removeEventListener("abort", cancelBody); + if (!body.destroyed && (signal.aborted || response.destroyed)) { + body.destroy(); + } + } +} + +export function writeRequestError(response: ServerResponse, error: unknown): void { + if (!response.headersSent) { + response.statusCode = 503; + response.setHeader("content-type", "application/json; charset=utf-8"); + } + if (!response.writableEnded && !response.destroyed) { + response.end(JSON.stringify({ error: toErrorMessage(error) })); + } +} + +export function closeServer(server: Server): Promise { + if (!server.listening) { + return Promise.resolve(); + } + + const closed = new Promise((resolve, reject) => { + server.close((error) => { + if (error === undefined) { + resolve(); + } else { + reject(error); + } + }); + }); + server.closeIdleConnections?.(); + return closed; +} + +async function endResponse(response: ServerResponse): Promise { + await new Promise((resolve, reject) => { + response.once("error", reject); + response.end(resolve); + }); +} diff --git a/packages/eve/src/internal/nitro/host/drained-nitro-dev-server.scenario.test.ts b/packages/eve/src/internal/nitro/host/drained-nitro-dev-server.scenario.test.ts new file mode 100644 index 000000000..5be6b1b87 --- /dev/null +++ b/packages/eve/src/internal/nitro/host/drained-nitro-dev-server.scenario.test.ts @@ -0,0 +1,242 @@ +import { connect } from "node:net"; + +import { describe, expect, it, vi } from "vitest"; + +import { + DrainedNitroDevServer, + type DrainedDevServerListener, +} from "#internal/nitro/host/drained-nitro-dev-server.js"; +import type { + DevelopmentRunner, + DevelopmentRunnerFactory, +} from "#internal/nitro/host/dev-runner.js"; + +const TEST_DEADLINE_MS = 5_000; +const LOGGER = { error: () => undefined }; + +interface TestRunner extends DevelopmentRunner { + crash(cause: Error): void; + readonly closeMock: ReturnType; +} + +function createRunnerFactory( + fetchHandler: (request: Request, runnerIndex: number) => Promise, + readiness: (runnerIndex: number) => Promise = async () => undefined, +): { readonly createRunner: DevelopmentRunnerFactory; readonly runners: TestRunner[] } { + const runners: TestRunner[] = []; + const createRunner: DevelopmentRunnerFactory = () => { + const runnerIndex = runners.length; + let closed = false; + const closedListeners = new Set<(cause?: unknown) => void>(); + const notifyClosed = (cause?: unknown) => { + const listeners = [...closedListeners]; + closedListeners.clear(); + for (const listener of listeners) { + listener(cause); + } + }; + const closeMock = vi.fn(async () => { + closed = true; + notifyClosed(); + }); + const runner: TestRunner = { + close: closeMock, + closeMock, + get closed() { + return closed; + }, + crash(cause) { + closed = true; + notifyClosed(cause); + }, + fetch: async (request) => await fetchHandler(request, runnerIndex), + onceClosed(listener) { + if (closed) { + listener(); + return; + } + closedListeners.add(listener); + }, + upgrade: vi.fn(async () => undefined), + waitForReady: vi.fn(async () => await readiness(runnerIndex)), + }; + runners.push(runner); + return runner; + }; + return { createRunner, runners }; +} + +function replacement(entry: string, dispose?: () => Promise) { + return { dispose, entry, workerData: {} }; +} + +async function listen(server: DrainedNitroDevServer): Promise { + const listener = server.listen({ hostname: "127.0.0.1", port: 0 }); + await listener.ready(); + if (listener.url === undefined) { + throw new Error("Listener did not expose a URL."); + } + return listener; +} + +async function withinDeadline(operation: Promise, message: string): Promise { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + operation, + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error(message)), TEST_DEADLINE_MS); + }), + ]); + } finally { + clearTimeout(timer); + } +} + +describe("drained Nitro dev server", () => { + it("keeps the previous worker serving when a candidate fails readiness", async () => { + const { createRunner, runners } = createRunnerFactory( + async (_request, runnerIndex) => new Response(`runner-${String(runnerIndex)}`), + async (runnerIndex) => { + if (runnerIndex === 1) { + throw new Error("candidate failed"); + } + }, + ); + const server = new DrainedNitroDevServer(LOGGER, createRunner); + const listener = await listen(server); + + await server.replaceWorker(replacement("/tmp/first.mjs")); + await expect(server.replaceWorker(replacement("/tmp/second.mjs"))).rejects.toThrow( + "candidate failed", + ); + expect(runners[1]?.closeMock).toHaveBeenCalled(); + await expect( + fetch(new URL("/", listener.url)).then(async (response) => await response.text()), + ).resolves.toBe("runner-0"); + + await server.close(); + }); + + it("drains the retired worker and disposes it after its last exchange settles", async () => { + let releaseFirstResponse: (() => void) | undefined; + const { createRunner, runners } = createRunnerFactory(async (_request, runnerIndex) => { + if (runnerIndex === 0) { + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("started\n")); + releaseFirstResponse = () => controller.close(); + }, + }); + return new Response(body); + } + return new Response("runner-1"); + }); + const server = new DrainedNitroDevServer(LOGGER, createRunner); + const listener = await listen(server); + const disposeFirst = vi.fn(async () => undefined); + + await server.replaceWorker(replacement("/tmp/first.mjs", disposeFirst)); + const streaming = await fetch(new URL("/", listener.url)); + const reader = streaming.body?.getReader(); + await reader?.read(); + + await server.replaceWorker(replacement("/tmp/second.mjs")); + await expect( + fetch(new URL("/", listener.url)).then(async (response) => await response.text()), + ).resolves.toBe("runner-1"); + expect(runners[0]?.closeMock).not.toHaveBeenCalled(); + expect(disposeFirst).not.toHaveBeenCalled(); + + releaseFirstResponse?.(); + await withinDeadline( + (async () => { + for (;;) { + const result = await reader?.read(); + if (result === undefined || result.done) { + return; + } + } + })(), + "Timed out waiting for the retired stream to finish.", + ); + await vi.waitFor(() => { + expect(runners[0]?.closeMock).toHaveBeenCalled(); + expect(disposeFirst).toHaveBeenCalledOnce(); + }); + + await server.close(); + }); + + it("restarts the worker with its own workspace when it exits unexpectedly", async () => { + const { createRunner, runners } = createRunnerFactory( + async (_request, runnerIndex) => new Response(`runner-${String(runnerIndex)}`), + ); + const server = new DrainedNitroDevServer(LOGGER, createRunner); + const listener = await listen(server); + const dispose = vi.fn(async () => undefined); + await server.replaceWorker(replacement("/tmp/first.mjs", dispose)); + + runners[0]?.crash(new Error("worker exploded")); + await withinDeadline( + vi.waitFor(async () => { + const response = await fetch(new URL("/", listener.url)); + expect(await response.text()).toBe("runner-1"); + }), + "Timed out waiting for the restarted worker.", + ); + expect(dispose).not.toHaveBeenCalled(); + + await server.close(); + expect(dispose).toHaveBeenCalledOnce(); + }); + + it("destroys the socket when an upgrade fails and closes idempotently", async () => { + const { createRunner, runners } = createRunnerFactory(async () => new Response("ok")); + const server = new DrainedNitroDevServer(LOGGER, createRunner); + const listener = await listen(server); + await server.replaceWorker(replacement("/tmp/first.mjs")); + (runners[0] as { upgrade: unknown }).upgrade = vi.fn(async () => { + throw new Error("upgrade rejected"); + }); + + const target = new URL(listener.url ?? ""); + await withinDeadline( + new Promise((resolve, reject) => { + const socket = connect({ host: target.hostname, port: Number(target.port) }, () => { + socket.write( + "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n", + ); + }); + socket.once("close", () => resolve()); + socket.once("error", reject); + }), + "Timed out waiting for the failed upgrade socket to close.", + ); + + await Promise.all([server.close(), server.close()]); + await server.close(); + expect(runners[0]?.closeMock).toHaveBeenCalledOnce(); + }); + + it("answers control requests without admitting them to the worker", async () => { + const fetchMock = vi.fn(async () => new Response("worker")); + const { createRunner } = createRunnerFactory(fetchMock); + const server = new DrainedNitroDevServer(LOGGER, createRunner); + server.setControlHandler(async (request) => + new URL(request.url).pathname === "/control" ? new Response("parent") : undefined, + ); + const listener = await listen(server); + await server.replaceWorker(replacement("/tmp/first.mjs")); + + await expect( + fetch(new URL("/control", listener.url)).then(async (response) => await response.text()), + ).resolves.toBe("parent"); + expect(fetchMock).not.toHaveBeenCalled(); + await expect( + fetch(new URL("/other", listener.url)).then(async (response) => await response.text()), + ).resolves.toBe("worker"); + + await server.close(); + }); +}); diff --git a/packages/eve/src/internal/nitro/host/drained-nitro-dev-server.ts b/packages/eve/src/internal/nitro/host/drained-nitro-dev-server.ts new file mode 100644 index 000000000..1fc4f40f3 --- /dev/null +++ b/packages/eve/src/internal/nitro/host/drained-nitro-dev-server.ts @@ -0,0 +1,383 @@ +import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import type { Socket } from "node:net"; + +import { + closeServer, + createPublicRequest, + writeRequestError, + writeResponse, +} from "#internal/nitro/host/dev-server-http.js"; +import { + createNodeDevelopmentRunner, + type DevelopmentRunner, + type DevelopmentRunnerFactory, +} from "#internal/nitro/host/dev-runner.js"; +import { toErrorMessage } from "#shared/errors.js"; + +const RUNNER_READY_TIMEOUT_MS = 60_000; + +export interface DrainedDevServerListener { + close(): Promise; + readonly node: { readonly server: Server }; + ready(): Promise; + readonly url: string | undefined; +} + +export interface DevelopmentWorkerReplacement { + readonly dispose?: () => Promise; + readonly entry: string; + readonly workerData: Readonly>; +} + +interface RunnerSlot { + activeExchanges: number; + readonly dispose?: () => Promise; + disposed: boolean; + readonly entry: string; + quietListeners: Array<() => void>; + released: boolean; + readonly runner: DevelopmentRunner; + readonly workerData: Readonly>; +} + +interface DevServerLogger { + error(message: unknown, ...details: unknown[]): unknown; +} + +/** + * Development server with drained worker replacement: `replaceWorker` swaps + * to a ready candidate while the retired worker keeps serving the responses + * and sockets it already admitted — without bound, a streaming turn can hold + * one for minutes — and is terminated once its last exchange settles. Stock + * Nitro's dev server terminates the previous worker as soon as the next one + * attaches, which resets admitted work; this class exists solely to close + * that gap and is intended to be deleted in favor of `createDevServer` once + * equivalent drain semantics are available upstream. + */ +export class DrainedNitroDevServer { + readonly #createRunner: DevelopmentRunnerFactory; + readonly #draining = new Set(); + readonly #listeners = new Set<{ beginClose(): Promise; destroySockets(): void }>(); + readonly #logger: DevServerLogger; + #active: RunnerSlot | undefined; + #activeWaiters: Array<() => void> = []; + #closePromise: Promise | undefined; + #closed = false; + #controlHandler: ((request: Request) => Promise) | undefined; + #replaceChain: Promise = Promise.resolve(); + #runnerCounter = 0; + + constructor( + logger: DevServerLogger = console, + createRunner: DevelopmentRunnerFactory = createNodeDevelopmentRunner, + ) { + this.#createRunner = createRunner; + this.#logger = logger; + } + + setControlHandler(handler: (request: Request) => Promise): void { + this.#controlHandler = handler; + } + + /** + * Swaps to a ready worker built from `entry`. Throws without touching the + * active worker when the candidate fails readiness. `dispose` is invoked + * once the worker created here has fully retired (drained, crashed, or + * closed at shutdown), so a caller can tie workspace cleanup to it. + */ + replaceWorker(replacement: DevelopmentWorkerReplacement): Promise { + const next = this.#replaceChain + .catch(() => undefined) + .then(async () => await this.#replaceWorker(replacement)); + this.#replaceChain = next.catch(() => undefined); + return next; + } + + async #replaceWorker(replacement: DevelopmentWorkerReplacement): Promise { + if (this.#closed) { + throw new Error("Development server is closed."); + } + + const slot: RunnerSlot = { + activeExchanges: 0, + dispose: replacement.dispose, + disposed: false, + entry: replacement.entry, + quietListeners: [], + released: false, + runner: this.#createRunner({ + entry: replacement.entry, + name: `eve-dev-${String(this.#runnerCounter++)}`, + workerData: replacement.workerData, + }), + workerData: replacement.workerData, + }; + + try { + await slot.runner.waitForReady(RUNNER_READY_TIMEOUT_MS); + } catch (error) { + await slot.runner.close(error).catch(() => undefined); + throw error; + } + + if (this.#closed) { + await this.#releaseSlot(slot); + throw new Error("Development server closed before the worker was ready."); + } + + const previous = this.#active; + this.#active = slot; + slot.runner.onceClosed(() => { + void this.#handleRunnerClose(slot); + }); + this.#wakeActiveWaiters(); + + if (previous !== undefined) { + this.#drainInBackground(previous); + } + } + + listen(input: { readonly hostname: string; readonly port: number }): DrainedDevServerListener { + if (this.#closed) { + throw new Error("Development server is closed."); + } + + const sockets = new Set(); + const server = createServer((request, response) => { + void this.#handleRequest(request, response); + }); + server.on("connection", (socket) => { + sockets.add(socket); + socket.once("close", () => sockets.delete(socket)); + }); + server.on("upgrade", (request, socket, head) => { + void this.#handleUpgrade(request, socket as Socket, head); + }); + + let url: string | undefined; + const ready = new Promise((resolvePromise, rejectPromise) => { + const onError = (error: Error) => { + server.off("listening", onListening); + rejectPromise(error); + }; + const onListening = () => { + server.off("error", onError); + const address = server.address(); + if (address === null || typeof address === "string") { + rejectPromise(new Error("Development server did not expose a TCP address.")); + return; + } + const host = input.hostname.includes(":") ? `[${input.hostname}]` : input.hostname; + url = `http://${host}:${String(address.port)}/`; + resolvePromise(); + }; + server.once("error", onError); + server.once("listening", onListening); + server.listen({ host: input.hostname, port: input.port }); + }); + + let closePromise: Promise | undefined; + const listenerState = { + beginClose: () => { + closePromise ??= closeServer(server).finally(() => { + this.#listeners.delete(listenerState); + }); + return closePromise; + }, + destroySockets: () => { + for (const socket of sockets) { + socket.destroy(); + } + }, + }; + this.#listeners.add(listenerState); + + return { + async close() { + const closed = listenerState.beginClose(); + listenerState.destroySockets(); + await closed; + }, + node: { server }, + ready: async () => await ready, + get url() { + return url; + }, + }; + } + + async waitForActiveRunner(timeoutMs: number = RUNNER_READY_TIMEOUT_MS): Promise { + const deadline = Date.now() + timeoutMs; + while (this.#active === undefined && !this.#closed) { + if (Date.now() >= deadline) { + throw new Error("Timed out waiting for the development worker to become ready."); + } + await new Promise((resolvePromise) => { + this.#activeWaiters.push(resolvePromise); + setTimeout(resolvePromise, 100); + }); + } + if (this.#closed) { + throw new Error("Development server is closed."); + } + } + + close(): Promise { + this.#closePromise ??= this.#close(); + return this.#closePromise; + } + + async #close(): Promise { + this.#closed = true; + this.#wakeActiveWaiters(); + + const listeners = [...this.#listeners]; + const listenerClosePromises = listeners.map((listener) => listener.beginClose()); + + const slots = [this.#active, ...this.#draining].filter( + (slot): slot is RunnerSlot => slot !== undefined, + ); + this.#active = undefined; + this.#draining.clear(); + await Promise.all(slots.map(async (slot) => await this.#releaseSlot(slot))); + + for (const listener of listeners) { + listener.destroySockets(); + } + await Promise.all(listenerClosePromises); + } + + async #releaseSlot(slot: RunnerSlot): Promise { + if (slot.released) { + return; + } + slot.released = true; + await slot.runner.close().catch(() => undefined); + if (slot.dispose !== undefined && !slot.disposed) { + slot.disposed = true; + await slot.dispose().catch((error) => { + this.#logger.error( + `[eve:dev] failed to dispose a retired dev host: ${toErrorMessage(error)}`, + ); + }); + } + } + + async #handleRunnerClose(slot: RunnerSlot): Promise { + if (this.#closed || this.#active !== slot) { + return; + } + this.#active = undefined; + this.#logger.error("[eve:dev] dev worker exited; restarting."); + // The crashed worker's workspace must survive for the restart, so the + // replacement inherits its dispose and the crashed slot releases nothing. + slot.disposed = true; + slot.released = true; + try { + await this.replaceWorker({ + dispose: slot.dispose, + entry: slot.entry, + workerData: slot.workerData, + }); + } catch (error) { + this.#logger.error(`[eve:dev] dev worker restart failed: ${toErrorMessage(error)}`); + } + } + + #drainInBackground(slot: RunnerSlot): void { + this.#draining.add(slot); + const finishDrain = () => { + this.#draining.delete(slot); + void this.#releaseSlot(slot); + }; + if (slot.activeExchanges === 0) { + finishDrain(); + return; + } + slot.quietListeners.push(finishDrain); + } + + #beginExchange(slot: RunnerSlot): () => void { + slot.activeExchanges += 1; + let settled = false; + return () => { + if (settled) { + return; + } + settled = true; + slot.activeExchanges -= 1; + if (slot.activeExchanges === 0) { + for (const listener of slot.quietListeners.splice(0)) { + listener(); + } + } + }; + } + + async #handleRequest(request: IncomingMessage, response: ServerResponse): Promise { + const requestAbort = new AbortController(); + const abortRequest = (cause?: unknown) => { + if (!requestAbort.signal.aborted) { + requestAbort.abort(cause); + } + }; + request.once("error", abortRequest); + response.once("close", () => { + if (!response.writableEnded) { + abortRequest(); + } + }); + + let settle: (() => void) | undefined; + try { + const publicRequest = createPublicRequest(request, requestAbort.signal); + const controlResponse = await this.#controlHandler?.(publicRequest); + if (controlResponse !== undefined) { + await writeResponse(response, controlResponse, requestAbort.signal); + return; + } + + await this.waitForActiveRunner(); + const slot = this.#active; + if (slot === undefined) { + throw new Error("Development worker is unavailable."); + } + settle = this.#beginExchange(slot); + const workerResponse = await slot.runner.fetch(publicRequest); + await writeResponse(response, workerResponse, requestAbort.signal); + } catch (error) { + if (!requestAbort.signal.aborted) { + writeRequestError(response, error); + } + } finally { + settle?.(); + request.off("error", abortRequest); + } + } + + async #handleUpgrade(request: IncomingMessage, socket: Socket, head: Buffer): Promise { + let settle: (() => void) | undefined; + try { + await this.waitForActiveRunner(); + const slot = this.#active; + if (slot === undefined) { + throw new Error("Development worker is unavailable."); + } + settle = this.#beginExchange(slot); + socket.once("close", settle); + socket.once("error", settle); + await slot.runner.upgrade({ node: { head, req: request, socket } }); + } catch { + settle?.(); + if (!socket.destroyed) { + socket.destroy(); + } + } + } + + #wakeActiveWaiters(): void { + for (const wake of this.#activeWaiters.splice(0)) { + wake(); + } + } +} diff --git a/packages/eve/src/internal/nitro/host/prepare-application-host.scenario.test.ts b/packages/eve/src/internal/nitro/host/prepare-application-host.scenario.test.ts index 9728e8377..4f0612f9e 100644 --- a/packages/eve/src/internal/nitro/host/prepare-application-host.scenario.test.ts +++ b/packages/eve/src/internal/nitro/host/prepare-application-host.scenario.test.ts @@ -1,5 +1,5 @@ import { existsSync } from "node:fs"; -import { readFile, writeFile } from "node:fs/promises"; +import { readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -9,10 +9,6 @@ import { createApplicationBuildWorkspace, removeApplicationBuildWorkspace, } from "#internal/application/build-workspace.js"; -import { - pruneDevelopmentRuntimeArtifactsSnapshots, - resolveDevelopmentRuntimeArtifactsPointerPath, -} from "#internal/nitro/dev-runtime-artifacts.js"; import { useTemporaryAppRoots } from "#internal/testing/use-temporary-app-roots.js"; import { prepareDevelopmentApplicationHost, @@ -21,17 +17,6 @@ import { const createAppRoot = useTemporaryAppRoots(); -interface DevelopmentRuntimePointer { - readonly runtimeAppRoot: string; - readonly snapshotRoot: string; -} - -async function readDevelopmentRuntimePointer(appRoot: string): Promise { - return JSON.parse( - await readFile(resolveDevelopmentRuntimeArtifactsPointerPath(appRoot), "utf8"), - ) as DevelopmentRuntimePointer; -} - describe("application host preparation", () => { afterEach(() => { vi.unstubAllEnvs(); @@ -105,25 +90,28 @@ describe("application host preparation", () => { await writeFile(instrumentationModulePath, "export default {};\n"); const firstHost = await prepareDevelopmentApplicationHost(appRoot); - const firstPointer = await readDevelopmentRuntimePointer(appRoot); - const stableHostDirectory = join(appRoot, ".eve", "host"); - const stableBootstrapPath = join(stableHostDirectory, "compiled-artifacts-bootstrap.mjs"); + const firstHostDirectory = firstHost.workspace.artifactsDir; + const firstBootstrapPath = join(firstHostDirectory, "compiled-artifacts-bootstrap.mjs"); const snapshotBootstrapPath = join( - firstPointer.runtimeAppRoot, + firstHost.generation.runtimeAppRoot, ".eve", "compile", "compiled-artifacts-bootstrap.mjs", ); - expect(firstHost.compiledArtifacts.bootstrapPath).toBe(stableBootstrapPath); + expect(firstHost.compileResult.paths.compileDirectoryPath).toBe( + join(firstHost.workspace.compilerArtifactsDir, "compile"), + ); + expect(existsSync(join(appRoot, ".eve", "compile"))).toBe(false); + expect(firstHost.compiledArtifacts.bootstrapPath).toBe(firstBootstrapPath); expect(firstHost.compiledArtifacts.workflowWorldPluginPath).toBe( - join(stableHostDirectory, "compiled-artifacts-workflow-world.mjs"), + join(firstHostDirectory, "compiled-artifacts-workflow-world.mjs"), ); expect(firstHost.compiledArtifacts.bootstrapPath).not.toContain("/.eve/dev-runtime/snapshots/"); expect(firstHost.compiledArtifacts.instrumentationSourcePath).toBe( - join(stableHostDirectory, "compiled-artifacts-instrumentation-source.mjs"), + join(firstHostDirectory, "compiled-artifacts-instrumentation-source.mjs"), ); - expect(await readFile(stableBootstrapPath, "utf8")).not.toContain( + expect(await readFile(firstBootstrapPath, "utf8")).not.toContain( normalizeEsmImportSpecifier(agentModulePath), ); await expect( @@ -136,19 +124,13 @@ describe("application host preparation", () => { 'export default { model: "openai/gpt-5.4" };\n// revision two\n', ); const nextHost = await prepareDevelopmentApplicationHost(appRoot); - const nextPointer = await readDevelopmentRuntimePointer(appRoot); - expect(nextHost.compiledArtifacts.bootstrapPath).toBe(stableBootstrapPath); - expect(nextPointer.snapshotRoot).not.toBe(firstPointer.snapshotRoot); + expect(nextHost.workspace.rootDir).not.toBe(firstHost.workspace.rootDir); + expect(nextHost.generation.snapshotRoot).not.toBe(firstHost.generation.snapshotRoot); - await pruneDevelopmentRuntimeArtifactsSnapshots({ - appRoot, - now: Date.now() + 1_000, - recentWindowMs: 0, - retainCount: 0, - }); + await rm(firstHost.generation.snapshotRoot, { force: true, recursive: true }); - expect(existsSync(firstPointer.snapshotRoot)).toBe(true); - expect(existsSync(stableBootstrapPath)).toBe(true); + expect(existsSync(firstHost.generation.snapshotRoot)).toBe(false); + expect(existsSync(firstBootstrapPath)).toBe(true); }); }); diff --git a/packages/eve/src/internal/nitro/host/prepare-application-host.ts b/packages/eve/src/internal/nitro/host/prepare-application-host.ts index c341cb8c4..b1d9d7583 100644 --- a/packages/eve/src/internal/nitro/host/prepare-application-host.ts +++ b/packages/eve/src/internal/nitro/host/prepare-application-host.ts @@ -1,13 +1,6 @@ -import { - compileAgent, - compileAgentInBuildWorkspace, - type CompileAgentResult, -} from "#compiler/compile-agent.js"; +import { compileAgentInWorkspace, type CompileAgentResult } from "#compiler/compile-agent.js"; import { createScheduleRegistrations } from "#runtime/schedules/register.js"; -import { - loadResolvedCompiledSchedules, - resolveSchedules, -} from "#runtime/schedules/resolve-schedule.js"; +import { resolveSchedules } from "#runtime/schedules/resolve-schedule.js"; import type { ResolvedScheduleDefinition } from "#runtime/types.js"; import type { ApplicationBuildWorkspace } from "#internal/application/build-workspace.js"; import { join } from "node:path"; @@ -17,53 +10,69 @@ import { writeDevelopmentCompiledArtifactsFiles, } from "#internal/application/compiled-artifacts.js"; import { - resolveApplicationHostArtifactsDirectory, - resolveWorkflowBuildDirectory, -} from "#internal/application/paths.js"; -import { createAuthoredSourceRuntimeCompiledArtifactsSource } from "#internal/application/runtime-compiled-artifacts-source.js"; -import { - activateDevelopmentGeneration, discardDevelopmentGeneration, stageDevelopmentGeneration, } from "#internal/nitro/development-generation.js"; -import type { PreparedApplicationHost } from "#internal/nitro/host/types.js"; +import { + createDevelopmentHostWorkspace, + removeDevelopmentHostWorkspace, +} from "#internal/nitro/host/dev-host-workspace.js"; +import type { + PreparedApplicationHost, + PreparedDevelopmentApplicationHost, +} from "#internal/nitro/host/types.js"; /** - * Compiles one authored app in place and stages the package-owned artifacts - * the dev-server Nitro host needs, activating a fresh runtime-artifacts - * snapshot for hot reload. + * Compiles one authored app and stages an isolated runtime generation and host + * candidate without changing the active development server. */ export async function prepareDevelopmentApplicationHost( - startPath: string, -): Promise { - const compileResult = await compileAgent({ - startPath, - }); - const schedules = await loadResolvedCompiledSchedules({ - compiledArtifactsSource: createAuthoredSourceRuntimeCompiledArtifactsSource( - compileResult.project.appRoot, - ), - }); - const generation = await stageDevelopmentGeneration(compileResult); + appRoot: string, +): Promise { + const workspace = await createDevelopmentHostWorkspace(appRoot); + let generation: Awaited> | undefined; try { + const compileResult = await compileAgentInWorkspace({ + artifactLocations: { + publishedRoot: join(appRoot, ".eve"), + writeRoot: workspace.compilerArtifactsDir, + }, + startPath: appRoot, + }); + const schedules = await resolveSchedules({ manifest: compileResult.manifest }); + generation = await stageDevelopmentGeneration(compileResult); const compiledArtifacts = await writeDevelopmentCompiledArtifactsFiles({ compileResult, - outDir: resolveApplicationHostArtifactsDirectory(compileResult.project.appRoot), + outDir: workspace.artifactsDir, runtimeAppRoot: generation.runtimeAppRoot, }); - await activateDevelopmentGeneration({ - appRoot: compileResult.project.appRoot, + return { + ...createPreparedApplicationHost({ + compileResult, + compiledArtifacts, + schedules, + workflowBuildDir: workspace.workflowBuildDir, + }), generation, - }); - return createPreparedApplicationHost({ - compileResult, - compiledArtifacts, - schedules, - workflowBuildDir: resolveWorkflowBuildDirectory(compileResult.project.appRoot), - }); + workspace, + }; } catch (error) { - await discardDevelopmentGeneration(generation); + const cleanupOperations: Promise[] = [removeDevelopmentHostWorkspace(workspace)]; + if (generation !== undefined) { + cleanupOperations.push(discardDevelopmentGeneration(generation)); + } + const cleanup = await Promise.allSettled(cleanupOperations); + const cleanupErrors = cleanup.flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + if (cleanupErrors.length > 0) { + throw new AggregateError( + [error, ...cleanupErrors], + "Failed to prepare and discard a development host candidate.", + { cause: error }, + ); + } throw error; } } @@ -78,7 +87,7 @@ export async function prepareDevelopmentApplicationHost( export async function prepareProductionApplicationHost( workspace: ApplicationBuildWorkspace, ): Promise { - const compileResult = await compileAgentInBuildWorkspace({ + const compileResult = await compileAgentInWorkspace({ artifactLocations: { publishedRoot: join(workspace.publication.output.finalDir, ".eve"), writeRoot: workspace.compiler.artifactsDir, diff --git a/packages/eve/src/internal/nitro/host/start-development-server.test.ts b/packages/eve/src/internal/nitro/host/start-development-server.test.ts index abc70fb2e..4bf56df7f 100644 --- a/packages/eve/src/internal/nitro/host/start-development-server.test.ts +++ b/packages/eve/src/internal/nitro/host/start-development-server.test.ts @@ -1,6 +1,3 @@ -import { EventEmitter } from "node:events"; -import type { IncomingMessage } from "node:http"; -import type { Socket } from "node:net"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -28,8 +25,12 @@ const mocks = vi.hoisted(() => { }; const devServer = { close: vi.fn(async () => undefined), + discardCandidate: vi.fn(async () => undefined), listen: vi.fn(() => listenerServer), + replaceWorker: vi.fn(async () => undefined), + setControlHandler: vi.fn(), upgrade: vi.fn(async (_req: unknown, _socket: unknown, _head: unknown) => undefined), + waitForActiveRunner: vi.fn(async () => undefined), }; const files = new Map(); const devHandlers: Nitro["options"]["devHandlers"] = []; @@ -49,9 +50,20 @@ const mocks = vi.hoisted(() => { return { authoredSourceWatcher, - buildNitro: vi.fn(async () => undefined), + buildDevelopmentHostCandidate: vi.fn(async (input: { nitro: { close(): Promise } }) => { + await input.nitro.close(); + return { + entry: "/tmp/eve-test/.eve/dev-hosts/test/output/server/index.mjs", + workerData: {}, + }; + }), + createDevelopmentAuthoredRebuildCoordinator: vi.fn(async () => ({ + rebuild: vi.fn(), + })), createDevelopmentApplicationNitro: vi.fn(async () => nitro), - createDevServer: vi.fn(() => devServer), + createDevServer: vi.fn(function createDevServerMock() { + return devServer; + }), devServer, fetch: vi.fn(async () => new Response(null, { status: 200 })), files, @@ -59,8 +71,27 @@ const mocks = vi.hoisted(() => { listenerServer, mkdir: vi.fn(async () => undefined), nitro, - prepareDevelopmentApplicationHost: vi.fn(async () => ({ appRoot: "/tmp/eve-test" })), - prepareNitro: vi.fn(async () => undefined), + prepareDevelopmentApplicationHost: vi.fn(async () => ({ + appRoot: "/tmp/eve-test", + generation: { + fingerprint: "test", + runtimeAppRoot: "/tmp/eve-test/.eve/dev-runtime/snapshots/test/source/app", + snapshotRoot: "/tmp/eve-test/.eve/dev-runtime/snapshots/test", + snapshotSourceRoot: "/tmp/eve-test/.eve/dev-runtime/snapshots/test/source", + sourceRoot: "/tmp/eve-test", + }, + workspace: { + artifactsDir: "/tmp/eve-test/.eve/dev-hosts/test/artifacts", + compilerArtifactsDir: "/tmp/eve-test/.eve/dev-hosts/test/compiler", + nitroBuildDir: "/tmp/eve-test/.eve/dev-hosts/test/nitro", + nitroOutputDir: "/tmp/eve-test/.eve/dev-hosts/test/output", + rootDir: "/tmp/eve-test/.eve/dev-hosts/test", + workflowBuildDir: "/tmp/eve-test/.eve/dev-hosts/test/workflow", + }, + })), + activateDevelopmentGeneration: vi.fn(async () => undefined), + discardDevelopmentGeneration: vi.fn(async () => undefined), + removeDevelopmentHostWorkspace: vi.fn(async () => undefined), readFile: vi.fn(async (path: string) => { if ( path.endsWith("/.eve/dev-server-state.v1.json") && @@ -113,10 +144,25 @@ vi.mock("node:fs/promises", () => ({ writeFile: mocks.writeFile, })); -vi.mock("nitro/builder", () => ({ - build: mocks.buildNitro, - createDevServer: mocks.createDevServer, - prepare: mocks.prepareNitro, +vi.mock("./drained-nitro-dev-server.js", () => ({ + DrainedNitroDevServer: mocks.createDevServer, +})); + +vi.mock("./dev-host-candidate.js", () => ({ + buildDevelopmentHostCandidate: mocks.buildDevelopmentHostCandidate, +})); + +vi.mock("./dev-authored-rebuild-coordinator.js", () => ({ + createDevelopmentAuthoredRebuildCoordinator: mocks.createDevelopmentAuthoredRebuildCoordinator, +})); + +vi.mock("./dev-host-workspace.js", () => ({ + removeDevelopmentHostWorkspace: mocks.removeDevelopmentHostWorkspace, +})); + +vi.mock("#internal/nitro/development-generation.js", () => ({ + activateDevelopmentGeneration: mocks.activateDevelopmentGeneration, + discardDevelopmentGeneration: mocks.discardDevelopmentGeneration, })); vi.mock("./create-application-nitro.js", () => ({ @@ -153,31 +199,6 @@ vi.mock("#execution/sandbox/bindings/local.js", async (importOriginal) => { }; }); -function createRequest(): IncomingMessage { - return { - headers: { - upgrade: "websocket", - }, - method: "GET", - } as IncomingMessage; -} - -function createSocket(): Socket { - const socket = new EventEmitter() as Socket; - Object.defineProperty(socket, "destroyed", { - configurable: true, - value: false, - }); - socket.destroy = vi.fn(() => { - Object.defineProperty(socket, "destroyed", { - configurable: true, - value: true, - }); - return socket; - }); - return socket; -} - const developmentServerStatePath = join("/tmp/eve-test", ".eve", "dev-server-state.v1.json"); function readStateRecord( @@ -244,15 +265,6 @@ async function loadStartDevelopmentServer(): Promise< }; } -async function startServer(): Promise<{ - close(): Promise; - url: string; -}> { - const startDevelopmentServer = await loadStartDevelopmentServer(); - - return await startDevelopmentServer("/tmp/eve-test"); -} - describe("normalizeDevelopmentServerClientUrl", () => { it("rewrites the IPv6 wildcard listen hostname to IPv6 loopback", async () => { const { normalizeDevelopmentServerClientUrl } = await import("./start-development-server.js"); @@ -393,7 +405,6 @@ describe("createDevelopmentServer", () => { expect(mocks.devServer.listen).toHaveBeenCalledWith({ hostname: "127.0.0.1", port: 2000, - silent: true, }); await server.close(); @@ -434,12 +445,10 @@ describe("createDevelopmentServer", () => { expect(mocks.devServer.listen).toHaveBeenNthCalledWith(1, { hostname: "127.0.0.1", port: 2000, - silent: true, }); expect(mocks.devServer.listen).toHaveBeenNthCalledWith(2, { hostname: "127.0.0.1", port: 2001, - silent: true, }); expect(server.url).toBe("http://127.0.0.1:2001/"); @@ -459,27 +468,31 @@ describe("createDevelopmentServer", () => { expect(mocks.files.has(developmentServerStatePath)).toBe(false); }); - async function callDevHandler( - handler: Nitro["options"]["devHandlers"][number]["handler"], - url: string, - ): Promise { - if (typeof handler !== "function") throw new Error("Expected a callable dev handler."); - return await handler({ - node: { req: { url } }, - } as Parameters[0]); + async function callControlHandler(url: string): Promise { + const call = mocks.devServer.setControlHandler.mock.calls.at(-1); + const handler = call?.[0] as ((request: Request) => Promise) | undefined; + if (handler === undefined) throw new Error("Missing runtime rebuild handler."); + return await handler(new Request(url)); } - it("registers a host-owned runtime rebuild handler that forces the live watcher", async () => { + it("serves runtime artifact state from the parent-owned control handler", async () => { const startDevelopmentServer = await loadStartDevelopmentServer(); const server = await startDevelopmentServer("/tmp/eve-test"); - const rebuildHandler = mocks.nitro.options.devHandlers.find( - (handler) => handler.route === "/eve/v1/dev/runtime-artifacts/rebuild", - ); - if (rebuildHandler === undefined) throw new Error("Missing runtime rebuild handler."); + const response = await callControlHandler("http://localhost/eve/v1/dev/runtime-artifacts"); - const response = await callDevHandler( - rebuildHandler.handler, - "/eve/v1/dev/runtime-artifacts/rebuild?force=1", + if (!(response instanceof Response)) throw new Error("Expected a Response."); + await expect(response.json()).resolves.toEqual({ revision: "/tmp/eve-test" }); + expect(mocks.authoredSourceWatcher.flush).not.toHaveBeenCalled(); + expect(mocks.authoredSourceWatcher.rebuild).not.toHaveBeenCalled(); + + await server.close(); + }); + + it("registers a host-owned runtime rebuild handler that forces the live watcher", async () => { + const startDevelopmentServer = await loadStartDevelopmentServer(); + const server = await startDevelopmentServer("/tmp/eve-test"); + const response = await callControlHandler( + "http://localhost/eve/v1/dev/runtime-artifacts/rebuild?force=1", ); expect(mocks.authoredSourceWatcher.rebuild).toHaveBeenCalledOnce(); @@ -493,12 +506,7 @@ describe("createDevelopmentServer", () => { it("registers a host-owned runtime rebuild handler that flushes queued changes", async () => { const startDevelopmentServer = await loadStartDevelopmentServer(); const server = await startDevelopmentServer("/tmp/eve-test"); - const rebuildHandler = mocks.nitro.options.devHandlers.find( - (handler) => handler.route === "/eve/v1/dev/runtime-artifacts/rebuild", - ); - if (rebuildHandler === undefined) throw new Error("Missing runtime rebuild handler."); - - await callDevHandler(rebuildHandler.handler, "/eve/v1/dev/runtime-artifacts/rebuild"); + await callControlHandler("http://localhost/eve/v1/dev/runtime-artifacts/rebuild"); expect(mocks.authoredSourceWatcher.flush).toHaveBeenCalledOnce(); expect(mocks.authoredSourceWatcher.rebuild).not.toHaveBeenCalled(); @@ -807,76 +815,4 @@ describe("createDevelopmentServer", () => { /Invalid PORT environment variable/, ); }); - - it("swallows websocket upgrade rejections from the Nitro dev server", async () => { - const originalUpgrade = vi.fn( - async (_req: unknown, _socket: unknown, _head: unknown): Promise => { - throw new Error("Upstream server did not upgrade the connection"); - }, - ); - Object.assign(mocks.nitro.options.features, { websocket: true }); - mocks.devServer.upgrade = originalUpgrade; - - const server = await startServer(); - - try { - const socket = createSocket(); - await expect( - mocks.devServer.upgrade(createRequest(), socket, Buffer.alloc(0)), - ).resolves.toBeUndefined(); - - expect(originalUpgrade).toHaveBeenCalledTimes(1); - expect(socket.destroy).toHaveBeenCalledTimes(1); - } finally { - await server.close(); - } - }); - - it("rejects websocket upgrades before Nitro proxying when websocket support is disabled", async () => { - const originalUpgrade = vi.fn( - async (_req: unknown, _socket: unknown, _head: unknown): Promise => undefined, - ); - mocks.devServer.upgrade = originalUpgrade; - - const server = await startServer(); - - try { - const socket = createSocket(); - await expect( - mocks.devServer.upgrade(createRequest(), socket, Buffer.alloc(0)), - ).resolves.toBeUndefined(); - - expect(originalUpgrade).not.toHaveBeenCalled(); - expect(socket.destroy).toHaveBeenCalledTimes(1); - } finally { - await server.close(); - } - }); - - it("handles socket errors emitted during websocket upgrade handling", async () => { - const originalUpgrade = vi.fn( - async (_req: unknown, socket: unknown, _head: unknown): Promise => { - const upgradeSocket = socket as Socket; - - upgradeSocket.emit("error", new Error("socket failure")); - throw new Error("socket failure"); - }, - ); - Object.assign(mocks.nitro.options.features, { websocket: true }); - mocks.devServer.upgrade = originalUpgrade; - - const server = await startServer(); - - try { - const socket = createSocket(); - await expect( - mocks.devServer.upgrade(createRequest(), socket, Buffer.alloc(0)), - ).resolves.toBeUndefined(); - - expect(originalUpgrade).toHaveBeenCalledTimes(1); - expect(socket.destroy).toHaveBeenCalledTimes(1); - } finally { - await server.close(); - } - }); }); diff --git a/packages/eve/src/internal/nitro/host/start-development-server.ts b/packages/eve/src/internal/nitro/host/start-development-server.ts index 4d3f84ece..d2ad4a0b2 100644 --- a/packages/eve/src/internal/nitro/host/start-development-server.ts +++ b/packages/eve/src/internal/nitro/host/start-development-server.ts @@ -1,16 +1,19 @@ -import type { IncomingMessage } from "node:http"; -import type { Socket } from "node:net"; - import { EVE_DEV_ENV_FLAG } from "#internal/application/optional-package-install.js"; -import { build as buildNitro, createDevServer, prepare } from "nitro/builder"; import type { Nitro } from "nitro/types"; import { createDevelopmentApplicationNitro } from "#internal/nitro/host/create-application-nitro.js"; +import { DrainedNitroDevServer } from "#internal/nitro/host/drained-nitro-dev-server.js"; import { createDevelopmentNitroArtifactsConfig } from "#internal/nitro/host/artifacts-config.js"; import type { AuthoredSourceWatcherHandle } from "#internal/nitro/host/dev-authored-source-watcher.js"; import { prepareDevelopmentApplicationHost } from "#internal/nitro/host/prepare-application-host.js"; -import { EVE_DEV_RUNTIME_ARTIFACTS_REBUILD_ROUTE_PATH } from "#protocol/routes.js"; +import { buildDevelopmentHostCandidate } from "#internal/nitro/host/dev-host-candidate.js"; +import { removeDevelopmentHostWorkspace } from "#internal/nitro/host/dev-host-workspace.js"; +import { createDevelopmentAuthoredRebuildCoordinator } from "#internal/nitro/host/dev-authored-rebuild-coordinator.js"; +import { + EVE_DEV_RUNTIME_ARTIFACTS_REBUILD_ROUTE_PATH, + EVE_DEV_RUNTIME_ARTIFACTS_ROUTE_PATH, +} from "#protocol/routes.js"; import { resolveDiscoveryProject } from "#discover/project.js"; import { DevelopmentServerState } from "#internal/nitro/host/dev-server-state.js"; import { toErrorMessage } from "#shared/errors.js"; @@ -34,6 +37,7 @@ import type { DevelopmentServerHandle, DevelopmentServerOptions, StartedDevelopmentServer, + PreparedDevelopmentApplicationHost, } from "#internal/nitro/host/types.js"; import { loadDevelopmentEnvironmentFiles } from "#cli/dev/environment.js"; import { @@ -43,6 +47,10 @@ import { import { detectPackageManager, type PackageManagerKind } from "#setup/package-manager.js"; import { eveDevArguments } from "#setup/primitives/index.js"; import { devBootPhase } from "#internal/dev-boot-progress.js"; +import { + activateDevelopmentGeneration, + discardDevelopmentGeneration, +} from "#internal/nitro/development-generation.js"; const MAX_ALLOWED_DEVELOPMENT_SERVER_PORT = 65_535; const WORKFLOW_LOCAL_BASE_URL_ENV = "WORKFLOW_LOCAL_BASE_URL"; @@ -108,8 +116,7 @@ function isAddressInUseError(error: unknown): error is NodeJS.ErrnoException { return error instanceof Error && "code" in error && error.code === "EADDRINUSE"; } -type NitroDevelopmentServer = ReturnType; -type NitroDevelopmentServerUpgrade = NitroDevelopmentServer["upgrade"]; +type NitroDevelopmentServer = DrainedNitroDevServer; function resolveDevelopmentServerPort(port: number | string | undefined): number { const resolvedPort = @@ -227,73 +234,25 @@ function installWorkflowLocalQueueEnvironment(serverUrl: string): () => void { }; } -function attachTemporarySocketErrorHandler(socket: Socket): () => void { - // Keep early socket failures from becoming uncaught EventEmitter errors - // while Nitro/httpxy installs its own upgrade-path listeners. - const onSocketError = () => {}; - - socket.once("error", onSocketError); - - return () => { - socket.off("error", onSocketError); - }; -} - -function shouldProxyDevelopmentServerWebSocketUpgrades(nitro: Nitro): boolean { - return nitro.options.features.websocket === true || nitro.options.experimental.websocket === true; -} - -function guardDevelopmentServerWebSocketUpgrades( - nitro: Nitro, - devServer: NitroDevelopmentServer, -): void { - const originalUpgrade = devServer.upgrade.bind(devServer) as NitroDevelopmentServerUpgrade; - const websocketEnabled = shouldProxyDevelopmentServerWebSocketUpgrades(nitro); - const guardedUpgrade: NitroDevelopmentServerUpgrade = async ( - req: IncomingMessage, - socket: Socket, - head: unknown, - ) => { - if (!websocketEnabled) { - if (!socket.destroyed) { - socket.destroy(); - } - return; - } - - const removeSocketErrorHandler = attachTemporarySocketErrorHandler(socket); - - try { - await originalUpgrade(req, socket, head); - } catch { - if (!socket.destroyed) { - socket.destroy(); - } - } finally { - removeSocketErrorHandler(); - } - }; - - devServer.upgrade = guardedUpgrade; -} - function addDevelopmentRuntimeArtifactsRebuildHandler(input: { readonly appRoot: string; - readonly nitro: Nitro; + readonly devServer: DrainedNitroDevServer; readonly watcher: AuthoredSourceWatcherHandle; }): void { - input.nitro.options.devHandlers.push({ - route: EVE_DEV_RUNTIME_ARTIFACTS_REBUILD_ROUTE_PATH, - handler: async (event) => { - const requestUrl = event.node?.req.url ?? ""; - const url = new URL(requestUrl, "http://localhost"); - if (url.searchParams.get("force") === "1") { - await input.watcher.rebuild(); - } else { - await input.watcher.flush(); - } + input.devServer.setControlHandler(async (request) => { + const url = new URL(request.url); + if (url.pathname === EVE_DEV_RUNTIME_ARTIFACTS_ROUTE_PATH && request.method === "GET") { return handleDevRuntimeArtifactsRequest({ appRoot: input.appRoot }); - }, + } + if (url.pathname !== EVE_DEV_RUNTIME_ARTIFACTS_REBUILD_ROUTE_PATH || request.method !== "GET") { + return undefined; + } + if (url.searchParams.get("force") === "1") { + await input.watcher.rebuild(); + } else { + await input.watcher.flush(); + } + return handleDevRuntimeArtifactsRequest({ appRoot: input.appRoot }); }); } @@ -365,7 +324,7 @@ function createDevelopmentServerStartupCleanupError( async function listenForDevelopmentServer(input: { readonly devServer: NitroDevelopmentServer; - readonly host?: string; + readonly host: string; readonly port: number | string | undefined; readonly retryOnAddressInUse: boolean; }) { @@ -379,7 +338,6 @@ async function listenForDevelopmentServer(input: { const server = input.devServer.listen({ hostname: input.host, port, - silent: true, }); try { @@ -453,6 +411,9 @@ async function startNitroDevelopmentServer( let devServer: NitroDevelopmentServer | undefined; let restoreWorkflowLocalQueueEnvironment: (() => void) | undefined; let authoredSourceWatcher: AuthoredSourceWatcherHandle | undefined; + let preparedDevelopmentHost: PreparedDevelopmentApplicationHost | undefined; + let initialGenerationPublished = false; + let initialWorkspaceTransferred = false; try { const preparedHost = await devBootPhase( @@ -460,15 +421,12 @@ async function startNitroDevelopmentServer( () => prepareDevelopmentApplicationHost(project.appRoot), options.onBootProgress, ); + preparedDevelopmentHost = preparedHost; const compiledArtifactsSource = resolveNitroCompiledArtifactsSource( createDevelopmentNitroArtifactsConfig({ appRoot: preparedHost.appRoot, }), ); - startDevelopmentSandboxPrewarmInBackground({ - appRoot: preparedHost.appRoot, - compiledArtifactsSource, - }); pruneLocalSandboxTemplatesInBackground(preparedHost.appRoot); const activeNitro = await devBootPhase( "creating dev server", @@ -476,9 +434,8 @@ async function startNitroDevelopmentServer( options.onBootProgress, ); nitro = activeNitro; - devServer = createDevServer(activeNitro); + devServer = new DrainedNitroDevServer(activeNitro.logger); const activeDevServer = devServer; - guardDevelopmentServerWebSocketUpgrades(activeNitro, devServer); const hostname = options.host ?? activeNitro.options.devServer.hostname ?? DEFAULT_DEVELOPMENT_SERVER_HOST; const retryOnAddressInUse = requestedPort === undefined; @@ -503,11 +460,35 @@ async function startNitroDevelopmentServer( await devBootPhase( "building dev bundle", async () => { - await prepare(activeNitro); - await buildNitro(activeNitro); + const payload = await buildDevelopmentHostCandidate({ + host: preparedHost, + nitro: activeNitro, + }); + nitro = undefined; + const workspace = preparedHost.workspace; + initialWorkspaceTransferred = true; + await activeDevServer.replaceWorker({ + dispose: async () => await removeDevelopmentHostWorkspace(workspace), + entry: payload.entry, + workerData: payload.workerData, + }); + await activateDevelopmentGeneration({ + appRoot: preparedHost.appRoot, + generation: preparedHost.generation, + }); + initialGenerationPublished = true; }, options.onBootProgress, ); + startDevelopmentSandboxPrewarmInBackground({ + appRoot: preparedHost.appRoot, + compiledArtifactsSource, + }); + + const rebuildCoordinator = await createDevelopmentAuthoredRebuildCoordinator({ + devServer: activeDevServer, + initialHost: preparedHost, + }); authoredSourceWatcher = await devBootPhase( "starting file watcher", @@ -515,7 +496,7 @@ async function startNitroDevelopmentServer( const { startAuthoredSourceWatcher } = await import("#internal/nitro/host/dev-authored-source-watcher.js"); return startAuthoredSourceWatcher({ - nitro: activeNitro, + coordinator: rebuildCoordinator, preparedHost, }); }, @@ -523,7 +504,7 @@ async function startNitroDevelopmentServer( ); addDevelopmentRuntimeArtifactsRebuildHandler({ appRoot: project.appRoot, - nitro: activeNitro, + devServer: activeDevServer, watcher: authoredSourceWatcher, }); await state.write(serverUrl); @@ -534,7 +515,6 @@ async function startNitroDevelopmentServer( const authoredSourceWatcherOnClose = authoredSourceWatcher; const devServerOnClose = devServer; - const nitroOnClose = activeNitro; let closePromise: Promise | undefined; const close = (): Promise => { closePromise ??= (async () => { @@ -542,7 +522,7 @@ async function startNitroDevelopmentServer( authoredSourceWatcher: authoredSourceWatcherOnClose, devServer: devServerOnClose, developmentSandboxRunId, - nitro: nitroOnClose, + nitro: undefined, }); if (cleanup.listenerClosed) { await state.remove().catch(() => {}); @@ -573,6 +553,20 @@ async function startNitroDevelopmentServer( nitro, }); const cleanupErrors = [...cleanup.errors]; + if (preparedDevelopmentHost !== undefined && !initialGenerationPublished) { + await discardDevelopmentGeneration(preparedDevelopmentHost.generation).catch( + (cleanupError) => { + cleanupErrors.push(cleanupError); + }, + ); + } + if (preparedDevelopmentHost !== undefined && !initialWorkspaceTransferred) { + await removeDevelopmentHostWorkspace(preparedDevelopmentHost.workspace).catch( + (cleanupError) => { + cleanupErrors.push(cleanupError); + }, + ); + } restoreWorkflowLocalQueueEnvironment?.(); clearInitializedDevelopmentSandboxBackendNames(developmentSandboxRunId); if (cleanup.listenerClosed) { diff --git a/packages/eve/src/internal/nitro/host/types.ts b/packages/eve/src/internal/nitro/host/types.ts index fcbc97a3a..feadd80e4 100644 --- a/packages/eve/src/internal/nitro/host/types.ts +++ b/packages/eve/src/internal/nitro/host/types.ts @@ -3,6 +3,8 @@ import type { ScheduleRegistration } from "#runtime/schedules/register.js"; import type { ResolvedScheduleDefinition } from "#runtime/types.js"; import type { GeneratedCompiledArtifactsFiles } from "#internal/application/compiled-artifacts.js"; import type { DevBootProgressReporter } from "#internal/dev-boot-progress.js"; +import type { DevelopmentGeneration } from "#internal/nitro/development-generation.js"; +import type { DevelopmentHostWorkspace } from "#internal/nitro/host/dev-host-workspace.js"; /** * Route surface included in one programmatic Nitro host build. @@ -74,3 +76,8 @@ export interface PreparedApplicationHost { schedules: readonly ResolvedScheduleDefinition[]; workflowBuildDir: string; } + +export interface PreparedDevelopmentApplicationHost extends PreparedApplicationHost { + generation: DevelopmentGeneration; + workspace: DevelopmentHostWorkspace; +} diff --git a/packages/eve/test/scenarios/dev-server-drain.scenario.test.ts b/packages/eve/test/scenarios/dev-server-drain.scenario.test.ts new file mode 100644 index 000000000..4625ba30a --- /dev/null +++ b/packages/eve/test/scenarios/dev-server-drain.scenario.test.ts @@ -0,0 +1,234 @@ +import { Agent } from "node:http"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { EVE_HEALTH_ROUTE_PATH } from "../../src/protocol/routes.js"; +import { WEATHER_AGENT_DESCRIPTOR } from "../../src/internal/testing/scenario-apps/weather-agent.js"; +import { + type ScenarioAppDescriptor, + useScenarioApp, +} from "../../src/internal/testing/scenario-app.js"; +import { sendDevelopmentMessage } from "../dev-client-harness/send-message.js"; +import { createDevelopmentSessionState } from "../dev-client-harness/session.js"; +import { + fetchText, + hasKnownDevServerFailure, + requestWithAgent, + startEveDev, + waitForCondition, + type RunningEveDev, +} from "./dev-server-harness.js"; + +const scenarioApp = useScenarioApp(); + +const DRAIN_SCENARIO_TIMEOUT_MS = 360_000; + +const DRAIN_CHANNEL_SOURCE = [ + 'import { randomUUID } from "node:crypto";', + 'import { defineChannel, GET } from "eve/channels";', + "", + "const workerId = randomUUID();", + "", + "export default defineChannel({", + " routes: [", + ' GET("/drain/worker-id", () => new Response(workerId)),', + ' GET("/drain/request-ip", (_request, context) => new Response(String(context.requestIp))),', + ' GET("/drain/crash", () => {', + " setTimeout(() => process.exit(7), 25);", + ' return new Response("crashing");', + " }),", + ' GET("/drain/slow-stream", () => {', + " let timer: ReturnType | undefined;", + " const body = new ReadableStream({", + " start(controller) {", + ' controller.enqueue(new TextEncoder().encode("chunk\\n"));', + " timer = setInterval(() => {", + ' controller.enqueue(new TextEncoder().encode("chunk\\n"));', + " }, 50);", + " },", + " cancel() {", + " clearInterval(timer);", + " },", + " });", + " return new Response(body);", + " }),", + " ],", + "});", + "", +].join("\n"); + +const DRAIN_DESCRIPTOR: ScenarioAppDescriptor = { + ...WEATHER_AGENT_DESCRIPTOR, + files: { + ...WEATHER_AGENT_DESCRIPTOR.files, + "agent/channels/drain.ts": DRAIN_CHANNEL_SOURCE, + }, + name: "weather-agent-drain", +}; + +async function triggerWorkerReplacement(server: RunningEveDev, appRoot: string): Promise { + const previousWorkerId = await fetchText(server.url, "/drain/worker-id"); + await writeFile(join(appRoot, ".env.local"), `EVE_DRAIN_RELOAD=${Date.now()}\n`); + await waitForCondition( + async () => { + try { + return (await fetchText(server.url, "/drain/worker-id")) !== previousWorkerId; + } catch { + return false; + } + }, + () => + [ + "Timed out waiting for a replacement worker.", + `stdout:\n${server.stdout()}`, + `stderr:\n${server.stderr()}`, + ].join("\n\n"), + ); +} + +describe("eve dev drained worker replacement", () => { + it( + "keeps an admitted stream flowing on the retired worker across a replacement", + async () => { + const app = await scenarioApp(DRAIN_DESCRIPTOR); + const server = await startEveDev(app.appRoot); + + try { + const streamResponse = await fetch(new URL("/drain/slow-stream", server.url)); + const reader = streamResponse.body?.getReader(); + await reader?.read(); + + await triggerWorkerReplacement(server, app.appRoot); + + // The retired worker must still be producing: several further chunks + // arrive after the swap, then the client walks away cleanly. + for (let i = 0; i < 5; i += 1) { + const result = await reader?.read(); + expect(result?.done).toBe(false); + } + await reader?.cancel(); + + const turn = await sendDevelopmentMessage({ + message: "What's the weather in Lisbon?", + session: createDevelopmentSessionState(), + serverUrl: server.url, + }); + expect(turn.events.some((event) => event.type === "message.completed")).toBe(true); + expect(hasKnownDevServerFailure(`${server.stdout()}\n${server.stderr()}`)).toBe(false); + } finally { + await server.stop(); + } + }, + DRAIN_SCENARIO_TIMEOUT_MS, + ); + + it( + "serves a kept-alive connection across a worker replacement", + async () => { + const app = await scenarioApp(DRAIN_DESCRIPTOR); + const server = await startEveDev(app.appRoot); + const agent = new Agent({ keepAlive: true, maxSockets: 1 }); + + try { + const first = await requestWithAgent(new URL("/drain/worker-id", server.url).href, agent); + expect(first.localPort).toBeDefined(); + + await triggerWorkerReplacement(server, app.appRoot); + + const second = await requestWithAgent(new URL("/drain/worker-id", server.url).href, agent); + expect(second.localPort).toBe(first.localPort); + expect(second.body).not.toBe(first.body); + expect(hasKnownDevServerFailure(`${server.stdout()}\n${server.stderr()}`)).toBe(false); + } finally { + agent.destroy(); + await server.stop(); + } + }, + DRAIN_SCENARIO_TIMEOUT_MS, + ); + + it( + "restarts the worker after a crash and keeps serving", + async () => { + const app = await scenarioApp(DRAIN_DESCRIPTOR); + const server = await startEveDev(app.appRoot); + + try { + const firstWorkerId = await fetchText(server.url, "/drain/worker-id"); + await fetch(new URL("/drain/crash", server.url)); + + await waitForCondition( + async () => { + try { + return (await fetchText(server.url, "/drain/worker-id")) !== firstWorkerId; + } catch { + return false; + } + }, + () => + [ + "Timed out waiting for the crashed worker to restart.", + `stdout:\n${server.stdout()}`, + `stderr:\n${server.stderr()}`, + ].join("\n\n"), + ); + const health = await fetch(new URL(EVE_HEALTH_ROUTE_PATH, server.url)); + expect(health.status).toBe(200); + } finally { + await server.stop(); + } + }, + DRAIN_SCENARIO_TIMEOUT_MS, + ); + + it( + "shuts down within a bounded deadline while a stream is open", + async () => { + const app = await scenarioApp(DRAIN_DESCRIPTOR); + const server = await startEveDev(app.appRoot); + const streamResponse = await fetch(new URL("/drain/slow-stream", server.url)); + const reader = streamResponse.body?.getReader(); + await reader?.read(); + + const stopStart = Date.now(); + await server.stop(); + + // The harness escalates to SIGKILL at 10s; a graceful exit must beat it. + expect(Date.now() - stopStart).toBeLessThan(9_000); + await expect( + (async () => { + for (;;) { + const result = await reader?.read(); + if (result === undefined || result.done) { + return "done"; + } + } + })().catch(() => "errored"), + ).resolves.toBeDefined(); + }, + DRAIN_SCENARIO_TIMEOUT_MS, + ); + + it( + "reports a socket-derived client address despite forged forwarding headers", + async () => { + const app = await scenarioApp(DRAIN_DESCRIPTOR); + const server = await startEveDev(app.appRoot); + + try { + const response = await fetch(new URL("/drain/request-ip", server.url), { + headers: { "x-forwarded-for": "203.0.113.7" }, + }); + expect(response.status).toBe(200); + const address = await response.text(); + expect(address).not.toBe("203.0.113.7"); + expect(address).toContain("127.0.0.1"); + } finally { + await server.stop(); + } + }, + DRAIN_SCENARIO_TIMEOUT_MS, + ); +}); diff --git a/packages/eve/test/scenarios/dev-server-harness.ts b/packages/eve/test/scenarios/dev-server-harness.ts new file mode 100644 index 000000000..95acabb81 --- /dev/null +++ b/packages/eve/test/scenarios/dev-server-harness.ts @@ -0,0 +1,433 @@ +import { spawn, spawnSync, type ChildProcessByStdio } from "node:child_process"; +import { request as requestHttp, type Agent } from "node:http"; +import { existsSync, watch as watchFileSystem } from "node:fs"; +import { dirname, join } from "node:path"; +import type { Readable } from "node:stream"; + +import { + EVE_DEV_RUNTIME_ARTIFACTS_REBUILD_ROUTE_PATH, + EVE_DEV_RUNTIME_ARTIFACTS_ROUTE_PATH, + EVE_INFO_ROUTE_PATH, +} from "../../src/protocol/routes.js"; +import type { AgentInfoResponse } from "../../src/internal/nitro/routes/agent-info/build-agent-info-response.js"; + +/** + * Handle to a spawned `eve dev --no-ui` process serving a scenario app. + */ +export interface RunningEveDev { + crash(): Promise; + readonly stderr: () => string; + readonly stdout: () => string; + readonly url: string; + stop(): Promise; +} + +export interface StartEveDevOptions { + readonly env?: Readonly>; + /** Runtime executing the CLI. Defaults to the current Node executable. */ + readonly runtime?: "bun" | "node"; +} + +/** + * Spawns `eve dev --no-ui` for the app and resolves once the server URL is + * printed and the state record exists. `NODE_ENV=test` activates the + * deterministic mock-model adapter so streamed turns complete without model + * credentials. + */ +export async function startEveDev( + appRoot: string, + options: StartEveDevOptions = {}, +): Promise { + const child = spawnEveDev(appRoot, options); + let stderr = ""; + let stdout = ""; + + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk: string) => { + stderr += chunk; + }); + + const url = await waitForServerUrl({ + child, + getOutput: () => ({ stderr, stdout }), + }); + await waitForPath(join(appRoot, ".eve", "dev-server-state.v1.json")); + + return { + async crash() { + if (child.exitCode !== null || child.signalCode !== null) { + return; + } + await withinDeadline( + new Promise((resolve) => { + child.once("exit", () => resolve()); + child.kill("SIGKILL"); + }), + "Timed out waiting for the dev server process to crash.", + ); + }, + stderr: () => stderr, + stdout: () => stdout, + async stop() { + if (child.exitCode !== null || child.signalCode !== null) { + return; + } + + await new Promise((resolve) => { + const timeout = setTimeout(() => { + child.kill("SIGKILL"); + resolve(); + }, 10_000); + + child.once("exit", () => { + clearTimeout(timeout); + resolve(); + }); + child.kill("SIGTERM"); + }); + }, + url, + }; +} + +/** + * Spawns `eve dev` and resolves with the process exit for flows that expect + * startup to fail (e.g. unsupported runtimes). + */ +export async function runEveDevToExit( + appRoot: string, + options: StartEveDevOptions = {}, +): Promise<{ readonly code: number | null; readonly output: string }> { + const child = spawnEveDev(appRoot, options); + let output = ""; + + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + output += chunk; + }); + child.stderr.on("data", (chunk: string) => { + output += chunk; + }); + + try { + return await withinDeadline( + new Promise<{ readonly code: number | null; readonly output: string }>((resolve, reject) => { + child.once("error", reject); + child.once("exit", (code) => resolve({ code, output })); + }), + `Timed out waiting for eve dev to exit.\n\noutput:\n${output}`, + 120_000, + ); + } finally { + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGKILL"); + } + } +} + +function spawnEveDev( + appRoot: string, + options: StartEveDevOptions, +): ChildProcessByStdio { + const eveBinPath = join(appRoot, "node_modules", "eve", "bin", "eve.js"); + const command = options.runtime === "bun" ? "bun" : process.execPath; + + return spawn(command, [eveBinPath, "dev", "--no-ui", "--host", "127.0.0.1", "--port", "0"], { + cwd: appRoot, + env: { + ...process.env, + ...options.env, + NODE_ENV: "test", + }, + stdio: ["ignore", "pipe", "pipe"], + }); +} + +/** Reports whether `bun` is runnable on this machine. */ +export function isBunAvailable(): boolean { + try { + return spawnSync("bun", ["--version"], { stdio: "ignore" }).status === 0; + } catch { + return false; + } +} + +export function hasUnsupportedWindowsEsmImport(text: string): boolean { + return ( + text.includes("ERR_UNSUPPORTED_ESM_URL_SCHEME") || + text.includes("Received protocol 'g:'") || + text.includes('Received protocol "g:"') + ); +} + +/** + * Matches output signatures that mean the dev server broke: reset sockets, + * unresolved generated imports, or a pruned module-map loader. + */ +export function hasKnownDevServerFailure(text: string): boolean { + return ( + hasUnsupportedWindowsEsmImport(text) || + text.includes("UNRESOLVED_IMPORT") || + text.includes("ECONNRESET") || + text.includes("socket hang up") || + (text.includes("ERR_MODULE_NOT_FOUND") && text.includes("authored-module-map-loader")) + ); +} + +export async function wait(ms: number): Promise { + await new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + +export async function waitForCondition( + condition: () => boolean | Promise, + failureMessage: string | (() => string), + timeoutMs: number = 60_000, +): Promise { + const deadline = Date.now() + timeoutMs; + + while (!(await condition())) { + if (Date.now() >= deadline) { + throw new Error(typeof failureMessage === "function" ? failureMessage() : failureMessage); + } + await wait(100); + } +} + +export async function withinDeadline( + operation: Promise, + failureMessage: string, + timeoutMs: number = 60_000, +): Promise { + let timeout: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + operation, + new Promise((_resolve, reject) => { + timeout = setTimeout(() => reject(new Error(failureMessage)), timeoutMs); + }), + ]); + } finally { + if (timeout !== undefined) { + clearTimeout(timeout); + } + } +} + +export async function waitForPath(path: string, timeoutMs: number = 60_000): Promise { + if (existsSync(path)) { + return; + } + await new Promise((resolve, reject) => { + let settled = false; + const watcher = watchFileSystem(dirname(path), () => { + if (existsSync(path)) { + settle(resolve); + } + }); + const timeout = setTimeout(() => { + settle(() => reject(new Error(`Timed out waiting for ${path}.`))); + }, timeoutMs); + function settle(complete: () => void) { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + watcher.close(); + complete(); + } + if (existsSync(path)) { + settle(resolve); + } + }); +} + +export async function fetchText(serverUrl: string, path: string): Promise { + const response = await fetch(new URL(path, serverUrl)); + if (!response.ok) { + throw new Error(`Expected ${path} to succeed, received ${String(response.status)}.`); + } + return await response.text(); +} + +export async function fetchAgentInfo(serverUrl: string): Promise { + const response = await fetch(new URL(EVE_INFO_ROUTE_PATH, serverUrl)); + if (!response.ok) { + throw new Error(`Expected agent info to succeed, received ${String(response.status)}.`); + } + return (await response.json()) as AgentInfoResponse; +} + +export async function forceDevelopmentRebuild(serverUrl: string): Promise { + const rebuildUrl = new URL(EVE_DEV_RUNTIME_ARTIFACTS_REBUILD_ROUTE_PATH, serverUrl); + rebuildUrl.searchParams.set("force", "1"); + const response = await fetch(rebuildUrl); + if (!response.ok) { + throw new Error(`Development rebuild failed with status ${String(response.status)}.`); + } +} + +export async function readDevelopmentRevision(serverUrl: string): Promise { + const response = await fetch(new URL(EVE_DEV_RUNTIME_ARTIFACTS_ROUTE_PATH, serverUrl)); + if (!response.ok) { + throw new Error(`Development runtime state failed with status ${String(response.status)}.`); + } + const body = (await response.json()) as { readonly revision?: unknown }; + if (typeof body.revision !== "string") { + throw new Error("Development runtime state did not include a revision."); + } + return body.revision; +} + +function stripAnsi(text: string): string { + return text + .split("\u001b[") + .map((segment, index) => { + if (index === 0) { + return segment; + } + + return segment.replace(/^[0-9;]*m/, ""); + }) + .join(""); +} + +function parseServerUrl(stdout: string): string | undefined { + const match = /server listening at (https?:\/\/\S+)/.exec(stripAnsi(stdout)); + + return match?.[1]; +} + +async function waitForServerUrl(input: { + readonly child: ChildProcessByStdio; + readonly getOutput: () => { + readonly stderr: string; + readonly stdout: string; + }; +}): Promise { + return await new Promise((resolve, reject) => { + let settled = false; + + const timeout = setTimeout(() => { + settleReject( + new Error( + [ + "Timed out waiting for eve dev to print its server URL.", + `stdout:\n${input.getOutput().stdout}`, + `stderr:\n${input.getOutput().stderr}`, + ].join("\n\n"), + ), + ); + }, 120_000); + + const cleanup = () => { + clearTimeout(timeout); + input.child.stdout.off("data", handleOutput); + input.child.stderr.off("data", handleOutput); + input.child.off("error", settleReject); + input.child.off("exit", handleExit); + }; + + const settleResolve = (url: string) => { + if (settled) { + return; + } + + settled = true; + cleanup(); + resolve(url); + }; + + function settleReject(error: unknown) { + if (settled) { + return; + } + + settled = true; + cleanup(); + reject(error); + } + + function handleOutput() { + const output = input.getOutput(); + const combinedOutput = `${output.stdout}\n${output.stderr}`; + + if (hasKnownDevServerFailure(combinedOutput)) { + settleReject( + new Error( + [ + "eve dev emitted a known reload or generated-bundle failure.", + `stdout:\n${output.stdout}`, + `stderr:\n${output.stderr}`, + ].join("\n\n"), + ), + ); + return; + } + + const url = parseServerUrl(output.stdout); + + if (url !== undefined) { + settleResolve(url); + } + } + + function handleExit(code: number | null, signal: NodeJS.Signals | null) { + const output = input.getOutput(); + + settleReject( + new Error( + [ + `eve dev exited before printing its server URL (code ${String(code)}, signal ${String(signal)}).`, + `stdout:\n${output.stdout}`, + `stderr:\n${output.stderr}`, + ].join("\n\n"), + ), + ); + } + + input.child.stdout.on("data", handleOutput); + input.child.stderr.on("data", handleOutput); + input.child.once("error", settleReject); + input.child.once("exit", handleExit); + handleOutput(); + }); +} + +/** + * Issues one GET through the provided keep-alive agent and reports the local + * port so callers can assert connection reuse across worker replacements. + */ +export async function requestWithAgent( + url: string, + agent: Agent, +): Promise<{ readonly body: string; readonly localPort: number | undefined }> { + return await new Promise((resolve, reject) => { + const target = new URL(url); + const request = requestHttp( + { + agent, + host: target.hostname, + path: `${target.pathname}${target.search}`, + port: Number(target.port), + }, + (response) => { + const chunks: Buffer[] = []; + const localPort = response.socket.localPort; + response.on("data", (chunk: Buffer) => chunks.push(chunk)); + response.on("end", () => { + resolve({ body: Buffer.concat(chunks).toString("utf8"), localPort }); + }); + }, + ); + request.once("error", reject); + request.end(); + }); +} diff --git a/packages/eve/test/scenarios/dev-server.scenario.test.ts b/packages/eve/test/scenarios/dev-server.scenario.test.ts index be149c9fe..028ae00fc 100644 --- a/packages/eve/test/scenarios/dev-server.scenario.test.ts +++ b/packages/eve/test/scenarios/dev-server.scenario.test.ts @@ -1,12 +1,18 @@ import { spawn, type ChildProcessByStdio } from "node:child_process"; -import { existsSync } from "node:fs"; +import { existsSync, watch as watchFileSystem } from "node:fs"; import { rm, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import type { Readable } from "node:stream"; import { describe, expect, it } from "vitest"; -import { EVE_HEALTH_ROUTE_PATH } from "../../src/protocol/routes.js"; +import { + EVE_DEV_RUNTIME_ARTIFACTS_REBUILD_ROUTE_PATH, + EVE_DEV_RUNTIME_ARTIFACTS_ROUTE_PATH, + EVE_HEALTH_ROUTE_PATH, + EVE_INFO_ROUTE_PATH, +} from "../../src/protocol/routes.js"; +import type { AgentInfoResponse } from "../../src/internal/nitro/routes/agent-info/build-agent-info-response.js"; import { readDevelopmentRuntimeArtifactsSnapshotRoot, resolveDevelopmentRuntimeArtifactsPointerPath, @@ -28,12 +34,170 @@ const scenarioApp = useScenarioApp(); const DEV_SERVER_SCENARIO_TIMEOUT_MS = 360_000; const DEV_SERVER_AGENT_DESCRIPTOR: ScenarioAppDescriptor = { ...WEATHER_AGENT_DESCRIPTOR, - files: Object.fromEntries( - Object.entries(WEATHER_AGENT_DESCRIPTOR.files).filter( - ([path]) => !path.startsWith("agent/channels/"), + files: { + ...Object.fromEntries( + Object.entries(WEATHER_AGENT_DESCRIPTOR.files).filter( + ([path]) => !path.startsWith("agent/channels/"), + ), ), - ), + "agent/channels/dev-generation.ts": [ + 'import { defineChannel, GET } from "eve/channels";', + "", + "export default defineChannel({", + ' routes: [GET("/dev-generation", () => new Response(process.env.EVE_SCENARIO_RELOAD ?? process.env.EVE_WEBSOCKET_RELOAD ?? "initial"))],', + "});", + "", + ].join("\n"), + }, +}; +const WEBSOCKET_DEV_SERVER_DESCRIPTOR: ScenarioAppDescriptor = { + ...DEV_SERVER_AGENT_DESCRIPTOR, + files: { + ...DEV_SERVER_AGENT_DESCRIPTOR.files, + "agent/channels/socket.ts": [ + 'import { defineChannel, WS } from "eve/channels";', + "", + "export default defineChannel({", + ' routes: [WS("/socket", () => ({', + " message(peer, message) {", + ' const transportHeader = peer.request.headers.has("x-eve-dev-worker-metadata") ? "exposed" : "hidden";', + ' peer.send(`${transportHeader}:${peer.remoteAddress ?? "missing"}:${message.text()}`);', + " },", + " }))],", + "});", + "", + ].join("\n"), + }, +}; +const TRANSACTIONAL_REBUILD_DESCRIPTOR: ScenarioAppDescriptor = { + ...DEV_SERVER_AGENT_DESCRIPTOR, + files: { + ...DEV_SERVER_AGENT_DESCRIPTOR.files, + "agent/channels/dev-generation.ts": createOverlappingChannelSource(), + "agent/instrumentation.ts": createInstrumentationSource("one"), + }, }; +const STREAM_PROMOTION_DESCRIPTOR: ScenarioAppDescriptor = { + ...TRANSACTIONAL_REBUILD_DESCRIPTOR, + files: { + ...TRANSACTIONAL_REBUILD_DESCRIPTOR.files, + "agent/channels/dev-generation.ts": [ + 'import { existsSync, watch } from "node:fs";', + 'import { join } from "node:path";', + 'import { defineChannel, GET } from "eve/channels";', + "", + "export default defineChannel({", + " routes: [", + ' GET("/instrumentation-marker", () => new Response(String(globalThis.__EVE_INSTRUMENTATION_MARKER__ ?? "missing"))),', + ' GET("/held-stream", () => {', + ' const releasePath = join(process.cwd(), ".stream-release");', + " let releaseWatcher: ReturnType | undefined;", + " let finished = false;", + " return new Response(new ReadableStream({", + " start(controller) {", + ' controller.enqueue(new TextEncoder().encode("first\\n"));', + " const finish = () => {", + " if (finished) return;", + " finished = true;", + " releaseWatcher?.close();", + ' controller.enqueue(new TextEncoder().encode("second\\n"));', + " controller.close();", + " };", + " if (existsSync(releasePath)) {", + " finish();", + " return;", + " }", + " releaseWatcher = watch(process.cwd(), (_event, filename) => {", + ' if (filename === ".stream-release") finish();', + " });", + " if (existsSync(releasePath)) finish();", + " },", + " cancel() { releaseWatcher?.close(); },", + " }));", + " }),", + " ],", + "});", + "", + ].join("\n"), + }, +}; + +function createInstrumentationSource(marker: string): string { + return [ + 'import { randomUUID } from "node:crypto";', + "", + "declare global {", + " var __EVE_INSTRUMENTATION_MARKER__: string | undefined;", + " var __EVE_WORKER_ID__: string | undefined;", + "}", + "", + `globalThis.__EVE_INSTRUMENTATION_MARKER__ = ${JSON.stringify(marker)};`, + "globalThis.__EVE_WORKER_ID__ = randomUUID();", + "export default {};", + "", + ].join("\n"); +} + +function createCandidateChannelSource(): string { + return createTransactionalChannelSource([ + ' GET("/candidate-only", () => new Response("candidate")),', + ]); +} + +function createOverlappingChannelSource(): string { + return createTransactionalChannelSource([ + ' GET("/overlap/:slug", (_request, context) => new Response(`parameter:${context.params.slug}`)),', + ' GET("/overlap/static", () => new Response("static")),', + ]); +} + +function createTransactionalChannelSource(routeLines: readonly string[]): string { + return [ + 'import { defineChannel, GET } from "eve/channels";', + "", + "export default defineChannel({", + " routes: [", + ' GET("/instrumentation-marker", () => new Response(String(globalThis.__EVE_INSTRUMENTATION_MARKER__ ?? "missing"))),', + ' GET("/worker-id", () => new Response(String(globalThis.__EVE_WORKER_ID__ ?? "missing"))),', + ...routeLines, + " ],", + "});", + "", + ].join("\n"); +} + +function createBlockedInstrumentationSource(): string { + return [ + 'import { existsSync, watch, writeFileSync } from "node:fs";', + 'import { join } from "node:path";', + "", + "declare global {", + " var __EVE_INSTRUMENTATION_MARKER__: string | undefined;", + "}", + "", + 'const startedPath = join(process.cwd(), ".candidate-started");', + 'const readyPath = join(process.cwd(), ".candidate-ready");', + 'writeFileSync(startedPath, "ready");', + "await new Promise((resolve) => {", + " if (existsSync(readyPath)) {", + " resolve();", + " return;", + " }", + " const readyWatcher = watch(process.cwd(), (_event, filename) => {", + ' if (filename !== ".candidate-ready") return;', + " readyWatcher.close();", + " resolve();", + " });", + " if (existsSync(readyPath)) {", + " readyWatcher.close();", + " resolve();", + " }", + "});", + 'globalThis.__EVE_INSTRUMENTATION_MARKER__ = "two";', + "export default {};", + "", + ].join("\n"); +} interface RunningEveDev { readonly stderr: () => string; @@ -63,10 +227,12 @@ function hasUnsupportedWindowsEsmImport(text: string): boolean { ); } -function hasKnownDevBundlingFailure(text: string): boolean { +function hasKnownDevServerFailure(text: string): boolean { return ( hasUnsupportedWindowsEsmImport(text) || text.includes("UNRESOLVED_IMPORT") || + text.includes("ECONNRESET") || + text.includes("socket hang up") || (text.includes("ERR_MODULE_NOT_FOUND") && text.includes("authored-module-map-loader")) ); } @@ -84,13 +250,13 @@ async function wait(ms: number): Promise { } async function waitForCondition( - condition: () => boolean, + condition: () => boolean | Promise, failureMessage: string, timeoutMs: number = 60_000, ): Promise { const deadline = Date.now() + timeoutMs; - while (!condition()) { + while (!(await condition())) { if (Date.now() >= deadline) { throw new Error(failureMessage); } @@ -152,11 +318,11 @@ async function waitForServerUrl(input: { const output = input.getOutput(); const combinedOutput = `${output.stdout}\n${output.stderr}`; - if (hasKnownDevBundlingFailure(combinedOutput)) { + if (hasKnownDevServerFailure(combinedOutput)) { settleReject( new Error( [ - "eve dev emitted a known generated dev bundle import failure.", + "eve dev emitted a known reload or generated-bundle failure.", `stdout:\n${output.stdout}`, `stderr:\n${output.stderr}`, ].join("\n\n"), @@ -194,6 +360,46 @@ async function waitForServerUrl(input: { }); } +async function waitForWebSocketOpen(socket: WebSocket): Promise { + await waitForWebSocketEvent(socket, "open", () => undefined); +} + +async function waitForWebSocketMessage(socket: WebSocket): Promise { + return await waitForWebSocketEvent(socket, "message", (event) => String(event.data)); +} + +async function waitForWebSocketClose(socket: WebSocket): Promise { + await waitForWebSocketEvent(socket, "close", () => undefined); +} + +async function waitForWebSocketEvent( + socket: WebSocket, + eventName: "close" | "message" | "open", + select: (event: MessageEvent) => T, +): Promise { + return await new Promise((resolve, reject) => { + const deadline = setTimeout(() => { + cleanup(); + reject(new Error(`Timed out waiting for WebSocket ${eventName}.`)); + }, 10_000); + const cleanup = () => { + clearTimeout(deadline); + socket.removeEventListener(eventName, onEvent as EventListener); + socket.removeEventListener("error", onError); + }; + const onEvent = (event: Event) => { + cleanup(); + resolve(select(event as MessageEvent)); + }; + const onError = () => { + cleanup(); + reject(new Error(`WebSocket failed while waiting for ${eventName}.`)); + }; + socket.addEventListener(eventName, onEvent as EventListener, { once: true }); + socket.addEventListener("error", onError, { once: true }); + }); +} + async function startEveDev(appRoot: string): Promise { const eveBinPath = join(appRoot, "node_modules", "eve", "bin", "eve.js"); const child = spawn( @@ -229,6 +435,7 @@ async function startEveDev(appRoot: string): Promise { stdout, }), }); + await waitForPath(join(appRoot, ".eve", "dev-server-state.v1.json")); return { stderr: () => stderr, @@ -256,6 +463,220 @@ async function startEveDev(appRoot: string): Promise { } describe("eve dev server", () => { + it( + "publishes authored tool removals without replacing the active host", + async () => { + const app = await scenarioApp(TRANSACTIONAL_REBUILD_DESCRIPTOR); + const server = await startEveDev(app.appRoot); + + try { + const initialRevision = await readDevelopmentRevision(server.url); + const initialWorkerId = await fetchText(server.url, "/worker-id"); + expect( + (await fetchAgentInfo(server.url)).tools.authored.map((tool) => tool.name), + ).toContain("get_weather"); + + await rm(join(app.appRoot, "agent", "tools", "get_weather.ts")); + await forceDevelopmentRebuild(server.url); + + await expect(readDevelopmentRevision(server.url)).resolves.not.toBe(initialRevision); + await expect(fetchText(server.url, "/worker-id")).resolves.toBe(initialWorkerId); + expect( + (await fetchAgentInfo(server.url)).tools.authored.map((tool) => tool.name), + ).not.toContain("get_weather"); + expect(hasKnownDevServerFailure(`${server.stdout()}\n${server.stderr()}`)).toBe(false); + } finally { + await server.stop(); + } + }, + DEV_SERVER_SCENARIO_TIMEOUT_MS, + ); + + it( + "replaces the worker for instrumentation changes and preserves Nitro's selected route", + async () => { + const app = await scenarioApp(TRANSACTIONAL_REBUILD_DESCRIPTOR); + const server = await startEveDev(app.appRoot); + + try { + await expect(fetchText(server.url, "/instrumentation-marker")).resolves.toBe("one"); + await expect(fetchText(server.url, "/overlap/static")).resolves.toBe("static"); + const initialWorkerId = await fetchText(server.url, "/worker-id"); + + await forceDevelopmentRebuild(server.url); + await expect(fetchText(server.url, "/worker-id")).resolves.toBe(initialWorkerId); + + await writeFile( + join(app.appRoot, "agent", "instrumentation.ts"), + createInstrumentationSource("two"), + ); + await forceDevelopmentRebuild(server.url); + + await expect(fetchText(server.url, "/instrumentation-marker")).resolves.toBe("two"); + await expect(fetchText(server.url, "/worker-id")).resolves.not.toBe(initialWorkerId); + await expect(fetchText(server.url, "/overlap/static")).resolves.toBe("static"); + expect(hasKnownDevServerFailure(`${server.stdout()}\n${server.stderr()}`)).toBe(false); + } finally { + await server.stop(); + } + }, + DEV_SERVER_SCENARIO_TIMEOUT_MS, + ); + + it( + "keeps the complete prior generation active when a structural candidate fails", + async () => { + const app = await scenarioApp(TRANSACTIONAL_REBUILD_DESCRIPTOR); + const server = await startEveDev(app.appRoot); + + try { + const initialRevision = await readDevelopmentRevision(server.url); + await writeFile( + join(app.appRoot, "agent", "channels", "dev-generation.ts"), + ['import "./missing-candidate-module.ts";', createCandidateChannelSource()].join("\n"), + ); + await forceDevelopmentRebuild(server.url); + await expect(readDevelopmentRevision(server.url)).resolves.toBe(initialRevision); + await expect(fetchText(server.url, "/instrumentation-marker")).resolves.toBe("one"); + + await writeFile( + join(app.appRoot, "agent", "instrumentation.ts"), + 'throw new Error("stage 4 rejected candidate");\nexport default {};\n', + ); + await writeFile( + join(app.appRoot, "agent", "channels", "dev-generation.ts"), + createCandidateChannelSource(), + ); + try { + await forceDevelopmentRebuild(server.url); + } catch (error) { + throw new Error( + `Rejected candidate rebuild request failed.\n\nstdout:\n${server.stdout()}\n\nstderr:\n${server.stderr()}`, + { cause: error }, + ); + } + + await expect(readDevelopmentRevision(server.url)).resolves.toBe(initialRevision); + await expect(fetchText(server.url, "/instrumentation-marker")).resolves.toBe("one"); + const candidateRoute = await fetch(new URL("/candidate-only", server.url)); + expect(candidateRoute.status).toBe(404); + + await writeFile( + join(app.appRoot, "agent", "instrumentation.ts"), + createInstrumentationSource("two"), + ); + await forceDevelopmentRebuild(server.url); + await expect(readDevelopmentRevision(server.url)).resolves.not.toBe(initialRevision); + await expect(fetchText(server.url, "/instrumentation-marker")).resolves.toBe("two"); + await expect(fetchText(server.url, "/candidate-only")).resolves.toBe("candidate"); + + await writeFile( + join(app.appRoot, "agent", "channels", "dev-generation.ts"), + createOverlappingChannelSource(), + ); + await writeFile( + join(app.appRoot, "agent", "instrumentation.ts"), + createInstrumentationSource("one"), + ); + await forceDevelopmentRebuild(server.url); + await expect(fetchText(server.url, "/instrumentation-marker")).resolves.toBe("one"); + await expect(fetchText(server.url, "/overlap/static")).resolves.toBe("static"); + const revertedCandidateRoute = await fetch(new URL("/candidate-only", server.url)); + expect(revertedCandidateRoute.status).toBe(404); + } finally { + await server.stop(); + } + }, + DEV_SERVER_SCENARIO_TIMEOUT_MS, + ); + + it( + "keeps streams and parent control routes alive while a structural candidate is prepared", + async () => { + const app = await scenarioApp(STREAM_PROMOTION_DESCRIPTOR); + const server = await startEveDev(app.appRoot); + const candidateStartedPath = join(app.appRoot, ".candidate-started"); + const candidateReadyPath = join(app.appRoot, ".candidate-ready"); + const streamReleasePath = join(app.appRoot, ".stream-release"); + + try { + const initialRevision = await readDevelopmentRevision(server.url); + const streamResponse = await fetch(new URL("/held-stream", server.url)); + if (streamResponse.body === null) { + throw new Error("Held stream response did not include a body."); + } + const reader = streamResponse.body.getReader(); + const firstChunk = await withinDeadline( + reader.read(), + "Timed out waiting for the first stream chunk.", + ); + expect(new TextDecoder().decode(firstChunk.value)).toBe("first\n"); + + await writeFile( + join(app.appRoot, "agent", "instrumentation.ts"), + createBlockedInstrumentationSource(), + ); + const rebuild = forceDevelopmentRebuild(server.url); + await waitForPath(candidateStartedPath); + + await expect(readDevelopmentRevision(server.url)).resolves.toBe(initialRevision); + await expect(fetchText(server.url, "/instrumentation-marker")).resolves.toBe("one"); + + await writeFile(candidateReadyPath, "ready"); + await withinDeadline(rebuild, "Timed out waiting for candidate promotion."); + await expect(fetchText(server.url, "/instrumentation-marker")).resolves.toBe("two"); + + await writeFile(streamReleasePath, "release"); + const secondChunk = await withinDeadline( + reader.read(), + "Timed out waiting for the retired worker stream.", + ); + expect(new TextDecoder().decode(secondChunk.value)).toBe("second\n"); + await expect(reader.read()).resolves.toEqual(expect.objectContaining({ done: true })); + expect(hasKnownDevServerFailure(`${server.stdout()}\n${server.stderr()}`)).toBe(false); + } finally { + await server.stop(); + } + }, + DEV_SERVER_SCENARIO_TIMEOUT_MS, + ); + + it( + "keeps an admitted websocket on its worker while a ready replacement is promoted", + async () => { + const app = await scenarioApp(WEBSOCKET_DEV_SERVER_DESCRIPTOR); + const server = await startEveDev(app.appRoot); + const socketUrl = new URL("/socket", server.url); + socketUrl.protocol = "ws:"; + const socket = new WebSocket(socketUrl); + + try { + await waitForWebSocketOpen(socket); + const firstMessage = waitForWebSocketMessage(socket); + socket.send("before"); + await expect(firstMessage).resolves.toBe("hidden:127.0.0.1:before"); + + await writeFile(join(app.appRoot, ".env.local"), "EVE_WEBSOCKET_RELOAD=1\n"); + await waitForCondition(async () => { + const generation = await fetch(new URL("/dev-generation", server.url)); + return (await generation.text()) === "1"; + }, `Timed out waiting for websocket worker replacement.\n\nstdout:\n${server.stdout()}\n\nstderr:\n${server.stderr()}`); + + const nextMessage = waitForWebSocketMessage(socket); + socket.send("after"); + await expect(nextMessage).resolves.toBe("hidden:127.0.0.1:after"); + } finally { + if (socket.readyState === WebSocket.OPEN) { + const closed = waitForWebSocketClose(socket); + socket.close(); + await closed; + } + await server.stop(); + } + }, + DEV_SERVER_SCENARIO_TIMEOUT_MS, + ); + it( "rebuilds after its startup runtime generation is force-pruned and completes a streamed turn", async () => { @@ -308,12 +729,11 @@ describe("eve dev server", () => { () => server.stdout().includes(STRUCTURAL_RELOAD_LOG_LINE), `Timed out waiting for a structural Nitro reload.\n\nstdout:\n${server.stdout()}\n\nstderr:\n${server.stderr()}`, ); - await waitForCondition(() => { - const currentRuntimeRoot = readDevelopmentRuntimeArtifactsSnapshotRoot(pointerPath); - return currentRuntimeRoot !== undefined && currentRuntimeRoot !== authoredRuntimeRoot; - }, `Timed out waiting for the structural reload snapshot.\n\nstdout:\n${server.stdout()}\n\nstderr:\n${server.stderr()}`); - // Nitro exposes no replacement-ready signal while it owns the public listener. - await wait(2_000); + expect(readDevelopmentRuntimeArtifactsSnapshotRoot(pointerPath)).toBe(authoredRuntimeRoot); + await waitForCondition(async () => { + const generation = await fetch(new URL("/dev-generation", server.url)); + return (await generation.text()) === "1"; + }, `Timed out waiting for a ready replacement worker.\n\nstdout:\n${server.stdout()}\n\nstderr:\n${server.stderr()}`); let messageResult: Awaited>; try { messageResult = await sendDevelopmentMessage({ @@ -342,7 +762,7 @@ describe("eve dev server", () => { ].join("\n\n"), ).toBe(true); const output = `${server.stdout()}\n${server.stderr()}`; - expect(hasKnownDevBundlingFailure(output)).toBe(false); + expect(hasKnownDevServerFailure(output)).toBe(false); } finally { await server.stop(); } @@ -350,3 +770,85 @@ describe("eve dev server", () => { DEV_SERVER_SCENARIO_TIMEOUT_MS, ); }); + +async function fetchText(serverUrl: string, path: string): Promise { + const response = await fetch(new URL(path, serverUrl)); + if (!response.ok) { + throw new Error(`Expected ${path} to succeed, received ${String(response.status)}.`); + } + return await response.text(); +} + +async function fetchAgentInfo(serverUrl: string): Promise { + const response = await fetch(new URL(EVE_INFO_ROUTE_PATH, serverUrl)); + if (!response.ok) { + throw new Error(`Expected agent info to succeed, received ${String(response.status)}.`); + } + return (await response.json()) as AgentInfoResponse; +} + +async function forceDevelopmentRebuild(serverUrl: string): Promise { + const rebuildUrl = new URL(EVE_DEV_RUNTIME_ARTIFACTS_REBUILD_ROUTE_PATH, serverUrl); + rebuildUrl.searchParams.set("force", "1"); + const response = await fetch(rebuildUrl); + if (!response.ok) { + throw new Error(`Development rebuild failed with status ${String(response.status)}.`); + } +} + +async function readDevelopmentRevision(serverUrl: string): Promise { + const response = await fetch(new URL(EVE_DEV_RUNTIME_ARTIFACTS_ROUTE_PATH, serverUrl)); + if (!response.ok) { + throw new Error(`Development runtime state failed with status ${String(response.status)}.`); + } + const body = (await response.json()) as { readonly revision?: unknown }; + if (typeof body.revision !== "string") { + throw new Error("Development runtime state did not include a revision."); + } + return body.revision; +} + +async function waitForPath(path: string): Promise { + if (existsSync(path)) { + return; + } + await new Promise((resolve, reject) => { + let settled = false; + const watcher = watchFileSystem(dirname(path), () => { + if (existsSync(path)) { + settle(resolve); + } + }); + const timeout = setTimeout(() => { + settle(() => reject(new Error(`Timed out waiting for ${path}.`))); + }, 60_000); + function settle(complete: () => void) { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + watcher.close(); + complete(); + } + if (existsSync(path)) { + settle(resolve); + } + }); +} + +async function withinDeadline(operation: Promise, failureMessage: string): Promise { + let timeout: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + operation, + new Promise((_resolve, reject) => { + timeout = setTimeout(() => reject(new Error(failureMessage)), 60_000); + }), + ]); + } finally { + if (timeout !== undefined) { + clearTimeout(timeout); + } + } +} diff --git a/packages/eve/test/scenarios/production-build-isolation.scenario.test.ts b/packages/eve/test/scenarios/production-build-isolation.scenario.test.ts index 4587148c3..726ca1149 100644 --- a/packages/eve/test/scenarios/production-build-isolation.scenario.test.ts +++ b/packages/eve/test/scenarios/production-build-isolation.scenario.test.ts @@ -1,18 +1,17 @@ import { spawn, type ChildProcessByStdio } from "node:child_process"; import { createHash } from "node:crypto"; -import { watch, type FSWatcher } from "node:fs"; +import { watch } from "node:fs"; import { lstat, mkdir, readFile, readdir, readlink, - realpath, rename, utimes, writeFile, } from "node:fs/promises"; -import { join, relative } from "node:path"; +import { join } from "node:path"; import type { Readable } from "node:stream"; import { pathToFileURL } from "node:url"; @@ -320,50 +319,32 @@ async function observeConcurrentBuildWorkspaces(input: { }); } -async function watchStableDevBuildSurfaces(appRoot: string): Promise<{ +async function watchActiveDevHostWorkspace(appRoot: string): Promise<{ readonly events: readonly string[]; + readonly initialHash: string; + readonly name: string; + readonly path: string; close(): void; }> { - const installedEveRoot = await realpath(join(appRoot, "node_modules", "eve")); - const candidates = [ - join(appRoot, ".eve", "compile"), - join(appRoot, ".eve", "host"), - join(appRoot, ".eve", "nitro"), - join(installedEveRoot, ".eve", "workflow-cache"), - ]; - const paths: string[] = []; - const missingPaths: string[] = []; - for (const path of candidates) { - try { - if ((await lstat(path)).isDirectory()) { - paths.push(path); - } - } catch (error) { - if (error instanceof Error && "code" in error && error.code === "ENOENT") { - missingPaths.push(path); - } else { - throw error; - } - } - } - if (missingPaths.length > 0) { - throw new Error( - `Expected eve dev to prepare every stable build surface before readiness:\n${missingPaths.join("\n")}`, - ); - } - const events: string[] = []; - const watchers: FSWatcher[] = paths.map((path) => - watch(path, { recursive: true }, (eventType, filename) => { - events.push(`${relative(appRoot, path)}:${eventType}:${filename?.toString() ?? ""}`); - }), + const devHostsRoot = join(appRoot, ".eve", "dev-hosts"); + const workspaces = (await readdir(devHostsRoot, { withFileTypes: true })).filter((entry) => + entry.isDirectory(), ); + expect(workspaces.map((entry) => entry.name)).toHaveLength(1); + const name = workspaces[0]!.name; + const path = join(devHostsRoot, name); + const events: string[] = []; + const watcher = watch(path, { recursive: true }, (eventType, filename) => { + events.push(`${eventType}:${filename?.toString() ?? ""}`); + }); return { events, + initialHash: await hashPath(path), + name, + path, close() { - for (const watcher of watchers) { - watcher.close(); - } + watcher.close(); }, }; } @@ -453,11 +434,11 @@ describe("production build isolation", () => { const processes: RunningProcess[] = []; let continueProbing = false; let healthProbe: Promise | undefined; - let stableSurfaceWatcher: Awaited> | undefined; + let devHostWatcher: Awaited> | undefined; try { await expectHealthy(server); - stableSurfaceWatcher = await watchStableDevBuildSurfaces(app.appRoot); + devHostWatcher = await watchActiveDevHostWorkspace(app.appRoot); const builds = [ startEveProcess({ appRoot: app.appRoot, args: ["build"] }), startEveProcess({ appRoot: app.appRoot, args: ["build"] }), @@ -488,19 +469,25 @@ describe("production build isolation", () => { ]); continueProbing = false; const healthChecks = await healthProbe; - stableSurfaceWatcher.close(); + devHostWatcher.close(); expect(workspaces).toHaveLength(2); expect(results.map((result) => result.code)).toEqual([0, 0]); expect(results.map((result) => result.signal)).toEqual([null, null]); expect(healthChecks).toBeGreaterThan(0); - expect(stableSurfaceWatcher.events).toEqual([]); + expect(devHostWatcher.events).toEqual([]); + expect(await hashPath(devHostWatcher.path)).toBe(devHostWatcher.initialHash); + expect( + (await readdir(join(app.appRoot, ".eve", "dev-hosts"), { withFileTypes: true })) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name), + ).toEqual([devHostWatcher.name]); expect(await listBuildWorkspaces(app.appRoot)).toEqual([]); await expectHealthy(server); } finally { continueProbing = false; await healthProbe?.catch(() => undefined); - stableSurfaceWatcher?.close(); + devHostWatcher?.close(); await Promise.all(processes.map((processHandle) => processHandle.stop())); await server.stop(); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 298710f9a..48d7a8760 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1180,6 +1180,9 @@ importers: emulate: specifier: 0.6.0 version: 0.6.0 + env-runner: + specifier: 0.1.12 + version: 0.1.12 eventsource-parser: specifier: 3.0.8 version: 3.0.8 @@ -8396,14 +8399,6 @@ packages: srvx: optional: true - crossws@0.4.5: - resolution: {integrity: sha512-wUR89x/Rw7/8t+vn0CmGDYM9TD6VtARGb0LD5jq2wjtMy1vCP4M+sm6N6TigWeTYvnA8MoW29NqqXD0ep0rfBA==} - peerDependencies: - srvx: '>=0.11.5' - peerDependenciesMeta: - srvx: - optional: true - crossws@0.4.6: resolution: {integrity: sha512-/Wxe9Z007EbJ496j88nToZEvyPZ8PY/wjZJ18Agh/GCA9cYHyLbxtrpdFlFzAw3TV20F0SUYGl0g6PzChbwUrg==} peerDependencies: @@ -9867,9 +9862,6 @@ packages: resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} engines: {node: '>= 14'} - httpxy@0.5.3: - resolution: {integrity: sha512-SMS9V6Sn7VWaS11lYhoAr0ceoaiolTWf4jYdJn0NJhCdKMu9R2H9Fh0LBDWBHQF6HRLI1PmaePYsjanSpE5PEw==} - httpxy@0.5.5: resolution: {integrity: sha512-uDjmnPyp1q4Sgzf3w+J/Fc6UqcCEj0x4Wjp7OqK5dGhNeDgpyrAmnS6ey8QWrX3SWDon2DMKf9sBa5X9+CVyMA==} @@ -20635,7 +20627,7 @@ snapshots: confbox: 0.2.4 defu: 6.1.7 dotenv: 17.4.2 - exsolve: 1.0.8 + exsolve: 1.1.0 giget: 3.2.0 jiti: 2.7.0 ohash: 2.0.11 @@ -20929,10 +20921,6 @@ snapshots: optionalDependencies: srvx: 0.11.22 - crossws@0.4.5(srvx@0.11.16): - optionalDependencies: - srvx: 0.11.16 - crossws@0.4.6(srvx@0.11.16): optionalDependencies: srvx: 0.11.16 @@ -21455,10 +21443,10 @@ snapshots: env-runner@0.1.12: dependencies: - crossws: 0.4.5(srvx@0.11.16) - exsolve: 1.0.8 - httpxy: 0.5.3 - srvx: 0.11.16 + crossws: 0.4.10(srvx@0.11.22) + exsolve: 1.1.0 + httpxy: 0.5.5 + srvx: 0.11.22 error-stack-parser-es@1.0.5: {} @@ -22665,7 +22653,7 @@ snapshots: h3@2.0.1-rc.22(crossws@0.4.6(srvx@0.11.16)): dependencies: rou3: 0.8.1 - srvx: 0.11.16 + srvx: 0.11.22 optionalDependencies: crossws: 0.4.6(srvx@0.11.16) @@ -22905,8 +22893,6 @@ snapshots: transitivePeerDependencies: - supports-color - httpxy@0.5.3: {} - httpxy@0.5.5: {} human-id@4.1.3: {} From 3f332ff58595dc292c887e053200196db054e41c Mon Sep 17 00:00:00 2001 From: Casey Gowrie Date: Tue, 14 Jul 2026 15:58:18 -0400 Subject: [PATCH 2/4] fix(dev): close review gaps in drain and rebuild transactionality - A crash restart now yields to a replacement that was already in flight instead of swapping retired code back in over the newer worker, and the crashed workspace is disposed when the restart loses or the server is closing. - The client abort signal rides the proxied worker request, so a client that disconnects before response headers arrive releases its exchange instead of pinning the worker until the handler returns. - close() terminates a candidate still waiting on readiness and awaits the replace chain, bounding shutdown during a structural rebuild. - A worker response that fails after headers were sent destroys the client connection instead of framing the truncated body as a complete response. - A candidate that fails readiness disposes its workspace instead of leaking it. - Pointer publication retries transient Windows rename failures (EPERM/EBUSY/EACCES) before surfacing an activation failure. Co-Authored-By: Claude Fable 5 Signed-off-by: Casey Gowrie --- .../internal/nitro/dev-runtime-artifacts.ts | 34 ++++- .../eve/src/internal/nitro/host/dev-runner.ts | 2 +- .../internal/nitro/host/dev-server-http.ts | 11 +- .../drained-nitro-dev-server.scenario.test.ts | 142 +++++++++++++++++- .../nitro/host/drained-nitro-dev-server.ts | 72 ++++++--- 5 files changed, 237 insertions(+), 24 deletions(-) diff --git a/packages/eve/src/internal/nitro/dev-runtime-artifacts.ts b/packages/eve/src/internal/nitro/dev-runtime-artifacts.ts index 04ed31638..b3062f993 100644 --- a/packages/eve/src/internal/nitro/dev-runtime-artifacts.ts +++ b/packages/eve/src/internal/nitro/dev-runtime-artifacts.ts @@ -549,6 +549,9 @@ async function writeDevelopmentRuntimeArtifactsPointer(input: { ); } +const POINTER_RENAME_ATTEMPTS = 5; +const POINTER_RENAME_RETRY_DELAY_MS = 50; + async function writeDevelopmentRuntimeArtifactsPointerSource( pointerPath: string, source: string, @@ -557,13 +560,42 @@ async function writeDevelopmentRuntimeArtifactsPointerSource( await mkdir(dirname(pointerPath), { recursive: true }); await writeFile(temporaryPointerPath, source); try { - await rename(temporaryPointerPath, pointerPath); + await renameWithWindowsRetry(temporaryPointerPath, pointerPath); } catch (error) { await rm(temporaryPointerPath, { force: true }).catch(() => {}); throw error; } } +/** + * Windows rejects `rename` with EPERM/EBUSY while another handle (a reader + * of the pointer, an antivirus scan) briefly holds the destination. Those + * failures are transient, so pointer publication retries before surfacing + * the activation failure. + */ +async function renameWithWindowsRetry(source: string, destination: string): Promise { + for (let attempt = 1; ; attempt += 1) { + try { + await rename(source, destination); + return; + } catch (error) { + const code = + error instanceof Error && "code" in error + ? (error as NodeJS.ErrnoException).code + : undefined; + if ( + attempt >= POINTER_RENAME_ATTEMPTS || + (code !== "EPERM" && code !== "EBUSY" && code !== "EACCES") + ) { + throw error; + } + await new Promise((resolvePromise) => { + setTimeout(resolvePromise, POINTER_RENAME_RETRY_DELAY_MS * attempt); + }); + } + } +} + async function readOptionalFile(path: string): Promise { try { return await readFile(path, "utf8"); diff --git a/packages/eve/src/internal/nitro/host/dev-runner.ts b/packages/eve/src/internal/nitro/host/dev-runner.ts index d5dc819b1..a30fd169d 100644 --- a/packages/eve/src/internal/nitro/host/dev-runner.ts +++ b/packages/eve/src/internal/nitro/host/dev-runner.ts @@ -7,7 +7,7 @@ import { resolvePackageCompiledFilePath } from "#internal/application/package.js export interface DevelopmentRunner { readonly closed: boolean; close(cause?: unknown): Promise; - fetch(request: Request): Promise; + fetch(request: Request, init?: RequestInit): Promise; onceClosed(listener: (cause?: unknown) => void): void; upgrade(input: { readonly node: { diff --git a/packages/eve/src/internal/nitro/host/dev-server-http.ts b/packages/eve/src/internal/nitro/host/dev-server-http.ts index 8f0527c19..ff16c30d9 100644 --- a/packages/eve/src/internal/nitro/host/dev-server-http.ts +++ b/packages/eve/src/internal/nitro/host/dev-server-http.ts @@ -84,10 +84,15 @@ export async function writeResponse( } export function writeRequestError(response: ServerResponse, error: unknown): void { - if (!response.headersSent) { - response.statusCode = 503; - response.setHeader("content-type", "application/json; charset=utf-8"); + if (response.headersSent) { + // The status line is already on the wire; ending now would frame the + // truncated body as a complete response. Destroying the socket is the + // only way to surface the failure to the client mid-stream. + response.destroy(); + return; } + response.statusCode = 503; + response.setHeader("content-type", "application/json; charset=utf-8"); if (!response.writableEnded && !response.destroyed) { response.end(JSON.stringify({ error: toErrorMessage(error) })); } diff --git a/packages/eve/src/internal/nitro/host/drained-nitro-dev-server.scenario.test.ts b/packages/eve/src/internal/nitro/host/drained-nitro-dev-server.scenario.test.ts index 5be6b1b87..817ad69ea 100644 --- a/packages/eve/src/internal/nitro/host/drained-nitro-dev-server.scenario.test.ts +++ b/packages/eve/src/internal/nitro/host/drained-nitro-dev-server.scenario.test.ts @@ -20,7 +20,7 @@ interface TestRunner extends DevelopmentRunner { } function createRunnerFactory( - fetchHandler: (request: Request, runnerIndex: number) => Promise, + fetchHandler: (request: Request, runnerIndex: number, init?: RequestInit) => Promise, readiness: (runnerIndex: number) => Promise = async () => undefined, ): { readonly createRunner: DevelopmentRunnerFactory; readonly runners: TestRunner[] } { const runners: TestRunner[] = []; @@ -49,7 +49,7 @@ function createRunnerFactory( closed = true; notifyClosed(cause); }, - fetch: async (request) => await fetchHandler(request, runnerIndex), + fetch: async (request, init) => await fetchHandler(request, runnerIndex, init), onceClosed(listener) { if (closed) { listener(); @@ -219,6 +219,144 @@ describe("drained Nitro dev server", () => { expect(runners[0]?.closeMock).toHaveBeenCalledOnce(); }); + it("lets an in-flight replacement win over a crash restart", async () => { + let releaseCandidate: (() => void) | undefined; + const { createRunner, runners } = createRunnerFactory( + async (_request, runnerIndex) => new Response(`runner-${String(runnerIndex)}`), + async (runnerIndex) => { + if (runnerIndex === 1) { + await new Promise((resolve) => { + releaseCandidate = resolve; + }); + } + }, + ); + const server = new DrainedNitroDevServer(LOGGER, createRunner); + const listener = await listen(server); + const disposeFirst = vi.fn(async () => undefined); + await server.replaceWorker(replacement("/tmp/first.mjs", disposeFirst)); + + const second = server.replaceWorker(replacement("/tmp/second.mjs")); + await withinDeadline( + vi.waitFor(() => { + expect(releaseCandidate).toBeDefined(); + }), + "Timed out waiting for the candidate to start readiness.", + ); + runners[0]?.crash(new Error("worker exploded")); + releaseCandidate?.(); + await second; + + // The queued crash restart must yield to the replacement that was + // already in flight instead of swapping retired code back in. + await expect( + fetch(new URL("/", listener.url)).then(async (response) => await response.text()), + ).resolves.toBe("runner-1"); + await vi.waitFor(() => { + expect(disposeFirst).toHaveBeenCalledOnce(); + }); + expect(runners).toHaveLength(2); + + await server.close(); + }); + + it("releases the exchange when a client aborts before response headers", async () => { + let sawRequest: (() => void) | undefined; + const requestSeen = new Promise((resolve) => { + sawRequest = resolve; + }); + const { createRunner, runners } = createRunnerFactory(async (_request, runnerIndex, init) => { + if (runnerIndex === 0) { + sawRequest?.(); + return await new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => reject(new Error("aborted")), { + once: true, + }); + }); + } + return new Response("runner-1"); + }); + const server = new DrainedNitroDevServer(LOGGER, createRunner); + const listener = await listen(server); + const disposeFirst = vi.fn(async () => undefined); + await server.replaceWorker(replacement("/tmp/first.mjs", disposeFirst)); + + const abort = new AbortController(); + const hung = fetch(new URL("/", listener.url), { signal: abort.signal }).catch(() => undefined); + await withinDeadline(requestSeen, "Timed out waiting for the hung request to be admitted."); + abort.abort(); + await hung; + + await server.replaceWorker(replacement("/tmp/second.mjs")); + await withinDeadline( + vi.waitFor(() => { + expect(runners[0]?.closeMock).toHaveBeenCalled(); + expect(disposeFirst).toHaveBeenCalledOnce(); + }), + "Timed out waiting for the aborted exchange to release the retired worker.", + ); + + await server.close(); + }); + + it("closes within a bounded interval while a candidate is mid-readiness", async () => { + const { createRunner } = createRunnerFactory( + async () => new Response("ok"), + async (runnerIndex) => { + if (runnerIndex === 1) { + await new Promise(() => undefined); + } + }, + ); + const server = new DrainedNitroDevServer(LOGGER, createRunner); + await listen(server); + await server.replaceWorker(replacement("/tmp/first.mjs")); + + const pending = server.replaceWorker(replacement("/tmp/second.mjs")); + pending.catch(() => undefined); + await withinDeadline(server.close(), "Timed out closing during a pending replacement."); + await expect(pending).rejects.toThrow(); + }); + + it("terminates the client connection when a worker response fails mid-stream", async () => { + let failStream: (() => void) | undefined; + const { createRunner } = createRunnerFactory(async () => { + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("started\n")); + failStream = () => controller.error(new Error("worker stream died")); + }, + }); + return new Response(body); + }); + const server = new DrainedNitroDevServer(LOGGER, createRunner); + const listener = await listen(server); + await server.replaceWorker(replacement("/tmp/first.mjs")); + + const response = await fetch(new URL("/", listener.url)); + const reader = response.body?.getReader(); + await reader?.read(); + failStream?.(); + + // A truncated stream must surface as a client-visible failure, never as + // a cleanly terminated response. + await expect( + withinDeadline( + (async () => { + for (;;) { + const result = await reader?.read(); + if (result === undefined || result.done) { + return; + } + } + })(), + "Timed out waiting for the failed stream to settle.", + ), + ).rejects.toThrow(); + + await server.close(); + }); + it("answers control requests without admitting them to the worker", async () => { const fetchMock = vi.fn(async () => new Response("worker")); const { createRunner } = createRunnerFactory(fetchMock); diff --git a/packages/eve/src/internal/nitro/host/drained-nitro-dev-server.ts b/packages/eve/src/internal/nitro/host/drained-nitro-dev-server.ts index 1fc4f40f3..49ad20dbc 100644 --- a/packages/eve/src/internal/nitro/host/drained-nitro-dev-server.ts +++ b/packages/eve/src/internal/nitro/host/drained-nitro-dev-server.ts @@ -64,6 +64,7 @@ export class DrainedNitroDevServer { #closePromise: Promise | undefined; #closed = false; #controlHandler: ((request: Request) => Promise) | undefined; + #pendingSlot: RunnerSlot | undefined; #replaceChain: Promise = Promise.resolve(); #runnerCounter = 0; @@ -113,11 +114,16 @@ export class DrainedNitroDevServer { workerData: replacement.workerData, }; + this.#pendingSlot = slot; try { await slot.runner.waitForReady(RUNNER_READY_TIMEOUT_MS); } catch (error) { await slot.runner.close(error).catch(() => undefined); + // The candidate never served; its workspace has no further use. + await this.#disposeSlot(slot); throw error; + } finally { + this.#pendingSlot = undefined; } if (this.#closed) { @@ -231,6 +237,13 @@ export class DrainedNitroDevServer { this.#closed = true; this.#wakeActiveWaiters(); + // Terminate a candidate still waiting on readiness so the replace chain + // settles within a bounded interval instead of the readiness timeout. + await this.#pendingSlot?.runner + .close(new Error("Development server is closing.")) + .catch(() => undefined); + await this.#replaceChain.catch(() => undefined); + const listeners = [...this.#listeners]; const listenerClosePromises = listeners.map((listener) => listener.beginClose()); @@ -253,14 +266,19 @@ export class DrainedNitroDevServer { } slot.released = true; await slot.runner.close().catch(() => undefined); - if (slot.dispose !== undefined && !slot.disposed) { - slot.disposed = true; - await slot.dispose().catch((error) => { - this.#logger.error( - `[eve:dev] failed to dispose a retired dev host: ${toErrorMessage(error)}`, - ); - }); + await this.#disposeSlot(slot); + } + + async #disposeSlot(slot: RunnerSlot): Promise { + if (slot.dispose === undefined || slot.disposed) { + return; } + slot.disposed = true; + await slot.dispose().catch((error) => { + this.#logger.error( + `[eve:dev] failed to dispose a retired dev host: ${toErrorMessage(error)}`, + ); + }); } async #handleRunnerClose(slot: RunnerSlot): Promise { @@ -269,16 +287,30 @@ export class DrainedNitroDevServer { } this.#active = undefined; this.#logger.error("[eve:dev] dev worker exited; restarting."); - // The crashed worker's workspace must survive for the restart, so the - // replacement inherits its dispose and the crashed slot releases nothing. - slot.disposed = true; - slot.released = true; - try { - await this.replaceWorker({ - dispose: slot.dispose, - entry: slot.entry, - workerData: slot.workerData, + const next = this.#replaceChain + .catch(() => undefined) + .then(async () => { + // A replacement that was already in flight when the worker crashed + // wins; restarting the crashed entry now would swap retired code back + // in over the newer worker. + if (this.#closed || this.#active !== undefined) { + await this.#releaseSlot(slot); + return; + } + // The crashed worker's workspace must survive for the restart, so the + // replacement inherits its dispose and the crashed slot releases + // nothing itself. + slot.disposed = true; + slot.released = true; + await this.#replaceWorker({ + dispose: slot.dispose, + entry: slot.entry, + workerData: slot.workerData, + }); }); + this.#replaceChain = next.catch(() => undefined); + try { + await next; } catch (error) { this.#logger.error(`[eve:dev] dev worker restart failed: ${toErrorMessage(error)}`); } @@ -343,7 +375,13 @@ export class DrainedNitroDevServer { throw new Error("Development worker is unavailable."); } settle = this.#beginExchange(slot); - const workerResponse = await slot.runner.fetch(publicRequest); + // The abort signal must ride the proxied request: without it a client + // that disconnects before response headers arrive leaves the worker + // handler running and the exchange pinned, so a retired worker holding + // one would never drain. + const workerResponse = await slot.runner.fetch(publicRequest, { + signal: requestAbort.signal, + }); await writeResponse(response, workerResponse, requestAbort.signal); } catch (error) { if (!requestAbort.signal.aborted) { From 1a2b81d266e29b376be069593765be5fb665b26f Mon Sep 17 00:00:00 2001 From: Casey Gowrie Date: Tue, 14 Jul 2026 19:43:59 -0400 Subject: [PATCH 3/4] fix(eve): retry busy publication lock renames Signed-off-by: Casey Gowrie --- .changeset/tidy-locks-retry.md | 5 ++ .../application/output-publication-lock.ts | 13 ++-- .../internal/nitro/dev-runtime-artifacts.ts | 37 +--------- packages/eve/src/shared/atomic-write-file.ts | 29 ++------ .../eve/src/shared/rename-with-retry.test.ts | 71 +++++++++++++++++++ packages/eve/src/shared/rename-with-retry.ts | 31 ++++++++ 6 files changed, 121 insertions(+), 65 deletions(-) create mode 100644 .changeset/tidy-locks-retry.md create mode 100644 packages/eve/src/shared/rename-with-retry.test.ts create mode 100644 packages/eve/src/shared/rename-with-retry.ts diff --git a/.changeset/tidy-locks-retry.md b/.changeset/tidy-locks-retry.md new file mode 100644 index 000000000..2060f5458 --- /dev/null +++ b/.changeset/tidy-locks-retry.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Retry transient Windows filesystem contention while atomically releasing build publication locks. diff --git a/packages/eve/src/internal/application/output-publication-lock.ts b/packages/eve/src/internal/application/output-publication-lock.ts index f703266f4..c73fcf7df 100644 --- a/packages/eve/src/internal/application/output-publication-lock.ts +++ b/packages/eve/src/internal/application/output-publication-lock.ts @@ -1,5 +1,5 @@ import { watch } from "node:fs"; -import { mkdir, readdir, rename, rm, stat, utimes } from "node:fs/promises"; +import { mkdir, readdir, rm, stat, utimes } from "node:fs/promises"; import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path"; import { @@ -17,6 +17,7 @@ import { } from "#internal/application/output-publication-journal.js"; import { isErrnoCode } from "#shared/guards.js"; import { pathExists } from "#shared/path-exists.js"; +import { renameWithTransientBusyRetry } from "#shared/rename-with-retry.js"; const PUBLICATION_LOCK_TIMEOUT_MS = 60_000; const INCOMPLETE_LOCK_STALE_MS = 5_000; @@ -89,7 +90,7 @@ export async function acquireOutputPublicationLock( } const releasedPath = `${lockPath}.released-${journal.token}`; try { - await rename(lockPath, releasedPath); + await renameWithTransientBusyRetry(lockPath, releasedPath); } catch (error) { if (isErrnoCode(error, "ENOENT")) { return; @@ -141,7 +142,7 @@ async function recoverStalePublication( if (await pathExists(lockPath)) { const recoveringJournalPath = join(recoveryPath, `owner-${recoveryJournal.token}`); try { - await rename(lockPath, recoveringJournalPath); + await renameWithTransientBusyRetry(lockPath, recoveringJournalPath); preserveRecovery = true; } catch (error) { if (!isErrnoCode(error, "ENOENT")) { @@ -197,7 +198,7 @@ async function acquireRecoveryLease( } const releasedPath = `${recoveryPath}.released-${token}`; try { - await rename(recoveryPath, releasedPath); + await renameWithTransientBusyRetry(recoveryPath, releasedPath); } catch (error) { if (isErrnoCode(error, "ENOENT")) { return; @@ -213,7 +214,7 @@ async function acquireRecoveryLease( } const releasedPath = `${leasePath}.released-${token}`; try { - await rename(leasePath, releasedPath); + await renameWithTransientBusyRetry(leasePath, releasedPath); } catch (error) { if (isErrnoCode(error, "ENOENT")) { return; @@ -248,7 +249,7 @@ async function acquireRecoveryLease( const staleLeasePath = `${leasePath}.stale-${token}`; try { - await rename(leasePath, staleLeasePath); + await renameWithTransientBusyRetry(leasePath, staleLeasePath); } catch (error) { if (isErrnoCode(error, "ENOENT")) { continue; diff --git a/packages/eve/src/internal/nitro/dev-runtime-artifacts.ts b/packages/eve/src/internal/nitro/dev-runtime-artifacts.ts index b3062f993..4c3348c69 100644 --- a/packages/eve/src/internal/nitro/dev-runtime-artifacts.ts +++ b/packages/eve/src/internal/nitro/dev-runtime-artifacts.ts @@ -1,11 +1,12 @@ import { randomUUID } from "node:crypto"; -import { cp, mkdir, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; +import { cp, mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; import { existsSync, readFileSync, type Dirent } from "node:fs"; import { dirname, join, relative, resolve, sep } from "node:path"; import type { CompileAgentResult } from "#compiler/compile-agent.js"; import { copyDevelopmentSourceSnapshot } from "#internal/nitro/dev-runtime-source-snapshot-copy.js"; import { createDevelopmentSourceSnapshotPlan } from "#internal/nitro/dev-runtime-source-snapshot.js"; +import { renameWithTransientBusyRetry } from "#shared/rename-with-retry.js"; const DEV_RUNTIME_ARTIFACTS_DIRECTORY = "dev-runtime"; const DEV_RUNTIME_ARTIFACTS_ACTIVATED_MARKER = "activated"; @@ -549,9 +550,6 @@ async function writeDevelopmentRuntimeArtifactsPointer(input: { ); } -const POINTER_RENAME_ATTEMPTS = 5; -const POINTER_RENAME_RETRY_DELAY_MS = 50; - async function writeDevelopmentRuntimeArtifactsPointerSource( pointerPath: string, source: string, @@ -560,42 +558,13 @@ async function writeDevelopmentRuntimeArtifactsPointerSource( await mkdir(dirname(pointerPath), { recursive: true }); await writeFile(temporaryPointerPath, source); try { - await renameWithWindowsRetry(temporaryPointerPath, pointerPath); + await renameWithTransientBusyRetry(temporaryPointerPath, pointerPath); } catch (error) { await rm(temporaryPointerPath, { force: true }).catch(() => {}); throw error; } } -/** - * Windows rejects `rename` with EPERM/EBUSY while another handle (a reader - * of the pointer, an antivirus scan) briefly holds the destination. Those - * failures are transient, so pointer publication retries before surfacing - * the activation failure. - */ -async function renameWithWindowsRetry(source: string, destination: string): Promise { - for (let attempt = 1; ; attempt += 1) { - try { - await rename(source, destination); - return; - } catch (error) { - const code = - error instanceof Error && "code" in error - ? (error as NodeJS.ErrnoException).code - : undefined; - if ( - attempt >= POINTER_RENAME_ATTEMPTS || - (code !== "EPERM" && code !== "EBUSY" && code !== "EACCES") - ) { - throw error; - } - await new Promise((resolvePromise) => { - setTimeout(resolvePromise, POINTER_RENAME_RETRY_DELAY_MS * attempt); - }); - } - } -} - async function readOptionalFile(path: string): Promise { try { return await readFile(path, "utf8"); diff --git a/packages/eve/src/shared/atomic-write-file.ts b/packages/eve/src/shared/atomic-write-file.ts index 87b8f0844..45c95f5f6 100644 --- a/packages/eve/src/shared/atomic-write-file.ts +++ b/packages/eve/src/shared/atomic-write-file.ts @@ -1,6 +1,6 @@ -import { rename, rm, writeFile } from "node:fs/promises"; +import { rm, writeFile } from "node:fs/promises"; -const REPLACE_RETRY_DELAYS_MS = [10, 20, 40, 80, 160, 320, 640]; +import { renameWithTransientBusyRetry } from "#shared/rename-with-retry.js"; /** * Writes `contents` so concurrent readers always observe either the old or @@ -10,7 +10,7 @@ const REPLACE_RETRY_DELAYS_MS = [10, 20, 40, 80, 160, 320, 640]; * * Windows refuses to replace a file while another handle is open on it * (concurrent readers, `utimes` heartbeats, antivirus scans), surfacing as - * `EPERM`/`EACCES`, so the replace is retried briefly before giving up. + * `EPERM`/`EACCES`/`EBUSY`, so the replace is retried briefly before giving up. */ export async function atomicWriteFile( targetPath: string, @@ -19,30 +19,9 @@ export async function atomicWriteFile( const tmpPath = `${targetPath}.tmp-${process.pid}-${Date.now().toString(36)}`; await writeFile(tmpPath, contents); try { - await renameReplacingBusyTarget(tmpPath, targetPath); + await renameWithTransientBusyRetry(tmpPath, targetPath); } catch (error) { await rm(tmpPath, { force: true }).catch(() => undefined); throw error; } } - -async function renameReplacingBusyTarget(fromPath: string, toPath: string): Promise { - for (const delayMs of REPLACE_RETRY_DELAYS_MS) { - try { - await rename(fromPath, toPath); - return; - } catch (error) { - if (!isBusyTargetError(error)) { - throw error; - } - await new Promise((resolve) => setTimeout(resolve, delayMs)); - } - } - await rename(fromPath, toPath); -} - -function isBusyTargetError(error: unknown): boolean { - return ( - error instanceof Error && "code" in error && (error.code === "EPERM" || error.code === "EACCES") - ); -} diff --git a/packages/eve/src/shared/rename-with-retry.test.ts b/packages/eve/src/shared/rename-with-retry.test.ts new file mode 100644 index 000000000..eb05152f9 --- /dev/null +++ b/packages/eve/src/shared/rename-with-retry.test.ts @@ -0,0 +1,71 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { renameWithTransientBusyRetry } from "#shared/rename-with-retry.js"; + +const fake = vi.hoisted(() => ({ + failureCode: "EPERM", + failures: 0, + renameCalls: [] as Array<{ sourcePath: string; destinationPath: string }>, +})); + +vi.mock("node:fs/promises", () => ({ + rename: async (sourcePath: string, destinationPath: string) => { + fake.renameCalls.push({ sourcePath, destinationPath }); + if (fake.failures > 0) { + fake.failures -= 1; + throw Object.assign(new Error(`injected ${fake.failureCode} rename failure`), { + code: fake.failureCode, + }); + } + }, +})); + +beforeEach(() => { + vi.useFakeTimers(); +}); + +afterEach(() => { + vi.useRealTimers(); + fake.failureCode = "EPERM"; + fake.failures = 0; + fake.renameCalls.length = 0; +}); + +async function settle(promise: Promise): Promise> { + const settled = Promise.allSettled([promise]); + await vi.runAllTimersAsync(); + return (await settled)[0]; +} + +describe("renameWithTransientBusyRetry", () => { + it.each(["EPERM", "EACCES", "EBUSY"])("retries a transient %s failure", async (code) => { + fake.failureCode = code; + fake.failures = 2; + + const result = await settle(renameWithTransientBusyRetry("source", "destination")); + + expect(result.status).toBe("fulfilled"); + expect(fake.renameCalls).toHaveLength(3); + }); + + it("gives up after bounded retries", async () => { + fake.failures = Number.MAX_SAFE_INTEGER; + + const result = await settle(renameWithTransientBusyRetry("source", "destination")); + + expect(result.status).toBe("rejected"); + expect((result as PromiseRejectedResult).reason).toMatchObject({ code: "EPERM" }); + expect(fake.renameCalls).toHaveLength(8); + }); + + it("does not retry a permanent rename failure", async () => { + fake.failureCode = "ENOENT"; + fake.failures = 1; + + const result = await settle(renameWithTransientBusyRetry("source", "destination")); + + expect(result.status).toBe("rejected"); + expect((result as PromiseRejectedResult).reason).toMatchObject({ code: "ENOENT" }); + expect(fake.renameCalls).toHaveLength(1); + }); +}); diff --git a/packages/eve/src/shared/rename-with-retry.ts b/packages/eve/src/shared/rename-with-retry.ts new file mode 100644 index 000000000..a334948ff --- /dev/null +++ b/packages/eve/src/shared/rename-with-retry.ts @@ -0,0 +1,31 @@ +import { rename } from "node:fs/promises"; + +import { isErrnoCode } from "#shared/guards.js"; + +const TRANSIENT_RENAME_RETRY_DELAYS_MS = [10, 20, 40, 80, 160, 320, 640]; + +/** + * Renames a path atomically, retrying the transient busy-handle errors that + * Windows can report while another process briefly has either path open. + */ +export async function renameWithTransientBusyRetry( + sourcePath: string, + destinationPath: string, +): Promise { + for (const delayMs of TRANSIENT_RENAME_RETRY_DELAYS_MS) { + try { + await rename(sourcePath, destinationPath); + return; + } catch (error) { + if (!isTransientRenameError(error)) { + throw error; + } + await new Promise((resolvePromise) => setTimeout(resolvePromise, delayMs)); + } + } + await rename(sourcePath, destinationPath); +} + +function isTransientRenameError(error: unknown): boolean { + return isErrnoCode(error, "EPERM") || isErrnoCode(error, "EACCES") || isErrnoCode(error, "EBUSY"); +} From 29a57b08059651246c7064c0e49ac52ca94794b3 Mon Sep 17 00:00:00 2001 From: Casey Gowrie Date: Tue, 14 Jul 2026 20:02:06 -0400 Subject: [PATCH 4/4] fix(dev): pin schedule dispatch to the active generation Signed-off-by: Casey Gowrie --- .../dev-schedule-dispatch-generation.md | 5 ++ .../host/dispatch-schedule-in-dev.test.ts | 85 +++++++++++++++++++ .../nitro/host/dispatch-schedule-in-dev.ts | 16 ++-- .../internal/nitro/routes/schedule-task.ts | 16 +++- 4 files changed, 115 insertions(+), 7 deletions(-) create mode 100644 .changeset/dev-schedule-dispatch-generation.md create mode 100644 packages/eve/src/internal/nitro/host/dispatch-schedule-in-dev.test.ts diff --git a/.changeset/dev-schedule-dispatch-generation.md b/.changeset/dev-schedule-dispatch-generation.md new file mode 100644 index 000000000..e34dbfb3a --- /dev/null +++ b/.changeset/dev-schedule-dispatch-generation.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Fix the dev-only schedule dispatch route to load compiled artifacts from the active development generation instead of the authored app root, which returned a 500 for every dispatch. diff --git a/packages/eve/src/internal/nitro/host/dispatch-schedule-in-dev.test.ts b/packages/eve/src/internal/nitro/host/dispatch-schedule-in-dev.test.ts new file mode 100644 index 000000000..24eb91024 --- /dev/null +++ b/packages/eve/src/internal/nitro/host/dispatch-schedule-in-dev.test.ts @@ -0,0 +1,85 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { dispatchScheduleInDev } from "#internal/nitro/host/dispatch-schedule-in-dev.js"; + +const mocks = vi.hoisted(() => { + const activeGeneration = { + appRoot: "/app/.eve/dev-runtime/snapshots/active/runtime", + kind: "disk" as const, + moduleMapLoaderPath: "/app/node_modules/eve/module-map-loader.js", + sandboxAppRoot: "/app", + }; + return { + activeGeneration, + createScheduleRegistrations: vi.fn(() => [ + { + cron: "0 0 * * *", + description: "heartbeat", + logicalPath: "schedules/heartbeat.md", + scheduleId: "heartbeat", + sourceId: "schedule:heartbeat", + taskName: "eve.schedule.heartbeat", + }, + ]), + dispatchScheduleTask: vi.fn(), + dispatchScheduleTaskFromArtifacts: vi.fn(async () => ({ + scheduleId: "heartbeat", + sessionIds: ["session-1"], + })), + loadResolvedCompiledSchedules: vi.fn(async () => []), + resolveNitroCompiledArtifactsSource: vi.fn(() => activeGeneration), + }; +}); + +vi.mock("#internal/nitro/routes/runtime-artifacts.js", () => ({ + resolveNitroCompiledArtifactsSource: mocks.resolveNitroCompiledArtifactsSource, +})); + +vi.mock("#runtime/schedules/register.js", () => ({ + createScheduleRegistrations: mocks.createScheduleRegistrations, +})); + +vi.mock("#runtime/schedules/resolve-schedule.js", () => ({ + loadResolvedCompiledSchedules: mocks.loadResolvedCompiledSchedules, +})); + +vi.mock("#internal/nitro/routes/schedule-task.js", () => ({ + dispatchScheduleTask: mocks.dispatchScheduleTask, + dispatchScheduleTaskFromArtifacts: mocks.dispatchScheduleTaskFromArtifacts, +})); + +const ARTIFACTS_CONFIG = { + appRoot: "/app", + devRuntimeArtifactsPointerPath: "/app/.eve/dev-runtime/current.json", + kind: "development", + moduleMapLoaderPath: "/app/node_modules/eve/module-map-loader.js", +} as const; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("dispatchScheduleInDev", () => { + it("pins schedule lookup and task dispatch to one active generation", async () => { + await expect( + dispatchScheduleInDev({ + artifactsConfig: ARTIFACTS_CONFIG, + scheduleId: "heartbeat", + }), + ).resolves.toEqual({ + scheduleId: "heartbeat", + sessionIds: ["session-1"], + }); + + expect(mocks.resolveNitroCompiledArtifactsSource).toHaveBeenCalledOnce(); + expect(mocks.resolveNitroCompiledArtifactsSource).toHaveBeenCalledWith(ARTIFACTS_CONFIG); + expect(mocks.loadResolvedCompiledSchedules).toHaveBeenCalledWith({ + compiledArtifactsSource: mocks.activeGeneration, + }); + expect(mocks.dispatchScheduleTaskFromArtifacts).toHaveBeenCalledWith( + "eve.schedule.heartbeat", + mocks.activeGeneration, + ); + expect(mocks.dispatchScheduleTask).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/eve/src/internal/nitro/host/dispatch-schedule-in-dev.ts b/packages/eve/src/internal/nitro/host/dispatch-schedule-in-dev.ts index 09326e5fb..515e8e253 100644 --- a/packages/eve/src/internal/nitro/host/dispatch-schedule-in-dev.ts +++ b/packages/eve/src/internal/nitro/host/dispatch-schedule-in-dev.ts @@ -1,5 +1,5 @@ import type { DevelopmentNitroArtifactsConfig } from "#internal/nitro/routes/runtime-artifacts.js"; -import { createDiskRuntimeCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js"; +import { resolveNitroCompiledArtifactsSource } from "#internal/nitro/routes/runtime-artifacts.js"; import { createScheduleRegistrations } from "#runtime/schedules/register.js"; import { loadResolvedCompiledSchedules } from "#runtime/schedules/resolve-schedule.js"; @@ -67,9 +67,9 @@ export async function dispatchScheduleInDev(input: { ); } - const compiledArtifactsSource = createDiskRuntimeCompiledArtifactsSource(appRoot, { - moduleMapLoaderPath, - }); + // Resolve the active generation once and reuse it for the task dispatch so + // a rebuild between the two loads cannot split them across generations. + const compiledArtifactsSource = resolveNitroCompiledArtifactsSource(input.artifactsConfig); const schedules = await loadResolvedCompiledSchedules({ compiledArtifactsSource }); const registrations = createScheduleRegistrations(schedules); const registration = registrations.find((candidate) => candidate.scheduleId === input.scheduleId); @@ -81,8 +81,12 @@ export async function dispatchScheduleInDev(input: { ); } - const { dispatchScheduleTask } = await import("#internal/nitro/routes/schedule-task.js"); - const result = await dispatchScheduleTask(registration.taskName, input.artifactsConfig); + const { dispatchScheduleTaskFromArtifacts } = + await import("#internal/nitro/routes/schedule-task.js"); + const result = await dispatchScheduleTaskFromArtifacts( + registration.taskName, + compiledArtifactsSource, + ); return { scheduleId: result.scheduleId, diff --git a/packages/eve/src/internal/nitro/routes/schedule-task.ts b/packages/eve/src/internal/nitro/routes/schedule-task.ts index a0958d235..77bf5287c 100644 --- a/packages/eve/src/internal/nitro/routes/schedule-task.ts +++ b/packages/eve/src/internal/nitro/routes/schedule-task.ts @@ -5,6 +5,7 @@ import { loadResolvedCompiledScheduleByTaskName } from "#runtime/schedules/resol import { getCompiledRuntimeAgentBundle } from "#runtime/sessions/compiled-agent-cache.js"; import type { NitroArtifactsConfig } from "#internal/nitro/routes/runtime-artifacts.js"; import { resolveNitroCompiledArtifactsSource } from "#internal/nitro/routes/runtime-artifacts.js"; +import type { RuntimeCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js"; /** * Dispatches one eve authored schedule via the execution engine. @@ -17,7 +18,20 @@ export async function dispatchScheduleTask( taskName: string, config: NitroArtifactsConfig, ): Promise<{ scheduleId: string; sessionIds: readonly string[] }> { - const compiledArtifactsSource = resolveNitroCompiledArtifactsSource(config); + return await dispatchScheduleTaskFromArtifacts( + taskName, + resolveNitroCompiledArtifactsSource(config), + ); +} + +/** + * Variant of {@link dispatchScheduleTask} for callers that already resolved + * an artifact source and need the dispatch to stay on that generation. + */ +export async function dispatchScheduleTaskFromArtifacts( + taskName: string, + compiledArtifactsSource: RuntimeCompiledArtifactsSource, +): Promise<{ scheduleId: string; sessionIds: readonly string[] }> { const schedule = await loadResolvedCompiledScheduleByTaskName(taskName, { compiledArtifactsSource, });