From 10d84e735b91bc68c6053d26a15cc04ab5e8a9f6 Mon Sep 17 00:00:00 2001 From: Casey Gowrie Date: Mon, 13 Jul 2026 23:32:42 -0400 Subject: [PATCH 1/4] feat(dev): preserve active Workflow generations Signed-off-by: Casey Gowrie --- .changeset/steady-worlds-wait.md | 5 + .../vendor-compiled/@workflow/world-local.mjs | 3 + .../declarations/workflow-world-local.d.ts | 14 + .../eve/src/execution/create-session-step.ts | 9 +- packages/eve/src/execution/workflow-entry.ts | 4 +- .../eve/src/execution/workflow-runtime.ts | 12 +- .../eve/src/execution/workflow-steps.test.ts | 14 + .../application/compiled-artifacts.test.ts | 29 +- .../application/compiled-artifacts.ts | 34 +- .../internal/application/dev-environment.ts | 7 + .../application/optional-package-install.ts | 14 +- .../dev-runtime-artifacts.integration.test.ts | 145 ++-- .../internal/nitro/dev-runtime-artifacts.ts | 199 +++--- .../nitro/dev-runtime-worker-artifacts.ts | 98 +++ .../nitro/host/configure-nitro-routes.test.ts | 26 +- .../nitro/host/configure-nitro-routes.ts | 8 +- .../host/dev-authored-rebuild-coordinator.ts | 12 +- .../internal/nitro/host/dev-host-candidate.ts | 3 +- .../nitro/host/dev-worker-http-server.ts | 246 +++++++ .../internal/nitro/host/dev-worker-http.ts | 32 +- .../internal/nitro/host/dev-worker-runner.ts | 2 +- .../nitro/host/dev-worker-server-types.ts | 91 +++ .../host/dev-worker-server.scenario.test.ts | 51 +- .../internal/nitro/host/dev-worker-server.ts | 669 ++++++++++-------- .../nitro/host/development-worker-recovery.ts | 25 + .../host/development-workflow-dispatch.ts | 89 +++ .../host/development-workflow-response.ts | 35 + .../nitro-development-worker-server.test.ts | 35 +- .../host/nitro-development-worker-server.ts | 65 +- .../host/start-development-server.test.ts | 52 +- .../nitro/host/start-development-server.ts | 19 +- .../development-generation-context.ts | 25 + .../workflow/development-world-client.ts | 218 ++++++ .../workflow/development-world-codec.test.ts | 39 + .../workflow/development-world-codec.ts | 108 +++ .../workflow/development-world-protocol.ts | 32 + ...velopment-world-server.integration.test.ts | 246 +++++++ .../workflow/development-world-server.ts | 361 ++++++++++ .../durable-compiled-artifacts-source.test.ts | 40 ++ .../durable-compiled-artifacts-source.ts | 35 + .../runtime/sessions/runtime-context-keys.ts | 12 +- .../scenarios/dev-server.scenario.test.ts | 222 +++++- 42 files changed, 2802 insertions(+), 583 deletions(-) create mode 100644 .changeset/steady-worlds-wait.md create mode 100644 packages/eve/scripts/vendor-compiled/declarations/workflow-world-local.d.ts create mode 100644 packages/eve/src/internal/application/dev-environment.ts create mode 100644 packages/eve/src/internal/nitro/dev-runtime-worker-artifacts.ts create mode 100644 packages/eve/src/internal/nitro/host/dev-worker-http-server.ts create mode 100644 packages/eve/src/internal/nitro/host/dev-worker-server-types.ts create mode 100644 packages/eve/src/internal/nitro/host/development-worker-recovery.ts create mode 100644 packages/eve/src/internal/nitro/host/development-workflow-dispatch.ts create mode 100644 packages/eve/src/internal/nitro/host/development-workflow-response.ts create mode 100644 packages/eve/src/internal/workflow/development-generation-context.ts create mode 100644 packages/eve/src/internal/workflow/development-world-client.ts create mode 100644 packages/eve/src/internal/workflow/development-world-codec.test.ts create mode 100644 packages/eve/src/internal/workflow/development-world-codec.ts create mode 100644 packages/eve/src/internal/workflow/development-world-protocol.ts create mode 100644 packages/eve/src/internal/workflow/development-world-server.integration.test.ts create mode 100644 packages/eve/src/internal/workflow/development-world-server.ts create mode 100644 packages/eve/src/runtime/durable-compiled-artifacts-source.test.ts create mode 100644 packages/eve/src/runtime/durable-compiled-artifacts-source.ts diff --git a/.changeset/steady-worlds-wait.md b/.changeset/steady-worlds-wait.md new file mode 100644 index 000000000..0146ccbf6 --- /dev/null +++ b/.changeset/steady-worlds-wait.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Keep active local Workflow turns on the development generation they selected across reloads, retries, and server restarts. New turns use the latest successful generation, while pruning retains only generations still owned by active work. diff --git a/packages/eve/scripts/vendor-compiled/@workflow/world-local.mjs b/packages/eve/scripts/vendor-compiled/@workflow/world-local.mjs index dbd48b6b3..6dcd268c4 100644 --- a/packages/eve/scripts/vendor-compiled/@workflow/world-local.mjs +++ b/packages/eve/scripts/vendor-compiled/@workflow/world-local.mjs @@ -2,6 +2,8 @@ import { readFileSync } from "node:fs"; import { createRequire } from "node:module"; import { dirname, join } from "node:path"; +import { loadDeclaration } from "../_shared.mjs"; + const require = createRequire(import.meta.url); function resolveWorkflowWorldLocalVersion() { @@ -53,5 +55,6 @@ export default { packageName: "@workflow/world-local", compiledPath: "@workflow/world-local", chunkGroup: "workflow", + declaration: await loadDeclaration("workflow-world-local.d.ts"), plugins: [workflowWorldLocalVersionPlugin], }; diff --git a/packages/eve/scripts/vendor-compiled/declarations/workflow-world-local.d.ts b/packages/eve/scripts/vendor-compiled/declarations/workflow-world-local.d.ts new file mode 100644 index 000000000..6eba378ce --- /dev/null +++ b/packages/eve/scripts/vendor-compiled/declarations/workflow-world-local.d.ts @@ -0,0 +1,14 @@ +import type { QueuePrefix, World } from "#compiled/@workflow/world/index.js"; + +export type LocalWorld = World & { + clear(): Promise; + registerHandler(prefix: QueuePrefix, handler: (request: Request) => Promise): void; +}; + +export function createWorld(options?: { + readonly baseUrl?: string; + readonly dataDir?: string; + readonly recoverActiveRuns?: boolean; + readonly streamFlushIntervalMs?: number; + readonly tag?: string; +}): LocalWorld; diff --git a/packages/eve/src/execution/create-session-step.ts b/packages/eve/src/execution/create-session-step.ts index df10de45d..d39e5525c 100644 --- a/packages/eve/src/execution/create-session-step.ts +++ b/packages/eve/src/execution/create-session-step.ts @@ -1,5 +1,8 @@ -import type { RuntimeCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js"; import { getCompiledRuntimeAgentBundle } from "#runtime/sessions/compiled-agent-cache.js"; +import { + type DurableCompiledArtifactsSource, + resolveDurableCompiledArtifactsSource, +} from "#runtime/durable-compiled-artifacts-source.js"; import { createDurableSessionState, type DurableSessionState, @@ -26,7 +29,7 @@ export interface CreateSessionStepResult { * the root agent. */ export async function createSessionStep(input: { - readonly compiledArtifactsSource: RuntimeCompiledArtifactsSource; + readonly compiledArtifactsSource: DurableCompiledArtifactsSource; readonly continuationToken: string; readonly inheritedLimits?: RunSessionLimits; readonly outputSchema?: JsonObject; @@ -38,7 +41,7 @@ export async function createSessionStep(input: { "use step"; const bundle = await getCompiledRuntimeAgentBundle({ - compiledArtifactsSource: input.compiledArtifactsSource, + compiledArtifactsSource: resolveDurableCompiledArtifactsSource(input.compiledArtifactsSource), nodeId: input.nodeId, }); diff --git a/packages/eve/src/execution/workflow-entry.ts b/packages/eve/src/execution/workflow-entry.ts index 7b10b55e6..29e60be99 100644 --- a/packages/eve/src/execution/workflow-entry.ts +++ b/packages/eve/src/execution/workflow-entry.ts @@ -10,7 +10,7 @@ import type { import { coalesceDeliveries } from "#harness/messages.js"; import { readChannelRequestId, readRootSessionId } from "#execution/eve-workflow-attributes.js"; import type { RunMode } from "#shared/run-mode.js"; -import type { RuntimeCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js"; +import type { DurableCompiledArtifactsSource } from "#runtime/durable-compiled-artifacts-source.js"; import { notifyDelegatedParentStep } from "#execution/delegated-parent-notification.js"; import { createDelegatedSubagentErrorResult, @@ -74,7 +74,7 @@ export async function workflowEntry(input: WorkflowEntryInput): Promise { ); }); + it("starts turn workflows on the latest promoted generation in local development", async () => { + vi.stubEnv("EVE_DEV", "1"); + const input = createTurnInput(); + startMock.mockResolvedValue({ runId: "turn-run" }); + + await expect(dispatchTurnStep(input)).resolves.toEqual({ runId: "turn-run" }); + + expect(startMock).toHaveBeenCalledWith( + turnWorkflowReference, + [createTurnWorkflowInput(input)], + expect.objectContaining({ deploymentId: "latest" }), + ); + }); + it("pins turn workflows to the current deployment off production", async () => { vi.stubEnv("VERCEL_ENV", "preview"); const input = createTurnInput(); diff --git a/packages/eve/src/internal/application/compiled-artifacts.test.ts b/packages/eve/src/internal/application/compiled-artifacts.test.ts index 10628bfe2..0c8b9bc24 100644 --- a/packages/eve/src/internal/application/compiled-artifacts.test.ts +++ b/packages/eve/src/internal/application/compiled-artifacts.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from "vitest"; -import { createWorkflowWorldPluginSource } from "#internal/application/compiled-artifacts.js"; +import { + createDevelopmentWorkflowWorldPluginSource, + createWorkflowWorldPluginSource, +} from "#internal/application/compiled-artifacts.js"; describe("createWorkflowWorldPluginSource", () => { it("imports a configured world package and delegates its construction to Workflow", () => { @@ -41,3 +44,27 @@ describe("createWorkflowWorldPluginSource", () => { ).toContain("/compiled/@workflow/world-vercel/index.js"); }); }); + +describe("createDevelopmentWorkflowWorldPluginSource", () => { + it("installs the parent-backed World without starting a local World in the worker", () => { + const source = createDevelopmentWorkflowWorldPluginSource({ + compiledArtifactsBootstrapPath: "/app/.eve/host/bootstrap.mjs", + configuredWorld: undefined, + }); + + expect(source).toContain("createDevelopmentWorkflowWorld"); + expect(source).toContain("setWorld(createDevelopmentWorkflowWorld());"); + expect(source).not.toContain("@workflow/world-local"); + expect(source).not.toContain("workflowWorld.start"); + }); + + it("keeps explicitly configured remote Worlds inside the worker", () => { + const source = createDevelopmentWorkflowWorldPluginSource({ + compiledArtifactsBootstrapPath: "/app/.eve/host/bootstrap.mjs", + configuredWorld: "@acme/eve-world", + }); + + expect(source).toContain('import * as workflowWorldModule from "@acme/eve-world";'); + expect(source).toContain("await workflowWorld.start?.();"); + }); +}); diff --git a/packages/eve/src/internal/application/compiled-artifacts.ts b/packages/eve/src/internal/application/compiled-artifacts.ts index 9554ea909..96ea155f8 100644 --- a/packages/eve/src/internal/application/compiled-artifacts.ts +++ b/packages/eve/src/internal/application/compiled-artifacts.ts @@ -129,10 +129,9 @@ export async function writeDevelopmentCompiledArtifactsFiles(input: { ); await writeFile( workflowWorldPluginPath, - createWorkflowWorldPluginSource({ + createDevelopmentWorkflowWorldPluginSource({ compiledArtifactsBootstrapPath: bootstrapPath, configuredWorld: input.compileResult.manifest.config.experimental?.workflow?.world, - defaultWorld: "local", }), ); @@ -279,6 +278,37 @@ export function createWorkflowWorldPluginSource(input: { ].join("\n"); } +export function createDevelopmentWorkflowWorldPluginSource(input: { + compiledArtifactsBootstrapPath: string; + configuredWorld: AgentWorkflowWorldDefinition | undefined; +}): string { + const packageName = getWorldImport({ WORKFLOW_TARGET_WORLD: input.configuredWorld ?? "local" }); + if (packageName !== "@workflow/world-local") { + return createWorkflowWorldPluginSource({ + ...input, + defaultWorld: "local", + }); + } + const workflowRuntimeImportSpecifier = resolvePackageCompiledFilePath( + "src/compiled/@workflow/core/runtime.js", + ); + const developmentWorldImportSpecifier = resolvePackageSourceFilePath( + "src/internal/workflow/development-world-client.ts", + ); + return [ + "// Generated by eve. Do not edit by hand.", + `import ${stringifyEsmImportSpecifier(input.compiledArtifactsBootstrapPath)};`, + `import { getWorld, setWorld } from ${stringifyEsmImportSpecifier(workflowRuntimeImportSpecifier)};`, + `import { createDevelopmentWorkflowWorld } from ${stringifyEsmImportSpecifier(developmentWorldImportSpecifier)};`, + "", + "setWorld(createDevelopmentWorkflowWorld());", + "await getWorld();", + "", + "export default function installDevelopmentWorkflowWorldPlugin() {}", + "", + ].join("\n"); +} + function createInstrumentationPluginSource(input: { agentName: string; instrumentationPath: string; diff --git a/packages/eve/src/internal/application/dev-environment.ts b/packages/eve/src/internal/application/dev-environment.ts new file mode 100644 index 000000000..acbd09cfc --- /dev/null +++ b/packages/eve/src/internal/application/dev-environment.ts @@ -0,0 +1,7 @@ +/** Environment flag set for processes that belong to an `eve dev` session. */ +export const EVE_DEV_ENV_FLAG = "EVE_DEV"; + +/** Reports whether this process belongs to an `eve dev` session. */ +export function isEveDevEnvironment(): boolean { + return process.env[EVE_DEV_ENV_FLAG] === "1"; +} diff --git a/packages/eve/src/internal/application/optional-package-install.ts b/packages/eve/src/internal/application/optional-package-install.ts index b16467a66..1b347f5dc 100644 --- a/packages/eve/src/internal/application/optional-package-install.ts +++ b/packages/eve/src/internal/application/optional-package-install.ts @@ -5,19 +5,9 @@ import { dirname, join } from "node:path"; import { pathToFileURL } from "node:url"; import { Worker } from "node:worker_threads"; -/** - * Environment flag set by `eve dev` so runtime code can distinguish the - * interactive development server from production processes. Optional - * engine packages are auto-installed only when this is set. - */ -export const EVE_DEV_ENV_FLAG = "EVE_DEV"; +import { EVE_DEV_ENV_FLAG, isEveDevEnvironment } from "#internal/application/dev-environment.js"; -/** - * Reports whether this process belongs to an `eve dev` session. - */ -export function isEveDevEnvironment(): boolean { - return process.env[EVE_DEV_ENV_FLAG] === "1"; -} +export { EVE_DEV_ENV_FLAG, isEveDevEnvironment }; export type ProjectPackageManager = "bun" | "npm" | "pnpm" | "yarn"; diff --git a/packages/eve/src/internal/nitro/dev-runtime-artifacts.integration.test.ts b/packages/eve/src/internal/nitro/dev-runtime-artifacts.integration.test.ts index 89d3b2e98..9b7b23334 100644 --- a/packages/eve/src/internal/nitro/dev-runtime-artifacts.integration.test.ts +++ b/packages/eve/src/internal/nitro/dev-runtime-artifacts.integration.test.ts @@ -1,6 +1,6 @@ import { existsSync } from "node:fs"; import { lstat, mkdir, readdir, readFile, symlink, utimes, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { basename, join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -18,11 +18,16 @@ import { activateDevelopmentRuntimeArtifactsSnapshot, activateDevelopmentRuntimeArtifactsSnapshotTransaction, pruneDevelopmentRuntimeArtifactsSnapshots, + readActivatedDevelopmentRuntimeArtifactGenerations, readDevelopmentRuntimeArtifactsSnapshotRoot, readDevelopmentRuntimeArtifactsRevision, resolveDevelopmentRuntimeArtifactsPointerPath, stageDevelopmentRuntimeArtifactsSnapshot, } from "#internal/nitro/dev-runtime-artifacts.js"; +import { + copyDevelopmentRuntimeArtifactsWorker, + writeDevelopmentRuntimeArtifactsWorker, +} from "#internal/nitro/dev-runtime-worker-artifacts.js"; import { resolveNitroCompiledArtifactsSource } from "#internal/nitro/routes/runtime-artifacts.js"; const createScratchDirectory = useTemporaryDirectories(); @@ -156,6 +161,67 @@ describe("development runtime artifact snapshots", () => { }); }); + it("restores complete activated generations and ignores snapshots from older formats", async () => { + const appRoot = await createScratchDirectory("eve-dev-runtime-generation-registry-"); + const compileDirectoryPath = join(appRoot, ".eve", "compile"); + await mkdir(compileDirectoryPath, { recursive: true }); + await writeFile(join(appRoot, "package.json"), '{"type":"module"}\n'); + await writeFile( + join(compileDirectoryPath, "compiled-agent-manifest.json"), + `${JSON.stringify({ agentRoot: appRoot, appRoot })}\n`, + ); + const snapshot = await stageDevelopmentRuntimeArtifactsSnapshot({ + paths: { compileDirectoryPath }, + project: { appRoot }, + } as CompileAgentResult); + const workspaceRoot = join(appRoot, ".eve", "dev-hosts", "host-a"); + const entry = join(workspaceRoot, "output", "server", "index.mjs"); + await mkdir(join(workspaceRoot, "output", "server"), { recursive: true }); + await writeFile(entry, "export default {};\n"); + await writeDevelopmentRuntimeArtifactsWorker({ + entry, + snapshotRoot: snapshot.snapshotRoot, + workerData: { bytes: new Uint8Array([1, 2, 3]), port: 4321 }, + workspaceRoot, + }); + await activateDevelopmentRuntimeArtifactsSnapshot({ appRoot, snapshot }); + const nextSnapshot = await stageDevelopmentRuntimeArtifactsSnapshot({ + paths: { compileDirectoryPath }, + project: { appRoot }, + } as CompileAgentResult); + await copyDevelopmentRuntimeArtifactsWorker({ + sourceSnapshotRoot: snapshot.snapshotRoot, + targetSnapshotRoot: nextSnapshot.snapshotRoot, + }); + await activateDevelopmentRuntimeArtifactsSnapshot({ appRoot, snapshot: nextSnapshot }); + const legacySnapshotRoot = join(appRoot, ".eve", "dev-runtime", "snapshots", "legacy"); + await mkdir(legacySnapshotRoot, { recursive: true }); + await writeFile(join(legacySnapshotRoot, "activated"), ""); + + const generations = await readActivatedDevelopmentRuntimeArtifactGenerations(appRoot); + expect(generations).toHaveLength(2); + expect(generations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: basename(snapshot.snapshotRoot), + worker: { + entry, + workerData: { bytes: new Uint8Array([1, 2, 3]), port: 4321 }, + workspaceRoot, + }, + }), + expect.objectContaining({ + id: basename(nextSnapshot.snapshotRoot), + worker: { + entry, + workerData: { bytes: new Uint8Array([1, 2, 3]), port: 4321 }, + workspaceRoot, + }, + }), + ]), + ); + }); + it("restores the prior runtime pointer when activation is rolled back", async () => { const appRoot = await createScratchDirectory("eve-dev-runtime-activation-rollback-"); const agentRoot = join(appRoot, "agent"); @@ -343,7 +409,7 @@ describe("development runtime artifact snapshots", () => { expect(existsSync(staleSnapshotRoot)).toBe(false); }); - it("retains every activated generation until lease-aware pruning exists", async () => { + it("prunes an inactive generation after its leases and durable references are released", async () => { const appRoot = await createScratchDirectory("eve-dev-runtime-activated-retention-"); const snapshotsRoot = join(appRoot, ".eve", "dev-runtime", "snapshots"); const firstSnapshotRoot = join(snapshotsRoot, "first"); @@ -351,7 +417,6 @@ describe("development runtime artifact snapshots", () => { for (const snapshotRoot of [firstSnapshotRoot, nextSnapshotRoot]) { await mkdir(snapshotRoot, { recursive: true }); - await utimes(snapshotRoot, new Date(1_000), new Date(1_000)); await activateDevelopmentRuntimeArtifactsSnapshot({ appRoot, snapshot: { @@ -361,6 +426,7 @@ describe("development runtime artifact snapshots", () => { sourceRoot: appRoot, }, }); + await utimes(snapshotRoot, new Date(1_000), new Date(1_000)); } await pruneDevelopmentRuntimeArtifactsSnapshots({ @@ -370,11 +436,11 @@ describe("development runtime artifact snapshots", () => { retainCount: 0, }); - expect(existsSync(firstSnapshotRoot)).toBe(true); + expect(existsSync(firstSnapshotRoot)).toBe(false); expect(existsSync(nextSnapshotRoot)).toBe(true); }); - it("preserves snapshots referenced by active durable workflow data", async () => { + it("preserves generations explicitly referenced by nonterminal Workflow records", async () => { const appRoot = await createScratchDirectory("eve-dev-runtime-artifacts-prune-durable-"); const snapshotsRoot = join(appRoot, ".eve", "dev-runtime", "snapshots"); const activeSnapshotRoot = join(snapshotsRoot, "active"); @@ -406,77 +472,10 @@ describe("development runtime artifact snapshots", () => { sourceRoot: appRoot, }, }); - await mkdir(join(appRoot, ".workflow-data", "default", "runs"), { recursive: true }); - await writeFile( - join(appRoot, ".workflow-data", "default", "runs", "parked-turn.json"), - `${JSON.stringify( - { - status: "running", - input: { - serializedContext: { - "eve.bundle": { - source: { - appRoot: join(posixParkedTurnSnapshotRoot, "source", "app").replaceAll("\\", "/"), - kind: "disk", - }, - }, - }, - }, - workflowId: "workflow//eve//turnWorkflow", - }, - null, - 2, - )}\n`, - ); - await writeFile( - join(appRoot, ".workflow-data", "default", "runs", "parked-turn-windows.json"), - `${JSON.stringify( - { - status: "running", - input: { - serializedContext: { - "eve.bundle": { - source: { - appRoot: join(windowsParkedTurnSnapshotRoot, "source", "app").replaceAll( - "/", - "\\", - ), - kind: "disk", - }, - }, - }, - }, - workflowId: "workflow//eve//turnWorkflow", - }, - null, - 2, - )}\n`, - ); - - await writeFile( - join(appRoot, ".workflow-data", "default", "runs", "completed-turn.json"), - `${JSON.stringify( - { - status: "completed", - input: { - serializedContext: { - "eve.bundle": { - source: { - appRoot: join(completedTurnSnapshotRoot, "source", "app").replaceAll("\\", "/"), - kind: "disk", - }, - }, - }, - }, - workflowId: "workflow//eve//turnWorkflow", - }, - null, - 2, - )}\n`, - ); await pruneDevelopmentRuntimeArtifactsSnapshots({ appRoot, now, + protectedGenerationIds: new Set(["parked-turn-posix", "parked-turn-windows"]), recentWindowMs: 0, retainCount: 0, }); diff --git a/packages/eve/src/internal/nitro/dev-runtime-artifacts.ts b/packages/eve/src/internal/nitro/dev-runtime-artifacts.ts index 04ed31638..1ff87aac7 100644 --- a/packages/eve/src/internal/nitro/dev-runtime-artifacts.ts +++ b/packages/eve/src/internal/nitro/dev-runtime-artifacts.ts @@ -6,14 +6,17 @@ import { dirname, join, relative, resolve, sep } from "node:path"; import type { CompileAgentResult } from "#compiler/compile-agent.js"; import { copyDevelopmentSourceSnapshot } from "#internal/nitro/dev-runtime-source-snapshot-copy.js"; import { createDevelopmentSourceSnapshotPlan } from "#internal/nitro/dev-runtime-source-snapshot.js"; +import { + readDevelopmentRuntimeArtifactsWorker, + type DevelopmentRuntimeArtifactsWorker, +} from "#internal/nitro/dev-runtime-worker-artifacts.js"; const DEV_RUNTIME_ARTIFACTS_DIRECTORY = "dev-runtime"; const DEV_RUNTIME_ARTIFACTS_ACTIVATED_MARKER = "activated"; +const DEV_RUNTIME_ARTIFACTS_GENERATION_METADATA = "generation.json"; const DEV_RUNTIME_ARTIFACTS_POINTER_VERSION = 2; const DEV_RUNTIME_SNAPSHOT_RECENT_WINDOW_MS = 15 * 60 * 1000; const DEV_RUNTIME_SNAPSHOT_RETAIN_COUNT = 5; -const DEV_RUNTIME_WORKFLOW_DATA_MAX_SCAN_BYTES = 1024 * 1024; -const TERMINAL_WORKFLOW_RUN_STATUSES = new Set(["completed", "failed", "cancelled", "canceled"]); interface DevelopmentRuntimeArtifactsPointerV1 { readonly appRoot: string; @@ -40,6 +43,13 @@ export interface DevelopmentRuntimeArtifactsSnapshot { readonly sourceRoot: string; } +export interface ActivatedDevelopmentRuntimeArtifactsGeneration { + readonly id: string; + readonly runtimeAppRoot: string; + readonly snapshotRoot: string; + readonly worker: DevelopmentRuntimeArtifactsWorker; +} + export interface ActiveDevelopmentRuntimeArtifactsSnapshot { readonly runtimeAppRoot: string; readonly snapshotRoot: string; @@ -105,6 +115,10 @@ export async function stageDevelopmentRuntimeArtifactsSnapshot( ), runtimeAppRoot: sourceSnapshotPlan.runtimeAppRoot, }); + await writeFile( + join(snapshotRoot, DEV_RUNTIME_ARTIFACTS_GENERATION_METADATA), + `${JSON.stringify({ runtimeAppRoot: sourceSnapshotPlan.runtimeAppRoot })}\n`, + ); } catch (error) { await rm(snapshotRoot, { force: true, recursive: true }).catch(() => {}); throw error; @@ -118,6 +132,72 @@ export async function stageDevelopmentRuntimeArtifactsSnapshot( }; } +export async function readActivatedDevelopmentRuntimeArtifactGenerations( + appRoot: string, +): Promise { + const snapshotsDirectory = resolveDevelopmentRuntimeArtifactsSnapshotsDirectory(appRoot); + let entries: Dirent[]; + try { + entries = await readdir(snapshotsDirectory, { withFileTypes: true }); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") { + return []; + } + throw error; + } + const generations = await Promise.all( + entries + .filter((entry) => entry.isDirectory()) + .map(async (entry): Promise => { + const snapshotRoot = join(snapshotsDirectory, entry.name); + if (!existsSync(join(snapshotRoot, DEV_RUNTIME_ARTIFACTS_ACTIVATED_MARKER))) { + return undefined; + } + const metadataSource = await readGenerationMetadata(snapshotRoot); + if (metadataSource === undefined) { + return undefined; + } + const metadata = parseJsonObject(metadataSource); + if (metadata === undefined || typeof metadata.runtimeAppRoot !== "string") { + throw new Error(`Activated development generation "${entry.name}" has invalid metadata.`); + } + if ( + !isPathInsideOrEqual(metadata.runtimeAppRoot, snapshotRoot) || + !existsSync( + join(metadata.runtimeAppRoot, ".eve", "compile", "compiled-agent-manifest.json"), + ) + ) { + throw new Error(`Activated development generation "${entry.name}" is incomplete.`); + } + const worker = await readDevelopmentRuntimeArtifactsWorker(snapshotRoot); + if (worker === undefined) { + return undefined; + } + return { + id: entry.name, + runtimeAppRoot: metadata.runtimeAppRoot, + snapshotRoot, + worker, + }; + }), + ); + return generations.filter( + (generation): generation is ActivatedDevelopmentRuntimeArtifactsGeneration => + generation !== undefined, + ); +} + +async function readGenerationMetadata(snapshotRoot: string): Promise { + try { + return await readFile(join(snapshotRoot, DEV_RUNTIME_ARTIFACTS_GENERATION_METADATA), "utf8"); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") { + return undefined; + } + throw error; + } +} + /** * Moves the dev runtime pointer so future sessions use a staged snapshot. */ @@ -222,17 +302,19 @@ export function readDevelopmentRuntimeArtifactsRevision( export async function pruneDevelopmentRuntimeArtifactsSnapshots(input: { readonly appRoot: string; readonly now?: number; + readonly protectAll?: boolean; + readonly protectedGenerationIds?: ReadonlySet; readonly recentWindowMs?: number; readonly retainCount?: number; }): Promise { + if (input.protectAll === true) { + return; + } const snapshotsDirectory = resolveDevelopmentRuntimeArtifactsSnapshotsDirectory(input.appRoot); const pointer = readDevelopmentRuntimeArtifactsPointer( resolveDevelopmentRuntimeArtifactsPointerPath(input.appRoot), ); - const protectedPaths = [ - ...collectProtectedSnapshotPaths(pointer), - ...(await collectWorkflowDataSnapshotPaths({ appRoot: input.appRoot, snapshotsDirectory })), - ]; + const protectedPaths = collectProtectedSnapshotPaths(pointer); const now = input.now ?? Date.now(); const recentWindowMs = input.recentWindowMs ?? DEV_RUNTIME_SNAPSHOT_RECENT_WINDOW_MS; const retainCount = input.retainCount ?? DEV_RUNTIME_SNAPSHOT_RETAIN_COUNT; @@ -254,6 +336,7 @@ export async function pruneDevelopmentRuntimeArtifactsSnapshots(input: { .map(async (entry) => { const path = join(snapshotsDirectory, entry.name); return { + name: entry.name, path, mtimeMs: (await stat(path)).mtimeMs, }; @@ -264,7 +347,7 @@ export async function pruneDevelopmentRuntimeArtifactsSnapshots(input: { await Promise.all( snapshots.map(async (snapshot, index) => { if ( - existsSync(join(snapshot.path, DEV_RUNTIME_ARTIFACTS_ACTIVATED_MARKER)) || + input.protectedGenerationIds?.has(snapshot.name) === true || index < retainCount || now - snapshot.mtimeMs <= recentWindowMs || protectedPaths.some((protectedPath) => pathsOverlap(snapshot.path, protectedPath)) @@ -345,104 +428,6 @@ function collectProtectedSnapshotPaths( return [pointer.runtimeAppRoot, pointer.snapshotRoot]; } -async function collectWorkflowDataSnapshotPaths(input: { - readonly appRoot: string; - readonly snapshotsDirectory: string; -}): Promise { - const workflowDataDirectory = join(input.appRoot, ".workflow-data"); - const snapshotPaths = new Set(); - - await collectSnapshotPathsFromDirectory({ - directory: workflowDataDirectory, - snapshotPaths, - snapshotsDirectory: input.snapshotsDirectory, - }); - - return [...snapshotPaths]; -} - -async function collectSnapshotPathsFromDirectory(input: { - readonly directory: string; - readonly snapshotPaths: Set; - readonly snapshotsDirectory: string; -}): Promise { - let entries: Dirent[]; - - try { - entries = await readdir(input.directory, { withFileTypes: true }); - } catch (error) { - if (error instanceof Error && "code" in error && error.code === "ENOENT") { - return; - } - - throw error; - } - - await Promise.all( - entries.map(async (entry) => { - const path = join(input.directory, entry.name); - - if (entry.isDirectory()) { - await collectSnapshotPathsFromDirectory({ - directory: path, - snapshotPaths: input.snapshotPaths, - snapshotsDirectory: input.snapshotsDirectory, - }); - return; - } - - if (!entry.isFile()) { - return; - } - - const fileStats = await stat(path); - // Local workflow run payloads are small; keep this best-effort scan cheap. - if (fileStats.size > DEV_RUNTIME_WORKFLOW_DATA_MAX_SCAN_BYTES) { - return; - } - - const source = await readFile(path, "utf8"); - if (!shouldScanWorkflowDataSource(source)) { - return; - } - - for (const snapshotPath of collectSnapshotPathsFromText(source, input.snapshotsDirectory)) { - input.snapshotPaths.add(snapshotPath); - } - }), - ); -} - -function collectSnapshotPathsFromText( - source: string, - snapshotsDirectory: string, -): readonly string[] { - const snapshotPaths = new Set(); - const normalizedSnapshotsDirectory = snapshotsDirectory.replaceAll("\\", "/"); - const pattern = new RegExp(`${escapeRegExp(normalizedSnapshotsDirectory)}/([^/"'\\s]+)`, "gu"); - const normalizedSource = source.replaceAll("\\\\", "/").replaceAll("\\", "/"); - - for (const match of normalizedSource.matchAll(pattern)) { - const snapshotName = match[1]; - - if (snapshotName !== undefined && snapshotName.length > 0) { - snapshotPaths.add(join(snapshotsDirectory, snapshotName)); - } - } - - return [...snapshotPaths]; -} - -function shouldScanWorkflowDataSource(source: string): boolean { - const value = parseJsonObject(source); - if (value === undefined) { - return true; - } - - const status = value.status; - return typeof status !== "string" || !TERMINAL_WORKFLOW_RUN_STATUSES.has(status); -} - function parseJsonObject(source: string): Record | undefined { try { const value = JSON.parse(source) as unknown; @@ -456,10 +441,6 @@ function isObjectRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } -function escapeRegExp(value: string): string { - return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); -} - function pathsOverlap(left: string, right: string): boolean { return isPathInsideOrEqual(left, right) || isPathInsideOrEqual(right, left); } diff --git a/packages/eve/src/internal/nitro/dev-runtime-worker-artifacts.ts b/packages/eve/src/internal/nitro/dev-runtime-worker-artifacts.ts new file mode 100644 index 000000000..3a48aef0e --- /dev/null +++ b/packages/eve/src/internal/nitro/dev-runtime-worker-artifacts.ts @@ -0,0 +1,98 @@ +import { existsSync } from "node:fs"; +import { readFile, writeFile } from "node:fs/promises"; +import { basename, join } from "node:path"; +import { deserialize, serialize } from "node:v8"; + +const WORKER_DATA_FILE = "worker-data.bin"; +const WORKER_METADATA_FILE = "worker.json"; + +export interface DevelopmentRuntimeArtifactsWorker { + readonly entry: string; + readonly workerData: Readonly>; + readonly workspaceRoot: string; +} + +export async function writeDevelopmentRuntimeArtifactsWorker(input: { + readonly entry: string; + readonly snapshotRoot: string; + readonly workerData: Readonly>; + readonly workspaceRoot: string; +}): Promise { + await writeFile(join(input.snapshotRoot, WORKER_DATA_FILE), serialize(input.workerData)); + await writeFile( + join(input.snapshotRoot, WORKER_METADATA_FILE), + `${JSON.stringify({ entry: input.entry, workspaceRoot: input.workspaceRoot })}\n`, + ); +} + +export async function copyDevelopmentRuntimeArtifactsWorker(input: { + readonly sourceSnapshotRoot: string; + readonly targetSnapshotRoot: string; +}): Promise { + const worker = await readDevelopmentRuntimeArtifactsWorker(input.sourceSnapshotRoot); + if (worker === undefined) { + throw new Error( + `Development generation "${basename(input.sourceSnapshotRoot)}" has no worker metadata.`, + ); + } + await writeDevelopmentRuntimeArtifactsWorker({ + ...worker, + snapshotRoot: input.targetSnapshotRoot, + }); +} + +export async function readDevelopmentRuntimeArtifactsWorker( + snapshotRoot: string, +): Promise { + const metadataPath = join(snapshotRoot, WORKER_METADATA_FILE); + const dataPath = join(snapshotRoot, WORKER_DATA_FILE); + let metadataSource: string; + let workerDataSource: Buffer; + try { + [metadataSource, workerDataSource] = await Promise.all([ + readFile(metadataPath, "utf8"), + readFile(dataPath), + ]); + } catch (error) { + if ( + error instanceof Error && + "code" in error && + error.code === "ENOENT" && + !existsSync(metadataPath) && + !existsSync(dataPath) + ) { + return undefined; + } + throw error; + } + const metadata = parseJsonObject(metadataSource); + const workerData = deserialize(workerDataSource) as unknown; + if ( + metadata === undefined || + typeof metadata.entry !== "string" || + typeof metadata.workspaceRoot !== "string" || + !isObjectRecord(workerData) + ) { + throw new Error( + `Development generation "${basename(snapshotRoot)}" has invalid worker metadata.`, + ); + } + return { + entry: metadata.entry, + workerData, + workspaceRoot: metadata.workspaceRoot, + }; +} + +function parseJsonObject(source: string): Record | undefined { + try { + const value = JSON.parse(source) as unknown; + return isObjectRecord(value) ? value : undefined; + } catch { + return undefined; + } +} + +function isObjectRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/eve/src/internal/nitro/host/configure-nitro-routes.test.ts b/packages/eve/src/internal/nitro/host/configure-nitro-routes.test.ts index 01d704e18..c007c027d 100644 --- a/packages/eve/src/internal/nitro/host/configure-nitro-routes.test.ts +++ b/packages/eve/src/internal/nitro/host/configure-nitro-routes.test.ts @@ -271,7 +271,7 @@ describe("Nitro route configuration", () => { expect(workflowBuilderMocks.build).toHaveBeenCalledOnce(); }); - it("registers direct workflow queue handlers in dev mode so the worker bypasses HTTP dispatch", async () => { + it("leaves development queue dispatch to the stable parent", async () => { const root = "/tmp/eve-nitro-direct-handlers"; const buildDir = `${root}/nitro`; const nitro = createNitroStub({ buildDir, dev: true, rootDir: root }); @@ -281,19 +281,25 @@ describe("Nitro route configuration", () => { const workflowHandlerSource = readWriteFileSourceMatching("/workflow/workflows-handler.mjs"); expect(workflowHandlerSource).toContain('import { POST } from "./workflows.mjs";'); - expect(workflowHandlerSource).toContain( - 'import "../../.eve/compiled-artifacts-workflow-world.mjs";', - ); - expect(workflowHandlerSource).toContain( - 'import { getWorld as __eveGetWorkflowWorld } from "file:///G:/projects/test-eve/node_modules/.pnpm/eve@0.3.0/node_modules/eve/dist/src/compiled/@workflow/core/runtime.js";', + expect(workflowHandlerSource).not.toContain("registerHandler"); + expect(workflowHandlerSource).not.toContain("__eveGetWorkflowWorld"); + expect(readWriteFileSourceMatching("/workflow/steps-handler.mjs")).toBeUndefined(); + }); + + it("keeps configured development World queue dispatch inside the worker", async () => { + const root = "/tmp/eve-nitro-configured-world-handlers"; + const nitro = createNitroStub({ buildDir: `${root}/nitro`, dev: true, rootDir: root }); + + await configureDevelopmentNitroRoutes( + nitro, + createPreparedHost({ appRoot: root, workflowWorld: "@workflow/world-postgres" }), ); + + const workflowHandlerSource = readWriteFileSourceMatching("/workflow/workflows-handler.mjs"); expect(workflowHandlerSource).toContain( "const __eveWorkflowWorld = await __eveGetWorkflowWorld();", ); - expect(workflowHandlerSource).toContain( - '__eveWorkflowWorld.registerHandler("__eve746573742d6167656e74_wkf_workflow_", POST);', - ); - expect(readWriteFileSourceMatching("/workflow/steps-handler.mjs")).toBeUndefined(); + expect(workflowHandlerSource).toContain("__eveWorkflowWorld.registerHandler"); }); it("bakes the module map loader into the dev schedule handler", async () => { diff --git a/packages/eve/src/internal/nitro/host/configure-nitro-routes.ts b/packages/eve/src/internal/nitro/host/configure-nitro-routes.ts index eb519db8e..b1ede5de7 100644 --- a/packages/eve/src/internal/nitro/host/configure-nitro-routes.ts +++ b/packages/eve/src/internal/nitro/host/configure-nitro-routes.ts @@ -392,9 +392,11 @@ export async function configureDevelopmentNitroRoutes( registerDevelopmentControlRoutes(nitro, artifactsConfig); const workflowBundlePath = join(workflowBuildDirectory, "workflows.mjs"); - await registerWorkflowRoute(nitro, preparedHost, workflowBundlePath, [ - createWorkflowDirectHandlerEntry(preparedHost, workflowBundlePath), - ]); + const directHandlers: WorkflowDirectHandlerEntry[] = []; + if (preparedHost.compileResult.manifest.config.experimental?.workflow?.world !== undefined) { + directHandlers.push(createWorkflowDirectHandlerEntry(preparedHost, workflowBundlePath)); + } + await registerWorkflowRoute(nitro, preparedHost, workflowBundlePath, directHandlers); nitro.routing.sync(); } diff --git a/packages/eve/src/internal/nitro/host/dev-authored-rebuild-coordinator.ts b/packages/eve/src/internal/nitro/host/dev-authored-rebuild-coordinator.ts index c76a6fbe6..ecce03f3c 100644 --- a/packages/eve/src/internal/nitro/host/dev-authored-rebuild-coordinator.ts +++ b/packages/eve/src/internal/nitro/host/dev-authored-rebuild-coordinator.ts @@ -85,11 +85,13 @@ class TransactionalDevelopmentAuthoredRebuildCoordinator implements DevelopmentA if (!hasStructuralChange) { await removeDevelopmentHostWorkspace(nextHost.workspace); const committedHost = retainActiveHostWorkspace(previousHost, nextHost); - await this.#devServer.publishRuntimeGeneration(async () => { - return await activateDevelopmentGenerationTransaction({ - appRoot: committedHost.appRoot, - generation: committedHost.generation, - }); + await this.#devServer.publishRuntimeGeneration({ + generation: committedHost.generation, + publish: async () => + await activateDevelopmentGenerationTransaction({ + appRoot: committedHost.appRoot, + generation: committedHost.generation, + }), }); this.#commitState(committedHost, nextHostFingerprint, nextRuntimeFingerprint); nextHost = undefined; diff --git a/packages/eve/src/internal/nitro/host/dev-host-candidate.ts b/packages/eve/src/internal/nitro/host/dev-host-candidate.ts index 050b038a7..298bdb7e8 100644 --- a/packages/eve/src/internal/nitro/host/dev-host-candidate.ts +++ b/packages/eve/src/internal/nitro/host/dev-host-candidate.ts @@ -6,7 +6,7 @@ import { NitroDevelopmentWorkerServer, toDevelopmentWorkerGeneration, } from "#internal/nitro/host/nitro-development-worker-server.js"; -import type { DevelopmentWorkerCandidate } from "#internal/nitro/host/dev-worker-server.js"; +import type { DevelopmentWorkerCandidate } from "#internal/nitro/host/dev-worker-server-types.js"; import type { PreparedDevelopmentApplicationHost } from "#internal/nitro/host/types.js"; export async function buildDevelopmentHostCandidate(input: { @@ -26,6 +26,7 @@ export async function buildDevelopmentHostCandidate(input: { await prepare(input.nitro); await buildNitro(input.nitro); }, + workspaceRoot: input.host.workspace.rootDir, }); } catch (error) { try { diff --git a/packages/eve/src/internal/nitro/host/dev-worker-http-server.ts b/packages/eve/src/internal/nitro/host/dev-worker-http-server.ts new file mode 100644 index 000000000..7ff762e5d --- /dev/null +++ b/packages/eve/src/internal/nitro/host/dev-worker-http-server.ts @@ -0,0 +1,246 @@ +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import type { Socket } from "node:net"; + +import { + closeServer, + createPublicRequest, + createWorkerRequest, + stampWorkerUpgradeRequest, + writeRequestError, + writeResponse, +} from "#internal/nitro/host/dev-worker-http.js"; +import type { + DevelopmentWorkerGeneration, + DevelopmentWorkerListener, + DevelopmentWorkerRunner, +} from "#internal/nitro/host/dev-worker-server-types.js"; + +interface ActiveExchange { + cancel(): void; + readonly runner: DevelopmentWorkerRunner; +} + +interface ListenerState { + readonly beginClose: () => Promise; + destroySockets(): void; +} + +export interface DevelopmentWorkerHttpLease { + readonly generation: DevelopmentWorkerGeneration; + release(): void; + readonly runner: DevelopmentWorkerRunner; +} + +export class DevelopmentWorkerHttpServer { + readonly #activeExchanges = new Set(); + readonly #admit: () => Promise; + readonly #handleParentRequest: (request: Request) => Promise; + readonly #listeners = new Set(); + readonly #transportSecret: string; + + constructor(input: { + readonly admit: () => Promise; + readonly handleParentRequest: (request: Request) => Promise; + readonly transportSecret: string; + }) { + this.#admit = input.admit; + this.#handleParentRequest = input.handleParentRequest; + this.#transportSecret = input.transportSecret; + } + + listen(input: { readonly hostname: string; readonly port: number }): DevelopmentWorkerListener { + let url: string | undefined; + const server = createServer((request, response) => { + void this.#handleRequest(request, response); + }); + server.on("upgrade", (request, socket, head) => { + void this.#handleUpgrade(request, socket as Socket, head); + }); + const sockets = new Set(); + server.on("connection", (socket) => { + sockets.add(socket); + socket.once("close", () => sockets.delete(socket)); + }); + const ready = new Promise((resolve, reject) => { + const onError = (error: Error) => { + server.off("listening", onListening); + reject(error); + }; + const onListening = () => { + server.off("error", onError); + const address = server.address(); + if (address === null || typeof address === "string") { + reject(new Error("Development worker server did not expose a TCP address.")); + return; + } + url = `http://${formatListenerHostname(input.hostname)}:${String(address.port)}/`; + resolve(); + }; + server.once("error", onError); + server.once("listening", onListening); + server.listen({ host: input.hostname, port: input.port }); + }); + + let listenerClosePromise: Promise | undefined; + const beginClose = () => { + listenerClosePromise ??= closeServer(server).finally(() => { + this.#listeners.delete(listenerState); + }); + return listenerClosePromise; + }; + const destroySockets = () => { + for (const socket of sockets) { + socket.destroy(); + } + }; + const listenerState: ListenerState = { beginClose, destroySockets }; + this.#listeners.add(listenerState); + + return { + async close() { + const closed = beginClose(); + destroySockets(); + await closed; + }, + node: { server }, + ready: async () => await ready, + get url() { + return url; + }, + }; + } + + cancelRunnerExchanges(runner: DevelopmentWorkerRunner): void { + for (const exchange of this.#activeExchanges) { + if (exchange.runner === runner) { + exchange.cancel(); + this.#activeExchanges.delete(exchange); + } + } + } + + async close(closeWorkers: () => Promise): Promise { + const listeners = [...this.#listeners]; + const listenerClosePromises = listeners.map((listener) => listener.beginClose()); + for (const exchange of this.#activeExchanges) { + exchange.cancel(); + } + this.#activeExchanges.clear(); + await closeWorkers(); + for (const listener of listeners) { + listener.destroySockets(); + } + await Promise.all(listenerClosePromises); + } + + async #handleRequest(request: IncomingMessage, response: ServerResponse): Promise { + const requestAbort = new AbortController(); + const abortRequest = (cause?: unknown) => { + if (!requestAbort.signal.aborted) { + requestAbort.abort(cause); + } + }; + request.once("aborted", abortRequest); + request.once("error", abortRequest); + response.once("close", () => { + if (!response.writableEnded) { + abortRequest(); + } + }); + + let exchange: ActiveExchange | undefined; + try { + const publicRequest = createPublicRequest(request, requestAbort.signal); + const parentResponse = await this.#handleParentRequest(publicRequest); + if (parentResponse !== undefined) { + await writeResponse(response, parentResponse, requestAbort.signal); + return; + } + const lease = await this.#admit(); + const admittedLease = lease; + exchange = { + cancel() { + abortRequest(new Error("Development worker request was cancelled.")); + if (!response.destroyed) { + response.destroy(); + } + admittedLease.release(); + }, + runner: lease.runner, + }; + this.#activeExchanges.add(exchange); + response.once("finish", admittedLease.release); + response.once("error", admittedLease.release); + response.once("close", admittedLease.release); + request.once("aborted", admittedLease.release); + const workerRequest = createWorkerRequest({ + generation: lease.generation, + request: publicRequest, + secret: this.#transportSecret, + socket: request.socket, + }); + const workerResponse = await lease.runner.fetch(workerRequest); + await writeResponse(response, workerResponse, requestAbort.signal); + } catch (error) { + if (!requestAbort.signal.aborted) { + writeRequestError(response, error); + } + } finally { + if (exchange !== undefined) { + exchange.cancel(); + this.#activeExchanges.delete(exchange); + } + request.off("aborted", abortRequest); + request.off("error", abortRequest); + } + } + + async #handleUpgrade(request: IncomingMessage, socket: Socket, head: Buffer): Promise { + let lease: DevelopmentWorkerHttpLease | undefined; + let exchange: ActiveExchange | undefined; + try { + lease = await this.#admit(); + stampWorkerUpgradeRequest({ + generation: lease.generation, + request, + secret: this.#transportSecret, + }); + const admittedLease = lease; + exchange = { + cancel() { + if (!socket.destroyed) { + socket.destroy(); + } + admittedLease.release(); + }, + runner: lease.runner, + }; + this.#activeExchanges.add(exchange); + const release = () => { + admittedLease.release(); + if (exchange !== undefined) { + this.#activeExchanges.delete(exchange); + exchange = undefined; + } + }; + socket.once("close", release); + socket.once("error", release); + await lease.runner.upgrade({ node: { head, req: request, socket } }); + } catch { + if (exchange !== undefined) { + exchange.cancel(); + } else { + // Failure before the exchange existed: no socket listener releases + // the lease, so release it here or the retired worker never closes. + lease?.release(); + } + if (!socket.destroyed) { + socket.destroy(); + } + } + } +} + +function formatListenerHostname(hostname: string): string { + return hostname.includes(":") ? `[${hostname}]` : hostname; +} diff --git a/packages/eve/src/internal/nitro/host/dev-worker-http.ts b/packages/eve/src/internal/nitro/host/dev-worker-http.ts index e79ef9c9b..e784889e7 100644 --- a/packages/eve/src/internal/nitro/host/dev-worker-http.ts +++ b/packages/eve/src/internal/nitro/host/dev-worker-http.ts @@ -6,7 +6,8 @@ import { DEVELOPMENT_WORKER_METADATA_HEADER, encodeDevelopmentWorkerMetadata, } from "#internal/nitro/host/dev-worker-metadata.js"; -import type { DevelopmentWorkerGeneration } from "#internal/nitro/host/dev-worker-server.js"; +import { DEVELOPMENT_WORKFLOW_DELIVERY_HEADER } from "#internal/workflow/development-world-protocol.js"; +import type { DevelopmentWorkerGeneration } from "#internal/nitro/host/dev-worker-server-types.js"; import { toErrorMessage } from "#shared/errors.js"; export function createPublicRequest(request: IncomingMessage, signal: AbortSignal): Request { @@ -39,13 +40,40 @@ export function createWorkerRequest(input: { readonly request: Request; readonly secret: string; readonly socket: Socket; +}): Request { + return createTrustedWorkerRequest({ + clientAddress: normalizeClientAddress(input.socket.remoteAddress), + generation: input.generation, + request: input.request, + secret: input.secret, + }); +} + +export function createWorkflowWorkerRequest(input: { + readonly generation: DevelopmentWorkerGeneration; + readonly request: Request; + readonly secret: string; +}): Request { + const request = createTrustedWorkerRequest({ + clientAddress: null, + ...input, + }); + request.headers.set(DEVELOPMENT_WORKFLOW_DELIVERY_HEADER, input.secret); + return request; +} + +function createTrustedWorkerRequest(input: { + readonly clientAddress: string | null; + readonly generation: DevelopmentWorkerGeneration; + readonly request: Request; + readonly secret: string; }): Request { const headers = new Headers(input.request.headers); headers.set( DEVELOPMENT_WORKER_METADATA_HEADER, encodeDevelopmentWorkerMetadata({ metadata: { - clientAddress: normalizeClientAddress(input.socket.remoteAddress), + clientAddress: input.clientAddress, generationId: input.generation.id, runtimeAppRoot: input.generation.runtimeAppRoot, }, diff --git a/packages/eve/src/internal/nitro/host/dev-worker-runner.ts b/packages/eve/src/internal/nitro/host/dev-worker-runner.ts index 0ea9faba2..cf5fb9254 100644 --- a/packages/eve/src/internal/nitro/host/dev-worker-runner.ts +++ b/packages/eve/src/internal/nitro/host/dev-worker-runner.ts @@ -10,7 +10,7 @@ import { import type { DevelopmentWorkerRunner, DevelopmentWorkerRunnerFactory, -} from "#internal/nitro/host/dev-worker-server.js"; +} from "#internal/nitro/host/dev-worker-server-types.js"; class NodeDevelopmentWorkerRunner extends BaseEnvRunner implements DevelopmentWorkerRunner { #closeCause: unknown; diff --git a/packages/eve/src/internal/nitro/host/dev-worker-server-types.ts b/packages/eve/src/internal/nitro/host/dev-worker-server-types.ts new file mode 100644 index 000000000..c4f9afaac --- /dev/null +++ b/packages/eve/src/internal/nitro/host/dev-worker-server-types.ts @@ -0,0 +1,91 @@ +import type { IncomingMessage, Server } from "node:http"; +import type { Socket } from "node:net"; + +export interface DevelopmentWorkerGeneration { + readonly id: string; + readonly runtimeAppRoot: string; + readonly snapshotRoot: string; +} + +export interface DevelopmentWorkerRunner { + readonly closed: boolean; + close(cause?: unknown): Promise; + fetch(request: Request): Promise; + upgrade(input: { + readonly node: { + readonly head: Buffer; + readonly req: IncomingMessage; + readonly socket: Socket; + }; + }): Promise; + waitForReady(timeout: number): Promise; +} + +export interface DevelopmentWorkerRunnerFactoryInput { + readonly appRoot: string; + readonly entry: string; + readonly name: string; + readonly onClose: (cause?: unknown) => void; + readonly transportSecret: string; + readonly workerData: Readonly>; +} + +export type DevelopmentWorkerRunnerFactory = ( + input: DevelopmentWorkerRunnerFactoryInput, +) => DevelopmentWorkerRunner; + +export type DevelopmentWorkerState = "candidate" | "active" | "retired" | "closed"; + +export interface DevelopmentWorkerSlot { + disposed: boolean; + readonly dispose: () => Promise; + readonly entry: string; + readonly generation: DevelopmentWorkerGeneration; + leases: number; + readonly runner: DevelopmentWorkerRunner; + state: DevelopmentWorkerState; + readonly workerData: Readonly>; +} + +export interface DevelopmentWorkerCandidate { + readonly slot: DevelopmentWorkerSlot; +} + +export interface DevelopmentWorkerPublication { + commit(): void; + rollback(): Promise; +} + +export interface DevelopmentWorkerListener { + close(): Promise; + readonly node: { readonly server: Server }; + ready(): Promise; + readonly url: string | undefined; +} + +export interface DevelopmentWorkerServer { + close(): Promise; + discardCandidate(candidate: DevelopmentWorkerCandidate): Promise; + listen(input: { readonly hostname: string; readonly port: number }): DevelopmentWorkerListener; + prepareCandidate(input: { + readonly dispose: () => Promise; + readonly entry: string; + readonly generation: DevelopmentWorkerGeneration; + readonly workerData: Readonly>; + }): Promise; + promote(candidate: DevelopmentWorkerCandidate): Promise; + publishRuntimeGeneration(input: { + readonly generation: DevelopmentWorkerGeneration; + readonly publish: () => Promise; + }): Promise; + publishStructuralCandidate(input: { + readonly candidate: DevelopmentWorkerCandidate; + readonly publish: () => Promise; + }): Promise; + setControlHandler(handler: (request: Request) => Promise): void; + startWorkflowWorld(): Promise; +} + +export type DevelopmentWorkflowWorldOwnership = + | { readonly agentName: string; readonly kind: "parent-local" } + | { readonly kind: "worker-configured" }; diff --git a/packages/eve/src/internal/nitro/host/dev-worker-server.scenario.test.ts b/packages/eve/src/internal/nitro/host/dev-worker-server.scenario.test.ts index eb580e9e8..d0a76693d 100644 --- a/packages/eve/src/internal/nitro/host/dev-worker-server.scenario.test.ts +++ b/packages/eve/src/internal/nitro/host/dev-worker-server.scenario.test.ts @@ -3,12 +3,12 @@ import { connect } from "node:net"; import { describe, expect, it, vi } from "vitest"; -import { - createDevelopmentWorkerServer, - type DevelopmentWorkerGeneration, - type DevelopmentWorkerRunner, - type DevelopmentWorkerRunnerFactory, -} from "#internal/nitro/host/dev-worker-server.js"; +import { createDevelopmentWorkerServer } from "#internal/nitro/host/dev-worker-server.js"; +import type { + DevelopmentWorkerGeneration, + DevelopmentWorkerRunner, + DevelopmentWorkerRunnerFactory, +} from "#internal/nitro/host/dev-worker-server-types.js"; import { decodeDevelopmentWorkerMetadata } from "#internal/nitro/host/dev-worker-metadata.js"; const TEST_DEADLINE_MS = 5_000; @@ -16,10 +16,12 @@ const TEST_DEADLINE_MS = 5_000; const FIRST_GENERATION: DevelopmentWorkerGeneration = { id: "first", runtimeAppRoot: "/tmp/eve-dev-runtime/first/source/app", + snapshotRoot: "/tmp/eve-dev-runtime/first", }; const SECOND_GENERATION: DevelopmentWorkerGeneration = { id: "second", runtimeAppRoot: "/tmp/eve-dev-runtime/second/source/app", + snapshotRoot: "/tmp/eve-dev-runtime/second", }; function createDeferred(): { @@ -115,6 +117,7 @@ async function listenWithGenerationResolver( appRoot: "/tmp/eve-dev-worker-test", createRunner, resolveAdmissionGeneration, + workflowWorld: { agentName: "worker-server-test", kind: "parent-local" }, }); const listener = server.listen({ hostname: "127.0.0.1", port: 0 }); await listener.ready(); @@ -163,6 +166,7 @@ describe("development worker server", () => { throw new Error("worker creation failed"); }, resolveAdmissionGeneration: (generation) => generation, + workflowWorld: { agentName: "worker-server-test", kind: "parent-local" }, }); await expect( @@ -686,6 +690,41 @@ describe("development worker server", () => { await closeWithinDeadline(() => server.close()); }); + it("leaves configured World queue requests with the worker", async () => { + const { createRunner } = createRunnerFactory(async (request) => + request.url.endsWith("/.well-known/workflow/v1/flow") + ? new Response("configured-world") + : new Response("unexpected", { status: 404 }), + ); + const server = createDevelopmentWorkerServer({ + appRoot: "/tmp/eve-dev-worker-configured-world-test", + createRunner, + resolveAdmissionGeneration: (generation) => generation, + workflowWorld: { kind: "worker-configured" }, + }); + const listener = server.listen({ hostname: "127.0.0.1", port: 0 }); + await listener.ready(); + if (listener.url === undefined) { + throw new Error("Development worker listener did not expose a URL."); + } + const candidate = await server.prepareCandidate({ + dispose: async () => undefined, + entry: "/tmp/configured-world.mjs", + generation: FIRST_GENERATION, + workerData: {}, + }); + await server.promote(candidate); + + await expect( + fetch(new URL("/.well-known/workflow/v1/flow", listener.url), { + body: "{}", + method: "POST", + }).then(async (response) => await response.text()), + ).resolves.toBe("configured-world"); + + await closeWithinDeadline(() => server.close()); + }); + it("closes workers and listeners only once when close is repeated", async () => { const { createRunner, runners } = createRunnerFactory(async () => new Response("ok")); const { server } = await listen(createRunner); diff --git a/packages/eve/src/internal/nitro/host/dev-worker-server.ts b/packages/eve/src/internal/nitro/host/dev-worker-server.ts index 559501461..554f2fbd5 100644 --- a/packages/eve/src/internal/nitro/host/dev-worker-server.ts +++ b/packages/eve/src/internal/nitro/host/dev-worker-server.ts @@ -1,113 +1,39 @@ import { randomBytes } from "node:crypto"; -import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; -import type { Socket } from "node:net"; +import { existsSync } from "node:fs"; +import { rm } from "node:fs/promises"; +import { DevelopmentWorkerHttpServer } from "#internal/nitro/host/dev-worker-http-server.js"; +import { DevelopmentWorkflowDispatcher } from "#internal/nitro/host/development-workflow-dispatch.js"; +import { isRestorableDevelopmentWorker } from "#internal/nitro/host/development-worker-recovery.js"; import { - closeServer, - createPublicRequest, - createWorkerRequest, - stampWorkerUpgradeRequest, - writeRequestError, - writeResponse, -} from "#internal/nitro/host/dev-worker-http.js"; + pruneDevelopmentRuntimeArtifactsSnapshots, + readActivatedDevelopmentRuntimeArtifactGenerations, +} from "#internal/nitro/dev-runtime-artifacts.js"; +import { copyDevelopmentRuntimeArtifactsWorker } from "#internal/nitro/dev-runtime-worker-artifacts.js"; +import type { + DevelopmentWorkerCandidate, + DevelopmentWorkerGeneration, + DevelopmentWorkerListener, + DevelopmentWorkerPublication, + DevelopmentWorkerRunner, + DevelopmentWorkerRunnerFactory, + DevelopmentWorkerServer, + DevelopmentWorkerSlot, + DevelopmentWorkflowWorldOwnership, +} from "#internal/nitro/host/dev-worker-server-types.js"; +import { + createParentDevelopmentWorkflowWorld, + type DevelopmentWorkflowGenerationReferences, + type ParentDevelopmentWorkflowWorld, +} from "#internal/workflow/development-world-server.js"; import { toErrorMessage } from "#shared/errors.js"; const DEVELOPMENT_WORKER_READY_TIMEOUT_MS = 60_000; -export interface DevelopmentWorkerGeneration { - readonly id: string; - readonly runtimeAppRoot: string; -} - -export interface DevelopmentWorkerRunner { - readonly closed: boolean; - close(cause?: unknown): Promise; - fetch(request: Request): Promise; - upgrade(input: { - readonly node: { - readonly head: Buffer; - readonly req: IncomingMessage; - readonly socket: Socket; - }; - }): Promise; - waitForReady(timeout: number): Promise; -} - -export interface DevelopmentWorkerRunnerFactoryInput { - readonly appRoot: string; - readonly entry: string; - readonly name: string; - readonly onClose: (cause?: unknown) => void; - readonly transportSecret: string; - readonly workerData: Readonly>; -} - -export type DevelopmentWorkerRunnerFactory = ( - input: DevelopmentWorkerRunnerFactoryInput, -) => DevelopmentWorkerRunner; - -type WorkerState = "candidate" | "active" | "retired" | "closed"; - -interface WorkerSlot { - disposed: boolean; - readonly dispose: () => Promise; - readonly entry: string; - readonly generation: DevelopmentWorkerGeneration; - leases: number; - readonly runner: DevelopmentWorkerRunner; - state: WorkerState; - readonly workerData: Readonly>; -} - -export interface DevelopmentWorkerCandidate { - readonly slot: WorkerSlot; -} - -export interface DevelopmentWorkerPublication { - commit(): void; - rollback(): Promise; -} - -export interface DevelopmentWorkerListener { - close(): Promise; - readonly node: { readonly server: Server }; - ready(): Promise; - readonly url: string | undefined; -} - interface RequestLease { readonly generation: DevelopmentWorkerGeneration; release(): void; - readonly slot: WorkerSlot; -} - -interface ActiveExchange { - cancel(): void; - readonly slot: WorkerSlot; -} - -interface ListenerState { - readonly beginClose: () => Promise; - destroySockets(): void; -} - -export interface DevelopmentWorkerServer { - close(): Promise; - discardCandidate(candidate: DevelopmentWorkerCandidate): Promise; - listen(input: { readonly hostname: string; readonly port: number }): DevelopmentWorkerListener; - prepareCandidate(input: { - readonly dispose: () => Promise; - readonly entry: string; - readonly generation: DevelopmentWorkerGeneration; - readonly workerData: Readonly>; - }): Promise; - promote(candidate: DevelopmentWorkerCandidate): Promise; - publishRuntimeGeneration(publish: () => Promise): Promise; - publishStructuralCandidate(input: { - readonly candidate: DevelopmentWorkerCandidate; - readonly publish: () => Promise; - }): Promise; - setControlHandler(handler: (request: Request) => Promise): void; + readonly slot: DevelopmentWorkerSlot; } export function createDevelopmentWorkerServer(input: { @@ -116,26 +42,34 @@ export function createDevelopmentWorkerServer(input: { readonly resolveAdmissionGeneration: ( workerGeneration: DevelopmentWorkerGeneration, ) => DevelopmentWorkerGeneration; + readonly workflowWorld: DevelopmentWorkflowWorldOwnership; }): DevelopmentWorkerServer { return new ParentDevelopmentWorkerServer(input); } class ParentDevelopmentWorkerServer implements DevelopmentWorkerServer { - readonly #activeExchanges = new Set(); readonly #activeWaiters = new Set<() => void>(); readonly #appRoot: string; readonly #createRunner: DevelopmentWorkerRunnerFactory; - readonly #listeners = new Set(); + readonly #generationRegistry = new Map(); + readonly #generationSlots = new Map(); + readonly #httpServer: DevelopmentWorkerHttpServer; + readonly #leasedGenerations = new Map(); + readonly #parentOwnsWorkflowWorld: boolean; readonly #resolveAdmissionGeneration: ( workerGeneration: DevelopmentWorkerGeneration, ) => DevelopmentWorkerGeneration; - readonly #slots = new Set(); + readonly #slots = new Set(); readonly #transportSecret = randomBytes(32).toString("base64url"); + readonly #workflowDispatcher: DevelopmentWorkflowDispatcher; + readonly #workflowWorld: ParentDevelopmentWorkflowWorld; #accepting = true; - #activeSlot: WorkerSlot | undefined; + #activeSlot: DevelopmentWorkerSlot | undefined; #closePromise: Promise | undefined; #controlHandler: ((request: Request) => Promise) | undefined; #promotion: Promise = Promise.resolve(); + #prunePromise: Promise | undefined; + #pruneRequested = false; #workerCounter = 0; constructor(input: { @@ -144,10 +78,96 @@ class ParentDevelopmentWorkerServer implements DevelopmentWorkerServer { readonly resolveAdmissionGeneration: ( workerGeneration: DevelopmentWorkerGeneration, ) => DevelopmentWorkerGeneration; + readonly workflowWorld: DevelopmentWorkflowWorldOwnership; }) { this.#appRoot = input.appRoot; this.#createRunner = input.createRunner; + this.#parentOwnsWorkflowWorld = input.workflowWorld.kind === "parent-local"; this.#resolveAdmissionGeneration = input.resolveAdmissionGeneration; + this.#workflowDispatcher = new DevelopmentWorkflowDispatcher({ + admit: async (generationId) => { + const lease = await this.#admitWorkflowGeneration(generationId); + return { + generation: lease.generation, + release: lease.release, + runner: lease.slot.runner, + }; + }, + onRelease: () => this.#pruneInBackground(), + transportSecret: this.#transportSecret, + }); + this.#workflowWorld = this.#createWorkflowWorld(input.workflowWorld); + this.#httpServer = new DevelopmentWorkerHttpServer({ + admit: async () => { + const lease = await this.#admit(); + return { + generation: lease.generation, + release: lease.release, + runner: lease.slot.runner, + }; + }, + handleParentRequest: async (request) => { + const workflowResponse = await this.#workflowWorld.handleRequest(request); + return workflowResponse ?? (await this.#controlHandler?.(request)); + }, + transportSecret: this.#transportSecret, + }); + } + + #createWorkflowWorld( + ownership: DevelopmentWorkflowWorldOwnership, + ): ParentDevelopmentWorkflowWorld { + if (ownership.kind === "worker-configured") { + return createWorkerOwnedDevelopmentWorkflowWorld(); + } + return createParentDevelopmentWorkflowWorld({ + agentName: ownership.agentName, + appRoot: this.#appRoot, + dispatch: async (request, generationId) => + await this.#workflowDispatcher.dispatch(request, generationId), + hasGeneration: (generationId) => this.#hasGeneration(generationId), + resolveActiveGenerationId: () => this.#readActiveGeneration().id, + transportSecret: this.#transportSecret, + }); + } + + async startWorkflowWorld(): Promise { + const generations = await readActivatedDevelopmentRuntimeArtifactGenerations(this.#appRoot); + for (const generation of generations) { + this.#generationRegistry.set(generation.id, generation); + } + if (this.#parentOwnsWorkflowWorld) { + const references = await this.#workflowWorld.collectGenerationReferences(); + if (!references.protectAll) { + for (const generationId of references.generationIds) { + if (this.#generationSlots.has(generationId)) { + continue; + } + const generation = generations.find((candidate) => candidate.id === generationId); + if (generation === undefined) { + throw new Error( + `Workflow run references missing development generation "${generationId}".`, + ); + } + const worker = generation.worker; + if (!isRestorableDevelopmentWorker(worker, this.#appRoot)) { + throw new Error( + `Workflow run references development generation "${generationId}" without a restorable worker.`, + ); + } + const candidate = await this.prepareCandidate({ + dispose: async () => await rm(worker.workspaceRoot, { force: true, recursive: true }), + entry: worker.entry, + generation, + workerData: worker.workerData, + }); + candidate.slot.state = "retired"; + this.#recordGeneration(generation, candidate.slot); + } + } + } + await this.#workflowWorld.start(); + this.#pruneInBackground(); } setControlHandler(handler: (request: Request) => Promise): void { @@ -158,67 +178,7 @@ class ParentDevelopmentWorkerServer implements DevelopmentWorkerServer { if (!this.#accepting) { throw new Error("Development worker server is closed."); } - - const sockets = new Set(); - const server = createServer((request, response) => { - void this.#handleRequest(request, response); - }); - server.on("connection", (socket) => { - sockets.add(socket); - socket.once("close", () => sockets.delete(socket)); - }); - server.on("upgrade", (request, socket, head) => { - void this.#handleUpgrade(request, socket as Socket, head); - }); - - let url: string | undefined; - const ready = new Promise((resolve, reject) => { - const onError = (error: Error) => { - server.off("listening", onListening); - reject(error); - }; - const onListening = () => { - server.off("error", onError); - const address = server.address(); - if (address === null || typeof address === "string") { - reject(new Error("Development worker server did not expose a TCP address.")); - return; - } - const urlHost = input.hostname.includes(":") ? `[${input.hostname}]` : input.hostname; - url = `http://${urlHost}:${String(address.port)}/`; - resolve(); - }; - server.once("error", onError); - server.once("listening", onListening); - server.listen({ host: input.hostname, port: input.port }); - }); - let listenerClosePromise: Promise | undefined; - const beginClose = (): Promise => { - listenerClosePromise ??= closeServer(server).finally(() => { - this.#listeners.delete(listenerState); - }); - return listenerClosePromise; - }; - const destroySockets = () => { - for (const socket of sockets) { - socket.destroy(); - } - }; - const listenerState: ListenerState = { beginClose, destroySockets }; - this.#listeners.add(listenerState); - - return { - async close() { - const closed = beginClose(); - destroySockets(); - await closed; - }, - node: { server }, - ready: async () => await ready, - get url() { - return url; - }, - }; + return this.#httpServer.listen(input); } async prepareCandidate(input: { @@ -231,7 +191,7 @@ class ParentDevelopmentWorkerServer implements DevelopmentWorkerServer { throw new Error("Development worker server is closed."); } - let slot: WorkerSlot | undefined; + let slot: DevelopmentWorkerSlot | undefined; const workerNumber = this.#workerCounter++; let runner: DevelopmentWorkerRunner; try { @@ -288,16 +248,29 @@ class ParentDevelopmentWorkerServer implements DevelopmentWorkerServer { async promote(candidate: DevelopmentWorkerCandidate): Promise { const previousSlot = await this.#gateAdmission(async () => this.#swapCandidate(candidate)); + this.#recordGeneration(candidate.slot.generation, candidate.slot); await this.#closeRetiredSlotWithoutFailingCommit(previousSlot); + this.#pruneInBackground(); } - async publishRuntimeGeneration( - publish: () => Promise, - ): Promise { + async publishRuntimeGeneration(input: { + readonly generation: DevelopmentWorkerGeneration; + readonly publish: () => Promise; + }): Promise { await this.#gateAdmission(async () => { - const publication = await publish(); + const slot = this.#activeSlot; + if (slot === undefined) { + throw new Error("Development worker is unavailable for runtime publication."); + } + await copyDevelopmentRuntimeArtifactsWorker({ + sourceSnapshotRoot: slot.generation.snapshotRoot, + targetSnapshotRoot: input.generation.snapshotRoot, + }); + const publication = await input.publish(); publication.commit(); + this.#recordGeneration(input.generation, slot); }); + this.#pruneInBackground(); } async publishStructuralCandidate(input: { @@ -310,6 +283,7 @@ class ParentDevelopmentWorkerServer implements DevelopmentWorkerServer { try { const slot = this.#swapCandidate(input.candidate); publication.commit(); + this.#recordGeneration(input.candidate.slot.generation, input.candidate.slot); return slot; } catch (error) { try { @@ -325,6 +299,7 @@ class ParentDevelopmentWorkerServer implements DevelopmentWorkerServer { } }); await this.#closeRetiredSlotWithoutFailingCommit(previousSlot); + this.#pruneInBackground(); } async #gateAdmission(operation: () => Promise): Promise { @@ -351,7 +326,7 @@ class ParentDevelopmentWorkerServer implements DevelopmentWorkerServer { } } - #swapCandidate(candidate: DevelopmentWorkerCandidate): WorkerSlot | undefined { + #swapCandidate(candidate: DevelopmentWorkerCandidate): DevelopmentWorkerSlot | undefined { this.#validateCandidate(candidate); const previousSlot = this.#activeSlot; candidate.slot.state = "active"; @@ -381,130 +356,29 @@ class ParentDevelopmentWorkerServer implements DevelopmentWorkerServer { this.#accepting = false; this.#wakeActiveWaiters(); - const listeners = [...this.#listeners]; - const listenerClosePromises = listeners.map((listener) => listener.beginClose()); - - for (const exchange of this.#activeExchanges) { - exchange.cancel(); - } - - await Promise.all([...this.#slots].map(async (slot) => await this.#closeSlot(slot))); - - for (const listener of listeners) { - listener.destroySockets(); - } - - await Promise.all(listenerClosePromises); - } - - async #handleRequest(request: IncomingMessage, response: ServerResponse): Promise { - const requestAbort = new AbortController(); - const abortRequest = (cause?: unknown) => { - if (!requestAbort.signal.aborted) { - requestAbort.abort(cause); - } - }; - request.once("aborted", abortRequest); - request.once("error", abortRequest); - response.once("close", () => { - if (!response.writableEnded) { - abortRequest(); - } - }); - - let lease: RequestLease | undefined; - let exchange: ActiveExchange | undefined; - try { - const publicRequest = createPublicRequest(request, requestAbort.signal); - const controlResponse = await this.#controlHandler?.(publicRequest); - if (controlResponse !== undefined) { - await writeResponse(response, controlResponse, requestAbort.signal); - return; - } - - lease = await this.#admit(); - const workerRequest = createWorkerRequest({ - generation: lease.generation, - request: publicRequest, - secret: this.#transportSecret, - socket: request.socket, - }); - const admittedLease = lease; - exchange = { - cancel() { - abortRequest(new Error("Development worker request was cancelled.")); - if (!response.destroyed) { - response.destroy(); - } - admittedLease.release(); - }, - slot: lease.slot, - }; - this.#activeExchanges.add(exchange); - response.once("finish", admittedLease.release); - response.once("error", admittedLease.release); - response.once("close", admittedLease.release); - request.once("aborted", admittedLease.release); - - const workerResponse = await lease.slot.runner.fetch(workerRequest); - await writeResponse(response, workerResponse, requestAbort.signal); - } catch (error) { - if (!requestAbort.signal.aborted) { - writeRequestError(response, error); - } - } finally { - if (exchange !== undefined) { - this.#activeExchanges.delete(exchange); - } - lease?.release(); - request.off("aborted", abortRequest); - request.off("error", abortRequest); - } - } - - async #handleUpgrade(request: IncomingMessage, socket: Socket, head: Buffer): Promise { - let lease: RequestLease | undefined; - let exchange: ActiveExchange | undefined; - try { - lease = await this.#admit(); - stampWorkerUpgradeRequest({ - generation: lease.generation, - request, - secret: this.#transportSecret, - }); - const admittedLease = lease; - exchange = { - cancel() { - if (!socket.destroyed) { - socket.destroy(); + this.#workflowDispatcher.close(); + await this.#httpServer.close(async () => { + await this.#prunePromise; + const workflowReferences = this.#parentOwnsWorkflowWorld + ? await this.#workflowWorld.collectGenerationReferences() + : undefined; + await Promise.all([ + ...[...this.#slots].map(async (slot) => { + if ( + workflowReferences !== undefined && + (workflowReferences.protectAll || + this.#readSlotGenerationIds(slot).some((generationId) => + workflowReferences.generationIds.has(generationId), + )) + ) { + await this.#stopSlotPreservingWorkspace(slot); + return; } - admittedLease.release(); - }, - slot: lease.slot, - }; - this.#activeExchanges.add(exchange); - const release = () => { - admittedLease.release(); - if (exchange !== undefined) { - this.#activeExchanges.delete(exchange); - exchange = undefined; - } - }; - socket.once("close", release); - socket.once("error", release); - await lease.slot.runner.upgrade({ node: { head, req: request, socket } }); - } catch { - if (exchange !== undefined) { - exchange.cancel(); - } else { - // Failure before the exchange existed: no socket listener releases - // the lease, so release it here or the retired worker never closes. - lease?.release(); - } - if (!socket.destroyed) { - socket.destroy(); - } - } + await this.#closeSlot(slot); + }), + this.#workflowWorld.close(), + ]); + }); } async #admit(): Promise { @@ -520,7 +394,15 @@ class ParentDevelopmentWorkerServer implements DevelopmentWorkerServer { } const generation = this.#resolveAdmissionGeneration(slot.generation); + return this.#createLease(slot, generation); + } + + #createLease(slot: DevelopmentWorkerSlot, generation: DevelopmentWorkerGeneration): RequestLease { slot.leases += 1; + this.#leasedGenerations.set( + generation.id, + (this.#leasedGenerations.get(generation.id) ?? 0) + 1, + ); let released = false; return { generation, @@ -530,6 +412,12 @@ class ParentDevelopmentWorkerServer implements DevelopmentWorkerServer { } released = true; slot.leases -= 1; + const generationLeases = (this.#leasedGenerations.get(generation.id) ?? 1) - 1; + if (generationLeases === 0) { + this.#leasedGenerations.delete(generation.id); + } else { + this.#leasedGenerations.set(generation.id, generationLeases); + } void this.#closeRetiredSlot(slot).catch((error) => { console.error(`[eve:dev] failed to close retired worker: ${toErrorMessage(error)}`); }); @@ -538,22 +426,112 @@ class ParentDevelopmentWorkerServer implements DevelopmentWorkerServer { }; } - async #handleWorkerClose(slot: WorkerSlot, cause: unknown): Promise { - if (slot.state !== "active" || !this.#accepting || this.#activeSlot !== slot) { + async #admitWorkflowGeneration(generationId: string): Promise { + await this.#promotion; + const generation = this.#generationRegistry.get(generationId); + const slot = this.#generationSlots.get(generationId); + if ( + !this.#accepting || + slot === undefined || + (slot.state !== "active" && slot.state !== "retired") + ) { + throw new Error("Development worker is unavailable for Workflow delivery."); + } + if (generation === undefined) { + throw new Error(`Workflow run references missing development generation "${generationId}".`); + } + return this.#createLease(slot, generation); + } + + #recordGeneration(generation: DevelopmentWorkerGeneration, slot: DevelopmentWorkerSlot): void { + this.#generationRegistry.set(generation.id, generation); + this.#generationSlots.set(generation.id, slot); + } + + #readActiveGeneration(): DevelopmentWorkerGeneration { + const slot = this.#activeSlot; + if (slot === undefined) { + throw new Error("Development runtime generation is unavailable."); + } + return this.#resolveAdmissionGeneration(slot.generation); + } + + #hasGeneration(generationId: string): boolean { + const generation = this.#generationRegistry.get(generationId); + return generation !== undefined && existsSync(generation.runtimeAppRoot); + } + + #pruneInBackground(): void { + if (!this.#accepting) { + return; + } + this.#pruneRequested = true; + if (this.#prunePromise !== undefined) { + return; + } + this.#prunePromise = this.#runScheduledPruning().finally(() => { + this.#prunePromise = undefined; + if (this.#pruneRequested) { + this.#pruneInBackground(); + } + }); + } + + async #runScheduledPruning(): Promise { + while (this.#accepting && this.#pruneRequested) { + this.#pruneRequested = false; + await this.#prune().catch((error) => { + console.error(`[eve:dev] failed to prune runtime generations: ${toErrorMessage(error)}`); + }); + } + } + + async #prune(): Promise { + const workflowReferences = await this.#workflowWorld.collectGenerationReferences(); + await Promise.all( + [...this.#slots].map(async (slot) => await this.#closeRetiredSlot(slot, workflowReferences)), + ); + const protectedGenerationIds = new Set(this.#leasedGenerations.keys()); + for (const slot of this.#slots) { + protectedGenerationIds.add(slot.generation.id); + } + for (const generationId of workflowReferences.generationIds) { + protectedGenerationIds.add(generationId); + } + await pruneDevelopmentRuntimeArtifactsSnapshots({ + appRoot: this.#appRoot, + protectAll: workflowReferences.protectAll, + protectedGenerationIds, + }); + for (const [generationId, generation] of this.#generationRegistry) { + if (!protectedGenerationIds.has(generationId) && !existsSync(generation.runtimeAppRoot)) { + this.#generationRegistry.delete(generationId); + } + } + } + + async #handleWorkerClose(slot: DevelopmentWorkerSlot, cause: unknown): Promise { + const wasActive = slot.state === "active" && this.#activeSlot === slot; + const wasRetired = slot.state === "retired"; + if ((!wasActive && !wasRetired) || !this.#accepting) { return; } + const generationIds = this.#readSlotGenerationIds(slot); slot.state = "closed"; this.#slots.delete(slot); - this.#activeSlot = undefined; - for (const exchange of this.#activeExchanges) { - if (exchange.slot === slot) { - exchange.cancel(); - this.#activeExchanges.delete(exchange); - } + this.#removeSlotGenerationMappings(slot); + if (wasActive) { + this.#activeSlot = undefined; } + this.#workflowDispatcher.cancelRunnerExchanges(slot.runner); + this.#httpServer.cancelRunnerExchanges(slot.runner); try { + if (wasRetired && !(await this.#isSlotProtected(generationIds))) { + await this.#disposeSlot(slot); + return; + } slot.disposed = true; const candidate = await this.prepareCandidate({ dispose: slot.dispose, @@ -561,7 +539,17 @@ class ParentDevelopmentWorkerServer implements DevelopmentWorkerServer { generation: slot.generation, workerData: slot.workerData, }); - await this.promote(candidate); + if (wasActive) { + await this.promote(candidate); + } else { + candidate.slot.state = "retired"; + } + for (const generationId of generationIds) { + const generation = this.#generationRegistry.get(generationId); + if (generation !== undefined) { + this.#generationSlots.set(generationId, candidate.slot); + } + } } catch (error) { console.error( `[eve:dev] worker restart failed after ${toErrorMessage(cause)}: ${toErrorMessage(error)}`, @@ -569,18 +557,55 @@ class ParentDevelopmentWorkerServer implements DevelopmentWorkerServer { } } - async #closeRetiredSlot(slot: WorkerSlot): Promise { - if (slot.state === "retired" && slot.leases === 0) { + async #closeRetiredSlot( + slot: DevelopmentWorkerSlot, + workflowReferences?: DevelopmentWorkflowGenerationReferences, + ): Promise { + if ( + slot.state === "retired" && + slot.leases === 0 && + !(await this.#isSlotProtected(this.#readSlotGenerationIds(slot), workflowReferences)) + ) { await this.#closeSlot(slot); } } - async #closeSlot(slot: WorkerSlot, cause?: unknown): Promise { + async #isSlotProtected( + generationIds: readonly string[], + workflowReferences?: DevelopmentWorkflowGenerationReferences, + ): Promise { + if (generationIds.some((generationId) => this.#leasedGenerations.has(generationId))) { + return true; + } + const references = + workflowReferences ?? (await this.#workflowWorld.collectGenerationReferences()); + return ( + references.protectAll || + generationIds.some((generationId) => references.generationIds.has(generationId)) + ); + } + + #readSlotGenerationIds(slot: DevelopmentWorkerSlot): readonly string[] { + return [...this.#generationSlots] + .filter(([, generationSlot]) => generationSlot === slot) + .map(([generationId]) => generationId); + } + + #removeSlotGenerationMappings(slot: DevelopmentWorkerSlot): void { + for (const [generationId, generationSlot] of this.#generationSlots) { + if (generationSlot === slot) { + this.#generationSlots.delete(generationId); + } + } + } + + async #closeSlot(slot: DevelopmentWorkerSlot, cause?: unknown): Promise { if (slot.state === "closed") { return; } slot.state = "closed"; this.#slots.delete(slot); + this.#removeSlotGenerationMappings(slot); if (this.#activeSlot === slot) { this.#activeSlot = undefined; } @@ -596,7 +621,20 @@ class ParentDevelopmentWorkerServer implements DevelopmentWorkerServer { } } - async #disposeSlot(slot: WorkerSlot): Promise { + async #stopSlotPreservingWorkspace(slot: DevelopmentWorkerSlot): Promise { + if (slot.state === "closed") { + return; + } + slot.state = "closed"; + this.#slots.delete(slot); + this.#removeSlotGenerationMappings(slot); + if (this.#activeSlot === slot) { + this.#activeSlot = undefined; + } + await slot.runner.close(); + } + + async #disposeSlot(slot: DevelopmentWorkerSlot): Promise { if (slot.disposed) { return; } @@ -604,7 +642,9 @@ class ParentDevelopmentWorkerServer implements DevelopmentWorkerServer { await slot.dispose(); } - async #closeRetiredSlotWithoutFailingCommit(slot: WorkerSlot | undefined): Promise { + async #closeRetiredSlotWithoutFailingCommit( + slot: DevelopmentWorkerSlot | undefined, + ): Promise { if (slot === undefined) { return; } @@ -620,3 +660,16 @@ class ParentDevelopmentWorkerServer implements DevelopmentWorkerServer { this.#activeWaiters.clear(); } } + +function createWorkerOwnedDevelopmentWorkflowWorld(): ParentDevelopmentWorkflowWorld { + return { + async close() {}, + async collectGenerationReferences() { + return { generationIds: new Set(), protectAll: true }; + }, + async handleRequest() { + return undefined; + }, + async start() {}, + }; +} diff --git a/packages/eve/src/internal/nitro/host/development-worker-recovery.ts b/packages/eve/src/internal/nitro/host/development-worker-recovery.ts new file mode 100644 index 000000000..fc5f1e979 --- /dev/null +++ b/packages/eve/src/internal/nitro/host/development-worker-recovery.ts @@ -0,0 +1,25 @@ +import { existsSync } from "node:fs"; +import { isAbsolute, join, relative, sep } from "node:path"; + +import type { DevelopmentRuntimeArtifactsWorker } from "#internal/nitro/dev-runtime-worker-artifacts.js"; + +export function isRestorableDevelopmentWorker( + worker: DevelopmentRuntimeArtifactsWorker, + appRoot: string, +): boolean { + const hostRoot = join(appRoot, ".eve", "dev-hosts"); + return ( + isPathInside(worker.workspaceRoot, hostRoot) && + isPathInside(worker.entry, worker.workspaceRoot) && + existsSync(worker.entry) + ); +} + +function isPathInside(path: string, parent: string): boolean { + const pathRelativeToParent = relative(parent, path); + return ( + pathRelativeToParent !== ".." && + !pathRelativeToParent.startsWith(`..${sep}`) && + !isAbsolute(pathRelativeToParent) + ); +} diff --git a/packages/eve/src/internal/nitro/host/development-workflow-dispatch.ts b/packages/eve/src/internal/nitro/host/development-workflow-dispatch.ts new file mode 100644 index 000000000..a71aa5e06 --- /dev/null +++ b/packages/eve/src/internal/nitro/host/development-workflow-dispatch.ts @@ -0,0 +1,89 @@ +import { createWorkflowWorkerRequest } from "#internal/nitro/host/dev-worker-http.js"; +import type { + DevelopmentWorkerGeneration, + DevelopmentWorkerRunner, +} from "#internal/nitro/host/dev-worker-server-types.js"; +import { wrapDevelopmentWorkflowResponse } from "#internal/nitro/host/development-workflow-response.js"; + +interface DevelopmentWorkflowExchange { + cancel(): void; + readonly runner: DevelopmentWorkerRunner; +} + +export interface DevelopmentWorkflowLease { + readonly generation: DevelopmentWorkerGeneration; + release(): void; + readonly runner: DevelopmentWorkerRunner; +} + +export class DevelopmentWorkflowDispatcher { + readonly #activeExchanges = new Set(); + readonly #admit: (generationId: string) => Promise; + readonly #onRelease: () => void; + readonly #transportSecret: string; + + constructor(input: { + readonly admit: (generationId: string) => Promise; + readonly onRelease: () => void; + readonly transportSecret: string; + }) { + this.#admit = input.admit; + this.#onRelease = input.onRelease; + this.#transportSecret = input.transportSecret; + } + + async dispatch(request: Request, generationId: string): Promise { + const lease = await this.#admit(generationId); + let released = false; + const release = () => { + if (released) { + return; + } + released = true; + lease.release(); + this.#onRelease(); + }; + const cancellation = new AbortController(); + const signal = AbortSignal.any([request.signal, cancellation.signal]); + const workerRequest = createWorkflowWorkerRequest({ + generation: lease.generation, + request: new Request(request, { signal }), + secret: this.#transportSecret, + }); + const exchange: DevelopmentWorkflowExchange = { + cancel() { + cancellation.abort(new Error("Development Workflow delivery was cancelled.")); + release(); + }, + runner: lease.runner, + }; + this.#activeExchanges.add(exchange); + try { + const response = await lease.runner.fetch(workerRequest); + return wrapDevelopmentWorkflowResponse(response, () => { + this.#activeExchanges.delete(exchange); + release(); + }); + } catch (error) { + this.#activeExchanges.delete(exchange); + release(); + throw error; + } + } + + cancelRunnerExchanges(runner: DevelopmentWorkerRunner): void { + for (const exchange of this.#activeExchanges) { + if (exchange.runner === runner) { + exchange.cancel(); + this.#activeExchanges.delete(exchange); + } + } + } + + close(): void { + for (const exchange of this.#activeExchanges) { + exchange.cancel(); + } + this.#activeExchanges.clear(); + } +} diff --git a/packages/eve/src/internal/nitro/host/development-workflow-response.ts b/packages/eve/src/internal/nitro/host/development-workflow-response.ts new file mode 100644 index 000000000..61f00abef --- /dev/null +++ b/packages/eve/src/internal/nitro/host/development-workflow-response.ts @@ -0,0 +1,35 @@ +export function wrapDevelopmentWorkflowResponse(response: Response, release: () => void): Response { + if (response.body === null) { + release(); + return response; + } + const reader = response.body.getReader(); + const body = new ReadableStream({ + async pull(controller) { + try { + const result = await reader.read(); + if (result.done) { + release(); + controller.close(); + return; + } + controller.enqueue(result.value); + } catch (error) { + release(); + controller.error(error); + } + }, + async cancel(reason) { + try { + await reader.cancel(reason); + } finally { + release(); + } + }, + }); + return new Response(body, { + headers: response.headers, + status: response.status, + statusText: response.statusText, + }); +} diff --git a/packages/eve/src/internal/nitro/host/nitro-development-worker-server.test.ts b/packages/eve/src/internal/nitro/host/nitro-development-worker-server.test.ts index fbf58025a..1e42aef6a 100644 --- a/packages/eve/src/internal/nitro/host/nitro-development-worker-server.test.ts +++ b/packages/eve/src/internal/nitro/host/nitro-development-worker-server.test.ts @@ -11,9 +11,20 @@ const mocks = vi.hoisted(() => ({ publishRuntimeGeneration: vi.fn(async () => undefined), publishStructuralCandidate: vi.fn(async () => undefined), setControlHandler: vi.fn(), + startWorkflowWorld: vi.fn(async () => undefined), }, + writeWorker: vi.fn(async () => undefined), })); +vi.mock("#internal/nitro/dev-runtime-worker-artifacts.js", async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + writeDevelopmentRuntimeArtifactsWorker: mocks.writeWorker, + }; +}); + vi.mock("#internal/nitro/host/dev-worker-server.js", async (importOriginal) => { const actual = await importOriginal(); return { @@ -63,12 +74,19 @@ describe("NitroDevelopmentWorkerServer", () => { }; const { NitroDevelopmentWorkerServer } = await import("#internal/nitro/host/nitro-development-worker-server.js"); - const server = new NitroDevelopmentWorkerServer({ appRoot: "/tmp/app" }); + const server = new NitroDevelopmentWorkerServer({ + appRoot: "/tmp/app", + workflowWorld: { agentName: "worker-server-test", kind: "parent-local" }, + }); const buildError = new Error("Nitro build failed"); const result = server.buildCandidate({ dispose: async () => undefined, - generation: { id: "generation-a", runtimeAppRoot: "/tmp/generation-a" }, + generation: { + id: "generation-a", + runtimeAppRoot: "/tmp/generation-a/source/app", + snapshotRoot: "/tmp/generation-a", + }, nitro, trigger: async () => { const reload = hooks.get("dev:reload")?.({ @@ -79,6 +97,7 @@ describe("NitroDevelopmentWorkerServer", () => { candidateReady.resolve(candidate); await reload; }, + workspaceRoot: "/tmp/dev-host-a", }); await expect(result).rejects.toThrow("Nitro build failed"); @@ -99,7 +118,10 @@ describe("NitroDevelopmentWorkerServer", () => { }; const { NitroDevelopmentWorkerServer } = await import("#internal/nitro/host/nitro-development-worker-server.js"); - const server = new NitroDevelopmentWorkerServer({ appRoot: "/tmp/app" }); + const server = new NitroDevelopmentWorkerServer({ + appRoot: "/tmp/app", + workflowWorld: { agentName: "worker-server-test", kind: "parent-local" }, + }); const buildError = new Error("Nitro build failed"); const cleanupError = new Error("candidate cleanup failed"); @@ -107,11 +129,16 @@ describe("NitroDevelopmentWorkerServer", () => { dispose: async () => { throw cleanupError; }, - generation: { id: "generation-a", runtimeAppRoot: "/tmp/generation-a" }, + generation: { + id: "generation-a", + runtimeAppRoot: "/tmp/generation-a/source/app", + snapshotRoot: "/tmp/generation-a", + }, nitro, trigger: async () => { throw buildError; }, + workspaceRoot: "/tmp/dev-host-a", }); const error = await result.catch((cause: unknown) => cause); diff --git a/packages/eve/src/internal/nitro/host/nitro-development-worker-server.ts b/packages/eve/src/internal/nitro/host/nitro-development-worker-server.ts index 477b70d74..5c3413f68 100644 --- a/packages/eve/src/internal/nitro/host/nitro-development-worker-server.ts +++ b/packages/eve/src/internal/nitro/host/nitro-development-worker-server.ts @@ -2,15 +2,18 @@ import { basename, resolve } from "node:path"; import type { Nitro } from "nitro/types"; +import type { DevelopmentGeneration } from "#internal/nitro/development-generation.js"; import { readActiveDevelopmentRuntimeArtifactsSnapshot } from "#internal/nitro/dev-runtime-artifacts.js"; -import { - createDevelopmentWorkerServer, - type DevelopmentWorkerCandidate, - type DevelopmentWorkerGeneration, - type DevelopmentWorkerListener, - type DevelopmentWorkerPublication, - type DevelopmentWorkerServer, -} from "#internal/nitro/host/dev-worker-server.js"; +import { writeDevelopmentRuntimeArtifactsWorker } from "#internal/nitro/dev-runtime-worker-artifacts.js"; +import { createDevelopmentWorkerServer } from "#internal/nitro/host/dev-worker-server.js"; +import type { + DevelopmentWorkflowWorldOwnership, + DevelopmentWorkerCandidate, + DevelopmentWorkerGeneration, + DevelopmentWorkerListener, + DevelopmentWorkerPublication, + DevelopmentWorkerServer, +} from "#internal/nitro/host/dev-worker-server-types.js"; import { createNodeDevelopmentWorkerRunner } from "#internal/nitro/host/dev-worker-runner.js"; const NITRO_CANDIDATE_BUILD_TIMEOUT_MS = 60_000; @@ -26,7 +29,10 @@ export class NitroDevelopmentWorkerServer { readonly #appRoot: string; readonly #server: DevelopmentWorkerServer; - constructor(input: { readonly appRoot: string }) { + constructor(input: { + readonly appRoot: string; + readonly workflowWorld: DevelopmentWorkflowWorldOwnership; + }) { this.#appRoot = input.appRoot; this.#server = createDevelopmentWorkerServer({ appRoot: input.appRoot, @@ -38,6 +44,7 @@ export class NitroDevelopmentWorkerServer { } return generation; }, + workflowWorld: input.workflowWorld, }); } @@ -49,11 +56,16 @@ export class NitroDevelopmentWorkerServer { this.#server.setControlHandler(handler); } + async startWorkflowWorld(): Promise { + await this.#server.startWorkflowWorld(); + } + async buildCandidate(input: { readonly dispose: () => Promise; readonly generation: DevelopmentWorkerGeneration; readonly nitro: NitroCandidateHost; readonly trigger: () => Promise; + readonly workspaceRoot: string; }): Promise { let buildError: unknown; let candidatePromise: Promise | undefined; @@ -70,12 +82,22 @@ export class NitroDevelopmentWorkerServer { candidateSignal.resolve(); return; } - candidatePromise = this.#server.prepareCandidate({ - dispose, - entry: payload?.entry ?? resolveDefaultEntry(input.nitro), - generation: input.generation, - workerData: payload?.workerData ?? {}, - }); + const entry = payload?.entry ?? resolveDefaultEntry(input.nitro); + const workerData = payload?.workerData ?? {}; + candidatePromise = (async () => { + await writeDevelopmentRuntimeArtifactsWorker({ + entry, + snapshotRoot: input.generation.snapshotRoot, + workerData, + workspaceRoot: input.workspaceRoot, + }); + return await this.#server.prepareCandidate({ + dispose, + entry, + generation: input.generation, + workerData, + }); + })(); try { await candidatePromise; } catch (error) { @@ -130,10 +152,14 @@ export class NitroDevelopmentWorkerServer { await this.#server.promote(candidate); } - async publishRuntimeGeneration( - publish: () => Promise, - ): Promise { - await this.#server.publishRuntimeGeneration(publish); + async publishRuntimeGeneration(input: { + readonly generation: DevelopmentGeneration; + readonly publish: () => Promise; + }): Promise { + await this.#server.publishRuntimeGeneration({ + generation: toDevelopmentWorkerGeneration(input.generation), + publish: input.publish, + }); } async publishStructuralCandidate(input: { @@ -155,6 +181,7 @@ export function toDevelopmentWorkerGeneration(input: { return { id: basename(input.snapshotRoot), runtimeAppRoot: input.runtimeAppRoot, + snapshotRoot: input.snapshotRoot, }; } diff --git a/packages/eve/src/internal/nitro/host/start-development-server.test.ts b/packages/eve/src/internal/nitro/host/start-development-server.test.ts index ed031f61f..b572f17da 100644 --- a/packages/eve/src/internal/nitro/host/start-development-server.test.ts +++ b/packages/eve/src/internal/nitro/host/start-development-server.test.ts @@ -28,10 +28,12 @@ const mocks = vi.hoisted(() => { discardCandidate: vi.fn(async () => undefined), listen: vi.fn(() => listenerServer), promote: vi.fn(async () => undefined), - publishRuntimeGeneration: vi.fn(async (publish: () => Promise<{ commit(): void }>) => { - const publication = await publish(); - publication.commit(); - }), + publishRuntimeGeneration: vi.fn( + async (input: { readonly publish: () => Promise<{ commit(): void }> }) => { + const publication = await input.publish(); + publication.commit(); + }, + ), publishStructuralCandidate: vi.fn( async (input: { publish: () => Promise<{ commit(): void }> }) => { const publication = await input.publish(); @@ -39,7 +41,11 @@ const mocks = vi.hoisted(() => { }, ), setControlHandler: vi.fn(), + startWorkflowWorld: vi.fn(async () => undefined), }; + const nitroDevelopmentWorkerServer = vi.fn(function NitroDevelopmentWorkerServer() { + return devServer; + }); const files = new Map(); const devHandlers: Nitro["options"]["devHandlers"] = []; const nitro = { @@ -77,8 +83,12 @@ const mocks = vi.hoisted(() => { listenerServer, mkdir: vi.fn(async () => undefined), nitro, + nitroDevelopmentWorkerServer, prepareDevelopmentApplicationHost: vi.fn(async () => ({ appRoot: "/tmp/eve-test", + compileResult: { + manifest: { config: { name: "test-agent" } }, + }, generation: { fingerprint: "test", runtimeAppRoot: "/tmp/eve-test/.eve/dev-runtime/snapshots/test/source/app", @@ -150,9 +160,7 @@ vi.mock("node:fs/promises", () => ({ })); vi.mock("./nitro-development-worker-server.js", () => ({ - NitroDevelopmentWorkerServer: function NitroDevelopmentWorkerServer() { - return mocks.devServer; - }, + NitroDevelopmentWorkerServer: mocks.nitroDevelopmentWorkerServer, })); vi.mock("./dev-host-candidate.js", () => ({ @@ -383,6 +391,11 @@ describe("createDevelopmentServer", () => { }, }); expect(mocks.pruneLocalSandboxTemplatesInBackground).toHaveBeenCalledWith("/tmp/eve-test"); + expect(mocks.nitroDevelopmentWorkerServer).toHaveBeenCalledWith({ + appRoot: "/tmp/eve-test", + workflowWorld: { agentName: "test-agent", kind: "parent-local" }, + }); + expect(mocks.devServer.startWorkflowWorld).toHaveBeenCalledOnce(); expect(process.env.WORKFLOW_LOCAL_BASE_URL).toBe("http://127.0.0.1:42123"); expect(process.env.PORT).toBe("42123"); @@ -398,6 +411,31 @@ describe("createDevelopmentServer", () => { expect(process.env.EVE_DEVELOPMENT_SANDBOX_RUN_ID).toBeUndefined(); }); + it("keeps an explicitly configured Workflow World inside the worker", async () => { + const preparedHost = await mocks.prepareDevelopmentApplicationHost(); + mocks.prepareDevelopmentApplicationHost.mockClear(); + mocks.prepareDevelopmentApplicationHost.mockResolvedValueOnce({ + ...preparedHost, + compileResult: { + manifest: { + config: { + experimental: { workflow: { world: "@workflow/world-postgres" } }, + name: "test-agent", + }, + }, + }, + } as never); + const startDevelopmentServer = await loadStartDevelopmentServer(); + + const server = await startDevelopmentServer("/tmp/eve-test"); + + expect(mocks.nitroDevelopmentWorkerServer).toHaveBeenCalledWith({ + appRoot: "/tmp/eve-test", + workflowWorld: { kind: "worker-configured" }, + }); + await server.close(); + }); + it("uses Eve's default port when no port is requested", async () => { const startDevelopmentServer = await loadStartDevelopmentServer(); Object.assign(mocks.nitro.options.devServer, { diff --git a/packages/eve/src/internal/nitro/host/start-development-server.ts b/packages/eve/src/internal/nitro/host/start-development-server.ts index 8719c406d..84620df0f 100644 --- a/packages/eve/src/internal/nitro/host/start-development-server.ts +++ b/packages/eve/src/internal/nitro/host/start-development-server.ts @@ -47,6 +47,7 @@ import { detectPackageManager, type PackageManagerKind } from "#setup/package-ma import { eveDevArguments } from "#setup/primitives/index.js"; import { devBootPhase } from "#internal/dev-boot-progress.js"; import { NitroDevelopmentWorkerServer } from "#internal/nitro/host/nitro-development-worker-server.js"; +import type { DevelopmentWorkflowWorldOwnership } from "#internal/nitro/host/dev-worker-server-types.js"; import { activateDevelopmentGenerationTransaction, discardDevelopmentGeneration, @@ -64,6 +65,18 @@ const IPV6_LOOPBACK_HOSTNAME = "[::1]"; */ const IPV6_WILDCARD_LISTEN_HOSTNAMES: ReadonlySet = new Set(["[::]", "::"]); +function resolveDevelopmentWorkflowWorldOwnership( + preparedHost: PreparedDevelopmentApplicationHost, +): DevelopmentWorkflowWorldOwnership { + if (preparedHost.compileResult.manifest.config.experimental?.workflow?.world !== undefined) { + return { kind: "worker-configured" }; + } + return { + agentName: preparedHost.compileResult.manifest.config.name, + kind: "parent-local", + }; +} + /** * Rewrites a server URL whose hostname is a wildcard listen address into a * loopback URL on the same address family. @@ -434,7 +447,10 @@ async function startNitroDevelopmentServer( options.onBootProgress, ); nitro = activeNitro; - devServer = new NitroDevelopmentWorkerServer({ appRoot: preparedHost.appRoot }); + devServer = new NitroDevelopmentWorkerServer({ + appRoot: preparedHost.appRoot, + workflowWorld: resolveDevelopmentWorkflowWorldOwnership(preparedHost), + }); const activeDevServer = devServer; const hostname = options.host ?? activeNitro.options.devServer.hostname ?? DEFAULT_DEVELOPMENT_SERVER_HOST; @@ -480,6 +496,7 @@ async function startNitroDevelopmentServer( }, options.onBootProgress, ); + await activeDevServer.startWorkflowWorld(); startDevelopmentSandboxPrewarmInBackground({ appRoot: preparedHost.appRoot, compiledArtifactsSource, diff --git a/packages/eve/src/internal/workflow/development-generation-context.ts b/packages/eve/src/internal/workflow/development-generation-context.ts new file mode 100644 index 000000000..d7da36b3a --- /dev/null +++ b/packages/eve/src/internal/workflow/development-generation-context.ts @@ -0,0 +1,25 @@ +import { ContextContainer, contextStorage } from "#context/container.js"; +import { ContextKey } from "#context/key.js"; +import type { RuntimeDiskCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js"; + +interface DevelopmentGenerationContext { + readonly generationId: string; + readonly source: RuntimeDiskCompiledArtifactsSource; +} + +const DevelopmentGenerationKey = new ContextKey( + "eve.internal.developmentGeneration", +); + +export function getDevelopmentWorkflowGeneration(): DevelopmentGenerationContext | undefined { + return contextStorage.getStore()?.get(DevelopmentGenerationKey); +} + +export async function withDevelopmentWorkflowGeneration( + context: DevelopmentGenerationContext, + operation: () => Promise, +): Promise { + const ctx = new ContextContainer(); + ctx.setVirtualContext(DevelopmentGenerationKey, context); + return await contextStorage.run(ctx, operation); +} diff --git a/packages/eve/src/internal/workflow/development-world-client.ts b/packages/eve/src/internal/workflow/development-world-client.ts new file mode 100644 index 000000000..b9b566bb9 --- /dev/null +++ b/packages/eve/src/internal/workflow/development-world-client.ts @@ -0,0 +1,218 @@ +import type { + MessageId, + QueuePrefix, + SpecVersion, + ValidQueueName, + World, +} from "#compiled/@workflow/world/index.js"; +import { resolvePackageSourceFilePath } from "#internal/application/package.js"; +import { + DEVELOPMENT_WORKER_APP_ROOT_ENV, + DEVELOPMENT_WORKER_TRANSPORT_SECRET_ENV, + readDevelopmentWorkerRequestMetadata, +} from "#internal/nitro/host/dev-worker-metadata.js"; +import { + decodeDevelopmentWorldJson, + decodeDevelopmentWorldValue, + deserializeDevelopmentWorldError, + encodeDevelopmentWorldValue, +} from "#internal/workflow/development-world-codec.js"; +import { + getDevelopmentWorkflowGeneration, + withDevelopmentWorkflowGeneration, +} from "#internal/workflow/development-generation-context.js"; +import { + DEVELOPMENT_WORKFLOW_DELIVERY_HEADER, + DEVELOPMENT_WORKFLOW_STREAM_ROUTE, + DEVELOPMENT_WORKFLOW_TRANSPORT_HEADER, + DEVELOPMENT_WORKFLOW_WORLD_ROUTE, + type DevelopmentWorldCall, + type DevelopmentWorldOperation, +} from "#internal/workflow/development-world-protocol.js"; +import { createDiskRuntimeCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js"; + +const WORKFLOW_LOCAL_BASE_URL_ENV = "WORKFLOW_LOCAL_BASE_URL"; + +export function createDevelopmentWorkflowWorld(): World { + const call = async ( + operation: DevelopmentWorldOperation, + args: readonly unknown[] = [], + ): Promise => { + const response = await fetchDevelopmentWorld(DEVELOPMENT_WORKFLOW_WORLD_ROUTE, { + body: encodeDevelopmentWorldValue({ + arguments: args, + operation, + } satisfies DevelopmentWorldCall), + method: "POST", + }); + return decodeDevelopmentWorldValue(await response.text()) as T; + }; + + const world = { + specVersion: 5 as SpecVersion, + processExitTriggersQueueRedelivery: false, + async getDeploymentId() { + return ( + getDevelopmentWorkflowGeneration()?.generationId ?? (await call("getDeploymentId")) + ); + }, + async resolveLatestDeploymentId() { + return await call("resolveLatestDeploymentId"); + }, + async queue(...args: Parameters) { + return await call>>("queue", args); + }, + createQueueHandler, + runs: { + get: async (...args: unknown[]) => await call("runs.get", args), + list: async (...args: unknown[]) => await call("runs.list", args), + experimentalSetAttributes: async (...args: unknown[]) => + await call("runs.experimentalSetAttributes", args), + } as World["runs"], + steps: { + get: async (...args: unknown[]) => await call("steps.get", args), + list: async (...args: unknown[]) => await call("steps.list", args), + } as World["steps"], + events: { + create: async (...args: unknown[]) => await call("events.create", args), + get: async (...args: unknown[]) => await call("events.get", args), + list: async (...args: unknown[]) => await call("events.list", args), + listByCorrelationId: async (...args: unknown[]) => + await call("events.listByCorrelationId", args), + } as World["events"], + hooks: { + get: async (...args: unknown[]) => await call("hooks.get", args), + getByToken: async (...args: unknown[]) => await call("hooks.getByToken", args), + list: async (...args: unknown[]) => await call("hooks.list", args), + } as World["hooks"], + streams: { + write: async (...args: unknown[]) => await call("streams.write", args), + writeMulti: async (...args: unknown[]) => await call("streams.writeMulti", args), + close: async (...args: unknown[]) => await call("streams.close", args), + get: async (runId: string, name: string, startIndex?: number) => { + const url = new URL(resolveDevelopmentWorldBaseUrl()); + url.pathname = DEVELOPMENT_WORKFLOW_STREAM_ROUTE; + url.searchParams.set("runId", runId); + url.searchParams.set("name", name); + if (startIndex !== undefined) { + url.searchParams.set("startIndex", String(startIndex)); + } + const response = await fetchDevelopmentWorld(url, { method: "GET" }); + if (response.body === null) { + throw new Error("Development Workflow stream response had no body."); + } + return response.body; + }, + list: async (...args: unknown[]) => await call("streams.list", args), + getChunks: async (...args: unknown[]) => await call("streams.getChunks", args), + getInfo: async (...args: unknown[]) => await call("streams.getInfo", args), + } as World["streams"], + async start() {}, + async close() {}, + } satisfies World; + + return world; +} + +function createQueueHandler( + prefix: QueuePrefix, + handler: ( + message: unknown, + metadata: { + attempt: number; + queueName: ValidQueueName; + messageId: MessageId; + requestId?: string; + }, + ) => Promise, +): (request: Request) => Promise { + return async (request) => { + const secret = readRequiredEnvironment(DEVELOPMENT_WORKER_TRANSPORT_SECRET_ENV); + if (request.headers.get(DEVELOPMENT_WORKFLOW_DELIVERY_HEADER) !== secret) { + return Response.json({ error: "Workflow delivery is not trusted." }, { status: 401 }); + } + const queueName = request.headers.get("x-vqs-queue-name"); + const messageId = request.headers.get("x-vqs-message-id"); + const attempt = Number(request.headers.get("x-vqs-message-attempt")); + if ( + queueName === null || + !queueName.startsWith(prefix) || + messageId === null || + !Number.isInteger(attempt) || + attempt < 1 || + request.body === null + ) { + return Response.json({ error: "Workflow delivery is malformed." }, { status: 400 }); + } + const metadata = readDevelopmentWorkerRequestMetadata(request); + if (metadata === undefined) { + return Response.json({ error: "Workflow delivery has no generation." }, { status: 400 }); + } + const message = decodeDevelopmentWorldJson(await request.text()); + try { + const result = await withDevelopmentWorkflowGeneration( + { + generationId: metadata.generationId, + source: createDiskRuntimeCompiledArtifactsSource(metadata.runtimeAppRoot, { + moduleMapLoaderPath: resolvePackageSourceFilePath( + "src/internal/authored-module-map-loader.ts", + ), + sandboxAppRoot: readRequiredEnvironment(DEVELOPMENT_WORKER_APP_ROOT_ENV), + }), + }, + async () => + await handler(message, { + attempt, + messageId: messageId as MessageId, + queueName: queueName as ValidQueueName, + }), + ); + return Response.json( + result === undefined ? { ok: true } : { timeoutSeconds: result.timeoutSeconds }, + ); + } catch (error) { + return Response.json(String(error), { status: 500 }); + } + }; +} + +async function fetchDevelopmentWorld(route: string | URL, init: RequestInit): Promise { + const url = route instanceof URL ? route : new URL(route, resolveDevelopmentWorldBaseUrl()); + const headers = new Headers(init.headers); + headers.set( + DEVELOPMENT_WORKFLOW_TRANSPORT_HEADER, + readRequiredEnvironment(DEVELOPMENT_WORKER_TRANSPORT_SECRET_ENV), + ); + const response = await fetch(url, { ...init, headers }); + if (!response.ok) { + const source = await response.text(); + const error = readDevelopmentWorldError(source); + if (error !== undefined) { + throw error; + } + throw new Error( + `Development Workflow World request failed (${String(response.status)}): ${source}`, + ); + } + return response; +} + +function readDevelopmentWorldError(source: string): Error | undefined { + try { + return deserializeDevelopmentWorldError(decodeDevelopmentWorldValue(source)); + } catch { + return undefined; + } +} + +function resolveDevelopmentWorldBaseUrl(): string { + return readRequiredEnvironment(WORKFLOW_LOCAL_BASE_URL_ENV); +} + +function readRequiredEnvironment(name: string): string { + const value = process.env[name]; + if (value === undefined || value.length === 0) { + throw new Error(`Development Workflow transport is missing ${name}.`); + } + return value; +} diff --git a/packages/eve/src/internal/workflow/development-world-codec.test.ts b/packages/eve/src/internal/workflow/development-world-codec.test.ts new file mode 100644 index 000000000..89282cb85 --- /dev/null +++ b/packages/eve/src/internal/workflow/development-world-codec.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; + +import { + decodeDevelopmentWorldValue, + deserializeDevelopmentWorldError, + encodeDevelopmentWorldValue, + serializeDevelopmentWorldError, +} from "#internal/workflow/development-world-codec.js"; + +describe("development Workflow World codec", () => { + it("preserves dates, bytes, and undefined values across the private transport", () => { + const createdAt = new Date("2026-07-13T12:00:00.000Z"); + + expect( + decodeDevelopmentWorldValue( + encodeDevelopmentWorldValue({ + createdAt, + input: new Uint8Array([1, 2, 3]), + output: undefined, + }), + ), + ).toEqual({ createdAt, input: new Uint8Array([1, 2, 3]), output: undefined }); + }); + + it("preserves World error identity and retry metadata", () => { + const source = Object.assign(new Error("try later"), { + name: "WorkflowWorldError", + retryAfter: 3, + }); + + const error = deserializeDevelopmentWorldError(serializeDevelopmentWorldError(source)); + + expect(error).toMatchObject({ + message: "try later", + name: "WorkflowWorldError", + retryAfter: 3, + }); + }); +}); diff --git a/packages/eve/src/internal/workflow/development-world-codec.ts b/packages/eve/src/internal/workflow/development-world-codec.ts new file mode 100644 index 000000000..054386405 --- /dev/null +++ b/packages/eve/src/internal/workflow/development-world-codec.ts @@ -0,0 +1,108 @@ +const VALUE_TYPE_KEY = "__eveDevelopmentWorldType"; +const DATE_MARKER = "date"; +const UINT8_ARRAY_MARKER = "uint8-array"; +const UNDEFINED_MARKER = "undefined"; + +interface EncodedValue { + readonly [VALUE_TYPE_KEY]: string; + readonly data?: string; +} + +export interface SerializedDevelopmentWorldError { + readonly details: Readonly>; + readonly message: string; + readonly name: string; + readonly stack?: string; +} + +export function encodeDevelopmentWorldValue(value: unknown): string { + return JSON.stringify({ value: encodeValue(value) }); +} + +export function decodeDevelopmentWorldValue(source: string): unknown { + const decoded = decodeDevelopmentWorldJson(source) as { readonly value?: unknown }; + return decoded.value; +} + +export function decodeDevelopmentWorldJson(source: string): unknown { + return decodeValue(JSON.parse(source) as unknown); +} + +export function serializeDevelopmentWorldError(error: unknown): SerializedDevelopmentWorldError { + if (!(error instanceof Error)) { + return { details: {}, message: String(error), name: "Error" }; + } + return { + details: Object.fromEntries(Object.entries(error)), + message: error.message, + name: error.name, + stack: error.stack, + }; +} + +export function deserializeDevelopmentWorldError(value: unknown): Error | undefined { + if (!isRecord(value) || !isRecord(value.details)) { + return undefined; + } + if (typeof value.message !== "string" || typeof value.name !== "string") { + return undefined; + } + const error = new Error(value.message); + error.name = value.name; + if (typeof value.stack === "string") { + error.stack = value.stack; + } + Object.assign(error, value.details); + return error; +} + +function encodeValue(value: unknown): unknown { + if (value === undefined) { + return { [VALUE_TYPE_KEY]: UNDEFINED_MARKER } satisfies EncodedValue; + } + if (value instanceof Date) { + return { + [VALUE_TYPE_KEY]: DATE_MARKER, + data: value.toISOString(), + } satisfies EncodedValue; + } + if (value instanceof Uint8Array) { + return { + [VALUE_TYPE_KEY]: UINT8_ARRAY_MARKER, + data: Buffer.from(value).toString("base64"), + } satisfies EncodedValue; + } + if (Array.isArray(value)) { + return value.map((item) => encodeValue(item)); + } + if (isRecord(value)) { + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, encodeValue(item)])); + } + return value; +} + +function decodeValue(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map((item) => decodeValue(item)); + } + if (!isRecord(value)) { + return value; + } + if (value[VALUE_TYPE_KEY] === UNDEFINED_MARKER) { + return undefined; + } + if (value[VALUE_TYPE_KEY] === DATE_MARKER && typeof value.data === "string") { + return new Date(value.data); + } + if ( + (value[VALUE_TYPE_KEY] === UINT8_ARRAY_MARKER || value.__type === "Uint8Array") && + typeof value.data === "string" + ) { + return Uint8Array.from(Buffer.from(value.data, "base64")); + } + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, decodeValue(item)])); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/eve/src/internal/workflow/development-world-protocol.ts b/packages/eve/src/internal/workflow/development-world-protocol.ts new file mode 100644 index 000000000..0bf8d3885 --- /dev/null +++ b/packages/eve/src/internal/workflow/development-world-protocol.ts @@ -0,0 +1,32 @@ +export const DEVELOPMENT_WORKFLOW_WORLD_ROUTE = "/eve/v1/dev/internal/workflow-world"; +export const DEVELOPMENT_WORKFLOW_STREAM_ROUTE = `${DEVELOPMENT_WORKFLOW_WORLD_ROUTE}/stream`; +export const DEVELOPMENT_WORKFLOW_TRANSPORT_HEADER = "x-eve-dev-workflow-transport"; +export const DEVELOPMENT_WORKFLOW_DELIVERY_HEADER = "x-eve-dev-workflow-delivery"; + +export type DevelopmentWorldOperation = + | "events.create" + | "events.get" + | "events.list" + | "events.listByCorrelationId" + | "hooks.get" + | "hooks.getByToken" + | "hooks.list" + | "getDeploymentId" + | "queue" + | "resolveLatestDeploymentId" + | "runs.experimentalSetAttributes" + | "runs.get" + | "runs.list" + | "steps.get" + | "steps.list" + | "streams.close" + | "streams.getChunks" + | "streams.getInfo" + | "streams.list" + | "streams.write" + | "streams.writeMulti"; + +export interface DevelopmentWorldCall { + readonly arguments: readonly unknown[]; + readonly operation: DevelopmentWorldOperation; +} diff --git a/packages/eve/src/internal/workflow/development-world-server.integration.test.ts b/packages/eve/src/internal/workflow/development-world-server.integration.test.ts new file mode 100644 index 000000000..f780e40ac --- /dev/null +++ b/packages/eve/src/internal/workflow/development-world-server.integration.test.ts @@ -0,0 +1,246 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +import { turnWorkflowReference, workflowEntryReference } from "#execution/workflow-runtime.js"; +import { useTemporaryDirectories } from "#internal/testing/use-temporary-app-roots.js"; +import { + decodeDevelopmentWorldValue, + encodeDevelopmentWorldValue, +} from "#internal/workflow/development-world-codec.js"; +import { + createParentDevelopmentWorkflowWorld, + type ParentDevelopmentWorkflowWorld, +} from "#internal/workflow/development-world-server.js"; +import { + DEVELOPMENT_WORKFLOW_TRANSPORT_HEADER, + DEVELOPMENT_WORKFLOW_WORLD_ROUTE, +} from "#internal/workflow/development-world-protocol.js"; + +const createScratchDirectory = useTemporaryDirectories(); +const SECRET = "workflow-transport-secret"; +const RUN_ID = "wrun_01J00000000000000000000000"; + +describe("parent development Workflow World", () => { + it("pins a child run to its recorded generation until it becomes terminal", async () => { + const appRoot = await createScratchDirectory("eve-parent-workflow-world-"); + await mkdir(join(appRoot, ".workflow-data"), { recursive: true }); + let activeGenerationId = "generation-a"; + const dispatchedGenerations: string[] = []; + const world = createWorld({ + activeGenerationId: () => activeGenerationId, + appRoot, + dispatch: async (_request, generationId) => { + dispatchedGenerations.push(generationId); + return Response.json({ ok: true }); + }, + hasGeneration: (generationId) => + generationId === "generation-a" || generationId === "generation-b", + }); + + try { + await world.start(); + const created = await callWorld(world, "events.create", [ + null, + { + eventData: { + deploymentId: "generation-a", + executionContext: {}, + input: new Uint8Array(), + workflowName: turnWorkflowReference.workflowId, + }, + eventType: "run_created", + specVersion: 5, + }, + ]); + const runId = readCreatedRunId(created); + + activeGenerationId = "generation-b"; + await deliver(world, { runId }); + expect(dispatchedGenerations).toEqual(["generation-a"]); + await expect(world.collectGenerationReferences()).resolves.toEqual({ + generationIds: new Set(["generation-a"]), + protectAll: false, + }); + + await callWorld(world, "events.create", [ + runId, + { + eventData: { result: new Uint8Array() }, + eventType: "run_completed", + specVersion: 5, + }, + ]); + await expect(world.collectGenerationReferences()).resolves.toEqual({ + generationIds: new Set(), + protectAll: false, + }); + } finally { + await world.close(); + } + }); + + it("routes the generation-neutral driver to active and rejects public queue spoofing", async () => { + const appRoot = await createScratchDirectory("eve-parent-workflow-routing-"); + const dispatch = vi.fn(async () => Response.json({ ok: true })); + const world = createWorld({ + activeGenerationId: () => "generation-b", + appRoot, + dispatch, + hasGeneration: () => true, + }); + + try { + await world.start(); + await deliver(world, { + runId: RUN_ID, + runInput: { + deploymentId: "generation-a", + input: new Uint8Array(), + specVersion: 5, + workflowName: workflowEntryReference.workflowId, + }, + }); + expect(dispatch).toHaveBeenCalledWith(expect.any(Request), "generation-b"); + + const response = await world.handleRequest( + new Request("http://localhost/.well-known/workflow/v1/flow", { + body: JSON.stringify({ + runId: RUN_ID, + runInput: { + deploymentId: "generation-a", + workflowName: turnWorkflowReference.workflowId, + }, + }), + method: "POST", + }), + ); + expect(response?.status).toBe(401); + expect(dispatch).toHaveBeenCalledTimes(1); + } finally { + await world.close(); + } + }); + + it("fails startup when a nonterminal turn references a missing generation", async () => { + const appRoot = await createScratchDirectory("eve-parent-workflow-missing-generation-"); + const first = createWorld({ + activeGenerationId: () => "generation-a", + appRoot, + dispatch: async () => Response.json({ ok: true }), + hasGeneration: () => true, + }); + await first.start(); + await callWorld(first, "events.create", [ + null, + { + eventData: { + deploymentId: "generation-a", + executionContext: {}, + input: new Uint8Array(), + workflowName: turnWorkflowReference.workflowId, + }, + eventType: "run_created", + specVersion: 5, + }, + ]); + await first.close(); + + const restarted = createWorld({ + activeGenerationId: () => "generation-b", + appRoot, + dispatch: async () => Response.json({ ok: true }), + hasGeneration: () => false, + }); + try { + await expect(restarted.start()).rejects.toThrow( + 'Workflow run references missing development generation "generation-a".', + ); + } finally { + await restarted.close(); + } + }); + + it("protects all generations when a persisted run record is unreadable", async () => { + const appRoot = await createScratchDirectory("eve-parent-workflow-unreadable-run-"); + await mkdir(join(appRoot, ".workflow-data", "runs"), { recursive: true }); + await writeFile(join(appRoot, ".workflow-data", "runs", "unreadable.json"), "{"); + const world = createWorld({ + activeGenerationId: () => "generation-a", + appRoot, + dispatch: async () => Response.json({ ok: true }), + hasGeneration: () => true, + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + try { + await expect(world.collectGenerationReferences()).resolves.toEqual({ + generationIds: new Set(), + protectAll: true, + }); + } finally { + warn.mockRestore(); + await world.close(); + } + }); +}); + +function readCreatedRunId(value: unknown): string { + if ( + typeof value === "object" && + value !== null && + "run" in value && + typeof value.run === "object" && + value.run !== null && + "runId" in value.run && + typeof value.run.runId === "string" + ) { + return value.run.runId; + } + throw new Error("Workflow World did not return the created run ID."); +} + +function createWorld(input: { + readonly activeGenerationId: () => string; + readonly appRoot: string; + readonly dispatch: (request: Request, generationId: string) => Promise; + readonly hasGeneration: (generationId: string) => boolean; +}): ParentDevelopmentWorkflowWorld { + return createParentDevelopmentWorkflowWorld({ + agentName: "workflow-world-test", + appRoot: input.appRoot, + dispatch: input.dispatch, + hasGeneration: input.hasGeneration, + resolveActiveGenerationId: input.activeGenerationId, + transportSecret: SECRET, + }); +} + +async function callWorld( + world: ParentDevelopmentWorkflowWorld, + operation: string, + args: readonly unknown[], +): Promise { + const response = await world.handleRequest( + new Request(`http://localhost${DEVELOPMENT_WORKFLOW_WORLD_ROUTE}`, { + body: encodeDevelopmentWorldValue({ arguments: args, operation }), + headers: { [DEVELOPMENT_WORKFLOW_TRANSPORT_HEADER]: SECRET }, + method: "POST", + }), + ); + expect(response).toBeDefined(); + const body = await response!.text(); + expect(response!.status, body).toBe(200); + return decodeDevelopmentWorldValue(body); +} + +async function deliver(world: ParentDevelopmentWorkflowWorld, payload: unknown): Promise { + const response = await world.handleRequest( + new Request("http://localhost/.well-known/workflow/v1/flow", { + body: JSON.stringify(payload), + headers: { [DEVELOPMENT_WORKFLOW_TRANSPORT_HEADER]: SECRET }, + method: "POST", + }), + ); + expect(response?.status).toBe(200); +} diff --git a/packages/eve/src/internal/workflow/development-world-server.ts b/packages/eve/src/internal/workflow/development-world-server.ts new file mode 100644 index 000000000..8fe2b8628 --- /dev/null +++ b/packages/eve/src/internal/workflow/development-world-server.ts @@ -0,0 +1,361 @@ +import { readdir } from "node:fs/promises"; +import { join } from "node:path"; + +import type { ValidQueueName, World } from "#compiled/@workflow/world/index.js"; +import { createWorld } from "#compiled/@workflow/world-local/index.js"; +import { turnWorkflowReference } from "#execution/workflow-runtime.js"; +import { deriveEveWorkflowQueuePrefix } from "#internal/workflow/queue-namespace.js"; +import { + decodeDevelopmentWorldJson, + decodeDevelopmentWorldValue, + encodeDevelopmentWorldValue, + serializeDevelopmentWorldError, +} from "#internal/workflow/development-world-codec.js"; +import { + DEVELOPMENT_WORKFLOW_STREAM_ROUTE, + DEVELOPMENT_WORKFLOW_TRANSPORT_HEADER, + DEVELOPMENT_WORKFLOW_WORLD_ROUTE, + type DevelopmentWorldCall, +} from "#internal/workflow/development-world-protocol.js"; + +const WORKFLOW_FLOW_ROUTE = "/.well-known/workflow/v1/flow"; +const WORKFLOW_STEP_ROUTE = "/.well-known/workflow/v1/step"; + +export interface DevelopmentWorkflowGenerationReferences { + readonly generationIds: ReadonlySet; + readonly protectAll: boolean; +} + +export interface ParentDevelopmentWorkflowWorld { + close(): Promise; + collectGenerationReferences(): Promise; + handleRequest(request: Request): Promise; + start(): Promise; +} + +export function createParentDevelopmentWorkflowWorld(input: { + readonly agentName: string; + readonly appRoot: string; + readonly dispatch: (request: Request, generationId: string) => Promise; + readonly hasGeneration: (generationId: string) => boolean; + readonly resolveActiveGenerationId: () => string; + readonly transportSecret: string; +}): ParentDevelopmentWorkflowWorld { + return new LocalParentDevelopmentWorkflowWorld(input); +} + +class LocalParentDevelopmentWorkflowWorld implements ParentDevelopmentWorkflowWorld { + readonly #agentName: string; + readonly #dataDir: string; + readonly #dispatch: (request: Request, generationId: string) => Promise; + readonly #hasGeneration: (generationId: string) => boolean; + readonly #resolveActiveGenerationId: () => string; + readonly #transportSecret: string; + readonly #world: World; + #closed = false; + #started = false; + + constructor(input: { + readonly agentName: string; + readonly appRoot: string; + readonly dispatch: (request: Request, generationId: string) => Promise; + readonly hasGeneration: (generationId: string) => boolean; + readonly resolveActiveGenerationId: () => string; + readonly transportSecret: string; + }) { + this.#agentName = input.agentName; + this.#dataDir = join(input.appRoot, ".workflow-data"); + this.#dispatch = input.dispatch; + this.#hasGeneration = input.hasGeneration; + this.#resolveActiveGenerationId = input.resolveActiveGenerationId; + this.#transportSecret = input.transportSecret; + this.#world = createWorld({ + dataDir: this.#dataDir, + recoverActiveRuns: false, + }); + } + + async start(): Promise { + if (this.#started) { + return; + } + const references = await this.collectGenerationReferences(); + if (!references.protectAll) { + for (const generationId of references.generationIds) { + this.#assertGenerationExists(generationId); + } + } + await this.#world.start?.(); + this.#started = true; + await reenqueueActiveDevelopmentRuns( + this.#world, + this.#queue.bind(this), + deriveEveWorkflowQueuePrefix(this.#agentName), + ); + } + + async close(): Promise { + if (this.#closed) { + return; + } + this.#closed = true; + this.#started = false; + await this.#world.close?.(); + } + + async handleRequest(request: Request): Promise { + const url = new URL(request.url); + if (url.pathname === DEVELOPMENT_WORKFLOW_WORLD_ROUTE) { + return await this.#handleCall(request); + } + if (url.pathname === DEVELOPMENT_WORKFLOW_STREAM_ROUTE) { + return await this.#handleStream(request, url); + } + if ( + request.method === "POST" && + (url.pathname === WORKFLOW_FLOW_ROUTE || url.pathname === WORKFLOW_STEP_ROUTE) + ) { + return await this.#handleDelivery(request); + } + return undefined; + } + + async collectGenerationReferences(): Promise { + const generationIds = new Set(); + try { + const runIds = new Set(); + let cursor: string | undefined; + do { + const page = await this.#world.runs.list({ + pagination: { cursor, limit: 1_000 }, + resolveData: "none", + }); + for (const run of page.data) { + runIds.add(run.runId); + if ( + (run.status === "pending" || run.status === "running") && + run.workflowName === turnWorkflowReference.workflowId + ) { + generationIds.add(run.deploymentId); + } + } + cursor = page.hasMore ? (page.cursor ?? undefined) : undefined; + } while (cursor !== undefined); + for (const persistedRunId of await this.#readPersistedRunIds()) { + if (!runIds.has(persistedRunId)) { + return { generationIds: new Set(), protectAll: true }; + } + } + return { generationIds, protectAll: false }; + } catch { + return { generationIds: new Set(), protectAll: true }; + } + } + + async #readPersistedRunIds(): Promise { + try { + const entries = await readdir(join(this.#dataDir, "runs"), { withFileTypes: true }); + return entries + .filter((entry) => entry.isFile() && entry.name.endsWith(".json")) + .map((entry) => entry.name.slice(0, -".json".length)); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") { + return []; + } + throw error; + } + } + + async #handleCall(request: Request): Promise { + if (!this.#isTrusted(request) || request.method !== "POST") { + return Response.json({ error: "Workflow World request is not trusted." }, { status: 401 }); + } + try { + const call = decodeDevelopmentWorldValue(await request.text()) as DevelopmentWorldCall; + const result = await this.#call(call); + return new Response(encodeDevelopmentWorldValue(result)); + } catch (error) { + return new Response(encodeDevelopmentWorldValue(serializeDevelopmentWorldError(error)), { + status: 500, + }); + } + } + + async #handleStream(request: Request, url: URL): Promise { + if (!this.#isTrusted(request) || request.method !== "GET") { + return Response.json({ error: "Workflow World request is not trusted." }, { status: 401 }); + } + const runId = url.searchParams.get("runId"); + const name = url.searchParams.get("name"); + const rawStartIndex = url.searchParams.get("startIndex"); + if (runId === null || name === null) { + return Response.json({ error: "Workflow stream request is malformed." }, { status: 400 }); + } + const startIndex = rawStartIndex === null ? undefined : Number(rawStartIndex); + if (startIndex !== undefined && !Number.isInteger(startIndex)) { + return Response.json({ error: "Workflow stream start index is invalid." }, { status: 400 }); + } + return new Response(await this.#world.streams.get(runId, name, startIndex)); + } + + async #handleDelivery(request: Request): Promise { + if (!this.#isTrusted(request)) { + return Response.json({ error: "Workflow delivery is not trusted." }, { status: 401 }); + } + try { + const payload = decodeDevelopmentWorldJson(await request.clone().text()); + const generationId = await this.#resolveDeliveryGeneration(payload); + this.#assertGenerationExists(generationId); + return await this.#dispatch(request, generationId); + } catch (error) { + return Response.json({ error: String(error) }, { status: 500 }); + } + } + + async #resolveDeliveryGeneration(payload: unknown): Promise { + if (!isObject(payload) || payload.__healthCheck === true) { + return this.#resolveActiveGenerationId(); + } + const initial = isObject(payload.runInput) ? payload.runInput : undefined; + if (initial !== undefined) { + return initial.workflowName === turnWorkflowReference.workflowId && + typeof initial.deploymentId === "string" + ? initial.deploymentId + : this.#resolveActiveGenerationId(); + } + const runId = + typeof payload.runId === "string" + ? payload.runId + : typeof payload.workflowRunId === "string" + ? payload.workflowRunId + : undefined; + if (runId === undefined) { + throw new Error("Workflow delivery does not identify a run."); + } + const run = await this.#world.runs.get(runId, { resolveData: "none" }); + return run.workflowName === turnWorkflowReference.workflowId + ? run.deploymentId + : this.#resolveActiveGenerationId(); + } + + async #queue(...args: Parameters): ReturnType { + const [queueName, message, options] = args; + return await this.#world.queue(queueName, message, { + ...options, + headers: { + ...options?.headers, + [DEVELOPMENT_WORKFLOW_TRANSPORT_HEADER]: this.#transportSecret, + }, + }); + } + + async #call(call: DevelopmentWorldCall): Promise { + if (!isDevelopmentWorldCall(call)) { + throw new Error("Development Workflow World call is malformed."); + } + const args = [...call.arguments]; + switch (call.operation) { + case "getDeploymentId": + case "resolveLatestDeploymentId": + return this.#resolveActiveGenerationId(); + case "queue": + return await this.#queue(...(args as Parameters)); + case "runs.get": + return await invoke(this.#world.runs.get, args, this.#world.runs); + case "runs.list": + return await invoke(this.#world.runs.list, args, this.#world.runs); + case "runs.experimentalSetAttributes": { + const operation = this.#world.runs.experimentalSetAttributes; + if (operation === undefined) { + return undefined; + } + return await invoke(operation, args, this.#world.runs); + } + case "steps.get": + return await invoke(this.#world.steps.get, args, this.#world.steps); + case "steps.list": + return await invoke(this.#world.steps.list, args, this.#world.steps); + case "events.create": + return await invoke(this.#world.events.create, args, this.#world.events); + case "events.get": + return await invoke(this.#world.events.get, args, this.#world.events); + case "events.list": + return await invoke(this.#world.events.list, args, this.#world.events); + case "events.listByCorrelationId": + return await invoke(this.#world.events.listByCorrelationId, args, this.#world.events); + case "hooks.get": + return await invoke(this.#world.hooks.get, args, this.#world.hooks); + case "hooks.getByToken": + return await invoke(this.#world.hooks.getByToken, args, this.#world.hooks); + case "hooks.list": + return await invoke(this.#world.hooks.list, args, this.#world.hooks); + case "streams.write": + return await invoke(this.#world.streams.write, args, this.#world.streams); + case "streams.writeMulti": { + const operation = this.#world.streams.writeMulti; + if (operation === undefined) { + for (const chunk of args[2] as readonly (string | Uint8Array)[]) { + await this.#world.streams.write(args[0] as string, args[1] as string, chunk); + } + return undefined; + } + return await invoke(operation, args, this.#world.streams); + } + case "streams.close": + return await invoke(this.#world.streams.close, args, this.#world.streams); + case "streams.list": + return await invoke(this.#world.streams.list, args, this.#world.streams); + case "streams.getChunks": + return await invoke(this.#world.streams.getChunks, args, this.#world.streams); + case "streams.getInfo": + return await invoke(this.#world.streams.getInfo, args, this.#world.streams); + } + } + + #isTrusted(request: Request): boolean { + return request.headers.get(DEVELOPMENT_WORKFLOW_TRANSPORT_HEADER) === this.#transportSecret; + } + + #assertGenerationExists(generationId: string): void { + if (!this.#hasGeneration(generationId)) { + throw new Error(`Workflow run references missing development generation "${generationId}".`); + } + } +} + +function invoke( + operation: (...args: never[]) => unknown, + args: readonly unknown[], + receiver: unknown, +): Promise | unknown { + return Reflect.apply(operation, receiver, args); +} + +function isDevelopmentWorldCall(value: unknown): value is DevelopmentWorldCall { + return isObject(value) && typeof value.operation === "string" && Array.isArray(value.arguments); +} + +function isObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +async function reenqueueActiveDevelopmentRuns( + world: World, + enqueue: World["queue"], + prefix: string, +): Promise { + for (const status of ["pending", "running"] as const) { + let cursor: string | undefined; + do { + const page = await world.runs.list({ + pagination: { cursor }, + resolveData: "none", + status, + }); + for (const run of page.data) { + await enqueue(`${prefix}${run.workflowName}` as ValidQueueName, { runId: run.runId }); + } + cursor = page.hasMore ? (page.cursor ?? undefined) : undefined; + } while (cursor !== undefined); + } +} diff --git a/packages/eve/src/runtime/durable-compiled-artifacts-source.test.ts b/packages/eve/src/runtime/durable-compiled-artifacts-source.test.ts new file mode 100644 index 000000000..425d49e2b --- /dev/null +++ b/packages/eve/src/runtime/durable-compiled-artifacts-source.test.ts @@ -0,0 +1,40 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { withDevelopmentWorkflowGeneration } from "#internal/workflow/development-generation-context.js"; +import { createDiskRuntimeCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js"; +import { + resolveDurableCompiledArtifactsSource, + serializeDurableCompiledArtifactsSource, +} from "#runtime/durable-compiled-artifacts-source.js"; + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe("durable compiled artifact sources", () => { + it("preserves deployed artifact sources", () => { + const source = createDiskRuntimeCompiledArtifactsSource("/var/task/.eve/compile"); + + expect(serializeDurableCompiledArtifactsSource(source)).toBe(source); + }); + + it("persists a logical selector and resolves it from the admitted child generation", async () => { + vi.stubEnv("EVE_DEV", "1"); + const original = createDiskRuntimeCompiledArtifactsSource( + "/app/.eve/dev-runtime/snapshots/generation-a/source/app", + ); + const admitted = createDiskRuntimeCompiledArtifactsSource( + "/app/.eve/dev-runtime/snapshots/generation-b/source/app", + ); + const durable = serializeDurableCompiledArtifactsSource(original); + + expect(durable).toEqual({ kind: "development" }); + expect(JSON.stringify(durable)).not.toContain("generation-a"); + await withDevelopmentWorkflowGeneration( + { generationId: "generation-b", source: admitted }, + async () => { + expect(resolveDurableCompiledArtifactsSource(durable)).toBe(admitted); + }, + ); + }); +}); diff --git a/packages/eve/src/runtime/durable-compiled-artifacts-source.ts b/packages/eve/src/runtime/durable-compiled-artifacts-source.ts new file mode 100644 index 000000000..aa5c4008b --- /dev/null +++ b/packages/eve/src/runtime/durable-compiled-artifacts-source.ts @@ -0,0 +1,35 @@ +import { getDevelopmentWorkflowGeneration } from "#internal/workflow/development-generation-context.js"; +import { isEveDevEnvironment } from "#internal/application/dev-environment.js"; +import type { RuntimeCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js"; + +interface DevelopmentCompiledArtifactsSelector { + readonly kind: "development"; +} + +export type DurableCompiledArtifactsSource = + | RuntimeCompiledArtifactsSource + | DevelopmentCompiledArtifactsSelector; + +export function serializeDurableCompiledArtifactsSource( + source: RuntimeCompiledArtifactsSource, +): DurableCompiledArtifactsSource { + if (source.kind === "disk" && isEveDevEnvironment()) { + return { kind: "development" }; + } + return source; +} + +export function resolveDurableCompiledArtifactsSource( + source: DurableCompiledArtifactsSource, +): RuntimeCompiledArtifactsSource { + if (source.kind !== "development") { + return source; + } + const generation = getDevelopmentWorkflowGeneration(); + if (generation === undefined) { + throw new Error( + "A development Workflow generation selector was resumed outside a generation-bound delivery.", + ); + } + return generation.source; +} diff --git a/packages/eve/src/runtime/sessions/runtime-context-keys.ts b/packages/eve/src/runtime/sessions/runtime-context-keys.ts index 9c6aa7a49..92ac6dc89 100644 --- a/packages/eve/src/runtime/sessions/runtime-context-keys.ts +++ b/packages/eve/src/runtime/sessions/runtime-context-keys.ts @@ -7,7 +7,11 @@ import type { ChannelAdapter } from "#channel/adapter.js"; import { getAdapterKind } from "#channel/adapter.js"; import { ContextKey } from "#context/key.js"; import { deserializeRuntimeAdapter } from "#runtime/channels/registry.js"; -import type { RuntimeCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js"; +import { + type DurableCompiledArtifactsSource, + resolveDurableCompiledArtifactsSource, + serializeDurableCompiledArtifactsSource, +} from "#runtime/durable-compiled-artifacts-source.js"; import { type CompiledRuntimeAgentBundle, getCompiledRuntimeAgentBundle, @@ -24,7 +28,7 @@ export type CompiledBundle = CompiledRuntimeAgentBundle; interface SerializedBundle { readonly nodeId?: string; - readonly source: RuntimeCompiledArtifactsSource; + readonly source: DurableCompiledArtifactsSource; } export const ChannelKey = new ContextKey("eve.channel", { @@ -53,12 +57,12 @@ export const BundleKey = new ContextKey("eve.bundle", { codec: { serialize: (bundle): SerializedBundle => ({ nodeId: bundle.nodeId, - source: bundle.compiledArtifactsSource, + source: serializeDurableCompiledArtifactsSource(bundle.compiledArtifactsSource), }), deserialize: (data) => { const { source, nodeId } = data as SerializedBundle; return getCompiledRuntimeAgentBundle({ - compiledArtifactsSource: source, + compiledArtifactsSource: resolveDurableCompiledArtifactsSource(source), nodeId, }); }, diff --git a/packages/eve/test/scenarios/dev-server.scenario.test.ts b/packages/eve/test/scenarios/dev-server.scenario.test.ts index 028ae00fc..f4ef1eb49 100644 --- a/packages/eve/test/scenarios/dev-server.scenario.test.ts +++ b/packages/eve/test/scenarios/dev-server.scenario.test.ts @@ -1,6 +1,6 @@ import { spawn, type ChildProcessByStdio } from "node:child_process"; import { existsSync, watch as watchFileSystem } from "node:fs"; -import { rm, writeFile } from "node:fs/promises"; +import { readFile, rm, writeFile } from "node:fs/promises"; import { dirname, join } from "node:path"; import type { Readable } from "node:stream"; @@ -121,6 +121,76 @@ const STREAM_PROMOTION_DESCRIPTOR: ScenarioAppDescriptor = { ].join("\n"), }, }; +const WORKFLOW_GENERATION_DESCRIPTOR: ScenarioAppDescriptor = { + ...TRANSACTIONAL_REBUILD_DESCRIPTOR, + files: { + ...Object.fromEntries( + Object.entries(TRANSACTIONAL_REBUILD_DESCRIPTOR.files).filter( + ([path]) => path !== "agent/tools/get_weather.ts" && !path.startsWith("agent/skills/"), + ), + ), + "agent/tools/get_marker.ts": createGenerationMarkerToolSource("generation-one", true), + }, +}; + +function createGenerationMarkerToolSource(marker: string, crashOnce: boolean): string { + const lifecycle = crashOnce + ? [ + 'import { existsSync, watch, writeFileSync } from "node:fs";', + 'import { basename, join } from "node:path";', + "", + "async function waitForPath(path: string) {", + " if (existsSync(path)) return;", + " await new Promise((resolve) => {", + " const watcher = watch(process.cwd(), (_event, filename) => {", + " if (filename !== basename(path)) return;", + " watcher.close();", + " resolve();", + " });", + " if (existsSync(path)) {", + " watcher.close();", + " resolve();", + " }", + " });", + "}", + "", + ] + : []; + const execute = crashOnce + ? [ + ' const startedPath = join(process.cwd(), ".turn-started");', + ' const crashPath = join(process.cwd(), ".crash-turn-worker");', + ' const crashedPath = join(process.cwd(), ".turn-worker-crashed");', + ' const restartPath = join(process.cwd(), ".restart-generation-test");', + ' writeFileSync(startedPath, "ready");', + " if (existsSync(restartPath)) {", + ` writeFileSync(join(process.cwd(), ".recovered-turn-started"), JSON.stringify({ instrumentation: String(globalThis.__EVE_INSTRUMENTATION_MARKER__ ?? "missing"), marker: ${JSON.stringify(marker)} }));`, + " } else {", + " await waitForPath(crashPath);", + " if (!existsSync(crashedPath)) {", + ' writeFileSync(crashedPath, "crashed");', + " process.exit(1);", + " }", + " }", + ] + : []; + + return [ + ...lifecycle, + 'import { defineTool } from "eve/tools";', + 'import { z } from "zod";', + "", + "export default defineTool({", + ' description: "Return the selected development generation marker.",', + " inputSchema: z.object({ city: z.string().optional() }),", + " async execute() {", + ...execute, + ` return { instrumentation: "instrumentation-" + String(globalThis.__EVE_INSTRUMENTATION_MARKER__ ?? "missing"), marker: ${JSON.stringify(marker)} };`, + " },", + "});", + "", + ].join("\n"); +} function createInstrumentationSource(marker: string): string { return [ @@ -200,6 +270,7 @@ function createBlockedInstrumentationSource(): string { } interface RunningEveDev { + crash(): Promise; readonly stderr: () => string; readonly stdout: () => string; readonly url: string; @@ -401,6 +472,19 @@ async function waitForWebSocketEvent( } async function startEveDev(appRoot: string): Promise { + return await startEveDevProcess(appRoot, {}); +} + +async function startEveDevWithBoundedWorkflowRecovery(appRoot: string): Promise { + return await startEveDevProcess(appRoot, { + WORKFLOW_INLINE_OWNERSHIP_LEASE_SECONDS: "1", + }); +} + +async function startEveDevProcess( + appRoot: string, + environment: Readonly>, +): Promise { const eveBinPath = join(appRoot, "node_modules", "eve", "bin", "eve.js"); const child = spawn( process.execPath, @@ -409,6 +493,7 @@ async function startEveDev(appRoot: string): Promise { cwd: appRoot, env: { ...process.env, + ...environment, // Activate the deterministic mock-model adapter in the spawned dev // server so the streamed turn completes without model credentials. NODE_ENV: "test", @@ -438,6 +523,18 @@ async function startEveDev(appRoot: string): Promise { await waitForPath(join(appRoot, ".eve", "dev-server-state.v1.json")); return { + async crash() { + if (child.exitCode !== null || child.signalCode !== null) { + return; + } + await withinDeadline( + new Promise((resolve) => { + child.once("exit", () => resolve()); + child.kill("SIGKILL"); + }), + "Timed out waiting for the dev server process to crash.", + ); + }, stderr: () => stderr, stdout: () => stdout, async stop() { @@ -769,8 +866,127 @@ describe("eve dev server", () => { }, DEV_SERVER_SCENARIO_TIMEOUT_MS, ); + + it( + "retries an active child Workflow on its selected generation after promotion", + async () => { + const app = await scenarioApp(WORKFLOW_GENERATION_DESCRIPTOR); + const server = await startEveDev(app.appRoot); + const turnStartedPath = join(app.appRoot, ".turn-started"); + const crashWorkerPath = join(app.appRoot, ".crash-turn-worker"); + + try { + const firstTurn = sendDevelopmentMessage({ + message: "Use get_marker.", + session: createDevelopmentSessionState(), + serverUrl: server.url, + }); + await waitForPath(turnStartedPath); + + await writeFile( + join(app.appRoot, "agent", "tools", "get_marker.ts"), + createGenerationMarkerToolSource("generation-two", false), + ); + await writeFile( + join(app.appRoot, "agent", "instrumentation.ts"), + createInstrumentationSource("two"), + ); + await forceDevelopmentRebuild(server.url); + await expect(fetchText(server.url, "/instrumentation-marker")).resolves.toBe("two"); + + await writeFile(crashWorkerPath, "crash"); + const firstResult = await withinDeadline( + firstTurn, + `Timed out waiting for the selected-generation retry.\n\nstdout:\n${server.stdout()}\n\nstderr:\n${server.stderr()}`, + ); + expect(readCompletedMessages(firstResult.events)).toContain("generation-one"); + expect(readCompletedMessages(firstResult.events)).toContain("instrumentation-one"); + + const secondResult = await sendDevelopmentMessage({ + message: "Use get_marker.", + session: firstResult.session, + serverUrl: server.url, + }); + expect(readCompletedMessages(secondResult.events)).toContain("generation-two"); + expect(readCompletedMessages(secondResult.events)).toContain("instrumentation-two"); + } finally { + await server.stop(); + } + }, + DEV_SERVER_SCENARIO_TIMEOUT_MS, + ); + + it( + "recovers a nonterminal child Workflow on its selected generation after restart", + async () => { + const app = await scenarioApp(WORKFLOW_GENERATION_DESCRIPTOR); + let server = await startEveDevWithBoundedWorkflowRecovery(app.appRoot); + const turnStartedPath = join(app.appRoot, ".turn-started"); + const restartPath = join(app.appRoot, ".restart-generation-test"); + const recoveredPath = join(app.appRoot, ".recovered-turn-started"); + + try { + await writeFile( + join(app.appRoot, "agent", "tools", "get_marker.ts"), + createGenerationMarkerToolSource("generation-one-runtime", true), + ); + await forceDevelopmentRebuild(server.url); + await expect(fetchText(server.url, "/instrumentation-marker")).resolves.toBe("one"); + + const interruptedTurn = sendDevelopmentMessage({ + message: "Use get_marker.", + session: createDevelopmentSessionState(), + serverUrl: server.url, + }).catch(() => undefined); + await waitForPath(turnStartedPath); + + await writeFile( + join(app.appRoot, "agent", "tools", "get_marker.ts"), + createGenerationMarkerToolSource("generation-two", false), + ); + await writeFile( + join(app.appRoot, "agent", "instrumentation.ts"), + createInstrumentationSource("two"), + ); + await forceDevelopmentRebuild(server.url); + await expect(fetchText(server.url, "/instrumentation-marker")).resolves.toBe("two"); + + await server.crash(); + await withinDeadline(interruptedTurn, "Interrupted client stream did not settle."); + await writeFile(restartPath, "restart"); + server = await startEveDevWithBoundedWorkflowRecovery(app.appRoot); + await waitForPath(recoveredPath); + await expect( + readFile(recoveredPath, "utf8").then((source) => JSON.parse(source) as unknown), + ).resolves.toEqual({ + instrumentation: "one", + marker: "generation-one-runtime", + }); + } finally { + await server.stop(); + } + }, + DEV_SERVER_SCENARIO_TIMEOUT_MS, + ); }); +function readCompletedMessages( + events: readonly { readonly data?: unknown; readonly type: string }[], +): string { + return events + .flatMap((event) => { + if (event.type !== "message.completed" || !isRecord(event.data)) { + return []; + } + return typeof event.data.message === "string" ? [event.data.message] : []; + }) + .join("\n"); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + async function fetchText(serverUrl: string, path: string): Promise { const response = await fetch(new URL(path, serverUrl)); if (!response.ok) { @@ -808,7 +1024,7 @@ async function readDevelopmentRevision(serverUrl: string): Promise { return body.revision; } -async function waitForPath(path: string): Promise { +async function waitForPath(path: string, timeoutMs: number = 60_000): Promise { if (existsSync(path)) { return; } @@ -821,7 +1037,7 @@ async function waitForPath(path: string): Promise { }); const timeout = setTimeout(() => { settle(() => reject(new Error(`Timed out waiting for ${path}.`))); - }, 60_000); + }, timeoutMs); function settle(complete: () => void) { if (settled) { return; From a27694c69e8c81e92fbe0026799b9d6b35bcf853 Mon Sep 17 00:00:00 2001 From: Casey Gowrie Date: Tue, 14 Jul 2026 08:50:50 -0400 Subject: [PATCH 2/4] fix(dev): restore one worker per persisted workspace after restart Runtime-only generations share one worker, and each generation snapshot persists that worker's entry and workspace. Restoring a worker per referenced generation gave several slots ownership of one workspace, so the first slot to close removed the workspace under the workers still serving the other generations. Deduplicate restoration by persisted workspace and map every referenced generation to the shared slot. Also point the startup integrity failures at removing .workflow-data so a user can discard the referencing runs deliberately. Signed-off-by: Casey Gowrie --- .../host/dev-worker-server.scenario.test.ts | 131 +++++++++++++++++- .../internal/nitro/host/dev-worker-server.ts | 18 ++- 2 files changed, 143 insertions(+), 6 deletions(-) diff --git a/packages/eve/src/internal/nitro/host/dev-worker-server.scenario.test.ts b/packages/eve/src/internal/nitro/host/dev-worker-server.scenario.test.ts index d0a76693d..d29b28228 100644 --- a/packages/eve/src/internal/nitro/host/dev-worker-server.scenario.test.ts +++ b/packages/eve/src/internal/nitro/host/dev-worker-server.scenario.test.ts @@ -1,15 +1,31 @@ import { Agent, request as requestHttp } from "node:http"; import { connect } from "node:net"; +import { mkdir, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; import { describe, expect, it, vi } from "vitest"; +import { turnWorkflowReference } from "#execution/workflow-runtime.js"; +import { writeDevelopmentRuntimeArtifactsWorker } from "#internal/nitro/dev-runtime-worker-artifacts.js"; import { createDevelopmentWorkerServer } from "#internal/nitro/host/dev-worker-server.js"; import type { DevelopmentWorkerGeneration, DevelopmentWorkerRunner, DevelopmentWorkerRunnerFactory, } from "#internal/nitro/host/dev-worker-server-types.js"; -import { decodeDevelopmentWorkerMetadata } from "#internal/nitro/host/dev-worker-metadata.js"; +import { + decodeDevelopmentWorkerMetadata, + DEVELOPMENT_WORKER_METADATA_HEADER, +} from "#internal/nitro/host/dev-worker-metadata.js"; +import { useTemporaryDirectories } from "#internal/testing/use-temporary-app-roots.js"; +import { encodeDevelopmentWorldValue } from "#internal/workflow/development-world-codec.js"; +import { + DEVELOPMENT_WORKFLOW_TRANSPORT_HEADER, + DEVELOPMENT_WORKFLOW_WORLD_ROUTE, +} from "#internal/workflow/development-world-protocol.js"; +import { createParentDevelopmentWorkflowWorld } from "#internal/workflow/development-world-server.js"; + +const createScratchDirectory = useTemporaryDirectories(); const TEST_DEADLINE_MS = 5_000; @@ -233,7 +249,7 @@ describe("development worker server", () => { it("restarts an active worker after a crash without moving its generation", async () => { const { createRunner, runners } = createRunnerFactory(async (request, runnerIndex, secret) => { const metadata = decodeDevelopmentWorkerMetadata({ - header: request.headers.get("x-eve-dev-worker-metadata"), + header: request.headers.get(DEVELOPMENT_WORKER_METADATA_HEADER), secret, }); return new Response(`${String(runnerIndex)}:${metadata.generationId}`); @@ -261,7 +277,7 @@ describe("development worker server", () => { let admissionGeneration = FIRST_GENERATION; const { createRunner, runners } = createRunnerFactory(async (request, runnerIndex, secret) => { const metadata = decodeDevelopmentWorkerMetadata({ - header: request.headers.get("x-eve-dev-worker-metadata"), + header: request.headers.get(DEVELOPMENT_WORKER_METADATA_HEADER), secret, }); return new Response(`${String(runnerIndex)}:${metadata.generationId}`); @@ -646,7 +662,7 @@ describe("development worker server", () => { it("preserves the socket client address in signed metadata and replaces public spoofing", async () => { const { createRunner } = createRunnerFactory(async (request, _runnerIndex, secret) => { const metadata = decodeDevelopmentWorkerMetadata({ - header: request.headers.get("x-eve-dev-worker-metadata"), + header: request.headers.get(DEVELOPMENT_WORKER_METADATA_HEADER), secret, }); return Response.json({ @@ -725,6 +741,92 @@ describe("development worker server", () => { await closeWithinDeadline(() => server.close()); }); + it("restores one retired worker for runtime generations that shared a workspace", async () => { + const appRoot = await createScratchDirectory("eve-dev-worker-shared-restore-"); + const workspaceRoot = join(appRoot, ".eve", "dev-hosts", "workspace-1"); + const entry = join(workspaceRoot, "output", "index.mjs"); + await mkdir(dirname(entry), { recursive: true }); + await writeFile(entry, "export default {};\n"); + for (const generationId of ["first", "second"]) { + await writeActivatedGenerationSnapshot({ appRoot, entry, generationId, workspaceRoot }); + } + + const seedSecret = "seed-transport-secret"; + const seedWorld = createParentDevelopmentWorkflowWorld({ + agentName: "shared-restore-test", + appRoot, + dispatch: async () => Response.json({ ok: true }), + hasGeneration: () => true, + resolveActiveGenerationId: () => "second", + transportSecret: seedSecret, + }); + await seedWorld.start(); + for (const generationId of ["first", "second"]) { + const response = await seedWorld.handleRequest( + new Request(`http://localhost${DEVELOPMENT_WORKFLOW_WORLD_ROUTE}`, { + body: encodeDevelopmentWorldValue({ + arguments: [ + null, + { + eventData: { + deploymentId: generationId, + executionContext: {}, + input: new Uint8Array(), + workflowName: turnWorkflowReference.workflowId, + }, + eventType: "run_created", + specVersion: 5, + }, + ], + operation: "events.create", + }), + headers: { [DEVELOPMENT_WORKFLOW_TRANSPORT_HEADER]: seedSecret }, + method: "POST", + }), + ); + expect(response?.status).toBe(200); + } + await seedWorld.close(); + + const dispatchedGenerationIds = new Set(); + const { createRunner, runners } = createRunnerFactory(async (request, _index, secret) => { + const metadata = decodeDevelopmentWorkerMetadata({ + header: request.headers.get(DEVELOPMENT_WORKER_METADATA_HEADER), + secret, + }); + dispatchedGenerationIds.add(metadata.generationId); + return Response.json({ ok: true }); + }); + const server = createDevelopmentWorkerServer({ + appRoot, + createRunner, + resolveAdmissionGeneration: (generation) => generation, + workflowWorld: { agentName: "shared-restore-test", kind: "parent-local" }, + }); + const listener = server.listen({ hostname: "127.0.0.1", port: 0 }); + await listener.ready(); + const previousBaseUrl = process.env.WORKFLOW_LOCAL_BASE_URL; + process.env.WORKFLOW_LOCAL_BASE_URL = listener.url; + try { + await server.startWorkflowWorld(); + + expect(runners).toHaveLength(1); + await withinDeadline( + vi.waitFor(() => { + expect(dispatchedGenerationIds).toEqual(new Set(["first", "second"])); + }), + "Timed out waiting for both restored generation deliveries.", + ); + } finally { + if (previousBaseUrl === undefined) { + delete process.env.WORKFLOW_LOCAL_BASE_URL; + } else { + process.env.WORKFLOW_LOCAL_BASE_URL = previousBaseUrl; + } + await closeWithinDeadline(() => server.close()); + } + }); + it("closes workers and listeners only once when close is repeated", async () => { const { createRunner, runners } = createRunnerFactory(async () => new Response("ok")); const { server } = await listen(createRunner); @@ -771,3 +873,24 @@ async function requestWithAgent( request.end(); }); } + +async function writeActivatedGenerationSnapshot(input: { + readonly appRoot: string; + readonly entry: string; + readonly generationId: string; + readonly workspaceRoot: string; +}): Promise { + const snapshotRoot = join(input.appRoot, ".eve", "dev-runtime", "snapshots", input.generationId); + const runtimeAppRoot = join(snapshotRoot, "source", "app"); + + await mkdir(join(runtimeAppRoot, ".eve", "compile"), { recursive: true }); + await writeFile(join(runtimeAppRoot, ".eve", "compile", "compiled-agent-manifest.json"), "{}\n"); + await writeFile(join(snapshotRoot, "activated"), ""); + await writeFile(join(snapshotRoot, "generation.json"), `${JSON.stringify({ runtimeAppRoot })}\n`); + await writeDevelopmentRuntimeArtifactsWorker({ + entry: input.entry, + snapshotRoot, + workerData: {}, + workspaceRoot: input.workspaceRoot, + }); +} diff --git a/packages/eve/src/internal/nitro/host/dev-worker-server.ts b/packages/eve/src/internal/nitro/host/dev-worker-server.ts index 554f2fbd5..61413f7dd 100644 --- a/packages/eve/src/internal/nitro/host/dev-worker-server.ts +++ b/packages/eve/src/internal/nitro/host/dev-worker-server.ts @@ -139,6 +139,11 @@ class ParentDevelopmentWorkerServer implements DevelopmentWorkerServer { if (this.#parentOwnsWorkflowWorld) { const references = await this.#workflowWorld.collectGenerationReferences(); if (!references.protectAll) { + // Runtime-only generations shared one worker before the restart, so + // restore one slot per persisted workspace. Restoring per generation + // would give several slots ownership of the same workspace, and the + // first one to close would remove it under the others. + const restoredSlotsByWorkspace = new Map(); for (const generationId of references.generationIds) { if (this.#generationSlots.has(generationId)) { continue; @@ -146,15 +151,23 @@ class ParentDevelopmentWorkerServer implements DevelopmentWorkerServer { const generation = generations.find((candidate) => candidate.id === generationId); if (generation === undefined) { throw new Error( - `Workflow run references missing development generation "${generationId}".`, + `Workflow run references missing development generation "${generationId}". ` + + `Remove ".workflow-data" to discard the app's active local Workflow runs.`, ); } const worker = generation.worker; if (!isRestorableDevelopmentWorker(worker, this.#appRoot)) { throw new Error( - `Workflow run references development generation "${generationId}" without a restorable worker.`, + `Workflow run references development generation "${generationId}" without a restorable worker. ` + + `Remove ".workflow-data" to discard the app's active local Workflow runs.`, ); } + const workspaceKey = `${worker.entry}\0${worker.workspaceRoot}`; + const restoredSlot = restoredSlotsByWorkspace.get(workspaceKey); + if (restoredSlot !== undefined) { + this.#recordGeneration(generation, restoredSlot); + continue; + } const candidate = await this.prepareCandidate({ dispose: async () => await rm(worker.workspaceRoot, { force: true, recursive: true }), entry: worker.entry, @@ -163,6 +176,7 @@ class ParentDevelopmentWorkerServer implements DevelopmentWorkerServer { }); candidate.slot.state = "retired"; this.#recordGeneration(generation, candidate.slot); + restoredSlotsByWorkspace.set(workspaceKey, candidate.slot); } } } From 5026846fd4b228a026ae364f669a471d3f0ca21a Mon Sep 17 00:00:00 2001 From: Casey Gowrie Date: Tue, 14 Jul 2026 08:52:42 -0400 Subject: [PATCH 3/4] docs(cli): describe reference-driven dev generation retention Signed-off-by: Casey Gowrie --- docs/reference/cli.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 369f83ade..2da14478c 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -158,7 +158,7 @@ For bearer tokens or custom schemes, pass explicit headers with `-H`. Local dev records the last ready URL per resolved app root in `.eve/dev-server-state.v1.json`. A second interactive `eve dev` reconnects only when that URL is loopback and healthy; each terminal UI creates a fresh client session while sharing the server process. A stale or malformed record is replaced when eve starts a new server. Passing `--host`, `--port`, or a `PORT` environment value skips reconnection and reports a healthy recorded server instead. -Local dev keeps immutable runtime source snapshots under `.eve/dev-runtime/snapshots/` so in-flight sessions hold a consistent code revision while new prompts pick up rebuilds. On startup, `eve dev` prunes stale runtime snapshots and old local sandbox templates in the background. For manual cleanup, stop `eve dev` and delete `.eve/dev-runtime/snapshots/` or `.eve/sandbox-cache/local/templates/`. +Local dev keeps immutable runtime generations under `.eve/dev-runtime/snapshots/` so an in-flight turn finishes on the code revision it started with while the next prompt picks up successful rebuilds. eve prunes generations in the background once no request, worker, or active local Workflow run references them, along with old local sandbox templates. For manual cleanup, stop `eve dev` and delete `.eve/dev-runtime/snapshots/` or `.eve/sandbox-cache/local/templates/`; also delete `.workflow-data` if it exists, since a Workflow run that references a deleted generation stops `eve dev` startup with an integrity error. ## `eve link` From 15ddd843aa4499102601f51bb3dd318b0c8db9a5 Mon Sep 17 00:00:00 2001 From: Casey Gowrie Date: Tue, 14 Jul 2026 09:49:37 -0400 Subject: [PATCH 4/4] test(dev): close audit gaps with real-server and classification coverage - Dispatch an authored schedule through the dev route on a real dev server, covering the baked artifacts-config path end to end. - Boot a dev server from an npm-installed app so the hoisted real-directory node_modules layout exercises generation dependency resolution alongside pnpm's symlinked store; the scenario harness gains a packageManager option. - Pin the compiled-manifest fingerprint's path independence and the host fingerprint's structural/runtime classification (instrumentation, channel topology, environment values, workflow world structural; schedules runtime-only). - Cover isRestorableDevelopmentWorker's workspace boundary checks. Signed-off-by: Casey Gowrie --- .../compiled-manifest-fingerprint.test.ts | 58 +++++++ .../dev-host-fingerprint.integration.test.ts | 148 ++++++++++++++++++ ...opment-worker-recovery.integration.test.ts | 72 +++++++++ .../eve/src/internal/testing/scenario-app.ts | 39 +++++ .../scenarios/dev-server.scenario.test.ts | 85 ++++++++++ 5 files changed, 402 insertions(+) create mode 100644 packages/eve/src/internal/compiled-manifest-fingerprint.test.ts create mode 100644 packages/eve/src/internal/nitro/host/dev-host-fingerprint.integration.test.ts create mode 100644 packages/eve/src/internal/nitro/host/development-worker-recovery.integration.test.ts diff --git a/packages/eve/src/internal/compiled-manifest-fingerprint.test.ts b/packages/eve/src/internal/compiled-manifest-fingerprint.test.ts new file mode 100644 index 000000000..ece211693 --- /dev/null +++ b/packages/eve/src/internal/compiled-manifest-fingerprint.test.ts @@ -0,0 +1,58 @@ +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { createCompiledAgentManifest } from "#compiler/manifest.js"; +import { serializeCompiledManifestForFingerprint } from "#internal/compiled-manifest-fingerprint.js"; + +function manifestWithRoot(runtimeAppRoot: string, agentRoot = join(runtimeAppRoot, "agent")) { + return createCompiledAgentManifest({ + agentRoot, + appRoot: runtimeAppRoot, + config: { + model: { id: "openai/gpt-5-mini", routing: { kind: "gateway", target: "openai" } }, + name: "fingerprint-test", + }, + }); +} + +describe("serializeCompiledManifestForFingerprint", () => { + it("serializes identical content under different snapshot roots identically", () => { + const firstRoot = "/tmp/snapshots/generation-a/source/app"; + const secondRoot = "/tmp/snapshots/generation-b/source/app"; + const first = serializeCompiledManifestForFingerprint({ + manifest: manifestWithRoot(firstRoot), + runtimeAppRoot: firstRoot, + }); + const second = serializeCompiledManifestForFingerprint({ + manifest: manifestWithRoot(secondRoot), + runtimeAppRoot: secondRoot, + }); + + expect(first).toBe(second); + expect(first).toContain("$runtime/agent"); + expect(first).not.toContain("generation-a"); + }); + + it("keeps absolute paths outside the runtime root verbatim", () => { + const runtimeAppRoot = "/tmp/snapshots/generation-a/source/app"; + const serialized = serializeCompiledManifestForFingerprint({ + manifest: manifestWithRoot(runtimeAppRoot, "/somewhere/else/agent"), + runtimeAppRoot, + }); + + expect(serialized).toContain("/somewhere/else/agent"); + }); + + it("canonicalizes object key order", () => { + const left = serializeCompiledManifestForFingerprint({ + manifest: { agentRoot: "/app/agent", config: { model: "m", name: "n" } } as never, + runtimeAppRoot: "/app", + }); + const right = serializeCompiledManifestForFingerprint({ + manifest: { config: { name: "n", model: "m" }, agentRoot: "/app/agent" } as never, + runtimeAppRoot: "/app", + }); + + expect(left).toBe(right); + }); +}); diff --git a/packages/eve/src/internal/nitro/host/dev-host-fingerprint.integration.test.ts b/packages/eve/src/internal/nitro/host/dev-host-fingerprint.integration.test.ts new file mode 100644 index 000000000..62e5d3621 --- /dev/null +++ b/packages/eve/src/internal/nitro/host/dev-host-fingerprint.integration.test.ts @@ -0,0 +1,148 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { createCompiledAgentManifest, type CompiledAgentManifest } from "#compiler/manifest.js"; +import { computeDevelopmentHostFingerprint } from "#internal/nitro/host/dev-host-fingerprint.js"; +import type { PreparedDevelopmentApplicationHost } from "#internal/nitro/host/types.js"; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + delete process.env.EVE_HOST_FINGERPRINT_TEST; + await Promise.all( + temporaryDirectories + .splice(0) + .map(async (dir) => await rm(dir, { force: true, recursive: true })), + ); +}); + +interface HostVariant { + readonly channels?: CompiledAgentManifest["channels"]; + readonly instrumentationSource?: string; + readonly schedules?: CompiledAgentManifest["schedules"]; + readonly workflowWorld?: "local" | "vercel"; +} + +async function createHost(variant: HostVariant = {}): Promise { + const appRoot = await mkdtemp(join(tmpdir(), "eve-host-fingerprint-")); + temporaryDirectories.push(appRoot); + const agentRoot = join(appRoot, "agent"); + await mkdir(agentRoot, { recursive: true }); + + let instrumentationSourcePath: string | undefined; + if (variant.instrumentationSource !== undefined) { + instrumentationSourcePath = join(appRoot, "instrumentation-source.mjs"); + await writeFile(instrumentationSourcePath, variant.instrumentationSource); + } + + const manifest = createCompiledAgentManifest({ + agentRoot, + appRoot, + channels: variant.channels ?? [], + config: { + model: { id: "openai/gpt-5-mini", routing: { kind: "gateway", target: "openai" } }, + name: "fingerprint-host", + ...(variant.workflowWorld === undefined + ? {} + : { experimental: { workflow: { world: variant.workflowWorld } } }), + }, + schedules: variant.schedules ?? [], + }); + + return { + appRoot, + compiledArtifacts: { + bootstrapPath: join(appRoot, "bootstrap.mjs"), + instrumentationSourcePath, + workflowWorldPluginPath: join(appRoot, "workflow-world.mjs"), + }, + compileResult: { manifest } as PreparedDevelopmentApplicationHost["compileResult"], + } as PreparedDevelopmentApplicationHost; +} + +describe("computeDevelopmentHostFingerprint", () => { + it("is stable for equivalent hosts in different app roots", async () => { + const first = await computeDevelopmentHostFingerprint(await createHost()); + const second = await computeDevelopmentHostFingerprint(await createHost()); + + expect(first).toBe(second); + }); + + it("treats instrumentation content as structural", async () => { + const base = await computeDevelopmentHostFingerprint( + await createHost({ instrumentationSource: 'export default { marker: "one" };\n' }), + ); + const changed = await computeDevelopmentHostFingerprint( + await createHost({ instrumentationSource: 'export default { marker: "two" };\n' }), + ); + + expect(changed).not.toBe(base); + }); + + it("treats channel route topology as structural", async () => { + const base = await computeDevelopmentHostFingerprint(await createHost()); + const withRoute = await computeDevelopmentHostFingerprint( + await createHost({ + channels: [ + { + kind: "channel", + logicalPath: "channels/smoke.ts", + method: "GET", + name: "smoke", + sourceId: "channels/smoke.ts", + sourceKind: "module", + urlPath: "/smoke", + }, + ] as CompiledAgentManifest["channels"], + }), + ); + + expect(withRoute).not.toBe(base); + }); + + it("treats configured environment values as structural", async () => { + const host = await createHost(); + await writeFile(join(host.appRoot, ".env"), "EVE_HOST_FINGERPRINT_TEST=one\n"); + process.env.EVE_HOST_FINGERPRINT_TEST = "one"; + const base = await computeDevelopmentHostFingerprint(host); + + process.env.EVE_HOST_FINGERPRINT_TEST = "two"; + const changed = await computeDevelopmentHostFingerprint(host); + + expect(changed).not.toBe(base); + }); + + it("treats the workflow world selection as structural", async () => { + const local = await computeDevelopmentHostFingerprint( + await createHost({ workflowWorld: "local" }), + ); + const vercel = await computeDevelopmentHostFingerprint( + await createHost({ workflowWorld: "vercel" }), + ); + + expect(vercel).not.toBe(local); + }); + + it("leaves schedule definitions runtime-only", async () => { + const base = await computeDevelopmentHostFingerprint(await createHost()); + const withSchedule = await computeDevelopmentHostFingerprint( + await createHost({ + schedules: [ + { + cron: "0 0 * * 0", + hasRun: false, + logicalPath: "schedules/heartbeat.md", + markdown: "Report the weather.", + name: "heartbeat", + sourceId: "schedules/heartbeat.md", + sourceKind: "markdown", + }, + ], + }), + ); + + expect(withSchedule).toBe(base); + }); +}); diff --git a/packages/eve/src/internal/nitro/host/development-worker-recovery.integration.test.ts b/packages/eve/src/internal/nitro/host/development-worker-recovery.integration.test.ts new file mode 100644 index 000000000..8acbed322 --- /dev/null +++ b/packages/eve/src/internal/nitro/host/development-worker-recovery.integration.test.ts @@ -0,0 +1,72 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { isRestorableDevelopmentWorker } from "#internal/nitro/host/development-worker-recovery.js"; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map(async (dir) => await rm(dir, { force: true, recursive: true })), + ); +}); + +async function createAppRoot(): Promise { + const appRoot = await mkdtemp(join(tmpdir(), "eve-worker-recovery-")); + temporaryDirectories.push(appRoot); + return appRoot; +} + +describe("isRestorableDevelopmentWorker", () => { + it("accepts a persisted worker whose entry exists inside its dev-hosts workspace", async () => { + const appRoot = await createAppRoot(); + const workspaceRoot = join(appRoot, ".eve", "dev-hosts", "workspace-1"); + const entry = join(workspaceRoot, "output", "index.mjs"); + await mkdir(join(workspaceRoot, "output"), { recursive: true }); + await writeFile(entry, "export default {};\n"); + + expect(isRestorableDevelopmentWorker({ entry, workerData: {}, workspaceRoot }, appRoot)).toBe( + true, + ); + }); + + it("rejects a workspace outside the app's dev-hosts directory", async () => { + const appRoot = await createAppRoot(); + const workspaceRoot = join(appRoot, "elsewhere"); + const entry = join(workspaceRoot, "index.mjs"); + await mkdir(workspaceRoot, { recursive: true }); + await writeFile(entry, "export default {};\n"); + + expect(isRestorableDevelopmentWorker({ entry, workerData: {}, workspaceRoot }, appRoot)).toBe( + false, + ); + }); + + it("rejects an entry that escapes its workspace", async () => { + const appRoot = await createAppRoot(); + const workspaceRoot = join(appRoot, ".eve", "dev-hosts", "workspace-1"); + const entry = join(appRoot, ".eve", "dev-hosts", "other", "index.mjs"); + await mkdir(join(appRoot, ".eve", "dev-hosts", "other"), { recursive: true }); + await mkdir(workspaceRoot, { recursive: true }); + await writeFile(entry, "export default {};\n"); + + expect(isRestorableDevelopmentWorker({ entry, workerData: {}, workspaceRoot }, appRoot)).toBe( + false, + ); + }); + + it("rejects a worker whose entry file no longer exists", async () => { + const appRoot = await createAppRoot(); + const workspaceRoot = join(appRoot, ".eve", "dev-hosts", "workspace-1"); + const entry = join(workspaceRoot, "output", "index.mjs"); + await mkdir(join(workspaceRoot, "output"), { recursive: true }); + + expect(isRestorableDevelopmentWorker({ entry, workerData: {}, workspaceRoot }, appRoot)).toBe( + false, + ); + }); +}); diff --git a/packages/eve/src/internal/testing/scenario-app.ts b/packages/eve/src/internal/testing/scenario-app.ts index 0e018db09..973420062 100644 --- a/packages/eve/src/internal/testing/scenario-app.ts +++ b/packages/eve/src/internal/testing/scenario-app.ts @@ -8,9 +8,11 @@ import { rm, writeFile, } from "node:fs/promises"; +import { execFile } from "node:child_process"; import { createRequire } from "node:module"; import { tmpdir } from "node:os"; import { basename, dirname, join } from "node:path"; +import { promisify } from "node:util"; import { afterEach } from "vitest"; @@ -76,6 +78,13 @@ export interface ScenarioAppDescriptor { * us having to maintain a bespoke template-then-copy cache. */ readonly installDependencies?: boolean; + /** + * Package manager used for {@link installDependencies}. Defaults to pnpm. + * npm installs are slower but produce the hoisted real-directory + * `node_modules/` layout, which exercises dependency resolution paths that + * pnpm's symlinked store never hits. + */ + readonly packageManager?: "npm" | "pnpm"; } /** @@ -262,6 +271,11 @@ async function installScenarioDependencies(input: { readonly appRoot: string; readonly descriptor: ScenarioAppDescriptor; }): Promise { + if (input.descriptor.packageManager === "npm") { + await runNpmInstall(input.appRoot); + return; + } + await runPnpmCommand({ args: [ "install", @@ -275,6 +289,31 @@ async function installScenarioDependencies(input: { }); } +const runFile = promisify(execFile); + +async function runNpmInstall(appRoot: string): Promise { + const args = ["install", "--no-audit", "--no-fund", "--ignore-scripts", "--prefer-offline"]; + + try { + await runFile(process.platform === "win32" ? "npm.cmd" : "npm", args, { + cwd: appRoot, + maxBuffer: 10 * 1024 * 1024, + shell: process.platform === "win32", + }); + } catch (error) { + const failure = error as { readonly stderr?: unknown; readonly stdout?: unknown }; + throw new Error( + [ + `Command failed: npm ${args.join(" ")}`, + `cwd: ${appRoot}`, + `stdout:\n${typeof failure.stdout === "string" ? failure.stdout : ""}`, + `stderr:\n${typeof failure.stderr === "string" ? failure.stderr : ""}`, + ].join("\n\n"), + { cause: error }, + ); + } +} + const EVE_SCENARIO_EVE_TARBALL_PATH_ENV = "EVE_SCENARIO_EVE_TARBALL_PATH"; let cachedScenarioEveTarballPromise: Promise | null = null; diff --git a/packages/eve/test/scenarios/dev-server.scenario.test.ts b/packages/eve/test/scenarios/dev-server.scenario.test.ts index f4ef1eb49..9688d3fb7 100644 --- a/packages/eve/test/scenarios/dev-server.scenario.test.ts +++ b/packages/eve/test/scenarios/dev-server.scenario.test.ts @@ -77,6 +77,26 @@ const TRANSACTIONAL_REBUILD_DESCRIPTOR: ScenarioAppDescriptor = { "agent/instrumentation.ts": createInstrumentationSource("one"), }, }; +const SCHEDULE_DISPATCH_DESCRIPTOR: ScenarioAppDescriptor = { + ...WEATHER_AGENT_DESCRIPTOR, + files: { + ...WEATHER_AGENT_DESCRIPTOR.files, + "agent/schedules/heartbeat.md": [ + "---", + 'cron: "0 0 * * 0"', + "---", + "", + "Report the weather in Lisbon.", + "", + ].join("\n"), + }, + name: "weather-agent-schedules", +}; +const NPM_LAYOUT_DESCRIPTOR: ScenarioAppDescriptor = { + ...WEATHER_AGENT_DESCRIPTOR, + name: "weather-agent-npm", + packageManager: "npm", +}; const STREAM_PROMOTION_DESCRIPTOR: ScenarioAppDescriptor = { ...TRANSACTIONAL_REBUILD_DESCRIPTOR, files: { @@ -968,6 +988,71 @@ describe("eve dev server", () => { }, DEV_SERVER_SCENARIO_TIMEOUT_MS, ); + + it( + "dispatches an authored schedule through the dev route on its generation", + async () => { + const app = await scenarioApp(SCHEDULE_DISPATCH_DESCRIPTOR); + const server = await startEveDev(app.appRoot); + + try { + const response = await fetch(new URL("/eve/v1/dev/schedules/heartbeat", server.url), { + method: "POST", + }); + const body = (await response.json()) as { + scheduleId?: string; + sessionIds?: readonly string[]; + }; + expect( + response.status, + [ + `Expected the dev schedule dispatch route to succeed: ${JSON.stringify(body)}`, + `stdout:\n${server.stdout()}`, + `stderr:\n${server.stderr()}`, + ].join("\n\n"), + ).toBe(200); + expect(body.scheduleId).toBe("heartbeat"); + expect(body.sessionIds).toHaveLength(1); + + const unknown = await fetch(new URL("/eve/v1/dev/schedules/missing", server.url), { + method: "POST", + }); + expect(unknown.status).toBe(404); + expect(hasKnownDevServerFailure(`${server.stdout()}\n${server.stderr()}`)).toBe(false); + } finally { + await server.stop(); + } + }, + DEV_SERVER_SCENARIO_TIMEOUT_MS, + ); + + it( + "serves an npm-installed app with hoisted real-directory dependencies", + async () => { + const app = await scenarioApp(NPM_LAYOUT_DESCRIPTOR); + const server = await startEveDev(app.appRoot); + + try { + const messageResult = await sendDevelopmentMessage({ + message: "What's the weather in Lisbon?", + session: createDevelopmentSessionState(), + serverUrl: server.url, + }); + expect( + messageResult.events.some((event) => event.type === "message.completed"), + [ + "Expected the npm-installed dev server to complete a streamed turn.", + `stdout:\n${server.stdout()}`, + `stderr:\n${server.stderr()}`, + ].join("\n\n"), + ).toBe(true); + expect(hasKnownDevServerFailure(`${server.stdout()}\n${server.stderr()}`)).toBe(false); + } finally { + await server.stop(); + } + }, + DEV_SERVER_SCENARIO_TIMEOUT_MS, + ); }); function readCompletedMessages(