From 50a0497913597814a3f6aa7b4b4be49f328790ba Mon Sep 17 00:00:00 2001 From: Casey Gowrie Date: Tue, 14 Jul 2026 10:30:11 -0400 Subject: [PATCH] test(dev): add bun and chaos coverage for the dev server - Extract the spawned dev-server helpers into a shared scenario harness so new suites reuse one RunningEveDev lifecycle. - Serve a bun-installed app under the Node runtime, and pin the current bun-runtime contract: `bun eve dev` fails fast with the worker readiness error (crossws's Node adapter refuses to initialize when the Bun global exists) instead of hanging. Both tests skip when bun is not installed; the scenario CI job now installs a pinned bun. - Chaos-test the dev server: an authored edit storm (rapid edits, broken TypeScript, deletes, concurrent forced rebuilds, a structural env change) with a continuous health probe that tolerates zero failed requests, and a crash/abort sequence (worker process.exit, aborted streaming response, crash racing a rebuild) that must recover to a completed streamed turn. Signed-off-by: Casey Gowrie --- .github/workflows/ci.yml | 5 + .../eve/src/internal/testing/scenario-app.ts | 28 +- .../scenarios/dev-server-bun.scenario.test.ts | 75 ++++ .../dev-server-chaos.scenario.test.ts | 238 +++++++++++ .../eve/test/scenarios/dev-server-harness.ts | 401 ++++++++++++++++++ .../scenarios/dev-server.scenario.test.ts | 365 +--------------- 6 files changed, 759 insertions(+), 353 deletions(-) create mode 100644 packages/eve/test/scenarios/dev-server-bun.scenario.test.ts create mode 100644 packages/eve/test/scenarios/dev-server-chaos.scenario.test.ts create mode 100644 packages/eve/test/scenarios/dev-server-harness.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3caf05fcf..a1d21a3bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -138,6 +138,11 @@ jobs: node-version-file: .nvmrc cache: "pnpm" + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: "1.3.14" + - name: Install dependencies run: pnpm install --frozen-lockfile diff --git a/packages/eve/src/internal/testing/scenario-app.ts b/packages/eve/src/internal/testing/scenario-app.ts index 973420062..7c349b240 100644 --- a/packages/eve/src/internal/testing/scenario-app.ts +++ b/packages/eve/src/internal/testing/scenario-app.ts @@ -80,11 +80,11 @@ export interface ScenarioAppDescriptor { readonly installDependencies?: boolean; /** * Package manager used for {@link installDependencies}. Defaults to pnpm. - * npm installs are slower but produce the hoisted real-directory + * npm and bun installs are slower but produce the hoisted real-directory * `node_modules/` layout, which exercises dependency resolution paths that * pnpm's symlinked store never hits. */ - readonly packageManager?: "npm" | "pnpm"; + readonly packageManager?: "bun" | "npm" | "pnpm"; } /** @@ -272,7 +272,17 @@ async function installScenarioDependencies(input: { readonly descriptor: ScenarioAppDescriptor; }): Promise { if (input.descriptor.packageManager === "npm") { - await runNpmInstall(input.appRoot); + await runInstallCommand(input.appRoot, "npm", [ + "install", + "--no-audit", + "--no-fund", + "--ignore-scripts", + "--prefer-offline", + ]); + return; + } + if (input.descriptor.packageManager === "bun") { + await runInstallCommand(input.appRoot, "bun", ["install", "--ignore-scripts"]); return; } @@ -291,11 +301,13 @@ async function installScenarioDependencies(input: { const runFile = promisify(execFile); -async function runNpmInstall(appRoot: string): Promise { - const args = ["install", "--no-audit", "--no-fund", "--ignore-scripts", "--prefer-offline"]; - +async function runInstallCommand( + appRoot: string, + command: "bun" | "npm", + args: readonly string[], +): Promise { try { - await runFile(process.platform === "win32" ? "npm.cmd" : "npm", args, { + await runFile(command, [...args], { cwd: appRoot, maxBuffer: 10 * 1024 * 1024, shell: process.platform === "win32", @@ -304,7 +316,7 @@ async function runNpmInstall(appRoot: string): Promise { const failure = error as { readonly stderr?: unknown; readonly stdout?: unknown }; throw new Error( [ - `Command failed: npm ${args.join(" ")}`, + `Command failed: ${command} ${args.join(" ")}`, `cwd: ${appRoot}`, `stdout:\n${typeof failure.stdout === "string" ? failure.stdout : ""}`, `stderr:\n${typeof failure.stderr === "string" ? failure.stderr : ""}`, diff --git a/packages/eve/test/scenarios/dev-server-bun.scenario.test.ts b/packages/eve/test/scenarios/dev-server-bun.scenario.test.ts new file mode 100644 index 000000000..7f2b1e931 --- /dev/null +++ b/packages/eve/test/scenarios/dev-server-bun.scenario.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; + +import { WEATHER_AGENT_DESCRIPTOR } from "../../src/internal/testing/scenario-apps/weather-agent.js"; +import { + type ScenarioAppDescriptor, + useScenarioApp, +} from "../../src/internal/testing/scenario-app.js"; +import { sendDevelopmentMessage } from "../dev-client-harness/send-message.js"; +import { createDevelopmentSessionState } from "../dev-client-harness/session.js"; +import { + hasKnownDevServerFailure, + isBunAvailable, + runEveDevToExit, + startEveDev, +} from "./dev-server-harness.js"; + +const scenarioApp = useScenarioApp(); + +const BUN_DEV_SERVER_TIMEOUT_MS = 360_000; +const BUN_LAYOUT_DESCRIPTOR: ScenarioAppDescriptor = { + ...WEATHER_AGENT_DESCRIPTOR, + name: "weather-agent-bun", + packageManager: "bun", +}; + +const bunAvailable = isBunAvailable(); + +describe("eve dev server with bun", () => { + it.skipIf(!bunAvailable)( + "serves a bun-installed app under the Node runtime", + async () => { + const app = await scenarioApp(BUN_LAYOUT_DESCRIPTOR); + const server = await startEveDev(app.appRoot); + + try { + const messageResult = await sendDevelopmentMessage({ + message: "What's the weather in Lisbon?", + session: createDevelopmentSessionState(), + serverUrl: server.url, + }); + expect( + messageResult.events.some((event) => event.type === "message.completed"), + [ + "Expected the bun-installed dev server to complete a streamed turn.", + `stdout:\n${server.stdout()}`, + `stderr:\n${server.stderr()}`, + ].join("\n\n"), + ).toBe(true); + expect(hasKnownDevServerFailure(`${server.stdout()}\n${server.stderr()}`)).toBe(false); + } finally { + await server.stop(); + } + }, + BUN_DEV_SERVER_TIMEOUT_MS, + ); + + // Bun cannot run the dev server today: Nitro wires crossws's Node adapter + // into every dev worker, and that adapter refuses to initialize when the + // Bun global exists. Until that support lands upstream, the contract is a + // fast, explanatory startup failure instead of a hang or a half-broken + // server. Replace this with a boot-and-stream assertion when `bun eve dev` + // becomes supported. + it.skipIf(!bunAvailable)( + "fails fast with the worker readiness error when the CLI runs under bun", + async () => { + const app = await scenarioApp(BUN_LAYOUT_DESCRIPTOR); + + const result = await runEveDevToExit(app.appRoot, { runtime: "bun" }); + + expect(result.code).not.toBe(0); + expect(result.output).toContain("failed before readiness"); + }, + BUN_DEV_SERVER_TIMEOUT_MS, + ); +}); diff --git a/packages/eve/test/scenarios/dev-server-chaos.scenario.test.ts b/packages/eve/test/scenarios/dev-server-chaos.scenario.test.ts new file mode 100644 index 000000000..47a9047bd --- /dev/null +++ b/packages/eve/test/scenarios/dev-server-chaos.scenario.test.ts @@ -0,0 +1,238 @@ +import { rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { EVE_HEALTH_ROUTE_PATH } from "../../src/protocol/routes.js"; +import { WEATHER_AGENT_DESCRIPTOR } from "../../src/internal/testing/scenario-apps/weather-agent.js"; +import { + type ScenarioAppDescriptor, + useScenarioApp, +} from "../../src/internal/testing/scenario-app.js"; +import { sendDevelopmentMessage } from "../dev-client-harness/send-message.js"; +import { createDevelopmentSessionState } from "../dev-client-harness/session.js"; +import { + fetchAgentInfo, + fetchText, + forceDevelopmentRebuild, + hasKnownDevServerFailure, + startEveDev, + wait, + waitForCondition, + type RunningEveDev, +} from "./dev-server-harness.js"; + +const scenarioApp = useScenarioApp(); + +const CHAOS_SCENARIO_TIMEOUT_MS = 360_000; + +const CHAOS_CHANNEL_SOURCE = [ + 'import { randomUUID } from "node:crypto";', + 'import { defineChannel, GET } from "eve/channels";', + "", + "const workerId = randomUUID();", + "", + "export default defineChannel({", + " routes: [", + ' GET("/chaos/worker-id", () => new Response(workerId)),', + ' GET("/chaos/crash", () => {', + " setTimeout(() => process.exit(7), 25);", + ' return new Response("crashing");', + " }),", + ' GET("/chaos/slow-stream", () => {', + " let timer: ReturnType | undefined;", + " const body = new ReadableStream({", + " start(controller) {", + ' controller.enqueue(new TextEncoder().encode("chunk\\n"));', + " timer = setInterval(() => {", + ' controller.enqueue(new TextEncoder().encode("chunk\\n"));', + " }, 50);", + " },", + " cancel() {", + " clearInterval(timer);", + " },", + " });", + " return new Response(body);", + " }),", + " ],", + "});", + "", +].join("\n"); + +const CHAOS_DESCRIPTOR: ScenarioAppDescriptor = { + ...WEATHER_AGENT_DESCRIPTOR, + files: { + ...WEATHER_AGENT_DESCRIPTOR.files, + "agent/channels/chaos.ts": CHAOS_CHANNEL_SOURCE, + }, + name: "weather-agent-chaos", +}; + +function toolSource(marker: string): string { + return [ + 'import { defineTool } from "eve/tools";', + 'import { z } from "zod";', + "", + "export default defineTool({", + ` description: ${JSON.stringify(`Report the weather (${marker}).`)},`, + " inputSchema: z.object({ city: z.string() }),", + " execute: ({ city }) => `sunny in ${city}`,", + "});", + "", + ].join("\n"); +} + +interface HealthProbe { + readonly failures: readonly string[]; + stop(): Promise; +} + +function startHealthProbe(serverUrl: string): HealthProbe { + const failures: string[] = []; + let probes = 0; + let stopped = false; + const run = (async () => { + while (!stopped) { + probes += 1; + try { + const response = await fetch(new URL(EVE_HEALTH_ROUTE_PATH, serverUrl)); + if (response.status !== 200) { + failures.push(`health returned ${String(response.status)}`); + } + await response.arrayBuffer(); + } catch (error) { + failures.push(`health request failed: ${String(error)}`); + } + await wait(50); + } + })(); + + return { + failures, + async stop() { + stopped = true; + await run; + return probes; + }, + }; +} + +async function completeStreamedTurn(server: RunningEveDev, message: string): Promise { + const result = await sendDevelopmentMessage({ + message, + session: createDevelopmentSessionState(), + serverUrl: server.url, + }); + expect( + result.events.some((event) => event.type === "message.completed"), + [ + "Expected a streamed turn to complete after the chaos sequence.", + `stdout:\n${server.stdout()}`, + `stderr:\n${server.stderr()}`, + ].join("\n\n"), + ).toBe(true); +} + +async function waitForToolMarker(serverUrl: string, marker: string): Promise { + await waitForCondition(async () => { + const info = await fetchAgentInfo(serverUrl); + return info.tools.authored.some((tool) => tool.description?.includes(marker)); + }, `Timed out waiting for the "${marker}" tool revision to publish.`); +} + +describe("eve dev server chaos", () => { + it( + "keeps every request succeeding through an authored edit storm", + async () => { + const app = await scenarioApp(CHAOS_DESCRIPTOR); + const toolPath = join(app.appRoot, "agent", "tools", "get_weather.ts"); + const server = await startEveDev(app.appRoot); + const probe = startHealthProbe(server.url); + + try { + for (let round = 1; round <= 3; round += 1) { + await writeFile(toolPath, toolSource(`storm-${String(round)}-a`)); + await writeFile(toolPath, toolSource(`storm-${String(round)}-b`)); + await writeFile(toolPath, "export default { this is not valid typescript\n"); + await wait(500); + await rm(toolPath); + await writeFile(toolPath, toolSource(`storm-${String(round)}-fixed`)); + await Promise.all([ + forceDevelopmentRebuild(server.url), + forceDevelopmentRebuild(server.url), + ]); + } + + await writeFile(join(app.appRoot, ".env.local"), "EVE_CHAOS_STRUCTURAL=1\n"); + await writeFile(toolPath, toolSource("storm-final")); + await forceDevelopmentRebuild(server.url); + await waitForToolMarker(server.url, "storm-final"); + await completeStreamedTurn(server, "What's the weather in Lisbon?"); + + const probes = await probe.stop(); + expect(probe.failures, probe.failures.join("\n")).toEqual([]); + expect(probes).toBeGreaterThan(50); + expect(hasKnownDevServerFailure(`${server.stdout()}\n${server.stderr()}`)).toBe(false); + } finally { + await probe.stop(); + await server.stop(); + } + }, + CHAOS_SCENARIO_TIMEOUT_MS, + ); + + it( + "recovers through worker crashes, aborted streams, and concurrent rebuilds", + async () => { + const app = await scenarioApp(CHAOS_DESCRIPTOR); + const server = await startEveDev(app.appRoot); + + try { + const firstWorkerId = await fetchText(server.url, "/chaos/worker-id"); + + const abort = new AbortController(); + const streamResponse = await fetch(new URL("/chaos/slow-stream", server.url), { + signal: abort.signal, + }); + const reader = streamResponse.body?.getReader(); + await reader?.read(); + abort.abort(); + + await expect(fetch(new URL("/chaos/crash", server.url))).resolves.toBeDefined(); + await waitForCondition(async () => { + try { + return (await fetchText(server.url, "/chaos/worker-id")) !== firstWorkerId; + } catch { + return false; + } + }, "Timed out waiting for a replacement worker after the crash."); + const burst = await Promise.all( + Array.from({ length: 10 }, async () => { + const response = await fetch(new URL(EVE_HEALTH_ROUTE_PATH, server.url)); + return response.status; + }), + ); + expect(burst).toEqual(Array.from({ length: 10 }, () => 200)); + + await Promise.all([ + fetch(new URL("/chaos/crash", server.url)), + forceDevelopmentRebuild(server.url), + ]); + await waitForCondition(async () => { + try { + const response = await fetch(new URL(EVE_HEALTH_ROUTE_PATH, server.url)); + return response.status === 200; + } catch { + return false; + } + }, "Timed out waiting for the dev server to recover from a crash during rebuild."); + + await completeStreamedTurn(server, "What's the weather in Lisbon?"); + expect(hasKnownDevServerFailure(`${server.stdout()}\n${server.stderr()}`)).toBe(false); + } finally { + await server.stop(); + } + }, + CHAOS_SCENARIO_TIMEOUT_MS, + ); +}); diff --git a/packages/eve/test/scenarios/dev-server-harness.ts b/packages/eve/test/scenarios/dev-server-harness.ts new file mode 100644 index 000000000..d662a0614 --- /dev/null +++ b/packages/eve/test/scenarios/dev-server-harness.ts @@ -0,0 +1,401 @@ +import { spawn, spawnSync, type ChildProcessByStdio } from "node:child_process"; +import { existsSync, watch as watchFileSystem } from "node:fs"; +import { dirname, join } from "node:path"; +import type { Readable } from "node:stream"; + +import { + EVE_DEV_RUNTIME_ARTIFACTS_REBUILD_ROUTE_PATH, + EVE_DEV_RUNTIME_ARTIFACTS_ROUTE_PATH, + EVE_INFO_ROUTE_PATH, +} from "../../src/protocol/routes.js"; +import type { AgentInfoResponse } from "../../src/internal/nitro/routes/agent-info/build-agent-info-response.js"; + +/** + * Handle to a spawned `eve dev --no-ui` process serving a scenario app. + */ +export interface RunningEveDev { + crash(): Promise; + readonly stderr: () => string; + readonly stdout: () => string; + readonly url: string; + stop(): Promise; +} + +export interface StartEveDevOptions { + readonly env?: Readonly>; + /** Runtime executing the CLI. Defaults to the current Node executable. */ + readonly runtime?: "bun" | "node"; +} + +/** + * Spawns `eve dev --no-ui` for the app and resolves once the server URL is + * printed and the state record exists. `NODE_ENV=test` activates the + * deterministic mock-model adapter so streamed turns complete without model + * credentials. + */ +export async function startEveDev( + appRoot: string, + options: StartEveDevOptions = {}, +): Promise { + const child = spawnEveDev(appRoot, options); + let stderr = ""; + let stdout = ""; + + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk: string) => { + stderr += chunk; + }); + + const url = await waitForServerUrl({ + child, + getOutput: () => ({ stderr, stdout }), + }); + await waitForPath(join(appRoot, ".eve", "dev-server-state.v1.json")); + + return { + async crash() { + if (child.exitCode !== null || child.signalCode !== null) { + return; + } + await withinDeadline( + new Promise((resolve) => { + child.once("exit", () => resolve()); + child.kill("SIGKILL"); + }), + "Timed out waiting for the dev server process to crash.", + ); + }, + stderr: () => stderr, + stdout: () => stdout, + async stop() { + if (child.exitCode !== null || child.signalCode !== null) { + return; + } + + await new Promise((resolve) => { + const timeout = setTimeout(() => { + child.kill("SIGKILL"); + resolve(); + }, 10_000); + + child.once("exit", () => { + clearTimeout(timeout); + resolve(); + }); + child.kill("SIGTERM"); + }); + }, + url, + }; +} + +/** + * Spawns `eve dev` and resolves with the process exit for flows that expect + * startup to fail (e.g. unsupported runtimes). + */ +export async function runEveDevToExit( + appRoot: string, + options: StartEveDevOptions = {}, +): Promise<{ readonly code: number | null; readonly output: string }> { + const child = spawnEveDev(appRoot, options); + let output = ""; + + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + output += chunk; + }); + child.stderr.on("data", (chunk: string) => { + output += chunk; + }); + + try { + return await withinDeadline( + new Promise<{ readonly code: number | null; readonly output: string }>((resolve, reject) => { + child.once("error", reject); + child.once("exit", (code) => resolve({ code, output })); + }), + `Timed out waiting for eve dev to exit.\n\noutput:\n${output}`, + 120_000, + ); + } finally { + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGKILL"); + } + } +} + +function spawnEveDev( + appRoot: string, + options: StartEveDevOptions, +): ChildProcessByStdio { + const eveBinPath = join(appRoot, "node_modules", "eve", "bin", "eve.js"); + const command = options.runtime === "bun" ? "bun" : process.execPath; + + return spawn(command, [eveBinPath, "dev", "--no-ui", "--host", "127.0.0.1", "--port", "0"], { + cwd: appRoot, + env: { + ...process.env, + ...options.env, + NODE_ENV: "test", + }, + stdio: ["ignore", "pipe", "pipe"], + }); +} + +/** Reports whether `bun` is runnable on this machine. */ +export function isBunAvailable(): boolean { + try { + return spawnSync("bun", ["--version"], { stdio: "ignore" }).status === 0; + } catch { + return false; + } +} + +export function hasUnsupportedWindowsEsmImport(text: string): boolean { + return ( + text.includes("ERR_UNSUPPORTED_ESM_URL_SCHEME") || + text.includes("Received protocol 'g:'") || + text.includes('Received protocol "g:"') + ); +} + +/** + * Matches output signatures that mean the dev server broke: reset sockets, + * unresolved generated imports, or a pruned module-map loader. + */ +export function hasKnownDevServerFailure(text: string): boolean { + return ( + hasUnsupportedWindowsEsmImport(text) || + text.includes("UNRESOLVED_IMPORT") || + text.includes("ECONNRESET") || + text.includes("socket hang up") || + (text.includes("ERR_MODULE_NOT_FOUND") && text.includes("authored-module-map-loader")) + ); +} + +export async function wait(ms: number): Promise { + await new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + +export async function waitForCondition( + condition: () => boolean | Promise, + failureMessage: string | (() => string), + timeoutMs: number = 60_000, +): Promise { + const deadline = Date.now() + timeoutMs; + + while (!(await condition())) { + if (Date.now() >= deadline) { + throw new Error(typeof failureMessage === "function" ? failureMessage() : failureMessage); + } + await wait(100); + } +} + +export async function withinDeadline( + operation: Promise, + failureMessage: string, + timeoutMs: number = 60_000, +): Promise { + let timeout: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + operation, + new Promise((_resolve, reject) => { + timeout = setTimeout(() => reject(new Error(failureMessage)), timeoutMs); + }), + ]); + } finally { + if (timeout !== undefined) { + clearTimeout(timeout); + } + } +} + +export async function waitForPath(path: string, timeoutMs: number = 60_000): Promise { + if (existsSync(path)) { + return; + } + await new Promise((resolve, reject) => { + let settled = false; + const watcher = watchFileSystem(dirname(path), () => { + if (existsSync(path)) { + settle(resolve); + } + }); + const timeout = setTimeout(() => { + settle(() => reject(new Error(`Timed out waiting for ${path}.`))); + }, timeoutMs); + function settle(complete: () => void) { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + watcher.close(); + complete(); + } + if (existsSync(path)) { + settle(resolve); + } + }); +} + +export async function fetchText(serverUrl: string, path: string): Promise { + const response = await fetch(new URL(path, serverUrl)); + if (!response.ok) { + throw new Error(`Expected ${path} to succeed, received ${String(response.status)}.`); + } + return await response.text(); +} + +export async function fetchAgentInfo(serverUrl: string): Promise { + const response = await fetch(new URL(EVE_INFO_ROUTE_PATH, serverUrl)); + if (!response.ok) { + throw new Error(`Expected agent info to succeed, received ${String(response.status)}.`); + } + return (await response.json()) as AgentInfoResponse; +} + +export async function forceDevelopmentRebuild(serverUrl: string): Promise { + const rebuildUrl = new URL(EVE_DEV_RUNTIME_ARTIFACTS_REBUILD_ROUTE_PATH, serverUrl); + rebuildUrl.searchParams.set("force", "1"); + const response = await fetch(rebuildUrl); + if (!response.ok) { + throw new Error(`Development rebuild failed with status ${String(response.status)}.`); + } +} + +export async function readDevelopmentRevision(serverUrl: string): Promise { + const response = await fetch(new URL(EVE_DEV_RUNTIME_ARTIFACTS_ROUTE_PATH, serverUrl)); + if (!response.ok) { + throw new Error(`Development runtime state failed with status ${String(response.status)}.`); + } + const body = (await response.json()) as { readonly revision?: unknown }; + if (typeof body.revision !== "string") { + throw new Error("Development runtime state did not include a revision."); + } + return body.revision; +} + +function stripAnsi(text: string): string { + return text + .split("\u001b[") + .map((segment, index) => { + if (index === 0) { + return segment; + } + + return segment.replace(/^[0-9;]*m/, ""); + }) + .join(""); +} + +function parseServerUrl(stdout: string): string | undefined { + const match = /server listening at (https?:\/\/\S+)/.exec(stripAnsi(stdout)); + + return match?.[1]; +} + +async function waitForServerUrl(input: { + readonly child: ChildProcessByStdio; + readonly getOutput: () => { + readonly stderr: string; + readonly stdout: string; + }; +}): Promise { + return await new Promise((resolve, reject) => { + let settled = false; + + const timeout = setTimeout(() => { + settleReject( + new Error( + [ + "Timed out waiting for eve dev to print its server URL.", + `stdout:\n${input.getOutput().stdout}`, + `stderr:\n${input.getOutput().stderr}`, + ].join("\n\n"), + ), + ); + }, 120_000); + + const cleanup = () => { + clearTimeout(timeout); + input.child.stdout.off("data", handleOutput); + input.child.stderr.off("data", handleOutput); + input.child.off("error", settleReject); + input.child.off("exit", handleExit); + }; + + const settleResolve = (url: string) => { + if (settled) { + return; + } + + settled = true; + cleanup(); + resolve(url); + }; + + function settleReject(error: unknown) { + if (settled) { + return; + } + + settled = true; + cleanup(); + reject(error); + } + + function handleOutput() { + const output = input.getOutput(); + const combinedOutput = `${output.stdout}\n${output.stderr}`; + + if (hasKnownDevServerFailure(combinedOutput)) { + settleReject( + new Error( + [ + "eve dev emitted a known reload or generated-bundle failure.", + `stdout:\n${output.stdout}`, + `stderr:\n${output.stderr}`, + ].join("\n\n"), + ), + ); + return; + } + + const url = parseServerUrl(output.stdout); + + if (url !== undefined) { + settleResolve(url); + } + } + + function handleExit(code: number | null, signal: NodeJS.Signals | null) { + const output = input.getOutput(); + + settleReject( + new Error( + [ + `eve dev exited before printing its server URL (code ${String(code)}, signal ${String(signal)}).`, + `stdout:\n${output.stdout}`, + `stderr:\n${output.stderr}`, + ].join("\n\n"), + ), + ); + } + + input.child.stdout.on("data", handleOutput); + input.child.stderr.on("data", handleOutput); + input.child.once("error", settleReject); + input.child.once("exit", handleExit); + handleOutput(); + }); +} diff --git a/packages/eve/test/scenarios/dev-server.scenario.test.ts b/packages/eve/test/scenarios/dev-server.scenario.test.ts index 9688d3fb7..f9f94dd61 100644 --- a/packages/eve/test/scenarios/dev-server.scenario.test.ts +++ b/packages/eve/test/scenarios/dev-server.scenario.test.ts @@ -1,18 +1,10 @@ -import { spawn, type ChildProcessByStdio } from "node:child_process"; -import { existsSync, watch as watchFileSystem } from "node:fs"; +import { existsSync } from "node:fs"; import { readFile, rm, writeFile } from "node:fs/promises"; -import { dirname, join } from "node:path"; -import type { Readable } from "node:stream"; +import { join } from "node:path"; import { describe, expect, it } from "vitest"; -import { - EVE_DEV_RUNTIME_ARTIFACTS_REBUILD_ROUTE_PATH, - EVE_DEV_RUNTIME_ARTIFACTS_ROUTE_PATH, - EVE_HEALTH_ROUTE_PATH, - EVE_INFO_ROUTE_PATH, -} from "../../src/protocol/routes.js"; -import type { AgentInfoResponse } from "../../src/internal/nitro/routes/agent-info/build-agent-info-response.js"; +import { EVE_HEALTH_ROUTE_PATH } from "../../src/protocol/routes.js"; import { readDevelopmentRuntimeArtifactsSnapshotRoot, resolveDevelopmentRuntimeArtifactsPointerPath, @@ -25,6 +17,17 @@ import { } from "../../src/internal/testing/scenario-app.js"; import { sendDevelopmentMessage } from "../dev-client-harness/send-message.js"; import { createDevelopmentSessionState } from "../dev-client-harness/session.js"; +import { + fetchAgentInfo, + fetchText, + forceDevelopmentRebuild, + hasKnownDevServerFailure, + readDevelopmentRevision, + startEveDev, + waitForCondition, + waitForPath, + withinDeadline, +} from "./dev-server-harness.js"; // Keep the dev TUI's glyph set deterministic across CI hosts so the // screen assertions below remain stable. @@ -289,168 +292,6 @@ function createBlockedInstrumentationSource(): string { ].join("\n"); } -interface RunningEveDev { - crash(): Promise; - readonly stderr: () => string; - readonly stdout: () => string; - readonly url: string; - stop(): Promise; -} - -function stripAnsi(text: string): string { - return text - .split("\u001b[") - .map((segment, index) => { - if (index === 0) { - return segment; - } - - return segment.replace(/^[0-9;]*m/, ""); - }) - .join(""); -} - -function hasUnsupportedWindowsEsmImport(text: string): boolean { - return ( - text.includes("ERR_UNSUPPORTED_ESM_URL_SCHEME") || - text.includes("Received protocol 'g:'") || - text.includes('Received protocol "g:"') - ); -} - -function hasKnownDevServerFailure(text: string): boolean { - return ( - hasUnsupportedWindowsEsmImport(text) || - text.includes("UNRESOLVED_IMPORT") || - text.includes("ECONNRESET") || - text.includes("socket hang up") || - (text.includes("ERR_MODULE_NOT_FOUND") && text.includes("authored-module-map-loader")) - ); -} - -function parseServerUrl(stdout: string): string | undefined { - const match = /server listening at (https?:\/\/\S+)/.exec(stripAnsi(stdout)); - - return match?.[1]; -} - -async function wait(ms: number): Promise { - await new Promise((resolve) => { - setTimeout(resolve, ms); - }); -} - -async function waitForCondition( - condition: () => boolean | Promise, - failureMessage: string, - timeoutMs: number = 60_000, -): Promise { - const deadline = Date.now() + timeoutMs; - - while (!(await condition())) { - if (Date.now() >= deadline) { - throw new Error(failureMessage); - } - await wait(100); - } -} - -async function waitForServerUrl(input: { - readonly child: ChildProcessByStdio; - readonly getOutput: () => { - readonly stderr: string; - readonly stdout: string; - }; -}): Promise { - return await new Promise((resolve, reject) => { - let settled = false; - - const timeout = setTimeout(() => { - settleReject( - new Error( - [ - "Timed out waiting for eve dev to print its server URL.", - `stdout:\n${input.getOutput().stdout}`, - `stderr:\n${input.getOutput().stderr}`, - ].join("\n\n"), - ), - ); - }, 120_000); - - const cleanup = () => { - clearTimeout(timeout); - input.child.stdout.off("data", handleOutput); - input.child.stderr.off("data", handleOutput); - input.child.off("error", settleReject); - input.child.off("exit", handleExit); - }; - - const settleResolve = (url: string) => { - if (settled) { - return; - } - - settled = true; - cleanup(); - resolve(url); - }; - - function settleReject(error: unknown) { - if (settled) { - return; - } - - settled = true; - cleanup(); - reject(error); - } - - function handleOutput() { - const output = input.getOutput(); - const combinedOutput = `${output.stdout}\n${output.stderr}`; - - if (hasKnownDevServerFailure(combinedOutput)) { - settleReject( - new Error( - [ - "eve dev emitted a known reload or generated-bundle failure.", - `stdout:\n${output.stdout}`, - `stderr:\n${output.stderr}`, - ].join("\n\n"), - ), - ); - return; - } - - const url = parseServerUrl(output.stdout); - - if (url !== undefined) { - settleResolve(url); - } - } - - function handleExit(code: number | null, signal: NodeJS.Signals | null) { - const output = input.getOutput(); - - settleReject( - new Error( - [ - `eve dev exited before printing its server URL (code ${String(code)}, signal ${String(signal)}).`, - `stdout:\n${output.stdout}`, - `stderr:\n${output.stderr}`, - ].join("\n\n"), - ), - ); - } - - input.child.stdout.on("data", handleOutput); - input.child.stderr.on("data", handleOutput); - input.child.once("error", settleReject); - input.child.once("exit", handleExit); - handleOutput(); - }); -} - async function waitForWebSocketOpen(socket: WebSocket): Promise { await waitForWebSocketEvent(socket, "open", () => undefined); } @@ -491,94 +332,6 @@ async function waitForWebSocketEvent( }); } -async function startEveDev(appRoot: string): Promise { - return await startEveDevProcess(appRoot, {}); -} - -async function startEveDevWithBoundedWorkflowRecovery(appRoot: string): Promise { - return await startEveDevProcess(appRoot, { - WORKFLOW_INLINE_OWNERSHIP_LEASE_SECONDS: "1", - }); -} - -async function startEveDevProcess( - appRoot: string, - environment: Readonly>, -): Promise { - const eveBinPath = join(appRoot, "node_modules", "eve", "bin", "eve.js"); - const child = spawn( - process.execPath, - [eveBinPath, "dev", "--no-ui", "--host", "127.0.0.1", "--port", "0"], - { - cwd: appRoot, - env: { - ...process.env, - ...environment, - // Activate the deterministic mock-model adapter in the spawned dev - // server so the streamed turn completes without model credentials. - NODE_ENV: "test", - }, - stdio: ["ignore", "pipe", "pipe"], - }, - ); - let stderr = ""; - let stdout = ""; - - child.stdout.setEncoding("utf8"); - child.stderr.setEncoding("utf8"); - child.stdout.on("data", (chunk: string) => { - stdout += chunk; - }); - child.stderr.on("data", (chunk: string) => { - stderr += chunk; - }); - - const url = await waitForServerUrl({ - child, - getOutput: () => ({ - stderr, - stdout, - }), - }); - await waitForPath(join(appRoot, ".eve", "dev-server-state.v1.json")); - - return { - async crash() { - if (child.exitCode !== null || child.signalCode !== null) { - return; - } - await withinDeadline( - new Promise((resolve) => { - child.once("exit", () => resolve()); - child.kill("SIGKILL"); - }), - "Timed out waiting for the dev server process to crash.", - ); - }, - stderr: () => stderr, - stdout: () => stdout, - async stop() { - if (child.exitCode !== null || child.signalCode !== null) { - return; - } - - await new Promise((resolve) => { - const timeout = setTimeout(() => { - child.kill("SIGKILL"); - resolve(); - }, 10_000); - - child.once("exit", () => { - clearTimeout(timeout); - resolve(); - }); - child.kill("SIGTERM"); - }); - }, - url, - }; -} - describe("eve dev server", () => { it( "publishes authored tool removals without replacing the active host", @@ -940,7 +693,9 @@ describe("eve dev server", () => { "recovers a nonterminal child Workflow on its selected generation after restart", async () => { const app = await scenarioApp(WORKFLOW_GENERATION_DESCRIPTOR); - let server = await startEveDevWithBoundedWorkflowRecovery(app.appRoot); + let server = await startEveDev(app.appRoot, { + env: { WORKFLOW_INLINE_OWNERSHIP_LEASE_SECONDS: "1" }, + }); const turnStartedPath = join(app.appRoot, ".turn-started"); const restartPath = join(app.appRoot, ".restart-generation-test"); const recoveredPath = join(app.appRoot, ".recovered-turn-started"); @@ -974,7 +729,9 @@ describe("eve dev server", () => { await server.crash(); await withinDeadline(interruptedTurn, "Interrupted client stream did not settle."); await writeFile(restartPath, "restart"); - server = await startEveDevWithBoundedWorkflowRecovery(app.appRoot); + server = await startEveDev(app.appRoot, { + env: { WORKFLOW_INLINE_OWNERSHIP_LEASE_SECONDS: "1" }, + }); await waitForPath(recoveredPath); await expect( readFile(recoveredPath, "utf8").then((source) => JSON.parse(source) as unknown), @@ -1071,85 +828,3 @@ function readCompletedMessages( function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } - -async function fetchText(serverUrl: string, path: string): Promise { - const response = await fetch(new URL(path, serverUrl)); - if (!response.ok) { - throw new Error(`Expected ${path} to succeed, received ${String(response.status)}.`); - } - return await response.text(); -} - -async function fetchAgentInfo(serverUrl: string): Promise { - const response = await fetch(new URL(EVE_INFO_ROUTE_PATH, serverUrl)); - if (!response.ok) { - throw new Error(`Expected agent info to succeed, received ${String(response.status)}.`); - } - return (await response.json()) as AgentInfoResponse; -} - -async function forceDevelopmentRebuild(serverUrl: string): Promise { - const rebuildUrl = new URL(EVE_DEV_RUNTIME_ARTIFACTS_REBUILD_ROUTE_PATH, serverUrl); - rebuildUrl.searchParams.set("force", "1"); - const response = await fetch(rebuildUrl); - if (!response.ok) { - throw new Error(`Development rebuild failed with status ${String(response.status)}.`); - } -} - -async function readDevelopmentRevision(serverUrl: string): Promise { - const response = await fetch(new URL(EVE_DEV_RUNTIME_ARTIFACTS_ROUTE_PATH, serverUrl)); - if (!response.ok) { - throw new Error(`Development runtime state failed with status ${String(response.status)}.`); - } - const body = (await response.json()) as { readonly revision?: unknown }; - if (typeof body.revision !== "string") { - throw new Error("Development runtime state did not include a revision."); - } - return body.revision; -} - -async function waitForPath(path: string, timeoutMs: number = 60_000): Promise { - if (existsSync(path)) { - return; - } - await new Promise((resolve, reject) => { - let settled = false; - const watcher = watchFileSystem(dirname(path), () => { - if (existsSync(path)) { - settle(resolve); - } - }); - const timeout = setTimeout(() => { - settle(() => reject(new Error(`Timed out waiting for ${path}.`))); - }, timeoutMs); - function settle(complete: () => void) { - if (settled) { - return; - } - settled = true; - clearTimeout(timeout); - watcher.close(); - complete(); - } - if (existsSync(path)) { - settle(resolve); - } - }); -} - -async function withinDeadline(operation: Promise, failureMessage: string): Promise { - let timeout: NodeJS.Timeout | undefined; - try { - return await Promise.race([ - operation, - new Promise((_resolve, reject) => { - timeout = setTimeout(() => reject(new Error(failureMessage)), 60_000); - }), - ]); - } finally { - if (timeout !== undefined) { - clearTimeout(timeout); - } - } -}