test(dev): add bun and chaos coverage for the dev server

.github/workflows/ci.yml

@@ -138,6 +138,11 @@ jobs:
           node-version-file: .nvmrc
           cache: "pnpm"
 
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
+        with:
+          bun-version: "1.3.14"
+
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 

packages/eve/src/internal/testing/scenario-app.ts

@@ -80,11 +80,11 @@ export interface ScenarioAppDescriptor {
   readonly installDependencies?: boolean;
   /**
    * Package manager used for {@link installDependencies}. Defaults to pnpm.
-   * npm installs are slower but produce the hoisted real-directory
+   * npm and bun installs are slower but produce the hoisted real-directory
    * `node_modules/` layout, which exercises dependency resolution paths that
    * pnpm's symlinked store never hits.
    */
-  readonly packageManager?: "npm" | "pnpm";
+  readonly packageManager?: "bun" | "npm" | "pnpm";
 }
 
 /**
@@ -272,7 +272,17 @@ async function installScenarioDependencies(input: {
   readonly descriptor: ScenarioAppDescriptor;
 }): Promise<void> {
   if (input.descriptor.packageManager === "npm") {
-    await runNpmInstall(input.appRoot);
+    await runInstallCommand(input.appRoot, "npm", [
+      "install",
+      "--no-audit",
+      "--no-fund",
+      "--ignore-scripts",
+      "--prefer-offline",
+    ]);
+    return;
+  }
+  if (input.descriptor.packageManager === "bun") {
+    await runInstallCommand(input.appRoot, "bun", ["install", "--ignore-scripts"]);
     return;
   }
 
@@ -291,11 +301,13 @@ async function installScenarioDependencies(input: {
 
 const runFile = promisify(execFile);
 
-async function runNpmInstall(appRoot: string): Promise<void> {
-  const args = ["install", "--no-audit", "--no-fund", "--ignore-scripts", "--prefer-offline"];
-
+async function runInstallCommand(
+  appRoot: string,
+  command: "bun" | "npm",
+  args: readonly string[],
+): Promise<void> {
   try {
-    await runFile(process.platform === "win32" ? "npm.cmd" : "npm", args, {
+    await runFile(command, [...args], {
       cwd: appRoot,
       maxBuffer: 10 * 1024 * 1024,
       shell: process.platform === "win32",
@@ -304,7 +316,7 @@ async function runNpmInstall(appRoot: string): Promise<void> {
     const failure = error as { readonly stderr?: unknown; readonly stdout?: unknown };
     throw new Error(
       [
-        `Command failed: npm ${args.join(" ")}`,
+        `Command failed: ${command} ${args.join(" ")}`,
         `cwd: ${appRoot}`,
         `stdout:\n${typeof failure.stdout === "string" ? failure.stdout : ""}`,
         `stderr:\n${typeof failure.stderr === "string" ? failure.stderr : ""}`,

packages/eve/test/scenarios/dev-server-bun.scenario.test.ts

@@ -0,0 +1,75 @@
+import { describe, expect, it } from "vitest";
+
+import { WEATHER_AGENT_DESCRIPTOR } from "../../src/internal/testing/scenario-apps/weather-agent.js";
+import {
+  type ScenarioAppDescriptor,
+  useScenarioApp,
+} from "../../src/internal/testing/scenario-app.js";
+import { sendDevelopmentMessage } from "../dev-client-harness/send-message.js";
+import { createDevelopmentSessionState } from "../dev-client-harness/session.js";
+import {
+  hasKnownDevServerFailure,
+  isBunAvailable,
+  runEveDevToExit,
+  startEveDev,
+} from "./dev-server-harness.js";
+
+const scenarioApp = useScenarioApp();
+
+const BUN_DEV_SERVER_TIMEOUT_MS = 360_000;
+const BUN_LAYOUT_DESCRIPTOR: ScenarioAppDescriptor = {
+  ...WEATHER_AGENT_DESCRIPTOR,
+  name: "weather-agent-bun",
+  packageManager: "bun",
+};
+
+const bunAvailable = isBunAvailable();
+
+describe("eve dev server with bun", () => {
+  it.skipIf(!bunAvailable)(
+    "serves a bun-installed app under the Node runtime",
+    async () => {
+      const app = await scenarioApp(BUN_LAYOUT_DESCRIPTOR);
+      const server = await startEveDev(app.appRoot);
+
+      try {
+        const messageResult = await sendDevelopmentMessage({
+          message: "What's the weather in Lisbon?",
+          session: createDevelopmentSessionState(),
+          serverUrl: server.url,
+        });
+        expect(
+          messageResult.events.some((event) => event.type === "message.completed"),
+          [
+            "Expected the bun-installed dev server to complete a streamed turn.",
+            `stdout:\n${server.stdout()}`,
+            `stderr:\n${server.stderr()}`,
+          ].join("\n\n"),
+        ).toBe(true);
+        expect(hasKnownDevServerFailure(`${server.stdout()}\n${server.stderr()}`)).toBe(false);
+      } finally {
+        await server.stop();
+      }
+    },
+    BUN_DEV_SERVER_TIMEOUT_MS,
+  );
+
+  // Bun cannot run the dev server today: Nitro wires crossws's Node adapter
+  // into every dev worker, and that adapter refuses to initialize when the
+  // Bun global exists. Until that support lands upstream, the contract is a
+  // fast, explanatory startup failure instead of a hang or a half-broken
+  // server. Replace this with a boot-and-stream assertion when `bun eve dev`
+  // becomes supported.
+  it.skipIf(!bunAvailable)(
+    "fails fast with the worker readiness error when the CLI runs under bun",
+    async () => {
+      const app = await scenarioApp(BUN_LAYOUT_DESCRIPTOR);
+
+      const result = await runEveDevToExit(app.appRoot, { runtime: "bun" });
+
+      expect(result.code).not.toBe(0);
+      expect(result.output).toContain("failed before readiness");
+    },
+    BUN_DEV_SERVER_TIMEOUT_MS,
+  );
+});

packages/eve/test/scenarios/dev-server-chaos.scenario.test.ts

@@ -0,0 +1,238 @@
+import { rm, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { describe, expect, it } from "vitest";
+
+import { EVE_HEALTH_ROUTE_PATH } from "../../src/protocol/routes.js";
+import { WEATHER_AGENT_DESCRIPTOR } from "../../src/internal/testing/scenario-apps/weather-agent.js";
+import {
+  type ScenarioAppDescriptor,
+  useScenarioApp,
+} from "../../src/internal/testing/scenario-app.js";
+import { sendDevelopmentMessage } from "../dev-client-harness/send-message.js";
+import { createDevelopmentSessionState } from "../dev-client-harness/session.js";
+import {
+  fetchAgentInfo,
+  fetchText,
+  forceDevelopmentRebuild,
+  hasKnownDevServerFailure,
+  startEveDev,
+  wait,
+  waitForCondition,
+  type RunningEveDev,
+} from "./dev-server-harness.js";
+
+const scenarioApp = useScenarioApp();
+
+const CHAOS_SCENARIO_TIMEOUT_MS = 360_000;
+
+const CHAOS_CHANNEL_SOURCE = [
+  'import { randomUUID } from "node:crypto";',
+  'import { defineChannel, GET } from "eve/channels";',
+  "",
+  "const workerId = randomUUID();",
+  "",
+  "export default defineChannel({",
+  "  routes: [",
+  '    GET("/chaos/worker-id", () => new Response(workerId)),',
+  '    GET("/chaos/crash", () => {',
+  "      setTimeout(() => process.exit(7), 25);",
+  '      return new Response("crashing");',
+  "    }),",
+  '    GET("/chaos/slow-stream", () => {',
+  "      let timer: ReturnType<typeof setInterval> | undefined;",
+  "      const body = new ReadableStream({",
+  "        start(controller) {",
+  '          controller.enqueue(new TextEncoder().encode("chunk\\n"));',
+  "          timer = setInterval(() => {",
+  '            controller.enqueue(new TextEncoder().encode("chunk\\n"));',
+  "          }, 50);",
+  "        },",
+  "        cancel() {",
+  "          clearInterval(timer);",
+  "        },",
+  "      });",
+  "      return new Response(body);",
+  "    }),",
+  "  ],",
+  "});",
+  "",
+].join("\n");
+
+const CHAOS_DESCRIPTOR: ScenarioAppDescriptor = {
+  ...WEATHER_AGENT_DESCRIPTOR,
+  files: {
+    ...WEATHER_AGENT_DESCRIPTOR.files,
+    "agent/channels/chaos.ts": CHAOS_CHANNEL_SOURCE,
+  },
+  name: "weather-agent-chaos",
+};
+
+function toolSource(marker: string): string {
+  return [
+    'import { defineTool } from "eve/tools";',
+    'import { z } from "zod";',
+    "",
+    "export default defineTool({",
+    `  description: ${JSON.stringify(`Report the weather (${marker}).`)},`,
+    "  inputSchema: z.object({ city: z.string() }),",
+    "  execute: ({ city }) => `sunny in ${city}`,",
+    "});",
+    "",
+  ].join("\n");
+}
+
+interface HealthProbe {
+  readonly failures: readonly string[];
+  stop(): Promise<number>;
+}
+
+function startHealthProbe(serverUrl: string): HealthProbe {
+  const failures: string[] = [];
+  let probes = 0;
+  let stopped = false;
+  const run = (async () => {
+    while (!stopped) {
+      probes += 1;
+      try {
+        const response = await fetch(new URL(EVE_HEALTH_ROUTE_PATH, serverUrl));
+        if (response.status !== 200) {
+          failures.push(`health returned ${String(response.status)}`);
+        }
+        await response.arrayBuffer();
+      } catch (error) {
+        failures.push(`health request failed: ${String(error)}`);
+      }
+      await wait(50);
+    }
+  })();
+
+  return {
+    failures,
+    async stop() {
+      stopped = true;
+      await run;
+      return probes;
+    },
+  };
+}
+
+async function completeStreamedTurn(server: RunningEveDev, message: string): Promise<void> {
+  const result = await sendDevelopmentMessage({
+    message,
+    session: createDevelopmentSessionState(),
+    serverUrl: server.url,
+  });
+  expect(
+    result.events.some((event) => event.type === "message.completed"),
+    [
+      "Expected a streamed turn to complete after the chaos sequence.",
+      `stdout:\n${server.stdout()}`,
+      `stderr:\n${server.stderr()}`,
+    ].join("\n\n"),
+  ).toBe(true);
+}
+
+async function waitForToolMarker(serverUrl: string, marker: string): Promise<void> {
+  await waitForCondition(async () => {
+    const info = await fetchAgentInfo(serverUrl);
+    return info.tools.authored.some((tool) => tool.description?.includes(marker));
+  }, `Timed out waiting for the "${marker}" tool revision to publish.`);
+}
+
+describe("eve dev server chaos", () => {
+  it(
+    "keeps every request succeeding through an authored edit storm",
+    async () => {
+      const app = await scenarioApp(CHAOS_DESCRIPTOR);
+      const toolPath = join(app.appRoot, "agent", "tools", "get_weather.ts");
+      const server = await startEveDev(app.appRoot);
+      const probe = startHealthProbe(server.url);
+
+      try {
+        for (let round = 1; round <= 3; round += 1) {
+          await writeFile(toolPath, toolSource(`storm-${String(round)}-a`));
+          await writeFile(toolPath, toolSource(`storm-${String(round)}-b`));
+          await writeFile(toolPath, "export default { this is not valid typescript\n");
+          await wait(500);
+          await rm(toolPath);
+          await writeFile(toolPath, toolSource(`storm-${String(round)}-fixed`));
+          await Promise.all([
+            forceDevelopmentRebuild(server.url),
+            forceDevelopmentRebuild(server.url),
+          ]);
+        }
+
+        await writeFile(join(app.appRoot, ".env.local"), "EVE_CHAOS_STRUCTURAL=1\n");
+        await writeFile(toolPath, toolSource("storm-final"));
+        await forceDevelopmentRebuild(server.url);
+        await waitForToolMarker(server.url, "storm-final");
+        await completeStreamedTurn(server, "What's the weather in Lisbon?");
+
+        const probes = await probe.stop();
+        expect(probe.failures, probe.failures.join("\n")).toEqual([]);
+        expect(probes).toBeGreaterThan(50);
+        expect(hasKnownDevServerFailure(`${server.stdout()}\n${server.stderr()}`)).toBe(false);
+      } finally {
+        await probe.stop();
+        await server.stop();
+      }
+    },
+    CHAOS_SCENARIO_TIMEOUT_MS,
+  );
+
+  it(
+    "recovers through worker crashes, aborted streams, and concurrent rebuilds",
+    async () => {
+      const app = await scenarioApp(CHAOS_DESCRIPTOR);
+      const server = await startEveDev(app.appRoot);
+
+      try {
+        const firstWorkerId = await fetchText(server.url, "/chaos/worker-id");
+
+        const abort = new AbortController();
+        const streamResponse = await fetch(new URL("/chaos/slow-stream", server.url), {
+          signal: abort.signal,
+        });
+        const reader = streamResponse.body?.getReader();
+        await reader?.read();
+        abort.abort();
+
+        await expect(fetch(new URL("/chaos/crash", server.url))).resolves.toBeDefined();
+        await waitForCondition(async () => {
+          try {
+            return (await fetchText(server.url, "/chaos/worker-id")) !== firstWorkerId;
+          } catch {
+            return false;
+          }
+        }, "Timed out waiting for a replacement worker after the crash.");
+        const burst = await Promise.all(
+          Array.from({ length: 10 }, async () => {
+            const response = await fetch(new URL(EVE_HEALTH_ROUTE_PATH, server.url));
+            return response.status;
+          }),
+        );
+        expect(burst).toEqual(Array.from({ length: 10 }, () => 200));
+
+        await Promise.all([
+          fetch(new URL("/chaos/crash", server.url)),
+          forceDevelopmentRebuild(server.url),
+        ]);
+        await waitForCondition(async () => {
+          try {
+            const response = await fetch(new URL(EVE_HEALTH_ROUTE_PATH, server.url));
+            return response.status === 200;
+          } catch {
+            return false;
+          }
+        }, "Timed out waiting for the dev server to recover from a crash during rebuild.");
+
+        await completeStreamedTurn(server, "What's the weather in Lisbon?");
+        expect(hasKnownDevServerFailure(`${server.stdout()}\n${server.stderr()}`)).toBe(false);
+      } finally {
+        await server.stop();
+      }
+    },
+    CHAOS_SCENARIO_TIMEOUT_MS,
+  );
+});