diff --git a/.changeset/calm-trees-build.md b/.changeset/calm-trees-build.md new file mode 100644 index 000000000..c4564b37a --- /dev/null +++ b/.changeset/calm-trees-build.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Keep development runtime generations executable after authored source or dependencies change by bundling ordinary dependencies and materializing configured external dependency closures. diff --git a/packages/eve/src/execution/sandbox/prewarm.scenario.test.ts b/packages/eve/src/execution/sandbox/prewarm.scenario.test.ts index d1667deca..a739fafe4 100644 --- a/packages/eve/src/execution/sandbox/prewarm.scenario.test.ts +++ b/packages/eve/src/execution/sandbox/prewarm.scenario.test.ts @@ -6,7 +6,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { compileAgent, compileAgentInBuildWorkspace } from "#compiler/compile-agent.js"; import { resolvePackageSourceFilePath } from "#internal/application/package.js"; import { createDevelopmentNitroArtifactsConfig } from "#internal/nitro/host/artifacts-config.js"; -import { publishDevelopmentRuntimeArtifactsSnapshot } from "#internal/nitro/dev-runtime-artifacts.js"; +import { publishDevelopmentGeneration } from "#internal/nitro/development-generation.js"; import { resolveNitroCompiledArtifactsSource } from "#internal/nitro/routes/runtime-artifacts.js"; import { useTemporaryDirectories } from "#internal/testing/use-temporary-app-roots.js"; import type { @@ -99,7 +99,7 @@ describe("prewarmAppSandboxes", () => { const compileResult = await compileAgent({ startPath: appRoot, }); - await publishDevelopmentRuntimeArtifactsSnapshot(compileResult); + await publishDevelopmentGeneration(compileResult); await prewarmAppSandboxes({ appRoot, diff --git a/packages/eve/src/internal/application/compiled-artifacts.ts b/packages/eve/src/internal/application/compiled-artifacts.ts index 843405f66..9554ea909 100644 --- a/packages/eve/src/internal/application/compiled-artifacts.ts +++ b/packages/eve/src/internal/application/compiled-artifacts.ts @@ -1,5 +1,5 @@ import { existsSync } from "node:fs"; -import { mkdir, writeFile } from "node:fs/promises"; +import { copyFile, mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; import type { CompileMetadata } from "#compiler/artifacts.js"; @@ -12,6 +12,7 @@ import { resolvePackageSourceFilePath, } from "#internal/application/package.js"; import type { AgentWorkflowWorldDefinition } from "#shared/agent-definition.js"; +import { readMaterializedAuthoredModuleIndex } from "#internal/materialized-authored-modules.js"; export type BuiltInWorkflowWorldTarget = "local" | "vercel"; @@ -99,6 +100,79 @@ export async function writeCompiledArtifactsFiles(input: { return generatedArtifacts; } +// The dev host's Nitro inputs outlive any single generation, so nothing +// written here may point into authored source or a prunable snapshot: the +// bootstrap references no authored module, and the instrumentation bundle is +// copied out of the generation into the stable host directory. +export async function writeDevelopmentCompiledArtifactsFiles(input: { + readonly compileResult: CompileAgentResult; + readonly outDir: string; + readonly runtimeAppRoot: string; +}): Promise { + const bootstrapPath = join(input.outDir, "compiled-artifacts-bootstrap.mjs"); + const instrumentationPluginPath = join(input.outDir, "compiled-artifacts-instrumentation.mjs"); + const instrumentationSourcePath = join( + input.outDir, + "compiled-artifacts-instrumentation-source.mjs", + ); + const workflowWorldPluginPath = join(input.outDir, "compiled-artifacts-workflow-world.mjs"); + const materializedIndex = await readMaterializedAuthoredModuleIndex(input.runtimeAppRoot); + + if (materializedIndex === undefined) { + throw new Error(`Development generation at "${input.runtimeAppRoot}" is not materialized.`); + } + + await mkdir(input.outDir, { recursive: true }); + await writeFile( + bootstrapPath, + createDevelopmentCompiledArtifactsBootstrapSource(input.compileResult.manifest.config.name), + ); + await writeFile( + workflowWorldPluginPath, + createWorkflowWorldPluginSource({ + compiledArtifactsBootstrapPath: bootstrapPath, + configuredWorld: input.compileResult.manifest.config.experimental?.workflow?.world, + defaultWorld: "local", + }), + ); + + const generatedArtifacts: GeneratedCompiledArtifactsFiles = { + bootstrapPath, + workflowWorldPluginPath, + }; + + if (materializedIndex.instrumentation !== undefined) { + await copyFile( + join(input.runtimeAppRoot, ".eve", "compile", materializedIndex.instrumentation), + instrumentationSourcePath, + ); + await writeFile( + instrumentationPluginPath, + createInstrumentationPluginSource({ + agentName: input.compileResult.manifest.config.name, + instrumentationPath: instrumentationSourcePath, + registerConfigPath: resolvePackageSourceFilePath("src/harness/instrumentation-config.ts"), + }), + ); + generatedArtifacts.instrumentationPluginPath = instrumentationPluginPath; + generatedArtifacts.instrumentationSourcePath = instrumentationSourcePath; + } + + return generatedArtifacts; +} + +function createDevelopmentCompiledArtifactsBootstrapSource(agentName: string): string { + return [ + "// Generated by eve. Do not edit by hand.", + `import { installEveWorkflowQueueNamespace } from ${stringifyEsmImportSpecifier(resolvePackageSourceFilePath("src/internal/workflow/queue-namespace.ts"))};`, + "", + `installEveWorkflowQueueNamespace(${JSON.stringify(agentName)});`, + "", + "export default function installDevelopmentCompiledArtifactsPlugin() {}", + "", + ].join("\n"); +} + const INSTRUMENTATION_EXTENSIONS = [".ts", ".mts", ".js", ".mjs"]; /** diff --git a/packages/eve/src/internal/authored-directive-prologue.test.ts b/packages/eve/src/internal/authored-directive-prologue.test.ts new file mode 100644 index 000000000..b6e579b0f --- /dev/null +++ b/packages/eve/src/internal/authored-directive-prologue.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; + +import { assertNoWorkflowDirectivePrologue } from "#internal/authored-directive-prologue.js"; + +describe("assertNoWorkflowDirectivePrologue", () => { + it.each(["use step", "use workflow"])("rejects an authored %s directive", async (directive) => { + await expect( + assertNoWorkflowDirectivePrologue({ + filePath: "/app/agent/tools/example.ts", + source: `'use strict';\n${JSON.stringify(directive)};\nexport const value = 1;\n`, + }), + ).rejects.toThrow(new RegExp(`actual .*${directive}.* directive`, "u")); + }); + + it("accepts comments and ordinary strings containing directive text", async () => { + await expect( + assertNoWorkflowDirectivePrologue({ + filePath: "/app/agent/tools/example.ts", + source: [ + '// "use step" is documentation, not a directive.', + 'const message = "use workflow";', + '"use step";', + "export { message };", + "", + ].join("\n"), + }), + ).resolves.toBeUndefined(); + }); +}); diff --git a/packages/eve/src/internal/authored-directive-prologue.ts b/packages/eve/src/internal/authored-directive-prologue.ts new file mode 100644 index 000000000..331f7118c --- /dev/null +++ b/packages/eve/src/internal/authored-directive-prologue.ts @@ -0,0 +1,35 @@ +import { parseWithNitroRolldownAst } from "#internal/bundler/nitro-rolldown.js"; + +const UNSUPPORTED_WORKFLOW_DIRECTIVES = new Set(["use step", "use workflow"]); + +interface DirectiveStatement { + readonly directive?: unknown; +} + +interface Program { + readonly body?: unknown; +} + +export async function assertNoWorkflowDirectivePrologue(input: { + readonly filePath: string; + readonly source: string; +}): Promise { + const program = (await parseWithNitroRolldownAst(input.filePath, input.source)) as Program; + + if (!Array.isArray(program.body)) { + throw new Error(`Failed to parse authored module "${input.filePath}".`); + } + + for (const statement of program.body as DirectiveStatement[]) { + if (typeof statement.directive !== "string") { + break; + } + + if (UNSUPPORTED_WORKFLOW_DIRECTIVES.has(statement.directive)) { + throw new Error( + `Authored module "${input.filePath}" contains an actual "${statement.directive}" directive. ` + + "Workflow directives are reserved for eve-generated workflow entrypoints.", + ); + } + } +} diff --git a/packages/eve/src/internal/authored-module-loader.ts b/packages/eve/src/internal/authored-module-loader.ts index a7698b7ac..d7b873c2c 100644 --- a/packages/eve/src/internal/authored-module-loader.ts +++ b/packages/eve/src/internal/authored-module-loader.ts @@ -1,18 +1,28 @@ import { createHash } from "node:crypto"; -import { existsSync, mkdirSync, realpathSync, statSync, writeFileSync } from "node:fs"; -import { createRequire } from "node:module"; -import { dirname, isAbsolute, join, resolve, sep } from "node:path"; +import { existsSync, mkdirSync, statSync, writeFileSync } from "node:fs"; +import { dirname, isAbsolute, join, resolve } from "node:path"; import { createAuthoredAssetImportPlugin } from "#internal/authored-asset-import-plugin.js"; +import { assertNoWorkflowDirectivePrologue } from "#internal/authored-directive-prologue.js"; import { createAuthoredModuleBundleError } from "#internal/authored-module-bundle.js"; import { createAuthoredPackageTsConfigPathsPlugin } from "#internal/authored-package-tsconfig-paths.js"; import { createFixedNamespaceScopePlugin } from "#internal/bundler/extension-scope-plugin.js"; +import { + CACHED_CHANNEL_PREFIX, + RESOLVE_EXTENSIONS, + createGenerationPackageBoundaryPlugin, + createRuntimeLoaderPackageBoundaryPlugin, + isNodeModulesPath, + isPathImport, + normalizeExternalDependencies, + type RolldownResolveContext, +} from "#internal/authored-package-boundary.js"; import { expectObjectRecord } from "#internal/authored-module.js"; import { buildWithNitroRolldown, getSingleRolldownChunk, } from "#internal/bundler/nitro-rolldown.js"; -import { SERVER_EXTERNAL_PACKAGES } from "#internal/nitro/host/server-external-packages.js"; +import type { ResolvedAuthoredExternalModule } from "#internal/materialize-authored-external-dependencies.js"; import { createNodeEsmCompatBannerPlugin } from "#internal/node-esm-compat-banner.js"; const AUTHORED_BUNDLED_MODULE_EXTENSION = /\.[cm]?[jt]sx?$/; @@ -22,32 +32,7 @@ const AUTHORED_MODULE_BUNDLE_DIRECTORY_PATH = join( "eve", "authored-modules", ); -const RESOLVE_EXTENSIONS = [ - ".ts", - ".tsx", - ".mts", - ".cts", - ".js", - ".jsx", - ".mjs", - ".cjs", - ".json", -] as const; - const CHANNEL_MODULE_CACHE_KEY = "__eveChannelModuleCache__"; -const CACHED_CHANNEL_PREFIX = "eve-cached-channel:"; - -type RolldownResolveResult = { - readonly id: string; -}; - -type RolldownResolveContext = { - resolve( - source: string, - importer: string | undefined, - options: { kind: string; skipSelf: boolean }, - ): Promise; -}; export interface AuthoredModuleLoadOptions { readonly externalDependencies?: readonly string[]; @@ -59,6 +44,11 @@ export interface AuthoredModuleLoadOptions { readonly extensionScopeNamespace?: string; } +export interface AuthoredGenerationModuleBundle { + readonly code: string; + readonly externalModules: readonly ResolvedAuthoredExternalModule[]; +} + function getChannelModuleCache(): Map | undefined { return (globalThis as Record)[CHANNEL_MODULE_CACHE_KEY] as | Map @@ -147,20 +137,78 @@ function createFileImportSpecifier(modulePath: string): string { } /** - * Bundles one authored entry module to a self-contained ESM string using the - * same plugin stack the dev/eval loader uses: `eve/*` and node_modules deps stay - * external, relative source is inlined, and (when `extensionScopeNamespace` is - * set) `defineState`/`defineExtension` are scoped to that namespace. Shared with - * `eve extension build`'s entrypoint compilation so both paths bundle identically. + * Bundles one authored entry for immediate dev/eval loading. Package dependencies + * remain external while relative authored source is inlined. */ export async function bundleAuthoredModuleCode( modulePath: string, options: AuthoredModuleLoadOptions = {}, ): Promise { - const channelCache = getChannelModuleCache(); + return await buildAuthoredModuleBundle(modulePath, options, { + channelIdentity: true, + packageBoundaryPlugin: createRuntimeLoaderPackageBoundaryPlugin({ + externalDependencies: normalizeExternalDependencies(options.externalDependencies), + packageRoot: resolveAuthoredPackageRoot(modulePath), + }), + plugins: [], + sourcemap: "inline", + }); +} + +/** + * Bundles one authored entry for an immutable development generation. Ordinary + * package dependencies are inlined so the emitted code stays executable after + * the original workspace changes; framework runtime imports and configured + * external dependencies stay external, and every configured external the + * bundle references is reported for closure materialization. + */ +export async function bundleAuthoredModuleForGeneration( + modulePath: string, + options: AuthoredModuleLoadOptions = {}, +): Promise { + const externalModules = new Map(); + const code = await buildAuthoredModuleBundle(modulePath, options, { + // Generation bundles must not reference process state: the channel + // identity plugin emits reads of a process-global cache keyed by live + // source paths, which an immutable retained artifact cannot depend on. + channelIdentity: false, + packageBoundaryPlugin: createGenerationPackageBoundaryPlugin({ + externalDependencies: normalizeExternalDependencies(options.externalDependencies), + packageRoot: resolveAuthoredPackageRoot(modulePath), + recordExternalModule(externalModule) { + externalModules.set( + `${externalModule.packageName}\0${externalModule.resolvedId}`, + externalModule, + ); + }, + }), + plugins: [createAuthoredDirectiveGuardPlugin()], + sourcemap: false, + }); + + return { + code: removeRolldownModuleRegionComments(code), + externalModules: [...externalModules.values()].sort((left, right) => + `${left.packageName}\0${left.resolvedId}`.localeCompare( + `${right.packageName}\0${right.resolvedId}`, + ), + ), + }; +} + +async function buildAuthoredModuleBundle( + modulePath: string, + options: AuthoredModuleLoadOptions, + configuration: { + readonly channelIdentity: boolean; + readonly packageBoundaryPlugin: Record; + readonly plugins: readonly Record[]; + readonly sourcemap: false | "inline"; + }, +): Promise { + const channelCache = configuration.channelIdentity ? getChannelModuleCache() : undefined; const packageRoot = resolveAuthoredPackageRoot(modulePath); const tsconfigPath = resolveAuthoredTsConfigPath(packageRoot); - const externalDependencies = normalizeExternalDependencies(options.externalDependencies); const channelIdentityPlugin = channelCache && channelCache.size > 0 ? { @@ -210,6 +258,7 @@ export async function bundleAuthoredModuleCode( : null; const plugins = [ channelIdentityPlugin, + ...configuration.plugins, options.extensionScopeNamespace === undefined ? null : createFixedNamespaceScopePlugin(options.extensionScopeNamespace), @@ -220,7 +269,7 @@ export async function bundleAuthoredModuleCode( extensions: RESOLVE_EXTENSIONS, }), createNodeEsmCompatBannerPlugin({ includeRequire: true }), - createPackageBoundaryPlugin(packageRoot, externalDependencies), + configuration.packageBoundaryPlugin, ].filter((plugin) => plugin !== null); try { @@ -237,7 +286,7 @@ export async function bundleAuthoredModuleCode( output: { comments: false, format: "esm", - sourcemap: "inline", + sourcemap: configuration.sourcemap, }, }); return getSingleRolldownChunk(result, `authored module for "${modulePath}"`).code; @@ -246,6 +295,27 @@ export async function bundleAuthoredModuleCode( } } +function createAuthoredDirectiveGuardPlugin(): Record { + return { + name: "eve-authored-directive-guard", + async transform(source: string, id: string) { + if (!AUTHORED_BUNDLED_MODULE_EXTENSION.test(id) || isNodeModulesPath(id)) { + return undefined; + } + + await assertNoWorkflowDirectivePrologue({ filePath: id, source }); + return undefined; + }, + }; +} + +function removeRolldownModuleRegionComments(code: string): string { + return code + .split("\n") + .filter((line) => !line.startsWith("//#region ") && line !== "//#endregion") + .join("\n"); +} + async function loadBundledAuthoredModule( modulePath: string, options: AuthoredModuleLoadOptions, @@ -303,126 +373,6 @@ function createAuthoredRelativeExtensionResolverPlugin(input: { }; } -function createPackageBoundaryPlugin( - packageRoot: string, - externalDependencies: readonly string[], -): Record { - // The bundler reports importers by realpath while `packageRoot` keeps the - // caller's spelling (e.g. macOS `/var` vs `/private/var`); compare - // canonical paths or the app-authored branch is skipped silently. - const canonicalPackageRoot = toCanonicalPath(packageRoot); - - return { - name: "eve-package-boundary", - async resolveId( - this: RolldownResolveContext, - source: string, - importer: string | undefined, - options: { kind: string }, - ) { - if (!isPackageImport(source)) { - return undefined; - } - - if (isEveFrameworkImport(source)) { - return { - external: true, - id: source, - }; - } - - const configuredExternalDependency = resolveConfiguredExternalDependency( - source, - externalDependencies, - ); - - if (configuredExternalDependency !== undefined) { - if (source !== configuredExternalDependency) { - const resolved = await this.resolve(source, importer, { - kind: options.kind, - skipSelf: true, - }); - - if (resolved !== null && typeof resolved.id === "string") { - return { - external: true, - id: resolveExternalFilePath({ - importer, - packageRoot, - resolvedId: resolved.id, - source, - }), - }; - } - - const resolvedSubpath = resolveExternalFilePath({ - importer, - packageRoot, - source, - }); - - if (resolvedSubpath !== undefined) { - return { - external: true, - id: resolvedSubpath, - }; - } - } - - return { - external: true, - id: source, - }; - } - - const importerPath = - importer === undefined || - importer.startsWith("\0") || - importer.startsWith(CACHED_CHANNEL_PREFIX) - ? undefined - : resolve(importer); - - // Keep package imports authored directly by the app external by - // default, but let symlinked/file workspace packages compile as - // source. Those packages often export `.ts` files and rely on the - // bundler's extension resolution for their own relative imports. - if ( - importerPath !== undefined && - isPathInsideOrEqual(toCanonicalPath(importerPath), canonicalPackageRoot) - ) { - const resolved = await this.resolve(source, importer, { - kind: options.kind, - skipSelf: true, - }); - - if (resolved === null || typeof resolved.id !== "string") { - // Failing here (instead of emitting the bare specifier as an - // external) is load-bearing: importing a bundle whose package is - // missing poisons Node's process-wide package-config cache with a - // negative entry, and once the package is installed the same - // long-running process keeps failing resolution until restart. - // The bundler's resolver is fresh on every rebuild, so failing at - // bundle time keeps the dev server able to recover after install. - throw new Error( - `Cannot resolve package "${source}" imported from "${importerPath}". ` + - `Install it with your package manager (e.g. \`pnpm install\`); ` + - `a running \`eve dev\` retries on the next rebuild.`, - ); - } - - if (isNodeModulesPath(resolved.id)) { - return { - external: true, - id: source, - }; - } - } - - return undefined; - }, - }; -} - function createInFlightModuleLoadKey( modulePath: string, options: AuthoredModuleLoadOptions, @@ -432,66 +382,6 @@ function createInFlightModuleLoadKey( return `${modulePath}\0${externalDependencies.join("\0")}\0${options.extensionScopeNamespace ?? ""}`; } -function normalizeExternalDependencies(externalDependencies: readonly string[] = []): string[] { - return [...new Set([...SERVER_EXTERNAL_PACKAGES, ...externalDependencies])].sort(); -} - -function resolveConfiguredExternalDependency( - source: string, - externalDependencies: readonly string[], -): string | undefined { - return externalDependencies.find( - (dependencyName) => source === dependencyName || source.startsWith(`${dependencyName}/`), - ); -} - -function resolveExternalFilePath(input: { - importer: string | undefined; - packageRoot: string; - resolvedId?: string; - source: string; -}): string | undefined { - if (input.resolvedId !== undefined) { - const resolvedPath = resolveExistingExternalFilePath(input.resolvedId); - - if (resolvedPath !== undefined) { - return resolvedPath; - } - } - - const importerPath = normalizeImporterPath(input.importer); - - if (importerPath !== undefined) { - try { - return createRequire(importerPath).resolve(input.source); - } catch { - // Fall back to the app package root below. - } - } - - try { - return createRequire(join(input.packageRoot, "package.json")).resolve(input.source); - } catch { - return input.resolvedId; - } -} - -function resolveExistingExternalFilePath(id: string): string | undefined { - if (existsSync(id)) { - return id; - } - - for (const extension of RESOLVE_EXTENSIONS) { - const candidate = `${id}${extension}`; - - if (existsSync(candidate)) { - return candidate; - } - } - - return undefined; -} - function resolveExistingImportPath( path: string, extensions: readonly string[], @@ -527,63 +417,6 @@ function isFile(path: string): boolean { } } -function normalizeImporterPath(importer: string | undefined): string | undefined { - if ( - importer === undefined || - importer.startsWith("\0") || - importer.startsWith(CACHED_CHANNEL_PREFIX) - ) { - return undefined; - } - - return resolve(importer); -} - -function isPackageImport(source: string): boolean { - if (isPathImport(source)) { - return false; - } - - if (/^(?:node|data|file):/.test(source)) { - return false; - } - - if (source.startsWith("@/")) { - return false; - } - - return !source.startsWith(CACHED_CHANNEL_PREFIX); -} - -function isPathImport(source: string): boolean { - return source.startsWith(".") || source.startsWith("/") || /^[A-Za-z]:[\\/]/.test(source); -} - -function isEveFrameworkImport(source: string): boolean { - return source === "eve" || source.startsWith("eve/"); -} - -function isNodeModulesPath(path: string): boolean { - return path.replaceAll("\\", "/").includes("/node_modules/"); -} - -function toCanonicalPath(path: string): string { - try { - return realpathSync(path); - } catch { - return resolve(path); - } -} - -function isPathInsideOrEqual(path: string, directory: string): boolean { - const resolvedPath = resolve(path); - const resolvedDirectory = resolve(directory); - - return ( - resolvedPath === resolvedDirectory || resolvedPath.startsWith(`${resolvedDirectory}${sep}`) - ); -} - function resolveAuthoredTsConfigPath(packageRoot: string): string | false { for (const fileName of ["tsconfig.json", "jsconfig.json"]) { const path = join(packageRoot, fileName); diff --git a/packages/eve/src/internal/authored-module-map-loader.ts b/packages/eve/src/internal/authored-module-map-loader.ts index 2ec5d3ec8..787b58ca6 100644 --- a/packages/eve/src/internal/authored-module-map-loader.ts +++ b/packages/eve/src/internal/authored-module-map-loader.ts @@ -1,4 +1,5 @@ import { join } from "node:path"; +import { pathToFileURL } from "node:url"; import type { CompiledAgentManifest, CompiledAgentNodeManifest } from "#compiler/manifest.js"; import { ROOT_COMPILED_AGENT_NODE_ID } from "#compiler/manifest.js"; @@ -6,6 +7,8 @@ import { collectModuleRefsForManifest, type CompiledModuleMap } from "#compiler/ import type { RuntimeDiskCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js"; import { loadCompiledManifest } from "#runtime/loaders/manifest.js"; import { loadAuthoredModuleNamespace } from "#internal/authored-module-loader.js"; +import { expectObjectRecord } from "#internal/authored-module.js"; +import { readMaterializedAuthoredModuleIndex } from "#internal/materialized-authored-modules.js"; /** * Ambient namespace read by `defineExtension` when it is evaluated from a module @@ -28,7 +31,10 @@ export async function loadCompiledModuleMapFromAuthoredSource(input: { compiledArtifactsSource: input.compiledArtifactsSource, }); - return await hydrateCompiledModuleMapFromManifest(manifest); + return await hydrateCompiledModuleMapFromManifest( + manifest, + input.compiledArtifactsSource.appRoot, + ); } interface ExtensionScopeIndex { @@ -40,6 +46,7 @@ interface ExtensionScopeIndex { async function hydrateCompiledModuleMapFromManifest( manifest: CompiledAgentManifest, + runtimeAppRoot: string, ): Promise { const nodes: CompiledModuleMap["nodes"] = {}; const nodeManifests: Array<{ @@ -69,12 +76,22 @@ async function hydrateCompiledModuleMapFromManifest( manifest.extensionMounts.map((mount) => [mount.mountSourceId, mount.packageNamespace]), ), }; + const materializedIndex = await readMaterializedAuthoredModuleIndex(runtimeAppRoot); for (const nodeManifest of nodeManifests) { + const materializedModules = materializedIndex?.nodes[nodeManifest.nodeId]?.modules; + if (materializedIndex !== undefined && materializedModules === undefined) { + throw new Error( + `Materialized authored module index is missing node "${nodeManifest.nodeId}".`, + ); + } + nodes[nodeManifest.nodeId] = { modules: await hydrateCompiledNodeScope({ agentRoot: nodeManifest.agentRoot, manifest: nodeManifest.manifest, + materializedModules, + runtimeAppRoot, scopeIndex, }), }; @@ -100,6 +117,8 @@ function extensionNamespaceForSourceId( async function hydrateCompiledNodeScope(input: { agentRoot: string; manifest: CompiledAgentNodeManifest; + materializedModules?: Readonly>; + runtimeAppRoot: string; scopeIndex: ExtensionScopeIndex; }): Promise { const refs = collectModuleRefsForManifest(input.manifest).sort((left, right) => @@ -123,6 +142,21 @@ async function hydrateCompiledNodeScope(input: { container[EXT_CONFIG_SCOPE] = mountConfigScope; } try { + const materializedPath = input.materializedModules?.[ref.sourceId]; + if (input.materializedModules !== undefined && materializedPath === undefined) { + throw new Error(`Materialized authored module index is missing source "${ref.sourceId}".`); + } + + if (materializedPath !== undefined) { + modules[ref.sourceId] = expectObjectRecord( + await import( + `${pathToFileURL(join(input.runtimeAppRoot, ".eve", "compile", materializedPath)).href}?generation=${encodeURIComponent(materializedPath)}` + ), + `Expected materialized authored module "${ref.sourceId}" to export a namespace object.`, + ); + continue; + } + modules[ref.sourceId] = await loadAuthoredModuleNamespace(modulePath, { externalDependencies, extensionScopeNamespace, diff --git a/packages/eve/src/internal/authored-package-boundary.ts b/packages/eve/src/internal/authored-package-boundary.ts new file mode 100644 index 000000000..28c96d3bb --- /dev/null +++ b/packages/eve/src/internal/authored-package-boundary.ts @@ -0,0 +1,278 @@ +import { existsSync, realpathSync } from "node:fs"; +import { join, resolve, sep } from "node:path"; + +import type { ResolvedAuthoredExternalModule } from "#internal/materialize-authored-external-dependencies.js"; +import { SERVER_EXTERNAL_PACKAGES } from "#internal/nitro/host/server-external-packages.js"; + +export const CACHED_CHANNEL_PREFIX = "eve-cached-channel:"; + +export const RESOLVE_EXTENSIONS = [ + ".ts", + ".tsx", + ".mts", + ".cts", + ".js", + ".jsx", + ".mjs", + ".cjs", + ".json", +] as const; + +type RolldownResolveResult = { + readonly id: string; +}; + +export type RolldownResolveContext = { + resolve( + source: string, + importer: string | undefined, + options: { kind: string; skipSelf: boolean }, + ): Promise; +}; + +export function createGenerationPackageBoundaryPlugin(input: { + readonly externalDependencies: readonly string[]; + readonly packageRoot: string; + readonly recordExternalModule: (externalModule: ResolvedAuthoredExternalModule) => void; +}): Record { + return { + name: "eve-generation-package-boundary", + async resolveId( + this: RolldownResolveContext, + source: string, + importer: string | undefined, + options: { kind: string }, + ) { + if (!isPackageImport(source)) { + return undefined; + } + + if (isFrameworkRuntimeImport(source, importer)) { + return { + external: true, + id: source, + }; + } + + const externalModule = await resolveConfiguredExternalModule.call(this, { + externalDependencies: input.externalDependencies, + importer, + kind: options.kind, + packageRoot: input.packageRoot, + source, + }); + if (externalModule === undefined) { + return undefined; + } + + input.recordExternalModule(externalModule); + return { external: true, id: source }; + }, + }; +} + +export function createRuntimeLoaderPackageBoundaryPlugin(input: { + readonly externalDependencies: readonly string[]; + readonly packageRoot: string; +}): Record { + const canonicalPackageRoot = toCanonicalPath(input.packageRoot); + + return { + name: "eve-runtime-loader-package-boundary", + async resolveId( + this: RolldownResolveContext, + source: string, + importer: string | undefined, + options: { kind: string }, + ) { + if (!isPackageImport(source)) { + return undefined; + } + + if (isFrameworkRuntimeImport(source, importer)) { + return { external: true, id: source }; + } + + const externalModule = await resolveConfiguredExternalModule.call(this, { + externalDependencies: input.externalDependencies, + importer, + kind: options.kind, + packageRoot: input.packageRoot, + source, + }); + if (externalModule !== undefined) { + return { + external: true, + id: + resolveExistingExternalFilePath(externalModule.resolvedId) ?? externalModule.resolvedId, + }; + } + + const importerPath = + importer === undefined || + importer.startsWith("\0") || + importer.startsWith(CACHED_CHANNEL_PREFIX) + ? undefined + : resolve(importer); + + // Keep package imports authored directly by the app external by + // default, but let symlinked/file workspace packages compile as + // source. Those packages often export `.ts` files and rely on the + // bundler's extension resolution for their own relative imports. + if ( + importerPath !== undefined && + isPathInsideOrEqual(toCanonicalPath(importerPath), canonicalPackageRoot) + ) { + const resolved = await this.resolve(source, importer, { + kind: options.kind, + skipSelf: true, + }); + + if (resolved === null || typeof resolved.id !== "string") { + // Failing here (instead of emitting the bare specifier as an + // external) is load-bearing: importing a bundle whose package is + // missing poisons Node's process-wide package-config cache with a + // negative entry, and once the package is installed the same + // long-running process keeps failing resolution until restart. + // The bundler's resolver is fresh on every rebuild, so failing at + // bundle time keeps the dev server able to recover after install. + throw new Error( + `Cannot resolve package "${source}" imported from "${importerPath}". ` + + `Install it with your package manager (e.g. \`pnpm install\`); ` + + `a running \`eve dev\` retries on the next rebuild.`, + ); + } + + if (isNodeModulesPath(resolved.id)) { + return { + external: true, + id: source, + }; + } + } + + return undefined; + }, + }; +} + +async function resolveConfiguredExternalModule( + this: RolldownResolveContext, + input: { + readonly externalDependencies: readonly string[]; + readonly importer: string | undefined; + readonly kind: string; + readonly packageRoot: string; + readonly source: string; + }, +): Promise { + const packageName = resolveConfiguredExternalDependency(input.source, input.externalDependencies); + if (packageName === undefined) { + return undefined; + } + + let resolved = await this.resolve(input.source, input.importer, { + kind: input.kind, + skipSelf: true, + }); + if (resolved === null) { + resolved = await this.resolve(input.source, join(input.packageRoot, "package.json"), { + kind: input.kind, + skipSelf: true, + }); + } + if (resolved === null || typeof resolved.id !== "string") { + throw new Error(`Cannot resolve external package "${input.source}".`); + } + + return { packageName, resolvedId: resolved.id }; +} + +export function normalizeExternalDependencies( + externalDependencies: readonly string[] = [], +): string[] { + return [...new Set([...SERVER_EXTERNAL_PACKAGES, ...externalDependencies])].sort(); +} + +function resolveConfiguredExternalDependency( + source: string, + externalDependencies: readonly string[], +): string | undefined { + return externalDependencies.find( + (dependencyName) => source === dependencyName || source.startsWith(`${dependencyName}/`), + ); +} + +function resolveExistingExternalFilePath(id: string): string | undefined { + if (existsSync(id)) { + return id; + } + + for (const extension of RESOLVE_EXTENSIONS) { + const candidate = `${id}${extension}`; + + if (existsSync(candidate)) { + return candidate; + } + } + + return undefined; +} + +function isPackageImport(source: string): boolean { + if (isPathImport(source)) { + return false; + } + + if (/^(?:node|data|file):/.test(source)) { + return false; + } + + if (source.startsWith("@/")) { + return false; + } + + return !source.startsWith(CACHED_CHANNEL_PREFIX); +} + +export function isPathImport(source: string): boolean { + return source.startsWith(".") || source.startsWith("/") || /^[A-Za-z]:[\\/]/.test(source); +} + +function isFrameworkRuntimeImport(source: string, importer: string | undefined): boolean { + if (source === "eve" || source.startsWith("eve/")) { + return true; + } + + // Workflow runtime imports in authored source must bind to the + // process-shared workflow runtime. Third-party packages inlined into a + // bundle keep their own copies instead: eve vendors `@workflow/*`, so a + // bare transitive import (e.g. `@ai-sdk/provider-utils` → `@workflow/serde`) + // is not resolvable from a materialized generation. + if (source === "workflow" || source.startsWith("workflow/") || source.startsWith("@workflow/")) { + return importer === undefined || !isNodeModulesPath(importer); + } + + return false; +} + +export function isNodeModulesPath(path: string): boolean { + return path.replaceAll("\\", "/").includes("/node_modules/"); +} + +function toCanonicalPath(path: string): string { + try { + return realpathSync(path); + } catch { + return resolve(path); + } +} + +function isPathInsideOrEqual(path: string, directory: string): boolean { + const resolvedPath = resolve(path); + const resolvedDirectory = resolve(directory); + + return ( + resolvedPath === resolvedDirectory || resolvedPath.startsWith(`${resolvedDirectory}${sep}`) + ); +} diff --git a/packages/eve/src/internal/bundler/nitro-node-file-trace.ts b/packages/eve/src/internal/bundler/nitro-node-file-trace.ts new file mode 100644 index 000000000..a419206a8 --- /dev/null +++ b/packages/eve/src/internal/bundler/nitro-node-file-trace.ts @@ -0,0 +1,49 @@ +import { createRequire } from "node:module"; +import { pathToFileURL } from "node:url"; + +interface NodeFileTraceResult { + readonly fileList: ReadonlySet; + readonly warnings: ReadonlySet; +} + +type NodeFileTrace = ( + files: string[], + options: { + readonly base: string; + readonly conditions: string[]; + readonly processCwd: string; + }, +) => Promise; + +interface NodeFileTraceModule { + readonly nodeFileTrace: NodeFileTrace; +} + +let nodeFileTraceModulePromise: Promise | undefined; + +async function loadNitroNodeFileTrace(): Promise { + nodeFileTraceModulePromise ??= (async () => { + const require = createRequire(import.meta.url); + const nitroRequire = createRequire(require.resolve("nitro/package.json")); + const modulePath = nitroRequire.resolve("@vercel/nft"); + return (await import(pathToFileURL(modulePath).href)) as NodeFileTraceModule; + })(); + + return await nodeFileTraceModulePromise; +} + +// Traces under `node` + `import` conditions: the runtime loads materialized +// externals with ESM `import()`, so tracing through `require` conditions +// would materialize files the runtime never resolves. +export async function traceNodeEsmFiles(input: { + readonly base: string; + readonly entries: readonly string[]; + readonly processCwd: string; +}): Promise { + const { nodeFileTrace } = await loadNitroNodeFileTrace(); + return await nodeFileTrace([...input.entries], { + base: input.base, + conditions: ["node", "import"], + processCwd: input.processCwd, + }); +} diff --git a/packages/eve/src/internal/materialize-authored-external-dependencies.ts b/packages/eve/src/internal/materialize-authored-external-dependencies.ts new file mode 100644 index 000000000..821b245e3 --- /dev/null +++ b/packages/eve/src/internal/materialize-authored-external-dependencies.ts @@ -0,0 +1,397 @@ +import { createHash } from "node:crypto"; +import { constants as fsConstants } from "node:fs"; +import { + copyFile, + lstat, + mkdir, + readFile, + readdir, + readlink, + realpath, + rm, + stat, + symlink, +} from "node:fs/promises"; +import { dirname, isAbsolute, join, parse, relative, resolve, sep } from "node:path"; + +import { traceNodeEsmFiles } from "#internal/bundler/nitro-node-file-trace.js"; + +const COPY_MODE = fsConstants.COPYFILE_FICLONE; + +export interface ResolvedAuthoredExternalModule { + readonly packageName: string; + readonly resolvedId: string; +} + +export async function materializeAuthoredExternalDependencies(input: { + readonly appRoot: string; + readonly externalModules: readonly ResolvedAuthoredExternalModule[]; + readonly snapshotSourceRoot: string; + readonly sourceRoot: string; +}): Promise { + if (input.externalModules.length === 0) { + return createHash("sha256").digest("hex"); + } + + const sourceRoot = await realpath(input.sourceRoot); + const appRoot = await realpath(input.appRoot); + const packages = await resolveExternalPackages(input.externalModules); + + const fingerprint = createHash("sha256"); + const tracedPaths = await traceExternalDependencyPaths(packages, appRoot); + const materializedPaths = new Set(); + const materializedDirectories = new Set(); + + for (const externalPackage of packages) { + await materializeDependencyDirectory({ + fingerprint, + materializedDirectories, + materializedPaths, + snapshotSourceRoot: input.snapshotSourceRoot, + sourcePath: externalPackage.packageRoot, + sourceRoot, + }); + } + + for (const tracedPath of tracedPaths) { + const packageRoot = await findNearestPackageRoot(tracedPath); + if (packageRoot !== undefined) { + await materializeDependencyDirectory({ + fingerprint, + materializedDirectories, + materializedPaths, + snapshotSourceRoot: input.snapshotSourceRoot, + sourcePath: packageRoot, + sourceRoot, + }); + } + } + + for (const tracedPath of tracedPaths) { + await materializeTracedPath({ + fingerprint, + materializedPaths, + snapshotSourceRoot: input.snapshotSourceRoot, + sourcePath: tracedPath, + sourceRoot, + }); + } + + for (const externalPackage of packages) { + const materializedPackageRoot = toMaterializedPath({ + snapshotSourceRoot: input.snapshotSourceRoot, + sourcePath: externalPackage.packageRoot, + sourceRoot, + }); + const runtimePackageRoot = join( + toMaterializedPath({ + snapshotSourceRoot: input.snapshotSourceRoot, + sourcePath: appRoot, + sourceRoot, + }), + "node_modules", + ...externalPackage.packageName.split("/"), + ); + + if (resolve(runtimePackageRoot) !== resolve(materializedPackageRoot)) { + await rm(runtimePackageRoot, { force: true, recursive: true }); + await mkdir(dirname(runtimePackageRoot), { recursive: true }); + await symlink( + relative(dirname(runtimePackageRoot), materializedPackageRoot) || ".", + runtimePackageRoot, + "junction", + ); + } + + fingerprint + .update("package\0") + .update(externalPackage.packageName) + .update("\0") + .update(toSemanticPath(externalPackage.packageRoot, sourceRoot)) + .update("\0"); + } + + return fingerprint.digest("hex"); +} + +async function traceExternalDependencyPaths( + packages: readonly ExternalPackage[], + processCwd: string, +): Promise { + const groups = new Map(); + + for (const externalPackage of packages) { + const base = parse(externalPackage.resolvedId).root; + const key = normalizePathRoot(base); + const group = groups.get(key) ?? { base, entries: [] }; + group.entries.push(externalPackage.resolvedId); + groups.set(key, group); + } + + const tracedPaths = new Set(); + const warnings: Error[] = []; + + for (const group of [...groups.values()].sort((left, right) => + normalizePathRoot(left.base).localeCompare(normalizePathRoot(right.base)), + )) { + const trace = await traceNodeEsmFiles({ + base: group.base, + entries: group.entries.sort(), + processCwd, + }); + warnings.push(...trace.warnings); + + for (const tracedPath of trace.fileList) { + tracedPaths.add(isAbsolute(tracedPath) ? tracedPath : resolve(group.base, tracedPath)); + } + } + + if (warnings.length > 0) { + const messages = warnings.map((warning) => warning.message).sort(); + throw new Error( + `Failed to trace the complete authored external dependency closure:\n${messages.join("\n")}`, + ); + } + + return [...tracedPaths].sort((left, right) => left.localeCompare(right)); +} + +async function materializeDependencyDirectory(input: { + readonly fingerprint: ReturnType; + readonly materializedDirectories: Set; + readonly materializedPaths: Set; + readonly snapshotSourceRoot: string; + readonly sourcePath: string; + readonly sourceRoot: string; +}): Promise { + const canonicalDirectory = await realpath(input.sourcePath); + if (input.materializedDirectories.has(canonicalDirectory)) { + return; + } + input.materializedDirectories.add(canonicalDirectory); + + for (const entry of (await readdir(canonicalDirectory, { withFileTypes: true })).sort( + (left, right) => left.name.localeCompare(right.name), + )) { + const sourcePath = join(canonicalDirectory, entry.name); + const sourceStats = await lstat(sourcePath); + + if (sourceStats.isDirectory()) { + await materializeDependencyDirectory({ ...input, sourcePath }); + continue; + } + + if (sourceStats.isSymbolicLink()) { + await materializeTracedPath({ + fingerprint: input.fingerprint, + materializedPaths: input.materializedPaths, + snapshotSourceRoot: input.snapshotSourceRoot, + sourcePath, + sourceRoot: input.sourceRoot, + }); + continue; + } + + await materializeTracedPath({ + fingerprint: input.fingerprint, + materializedPaths: input.materializedPaths, + snapshotSourceRoot: input.snapshotSourceRoot, + sourcePath, + sourceRoot: input.sourceRoot, + }); + } +} + +interface ExternalPackage { + readonly packageName: string; + readonly packageRoot: string; + readonly resolvedId: string; +} + +async function resolveExternalPackages( + modules: readonly ResolvedAuthoredExternalModule[], +): Promise { + const packagesByName = new Map(); + + for (const module of modules) { + const packageRoot = await findPackageRoot(module.resolvedId, module.packageName); + const existing = packagesByName.get(module.packageName); + + if (existing !== undefined && resolve(existing.packageRoot) !== resolve(packageRoot)) { + throw new Error( + `Authored external dependency "${module.packageName}" resolves to multiple package instances.`, + ); + } + + packagesByName.set(module.packageName, { + packageName: module.packageName, + packageRoot, + resolvedId: module.resolvedId, + }); + } + + return [...packagesByName.values()].sort((left, right) => + left.packageName.localeCompare(right.packageName), + ); +} + +async function findPackageRoot(resolvedId: string, packageName: string): Promise { + let current = dirname(await realpath(resolvedId)); + + while (true) { + const packageJsonPath = join(current, "package.json"); + + try { + const packageJson = JSON.parse(await readFile(packageJsonPath, "utf8")) as { + readonly name?: unknown; + }; + + if (packageJson.name === packageName) { + return current; + } + } catch (error) { + if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) { + throw error; + } + } + + const parent = dirname(current); + if (parent === current) { + throw new Error( + `Cannot find package root for authored external dependency "${packageName}" from "${resolvedId}".`, + ); + } + current = parent; + } +} + +async function findNearestPackageRoot(path: string): Promise { + const canonicalPath = await realpath(path); + let current = (await stat(canonicalPath)).isDirectory() ? canonicalPath : dirname(canonicalPath); + + while (true) { + try { + const packageJson = JSON.parse(await readFile(join(current, "package.json"), "utf8")) as { + readonly name?: unknown; + }; + if (typeof packageJson.name === "string" && packageJson.name.length > 0) { + return current; + } + } catch (error) { + if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) { + throw error; + } + } + + const parent = dirname(current); + if (parent === current) { + return undefined; + } + current = parent; + } +} + +async function materializeTracedPath(input: { + readonly fingerprint: ReturnType; + readonly materializedPaths: Set; + readonly snapshotSourceRoot: string; + readonly sourcePath: string; + readonly sourceRoot: string; +}): Promise { + const materializedPathKey = toSemanticPath(input.sourcePath, input.sourceRoot); + if (input.materializedPaths.has(materializedPathKey)) { + return; + } + input.materializedPaths.add(materializedPathKey); + + const targetPath = toMaterializedPath(input); + const sourceStats = await lstat(input.sourcePath); + const portablePath = toSemanticPath(input.sourcePath, input.sourceRoot); + + if (sourceStats.isSymbolicLink()) { + const declaredTarget = await readlink(input.sourcePath); + const resolvedTarget = await realpath(resolve(dirname(input.sourcePath), declaredTarget)); + const targetLinkTarget = toMaterializedPath({ + snapshotSourceRoot: input.snapshotSourceRoot, + sourcePath: resolvedTarget, + sourceRoot: input.sourceRoot, + }); + const targetStats = await stat(input.sourcePath); + + await rm(targetPath, { force: true, recursive: true }); + await mkdir(dirname(targetPath), { recursive: true }); + await symlink( + relative(dirname(targetPath), targetLinkTarget) || ".", + targetPath, + targetStats.isDirectory() ? "junction" : "file", + ); + input.fingerprint + .update(portablePath) + .update("\0link\0") + .update(toSemanticPath(resolvedTarget, input.sourceRoot)) + .update("\0"); + return; + } + + if (!sourceStats.isFile()) { + throw new Error(`Unsupported traced authored dependency path "${input.sourcePath}".`); + } + + await mkdir(dirname(targetPath), { recursive: true }); + await copyFile(input.sourcePath, targetPath, COPY_MODE); + input.fingerprint + .update(portablePath) + .update("\0file\0") + .update(await readFile(input.sourcePath)) + .update("\0"); +} + +function toMaterializedPath(input: { + readonly snapshotSourceRoot: string; + readonly sourcePath: string; + readonly sourceRoot: string; +}): string { + if (isPathInsideOrEqual(input.sourcePath, input.sourceRoot)) { + return join(input.snapshotSourceRoot, relative(input.sourceRoot, input.sourcePath)); + } + + const pathRoot = parse(resolve(input.sourcePath)).root; + return join( + input.snapshotSourceRoot, + ".eve", + "external-dependencies", + createPathRootKey(pathRoot), + relative(pathRoot, input.sourcePath), + ); +} + +function toSemanticPath(path: string, sourceRoot: string): string { + if (isPathInsideOrEqual(path, sourceRoot)) { + return `source/${toPortablePath(relative(sourceRoot, path))}`; + } + + const pathRoot = parse(resolve(path)).root; + return `external/${createPathRootKey(pathRoot)}/${toPortablePath(relative(pathRoot, path))}`; +} + +function createPathRootKey(pathRoot: string): string { + return createHash("sha256").update(normalizePathRoot(pathRoot)).digest("hex").slice(0, 12); +} + +function isPathInsideOrEqual(path: string, directory: string): boolean { + const relativePath = relative(resolve(directory), resolve(path)); + return !( + isAbsolute(relativePath) || + relativePath === ".." || + relativePath.startsWith(`..${sep}`) || + relativePath.startsWith(sep) + ); +} + +function normalizePathRoot(pathRoot: string): string { + return toPortablePath(pathRoot).toLowerCase(); +} + +function toPortablePath(path: string): string { + return path.split(sep).join("/"); +} diff --git a/packages/eve/src/internal/materialized-authored-modules.ts b/packages/eve/src/internal/materialized-authored-modules.ts new file mode 100644 index 000000000..b65d14dd5 --- /dev/null +++ b/packages/eve/src/internal/materialized-authored-modules.ts @@ -0,0 +1,260 @@ +import { createHash } from "node:crypto"; +import { existsSync } from "node:fs"; +import { lstat, mkdir, readFile, readdir, readlink, writeFile } from "node:fs/promises"; +import { join, relative, sep } from "node:path"; + +import type { CompiledAgentManifest, CompiledAgentNodeManifest } from "#compiler/manifest.js"; +import { COMPILED_AGENT_MANIFEST_KIND, ROOT_COMPILED_AGENT_NODE_ID } from "#compiler/manifest.js"; +import { collectModuleRefsForManifest } from "#compiler/module-map.js"; +import { bundleAuthoredModuleForGeneration } from "#internal/authored-module-loader.js"; +import { + materializeAuthoredExternalDependencies, + type ResolvedAuthoredExternalModule, +} from "#internal/materialize-authored-external-dependencies.js"; + +const MATERIALIZED_MODULES_DIRECTORY = "authored-modules"; +const MATERIALIZED_MODULES_INDEX = "authored-modules.json"; +const INSTRUMENTATION_EXTENSIONS = [".ts", ".mts", ".js", ".mjs"] as const; + +export interface MaterializedAuthoredModuleIndex { + readonly fingerprint: string; + readonly instrumentation?: string; + readonly nodes: Readonly> }>>; + readonly version: 1; +} + +export async function materializeAuthoredModules(input: { + readonly appRoot: string; + readonly runtimeAppRoot: string; + readonly snapshotSourceRoot: string; + readonly sourceRoot: string; +}): Promise { + const compileRoot = join(input.runtimeAppRoot, ".eve", "compile"); + const manifest = await readCompiledManifest(join(compileRoot, "compiled-agent-manifest.json")); + const modulesRoot = join(compileRoot, MATERIALIZED_MODULES_DIRECTORY); + const scopeIndex = createExtensionScopeIndex(manifest); + const nodes: Record }> = {}; + const externalModules: ResolvedAuthoredExternalModule[] = []; + const fingerprint = createHash("sha256"); + + await mkdir(modulesRoot, { recursive: true }); + for (const node of collectNodeManifests(manifest)) { + const modules: Record = {}; + + for (const ref of collectModuleRefsForManifest(node.manifest).sort((left, right) => + left.sourceId.localeCompare(right.sourceId), + )) { + const bundle = await bundleAuthoredModuleForGeneration( + join(node.agentRoot, ref.logicalPath), + { + externalDependencies: node.manifest.config.build?.externalDependencies ?? [], + extensionScopeNamespace: extensionNamespaceForSourceId(ref.sourceId, scopeIndex), + }, + ); + const fileName = createMaterializedModuleFileName(node.nodeId, ref.sourceId, bundle.code); + + await writeFile(join(modulesRoot, fileName), bundle.code); + externalModules.push(...bundle.externalModules); + fingerprint + .update("module\0") + .update(node.nodeId) + .update("\0") + .update(ref.sourceId) + .update("\0") + .update(bundle.code) + .update("\0"); + modules[ref.sourceId] = join(MATERIALIZED_MODULES_DIRECTORY, fileName); + } + + nodes[node.nodeId] = { modules }; + } + + const instrumentation = resolveInstrumentationModule(manifest.agentRoot); + let instrumentationPath: string | undefined; + + if (instrumentation !== undefined) { + const bundle = await bundleAuthoredModuleForGeneration(instrumentation, { + externalDependencies: manifest.config.build?.externalDependencies ?? [], + }); + const fileName = createMaterializedModuleFileName( + ROOT_COMPILED_AGENT_NODE_ID, + "instrumentation", + bundle.code, + ); + + await writeFile(join(modulesRoot, fileName), bundle.code); + externalModules.push(...bundle.externalModules); + instrumentationPath = join(MATERIALIZED_MODULES_DIRECTORY, fileName); + fingerprint.update("instrumentation\0").update(bundle.code).update("\0"); + } + + await hashDirectoryIfPresent({ + fingerprint, + path: join(compileRoot, "workspace-resources"), + root: join(compileRoot, "workspace-resources"), + }); + fingerprint + .update("external-dependencies\0") + .update( + await materializeAuthoredExternalDependencies({ + appRoot: input.appRoot, + externalModules, + snapshotSourceRoot: input.snapshotSourceRoot, + sourceRoot: input.sourceRoot, + }), + ) + .update("\0"); + + const index: { + fingerprint: string; + instrumentation?: string; + nodes: MaterializedAuthoredModuleIndex["nodes"]; + version: 1; + } = { + fingerprint: fingerprint.digest("hex"), + nodes, + version: 1, + }; + if (instrumentationPath !== undefined) { + index.instrumentation = instrumentationPath; + } + await writeFile(join(compileRoot, MATERIALIZED_MODULES_INDEX), `${JSON.stringify(index)}\n`); + return index; +} + +export async function readMaterializedAuthoredModuleIndex( + runtimeAppRoot: string, +): Promise { + const indexPath = join(runtimeAppRoot, ".eve", "compile", MATERIALIZED_MODULES_INDEX); + if (!existsSync(indexPath)) { + return undefined; + } + + const parsed = JSON.parse( + await readFile(indexPath, "utf8"), + ) as Partial; + if ( + parsed.version !== 1 || + typeof parsed.fingerprint !== "string" || + parsed.fingerprint.length === 0 || + typeof parsed.nodes !== "object" || + parsed.nodes === null || + (parsed.instrumentation !== undefined && typeof parsed.instrumentation !== "string") + ) { + throw new Error(`Invalid materialized authored module index at "${indexPath}".`); + } + + return parsed as MaterializedAuthoredModuleIndex; +} + +interface ExtensionScopeIndex { + readonly byMountNamespace: ReadonlyMap; +} + +function collectNodeManifests(manifest: CompiledAgentManifest): Array<{ + readonly agentRoot: string; + readonly manifest: CompiledAgentNodeManifest; + readonly nodeId: string; +}> { + return [ + { agentRoot: manifest.agentRoot, manifest, nodeId: ROOT_COMPILED_AGENT_NODE_ID }, + ...[...manifest.subagents] + .sort((left, right) => left.nodeId.localeCompare(right.nodeId)) + .map((subagent) => ({ + agentRoot: subagent.agent.agentRoot, + manifest: subagent.agent, + nodeId: subagent.nodeId, + })), + ]; +} + +function createExtensionScopeIndex(manifest: CompiledAgentManifest): ExtensionScopeIndex { + return { + byMountNamespace: new Map( + manifest.extensionMounts.map((mount) => [mount.namespace, mount.packageNamespace]), + ), + }; +} + +function extensionNamespaceForSourceId( + sourceId: string, + index: ExtensionScopeIndex, +): string | undefined { + const match = sourceId.match(/^ext:([^:]+):/u); + return match === null ? undefined : index.byMountNamespace.get(match[1]!); +} + +async function readCompiledManifest(path: string): Promise { + const manifest = JSON.parse(await readFile(path, "utf8")) as CompiledAgentManifest; + + if (manifest.kind !== COMPILED_AGENT_MANIFEST_KIND) { + throw new Error(`Invalid compiled agent manifest at "${path}".`); + } + + return manifest; +} + +function resolveInstrumentationModule(agentRoot: string): string | undefined { + for (const extension of INSTRUMENTATION_EXTENSIONS) { + const candidate = join(agentRoot, `instrumentation${extension}`); + if (existsSync(candidate)) { + return candidate; + } + } + + return undefined; +} + +function createMaterializedModuleFileName(nodeId: string, sourceId: string, code: string): string { + return `${createHash("sha256") + .update(nodeId) + .update("\0") + .update(sourceId) + .update("\0") + .update(code) + .digest("hex")}.mjs`; +} + +async function hashDirectoryIfPresent(input: { + readonly fingerprint: ReturnType; + readonly path: string; + readonly root: string; +}): Promise { + if (!existsSync(input.path)) { + return; + } + + const stats = await lstat(input.path); + const relativePath = toPortablePath(relative(input.root, input.path)); + + if (stats.isSymbolicLink()) { + input.fingerprint + .update(relativePath) + .update("\0link\0") + .update(await readlink(input.path)) + .update("\0"); + return; + } + + if (stats.isDirectory()) { + for (const entry of (await readdir(input.path)).sort()) { + await hashDirectoryIfPresent({ + ...input, + path: join(input.path, entry), + }); + } + return; + } + + if (stats.isFile()) { + input.fingerprint + .update(relativePath) + .update("\0file\0") + .update(await readFile(input.path)) + .update("\0"); + } +} + +function toPortablePath(path: string): string { + return path.split(sep).join("/"); +} diff --git a/packages/eve/src/internal/nitro/dev-generation-artifacts.scenario.test.ts b/packages/eve/src/internal/nitro/dev-generation-artifacts.scenario.test.ts new file mode 100644 index 000000000..02894800e --- /dev/null +++ b/packages/eve/src/internal/nitro/dev-generation-artifacts.scenario.test.ts @@ -0,0 +1,293 @@ +import { existsSync } from "node:fs"; +import { mkdir, readFile, readdir, realpath, rm, symlink, writeFile } from "node:fs/promises"; +import { basename, dirname, join, relative } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { compileAgent } from "#compiler/compile-agent.js"; +import { ROOT_COMPILED_AGENT_NODE_ID } from "#compiler/manifest.js"; +import { loadCompiledModuleMapFromAuthoredSource } from "#internal/authored-module-map-loader.js"; +import { + discardDevelopmentGeneration, + stageDevelopmentGeneration, +} from "#internal/nitro/development-generation.js"; +import { createAuthoredSourceRuntimeCompiledArtifactsSource } from "#internal/application/runtime-compiled-artifacts-source.js"; +import { useScenarioApp } from "#internal/testing/scenario-app.js"; + +describe("development generation artifacts", () => { + const scenarioApp = useScenarioApp(); + + it("executes its ESM external closure after the original source and dependencies are removed", async () => { + const app = await scenarioApp({ + dependencies: { + "fixture-bundled": "1.0.0", + "fixture-external": "1.0.0", + }, + files: { + "agent/agent.mjs": [ + "export default {", + ' model: "openai/gpt-5.4",', + ' build: { externalDependencies: ["fixture-external"] },', + "};", + "", + ].join("\n"), + "agent/instructions.md": "Use the available tools.", + "agent/tools/read_value.mjs": [ + 'import { bundledValue } from "fixture-bundled";', + 'import { externalValue } from "fixture-external/feature";', + "", + "export default {", + ' description: "Read dependency values.",', + " execute() {", + " return `${bundledValue}:${externalValue}`;", + " },", + "};", + "", + ].join("\n"), + }, + name: "immutable-generation-closure", + }); + const externalFixtureRoot = await writeDependencyFixture(app.appRoot); + + const compileResult = await compileAgent({ startPath: app.appRoot }); + const snapshot = await stageDevelopmentGeneration(compileResult); + const externalPackagePath = join(snapshot.runtimeAppRoot, "node_modules", "fixture-external"); + const materializedExternalPath = await realpath(externalPackagePath); + const canonicalSnapshotRoot = await realpath(snapshot.snapshotRoot); + + expect(relative(canonicalSnapshotRoot, materializedExternalPath)).not.toMatch( + /^\.\.(?:[\\/]|$)/u, + ); + expect(existsSync(join(materializedExternalPath, "binding.node"))).toBe(true); + + await writeFile( + join(externalFixtureRoot, "fixture-transitive", "index.mjs"), + 'export const transitiveValue = "external-changed";\n', + ); + const changedDependencies = await stageDevelopmentGeneration( + await compileAgent({ startPath: app.appRoot }), + ); + expect(changedDependencies.fingerprint).not.toBe(snapshot.fingerprint); + + await rm(join(app.appRoot, "node_modules"), { force: true, recursive: true }); + await rm(externalFixtureRoot, { force: true, recursive: true }); + await writeFile( + join(app.appRoot, "agent", "tools", "read_value.mjs"), + 'throw new Error("mutated source loaded");\n', + ); + + const moduleMap = await loadCompiledModuleMapFromAuthoredSource({ + compiledArtifactsSource: createAuthoredSourceRuntimeCompiledArtifactsSource( + snapshot.runtimeAppRoot, + ), + }); + const toolSourceId = compileResult.manifest.tools[0]?.sourceId; + expect(toolSourceId).toBeDefined(); + const tool = moduleMap.nodes[ROOT_COMPILED_AGENT_NODE_ID]?.modules[toolSourceId!] as { + default: { execute(): string }; + }; + + expect(tool.default.execute()).toBe( + "bundled-original:serde-original:external-original:payload-original", + ); + }); + + it("uses a path-independent fingerprint that includes instrumentation", async () => { + const app = await scenarioApp({ + files: { + "agent/agent.mjs": 'export default { model: "openai/gpt-5.4" };\n', + "agent/instrumentation.mjs": 'export default { marker: "one" };\n', + "agent/instructions.md": "Use the configured model.", + "agent/skills/guide.md": [ + "---", + "description: Follow the guide.", + "---", + "", + "Use the original guidance.", + "", + ].join("\n"), + }, + name: "generation-fingerprint", + }); + const firstCompile = await compileAgent({ startPath: app.appRoot }); + const first = await stageDevelopmentGeneration(firstCompile); + const identical = await stageDevelopmentGeneration(firstCompile); + const firstIndex = JSON.parse( + await readFile( + join(first.runtimeAppRoot, ".eve", "compile", "authored-modules.json"), + "utf8", + ), + ) as { readonly instrumentation?: string }; + const materializedInstrumentation = await readFile( + join(first.runtimeAppRoot, ".eve", "compile", firstIndex.instrumentation!), + "utf8", + ); + + expect(first.fingerprint).toBe(identical.fingerprint); + expect(materializedInstrumentation).not.toContain("/.eve/dev-runtime/snapshots/"); + + await writeFile( + join(app.appRoot, "agent", "instrumentation.mjs"), + 'export default { marker: "two" };\n', + ); + const changedCompile = await compileAgent({ startPath: app.appRoot }); + const changed = await stageDevelopmentGeneration(changedCompile); + + expect(changed.fingerprint).not.toBe(first.fingerprint); + await expect( + readFile(join(changed.runtimeAppRoot, ".eve", "compile", "authored-modules.json"), "utf8"), + ).resolves.toContain('"instrumentation"'); + + await writeFile( + join(app.appRoot, "agent", "skills", "guide.md"), + ["---", "description: Follow the guide.", "---", "", "Use the changed guidance.", ""].join( + "\n", + ), + ); + const changedResources = await stageDevelopmentGeneration( + await compileAgent({ startPath: app.appRoot }), + ); + expect(changedResources.fingerprint).not.toBe(changed.fingerprint); + }); + + it("rejects only authored workflow directives", async () => { + const app = await scenarioApp({ + files: { + "agent/agent.mjs": 'export default { model: "openai/gpt-5.4" };\n', + "agent/instructions.md": "Use the available tools.", + "agent/tools/directive_text.mjs": [ + '// "use workflow" is documentation.', + 'const message = "use step";', + '"use workflow";', + 'export default { description: "Return text.", execute: () => message };', + "", + ].join("\n"), + }, + name: "authored-directive-guard", + }); + const validCompile = await compileAgent({ startPath: app.appRoot }); + + await expect(stageDevelopmentGeneration(validCompile)).resolves.toBeDefined(); + const snapshotsRoot = join(app.appRoot, ".eve", "dev-runtime", "snapshots"); + const stagedGenerations = await readdir(snapshotsRoot); + + await writeFile( + join(app.appRoot, "agent", "tools", "directive_text.mjs"), + [ + '"use step";', + 'export default { description: "Return text.", execute: () => "invalid" };', + "", + ].join("\n"), + ); + const invalidCompile = await compileAgent({ startPath: app.appRoot }); + + await expect(stageDevelopmentGeneration(invalidCompile)).rejects.toThrow( + /actual "use step" directive/u, + ); + await expect(readdir(snapshotsRoot)).resolves.toEqual(stagedGenerations); + }); + + it("discards a staged generation that was never activated", async () => { + const app = await scenarioApp({ + files: { + "agent/agent.mjs": 'export default { model: "openai/gpt-5.4" };\n', + "agent/instructions.md": "Use the configured model.", + }, + name: "discard-generation", + }); + const compileResult = await compileAgent({ startPath: app.appRoot }); + const snapshot = await stageDevelopmentGeneration(compileResult); + + await discardDevelopmentGeneration(snapshot); + + expect(existsSync(snapshot.snapshotRoot)).toBe(false); + }); +}); + +async function writeDependencyFixture(appRoot: string): Promise { + const nodeModulesRoot = join(appRoot, "node_modules"); + const bundledRoot = join(nodeModulesRoot, "fixture-bundled"); + const externalFixtureRoot = join(dirname(appRoot), `${basename(appRoot)}-linked-dependencies`); + const externalRoot = join(externalFixtureRoot, "fixture-external"); + const transitiveRoot = join(externalFixtureRoot, "fixture-transitive"); + + await mkdir(bundledRoot, { recursive: true }); + await writeFile( + join(bundledRoot, "package.json"), + `${JSON.stringify({ exports: "./index.mjs", name: "fixture-bundled", type: "module" })}\n`, + ); + // eve vendors `@workflow/*`, so a transitive workflow import reachable only + // through an inlined dependency must be inlined with it — a bare specifier + // would be unresolvable once the generation outlives `node_modules`. + await writeFile( + join(bundledRoot, "index.mjs"), + [ + '"use workflow";', + 'import { serdeMarker } from "@workflow/serde";', + "export const bundledValue = `bundled-original:${serdeMarker}`;", + "", + ].join("\n"), + ); + + const vendoredWorkflowRoot = join(nodeModulesRoot, "@workflow", "serde"); + await mkdir(vendoredWorkflowRoot, { recursive: true }); + await writeFile( + join(vendoredWorkflowRoot, "package.json"), + `${JSON.stringify({ exports: "./index.mjs", name: "@workflow/serde", type: "module" })}\n`, + ); + await writeFile( + join(vendoredWorkflowRoot, "index.mjs"), + 'export const serdeMarker = "serde-original";\n', + ); + + await mkdir(externalRoot, { recursive: true }); + await writeFile( + join(externalRoot, "package.json"), + `${JSON.stringify({ + dependencies: { "fixture-transitive": "1.0.0" }, + exports: { + "./feature": { + import: "./feature.mjs", + require: "./wrong.cjs", + }, + }, + name: "fixture-external", + type: "module", + })}\n`, + ); + await writeFile( + join(externalRoot, "feature.mjs"), + [ + 'import { readFileSync } from "node:fs";', + 'import { transitiveValue } from "fixture-transitive";', + 'export const externalValue = `${transitiveValue}:${readFileSync(new URL("./payload.txt", import.meta.url), "utf8").trim()}`;', + "", + ].join("\n"), + ); + await writeFile(join(externalRoot, "wrong.cjs"), 'throw new Error("require export used");\n'); + await writeFile(join(externalRoot, "binding.node"), "native-addon-placeholder\n"); + await writeFile(join(externalRoot, "payload.txt"), "payload-original\n"); + await mkdir(join(externalRoot, "node_modules"), { recursive: true }); + + await mkdir(transitiveRoot, { recursive: true }); + await writeFile( + join(transitiveRoot, "package.json"), + `${JSON.stringify({ + exports: "./index.mjs", + name: "fixture-transitive", + type: "module", + })}\n`, + ); + await writeFile( + join(transitiveRoot, "index.mjs"), + 'export const transitiveValue = "external-original";\n', + ); + + await symlink( + transitiveRoot, + join(externalRoot, "node_modules", "fixture-transitive"), + "junction", + ); + await symlink(externalRoot, join(nodeModulesRoot, "fixture-external"), "junction"); + return externalFixtureRoot; +} diff --git a/packages/eve/src/internal/nitro/dev-runtime-artifacts.integration.test.ts b/packages/eve/src/internal/nitro/dev-runtime-artifacts.integration.test.ts index 10e48e56d..2c3aa9bd9 100644 --- a/packages/eve/src/internal/nitro/dev-runtime-artifacts.integration.test.ts +++ b/packages/eve/src/internal/nitro/dev-runtime-artifacts.integration.test.ts @@ -13,10 +13,10 @@ import { createDiskRuntimeCompiledArtifactsSource } from "#runtime/compiled-arti import { getCompiledRuntimeAgentBundle } from "#runtime/sessions/compiled-agent-cache.js"; import { createRuntimeSession, withRuntimeSession } from "#runtime/sessions/runtime-session.js"; import { createDevelopmentNitroArtifactsConfig } from "#internal/nitro/host/artifacts-config.js"; +import { publishDevelopmentGeneration } from "#internal/nitro/development-generation.js"; import { activateDevelopmentRuntimeArtifactsSnapshot, pruneDevelopmentRuntimeArtifactsSnapshots, - publishDevelopmentRuntimeArtifactsSnapshot, readDevelopmentRuntimeArtifactsSnapshotRoot, readDevelopmentRuntimeArtifactsRevision, resolveDevelopmentRuntimeArtifactsPointerPath, @@ -112,7 +112,7 @@ async function createNextStyleImportSnapshotFixture(): Promise<{ readonly appRoo }), ); - await publishDevelopmentRuntimeArtifactsSnapshot({ + await publishDevelopmentGeneration({ paths: { compileDirectoryPath }, project: { appRoot }, } as CompileAgentResult); @@ -294,8 +294,10 @@ describe("development runtime artifact snapshots", () => { runtimeAppRoot: join(activeSnapshotRoot, "source", "app"), snapshotRoot: activeSnapshotRoot, snapshotSourceRoot: join(activeSnapshotRoot, "source"), + sourceRoot: appRoot, }, }); + await utimes(activeSnapshotRoot, oldActiveTime, oldActiveTime); await pruneDevelopmentRuntimeArtifactsSnapshots({ appRoot, @@ -310,6 +312,37 @@ describe("development runtime artifact snapshots", () => { expect(existsSync(staleSnapshotRoot)).toBe(false); }); + it("retains every activated generation until lease-aware pruning exists", async () => { + const appRoot = await createScratchDirectory("eve-dev-runtime-activated-retention-"); + const snapshotsRoot = join(appRoot, ".eve", "dev-runtime", "snapshots"); + const firstSnapshotRoot = join(snapshotsRoot, "first"); + const nextSnapshotRoot = join(snapshotsRoot, "next"); + + for (const snapshotRoot of [firstSnapshotRoot, nextSnapshotRoot]) { + await mkdir(snapshotRoot, { recursive: true }); + await utimes(snapshotRoot, new Date(1_000), new Date(1_000)); + await activateDevelopmentRuntimeArtifactsSnapshot({ + appRoot, + snapshot: { + runtimeAppRoot: join(snapshotRoot, "source"), + snapshotRoot, + snapshotSourceRoot: join(snapshotRoot, "source"), + sourceRoot: appRoot, + }, + }); + } + + await pruneDevelopmentRuntimeArtifactsSnapshots({ + appRoot, + now: 1_000_000, + recentWindowMs: 0, + retainCount: 0, + }); + + expect(existsSync(firstSnapshotRoot)).toBe(true); + expect(existsSync(nextSnapshotRoot)).toBe(true); + }); + it("preserves snapshots referenced by active durable workflow data", async () => { const appRoot = await createScratchDirectory("eve-dev-runtime-artifacts-prune-durable-"); const snapshotsRoot = join(appRoot, ".eve", "dev-runtime", "snapshots"); @@ -339,6 +372,7 @@ describe("development runtime artifact snapshots", () => { runtimeAppRoot: join(activeSnapshotRoot, "source", "app"), snapshotRoot: activeSnapshotRoot, snapshotSourceRoot: join(activeSnapshotRoot, "source"), + sourceRoot: appRoot, }, }); await mkdir(join(appRoot, ".workflow-data", "default", "runs"), { recursive: true }); @@ -478,10 +512,11 @@ describe("development runtime artifact snapshots", () => { )}\n`, ); - const snapshot = await publishDevelopmentRuntimeArtifactsSnapshot({ + const snapshot = await stageDevelopmentRuntimeArtifactsSnapshot({ paths: { compileDirectoryPath }, project: { appRoot }, } as CompileAgentResult); + await activateDevelopmentRuntimeArtifactsSnapshot({ appRoot, snapshot }); await writeFile(toolPath, "export const temperature = 73;\n"); diff --git a/packages/eve/src/internal/nitro/dev-runtime-artifacts.ts b/packages/eve/src/internal/nitro/dev-runtime-artifacts.ts index 88c896d65..decaa00f6 100644 --- a/packages/eve/src/internal/nitro/dev-runtime-artifacts.ts +++ b/packages/eve/src/internal/nitro/dev-runtime-artifacts.ts @@ -8,6 +8,7 @@ import { copyDevelopmentSourceSnapshot } from "#internal/nitro/dev-runtime-sourc import { createDevelopmentSourceSnapshotPlan } from "#internal/nitro/dev-runtime-source-snapshot.js"; const DEV_RUNTIME_ARTIFACTS_DIRECTORY = "dev-runtime"; +const DEV_RUNTIME_ARTIFACTS_ACTIVATED_MARKER = "activated"; const DEV_RUNTIME_ARTIFACTS_POINTER_VERSION = 2; const DEV_RUNTIME_SNAPSHOT_RECENT_WINDOW_MS = 15 * 60 * 1000; const DEV_RUNTIME_SNAPSHOT_RETAIN_COUNT = 5; @@ -36,6 +37,7 @@ export interface DevelopmentRuntimeArtifactsSnapshot { readonly runtimeAppRoot: string; readonly snapshotRoot: string; readonly snapshotSourceRoot: string; + readonly sourceRoot: string; } /** @@ -50,20 +52,6 @@ function resolveDevelopmentRuntimeArtifactsSnapshotsDirectory(appRoot: string): return join(appRoot, ".eve", DEV_RUNTIME_ARTIFACTS_DIRECTORY, "snapshots"); } -/** - * Publishes one immutable dev runtime snapshot and points future sessions at it. - */ -export async function publishDevelopmentRuntimeArtifactsSnapshot( - compileResult: CompileAgentResult, -): Promise { - const snapshot = await stageDevelopmentRuntimeArtifactsSnapshot(compileResult); - await activateDevelopmentRuntimeArtifactsSnapshot({ - appRoot: compileResult.project.appRoot, - snapshot, - }); - return snapshot; -} - /** * Stages one immutable dev runtime snapshot without moving the latest pointer. */ @@ -116,6 +104,7 @@ export async function stageDevelopmentRuntimeArtifactsSnapshot( runtimeAppRoot: sourceSnapshotPlan.runtimeAppRoot, snapshotRoot, snapshotSourceRoot: sourceSnapshotPlan.snapshotSourceRoot, + sourceRoot: sourceSnapshotPlan.sourceRoot, }; } @@ -126,6 +115,7 @@ export async function activateDevelopmentRuntimeArtifactsSnapshot(input: { readonly appRoot: string; readonly snapshot: DevelopmentRuntimeArtifactsSnapshot; }): Promise { + await writeFile(join(input.snapshot.snapshotRoot, DEV_RUNTIME_ARTIFACTS_ACTIVATED_MARKER), ""); await writeDevelopmentRuntimeArtifactsPointer(input); } @@ -162,16 +152,6 @@ export function readDevelopmentRuntimeArtifactsRevision( }; } -/** - * Starts best-effort cleanup for stale dev-runtime snapshots without delaying - * `eve dev` startup or rebuild handling. - */ -export function pruneDevelopmentRuntimeArtifactsSnapshotsInBackground(appRoot: string): void { - void pruneDevelopmentRuntimeArtifactsSnapshots({ appRoot }).catch((error) => { - console.warn(`[eve:dev] failed to prune stale runtime snapshots: ${formatErrorMessage(error)}`); - }); -} - export async function pruneDevelopmentRuntimeArtifactsSnapshots(input: { readonly appRoot: string; readonly now?: number; @@ -217,6 +197,7 @@ export async function pruneDevelopmentRuntimeArtifactsSnapshots(input: { await Promise.all( snapshots.map(async (snapshot, index) => { if ( + existsSync(join(snapshot.path, DEV_RUNTIME_ARTIFACTS_ACTIVATED_MARKER)) || index < retainCount || now - snapshot.mtimeMs <= recentWindowMs || protectedPaths.some((protectedPath) => pathsOverlap(snapshot.path, protectedPath)) @@ -416,10 +397,6 @@ function pathsOverlap(left: string, right: string): boolean { return isPathInsideOrEqual(left, right) || isPathInsideOrEqual(right, left); } -function formatErrorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} - async function rewriteSnapshotCompiledManifest(input: { readonly appRoot: string; readonly manifestPath: string; diff --git a/packages/eve/src/internal/nitro/development-generation.ts b/packages/eve/src/internal/nitro/development-generation.ts new file mode 100644 index 000000000..6bbe2c464 --- /dev/null +++ b/packages/eve/src/internal/nitro/development-generation.ts @@ -0,0 +1,70 @@ +import { rm } from "node:fs/promises"; + +import type { CompileAgentResult } from "#compiler/compile-agent.js"; +import { materializeAuthoredModules } from "#internal/materialized-authored-modules.js"; +import { + activateDevelopmentRuntimeArtifactsSnapshot, + stageDevelopmentRuntimeArtifactsSnapshot, + type DevelopmentRuntimeArtifactsSnapshot, +} from "#internal/nitro/dev-runtime-artifacts.js"; + +export interface DevelopmentGeneration extends DevelopmentRuntimeArtifactsSnapshot { + readonly fingerprint: string; +} + +export async function stageDevelopmentGeneration( + compileResult: CompileAgentResult, +): Promise { + const snapshot = await stageDevelopmentRuntimeArtifactsSnapshot(compileResult); + + try { + const materialized = await materializeAuthoredModules({ + appRoot: compileResult.project.appRoot, + runtimeAppRoot: snapshot.runtimeAppRoot, + snapshotSourceRoot: snapshot.snapshotSourceRoot, + sourceRoot: snapshot.sourceRoot, + }); + + return { + ...snapshot, + fingerprint: materialized.fingerprint, + }; + } catch (error) { + try { + await rm(snapshot.snapshotRoot, { force: true, recursive: true }); + } catch (cleanupError) { + throw new AggregateError( + [error, cleanupError], + `Failed to materialize and discard development generation "${snapshot.snapshotRoot}".`, + ); + } + throw error; + } +} + +export async function publishDevelopmentGeneration( + compileResult: CompileAgentResult, +): Promise { + const generation = await stageDevelopmentGeneration(compileResult); + await activateDevelopmentGeneration({ + appRoot: compileResult.project.appRoot, + generation, + }); + return generation; +} + +export async function activateDevelopmentGeneration(input: { + readonly appRoot: string; + readonly generation: DevelopmentGeneration; +}): Promise { + await activateDevelopmentRuntimeArtifactsSnapshot({ + appRoot: input.appRoot, + snapshot: input.generation, + }); +} + +export async function discardDevelopmentGeneration( + generation: DevelopmentGeneration, +): Promise { + await rm(generation.snapshotRoot, { force: true, recursive: true }); +} diff --git a/packages/eve/src/internal/nitro/host/dev-authored-source-watcher.scenario.test.ts b/packages/eve/src/internal/nitro/host/dev-authored-source-watcher.scenario.test.ts index 44e919181..3678787e9 100644 --- a/packages/eve/src/internal/nitro/host/dev-authored-source-watcher.scenario.test.ts +++ b/packages/eve/src/internal/nitro/host/dev-authored-source-watcher.scenario.test.ts @@ -105,7 +105,6 @@ const mockedWatcher = vi.hoisted(() => { const prepareDevelopmentApplicationHostMock = vi.hoisted(() => vi.fn()); const clearCompiledRuntimeAgentBundleCacheMock = vi.hoisted(() => vi.fn()); const startDevelopmentSandboxPrewarmInBackgroundMock = vi.hoisted(() => vi.fn()); -const pruneDevelopmentRuntimeArtifactsSnapshotsInBackgroundMock = vi.hoisted(() => vi.fn()); const resolveNitroCompiledArtifactsSourceMock = vi.hoisted(() => vi.fn((config: { readonly appRoot?: string }) => ({ appRoot: `${config.appRoot ?? "/tmp/eve-test"}/.eve/dev-runtime-test`, @@ -127,17 +126,6 @@ vi.mock("#execution/sandbox/development-prewarm.js", () => ({ startDevelopmentSandboxPrewarmInBackground: startDevelopmentSandboxPrewarmInBackgroundMock, })); -vi.mock("#internal/nitro/dev-runtime-artifacts.js", async () => { - const actual = await vi.importActual( - "#internal/nitro/dev-runtime-artifacts.js", - ); - return { - ...actual, - pruneDevelopmentRuntimeArtifactsSnapshotsInBackground: - pruneDevelopmentRuntimeArtifactsSnapshotsInBackgroundMock, - }; -}); - vi.mock("#internal/nitro/routes/runtime-artifacts.js", () => ({ resolveNitroCompiledArtifactsSource: resolveNitroCompiledArtifactsSourceMock, })); @@ -171,7 +159,6 @@ beforeEach(() => { prepareDevelopmentApplicationHostMock.mockReset(); clearCompiledRuntimeAgentBundleCacheMock.mockReset(); startDevelopmentSandboxPrewarmInBackgroundMock.mockReset(); - pruneDevelopmentRuntimeArtifactsSnapshotsInBackgroundMock.mockReset(); resolveNitroCompiledArtifactsSourceMock.mockClear(); }); @@ -204,9 +191,6 @@ describe("startAuthoredSourceWatcher", () => { await watcher.rebuild(); expect(prepareDevelopmentApplicationHostMock).toHaveBeenCalledWith(previousHost.appRoot); - expect(pruneDevelopmentRuntimeArtifactsSnapshotsInBackgroundMock).toHaveBeenCalledWith( - nextHost.appRoot, - ); expect(clearCompiledRuntimeAgentBundleCacheMock).toHaveBeenCalledTimes(1); } finally { await watcher.close(); diff --git a/packages/eve/src/internal/nitro/host/dev-authored-source-watcher.ts b/packages/eve/src/internal/nitro/host/dev-authored-source-watcher.ts index f36b75bdb..030cec474 100644 --- a/packages/eve/src/internal/nitro/host/dev-authored-source-watcher.ts +++ b/packages/eve/src/internal/nitro/host/dev-authored-source-watcher.ts @@ -7,7 +7,6 @@ import { toErrorMessage } from "#shared/errors.js"; import { resolveTsConfigDependencyPaths } from "#internal/application/tsconfig-dependencies.js"; import { createDevelopmentNitroArtifactsConfig } from "#internal/nitro/host/artifacts-config.js"; import { resolveDevelopmentSourceSnapshotWatchPaths } from "#internal/nitro/dev-runtime-source-snapshot.js"; -import { pruneDevelopmentRuntimeArtifactsSnapshotsInBackground } from "#internal/nitro/dev-runtime-artifacts.js"; import { startDevelopmentSandboxPrewarmInBackground } from "#execution/sandbox/development-prewarm.js"; import { computeChannelRouteRegistrations, @@ -143,7 +142,6 @@ export async function startAuthoredSourceWatcher(input: { } const nextHost = await prepareDevelopmentApplicationHost(previousHost.appRoot); - pruneDevelopmentRuntimeArtifactsSnapshotsInBackground(nextHost.appRoot); const artifactsConfig = createDevelopmentNitroArtifactsConfig({ appRoot: nextHost.appRoot, }); diff --git a/packages/eve/src/internal/nitro/host/prepare-application-host.scenario.test.ts b/packages/eve/src/internal/nitro/host/prepare-application-host.scenario.test.ts index 587d916ad..9728e8377 100644 --- a/packages/eve/src/internal/nitro/host/prepare-application-host.scenario.test.ts +++ b/packages/eve/src/internal/nitro/host/prepare-application-host.scenario.test.ts @@ -92,7 +92,7 @@ describe("application host preparation", () => { } }); - it("keeps Nitro host inputs stable when their runtime snapshot is pruned", async () => { + it("keeps Nitro host inputs outside retained runtime generations", async () => { const { agentRoot, appRoot } = await createAppRoot("eve-stable-dev-host-artifacts-", { files: { "agent/instructions.md": "Use the configured model.", @@ -100,7 +100,9 @@ describe("application host preparation", () => { packageName: "stable-dev-host-artifacts", }); const agentModulePath = join(agentRoot, "agent.mjs"); + const instrumentationModulePath = join(agentRoot, "instrumentation.mjs"); await writeFile(agentModulePath, 'export default { model: "openai/gpt-5.4" };\n'); + await writeFile(instrumentationModulePath, "export default {};\n"); const firstHost = await prepareDevelopmentApplicationHost(appRoot); const firstPointer = await readDevelopmentRuntimePointer(appRoot); @@ -118,9 +120,15 @@ describe("application host preparation", () => { join(stableHostDirectory, "compiled-artifacts-workflow-world.mjs"), ); expect(firstHost.compiledArtifacts.bootstrapPath).not.toContain("/.eve/dev-runtime/snapshots/"); - expect(await readFile(stableBootstrapPath, "utf8")).toContain( + expect(firstHost.compiledArtifacts.instrumentationSourcePath).toBe( + join(stableHostDirectory, "compiled-artifacts-instrumentation-source.mjs"), + ); + expect(await readFile(stableBootstrapPath, "utf8")).not.toContain( normalizeEsmImportSpecifier(agentModulePath), ); + await expect( + readFile(firstHost.compiledArtifacts.instrumentationPluginPath!, "utf8"), + ).resolves.not.toContain(normalizeEsmImportSpecifier(instrumentationModulePath)); expect(existsSync(snapshotBootstrapPath)).toBe(false); await writeFile( @@ -140,7 +148,7 @@ describe("application host preparation", () => { retainCount: 0, }); - expect(existsSync(firstPointer.snapshotRoot)).toBe(false); + expect(existsSync(firstPointer.snapshotRoot)).toBe(true); expect(existsSync(stableBootstrapPath)).toBe(true); }); }); diff --git a/packages/eve/src/internal/nitro/host/prepare-application-host.ts b/packages/eve/src/internal/nitro/host/prepare-application-host.ts index 51d3842e0..c341cb8c4 100644 --- a/packages/eve/src/internal/nitro/host/prepare-application-host.ts +++ b/packages/eve/src/internal/nitro/host/prepare-application-host.ts @@ -14,6 +14,7 @@ import { join } from "node:path"; import { type BuiltInWorkflowWorldTarget, writeCompiledArtifactsFiles, + writeDevelopmentCompiledArtifactsFiles, } from "#internal/application/compiled-artifacts.js"; import { resolveApplicationHostArtifactsDirectory, @@ -21,9 +22,10 @@ import { } from "#internal/application/paths.js"; import { createAuthoredSourceRuntimeCompiledArtifactsSource } from "#internal/application/runtime-compiled-artifacts-source.js"; import { - activateDevelopmentRuntimeArtifactsSnapshot, - stageDevelopmentRuntimeArtifactsSnapshot, -} from "#internal/nitro/dev-runtime-artifacts.js"; + activateDevelopmentGeneration, + discardDevelopmentGeneration, + stageDevelopmentGeneration, +} from "#internal/nitro/development-generation.js"; import type { PreparedApplicationHost } from "#internal/nitro/host/types.js"; /** @@ -42,20 +44,28 @@ export async function prepareDevelopmentApplicationHost( compileResult.project.appRoot, ), }); - const runtimeArtifactsSnapshot = await stageDevelopmentRuntimeArtifactsSnapshot(compileResult); - const preparedHost = await materializeApplicationHost({ - compileResult, - defaultWorkflowWorld: "local", - hostArtifactsDir: resolveApplicationHostArtifactsDirectory(compileResult.project.appRoot), - schedules, - workflowBuildDir: resolveWorkflowBuildDirectory(compileResult.project.appRoot), - }); - await activateDevelopmentRuntimeArtifactsSnapshot({ - appRoot: compileResult.project.appRoot, - snapshot: runtimeArtifactsSnapshot, - }); + const generation = await stageDevelopmentGeneration(compileResult); - return preparedHost; + try { + const compiledArtifacts = await writeDevelopmentCompiledArtifactsFiles({ + compileResult, + outDir: resolveApplicationHostArtifactsDirectory(compileResult.project.appRoot), + runtimeAppRoot: generation.runtimeAppRoot, + }); + await activateDevelopmentGeneration({ + appRoot: compileResult.project.appRoot, + generation, + }); + return createPreparedApplicationHost({ + compileResult, + compiledArtifacts, + schedules, + workflowBuildDir: resolveWorkflowBuildDirectory(compileResult.project.appRoot), + }); + } catch (error) { + await discardDevelopmentGeneration(generation); + throw error; + } } /** @@ -77,32 +87,30 @@ export async function prepareProductionApplicationHost( }); const schedules = await resolveSchedules({ manifest: compileResult.manifest }); - return await materializeApplicationHost({ + const compiledArtifacts = await writeCompiledArtifactsFiles({ compileResult, defaultWorkflowWorld: resolveProductionWorkflowWorldTarget(), - hostArtifactsDir: workspace.host.artifactsDir, + outDir: workspace.host.artifactsDir, + }); + + return createPreparedApplicationHost({ + compileResult, + compiledArtifacts, schedules, workflowBuildDir: workspace.workflow.buildDir, }); } -async function materializeApplicationHost(input: { +function createPreparedApplicationHost(input: { readonly compileResult: CompileAgentResult; - readonly defaultWorkflowWorld: BuiltInWorkflowWorldTarget; - readonly hostArtifactsDir: string; + readonly compiledArtifacts: PreparedApplicationHost["compiledArtifacts"]; readonly schedules: readonly ResolvedScheduleDefinition[]; readonly workflowBuildDir: string; -}): Promise { - const compiledArtifacts = await writeCompiledArtifactsFiles({ - compileResult: input.compileResult, - defaultWorkflowWorld: input.defaultWorkflowWorld, - outDir: input.hostArtifactsDir, - }); - +}): PreparedApplicationHost { return { appRoot: input.compileResult.project.appRoot, compileResult: input.compileResult, - compiledArtifacts, + compiledArtifacts: input.compiledArtifacts, scheduleRegistrations: createScheduleRegistrations(input.schedules), schedules: input.schedules, workflowBuildDir: input.workflowBuildDir, diff --git a/packages/eve/src/internal/nitro/host/start-development-server.test.ts b/packages/eve/src/internal/nitro/host/start-development-server.test.ts index c9a06637e..abc70fb2e 100644 --- a/packages/eve/src/internal/nitro/host/start-development-server.test.ts +++ b/packages/eve/src/internal/nitro/host/start-development-server.test.ts @@ -83,7 +83,6 @@ const mocks = vi.hoisted(() => { startDevelopmentSandboxPrewarmInBackground: vi.fn(() => undefined), pruneLocalSandboxTemplatesInBackground: vi.fn(() => undefined), stopDevelopmentSandboxResources: vi.fn(async () => undefined), - pruneDevelopmentRuntimeArtifactsSnapshotsInBackground: vi.fn(() => undefined), resolveDiscoveryProject: vi.fn(async () => ({ agentRoot: "/tmp/eve-test/agent", appRoot: "/tmp/eve-test", @@ -154,16 +153,6 @@ vi.mock("#execution/sandbox/bindings/local.js", async (importOriginal) => { }; }); -vi.mock("#internal/nitro/dev-runtime-artifacts.js", async (importOriginal) => { - const actual = await importOriginal(); - - return { - ...actual, - pruneDevelopmentRuntimeArtifactsSnapshotsInBackground: - mocks.pruneDevelopmentRuntimeArtifactsSnapshotsInBackground, - }; -}); - function createRequest(): IncomingMessage { return { headers: { @@ -369,9 +358,6 @@ describe("createDevelopmentServer", () => { const server = await startDevelopmentServer("/tmp/eve-test"); expect(mocks.prepareDevelopmentApplicationHost).toHaveBeenCalledWith("/tmp/eve-test"); - expect(mocks.pruneDevelopmentRuntimeArtifactsSnapshotsInBackground).toHaveBeenCalledWith( - "/tmp/eve-test", - ); expect(mocks.startDevelopmentSandboxPrewarmInBackground).toHaveBeenCalledWith({ appRoot: "/tmp/eve-test", compiledArtifactsSource: { diff --git a/packages/eve/src/internal/nitro/host/start-development-server.ts b/packages/eve/src/internal/nitro/host/start-development-server.ts index 411f2693c..4d3f84ece 100644 --- a/packages/eve/src/internal/nitro/host/start-development-server.ts +++ b/packages/eve/src/internal/nitro/host/start-development-server.ts @@ -36,7 +36,6 @@ import type { StartedDevelopmentServer, } from "#internal/nitro/host/types.js"; import { loadDevelopmentEnvironmentFiles } from "#cli/dev/environment.js"; -import { pruneDevelopmentRuntimeArtifactsSnapshotsInBackground } from "#internal/nitro/dev-runtime-artifacts.js"; import { DEFAULT_DEVELOPMENT_SERVER_PORT, MAX_DEVELOPMENT_SERVER_PORT_ATTEMPTS, @@ -461,7 +460,6 @@ async function startNitroDevelopmentServer( () => prepareDevelopmentApplicationHost(project.appRoot), options.onBootProgress, ); - pruneDevelopmentRuntimeArtifactsSnapshotsInBackground(preparedHost.appRoot); const compiledArtifactsSource = resolveNitroCompiledArtifactsSource( createDevelopmentNitroArtifactsConfig({ appRoot: preparedHost.appRoot, diff --git a/packages/eve/test/scenarios/dev-server.scenario.test.ts b/packages/eve/test/scenarios/dev-server.scenario.test.ts index e207c02e6..be149c9fe 100644 --- a/packages/eve/test/scenarios/dev-server.scenario.test.ts +++ b/packages/eve/test/scenarios/dev-server.scenario.test.ts @@ -1,6 +1,6 @@ import { spawn, type ChildProcessByStdio } from "node:child_process"; import { existsSync } from "node:fs"; -import { writeFile } from "node:fs/promises"; +import { rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import type { Readable } from "node:stream"; @@ -8,7 +8,6 @@ import { describe, expect, it } from "vitest"; import { EVE_HEALTH_ROUTE_PATH } from "../../src/protocol/routes.js"; import { - pruneDevelopmentRuntimeArtifactsSnapshots, readDevelopmentRuntimeArtifactsSnapshotRoot, resolveDevelopmentRuntimeArtifactsPointerPath, } from "../../src/internal/nitro/dev-runtime-artifacts.js"; @@ -258,7 +257,7 @@ async function startEveDev(appRoot: string): Promise { describe("eve dev server", () => { it( - "rebuilds after pruning its startup runtime snapshot and completes a streamed turn", + "rebuilds after its startup runtime generation is force-pruned and completes a streamed turn", async () => { const app = await scenarioApp(DEV_SERVER_AGENT_DESCRIPTOR); const server = await startEveDev(app.appRoot); @@ -301,12 +300,7 @@ describe("eve dev server", () => { throw new Error("Expected authored HMR to publish a runtime snapshot."); } - await pruneDevelopmentRuntimeArtifactsSnapshots({ - appRoot: app.appRoot, - now: Date.now() + 1_000, - recentWindowMs: 0, - retainCount: 0, - }); + await rm(startupRuntimeRoot, { force: true, recursive: true }); expect(existsSync(startupRuntimeRoot)).toBe(false); await writeFile(join(app.appRoot, ".env.local"), "EVE_SCENARIO_RELOAD=1\n"); @@ -318,8 +312,8 @@ describe("eve dev server", () => { const currentRuntimeRoot = readDevelopmentRuntimeArtifactsSnapshotRoot(pointerPath); return currentRuntimeRoot !== undefined && currentRuntimeRoot !== authoredRuntimeRoot; }, `Timed out waiting for the structural reload snapshot.\n\nstdout:\n${server.stdout()}\n\nstderr:\n${server.stderr()}`); + // Nitro exposes no replacement-ready signal while it owns the public listener. await wait(2_000); - let messageResult: Awaited>; try { messageResult = await sendDevelopmentMessage({ @@ -347,8 +341,6 @@ describe("eve dev server", () => { `stderr:\n${server.stderr()}`, ].join("\n\n"), ).toBe(true); - await wait(1_000); - const output = `${server.stdout()}\n${server.stderr()}`; expect(hasKnownDevBundlingFailure(output)).toBe(false); } finally {