diff --git a/.changeset/fast-workflow-discovery.md b/.changeset/fast-workflow-discovery.md new file mode 100644 index 0000000000..030302e9c9 --- /dev/null +++ b/.changeset/fast-workflow-discovery.md @@ -0,0 +1,6 @@ +--- +'@workflow/builders': patch +'@workflow/next': patch +--- + +Optimize eager workflow discovery and improve default eager build compatibility. diff --git a/packages/builders/src/base-builder.ts b/packages/builders/src/base-builder.ts index 044142a23f..da90f757c5 100644 --- a/packages/builders/src/base-builder.ts +++ b/packages/builders/src/base-builder.ts @@ -15,8 +15,11 @@ import { type WorkflowManifest, } from './apply-swc-transform.js'; import { createWorkflowEntrypointOptionsCode } from './constants.js'; -import { createDiscoverEntriesPlugin } from './discover-entries-esbuild-plugin.js'; import { getEsbuildTsconfigOptions } from './esbuild-tsconfig.js'; +import { + fastDiscoverEntries, + type DiscoveredEntries, +} from './fast-discovery.js'; import { getImportPath, resolveModuleSpecifier, @@ -32,6 +35,8 @@ import { extractWorkflowGraphs } from './workflows-extractor.js'; const enhancedResolve = promisify(enhancedResolveOriginal); const require = createRequire(import.meta.url); +export type { DiscoveredEntries } from './fast-discovery.js'; + /** * Legacy opt-in for source maps on the final workflow wrapper + webhook * bundles (which default to off, unlike the step/interim workflow bundles @@ -120,10 +125,73 @@ function moduleIdentityKey(file: string, moduleSpecifierRoot: string): string { return file.replace(/\\/g, '/'); } -export interface DiscoveredEntries { - discoveredSteps: Set; - discoveredWorkflows: Set; - discoveredSerdeFiles: Set; +type ManifestEntryLocation = { + filePath: string; + name: string; +}; + +function formatIdLocation(location: ManifestEntryLocation): string { + return `${location.filePath}#${location.name}`; +} + +function assertUniqueManifestIds( + entriesByFile: Record> | undefined, + ids: Map, + getId: (entry: TEntry) => string, + label: 'step' | 'workflow' +): void { + for (const [filePath, entries] of Object.entries(entriesByFile || {})) { + for (const [name, data] of Object.entries(entries)) { + const id = getId(data); + const existing = ids.get(id); + const current = { filePath, name }; + if ( + existing && + (existing.filePath !== current.filePath || + existing.name !== current.name) + ) { + const idName = label === 'step' ? 'workflow step ID' : 'workflow ID'; + const functionName = `${label} function`; + const capitalizedLabel = label === 'step' ? 'Step' : 'Workflow'; + throw new WorkflowBuildError( + `Duplicate ${idName} "${id}" generated for ${formatIdLocation(existing)} and ${formatIdLocation(current)}.`, + { + hint: + `${capitalizedLabel} IDs must be unique across a build. ` + + `If you own one of the colliding files, rename the ${functionName} or export ` + + `the package file through a unique package subpath. If the collision is in a ` + + `transitive dependency you don't control, file an issue with the upstream ` + + `package or pin to a non-colliding version.`, + } + ); + } + ids.set(id, current); + } + } +} + +function mergeWorkflowManifest( + target: WorkflowManifest, + incoming: WorkflowManifest, + stepIds: Map, + workflowIds: Map +): void { + assertUniqueManifestIds( + incoming.steps, + stepIds, + (data) => data.stepId, + 'step' + ); + assertUniqueManifestIds( + incoming.workflows, + workflowIds, + (data) => data.workflowId, + 'workflow' + ); + + target.workflows = Object.assign(target.workflows || {}, incoming.workflows); + target.steps = Object.assign(target.steps || {}, incoming.steps); + target.classes = Object.assign(target.classes || {}, incoming.classes); } /** @@ -288,6 +356,10 @@ export abstract class BaseBuilder { private discoveredEntries: WeakMap = new WeakMap(); + public clearDiscoveredEntriesCache(): void { + this.discoveredEntries = new WeakMap(); + } + /** * Pseudo-packages that should not be checked for workflow patterns. */ @@ -414,17 +486,16 @@ export abstract class BaseBuilder { const discoverStart = Date.now(); - // Resolve the SDK runtime entry point so that the discovery pass - // traces through it and discovers serde classes (like `Run`) that - // live inside SDK packages. Without this, files like `run.js` are - // only discovered when user code happens to import them. + // Resolve the SDK runtime serde entry point so that the discovery pass + // discovers classes like `Run` that live inside SDK packages. Without this, + // files like `run.js` are only discovered when user code imports them. // This is resolved here (rather than in callers) so that the original // `inputs` array reference is preserved for WeakMap caching — callers // like createWorkflowsBundle and createStepsBundle can share the same // cache entry when they pass the same inputFiles array. const resolvedWorkflowRuntime = await enhancedResolve( outdir, - 'workflow/runtime' + '@workflow/core/runtime/run' ).catch(() => undefined); const entryPoints = resolvedWorkflowRuntime ? [...inputs, resolvedWorkflowRuntime] @@ -432,28 +503,13 @@ export abstract class BaseBuilder { const effectiveTsconfigPath = tsconfigPath ?? (await this.findTsConfigPath()); - const esbuildTsconfigOptions = await getEsbuildTsconfigOptions( - effectiveTsconfigPath - ); - try { - await esbuild.build({ - treeShaking: true, - entryPoints, - plugins: [ - createDiscoverEntriesPlugin(state, this.transformProjectRoot), - ], - platform: 'node', - write: false, - outdir, - bundle: true, - sourcemap: false, - absWorkingDir: this.config.workingDir, - logLevel: 'silent', - ...esbuildTsconfigOptions, - // External packages that should not be bundled during discovery - external: this.config.externalPackages || [], - }); - } catch (_) {} + + await fastDiscoverEntries({ + entryPoints, + state, + defaultTsconfigPath: effectiveTsconfigPath, + workingDir: this.config.workingDir, + }); this.logBaseBuilderInfo( `Discovering workflow directives`, @@ -587,11 +643,139 @@ export abstract class BaseBuilder { return relativePath; } + protected createRouteImportSpecifier(file: string, routeDir: string): string { + const { importPath, isPackage } = getImportPath( + file, + this.config.workingDir + ); + if (isPackage) { + return importPath; + } + + let relativePath = relative(routeDir, file).replace(/\\/g, '/'); + if (!relativePath.startsWith('./') && !relativePath.startsWith('../')) { + relativePath = `./${relativePath}`; + } + return relativePath; + } + + private async createStepSourceRegistrationFile({ + inputFiles, + outfile, + tsconfigPath, + discoveredEntries, + }: { + inputFiles: string[]; + outfile: string; + tsconfigPath?: string; + discoveredEntries?: DiscoveredEntries; + }): Promise { + const stepsBundleStart = Date.now(); + const workflowManifest: WorkflowManifest = {}; + const builtInSteps = 'workflow/internal/builtins'; + const resolvedBuiltInSteps = (await enhancedResolve( + dirname(outfile), + builtInSteps + ).catch((err) => { + throw new WorkflowBuildError( + `Failed to resolve built-in steps sources.\n\nCaused by: ${String(err)}`, + { + hint: 'run `pnpm install workflow` to resolve this issue.', + cause: err, + } + ); + })) as string; + + const discovered = + discoveredEntries ?? + (await this.discoverEntries(inputFiles, dirname(outfile), tsconfigPath)); + const stepFiles = [...discovered.discoveredSteps].sort(); + const workflowFiles = [...discovered.discoveredWorkflows].sort(); + const serdeFiles = [...discovered.discoveredSerdeFiles].sort(); + const stepFilesSet = new Set(stepFiles); + const serdeOnlyFiles = serdeFiles.filter((f) => !stepFilesSet.has(f)); + + await this.writeDebugFile(outfile, { + stepFiles, + workflowFiles, + serdeOnlyFiles, + sourceImports: true, + }); + + const emittedImportIdentities = new Set([builtInSteps]); + const importStatements: string[] = []; + const routeDir = dirname(outfile); + const addRegistrationImport = (specifier: string): void => { + importStatements.push(`import ${JSON.stringify(specifier)};`); + }; + const addRegistrationFileImport = (file: string): void => { + const identity = moduleIdentityKey(file, this.moduleSpecifierRoot); + if (emittedImportIdentities.has(identity)) { + return; + } + emittedImportIdentities.add(identity); + addRegistrationImport(this.createRouteImportSpecifier(file, routeDir)); + }; + + addRegistrationImport(builtInSteps); + for (const file of stepFiles) { + addRegistrationFileImport(file); + } + for (const file of serdeOnlyFiles) { + addRegistrationFileImport(file); + } + + const output = `// biome-ignore-all lint: generated file +/* eslint-disable */ +${importStatements.join('\n')} + +export const __steps_registered = true; +`; + await mkdir(dirname(outfile), { recursive: true }); + const tempPath = `${outfile}.${randomUUID()}.tmp`; + await writeFile(tempPath, output); + await rename(tempPath, outfile); + + const manifestFiles = Array.from( + new Set([...stepFiles, ...serdeOnlyFiles, resolvedBuiltInSteps]) + ).sort(); + const stepIds = new Map(); + const workflowIds = new Map(); + await Promise.all( + manifestFiles.map(async (file) => { + const source = await readFile(file, 'utf8'); + const relativeFilepath = this.getRelativeFilepath(file); + const { workflowManifest: fileManifest } = await applySwcTransform( + relativeFilepath, + source, + 'step', + file, + this.transformProjectRoot, + this.moduleSpecifierRoot + ); + mergeWorkflowManifest( + workflowManifest, + fileManifest, + stepIds, + workflowIds + ); + }) + ); + + await this.ensureSwcIgnored(); + this.logBaseBuilderInfo( + 'Created step registrations', + `${Date.now() - stepsBundleStart}ms` + ); + return workflowManifest; + } + /** * Creates a bundle for workflow step functions. * Steps have full Node.js runtime access and handle side effects, API calls, etc. * * @param externalizeNonSteps - If true, only bundles step entry points and externalizes other code + * @param sourceStepRegistrationImports - If true, emits a source import registration file instead of bundling step registrations * @param bundleTransitiveLocalStepDependencies - If true, also bundles project-local files imported by step entries for direct runtime loading * @returns Build context (for watch mode) and the collected workflow manifest */ @@ -601,6 +785,7 @@ export abstract class BaseBuilder { outfile, externalizeNonSteps, bundleTransitiveLocalStepDependencies, + sourceStepRegistrationImports, rewriteTsExtensions, tsconfigPath, discoveredEntries, @@ -612,6 +797,7 @@ export abstract class BaseBuilder { format?: 'cjs' | 'esm'; externalizeNonSteps?: boolean; bundleTransitiveLocalStepDependencies?: boolean; + sourceStepRegistrationImports?: boolean; rewriteTsExtensions?: boolean; discoveredEntries?: DiscoveredEntries; /** @@ -659,6 +845,22 @@ export abstract class BaseBuilder { const stepFilesSet = new Set(stepFiles); const serdeOnlyFiles = serdeFiles.filter((f) => !stepFilesSet.has(f)); + if ( + sourceStepRegistrationImports && + externalizeNonSteps && + !bundleTransitiveLocalStepDependencies + ) { + return { + context: undefined, + manifest: await this.createStepSourceRegistrationFile({ + inputFiles, + outfile, + tsconfigPath, + discoveredEntries: discovered, + }), + }; + } + // log the step files for debugging await this.writeDebugFile(outfile, { stepFiles, @@ -1312,6 +1514,7 @@ export const POST = workflowEntrypoint(workflowCode${workflowEntrypointOptionsCo tsconfigPath, externalizeNonSteps, bundleTransitiveLocalStepDependencies, + sourceStepRegistrationImports, discoveredEntries, }: { inputFiles: string[]; @@ -1324,6 +1527,7 @@ export const POST = workflowEntrypoint(workflowCode${workflowEntrypointOptionsCo tsconfigPath?: string; externalizeNonSteps?: boolean; bundleTransitiveLocalStepDependencies?: boolean; + sourceStepRegistrationImports?: boolean; discoveredEntries?: DiscoveredEntries; }): Promise<{ manifest: WorkflowManifest; @@ -1346,6 +1550,7 @@ export const POST = workflowEntrypoint(workflowCode${workflowEntrypointOptionsCo format: bundleFinalOutput ? 'esm' : format, externalizeNonSteps, bundleTransitiveLocalStepDependencies, + sourceStepRegistrationImports, tsconfigPath, discoveredEntries, // Skip the createRequire banner here — when bundleFinalOutput is true diff --git a/packages/builders/src/fast-discovery.test.ts b/packages/builders/src/fast-discovery.test.ts new file mode 100644 index 0000000000..c7966d895c --- /dev/null +++ b/packages/builders/src/fast-discovery.test.ts @@ -0,0 +1,515 @@ +import { + mkdirSync, + mkdtempSync, + realpathSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { BaseBuilder, type DiscoveredEntries } from './base-builder.js'; +import { + importParents, + parentHasChild, +} from './discover-entries-esbuild-plugin.js'; +import type { StandaloneConfig } from './types.js'; + +class TestBuilder extends BaseBuilder { + async build(): Promise { + // no-op + } + + public discoverEntriesPublic( + inputs: string[], + outdir: string, + tsconfigPath?: string + ): Promise { + return this.discoverEntries(inputs, outdir, tsconfigPath); + } + + public createRouteImportSpecifierPublic( + file: string, + routeDir: string + ): string { + return this.createRouteImportSpecifier(file, routeDir); + } +} + +const realTmpdir = realpathSync(tmpdir()); + +function normalize(path: string): string { + return path.replace(/\\/g, '/'); +} + +function writeFile(path: string, contents: string): void { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, contents, 'utf-8'); +} + +function createBuilder(workingDir: string): TestBuilder { + const config: StandaloneConfig = { + buildTarget: 'standalone', + workingDir, + dirs: ['.'], + stepsBundlePath: join(workingDir, 'steps.js'), + workflowsBundlePath: join(workingDir, 'workflows.js'), + webhookBundlePath: join(workingDir, 'webhook.js'), + }; + return new TestBuilder(config); +} + +describe('fast workflow discovery', () => { + let testRoot: string; + + beforeEach(() => { + testRoot = mkdtempSync(join(realTmpdir, 'workflow-fast-discovery-')); + importParents.clear(); + }); + + afterEach(() => { + importParents.clear(); + rmSync(testRoot, { recursive: true, force: true }); + }); + + it('discovers transitive relative step imports and tracks the parent chain', async () => { + const entryFile = join(testRoot, 'src', 'entry.ts'); + const workflowFile = join(testRoot, 'src', 'workflow.ts'); + const stepFile = join(testRoot, 'src', 'step.ts'); + + writeFile(entryFile, `import './workflow';\n`); + writeFile(workflowFile, `import { doStep } from './step';\nvoid doStep;\n`); + writeFile( + stepFile, + `export async function doStep() { + 'use step'; + return 1; +} +` + ); + + const discovered = await createBuilder(testRoot).discoverEntriesPublic( + [entryFile], + join(testRoot, 'out') + ); + + expect(discovered.discoveredSteps).toEqual(new Set([normalize(stepFile)])); + expect(parentHasChild(normalize(entryFile), normalize(stepFile))).toBe( + true + ); + }); + + it('discovers workflow files reached through an imported package re-export', async () => { + const entryFile = join(testRoot, 'src', 'entry.ts'); + const packageRoot = join(testRoot, 'node_modules', 'workflow-pkg'); + const packageIndex = join(packageRoot, 'index.js'); + const packageWorkflow = join(packageRoot, 'workflow.js'); + + writeFile(entryFile, `import { run } from 'workflow-pkg';\nvoid run;\n`); + writeFile( + join(packageRoot, 'package.json'), + JSON.stringify({ + name: 'workflow-pkg', + version: '1.0.0', + main: 'index.js', + dependencies: { + workflow: '^1.0.0', + }, + }) + ); + writeFile(packageIndex, `export { run } from './workflow.js';\n`); + writeFile( + packageWorkflow, + `export async function run() { + "use workflow"; + return "ok"; +} +` + ); + + const discovered = await createBuilder(testRoot).discoverEntriesPublic( + [entryFile], + join(testRoot, 'out') + ); + + expect(discovered.discoveredWorkflows).toEqual( + new Set([normalize(packageWorkflow)]) + ); + expect( + parentHasChild(normalize(packageIndex), normalize(packageWorkflow)) + ).toBe(true); + }); + + it('discovers files reached through tsconfig path aliases', async () => { + const entryFile = join(testRoot, 'src', 'entry.ts'); + const registryFile = join(testRoot, 'src', '_workflows.ts'); + const workflowFile = join(testRoot, 'src', 'workflows', 'workflow.ts'); + const tsconfigFile = join(testRoot, 'tsconfig.json'); + + writeFile( + tsconfigFile, + JSON.stringify({ + compilerOptions: { + paths: { + '@/*': ['./src/*'], + }, + }, + }) + ); + writeFile(entryFile, `import { allWorkflows } from '@/_workflows';\n`); + writeFile( + registryFile, + `import * as workflow from './workflows/workflow'; +export const allWorkflows = { workflow }; +` + ); + writeFile( + workflowFile, + `export async function run() { + 'use workflow'; + return 'ok'; +} +` + ); + + const discovered = await createBuilder(testRoot).discoverEntriesPublic( + [entryFile], + join(testRoot, 'out'), + tsconfigFile + ); + + expect(discovered.discoveredWorkflows).toEqual( + new Set([normalize(workflowFile)]) + ); + expect(parentHasChild(normalize(entryFile), normalize(workflowFile))).toBe( + true + ); + }); + + it('discovers path aliases inherited through tsconfig extends', async () => { + const entryFile = join(testRoot, 'src', 'entry.ts'); + const workflowFile = join(testRoot, 'src', 'workflows', 'workflow.ts'); + const baseTsconfigFile = join(testRoot, 'tsconfig.base.json'); + const tsconfigFile = join(testRoot, 'tsconfig.json'); + + writeFile( + baseTsconfigFile, + JSON.stringify({ + compilerOptions: { + baseUrl: '.', + paths: { + '@base/*': ['./src/*'], + }, + }, + }) + ); + writeFile( + tsconfigFile, + JSON.stringify({ + extends: './tsconfig.base.json', + }) + ); + writeFile(entryFile, `import { run } from '@base/workflows/workflow';\n`); + writeFile( + workflowFile, + `export async function run() { + 'use workflow'; + return 'ok'; +} +` + ); + + const discovered = await createBuilder(testRoot).discoverEntriesPublic( + [entryFile], + join(testRoot, 'out'), + tsconfigFile + ); + + expect(discovered.discoveredWorkflows).toEqual( + new Set([normalize(workflowFile)]) + ); + }); + + it('discovers path aliases with multiple wildcards', async () => { + const entryFile = join(testRoot, 'src', 'entry.ts'); + const workflowFile = join( + testRoot, + 'src', + 'features', + 'billing', + 'flows', + 'charge.ts' + ); + const tsconfigFile = join(testRoot, 'tsconfig.json'); + + writeFile( + tsconfigFile, + JSON.stringify({ + compilerOptions: { + paths: { + '@feature/*/workflow/*': ['./src/features/*/flows/*'], + }, + }, + }) + ); + writeFile( + entryFile, + `import { charge } from '@feature/billing/workflow/charge';\n` + ); + writeFile( + workflowFile, + `export async function charge() { + 'use workflow'; + return 'ok'; +} +` + ); + + const discovered = await createBuilder(testRoot).discoverEntriesPublic( + [entryFile], + join(testRoot, 'out'), + tsconfigFile + ); + + expect(discovered.discoveredWorkflows).toEqual( + new Set([normalize(workflowFile)]) + ); + }); + + it('uses nearest nested jsconfig aliases in monorepo packages', async () => { + const rootTsconfigFile = join(testRoot, 'tsconfig.json'); + const packageRoot = join(testRoot, 'packages', 'app'); + const entryFile = join(packageRoot, 'src', 'entry.js'); + const workflowFile = join(packageRoot, 'src', 'workflow.js'); + + writeFile( + rootTsconfigFile, + JSON.stringify({ + compilerOptions: { + paths: { + '@root/*': ['./root/*'], + }, + }, + }) + ); + writeFile( + join(packageRoot, 'jsconfig.json'), + JSON.stringify({ + compilerOptions: { + paths: { + '#/*': ['./src/*'], + }, + }, + }) + ); + writeFile(entryFile, `import { run } from '#/workflow';\n`); + writeFile( + workflowFile, + `export async function run() { + "use workflow"; + return "ok"; +} +` + ); + + const discovered = await createBuilder(testRoot).discoverEntriesPublic( + [entryFile], + join(testRoot, 'out') + ); + + expect(discovered.discoveredWorkflows).toEqual( + new Set([normalize(workflowFile)]) + ); + }); + + it('only treats serde files as registration candidates when they define static serde methods', async () => { + const entryFile = join(testRoot, 'src', 'entry.ts'); + const reducerFile = join(testRoot, 'src', 'reducer.ts'); + const serdeFile = join(testRoot, 'src', 'serde.ts'); + + writeFile( + entryFile, + `import './reducer'; +import './serde'; +` + ); + writeFile( + reducerFile, + `import { WORKFLOW_SERIALIZE } from '@workflow/serde'; + +export function reducer(value: unknown) { + return value?.constructor?.[WORKFLOW_SERIALIZE]; +} +` + ); + writeFile( + serdeFile, + `import { WORKFLOW_SERIALIZE as WS } from '@workflow/serde'; + +export class Value { + static classId = 'Value'; + static [WS](value: Value) { + return value; + } +} +` + ); + + const discovered = await createBuilder(testRoot).discoverEntriesPublic( + [entryFile], + join(testRoot, 'out') + ); + + expect(discovered.discoveredSerdeFiles).toEqual( + new Set([normalize(serdeFile)]) + ); + }); + + it('categorizes step, workflow, and serde usage independently', async () => { + const entryFile = join(testRoot, 'src', 'entry.ts'); + const stepFile = join(testRoot, 'src', 'step.ts'); + const workflowFile = join(testRoot, 'src', 'workflow.ts'); + const serdeFile = join(testRoot, 'src', 'serde.ts'); + + writeFile( + entryFile, + `import './step'; +import './workflow'; +import './serde'; +` + ); + writeFile( + stepFile, + `export async function runStep() { + 'use step'; + return 'ok'; +} +` + ); + writeFile( + workflowFile, + `export async function runWorkflow() { + 'use workflow'; + return 'ok'; +} +` + ); + writeFile( + serdeFile, + `export class Value { + static classId = 'Value'; + static [Symbol.for('workflow-serialize')](value: Value) { + return value; + } +} +` + ); + + const discovered = await createBuilder(testRoot).discoverEntriesPublic( + [entryFile], + join(testRoot, 'out') + ); + + expect(discovered.discoveredSteps).toEqual(new Set([normalize(stepFile)])); + expect(discovered.discoveredWorkflows).toEqual( + new Set([normalize(workflowFile)]) + ); + expect(discovered.discoveredSerdeFiles).toEqual( + new Set([normalize(serdeFile)]) + ); + }); + + it('ignores serde examples that only appear inside comments', async () => { + const entryFile = join(testRoot, 'src', 'entry.ts'); + const docsFile = join(testRoot, 'src', 'docs.ts'); + + writeFile(entryFile, `import './docs';\n`); + writeFile( + docsFile, + `/** + * import { WORKFLOW_SERIALIZE } from '@workflow/serde'; + * + * class Example { + * static [WORKFLOW_SERIALIZE](value) { + * return value; + * } + * } + */ +export const WORKFLOW_SERIALIZE = Symbol.for('workflow-serialize'); +` + ); + + const discovered = await createBuilder(testRoot).discoverEntriesPublic( + [entryFile], + join(testRoot, 'out') + ); + + expect(discovered.discoveredSerdeFiles).toEqual(new Set()); + }); + + it('relativizes nested package step registration imports', () => { + const routeDir = join(testRoot, 'app', '.well-known', 'workflow', 'v1'); + const directPackageFile = join( + testRoot, + 'node_modules', + 'direct-pkg', + 'step.js' + ); + const nestedPackageFile = join( + testRoot, + 'node_modules', + 'parent-pkg', + 'node_modules', + 'nested-pkg', + 'step.js' + ); + + writeFile( + join(testRoot, 'package.json'), + JSON.stringify({ + dependencies: { + 'direct-pkg': '1.0.0', + }, + }) + ); + writeFile( + join(testRoot, 'node_modules', 'direct-pkg', 'package.json'), + JSON.stringify({ + name: 'direct-pkg', + version: '1.0.0', + exports: { + './step': './step.js', + }, + }) + ); + writeFile(directPackageFile, `export const step = true;\n`); + writeFile( + join( + testRoot, + 'node_modules', + 'parent-pkg', + 'node_modules', + 'nested-pkg', + 'package.json' + ), + JSON.stringify({ + name: 'nested-pkg', + version: '1.0.0', + exports: { + './step': './step.js', + }, + }) + ); + writeFile(nestedPackageFile, `export const step = true;\n`); + + const builder = createBuilder(testRoot); + expect( + builder.createRouteImportSpecifierPublic(directPackageFile, routeDir) + ).toBe('direct-pkg/step'); + expect( + builder.createRouteImportSpecifierPublic(nestedPackageFile, routeDir) + ).toBe( + '../../../../node_modules/parent-pkg/node_modules/nested-pkg/step.js' + ); + }); +}); diff --git a/packages/builders/src/fast-discovery.ts b/packages/builders/src/fast-discovery.ts new file mode 100644 index 0000000000..e57f0864dc --- /dev/null +++ b/packages/builders/src/fast-discovery.ts @@ -0,0 +1,915 @@ +import { access, readFile } from 'node:fs/promises'; +import { builtinModules, createRequire } from 'node:module'; +import { dirname, extname, isAbsolute, join, resolve } from 'node:path'; +import { promisify } from 'node:util'; +import enhancedResolveOriginal from 'enhanced-resolve'; +import { findUp } from 'find-up'; +import JSON5 from 'json5'; +import { importParents } from './discover-entries-esbuild-plugin.js'; +import { detectWorkflowPatterns } from './transform-utils.js'; + +const FAST_DISCOVERY_SOURCE_EXTENSIONS = [ + '.ts', + '.tsx', + '.mts', + '.cts', + '.js', + '.jsx', + '.mjs', + '.cjs', +]; +const FAST_DISCOVERY_SOURCE_EXTENSION_SET = new Set( + FAST_DISCOVERY_SOURCE_EXTENSIONS +); +const fastDiscoveryResolve = promisify( + enhancedResolveOriginal.create({ + extensions: [...FAST_DISCOVERY_SOURCE_EXTENSIONS, '.json', '.node'], + fullySpecified: false, + conditionNames: ['node', 'import', 'require'], + }) +); +const require = createRequire(import.meta.url); + +const FAST_DISCOVERY_READ_CONCURRENCY = 32; +const FAST_DISCOVERY_RESOLVE_CONCURRENCY = 32; +const FAST_DISCOVERY_FILE_CONCURRENCY = 128; +const PACKAGE_JSON = 'package.json'; +const NODE_BUILTIN_SPECIFIERS = new Set([ + ...builtinModules, + ...builtinModules.map((moduleName) => `node:${moduleName}`), +]); +const IMPORT_SPECIFIER_PATTERNS = [ + /\bfrom\s+['"]([^'"]+)['"]/g, + /(?:^|[;\n])\s*import\s+['"]([^'"]+)['"]/g, + /\bimport\s*\(\s*['"]([^'"]+)['"]\s*\)/g, + /\brequire\s*\(\s*['"]([^'"]+)['"]\s*\)/g, +]; + +export interface DiscoveredEntries { + discoveredSteps: Set; + discoveredWorkflows: Set; + discoveredSerdeFiles: Set; +} + +interface FastDiscoverEntriesOptions { + entryPoints: string[]; + state: DiscoveredEntries; + defaultTsconfigPath: string | undefined; + workingDir: string; +} + +interface PackageInfo { + root: string; + hasWorkflowDependency: boolean; +} + +interface TsconfigPathAlias { + pattern: string; + patternParts: string[]; + targets: Array<{ + template: string; + parts: string[]; + }>; +} + +interface TsconfigPathAliasLoadResult { + aliases: TsconfigPathAlias[]; + baseUrl: string | undefined; +} + +function createLimiter(concurrency: number) { + let activeCount = 0; + const queue: Array<() => void> = []; + + const acquire = async () => { + await new Promise((resolve) => { + const run = () => { + activeCount++; + resolve(); + }; + + if (activeCount < concurrency) { + run(); + } else { + queue.push(run); + } + }); + }; + + const release = () => { + activeCount--; + const next = queue.shift(); + if (next) { + next(); + } + }; + + return async function limit(fn: () => Promise): Promise { + await acquire(); + try { + return await fn(); + } finally { + release(); + } + }; +} + +function normalizePath(filePath: string): string { + return filePath.replace(/\\/g, '/'); +} + +function isJsTsFile(filePath: string): boolean { + return FAST_DISCOVERY_SOURCE_EXTENSION_SET.has(extname(filePath)); +} + +function isRelativeOrAbsoluteSpecifier(specifier: string): boolean { + return specifier.startsWith('.') || isAbsolute(specifier); +} + +function getPackageNameFromSpecifier(specifier: string): string | null { + const strippedSpecifier = stripImportSpecifierQuery(specifier); + if (isRelativeOrAbsoluteSpecifier(strippedSpecifier)) { + return null; + } + + if (strippedSpecifier.startsWith('@')) { + const [scope, name] = strippedSpecifier.split('/'); + return scope && name ? `${scope}/${name}` : null; + } + + return strippedSpecifier.split('/')[0] || null; +} + +function stripImportSpecifierQuery(specifier: string): string { + const queryIndex = specifier.indexOf('?'); + const hashIndex = specifier.indexOf('#'); + const endIndex = + queryIndex === -1 + ? hashIndex + : hashIndex === -1 + ? queryIndex + : Math.min(queryIndex, hashIndex); + return endIndex === -1 ? specifier : specifier.slice(0, endIndex); +} + +function shouldSkipFastDiscoveryImport(specifier: string): boolean { + if (NODE_BUILTIN_SPECIFIERS.has(specifier)) { + return true; + } + + const pathLikeSpecifier = stripImportSpecifierQuery(specifier); + if ( + !pathLikeSpecifier.startsWith('.') && + !isAbsolute(pathLikeSpecifier) && + !pathLikeSpecifier.includes('/') + ) { + return false; + } + + const extension = extname(pathLikeSpecifier); + return ( + extension !== '' && !FAST_DISCOVERY_SOURCE_EXTENSION_SET.has(extension) + ); +} + +function matchTsconfigPathAlias( + specifier: string, + alias: TsconfigPathAlias +): string[] | null { + if (alias.patternParts.length === 1) { + return specifier === alias.pattern ? [] : null; + } + + const captures: string[] = []; + let position = 0; + const firstPart = alias.patternParts[0]; + if (!specifier.startsWith(firstPart)) { + return null; + } + position = firstPart.length; + + for (let i = 1; i < alias.patternParts.length; i++) { + const part = alias.patternParts[i]; + if (i === alias.patternParts.length - 1) { + if (!specifier.endsWith(part)) { + return null; + } + captures.push(specifier.slice(position, specifier.length - part.length)); + return captures; + } + + const nextIndex = specifier.indexOf(part, position); + if (nextIndex === -1) { + return null; + } + captures.push(specifier.slice(position, nextIndex)); + position = nextIndex + part.length; + } + + return captures; +} + +function applyTsconfigPathTarget( + target: TsconfigPathAlias['targets'][number], + captures: string[] +): string { + if (target.parts.length === 1) { + return target.template; + } + + let resolved = target.parts[0]; + for (let i = 1; i < target.parts.length; i++) { + resolved += (captures[i - 1] ?? '') + target.parts[i]; + } + return resolved; +} + +function isGeneratedBuildArtifactPath(filePath: string): boolean { + const normalizedPath = normalizePath(filePath); + return ( + normalizedPath.includes('/.nitro/') || + normalizedPath.includes('/.output/') || + normalizedPath.includes('/.next/') || + normalizedPath.includes('/.nuxt/') || + normalizedPath.includes('/.svelte-kit/') || + normalizedPath.includes('/.vercel/') || + normalizedPath.includes('/.well-known/workflow/') + ); +} + +function isNodeModulesPath(filePath: string): boolean { + const normalizedPath = normalizePath(filePath); + return ( + normalizedPath.includes('/node_modules/') || + normalizedPath.includes('/.pnpm/') + ); +} + +function addImportParent(parent: string, child: string): void { + const normalizedParent = normalizePath(parent); + const normalizedChild = normalizePath(child); + let children = importParents.get(normalizedParent); + if (!children) { + children = new Set(); + importParents.set(normalizedParent, children); + } + children.add(normalizedChild); +} + +function extractImportSpecifiers(source: string): string[] { + if ( + !source.includes('import') && + !source.includes('require') && + !source.includes('from') + ) { + return []; + } + + const specifiers = new Set(); + + for (const importPattern of IMPORT_SPECIFIER_PATTERNS) { + for (const match of source.matchAll(importPattern)) { + const specifier = match[1]; + if (specifier) { + specifiers.add(specifier); + } + } + } + + return Array.from(specifiers); +} + +function hasWorkflowDependency(dependencies: unknown): boolean { + if ( + typeof dependencies !== 'object' || + dependencies === null || + Array.isArray(dependencies) + ) { + return false; + } + + return Object.keys(dependencies).some( + (dependency) => + dependency === 'workflow' || dependency.startsWith('@workflow/') + ); +} + +function stripComments(source: string): string { + return source + .replace(/\/\*[\s\S]*?\*\//g, '') + .replace(/(^|[^:])\/\/.*$/gm, '$1'); +} + +function hasLikelySerdeClass(source: string): boolean { + if (!source.includes('static') || !source.includes('[')) { + return false; + } + + const uncommentedSource = stripComments(source); + if ( + /static\s+\[\s*(?:WORKFLOW_(?:SERIALIZE|DESERIALIZE)|Symbol\.for\s*\(\s*['"]workflow-(?:serialize|deserialize)['"]\s*\))\s*\]\s*\(/.test( + uncommentedSource + ) + ) { + return true; + } + + if ( + !/from\s+['"]@workflow\/serde['"]|require\s*\(\s*['"]@workflow\/serde['"]\s*\)/.test( + uncommentedSource + ) + ) { + return false; + } + + return /static\s+\[\s*[$A-Z_a-z][$\w]*\s*\]\s*\(/.test(uncommentedSource); +} + +async function loadTsconfigPathAliases( + tsconfigPath: string | undefined, + seen = new Set() +): Promise { + if (!tsconfigPath) { + return []; + } + + return (await loadTsconfigPathAliasConfig(tsconfigPath, seen)).aliases; +} + +async function loadTsconfigPathAliasConfig( + tsconfigPath: string, + seen: Set +): Promise { + const normalizedTsconfigPath = resolve(tsconfigPath); + if (seen.has(normalizedTsconfigPath)) { + return { aliases: [], baseUrl: undefined }; + } + + seen.add(normalizedTsconfigPath); + + try { + const source = await readFile(normalizedTsconfigPath, 'utf8'); + const parsed = JSON5.parse(source) as { + extends?: unknown; + compilerOptions?: { + baseUrl?: unknown; + paths?: unknown; + }; + }; + const compilerOptions = parsed.compilerOptions; + + let baseConfig: TsconfigPathAliasLoadResult = { + aliases: [], + baseUrl: undefined, + }; + if (typeof parsed.extends === 'string') { + const baseTsconfigPath = await resolveTsconfigExtendsPath( + parsed.extends, + normalizedTsconfigPath + ); + if (baseTsconfigPath) { + baseConfig = await loadTsconfigPathAliasConfig(baseTsconfigPath, seen); + } + } + + const baseUrl = + typeof compilerOptions?.baseUrl === 'string' + ? resolve(dirname(normalizedTsconfigPath), compilerOptions.baseUrl) + : baseConfig.baseUrl; + + if ( + !compilerOptions || + typeof compilerOptions.paths !== 'object' || + compilerOptions.paths === null || + Array.isArray(compilerOptions.paths) + ) { + return { + aliases: baseConfig.aliases, + baseUrl, + }; + } + + const baseDir = baseUrl ?? dirname(normalizedTsconfigPath); + const aliases: TsconfigPathAlias[] = []; + + for (const [pattern, rawTargets] of Object.entries(compilerOptions.paths)) { + if (!Array.isArray(rawTargets)) { + continue; + } + + const targets = rawTargets + .filter((target): target is string => typeof target === 'string') + .map((target) => { + const template = resolve(baseDir, target); + return { + template, + parts: template.split('*'), + }; + }); + if (targets.length === 0) { + continue; + } + + aliases.push({ + pattern, + patternParts: pattern.split('*'), + targets, + }); + } + + return { aliases, baseUrl }; + } catch { + return { aliases: [], baseUrl: undefined }; + } finally { + seen.delete(normalizedTsconfigPath); + } +} + +async function resolveTsconfigExtendsPath( + extendsValue: string, + tsconfigPath: string +): Promise { + const configDir = dirname(tsconfigPath); + if (extendsValue.startsWith('.') || isAbsolute(extendsValue)) { + const resolved = isAbsolute(extendsValue) + ? extendsValue + : resolve(configDir, extendsValue); + return findExistingTsconfigPath(resolved); + } + + try { + return require.resolve(extendsValue, { paths: [configDir] }); + } catch {} + + try { + return require.resolve(`${extendsValue}/tsconfig.json`, { + paths: [configDir], + }); + } catch { + return undefined; + } +} + +async function findExistingTsconfigPath( + candidatePath: string +): Promise { + const candidates = + extname(candidatePath) === '' + ? [ + `${candidatePath}.json`, + join(candidatePath, 'tsconfig.json'), + candidatePath, + ] + : [candidatePath]; + + for (const candidate of candidates) { + try { + await access(candidate); + return candidate; + } catch {} + } + + return undefined; +} + +async function findPackageInfo( + filePath: string, + packageInfoCache: Map> +): Promise { + let currentDir = dirname(filePath); + + while (currentDir && currentDir !== dirname(currentDir)) { + const cached = packageInfoCache.get(currentDir); + if (cached) { + const cachedInfo = await cached; + if (cachedInfo) { + return cachedInfo; + } + currentDir = dirname(currentDir); + continue; + } + + const packageJsonPath = join(currentDir, PACKAGE_JSON); + const packageInfoPromise = readFile(packageJsonPath, 'utf8') + .then((source): PackageInfo => { + const parsed = JSON.parse(source) as { + name?: unknown; + dependencies?: unknown; + peerDependencies?: unknown; + optionalDependencies?: unknown; + devDependencies?: unknown; + }; + const packageName = typeof parsed.name === 'string' ? parsed.name : ''; + return { + root: normalizePath(currentDir), + hasWorkflowDependency: + packageName === 'workflow' || + packageName.startsWith('@workflow/') || + hasWorkflowDependency(parsed.dependencies) || + hasWorkflowDependency(parsed.peerDependencies) || + hasWorkflowDependency(parsed.optionalDependencies) || + hasWorkflowDependency(parsed.devDependencies), + }; + }) + .catch(() => null); + + packageInfoCache.set(currentDir, packageInfoPromise); + const packageInfo = await packageInfoPromise; + if (packageInfo) { + return packageInfo; + } + + currentDir = dirname(currentDir); + } + + return null; +} + +export async function fastDiscoverEntries({ + entryPoints, + state, + defaultTsconfigPath, + workingDir, +}: FastDiscoverEntriesOptions): Promise { + const readLimit = createLimiter(FAST_DISCOVERY_READ_CONCURRENCY); + const resolveLimit = createLimiter(FAST_DISCOVERY_RESOLVE_CONCURRENCY); + const resolveCache = new Map>(); + const fileExistsCache = new Map>(); + const tsconfigPathByDirCache = new Map>(); + const tsconfigAliasesCache = new Map>(); + const packageInfoCache = new Map>(); + const packageSpecifierInfoCache = new Map< + string, + Promise + >(); + const queuedFiles = new Set(); + const processedFiles = new Set(); + const queue: string[] = []; + const enqueueFile = (filePath: string | undefined | null): void => { + if (!filePath) return; + const normalizedPath = normalizePath(filePath); + if ( + queuedFiles.has(normalizedPath) || + processedFiles.has(normalizedPath) || + !isJsTsFile(normalizedPath) || + isGeneratedBuildArtifactPath(normalizedPath) + ) { + return; + } + queuedFiles.add(normalizedPath); + queue.push(normalizedPath); + }; + + const readSource = async (filePath: string): Promise => { + return await readLimit(async () => { + try { + return await readFile(filePath, 'utf8'); + } catch { + return null; + } + }); + }; + + const fileExists = (filePath: string): Promise => { + const cached = fileExistsCache.get(filePath); + if (cached) { + return cached; + } + + const promise = readLimit(async () => { + try { + await access(filePath); + return true; + } catch { + return false; + } + }); + fileExistsCache.set(filePath, promise); + return promise; + }; + + const findTsconfigPathForImporter = ( + importer: string + ): Promise => { + if (isNodeModulesPath(importer)) { + return Promise.resolve(defaultTsconfigPath); + } + + const importerDir = dirname(importer); + const cached = tsconfigPathByDirCache.get(importerDir); + if (cached) { + return cached; + } + + const promise = findUp(['tsconfig.json', 'jsconfig.json'], { + cwd: importerDir, + }).then((found) => found ?? defaultTsconfigPath); + tsconfigPathByDirCache.set(importerDir, promise); + return promise; + }; + + const loadAliasesForTsconfig = ( + configPath: string | undefined + ): Promise => { + if (!configPath) { + return Promise.resolve([]); + } + + const cached = tsconfigAliasesCache.get(configPath); + if (cached) { + return cached; + } + + const promise = loadTsconfigPathAliases(configPath); + tsconfigAliasesCache.set(configPath, promise); + return promise; + }; + + const resolvePathLikeSpecifier = async ( + importer: string, + specifier: string + ): Promise => { + const strippedSpecifier = stripImportSpecifierQuery(specifier); + const basePath = isAbsolute(strippedSpecifier) + ? strippedSpecifier + : resolve(dirname(importer), strippedSpecifier); + const extension = extname(basePath); + if (extension !== '') { + return FAST_DISCOVERY_SOURCE_EXTENSION_SET.has(extension) && + (await fileExists(basePath)) + ? normalizePath(basePath) + : null; + } + + for (const candidate of [ + ...FAST_DISCOVERY_SOURCE_EXTENSIONS.map( + (candidateExtension) => `${basePath}${candidateExtension}` + ), + ...FAST_DISCOVERY_SOURCE_EXTENSIONS.map((candidateExtension) => + join(basePath, `index${candidateExtension}`) + ), + ]) { + if (await fileExists(candidate)) { + return normalizePath(candidate); + } + } + + return null; + }; + + const resolveWithTsconfigPaths = async ( + importer: string, + specifier: string + ): Promise => { + if (specifier.startsWith('.') || isAbsolute(specifier)) { + return null; + } + + const tsconfigPath = await findTsconfigPathForImporter(importer); + const tsconfigPathAliases = await loadAliasesForTsconfig(tsconfigPath); + if (tsconfigPathAliases.length === 0) { + return null; + } + + for (const alias of tsconfigPathAliases) { + const captures = matchTsconfigPathAlias(specifier, alias); + if (!captures) { + continue; + } + + for (const target of alias.targets) { + const targetPath = applyTsconfigPathTarget(target, captures); + try { + const resolved = await resolvePathLikeSpecifier(importer, targetPath); + if (resolved) { + return resolved; + } + } catch {} + } + } + + return null; + }; + + const findPackageInfoBySpecifier = ( + packageName: string + ): Promise => { + const cached = packageSpecifierInfoCache.get(packageName); + if (cached) { + return cached; + } + + const packageInfoPromise = (async () => { + let packageJsonPath: string; + try { + packageJsonPath = require.resolve(`${packageName}/package.json`, { + paths: [workingDir], + }); + } catch { + return null; + } + + try { + const source = await readLimit(() => readFile(packageJsonPath, 'utf8')); + const parsed = JSON.parse(source) as { + name?: unknown; + dependencies?: unknown; + peerDependencies?: unknown; + optionalDependencies?: unknown; + devDependencies?: unknown; + }; + const parsedPackageName = + typeof parsed.name === 'string' ? parsed.name : packageName; + return { + root: normalizePath(dirname(packageJsonPath)), + hasWorkflowDependency: + parsedPackageName === 'workflow' || + parsedPackageName.startsWith('@workflow/') || + hasWorkflowDependency(parsed.dependencies) || + hasWorkflowDependency(parsed.peerDependencies) || + hasWorkflowDependency(parsed.optionalDependencies) || + hasWorkflowDependency(parsed.devDependencies), + }; + } catch { + return null; + } + })(); + packageSpecifierInfoCache.set(packageName, packageInfoPromise); + return packageInfoPromise; + }; + + const shouldResolveBareSpecifier = async ( + specifier: string + ): Promise => { + const packageName = getPackageNameFromSpecifier(specifier); + if (!packageName) { + return true; + } + if (packageName === 'workflow' || packageName.startsWith('@workflow/')) { + return true; + } + + const packageInfo = await findPackageInfoBySpecifier(packageName); + if (!packageInfo) { + return true; + } + if (packageInfo.hasWorkflowDependency) { + return true; + } + + return false; + }; + + const resolveImport = ( + importer: string, + specifier: string + ): Promise => { + const cacheKey = `${dirname(importer)}\0${specifier}`; + const cached = resolveCache.get(cacheKey); + if (cached) { + return cached; + } + + const resolvedPromise = resolveLimit(async () => { + if (isRelativeOrAbsoluteSpecifier(specifier)) { + return resolvePathLikeSpecifier(importer, specifier); + } + + const resolvedAlias = await resolveWithTsconfigPaths(importer, specifier); + if (resolvedAlias) { + return resolvedAlias; + } + + if (!(await shouldResolveBareSpecifier(specifier))) { + return null; + } + + try { + const resolved = await fastDiscoveryResolve( + dirname(importer), + specifier + ); + return typeof resolved === 'string' ? normalizePath(resolved) : null; + } catch { + return null; + } + }); + resolveCache.set(cacheKey, resolvedPromise); + return resolvedPromise; + }; + + const shouldFollowImportsFromFile = async ( + importer: string, + forceFollow: boolean + ): Promise => { + if (forceFollow) { + return true; + } + if (!isNodeModulesPath(importer)) { + return true; + } + + const packageInfo = await findPackageInfo(importer, packageInfoCache); + return packageInfo?.hasWorkflowDependency === true; + }; + + const processImportSpecifier = async ( + filePath: string, + specifier: string, + forceFollowImports: boolean + ): Promise => { + if (shouldSkipFastDiscoveryImport(specifier)) { + return; + } + if (!(await shouldFollowImportsFromFile(filePath, forceFollowImports))) { + return; + } + + const resolved = await resolveImport(filePath, specifier); + if (!resolved) { + return; + } + + addImportParent(filePath, resolved); + if (!isJsTsFile(resolved) || isGeneratedBuildArtifactPath(resolved)) { + return; + } + + if (specifier.startsWith('.')) { + enqueueFile(resolved); + return; + } + + enqueueFile(resolved); + }; + + const processFile = async (filePath: string): Promise => { + queuedFiles.delete(filePath); + if (processedFiles.has(filePath)) { + return; + } + processedFiles.add(filePath); + const source = await readSource(filePath); + if (source === null) { + return; + } + + const patterns = detectWorkflowPatterns(source); + if (patterns.hasUseWorkflow) { + state.discoveredWorkflows.add(filePath); + } + if (patterns.hasUseStep) { + state.discoveredSteps.add(filePath); + } + if (patterns.hasSerde && hasLikelySerdeClass(source)) { + state.discoveredSerdeFiles.add(filePath); + } + + const forceFollowImports = patterns.hasDirective || patterns.hasSerde; + if ( + !forceFollowImports && + !(await shouldFollowImportsFromFile(filePath, false)) + ) { + return; + } + + const specifiers = extractImportSpecifiers(source); + if (specifiers.length === 0) { + return; + } + + await Promise.all( + specifiers.map((specifier) => + processImportSpecifier(filePath, specifier, forceFollowImports) + ) + ); + }; + + for (const entryPoint of entryPoints) { + enqueueFile(entryPoint); + } + + const inFlight = new Set>(); + const scheduleFiles = () => { + while ( + queue.length > 0 && + inFlight.size < FAST_DISCOVERY_FILE_CONCURRENCY + ) { + const filePath = queue.shift(); + if (!filePath) { + continue; + } + + const promise = processFile(filePath).finally(() => { + inFlight.delete(promise); + }); + inFlight.add(promise); + } + }; + + scheduleFiles(); + while (inFlight.size > 0) { + await Promise.race(inFlight); + scheduleFiles(); + } +} diff --git a/packages/builders/src/step-source-registration.test.ts b/packages/builders/src/step-source-registration.test.ts new file mode 100644 index 0000000000..91c3062621 --- /dev/null +++ b/packages/builders/src/step-source-registration.test.ts @@ -0,0 +1,124 @@ +import { + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { BaseBuilder, type DiscoveredEntries } from './base-builder.js'; +import type { StandaloneConfig } from './types.js'; + +class TestBuilder extends BaseBuilder { + async build(): Promise { + // no-op + } + + protected get shouldLogBaseBuilderInfo(): boolean { + return false; + } + + public createSourceStepRegistrations( + inputFiles: string[], + outfile: string, + discoveredEntries: DiscoveredEntries + ) { + return this.createStepsBundle({ + inputFiles, + outfile, + externalizeNonSteps: true, + bundleTransitiveLocalStepDependencies: false, + sourceStepRegistrationImports: true, + discoveredEntries, + }); + } +} + +const realTmpdir = realpathSync(tmpdir()); + +function writeFile(path: string, contents: string): void { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, contents, 'utf-8'); +} + +function createBuilder(workingDir: string): TestBuilder { + const config: StandaloneConfig = { + buildTarget: 'standalone', + workingDir, + dirs: ['.'], + stepsBundlePath: join(workingDir, '.workflow', 'steps.js'), + workflowsBundlePath: join(workingDir, '.workflow', 'workflows.js'), + webhookBundlePath: join(workingDir, '.workflow', 'webhook.js'), + }; + return new TestBuilder(config); +} + +describe('step source registration', () => { + let testRoot: string; + + beforeEach(() => { + testRoot = mkdtempSync(join(realTmpdir, 'workflow-step-registration-')); + writeFile( + join(testRoot, 'node_modules', 'workflow', 'package.json'), + JSON.stringify({ name: 'workflow', version: '1.0.0' }) + ); + writeFile( + join(testRoot, 'node_modules', 'workflow', 'internal', 'builtins.js'), + 'export const __builtins = true;\n' + ); + }); + + afterEach(() => { + rmSync(testRoot, { recursive: true, force: true }); + }); + + it('imports serde-only files for step context class registration', async () => { + const entryFile = join(testRoot, 'src', 'entry.ts'); + const stepFile = join(testRoot, 'src', 'step.ts'); + const serdeFile = join(testRoot, 'src', 'serde.ts'); + const outfile = join(testRoot, '.workflow', 'steps.js'); + + mkdirSync(dirname(outfile), { recursive: true }); + writeFile(entryFile, `export { runStep } from './step';\n`); + writeFile( + stepFile, + `export async function runStep() { + 'use step'; + return 1; +} +` + ); + writeFile( + serdeFile, + `export class Value { + static classId = 'Value'; + static [Symbol.for('workflow-serialize')](value: Value) { + return value; + } + static [Symbol.for('workflow-deserialize')](value: Value) { + return value; + } +} +` + ); + + const discoveredEntries: DiscoveredEntries = { + discoveredSteps: new Set([stepFile]), + discoveredWorkflows: new Set(), + discoveredSerdeFiles: new Set([serdeFile]), + }; + + const { manifest } = await createBuilder( + testRoot + ).createSourceStepRegistrations([entryFile], outfile, discoveredEntries); + const generated = readFileSync(outfile, 'utf-8'); + + expect(generated).toContain('import "workflow/internal/builtins";'); + expect(generated).toContain('import "../src/step.ts";'); + expect(generated).toContain('import "../src/serde.ts";'); + expect(Object.keys(manifest.classes ?? {})).toContain('src/serde.ts'); + }); +}); diff --git a/packages/builders/src/transform-utils.test.ts b/packages/builders/src/transform-utils.test.ts index 5ce399e4e2..44d1462ddd 100644 --- a/packages/builders/src/transform-utils.test.ts +++ b/packages/builders/src/transform-utils.test.ts @@ -333,11 +333,7 @@ describe('transform-utils patterns', () => { expect(result.hasSerde).toBe(true); }); - it('regexp detection matches directives inside template literals (false positive)', () => { - // detectWorkflowPatterns uses regexp and will match directive-like - // strings inside template literals. The discover-entries plugin - // handles this by using the SWC plugin manifest (AST-level) for - // directive discovery instead of relying on regexp alone. + it('does not detect directives inside template literals', () => { const source = `'use client'; const CODE_SNIPPET = \`import { sleep } from "workflow"; @@ -347,6 +343,88 @@ export async function handleUserSignup(email: string) { } \`; export default function Page() { return null; } +`; + const result = detectWorkflowPatterns(source); + expect(result.hasUseWorkflow).toBe(false); + expect(result.hasDirective).toBe(false); + }); + + it('does not detect single-quoted step directives inside template literals', () => { + const source = `const CODE_SNIPPET = \` +export async function doThing() { + 'use step'; +} +\`; +`; + const result = detectWorkflowPatterns(source); + expect(result.hasUseStep).toBe(false); + expect(result.hasDirective).toBe(false); + }); + + it('does not detect directives inside comments', () => { + const source = `/* +export async function run() { + "use workflow"; +} +*/ + +// 'use step'; +export const value = 1; +`; + const result = detectWorkflowPatterns(source); + expect(result.hasUseWorkflow).toBe(false); + expect(result.hasUseStep).toBe(false); + expect(result.hasDirective).toBe(false); + }); + + it('does not detect directive-looking strings inside multiline calls', () => { + const source = `console.log( + "use step" +); + +console.log( + 'use workflow' +); +`; + const result = detectWorkflowPatterns(source); + expect(result.hasUseWorkflow).toBe(false); + expect(result.hasUseStep).toBe(false); + expect(result.hasDirective).toBe(false); + }); + + it('does not detect quoted directive-looking strings inside multiline calls', () => { + const source = `console.log( + '"use step"' +); + +console.log( + "'use step'" +); +`; + const result = detectWorkflowPatterns(source); + expect(result.hasUseStep).toBe(false); + expect(result.hasDirective).toBe(false); + }); + + it('still detects real directives after template literals', () => { + const source = `const CODE_SNIPPET = \` + "use workflow"; +\`; + +export async function run() { + "use workflow"; +} +`; + const result = detectWorkflowPatterns(source); + expect(result.hasUseWorkflow).toBe(true); + expect(result.hasDirective).toBe(true); + }); + + it('still detects directives after other directive prologue entries', () => { + const source = `export async function run() { + "use strict"; + "use workflow"; +} `; const result = detectWorkflowPatterns(source); expect(result.hasUseWorkflow).toBe(true); diff --git a/packages/builders/src/transform-utils.ts b/packages/builders/src/transform-utils.ts index eedb372efc..7ac1733bdb 100644 --- a/packages/builders/src/transform-utils.ts +++ b/packages/builders/src/transform-utils.ts @@ -22,11 +22,45 @@ export const workflowSerdeSymbolPattern = export const workflowSerdeComputedPropertyPattern = /\[\s*WORKFLOW_(?:SERIALIZE|DESERIALIZE)\s*\]/; +const templateLiteralPattern = /`(?:\\[\s\S]|[^`\\])*`/g; +const commentPattern = /\/\*[\s\S]*?\*\/|\/\/[^\r\n]*/g; +const directiveLinePattern = /^\s*(['"])(use workflow|use step)\1;?\s*$/; +const stringDirectiveLinePattern = /^\s*(['"])[^'"]+\1;?\s*$/; + // Pattern to detect generated workflow route files that should be excluded // These files are generated by the build process and should not be re-processed export const generatedWorkflowPathPattern = /[/\\]\.well-known[/\\]workflow[/\\]/; +function hasDirective(source: string, directive: 'use workflow' | 'use step') { + let previousMeaningfulLine: string | undefined; + + for (const line of source.split(/\r?\n/)) { + const trimmedLine = line.trim(); + if (trimmedLine === '') { + continue; + } + + const directiveMatch = directiveLinePattern.exec(trimmedLine); + if (directiveMatch) { + if ( + directiveMatch[2] === directive && + (previousMeaningfulLine === undefined || + previousMeaningfulLine.endsWith('{') || + stringDirectiveLinePattern.test(previousMeaningfulLine)) + ) { + return true; + } + previousMeaningfulLine = trimmedLine; + continue; + } + + previousMeaningfulLine = trimmedLine; + } + + return false; +} + /** * Detects workflow-related patterns in source code. */ @@ -51,11 +85,32 @@ export interface WorkflowPatternMatch { * @returns Object with flags for each detected pattern */ export function detectWorkflowPatterns(source: string): WorkflowPatternMatch { - const hasUseWorkflow = useWorkflowPattern.test(source); - const hasUseStep = useStepPattern.test(source); - const hasSerdeImport = workflowSerdeImportPattern.test(source); - const hasSerdeSymbol = workflowSerdeSymbolPattern.test(source); + const hasDirectiveSubstring = + source.includes('use workflow') || source.includes('use step'); + const sourceForDirectives = + hasDirectiveSubstring && (source.includes('`') || source.includes('/')) + ? source + .replace(templateLiteralPattern, (match) => + match.replace(/[^\r\n]/g, ' ') + ) + .replace(commentPattern, (match) => match.replace(/[^\r\n]/g, ' ')) + : source; + const hasUseWorkflow = + source.includes('use workflow') && + hasDirective(sourceForDirectives, 'use workflow'); + const hasUseStep = + source.includes('use step') && + hasDirective(sourceForDirectives, 'use step'); + const hasSerdeImport = + source.includes('@workflow/serde') && + workflowSerdeImportPattern.test(source); + const hasSerdeSymbol = + (source.includes('workflow-serialize') || + source.includes('workflow-deserialize')) && + workflowSerdeSymbolPattern.test(source); const hasSerdeComputedProperty = + (source.includes('WORKFLOW_SERIALIZE') || + source.includes('WORKFLOW_DESERIALIZE')) && workflowSerdeComputedPropertyPattern.test(source); return { diff --git a/packages/core/e2e/utils.test.ts b/packages/core/e2e/utils.test.ts index 4952ceabfc..f85247e488 100644 --- a/packages/core/e2e/utils.test.ts +++ b/packages/core/e2e/utils.test.ts @@ -43,14 +43,14 @@ describe('hasStepSourceMaps', () => { expect(hasStepSourceMaps()).toBe(true); }); - test('does not expect source filenames for webpack local dev with lazy discovery disabled', () => { + test('expects source filenames for webpack local dev with lazy discovery disabled', () => { setStepSourceMapEnv({ appName: 'nextjs-webpack', dev: true, lazyDiscovery: false, }); - expect(hasStepSourceMaps()).toBe(false); + expect(hasStepSourceMaps()).toBe(true); }); test('does not expect source filenames for turbopack local dev with lazy discovery disabled', () => { diff --git a/packages/core/e2e/utils.ts b/packages/core/e2e/utils.ts index b669393089..0ad4a7cdbe 100644 --- a/packages/core/e2e/utils.ts +++ b/packages/core/e2e/utils.ts @@ -135,12 +135,6 @@ export function hasStepSourceMaps(): boolean { if (appName === 'nextjs-turbopack') { return false; } - // Webpack's eager Next flow route executes steps from the generated - // __step_registrations.js bundle. Lazy discovery imports step sources through - // the flow route and preserves source filenames in local dev stacks. - if (appName === 'nextjs-webpack' && !isNextLazyDiscoveryEnabledForTest()) { - return false; - } // V2 carve-out: the V2 combined flow handler does not yet wire up inline // source maps for step bundles across the framework integrations on Vercel. // To unblock CI while V2 source-map coverage catches up, treat every diff --git a/packages/next/src/builder-eager.ts b/packages/next/src/builder-eager.ts index 93d587d951..05e73587b6 100644 --- a/packages/next/src/builder-eager.ts +++ b/packages/next/src/builder-eager.ts @@ -96,17 +96,18 @@ export async function getNextBuilderEager() { if (this.config.watch) { // TODO: implement watch mode for combined bundle // For now, fall back to full rebuild on file changes - let stepsCtx = combinedResult?.stepsContext; - if (!stepsCtx) { + if (!combinedResult?.interimBundleCtx || !combinedResult.bundleFinal) { throw new Error( - 'Invariant: expected steps build context in watch mode' + 'Invariant: expected workflow build context in watch mode' ); } - // Use stepsCtx for the watch rebuild (workflow interim ctx from combined) + // Step registrations may be emitted as source imports without an + // esbuild context when externalizeNonSteps is enabled. + let stepsCtx = combinedResult.stepsContext; let workflowsCtx = { - interimBundleCtx: combinedResult?.interimBundleCtx!, - bundleFinal: combinedResult?.bundleFinal!, + interimBundleCtx: combinedResult.interimBundleCtx, + bundleFinal: combinedResult.bundleFinal, }; const normalizePath = (pathname: string) => @@ -195,18 +196,14 @@ export async function getNextBuilderEager() { }; const fullRebuild = async () => { + this.clearDiscoveredEntriesCache(); const newInputFiles = await this.getInputFiles(); options.inputFiles = newInputFiles; - await stepsCtx!.dispose(); + await stepsCtx?.dispose(); await workflowsCtx.interimBundleCtx.dispose(); const newCombined = await this.buildCombinedFunction(options); - if (!newCombined?.stepsContext) { - throw new Error( - 'Invariant: expected steps build context after rebuild' - ); - } stepsCtx = newCombined.stepsContext; if (!newCombined?.interimBundleCtx || !newCombined?.bundleFinal) { @@ -222,60 +219,6 @@ export async function getNextBuilderEager() { await writeManifest(newCombined.manifest); }; - const logBuildMessages = ( - result: { - errors?: import('esbuild').Message[]; - warnings?: import('esbuild').Message[]; - }, - label: string - ) => { - const logByType = ( - messages: import('esbuild').Message[] | undefined, - method: 'error' | 'warn' - ) => { - if (!messages || messages.length === 0) { - return; - } - const descriptor = method === 'error' ? 'errors' : 'warnings'; - console[method](`${descriptor} while rebuilding ${label}`); - for (const message of messages) { - console[method](message); - } - }; - - logByType(result.errors, 'error'); - logByType(result.warnings, 'warn'); - }; - - const rebuildExistingFiles = async () => { - const rebuiltStepStart = Date.now(); - const stepsResult = await stepsCtx!.rebuild(); - logBuildMessages(stepsResult, 'steps bundle'); - console.log( - 'Rebuilt steps bundle', - `${Date.now() - rebuiltStepStart}ms` - ); - - const rebuiltWorkflowStart = Date.now(); - const workflowResult = await workflowsCtx.interimBundleCtx.rebuild(); - logBuildMessages(workflowResult, 'workflows bundle'); - - if ( - !workflowResult.outputFiles || - workflowResult.outputFiles.length === 0 - ) { - console.error( - 'No output generated while rebuilding workflows bundle' - ); - return; - } - await workflowsCtx.bundleFinal(workflowResult.outputFiles[0].text); - console.log( - 'Rebuilt workflow bundle', - `${Date.now() - rebuiltWorkflowStart}ms` - ); - }; - const isWatchableFile = (path: string) => watchableExtensions.has(extname(path)); @@ -373,13 +316,12 @@ export async function getNextBuilderEager() { } enqueue(async () => { - if (addedFiles.length > 0 || removedFiles.length > 0) { + if ( + addedFiles.length > 0 || + modifiedFiles.length > 0 || + removedFiles.length > 0 + ) { await fullRebuild(); - return; - } - - if (modifiedFiles.length > 0) { - await rebuildExistingFiles(); } }); }); @@ -456,6 +398,7 @@ export async function getNextBuilderEager() { flowOutfile: join(flowRouteDir, 'route.js'), bundleFinalOutput: false, externalizeNonSteps: true, + sourceStepRegistrationImports: true, tsconfigPath, }); }