Cache package dependency collection results to avoid redundant filesystem walks

.changeset/cache-package-dependencies.md

@@ -0,0 +1,7 @@
+---
+"@cloudflare/deploy-helpers": patch
+---
+
+Cache package dependency collection results to avoid redundant filesystem walks
+
+`collectPackageDependencies` now accepts an optional options object as its second parameter instead of a positional `excludePackages` array. The options object supports `excludePackages` (moved from the previous positional parameter) and a new `cacheDir` field. When `cacheDir` is provided, collected results are persisted to disk and reused on subsequent calls as long as neither `package.json` nor the project's lockfile have been modified. This speeds up repeated deploys when dependencies haven't changed.

packages/deploy-helpers/package.json

@@ -62,6 +62,7 @@
 		"@types/node": "catalog:default",
 		"concurrently": "^8.2.2",
 		"devtools-protocol": "^0.0.1182435",
+		"empathic": "^2.0.0",
 		"esbuild": "catalog:default",
 		"json-diff": "^1.0.6",
 		"ts-dedent": "^2.2.0",

packages/deploy-helpers/src/deploy/deploy.ts

@@ -328,10 +328,11 @@ export default async function deploy(
 		cache: config.cache,
 		package_dependencies:
 			config.dependencies_instrumentation?.enabled !== false && projectRoot
-				? await collectPackageDependencies(
-						projectRoot,
-						config.dependencies_instrumentation?.exclude_packages
-					)
+				? await collectPackageDependencies(projectRoot, {
+						excludePackages:
+							config.dependencies_instrumentation?.exclude_packages,
+						cacheDir: props.cacheDir,
+					})
 				: undefined,
 	};
 

packages/deploy-helpers/src/deploy/helpers/package-dependencies.ts

@@ -1,9 +1,11 @@
-import { access, readFile } from "node:fs/promises";
+import { createHash } from "node:crypto";
+import { access, mkdir, readFile, writeFile } from "node:fs/promises";
 import path from "node:path";
 import {
 	getInstalledPackageVersion,
 	parsePackageJSON,
 } from "@cloudflare/workers-utils";
+import * as find from "empathic/find";
 import { logger } from "../../shared/context";
 
 /**
@@ -26,6 +28,156 @@ export type PackageDependency = {
  */
 const MAX_PACKAGE_DEPENDENCIES = 200;
 
+/** Filename used for the cached dependency results. */
+const CACHE_FILENAME = "package-dependencies.json";
+
+/**
+ * Lockfile names to look for when computing the cache key.
+ * Checked in order; the first match wins.
+ */
+const KNOWN_LOCKFILES = [
+	"pnpm-lock.yaml",
+	"package-lock.json",
+	"yarn.lock",
+	"bun.lock",
+	"bun.lockb",
+] as const;
+
+/**
+ * Shape of the on-disk cache file written to the caller-supplied cache directory.
+ */
+type PackageDependenciesCache = {
+	/** SHA-256 hex digest of the project's package.json content. */
+	packageJsonHash: string;
+	/** SHA-256 hex digest of the lockfile content, or `null` if none was found. */
+	lockfileHash: string | null;
+	/** Absolute path of the lockfile that was hashed, or `null` if none. */
+	lockfilePath: string | null;
+	/** The collected dependency entries. */
+	dependencies: PackageDependency[];
+};
+
+/**
+ * Computes the SHA-256 hex digest of a buffer or string.
+ *
+ * @param content - The content to hash
+ * @returns The hex-encoded SHA-256 digest
+ */
+function sha256(content: Buffer | string): string {
+	return createHash("sha256").update(content).digest("hex");
+}
+
+/**
+ * Searches for the first known lockfile starting at `projectPath` and walking
+ * up parent directories toward the filesystem root. This ensures lockfiles at
+ * a monorepo/workspace root are detected even when the Worker lives in a
+ * sub-package.
+ *
+ * @param projectPath - The project directory to start searching from
+ * @returns An object with the lockfile's absolute `filePath` and its content `hash`,
+ *          or `null` if no lockfile is found
+ */
+async function getLockfileInfo(
+	projectPath: string
+): Promise<{ filePath: string; hash: string } | null> {
+	for (const name of KNOWN_LOCKFILES) {
+		const filePath = find.file(name, { cwd: projectPath });
+		if (filePath) {
+			const content = await readFile(filePath);
+			return { filePath, hash: sha256(content) };
+		}
+	}
+	return null;
+}
+
+/**
+ * Validates that a parsed JSON value conforms to the {@link PackageDependenciesCache} shape.
+ * Returns `false` if any field is missing or has the wrong type, causing the
+ * entire cache to be discarded.
+ *
+ * @param value - The parsed JSON value to validate
+ * @returns `true` if the value matches the expected cache structure, `false` otherwise
+ */
+function isValidDependenciesCache(
+	value: unknown
+): value is PackageDependenciesCache {
+	if (typeof value !== "object" || value === null || Array.isArray(value)) {
+		return false;
+	}
+
+	const obj = value as Record<string, unknown>;
+
+	if (typeof obj.packageJsonHash !== "string") {
+		return false;
+	}
+	if (obj.lockfileHash !== null && typeof obj.lockfileHash !== "string") {
+		return false;
+	}
+	if (obj.lockfilePath !== null && typeof obj.lockfilePath !== "string") {
+		return false;
+	}
+	if (!Array.isArray(obj.dependencies)) {
+		return false;
+	}
+
+	for (const dep of obj.dependencies) {
+		if (typeof dep !== "object" || dep === null || Array.isArray(dep)) {
+			return false;
+		}
+		const d = dep as Record<string, unknown>;
+		if (
+			typeof d.name !== "string" ||
+			typeof d.packageJsonVersion !== "string" ||
+			typeof d.installedVersion !== "string"
+		) {
+			return false;
+		}
+	}
+
+	return true;
+}
+
+/**
+ * Attempts to read a previously-written cache file.
+ *
+ * @param cachePath - Full path to the cache JSON file
+ * @returns The parsed cache contents, or `null` if the file is missing or malformed
+ */
+async function readDependenciesCache(
+	cachePath: string
+): Promise<PackageDependenciesCache | null> {
+	try {
+		const raw = await readFile(cachePath, "utf-8");
+		const parsed: unknown = JSON.parse(raw);
+
+		if (!isValidDependenciesCache(parsed)) {
+			return null;
+		}
+
+		return parsed;
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * Writes the dependency cache to disk (best-effort; failures are silently ignored).
+ *
+ * @param cachePath - Full path to the cache JSON file
+ * @param data - The cache payload to persist
+ */
+async function writeDependenciesCache(
+	cachePath: string,
+	data: PackageDependenciesCache
+): Promise<void> {
+	try {
+		await mkdir(path.dirname(cachePath), { recursive: true });
+		await writeFile(cachePath, JSON.stringify(data, null, 2));
+	} catch {
+		// best-effort — a failure to write cache should never block a deploy
+	}
+}
+
 /**
  * Converts a glob pattern (supporting `*` wildcards) to a regular expression.
  *
@@ -67,15 +219,24 @@ function isExcludedPackage(name: string, excludePatterns: string[]): boolean {
  *
  * The result is capped at {@link MAX_PACKAGE_DEPENDENCIES} entries.
  *
+ * When `opts.cacheDir` is provided the results are cached to disk and reused
+ * on subsequent calls as long as neither `package.json` nor the project's
+ * lockfile have been modified (compared by content hash).
+ *
  * @param projectPath - Path to the project directory (where package.json is located)
- * @param excludePackages - Optional list of package name patterns (glob-style with
+ * @param opts - Optional settings
+ * @param opts.excludePackages - Optional list of package name patterns (glob-style with
  *        `*` wildcards) to exclude from the collected dependencies
+ * @param opts.cacheDir - Directory to store the cache file. If omitted, caching is disabled.
  * @returns An array of package dependency entries, or `undefined` if package.json
  *          cannot be read or no valid dependencies are found
  */
 export async function collectPackageDependencies(
 	projectPath: string,
-	excludePackages?: string[]
+	opts?: {
+		excludePackages?: string[];
+		cacheDir?: string;
+	}
 ): Promise<PackageDependency[] | undefined> {
 	const packageJsonPath = path.join(projectPath, "package.json");
 
@@ -86,8 +247,35 @@ export async function collectPackageDependencies(
 	}
 
 	try {
-		const content = await readFile(packageJsonPath, "utf-8");
-		const packageJson = parsePackageJSON(content, packageJsonPath);
+		const packageJsonContent = await readFile(packageJsonPath, "utf-8");
+
+		const cachePath = opts?.cacheDir
+			? path.join(opts.cacheDir, CACHE_FILENAME)
+			: null;
+
+		// Only compute hashes and resolve lockfile info when caching is enabled,
+		// to avoid unnecessary filesystem I/O (directory walks and large file
+		// reads) when cacheDir is not provided.
+		const packageJsonHash = cachePath ? sha256(packageJsonContent) : undefined;
+		const lockfileInfo = cachePath
+			? await getLockfileInfo(projectPath)
+			: undefined;
+
+		if (cachePath && packageJsonHash !== undefined) {
+			// Attempt to return cached results when available
+			const cached = await readDependenciesCache(cachePath);
+			if (
+				cached &&
+				cached.packageJsonHash === packageJsonHash &&
+				cached.lockfileHash === (lockfileInfo?.hash ?? null) &&
+				cached.lockfilePath === (lockfileInfo?.filePath ?? null)
+			) {
+				logger?.debug("Using cached package dependency results");
+				return cached.dependencies.length > 0 ? cached.dependencies : undefined;
+			}
+		}
+
+		const packageJson = parsePackageJSON(packageJsonContent, packageJsonPath);
 
 		const allDependencies = {
 			...packageJson.dependencies,
@@ -113,8 +301,8 @@ export async function collectPackageDependencies(
 			}
 
 			if (
-				excludePackages?.length &&
-				isExcludedPackage(dependencyName, excludePackages)
+				opts?.excludePackages?.length &&
+				isExcludedPackage(dependencyName, opts.excludePackages)
 			) {
 				continue;
 			}
@@ -135,9 +323,19 @@ export async function collectPackageDependencies(
 			});
 		}
 
+		// Persist the cache for future calls
+		if (cachePath && packageJsonHash !== undefined) {
+			await writeDependenciesCache(cachePath, {
+				packageJsonHash,
+				lockfileHash: lockfileInfo?.hash ?? null,
+				lockfilePath: lockfileInfo?.filePath ?? null,
+				dependencies: result,
+			});
+		}
+
 		return result.length > 0 ? result : undefined;
 	} catch (error) {
-		logger.debug(
+		logger?.debug(
 			`Failed to collect package dependencies: ${error instanceof Error ? error.message : error}`
 		);
 		return undefined;

packages/deploy-helpers/src/deploy/versions-upload.ts

@@ -192,10 +192,11 @@ export default async function versionsUpload(
 		cache: config.cache, // cache is a versioned setting
 		package_dependencies:
 			config.dependencies_instrumentation?.enabled !== false && projectRoot
-				? await collectPackageDependencies(
-						projectRoot,
-						config.dependencies_instrumentation?.exclude_packages
-					)
+				? await collectPackageDependencies(projectRoot, {
+						excludePackages:
+							config.dependencies_instrumentation?.exclude_packages,
+						cacheDir: props.cacheDir,
+					})
 				: undefined,
 	};
 

packages/deploy-helpers/src/shared/types.ts

@@ -98,6 +98,8 @@ export type SharedDeployVersionsProps = {
 	accountId: string | undefined;
 	/** Resolved from getMetricsUsageHeaders() / config.send_metrics. Controls whether usage metrics headers are sent with upload requests. */
 	sendMetrics: boolean;
+	/** Optional directory for caching intermediate results (e.g. package dependency collection). Callers set this to a project-local cache directory. */
+	cacheDir?: string;
 	/** Resolved from getFlag("RESOURCES_PROVISION"). Controls whether bindings are auto-provisioned before upload. */
 	resourcesProvision: boolean;
 	/** Controls whether provisioned resource IDs are written back to the config file. */