Hash cache keys to limit their length (#553)

vicb · web-flow · commit 930ab833a5d6 · 2025-04-09T10:59:48.000+02:00
diff --git a/packages/cloudflare/src/api/config.ts b/packages/cloudflare/src/api/config.ts
@@ -45,6 +45,8 @@ export function defineCloudflareConfig(config: CloudflareOverrides = {}): OpenNe
         queue: resolveQueue(queue),
       },
     },
+    // node:crypto is used to compute cache keys
+    edgeExternals: ["node:crypto"],
   };
 }
 
diff --git a/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts b/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts
@@ -1,3 +1,5 @@
+import { createHash } from "node:crypto";
+
 import { error } from "@opennextjs/aws/adapters/logger.js";
 import type { CacheValue, IncrementalCache, WithLastModified } from "@opennextjs/aws/types/overrides.js";
 import { IgnorableError } from "@opennextjs/aws/utils/error.js";
@@ -9,6 +11,17 @@ export const NAME = "cf-kv-incremental-cache";
 
 export const BINDING_NAME = "NEXT_INC_CACHE_KV";
 
+export type KeyOptions = {
+  isFetch?: boolean;
+  buildId?: string;
+};
+
+export function computeCacheKey(key: string, options: KeyOptions) {
+  const { isFetch = false, buildId = FALLBACK_BUILD_ID } = options;
+  const hash = createHash("sha256").update(key).digest("hex");
+  return `${buildId}/${hash}.${isFetch ? "fetch" : "cache"}`.replace(/\/+/g, "/");
+}
+
 /**
  * Open Next cache based on Cloudflare KV.
  *
@@ -93,8 +106,10 @@ class KVIncrementalCache implements IncrementalCache {
   }
 
   protected getKVKey(key: string, isFetch?: boolean): string {
-    const buildId = process.env.NEXT_BUILD_ID ?? FALLBACK_BUILD_ID;
-    return `${buildId}/${key}.${isFetch ? "fetch" : "cache"}`.replace(/\/+/g, "/");
+    return computeCacheKey(key, {
+      buildId: process.env.NEXT_BUILD_ID,
+      isFetch,
+    });
   }
 }
 
diff --git a/packages/cloudflare/src/api/overrides/incremental-cache/r2-incremental-cache.ts b/packages/cloudflare/src/api/overrides/incremental-cache/r2-incremental-cache.ts
@@ -1,3 +1,5 @@
+import { createHash } from "node:crypto";
+
 import { error } from "@opennextjs/aws/adapters/logger.js";
 import type { CacheValue, IncrementalCache, WithLastModified } from "@opennextjs/aws/types/overrides.js";
 import { IgnorableError } from "@opennextjs/aws/utils/error.js";
@@ -12,6 +14,18 @@ export const BINDING_NAME = "NEXT_INC_CACHE_R2_BUCKET";
 export const PREFIX_ENV_NAME = "NEXT_INC_CACHE_R2_PREFIX";
 export const DEFAULT_PREFIX = "incremental-cache";
 
+export type KeyOptions = {
+  isFetch?: boolean;
+  directory?: string;
+  buildId?: string;
+};
+
+export function computeCacheKey(key: string, options: KeyOptions) {
+  const { isFetch = false, directory = DEFAULT_PREFIX, buildId = FALLBACK_BUILD_ID } = options;
+  const hash = createHash("sha256").update(key).digest("hex");
+  return `${directory}/${buildId}/${hash}.${isFetch ? "fetch" : "cache"}`.replace(/\/+/g, "/");
+}
+
 /**
  * An instance of the Incremental Cache that uses an R2 bucket (`NEXT_INC_CACHE_R2_BUCKET`) as it's
  * underlying data store.
@@ -76,12 +90,11 @@ class R2IncrementalCache implements IncrementalCache {
   }
 
   protected getR2Key(key: string, isFetch?: boolean): string {
-    const directory = getCloudflareContext().env[PREFIX_ENV_NAME] ?? DEFAULT_PREFIX;
-
-    return `${directory}/${process.env.NEXT_BUILD_ID ?? FALLBACK_BUILD_ID}/${key}.${isFetch ? "fetch" : "cache"}`.replace(
-      /\/+/g,
-      "/"
-    );
+    return computeCacheKey(key, {
+      directory: getCloudflareContext().env[PREFIX_ENV_NAME],
+      buildId: process.env.NEXT_BUILD_ID,
+      isFetch,
+    });
   }
 }
 
diff --git a/packages/cloudflare/src/cli/build/utils/ensure-cf-config.ts b/packages/cloudflare/src/cli/build/utils/ensure-cf-config.ts
@@ -22,6 +22,7 @@ export function ensureCloudflareConfig(config: OpenNextConfig) {
       config.default?.override?.queue === "direct" ||
       typeof config.default?.override?.queue === "function",
     mwIsMiddlewareIntegrated: config.middleware === undefined,
+    hasCryptoExternal: config.edgeExternals?.includes("node:crypto"),
   };
 
   if (config.default?.override?.queue === "direct") {
@@ -42,6 +43,7 @@ export function ensureCloudflareConfig(config: OpenNextConfig) {
               queue: "dummy" | "direct" | function,
             },
           },
+          edgeExternals: ["node:crypto"],
         }\n\n`.replace(/^ {8}/gm, "")
     );
   }
diff --git a/packages/cloudflare/src/cli/commands/populate-cache.spec.ts b/packages/cloudflare/src/cli/commands/populate-cache.spec.ts
@@ -0,0 +1,70 @@
+import { mkdirSync, writeFileSync } from "node:fs";
+import path from "node:path";
+
+import type { BuildOptions } from "@opennextjs/aws/build/helper";
+import mockFs from "mock-fs";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+
+import { getCacheAssets } from "./populate-cache";
+
+describe("getCacheAssets", () => {
+  beforeAll(() => {
+    mockFs();
+
+    const fetchBaseDir = "/base/path/cache/__fetch/buildID";
+    const cacheDir = "/base/path/cache/buildID/path/to";
+
+    mkdirSync(fetchBaseDir, { recursive: true });
+    mkdirSync(cacheDir, { recursive: true });
+
+    for (let i = 0; i < 3; i++) {
+      writeFileSync(path.join(fetchBaseDir, `${i}`), "", { encoding: "utf-8" });
+      writeFileSync(path.join(cacheDir, `${i}.cache`), "", { encoding: "utf-8" });
+    }
+  });
+
+  afterAll(() => mockFs.restore());
+
+  test("list cache assets", () => {
+    expect(getCacheAssets({ outputDir: "/base/path" } as BuildOptions)).toMatchInlineSnapshot(`
+      [
+        {
+          "buildId": "buildID",
+          "fullPath": "/base/path/cache/buildID/path/to/2.cache",
+          "isFetch": false,
+          "key": "/path/to/2",
+        },
+        {
+          "buildId": "buildID",
+          "fullPath": "/base/path/cache/buildID/path/to/1.cache",
+          "isFetch": false,
+          "key": "/path/to/1",
+        },
+        {
+          "buildId": "buildID",
+          "fullPath": "/base/path/cache/buildID/path/to/0.cache",
+          "isFetch": false,
+          "key": "/path/to/0",
+        },
+        {
+          "buildId": "buildID",
+          "fullPath": "/base/path/cache/__fetch/buildID/2",
+          "isFetch": true,
+          "key": "/2",
+        },
+        {
+          "buildId": "buildID",
+          "fullPath": "/base/path/cache/__fetch/buildID/1",
+          "isFetch": true,
+          "key": "/1",
+        },
+        {
+          "buildId": "buildID",
+          "fullPath": "/base/path/cache/__fetch/buildID/0",
+          "isFetch": true,
+          "key": "/0",
+        },
+      ]
+    `);
+  });
+});
diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts
@@ -16,11 +16,12 @@ import { unstable_readConfig } from "wrangler";
 
 import {
   BINDING_NAME as KV_CACHE_BINDING_NAME,
+  computeCacheKey as computeKVCacheKey,
   NAME as KV_CACHE_NAME,
 } from "../../api/overrides/incremental-cache/kv-incremental-cache.js";
 import {
   BINDING_NAME as R2_CACHE_BINDING_NAME,
-  DEFAULT_PREFIX as R2_CACHE_DEFAULT_PREFIX,
+  computeCacheKey as computeR2CacheKey,
   NAME as R2_CACHE_NAME,
   PREFIX_ENV_NAME as R2_CACHE_PREFIX_ENV_NAME,
 } from "../../api/overrides/incremental-cache/r2-incremental-cache.js";
@@ -45,22 +46,50 @@ async function resolveCacheName(
   return typeof value === "function" ? (await value()).name : value;
 }
 
-function getCacheAssetPaths(opts: BuildOptions) {
-  return globSync(path.join(opts.outputDir, "cache/**/*"), {
+export type CacheAsset = { isFetch: boolean; fullPath: string; key: string; buildId: string };
+
+export function getCacheAssets(opts: BuildOptions): CacheAsset[] {
+  const allFiles = globSync(path.join(opts.outputDir, "cache/**/*"), {
     withFileTypes: true,
     windowsPathsNoEscape: true,
-  })
-    .filter((f) => f.isFile())
-    .map((f) => {
-      const relativePath = path.relative(path.join(opts.outputDir, "cache"), f.fullpathPosix());
-
-      return {
-        fsPath: f.fullpathPosix(),
-        destPath: relativePath.startsWith("__fetch")
-          ? `${relativePath.replace("__fetch/", "")}.fetch`
-          : relativePath,
-      };
-    });
+  }).filter((f) => f.isFile());
+
+  const assets: CacheAsset[] = [];
+
+  for (const file of allFiles) {
+    const fullPath = file.fullpathPosix();
+    const relativePath = path.relative(path.join(opts.outputDir, "cache"), fullPath);
+
+    if (relativePath.startsWith("__fetch")) {
+      const [__fetch, buildId, ...keyParts] = relativePath.split("/");
+
+      if (__fetch !== "__fetch" || buildId === undefined || keyParts.length === 0) {
+        throw new Error(`Invalid path for a Cache Asset file: ${relativePath}`);
+      }
+
+      assets.push({
+        isFetch: true,
+        fullPath,
+        key: `/${keyParts.join("/")}`,
+        buildId,
+      });
+    } else {
+      const [buildId, ...keyParts] = relativePath.slice(0, -".cache".length).split("/");
+
+      if (!relativePath.endsWith(".cache") || buildId === undefined || keyParts.length === 0) {
+        throw new Error(`Invalid path for a Cache Asset file: ${relativePath}`);
+      }
+
+      assets.push({
+        isFetch: false,
+        fullPath,
+        key: `/${keyParts.join("/")}`,
+        buildId,
+      });
+    }
+  }
+
+  return assets;
 }
 
 function populateR2IncrementalCache(
@@ -81,17 +110,18 @@ function populateR2IncrementalCache(
     throw new Error(`R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} should have a 'bucket_name'`);
   }
 
-  const assets = getCacheAssetPaths(options);
-  for (const { fsPath, destPath } of tqdm(assets)) {
-    const fullDestPath = path.join(
-      bucket,
-      process.env[R2_CACHE_PREFIX_ENV_NAME] ?? R2_CACHE_DEFAULT_PREFIX,
-      destPath
-    );
+  const assets = getCacheAssets(options);
+
+  for (const { fullPath, key, buildId, isFetch } of tqdm(assets)) {
+    const cacheKey = computeR2CacheKey(key, {
+      directory: process.env[R2_CACHE_PREFIX_ENV_NAME],
+      buildId,
+      isFetch,
+    });
 
     runWrangler(
       options,
-      ["r2 object put", JSON.stringify(fullDestPath), `--file ${JSON.stringify(fsPath)}`],
+      ["r2 object put", JSON.stringify(path.join(bucket, cacheKey)), `--file ${JSON.stringify(fullPath)}`],
       // NOTE: R2 does not support the environment flag and results in the following error:
       // Incorrect type for the 'cacheExpiry' field on 'HttpMetadata': the provided value is not of type 'date'.
       { target: populateCacheOptions.target, excludeRemoteFlag: true, logging: "error" }
@@ -113,15 +143,21 @@ function populateKVIncrementalCache(
     throw new Error(`No KV binding ${JSON.stringify(KV_CACHE_BINDING_NAME)} found!`);
   }
 
-  const assets = getCacheAssetPaths(options);
-  for (const { fsPath, destPath } of tqdm(assets)) {
+  const assets = getCacheAssets(options);
+
+  for (const { fullPath, key, buildId, isFetch } of tqdm(assets)) {
+    const cacheKey = computeKVCacheKey(key, {
+      buildId,
+      isFetch,
+    });
+
     runWrangler(
       options,
       [
         "kv key put",
-        JSON.stringify(destPath),
+        JSON.stringify(cacheKey),
         `--binding ${JSON.stringify(KV_CACHE_BINDING_NAME)}`,
-        `--path ${JSON.stringify(fsPath)}`,
+        `--path ${JSON.stringify(fullPath)}`,
       ],
       { ...populateCacheOptions, logging: "error" }
     );

Original file line number	Diff line number	Diff line change
`@@ -45,6 +45,8 @@ export function defineCloudflareConfig(config: CloudflareOverrides = {}): OpenNe`
`45`	`45`	`queue: resolveQueue(queue),`
`46`	`46`	`},`
`47`	`47`	`},`
	`48`	`+ // node:crypto is used to compute cache keys`
	`49`	`+ edgeExternals: ["node:crypto"],`
`48`	`50`	`};`
`49`	`51`	`}`
`50`	`52`