From 81aae0e57e6caffc369381f9d28de2f6535464e3 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 3 Mar 2025 22:17:29 +0000 Subject: [PATCH 01/26] feat: auto-populating d1 cache data --- .changeset/tame-icons-shave.md | 5 ++ examples/e2e/app-router/package.json | 3 +- packages/cloudflare/src/cli/args.ts | 63 ++++++++++++------- packages/cloudflare/src/cli/build/build.ts | 10 +++ .../cloudflare/src/cli/build/utils/index.ts | 1 + .../src/cli/build/utils/populate-cache.ts | 61 ++++++++++++++++++ packages/cloudflare/src/cli/index.ts | 3 +- .../cloudflare/src/cli/project-options.ts | 3 + 8 files changed, 125 insertions(+), 24 deletions(-) create mode 100644 .changeset/tame-icons-shave.md create mode 100644 packages/cloudflare/src/cli/build/utils/populate-cache.ts diff --git a/.changeset/tame-icons-shave.md b/.changeset/tame-icons-shave.md new file mode 100644 index 000000000..8dfbe7871 --- /dev/null +++ b/.changeset/tame-icons-shave.md @@ -0,0 +1,5 @@ +--- +"@opennextjs/cloudflare": patch +--- + +feat: auto-populating d1 cache data diff --git a/examples/e2e/app-router/package.json b/examples/e2e/app-router/package.json index 3d1037d12..9d26a4c6e 100644 --- a/examples/e2e/app-router/package.json +++ b/examples/e2e/app-router/package.json @@ -10,8 +10,7 @@ "lint": "next lint", "clean": "rm -rf .turbo node_modules .next .open-next", "d1:clean": "wrangler d1 execute NEXT_CACHE_D1 --command \"DROP TABLE IF EXISTS tags; DROP TABLE IF EXISTS revalidations\"", - "d1:setup": "wrangler d1 execute NEXT_CACHE_D1 --file .open-next/cloudflare/cache-assets-manifest.sql", - "build:worker": "pnpm opennextjs-cloudflare && pnpm d1:clean && pnpm d1:setup", + "build:worker": "pnpm d1:clean && pnpm opennextjs-cloudflare --populateCache=local", "preview": "pnpm build:worker && pnpm wrangler dev", "e2e": "playwright test -c e2e/playwright.config.ts" }, diff --git a/packages/cloudflare/src/cli/args.ts b/packages/cloudflare/src/cli/args.ts index f64d0d731..b29ce67c2 100644 --- a/packages/cloudflare/src/cli/args.ts +++ b/packages/cloudflare/src/cli/args.ts @@ -2,34 +2,45 @@ import { mkdirSync, type Stats, statSync } from "node:fs"; import { resolve } from "node:path"; import { parseArgs } from "node:util"; +import { CacheBindingMode } from "./build/utils/index.js"; + export function getArgs(): { skipNextBuild: boolean; skipWranglerConfigCheck: boolean; outputDir?: string; minify: boolean; + populateCache?: { mode: "local" | "remote"; onlyPopulate: boolean }; } { - const { skipBuild, skipWranglerConfigCheck, output, noMinify } = parseArgs({ - options: { - skipBuild: { - type: "boolean", - short: "s", - default: false, - }, - output: { - type: "string", - short: "o", - }, - noMinify: { - type: "boolean", - default: false, + const { skipBuild, skipWranglerConfigCheck, output, noMinify, populateCache, onlyPopulateCache } = + parseArgs({ + options: { + skipBuild: { + type: "boolean", + short: "s", + default: false, + }, + output: { + type: "string", + short: "o", + }, + noMinify: { + type: "boolean", + default: false, + }, + skipWranglerConfigCheck: { + type: "boolean", + default: false, + }, + populateCache: { + type: "string", + }, + onlyPopulateCache: { + type: "boolean", + default: false, + }, }, - skipWranglerConfigCheck: { - type: "boolean", - default: false, - }, - }, - allowPositionals: false, - }).values; + allowPositionals: false, + }).values; const outputDir = output ? resolve(output) : undefined; @@ -37,6 +48,13 @@ export function getArgs(): { assertDirArg(outputDir, "output", true); } + if ( + (populateCache !== undefined || onlyPopulateCache) && + (!populateCache?.length || !["local", "remote"].includes(populateCache)) + ) { + throw new Error(`Error: missing mode for populate cache flag, expected 'local' | 'remote'`); + } + return { outputDir, skipNextBuild: skipBuild || ["1", "true", "yes"].includes(String(process.env.SKIP_NEXT_APP_BUILD)), @@ -44,6 +62,9 @@ export function getArgs(): { skipWranglerConfigCheck || ["1", "true", "yes"].includes(String(process.env.SKIP_WRANGLER_CONFIG_CHECK)), minify: !noMinify, + populateCache: populateCache + ? { mode: populateCache as CacheBindingMode, onlyPopulate: !!onlyPopulateCache } + : undefined, }; } diff --git a/packages/cloudflare/src/cli/build/build.ts b/packages/cloudflare/src/cli/build/build.ts index 110f563b8..4148b92be 100644 --- a/packages/cloudflare/src/cli/build/build.ts +++ b/packages/cloudflare/src/cli/build/build.ts @@ -20,6 +20,7 @@ import { createOpenNextConfigIfNotExistent, createWranglerConfigIfNotExistent, ensureCloudflareConfig, + populateCache, } from "./utils/index.js"; import { getVersion } from "./utils/version.js"; @@ -62,6 +63,11 @@ export async function build(projectOpts: ProjectOptions): Promise { logger.info(`@opennextjs/cloudflare version: ${cloudflare}`); logger.info(`@opennextjs/aws version: ${aws}`); + if (projectOpts.populateCache?.onlyPopulate) { + populateCache(options, config, projectOpts.populateCache.mode); + return; + } + if (projectOpts.skipNextBuild) { logger.warn("Skipping Next.js build"); } else { @@ -103,6 +109,10 @@ export async function build(projectOpts: ProjectOptions): Promise { await createWranglerConfigIfNotExistent(projectOpts); } + if (projectOpts.populateCache) { + populateCache(options, config, projectOpts.populateCache.mode); + } + logger.info("OpenNext build complete."); } diff --git a/packages/cloudflare/src/cli/build/utils/index.ts b/packages/cloudflare/src/cli/build/utils/index.ts index cca97f023..e7fb383b6 100644 --- a/packages/cloudflare/src/cli/build/utils/index.ts +++ b/packages/cloudflare/src/cli/build/utils/index.ts @@ -4,3 +4,4 @@ export * from "./ensure-cf-config.js"; export * from "./extract-project-env-vars.js"; export * from "./needs-experimental-react.js"; export * from "./normalize-path.js"; +export * from "./populate-cache.js"; diff --git a/packages/cloudflare/src/cli/build/utils/populate-cache.ts b/packages/cloudflare/src/cli/build/utils/populate-cache.ts new file mode 100644 index 000000000..c9aa07335 --- /dev/null +++ b/packages/cloudflare/src/cli/build/utils/populate-cache.ts @@ -0,0 +1,61 @@ +import { spawnSync } from "node:child_process"; +import path from "node:path"; + +import type { BuildOptions } from "@opennextjs/aws/build/helper.js"; +import logger from "@opennextjs/aws/logger.js"; +import type { + IncludedIncrementalCache, + IncludedTagCache, + LazyLoadedOverride, + OpenNextConfig, +} from "@opennextjs/aws/types/open-next.js"; +import type { IncrementalCache, TagCache } from "@opennextjs/aws/types/overrides.js"; + +export type CacheBindingMode = "local" | "remote"; + +async function resolveCacheName( + value: + | IncludedIncrementalCache + | IncludedTagCache + | LazyLoadedOverride + | LazyLoadedOverride +) { + return typeof value === "function" ? (await value()).name : value; +} + +function runWrangler(opts: BuildOptions, mode: CacheBindingMode, args: string[]) { + const result = spawnSync( + opts.packager, + ["exec", "wrangler", ...args, mode === "remote" && "--remote"].filter((v): v is string => !!v), + { + shell: true, + stdio: ["ignore", "ignore", "inherit"], + } + ); + + if (result.status !== 0) { + logger.error("Failed to populate cache"); + } else { + logger.info("Successfully populated cache"); + } +} + +export async function populateCache(opts: BuildOptions, config: OpenNextConfig, mode: CacheBindingMode) { + const { tagCache } = config.default.override ?? {}; + + if (tagCache) { + const name = await resolveCacheName(tagCache); + switch (name) { + case "d1-tag-cache": { + logger.info("\nPopulating D1 tag cache..."); + + runWrangler(opts, mode, [ + "d1 execute", + "NEXT_CACHE_D1", + `--file ${JSON.stringify(path.join(opts.outputDir, "cloudflare/cache-assets-manifest.sql"))}`, + ]); + break; + } + } + } +} diff --git a/packages/cloudflare/src/cli/index.ts b/packages/cloudflare/src/cli/index.ts index 2a6c654e4..be3c1e763 100644 --- a/packages/cloudflare/src/cli/index.ts +++ b/packages/cloudflare/src/cli/index.ts @@ -6,7 +6,7 @@ import { build } from "./build/build.js"; const nextAppDir = process.cwd(); -const { skipNextBuild, skipWranglerConfigCheck, outputDir, minify } = getArgs(); +const { skipNextBuild, skipWranglerConfigCheck, outputDir, minify, populateCache } = getArgs(); await build({ sourceDir: nextAppDir, @@ -14,4 +14,5 @@ await build({ skipNextBuild, skipWranglerConfigCheck, minify, + populateCache, }); diff --git a/packages/cloudflare/src/cli/project-options.ts b/packages/cloudflare/src/cli/project-options.ts index 0ceb3d96c..f83beef4d 100644 --- a/packages/cloudflare/src/cli/project-options.ts +++ b/packages/cloudflare/src/cli/project-options.ts @@ -1,3 +1,5 @@ +import type { CacheBindingMode } from "./build/utils/index.js"; + export type ProjectOptions = { // Next app root folder sourceDir: string; @@ -9,4 +11,5 @@ export type ProjectOptions = { skipWranglerConfigCheck: boolean; // Whether minification of the worker should be enabled minify: boolean; + populateCache?: { mode: CacheBindingMode; onlyPopulate: boolean }; }; From c19f43fea25cf1128f2575ab942ae6e439259672 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 4 Mar 2025 08:57:00 +0000 Subject: [PATCH 02/26] checks for output directory / enabled --- .../src/cli/build/utils/populate-cache.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/packages/cloudflare/src/cli/build/utils/populate-cache.ts b/packages/cloudflare/src/cli/build/utils/populate-cache.ts index c9aa07335..27f5c4ae9 100644 --- a/packages/cloudflare/src/cli/build/utils/populate-cache.ts +++ b/packages/cloudflare/src/cli/build/utils/populate-cache.ts @@ -1,4 +1,5 @@ import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; import path from "node:path"; import type { BuildOptions } from "@opennextjs/aws/build/helper.js"; @@ -35,15 +36,25 @@ function runWrangler(opts: BuildOptions, mode: CacheBindingMode, args: string[]) if (result.status !== 0) { logger.error("Failed to populate cache"); + process.exit(1); } else { logger.info("Successfully populated cache"); } } export async function populateCache(opts: BuildOptions, config: OpenNextConfig, mode: CacheBindingMode) { - const { tagCache } = config.default.override ?? {}; + const { incrementalCache, tagCache } = config.default.override ?? {}; - if (tagCache) { + if (!existsSync(opts.outputDir)) { + logger.error("Unable to populate cache: Open Next build not found"); + process.exit(1); + } + + if (!config.dangerous?.disableIncrementalCache && incrementalCache) { + logger.info("Incremental cache does not need populating"); + } + + if (!config.dangerous?.disableTagCache && tagCache) { const name = await resolveCacheName(tagCache); switch (name) { case "d1-tag-cache": { @@ -56,6 +67,8 @@ export async function populateCache(opts: BuildOptions, config: OpenNextConfig, ]); break; } + default: + logger.info("Tag cache does not need populating"); } } } From 697e2f8e0434c0f6c250d9b201c83bd27616988d Mon Sep 17 00:00:00 2001 From: James Anderson Date: Tue, 4 Mar 2025 09:50:26 +0000 Subject: [PATCH 03/26] Update packages/cloudflare/src/cli/build/utils/populate-cache.ts Co-authored-by: conico974 --- packages/cloudflare/src/cli/build/utils/populate-cache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cloudflare/src/cli/build/utils/populate-cache.ts b/packages/cloudflare/src/cli/build/utils/populate-cache.ts index 27f5c4ae9..f5533289d 100644 --- a/packages/cloudflare/src/cli/build/utils/populate-cache.ts +++ b/packages/cloudflare/src/cli/build/utils/populate-cache.ts @@ -54,7 +54,7 @@ export async function populateCache(opts: BuildOptions, config: OpenNextConfig, logger.info("Incremental cache does not need populating"); } - if (!config.dangerous?.disableTagCache && tagCache) { + if (!config.dangerous?.disableTagCache && !config.dangerous?.disableIncrementalCache && tagCache) { const name = await resolveCacheName(tagCache); switch (name) { case "d1-tag-cache": { From 87b39b0e3fbfe53c64094a2bdf312f0f939b2d7e Mon Sep 17 00:00:00 2001 From: James Date: Sat, 8 Mar 2025 20:46:04 +0000 Subject: [PATCH 04/26] add suggestions --- packages/cloudflare/src/cli/args.ts | 11 +++++------ .../cloudflare/src/cli/build/utils/populate-cache.ts | 4 ++++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/cloudflare/src/cli/args.ts b/packages/cloudflare/src/cli/args.ts index b29ce67c2..f21907694 100644 --- a/packages/cloudflare/src/cli/args.ts +++ b/packages/cloudflare/src/cli/args.ts @@ -2,14 +2,15 @@ import { mkdirSync, type Stats, statSync } from "node:fs"; import { resolve } from "node:path"; import { parseArgs } from "node:util"; -import { CacheBindingMode } from "./build/utils/index.js"; +import type { CacheBindingMode } from "./build/utils/index.js"; +import { isCacheBindingMode } from "./build/utils/index.js"; export function getArgs(): { skipNextBuild: boolean; skipWranglerConfigCheck: boolean; outputDir?: string; minify: boolean; - populateCache?: { mode: "local" | "remote"; onlyPopulate: boolean }; + populateCache?: { mode: CacheBindingMode; onlyPopulate: boolean }; } { const { skipBuild, skipWranglerConfigCheck, output, noMinify, populateCache, onlyPopulateCache } = parseArgs({ @@ -50,7 +51,7 @@ export function getArgs(): { if ( (populateCache !== undefined || onlyPopulateCache) && - (!populateCache?.length || !["local", "remote"].includes(populateCache)) + (!populateCache?.length || !isCacheBindingMode(populateCache)) ) { throw new Error(`Error: missing mode for populate cache flag, expected 'local' | 'remote'`); } @@ -62,9 +63,7 @@ export function getArgs(): { skipWranglerConfigCheck || ["1", "true", "yes"].includes(String(process.env.SKIP_WRANGLER_CONFIG_CHECK)), minify: !noMinify, - populateCache: populateCache - ? { mode: populateCache as CacheBindingMode, onlyPopulate: !!onlyPopulateCache } - : undefined, + populateCache: populateCache ? { mode: populateCache, onlyPopulate: !!onlyPopulateCache } : undefined, }; } diff --git a/packages/cloudflare/src/cli/build/utils/populate-cache.ts b/packages/cloudflare/src/cli/build/utils/populate-cache.ts index f5533289d..ad38ede79 100644 --- a/packages/cloudflare/src/cli/build/utils/populate-cache.ts +++ b/packages/cloudflare/src/cli/build/utils/populate-cache.ts @@ -72,3 +72,7 @@ export async function populateCache(opts: BuildOptions, config: OpenNextConfig, } } } + +export function isCacheBindingMode(v: string | undefined): v is CacheBindingMode { + return !!v && ["local", "remote"].includes(v); +} From 76e2c5d92e8ab43c8ef83e8833258883d528ff46 Mon Sep 17 00:00:00 2001 From: James Date: Sat, 8 Mar 2025 20:49:16 +0000 Subject: [PATCH 05/26] rename onlyPopulate to onlyPopulateWithoutBuilding --- packages/cloudflare/src/cli/args.ts | 6 ++++-- packages/cloudflare/src/cli/build/build.ts | 2 +- packages/cloudflare/src/cli/project-options.ts | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/cloudflare/src/cli/args.ts b/packages/cloudflare/src/cli/args.ts index f21907694..5a546ad99 100644 --- a/packages/cloudflare/src/cli/args.ts +++ b/packages/cloudflare/src/cli/args.ts @@ -10,7 +10,7 @@ export function getArgs(): { skipWranglerConfigCheck: boolean; outputDir?: string; minify: boolean; - populateCache?: { mode: CacheBindingMode; onlyPopulate: boolean }; + populateCache?: { mode: CacheBindingMode; onlyPopulateWithoutBuilding: boolean }; } { const { skipBuild, skipWranglerConfigCheck, output, noMinify, populateCache, onlyPopulateCache } = parseArgs({ @@ -63,7 +63,9 @@ export function getArgs(): { skipWranglerConfigCheck || ["1", "true", "yes"].includes(String(process.env.SKIP_WRANGLER_CONFIG_CHECK)), minify: !noMinify, - populateCache: populateCache ? { mode: populateCache, onlyPopulate: !!onlyPopulateCache } : undefined, + populateCache: populateCache + ? { mode: populateCache, onlyPopulateWithoutBuilding: !!onlyPopulateCache } + : undefined, }; } diff --git a/packages/cloudflare/src/cli/build/build.ts b/packages/cloudflare/src/cli/build/build.ts index 4148b92be..c2b8de995 100644 --- a/packages/cloudflare/src/cli/build/build.ts +++ b/packages/cloudflare/src/cli/build/build.ts @@ -63,7 +63,7 @@ export async function build(projectOpts: ProjectOptions): Promise { logger.info(`@opennextjs/cloudflare version: ${cloudflare}`); logger.info(`@opennextjs/aws version: ${aws}`); - if (projectOpts.populateCache?.onlyPopulate) { + if (projectOpts.populateCache?.onlyPopulateWithoutBuilding) { populateCache(options, config, projectOpts.populateCache.mode); return; } diff --git a/packages/cloudflare/src/cli/project-options.ts b/packages/cloudflare/src/cli/project-options.ts index f83beef4d..15b23259d 100644 --- a/packages/cloudflare/src/cli/project-options.ts +++ b/packages/cloudflare/src/cli/project-options.ts @@ -11,5 +11,5 @@ export type ProjectOptions = { skipWranglerConfigCheck: boolean; // Whether minification of the worker should be enabled minify: boolean; - populateCache?: { mode: CacheBindingMode; onlyPopulate: boolean }; + populateCache?: { mode: CacheBindingMode; onlyPopulateWithoutBuilding: boolean }; }; From 46d8047e8abcc46cc7d6b6a752978216e944705e Mon Sep 17 00:00:00 2001 From: James Date: Sun, 9 Mar 2025 00:14:27 +0000 Subject: [PATCH 06/26] feat: r2 adapter for the incremental cache --- .changeset/weak-houses-divide.md | 5 + examples/e2e/app-router/open-next.config.ts | 5 +- examples/e2e/app-router/wrangler.jsonc | 7 + .../cloudflare/src/api/cloudflare-context.ts | 2 + .../src/api/r2-incremental-cache.ts | 171 ++++++++++++++++++ .../src/cli/build/utils/populate-cache.ts | 61 ++++++- 6 files changed, 243 insertions(+), 8 deletions(-) create mode 100644 .changeset/weak-houses-divide.md create mode 100644 packages/cloudflare/src/api/r2-incremental-cache.ts diff --git a/.changeset/weak-houses-divide.md b/.changeset/weak-houses-divide.md new file mode 100644 index 000000000..ec7f9db41 --- /dev/null +++ b/.changeset/weak-houses-divide.md @@ -0,0 +1,5 @@ +--- +"@opennextjs/cloudflare": patch +--- + +feat: r2 adapter for the incremental cache diff --git a/examples/e2e/app-router/open-next.config.ts b/examples/e2e/app-router/open-next.config.ts index 00db54284..e01a5d58c 100644 --- a/examples/e2e/app-router/open-next.config.ts +++ b/examples/e2e/app-router/open-next.config.ts @@ -1,10 +1,11 @@ import { defineCloudflareConfig } from "@opennextjs/cloudflare"; import d1TagCache from "@opennextjs/cloudflare/d1-tag-cache"; -import kvIncrementalCache from "@opennextjs/cloudflare/kv-cache"; +// import kvIncrementalCache from "@opennextjs/cloudflare/kv-cache"; +import r2IncrementalCache from "@opennextjs/cloudflare/r2-incremental-cache"; import memoryQueue from "@opennextjs/cloudflare/memory-queue"; export default defineCloudflareConfig({ - incrementalCache: kvIncrementalCache, + incrementalCache: r2IncrementalCache, tagCache: d1TagCache, queue: memoryQueue, }); diff --git a/examples/e2e/app-router/wrangler.jsonc b/examples/e2e/app-router/wrangler.jsonc index 25be2dde5..f7de1ca39 100644 --- a/examples/e2e/app-router/wrangler.jsonc +++ b/examples/e2e/app-router/wrangler.jsonc @@ -26,5 +26,12 @@ "binding": "NEXT_CACHE_REVALIDATION_WORKER", "service": "app-router" } + ], + "r2_buckets": [ + { + "binding": "NEXT_CACHE_R2_BUCKET", + "bucket_name": "NEXT_CACHE_R2_BUCKET", + "preview_bucket_name": "NEXT_CACHE_R2_BUCKET" + } ] } diff --git a/packages/cloudflare/src/api/cloudflare-context.ts b/packages/cloudflare/src/api/cloudflare-context.ts index f13a6a5b6..6e866b52c 100644 --- a/packages/cloudflare/src/api/cloudflare-context.ts +++ b/packages/cloudflare/src/api/cloudflare-context.ts @@ -7,6 +7,8 @@ declare global { NEXT_CACHE_D1_TAGS_TABLE?: string; NEXT_CACHE_D1_REVALIDATIONS_TABLE?: string; NEXT_CACHE_REVALIDATION_WORKER?: Service; + NEXT_CACHE_R2_BUCKET?: R2Bucket; + NEXT_CACHE_R2_DIRECTORY?: string; ASSETS?: Fetcher; } } diff --git a/packages/cloudflare/src/api/r2-incremental-cache.ts b/packages/cloudflare/src/api/r2-incremental-cache.ts new file mode 100644 index 000000000..78bfb807d --- /dev/null +++ b/packages/cloudflare/src/api/r2-incremental-cache.ts @@ -0,0 +1,171 @@ +import { debug, error } from "@opennextjs/aws/adapters/logger.js"; +import type { CacheValue, IncrementalCache, WithLastModified } from "@opennextjs/aws/types/overrides.js"; +import { IgnorableError } from "@opennextjs/aws/utils/error.js"; + +import { getCloudflareContext } from "./cloudflare-context.js"; + +type Entry = { + value: CacheValue; + lastModified: number; +}; + +const ONE_YEAR_IN_SECONDS = 31536000; + +/** + * An instance of the Incremental Cache that uses an R2 bucket (`NEXT_CACHE_R2_BUCKET`) as it's + * underlying data store. + * + * The directory that the cache entries are stored in can be confused with the `NEXT_CACHE_R2_DIRECTORY` + * environment variable, and defaults to `incremental-cache`. + * + * The cache uses an instance of the Cache API (`incremental-cache`) to store a local version of the + * R2 cache entry to enable fast retrieval, with the cache being updated from R2 in the background. + */ +class R2IncrementalCache implements IncrementalCache { + readonly name = "r2-incremental-cache"; + + protected localCache: Cache | undefined; + + async get( + key: string, + isFetch?: IsFetch + ): Promise> | null> { + const r2 = getCloudflareContext().env.NEXT_CACHE_R2_BUCKET; + if (!r2) throw new IgnorableError("No R2 bucket"); + + debug(`Get ${key}`); + + try { + const r2Response = r2.get(this.getR2Key(key)); + + const localCacheKey = this.getLocalCacheKey(key, isFetch); + + // Check for a cached entry as this will be faster than R2. + const cachedResponse = await this.getFromLocalCache(localCacheKey); + if (cachedResponse) { + debug(` -> Cached response`); + // Update the local cache after the R2 fetch has completed. + getCloudflareContext().ctx.waitUntil( + Promise.resolve(r2Response).then(async (res) => { + if (res) { + const entry: Entry = await res.json(); + await this.putToLocalCache(localCacheKey, JSON.stringify(entry), entry.value.revalidate); + } + }) + ); + + return cachedResponse.json(); + } + + const r2Object = await r2Response; + if (!r2Object) return null; + const entry: Entry = await r2Object.json(); + + // Update the locale cache after retrieving from R2. + getCloudflareContext().ctx.waitUntil( + this.putToLocalCache(localCacheKey, JSON.stringify(entry), entry.value.revalidate) + ); + + return entry; + } catch (e) { + error(`Failed to get from cache`, e); + return null; + } + } + + async set( + key: string, + value: CacheValue, + isFetch?: IsFetch + ): Promise { + const r2 = getCloudflareContext().env.NEXT_CACHE_R2_BUCKET; + if (!r2) throw new IgnorableError("No R2 bucket"); + + debug(`Set ${key}`); + + try { + const entry: Entry = { + value, + // Note: `Date.now()` returns the time of the last IO rather than the actual time. + // See https://developers.cloudflare.com/workers/reference/security-model/ + lastModified: Date.now(), + }; + + await Promise.all([ + r2.put(this.getR2Key(key, isFetch), JSON.stringify(entry)), + // Update the locale cache for faster retrieval. + this.putToLocalCache( + this.getLocalCacheKey(key, isFetch), + JSON.stringify(entry), + entry.value.revalidate + ), + ]); + } catch (e) { + error(`Failed to set to cache`, e); + } + } + + async delete(key: string): Promise { + const r2 = getCloudflareContext().env.NEXT_CACHE_R2_BUCKET; + if (!r2) throw new IgnorableError("No R2 bucket"); + + debug(`Delete ${key}`); + + try { + await Promise.all([ + r2.delete(this.getR2Key(key)), + this.deleteFromLocalCache(this.getLocalCacheKey(key)), + ]); + } catch (e) { + error(`Failed to delete from cache`, e); + } + } + + protected getBaseCacheKey(key: string, isFetch?: boolean): string { + return `${process.env.NEXT_BUILD_ID ?? "no-build-id"}/${key}.${isFetch ? "fetch" : "cache"}`; + } + + protected getR2Key(key: string, isFetch?: boolean): string { + const directory = getCloudflareContext().env.NEXT_CACHE_R2_DIRECTORY ?? "incremental-cache"; + return `${directory}/${this.getBaseCacheKey(key, isFetch)}`; + } + + protected getLocalCacheKey(key: string, isFetch?: boolean) { + return new Request(new URL(this.getBaseCacheKey(key, isFetch), "http://cache.local")); + } + + protected async getLocalCacheInstance(): Promise { + if (this.localCache) return this.localCache; + + this.localCache = await caches.open("incremental-cache"); + return this.localCache; + } + + protected async getFromLocalCache(key: Request) { + const cache = await this.getLocalCacheInstance(); + return cache.match(key); + } + + protected async putToLocalCache( + key: Request, + entry: string, + revalidate: number | false | undefined + ): Promise { + const cache = await this.getLocalCacheInstance(); + await cache.put( + key, + new Response(entry, { + headers: new Headers({ + "cache-control": `max-age=${revalidate || ONE_YEAR_IN_SECONDS}`, + }), + }) + ); + } + + protected async deleteFromLocalCache(key: Request) { + const cache = await this.getLocalCacheInstance(); + await cache.delete(key); + } +} + +export default new R2IncrementalCache(); diff --git a/packages/cloudflare/src/cli/build/utils/populate-cache.ts b/packages/cloudflare/src/cli/build/utils/populate-cache.ts index ad38ede79..d8545cbdd 100644 --- a/packages/cloudflare/src/cli/build/utils/populate-cache.ts +++ b/packages/cloudflare/src/cli/build/utils/populate-cache.ts @@ -11,6 +11,7 @@ import type { OpenNextConfig, } from "@opennextjs/aws/types/open-next.js"; import type { IncrementalCache, TagCache } from "@opennextjs/aws/types/overrides.js"; +import { globSync } from "glob"; export type CacheBindingMode = "local" | "remote"; @@ -24,10 +25,20 @@ async function resolveCacheName( return typeof value === "function" ? (await value()).name : value; } -function runWrangler(opts: BuildOptions, mode: CacheBindingMode, args: string[]) { +function runWrangler( + opts: BuildOptions, + wranglerOpts: { mode: CacheBindingMode; excludeRemoteFlag?: boolean }, + args: string[] +) { const result = spawnSync( opts.packager, - ["exec", "wrangler", ...args, mode === "remote" && "--remote"].filter((v): v is string => !!v), + [ + "exec", + "wrangler", + ...args, + wranglerOpts.mode === "remote" && !wranglerOpts.excludeRemoteFlag && "--remote", + wranglerOpts.mode === "local" && "--local", + ].filter((v): v is string => !!v), { shell: true, stdio: ["ignore", "ignore", "inherit"], @@ -37,11 +48,24 @@ function runWrangler(opts: BuildOptions, mode: CacheBindingMode, args: string[]) if (result.status !== 0) { logger.error("Failed to populate cache"); process.exit(1); - } else { - logger.info("Successfully populated cache"); } } +function getCacheAssetPaths(opts: BuildOptions) { + return globSync(path.join(opts.outputDir, "cache/**/*"), { withFileTypes: true }) + .filter((f) => f.isFile()) + .map((f) => { + const relativePath = path.relative(path.join(opts.outputDir, "cache"), f.fullpathPosix()); + + return { + fsPath: f.fullpathPosix(), + destPath: relativePath.startsWith("__fetch") + ? `${relativePath.replace("__fetch/", "")}.fetch` + : relativePath, + }; + }); +} + export async function populateCache(opts: BuildOptions, config: OpenNextConfig, mode: CacheBindingMode) { const { incrementalCache, tagCache } = config.default.override ?? {}; @@ -51,7 +75,31 @@ export async function populateCache(opts: BuildOptions, config: OpenNextConfig, } if (!config.dangerous?.disableIncrementalCache && incrementalCache) { - logger.info("Incremental cache does not need populating"); + const name = await resolveCacheName(incrementalCache); + switch (name) { + case "r2-incremental-cache": { + logger.info("\nPopulating R2 incremental cache..."); + + const assets = getCacheAssetPaths(opts); + assets.forEach(({ fsPath, destPath }) => { + const fullDestPath = path.join( + "NEXT_CACHE_R2_BUCKET", + process.env.NEXT_CACHE_R2_DIRECTORY ?? "incremental-cache", + destPath + ); + + runWrangler(opts, { mode, excludeRemoteFlag: true }, [ + "r2 object put", + JSON.stringify(fullDestPath), + `--file ${JSON.stringify(fsPath)}`, + ]); + }); + logger.info(`Successfully populated cache with ${assets.length} assets`); + break; + } + default: + logger.info("INcremental cache does not need populating"); + } } if (!config.dangerous?.disableTagCache && !config.dangerous?.disableIncrementalCache && tagCache) { @@ -60,11 +108,12 @@ export async function populateCache(opts: BuildOptions, config: OpenNextConfig, case "d1-tag-cache": { logger.info("\nPopulating D1 tag cache..."); - runWrangler(opts, mode, [ + runWrangler(opts, { mode }, [ "d1 execute", "NEXT_CACHE_D1", `--file ${JSON.stringify(path.join(opts.outputDir, "cloudflare/cache-assets-manifest.sql"))}`, ]); + logger.info("Successfully populated cache"); break; } default: From ecc83aa94dbef6668ceeb64b3a6c4dcecf6fbbce Mon Sep 17 00:00:00 2001 From: James Date: Wed, 12 Mar 2025 23:43:31 +0000 Subject: [PATCH 07/26] directory -> prefix --- packages/cloudflare/src/api/cloudflare-context.ts | 2 +- packages/cloudflare/src/api/r2-incremental-cache.ts | 4 ++-- packages/cloudflare/src/cli/build/utils/populate-cache.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/cloudflare/src/api/cloudflare-context.ts b/packages/cloudflare/src/api/cloudflare-context.ts index 6e866b52c..d7c039974 100644 --- a/packages/cloudflare/src/api/cloudflare-context.ts +++ b/packages/cloudflare/src/api/cloudflare-context.ts @@ -8,7 +8,7 @@ declare global { NEXT_CACHE_D1_REVALIDATIONS_TABLE?: string; NEXT_CACHE_REVALIDATION_WORKER?: Service; NEXT_CACHE_R2_BUCKET?: R2Bucket; - NEXT_CACHE_R2_DIRECTORY?: string; + NEXT_CACHE_R2_PREFIX?: string; ASSETS?: Fetcher; } } diff --git a/packages/cloudflare/src/api/r2-incremental-cache.ts b/packages/cloudflare/src/api/r2-incremental-cache.ts index 78bfb807d..9f41fcf97 100644 --- a/packages/cloudflare/src/api/r2-incremental-cache.ts +++ b/packages/cloudflare/src/api/r2-incremental-cache.ts @@ -15,7 +15,7 @@ const ONE_YEAR_IN_SECONDS = 31536000; * An instance of the Incremental Cache that uses an R2 bucket (`NEXT_CACHE_R2_BUCKET`) as it's * underlying data store. * - * The directory that the cache entries are stored in can be confused with the `NEXT_CACHE_R2_DIRECTORY` + * The directory that the cache entries are stored in can be confused with the `NEXT_CACHE_R2_PREFIX` * environment variable, and defaults to `incremental-cache`. * * The cache uses an instance of the Cache API (`incremental-cache`) to store a local version of the @@ -126,7 +126,7 @@ class R2IncrementalCache implements IncrementalCache { } protected getR2Key(key: string, isFetch?: boolean): string { - const directory = getCloudflareContext().env.NEXT_CACHE_R2_DIRECTORY ?? "incremental-cache"; + const directory = getCloudflareContext().env.NEXT_CACHE_R2_PREFIX ?? "incremental-cache"; return `${directory}/${this.getBaseCacheKey(key, isFetch)}`; } diff --git a/packages/cloudflare/src/cli/build/utils/populate-cache.ts b/packages/cloudflare/src/cli/build/utils/populate-cache.ts index d8545cbdd..f678ffcbf 100644 --- a/packages/cloudflare/src/cli/build/utils/populate-cache.ts +++ b/packages/cloudflare/src/cli/build/utils/populate-cache.ts @@ -84,7 +84,7 @@ export async function populateCache(opts: BuildOptions, config: OpenNextConfig, assets.forEach(({ fsPath, destPath }) => { const fullDestPath = path.join( "NEXT_CACHE_R2_BUCKET", - process.env.NEXT_CACHE_R2_DIRECTORY ?? "incremental-cache", + process.env.NEXT_CACHE_R2_PREFIX ?? "incremental-cache", destPath ); From c13ca95c8ba6656ea44804795fd1072faec5f310 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 13 Mar 2025 00:42:21 +0000 Subject: [PATCH 08/26] split r2 cache and regional cache into separate things --- examples/e2e/app-router/open-next.config.ts | 3 +- .../src/api/internal/incremental-cache.ts | 6 + .../src/api/r2-incremental-cache.ts | 106 ++------------ packages/cloudflare/src/api/regional-cache.ts | 133 ++++++++++++++++++ 4 files changed, 151 insertions(+), 97 deletions(-) create mode 100644 packages/cloudflare/src/api/internal/incremental-cache.ts create mode 100644 packages/cloudflare/src/api/regional-cache.ts diff --git a/examples/e2e/app-router/open-next.config.ts b/examples/e2e/app-router/open-next.config.ts index e01a5d58c..81590e11a 100644 --- a/examples/e2e/app-router/open-next.config.ts +++ b/examples/e2e/app-router/open-next.config.ts @@ -3,9 +3,10 @@ import d1TagCache from "@opennextjs/cloudflare/d1-tag-cache"; // import kvIncrementalCache from "@opennextjs/cloudflare/kv-cache"; import r2IncrementalCache from "@opennextjs/cloudflare/r2-incremental-cache"; import memoryQueue from "@opennextjs/cloudflare/memory-queue"; +import { withRegionalCache } from "@opennextjs/cloudflare/regional-cache"; export default defineCloudflareConfig({ - incrementalCache: r2IncrementalCache, + incrementalCache: withRegionalCache(r2IncrementalCache, { mode: "long-lived" }), tagCache: d1TagCache, queue: memoryQueue, }); diff --git a/packages/cloudflare/src/api/internal/incremental-cache.ts b/packages/cloudflare/src/api/internal/incremental-cache.ts new file mode 100644 index 000000000..2407fef8a --- /dev/null +++ b/packages/cloudflare/src/api/internal/incremental-cache.ts @@ -0,0 +1,6 @@ +import { CacheValue } from "@opennextjs/aws/types/overrides.js"; + +export type IncrementalCacheEntry = { + value: CacheValue; + lastModified: number; +}; diff --git a/packages/cloudflare/src/api/r2-incremental-cache.ts b/packages/cloudflare/src/api/r2-incremental-cache.ts index 9f41fcf97..f230a9eb3 100644 --- a/packages/cloudflare/src/api/r2-incremental-cache.ts +++ b/packages/cloudflare/src/api/r2-incremental-cache.ts @@ -3,13 +3,7 @@ import type { CacheValue, IncrementalCache, WithLastModified } from "@opennextjs import { IgnorableError } from "@opennextjs/aws/utils/error.js"; import { getCloudflareContext } from "./cloudflare-context.js"; - -type Entry = { - value: CacheValue; - lastModified: number; -}; - -const ONE_YEAR_IN_SECONDS = 31536000; +import { IncrementalCacheEntry } from "./internal/incremental-cache.js"; /** * An instance of the Incremental Cache that uses an R2 bucket (`NEXT_CACHE_R2_BUCKET`) as it's @@ -24,8 +18,6 @@ const ONE_YEAR_IN_SECONDS = 31536000; class R2IncrementalCache implements IncrementalCache { readonly name = "r2-incremental-cache"; - protected localCache: Cache | undefined; - async get( key: string, isFetch?: IsFetch @@ -36,39 +28,12 @@ class R2IncrementalCache implements IncrementalCache { debug(`Get ${key}`); try { - const r2Response = r2.get(this.getR2Key(key)); - - const localCacheKey = this.getLocalCacheKey(key, isFetch); - - // Check for a cached entry as this will be faster than R2. - const cachedResponse = await this.getFromLocalCache(localCacheKey); - if (cachedResponse) { - debug(` -> Cached response`); - // Update the local cache after the R2 fetch has completed. - getCloudflareContext().ctx.waitUntil( - Promise.resolve(r2Response).then(async (res) => { - if (res) { - const entry: Entry = await res.json(); - await this.putToLocalCache(localCacheKey, JSON.stringify(entry), entry.value.revalidate); - } - }) - ); - - return cachedResponse.json(); - } - - const r2Object = await r2Response; + const r2Object = await r2.get(this.getR2Key(key, isFetch)); if (!r2Object) return null; - const entry: Entry = await r2Object.json(); - // Update the locale cache after retrieving from R2. - getCloudflareContext().ctx.waitUntil( - this.putToLocalCache(localCacheKey, JSON.stringify(entry), entry.value.revalidate) - ); - - return entry; + return r2Object.json(); } catch (e) { - error(`Failed to get from cache`, e); + error("Failed to get from cache", e); return null; } } @@ -84,24 +49,16 @@ class R2IncrementalCache implements IncrementalCache { debug(`Set ${key}`); try { - const entry: Entry = { + const entry: IncrementalCacheEntry = { value, // Note: `Date.now()` returns the time of the last IO rather than the actual time. // See https://developers.cloudflare.com/workers/reference/security-model/ lastModified: Date.now(), }; - await Promise.all([ - r2.put(this.getR2Key(key, isFetch), JSON.stringify(entry)), - // Update the locale cache for faster retrieval. - this.putToLocalCache( - this.getLocalCacheKey(key, isFetch), - JSON.stringify(entry), - entry.value.revalidate - ), - ]); + await r2.put(this.getR2Key(key, isFetch), JSON.stringify(entry)); } catch (e) { - error(`Failed to set to cache`, e); + error("Failed to set to cache", e); } } @@ -112,59 +69,16 @@ class R2IncrementalCache implements IncrementalCache { debug(`Delete ${key}`); try { - await Promise.all([ - r2.delete(this.getR2Key(key)), - this.deleteFromLocalCache(this.getLocalCacheKey(key)), - ]); + await r2.delete(this.getR2Key(key)); } catch (e) { - error(`Failed to delete from cache`, e); + error("Failed to delete from cache", e); } } - protected getBaseCacheKey(key: string, isFetch?: boolean): string { - return `${process.env.NEXT_BUILD_ID ?? "no-build-id"}/${key}.${isFetch ? "fetch" : "cache"}`; - } - protected getR2Key(key: string, isFetch?: boolean): string { const directory = getCloudflareContext().env.NEXT_CACHE_R2_PREFIX ?? "incremental-cache"; - return `${directory}/${this.getBaseCacheKey(key, isFetch)}`; - } - - protected getLocalCacheKey(key: string, isFetch?: boolean) { - return new Request(new URL(this.getBaseCacheKey(key, isFetch), "http://cache.local")); - } - - protected async getLocalCacheInstance(): Promise { - if (this.localCache) return this.localCache; - - this.localCache = await caches.open("incremental-cache"); - return this.localCache; - } - - protected async getFromLocalCache(key: Request) { - const cache = await this.getLocalCacheInstance(); - return cache.match(key); - } - - protected async putToLocalCache( - key: Request, - entry: string, - revalidate: number | false | undefined - ): Promise { - const cache = await this.getLocalCacheInstance(); - await cache.put( - key, - new Response(entry, { - headers: new Headers({ - "cache-control": `max-age=${revalidate || ONE_YEAR_IN_SECONDS}`, - }), - }) - ); - } - protected async deleteFromLocalCache(key: Request) { - const cache = await this.getLocalCacheInstance(); - await cache.delete(key); + return `${directory}/${process.env.NEXT_BUILD_ID ?? "no-build-id"}/${key}.${isFetch ? "fetch" : "cache"}`; } } diff --git a/packages/cloudflare/src/api/regional-cache.ts b/packages/cloudflare/src/api/regional-cache.ts new file mode 100644 index 000000000..552242218 --- /dev/null +++ b/packages/cloudflare/src/api/regional-cache.ts @@ -0,0 +1,133 @@ +import { debug, error } from "@opennextjs/aws/adapters/logger.js"; +import { CacheValue, IncrementalCache, WithLastModified } from "@opennextjs/aws/types/overrides.js"; + +import { getCloudflareContext } from "./cloudflare-context.js"; +import { IncrementalCacheEntry } from "./internal/incremental-cache.js"; + +const ONE_YEAR_IN_SECONDS = 31536000; +const ONE_MINUTE_IN_SECONDS = 60; + +type Options = { + mode: "short-lived" | "long-lived"; +}; + +class RegionalCache implements IncrementalCache { + public name: string; + + protected localCache: Cache | undefined; + + constructor( + private store: IncrementalCache, + private opts: Options + ) { + this.name = this.store.name; + } + + async get( + key: string, + isFetch?: IsFetch + ): Promise> | null> { + try { + const storeResponse = this.store.get(key, isFetch); + + const localCacheKey = this.getCacheKey(key, isFetch); + + // Check for a cached entry as this will be faster than the store response. + const cache = await this.getCacheInstance(); + const cachedResponse = await cache.match(localCacheKey); + if (cachedResponse) { + debug("Get - cached response"); + + // Update the local cache after the R2 fetch has completed. + getCloudflareContext().ctx.waitUntil( + Promise.resolve(storeResponse).then(async (rawEntry) => { + const { value, lastModified } = rawEntry ?? {}; + + if (value && typeof lastModified === "number") { + await this.putToCache(localCacheKey, { value, lastModified }); + } + }) + ); + + return cachedResponse.json(); + } + + const rawEntry = await storeResponse; + const { value, lastModified } = rawEntry ?? {}; + if (!value || typeof lastModified !== "number") return null; + + // Update the locale cache after retrieving from the store. + getCloudflareContext().ctx.waitUntil(this.putToCache(localCacheKey, { value, lastModified })); + + return { value, lastModified }; + } catch (e) { + error("Failed to get from regional cache", e); + return null; + } + } + + async set( + key: string, + value: CacheValue, + isFetch?: IsFetch + ): Promise { + try { + await Promise.all([ + this.store.set(key, value, isFetch), + this.putToCache(this.getCacheKey(key, isFetch), { + value, + // Note: `Date.now()` returns the time of the last IO rather than the actual time. + // See https://developers.cloudflare.com/workers/reference/security-model/ + lastModified: Date.now(), + }), + ]); + } catch (e) { + error(`Failed to get from regional cache`, e); + } + } + + async delete(key: string): Promise { + try { + const cache = await this.getCacheInstance(); + await Promise.all([this.store.delete(key), cache.delete(this.getCacheKey(key))]); + } catch (e) { + error("Failed to delete from regional cache", e); + } + } + + protected async getCacheInstance(): Promise { + if (this.localCache) return this.localCache; + + this.localCache = await caches.open("incremental-cache"); + return this.localCache; + } + + protected getCacheKey(key: string, isFetch?: boolean) { + return new Request( + new URL( + `${process.env.NEXT_BUILD_ID ?? "no-build-id"}/${key}.${isFetch ? "fetch" : "cache"}`, + "http://cache.local" + ) + ); + } + + protected async putToCache(key: Request, entry: IncrementalCacheEntry): Promise { + const cache = await this.getCacheInstance(); + + const age = + this.opts.mode === "short-lived" + ? ONE_MINUTE_IN_SECONDS + : entry.value.revalidate || ONE_YEAR_IN_SECONDS; + + await cache.put( + key, + new Response(JSON.stringify(entry), { + headers: new Headers({ "cache-control": `max-age=${age}` }), + }) + ); + } +} + +export function withRegionalCache(cache: IncrementalCache, opts: Options) { + return new RegionalCache(cache, opts); +} From 28c27a954b1f80b4d0a448f36cce74dc7d791d75 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 13 Mar 2025 00:42:41 +0000 Subject: [PATCH 09/26] add timeout in revalidate path test as it runs too fast locally From e018c711292ec346d2c715d377dde72ac023b603 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 13 Mar 2025 20:16:44 +0000 Subject: [PATCH 10/26] incorporate suggestions --- .../src/api/r2-incremental-cache.ts | 15 +++++-------- packages/cloudflare/src/api/regional-cache.ts | 21 ++++++++++--------- .../src/cli/build/utils/populate-cache.ts | 2 +- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/packages/cloudflare/src/api/r2-incremental-cache.ts b/packages/cloudflare/src/api/r2-incremental-cache.ts index f230a9eb3..4ab451982 100644 --- a/packages/cloudflare/src/api/r2-incremental-cache.ts +++ b/packages/cloudflare/src/api/r2-incremental-cache.ts @@ -3,7 +3,6 @@ import type { CacheValue, IncrementalCache, WithLastModified } from "@opennextjs import { IgnorableError } from "@opennextjs/aws/utils/error.js"; import { getCloudflareContext } from "./cloudflare-context.js"; -import { IncrementalCacheEntry } from "./internal/incremental-cache.js"; /** * An instance of the Incremental Cache that uses an R2 bucket (`NEXT_CACHE_R2_BUCKET`) as it's @@ -31,7 +30,10 @@ class R2IncrementalCache implements IncrementalCache { const r2Object = await r2.get(this.getR2Key(key, isFetch)); if (!r2Object) return null; - return r2Object.json(); + return { + value: await r2Object.json(), + lastModified: r2Object.uploaded.getTime(), + }; } catch (e) { error("Failed to get from cache", e); return null; @@ -49,14 +51,7 @@ class R2IncrementalCache implements IncrementalCache { debug(`Set ${key}`); try { - const entry: IncrementalCacheEntry = { - value, - // Note: `Date.now()` returns the time of the last IO rather than the actual time. - // See https://developers.cloudflare.com/workers/reference/security-model/ - lastModified: Date.now(), - }; - - await r2.put(this.getR2Key(key, isFetch), JSON.stringify(entry)); + await r2.put(this.getR2Key(key, isFetch), JSON.stringify(value)); } catch (e) { error("Failed to set to cache", e); } diff --git a/packages/cloudflare/src/api/regional-cache.ts b/packages/cloudflare/src/api/regional-cache.ts index 552242218..5877231e7 100644 --- a/packages/cloudflare/src/api/regional-cache.ts +++ b/packages/cloudflare/src/api/regional-cache.ts @@ -72,15 +72,14 @@ class RegionalCache implements IncrementalCache { isFetch?: IsFetch ): Promise { try { - await Promise.all([ - this.store.set(key, value, isFetch), - this.putToCache(this.getCacheKey(key, isFetch), { - value, - // Note: `Date.now()` returns the time of the last IO rather than the actual time. - // See https://developers.cloudflare.com/workers/reference/security-model/ - lastModified: Date.now(), - }), - ]); + await this.store.set(key, value, isFetch); + + await this.putToCache(this.getCacheKey(key, isFetch), { + value, + // Note: `Date.now()` returns the time of the last IO rather than the actual time. + // See https://developers.cloudflare.com/workers/reference/security-model/ + lastModified: Date.now(), + }); } catch (e) { error(`Failed to get from regional cache`, e); } @@ -88,8 +87,10 @@ class RegionalCache implements IncrementalCache { async delete(key: string): Promise { try { + await this.store.delete(key); + const cache = await this.getCacheInstance(); - await Promise.all([this.store.delete(key), cache.delete(this.getCacheKey(key))]); + await cache.delete(this.getCacheKey(key)); } catch (e) { error("Failed to delete from regional cache", e); } diff --git a/packages/cloudflare/src/cli/build/utils/populate-cache.ts b/packages/cloudflare/src/cli/build/utils/populate-cache.ts index f678ffcbf..d886d975e 100644 --- a/packages/cloudflare/src/cli/build/utils/populate-cache.ts +++ b/packages/cloudflare/src/cli/build/utils/populate-cache.ts @@ -98,7 +98,7 @@ export async function populateCache(opts: BuildOptions, config: OpenNextConfig, break; } default: - logger.info("INcremental cache does not need populating"); + logger.info("Incremental cache does not need populating"); } } From a2b26670c93cbb2f65030ed4b74f5a7d9372a65c Mon Sep 17 00:00:00 2001 From: James Date: Thu, 13 Mar 2025 20:33:26 +0000 Subject: [PATCH 11/26] lazily update the regional cache in the background via option --- examples/e2e/app-router/open-next.config.ts | 5 +++- packages/cloudflare/src/api/regional-cache.ts | 27 ++++++++++--------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/examples/e2e/app-router/open-next.config.ts b/examples/e2e/app-router/open-next.config.ts index 81590e11a..2f9054793 100644 --- a/examples/e2e/app-router/open-next.config.ts +++ b/examples/e2e/app-router/open-next.config.ts @@ -6,7 +6,10 @@ import memoryQueue from "@opennextjs/cloudflare/memory-queue"; import { withRegionalCache } from "@opennextjs/cloudflare/regional-cache"; export default defineCloudflareConfig({ - incrementalCache: withRegionalCache(r2IncrementalCache, { mode: "long-lived" }), + incrementalCache: withRegionalCache(r2IncrementalCache, { + mode: "long-lived", + shouldLazilyUpdateOnCacheHit: true, + }), tagCache: d1TagCache, queue: memoryQueue, }); diff --git a/packages/cloudflare/src/api/regional-cache.ts b/packages/cloudflare/src/api/regional-cache.ts index 5877231e7..ccc0deb76 100644 --- a/packages/cloudflare/src/api/regional-cache.ts +++ b/packages/cloudflare/src/api/regional-cache.ts @@ -9,6 +9,7 @@ const ONE_MINUTE_IN_SECONDS = 60; type Options = { mode: "short-lived" | "long-lived"; + shouldLazilyUpdateOnCacheHit?: boolean; }; class RegionalCache implements IncrementalCache { @@ -28,31 +29,31 @@ class RegionalCache implements IncrementalCache { isFetch?: IsFetch ): Promise> | null> { try { - const storeResponse = this.store.get(key, isFetch); - + const cache = await this.getCacheInstance(); const localCacheKey = this.getCacheKey(key, isFetch); // Check for a cached entry as this will be faster than the store response. - const cache = await this.getCacheInstance(); const cachedResponse = await cache.match(localCacheKey); if (cachedResponse) { debug("Get - cached response"); - // Update the local cache after the R2 fetch has completed. - getCloudflareContext().ctx.waitUntil( - Promise.resolve(storeResponse).then(async (rawEntry) => { - const { value, lastModified } = rawEntry ?? {}; + // Re-fetch from the store and update the regional cache in the background + if (this.opts.shouldLazilyUpdateOnCacheHit) { + getCloudflareContext().ctx.waitUntil( + this.store.get(key, isFetch).then(async (rawEntry) => { + const { value, lastModified } = rawEntry ?? {}; - if (value && typeof lastModified === "number") { - await this.putToCache(localCacheKey, { value, lastModified }); - } - }) - ); + if (value && typeof lastModified === "number") { + await this.putToCache(localCacheKey, { value, lastModified }); + } + }) + ); + } return cachedResponse.json(); } - const rawEntry = await storeResponse; + const rawEntry = await this.store.get(key, isFetch); const { value, lastModified } = rawEntry ?? {}; if (!value || typeof lastModified !== "number") return null; From 03ffc01138cf03e92f0b9cbd1e5f0d23bf447728 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 13 Mar 2025 20:46:21 +0000 Subject: [PATCH 12/26] add comments and make the lazy boolean non-optional --- packages/cloudflare/src/api/regional-cache.ts | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/packages/cloudflare/src/api/regional-cache.ts b/packages/cloudflare/src/api/regional-cache.ts index ccc0deb76..bd434b851 100644 --- a/packages/cloudflare/src/api/regional-cache.ts +++ b/packages/cloudflare/src/api/regional-cache.ts @@ -8,8 +8,18 @@ const ONE_YEAR_IN_SECONDS = 31536000; const ONE_MINUTE_IN_SECONDS = 60; type Options = { + /** + * The mode to use for the regional cache. + * + * - `short-lived`: Re-use a cache entry for up to a minute after it has been retrieved. + * - `long-lived`: Re-use a cache entry until it is revalidated. + */ mode: "short-lived" | "long-lived"; - shouldLazilyUpdateOnCacheHit?: boolean; + /** + * Whether the regional cache entry should be updated in the background or not when it experiences + * a cache hit. + */ + shouldLazilyUpdateOnCacheHit: boolean; }; class RegionalCache implements IncrementalCache { @@ -130,6 +140,19 @@ class RegionalCache implements IncrementalCache { } } +/** + * A regional cache will wrap an incremental cache and provide faster cache lookups for an entry + * when making requests within the region. + * + * The regional cache uses the Cache API. + * + * @param cache - Incremental cache instance. + * @param opts.mode - The mode to use for the regional cache. + * - `short-lived`: Re-use a cache entry for up to a minute after it has been retrieved. + * - `long-lived`: Re-use a cache entry until it is revalidated. + * @param opts.shouldLazilyUpdateOnCacheHit - Whether the regional cache entry should be updated in + * the background or not when it experiences a cache hit. + */ export function withRegionalCache(cache: IncrementalCache, opts: Options) { return new RegionalCache(cache, opts); } From 3c3b7f3105d4ef4b739dc7401bd5f5315e7141a9 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 13 Mar 2025 20:48:52 +0000 Subject: [PATCH 13/26] add warning comment --- packages/cloudflare/src/api/regional-cache.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/cloudflare/src/api/regional-cache.ts b/packages/cloudflare/src/api/regional-cache.ts index bd434b851..b55ed22b6 100644 --- a/packages/cloudflare/src/api/regional-cache.ts +++ b/packages/cloudflare/src/api/regional-cache.ts @@ -146,6 +146,9 @@ class RegionalCache implements IncrementalCache { * * The regional cache uses the Cache API. * + * **WARNING:** If an entry is revalidated in one region, it will trigger an additional revalidation if + * a request is made to another region that has an entry stored in its regional cache. + * * @param cache - Incremental cache instance. * @param opts.mode - The mode to use for the regional cache. * - `short-lived`: Re-use a cache entry for up to a minute after it has been retrieved. From d676d1d5ded28cd57c34bd498ab6fcc3fb4d631b Mon Sep 17 00:00:00 2001 From: James Date: Thu, 13 Mar 2025 20:51:38 +0000 Subject: [PATCH 14/26] add comments to env vars --- packages/cloudflare/src/api/cloudflare-context.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/cloudflare/src/api/cloudflare-context.ts b/packages/cloudflare/src/api/cloudflare-context.ts index d7c039974..3a0328ca2 100644 --- a/packages/cloudflare/src/api/cloudflare-context.ts +++ b/packages/cloudflare/src/api/cloudflare-context.ts @@ -7,7 +7,9 @@ declare global { NEXT_CACHE_D1_TAGS_TABLE?: string; NEXT_CACHE_D1_REVALIDATIONS_TABLE?: string; NEXT_CACHE_REVALIDATION_WORKER?: Service; + // R2 bucket used for the incremental cache NEXT_CACHE_R2_BUCKET?: R2Bucket; + // Prefix used for the R2 incremental cache bucket NEXT_CACHE_R2_PREFIX?: string; ASSETS?: Fetcher; } From 0c1887e2251a8b8cb13e87ae054c89d170f14601 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 14 Mar 2025 22:36:07 +0000 Subject: [PATCH 15/26] change lazy update defaults --- packages/cloudflare/src/api/regional-cache.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/cloudflare/src/api/regional-cache.ts b/packages/cloudflare/src/api/regional-cache.ts index b55ed22b6..bf8056b19 100644 --- a/packages/cloudflare/src/api/regional-cache.ts +++ b/packages/cloudflare/src/api/regional-cache.ts @@ -18,8 +18,10 @@ type Options = { /** * Whether the regional cache entry should be updated in the background or not when it experiences * a cache hit. + * + * Defaults to `false` for the `short-lived` mode, and `true` for the `long-lived` mode. */ - shouldLazilyUpdateOnCacheHit: boolean; + shouldLazilyUpdateOnCacheHit?: boolean; }; class RegionalCache implements IncrementalCache { @@ -32,6 +34,8 @@ class RegionalCache implements IncrementalCache { private opts: Options ) { this.name = this.store.name; + + this.opts.shouldLazilyUpdateOnCacheHit ??= this.opts.mode === "long-lived"; } async get( @@ -155,6 +159,8 @@ class RegionalCache implements IncrementalCache { * - `long-lived`: Re-use a cache entry until it is revalidated. * @param opts.shouldLazilyUpdateOnCacheHit - Whether the regional cache entry should be updated in * the background or not when it experiences a cache hit. + * + * Defaults to `false` for the `short-lived` mode, and `true` for the `long-lived` mode. */ export function withRegionalCache(cache: IncrementalCache, opts: Options) { return new RegionalCache(cache, opts); From 5ff320b0e5ea97aab133cc3a500b556f8c69f849 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 20 Mar 2025 21:51:37 +0000 Subject: [PATCH 16/26] change fallback ttl to 30 mins --- packages/cloudflare/src/api/regional-cache.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/cloudflare/src/api/regional-cache.ts b/packages/cloudflare/src/api/regional-cache.ts index bf8056b19..fbe8c7b4e 100644 --- a/packages/cloudflare/src/api/regional-cache.ts +++ b/packages/cloudflare/src/api/regional-cache.ts @@ -4,15 +4,15 @@ import { CacheValue, IncrementalCache, WithLastModified } from "@opennextjs/aws/ import { getCloudflareContext } from "./cloudflare-context.js"; import { IncrementalCacheEntry } from "./internal/incremental-cache.js"; -const ONE_YEAR_IN_SECONDS = 31536000; const ONE_MINUTE_IN_SECONDS = 60; +const THIRTY_MINUTES_IN_SECONDS = ONE_MINUTE_IN_SECONDS * 30; type Options = { /** * The mode to use for the regional cache. * * - `short-lived`: Re-use a cache entry for up to a minute after it has been retrieved. - * - `long-lived`: Re-use a cache entry until it is revalidated. + * - `long-lived`: Re-use a fetch cache entry until it is revalidated, or an ISR/SSG entry for up to 30 minutes. */ mode: "short-lived" | "long-lived"; /** @@ -133,7 +133,7 @@ class RegionalCache implements IncrementalCache { const age = this.opts.mode === "short-lived" ? ONE_MINUTE_IN_SECONDS - : entry.value.revalidate || ONE_YEAR_IN_SECONDS; + : entry.value.revalidate || THIRTY_MINUTES_IN_SECONDS; await cache.put( key, @@ -156,7 +156,7 @@ class RegionalCache implements IncrementalCache { * @param cache - Incremental cache instance. * @param opts.mode - The mode to use for the regional cache. * - `short-lived`: Re-use a cache entry for up to a minute after it has been retrieved. - * - `long-lived`: Re-use a cache entry until it is revalidated. + * - `long-lived`: Re-use a fetch cache entry until it is revalidated, or an ISR/SSG entry for up to 30 minutes. * @param opts.shouldLazilyUpdateOnCacheHit - Whether the regional cache entry should be updated in * the background or not when it experiences a cache hit. * From 0da9705a4f09a011523c581dea289ddb632307cd Mon Sep 17 00:00:00 2001 From: James Date: Thu, 20 Mar 2025 10:19:27 +0000 Subject: [PATCH 17/26] move opennext setup to be universal and add commands --- packages/cloudflare/src/cli/args.ts | 107 ++++++++---------- packages/cloudflare/src/cli/build/build.ts | 50 ++------ .../cli/build/utils/create-config-files.ts | 6 +- packages/cloudflare/src/cli/index.ts | 53 +++++++-- 4 files changed, 103 insertions(+), 113 deletions(-) diff --git a/packages/cloudflare/src/cli/args.ts b/packages/cloudflare/src/cli/args.ts index 5a546ad99..529015e54 100644 --- a/packages/cloudflare/src/cli/args.ts +++ b/packages/cloudflare/src/cli/args.ts @@ -2,71 +2,58 @@ import { mkdirSync, type Stats, statSync } from "node:fs"; import { resolve } from "node:path"; import { parseArgs } from "node:util"; -import type { CacheBindingMode } from "./build/utils/index.js"; -import { isCacheBindingMode } from "./build/utils/index.js"; +import type { CacheBindingTarget } from "./build/utils/index.js"; +import { isCacheBindingTarget } from "./build/utils/index.js"; -export function getArgs(): { - skipNextBuild: boolean; - skipWranglerConfigCheck: boolean; - outputDir?: string; - minify: boolean; - populateCache?: { mode: CacheBindingMode; onlyPopulateWithoutBuilding: boolean }; -} { - const { skipBuild, skipWranglerConfigCheck, output, noMinify, populateCache, onlyPopulateCache } = - parseArgs({ - options: { - skipBuild: { - type: "boolean", - short: "s", - default: false, - }, - output: { - type: "string", - short: "o", - }, - noMinify: { - type: "boolean", - default: false, - }, - skipWranglerConfigCheck: { - type: "boolean", - default: false, - }, - populateCache: { - type: "string", - }, - onlyPopulateCache: { - type: "boolean", - default: false, - }, - }, - allowPositionals: false, - }).values; +export type Arguments = ( + | { + command: "build"; + skipNextBuild: boolean; + skipWranglerConfigCheck: boolean; + minify: boolean; + } + | { command: "preview" | "deploy" } + | { command: "populateCache"; target: CacheBindingTarget } +) & { outputDir?: string }; - const outputDir = output ? resolve(output) : undefined; +export function getArgs(): Arguments { + const { positionals, values } = parseArgs({ + options: { + skipBuild: { type: "boolean", short: "s", default: false }, + output: { type: "string", short: "o" }, + noMinify: { type: "boolean", default: false }, + skipWranglerConfigCheck: { type: "boolean", default: false }, + }, + allowPositionals: true, + }); - if (outputDir) { - assertDirArg(outputDir, "output", true); - } + const outputDir = values.output ? resolve(values.output) : undefined; + if (outputDir) assertDirArg(outputDir, "output", true); - if ( - (populateCache !== undefined || onlyPopulateCache) && - (!populateCache?.length || !isCacheBindingMode(populateCache)) - ) { - throw new Error(`Error: missing mode for populate cache flag, expected 'local' | 'remote'`); + switch (positionals[0]) { + case "build": + return { + command: "build", + outputDir, + skipNextBuild: + values.skipBuild || ["1", "true", "yes"].includes(String(process.env.SKIP_NEXT_APP_BUILD)), + skipWranglerConfigCheck: + values.skipWranglerConfigCheck || + ["1", "true", "yes"].includes(String(process.env.SKIP_WRANGLER_CONFIG_CHECK)), + minify: !values.noMinify, + }; + case "preview": + return { command: "preview", outputDir }; + case "deploy": + return { command: "preview", outputDir }; + case "populateCache": + if (!isCacheBindingTarget(positionals[1])) { + throw new Error(`Error: invalid target for populating the cache, expected 'local' | 'remote'`); + } + return { command: "populateCache", outputDir, target: positionals[1] }; + default: + throw new Error("Error: invalid command, expected 'build' | 'preview' | 'deploy' | 'populateCache'"); } - - return { - outputDir, - skipNextBuild: skipBuild || ["1", "true", "yes"].includes(String(process.env.SKIP_NEXT_APP_BUILD)), - skipWranglerConfigCheck: - skipWranglerConfigCheck || - ["1", "true", "yes"].includes(String(process.env.SKIP_WRANGLER_CONFIG_CHECK)), - minify: !noMinify, - populateCache: populateCache - ? { mode: populateCache, onlyPopulateWithoutBuilding: !!onlyPopulateCache } - : undefined, - }; } function assertDirArg(path: string, argName?: string, make?: boolean) { diff --git a/packages/cloudflare/src/cli/build/build.ts b/packages/cloudflare/src/cli/build/build.ts index c2b8de995..afa82cf89 100644 --- a/packages/cloudflare/src/cli/build/build.ts +++ b/packages/cloudflare/src/cli/build/build.ts @@ -1,14 +1,12 @@ -import { createRequire } from "node:module"; -import { dirname } from "node:path"; - import { buildNextjsApp, setStandaloneBuildMode } from "@opennextjs/aws/build/buildNextApp.js"; import { compileCache } from "@opennextjs/aws/build/compileCache.js"; -import { compileOpenNextConfig } from "@opennextjs/aws/build/compileConfig.js"; import { createCacheAssets, createStaticAssets } from "@opennextjs/aws/build/createAssets.js"; import { createMiddleware } from "@opennextjs/aws/build/createMiddleware.js"; import * as buildHelper from "@opennextjs/aws/build/helper.js"; -import { printHeader, showWarningOnWindows } from "@opennextjs/aws/build/utils.js"; +import { BuildOptions } from "@opennextjs/aws/build/helper.js"; +import { printHeader } from "@opennextjs/aws/build/utils.js"; import logger from "@opennextjs/aws/logger.js"; +import { OpenNextConfig } from "@opennextjs/aws/types/open-next.js"; import type { ProjectOptions } from "../project-options.js"; import { bundleServer } from "./bundle-server.js"; @@ -16,12 +14,7 @@ import { compileCacheAssetsManifestSqlFile } from "./open-next/compile-cache-ass import { compileEnvFiles } from "./open-next/compile-env-files.js"; import { copyCacheAssets } from "./open-next/copyCacheAssets.js"; import { createServerBundle } from "./open-next/createServerBundle.js"; -import { - createOpenNextConfigIfNotExistent, - createWranglerConfigIfNotExistent, - ensureCloudflareConfig, - populateCache, -} from "./utils/index.js"; +import { createWranglerConfigIfNotExistent } from "./utils/index.js"; import { getVersion } from "./utils/version.js"; /** @@ -29,27 +22,15 @@ import { getVersion } from "./utils/version.js"; * * It saves the output in a `.worker-next` directory * + * @param options The OpenNext options + * @param config The OpenNext config * @param projectOpts The options for the project */ -export async function build(projectOpts: ProjectOptions): Promise { - printHeader("Cloudflare build"); - - showWarningOnWindows(); - - const baseDir = projectOpts.sourceDir; - const require = createRequire(import.meta.url); - const openNextDistDir = dirname(require.resolve("@opennextjs/aws/index.js")); - - await createOpenNextConfigIfNotExistent(projectOpts); - - const { config, buildDir } = await compileOpenNextConfig(baseDir); - - ensureCloudflareConfig(config); - - // Initialize options - const options = buildHelper.normalizeOptions(config, openNextDistDir, buildDir); - logger.setLevel(options.debug ? "debug" : "info"); - +export async function build( + options: BuildOptions, + config: OpenNextConfig, + projectOpts: ProjectOptions +): Promise { // Do not minify the code so that we can apply string replacement patch. // Note that wrangler will still minify the bundle. options.minify = false; @@ -63,11 +44,6 @@ export async function build(projectOpts: ProjectOptions): Promise { logger.info(`@opennextjs/cloudflare version: ${cloudflare}`); logger.info(`@opennextjs/aws version: ${aws}`); - if (projectOpts.populateCache?.onlyPopulateWithoutBuilding) { - populateCache(options, config, projectOpts.populateCache.mode); - return; - } - if (projectOpts.skipNextBuild) { logger.warn("Skipping Next.js build"); } else { @@ -109,10 +85,6 @@ export async function build(projectOpts: ProjectOptions): Promise { await createWranglerConfigIfNotExistent(projectOpts); } - if (projectOpts.populateCache) { - populateCache(options, config, projectOpts.populateCache.mode); - } - logger.info("OpenNext build complete."); } diff --git a/packages/cloudflare/src/cli/build/utils/create-config-files.ts b/packages/cloudflare/src/cli/build/utils/create-config-files.ts index b13f735e2..8515bfd1c 100644 --- a/packages/cloudflare/src/cli/build/utils/create-config-files.ts +++ b/packages/cloudflare/src/cli/build/utils/create-config-files.ts @@ -92,10 +92,10 @@ export async function getLatestCompatDate(): Promise { * * If the user refuses an error is thrown (since the file is mandatory). * - * @param projectOpts The options for the project + * @param sourceDir The source directory for the project */ -export async function createOpenNextConfigIfNotExistent(projectOpts: ProjectOptions): Promise { - const openNextConfigPath = join(projectOpts.sourceDir, "open-next.config.ts"); +export async function createOpenNextConfigIfNotExistent(sourceDir: string): Promise { + const openNextConfigPath = join(sourceDir, "open-next.config.ts"); if (!existsSync(openNextConfigPath)) { const answer = await askConfirmation( diff --git a/packages/cloudflare/src/cli/index.ts b/packages/cloudflare/src/cli/index.ts index be3c1e763..345e05346 100644 --- a/packages/cloudflare/src/cli/index.ts +++ b/packages/cloudflare/src/cli/index.ts @@ -1,18 +1,49 @@ #!/usr/bin/env node -import { resolve } from "node:path"; +import { createRequire } from "node:module"; +import path from "node:path"; -import { getArgs } from "./args.js"; +import { compileOpenNextConfig } from "@opennextjs/aws/build/compileConfig.js"; +import { normalizeOptions } from "@opennextjs/aws/build/helper.js"; +import { printHeader, showWarningOnWindows } from "@opennextjs/aws/build/utils.js"; +import logger from "@opennextjs/aws/logger.js"; + +import { Arguments, getArgs } from "./args.js"; import { build } from "./build/build.js"; +import { createOpenNextConfigIfNotExistent, ensureCloudflareConfig } from "./build/utils/index.js"; +import { deploy } from "./deploy/deploy.js"; +import { populateCache } from "./populate-cache/populate-cache.js"; +import { preview } from "./preview/preview.js"; const nextAppDir = process.cwd(); -const { skipNextBuild, skipWranglerConfigCheck, outputDir, minify, populateCache } = getArgs(); +async function runCommand(args: Arguments) { + printHeader(`Cloudflare ${args.command}`); + + showWarningOnWindows(); + + const baseDir = nextAppDir; + const require = createRequire(import.meta.url); + const openNextDistDir = path.dirname(require.resolve("@opennextjs/aws/index.js")); + + await createOpenNextConfigIfNotExistent(baseDir); + const { config, buildDir } = await compileOpenNextConfig(baseDir); + + ensureCloudflareConfig(config); + + // Initialize options + const options = normalizeOptions(config, openNextDistDir, buildDir); + logger.setLevel(options.debug ? "debug" : "info"); + + switch (args.command) { + case "build": + return build(options, config, args); + case "preview": + return preview(options, config); + case "deploy": + return deploy(options, config); + case "populateCache": + return populateCache(options, config, { target: args.target }); + } +} -await build({ - sourceDir: nextAppDir, - outputDir: resolve(outputDir ?? nextAppDir, ".open-next"), - skipNextBuild, - skipWranglerConfigCheck, - minify, - populateCache, -}); +await runCommand(getArgs()); From 497855a2836840b9b0d0fafa87f845ad38abe5ae Mon Sep 17 00:00:00 2001 From: James Date: Thu, 20 Mar 2025 10:20:32 +0000 Subject: [PATCH 18/26] move populate cache location --- .../src/cli/{build/utils => populate-cache}/populate-cache.ts | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/cloudflare/src/cli/{build/utils => populate-cache}/populate-cache.ts (100%) diff --git a/packages/cloudflare/src/cli/build/utils/populate-cache.ts b/packages/cloudflare/src/cli/populate-cache/populate-cache.ts similarity index 100% rename from packages/cloudflare/src/cli/build/utils/populate-cache.ts rename to packages/cloudflare/src/cli/populate-cache/populate-cache.ts From 3484a59c51cd9a912a2477c92495ca07b6d280a3 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 20 Mar 2025 10:26:11 +0000 Subject: [PATCH 19/26] cleanup and move around some utils --- packages/cloudflare/src/cli/args.ts | 7 ++- packages/cloudflare/src/cli/deploy/deploy.ts | 4 ++ .../src/cli/populate-cache/populate-cache.ts | 51 +++++-------------- .../cloudflare/src/cli/preview/preview.ts | 4 ++ .../cloudflare/src/cli/project-options.ts | 7 --- .../cloudflare/src/cli/utils/run-wrangler.ts | 35 +++++++++++++ 6 files changed, 58 insertions(+), 50 deletions(-) create mode 100644 packages/cloudflare/src/cli/deploy/deploy.ts create mode 100644 packages/cloudflare/src/cli/preview/preview.ts create mode 100644 packages/cloudflare/src/cli/utils/run-wrangler.ts diff --git a/packages/cloudflare/src/cli/args.ts b/packages/cloudflare/src/cli/args.ts index 529015e54..7951dc6c5 100644 --- a/packages/cloudflare/src/cli/args.ts +++ b/packages/cloudflare/src/cli/args.ts @@ -2,8 +2,7 @@ import { mkdirSync, type Stats, statSync } from "node:fs"; import { resolve } from "node:path"; import { parseArgs } from "node:util"; -import type { CacheBindingTarget } from "./build/utils/index.js"; -import { isCacheBindingTarget } from "./build/utils/index.js"; +import { isWranglerTarget, WranglerTarget } from "./utils/run-wrangler"; export type Arguments = ( | { @@ -13,7 +12,7 @@ export type Arguments = ( minify: boolean; } | { command: "preview" | "deploy" } - | { command: "populateCache"; target: CacheBindingTarget } + | { command: "populateCache"; target: WranglerTarget } ) & { outputDir?: string }; export function getArgs(): Arguments { @@ -47,7 +46,7 @@ export function getArgs(): Arguments { case "deploy": return { command: "preview", outputDir }; case "populateCache": - if (!isCacheBindingTarget(positionals[1])) { + if (!isWranglerTarget(positionals[1])) { throw new Error(`Error: invalid target for populating the cache, expected 'local' | 'remote'`); } return { command: "populateCache", outputDir, target: positionals[1] }; diff --git a/packages/cloudflare/src/cli/deploy/deploy.ts b/packages/cloudflare/src/cli/deploy/deploy.ts new file mode 100644 index 000000000..c4bb5b804 --- /dev/null +++ b/packages/cloudflare/src/cli/deploy/deploy.ts @@ -0,0 +1,4 @@ +import { BuildOptions } from "@opennextjs/aws/build/helper.js"; +import { OpenNextConfig } from "@opennextjs/aws/types/open-next.js"; + +export async function deploy(options: BuildOptions, config: OpenNextConfig) {} diff --git a/packages/cloudflare/src/cli/populate-cache/populate-cache.ts b/packages/cloudflare/src/cli/populate-cache/populate-cache.ts index d886d975e..620cf600f 100644 --- a/packages/cloudflare/src/cli/populate-cache/populate-cache.ts +++ b/packages/cloudflare/src/cli/populate-cache/populate-cache.ts @@ -1,8 +1,7 @@ -import { spawnSync } from "node:child_process"; import { existsSync } from "node:fs"; import path from "node:path"; -import type { BuildOptions } from "@opennextjs/aws/build/helper.js"; +import { BuildOptions } from "@opennextjs/aws/build/helper.js"; import logger from "@opennextjs/aws/logger.js"; import type { IncludedIncrementalCache, @@ -13,7 +12,7 @@ import type { import type { IncrementalCache, TagCache } from "@opennextjs/aws/types/overrides.js"; import { globSync } from "glob"; -export type CacheBindingMode = "local" | "remote"; +import { runWrangler, WranglerTarget } from "../utils/run-wrangler.js"; async function resolveCacheName( value: @@ -25,32 +24,6 @@ async function resolveCacheName( return typeof value === "function" ? (await value()).name : value; } -function runWrangler( - opts: BuildOptions, - wranglerOpts: { mode: CacheBindingMode; excludeRemoteFlag?: boolean }, - args: string[] -) { - const result = spawnSync( - opts.packager, - [ - "exec", - "wrangler", - ...args, - wranglerOpts.mode === "remote" && !wranglerOpts.excludeRemoteFlag && "--remote", - wranglerOpts.mode === "local" && "--local", - ].filter((v): v is string => !!v), - { - shell: true, - stdio: ["ignore", "ignore", "inherit"], - } - ); - - if (result.status !== 0) { - logger.error("Failed to populate cache"); - process.exit(1); - } -} - function getCacheAssetPaths(opts: BuildOptions) { return globSync(path.join(opts.outputDir, "cache/**/*"), { withFileTypes: true }) .filter((f) => f.isFile()) @@ -66,10 +39,14 @@ function getCacheAssetPaths(opts: BuildOptions) { }); } -export async function populateCache(opts: BuildOptions, config: OpenNextConfig, mode: CacheBindingMode) { +export async function populateCache( + options: BuildOptions, + config: OpenNextConfig, + populateCacheOptions: { target: WranglerTarget } +) { const { incrementalCache, tagCache } = config.default.override ?? {}; - if (!existsSync(opts.outputDir)) { + if (!existsSync(options.outputDir)) { logger.error("Unable to populate cache: Open Next build not found"); process.exit(1); } @@ -80,7 +57,7 @@ export async function populateCache(opts: BuildOptions, config: OpenNextConfig, case "r2-incremental-cache": { logger.info("\nPopulating R2 incremental cache..."); - const assets = getCacheAssetPaths(opts); + const assets = getCacheAssetPaths(options); assets.forEach(({ fsPath, destPath }) => { const fullDestPath = path.join( "NEXT_CACHE_R2_BUCKET", @@ -88,7 +65,7 @@ export async function populateCache(opts: BuildOptions, config: OpenNextConfig, destPath ); - runWrangler(opts, { mode, excludeRemoteFlag: true }, [ + runWrangler(options.packager, { ...populateCacheOptions, excludeRemoteFlag: true }, [ "r2 object put", JSON.stringify(fullDestPath), `--file ${JSON.stringify(fsPath)}`, @@ -108,10 +85,10 @@ export async function populateCache(opts: BuildOptions, config: OpenNextConfig, case "d1-tag-cache": { logger.info("\nPopulating D1 tag cache..."); - runWrangler(opts, { mode }, [ + runWrangler(options.packager, populateCacheOptions, [ "d1 execute", "NEXT_CACHE_D1", - `--file ${JSON.stringify(path.join(opts.outputDir, "cloudflare/cache-assets-manifest.sql"))}`, + `--file ${JSON.stringify(path.join(options.outputDir, "cloudflare/cache-assets-manifest.sql"))}`, ]); logger.info("Successfully populated cache"); break; @@ -121,7 +98,3 @@ export async function populateCache(opts: BuildOptions, config: OpenNextConfig, } } } - -export function isCacheBindingMode(v: string | undefined): v is CacheBindingMode { - return !!v && ["local", "remote"].includes(v); -} diff --git a/packages/cloudflare/src/cli/preview/preview.ts b/packages/cloudflare/src/cli/preview/preview.ts new file mode 100644 index 000000000..61afaae13 --- /dev/null +++ b/packages/cloudflare/src/cli/preview/preview.ts @@ -0,0 +1,4 @@ +import { BuildOptions } from "@opennextjs/aws/build/helper.js"; +import { OpenNextConfig } from "@opennextjs/aws/types/open-next.js"; + +export async function preview(options: BuildOptions, config: OpenNextConfig) {} diff --git a/packages/cloudflare/src/cli/project-options.ts b/packages/cloudflare/src/cli/project-options.ts index 15b23259d..bc4cfed5f 100644 --- a/packages/cloudflare/src/cli/project-options.ts +++ b/packages/cloudflare/src/cli/project-options.ts @@ -1,15 +1,8 @@ -import type { CacheBindingMode } from "./build/utils/index.js"; - export type ProjectOptions = { - // Next app root folder - sourceDir: string; - // The directory to save the output to (defaults to the app's directory) - outputDir: string; // Whether the Next.js build should be skipped (i.e. if the `.next` dir is already built) skipNextBuild: boolean; // Whether the check to see if a wrangler config file exists should be skipped skipWranglerConfigCheck: boolean; // Whether minification of the worker should be enabled minify: boolean; - populateCache?: { mode: CacheBindingMode; onlyPopulateWithoutBuilding: boolean }; }; diff --git a/packages/cloudflare/src/cli/utils/run-wrangler.ts b/packages/cloudflare/src/cli/utils/run-wrangler.ts new file mode 100644 index 000000000..e901a5252 --- /dev/null +++ b/packages/cloudflare/src/cli/utils/run-wrangler.ts @@ -0,0 +1,35 @@ +import { spawnSync } from "node:child_process"; + +import logger from "@opennextjs/aws/logger.js"; + +export type WranglerTarget = "local" | "remote"; + +export function runWrangler( + pm: string, + wranglerOpts: { target: WranglerTarget; excludeRemoteFlag?: boolean }, + args: string[] +) { + const result = spawnSync( + pm, + [ + "exec", + "wrangler", + ...args, + wranglerOpts.target === "remote" && !wranglerOpts.excludeRemoteFlag && "--remote", + wranglerOpts.target === "local" && "--local", + ].filter((v): v is string => !!v), + { + shell: true, + stdio: ["ignore", "ignore", "inherit"], + } + ); + + if (result.status !== 0) { + logger.error("Failed to populate cache"); + process.exit(1); + } +} + +export function isWranglerTarget(v: string | undefined): v is WranglerTarget { + return !!v && ["local", "remote"].includes(v); +} From 5ad723ecc6f4bb2ecd1b4190019cc591e316b16c Mon Sep 17 00:00:00 2001 From: James Date: Thu, 20 Mar 2025 10:41:09 +0000 Subject: [PATCH 20/26] add other commands --- .../cloudflare/src/cli/build/utils/index.ts | 1 - packages/cloudflare/src/cli/deploy/deploy.ts | 8 ++++++- packages/cloudflare/src/cli/index.ts | 2 +- .../src/cli/populate-cache/populate-cache.ts | 24 +++++++++++-------- .../cloudflare/src/cli/preview/preview.ts | 8 ++++++- .../cloudflare/src/cli/project-options.ts | 2 ++ .../cloudflare/src/cli/utils/run-wrangler.ts | 11 +++++---- 7 files changed, 37 insertions(+), 19 deletions(-) diff --git a/packages/cloudflare/src/cli/build/utils/index.ts b/packages/cloudflare/src/cli/build/utils/index.ts index e7fb383b6..cca97f023 100644 --- a/packages/cloudflare/src/cli/build/utils/index.ts +++ b/packages/cloudflare/src/cli/build/utils/index.ts @@ -4,4 +4,3 @@ export * from "./ensure-cf-config.js"; export * from "./extract-project-env-vars.js"; export * from "./needs-experimental-react.js"; export * from "./normalize-path.js"; -export * from "./populate-cache.js"; diff --git a/packages/cloudflare/src/cli/deploy/deploy.ts b/packages/cloudflare/src/cli/deploy/deploy.ts index c4bb5b804..6fe807cb4 100644 --- a/packages/cloudflare/src/cli/deploy/deploy.ts +++ b/packages/cloudflare/src/cli/deploy/deploy.ts @@ -1,4 +1,10 @@ import { BuildOptions } from "@opennextjs/aws/build/helper.js"; import { OpenNextConfig } from "@opennextjs/aws/types/open-next.js"; -export async function deploy(options: BuildOptions, config: OpenNextConfig) {} +import { populateCache } from "../populate-cache/populate-cache.js"; +import { runWrangler } from "../utils/run-wrangler.js"; + +export async function deploy(options: BuildOptions, config: OpenNextConfig) { + await populateCache(options, config, { target: "remote" }); + runWrangler(options, ["dev"], { logging: "all" }); +} diff --git a/packages/cloudflare/src/cli/index.ts b/packages/cloudflare/src/cli/index.ts index 345e05346..fcf957934 100644 --- a/packages/cloudflare/src/cli/index.ts +++ b/packages/cloudflare/src/cli/index.ts @@ -36,7 +36,7 @@ async function runCommand(args: Arguments) { switch (args.command) { case "build": - return build(options, config, args); + return build(options, config, { ...args, sourceDir: baseDir }); case "preview": return preview(options, config); case "deploy": diff --git a/packages/cloudflare/src/cli/populate-cache/populate-cache.ts b/packages/cloudflare/src/cli/populate-cache/populate-cache.ts index 620cf600f..896306a43 100644 --- a/packages/cloudflare/src/cli/populate-cache/populate-cache.ts +++ b/packages/cloudflare/src/cli/populate-cache/populate-cache.ts @@ -65,11 +65,11 @@ export async function populateCache( destPath ); - runWrangler(options.packager, { ...populateCacheOptions, excludeRemoteFlag: true }, [ - "r2 object put", - JSON.stringify(fullDestPath), - `--file ${JSON.stringify(fsPath)}`, - ]); + runWrangler( + options, + ["r2 object put", JSON.stringify(fullDestPath), `--file ${JSON.stringify(fsPath)}`], + { ...populateCacheOptions, excludeRemoteFlag: true, logging: "error" } + ); }); logger.info(`Successfully populated cache with ${assets.length} assets`); break; @@ -85,11 +85,15 @@ export async function populateCache( case "d1-tag-cache": { logger.info("\nPopulating D1 tag cache..."); - runWrangler(options.packager, populateCacheOptions, [ - "d1 execute", - "NEXT_CACHE_D1", - `--file ${JSON.stringify(path.join(options.outputDir, "cloudflare/cache-assets-manifest.sql"))}`, - ]); + runWrangler( + options, + [ + "d1 execute", + "NEXT_CACHE_D1", + `--file ${JSON.stringify(path.join(options.outputDir, "cloudflare/cache-assets-manifest.sql"))}`, + ], + { ...populateCacheOptions, logging: "error" } + ); logger.info("Successfully populated cache"); break; } diff --git a/packages/cloudflare/src/cli/preview/preview.ts b/packages/cloudflare/src/cli/preview/preview.ts index 61afaae13..b06489a78 100644 --- a/packages/cloudflare/src/cli/preview/preview.ts +++ b/packages/cloudflare/src/cli/preview/preview.ts @@ -1,4 +1,10 @@ import { BuildOptions } from "@opennextjs/aws/build/helper.js"; import { OpenNextConfig } from "@opennextjs/aws/types/open-next.js"; -export async function preview(options: BuildOptions, config: OpenNextConfig) {} +import { populateCache } from "../populate-cache/populate-cache"; +import { runWrangler } from "../utils/run-wrangler"; + +export async function preview(options: BuildOptions, config: OpenNextConfig) { + await populateCache(options, config, { target: "local" }); + runWrangler(options, ["dev"], { logging: "all" }); +} diff --git a/packages/cloudflare/src/cli/project-options.ts b/packages/cloudflare/src/cli/project-options.ts index bc4cfed5f..867ca2a06 100644 --- a/packages/cloudflare/src/cli/project-options.ts +++ b/packages/cloudflare/src/cli/project-options.ts @@ -1,4 +1,6 @@ export type ProjectOptions = { + // Next app root folder + sourceDir: string; // Whether the Next.js build should be skipped (i.e. if the `.next` dir is already built) skipNextBuild: boolean; // Whether the check to see if a wrangler config file exists should be skipped diff --git a/packages/cloudflare/src/cli/utils/run-wrangler.ts b/packages/cloudflare/src/cli/utils/run-wrangler.ts index e901a5252..8d64952de 100644 --- a/packages/cloudflare/src/cli/utils/run-wrangler.ts +++ b/packages/cloudflare/src/cli/utils/run-wrangler.ts @@ -1,16 +1,17 @@ import { spawnSync } from "node:child_process"; +import type { BuildOptions } from "@opennextjs/aws/build/helper.js"; import logger from "@opennextjs/aws/logger.js"; export type WranglerTarget = "local" | "remote"; export function runWrangler( - pm: string, - wranglerOpts: { target: WranglerTarget; excludeRemoteFlag?: boolean }, - args: string[] + options: BuildOptions, + args: string[], + wranglerOpts: { target?: WranglerTarget; excludeRemoteFlag?: boolean; logging?: "all" | "error" } = {} ) { const result = spawnSync( - pm, + options.packager, [ "exec", "wrangler", @@ -20,7 +21,7 @@ export function runWrangler( ].filter((v): v is string => !!v), { shell: true, - stdio: ["ignore", "ignore", "inherit"], + stdio: wranglerOpts.logging === "error" ? ["ignore", "ignore", "inherit"] : "inherit", } ); From 1aafacd309511b9cdce0011b618f1ae0f103d39a Mon Sep 17 00:00:00 2001 From: James Date: Thu, 20 Mar 2025 11:02:33 +0000 Subject: [PATCH 21/26] update command usage --- examples/bugs/gh-119/package.json | 5 +++-- examples/bugs/gh-219/package.json | 5 +++-- examples/bugs/gh-223/package.json | 5 +++-- examples/common/config-e2e.ts | 4 ++-- examples/create-next-app/package.json | 5 +++-- examples/e2e/app-pages-router/package.json | 5 +++-- examples/e2e/app-router/package.json | 5 +++-- examples/e2e/pages-router/package.json | 5 +++-- examples/middleware/package.json | 5 +++-- examples/next-partial-prerendering/package.json | 5 +++-- examples/playground14/package.json | 5 +++-- examples/playground15/package.json | 5 +++-- examples/ssg-app/package.json | 5 +++-- examples/vercel-blog-starter/package.json | 5 +++-- examples/vercel-commerce/package.json | 5 +++-- packages/cloudflare/README.md | 16 ++++++++-------- packages/cloudflare/src/cli/args.ts | 10 +++++++--- 17 files changed, 59 insertions(+), 41 deletions(-) diff --git a/examples/bugs/gh-119/package.json b/examples/bugs/gh-119/package.json index d5ce02131..a951ae4c1 100644 --- a/examples/bugs/gh-119/package.json +++ b/examples/bugs/gh-119/package.json @@ -7,8 +7,9 @@ "build": "next build", "start": "next start", "lint": "next lint", - "build:worker": "pnpm opennextjs-cloudflare", - "preview": "pnpm build:worker && pnpm wrangler dev", + "build:worker": "pnpm opennextjs-cloudflare build", + "preview:worker": "pnpm opennextjs-cloudflare preview", + "preview": "pnpm build:worker && pnpm preview:worker", "e2e": "playwright test -c e2e/playwright.config.ts", "cf-typegen": "wrangler types --env-interface CloudflareEnv" }, diff --git a/examples/bugs/gh-219/package.json b/examples/bugs/gh-219/package.json index 68f099f70..abaacd4b5 100644 --- a/examples/bugs/gh-219/package.json +++ b/examples/bugs/gh-219/package.json @@ -7,8 +7,9 @@ "build": "next build", "start": "next start", "lint": "next lint", - "build:worker": "opennextjs-cloudflare", - "preview": "pnpm run build:worker && pnpm wrangler dev", + "build:worker": "pnpm opennextjs-cloudflare build", + "preview:worker": "pnpm opennextjs-cloudflare preview", + "preview": "pnpm build:worker && pnpm preview:worker", "e2e": "playwright test -c e2e/playwright.config.ts", "deploy:worker": "pnpm run build:worker && pnpm wrangler deploy" }, diff --git a/examples/bugs/gh-223/package.json b/examples/bugs/gh-223/package.json index 0b6bae934..a7156dc2c 100644 --- a/examples/bugs/gh-223/package.json +++ b/examples/bugs/gh-223/package.json @@ -7,8 +7,9 @@ "build": "next build", "start": "next start", "lint": "next lint", - "build:worker": "opennextjs-cloudflare", - "preview": "pnpm run build:worker && pnpm wrangler dev", + "build:worker": "pnpm opennextjs-cloudflare build", + "preview:worker": "pnpm opennextjs-cloudflare preview", + "preview": "pnpm build:worker && pnpm preview:worker", "e2e": "playwright test -c e2e/playwright.config.ts", "deploy:worker": "pnpm run build:worker && pnpm wrangler deploy" }, diff --git a/examples/common/config-e2e.ts b/examples/common/config-e2e.ts index 17c8cfc0d..95c090724 100644 --- a/examples/common/config-e2e.ts +++ b/examples/common/config-e2e.ts @@ -23,11 +23,11 @@ export function configurePlaywright( if (isWorker) { if (isCI) { // Do not build on CI - there is a preceding build step - command = `pnpm wrangler dev --port ${port} --inspector-port ${inspectorPort}`; + command = `pnpm preview:worker -- --port ${port} --inspector-port ${inspectorPort}`; timeout = 100_000; } else { timeout = 500_000; - command = `pnpm preview --port ${port} --inspector-port ${inspectorPort}`; + command = `pnpm preview -- --port ${port} --inspector-port ${inspectorPort}`; } } else { timeout = 100_000; diff --git a/examples/create-next-app/package.json b/examples/create-next-app/package.json index 53062ea53..55e8b71b1 100644 --- a/examples/create-next-app/package.json +++ b/examples/create-next-app/package.json @@ -7,8 +7,9 @@ "build": "next build", "start": "next start", "lint": "next lint", - "build:worker": "opennextjs-cloudflare", - "preview": "pnpm build:worker && pnpm wrangler dev", + "build:worker": "pnpm opennextjs-cloudflare build", + "preview:worker": "pnpm opennextjs-cloudflare preview", + "preview": "pnpm build:worker && pnpm preview:worker", "e2e": "playwright test -c e2e/playwright.config.ts" }, "dependencies": { diff --git a/examples/e2e/app-pages-router/package.json b/examples/e2e/app-pages-router/package.json index 5e0eec36e..23e906b2b 100644 --- a/examples/e2e/app-pages-router/package.json +++ b/examples/e2e/app-pages-router/package.json @@ -9,8 +9,9 @@ "start": "next start --port 3003", "lint": "next lint", "clean": "rm -rf .turbo node_modules .next .open-next", - "build:worker": "pnpm opennextjs-cloudflare", - "preview": "pnpm build:worker && pnpm wrangler dev", + "build:worker": "pnpm opennextjs-cloudflare build", + "preview:worker": "pnpm opennextjs-cloudflare preview", + "preview": "pnpm build:worker && pnpm preview:worker", "e2e": "playwright test -c e2e/playwright.config.ts" }, "dependencies": { diff --git a/examples/e2e/app-router/package.json b/examples/e2e/app-router/package.json index 9d26a4c6e..5c71e9a51 100644 --- a/examples/e2e/app-router/package.json +++ b/examples/e2e/app-router/package.json @@ -10,8 +10,9 @@ "lint": "next lint", "clean": "rm -rf .turbo node_modules .next .open-next", "d1:clean": "wrangler d1 execute NEXT_CACHE_D1 --command \"DROP TABLE IF EXISTS tags; DROP TABLE IF EXISTS revalidations\"", - "build:worker": "pnpm d1:clean && pnpm opennextjs-cloudflare --populateCache=local", - "preview": "pnpm build:worker && pnpm wrangler dev", + "build:worker": "pnpm d1:clean && pnpm opennextjs-cloudflare build", + "preview:worker": "pnpm opennextjs-cloudflare preview", + "preview": "pnpm build:worker && pnpm preview:worker", "e2e": "playwright test -c e2e/playwright.config.ts" }, "dependencies": { diff --git a/examples/e2e/pages-router/package.json b/examples/e2e/pages-router/package.json index cfcc5b9fd..96b3a8cd1 100644 --- a/examples/e2e/pages-router/package.json +++ b/examples/e2e/pages-router/package.json @@ -9,8 +9,9 @@ "start": "next start --port 3002", "lint": "next lint", "clean": "rm -rf .turbo node_modules .next .open-next", - "build:worker": "pnpm opennextjs-cloudflare", - "preview": "pnpm build:worker && pnpm wrangler dev", + "build:worker": "pnpm opennextjs-cloudflare build", + "preview:worker": "pnpm opennextjs-cloudflare preview", + "preview": "pnpm build:worker && pnpm preview:worker", "e2e": "playwright test -c e2e/playwright.config.ts" }, "dependencies": { diff --git a/examples/middleware/package.json b/examples/middleware/package.json index 33e62aa8b..f712c4dcd 100644 --- a/examples/middleware/package.json +++ b/examples/middleware/package.json @@ -6,8 +6,9 @@ "build": "next build", "start": "next start", "lint": "next lint", - "build:worker": "pnpm opennextjs-cloudflare", - "preview": "pnpm build:worker && pnpm wrangler dev", + "build:worker": "pnpm opennextjs-cloudflare build", + "preview:worker": "pnpm opennextjs-cloudflare preview", + "preview": "pnpm build:worker && pnpm preview:worker", "e2e": "playwright test -c e2e/playwright.config.ts", "e2e:dev": "playwright test -c e2e/playwright.dev.config.ts" }, diff --git a/examples/next-partial-prerendering/package.json b/examples/next-partial-prerendering/package.json index 79582b0e4..1da41fa47 100644 --- a/examples/next-partial-prerendering/package.json +++ b/examples/next-partial-prerendering/package.json @@ -5,8 +5,9 @@ "build": "next build", "dev": "next dev --turbo", "start": "next start", - "build:worker": "opennextjs-cloudflare", - "preview": "pnpm build:worker && pnpm wrangler dev" + "build:worker": "pnpm opennextjs-cloudflare build", + "preview:worker": "pnpm opennextjs-cloudflare preview", + "preview": "pnpm build:worker && pnpm preview:worker" }, "dependencies": { "@heroicons/react": "2.1.5", diff --git a/examples/playground14/package.json b/examples/playground14/package.json index 6dbec953d..8c7d26149 100644 --- a/examples/playground14/package.json +++ b/examples/playground14/package.json @@ -8,8 +8,9 @@ "build": "next build", "start": "next start", "lint": "next lint", - "build:worker": "pnpm opennextjs-cloudflare", - "preview": "pnpm build:worker && pnpm wrangler dev", + "build:worker": "pnpm opennextjs-cloudflare build", + "preview:worker": "pnpm opennextjs-cloudflare preview", + "preview": "pnpm build:worker && pnpm preview:worker", "e2e": "playwright test -c e2e/playwright.config.ts", "e2e:dev": "playwright test -c e2e/playwright.dev.config.ts", "cf-typegen": "wrangler types --env-interface CloudflareEnv" diff --git a/examples/playground15/package.json b/examples/playground15/package.json index 299673955..449e996cf 100644 --- a/examples/playground15/package.json +++ b/examples/playground15/package.json @@ -8,8 +8,9 @@ "build": "next build", "start": "next start", "lint": "next lint", - "build:worker": "pnpm opennextjs-cloudflare", - "preview": "pnpm build:worker && pnpm wrangler dev", + "build:worker": "pnpm opennextjs-cloudflare build", + "preview:worker": "pnpm opennextjs-cloudflare preview", + "preview": "pnpm build:worker && pnpm preview:worker", "e2e": "playwright test -c e2e/playwright.config.ts", "e2e:dev": "playwright test -c e2e/playwright.dev.config.ts", "cf-typegen": "wrangler types --env-interface CloudflareEnv" diff --git a/examples/ssg-app/package.json b/examples/ssg-app/package.json index da7166404..681ded13a 100644 --- a/examples/ssg-app/package.json +++ b/examples/ssg-app/package.json @@ -7,8 +7,9 @@ "build": "next build", "start": "next start", "lint": "next lint", - "build:worker": "opennextjs-cloudflare", - "preview": "pnpm build:worker && pnpm wrangler dev", + "build:worker": "pnpm opennextjs-cloudflare build", + "preview:worker": "pnpm opennextjs-cloudflare preview", + "preview": "pnpm build:worker && pnpm preview:worker", "e2e": "playwright test -c e2e/playwright.config.ts" }, "dependencies": { diff --git a/examples/vercel-blog-starter/package.json b/examples/vercel-blog-starter/package.json index 7c9a4b8d4..5cf746e36 100644 --- a/examples/vercel-blog-starter/package.json +++ b/examples/vercel-blog-starter/package.json @@ -5,8 +5,9 @@ "dev": "next", "build": "next build", "start": "next start", - "build:worker": "opennextjs-cloudflare", - "preview": "pnpm build:worker && pnpm wrangler dev" + "build:worker": "pnpm opennextjs-cloudflare build", + "preview:worker": "pnpm opennextjs-cloudflare preview", + "preview": "pnpm build:worker && pnpm preview:worker" }, "dependencies": { "classnames": "^2.5.1", diff --git a/examples/vercel-commerce/package.json b/examples/vercel-commerce/package.json index e50d63bcb..d64b0d847 100644 --- a/examples/vercel-commerce/package.json +++ b/examples/vercel-commerce/package.json @@ -12,8 +12,9 @@ "prettier": "prettier --write --ignore-unknown .", "prettier:check": "prettier --check --ignore-unknown .", "test": "pnpm prettier:check", - "tofix-build:worker": "opennextjs-cloudflare", - "tofix-preview": "pnpm build:worker && pnpm wrangler dev" + "build:worker": "pnpm opennextjs-cloudflare build", + "preview:worker": "pnpm opennextjs-cloudflare preview", + "preview": "pnpm build:worker && pnpm preview:worker" }, "dependencies": { "@headlessui/react": "^2.1.2", diff --git a/packages/cloudflare/README.md b/packages/cloudflare/README.md index bfc0336ac..cc1f5e72a 100644 --- a/packages/cloudflare/README.md +++ b/packages/cloudflare/README.md @@ -19,13 +19,13 @@ Run the following commands to preview the production build of your application l - build the app and adapt it for Cloudflare ```bash - npx opennextjs-cloudflare + npx opennextjs-cloudflare build # or - pnpm opennextjs-cloudflare + pnpm opennextjs-cloudflare build # or - yarn opennextjs-cloudflare + yarn opennextjs-cloudflare build # or - bun opennextjs-cloudflare + bun opennextjs-cloudflare build ``` - Preview the app in Wrangler @@ -47,11 +47,11 @@ Deploy your application to production with the following: - build the app and adapt it for Cloudflare ```bash - npx opennextjs-cloudflare && npx wrangler deploy + npx opennextjs-cloudflare build && npx opennextjs-cloudflare deploy # or - pnpm opennextjs-cloudflare && pnpm wrangler deploy + pnpm opennextjs-cloudflare build && pnpm opennextjs-cloudflare deploy # or - yarn opennextjs-cloudflare && yarn wrangler deploy + yarn opennextjs-cloudflare build && yarn opennextjs-cloudflare deploy # or - bun opennextjs-cloudflare && bun wrangler deploy + bun opennextjs-cloudflare build && bun opennextjs-cloudflare deploy ``` diff --git a/packages/cloudflare/src/cli/args.ts b/packages/cloudflare/src/cli/args.ts index 7951dc6c5..b2d75ddd6 100644 --- a/packages/cloudflare/src/cli/args.ts +++ b/packages/cloudflare/src/cli/args.ts @@ -11,7 +11,7 @@ export type Arguments = ( skipWranglerConfigCheck: boolean; minify: boolean; } - | { command: "preview" | "deploy" } + | { command: "preview" | "deploy"; passthroughArgs: string[] } | { command: "populateCache"; target: WranglerTarget } ) & { outputDir?: string }; @@ -42,9 +42,8 @@ export function getArgs(): Arguments { minify: !values.noMinify, }; case "preview": - return { command: "preview", outputDir }; case "deploy": - return { command: "preview", outputDir }; + return { command: positionals[0], outputDir, passthroughArgs: getPassthroughArgs() }; case "populateCache": if (!isWranglerTarget(positionals[1])) { throw new Error(`Error: invalid target for populating the cache, expected 'local' | 'remote'`); @@ -55,6 +54,11 @@ export function getArgs(): Arguments { } } +function getPassthroughArgs() { + const passthroughPos = process.argv.indexOf("--"); + return passthroughPos === -1 ? [] : process.argv.slice(passthroughPos + 1); +} + function assertDirArg(path: string, argName?: string, make?: boolean) { let dirStats: Stats; try { From 655d4605ad11b20f7053db06ea0d43e29fda8059 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 20 Mar 2025 11:05:35 +0000 Subject: [PATCH 22/26] changeset --- .changeset/silly-jokes-hammer.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .changeset/silly-jokes-hammer.md diff --git a/.changeset/silly-jokes-hammer.md b/.changeset/silly-jokes-hammer.md new file mode 100644 index 000000000..4adf4f121 --- /dev/null +++ b/.changeset/silly-jokes-hammer.md @@ -0,0 +1,12 @@ +--- +"@opennextjs/cloudflare": minor +--- + +feat: commands for cli actions + +The OpenNext Cloudflare CLI now uses the following commands; + +- `build`: build the application +- `populateCache`: populate either the local or remote cache +- `preview`: populate the local cache and start a dev server +- `deploy`: populate the remote cache and deploy to production From 64baa1ac20d974ce902a12d60ce6fd79d59f8e6d Mon Sep 17 00:00:00 2001 From: James Date: Thu, 20 Mar 2025 11:14:12 +0000 Subject: [PATCH 23/26] fix import --- examples/vercel-commerce/package.json | 6 +++--- packages/cloudflare/src/cli/args.ts | 2 +- packages/cloudflare/src/cli/preview/preview.ts | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/vercel-commerce/package.json b/examples/vercel-commerce/package.json index d64b0d847..6341cb5dd 100644 --- a/examples/vercel-commerce/package.json +++ b/examples/vercel-commerce/package.json @@ -12,9 +12,9 @@ "prettier": "prettier --write --ignore-unknown .", "prettier:check": "prettier --check --ignore-unknown .", "test": "pnpm prettier:check", - "build:worker": "pnpm opennextjs-cloudflare build", - "preview:worker": "pnpm opennextjs-cloudflare preview", - "preview": "pnpm build:worker && pnpm preview:worker" + "tofix-build:worker": "pnpm opennextjs-cloudflare build", + "tofix-preview:worker": "pnpm opennextjs-cloudflare preview", + "tofix-preview": "pnpm build:worker && pnpm preview:worker" }, "dependencies": { "@headlessui/react": "^2.1.2", diff --git a/packages/cloudflare/src/cli/args.ts b/packages/cloudflare/src/cli/args.ts index b2d75ddd6..1dbec4f24 100644 --- a/packages/cloudflare/src/cli/args.ts +++ b/packages/cloudflare/src/cli/args.ts @@ -2,7 +2,7 @@ import { mkdirSync, type Stats, statSync } from "node:fs"; import { resolve } from "node:path"; import { parseArgs } from "node:util"; -import { isWranglerTarget, WranglerTarget } from "./utils/run-wrangler"; +import { isWranglerTarget, WranglerTarget } from "./utils/run-wrangler.js"; export type Arguments = ( | { diff --git a/packages/cloudflare/src/cli/preview/preview.ts b/packages/cloudflare/src/cli/preview/preview.ts index b06489a78..58dd21bd4 100644 --- a/packages/cloudflare/src/cli/preview/preview.ts +++ b/packages/cloudflare/src/cli/preview/preview.ts @@ -1,8 +1,8 @@ import { BuildOptions } from "@opennextjs/aws/build/helper.js"; import { OpenNextConfig } from "@opennextjs/aws/types/open-next.js"; -import { populateCache } from "../populate-cache/populate-cache"; -import { runWrangler } from "../utils/run-wrangler"; +import { populateCache } from "../populate-cache/populate-cache.js"; +import { runWrangler } from "../utils/run-wrangler.js"; export async function preview(options: BuildOptions, config: OpenNextConfig) { await populateCache(options, config, { target: "local" }); From 2247ed4b93e0ee662a487e73275296eb80787b95 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 20 Mar 2025 15:05:52 +0000 Subject: [PATCH 24/26] add missing passthrough args --- packages/cloudflare/src/cli/deploy/deploy.ts | 8 ++++++-- packages/cloudflare/src/cli/index.ts | 6 +++--- packages/cloudflare/src/cli/preview/preview.ts | 8 ++++++-- packages/cloudflare/src/cli/utils/run-wrangler.ts | 2 +- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/packages/cloudflare/src/cli/deploy/deploy.ts b/packages/cloudflare/src/cli/deploy/deploy.ts index 6fe807cb4..31f714778 100644 --- a/packages/cloudflare/src/cli/deploy/deploy.ts +++ b/packages/cloudflare/src/cli/deploy/deploy.ts @@ -4,7 +4,11 @@ import { OpenNextConfig } from "@opennextjs/aws/types/open-next.js"; import { populateCache } from "../populate-cache/populate-cache.js"; import { runWrangler } from "../utils/run-wrangler.js"; -export async function deploy(options: BuildOptions, config: OpenNextConfig) { +export async function deploy( + options: BuildOptions, + config: OpenNextConfig, + deployOptions: { passthroughArgs: string[] } +) { await populateCache(options, config, { target: "remote" }); - runWrangler(options, ["dev"], { logging: "all" }); + runWrangler(options, ["deploy", ...deployOptions.passthroughArgs], { logging: "all" }); } diff --git a/packages/cloudflare/src/cli/index.ts b/packages/cloudflare/src/cli/index.ts index fcf957934..9c966807e 100644 --- a/packages/cloudflare/src/cli/index.ts +++ b/packages/cloudflare/src/cli/index.ts @@ -38,11 +38,11 @@ async function runCommand(args: Arguments) { case "build": return build(options, config, { ...args, sourceDir: baseDir }); case "preview": - return preview(options, config); + return preview(options, config, args); case "deploy": - return deploy(options, config); + return deploy(options, config, args); case "populateCache": - return populateCache(options, config, { target: args.target }); + return populateCache(options, config, args); } } diff --git a/packages/cloudflare/src/cli/preview/preview.ts b/packages/cloudflare/src/cli/preview/preview.ts index 58dd21bd4..b4df9d0d1 100644 --- a/packages/cloudflare/src/cli/preview/preview.ts +++ b/packages/cloudflare/src/cli/preview/preview.ts @@ -4,7 +4,11 @@ import { OpenNextConfig } from "@opennextjs/aws/types/open-next.js"; import { populateCache } from "../populate-cache/populate-cache.js"; import { runWrangler } from "../utils/run-wrangler.js"; -export async function preview(options: BuildOptions, config: OpenNextConfig) { +export async function preview( + options: BuildOptions, + config: OpenNextConfig, + previewOptions: { passthroughArgs: string[] } +) { await populateCache(options, config, { target: "local" }); - runWrangler(options, ["dev"], { logging: "all" }); + runWrangler(options, ["dev", ...previewOptions.passthroughArgs], { logging: "all" }); } diff --git a/packages/cloudflare/src/cli/utils/run-wrangler.ts b/packages/cloudflare/src/cli/utils/run-wrangler.ts index 8d64952de..6852d7b61 100644 --- a/packages/cloudflare/src/cli/utils/run-wrangler.ts +++ b/packages/cloudflare/src/cli/utils/run-wrangler.ts @@ -26,7 +26,7 @@ export function runWrangler( ); if (result.status !== 0) { - logger.error("Failed to populate cache"); + logger.error("Wrangler command failed"); process.exit(1); } } From 631ce2931da69aae5b0c56b3a85c4de4d187cec2 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 21 Mar 2025 11:24:36 +0000 Subject: [PATCH 25/26] update projects --- examples/overrides/d1-tag-next/package.json | 5 +++-- examples/overrides/memory-queue/package.json | 5 +++-- examples/overrides/r2-incremental-cache/package.json | 5 +++-- packages/cloudflare/src/cli/build/utils/index.ts | 1 - packages/cloudflare/src/cli/populate-cache/populate-cache.ts | 5 +++-- packages/cloudflare/src/cli/project-options.ts | 4 ++-- 6 files changed, 14 insertions(+), 11 deletions(-) diff --git a/examples/overrides/d1-tag-next/package.json b/examples/overrides/d1-tag-next/package.json index 01bdf5662..7909151ba 100644 --- a/examples/overrides/d1-tag-next/package.json +++ b/examples/overrides/d1-tag-next/package.json @@ -9,8 +9,9 @@ "lint": "next lint", "d1:clean": "wrangler d1 execute NEXT_CACHE_D1 --command \"DROP TABLE IF EXISTS revalidations\"", "d1:setup": "wrangler d1 execute NEXT_CACHE_D1 --command \"CREATE TABLE IF NOT EXISTS revalidations (tag TEXT NOT NULL, revalidatedAt INTEGER NOT NULL, UNIQUE(tag) ON CONFLICT REPLACE);\"", - "build:worker": "opennextjs-cloudflare && pnpm d1:clean && pnpm d1:setup", - "preview": "pnpm build:worker && pnpm wrangler dev", + "build:worker": "pnpm d1:clean && pnpm opennextjs-cloudflare build && pnpm d1:setup", + "preview:worker": "pnpm opennextjs-cloudflare preview", + "preview": "pnpm build:worker && pnpm preview:worker", "e2e": "playwright test -c e2e/playwright.config.ts" }, "dependencies": { diff --git a/examples/overrides/memory-queue/package.json b/examples/overrides/memory-queue/package.json index 0ed10a102..cfc9a5cff 100644 --- a/examples/overrides/memory-queue/package.json +++ b/examples/overrides/memory-queue/package.json @@ -7,8 +7,9 @@ "build": "next build", "start": "next start", "lint": "next lint", - "build:worker": "opennextjs-cloudflare", - "preview": "pnpm build:worker && pnpm wrangler dev", + "build:worker": "pnpm opennextjs-cloudflare build", + "preview:worker": "pnpm opennextjs-cloudflare preview", + "preview": "pnpm build:worker && pnpm preview:worker", "e2e": "playwright test -c e2e/playwright.config.ts" }, "dependencies": { diff --git a/examples/overrides/r2-incremental-cache/package.json b/examples/overrides/r2-incremental-cache/package.json index 8eda7f825..f6bd17255 100644 --- a/examples/overrides/r2-incremental-cache/package.json +++ b/examples/overrides/r2-incremental-cache/package.json @@ -8,8 +8,9 @@ "start": "next start", "lint": "next lint", "d1:clean": "wrangler d1 execute NEXT_CACHE_D1 --command \"DROP TABLE IF EXISTS tags; DROP TABLE IF EXISTS revalidations\"", - "build:worker": "pnpm d1:clean && pnpm opennextjs-cloudflare --populateCache=local", - "preview": "pnpm build:worker && pnpm wrangler dev", + "build:worker": "pnpm d1:clean && pnpm opennextjs-cloudflare build", + "preview:worker": "pnpm opennextjs-cloudflare preview", + "preview": "pnpm build:worker && pnpm preview:worker", "e2e": "playwright test -c e2e/playwright.config.ts" }, "dependencies": { diff --git a/packages/cloudflare/src/cli/build/utils/index.ts b/packages/cloudflare/src/cli/build/utils/index.ts index e7fb383b6..cca97f023 100644 --- a/packages/cloudflare/src/cli/build/utils/index.ts +++ b/packages/cloudflare/src/cli/build/utils/index.ts @@ -4,4 +4,3 @@ export * from "./ensure-cf-config.js"; export * from "./extract-project-env-vars.js"; export * from "./needs-experimental-react.js"; export * from "./normalize-path.js"; -export * from "./populate-cache.js"; diff --git a/packages/cloudflare/src/cli/populate-cache/populate-cache.ts b/packages/cloudflare/src/cli/populate-cache/populate-cache.ts index 527ad2ae6..949a04f73 100644 --- a/packages/cloudflare/src/cli/populate-cache/populate-cache.ts +++ b/packages/cloudflare/src/cli/populate-cache/populate-cache.ts @@ -1,7 +1,7 @@ import { existsSync } from "node:fs"; import path from "node:path"; -import { BuildOptions } from "@opennextjs/aws/build/helper.js"; +import type { BuildOptions } from "@opennextjs/aws/build/helper.js"; import logger from "@opennextjs/aws/logger.js"; import type { IncludedIncrementalCache, @@ -12,7 +12,8 @@ import type { import type { IncrementalCache, TagCache } from "@opennextjs/aws/types/overrides.js"; import { globSync } from "glob"; -import { runWrangler, WranglerTarget } from "../utils/run-wrangler.js"; +import type { WranglerTarget } from "../utils/run-wrangler.js"; +import { runWrangler } from "../utils/run-wrangler.js"; async function resolveCacheName( value: diff --git a/packages/cloudflare/src/cli/project-options.ts b/packages/cloudflare/src/cli/project-options.ts index 3176995c2..7e41d8baf 100644 --- a/packages/cloudflare/src/cli/project-options.ts +++ b/packages/cloudflare/src/cli/project-options.ts @@ -1,4 +1,4 @@ -import type { CacheBindingMode } from "./build/utils/index.js"; +import type { WranglerTarget } from "./utils/run-wrangler.js"; export type ProjectOptions = { // Next app root folder @@ -9,5 +9,5 @@ export type ProjectOptions = { skipWranglerConfigCheck: boolean; // Whether minification of the worker should be enabled minify: boolean; - populateCache?: { mode: CacheBindingMode; onlyPopulateWithoutBuilding: boolean }; + populateCache?: { mode: WranglerTarget; onlyPopulateWithoutBuilding: boolean }; }; From a147d77a2a1c79b5614b79e14dc8680d449db104 Mon Sep 17 00:00:00 2001 From: James Anderson Date: Fri, 21 Mar 2025 14:04:54 +0000 Subject: [PATCH 26/26] Update examples/e2e/app-router/wrangler.jsonc --- examples/e2e/app-router/wrangler.jsonc | 7 ------- 1 file changed, 7 deletions(-) diff --git a/examples/e2e/app-router/wrangler.jsonc b/examples/e2e/app-router/wrangler.jsonc index 8c50442fa..abb7ff277 100644 --- a/examples/e2e/app-router/wrangler.jsonc +++ b/examples/e2e/app-router/wrangler.jsonc @@ -44,12 +44,5 @@ "binding": "NEXT_CACHE_REVALIDATION_WORKER", "service": "app-router" } - ], - "r2_buckets": [ - { - "binding": "NEXT_CACHE_R2_BUCKET", - "bucket_name": "NEXT_CACHE_R2_BUCKET", - "preview_bucket_name": "NEXT_CACHE_R2_BUCKET" - } ] }