reimplement getMarkdownFiles in next.helpers.mjs to be generated at build time

Author: dario-piotrowicz

.gitignore

@@ -36,3 +36,5 @@ dist/
 # Ignore worker artifacts
 apps/site/.open-next
 apps/site/.wrangler
+
+.next.helpers.mjs

apps/site/.cloudflare/node/fs.mjs

@@ -1,4 +1,4 @@
-import { files } from '../../next.helpers.mjs';
+import { files } from '../../.next.helpers.mjs';
 
 export function readdir(params, cb) {
   console.log('fs#readdir', params);

apps/site/CLOUDFLARE.md

@@ -72,8 +72,9 @@ You can monitor and configure the project at https://dash.cloudflare.com/fb4a2d0
 The following is an incomplete list of tasks and problems that still need to be resolved:
 
 - [ ] sort out issues with `eval` and MDX and undo edits in `./app/[locale]/[[...path]]/page.tsx`
-- [ ] reimplement `getMarkdownFiles` in `next.helpers.mjs` to be generated at build time
-  - this can be accomplished either via a npm/turbo prebuild task, or possibly as part of next.js SSG/staticProps (but we need to ensure that we don't end up accidentaly downloading this big file to the client as part of hydration)
+- [x] reimplement `getMarkdownFiles` in `next.helpers.mjs` to be generated at build time
+  - this can be accomplished either via a npm/turbo prebuild task, or possibly as part of next.js SSG/staticProps but
+  - [ ] we need to ensure that we don't end up accidentally downloading this big file to the client as part of hydration
   - [ ] once we have easy access to the list of files, we should roll back changes to `next-data/providers/blogData.ts`
 - [ ] back out most changes from `next.dynamic.mjs`
   - [ ] instead of using runtime detection via `globalThis.navigator?.userAgent`, we should instead use `alias` feature in `wrangler.toml` to override the implementation of `node:fs` calls but only when running in workerd as we need the build to keep on running in node.js for SSG to work

apps/site/next-data/generators/blogData.mjs

@@ -6,7 +6,7 @@ import readline from 'node:readline';
 
 import graymatter from 'gray-matter';
 
-import { getMarkdownFiles } from '../../next.helpers.mjs';
+import { getMarkdownFiles } from '../../.next.helpers.mjs';
 
 // gets the current blog path based on local module path
 const blogPath = join(process.cwd(), 'pages/en/blog');

apps/site/next.dynamic.mjs

@@ -10,13 +10,13 @@ import { VFile } from 'vfile';
 
 import { readFile as runtimeReadFile } from './.cloudflare/node/fs/promises.mjs';
 import { exists as runtimeExists } from './.cloudflare/node/fs.mjs';
+import { getMarkdownFiles } from './.next.helpers.mjs';
 import { BASE_URL, BASE_PATH, IS_DEVELOPMENT } from './next.constants.mjs';
 import {
   IGNORED_ROUTES,
   DYNAMIC_ROUTES,
   PAGE_METADATA,
 } from './next.dynamic.constants.mjs';
-import { getMarkdownFiles } from './next.helpers.mjs';
 import { siteConfig } from './next.json.mjs';
 import { availableLocaleCodes, defaultLocale } from './next.locales.mjs';
 import { compileMDX } from './next.mdx.compiler.mjs';

apps/site/next.helpers.mjs

@@ -19,6 +19,7 @@
     "scripts:release-post": "cross-env NODE_NO_WARNINGS=1 node scripts/release-post/index.mjs",
     "dev": "cross-env NODE_NO_WARNINGS=1 next dev --turbo",
     "serve": "npm run dev",
+    "prebuild": "node scripts/build.helpers.mjs",
     "build": "cross-env NODE_NO_WARNINGS=1 next build",
     "start": "cross-env NODE_NO_WARNINGS=1 next start",
     "deploy": "cross-env NEXT_PUBLIC_STATIC_EXPORT=true npm run build",

apps/site/package.json

@@ -0,0 +1,28 @@
+'use strict';
+
+import { readFileSync, writeFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+
+import { glob } from 'glob';
+
+import { getRelativePath } from '../next.helpers.mjs';
+
+const __dirname = getRelativePath(import.meta.url);
+const nextHelpersMjs = readFileSync(
+    resolve(__dirname, '..', 'next.helpers.mjs'),
+    'utf8'
+);
+
+const files = await glob('**/*.{md,mdx}', { root: 'pages' });
+
+const outputNextHelpersMjs = nextHelpersMjs.replace(
+    /export const files = \[\s*\/\* generated at build time \*\/\s*\];/,
+    `export const files = [\n${files
+        .map(file => `  ${JSON.stringify(file)},\n`)
+        .join('')}];`
+);
+
+writeFileSync(
+    resolve(__dirname, '..', '.next.helpers.mjs'),
+    outputNextHelpersMjs
+);

apps/site/scripts/build.helpers.mjs

@@ -4,7 +4,7 @@ import zlib from 'node:zlib';
 
 import { slug } from 'github-slugger';
 
-import { getRelativePath } from '../../next.helpers.mjs';
+import { getRelativePath } from '../../.next.helpers.mjs';
 
 const currentRoot = getRelativePath(import.meta.url);
 const dataBasePath = join(currentRoot, '../../.next/server/app/en/next-data');

apps/site/scripts/orama-search/get-documents.mjs

@@ -29,7 +29,7 @@ import { format } from 'prettier';
 
 import { downloadsTable } from './downloadsTable.mjs';
 import prettierConfig from '../../../../.prettierrc.json' with { type: 'json' };
-import { getRelativePath } from '../../next.helpers.mjs';
+import { getRelativePath } from '../../.next.helpers.mjs';
 
 const URLS = {
   NODE_DIST_JSON: 'https://nodejs.org/dist/index.json',