zotero-plugin-dev · northword · Apr 19, 2025 · Apr 19, 2025 · Apr 19, 2025 · Apr 19, 2025
diff --git a/packages/scaffold/package.json b/packages/scaffold/package.json
@@ -69,6 +69,7 @@
     }
   },
   "dependencies": {
+    "@fluent/syntax": "^0.19.0",
     "@swc/core": "^1.11.16",
     "adm-zip": "^0.5.16",
     "bumpp": "^10.1.0",

diff --git a/packages/scaffold/src/core/builder.ts b/packages/scaffold/src/core/builder.ts
@@ -2,20 +2,21 @@ import type { Context } from "../types/index.js";
 import type { Manifest } from "../types/manifest.js";
 import type { UpdateJSON } from "../types/update-json.js";
 import { existsSync } from "node:fs";
-import { readFile, writeFile } from "node:fs/promises";
-import { basename, dirname, join, resolve } from "node:path";
+import { readFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
 import process from "node:process";
 import AdmZip from "adm-zip";
-import { escapeRegExp, toMerged } from "es-toolkit";
+import { toMerged } from "es-toolkit";
 import { build as buildAsync } from "esbuild";
-import { copy, emptyDir, move, outputFile, outputJSON, readJSON, writeJson } from "fs-extra/esm";
+import { copy, emptyDir, outputFile, outputJSON, readJSON, writeJson } from "fs-extra/esm";
 import styleText from "node-style-text";
 import { glob } from "tinyglobby";
 import { generateHash } from "../utils/crypto.js";
 import { is32BitNumber } from "../utils/number.js";
 import { PrefsManager, renderPluginPrefsDts } from "../utils/prefs-manager.js";
 import { dateFormat, replaceInFile, toArray } from "../utils/string.js";
 import { Base } from "./base.js";
+import buildLocale from "./builder/fluent.js";
 
 export default class Build extends Base {
   private buildTime: string;
@@ -145,109 +146,7 @@ export default class Build extends Base {
 
   async prepareLocaleFiles() {
     const { dist, namespace, build } = this.ctx;
-
-    const ignores = toArray(build.fluent.ignore);
-
-    // https://regex101.com/r/lQ9x5p/1
-    // eslint-disable-next-line regexp/no-super-linear-backtracking
-    const FTL_MESSAGE_PATTERN = /^(?<message>[a-z]\S*)( *= *)(?<pattern>.*)$/gim;
-    const HTML_DATAI10NID_PATTERN = new RegExp(`(data-l10n-id)="((?!${namespace})\\S*)"`, "g");
-
-    // Get locale names
-    const localePaths = await glob(`${dist}/addon/locale/*`, { onlyDirectories: true });
-    const localeNames = localePaths.map(locale => basename(locale));
-    this.logger.debug(`Locale names:", ${localeNames}`);
-
-    const allMessages = new Set<string>();
-    const messagesByLocale = new Map<string, Set<string>>();
-
-    for (const localeName of localeNames) {
-      // Prefix Fluent messages in each ftl, add message to set.
-      const localeMessages = new Set<string>();
-      const ftlPaths = await glob(`${dist}/addon/locale/${localeName}/**/*.ftl`);
-
-      await Promise.all(ftlPaths.map(async (ftlPath: string) => {
-        let ftlContent = await readFile(ftlPath, "utf-8");
-        const matches = [...ftlContent.matchAll(FTL_MESSAGE_PATTERN)];
-
-        for (const match of matches) {
-          const [matched, message, _pattern] = match;
-          if (message) {
-            localeMessages.add(message);
-            allMessages.add(message);
-            ftlContent = ftlContent.replace(new RegExp(`^${escapeRegExp(matched)}`, "gm"), `${namespace}-${matched}`);
-          }
-        }
-
-        // If prefixFluentMessages===true, we save the changed ftl file,
-        // otherwise discard the changes
-        if (build.fluent.prefixFluentMessages)
-          await writeFile(ftlPath, ftlContent);
-
-        // rename *.ftl to addonRef-*.ftl
-        if (build.fluent.prefixLocaleFiles === true) {
-          await move(ftlPath, `${dirname(ftlPath)}/${namespace}-${basename(ftlPath)}`);
-          this.logger.debug(`FTL file '${ftlPath}' is renamed to '${namespace}-${basename(ftlPath)}'.`);
-        }
-      }));
-
-      messagesByLocale.set(localeName, localeMessages);
-    }
-
-    // Prefix Fluent messages in xhtml
-    const messagesInHTML = new Set<string>();
-    const htmlPaths = await glob([
-      `${dist}/addon/**/*.xhtml`,
-      `${dist}/addon/**/*.html`,
-    ]);
-    await Promise.all(htmlPaths.map(async (htmlPath) => {
-      let htmlContent = await readFile(htmlPath, "utf-8");
-      const matches = [...htmlContent.matchAll(HTML_DATAI10NID_PATTERN)];
-
-      for (const match of matches) {
-        const [matched, attrKey, attrVal] = match;
-
-        if (ignores.includes(attrVal)) {
-          this.logger.debug(`HTML data-i10n-id ${attrVal} is in ignore list, skip to namespace`);
-          continue;
-        }
-
-        if (!allMessages.has(attrVal)) {
-          this.logger.warn(`HTML data-i10n-id '${styleText.blue(attrVal)}' in ${styleText.gray(htmlPath)} do not exist in any FTL message, skip to namespace it.`);
-          continue;
-        }
-
-        messagesInHTML.add(attrVal);
-        const namespacedAttr = `${namespace}-${attrVal}`;
-        htmlContent = htmlContent.replace(matched, `${attrKey}="${namespacedAttr}"`);
-        this.logger.debug(`HTML data-i10n-id '${styleText.blue(attrVal)}' in ${styleText.gray(htmlPath)} is namespaced to ${styleText.blue(namespacedAttr)}.`);
-      }
-
-      if (build.fluent.prefixFluentMessages)
-        await writeFile(htmlPath, htmlContent);
-    }));
-
-    // Check miss 1: Cross check in diff locale - seems no need
-    // messagesByLocale.forEach((messageInThisLang, lang) => {
-    //   // Needs Nodejs 22
-    //   const diff = allMessages.difference(messageInThisLang);
-    //   if (diff.size)
-    //     this.logger.warn(`FTL messages '${Array.from(diff).join(", ")}' don't exist the locale '${lang}'`);
-    // });
-
-    // Check miss 2: Check ids in HTML but not in ftl
-    messagesInHTML.forEach((messageInHTML) => {
-      if (ignores.includes(messageInHTML))
-        return;
-
-      const missingLocales = [...messagesByLocale.entries()]
-        .filter(([_, messages]) => !messages.has(messageInHTML))
-        .map(([locale]) => locale);
-
-      if (missingLocales.length > 0) {
-        this.logger.warn(`HTML data-l10n-id '${styleText.blue(messageInHTML)}' is missing in locales: ${missingLocales.join(", ")}.`);
-      }
-    });
+    await buildLocale(dist, namespace, build.fluent);
   }
 
   async preparePrefs() {

diff --git a/packages/scaffold/src/core/builder/fluent.ts b/packages/scaffold/src/core/builder/fluent.ts
@@ -0,0 +1,242 @@
+import type { BaseNode, Message, MessageReference, Resource } from "@fluent/syntax";
+import type { BuildConfig } from "../../types/config.js";
+import { readFile, writeFile } from "node:fs/promises";
+import { basename, dirname } from "node:path";
+import { parse, serialize, Transformer } from "@fluent/syntax";
+import { move } from "fs-extra/esm";
+import styleText from "node-style-text";
+import { glob } from "tinyglobby";
+import { logger } from "../../utils/logger.js";
+import { toArray } from "../../utils/string.js";
+
+export default async function buildLocale(
+  dist: string,
+  namespace: string,
+  options: BuildConfig["fluent"],
+) {
+  const ignores = toArray(options.ignore);
+  const localeNames = await getLocales(dist);
+  const messageManager = new MessageManager(ignores);
+
+  // Process FTL files and add messages to the manager
+  await Promise.all(localeNames.map(async (locale) => {
+    const paths = await glob(`${dist}/addon/locale/${locale}/**/*.ftl`);
+    await Promise.all(paths.map(async (path) => {
+      const fm = new FluentManager();
+      await fm.read(path);
+      messageManager.addMessages(locale, fm.getMessages());
+
+      if (options.prefixFluentMessages) {
+        fm.prefixMessages(namespace);
+        await fm.write(path);
+      }
+
+      if (options.prefixLocaleFiles) {
+        const newPath = `${dirname(path)}/${namespace}-${basename(path)}`;
+        await move(path, newPath);
+        logger.debug(`Renamed FTL: ${path} → ${newPath}`);
+      }
+    }));
+  }));
+
+  // Process HTML files and add messages to the manager
+  const htmlPaths = await glob([`${dist}/addon/**/*.xhtml`, `${dist}/addon/**/*.html`]);
+  await Promise.all(htmlPaths.map(async (htmlPath) => {
+    const content = await readFile(htmlPath, "utf-8");
+    const { processedContent, foundMessages } = processHTMLFile(
+      content,
+      namespace,
+      messageManager.getFTLMessages(),
+      ignores,
+      htmlPath,
+    );
+
+    // Add all found HTML messages
+    messageManager.addMessages("html", foundMessages);
+
+    if (options.prefixFluentMessages) {
+      await writeFile(htmlPath, processedContent);
+    }
+  }));
+
+  // Validate that all HTML messages exist in all locales
+  messageManager.validateMessages();
+}
+
+async function getLocales(dist: string): Promise<string[]> {
+  const localePaths = await glob(`${dist}/addon/locale/*`, { onlyDirectories: true });
+  return localePaths.map(p => basename(p));
+}
+
+export class FluentManager {
+  private source?: string;
+  private resource?: Resource;
+  public readonly messages: string[] = [];
+
+  constructor() {}
+
+  // Parse Fluent source into an AST and extract messages
+  public parse(source: string) {
+    this.source = source;
+    this.resource = parse(source, {});
+  }
+
+  // Read a file, parse its content, and extract messages
+  public async read(path: string) {
+    const content = await readFile(path, "utf-8");
+    this.parse(content);
+  }
+
+  // Extract message IDs from the parsed resource
+  public getMessages(): string[] {
+    if (!this.resource) {
+      throw new Error("Resource must be parsed first.");
+    }
+    this.messages.length = 0; // Clear the previous messages
+    this.messages.push(
+      ...this.resource.body.filter(entry => entry.type === "Message")
+        .map(message => message.id.name),
+    );
+    return this.messages;
+  }
+
+  // Apply namespace prefix to message IDs in the resource
+  public prefixMessages(namespace: string) {
+    if (!this.resource) {
+      throw new Error("Resource must be parsed before applying prefix.");
+    }
+    new FluentTransformer(namespace).genericVisit(this.resource);
+  }
+
+  // Serialize the resource back into a string
+  public serialize(): string {
+    if (!this.resource) {
+      throw new Error("Resource not parsed. Cannot serialize.");
+    }
+    return serialize(this.resource, {});
+  }
+
+  // Write the serialized resource to a file
+  public async write(path: string) {
+    const result = this.serialize();
+    if (result !== this.source)
+      await writeFile(path, this.serialize());
+  }
+}
+// Custom Fluent AST transformer to apply message ID prefix
+class FluentTransformer extends Transformer {
+  constructor(private readonly prefix: string | false) {
+    super();
+  }
+
+  private needsPrefix(name: string): boolean {
+    return !!this.prefix && !name.startsWith(this.prefix);
+  }
+
+  visitMessage(node: Message): BaseNode {
+    if (this.needsPrefix(node.id.name)) {
+      node.id.name = `${this.prefix}-${node.id.name}`;
+    }
+    return this.genericVisit(node);
+  }
+
+  visitMessageReference(node: MessageReference): BaseNode {
+    if (this.needsPrefix(node.id.name)) {
+      node.id.name = `${this.prefix}-${node.id.name}`;
+    }
+    return this.genericVisit(node);
+  }
+}
+
+export class MessageManager {
+  private ftlMessages: Map<string, Set<string>> = new Map();
+  private htmlMessages: Set<string> = new Set();
+  private ignores: string[];
+
+  constructor(ignores: string[]) {
+    this.ignores = ignores;
+  }
+
+  // Add a set of messages (FTL or HTML) for a specific locale or for HTML globally
+  addMessages(target: string | "html", messages: string[]) {
+    if (target === "html") {
+      messages.forEach(msg => this.htmlMessages.add(msg));
+    }
+    else {
+      let ftlLocaleMessages = this.ftlMessages.get(target);
+      if (!ftlLocaleMessages) {
+        ftlLocaleMessages = new Set();
+        this.ftlMessages.set(target, ftlLocaleMessages);
+      }
+      messages.forEach(msg => ftlLocaleMessages.add(msg));
+    }
+  }
+
+  validateMessages() {
+    // Check miss 1: Cross check in diff locale - seems no need
+    // messagesByLocale.forEach((messageInThisLang, lang) => {
+    //   // Needs Nodejs 22
+    //   const diff = allMessages.difference(messageInThisLang);
+    //   if (diff.size)
+    //     this.logger.warn(`FTL messages '${Array.from(diff).join(", ")}' don't exist the locale '${lang}'`);
+    // });
+
+    // Check miss 2: Check ids in HTML but not in ftl
+    this.htmlMessages.forEach((msg) => {
+      if (this.ignores.includes(msg))
+        return;
+
+      const missingLocales = [...this.ftlMessages.entries()]
+        .filter(([_, messages]) => !messages.has(msg))
+        .map(([locale]) => locale);
+      if (missingLocales.length > 0)
+        logger.warn(`I10N id ${styleText.blue(msg)} missing in locale: ${missingLocales.join(", ")}`);
+    });
+  }
+
+  getFTLMessages(): Set<string> {
+    const allMessages = new Set<string>();
+    this.ftlMessages.forEach(messages => messages.forEach(msg => allMessages.add(msg)));
+    return allMessages;
+  }
+
+  // Get all FTL messages for a specific locale
+  getFTLMessagesByLocale(locale: string): Set<string> {
+    return this.ftlMessages.get(locale) || new Set();
+  }
+
+  // Get all HTML messages
+  getHTMLMessages(): Set<string> {
+    return this.htmlMessages;
+  }
+}
+
+// Scan HTML content for l10n references and apply namespace prefix
+export function processHTMLFile(
+  content: string,
+  namespace: string,
+  allMessages: Set<string>,
+  ignores: string[],
+  filePath: string,
+) {
+  const foundMessages = new Set<string>();
+
+  const L10N_PATTERN = new RegExp(`(data-l10n-id)="((?!${namespace})\\S*)"`, "g");
+  const processed = content.replace(L10N_PATTERN, (match, attr, id) => {
+    foundMessages.add(id);
+
+    if (ignores.includes(id)) {
+      logger.debug(`Skipped ignored ID: ${styleText.blue(id)} in ${styleText.gray(filePath)}`);
+      return match;
+    }
+
+    if (!allMessages.has(id)) {
+      logger.warn(`I10N id ${styleText.blue(id)} in path ${styleText.gray(filePath)} does not exist in any locale, skip renaming it.`);
+      return match;
+    }
+
+    return `${attr}="${namespace}-${id}"`;
+  });
+
+  return { processedContent: processed, foundMessages: [...foundMessages] };
+}
diff --git a/packages/scaffold/test/e2e/snap/dist/addon/locale/en-US/test-plugin-prefs.ftl b/packages/scaffold/test/e2e/snap/dist/addon/locale/en-US/test-plugin-prefs.ftl
@@ -1,3 +1,3 @@
 test-plugin-meaasge-1 = General Settings
-test-plugin-meaasge-2-message-1 = 
+test-plugin-meaasge-2-message-1 =
     .label = example