diff --git a/packages/scaffold/package.json b/packages/scaffold/package.json index e7aa66e..fb177a7 100644 --- a/packages/scaffold/package.json +++ b/packages/scaffold/package.json @@ -69,6 +69,7 @@ } }, "dependencies": { + "@fluent/syntax": "^0.19.0", "@swc/core": "^1.11.16", "adm-zip": "^0.5.16", "bumpp": "^10.1.0", diff --git a/packages/scaffold/src/core/builder.ts b/packages/scaffold/src/core/builder.ts index ee564ec..1738200 100644 --- a/packages/scaffold/src/core/builder.ts +++ b/packages/scaffold/src/core/builder.ts @@ -2,13 +2,13 @@ import type { Context } from "../types/index.js"; import type { Manifest } from "../types/manifest.js"; import type { UpdateJSON } from "../types/update-json.js"; import { existsSync } from "node:fs"; -import { readFile, writeFile } from "node:fs/promises"; -import { basename, dirname, join, resolve } from "node:path"; +import { readFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; import process from "node:process"; import AdmZip from "adm-zip"; -import { escapeRegExp, toMerged } from "es-toolkit"; +import { toMerged } from "es-toolkit"; import { build as buildAsync } from "esbuild"; -import { copy, emptyDir, move, outputFile, outputJSON, readJSON, writeJson } from "fs-extra/esm"; +import { copy, emptyDir, outputFile, outputJSON, readJSON, writeJson } from "fs-extra/esm"; import styleText from "node-style-text"; import { glob } from "tinyglobby"; import { generateHash } from "../utils/crypto.js"; @@ -16,6 +16,7 @@ import { is32BitNumber } from "../utils/number.js"; import { PrefsManager, renderPluginPrefsDts } from "../utils/prefs-manager.js"; import { dateFormat, replaceInFile, toArray } from "../utils/string.js"; import { Base } from "./base.js"; +import buildLocale from "./builder/fluent.js"; export default class Build extends Base { private buildTime: string; @@ -145,109 +146,7 @@ export default class Build extends Base { async prepareLocaleFiles() { const { dist, namespace, build } = this.ctx; - - const ignores = toArray(build.fluent.ignore); - - // https://regex101.com/r/lQ9x5p/1 - // eslint-disable-next-line regexp/no-super-linear-backtracking - const FTL_MESSAGE_PATTERN = /^(?[a-z]\S*)( *= *)(?.*)$/gim; - const HTML_DATAI10NID_PATTERN = new RegExp(`(data-l10n-id)="((?!${namespace})\\S*)"`, "g"); - - // Get locale names - const localePaths = await glob(`${dist}/addon/locale/*`, { onlyDirectories: true }); - const localeNames = localePaths.map(locale => basename(locale)); - this.logger.debug(`Locale names:", ${localeNames}`); - - const allMessages = new Set(); - const messagesByLocale = new Map>(); - - for (const localeName of localeNames) { - // Prefix Fluent messages in each ftl, add message to set. - const localeMessages = new Set(); - const ftlPaths = await glob(`${dist}/addon/locale/${localeName}/**/*.ftl`); - - await Promise.all(ftlPaths.map(async (ftlPath: string) => { - let ftlContent = await readFile(ftlPath, "utf-8"); - const matches = [...ftlContent.matchAll(FTL_MESSAGE_PATTERN)]; - - for (const match of matches) { - const [matched, message, _pattern] = match; - if (message) { - localeMessages.add(message); - allMessages.add(message); - ftlContent = ftlContent.replace(new RegExp(`^${escapeRegExp(matched)}`, "gm"), `${namespace}-${matched}`); - } - } - - // If prefixFluentMessages===true, we save the changed ftl file, - // otherwise discard the changes - if (build.fluent.prefixFluentMessages) - await writeFile(ftlPath, ftlContent); - - // rename *.ftl to addonRef-*.ftl - if (build.fluent.prefixLocaleFiles === true) { - await move(ftlPath, `${dirname(ftlPath)}/${namespace}-${basename(ftlPath)}`); - this.logger.debug(`FTL file '${ftlPath}' is renamed to '${namespace}-${basename(ftlPath)}'.`); - } - })); - - messagesByLocale.set(localeName, localeMessages); - } - - // Prefix Fluent messages in xhtml - const messagesInHTML = new Set(); - const htmlPaths = await glob([ - `${dist}/addon/**/*.xhtml`, - `${dist}/addon/**/*.html`, - ]); - await Promise.all(htmlPaths.map(async (htmlPath) => { - let htmlContent = await readFile(htmlPath, "utf-8"); - const matches = [...htmlContent.matchAll(HTML_DATAI10NID_PATTERN)]; - - for (const match of matches) { - const [matched, attrKey, attrVal] = match; - - if (ignores.includes(attrVal)) { - this.logger.debug(`HTML data-i10n-id ${attrVal} is in ignore list, skip to namespace`); - continue; - } - - if (!allMessages.has(attrVal)) { - this.logger.warn(`HTML data-i10n-id '${styleText.blue(attrVal)}' in ${styleText.gray(htmlPath)} do not exist in any FTL message, skip to namespace it.`); - continue; - } - - messagesInHTML.add(attrVal); - const namespacedAttr = `${namespace}-${attrVal}`; - htmlContent = htmlContent.replace(matched, `${attrKey}="${namespacedAttr}"`); - this.logger.debug(`HTML data-i10n-id '${styleText.blue(attrVal)}' in ${styleText.gray(htmlPath)} is namespaced to ${styleText.blue(namespacedAttr)}.`); - } - - if (build.fluent.prefixFluentMessages) - await writeFile(htmlPath, htmlContent); - })); - - // Check miss 1: Cross check in diff locale - seems no need - // messagesByLocale.forEach((messageInThisLang, lang) => { - // // Needs Nodejs 22 - // const diff = allMessages.difference(messageInThisLang); - // if (diff.size) - // this.logger.warn(`FTL messages '${Array.from(diff).join(", ")}' don't exist the locale '${lang}'`); - // }); - - // Check miss 2: Check ids in HTML but not in ftl - messagesInHTML.forEach((messageInHTML) => { - if (ignores.includes(messageInHTML)) - return; - - const missingLocales = [...messagesByLocale.entries()] - .filter(([_, messages]) => !messages.has(messageInHTML)) - .map(([locale]) => locale); - - if (missingLocales.length > 0) { - this.logger.warn(`HTML data-l10n-id '${styleText.blue(messageInHTML)}' is missing in locales: ${missingLocales.join(", ")}.`); - } - }); + await buildLocale(dist, namespace, build.fluent); } async preparePrefs() { diff --git a/packages/scaffold/src/core/builder/fluent.ts b/packages/scaffold/src/core/builder/fluent.ts new file mode 100644 index 0000000..ab7e72e --- /dev/null +++ b/packages/scaffold/src/core/builder/fluent.ts @@ -0,0 +1,242 @@ +import type { BaseNode, Message, MessageReference, Resource } from "@fluent/syntax"; +import type { BuildConfig } from "../../types/config.js"; +import { readFile, writeFile } from "node:fs/promises"; +import { basename, dirname } from "node:path"; +import { parse, serialize, Transformer } from "@fluent/syntax"; +import { move } from "fs-extra/esm"; +import styleText from "node-style-text"; +import { glob } from "tinyglobby"; +import { logger } from "../../utils/logger.js"; +import { toArray } from "../../utils/string.js"; + +export default async function buildLocale( + dist: string, + namespace: string, + options: BuildConfig["fluent"], +) { + const ignores = toArray(options.ignore); + const localeNames = await getLocales(dist); + const messageManager = new MessageManager(ignores); + + // Process FTL files and add messages to the manager + await Promise.all(localeNames.map(async (locale) => { + const paths = await glob(`${dist}/addon/locale/${locale}/**/*.ftl`); + await Promise.all(paths.map(async (path) => { + const fm = new FluentManager(); + await fm.read(path); + messageManager.addMessages(locale, fm.getMessages()); + + if (options.prefixFluentMessages) { + fm.prefixMessages(namespace); + await fm.write(path); + } + + if (options.prefixLocaleFiles) { + const newPath = `${dirname(path)}/${namespace}-${basename(path)}`; + await move(path, newPath); + logger.debug(`Renamed FTL: ${path} → ${newPath}`); + } + })); + })); + + // Process HTML files and add messages to the manager + const htmlPaths = await glob([`${dist}/addon/**/*.xhtml`, `${dist}/addon/**/*.html`]); + await Promise.all(htmlPaths.map(async (htmlPath) => { + const content = await readFile(htmlPath, "utf-8"); + const { processedContent, foundMessages } = processHTMLFile( + content, + namespace, + messageManager.getFTLMessages(), + ignores, + htmlPath, + ); + + // Add all found HTML messages + messageManager.addMessages("html", foundMessages); + + if (options.prefixFluentMessages) { + await writeFile(htmlPath, processedContent); + } + })); + + // Validate that all HTML messages exist in all locales + messageManager.validateMessages(); +} + +async function getLocales(dist: string): Promise { + const localePaths = await glob(`${dist}/addon/locale/*`, { onlyDirectories: true }); + return localePaths.map(p => basename(p)); +} + +export class FluentManager { + private source?: string; + private resource?: Resource; + public readonly messages: string[] = []; + + constructor() {} + + // Parse Fluent source into an AST and extract messages + public parse(source: string) { + this.source = source; + this.resource = parse(source, {}); + } + + // Read a file, parse its content, and extract messages + public async read(path: string) { + const content = await readFile(path, "utf-8"); + this.parse(content); + } + + // Extract message IDs from the parsed resource + public getMessages(): string[] { + if (!this.resource) { + throw new Error("Resource must be parsed first."); + } + this.messages.length = 0; // Clear the previous messages + this.messages.push( + ...this.resource.body.filter(entry => entry.type === "Message") + .map(message => message.id.name), + ); + return this.messages; + } + + // Apply namespace prefix to message IDs in the resource + public prefixMessages(namespace: string) { + if (!this.resource) { + throw new Error("Resource must be parsed before applying prefix."); + } + new FluentTransformer(namespace).genericVisit(this.resource); + } + + // Serialize the resource back into a string + public serialize(): string { + if (!this.resource) { + throw new Error("Resource not parsed. Cannot serialize."); + } + return serialize(this.resource, {}); + } + + // Write the serialized resource to a file + public async write(path: string) { + const result = this.serialize(); + if (result !== this.source) + await writeFile(path, this.serialize()); + } +} +// Custom Fluent AST transformer to apply message ID prefix +class FluentTransformer extends Transformer { + constructor(private readonly prefix: string | false) { + super(); + } + + private needsPrefix(name: string): boolean { + return !!this.prefix && !name.startsWith(this.prefix); + } + + visitMessage(node: Message): BaseNode { + if (this.needsPrefix(node.id.name)) { + node.id.name = `${this.prefix}-${node.id.name}`; + } + return this.genericVisit(node); + } + + visitMessageReference(node: MessageReference): BaseNode { + if (this.needsPrefix(node.id.name)) { + node.id.name = `${this.prefix}-${node.id.name}`; + } + return this.genericVisit(node); + } +} + +export class MessageManager { + private ftlMessages: Map> = new Map(); + private htmlMessages: Set = new Set(); + private ignores: string[]; + + constructor(ignores: string[]) { + this.ignores = ignores; + } + + // Add a set of messages (FTL or HTML) for a specific locale or for HTML globally + addMessages(target: string | "html", messages: string[]) { + if (target === "html") { + messages.forEach(msg => this.htmlMessages.add(msg)); + } + else { + let ftlLocaleMessages = this.ftlMessages.get(target); + if (!ftlLocaleMessages) { + ftlLocaleMessages = new Set(); + this.ftlMessages.set(target, ftlLocaleMessages); + } + messages.forEach(msg => ftlLocaleMessages.add(msg)); + } + } + + validateMessages() { + // Check miss 1: Cross check in diff locale - seems no need + // messagesByLocale.forEach((messageInThisLang, lang) => { + // // Needs Nodejs 22 + // const diff = allMessages.difference(messageInThisLang); + // if (diff.size) + // this.logger.warn(`FTL messages '${Array.from(diff).join(", ")}' don't exist the locale '${lang}'`); + // }); + + // Check miss 2: Check ids in HTML but not in ftl + this.htmlMessages.forEach((msg) => { + if (this.ignores.includes(msg)) + return; + + const missingLocales = [...this.ftlMessages.entries()] + .filter(([_, messages]) => !messages.has(msg)) + .map(([locale]) => locale); + if (missingLocales.length > 0) + logger.warn(`I10N id ${styleText.blue(msg)} missing in locale: ${missingLocales.join(", ")}`); + }); + } + + getFTLMessages(): Set { + const allMessages = new Set(); + this.ftlMessages.forEach(messages => messages.forEach(msg => allMessages.add(msg))); + return allMessages; + } + + // Get all FTL messages for a specific locale + getFTLMessagesByLocale(locale: string): Set { + return this.ftlMessages.get(locale) || new Set(); + } + + // Get all HTML messages + getHTMLMessages(): Set { + return this.htmlMessages; + } +} + +// Scan HTML content for l10n references and apply namespace prefix +export function processHTMLFile( + content: string, + namespace: string, + allMessages: Set, + ignores: string[], + filePath: string, +) { + const foundMessages = new Set(); + + const L10N_PATTERN = new RegExp(`(data-l10n-id)="((?!${namespace})\\S*)"`, "g"); + const processed = content.replace(L10N_PATTERN, (match, attr, id) => { + foundMessages.add(id); + + if (ignores.includes(id)) { + logger.debug(`Skipped ignored ID: ${styleText.blue(id)} in ${styleText.gray(filePath)}`); + return match; + } + + if (!allMessages.has(id)) { + logger.warn(`I10N id ${styleText.blue(id)} in path ${styleText.gray(filePath)} does not exist in any locale, skip renaming it.`); + return match; + } + + return `${attr}="${namespace}-${id}"`; + }); + + return { processedContent: processed, foundMessages: [...foundMessages] }; +} diff --git a/packages/scaffold/test/e2e/snap/dist/addon/locale/en-US/test-plugin-prefs.ftl b/packages/scaffold/test/e2e/snap/dist/addon/locale/en-US/test-plugin-prefs.ftl index d9a4efa..ee7474f 100644 --- a/packages/scaffold/test/e2e/snap/dist/addon/locale/en-US/test-plugin-prefs.ftl +++ b/packages/scaffold/test/e2e/snap/dist/addon/locale/en-US/test-plugin-prefs.ftl @@ -1,3 +1,3 @@ test-plugin-meaasge-1 = General Settings -test-plugin-meaasge-2-message-1 = +test-plugin-meaasge-2-message-1 = .label = example diff --git a/packages/scaffold/test/unit/fluent.test.ts b/packages/scaffold/test/unit/fluent.test.ts new file mode 100644 index 0000000..a6b600e --- /dev/null +++ b/packages/scaffold/test/unit/fluent.test.ts @@ -0,0 +1,175 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + FluentManager, + MessageManager, + processHTMLFile, +} from "../../src/core/builder/fluent.js"; +import { logger } from "../../src/utils/logger.js"; + +vi.mock("../../src/utils/logger.js"); + +describe("fluent-manager", () => { + let manager: FluentManager; + + beforeEach(() => { + manager = new FluentManager(); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + describe("getMessages()", () => { + it("should return all message IDs", () => { + manager.parse(` +welcome = Welcome +about = About { welcome } +`); + const messages = manager.getMessages(); + expect(messages).toEqual(["welcome", "about"]); + }); + }); + + describe("prefix()", () => { + it("should prefix message IDs correctly", () => { + manager.parse(` +welcome = Welcome +about = About { welcome } +test-prefixed = Test { welcome } +`); + manager.prefixMessages("test"); + expect(manager.serialize()).toBe([ + "test-welcome = Welcome", + "test-about = About { test-welcome }", + "test-prefixed = Test { test-welcome }", + "", + ] + .join("\n")); + }); + }); +}); + +describe("processHTMLFile", () => { + const namespace = "myNamespace"; + const ignores = ["ignoredMessage"]; + const allMessages = new Set(["validMessage1", "validMessage2"]); + const filePath = "path/to/file.html"; + + it("should replace data-l10n-id with namespace-prefixed ID", () => { + const inputContent = `
`; + + const { processedContent, foundMessages } = processHTMLFile(inputContent, namespace, allMessages, ignores, filePath); + + expect(processedContent).toBe("
"); + expect(foundMessages.includes("validMessage1")).toBe(true); + }); + + it("should skip ignored IDs", () => { + const inputContent = `
`; + const { processedContent, foundMessages } = processHTMLFile(inputContent, namespace, allMessages, ignores, filePath); + + expect(processedContent).toBe("
"); + expect(foundMessages.includes("ignoredMessage")).toBe(true); + expect(logger.debug).toHaveBeenCalledWith(expect.stringContaining("Skipped ignored ID:")); + }); + + it("should warn when an FTL message is missing", () => { + const inputContent = `
`; + + const { processedContent, foundMessages } = processHTMLFile(inputContent, namespace, allMessages, ignores, filePath); + + expect(processedContent).toBe("
"); + expect(foundMessages.includes("missingMessage")).toBe(true); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining("does not exist in any locale, skip renaming it")); + }); + + it("should not modify the content if no matching messages are found", () => { + const inputContent = `
`; + const { processedContent, foundMessages } = processHTMLFile(inputContent, namespace, allMessages, ignores, filePath); + + expect(processedContent).toBe("
"); // No replacement + expect(foundMessages.includes("validMessage3")).toBe(true); // No message found + }); +}); + +describe("message-manager", () => { + let messageManager: MessageManager; + const ignores = ["ignore"]; + + beforeEach(() => { + messageManager = new MessageManager(ignores); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it("should add FTL messages for a specific locale", () => { + const locale = "en"; + const messages = ["welcome", "goodbye"]; + + messageManager.addMessages(locale, messages); + + const ftlMessages = messageManager.getFTLMessagesByLocale(locale); + + expect(ftlMessages.has("welcome")).toBe(true); + expect(ftlMessages.has("goodbye")).toBe(true); + }); + + it("should add HTML messages", () => { + const messages = ["about", "contact"]; + + messageManager.addMessages("html", messages); + + const htmlMessages = messageManager.getHTMLMessages(); + + expect(htmlMessages.has("about")).toBe(true); + expect(htmlMessages.has("contact")).toBe(true); + }); + + it("should validate missing HTML messages in FTL locales", () => { + messageManager.addMessages("en", ["welcome", "goodbye"]); + messageManager.addMessages("fr", ["welcome"]); + messageManager.addMessages("html", ["welcome", "goodbye"]); + + messageManager.validateMessages(); + + expect(logger.warn).toHaveBeenCalledWith( + expect.stringMatching(/I10N id .*goodbye.* missing in locale: fr/), + ); + }); + + it("should not trigger a warning when HTML messages are present in all FTL locales", () => { + messageManager.addMessages("en", ["welcome", "about"]); + messageManager.addMessages("fr", ["welcome", "about"]); + messageManager.addMessages("html", ["about", "welcome"]); + + messageManager.validateMessages(); + + // No warnings should be logged for "about" and "welcome" + expect(logger.warn).not.toHaveBeenCalled(); + }); + + it("should return all FTL messages", () => { + messageManager.addMessages("en", ["welcome", "about"]); + messageManager.addMessages("fr", ["welcome"]); + + const allMessages = messageManager.getFTLMessages(); + + expect(allMessages).toEqual(new Set(["welcome", "about"])); + }); + + it("should return an empty set when no FTL messages exist for a locale", () => { + messageManager.addMessages("en", ["welcome", "about"]); + + const frMessages = messageManager.getFTLMessagesByLocale("fr"); + + expect(frMessages.size).toBe(0); + }); + + it("should return empty HTML messages if none have been added", () => { + const htmlMessages = messageManager.getHTMLMessages(); + + expect(htmlMessages.size).toBe(0); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ce65bdb..e18d873 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -92,6 +92,9 @@ importers: packages/scaffold: dependencies: + '@fluent/syntax': + specifier: ^0.19.0 + version: 0.19.0 '@swc/core': specifier: ^1.11.16 version: 1.11.16 @@ -747,6 +750,10 @@ packages: '@floating-ui/utils@0.2.9': resolution: {integrity: sha512-MDWhGtE+eHw5JW7lq4qhc5yRLS11ERl1c7Z6Xd0a58DozHES6EnNNwUWbMiG4J9Cgj053Bhk8zvlhFYKVhULwg==} + '@fluent/syntax@0.19.0': + resolution: {integrity: sha512-5D2qVpZrgpjtqU4eNOcWGp1gnUCgjfM+vKGE2y03kKN6z5EBhtx0qdRFbg8QuNNj8wXNoX93KJoYb+NqoxswmQ==} + engines: {node: '>=14.0.0', npm: '>=7.0.0'} + '@humanfs/core@0.19.1': resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} engines: {node: '>=18.18.0'} @@ -4450,6 +4457,8 @@ snapshots: '@floating-ui/utils@0.2.9': {} + '@fluent/syntax@0.19.0': {} + '@humanfs/core@0.19.1': {} '@humanfs/node@0.16.6':