Support globs for src and ignoring paths (#165)

* Add glob and is-glob dependencies, regenerates package-lock

* Add path resolution util with optional ignore argument

* Update extract command to support ignore flag

Includes internal refactor to align argument names with CLI flag naming

* Update check command to support ignore flag

Internal refactor to align argument names with CLI flag naming

* Update update command to support ignore flag

Author: aperkaz

package-lock.json

@@ -71,8 +71,10 @@
     "cross-spawn": "^7.0.6",
     "estree-walker": "^2.0.1",
     "gettext-parser": "^6.0.0",
+    "glob": "13.0.0",
     "hunspell-spellchecker": "^1.0.2",
     "ignore": "^5.1.8",
+    "is-glob": "^4.0.3",
     "koa": "^2.16.4",
     "koa-body": "^7.0.1",
     "koa-router": "^14.0.0",

package.json

@@ -8,6 +8,7 @@ import { extractAll } from "../lib/extract";
 import { checkDuplicateKeys } from "../lib/checkDuplicateKeys";
 import * as c3poTypes from "../types";
 import { parse, PoData } from "../lib/parser";
+import { resolvePaths } from "../lib/utils";
 
 /*
 Run any string in stream through warning first
@@ -61,12 +62,15 @@ Check all keys from pots(keys only files) are present in pofile(files with trans
 */
 async function check(
     pofile: string,
-    paths: string[],
+    src: string[],
+    ignore: string[] | string | undefined,
     lang: string,
     overrideOpts?: c3poTypes.TtagOpts,
     ttagRcOpts?: c3poTypes.TtagRc,
     skip?: "translation"
 ) {
+    const paths = resolvePaths(src, ignore);
+
     const progress: c3poTypes.Progress = ora(
         `[ttag] checking translations from ${paths} ...`
     );

src/commands/check.ts

@@ -2,20 +2,23 @@ import * as ora from "ora";
 import * as fs from "fs";
 import * as c3poTypes from "../types";
 import { extractAll } from "../lib/extract";
+import { resolvePaths } from "../lib/utils";
 
 async function extract(
     output: string,
-    paths: string[],
+    src: string[],
+    ignore: string[] | string | undefined,
     lang: string = "en",
     ttagOverrideOpts?: c3poTypes.TtagOpts,
     ttagRcOpts?: c3poTypes.TtagRc
 ) {
     const progress: c3poTypes.Progress = ora(
         `[ttag] extracting translations to ${output} ...`
     );
+
     progress.start();
     const result = await extractAll(
-        paths,
+        resolvePaths(src, ignore),
         lang,
         progress,
         ttagOverrideOpts,

src/commands/extract.ts

@@ -6,20 +6,30 @@ import { updatePo } from "../lib/update";
 import { parse } from "../lib/parser";
 import { serialize, SerializeOptions } from "../lib/serializer";
 import { checkDuplicateKeys } from "../lib/checkDuplicateKeys";
+import { resolvePaths } from "../lib/utils";
 
 async function update(
     pofile: string,
     src: string[],
+    ignore: string[] | string | undefined,
     lang: string,
     ttagOverrideOpts?: ttagTypes.TtagOpts,
     ttagRcOpts?: ttagTypes.TtagRc,
     serializeOpts?: SerializeOptions
 ) {
+    const paths = resolvePaths(src, ignore);
+
     const progress: ttagTypes.Progress = ora(`[ttag] updating ${pofile} ...`);
     progress.start();
     try {
         const pot = parse(
-            await extractAll(src, lang, progress, ttagOverrideOpts, ttagRcOpts)
+            await extractAll(
+                paths,
+                lang,
+                progress,
+                ttagOverrideOpts,
+                ttagRcOpts
+            )
         );
         const errMessage = checkDuplicateKeys(pot);
 

src/commands/update.ts

@@ -84,12 +84,22 @@ yargs
                 default: "en",
                 description: "sets default lang (ISO format)"
             },
+            src: {
+                description:
+                    "path to source files/directories (supports glob, but needs to be quoted)"
+            },
+            ignore: {
+                alias: "i",
+                description:
+                    "paths to ignore (supports glob, but needs to be quoted)"
+            },
             ...getTtagOptsForYargs()
         },
         (argv: any) => {
             extract(
                 argv.output,
                 argv.src,
+                argv.ignore,
                 argv.lang,
                 parseTtagPluginOpts(argv),
                 parseTtagRcOpts()
@@ -110,12 +120,22 @@ yargs
                 choices: ["translation"],
                 default: undefined
             },
+            src: {
+                description:
+                    "path to source files/directories (supports glob, but needs to be quoted)"
+            },
+            ignore: {
+                alias: "i",
+                description:
+                    "paths to ignore (supports glob, but needs to be quoted)"
+            },
             ...getTtagOptsForYargs()
         },
         (argv: any) => {
             check(
                 argv.pofile,
                 argv.src,
+                argv.ignore,
                 argv.lang,
                 parseTtagPluginOpts(argv),
                 parseTtagRcOpts(),
@@ -226,7 +246,13 @@ yargs
                 description: "path to .po file with translations"
             },
             src: {
-                description: "path to source files/directories"
+                description:
+                    "path to source files/directories (supports glob, but needs to be quoted)"
+            },
+            ignore: {
+                alias: "i",
+                description:
+                    "paths to ignore (supports glob, but needs to be quoted)"
             },
             ...getTtagOptsForYargs(),
             foldLength: {
@@ -238,6 +264,7 @@ yargs
             update(
                 argv.pofile,
                 argv.src,
+                argv.ignore,
                 argv.lang,
                 parseTtagPluginOpts(argv),
                 parseTtagRcOpts(),

src/index.ts

@@ -1,6 +1,9 @@
 import { Translations, Message, PoData, PoDataCompact } from "./parser";
 import generate from "@babel/generator";
 import { Node } from "@babel/types";
+import { globSync } from "glob";
+import * as fs from "fs";
+const isGlob = require("is-glob");
 
 const pluralNumRegex = /^nplurals ?= ?(\d);/;
 
@@ -70,3 +73,46 @@ export function convert2Compact(poData: PoData): PoDataCompact {
     delete compactPo.contexts[""][""];
     return compactPo;
 }
+
+/**
+ * Helper function that calculates all the file paths from a src array with optional ignore paths.
+ * The folder paths are ignored, since further directory traversing is not necessary.
+ */
+export function resolvePaths(
+    /** Regular paths or globs */
+    src: string[],
+    /**
+     * Could be a string or an array of strings, depending on how many paths are passed by the cli.
+     *
+     * `--ignore path1` => 'path1'
+     *
+     * `--ignore path1 --ignore path2` => ['path1', 'path2']
+     */
+    ignore?: string | string[]
+): string[] {
+    const toGlob = (path: string): string => {
+        if (fs.lstatSync(path).isDirectory()) {
+            if (!path.endsWith("/")) {
+                path = `${path}/`;
+            }
+            return `${path}**/*`;
+        } else {
+            return path;
+        }
+    };
+
+    /** All paths are internally transformed to globs, to keep backwards compatibility with 'src' paths */
+    const srcGlob = src.map(path => (isGlob(path) ? path : toGlob(path)));
+    if (typeof ignore === "string") {
+        ignore = [ignore];
+    }
+    const ignoreGlob = ignore?.map(path =>
+        isGlob(path) ? path : toGlob(path)
+    );
+
+    return globSync(srcGlob, {
+        absolute: true,
+        nodir: true,
+        ignore: ignoreGlob
+    });
+}

src/lib/utils.ts

@@ -1,3 +1,7 @@
 // Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing
 
+exports[`check from glob source 1`] = `""`;
+
+exports[`check from glob source with ignore 1`] = `""`;
+
 exports[`check when all string are translated 1`] = `""`;

tests/commands/__snapshots__/test_check.ts.snap

@@ -18,6 +18,37 @@ msgstr \\"\\"
 "
 `;
 
+exports[`extract from glob source with filter (ignore all) 1`] = `
+"msgid \\"\\"
+msgstr \\"\\"
+\\"Content-Type: text/plain; charset=utf-8\\\\n\\"
+\\"Plural-Forms: nplurals=2; plural=(n!=1);\\\\n\\"
+
+#: tests/fixtures/baseTest/test1.js:4
+#, javascript-format
+msgid \\"test translation \${ name }\\"
+msgstr \\"\\"
+
+#: tests/fixtures/baseTest/nested/test2.js:4
+#, javascript-format
+msgid \\"test translation 2 \${ name }\\"
+msgstr \\"\\"
+"
+`;
+
+exports[`extract from glob source with filter 1`] = `
+"msgid \\"\\"
+msgstr \\"\\"
+\\"Content-Type: text/plain; charset=utf-8\\\\n\\"
+\\"Plural-Forms: nplurals=2; plural=(n!=1);\\\\n\\"
+
+#: tests/fixtures/baseTest/nested/test2.js:4
+#, javascript-format
+msgid \\"test translation 2 \${ name }\\"
+msgstr \\"\\"
+"
+`;
+
 exports[`extract from js with another default locale 1`] = `
 "msgid \\"\\"
 msgstr \\"\\"

tests/commands/__snapshots__/test_extract.ts.snap

@@ -45,3 +45,26 @@ msgstr[0] \\"\\"
 msgstr[1] \\"\\"
 "
 `;
+
+exports[`test update po with glob 1`] = `
+"msgid \\"\\"
+msgstr \\"\\"
+\\"Content-Type: text/plain; charset=utf-8\\\\n\\"
+\\"Plural-Forms: nplurals=2; plural=(n!=1);\\\\n\\"
+
+#: tests/fixtures/updateTest/test.js:3
+msgid \\"test\\"
+msgstr \\"\\"
+
+#: tests/fixtures/updateTest/test.js:4
+msgid \\"new\\"
+msgstr \\"\\"
+
+#: tests/fixtures/updateTest/test.js:6
+#, javascript-format
+msgid \\"\${ n } banana\\"
+msgid_plural \\"\${ n } bananas\\"
+msgstr[0] \\"\\"
+msgstr[1] \\"\\"
+"
+`;