Selective typst package staging using typst-gather analyze

Instead of copying ALL packages from built-in resources and extensions
into the scratch directory, use typst-gather analyze to determine the
exact set of packages needed (including transitive deps), then stage
only those.

- Create src/core/typst-gather.ts shared module (moved from cmd.ts)
- Decompose stageTypstPackages into collectPackageSources,
  analyzeNeededPackages, stageSelectedPackages, stageAllPackages
- Change from first-write-wins to last-write-wins so extensions can
  override built-in packages
- Falls back to staging everything if typst-gather binary is missing
  or analyze fails
- 14 unit tests for staging functions and TOML building
- 1 smoke test verifying no packages staged for import-free documents

Fixes #14157

Author: gordonwoodhull

src/command/call/typst-gather/cmd.ts

@@ -7,19 +7,18 @@
 import { Command } from "cliffy/command/mod.ts";
 import { info } from "../../../deno_ral/log.ts";
 
-import { architectureToolsPath } from "../../../core/resources.ts";
 import { execProcess } from "../../../core/process.ts";
 import { dirname, join, relative } from "../../../deno_ral/path.ts";
 import { existsSync } from "../../../deno_ral/fs.ts";
-import { isWindows } from "../../../deno_ral/platform.ts";
 import { expandGlobSync } from "../../../core/deno/expand-glob.ts";
 import { readYaml } from "../../../core/yaml.ts";
-
-// Convert path to use forward slashes for TOML compatibility
-// TOML treats backslash as escape character, so Windows paths must use forward slashes
-function toTomlPath(p: string): string {
-  return p.replace(/\\/g, "/");
-}
+import {
+  type AnalyzeImport,
+  type AnalyzeResult,
+  runAnalyze,
+  toTomlPath,
+  typstGatherBinaryPath,
+} from "../../../core/typst-gather.ts";
 
 interface ExtensionYml {
   contributes?: {
@@ -163,55 +162,7 @@ async function resolveConfig(
   };
 }
 
-export interface AnalyzeImport {
-  namespace: string;
-  name: string;
-  version: string;
-  source: string;
-  direct: boolean;
-}
-
-export interface AnalyzeResult {
-  imports: AnalyzeImport[];
-  files: string[];
-}
-
-function typstGatherBinaryPath(): string {
-  const binaryName = isWindows ? "typst-gather.exe" : "typst-gather";
-  const binary = Deno.env.get("QUARTO_TYPST_GATHER") ||
-    architectureToolsPath(binaryName);
-
-  if (!existsSync(binary)) {
-    throw new Error(
-      `typst-gather binary not found.\n` +
-        `Run ./configure.sh to build and install it.`,
-    );
-  }
-
-  return binary;
-}
-
-async function runAnalyze(tomlConfig: string): Promise<AnalyzeResult> {
-  const binary = typstGatherBinaryPath();
-
-  const result = await execProcess(
-    {
-      cmd: binary,
-      args: ["analyze", "-"],
-      stdout: "piped",
-      stderr: "piped",
-    },
-    tomlConfig,
-  );
-
-  if (!result.success) {
-    throw new Error(
-      result.stderr || "typst-gather analyze failed",
-    );
-  }
-
-  return JSON.parse(result.stdout!) as AnalyzeResult;
-}
+export type { AnalyzeImport, AnalyzeResult };
 
 export function generateConfigFromAnalysis(
   result: AnalyzeResult,

src/command/render/output-typst.ts

@@ -49,70 +49,150 @@ import {
   TypstCompileOptions,
   validateRequiredTypstVersion,
 } from "../../core/typst.ts";
+import { runAnalyze, toTomlPath } from "../../core/typst-gather.ts";
 import { asArray } from "../../core/array.ts";
 import { ProjectContext } from "../../project/types.ts";
 import { validatePdfStandards } from "../../core/verapdf.ts";
 
-// Stage typst packages to .quarto/typst-packages/
-// First stages built-in packages, then extension packages (which can override)
-async function stageTypstPackages(
-  input: string,
-  projectDir?: string,
-): Promise<string | undefined> {
-  if (!projectDir) {
-    return undefined;
-  }
+export interface NeededPackage {
+  namespace: string;
+  name: string;
+  version: string;
+}
 
-  const packageSources: string[] = [];
+// Collect all package source directories (built-in + extensions)
+async function collectPackageSources(
+  input: string,
+  projectDir: string,
+): Promise<string[]> {
+  const sources: string[] = [];
 
-  // 1. Add built-in packages from quarto resources
+  // 1. Built-in packages
   const builtinPackages = resourcePath("formats/typst/packages");
   if (existsSync(builtinPackages)) {
-    packageSources.push(builtinPackages);
+    sources.push(builtinPackages);
   }
 
-  // 2. Add packages from extensions (can override built-in)
+  // 2. Extension packages
   const extensionDirs = inputExtensionDirs(input, projectDir);
   const subtreePath = builtinSubtreeExtensions();
   for (const extDir of extensionDirs) {
-    // Use readSubtreeExtensions for subtree directory, readExtensions for others
     const extensions = extDir === subtreePath
       ? await readSubtreeExtensions(extDir)
       : await readExtensions(extDir);
     for (const ext of extensions) {
       const packagesDir = join(ext.path, "typst/packages");
       if (existsSync(packagesDir)) {
-        packageSources.push(packagesDir);
+        sources.push(packagesDir);
       }
     }
   }
 
-  if (packageSources.length === 0) {
-    return undefined;
+  return sources;
+}
+
+// Build the TOML config string for typst-gather analyze
+export function buildAnalyzeToml(
+  typstInput: string,
+  packageSources: string[],
+): string {
+  const discoverPath = toTomlPath(typstInput);
+  const cachePaths = packageSources.map((p) => `"${toTomlPath(p)}"`).join(", ");
+
+  return [
+    `discover = ["${discoverPath}"]`,
+    `package-cache = [${cachePaths}]`,
+  ].join("\n");
+}
+
+// Run typst-gather analyze on the .typ file to determine needed packages
+async function analyzeNeededPackages(
+  typstInput: string,
+  packageSources: string[],
+): Promise<NeededPackage[] | null> {
+  const tomlConfig = buildAnalyzeToml(typstInput, packageSources);
+
+  try {
+    const result = await runAnalyze(tomlConfig);
+    return result.imports.map(({ namespace, name, version }) => ({
+      namespace,
+      name,
+      version,
+    }));
+  } catch {
+    // Fallback: if analyze fails, stage everything (current behavior)
+    warning("typst-gather analyze failed; staging all packages as fallback");
+    return null;
   }
+}
 
-  // Stage to .quarto/typst/packages/
-  const cacheDir = projectScratchPath(projectDir, "typst/packages");
+// Stage only the needed packages from source dirs into the cache dir.
+// Last write wins — extensions (listed after built-in) override built-in packages.
+export function stageSelectedPackages(
+  sources: string[],
+  cacheDir: string,
+  needed: NeededPackage[] | null,
+): void {
+  if (needed === null) {
+    stageAllPackages(sources, cacheDir);
+    return;
+  }
 
-  // Copy contents of each source directory (merging namespaces like "preview", "local")
-  for (const source of packageSources) {
-    for (const entry of Deno.readDirSync(source)) {
-      const srcPath = join(source, entry.name);
-      const destPath = join(cacheDir, entry.name);
-      if (!existsSync(destPath)) {
-        copySync(srcPath, destPath);
-      } else if (entry.isDirectory) {
-        // Merge directory contents (e.g., merge packages within "preview" namespace)
-        for (const subEntry of Deno.readDirSync(srcPath)) {
-          const subSrcPath = join(srcPath, subEntry.name);
-          const subDestPath = join(destPath, subEntry.name);
-          if (!existsSync(subDestPath)) {
-            copySync(subSrcPath, subDestPath);
-          }
-        }
+  for (const pkg of needed) {
+    const relPath = join(pkg.namespace, pkg.name, pkg.version);
+    const destPath = join(cacheDir, relPath);
+
+    for (const source of sources) {
+      const srcPath = join(source, relPath);
+      if (existsSync(srcPath)) {
+        ensureDirSync(dirname(destPath));
+        copySync(srcPath, destPath, { overwrite: true });
       }
     }
   }
+}
+
+// Fallback: copy all packages from all sources. Last write wins at the
+// package directory level. Built-in listed first, extensions after.
+export function stageAllPackages(sources: string[], cacheDir: string): void {
+  for (const source of sources) {
+    for (const nsEntry of Deno.readDirSync(source)) {
+      if (!nsEntry.isDirectory) continue;
+      const nsSrc = join(source, nsEntry.name);
+      const nsDest = join(cacheDir, nsEntry.name);
+      ensureDirSync(nsDest);
+      for (const pkgEntry of Deno.readDirSync(nsSrc)) {
+        const pkgSrc = join(nsSrc, pkgEntry.name);
+        const pkgDest = join(nsDest, pkgEntry.name);
+        copySync(pkgSrc, pkgDest, { overwrite: true });
+      }
+    }
+  }
+}
+
+// Stage typst packages to .quarto/typst-packages/
+// First stages built-in packages, then extension packages (which can override)
+async function stageTypstPackages(
+  input: string,
+  typstInput: string,
+  projectDir?: string,
+): Promise<string | undefined> {
+  if (!projectDir) {
+    return undefined;
+  }
+
+  const packageSources = await collectPackageSources(input, projectDir);
+  if (packageSources.length === 0) {
+    return undefined;
+  }
+
+  const neededPackages = await analyzeNeededPackages(
+    typstInput,
+    packageSources,
+  );
+
+  const cacheDir = projectScratchPath(projectDir, "typst/packages");
+  stageSelectedPackages(packageSources, cacheDir, neededPackages);
 
   return cacheDir;
 }
@@ -168,7 +248,11 @@ export function typstPdfOutputRecipe(
       typstOptions.rootDir = project.dir;
 
       // Stage extension typst packages
-      const packagePath = await stageTypstPackages(input, project.dir);
+      const packagePath = await stageTypstPackages(
+        input,
+        typstInput,
+        project.dir,
+      );
       if (packagePath) {
         typstOptions.packagePath = packagePath;
       }

src/core/typst-gather.ts

@@ -0,0 +1,68 @@
+/*
+ * typst-gather.ts
+ *
+ * Shared infrastructure for typst-gather binary integration.
+ *
+ * Copyright (C) 2025 Posit Software, PBC
+ */
+
+import { existsSync } from "../deno_ral/fs.ts";
+import { isWindows } from "../deno_ral/platform.ts";
+import { architectureToolsPath } from "./resources.ts";
+import { execProcess } from "./process.ts";
+
+// Convert path to use forward slashes for TOML compatibility
+// TOML treats backslash as escape character, so Windows paths must use forward slashes
+export function toTomlPath(p: string): string {
+  return p.replace(/\\/g, "/");
+}
+
+export interface AnalyzeImport {
+  namespace: string;
+  name: string;
+  version: string;
+  source: string;
+  direct: boolean;
+}
+
+export interface AnalyzeResult {
+  imports: AnalyzeImport[];
+  files: string[];
+}
+
+export function typstGatherBinaryPath(): string {
+  const binaryName = isWindows ? "typst-gather.exe" : "typst-gather";
+  const binary = Deno.env.get("QUARTO_TYPST_GATHER") ||
+    architectureToolsPath(binaryName);
+
+  if (!existsSync(binary)) {
+    throw new Error(
+      `typst-gather binary not found.\n` +
+        `Run ./configure.sh to build and install it.`,
+    );
+  }
+
+  return binary;
+}
+
+export async function runAnalyze(tomlConfig: string): Promise<AnalyzeResult> {
+  const binary = typstGatherBinaryPath();
+
+  const result = await execProcess(
+    {
+      cmd: binary,
+      args: ["analyze", "-"],
+      stdout: "piped",
+      stderr: "piped",
+    },
+    tomlConfig,
+  );
+
+  if (!result.success) {
+    throw new Error(
+      result.stderr || "typst-gather analyze failed",
+    );
+  }
+
+  return JSON.parse(result.stdout!) as AnalyzeResult;
+}

tests/docs/smoke-all/typst/marginalia-only-project/.gitignore

@@ -0,0 +1,5 @@
+/.quarto/
+**/*.quarto_ipynb
+*.html
+*_files/
+*.pdf

tests/docs/smoke-all/typst/marginalia-only-project/_quarto.yml

@@ -0,0 +1,6 @@
+project:
+  type: default
+
+format:
+  typst:
+    keep-typ: true

tests/docs/smoke-all/typst/marginalia-only-project/index.qmd

@@ -0,0 +1,8 @@
+---
+title: "Marginalia Only Test"
+papersize: us-letter
+---
+
+Main content here.
+
+[This note goes in the margin.]{.column-margin}

tests/docs/smoke-all/typst/no-packages-project/_quarto.yml

@@ -0,0 +1,6 @@
+project:
+  type: default
+
+format:
+  typst:
+    keep-typ: true

tests/docs/smoke-all/typst/no-packages-project/index.qmd

@@ -0,0 +1,5 @@
+---
+title: "No Packages Test"
+---
+
+Hello world. This document uses no typst packages.