Merge pull request #2632 from NullVoxPopuli-ai-agent/nvp/add-benchmark

Add mitata benchmark for recommended config linting

Author: NullVoxPopuli

.github/workflows/bench-compare.yml

@@ -0,0 +1,92 @@
+name: Benchmark Comparison
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+concurrency:
+  group: bench-${{ github.head_ref }}
+  cancel-in-progress: true
+
+jobs:
+  bench-compare:
+    name: 'Benchmark Comparison'
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v6
+        id: checkout
+        with:
+          # Full history so the script can git-archive the base branch.
+          fetch-depth: 0
+          # Use the PR head SHA so fork PRs resolve correctly
+          # (github.head_ref is a branch name that only exists on the fork remote).
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - uses: wyvox/action-setup-pnpm@v3
+
+      - name: Run benchmark comparison
+        env:
+          BENCH_JSON_OUTPUT: ${{ runner.temp }}/bench-results.json
+        run: |
+          set -o pipefail
+          pnpm bench:compare | sed 's/\x1b\[[0-9;]*m//g' > "$RUNNER_TEMP/bench-output.txt"
+
+      - name: Format PR comment
+        if: always() && steps.checkout.outcome == 'success'
+        env:
+          BENCH_OUTPUT_FILE: ${{ runner.temp }}/bench-output.txt
+          BENCH_JSON_OUTPUT: ${{ runner.temp }}/bench-results.json
+          BENCH_JOB_SUCCESS: ${{ job.status == 'success' }}
+        run: node scripts/format-bench-comment.mjs > "$RUNNER_TEMP/bench-comment.md"
+
+      - name: Write job summary
+        if: always() && steps.checkout.outcome == 'success'
+        run: cat "$RUNNER_TEMP/bench-comment.md" >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Post PR comment
+        if: always() && steps.checkout.outcome == 'success'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const marker = '<!-- bench-compare -->';
+
+            const body = fs.readFileSync(process.env.RUNNER_TEMP + '/bench-comment.md', 'utf8');
+
+            const headFullName = context.payload.pull_request?.head?.repo?.full_name;
+            const isFork = !headFullName || headFullName !== context.repo.owner + '/' + context.repo.repo;
+
+            if (isFork) {
+              core.info('PR is from a fork — skipping PR comment (results are in the job summary).');
+              core.info('--- Comment body start ---');
+              core.info(body);
+              core.info('--- Comment body end ---');
+            } else {
+              const { data: comments } = await github.rest.issues.listComments({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+              });
+
+              const existing = comments.find(c => c.body.includes(marker));
+
+              if (existing) {
+                await github.rest.issues.updateComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  comment_id: existing.id,
+                  body,
+                });
+              } else {
+                await github.rest.issues.createComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: context.issue.number,
+                  body,
+                });
+              }
+            }

.gitignore

@@ -15,5 +15,8 @@ npm-debug.log
 # eslint-remote-tester
 eslint-remote-tester-results
 
+# Benchmark output
+bench-results.json
+
 # Lock file generated by npm install (project uses pnpm)
 package-lock.json

eslint.config.js

@@ -239,6 +239,8 @@ module.exports = [
       'lib/recommended-rules-gjs.js',
       'lib/recommended-rules-gts.js',
       'tests/__snapshots__/',
+      'tests/bench/',
+      'tests/lint.bench.mjs',
 
       // # Contains <template> in js markdown
       'docs/rules/no-empty-glimmer-component-classes.md',

package.json

@@ -37,6 +37,9 @@
     "lib"
   ],
   "scripts": {
+    "bench": "./scripts/run-bench.sh tests/lint.bench.mjs",
+    "bench:compare": "node scripts/bench-compare.mjs",
+    "bench:summary": "./scripts/local-bench-summary.sh",
     "format": "prettier . --write",
     "lint": "npm-run-all --continue-on-error --aggregate-output --parallel \"lint:!(fix)\"",
     "lint:docs": "markdownlint \"**/*.md\"",
@@ -99,6 +102,7 @@
     "jquery": "^3.7.1",
     "jsdom": "^24.0.0",
     "markdownlint-cli": "^0.48.0",
+    "mitata": "^1.0.34",
     "npm-package-json-lint": "^7.0.0",
     "npm-run-all2": "^5.0.0",
     "prettier": "^3.0.3",

pnpm-lock.yaml

@@ -0,0 +1,144 @@
+/* eslint-disable n/no-process-exit */
+/**
+ * Benchmark comparison script using mitata.
+ *
+ * Copies the base branch's source to a temp directory, installs its
+ * dependencies, then runs the mitata bench script with --control-dir so that
+ * both control (base) and experiment (current) plugins are benchmarked in the
+ * same process — giving mitata a fair, head-to-head comparison with built-in
+ * summary tables and boxplots.
+ *
+ * Usage:
+ *   node scripts/bench-compare.mjs [--base <branch>]
+ *
+ * Options:
+ *   --base <branch>   Branch to compare against (default: master)
+ */
+
+import { execSync, spawnSync } from 'node:child_process';
+import { existsSync, mkdirSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+// ---------------------------------------------------------------------------
+// CLI args
+// ---------------------------------------------------------------------------
+
+const args = process.argv.slice(2);
+const baseIdx = args.indexOf('--base');
+const BASE_BRANCH = baseIdx !== -1 ? args[baseIdx + 1] : 'master';
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function run(cmd, opts = {}) {
+  return execSync(cmd, { stdio: 'inherit', ...opts });
+}
+
+/**
+ * Resolve a branch name to a commit SHA. Tries `origin/<branch>` first (for CI
+ * where only the PR branch is checked out locally), then falls back to `<branch>`.
+ */
+function resolveRef(branch) {
+  for (const candidate of [`origin/${branch}`, branch]) {
+    const result = spawnSync('git', ['rev-parse', '--verify', candidate], {
+      encoding: 'utf8',
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+    if (result.status === 0) return result.stdout.trim();
+  }
+  throw new Error(`Could not resolve ref for branch "${branch}". Is it fetched?`);
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
+const ROOT = process.cwd();
+const CONTROL_DIR = join(tmpdir(), `bench-control-${BASE_BRANCH}-${Date.now()}`);
+
+console.error(`\n🔧  Setting up control (${BASE_BRANCH}) in ${CONTROL_DIR}\n`);
+
+const BASE_REF = resolveRef(BASE_BRANCH);
+console.error(`   Resolved ${BASE_BRANCH} → ${BASE_REF.slice(0, 10)}\n`);
+
+// Clean up temp dir on exit
+function cleanup() {
+  if (existsSync(CONTROL_DIR)) {
+    try {
+      rmSync(CONTROL_DIR, { recursive: true, force: true });
+    } catch {
+      // best-effort cleanup
+    }
+  }
+}
+process.on('exit', cleanup);
+process.on('SIGINT', () => process.exit(130));
+process.on('SIGTERM', () => process.exit(143));
+
+try {
+  // ── 1. Export base branch source to temp dir ─────────────────────────────
+  mkdirSync(CONTROL_DIR, { recursive: true });
+
+  // Copy package manifests and source (use resolved SHA for reliability)
+  run(
+    `git archive ${BASE_REF} -- package.json pnpm-lock.yaml .npmrc lib/ | tar -x -C "${CONTROL_DIR}"`
+  );
+
+  // ── 2. Install dependencies in control dir ───────────────────────────────
+  console.error(`\n📦  Installing dependencies for control (${BASE_BRANCH})…\n`);
+  try {
+    run('pnpm install --frozen-lockfile', {
+      cwd: CONTROL_DIR,
+      stdio: ['inherit', 'pipe', 'inherit'],
+    });
+  } catch {
+    console.error('⚠️  Frozen install failed, retrying without --frozen-lockfile…\n');
+    run('pnpm install --no-frozen-lockfile', {
+      cwd: CONTROL_DIR,
+      stdio: ['inherit', 'pipe', 'inherit'],
+    });
+  }
+
+  // ── 3. Run mitata bench with --control-dir ───────────────────────────────
+  console.error(`\n🏎️  Running benchmarks (experiment vs control)…\n`);
+
+  const benchScript = join(ROOT, 'tests/lint.bench.mjs');
+  const benchArgs = [
+    '--expose-gc',
+    '--max-old-space-size=4096',
+    benchScript,
+    '--control-dir',
+    CONTROL_DIR,
+  ];
+
+  // CPU pinning on Linux to reduce cross-core migration variance
+  const IS_LINUX = process.platform === 'linux';
+  const HAS_TASKSET = IS_LINUX && spawnSync('which', ['taskset'], { stdio: 'pipe' }).status === 0;
+
+  let cmd = 'node';
+  let fullArgs = benchArgs;
+
+  if (HAS_TASKSET) {
+    cmd = 'taskset';
+    fullArgs = ['-c', '0', 'node', ...benchArgs];
+    console.error('📌  CPU pinning enabled (taskset -c 0)\n');
+  }
+
+  const result = spawnSync(cmd, fullArgs, {
+    stdio: 'inherit',
+    cwd: ROOT,
+    env: { ...process.env },
+  });
+
+  if (result.status !== 0) {
+    console.error('\n❌  Benchmark run failed.');
+    process.exit(1);
+  }
+
+  console.error('\n✅  Benchmark comparison complete.\n');
+} catch (e) {
+  console.error('❌  Error:', e.message);
+  process.exit(1);
+}

scripts/bench-compare.mjs

@@ -0,0 +1,57 @@
+/**
+ * Shared utilities for benchmark formatting scripts.
+ */
+
+import { readFileSync } from 'node:fs';
+
+export function formatTime(ns) {
+  if (ns >= 1e6) return `${(ns / 1e6).toFixed(2)} ms`;
+  if (ns >= 1e3) return `${(ns / 1e3).toFixed(2)} µs`;
+  return `${ns.toFixed(2)} ns`;
+}
+
+export function deltaEmoji(pct) {
+  const abs = Math.abs(pct);
+  if (abs < 2) return '⚪';
+  if (pct <= -5) return '🟢';
+  if (pct >= 5) return '🔴';
+  if (pct < 0) return '🟢';
+  return '🟠';
+}
+
+/**
+ * Parse benchmark JSON results into control/experiment pairs with deltas.
+ * Uses p50 (median) which is more robust to outliers than avg.
+ */
+export function parsePairs(json) {
+  const pairs = new Map();
+
+  for (const trial of json.benchmarks || []) {
+    for (const r of trial.runs || []) {
+      if (!r.stats) continue;
+      const m = r.name.match(/^(.+)\s+\((control|experiment)\)$/);
+      if (!m) continue;
+      const [, key, role] = m;
+      if (!pairs.has(key)) pairs.set(key, {});
+      pairs.get(key)[role] = r.stats;
+    }
+  }
+
+  const rows = [];
+  for (const [name, { control, experiment }] of pairs) {
+    if (!control || !experiment) continue;
+    const ctrlVal = control.p50 ?? control.avg;
+    const expVal = experiment.p50 ?? experiment.avg;
+    const delta = ((expVal - ctrlVal) / ctrlVal) * 100;
+    rows.push({ name, control: ctrlVal, experiment: expVal, delta });
+  }
+
+  return rows;
+}
+
+/**
+ * Read and parse the benchmark JSON results file.
+ */
+export function readBenchJSON(path) {
+  return JSON.parse(readFileSync(path, 'utf8'));
+}