Skip to content

Commit 0282c23

Browse files
NullVoxPopuliclaude
NullVoxPopuli
and
claude
committed
Add mitata benchmark for recommended config linting performance
Introduces a benchmark suite that measures how fast the recommended config lints files of various sizes (.js, .gjs, .gts), with automatic PR comparison comments so performance regressions are visible on every PR. Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
1 parent bf3a672 commit 0282c23

19 files changed

Lines changed: 1847 additions & 0 deletions

‎.github/workflows/bench-compare.yml‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
name: Benchmark Comparison
2+
3+
on:
4+
pull_request:
5+
types: [opened, synchronize, reopened]
6+
7+
concurrency:
8+
group: bench-${{ github.head_ref }}
9+
cancel-in-progress: true
10+
11+
jobs:
12+
bench-compare:
13+
name: 'Benchmark Comparison'
14+
runs-on: ubuntu-latest
15+
permissions:
16+
pull-requests: write
17+
contents: read
18+
19+
steps:
20+
- uses: actions/checkout@v6
21+
id: checkout
22+
with:
23+
# Full history so the script can git-archive the base branch.
24+
fetch-depth: 0
25+
# Use the PR head SHA so fork PRs resolve correctly
26+
# (github.head_ref is a branch name that only exists on the fork remote).
27+
ref: ${{ github.event.pull_request.head.sha }}
28+
29+
- uses: wyvox/action-setup-pnpm@v3
30+
31+
- name: Run benchmark comparison
32+
env:
33+
BENCH_JSON_OUTPUT: ${{ runner.temp }}/bench-results.json
34+
run: |
35+
set -o pipefail
36+
pnpm bench:compare | sed 's/\x1b\[[0-9;]*m//g' > "$RUNNER_TEMP/bench-output.txt"
37+
38+
- name: Format PR comment
39+
if: always() && steps.checkout.outcome == 'success'
40+
env:
41+
BENCH_OUTPUT_FILE: ${{ runner.temp }}/bench-output.txt
42+
BENCH_JSON_OUTPUT: ${{ runner.temp }}/bench-results.json
43+
BENCH_JOB_SUCCESS: ${{ job.status == 'success' }}
44+
run: node scripts/format-bench-comment.mjs > "$RUNNER_TEMP/bench-comment.md"
45+
46+
- name: Write job summary
47+
if: always() && steps.checkout.outcome == 'success'
48+
run: cat "$RUNNER_TEMP/bench-comment.md" >> "$GITHUB_STEP_SUMMARY"
49+
50+
- name: Post PR comment
51+
if: always() && steps.checkout.outcome == 'success'
52+
uses: actions/github-script@v7
53+
with:
54+
script: |
55+
const fs = require('fs');
56+
const marker = '<!-- bench-compare -->';
57+
58+
const body = fs.readFileSync(process.env.RUNNER_TEMP + '/bench-comment.md', 'utf8');
59+
60+
const headFullName = context.payload.pull_request?.head?.repo?.full_name;
61+
const isFork = !headFullName || headFullName !== context.repo.owner + '/' + context.repo.repo;
62+
63+
if (isFork) {
64+
core.info('PR is from a fork — skipping PR comment (results are in the job summary).');
65+
core.info('--- Comment body start ---');
66+
core.info(body);
67+
core.info('--- Comment body end ---');
68+
} else {
69+
const { data: comments } = await github.rest.issues.listComments({
70+
owner: context.repo.owner,
71+
repo: context.repo.repo,
72+
issue_number: context.issue.number,
73+
});
74+
75+
const existing = comments.find(c => c.body.includes(marker));
76+
77+
if (existing) {
78+
await github.rest.issues.updateComment({
79+
owner: context.repo.owner,
80+
repo: context.repo.repo,
81+
comment_id: existing.id,
82+
body,
83+
});
84+
} else {
85+
await github.rest.issues.createComment({
86+
owner: context.repo.owner,
87+
repo: context.repo.repo,
88+
issue_number: context.issue.number,
89+
body,
90+
});
91+
}
92+
}

‎package.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@
5151
"lint:package-json-sorting:fix": "sort-package-json package.json",
5252
"lint:prettier": "prettier . --check",
5353
"lint:remote": "eslint-remote-tester",
54+
"bench": "./scripts/run-bench.sh tests/lint.bench.mjs",
55+
"bench:compare": "node scripts/bench-compare.mjs",
56+
"bench:summary": "./scripts/local-bench-summary.sh",
5457
"start": "pnpm test:watch",
5558
"test": "vitest run",
5659
"test:coverage": "vitest --coverage",
@@ -99,6 +102,7 @@
99102
"jquery": "^3.7.1",
100103
"jsdom": "^24.0.0",
101104
"markdownlint-cli": "^0.48.0",
105+
"mitata": "^1.0.34",
102106
"npm-package-json-lint": "^7.0.0",
103107
"npm-run-all2": "^5.0.0",
104108
"prettier": "^3.0.3",

‎pnpm-lock.yaml‎

Lines changed: 8 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎scripts/bench-compare.mjs‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
/* eslint-disable n/no-process-exit */
2+
/**
3+
* Benchmark comparison script using mitata.
4+
*
5+
* Copies the base branch's source to a temp directory, installs its
6+
* dependencies, then runs the mitata bench script with --control-dir so that
7+
* both control (base) and experiment (current) plugins are benchmarked in the
8+
* same process — giving mitata a fair, head-to-head comparison with built-in
9+
* summary tables and boxplots.
10+
*
11+
* Usage:
12+
* node scripts/bench-compare.mjs [--base <branch>]
13+
*
14+
* Options:
15+
* --base <branch> Branch to compare against (default: master)
16+
*/
17+
18+
import { execSync, spawnSync } from 'node:child_process';
19+
import { existsSync, mkdirSync, rmSync } from 'node:fs';
20+
import { tmpdir } from 'node:os';
21+
import { join } from 'node:path';
22+
23+
// ---------------------------------------------------------------------------
24+
// CLI args
25+
// ---------------------------------------------------------------------------
26+
27+
const args = process.argv.slice(2);
28+
const baseIdx = args.indexOf('--base');
29+
const BASE_BRANCH = baseIdx !== -1 ? args[baseIdx + 1] : 'master';
30+
31+
// ---------------------------------------------------------------------------
32+
// Helpers
33+
// ---------------------------------------------------------------------------
34+
35+
function run(cmd, opts = {}) {
36+
return execSync(cmd, { stdio: 'inherit', ...opts });
37+
}
38+
39+
/**
40+
* Resolve a branch name to a commit SHA. Tries `origin/<branch>` first (for CI
41+
* where only the PR branch is checked out locally), then falls back to `<branch>`.
42+
*/
43+
function resolveRef(branch) {
44+
for (const candidate of [`origin/${branch}`, branch]) {
45+
const result = spawnSync('git', ['rev-parse', '--verify', candidate], {
46+
encoding: 'utf8',
47+
stdio: ['pipe', 'pipe', 'pipe'],
48+
});
49+
if (result.status === 0) return result.stdout.trim();
50+
}
51+
throw new Error(`Could not resolve ref for branch "${branch}". Is it fetched?`);
52+
}
53+
54+
// ---------------------------------------------------------------------------
55+
// Main
56+
// ---------------------------------------------------------------------------
57+
58+
const ROOT = process.cwd();
59+
const CONTROL_DIR = join(tmpdir(), `bench-control-${BASE_BRANCH}-${Date.now()}`);
60+
61+
console.error(`\n🔧 Setting up control (${BASE_BRANCH}) in ${CONTROL_DIR}\n`);
62+
63+
const BASE_REF = resolveRef(BASE_BRANCH);
64+
console.error(` Resolved ${BASE_BRANCH} → ${BASE_REF.slice(0, 10)}\n`);
65+
66+
// Clean up temp dir on exit
67+
function cleanup() {
68+
if (existsSync(CONTROL_DIR)) {
69+
try {
70+
rmSync(CONTROL_DIR, { recursive: true, force: true });
71+
} catch {
72+
// best-effort cleanup
73+
}
74+
}
75+
}
76+
process.on('exit', cleanup);
77+
process.on('SIGINT', () => process.exit(130));
78+
process.on('SIGTERM', () => process.exit(143));
79+
80+
try {
81+
// ── 1. Export base branch source to temp dir ─────────────────────────────
82+
mkdirSync(CONTROL_DIR, { recursive: true });
83+
84+
// Copy package manifests and source (use resolved SHA for reliability)
85+
run(
86+
`git archive ${BASE_REF} -- package.json pnpm-lock.yaml lib/ | tar -x -C "${CONTROL_DIR}"`
87+
);
88+
89+
// ── 2. Install dependencies in control dir ───────────────────────────────
90+
console.error(`\n📦 Installing dependencies for control (${BASE_BRANCH})…\n`);
91+
run('pnpm install --frozen-lockfile', {
92+
cwd: CONTROL_DIR,
93+
stdio: ['inherit', 'pipe', 'inherit'],
94+
});
95+
96+
// ── 3. Run mitata bench with --control-dir ───────────────────────────────
97+
console.error(`\n🏎️ Running benchmarks (experiment vs control)…\n`);
98+
99+
const benchScript = join(ROOT, 'tests/lint.bench.mjs');
100+
const benchArgs = [
101+
'--expose-gc',
102+
'--max-old-space-size=4096',
103+
benchScript,
104+
'--control-dir',
105+
CONTROL_DIR,
106+
];
107+
108+
// CPU pinning on Linux to reduce cross-core migration variance
109+
const IS_LINUX = process.platform === 'linux';
110+
const HAS_TASKSET = IS_LINUX && spawnSync('which', ['taskset'], { stdio: 'pipe' }).status === 0;
111+
112+
let cmd = 'node';
113+
let fullArgs = benchArgs;
114+
115+
if (HAS_TASKSET) {
116+
cmd = 'taskset';
117+
fullArgs = ['-c', '0', 'node', ...benchArgs];
118+
console.error('📌 CPU pinning enabled (taskset -c 0)\n');
119+
}
120+
121+
const result = spawnSync(cmd, fullArgs, {
122+
stdio: 'inherit',
123+
cwd: ROOT,
124+
env: { ...process.env },
125+
});
126+
127+
if (result.status !== 0) {
128+
console.error('\n❌ Benchmark run failed.');
129+
process.exit(1);
130+
}
131+
132+
console.error('\n✅ Benchmark comparison complete.\n');
133+
} catch (e) {
134+
console.error('❌ Error:', e.message);
135+
process.exit(1);
136+
}

‎scripts/bench-utils.mjs‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/**
2+
* Shared utilities for benchmark formatting scripts.
3+
*/
4+
5+
import { readFileSync } from 'node:fs';
6+
7+
export function formatTime(ns) {
8+
if (ns >= 1e6) return `${(ns / 1e6).toFixed(2)} ms`;
9+
if (ns >= 1e3) return `${(ns / 1e3).toFixed(2)} µs`;
10+
return `${ns.toFixed(2)} ns`;
11+
}
12+
13+
export function deltaEmoji(pct) {
14+
const abs = Math.abs(pct);
15+
if (abs < 2) return '⚪';
16+
if (pct <= -5) return '🟢';
17+
if (pct >= 5) return '🔴';
18+
if (pct < 0) return '🟢';
19+
return '🟠';
20+
}
21+
22+
/**
23+
* Parse benchmark JSON results into control/experiment pairs with deltas.
24+
* Uses p50 (median) which is more robust to outliers than avg.
25+
*/
26+
export function parsePairs(json) {
27+
const pairs = new Map();
28+
29+
for (const trial of json.benchmarks || []) {
30+
for (const r of trial.runs || []) {
31+
if (!r.stats) continue;
32+
const m = r.name.match(/^(.+)\s+\((control|experiment)\)$/);
33+
if (!m) continue;
34+
const [, key, role] = m;
35+
if (!pairs.has(key)) pairs.set(key, {});
36+
pairs.get(key)[role] = r.stats;
37+
}
38+
}
39+
40+
const rows = [];
41+
for (const [name, { control, experiment }] of pairs) {
42+
if (!control || !experiment) continue;
43+
const ctrlVal = control.p50 ?? control.avg;
44+
const expVal = experiment.p50 ?? experiment.avg;
45+
const delta = ((expVal - ctrlVal) / ctrlVal) * 100;
46+
rows.push({ name, control: ctrlVal, experiment: expVal, delta });
47+
}
48+
49+
return rows;
50+
}
51+
52+
/**
53+
* Read and parse the benchmark JSON results file.
54+
*/
55+
export function readBenchJSON(path) {
56+
return JSON.parse(readFileSync(path, 'utf8'));
57+
}

0 commit comments

Comments
 (0)