runner.mts

/* eslint-disable no-console */
import fs from 'node:fs/promises';
import path from 'node:path';
import process from 'node:process';

import { metrics, snakeToCamel } from './driver.mjs';

const [, , benchmarkFile] = process.argv;

type BenchmarkModule = {
  taskSize: number;
  tags: ReadonlyArray<string>;

  before?: () => Promise<void>;
  beforeEach?: () => Promise<void>;
  run: () => Promise<void>;
  afterEach?: () => Promise<void>;
  after?: () => Promise<void>;
};

const benchmarkName = snakeToCamel(path.basename(benchmarkFile, '.mjs'));
const benchmark: BenchmarkModule = await import(`./${benchmarkFile}`);

if (typeof benchmark.taskSize !== 'number') throw new Error('missing taskSize');
if (typeof benchmark.run !== 'function') throw new Error('missing run');
if (!Array.isArray(benchmark.tags)) throw new Error('tags must be an array');
if (benchmark.tags.length === 0 || !benchmark.tags.every(t => typeof t === 'string')) {
  throw new Error('must have more than one tag and all tags must be strings');
}

/** CRITICAL SECTION: time task took in seconds */
async function timeTask() {
  const start = performance.now();
  await benchmark.run();
  const end = performance.now();
  return (end - start) / 1000;
}

/** 1 min in seconds */
const ONE_MIN = 1 * 60;
/** 5 min in seconds */
const FIVE_MIN = 5 * 60;
/** Don't run more than 100 iterations */
const MAX_COUNT = 100;

await benchmark.before?.();

// Allocate an obscene amount of space
const data = new Float64Array(10_000_000);

// Test.
let totalDuration = 0;
let count = 0;
do {
  await benchmark.beforeEach?.();

  data[count] = await timeTask();

  await benchmark.afterEach?.();

  totalDuration += data[count]; // time moves up by benchmark exec time not wall clock
  count += 1;

  // must run for at least one minute
  if (totalDuration < ONE_MIN) continue;

  // 100 runs OR five minutes
  if (count >= MAX_COUNT || totalDuration >= FIVE_MIN) break;

  // count exceeds data space, we never intend to have more than a million data points let alone 10M
  if (count === data.length) break;

  // else: more than one min, less than 100 iterations, less than 5min

  // eslint-disable-next-line no-constant-condition
} while (true);

await benchmark.after?.();

const durations = data.subarray(0, count).toSorted((a, b) => a - b);

function percentileIndex(percentile: number, count: number) {
  return Math.max(Math.floor((count * percentile) / 100 - 1), 0);
}

const medianExecution = durations[percentileIndex(50, count)];
const megabytesPerSecond = benchmark.taskSize / medianExecution;

console.log(
  ' '.repeat(3),
  ...['total time:', totalDuration, 'sec,'],
  ...['ran:', count, 'times,'],
  ...['median time per run:', medianExecution, 'sec,'],
  ...['taskSize:', benchmark.taskSize, 'mb,'],
  ...['throughput:', megabytesPerSecond, 'mb/sec']
);

await fs.writeFile(
  `results_${path.basename(benchmarkFile, '.mjs')}.json`,
  JSON.stringify(metrics(benchmarkName, megabytesPerSecond, benchmark.tags), undefined, 2) + '\n',
  'utf8'
);