diff --git a/.eslintrc.js b/.eslintrc.js index bfd204adc..5f47bb5be 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -407,6 +407,7 @@ module.exports = { 'packages/plugins/**/scripts/**/*', 'packages/tests/src/_jest/**/*', 'packages/tests/src/_playwright/**/*', + 'packages/tests/src/bench/**/*', 'packages/tests/src/e2e/**/*', ], rules: { diff --git a/.github/actions/setup-playwright-build/action.yaml b/.github/actions/setup-playwright-build/action.yaml new file mode 100644 index 000000000..38c6b3ed9 --- /dev/null +++ b/.github/actions/setup-playwright-build/action.yaml @@ -0,0 +1,50 @@ +name: Setup Playwright build +description: Install dependencies, Playwright browsers, and prebuilt plugins for Playwright jobs. +runs: + using: composite + steps: + - name: Install Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: "package.json" + + - name: Cache build:all + id: cache-build + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: packages/published/**/dist + key: node18-cache-build-${{ hashFiles('packages/core/**', 'packages/factory/**', + 'packages/plugins/**', 'packages/published/**', + 'packages/tools/src/**', 'yarn.lock') }} + + - name: Cache playwright binaries + id: cache-playwright-binaries + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: | + ~/.cache/ms-playwright + ~/Library/Caches/ms-playwright + %USERPROFILE%\AppData\Local\ms-playwright + key: cache-playwright-binaries-${{ hashFiles('yarn.lock') }} + + - run: yarn install + shell: bash + + - name: Install playwright + run: yarn workspace @dd/tests playwright install --with-deps + shell: bash + + - name: Build all plugins + if: steps.cache-build.outputs.cache-hit != 'true' + run: yarn build:all-no-types + shell: bash + + - name: Save playwright cache + if: always() && steps.cache-playwright-binaries.outputs.cache-hit != 'true' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: | + ~/.cache/ms-playwright + ~/Library/Caches/ms-playwright + %USERPROFILE%\AppData\Local\ms-playwright + key: cache-playwright-binaries-${{ hashFiles('yarn.lock') }} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 48f9d26b4..7c7776288 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -64,29 +64,7 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Install Node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version-file: "package.json" - - - name: Cache build:all - id: cache-build - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: packages/published/**/dist - key: node18-cache-build-${{ hashFiles('packages/core/**', 'packages/factory/**', - 'packages/plugins/**', 'packages/published/**', - 'packages/tools/src/**', 'yarn.lock') }} - - - name: Cache playwright binaries - id: cache-playwright-binaries - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: | - ~/.cache/ms-playwright - ~/Library/Caches/ms-playwright - %USERPROFILE%\AppData\Local\ms-playwright - key: cache-playwright-binaries-${{ hashFiles('yarn.lock') }} + - uses: ./.github/actions/setup-playwright-build - name: Configure Datadog Test Optimization uses: datadog/test-visibility-github-action@f76512a963e7375dab9ad7f1abc0cacd41806c5c # v2.6.0 @@ -96,37 +74,146 @@ jobs: api_key: ${{secrets.DATADOG_API_KEY}} site: datadoghq.com - - run: yarn install - - - name: Install playwright - run: yarn workspace @dd/tests playwright install --with-deps - - - name: Build all plugins - if: steps.cache-build.outputs.cache-hit != 'true' - run: yarn build:all-no-types - - run: yarn test:e2e env: NODE_OPTIONS: -r ${{env.DD_TRACE_PACKAGE}} DD_TAGS: type:e2e DD_ENV: ci - - name: Save playwright cache - if: always() && steps.cache-playwright-binaries.outputs.cache-hit != 'true' - id: save-playwright-cache - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: ${{ failure() }} with: + name: playwright path: | - ~/.cache/ms-playwright - ~/Library/Caches/ms-playwright - %USERPROFILE%\AppData\Local\ms-playwright - key: cache-playwright-binaries-${{ hashFiles('yarn.lock') }} + packages/tests/playwright-report + packages/tests/test-results + retention-days: 3 + + runtime-bench-preflight: + timeout-minutes: 20 + + name: Live Debugger runtime benchmark preflight + runs-on: ubuntu-latest + if: ${{ github.event_name == 'pull_request' && !startsWith(github.ref_name, 'mq-working-branch-') }} + outputs: + should-run: ${{ steps.output-check.outputs.should-run || steps.changed-files.outputs.should-run }} + reason: ${{ steps.output-check.outputs.reason || steps.changed-files.outputs.reason }} + env: + FORCE_COLOR: true + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + + - name: Install Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: "package.json" + + - run: yarn install + + - name: Check benchmark and CI changes + id: changed-files + run: node packages/tests/src/bench/liveDebuggerRuntime/preflight.js --changed-files --base-ref=origin/${{ github.base_ref }} + + - name: Check benchmark output changes + if: steps.changed-files.outputs.should-run != 'true' + id: output-check + run: node packages/tests/src/bench/liveDebuggerRuntime/preflight.js --compare-output --base-ref=origin/${{ github.base_ref }} + + runtime-bench: + timeout-minutes: 30 + + name: Live Debugger runtime benchmark + runs-on: ubuntu-latest + needs: runtime-bench-preflight + if: ${{ needs.runtime-bench-preflight.outputs.should-run == 'true' }} + continue-on-error: true + permissions: + contents: read + pull-requests: write + env: + FORCE_COLOR: true + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - uses: ./.github/actions/setup-playwright-build + + - name: Run runtime benchmark + run: yarn workspace @dd/tests bench:live-debugger:runtime + + - name: Render benchmark comment + if: always() + run: | + BENCH_TMP_DIR="$(node -e "process.stdout.write(require('os').tmpdir())")" + COMMENT_PATH="$BENCH_TMP_DIR/live-debugger-runtime-bench-comment.md" + RESULTS_PATH="$(ls -t "$BENCH_TMP_DIR"/live-debugger-runtime-bench-results-*.json 2>/dev/null | head -n 1)" + if [ -f "$COMMENT_PATH" ]; then + cp "$COMMENT_PATH" runtime-bench-comment.md + else + { + echo '' + echo '## Live Debugger Runtime Benchmark' + echo + echo 'Benchmark results were not produced. Check the workflow logs for details.' + } > runtime-bench-comment.md + fi + if [ -n "$RESULTS_PATH" ] && [ -f "$RESULTS_PATH" ]; then + cp "$RESULTS_PATH" runtime-bench-results.json + fi + + - name: Find benchmark comment + if: always() + id: benchmark-comment + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + with: + script: | + const marker = ''; + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + const comment = comments.find((candidate) => candidate.body?.includes(marker)); + core.setOutput('comment-id', comment?.id || ''); + + - name: Create benchmark comment + if: always() && steps.benchmark-comment.outputs.comment-id == '' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + with: + script: | + const fs = require('fs'); + const body = fs.readFileSync('runtime-bench-comment.md', 'utf8'); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + + - name: Update benchmark comment + if: always() && steps.benchmark-comment.outputs.comment-id != '' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + with: + script: | + const fs = require('fs'); + const body = fs.readFileSync('runtime-bench-comment.md', 'utf8'); + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: Number('${{ steps.benchmark-comment.outputs.comment-id }}'), + body, + }); - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - if: ${{ failure() }} + if: always() with: - name: playwright + name: live-debugger-runtime-bench-results path: | + runtime-bench-comment.md + runtime-bench-results.json packages/tests/playwright-report packages/tests/test-results retention-days: 3 diff --git a/packages/plugins/live-debugger/CONTRIBUTING.md b/packages/plugins/live-debugger/CONTRIBUTING.md new file mode 100644 index 000000000..231439d49 --- /dev/null +++ b/packages/plugins/live-debugger/CONTRIBUTING.md @@ -0,0 +1,123 @@ +# Contributing to Live Debugger + +Developer notes for the Live Debugger plugin. + +## Table of content + + +- [Development workflow](#development-workflow) +- [Runtime benchmark](#runtime-benchmark) + - [Running it](#running-it) + - [What it measures](#what-it-measures) + - [How to interpret the results](#how-to-interpret-the-results) + - [Methodology](#methodology) + - [Caveats](#caveats) + + +## Development workflow + +Use the [root contributor guide](../../../CONTRIBUTING.md) for repository setup, formatting, and release process. This page covers the Live Debugger-specific checks that are easy to miss when changing the plugin. + +Run the focused unit suite while iterating: + +```bash +yarn test:unit packages/plugins/live-debugger +``` + +Run the package typecheck when changing exported types, option handling, or transform internals: + +```bash +yarn workspace @dd/live-debugger-plugin typecheck +``` + +When changing instrumentation output, add or update cases in [`src/transform/index.test.ts`](./src/transform/index.test.ts). If the generated before/after shape changes in a way users or reviewers should understand, update [`EXAMPLES.md`](./EXAMPLES.md) alongside the tests. + +Changes that affect source positions, injected wrappers, return rewriting, or error handling should preserve source maps. Cover those cases in [`src/sourcemap.integration.test.ts`](./src/sourcemap.integration.test.ts). + +The Babel packages and `magic-string` are optional peer dependencies for consumers. Keep the transform dependencies lazy-loaded and preserve the user-facing missing-dependency error path. Update [`src/transform/lazy-deps.test.ts`](./src/transform/lazy-deps.test.ts) when touching dependency loading. + +Generated code should keep the dormant runtime path small: call `$dd_probes(functionId)` first, and only call `$dd_entry`, `$dd_return`, or `$dd_throw` when a probe is active. Preserve the no-SDK fallback injected from [`src/runtime-bootstrap.ts`](./src/runtime-bootstrap.ts). + +When adding or changing `liveDebugger` configuration, update [`src/types.ts`](./src/types.ts), [`src/validate.ts`](./src/validate.ts), [`src/validate.test.ts`](./src/validate.test.ts), and the consumer-facing [`README.md`](./README.md). + +## Runtime benchmark + +The opt-in browser benchmark measures the dormant runtime overhead added by Live Debugger instrumentation. It compares instrumented code against equivalent uninstrumented code, back-to-back in the same browser session, while SDK-like dormant probe hooks are installed. + +### Running it + +Run it locally with: + +```bash +yarn workspace @dd/tests bench:live-debugger:runtime +``` + +For a faster loop, pass a browser project: + +```bash +yarn workspace @dd/tests bench:live-debugger:runtime --project chrome +``` + +The terminal output prints one row per browser and workload (`Tiny`, `Hot`). Browser projects run serially to reduce CPU contention. The benchmark uses one fixed bundler so the report focuses on runtime overhead, not on bundler-to-bundler differences. + +### What it measures + +Each sample measures three variants: + +- **baseline**: the uninstrumented workload. +- **control**: the same baseline function measured a second time. This is an A/A diagnostic for timing noise in the benchmark apparatus. +- **instrumented**: the same workload after Live Debugger instrumentation, with dormant SDK hooks installed. + +The reporter estimates overhead from `instrumented - control`. That direct paired difference avoids the old correlated-interval comparison against the shared baseline sample. The `control - baseline` result is still shown as the A/A diagnostic; it should be centered around zero if the browser session is quiet enough to trust. + +There are two workloads because one number cannot describe every runtime shape: + +- **Tiny** calls one very small instrumented function. It is the best row for answering: "what is the smallest cost we can measure for one dormant instrumented call?" Since the function does almost no work, its baseline time is tiny too. That means a small nanosecond cost can look like a large percentage. +- **Hot** runs an uninstrumented loop that calls a small instrumented kernel many times. It is the best row for answering: "what happens when an instrumented function sits on a hot path?" This row includes the cost of the dormant hooks and any optimizer disruption from the instrumented function shape, such as losing an inlining opportunity. + +Read them together. `Tiny` shows the minimum cost and the measurement floor. `Hot` shows the repeated-call hot-path cost. If `Hot` is higher than `Tiny` in nanoseconds per call, the gap is the extra cost from the hot-path shape in this benchmark. If `Tiny` is higher in percentage, that usually means the denominator is much smaller, not that `Tiny` has a larger absolute cost. + +### How to interpret the results + +Start with three columns: + +- **per-call overhead upper**: the headline number. It is the conservative upper bound for dormant overhead per instrumented function call, reported in nanoseconds. +- **quality**: whether the row is safe to read. `clean` means use the row, `caution` means it is usable but worth rerunning if the number matters, and `unreliable` means rerun before drawing conclusions. +- **overhead upper**: the same result as a workload-level percentage. Use this as context, not as the main comparison between `Tiny` and `Hot`. + +Prefer the nanosecond number when comparing workloads. It puts `Tiny` and `Hot` on the same per-call scale. The percentage can look inverted because it divides by the workload's baseline time. `Tiny` does almost no work, so a small absolute cost can become a large percentage. `Hot` does more baseline work, so a larger absolute cost can still be a smaller percentage. + +Example: if `Tiny` reports `<= 1.5 ns` and `Hot` reports `<= 5.0 ns`, the benchmark is saying the hot-path shape costs more per instrumented call. If those same rows report `Tiny <= 40%` and `Hot <= 5%`, that does not contradict the nanosecond result. It only means `Tiny` started from a much smaller baseline. + +The other diagnostic columns explain why a row got its `quality` verdict: + +- **95% CI**: the signed estimate range for `instrumented - control`. If it sits near zero, the overhead was too small to resolve clearly. +- **A/A diag**: `control - baseline`. This is the benchmark checking itself by timing the baseline code twice. A small value is fine; a value as large as the measured effect means the browser session was noisy. +- **Block CI** and **acf(1)**: checks for timing drift across samples. If the block interval tells the same story as the main interval, the row is usually fine even when `acf(1)` is non-zero. +- **Samples**: how many samples were recorded, plus trimming and outlier diagnostics. A few outliers are expected in browser timing. The row only becomes suspect when one side has enough outliers to survive the 20% trim. + +The benchmark treats tiny "speed-ups" as measurement noise. Instrumentation only adds work, but separate baseline and instrumented bundles can land in slightly different code layouts. Below about `0.5 ns/call`, that layout noise is roughly the same size as the effect being measured, so the row is reported as clean but unresolved rather than as a real speed-up. + +### Methodology + +The benchmark tries to make each browser comparison fair and repeatable: + +- It serves the page with cross-origin isolation headers so `performance.now()` has better precision. +- It warms up each workload before measuring, then calibrates the batch size, then warms up again with the calibrated size. +- It calibrates against the slowest variant. This keeps the slow instrumented batches from becoming much longer than the baseline batches, which would make the run more vulnerable to JIT warm-up or thermal drift. +- It records `baseline`, `control`, and `instrumented` back-to-back in rotating forward/reverse order. That keeps each variant from always running first, middle, or last. +- It rounds the sample count to a full counterbalancing period, `2 * variantCount`, so every timing position is represented evenly. + +The reported point estimate is a trimmed mean of `instrumented - control`: the benchmark drops the noisiest 20% on each side and averages the middle. Confidence intervals are bootstrapped from the same paired samples. The percentage column uses the same paired data, but divides by the baseline workload time. + +The code uses more specific statistical machinery than this section describes, but the practical rule is simple: trust `clean` rows, rerun `unreliable` rows, and compare `Tiny` and `Hot` primarily on `per-call overhead upper`. + +### Caveats + +Do not compare absolute timings across unrelated machines. Treat the report as a back-to-back comparison from one browser session. + +The benchmark builds separate baseline and instrumented bundles. That is necessary for the comparison, but it also means the browser may lay out or optimize the two bundles slightly differently. The A/A diagnostic can catch noise in the baseline path, but it cannot see every instrumented-bundle-specific effect. This is why `Tiny` and `Hot` should be read as a bracket rather than as one universal overhead number. + +`Tiny` is close to the measurement floor on fast engines. A tiny negative result, especially below about `0.5 ns/call`, should be read as "too small to resolve", not as instrumentation making code faster. + +Browser timings can be spiky or coarsely quantized. The trimmed mean handles ordinary spikes, and the `outliers` reason appears only when the spike pattern is large enough to threaten the estimate. If a row says `unreliable (outliers)`, rerun before trusting it. diff --git a/packages/tests/package.json b/packages/tests/package.json index 37b75eab9..a67d99821 100644 --- a/packages/tests/package.json +++ b/packages/tests/package.json @@ -17,6 +17,7 @@ "./_jest/helpers/*": "./src/_jest/helpers/*.ts" }, "scripts": { + "bench:live-debugger:runtime": "BUILD_PLUGINS_ENV=test FORCE_COLOR=true PLAYWRIGHT_REQUESTED_BUNDLERS=rspack playwright test --config=playwright.live-debugger-runtime.config.ts", "build": "yarn clean && tsc", "clean": "rm -rf dist", "test:e2e": "BUILD_PLUGINS_ENV=test FORCE_COLOR=true playwright test", diff --git a/packages/tests/playwright.live-debugger-runtime.config.ts b/packages/tests/playwright.live-debugger-runtime.config.ts new file mode 100644 index 000000000..e8fdf7df4 --- /dev/null +++ b/packages/tests/playwright.live-debugger-runtime.config.ts @@ -0,0 +1,53 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { PUBLIC_DIR } from '@dd/tests/_playwright/constants'; +import type { TestOptions } from '@dd/tests/_playwright/testParams'; +import { ROOT } from '@dd/tools/constants'; +import { defineConfig, devices } from '@playwright/test'; + +const BENCH_BUNDLER = 'rspack'; +const BENCH_DEV_SERVER_PORT = 8001; +const BENCH_DEV_SERVER_URL = `http://localhost:${BENCH_DEV_SERVER_PORT}`; +const BENCH_BROWSERS = ['chrome', 'firefox', 'safari'] as const; + +const DEVICE_BY_BROWSER = { + chrome: devices['Desktop Chrome'], + firefox: devices['Desktop Firefox'], + safari: devices['Desktop Safari'], +}; + +/** + * Live Debugger runtime benchmarks are opt-in and intentionally live outside + * src/e2e so the normal E2E run does not collect noisy performance data. + */ +export default defineConfig({ + testDir: './src/bench/liveDebuggerRuntime', + testMatch: '**/*.bench.ts', + fullyParallel: false, + workers: 1, + forbidOnly: !!process.env.CI, + retries: 0, + reporter: [['list'], ['./src/bench/liveDebuggerRuntime/reporter/benchReporter.ts']], + globalSetup: require.resolve('./src/_playwright/globalSetup.ts'), + use: { + bundlers: [BENCH_BUNDLER], + trace: 'off', + }, + globalTimeout: process.env.CI ? 20 * 60 * 1000 : undefined, + timeout: 120_000, + projects: BENCH_BROWSERS.map((browserName) => ({ + name: browserName, + use: { + ...DEVICE_BY_BROWSER[browserName], + bundler: BENCH_BUNDLER, + }, + })), + webServer: { + command: `yarn cli dev-server --root=${PUBLIC_DIR} --port=${BENCH_DEV_SERVER_PORT} --cross-origin-isolated`, + cwd: ROOT, + url: BENCH_DEV_SERVER_URL, + reuseExistingServer: !process.env.CI, + }, +}); diff --git a/packages/tests/src/bench/liveDebuggerRuntime/liveDebuggerBenchConfig.js b/packages/tests/src/bench/liveDebuggerRuntime/liveDebuggerBenchConfig.js new file mode 100644 index 000000000..7d0f5943f --- /dev/null +++ b/packages/tests/src/bench/liveDebuggerRuntime/liveDebuggerBenchConfig.js @@ -0,0 +1,22 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +// Live Debugger plugin options shared by the benchmark build (run through the +// bundler harness in liveDebuggerRuntime.bench.ts) and the CI preflight build +// (run through Rspack directly in preflight-build.js). Sharing the object is +// what keeps the preflight's "did the instrumented output change" hash gated on +// the same instrumentation the benchmark actually measures. + +/** + * @param {boolean} enable + */ +const getLiveDebuggerBenchConfig = (enable) => { + return { + enable, + include: [/workload\.js$/], + namedOnly: true, + }; +}; + +module.exports = { getLiveDebuggerBenchConfig }; diff --git a/packages/tests/src/bench/liveDebuggerRuntime/liveDebuggerRuntime.bench.ts b/packages/tests/src/bench/liveDebuggerRuntime/liveDebuggerRuntime.bench.ts new file mode 100644 index 000000000..5de42dc84 --- /dev/null +++ b/packages/tests/src/bench/liveDebuggerRuntime/liveDebuggerRuntime.bench.ts @@ -0,0 +1,234 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +/* eslint-env browser */ +/* global globalThis */ +import { existsSync, mkdir, outputFile, rm } from '@dd/core/helpers/fs'; +import { verifyProjectBuild } from '@dd/tests/_playwright/helpers/buildProject'; +import type { TestOptions } from '@dd/tests/_playwright/testParams'; +import { test } from '@dd/tests/_playwright/testParams'; +import { defaultConfig } from '@dd/tools/plugins'; +import type { Page } from '@playwright/test'; +import fs from 'fs'; +import path from 'path'; + +import { getLiveDebuggerBenchConfig } from './liveDebuggerBenchConfig'; +import type { BenchVariant, BrowserBenchApi } from './types'; + +/* eslint-disable no-var, vars-on-top */ +declare global { + var ddBench: Record | undefined; + var ddActiveProbes: Record; + var crossOriginIsolated: boolean; + var $dd_probes: (functionId: string) => unknown[] | undefined; + var $dd_entry: (probes: unknown[], self: unknown, args?: Record) => void; + var $dd_return: ( + probes: unknown[], + value: unknown, + self: unknown, + args?: Record, + locals?: Record, + ) => unknown; + var $dd_throw: ( + probes: unknown[], + error: unknown, + self: unknown, + args?: Record, + ) => void; +} +/* eslint-enable no-var, vars-on-top */ + +const { expect, beforeAll, describe } = test; + +const BENCH_OPTIONS = { + calibrationAttempts: 8, + minBatchMs: 50, + warmupMs: process.env.CI ? 250 : 300, +}; + +const copyFixtureShell = async (source: string, destination: string) => { + await fs.promises.cp(`${source}/`, `${destination}/`, { + recursive: true, + force: true, + }); +}; + +// Build baseline and instrumented variants separately, then serve their dist outputs +// from one fixture root so both bundles run back-to-back in the same browser page. +const buildVariant = async ( + source: string, + rootDestination: string, + variant: BenchVariant, + bundlers: TestOptions['bundlers'], +) => { + const variantDestination = path.resolve(rootDestination, variant); + const enableLiveDebugger = variant === 'instrumented'; + const entry = `./${variant}.js`; + + await verifyProjectBuild( + source, + variantDestination, + bundlers, + { + ...defaultConfig, + liveDebugger: getLiveDebuggerBenchConfig(enableLiveDebugger), + }, + { + entry: { + webpack: entry, + vite: entry, + esbuild: entry, + rollup: entry, + rspack: entry, + }, + }, + ); + + const sourceDist = path.resolve(variantDestination, 'dist'); + if (!existsSync(sourceDist)) { + throw new Error(`Live Debugger benchmark ${variant} build did not produce ${sourceDist}`); + } + + const destinationDist = path.resolve(rootDestination, 'dist', variant); + await rm(destinationDist); + await mkdir(path.dirname(destinationDist)); + await fs.promises.rename(sourceDist, destinationDist); + await rm(variantDestination); +}; + +const buildBenchProject = async ( + source: string, + destination: string, + bundlers: TestOptions['bundlers'], +) => { + if (existsSync(path.resolve(destination, 'built'))) { + return; + } + + try { + await rm(destination); + await copyFixtureShell(source, destination); + await buildVariant(source, destination, 'baseline', bundlers); + await buildVariant(source, destination, 'instrumented', bundlers); + await outputFile(path.resolve(destination, 'built'), ''); + } catch (error) { + await rm(destination); + throw error; + } +}; + +const userFlow = async (url: string, page: Page, bundler: TestOptions['bundler']) => { + await page.goto(`${url}/index.html?context_bundler=${bundler}`); + await page.waitForSelector('#status'); + await page.waitForFunction(() => { + const bench = globalThis.ddBench; + + return Boolean(bench?.baseline && bench.instrumented); + }); + + return page.evaluate(() => { + return globalThis.crossOriginIsolated; + }); +}; + +const installDormantDebuggerSdkHooks = async (page: Page) => { + await page.addInitScript(() => { + globalThis.ddActiveProbes = { + // Mirror the Browser SDK by adding a __placeholder__ key. + __placeholder__: undefined, + }; + globalThis.$dd_probes = (functionId) => globalThis.ddActiveProbes[functionId]; + globalThis.$dd_entry = () => {}; + globalThis.$dd_return = (_probes, value) => value; + globalThis.$dd_throw = () => {}; + }); +}; + +describe('Live Debugger Runtime Benchmark', () => { + beforeAll(async ({ publicDir, bundlers, suiteName }) => { + const source = path.resolve(__dirname, 'project'); + const destination = path.resolve(publicDir, suiteName); + await buildBenchProject(source, destination, bundlers); + }); + + test('Measures SDK-loaded dormant runtime overhead', async ({ + page, + bundler, + browserName, + suiteName, + devServerUrl, + }, testInfo) => { + const errors: string[] = []; + const projectName = testInfo.project.name; + const testBaseUrl = `${devServerUrl}/${suiteName}`; + + page.on('pageerror', (error) => errors.push(error.message)); + page.on('console', (msg) => { + if (msg.type() === 'error') { + errors.push(`[console error] ${msg.text()}`); + } + }); + page.on('response', async (response) => { + if (!response.ok()) { + const url = response.request().url(); + const prefix = `[${browserName} ${response.status()}]`; + errors.push(`${prefix} ${url}`); + } + }); + + await installDormantDebuggerSdkHooks(page); + + const crossOriginIsolated = await userFlow(testBaseUrl, page, bundler); + expect(crossOriginIsolated).toBe(true); + + const results = await page.evaluate((options) => { + const bench = globalThis.ddBench; + if (!bench) { + throw new Error('Missing benchmark globals'); + } + + return bench.baseline.workloads.map((workload) => { + const baseline = bench.baseline.workloads.find( + (candidate) => candidate.id === workload.id, + ); + const instrumented = bench.instrumented.workloads.find( + (candidate) => candidate.id === workload.id, + ); + + if (!baseline || !instrumented) { + throw new Error(`Missing benchmark workload: ${workload.id}`); + } + + return bench.baseline.runBenchPair( + workload, + [ + { id: 'baseline', fn: baseline.fn }, + { id: 'control', fn: baseline.fn }, + { id: 'instrumented', fn: instrumented.fn }, + ], + { + ...options, + batchSize: workload.batchSize, + samples: workload.samples, + }, + ); + }); + }, BENCH_OPTIONS); + + await testInfo.attach('live-debugger-runtime-bench', { + body: JSON.stringify( + { + browserName: projectName || browserName, + results, + }, + null, + 2, + ), + contentType: 'application/json', + }); + + expect(results).toHaveLength(2); + expect(errors).toEqual([]); + }); +}); diff --git a/packages/tests/src/bench/liveDebuggerRuntime/preflight-build.js b/packages/tests/src/bench/liveDebuggerRuntime/preflight-build.js new file mode 100644 index 000000000..958e81489 --- /dev/null +++ b/packages/tests/src/bench/liveDebuggerRuntime/preflight-build.js @@ -0,0 +1,74 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +/* eslint-disable no-console */ + +// Standalone Rspack build for the preflight output hash. Kept as a real file +// (invoked as `node preflight-build.js `) rather than an +// inline `node -e` string so the build config stays lintable and the live +// debugger options can be shared with the benchmark. + +const path = require('path'); + +const { datadogRspackPlugin } = require('@datadog/rspack-plugin/dist/src'); +const { rspack } = require('@rspack/core'); + +const { getLiveDebuggerBenchConfig } = require('./liveDebuggerBenchConfig'); + +const fixturePath = process.argv[2]; +const bundler = process.argv[3]; + +if (!fixturePath || !bundler) { + console.error('Usage: node preflight-build.js '); + process.exit(1); +} + +const plugin = datadogRspackPlugin({ + auth: { apiKey: '123', appKey: '123' }, + metadata: { name: path.basename(fixturePath) }, + liveDebugger: getLiveDebuggerBenchConfig(true), +}); + +const config = { + context: fixturePath, + entry: { [bundler]: path.resolve(fixturePath, 'instrumented.js') }, + experiments: { + css: true, + }, + mode: 'none', + output: { + path: path.resolve(fixturePath, 'dist'), + filename: '[name].js', + chunkFilename: 'chunk.[contenthash].js', + }, + devtool: 'source-map', + optimization: { + minimize: false, + }, + plugins: [plugin], + resolve: { + extensions: ['.tsx', '.ts', '.js'], + }, + module: { + rules: [{ test: /\.([cm]?ts|tsx)$/, loader: 'ts-loader' }], + }, +}; + +rspack(config, (error, stats) => { + if (error) { + console.error(error); + process.exit(1); + } + + if (!stats) { + console.error('No Rspack stats returned.'); + process.exit(1); + } + + if (stats.hasErrors()) { + const info = stats.toJson(); + console.error((info.errors || []).join('\n')); + process.exit(1); + } +}); diff --git a/packages/tests/src/bench/liveDebuggerRuntime/preflight.js b/packages/tests/src/bench/liveDebuggerRuntime/preflight.js new file mode 100644 index 000000000..72a098726 --- /dev/null +++ b/packages/tests/src/bench/liveDebuggerRuntime/preflight.js @@ -0,0 +1,199 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +/* eslint-disable no-console */ + +const childProcess = require('child_process'); +const crypto = require('crypto'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const BENCH_BUNDLER = 'rspack'; +const BENCH_OUTPUT_ENTRY = `${BENCH_BUNDLER}.js`; +const FIXTURE_SOURCE = path.resolve(__dirname, 'project'); +const ROOT = path.resolve(__dirname, '../../../../..'); +const WORKSPACE_NAME = '@datadog/rspack-plugin'; + +const forceRunPaths = [ + '.github/actions/setup-playwright-build/', + '.github/workflows/ci.yaml', + 'package.json', + 'packages/tests/package.json', + 'packages/tests/playwright.live-debugger-runtime.config.ts', + 'packages/tests/src/_playwright/', + 'packages/tests/src/bench/liveDebuggerRuntime/', + 'packages/tools/src/bundlers.ts', + 'packages/tools/src/commands/dev-server/', + 'packages/tools/src/plugins.ts', + 'yarn.lock', +]; + +function exec(command, options = {}) { + return childProcess.execFileSync(command[0], command.slice(1), { + cwd: options.cwd || ROOT, + encoding: 'utf8', + env: { + ...process.env, + FORCE_COLOR: 'true', + PROJECT_CWD: ROOT, + ...options.env, + }, + stdio: options.stdio || 'pipe', + }); +} + +function getArgValue(name) { + const prefix = `${name}=`; + const match = process.argv.slice(2).find((arg) => arg.startsWith(prefix)); + + return match ? match.slice(prefix.length) : undefined; +} + +function hasArg(name) { + return process.argv.includes(name); +} + +function appendGithubOutput(values) { + if (!process.env.GITHUB_OUTPUT) { + return; + } + + const lines = Object.entries(values).map(([key, value]) => `${key}=${value}`); + fs.appendFileSync(process.env.GITHUB_OUTPUT, `${lines.join('\n')}\n`); +} + +function normalizeChangedPath(filePath) { + return filePath.replace(/\\/g, '/'); +} + +function getChangedFiles(baseRef) { + const diffOutput = exec(['git', 'diff', '--name-only', `${baseRef}...HEAD`]); + + return diffOutput.split('\n').filter(Boolean).map(normalizeChangedPath); +} + +function getForceRunReason(changedFiles) { + const matchingPath = changedFiles.find((filePath) => + forceRunPaths.some((forceRunPath) => { + if (forceRunPath.endsWith('/')) { + return filePath.startsWith(forceRunPath); + } + + return filePath === forceRunPath; + }), + ); + + return matchingPath ? `changed:${matchingPath}` : ''; +} + +function copyFixture(destination) { + fs.cpSync(FIXTURE_SOURCE, destination, { + recursive: true, + force: true, + }); +} + +function cleanPublishedPlugin() { + exec(['yarn', 'workspace', WORKSPACE_NAME, 'clean'], { stdio: 'inherit' }); +} + +function buildPublishedPlugin() { + exec(['yarn', 'workspace', WORKSPACE_NAME, 'build'], { + env: { + NO_TYPES: '1', + }, + stdio: 'inherit', + }); +} + +function buildFixture(fixturePath) { + const buildScript = path.resolve(__dirname, 'preflight-build.js'); + + exec(['node', buildScript, fixturePath, BENCH_BUNDLER], { stdio: 'inherit' }); +} + +function hashOutput(fixturePath) { + const outputPath = path.resolve(fixturePath, 'dist', BENCH_OUTPUT_ENTRY); + const code = fs.readFileSync(outputPath); + + return crypto.createHash('sha256').update(code).digest('hex'); +} + +function captureOutputHash(worktreeRef, tempRoot) { + const fixturePath = path.resolve(tempRoot, 'fixture'); + + exec(['git', 'checkout', '--quiet', worktreeRef], { stdio: 'inherit' }); + fs.rmSync(fixturePath, { recursive: true, force: true }); + copyFixture(fixturePath); + cleanPublishedPlugin(); + buildPublishedPlugin(); + buildFixture(fixturePath); + + return hashOutput(fixturePath); +} + +function writeResult(shouldRun, reason) { + console.log(`should-run=${shouldRun}`); + console.log(`reason=${reason}`); + appendGithubOutput({ + 'should-run': shouldRun ? 'true' : 'false', + reason, + }); +} + +function runChangedFilesMode(baseRef) { + const changedFiles = getChangedFiles(baseRef); + const reason = getForceRunReason(changedFiles); + + writeResult(Boolean(reason), reason); +} + +function runCompareOutputMode(baseRef) { + const originalRef = exec(['git', 'rev-parse', 'HEAD']).trim(); + const originalBranch = exec(['git', 'branch', '--show-current']).trim(); + const restoreRef = originalBranch || originalRef; + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ld-runtime-bench-preflight-')); + + try { + const baseHash = captureOutputHash(baseRef, tempRoot); + const headHash = captureOutputHash(originalRef, tempRoot); + const changed = baseHash !== headHash; + const reason = changed ? 'build-output-changed' : 'build-output-unchanged'; + + console.log(`base-hash=${baseHash}`); + console.log(`head-hash=${headHash}`); + writeResult(changed, reason); + } finally { + exec(['git', 'checkout', '--quiet', restoreRef], { stdio: 'inherit' }); + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +} + +function main() { + const baseRef = getArgValue('--base-ref') || 'origin/main'; + + if (hasArg('--changed-files')) { + runChangedFilesMode(baseRef); + return; + } + + if (hasArg('--compare-output')) { + runCompareOutputMode(baseRef); + return; + } + + throw new Error('Expected --changed-files or --compare-output.'); +} + +try { + main(); +} catch (error) { + if (!process.env.CI) { + throw error; + } + + console.error(error); + writeResult(true, 'preflight-error'); +} diff --git a/packages/tests/src/bench/liveDebuggerRuntime/project/baseline.js b/packages/tests/src/bench/liveDebuggerRuntime/project/baseline.js new file mode 100644 index 000000000..3b2a70fa6 --- /dev/null +++ b/packages/tests/src/bench/liveDebuggerRuntime/project/baseline.js @@ -0,0 +1,9 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +/* eslint-env browser */ + +import { registerBenchVariant } from './index.js'; + +registerBenchVariant('baseline'); diff --git a/packages/tests/src/bench/liveDebuggerRuntime/project/harness.js b/packages/tests/src/bench/liveDebuggerRuntime/project/harness.js new file mode 100644 index 000000000..db45d40f0 --- /dev/null +++ b/packages/tests/src/bench/liveDebuggerRuntime/project/harness.js @@ -0,0 +1,159 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +let sink = 0; + +// Kept in the browser harness so the benchmark fixture remains self-contained. +const median = (values) => { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + + if (sorted.length % 2 === 0) { + return (sorted[mid - 1] + sorted[mid]) / 2; + } + + return sorted[mid]; +}; + +const runBatch = (workloadFn, batchSize, sampleIndex) => { + let localSink = sink; + + for (let i = 0; i < batchSize; i++) { + localSink += workloadFn(sampleIndex + i); + } + + sink = localSink; +}; + +const timeBatch = (workloadFn, batchSize, sampleIndex) => { + const start = performance.now(); + runBatch(workloadFn, batchSize, sampleIndex); + + return performance.now() - start; +}; + +const calibrateBatchSize = (variants, options) => { + let batchSize = options.batchSize; + + for (let attempt = 0; attempt < options.calibrationAttempts; attempt++) { + const durations = []; + for (let index = 0; index < variants.length; index++) { + durations.push(timeBatch(variants[index].fn, batchSize, index)); + } + // Calibrate against the slowest variant rather than the median. The + // three variants share one batch size, but two of them run the cheap + // baseline function, so a median-based target grows the batch until the + // baseline reaches minBatchMs. The instrumented variant can be an order + // of magnitude slower, so its batches then run for hundreds of + // milliseconds and the whole sweep lasts long enough for the + // instrumented path to drift (JIT warm-up then thermal throttling). + // That drift does not cancel in the within-sample pairing because the + // baseline batches stay short, which shows up as severe + // autocorrelation. Sizing to the slowest variant keeps every batch + // near the target and the sweep short enough to stay stationary. + const slowestDuration = Math.max(...durations); + + if (slowestDuration >= options.minBatchMs) { + return batchSize; + } + + const multiplier = Math.max( + 2, + Math.ceil(options.minBatchMs / Math.max(slowestDuration, 0.1)), + ); + batchSize *= multiplier; + } + + return batchSize; +}; + +const warmup = (variants, options) => { + const start = performance.now(); + let iteration = 0; + + while (performance.now() - start < options.warmupMs) { + const rotatedVariants = getCounterbalancedVariantOrder(variants, iteration); + for (const variant of rotatedVariants) { + runBatch(variant.fn, options.batchSize, iteration); + } + iteration += options.batchSize; + } +}; + +const rotateVariants = (variants, rotationIndex) => { + const rotation = rotationIndex % variants.length; + + return variants.slice(rotation).concat(variants.slice(0, rotation)); +}; + +export const getCounterbalancingPeriod = (variantCount) => { + return 2 * variantCount; +}; + +export const roundSamplesToCounterbalancingPeriod = (samples, variantCount) => { + const period = getCounterbalancingPeriod(variantCount); + + return Math.ceil(samples / period) * period; +}; + +export const getCounterbalancedVariantOrder = (variants, sampleIndex) => { + const rotation = sampleIndex % variants.length; + const rotatedVariants = rotateVariants(variants, rotation); + + if (Math.floor(sampleIndex / variants.length) % 2 === 0) { + return rotatedVariants; + } + + return [rotatedVariants[0], ...rotatedVariants.slice(1).reverse()]; +}; + +export const runBenchPair = (workload, variants, options = {}) => { + const benchmarkOptions = { + warmupMs: options.warmupMs ?? 300, + batchSize: options.batchSize ?? workload.batchSize, + calibrationAttempts: options.calibrationAttempts ?? 8, + minBatchMs: options.minBatchMs ?? 50, + samples: roundSamplesToCounterbalancingPeriod(options.samples ?? 35, variants.length), + }; + const counterbalancingPeriod = getCounterbalancingPeriod(variants.length); + if (benchmarkOptions.samples % counterbalancingPeriod !== 0) { + throw new Error( + `Benchmark sample count must be a multiple of the counterbalancing period (${counterbalancingPeriod})`, + ); + } + + const samplesByVariant = Object.fromEntries(variants.map((variant) => [variant.id, []])); + + warmup(variants, benchmarkOptions); + benchmarkOptions.batchSize = calibrateBatchSize(variants, benchmarkOptions); + warmup(variants, benchmarkOptions); + + for (let sampleIndex = 0; sampleIndex < benchmarkOptions.samples; sampleIndex++) { + const rotatedVariants = getCounterbalancedVariantOrder(variants, sampleIndex); + for (const variant of rotatedVariants) { + const elapsedMs = timeBatch(variant.fn, benchmarkOptions.batchSize, sampleIndex); + samplesByVariant[variant.id].push(elapsedMs); + } + } + + const instrumentedCallsPerBatch = + workload.instrumentedCallsPerInvocation * benchmarkOptions.batchSize; + + return { + workloadId: workload.id, + workloadLabel: workload.label, + batchSize: benchmarkOptions.batchSize, + instrumentedCallsPerBatch, + sink, + variants: Object.fromEntries( + variants.map((variant) => [ + variant.id, + { + samplesMs: samplesByVariant[variant.id], + medianMs: median(samplesByVariant[variant.id]), + }, + ]), + ), + }; +}; diff --git a/packages/tests/src/bench/liveDebuggerRuntime/project/index.html b/packages/tests/src/bench/liveDebuggerRuntime/project/index.html new file mode 100644 index 000000000..6653cec36 --- /dev/null +++ b/packages/tests/src/bench/liveDebuggerRuntime/project/index.html @@ -0,0 +1,17 @@ + + + + + + + Live Debugger Runtime Benchmark + + + +

Live Debugger Runtime Benchmark - {{bundler}}

+

Ready

+ + + + + diff --git a/packages/tests/src/bench/liveDebuggerRuntime/project/index.js b/packages/tests/src/bench/liveDebuggerRuntime/project/index.js new file mode 100644 index 000000000..a8bf199ab --- /dev/null +++ b/packages/tests/src/bench/liveDebuggerRuntime/project/index.js @@ -0,0 +1,21 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +/* eslint-env browser */ +/* global globalThis */ + +import { runBenchPair } from './harness.js'; +import { workloads } from './workload.js'; + +export function registerBenchVariant(variant) { + if (!globalThis['ddBench']) { + globalThis['ddBench'] = {}; + } + + globalThis['ddBench'][variant] = { + runBenchPair, + workloads, + variant, + }; +} diff --git a/packages/tests/src/bench/liveDebuggerRuntime/project/instrumented.js b/packages/tests/src/bench/liveDebuggerRuntime/project/instrumented.js new file mode 100644 index 000000000..dfac1980f --- /dev/null +++ b/packages/tests/src/bench/liveDebuggerRuntime/project/instrumented.js @@ -0,0 +1,9 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +/* eslint-env browser */ + +import { registerBenchVariant } from './index.js'; + +registerBenchVariant('instrumented'); diff --git a/packages/tests/src/bench/liveDebuggerRuntime/project/workload.js b/packages/tests/src/bench/liveDebuggerRuntime/project/workload.js new file mode 100644 index 000000000..e3fb81cda --- /dev/null +++ b/packages/tests/src/bench/liveDebuggerRuntime/project/workload.js @@ -0,0 +1,52 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +const HOT_CALLS = 256; + +// The harness counterbalances baseline, control, and instrumented over +// 2 * variantCount samples. With 3 variants, the period is 6, so the +// closest complete period above 100 samples is ceil(100 / 6) * 6 = 102. +const WORKLOAD_SAMPLES = 102; + +function tinyWorkload(iteration) { + const value = (iteration + 3) * 1.00001; + + return value; +} + +function hotKernel(a, b, c) { + const blended = a * 1.5 + b - c; + const folded = (blended * 0.25 + a) % 9_973; + + return folded + b * 0.5; +} + +// @dd-no-instrumentation +function hotLoopWorkload(iteration) { + let acc = 0; + for (let i = 0; i < HOT_CALLS; i++) { + acc += hotKernel(iteration + i, acc, i); + } + + return acc; +} + +export const workloads = [ + { + id: 'tiny', + label: 'Tiny', + instrumentedCallsPerInvocation: 1, + batchSize: 20_000, + samples: WORKLOAD_SAMPLES, + fn: tinyWorkload, + }, + { + id: 'hot', + label: 'Hot', + instrumentedCallsPerInvocation: HOT_CALLS, + batchSize: 2_000, + samples: WORKLOAD_SAMPLES, + fn: hotLoopWorkload, + }, +]; diff --git a/packages/tests/src/bench/liveDebuggerRuntime/reporter/benchReporter.test.ts b/packages/tests/src/bench/liveDebuggerRuntime/reporter/benchReporter.test.ts new file mode 100644 index 000000000..1348e9dfd --- /dev/null +++ b/packages/tests/src/bench/liveDebuggerRuntime/reporter/benchReporter.test.ts @@ -0,0 +1,100 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import type { BenchResultRow } from '../types'; + +import { buildAlignedTable, renderMarkdownComment } from './benchReporter'; + +const createBenchResultRow = (): BenchResultRow => { + return { + browserName: 'chrome', + workloadId: 'tiny', + workloadLabel: 'Tiny', + batchSize: 1_000, + instrumentedCallsPerBatch: 1_000, + baseline: { + samplesMs: [1], + medianMs: 1.2345, + }, + control: { + samplesMs: [1], + medianMs: 1.2345, + }, + instrumented: { + samplesMs: [2], + medianMs: 2.3456, + }, + perCallNs: { + point: 1, + direct: { + low: -0.04, + high: 0.03, + }, + block: { + low: -0.05, + high: 0.04, + }, + aaFloor: { + low: -0.03, + high: 0.03, + }, + upperBound: 0.05, + }, + overheadUpperPercent: 1.23, + sampleCount: 30, + trimFraction: 0.2, + outlierFraction: 0.1, + lag1Autocorrelation: 0.01, + quality: { level: 'caution', reasons: ['outliers'] }, + }; +}; + +describe('Live Debugger runtime benchmark reporter', () => { + test('should render CLI diagnostics with units in row values', () => { + const row = createBenchResultRow(); + const table = buildAlignedTable([row]); + const header = table.split('\n')[0]; + const headerCells = header.trim().split(/\s{2,}/); + + expect(headerCells).toEqual([ + 'browser', + 'workload', + 'quality', + 'per-call overhead upper', + 'overhead upper', + '95% CI', + 'A/A diag', + 'block CI', + 'acf(1)', + 'baseline', + 'instrumented', + 'samples', + ]); + expect(table).toContain('chrome Tiny caution (outliers)'); + expect(table).toContain('<= 0.05 ns'); + expect(table).toContain('<= 1.23%'); + expect(table).toContain('-0.04..0.03 ns'); + expect(table).toContain('1.235 ms'); + expect(table).toContain('2.346 ms'); + }); + + test('should render GitHub comment summary and diagnostics', () => { + const row = createBenchResultRow(); + const comment = renderMarkdownComment([row], []); + const summary = comment.split('
')[0]; + + expect(comment).toContain( + 'SDK-loaded dormant-probe runtime overhead, measured against an uninstrumented bundle in the same browser session.', + ); + expect(comment).toContain('| Browser | Workload | Quality | Per-call overhead upper |'); + expect(comment).toContain('Full diagnostics'); + expect(comment).toContain('| chrome | Tiny | caution (outliers) | <= 0.05 ns |'); + expect(summary).not.toContain('<= 1.23%'); + expect(summary).not.toContain('| overhead upper |'); + expect(comment).toContain('1.235 ms'); + expect(comment).toContain('-0.04..0.03 ns'); + expect(comment).toContain('<= 1.23%'); + expect(comment).toContain('overhead upper'); + }); +}); diff --git a/packages/tests/src/bench/liveDebuggerRuntime/reporter/benchReporter.ts b/packages/tests/src/bench/liveDebuggerRuntime/reporter/benchReporter.ts new file mode 100644 index 000000000..be7982ec9 --- /dev/null +++ b/packages/tests/src/bench/liveDebuggerRuntime/reporter/benchReporter.ts @@ -0,0 +1,463 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { outputFileSync, readFileSync } from '@dd/core/helpers/fs'; +import type { FullResult, Reporter, TestCase, TestResult } from '@playwright/test/reporter'; +import os from 'os'; +import path from 'path'; + +import type { + BenchFailure, + BenchResultRow, + MetricUnit, + RawBenchAttachment, + RawVariantResult, + RawWorkloadResult, +} from '../types'; + +import { + BOOTSTRAP_ITERATIONS, + TRIM_FRACTION, + bcaConfidenceInterval, + bcaIndexConfidenceInterval, + getQuality, + hashString, + lag1Autocorrelation, + madOutlierFraction, + movingBlockBootstrapConfidenceInterval, + movingBlockIndexBootstrapConfidenceInterval, + overheadUpperBound, + pick, + trimmedMean, +} from './stats'; + +const ATTACHMENT_NAME = 'live-debugger-runtime-bench'; +const COMMENT_MARKER = ''; +const COMMENT_FILE = path.resolve(os.tmpdir(), 'live-debugger-runtime-bench-comment.md'); + +const buildResultsFilePath = (generatedAt: string) => { + const safeTimestamp = generatedAt.replace(/[:.]/g, '-'); + return path.resolve(os.tmpdir(), `live-debugger-runtime-bench-results-${safeTimestamp}.json`); +}; + +const parseAttachmentBody = (result: TestResult): RawBenchAttachment[] => { + const attachments = result.attachments.filter( + (attachment) => attachment.name === ATTACHMENT_NAME, + ); + const parsedAttachments: RawBenchAttachment[] = []; + + for (const attachment of attachments) { + if (attachment.body) { + parsedAttachments.push(JSON.parse(attachment.body.toString())); + } else if (attachment.path) { + const fileContent = readFileSync(attachment.path); + parsedAttachments.push(JSON.parse(fileContent)); + } + } + + return parsedAttachments; +}; + +const getSampleDeltasMs = (left: RawVariantResult, right: RawVariantResult) => { + return left.samplesMs.map((sample, index) => { + return sample - right.samplesMs[index]; + }); +}; + +const getBlockLength = (sampleCount: number) => { + return Math.max(1, Math.round(Math.sqrt(sampleCount))); +}; + +const toRatioPercent = (deltasMs: number[], baselineSamplesMs: number[]) => { + const baselineLevelMs = trimmedMean(baselineSamplesMs, TRIM_FRACTION); + + if (baselineLevelMs <= 0) { + return 0; + } + + return (trimmedMean(deltasMs, TRIM_FRACTION) / baselineLevelMs) * 100; +}; + +type MetricEstimate = Omit; + +const withUpperBound = (estimate: MetricEstimate): MetricUnit => { + return { + ...estimate, + upperBound: overheadUpperBound(estimate.direct, estimate.block), + }; +}; + +const mapMetricUnit = (unit: MetricEstimate, mapper: (value: number) => number): MetricUnit => { + const direct = { + low: mapper(unit.direct.low), + high: mapper(unit.direct.high), + }; + const block = { + low: mapper(unit.block.low), + high: mapper(unit.block.high), + }; + + return withUpperBound({ + point: mapper(unit.point), + direct, + block, + aaFloor: { + low: mapper(unit.aaFloor.low), + high: mapper(unit.aaFloor.high), + }, + }); +}; + +type SeedFor = (suffix: string) => number; + +// Per-call overhead in ns: a 20% trimmed mean of `instrumented - control` with +// a naive (BCa) and dependence-robust (moving-block) interval, plus the +// `control - baseline` A/A floor, all rescaled from ms-per-batch to ns-per-call. +const buildPerCallNsMetric = ( + directDeltasMs: number[], + aaDeltasMs: number[], + instrumentedCallsPerBatch: number, + blockLength: number, + seedFor: SeedFor, +): MetricUnit => { + const toNsPerCall = (valueMs: number) => { + return (valueMs * 1_000_000) / instrumentedCallsPerBatch; + }; + const directSeed = seedFor(''); + const blockSeed = seedFor(':block'); + const aaSeed = seedFor(':control'); + const msMetric: MetricEstimate = { + point: trimmedMean(directDeltasMs, TRIM_FRACTION), + direct: bcaConfidenceInterval(directDeltasMs, BOOTSTRAP_ITERATIONS, directSeed), + block: movingBlockBootstrapConfidenceInterval( + directDeltasMs, + BOOTSTRAP_ITERATIONS, + blockSeed, + blockLength, + ), + aaFloor: bcaConfidenceInterval(aaDeltasMs, BOOTSTRAP_ITERATIONS, aaSeed), + }; + + return mapMetricUnit(msMetric, toNsPerCall); +}; + +// Workload-level upper bound expressed as a percentage of the baseline. Only the +// bound is published, so the percent direct/block intervals exist solely to feed +// it; the paired ratio is bootstrapped directly so the baseline denominator's +// own sampling uncertainty is propagated. +const computeOverheadUpperPercent = ( + directDeltasMs: number[], + baselineSamplesMs: number[], + blockLength: number, + seedFor: SeedFor, +): number => { + const ratioEstimator = (indices: number[]) => { + return toRatioPercent(pick(directDeltasMs, indices), pick(baselineSamplesMs, indices)); + }; + const directSeed = seedFor(':percent'); + const blockSeed = seedFor(':percent:block'); + const direct = bcaIndexConfidenceInterval( + directDeltasMs.length, + BOOTSTRAP_ITERATIONS, + directSeed, + 0.95, + ratioEstimator, + ); + const block = movingBlockIndexBootstrapConfidenceInterval( + directDeltasMs.length, + BOOTSTRAP_ITERATIONS, + blockSeed, + blockLength, + 0.95, + ratioEstimator, + ); + + return overheadUpperBound(direct, block); +}; + +const toRow = (attachment: RawBenchAttachment, result: RawWorkloadResult): BenchResultRow => { + const baselineSamplesMs = result.variants.baseline.samplesMs; + const directDeltasMs = getSampleDeltasMs(result.variants.instrumented, result.variants.control); + const aaDeltasMs = getSampleDeltasMs(result.variants.control, result.variants.baseline); + const blockLength = getBlockLength(directDeltasMs.length); + // Every bootstrap for this row draws from a distinct but reproducible seed + // derived from the browser/workload pair, so reruns are byte-stable. + const seedFor: SeedFor = (suffix) => { + return hashString(`${attachment.browserName}:${result.workloadId}${suffix}`); + }; + + const perCallNs = buildPerCallNsMetric( + directDeltasMs, + aaDeltasMs, + result.instrumentedCallsPerBatch, + blockLength, + seedFor, + ); + const overheadUpperPercent = computeOverheadUpperPercent( + directDeltasMs, + baselineSamplesMs, + blockLength, + seedFor, + ); + const outlierFraction = madOutlierFraction(directDeltasMs); + const acf1 = lag1Autocorrelation(directDeltasMs); + // Quality is judged on the per-call ns scale, not percent: the noise it + // tolerates (cross-bundle layout, codegen, timer granularity) is an + // absolute per-call quantity, and percent collapses a real 3.6 ns/call + // effect and 0.15 ns/call noise onto the same ~3.5% (see stats.ts). + const quality = getQuality({ + direct: perCallNs.direct, + block: perCallNs.block, + aaFloor: perCallNs.aaFloor, + outlierFraction, + }); + + return { + browserName: attachment.browserName, + workloadId: result.workloadId, + workloadLabel: result.workloadLabel, + batchSize: result.batchSize, + instrumentedCallsPerBatch: result.instrumentedCallsPerBatch, + baseline: result.variants.baseline, + control: result.variants.control, + instrumented: result.variants.instrumented, + perCallNs, + overheadUpperPercent, + sampleCount: directDeltasMs.length, + trimFraction: TRIM_FRACTION, + outlierFraction, + lag1Autocorrelation: acf1, + quality, + }; +}; + +const toRows = (attachment: RawBenchAttachment): BenchResultRow[] => { + return attachment.results.map((result) => toRow(attachment, result)); +}; + +const formatNumber = (value: number, fractionDigits: number) => { + return value.toLocaleString('en-US', { + maximumFractionDigits: fractionDigits, + minimumFractionDigits: fractionDigits, + }); +}; + +const formatCallOverhead = (row: BenchResultRow) => { + return `<= ${formatNumber(row.perCallNs.upperBound, 2)} ns`; +}; + +const formatCallOverheadConfidenceInterval = (row: BenchResultRow) => { + const low = formatNumber(row.perCallNs.direct.low, 2); + const high = formatNumber(row.perCallNs.direct.high, 2); + + return `${low}..${high} ns`; +}; + +const formatUpperBoundPercent = (row: BenchResultRow) => { + return `<= ${formatNumber(row.overheadUpperPercent, 2)}%`; +}; + +const formatNoiseFloorConfidenceInterval = (row: BenchResultRow) => { + const low = formatNumber(row.perCallNs.aaFloor.low, 2); + const high = formatNumber(row.perCallNs.aaFloor.high, 2); + + return `${low}..${high} ns`; +}; + +const formatBlockConfidenceInterval = (row: BenchResultRow) => { + const low = formatNumber(row.perCallNs.block.low, 2); + const high = formatNumber(row.perCallNs.block.high, 2); + + return `${low}..${high} ns`; +}; + +const formatSampleCount = (row: BenchResultRow) => { + const trimPercent = row.trimFraction * 100; + const outlierPercent = row.outlierFraction * 100; + const formattedTrimPercent = formatNumber(trimPercent, 0); + const formattedOutlierPercent = formatNumber(outlierPercent, 1); + + return `${row.sampleCount} (trim ${formattedTrimPercent}%, outliers ${formattedOutlierPercent}%)`; +}; + +const formatQuality = (row: BenchResultRow) => { + if (row.quality.level === 'clean') { + return 'clean'; + } + + const dominantReason = row.quality.reasons[0]; + + return `${row.quality.level} (${dominantReason})`; +}; + +const pad = (value: string, width: number) => { + return value.padEnd(width, ' '); +}; + +const padLeft = (value: string, width: number) => { + return value.padStart(width, ' '); +}; + +export const buildAlignedTable = (rows: BenchResultRow[]) => { + const headers = [ + 'browser', + 'workload', + 'quality', + 'per-call overhead upper', + 'overhead upper', + '95% CI', + 'A/A diag', + 'block CI', + 'acf(1)', + 'baseline', + 'instrumented', + 'samples', + ]; + const rightAlignedColumns = new Set([3, 4, 5, 6, 7, 8, 9, 10, 11]); + const body = rows.map((row) => [ + row.browserName, + row.workloadLabel, + formatQuality(row), + formatCallOverhead(row), + formatUpperBoundPercent(row), + formatCallOverheadConfidenceInterval(row), + formatNoiseFloorConfidenceInterval(row), + formatBlockConfidenceInterval(row), + formatNumber(row.lag1Autocorrelation, 2), + `${formatNumber(row.baseline.medianMs, 3)} ms`, + `${formatNumber(row.instrumented.medianMs, 3)} ms`, + formatSampleCount(row), + ]); + const widths = headers.map((header, index) => { + const bodyWidths = body.map((row) => row[index].length); + return Math.max(header.length, ...bodyWidths); + }); + const formatLine = (cells: string[]) => { + return cells + .map((value, index) => + rightAlignedColumns.has(index) + ? padLeft(value, widths[index]) + : pad(value, widths[index]), + ) + .join(' '); + }; + const separator = widths.map((width) => ''.padEnd(width, '-')).join(' '); + const lines = [formatLine(headers), separator, ...body.map(formatLine)]; + + return lines.join('\n'); +}; + +const printRows = (rows: BenchResultRow[]) => { + if (rows.length === 0) { + console.log('\nLive Debugger runtime benchmark produced no results.'); + return; + } + + console.log('\nLive Debugger runtime benchmark'); + console.log(buildAlignedTable(rows)); +}; + +const printFailures = (failures: BenchFailure[]) => { + if (failures.length === 0) { + return; + } + + console.log('\nLive Debugger runtime benchmark failures'); + for (const failure of failures) { + console.log(`- [${failure.projectName}] ${failure.title}: ${failure.error}`); + } +}; + +const printOutputPaths = (resultsFile: string) => { + console.log(`\nRaw results written to ${resultsFile}`); +}; + +export const renderMarkdownComment = (rows: BenchResultRow[], failures: BenchFailure[]) => { + let body = `${COMMENT_MARKER}\n## Live Debugger Runtime Benchmark\n\n`; + + if (rows.length === 0) { + body += 'Benchmark results were not produced. Check the workflow logs for details.\n'; + } else { + body += + 'SDK-loaded dormant-probe runtime overhead, measured against an uninstrumented bundle in the same browser session.\n\n'; + body += '| Browser | Workload | Quality | Per-call overhead upper |\n'; + body += '| --- | --- | --- | ---: |\n'; + + for (const row of rows) { + body += `| ${row.browserName} | ${row.workloadLabel} | ${formatQuality(row)} | ${formatCallOverhead(row)} |\n`; + } + + body += '\n
\nFull diagnostics\n\n'; + body += '```\n'; + body += `${buildAlignedTable(rows)}\n`; + body += '```\n'; + body += '\n
\n'; + } + + if (failures.length > 0) { + body += '\n### Benchmark failures\n\n'; + for (const failure of failures) { + body += `- **${failure.projectName}** (${failure.status}): ${failure.error}\n`; + } + } + + if (rows.length > 0 || failures.length > 0) { + body += '\nRaw samples are in the `live-debugger-runtime-bench-results` artifact.\n'; + } + + return body; +}; + +export default class BenchReporter implements Reporter { + private rows: BenchResultRow[] = []; + private failures: BenchFailure[] = []; + + onTestEnd(test: TestCase, result: TestResult) { + if (result.status !== 'passed') { + const errorMessage = result.error?.message || result.error?.value || result.status; + this.failures.push({ + projectName: test.parent.project()?.name || 'unknown', + title: test.title, + status: result.status, + error: errorMessage, + }); + return; + } + + const attachments = parseAttachmentBody(result); + for (const attachment of attachments) { + this.rows.push(...toRows(attachment)); + } + } + + onEnd(result: FullResult) { + this.rows.sort((a, b) => { + const aKey = `${a.browserName} | ${a.workloadId}`; + const bKey = `${b.browserName} | ${b.workloadId}`; + + return aKey.localeCompare(bKey); + }); + + printRows(this.rows); + printFailures(this.failures); + const markdownComment = renderMarkdownComment(this.rows, this.failures); + outputFileSync(COMMENT_FILE, markdownComment); + const generatedAt = new Date().toISOString(); + const resultsFile = buildResultsFilePath(generatedAt); + outputFileSync( + resultsFile, + `${JSON.stringify( + { + status: result.status, + generatedAt, + rows: this.rows, + failures: this.failures, + }, + null, + 2, + )}\n`, + ); + printOutputPaths(resultsFile); + } +} diff --git a/packages/tests/src/bench/liveDebuggerRuntime/reporter/stats.test.ts b/packages/tests/src/bench/liveDebuggerRuntime/reporter/stats.test.ts new file mode 100644 index 000000000..0b1d87fb7 --- /dev/null +++ b/packages/tests/src/bench/liveDebuggerRuntime/reporter/stats.test.ts @@ -0,0 +1,416 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { + getCounterbalancedVariantOrder, + getCounterbalancingPeriod, + roundSamplesToCounterbalancingPeriod, +} from '../project/harness'; + +import type { BenchQuality, QualityInputs } from './stats'; +import { + autocorrelation, + bcaConfidenceInterval, + bcaIndexConfidenceInterval, + createDeterministicRandom, + getQuality, + madOutlierFraction, + median, + movingBlockBootstrapConfidenceInterval, + movingBlockIndexBootstrapConfidenceInterval, + normalCdf, + normalQuantile, + pick, + percentile, + trimmedMean, +} from './stats'; + +type TestVariant = { + id: string; + fn: () => number; +}; + +describe('Live Debugger runtime benchmark stats', () => { + describe('median', () => { + const cases = [ + { + description: 'returns the middle value for odd-length input', + values: [9, 1, 5], + expected: 5, + }, + { + description: 'averages the middle values for even-length input', + values: [10, 2, 6, 4], + expected: 5, + }, + ]; + + test.each(cases)('should $description', ({ values, expected }) => { + expect(median(values)).toBe(expected); + }); + }); + + describe('percentile', () => { + const cases = [ + { + description: 'clamps the zero percentile to the first value', + percentileRank: 0, + expected: 1, + }, + { + description: 'returns an interpolated percentile', + percentileRank: 75, + expected: 7.5, + }, + { + description: 'clamps ranks above one hundred to the last value', + percentileRank: 101, + expected: 9, + }, + ]; + + test.each(cases)('should $description', ({ percentileRank, expected }) => { + expect(percentile([9, 1, 3, 7], percentileRank)).toBe(expected); + }); + }); + + describe('trimmedMean', () => { + const cases = [ + { + description: 'trims both tails before averaging', + values: [100, 1, 2, 3, 4], + trimFraction: 0.2, + expected: 3, + }, + { + description: 'falls back to the full sample when trimming removes everything', + values: [2, 4], + trimFraction: 0.5, + expected: 3, + }, + ]; + + test.each(cases)('should $description', ({ values, trimFraction, expected }) => { + expect(trimmedMean(values, trimFraction)).toBe(expected); + }); + }); + + describe('madOutlierFraction', () => { + const cases = [ + { + description: 'reports the scaled-MAD diagnostic fraction', + values: [10, 11, 12, 13, 100], + expected: 0.2, + }, + { + description: 'counts a single off-median spike when MAD implodes to zero', + values: [5, 5, 5, 50], + expected: 0.25, + }, + { + description: 'counts a clustered off-mode when MAD implodes to zero', + values: [5, 5, 5, 5, 5, 5, 5, 50, 50, 50], + expected: 0.3, + }, + { + description: 'reports the busier tail rather than pooling both tails', + values: [-100, 1, 2, 3, 4, 5, 6, 7, 8, 100], + expected: 0.1, + }, + { + description: 'returns zero when every sample is identical', + values: [5, 5, 5, 5], + expected: 0, + }, + ]; + + test.each(cases)('should $description', ({ values, expected }) => { + expect(madOutlierFraction(values)).toBe(expected); + }); + }); + + describe('createDeterministicRandom', () => { + test('should return the same sequence for the same seed', () => { + const left = createDeterministicRandom(123); + const right = createDeterministicRandom(123); + const leftValues = [left(), left(), left()]; + const rightValues = [right(), right(), right()]; + + expect(leftValues).toEqual(rightValues); + }); + + test('should use the pinned deterministic sequence', () => { + const random = createDeterministicRandom(123); + const values = [random(), random(), random()]; + + expect(values).toEqual([0.47988681646529585, 0.06268894905224442, 0.463917750865221]); + }); + }); + + describe('bcaConfidenceInterval', () => { + test('should return a stable interval for a fixed seed', () => { + const interval = bcaConfidenceInterval([1, 2, 3, 4, 5], 200, 42); + + expect(interval).toEqual({ + low: 1.563081445492187, + high: 4.333333333333333, + }); + }); + }); + + describe('movingBlockBootstrapConfidenceInterval', () => { + test('should return a stable interval for a fixed seed and block length', () => { + const interval = movingBlockBootstrapConfidenceInterval([1, 2, 3, 4, 5], 200, 42, 2); + + expect(interval).toEqual({ + low: 1.6666666666666667, + high: 4.333333333333333, + }); + }); + }); + + describe('paired ratio confidence intervals', () => { + const leftValues = [1, 2, 3, 4, 5]; + const rightValues = [10, 10, 11, 11, 12]; + const ratioEstimator = (indices: number[]) => { + const leftSample = pick(leftValues, indices); + const rightSample = pick(rightValues, indices); + + return (trimmedMean(leftSample, 0.2) / trimmedMean(rightSample, 0.2)) * 100; + }; + + test('should resample paired rows for BCa intervals', () => { + const interval = bcaIndexConfidenceInterval( + leftValues.length, + 200, + 42, + 0.95, + ratioEstimator, + ); + + expect(interval).toEqual({ + low: 16.129032258064516, + high: 38.23529411764706, + }); + }); + + test('should resample paired rows for moving-block intervals', () => { + const interval = movingBlockIndexBootstrapConfidenceInterval( + leftValues.length, + 200, + 42, + 2, + 0.95, + ratioEstimator, + ); + + expect(interval).toEqual({ + low: 16.666666666666668, + high: 38.23529411764706, + }); + }); + }); + + describe('normalCdf and normalQuantile', () => { + test('should approximate standard normal values', () => { + expect(normalCdf(0)).toBeCloseTo(0.5, 8); + expect(normalQuantile(0.975)).toBeCloseTo(1.959963986120195, 12); + }); + }); + + describe('autocorrelation', () => { + test('should compute lagged sample correlation against the full-series variance', () => { + expect(autocorrelation([1, 2, 3, 4], 1)).toBe(0.25); + }); + + test('should return zero for invalid lags', () => { + expect(autocorrelation([1, 2, 3], 0)).toBe(0); + expect(autocorrelation([1, 2, 3], 3)).toBe(0); + }); + }); + + describe('getQuality', () => { + // All intervals are per-call overhead in ns; OVERHEAD_RESOLUTION_NS is + // 0.5, so an effect below ~0.5 ns/call is treated as unresolvable. + const cleanInterval = { low: -0.4, high: 0.3 }; + const cleanInputs: QualityInputs = { + direct: cleanInterval, + block: cleanInterval, + aaFloor: cleanInterval, + outlierFraction: 0.02, + }; + const cases: { description: string; inputs: QualityInputs; expected: BenchQuality }[] = [ + { + description: 'reports clean when every diagnostic passes', + inputs: cleanInputs, + expected: { level: 'clean', reasons: [] }, + }, + { + description: + 'flags A/A drift as unreliable when the apparatus bias rivals a resolved effect', + inputs: { + ...cleanInputs, + aaFloor: { low: 0.8, high: 1.2 }, + direct: { low: 0.6, high: 1 }, + block: { low: 0.6, high: 1 }, + }, + expected: { level: 'unreliable', reasons: ['A/A drift'] }, + }, + { + description: + 'stays clean when a resolved A/A floor is negligible against the effect', + inputs: { + ...cleanInputs, + aaFloor: { low: 0.01, high: 0.03 }, + direct: { low: 43, high: 44 }, + block: { low: 43, high: 44 }, + }, + expected: { level: 'clean', reasons: [] }, + }, + { + description: 'cautions when a resolved A/A floor reaches half the effect', + inputs: { + ...cleanInputs, + aaFloor: { low: 0.6, high: 1 }, + direct: { low: 1.8, high: 2 }, + block: { low: 1.8, high: 2 }, + }, + expected: { level: 'caution', reasons: ['A/A drift'] }, + }, + { + description: + 'flags A/A drift when the apparatus bias clears the floor but the effect does not', + inputs: { + ...cleanInputs, + aaFloor: { low: 0.6, high: 0.9 }, + direct: { low: -0.1, high: 0.1 }, + block: { low: -0.1, high: 0.1 }, + }, + expected: { level: 'unreliable', reasons: ['A/A drift'] }, + }, + { + description: + 'flags a confident speed-up past the floor as negative overhead even inside a wide A/A floor', + inputs: { + ...cleanInputs, + aaFloor: { low: -0.6, high: 0.6 }, + direct: { low: -1.5, high: -0.8 }, + block: { low: -1.6, high: -0.9 }, + }, + expected: { level: 'unreliable', reasons: ['negative overhead'] }, + }, + { + description: + 'stays clean on a sub-floor confident speed-up (cross-bundle layout noise)', + inputs: { + ...cleanInputs, + aaFloor: { low: -0.06, high: 0.05 }, + direct: { low: -0.16, high: -0.05 }, + block: { low: -0.15, high: -0.05 }, + }, + expected: { level: 'clean', reasons: [] }, + }, + { + description: + 'stays clean when both the sub-floor effect and a resolved A/A floor are below the floor', + inputs: { + ...cleanInputs, + aaFloor: { low: 0.001, high: 0.115 }, + direct: { low: -0.16, high: -0.05 }, + block: { low: -0.15, high: -0.05 }, + }, + expected: { level: 'clean', reasons: [] }, + }, + { + description: 'stays clean when a wide negative tail still brackets zero', + inputs: { + ...cleanInputs, + aaFloor: { low: -0.02, high: 0.02 }, + direct: { low: -0.5, high: 0.05 }, + }, + expected: { level: 'clean', reasons: [] }, + }, + { + description: 'stays clean on a moderate outlier fraction the trim absorbs', + inputs: { ...cleanInputs, outlierFraction: 0.15 }, + expected: { level: 'clean', reasons: [] }, + }, + { + description: 'escalates an outlier fraction at the trim fraction to unreliable', + inputs: { ...cleanInputs, outlierFraction: 0.2 }, + expected: { level: 'unreliable', reasons: ['outliers'] }, + }, + { + description: 'stays clean on autocorrelation alone when the intervals agree', + inputs: { + ...cleanInputs, + direct: { low: 9.1, high: 9.3 }, + block: { low: 9, high: 9.4 }, + }, + expected: { level: 'clean', reasons: [] }, + }, + { + description: + 'cautions when the block interval disagrees on bracketing zero for a resolved effect', + inputs: { + ...cleanInputs, + direct: { low: -0.4, high: 0.7 }, + block: { low: 0.6, high: 0.9 }, + }, + expected: { level: 'caution', reasons: ['block disagreement'] }, + }, + { + description: 'lists the most severe reason first when several fire', + inputs: { + ...cleanInputs, + aaFloor: { low: 0.8, high: 1.2 }, + direct: { low: 0.6, high: 1 }, + block: { low: 0.6, high: 1 }, + outlierFraction: 0.2, + }, + expected: { level: 'unreliable', reasons: ['A/A drift', 'outliers'] }, + }, + ]; + + test.each(cases)('should $description', ({ inputs, expected }) => { + expect(getQuality(inputs)).toEqual(expected); + }); + }); + + describe('counterbalanced variant order', () => { + const variants: TestVariant[] = [ + { id: 'baseline', fn: () => 0 }, + { id: 'control', fn: () => 0 }, + { id: 'instrumented', fn: () => 0 }, + ]; + + test('should round sample counts to the full counterbalancing period', () => { + expect(getCounterbalancingPeriod(variants.length)).toBe(6); + expect(roundSamplesToCounterbalancingPeriod(100, variants.length)).toBe(102); + expect(roundSamplesToCounterbalancingPeriod(33, variants.length)).toBe(36); + }); + + test('should put every variant in every position across one rotation cycle', () => { + const positions = variants.map((variant) => { + return { + id: variant.id, + indexes: [0, 1, 2].map((sampleIndex) => { + const order = getCounterbalancedVariantOrder( + variants, + sampleIndex, + ) as TestVariant[]; + + return order.findIndex((candidate) => candidate.id === variant.id); + }), + }; + }); + + expect(positions).toEqual([ + { id: 'baseline', indexes: [0, 2, 1] }, + { id: 'control', indexes: [1, 0, 2] }, + { id: 'instrumented', indexes: [2, 1, 0] }, + ]); + }); + }); +}); diff --git a/packages/tests/src/bench/liveDebuggerRuntime/reporter/stats.ts b/packages/tests/src/bench/liveDebuggerRuntime/reporter/stats.ts new file mode 100644 index 000000000..e9088fe8e --- /dev/null +++ b/packages/tests/src/bench/liveDebuggerRuntime/reporter/stats.ts @@ -0,0 +1,630 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +export const BOOTSTRAP_ITERATIONS = 10_000; +export const MAD_OUTLIER_MULTIPLIER = 3; +export const MAD_NORMAL_SCALE = 1.4826; +export const TRIM_FRACTION = 0.2; + +// Quality is judged on the per-call nanosecond scale, not on percent. The +// benchmark's headline metric is overhead per instrumented call, and the noise +// it has to tolerate -- cross-bundle code layout, JIT codegen, timer +// granularity -- is fundamentally an absolute per-call quantity. Percent hides +// this: on V8 the dormant hooks are nearly free, so `chrome/Tiny` (a genuine +// ~0.15 ns/call effect) and `chrome/Hot` (a genuine ~3.6 ns/call effect) both +// read as ~3.5% because the Tiny baseline per call is tiny. No percentage +// threshold can separate the 0.15 ns noise from the 3.6 ns signal; the ns scale +// separates them trivially. +// +// Below this floor a per-call effect cannot be distinguished from cross-bundle +// layout noise: the baseline and instrumented bundles are built separately, so +// their "same" code differs in layout, and that difference (a fraction of a CPU +// cycle per call) makes the measured overhead drift by a sub-nanosecond amount +// in either direction around a true ~zero effect. ~0.5 ns is roughly one to two +// cycles on a multi-GHz core; the smallest genuinely-resolved overhead we see +// (firefox/Tiny, ~1.7 ns) sits comfortably above it. +export const OVERHEAD_RESOLUTION_NS = 0.5; + +// Once the effect clears the resolution floor it is real, and the A/A floor +// (control vs. baseline, same code) -- the apparatus's own order/scheduling +// bias -- is weighed against it as a fraction rather than tested against a hard +// zero. Below the caution ratio the resolved bias is negligible against the +// effect; at or above the severe ratio the apparatus bias is as large as the +// effect and the row cannot be trusted. +export const AA_DRIFT_CAUTION_RATIO = 0.5; +export const AA_DRIFT_SEVERE_RATIO = 1; + +// The per-tail MAD outlier fraction (see madOutlierFraction) is only acted on +// once it reaches the trim fraction, because that is exactly the 20% +// trimmed-mean estimator's per-tail breakdown point: below it the trim absorbs +// the outliers on that side without bias, at or beyond it the outliers start +// surviving the trim and moving the point estimate itself. +export const MAD_SEVERE_FRACTION = TRIM_FRACTION; + +export type ConfidenceInterval = { + low: number; + high: number; +}; + +export type BenchQualityLevel = 'clean' | 'caution' | 'unreliable'; + +export type BenchQualityFlag = + | 'A/A drift' + | 'negative overhead' + | 'outliers' + | 'block disagreement'; + +export type BenchQuality = { + level: BenchQualityLevel; + reasons: BenchQualityFlag[]; +}; + +// All three intervals are per-call overhead in nanoseconds (see +// OVERHEAD_RESOLUTION_NS for why the ns scale, not percent, is the right one to +// judge quality on): `direct`/`block` are the naive and dependence-robust +// `instrumented - control` intervals, `aaFloor` is the `control - baseline` +// apparatus floor. +export type QualityInputs = { + direct: ConfidenceInterval; + block: ConfidenceInterval; + aaFloor: ConfidenceInterval; + outlierFraction: number; +}; + +export type Estimator = (values: number[]) => number; + +export type IndexEstimator = (indices: number[]) => number; + +type IndexResampler = (sampleCount: number, random: () => number) => number[]; + +export const percentile = (values: number[], percentileRank: number) => { + const sorted = [...values].sort((a, b) => a - b); + if (sorted.length === 0) { + throw new Error('Cannot compute a percentile of an empty sample'); + } + + if (percentileRank <= 0) { + return sorted[0]; + } + + if (percentileRank >= 100) { + return sorted[sorted.length - 1]; + } + + const rank = (percentileRank / 100) * (sorted.length - 1); + const lowerIndex = Math.floor(rank); + const upperIndex = Math.ceil(rank); + const fraction = rank - lowerIndex; + + return sorted[lowerIndex] + (sorted[upperIndex] - sorted[lowerIndex]) * fraction; +}; + +export const mean = (values: number[]) => { + if (values.length === 0) { + throw new Error('Cannot compute a mean of an empty sample'); + } + + return values.reduce((sum, value) => sum + value, 0) / values.length; +}; + +export const median = (values: number[]) => { + const sorted = [...values].sort((a, b) => a - b); + if (sorted.length === 0) { + throw new Error('Cannot compute a median of an empty sample'); + } + + const mid = Math.floor(sorted.length / 2); + + if (sorted.length % 2 === 0) { + return (sorted[mid - 1] + sorted[mid]) / 2; + } + + return sorted[mid]; +}; + +export const trimmedMean = (values: number[], trimFraction: number) => { + const sorted = [...values].sort((a, b) => a - b); + const trimCount = Math.floor(sorted.length * trimFraction); + const trimmed = sorted.slice(trimCount, sorted.length - trimCount); + + return mean(trimmed.length > 0 ? trimmed : sorted); +}; + +// The fraction of samples in the busier tail that sit beyond 3 scaled-MADs of +// the median. It is measured per tail rather than pooled across both, because +// the point estimate is a 20% trimmed mean: it trims each end independently and +// stays unbiased as long as neither tail's contamination exceeds the trim +// fraction. The worst single tail is therefore what decides whether the +// estimate is still trustworthy -- a symmetric spread of, say, 12% per side +// (24% pooled) is fully absorbed by the trim, while 24% on one side is not. +export const madOutlierFraction = (values: number[]) => { + const medianValue = median(values); + const deviations = values.map((value) => Math.abs(value - medianValue)); + const scaledMad = median(deviations) * MAD_NORMAL_SCALE; + const maxDeviation = MAD_OUTLIER_MULTIPLIER * scaledMad; + + // The MAD implodes to zero once more than half the samples sit on the + // median, which is common with quantized browser timings. In that shape any + // non-median sample is outside the robust central mass, so count it directly + // instead of letting an off-mode cluster inflate its own spread estimate. + const isOutlier = (value: number) => { + return scaledMad === 0 + ? value !== medianValue + : Math.abs(value - medianValue) > maxDeviation; + }; + + let upperTail = 0; + let lowerTail = 0; + for (const value of values) { + if (!isOutlier(value)) { + continue; + } + if (value > medianValue) { + upperTail++; + } else { + lowerTail++; + } + } + + return Math.max(upperTail, lowerTail) / values.length; +}; + +export const createDeterministicRandom = (seed: number) => { + const mash = createMash(); + let state0 = mash(' '); + let state1 = mash(' '); + let state2 = mash(' '); + state0 -= mash(seed.toString()); + if (state0 < 0) { + state0 += 1; + } + state1 -= mash(seed.toString()); + if (state1 < 0) { + state1 += 1; + } + state2 -= mash(seed.toString()); + if (state2 < 0) { + state2 += 1; + } + let carry = 1; + + return () => { + const nextValue = 2_091_639 * state0 + carry * 2.328_306_436_538_696_3e-10; + state0 = state1; + state1 = state2; + carry = Math.floor(nextValue); + state2 = nextValue - carry; + + return state2; + }; +}; + +const createMash = () => { + let state = 4_022_871_197; + + return (value: string) => { + for (let index = 0; index < value.length; index++) { + state += value.charCodeAt(index); + let hash = 0.025_196_032_824_169_38 * state; + state = Math.floor(hash); + hash -= state; + hash *= state; + state = Math.floor(hash); + hash -= state; + state += hash * 4_294_967_296; + } + + const unsignedState = state < 0 ? state + 4_294_967_296 : state; + + return (unsignedState % 4_294_967_296) * 2.328_306_436_538_696_3e-10; + }; +}; + +export const hashString = (value: string) => { + let hash = 0; + + for (let i = 0; i < value.length; i++) { + hash = (hash * 31 + value.charCodeAt(i)) % 4_294_967_296; + } + + return hash; +}; + +export const normalCdf = (value: number) => { + const sign = value < 0 ? -1 : 1; + const x = Math.abs(value) / Math.sqrt(2); + const t = 1 / (1 + 0.3275911 * x); + const coefficients = [0.254829592, -0.284496736, 1.421413741, -1.453152027, 1.061405429]; + const polynomial = coefficients.reduceRight((accumulator, coefficient) => { + return (accumulator + coefficient) * t; + }, 0); + const erf = sign * (1 - polynomial * Math.exp(-x * x)); + + return 0.5 * (1 + erf); +}; + +export const normalQuantile = (probability: number) => { + if (probability <= 0 || probability >= 1) { + throw new Error('Normal quantile probability must be between 0 and 1'); + } + + const a = [ + -3.969683028665376e1, 2.209460984245205e2, -2.759285104469687e2, 1.38357751867269e2, + -3.066479806614716e1, 2.506628277459239, + ]; + const b = [ + -5.447609879822406e1, 1.615858368580409e2, -1.556989798598866e2, 6.680131188771972e1, + -1.328068155288572e1, + ]; + const c = [ + -7.784894002430293e-3, -3.223964580411365e-1, -2.400758277161838, -2.549732539343734, + 4.374664141464968, 2.938163982698783, + ]; + const d = [7.784695709041462e-3, 3.224671290700398e-1, 2.445134137142996, 3.754408661907416]; + const lower = 0.02425; + const upper = 1 - lower; + + if (probability < lower) { + const q = Math.sqrt(-2 * Math.log(probability)); + const numerator = ((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]; + const denominator = (((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1; + + return numerator / denominator; + } + + if (probability > upper) { + const q = Math.sqrt(-2 * Math.log(1 - probability)); + const numerator = ((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]; + const denominator = (((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1; + + return -(numerator / denominator); + } + + const q = probability - 0.5; + const r = q * q; + const numerator = (((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r + a[5]) * q; + const denominator = ((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1; + + return numerator / denominator; +}; + +const defaultEstimator: Estimator = (values) => { + return trimmedMean(values, TRIM_FRACTION); +}; + +const getConfidenceBounds = (confidenceLevel: number) => { + const alpha = 1 - confidenceLevel; + + return { + low: alpha / 2, + high: 1 - alpha / 2, + }; +}; + +export const pick = (values: number[], indices: number[]) => { + return indices.map((index) => values[index]); +}; + +const createSequentialIndices = (sampleCount: number) => { + return Array.from({ length: sampleCount }, (_value, index) => index); +}; + +const resampleIndices: IndexResampler = (sampleCount, random) => { + const resampled: number[] = []; + + for (let sampleIndex = 0; sampleIndex < sampleCount; sampleIndex++) { + resampled.push(Math.floor(random() * sampleCount)); + } + + return resampled; +}; + +const createMovingBlockIndexResampler = (blockLength: number): IndexResampler => { + return (sampleCount, random) => { + const resampled: number[] = []; + const normalizedBlockLength = Math.max(1, Math.min(sampleCount, Math.round(blockLength))); + + while (resampled.length < sampleCount) { + const blockStart = Math.floor(random() * sampleCount); + for (let offset = 0; offset < normalizedBlockLength; offset++) { + resampled.push((blockStart + offset) % sampleCount); + + if (resampled.length === sampleCount) { + break; + } + } + } + + return resampled; + }; +}; + +const createValueEstimator = (values: number[], estimator: Estimator): IndexEstimator => { + return (indices) => estimator(pick(values, indices)); +}; + +const bootstrapEstimates = ( + sampleCount: number, + iterations: number, + seed: number, + estimator: IndexEstimator, + resampler: IndexResampler, +) => { + const random = createDeterministicRandom(seed); + const estimates: number[] = []; + + for (let iteration = 0; iteration < iterations; iteration++) { + estimates.push(estimator(resampler(sampleCount, random))); + } + + return estimates; +}; + +const getBcaAcceleration = (sampleCount: number, estimator: IndexEstimator) => { + if (sampleCount < 3) { + return 0; + } + + const indices = createSequentialIndices(sampleCount); + const jackknifeEstimates = indices.map((omittedIndex) => { + const jackknifeSample = indices.filter((index) => { + return index !== omittedIndex; + }); + + return estimator(jackknifeSample); + }); + const jackknifeMean = mean(jackknifeEstimates); + const numerator = jackknifeEstimates.reduce((sum, estimate) => { + return sum + (jackknifeMean - estimate) ** 3; + }, 0); + const denominatorTerm = jackknifeEstimates.reduce((sum, estimate) => { + return sum + (jackknifeMean - estimate) ** 2; + }, 0); + + if (denominatorTerm === 0) { + return 0; + } + + return numerator / (6 * denominatorTerm ** 1.5); +}; + +const getBcaPercentile = (alpha: number, biasCorrection: number, acceleration: number) => { + const zAlpha = normalQuantile(alpha); + const numerator = biasCorrection + zAlpha; + const denominator = 1 - acceleration * numerator; + + if (denominator === 0) { + return alpha; + } + + return normalCdf(biasCorrection + numerator / denominator); +}; + +export const bcaIndexConfidenceInterval = ( + sampleCount: number, + iterations: number, + seed: number, + confidenceLevel: number, + estimator: IndexEstimator, + resampler: IndexResampler = resampleIndices, +): ConfidenceInterval => { + const thetaHat = estimator(createSequentialIndices(sampleCount)); + const estimates = bootstrapEstimates(sampleCount, iterations, seed, estimator, resampler); + + const estimatesBelowTheta = estimates.filter((estimate) => { + return estimate < thetaHat; + }).length; + const biasProbability = (estimatesBelowTheta + 0.5) / (estimates.length + 1); + const biasCorrection = normalQuantile(biasProbability); + const acceleration = getBcaAcceleration(sampleCount, estimator); + const bounds = getConfidenceBounds(confidenceLevel); + const lowPercentile = getBcaPercentile(bounds.low, biasCorrection, acceleration); + const highPercentile = getBcaPercentile(bounds.high, biasCorrection, acceleration); + + return { + low: percentile(estimates, lowPercentile * 100), + high: percentile(estimates, highPercentile * 100), + }; +}; + +export const bcaConfidenceInterval = ( + values: number[], + iterations: number, + seed: number, + confidenceLevel = 0.95, + estimator: Estimator = defaultEstimator, +): ConfidenceInterval => { + return bcaIndexConfidenceInterval( + values.length, + iterations, + seed, + confidenceLevel, + createValueEstimator(values, estimator), + ); +}; + +const indexBootstrapConfidenceInterval = ( + sampleCount: number, + iterations: number, + seed: number, + confidenceLevel: number, + estimator: IndexEstimator, + resampler: IndexResampler = resampleIndices, +): ConfidenceInterval => { + const bounds = getConfidenceBounds(confidenceLevel); + const estimates = bootstrapEstimates(sampleCount, iterations, seed, estimator, resampler); + + return { + low: percentile(estimates, bounds.low * 100), + high: percentile(estimates, bounds.high * 100), + }; +}; + +export const movingBlockIndexBootstrapConfidenceInterval = ( + sampleCount: number, + iterations: number, + seed: number, + blockLength: number, + confidenceLevel: number, + estimator: IndexEstimator, +): ConfidenceInterval => { + const resampler = createMovingBlockIndexResampler(blockLength); + + return indexBootstrapConfidenceInterval( + sampleCount, + iterations, + seed, + confidenceLevel, + estimator, + resampler, + ); +}; + +export const movingBlockBootstrapConfidenceInterval = ( + values: number[], + iterations: number, + seed: number, + blockLength: number, + confidenceLevel = 0.95, + estimator: Estimator = defaultEstimator, +): ConfidenceInterval => { + return movingBlockIndexBootstrapConfidenceInterval( + values.length, + iterations, + seed, + blockLength, + confidenceLevel, + createValueEstimator(values, estimator), + ); +}; + +export const autocorrelation = (values: number[], lag: number) => { + if (lag <= 0 || lag >= values.length) { + return 0; + } + + const average = mean(values); + const denominator = values.reduce((sum, value) => { + return sum + (value - average) ** 2; + }, 0); + + if (denominator === 0) { + return 0; + } + + let numerator = 0; + for (let index = 0; index < values.length - lag; index++) { + numerator += (values[index] - average) * (values[index + lag] - average); + } + + return numerator / denominator; +}; + +export const lag1Autocorrelation = (values: number[]) => { + return autocorrelation(values, 1); +}; + +const bracketsZero = (interval: ConfidenceInterval) => { + return interval.low <= 0 && interval.high >= 0; +}; + +// The largest plausible apparatus bias from the A/A floor: the bound furthest +// from zero, since either tail could be the systematic offset. +const apparatusBiasMagnitude = (aaFloor: ConfidenceInterval) => { + return Math.max(Math.abs(aaFloor.low), Math.abs(aaFloor.high)); +}; + +// The single definition of the published overhead upper bound: the +// dependence-robust upper bound of the `instrumented - control` effect, floored +// at zero. The reporter's `withUpperBound` derives the displayed number from +// this same helper, so the quality verdict and the printed bound can never +// drift apart. +export const overheadUpperBound = (direct: ConfidenceInterval, block: ConfidenceInterval) => { + return Math.max(direct.high, block.high, 0); +}; + +// Aggregates the per-row diagnostics into one plain-language verdict about +// whether the timing apparatus and the sampled `instrumented - control` delta +// are statistically sound. This is orthogonal to how large the overhead is: a +// quiet run whose effect is below the resolution floor is `clean`, while a +// contaminated run is flagged regardless of its number. It does not certify the +// absence of variant-specific bias such as code-layout or inlining differences +// between the two bundles: the A/A floor only re-runs the baseline path, so +// that class of bias is bracketed by the Tiny/Hot workloads, not detected here. +export const getQuality = (inputs: QualityInputs): BenchQuality => { + const severe: BenchQualityFlag[] = []; + const caution: BenchQualityFlag[] = []; + + // Everything below is judged in ns/call against OVERHEAD_RESOLUTION_NS. An + // effect at or above the floor is "resolved": large enough to stand clear + // of cross-bundle layout noise. Below it the per-call overhead cannot be + // told apart from that noise, so a small negative excursion or a small + // resolved A/A floor is the expected ~zero outcome, not a defect. + const overhead = overheadUpperBound(inputs.direct, inputs.block); + const overheadResolved = overhead >= OVERHEAD_RESOLUTION_NS; + + // The A/A floor (control vs. baseline) captures the apparatus's own bias. + // It is expected to resolve to a small non-zero number, so testing it + // against a hard zero condemns solid runs over picosecond offsets. + if (!bracketsZero(inputs.aaFloor)) { + const bias = apparatusBiasMagnitude(inputs.aaFloor); + + if (overheadResolved) { + // Weigh the resolved bias against the effect it could swamp: a bias + // far below the effect is harmless, one that rivals it means the + // session drifted enough to swamp what we are measuring. + const biasRatio = bias / overhead; + if (biasRatio >= AA_DRIFT_SEVERE_RATIO) { + severe.push('A/A drift'); + } else if (biasRatio >= AA_DRIFT_CAUTION_RATIO) { + caution.push('A/A drift'); + } + } else if (bias >= OVERHEAD_RESOLUTION_NS) { + // The effect is below the floor, so there is nothing to weigh the + // bias against; the floor is only alarming if the apparatus bias + // itself clears it -- the session drifted by more than a resolvable + // per-call amount while we could not even resolve the effect. + severe.push('A/A drift'); + } + } + + // Instrumentation only adds instructions, so it can never be faster. A + // confident speed-up (the whole direct interval below zero) is impossible, + // but only material once it clears the resolution floor: a sub-floor + // "speed-up" is the cross-bundle layout noise around a true ~zero effect, + // not contamination, so it stays clean. + if (inputs.direct.high <= -OVERHEAD_RESOLUTION_NS) { + severe.push('negative overhead'); + } + + // Only contamination heavy enough to defeat the trimmed-mean estimator is a + // reliability problem. The fraction is per-tail (the busier side), so a + // symmetric spread the trim absorbs from both ends stays clean; browsers + // like WebKit are routinely spiky on the hot path without being unreliable. + if (inputs.outlierFraction >= MAD_SEVERE_FRACTION) { + severe.push('outliers'); + } + + // Autocorrelation is not gated on directly: the moving-block bootstrap is + // the dependence-robust interval and is what the row reports, so stationary + // autocorrelation (a wander whose mean is still well resolved) is already + // handled. The lag-1 value is shown as a diagnostic. The check that matters + // is whether accounting for dependence changes the verdict: if the naive + // and block intervals disagree on bracketing zero, the naive width lied. + // Only meaningful once the effect is resolved; below the floor a zero- + // bracket flip is noise around zero, not a dependence problem. + if (overheadResolved && bracketsZero(inputs.direct) !== bracketsZero(inputs.block)) { + caution.push('block disagreement'); + } + + if (severe.length > 0) { + return { level: 'unreliable', reasons: [...severe, ...caution] }; + } + + if (caution.length > 0) { + return { level: 'caution', reasons: caution }; + } + + return { level: 'clean', reasons: [] }; +}; diff --git a/packages/tests/src/bench/liveDebuggerRuntime/types.ts b/packages/tests/src/bench/liveDebuggerRuntime/types.ts new file mode 100644 index 000000000..43838d949 --- /dev/null +++ b/packages/tests/src/bench/liveDebuggerRuntime/types.ts @@ -0,0 +1,92 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import type { TestResult } from '@playwright/test/reporter'; + +import type { BenchQuality, ConfidenceInterval } from './reporter/stats'; + +export type BenchVariant = 'baseline' | 'instrumented'; + +export type BenchVariantId = BenchVariant | 'control'; + +export type BrowserBenchVariant = { + id: BenchVariantId; + fn: (iteration: number) => number; +}; + +export type BrowserBenchWorkload = { + id: string; + label: string; + batchSize: number; + instrumentedCallsPerInvocation: number; + samples: number; + fn: (iteration: number) => number; +}; + +export type RawVariantResult = { + samplesMs: number[]; + medianMs: number; +}; + +export type RawWorkloadResult = { + workloadId: string; + workloadLabel: string; + batchSize: number; + instrumentedCallsPerBatch: number; + sink: number; + variants: Record; +}; + +export type BrowserBenchApi = { + runBenchPair: ( + workload: BrowserBenchWorkload, + variants: BrowserBenchVariant[], + options?: { + warmupMs?: number; + batchSize?: number; + calibrationAttempts?: number; + minBatchMs?: number; + samples?: number; + }, + ) => RawWorkloadResult; + workloads: BrowserBenchWorkload[]; +}; + +export type RawBenchAttachment = { + browserName: string; + results: RawWorkloadResult[]; +}; + +export type MetricUnit = { + point: number; + direct: ConfidenceInterval; + block: ConfidenceInterval; + aaFloor: ConfidenceInterval; + upperBound: number; +}; + +export type BenchResultRow = { + browserName: string; + workloadId: string; + workloadLabel: string; + batchSize: number; + instrumentedCallsPerBatch: number; + baseline: RawVariantResult; + control: RawVariantResult; + instrumented: RawVariantResult; + perCallNs: MetricUnit; + overheadUpperPercent: number; + sampleCount: number; + trimFraction: number; + outlierFraction: number; + lag1Autocorrelation: number; + quality: BenchQuality; +}; + +export type BenchFailure = { + projectName: string; + title: string; + status: TestResult['status']; + error: string; +}; diff --git a/packages/tools/src/commands/dev-server/index.ts b/packages/tools/src/commands/dev-server/index.ts index 2e95cc04e..deb717691 100644 --- a/packages/tools/src/commands/dev-server/index.ts +++ b/packages/tools/src/commands/dev-server/index.ts @@ -45,6 +45,10 @@ class DevServer extends Command { description: 'The root directory the server will serve.', }); + crossOriginIsolated = Option.Boolean('--cross-origin-isolated', false, { + description: 'Serve pages with cross-origin isolation headers.', + }); + parseCookie(cookieHeader?: string): Record { if (!cookieHeader) { return {}; @@ -106,10 +110,16 @@ class DevServer extends Command { const content = template(resp.body, { interpolate: INTERPOLATE_RX, })(context); - const headers = { + const headers: Record = { 'Set-Cookie': `context_cookie=${encodeURIComponent(JSON.stringify(context))};SameSite=Strict;`, }; + if (this.crossOriginIsolated) { + headers['Cross-Origin-Opener-Policy'] = 'same-origin'; + headers['Cross-Origin-Embedder-Policy'] = 'require-corp'; + headers['Cross-Origin-Resource-Policy'] = 'same-origin'; + } + const c = { 200: chalk.green,