From 04af8238d242178175169741b4d26cb631797769 Mon Sep 17 00:00:00 2001 From: Oliver Nordbjerg Date: Thu, 30 Apr 2026 16:03:13 +0200 Subject: [PATCH 01/18] feat(bench): add txgen benchmark runner scripts --- .github/scripts/bench-txgen-build.sh | 59 ++++ .github/scripts/bench-txgen-install.sh | 18 ++ .../scripts/bench-txgen-report-to-reth-csv.py | 107 ++++++++ .github/scripts/bench-txgen-run.sh | 258 ++++++++++++++++++ 4 files changed, 442 insertions(+) create mode 100755 .github/scripts/bench-txgen-build.sh create mode 100755 .github/scripts/bench-txgen-install.sh create mode 100755 .github/scripts/bench-txgen-report-to-reth-csv.py create mode 100755 .github/scripts/bench-txgen-run.sh diff --git a/.github/scripts/bench-txgen-build.sh b/.github/scripts/bench-txgen-build.sh new file mode 100755 index 00000000000..80b1ab3a753 --- /dev/null +++ b/.github/scripts/bench-txgen-build.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# +# Builds the node binary for the txgen-backed PR benchmark path. +# +# Usage: bench-txgen-build.sh +# +# This intentionally does not build or install reth-bench. Big-block benchmarks +# still use the legacy reth-bench path because txgen does not yet replay the +# reth-bb payload/env-switch/BAL format. +set -euxo pipefail + +MODE="$1" +SOURCE_DIR="$2" +COMMIT="$3" + +if [ "${BENCH_BIG_BLOCKS:-false}" = "true" ]; then + echo "::error::txgen path does not support big-block benchmarks yet; use the reth-bench driver" + exit 1 +fi + +EXTRA_FEATURES="" +EXTRA_RUSTFLAGS="" +if [ "${BENCH_TRACY:-off}" != "off" ]; then + EXTRA_FEATURES="tracy,tracy-client/ondemand" + EXTRA_RUSTFLAGS=" -C force-frame-pointers=yes" +fi + +build_node_binary() { + local features_arg="" + local workspace_arg="" + + cd "$SOURCE_DIR" + if [ -n "$EXTRA_FEATURES" ]; then + features_arg="--features ${EXTRA_FEATURES}" + workspace_arg="--workspace" + fi + + # shellcheck disable=SC2086 + RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \ + cargo build --locked --profile profiling --bin reth $workspace_arg $features_arg +} + +case "$MODE" in + baseline|main) + echo "Building baseline reth (${COMMIT}) from source for txgen benchmark..." + build_node_binary + ;; + + feature|branch) + echo "Building feature reth (${COMMIT}) from source for txgen benchmark..." + rustup show active-toolchain || rustup default stable + build_node_binary + ;; + + *) + echo "Usage: $0 " + exit 1 + ;; +esac diff --git a/.github/scripts/bench-txgen-install.sh b/.github/scripts/bench-txgen-install.sh new file mode 100755 index 00000000000..dd5ce58b357 --- /dev/null +++ b/.github/scripts/bench-txgen-install.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# +# Installs the txgen tools used by the txgen-backed PR benchmark path. +# Keep this separate from bench-reth-build.sh so scheduled benchmarks can keep +# using the legacy reth-bench runner until they are migrated explicitly. +# +# Required env: +# TXGEN_REV – pinned txgen git revision +# Optional env: +# TXGEN_REPO – txgen repository URL (default: https://github.com/tempoxyz/txgen) +set -euxo pipefail + +: "${TXGEN_REV:?TXGEN_REV must be set to a pinned txgen revision}" + +TXGEN_REPO="${TXGEN_REPO:-https://github.com/tempoxyz/txgen}" + +cargo install --git "$TXGEN_REPO" --rev "$TXGEN_REV" --bin txgen-ethereum --locked +cargo install --git "$TXGEN_REPO" --rev "$TXGEN_REV" --bin bench --locked diff --git a/.github/scripts/bench-txgen-report-to-reth-csv.py b/.github/scripts/bench-txgen-report-to-reth-csv.py new file mode 100755 index 00000000000..01620d94260 --- /dev/null +++ b/.github/scripts/bench-txgen-report-to-reth-csv.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Convert txgen `bench send-blocks` JSON into the legacy reth-bench CSVs. + +The PR benchmark rendering pipeline still consumes `combined_latency.csv` and +`total_gas.csv`. This adapter lets the txgen-backed runner reuse the existing +summary/charts/slack code while we migrate those consumers to txgen JSON. +""" + +import argparse +import csv +import json +from pathlib import Path + + +def opt_int(value, default=None): + if value is None: + return default + return int(value) + + +def block_latency_us(block: dict) -> tuple[int, int, int]: + # txgen currently records server newPayload latency in microseconds but + # client-side forkchoiceUpdated latency in milliseconds. + new_payload_us = opt_int(block.get("new_payload_server_latency_us")) + if new_payload_us is None: + new_payload_us = opt_int(block.get("new_payload_ms"), 0) * 1000 + fcu_us = opt_int(block.get("forkchoice_updated_ms"), 0) * 1000 + return new_payload_us, fcu_us, new_payload_us + fcu_us + + +def main() -> None: + parser = argparse.ArgumentParser(description="Convert txgen JSON report to reth-bench CSVs") + parser.add_argument("report", help="txgen JSON report path") + parser.add_argument("output_dir", help="directory for combined_latency.csv and total_gas.csv") + args = parser.parse_args() + + report_path = Path(args.report) + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + with report_path.open() as f: + report = json.load(f) + + blocks = report.get("blocks") or [] + if not blocks: + raise SystemExit(f"txgen report {report_path} does not contain any blocks") + + combined_path = output_dir / "combined_latency.csv" + with combined_path.open("w", newline="") as f: + writer = csv.DictWriter( + f, + fieldnames=[ + "block_number", + "gas_limit", + "transaction_count", + "gas_used", + "new_payload_latency", + "fcu_latency", + "total_latency", + "persistence_wait", + "execution_cache_wait", + "sparse_trie_wait", + ], + ) + writer.writeheader() + for block in blocks: + new_payload_us, fcu_us, total_us = block_latency_us(block) + writer.writerow( + { + "block_number": block["number"], + "gas_limit": block["gas_limit"], + "transaction_count": block["tx_count"], + "gas_used": block["gas_used"], + "new_payload_latency": new_payload_us, + "fcu_latency": fcu_us, + "total_latency": total_us, + "persistence_wait": block.get("persistence_wait_us") or 0, + "execution_cache_wait": block.get("execution_cache_wait_us") or 0, + "sparse_trie_wait": block.get("sparse_trie_wait_us") or 0, + } + ) + + total_gas_path = output_dir / "total_gas.csv" + elapsed_us = 0 + with total_gas_path.open("w", newline="") as f: + writer = csv.DictWriter( + f, + fieldnames=["block_number", "transaction_count", "gas_used", "time"], + ) + writer.writeheader() + for block in blocks: + _, _, total_us = block_latency_us(block) + elapsed_us += total_us + writer.writerow( + { + "block_number": block["number"], + "transaction_count": block["tx_count"], + "gas_used": block["gas_used"], + "time": elapsed_us, + } + ) + + print(f"Wrote legacy CSVs from {report_path} to {output_dir}") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/bench-txgen-run.sh b/.github/scripts/bench-txgen-run.sh new file mode 100755 index 00000000000..264bc1e0bdc --- /dev/null +++ b/.github/scripts/bench-txgen-run.sh @@ -0,0 +1,258 @@ +#!/usr/bin/env bash +# +# Runs a single txgen-backed Engine API benchmark cycle: +# mount snapshot → start node → extract source blocks → warmup → send-blocks → +# convert txgen JSON report into the legacy reth-bench CSVs. +# +# Usage: bench-txgen-run.sh