Benchmark #17
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Scheduled benchmark runs (design/architecture.md). | |
| # | |
| # Loop avoidance (this workflow commits to the repo it runs in), three layers: | |
| # 1. Triggers are schedule + dispatch ONLY — its own commits can never retrigger it. | |
| # 2. The bot commit message carries [skip ci]. | |
| # 3. test.yml ignores data/results/** on push. | |
| # | |
| # Tests gate the benchmarks: if any test fails, nothing is recorded. | |
| name: Benchmark | |
| on: | |
| schedule: | |
| - cron: "0 3 * * 1" # Monday 03:00 UTC | |
| workflow_dispatch: | |
| inputs: | |
| smoke: | |
| description: "Smoke mode (tiny sizes, validates the pipeline without recording)" | |
| type: boolean | |
| default: false | |
| concurrency: | |
| group: benchmark | |
| cancel-in-progress: false | |
| # Run JS-based actions (checkout, setup-julia, and the actions/cache + pyTooling | |
| # steps vendored inside julia-actions/cache) on Node 24 — Node 20 is deprecated. | |
| env: | |
| FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" | |
| jobs: | |
| tests: | |
| name: Gate — full test suite | |
| uses: ./.github/workflows/test.yml | |
| permissions: | |
| contents: read | |
| bench: | |
| name: Benchmark (${{ matrix.hardware }} · Julia ${{ matrix.julia }}) | |
| needs: tests | |
| runs-on: ${{ matrix.runner }} | |
| permissions: | |
| contents: write # pushes result files | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 # serialize commits across every hardware/Julia leg | |
| matrix: | |
| include: | |
| - { julia: "1.10", runner: ubuntu-latest, hardware: github-actions-ubuntu } | |
| - { julia: "1.11", runner: ubuntu-latest, hardware: github-actions-ubuntu } | |
| - { julia: "1.12", runner: ubuntu-latest, hardware: github-actions-ubuntu } | |
| - { julia: "1.10", runner: [self-hosted, ARM64, pi5], hardware: rpi5-8gb } | |
| - { julia: "1.11", runner: [self-hosted, ARM64, pi5], hardware: rpi5-8gb } | |
| - { julia: "1.12", runner: [self-hosted, ARM64, pi5], hardware: rpi5-8gb } | |
| env: | |
| RXBENCH_HARDWARE_ID: ${{ matrix.hardware }} | |
| USE_DEV: "false" | |
| LOG_USING_RXINFER: "false" | |
| JULIA_FASTCHOLESKY_THROW_ERROR_NON_SYMMETRIC: "1" | |
| THROW_ON_INFERENCE_ERROR_HINT: "true" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: julia-actions/setup-julia@v2 | |
| with: | |
| version: ${{ matrix.julia }} | |
| - uses: julia-actions/cache@v2 | |
| - name: Instantiate harness | |
| run: julia --project=benchmarks/harness -e 'using Pkg; Pkg.instantiate()' | |
| - name: Update models to the latest released RxInfer | |
| # The resolved manifests define the environment fingerprint for this run. | |
| run: | | |
| for d in models/*/; do m=$(basename "$d"); | |
| echo "==> models/$m" | |
| julia --project=models/$m -e 'using Pkg; Pkg.update()' | |
| done | |
| - name: Run benchmarks | |
| env: | |
| RXBENCH_SMOKE: ${{ inputs.smoke && '1' || '0' }} | |
| run: | | |
| if [ "$RXBENCH_SMOKE" = "1" ]; then | |
| # smoke mode writes to a temp dir and records nothing | |
| julia --project=benchmarks/harness benchmarks/harness/bin/run_benchmarks.jl | |
| else | |
| # --output data/results is the ONLY path that writes the public REAL dataset; | |
| # without it a run defaults to the local seed tree (design/data.md). | |
| julia --project=benchmarks/harness benchmarks/harness/bin/run_benchmarks.jl --output data/results | |
| julia --project=benchmarks/harness benchmarks/harness/bin/build_index.jl | |
| fi | |
| - name: Commit results | |
| if: ${{ !inputs.smoke }} | |
| run: | | |
| git config user.name "rxinfer-bench-bot" | |
| git config user.email "[email protected]" | |
| git add data/results data/experiments.json data/hardware.json data/metrics.json | |
| if git diff --staged --quiet; then | |
| echo "no new results" | |
| exit 0 | |
| fi | |
| git commit -m "bench: results for ${GITHUB_SHA::7} (${{ matrix.hardware }} julia ${{ matrix.julia }}) [skip ci]" | |
| # Another matrix leg may have pushed first. Result files live in disjoint | |
| # folders, but the regenerated index.json/mirrors can conflict — resolve by | |
| # regenerating them from the union (they are wholesale-generated artifacts). | |
| if ! git pull --rebase origin main; then | |
| julia --project=benchmarks/harness benchmarks/harness/bin/build_index.jl | |
| git add data/results/index.json data/experiments.json data/hardware.json data/metrics.json | |
| GIT_EDITOR=true git rebase --continue | |
| fi | |
| julia --project=benchmarks/harness benchmarks/harness/bin/build_index.jl | |
| git add data/results/index.json | |
| git diff --staged --quiet || git commit --amend --no-edit | |
| git push origin HEAD:main |