CLI Scale Benchmark #3
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CLI Scale Benchmark | |
| on: | |
| schedule: | |
| - cron: "17 9 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| scale_files: | |
| description: "Comma-separated file counts, for example 10000,100000,1000000" | |
| required: true | |
| default: "10000,100000,1000000" | |
| run_materialized: | |
| description: "Run materialized/sparse workdir benchmark cases" | |
| required: true | |
| default: "0" | |
| run_backup: | |
| description: "Run backup benchmark cases" | |
| required: true | |
| default: "0" | |
| run_daemon: | |
| description: "Run daemon hot-path benchmark cases" | |
| required: true | |
| default: "1" | |
| run_git_import: | |
| description: "Run Git import/update benchmark cases" | |
| required: true | |
| default: "1" | |
| jobs: | |
| scale: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 360 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Run CLI scale benchmark | |
| run: scripts/cli-scale-bench.sh | |
| env: | |
| CRABDB_SCALE_BASE: ${{ runner.temp }} | |
| CRABDB_SCALE_LABEL: scale-${{ github.run_id }} | |
| CRABDB_SCALE_FILES: ${{ github.event.inputs.scale_files || '10000,100000,1000000' }} | |
| CRABDB_SCALE_MATERIALIZED: ${{ github.event.inputs.run_materialized || '0' }} | |
| CRABDB_SCALE_BACKUP: ${{ github.event.inputs.run_backup || '0' }} | |
| CRABDB_SCALE_DAEMON: ${{ github.event.inputs.run_daemon || '1' }} | |
| CRABDB_SCALE_GIT_IMPORT: ${{ github.event.inputs.run_git_import || '1' }} | |
| - name: Summarize scale results | |
| if: always() | |
| run: | | |
| find "${{ runner.temp }}/crabdb-cli-scale-scale-${{ github.run_id }}" -name results.tsv -print -exec cat {} \; || true | |
| find "${{ runner.temp }}/crabdb-cli-scale-scale-${{ github.run_id }}" -name metrics.tsv -print -exec cat {} \; || true | |
| - name: Check large-agent hot path thresholds | |
| if: always() | |
| run: | | |
| if [ "${{ github.event.inputs.run_daemon || '1' }}" != "1" ]; then | |
| echo "skipping daemon hot-path thresholds because run_daemon is disabled" | |
| exit 0 | |
| fi | |
| root="${{ runner.temp }}/crabdb-cli-scale-scale-${{ github.run_id }}" | |
| mapfile -t result_files < <(find "$root" -name results.tsv -print | sort) | |
| if [ "${#result_files[@]}" -eq 0 ]; then | |
| echo "no scale benchmark results found under $root" >&2 | |
| exit 1 | |
| fi | |
| git_thresholds=() | |
| if [ "${{ github.event.inputs.run_git_import || '1' }}" = "1" ]; then | |
| git_thresholds=( | |
| git_dirty_status=120 | |
| git_dirty_diff=120 | |
| git_dirty_record=120 | |
| git_status_after_dirty_record=90 | |
| ) | |
| fi | |
| materialized_thresholds=() | |
| if [ "${{ github.event.inputs.run_materialized || '0' }}" = "1" ]; then | |
| materialized_thresholds=( | |
| agent_spawn_sparse=30 | |
| agent_read_sparse_hydrate_neighbors=30 | |
| agent_sync_sparse_dir=30 | |
| agent_spawn_materialized=120 | |
| agent_status_materialized=60 | |
| agent_record_materialized=120 | |
| ) | |
| fi | |
| backup_thresholds=() | |
| if [ "${{ github.event.inputs.run_backup || '0' }}" = "1" ]; then | |
| backup_thresholds=( | |
| backup_create=180 | |
| backup_verify=180 | |
| ) | |
| fi | |
| for results in "${result_files[@]}"; do | |
| metrics="$(dirname "$results")/metrics.tsv" | |
| python3 scripts/check-cli-scale-thresholds.py "$results" \ | |
| daemon_wait_for_health=60 \ | |
| daemon_wait_for_hot_cache=120 \ | |
| daemon_status=5 \ | |
| daemon_persisted_snapshot_status=5 \ | |
| daemon_persisted_snapshot_record_clean=5 \ | |
| daemon_persisted_snapshot_diff_dirty=10 \ | |
| daemon_cli_status=5 \ | |
| daemon_cli_record_dirty=10 \ | |
| daemon_cli_agent_readiness=5 \ | |
| daemon_cli_agent_trace_summary=5 \ | |
| daemon_cli_merge_dry_run=10 \ | |
| daemon_cli_session_start=10 \ | |
| daemon_cli_approval_request=10 \ | |
| daemon_cli_lease_acquire=10 \ | |
| daemon_cli_timeline=10 \ | |
| daemon_cli_why=10 \ | |
| daemon_cli_history=10 \ | |
| daemon_cli_code_from=10 \ | |
| agent_apply_patch=10 \ | |
| agent_readiness=10 \ | |
| merge_agent_dry_run=10 \ | |
| merge_agent_apply=10 \ | |
| merge_queue_run=10 \ | |
| "${git_thresholds[@]}" \ | |
| "${materialized_thresholds[@]}" \ | |
| "${backup_thresholds[@]}" \ | |
| --metrics "$metrics" \ | |
| sqlite_bytes=4000000000 \ | |
| dbstat_repo_prolly_nodes=2500000000 \ | |
| object_kind_repo_TextContent_bytes=1200000000 \ | |
| object_count=2000000 | |
| done | |
| - name: Upload scale results | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: crabdb-cli-scale-results | |
| path: ${{ runner.temp }}/crabdb-cli-scale-scale-${{ github.run_id }} |