Loadtest Live #14
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Loadtest Live | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| base_url: | |
| description: "Optional override base URL (fallback: secret NULL_LOADTEST_BASE_URL)" | |
| required: false | |
| type: string | |
| dry_run: | |
| description: "Force dry-run mode" | |
| required: false | |
| type: boolean | |
| default: false | |
| requests: | |
| description: "Total requests" | |
| required: false | |
| type: string | |
| default: "600" | |
| concurrency: | |
| description: "Concurrent workers" | |
| required: false | |
| type: string | |
| default: "30" | |
| history_window: | |
| description: "Trend table window size" | |
| required: false | |
| type: string | |
| default: "30" | |
| target_success_rate: | |
| description: "Alert threshold: success rate" | |
| required: false | |
| type: string | |
| default: "0.98" | |
| target_p95_ms: | |
| description: "Alert threshold: p95 latency (ms)" | |
| required: false | |
| type: string | |
| default: "1000" | |
| schedule: | |
| - cron: "0 3 * * 1" | |
| jobs: | |
| loadtest-live: | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: backend | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Prepare artifacts dir | |
| run: mkdir -p ../artifacts | |
| - uses: actions/cache/restore@v4 | |
| id: history_cache | |
| with: | |
| path: artifacts/loadtest-history.jsonl | |
| key: loadtest-history-${{ github.ref_name }}-${{ github.run_id }} | |
| restore-keys: | | |
| loadtest-history-${{ github.ref_name }}- | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - run: pip install poetry | |
| - run: poetry install --no-interaction | |
| - name: Run live-or-dry benchmark | |
| id: benchmark | |
| env: | |
| INPUT_BASE_URL: ${{ github.event.inputs.base_url }} | |
| INPUT_DRY_RUN: ${{ github.event.inputs.dry_run }} | |
| INPUT_REQUESTS: ${{ github.event.inputs.requests }} | |
| INPUT_CONCURRENCY: ${{ github.event.inputs.concurrency }} | |
| INPUT_HISTORY_WINDOW: ${{ github.event.inputs.history_window }} | |
| INPUT_TARGET_SUCCESS_RATE: ${{ github.event.inputs.target_success_rate }} | |
| INPUT_TARGET_P95_MS: ${{ github.event.inputs.target_p95_ms }} | |
| SECRET_BASE_URL: ${{ secrets.NULL_LOADTEST_BASE_URL }} | |
| run: | | |
| set -euo pipefail | |
| requests="${INPUT_REQUESTS:-600}" | |
| concurrency="${INPUT_CONCURRENCY:-30}" | |
| history_window="${INPUT_HISTORY_WINDOW:-30}" | |
| target_success_rate="${INPUT_TARGET_SUCCESS_RATE:-0.98}" | |
| target_p95_ms="${INPUT_TARGET_P95_MS:-1000}" | |
| force_dry_run="${INPUT_DRY_RUN:-false}" | |
| base_url="${INPUT_BASE_URL:-$SECRET_BASE_URL}" | |
| base_url_status="n/a" | |
| if [ -n "$base_url" ]; then | |
| base_url_status="configured" | |
| fi | |
| args=( | |
| --requests "$requests" | |
| --concurrency "$concurrency" | |
| --out ../artifacts/loadtest-report.json | |
| --history-out ../artifacts/loadtest-history.jsonl | |
| --trend-out ../artifacts/loadtest-trend.md | |
| --history-window "$history_window" | |
| --target-success-rate "$target_success_rate" | |
| --target-p95-ms "$target_p95_ms" | |
| ) | |
| run_mode="live" | |
| if [ -z "$base_url" ] || [ "$force_dry_run" = "true" ]; then | |
| run_mode="dry-run" | |
| args+=(--dry-run --base-url "${base_url:-http://localhost:3301}" --no-fail-on-alert) | |
| else | |
| args+=(--base-url "$base_url") | |
| fi | |
| echo "run_mode=$run_mode" >> "$GITHUB_OUTPUT" | |
| echo "base_url_status=$base_url_status" >> "$GITHUB_OUTPUT" | |
| echo "requests=$requests" >> "$GITHUB_OUTPUT" | |
| echo "concurrency=$concurrency" >> "$GITHUB_OUTPUT" | |
| echo "target_success_rate=$target_success_rate" >> "$GITHUB_OUTPUT" | |
| echo "target_p95_ms=$target_p95_ms" >> "$GITHUB_OUTPUT" | |
| poetry run python scripts/loadtest.py "${args[@]}" | |
| - name: Publish run summary | |
| if: always() | |
| run: | | |
| { | |
| echo "## Loadtest Live Summary" | |
| echo "" | |
| echo "- Mode: ${{ steps.benchmark.outputs.run_mode }}" | |
| echo "- Base URL: ${{ steps.benchmark.outputs.base_url_status }}" | |
| echo "- Requests: ${{ steps.benchmark.outputs.requests }}" | |
| echo "- Concurrency: ${{ steps.benchmark.outputs.concurrency }}" | |
| echo "- Target Success Rate: ${{ steps.benchmark.outputs.target_success_rate }}" | |
| echo "- Target P95 (ms): ${{ steps.benchmark.outputs.target_p95_ms }}" | |
| echo "" | |
| if [ -f ../artifacts/loadtest-trend.md ]; then | |
| cat ../artifacts/loadtest-trend.md | |
| else | |
| echo "No trend output generated." | |
| fi | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Send trend report to external webhook | |
| if: always() | |
| env: | |
| WEBHOOK_URL: ${{ secrets.NULL_LOADTEST_REPORT_WEBHOOK_URL }} | |
| RUN_MODE: ${{ steps.benchmark.outputs.run_mode }} | |
| BASE_URL_STATUS: ${{ steps.benchmark.outputs.base_url_status }} | |
| REQUESTS: ${{ steps.benchmark.outputs.requests }} | |
| CONCURRENCY: ${{ steps.benchmark.outputs.concurrency }} | |
| TARGET_SUCCESS_RATE: ${{ steps.benchmark.outputs.target_success_rate }} | |
| TARGET_P95_MS: ${{ steps.benchmark.outputs.target_p95_ms }} | |
| REPOSITORY: ${{ github.repository }} | |
| RUN_ID: ${{ github.run_id }} | |
| RUN_NUMBER: ${{ github.run_number }} | |
| RUN_ATTEMPT: ${{ github.run_attempt }} | |
| SERVER_URL: ${{ github.server_url }} | |
| run: | | |
| set -euo pipefail | |
| if [ -z "${WEBHOOK_URL:-}" ]; then | |
| echo "NULL_LOADTEST_REPORT_WEBHOOK_URL is not set; skipping webhook publish." | |
| exit 0 | |
| fi | |
| python - <<'PY' | |
| import json | |
| import os | |
| from pathlib import Path | |
| from urllib.request import Request, urlopen | |
| report_path = Path("../artifacts/loadtest-report.json") | |
| trend_path = Path("../artifacts/loadtest-trend.md") | |
| report: dict[str, object] = {} | |
| if report_path.exists(): | |
| report = json.loads(report_path.read_text(encoding="utf-8")) | |
| trend_text = "No trend output generated." | |
| if trend_path.exists(): | |
| trend_text = trend_path.read_text(encoding="utf-8") | |
| if len(trend_text) > 12000: | |
| trend_text = trend_text[:12000] + "\n...(truncated)" | |
| overall = report.get("overall", {}) | |
| latency = {} | |
| if isinstance(overall, dict): | |
| latency = overall.get("latency_ms", {}) | |
| alerts = report.get("alerts", []) | |
| if not isinstance(alerts, list): | |
| alerts = [] | |
| run_url = f"{os.environ['SERVER_URL']}/{os.environ['REPOSITORY']}/actions/runs/{os.environ['RUN_ID']}" | |
| success_rate = overall.get("success_rate", "n/a") if isinstance(overall, dict) else "n/a" | |
| p95_ms = latency.get("p95", "n/a") if isinstance(latency, dict) else "n/a" | |
| throughput_rps = report.get("throughput_rps", "n/a") | |
| text = "\n".join( | |
| [ | |
| "Loadtest Live Report", | |
| f"Repository: {os.environ['REPOSITORY']}", | |
| f"Run: #{os.environ['RUN_NUMBER']} attempt {os.environ['RUN_ATTEMPT']} ({run_url})", | |
| f"Mode: {os.environ['RUN_MODE']} | Base URL: {os.environ['BASE_URL_STATUS']}", | |
| f"Requests/Concurrency: {os.environ['REQUESTS']}/{os.environ['CONCURRENCY']}", | |
| f"Overall success_rate={success_rate}, p95_ms={p95_ms}, throughput_rps={throughput_rps}", | |
| f"Alerts: {len(alerts)}", | |
| ] | |
| ) | |
| payload = { | |
| "text": text, | |
| "loadtest": { | |
| "repository": os.environ["REPOSITORY"], | |
| "run_id": os.environ["RUN_ID"], | |
| "run_number": os.environ["RUN_NUMBER"], | |
| "run_attempt": os.environ["RUN_ATTEMPT"], | |
| "run_url": run_url, | |
| "mode": os.environ["RUN_MODE"], | |
| "base_url_status": os.environ["BASE_URL_STATUS"], | |
| "requests": os.environ["REQUESTS"], | |
| "concurrency": os.environ["CONCURRENCY"], | |
| "targets": { | |
| "success_rate": os.environ["TARGET_SUCCESS_RATE"], | |
| "p95_ms": os.environ["TARGET_P95_MS"], | |
| }, | |
| "overall": { | |
| "ok": overall.get("ok") if isinstance(overall, dict) else None, | |
| "failed": overall.get("failed") if isinstance(overall, dict) else None, | |
| "success_rate": success_rate, | |
| "p95_ms": p95_ms, | |
| "throughput_rps": throughput_rps, | |
| }, | |
| "alerts": alerts, | |
| "trend_markdown": trend_text, | |
| }, | |
| } | |
| req = Request( | |
| os.environ["WEBHOOK_URL"], | |
| data=json.dumps(payload, ensure_ascii=True).encode("utf-8"), | |
| headers={"Content-Type": "application/json"}, | |
| method="POST", | |
| ) | |
| with urlopen(req, timeout=20) as resp: | |
| status = getattr(resp, "status", resp.getcode()) | |
| if status >= 300: | |
| raise RuntimeError(f"Webhook returned non-success status: {status}") | |
| print(f"Sent loadtest webhook payload (status={status}).") | |
| PY | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: loadtest-live-report | |
| path: | | |
| artifacts/loadtest-report.json | |
| artifacts/loadtest-history.jsonl | |
| artifacts/loadtest-trend.md | |
| if-no-files-found: warn | |
| - uses: actions/cache/save@v4 | |
| if: always() && steps.history_cache.outputs.cache-primary-key != '' | |
| with: | |
| path: artifacts/loadtest-history.jsonl | |
| key: ${{ steps.history_cache.outputs.cache-primary-key }} |