feat(core): reject empty extract outputSchema at runtime #59
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Trigger Pilo Eval via Repository Dispatch | |
| # Triggers evaluation in pilo-evals-judge via repository_dispatch event | |
| # when pushing to evals/** branches | |
| on: | |
| push: | |
| branches: | |
| - "evals/**" | |
| permissions: | |
| contents: read | |
| jobs: | |
| trigger-eval: | |
| name: Trigger Eval in pilo-evals-judge | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Parse eval name from branch | |
| id: parse | |
| run: | | |
| BRANCH="${{ github.ref }}" | |
| echo "Branch: $BRANCH" | |
| # Extract eval name from branch: refs/heads/evals/{eval_name}/... | |
| if [[ "$BRANCH" =~ ^refs/heads/evals/([^/]+)/ ]]; then | |
| EVAL_NAME="${BASH_REMATCH[1]}" | |
| echo "eval_name=$EVAL_NAME" >> $GITHUB_OUTPUT | |
| echo "✅ Parsed eval name: $EVAL_NAME" | |
| else | |
| echo "❌ Error: Branch does not match evals/{eval_name}/* pattern" | |
| echo " Expected format: evals/single/*, evals/partial/*, evals/full/*" | |
| exit 1 | |
| fi | |
| - name: Trigger repository_dispatch | |
| env: | |
| COMMIT_MESSAGE: ${{ github.event.head_commit.message }} | |
| run: | | |
| echo "Sending repository_dispatch event to pilo-evals-judge..." | |
| echo "" | |
| echo "Payload:" | |
| echo " eval_name: ${{ steps.parse.outputs.eval_name }}" | |
| echo " git_sha: ${{ github.sha }}" | |
| echo " git_ref: ${{ github.ref }}" | |
| echo " repo: ${{ github.repository }}" | |
| echo " commit_timestamp: ${{ github.event.head_commit.timestamp }}" | |
| echo " run_id: ${{ github.run_id }}" | |
| echo "" | |
| # Escape commit message for JSON (replace newlines, quotes, backslashes) | |
| COMMIT_MSG=$(echo "$COMMIT_MESSAGE" | jq -Rs .) | |
| # Generate current timestamp | |
| TRIGGER_TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ) | |
| # Create payload file with proper JSON escaping | |
| cat > payload.json <<EOF | |
| { | |
| "event_type": "pilo_eval", | |
| "client_payload": { | |
| "eval_name": "${{ steps.parse.outputs.eval_name }}", | |
| "git_sha": "${{ github.sha }}", | |
| "git_ref": "${{ github.ref }}", | |
| "repo": "${{ github.repository }}", | |
| "commit_timestamp": "${{ github.event.head_commit.timestamp }}", | |
| "commit_message": $COMMIT_MSG, | |
| "trigger_timestamp": "$TRIGGER_TIMESTAMP", | |
| "workflow_name": "${{ github.workflow }}", | |
| "run_id": "${{ github.run_id }}", | |
| "actor": "${{ github.actor }}" | |
| } | |
| } | |
| EOF | |
| # Send repository_dispatch event | |
| HTTP_STATUS=$(curl -s -o response.json -w "%{http_code}" \ | |
| -X POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer ${{ secrets.PILO_EVALS_DISPATCH_TOKEN }}" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| https://api.github.com/repos/Mozilla-Ocho/pilo-evals-judge/dispatches \ | |
| --data @payload.json) | |
| echo "" | |
| echo "GitHub API HTTP status: $HTTP_STATUS" | |
| # Check HTTP status code | |
| if [[ "$HTTP_STATUS" -lt 200 || "$HTTP_STATUS" -ge 300 ]]; then | |
| echo "❌ Error: Failed to send repository_dispatch event (HTTP $HTTP_STATUS)" | |
| if [[ -s response.json ]]; then | |
| echo "Response body:" | |
| cat response.json | |
| fi | |
| exit 1 | |
| fi | |
| # Even with 200 OK, check for error in response body | |
| if [[ -s response.json ]]; then | |
| if grep -q '"message"' response.json || grep -q '"error"' response.json; then | |
| echo "⚠️ Warning: Response contains error/message field:" | |
| cat response.json | |
| # GitHub API sometimes returns errors with 200 OK | |
| if grep -qi '"Not Found"' response.json || grep -qi '"Bad credentials"' response.json; then | |
| echo "❌ Error: API call failed despite 200 OK status" | |
| exit 1 | |
| fi | |
| fi | |
| fi | |
| echo "" | |
| echo "✅ Repository dispatch event sent" | |
| echo "" | |
| echo "Monitor the eval at:" | |
| echo " https://github.com/Mozilla-Ocho/pilo-evals-judge/actions" | |
| echo "" | |
| echo "Check commit status at:" | |
| echo " https://github.com/${{ github.repository }}/commit/${{ github.sha }}" |