fix(evals): portable hooks + OpenRouter CI + code-exec grader — eval system review-ready

.github/scripts/run-evals.ts

@@ -0,0 +1,98 @@
+#!/usr/bin/env bun
+
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+// CI eval runner for OpenRouterTeam/skills.
+// Runs every authored eval suite under evals/ through waza (driven against
+// OpenRouter — see .github/workflows/eval.yml for the COPILOT_PROVIDER_* env).
+// Skips stub suites (description starts with "TODO: scaffolding only"), so CI
+// only grades suites with real tasks. Exits non-zero if any suite fails.
+
+const STUB_MARKER = "TODO: scaffolding only";
+const REPO_ROOT = process.cwd();
+const EVALS_DIR = join(REPO_ROOT, "evals");
+
+function parseOutputDir(argv: string[]): string {
+  const i = argv.indexOf("--output-dir");
+  return i >= 0 && argv[i + 1] ? argv[i + 1] : "./results";
+}
+
+function isStub(specPath: string): boolean {
+  const body = readFileSync(specPath, "utf-8");
+  return body.includes(STUB_MARKER);
+}
+
+function discoverSuites(): { skill: string; spec: string }[] {
+  const out: { skill: string; spec: string }[] = [];
+  for (const entry of readdirSync(EVALS_DIR)) {
+    if (entry.startsWith("_")) {
+      continue;
+    }
+    const spec = join(EVALS_DIR, entry, "eval.yaml");
+    if (!existsSync(spec)) {
+      continue;
+    }
+    if (isStub(spec)) {
+      console.error(`⏭  skip (stub): ${entry}`);
+      continue;
+    }
+    out.push({ skill: entry, spec });
+  }
+  return out.sort((a, b) => a.skill.localeCompare(b.skill));
+}
+
+function main(): void {
+  const outputDir = parseOutputDir(process.argv.slice(2));
+  mkdirSync(outputDir, { recursive: true });
+
+  const model = process.env.EVAL_MODEL ?? "anthropic/claude-opus-4.8";
+  const suites = discoverSuites();
+  if (suites.length === 0) {
+    console.error("❌ no authored eval suites found under evals/");
+    process.exit(1);
+  }
+
+  console.error(`▶ running ${suites.length} suite(s) on ${model}\n`);
+  const results: { skill: string; code: number }[] = [];
+
+  for (const { skill, spec } of suites) {
+    console.error(`\n${"=".repeat(60)}\n▶ ${skill}\n${"=".repeat(60)}`);
+    const run = spawnSync(
+      "waza",
+      [
+        "run",
+        spec,
+        "--judge-model",
+        model,
+        "--output-dir",
+        join(outputDir, skill),
+        "--no-update-check",
+        "-v",
+      ],
+      { cwd: REPO_ROOT, env: process.env, stdio: ["ignore", "inherit", "inherit"] },
+    );
+    results.push({ skill, code: run.status ?? 1 });
+  }
+
+  const failed = results.filter((r) => r.code !== 0);
+  const summary = {
+    model,
+    total: results.length,
+    passed: results.length - failed.length,
+    failed: failed.length,
+    suites: results,
+  };
+  writeFileSync(join(outputDir, "ci-summary.json"), JSON.stringify(summary, null, 2));
+
+  console.error(`\n${"=".repeat(60)}\n CI EVAL SUMMARY\n${"=".repeat(60)}`);
+  for (const r of results) {
+    console.error(`  ${r.code === 0 ? "✓" : "✗"} ${r.skill}`);
+  }
+  console.error(`\n${summary.passed}/${summary.total} suites passed on ${model}`);
+
+  process.exit(failed.length > 0 ? 1 : 0);
+}
+
+main();

.github/workflows/eval.yml

@@ -6,25 +6,75 @@ on:
     paths:
       - 'evals/**'
       - 'skills/**'
+  workflow_dispatch:
+    inputs:
+      model:
+        description: 'OpenRouter model slug for agent + judge'
+        required: false
+        default: 'anthropic/claude-opus-4.8'
 
 permissions:
   contents: read
 
+concurrency:
+  group: eval-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  # waza drives its embedded GitHub Copilot CLI through OpenRouter via BYOK —
+  # no GitHub Copilot account, no adapter. OpenRouter is OpenAI-compatible.
+  COPILOT_PROVIDER_TYPE: openai
+  COPILOT_PROVIDER_BASE_URL: https://openrouter.ai/api/v1
+  COPILOT_PROVIDER_WIRE_API: completions
+  COPILOT_PROVIDER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+  COPILOT_MODEL: ${{ github.event.inputs.model || 'anthropic/claude-opus-4.8' }}
+  COPILOT_DISABLE_KEYTARCLI: '1'
+  EVAL_MODEL: ${{ github.event.inputs.model || 'anthropic/claude-opus-4.8' }}
+
 jobs:
+  guard:
+    name: Check secret present
+    runs-on: ubuntu-latest
+    outputs:
+      has_key: ${{ steps.check.outputs.has_key }}
+    steps:
+      - id: check
+        env:
+          KEY: ${{ secrets.OPENROUTER_API_KEY }}
+        run: |
+          if [ -n "$KEY" ]; then
+            echo "has_key=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "has_key=false" >> "$GITHUB_OUTPUT"
+            echo "::warning::OPENROUTER_API_KEY repo secret is not set — eval job skipped. A repo admin must add it under Settings → Secrets and variables → Actions."
+          fi
+
   eval:
     name: Run Evaluations
+    needs: guard
+    if: needs.guard.outputs.has_key == 'true'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Install Azure Developer CLI
-        uses: Azure/setup-azd@v2
-      - name: Install waza extension
+
+      - name: Install Bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+
+      - name: Install Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+
+      - name: Install waza
         run: |
-          azd config set alpha.extensions on
-          azd ext source add -n waza -t url -l https://raw.githubusercontent.com/microsoft/waza/main/registry.json
-          azd ext install microsoft.azd.waza
+          curl -fsSL https://raw.githubusercontent.com/microsoft/waza/main/install.sh | bash
+          waza --version
+
       - name: Run evaluations
-        run: azd waza run --output-dir ./results
+        run: bun .github/scripts/run-evals.ts --output-dir ./results
+
       - name: Upload results
         if: always()
         uses: actions/upload-artifact@v4

evals/_graders/README.md

@@ -0,0 +1,44 @@
+# `_graders` — shared waza grader helpers
+
+Reusable `program`-grader scripts that any task can reference.
+
+## `verify-code-runs.ts` — does the produced code actually run?
+
+Closes the "does the artifact run" gap from OpenAI's
+[skill-eval methodology](https://developers.openai.com/blog/eval-skills): for
+*building* skills, grading the response text for the right patterns isn't enough
+— the code the agent produces should actually parse/type-check.
+
+```yaml
+graders:
+  - type: program
+    name: produced_code_parses
+    config:
+      command: "bun"
+      args: ["evals/_graders/verify-code-runs.ts"]
+      timeout: 60
+```
+
+### How it works
+
+waza pipes the agent's full final response to a `program` grader on **stdin**
+(exit 0 = pass, non-zero = fail). This helper:
+
+1. Extracts fenced code blocks from the response.
+2. Writes JS/TS blocks to temp files.
+3. Syntax-checks JS with `node --check`; type-checks TS with
+   `npx tsc --noEmit --skipLibCheck` (only fails on `TS1xxx` *syntax* errors, so
+   missing-import noise in a snippet doesn't false-fail).
+4. Exits 0 only if every code block parses; 1 otherwise (including "no code
+   block found", which is itself a failure for a code-producing task).
+
+### waza `program` grader facts (learned the hard way)
+
+- The agent's response arrives on **stdin** — not as workspace files. The
+  copilot agent-under-test answers in text and does **not** populate
+  `$WAZA_WORKSPACE_DIR` by default.
+- waza **swallows grader stderr on success** and only surfaces it on failure.
+  Exit non-zero (temporarily) if you need to see a grader's diagnostics.
+- Reference a committed script via a single clean `args` entry rather than
+  inlining bash — inline shell/YAML escaping is brittle (same lesson as the
+  `_hooks/` helpers).

evals/_graders/verify-code-runs.ts

@@ -0,0 +1,93 @@
+#!/usr/bin/env bun
+
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+// waza `program` grader: verifies that the code the agent produced actually
+// parses / type-checks, not just that it looks right to a judge (gap #4 from
+// the OpenAI eval methodology — "does the artifact run").
+//
+// waza pipes the agent's full response to this grader on STDIN. We extract
+// fenced code blocks, write JS/TS to temp files, and syntax/type-check them.
+// Exit 0 = all blocks valid (pass); exit 1 = a block failed to parse (fail).
+//
+// Usage in a task:
+//   - type: program
+//     name: produced_code_parses
+//     config:
+//       command: "bun"
+//       args: ["evals/_graders/verify-code-runs.ts"]
+//       timeout: 60
+
+interface Block {
+  lang: string;
+  code: string;
+}
+
+function extractBlocks(md: string): Block[] {
+  const blocks: Block[] = [];
+  const fence = /```([a-zA-Z0-9_+-]*)\n([\s\S]*?)```/g;
+  let m: RegExpExecArray | null;
+  // biome-ignore lint/suspicious/noAssignInExpressions: standard regex exec loop
+  while ((m = fence.exec(md)) !== null) {
+    blocks.push({ lang: (m[1] || "").toLowerCase(), code: m[2] });
+  }
+  return blocks;
+}
+
+const JS_LANGS = new Set(["js", "javascript", "jsx", "mjs", "cjs", "node"]);
+const TS_LANGS = new Set(["ts", "typescript", "tsx"]);
+
+function checkJs(file: string): { ok: boolean; err: string } {
+  const r = spawnSync("node", ["--check", file], { encoding: "utf-8" });
+  return { ok: r.status === 0, err: r.stderr || "" };
+}
+
+function checkTs(file: string): { ok: boolean; err: string } {
+  // Type-check without emitting; skip lib checks for speed and to tolerate
+  // missing ambient deps in snippet-sized code.
+  const r = spawnSync(
+    "npx",
+    ["--yes", "tsc", "--noEmit", "--skipLibCheck", "--allowJs", "--moduleResolution", "node", file],
+    { encoding: "utf-8", timeout: 50_000 },
+  );
+  // tsc errors about missing imports are common in snippets and not what we're
+  // testing; we only fail on genuine syntax errors (TS1xxx).
+  const syntaxError = /error TS1\d{3}/.test(r.stdout || "");
+  return { ok: !syntaxError, err: syntaxError ? r.stdout : "" };
+}
+
+async function main(): Promise<void> {
+  const input = await Bun.stdin.text();
+  const blocks = extractBlocks(input);
+  const codeBlocks = blocks.filter((b) => JS_LANGS.has(b.lang) || TS_LANGS.has(b.lang));
+
+  if (codeBlocks.length === 0) {
+    process.stderr.write("verify-code-runs: no JS/TS code blocks found in agent output\n");
+    process.exit(1);
+  }
+
+  const dir = mkdtempSync(join(tmpdir(), "verify-code-"));
+  let failures = 0;
+  for (const [i, b] of codeBlocks.entries()) {
+    const isTs = TS_LANGS.has(b.lang);
+    const file = join(dir, `block-${i}.${isTs ? "ts" : "js"}`);
+    writeFileSync(file, b.code);
+    const res = isTs ? checkTs(file) : checkJs(file);
+    if (!res.ok) {
+      failures++;
+      process.stderr.write(`verify-code-runs: block ${i} (${b.lang}) FAILED:\n${res.err.slice(0, 500)}\n`);
+    }
+  }
+
+  if (failures > 0) {
+    process.stderr.write(`verify-code-runs: ${failures}/${codeBlocks.length} block(s) failed to parse\n`);
+    process.exit(1);
+  }
+  process.stderr.write(`verify-code-runs: all ${codeBlocks.length} code block(s) parse cleanly\n`);
+  process.exit(0);
+}
+
+main();

evals/_hooks/README.md

@@ -0,0 +1,41 @@
+# `_hooks` — shared waza `before_run` helpers
+
+## `prepare-skill.ts`
+
+Syncs a skill into `~/.agents/skills/<name>/` (where the copilot agent discovers
+skills) and runs `npm install` if the skill ships bundled scripts.
+
+```yaml
+hooks:
+  before_run:
+    - command: "bun evals/_hooks/prepare-skill.ts <skill-name>"
+```
+
+### Two gotchas this helper exists to handle
+
+1. **waza runs hook `command:` entries WITHOUT a shell.** `cd X && cmd` fails with
+   `cd: executable file not found in $PATH`. Each hook must be a single executable
+   invocation — no `&&`, no shell builtins. This helper does the multi-step work
+   (mkdir + rsync + npm install) in one process.
+
+2. **Hooks run from waza's invocation cwd (the repo root), not the eval dir.** So the
+   path is `evals/_hooks/prepare-skill.ts`, not `../_hooks/...`. The helper itself
+   resolves the repo root from its own file location, so it's robust regardless.
+
+## Running evals against OpenRouter
+
+waza's embedded GitHub Copilot CLI has native BYOK custom-provider support, and
+OpenRouter is OpenAI-compatible — so waza runs on OpenRouter with **zero adapter
+code**. Set these env vars before `waza run` (the skillet `skill-eval` skill's
+`run-waza.ts` sets them automatically):
+
+```
+COPILOT_PROVIDER_TYPE=openai
+COPILOT_PROVIDER_BASE_URL=https://openrouter.ai/api/v1
+COPILOT_PROVIDER_WIRE_API=completions          # NOT "openai" — waza rejects that
+COPILOT_PROVIDER_API_KEY=$OPENROUTER_API_KEY
+COPILOT_MODEL=anthropic/claude-opus-4.8        # BYOK refuses to start without an explicit model
+COPILOT_DISABLE_KEYTARCLI=1
+```
+
+Model slugs must be real OpenRouter IDs (family-first, e.g. `anthropic/claude-opus-4.8`).