feat: improve agent migration skill with measured evals

skills/openrouter-agent-migration/README.md

@@ -19,7 +19,12 @@ For other install methods (Claude Code plugin marketplace, Cursor Rules, etc.) s
 See [SKILL.md](SKILL.md) for the full reference, including:
 
 - When to migrate (which imports trigger it)
-- Package install changes (`@openrouter/sdk` → `@openrouter/agent`)
+- Pre-edit inventory: inspect package versions, identify agent vs platform uses, locate converter call sites, and find test mocks before touching any file
+- Evidence-driven Zod compatibility: preserve the current setup first, typecheck the minimal migration, and upgrade only when the documented nominal mismatch occurs and project-wide compatibility permits
+- Package install changes (`@openrouter/sdk` → `@openrouter/agent`), including lockfile update
 - Import rewrites for `callModel`, `tool()`, `stepCountIs`, `hasToolCall`, `maxCost`, `maxTokensUsed`, `finishReasonIs`
 - Format converter renames (`fromClaudeMessages`, `toClaudeMessage`, `fromChatMessages`, `toChatMessage`)
 - Type renames (`Tool`, `ToolWithExecute`, `ManualTool`, `CallModelInput`, `ModelResult`)
+- Converter type mismatch: `fromClaudeMessages`/`fromChatMessages` return `InputsUnion`, not `Item[]` — prohibited workarounds (`as any`, etc.) and safe fallback paths
+- Test mock path updates and how to handle mixed-project mocks
+- Stale-import search, type-suppression check, typecheck, and diff review before finishing

skills/openrouter-agent-migration/evals/evals.json

@@ -0,0 +1,63 @@
+{
+  "skill_name": "openrouter-agent-migration",
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "Migrate the TypeScript project at evals/fixtures/agent-only from @openrouter/sdk to @openrouter/agent. The project imports OpenRouter from '@openrouter/sdk', tool from '@openrouter/sdk/lib/tool', stop conditions (stepCountIs, hasToolCall, maxCost) from '@openrouter/sdk/lib/stop-conditions', and types (CallModelInput, ModelResult, ToolWithGenerator, Tool, ToolWithExecute, ManualTool) from '@openrouter/sdk/lib/*' subpaths. No platform API features are used. Remove @openrouter/sdk entirely and update all imports to use @openrouter/agent and its subpaths.",
+      "expected_output": "The project's package.json replaces @openrouter/sdk with @openrouter/agent. All imports in src/ are updated: OpenRouter from '@openrouter/agent', tool from '@openrouter/agent/tool', stop conditions from '@openrouter/agent/stop-conditions', type imports (Tool, ToolWithExecute, ManualTool, ToolWithGenerator) from '@openrouter/agent/tool-types', CallModelInput from '@openrouter/agent/async-params', ModelResult from '@openrouter/agent/model-result'. No @openrouter/sdk imports remain anywhere in src/. The bun test and tsc --noEmit commands pass.",
+      "expectations": [
+        "@openrouter/sdk is removed from package.json dependencies",
+        "@openrouter/agent is added to package.json dependencies",
+        "src/agent.ts imports OpenRouter from '@openrouter/agent' as a named import",
+        "src/agent.ts imports stepCountIs, hasToolCall, and maxCost from '@openrouter/agent/stop-conditions'",
+        "src/tools.ts imports tool from '@openrouter/agent/tool'",
+        "src/tools.ts imports Tool, ToolWithExecute, and ManualTool from '@openrouter/agent/tool-types'",
+        "src/types.ts imports CallModelInput from '@openrouter/agent/async-params'",
+        "src/types.ts imports ModelResult from '@openrouter/agent/model-result'",
+        "src/types.ts imports ToolWithGenerator from '@openrouter/agent/tool-types'",
+        "No file in src/ contains any import from '@openrouter/sdk'",
+        "The project's test suite passes (bun test exits 0)",
+        "TypeScript type check passes (tsc --noEmit exits 0)"
+      ]
+    },
+    {
+      "id": 2,
+      "prompt": "Migrate the TypeScript project at evals/fixtures/mixed-platform-agent. The project uses @openrouter/sdk for platform features in src/platform.ts (OpenRouter client for models.list, credits.getCredits, chat.send). The src/agent.ts file uses @openrouter/sdk for agent features (OpenRouter client, tool from '@openrouter/sdk/lib/tool', stop conditions from '@openrouter/sdk/lib/stop-conditions'). Keep @openrouter/sdk for platform.ts but migrate agent.ts to use @openrouter/agent and its subpaths for all agent functionality.",
+      "expected_output": "package.json retains @openrouter/sdk (for platform features) and adds @openrouter/agent (for agent features). src/platform.ts is unchanged and still imports from '@openrouter/sdk'. src/agent.ts is updated: OpenRouter (or aliased as Agent) from '@openrouter/agent', tool from '@openrouter/agent/tool', stepCountIs/maxTokensUsed/finishReasonIs from '@openrouter/agent/stop-conditions'. src/agent.ts no longer imports from '@openrouter/sdk'. The bun test and tsc --noEmit commands pass.",
+      "expectations": [
+        "@openrouter/sdk is retained in package.json dependencies",
+        "@openrouter/agent is added to package.json dependencies",
+        "src/platform.ts still imports from '@openrouter/sdk' (file unchanged)",
+        "src/agent.ts no longer imports from '@openrouter/sdk'",
+        "src/agent.ts imports OpenRouter (or an alias like Agent) from '@openrouter/agent'",
+        "src/agent.ts imports tool from '@openrouter/agent/tool'",
+        "src/agent.ts imports stepCountIs from '@openrouter/agent/stop-conditions'",
+        "src/agent.ts imports maxTokensUsed from '@openrouter/agent/stop-conditions'",
+        "src/agent.ts imports finishReasonIs from '@openrouter/agent/stop-conditions'",
+        "The project's test suite passes (bun test exits 0)",
+        "TypeScript type check passes (tsc --noEmit exits 0)"
+      ]
+    },
+    {
+      "id": 3,
+      "prompt": "Migrate the TypeScript project at evals/fixtures/streaming-converters from @openrouter/sdk to @openrouter/agent. The project uses format converters in src/converters.ts (fromClaudeMessages, toClaudeMessage from '@openrouter/sdk/lib/anthropic-compat'; fromChatMessages, toChatMessage from '@openrouter/sdk/lib/chat-compat'), type guards and type imports in src/type-guards.ts (hasExecuteFunction, isGeneratorTool, isRegularExecuteTool, Tool, ToolWithExecute, ToolWithGenerator from '@openrouter/sdk/lib/tool-types'; tool from '@openrouter/sdk/lib/tool'), and streaming in src/stream.ts (OpenRouter from '@openrouter/sdk'; CallModelInput from '@openrouter/sdk/lib/async-params'). Update all imports and remove @openrouter/sdk entirely.",
+      "expected_output": "package.json replaces @openrouter/sdk with @openrouter/agent. src/converters.ts imports fromClaudeMessages and toClaudeMessage from '@openrouter/agent' (barrel) and fromChatMessages and toChatMessage from '@openrouter/agent' (barrel). src/type-guards.ts imports hasExecuteFunction, isGeneratorTool, isRegularExecuteTool, Tool, ToolWithExecute, ToolWithGenerator from '@openrouter/agent/tool-types' and tool from '@openrouter/agent/tool'. src/stream.ts imports OpenRouter from '@openrouter/agent' and CallModelInput from '@openrouter/agent/async-params'. No @openrouter/sdk imports remain. The bun test and tsc --noEmit commands pass.",
+      "expectations": [
+        "@openrouter/sdk is removed from package.json dependencies",
+        "@openrouter/agent is added to package.json dependencies",
+        "src/converters.ts imports fromClaudeMessages from '@openrouter/agent'",
+        "src/converters.ts imports toClaudeMessage from '@openrouter/agent'",
+        "src/converters.ts imports fromChatMessages from '@openrouter/agent'",
+        "src/converters.ts imports toChatMessage from '@openrouter/agent'",
+        "src/type-guards.ts imports hasExecuteFunction from '@openrouter/agent/tool-types'",
+        "src/type-guards.ts imports isGeneratorTool from '@openrouter/agent/tool-types'",
+        "src/type-guards.ts imports isRegularExecuteTool from '@openrouter/agent/tool-types'",
+        "src/type-guards.ts imports tool from '@openrouter/agent/tool'",
+        "src/stream.ts imports CallModelInput from '@openrouter/agent/async-params'",
+        "No file in src/ contains any import from '@openrouter/sdk'",
+        "The project's test suite passes (bun test exits 0)",
+        "TypeScript type check passes (tsc --noEmit exits 0)"
+      ]
+    }
+  ]
+}

skills/openrouter-agent-migration/evals/fixtures/agent-only/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "agent-only-fixture",
+  "type": "module",
+  "scripts": {
+    "test": "bun test",
+    "build": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@openrouter/sdk": "0.13.43",
+    "zod": "3.25.76"
+  },
+  "devDependencies": {
+    "@types/bun": "latest",
+    "typescript": "^5.4.0"
+  },
+  "resolutions": {
+    "zod": "3.25.76"
+  }
+}

skills/openrouter-agent-migration/evals/fixtures/agent-only/src/agent.ts

@@ -0,0 +1,27 @@
+import { OpenRouter } from '@openrouter/sdk';
+import { hasToolCall, maxCost, stepCountIs } from '@openrouter/sdk/lib/stop-conditions';
+import { finishTool, searchTool } from './tools.js';
+
+const client = new OpenRouter({
+  apiKey: process.env.OPENROUTER_API_KEY ?? 'sk-or-test',
+});
+
+export async function runResearchAgent(task: string): Promise<string> {
+  const result = client.callModel({
+    model: 'openai/gpt-5-nano',
+    instructions: 'You are a research assistant. Search for information and finish when done.',
+    input: task,
+    tools: [searchTool, finishTool],
+    stopWhen: [stepCountIs(10), hasToolCall('finish'), maxCost(0.5)],
+  });
+
+  return await result.getText();
+}
+
+export async function runSimpleQuery(prompt: string): Promise<string> {
+  const result = client.callModel({
+    model: 'openai/gpt-5-nano',
+    input: prompt,
+  });
+  return await result.getText();
+}

skills/openrouter-agent-migration/evals/fixtures/agent-only/src/tools.ts

@@ -0,0 +1,34 @@
+import { tool } from '@openrouter/sdk/lib/tool';
+import type { ManualTool, Tool, ToolWithExecute } from '@openrouter/sdk/lib/tool-types';
+import { z } from 'zod/v4';
+
+export const searchTool = tool({
+  name: 'web_search',
+  description: 'Search the web for information',
+  inputSchema: z.object({
+    query: z.string().describe('Search query'),
+  }),
+  outputSchema: z.object({
+    results: z.array(z.string()),
+  }),
+  execute: async ({ query }) => {
+    return { results: [`Result for: ${query}`] };
+  },
+});
+
+export const finishTool = tool({
+  name: 'finish',
+  description: 'Signal task completion with final answer',
+  inputSchema: z.object({ answer: z.string() }),
+  execute: async ({ answer }) => ({ answer }),
+});
+
+export const confirmTool: ManualTool = tool({
+  name: 'confirm',
+  description: 'Request manual confirmation',
+  inputSchema: z.object({ message: z.string() }),
+  execute: false,
+});
+
+export type SearchToolFn = ToolWithExecute;
+export type MyTools = Tool[];

skills/openrouter-agent-migration/evals/fixtures/agent-only/src/types.ts

@@ -0,0 +1,12 @@
+import type { CallModelInput } from '@openrouter/sdk/lib/async-params';
+import { ModelResult } from '@openrouter/sdk/lib/model-result';
+import type { Tool, ToolWithGenerator } from '@openrouter/sdk/lib/tool-types';
+
+export type AgentInput = CallModelInput;
+
+export function isModelResult(value: unknown): value is ModelResult<readonly Tool[]> {
+  return value instanceof ModelResult;
+}
+
+// NOTE: ToolWithGenerator uses the SDK's internal zod v4 types via @openrouter/sdk/lib/tool-types
+export type StreamingTool = ToolWithGenerator;

skills/openrouter-agent-migration/evals/fixtures/agent-only/test/agent.test.ts

@@ -0,0 +1,91 @@
+import { describe, expect, mock, test } from 'bun:test';
+
+const mockGetText = mock(async () => 'mocked response');
+const mockCallModel = mock(() => ({
+  getText: mockGetText,
+  getResponse: mock(async () => ({ text: 'mocked', usage: null })),
+}));
+
+mock.module('@openrouter/sdk', () => ({
+  OpenRouter: class MockOpenRouter {
+    callModel = mockCallModel;
+  },
+}));
+
+mock.module('@openrouter/sdk/lib/stop-conditions', () => ({
+  stepCountIs: (n: number) => ({ type: 'stepCount', n }),
+  hasToolCall: (name: string) => ({ type: 'hasToolCall', name }),
+  maxCost: (amount: number) => ({ type: 'maxCost', amount }),
+}));
+
+mock.module('@openrouter/sdk/lib/tool', () => ({
+  tool: (config: unknown) => ({ type: 'function', function: config }),
+}));
+
+mock.module('@openrouter/sdk/lib/tool-types', () => ({}));
+mock.module('@openrouter/sdk/lib/async-params', () => ({}));
+mock.module('@openrouter/sdk/lib/model-result', () => ({
+  ModelResult: class MockModelResult {},
+}));
+
+describe('agent-only fixture', () => {
+  test('searchTool has expected name and execute function', async () => {
+    const { searchTool } = await import('../src/tools.js');
+    const fn = (searchTool as unknown as { function: { name: string; execute: unknown } }).function;
+    expect(fn.name).toBe('web_search');
+    expect(typeof fn.execute).toBe('function');
+  });
+
+  test('finishTool execute returns answer', async () => {
+    const { finishTool } = await import('../src/tools.js');
+    const fn = (finishTool as unknown as {
+      function: { execute: (a: { answer: string }) => Promise<{ answer: string }> };
+    }).function;
+    const result = await fn.execute({ answer: 'test answer' });
+    expect(result).toEqual({ answer: 'test answer' });
+  });
+
+  test('confirmTool has execute: false (manual tool)', async () => {
+    const { confirmTool } = await import('../src/tools.js');
+    const fn = (confirmTool as unknown as { function: { execute: unknown } }).function;
+    expect(fn.execute).toBe(false);
+  });
+
+  test('searchTool execute returns results array for query', async () => {
+    const { searchTool } = await import('../src/tools.js');
+    const fn = (searchTool as unknown as {
+      function: { execute: (a: { query: string }) => Promise<{ results: string[] }> };
+    }).function;
+    const result = await fn.execute({ query: 'quantum computing' });
+    expect(Array.isArray(result.results)).toBe(true);
+    expect(result.results[0]).toContain('quantum computing');
+  });
+
+  test('runResearchAgent calls callModel with three stop conditions', async () => {
+    mockCallModel.mockClear();
+    mockGetText.mockClear();
+    const { runResearchAgent } = await import('../src/agent.js');
+    const text = await runResearchAgent('test task');
+    expect(mockCallModel).toHaveBeenCalled();
+    const callArgs = (mockCallModel.mock.calls as unknown as Array<[{
+      stopWhen: unknown[];
+      input: string;
+    }]>)[0][0];
+    expect(callArgs.stopWhen).toHaveLength(3);
+    expect(callArgs.input).toBe('test task');
+    expect(text).toBe('mocked response');
+  });
+
+  test('runSimpleQuery calls callModel without stop conditions', async () => {
+    mockCallModel.mockClear();
+    const { runSimpleQuery } = await import('../src/agent.js');
+    await runSimpleQuery('hello');
+    expect(mockCallModel).toHaveBeenCalled();
+    const callArgs = (mockCallModel.mock.calls as unknown as Array<[{
+      input: string;
+      stopWhen?: unknown[];
+    }]>)[0][0];
+    expect(callArgs.input).toBe('hello');
+    expect(callArgs.stopWhen).toBeUndefined();
+  });
+});

skills/openrouter-agent-migration/evals/fixtures/agent-only/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "types": ["bun-types"]
+  },
+  "include": ["src", "test"]
+}

skills/openrouter-agent-migration/evals/fixtures/mixed-platform-agent/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "mixed-platform-agent-fixture",
+  "type": "module",
+  "scripts": {
+    "test": "bun test",
+    "build": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@openrouter/sdk": "0.13.43",
+    "zod": "3.25.76"
+  },
+  "devDependencies": {
+    "@types/bun": "latest",
+    "typescript": "^5.4.0"
+  }
+}