diff --git a/skills/openrouter-agent-migration/README.md b/skills/openrouter-agent-migration/README.md index 6a35a33..a8bc02b 100644 --- a/skills/openrouter-agent-migration/README.md +++ b/skills/openrouter-agent-migration/README.md @@ -19,7 +19,12 @@ For other install methods (Claude Code plugin marketplace, Cursor Rules, etc.) s See [SKILL.md](SKILL.md) for the full reference, including: - When to migrate (which imports trigger it) -- Package install changes (`@openrouter/sdk` → `@openrouter/agent`) +- Pre-edit inventory: inspect package versions, identify agent vs platform uses, locate converter call sites, and find test mocks before touching any file +- Evidence-driven Zod compatibility: preserve the current setup first, typecheck the minimal migration, and upgrade only when the documented nominal mismatch occurs and project-wide compatibility permits +- Package install changes (`@openrouter/sdk` → `@openrouter/agent`), including lockfile update - Import rewrites for `callModel`, `tool()`, `stepCountIs`, `hasToolCall`, `maxCost`, `maxTokensUsed`, `finishReasonIs` - Format converter renames (`fromClaudeMessages`, `toClaudeMessage`, `fromChatMessages`, `toChatMessage`) - Type renames (`Tool`, `ToolWithExecute`, `ManualTool`, `CallModelInput`, `ModelResult`) +- Converter type mismatch: `fromClaudeMessages`/`fromChatMessages` return `InputsUnion`, not `Item[]` — prohibited workarounds (`as any`, etc.) and safe fallback paths +- Test mock path updates and how to handle mixed-project mocks +- Stale-import search, type-suppression check, typecheck, and diff review before finishing diff --git a/skills/openrouter-agent-migration/SKILL.md b/skills/openrouter-agent-migration/SKILL.md index 87e8f98..77e721d 100644 --- a/skills/openrouter-agent-migration/SKILL.md +++ b/skills/openrouter-agent-migration/SKILL.md @@ -1,7 +1,7 @@ --- name: openrouter-agent-migration description: Migration guide from @openrouter/sdk to @openrouter/agent for callModel, tool(), stop conditions, and agent features. This skill should be used when code imports callModel, tool(), or stop conditions from @openrouter/sdk and needs to migrate to @openrouter/agent. -version: 1.0.0 +version: 2.0.0 --- # Migrating from @openrouter/sdk to @openrouter/agent @@ -22,15 +22,61 @@ Migrate if your code imports any of these from `@openrouter/sdk`: --- -## Quick Migration +## Before You Edit Any File -### Step 1: Install +Run these four inventory steps first. Do not skip them — they determine the migration scope and expose dependency constraints before edits. + +### 1. Inspect package versions and compatibility + +```bash +# Check declared OpenRouter and zod versions. +grep -E '"zod"|"@openrouter/' package.json + +# If @openrouter/agent is installed, inspect its actual version and dependency constraints. +bun -e "const p=await Bun.file('node_modules/@openrouter/agent/package.json').json(); console.log({version:p.version,zod:p.dependencies?.zod,peerDependencies:p.peerDependencies})" 2>/dev/null || printf '%s\n' '(not installed yet — inspect again after installation)' +``` + +Record the current package versions and lockfile. Do not infer that zod must change merely because the agent package declares native zod v4; first perform the minimal agent migration and typecheck it (see §Zod Version Compatibility). + +### 2. Inventory agent vs platform uses + +```bash +# Find every file importing from @openrouter/sdk (agent features vs platform features) +grep -rn "@openrouter/sdk" src/ test/ --include="*.ts" | grep "^[^:]*import" +``` + +For each file, determine: +- **Agent file** (uses `callModel`, `tool`, stop conditions, converters) → migrate to `@openrouter/agent` +- **Platform file** (uses `models.list`, `chat.send`, `credits`, `oauth`, `apiKeys`) → keep `@openrouter/sdk` + +Files using both must be split or kept on `@openrouter/sdk` (adding `@openrouter/agent` alongside). + +### 3. Check for converter call sites + +```bash +# Find fromClaudeMessages / fromChatMessages call sites — they need special handling (see §Converter Type Mismatch) +grep -rn "fromClaudeMessages\|fromChatMessages" src/ --include="*.ts" +``` + +If any results are found, read §Converter Type Mismatch before editing those files. + +### 4. Check for test mocks that reference old paths + +```bash +grep -rn "mock.module.*@openrouter/sdk" test/ --include="*.ts" +``` + +These need parallel updates when source imports change (see §Updating Tests). + +--- + +## Step 1: Install ```bash npm install @openrouter/agent ``` -If you only use agent features, you can remove `@openrouter/sdk`: +If you only use agent features, you can also remove `@openrouter/sdk`: ```bash npm uninstall @openrouter/sdk @@ -39,7 +85,46 @@ npm install @openrouter/agent If you also use non-agent SDK features (models list, chat completions, credits, OAuth, API keys), keep both packages installed. -### Step 2: Update Imports +**Also update the lockfile.** After editing `package.json`, run your package manager's install command (`npm install`, `bun install`, `yarn install`, `pnpm install`) so the lockfile reflects the new deps. Never edit `package.json` without also running install. + +--- + +## Step 2: Zod Version Compatibility + +`@openrouter/agent@0.7.2` depends on native `zod@^4.0.0`. A project may still compile while declaring Zod v3 and importing its `zod/v4` compatibility entry point: two of three agent-only eval runs did exactly that. Other fixtures exposed nominal incompatibility between the compatibility import and the agent package's native-v4 types. Therefore, package declarations alone do not justify upgrading zod. + +### Evidence-driven decision procedure + +1. **Preserve the project's zod dependency and imports initially.** Install `@openrouter/agent`, migrate only the OpenRouter agent imports, update the lockfile, and run the project's typecheck. + +2. **If typecheck passes, leave zod unchanged.** Do not rewrite `zod/v4` to `zod`, remove resolutions, or widen the zod version merely to align package declarations. + +3. **If typecheck fails, inspect the exact diagnostic before editing zod.** The relevant nominal mismatch usually mentions Zod internals such as: + + ```text + Type '0' is not assignable to type '4' + ``` + + on `_zod.version.minor` at a `tool()` schema boundary. Confirm all of the following: + - The error appeared only after the agent migration. + - The project declares Zod v3 and imports `zod/v4` at the failing schema. + - The diagnostic is a Zod nominal/type-identity mismatch, not an unrelated schema error. + +4. **Only after that confirmation, check project-wide compatibility with native Zod v4.** Inspect all zod consumers, dependency constraints, package-manager overrides (`resolutions`, `overrides`, or equivalent), and tests. Do not remove a v3 pin whose purpose is unknown. + +5. **If the project can adopt native Zod v4**, make the smallest compatible change: + - Change the project's zod dependency to a compatible v4 range. + - Change affected `zod/v4` imports to `zod`. + - Adjust an override only when it is the confirmed source of the mismatch. + - Reinstall with the project's package manager, then rerun typecheck and tests. + +6. **If project compatibility is uncertain or another dependency requires Zod v3**, keep zod unchanged and report the blocker. Do not hide the failure with casts, ignored diagnostics, or lint suppressions. + +This sequence avoids the observed over-migration: an agent-only project that already typechecks keeps its existing zod setup, while a project with the documented nominal incompatibility can upgrade deliberately after proving it is necessary. + +--- + +## Step 3: Update Imports The `OpenRouter` client class and `client.callModel()` pattern work identically. Only the import source changes: @@ -72,17 +157,6 @@ const text = await result.getText(); | `import OpenRouter from '@openrouter/sdk'` | `import { OpenRouter } from '@openrouter/agent'` | | `import OpenRouter, { tool, stepCountIs } from '@openrouter/sdk'` | `import { OpenRouter } from '@openrouter/agent'`
`import { tool } from '@openrouter/agent/tool'`
`import { stepCountIs } from '@openrouter/agent/stop-conditions'` | -A standalone `callModel` function is also available for advanced use cases where a pre-existing `OpenRouterCore` instance is available: - -```typescript -import { callModel } from '@openrouter/agent/call-model'; - -// Requires an OpenRouterCore instance (from @openrouter/sdk/core) -const result = callModel(coreInstance, { model: 'openai/gpt-5-nano', input: 'Hello' }); -``` - -For most use cases, prefer the `client.callModel()` method shown above. - ### Tool Creation | Old | New | @@ -106,10 +180,14 @@ For most use cases, prefer the `client.callModel()` method shown above. ### Format Converters +The barrel import path has changed, but also see §Converter Type Mismatch below before using them. + | Old | New | |-----|-----| | `import { fromClaudeMessages, toClaudeMessage } from '@openrouter/sdk'` | `import { fromClaudeMessages, toClaudeMessage } from '@openrouter/agent'` | | `import { fromChatMessages, toChatMessage } from '@openrouter/sdk'` | `import { fromChatMessages, toChatMessage } from '@openrouter/agent'` | +| `import { fromClaudeMessages, toClaudeMessage } from '@openrouter/sdk/lib/anthropic-compat'` | `import { fromClaudeMessages, toClaudeMessage } from '@openrouter/agent'` | +| `import { fromChatMessages, toChatMessage } from '@openrouter/sdk/lib/chat-compat'` | `import { fromChatMessages, toChatMessage } from '@openrouter/agent'` | ### Type Guards @@ -119,6 +197,106 @@ For most use cases, prefer the `client.callModel()` method shown above. --- +## Converter Type Mismatch (Known Upstream Issue) + +**This section applies when your code passes `fromClaudeMessages(...)` or `fromChatMessages(...)` directly to `callModel({ input: ... })`.** + +In `@openrouter/agent` v0.7.x, `fromClaudeMessages` and `fromChatMessages` are declared as returning `models.InputsUnion` (the SDK's broad response-input union) while `callModel`'s `input` parameter is typed as `FieldOrAsyncFunction | string`. These types are structurally compatible at runtime — the converters produce arrays that satisfy the shape — but TypeScript cannot prove the assignment without a cast. + +This is a type-definition gap in the upstream package, not an error in your migration. + +### Prohibited workarounds + +**Do not** use `as any`, `as unknown as Item[]`, `as unknown as string`, `@ts-ignore`, `@ts-expect-error`, or any lint/type suppression comment to silence this error. These erase type safety in the surrounding code and are harder to remove later. A migration that introduces type suppressions is worse than keeping the old import path. + +### What to do instead + +**Option A — Preserve the old converter path.** + +Keep the converter-dependent path on `@openrouter/sdk`, including the SDK client used by that call site, and migrate independent agent-only files separately. This preserves the previously typechecked behavior instead of turning the migration into a type-suppression exercise. Keep both packages installed and report that this path is blocked on the upstream type mismatch. + +**Option B — Implement an explicit validated conversion.** + +If the project already has runtime schemas or exhaustive type guards for response-input items, add a small named adapter that: + +1. Rejects the string branch when the converter is expected to return an array. +2. Validates every array element against the exact `Item` variants accepted by the installed agent version. +3. Returns `Item[]` only after that validation succeeds. +4. Has unit tests for accepted variants and rejection paths. + +Do not call an `Array.isArray` check followed by `as Item[]` a validated conversion: it verifies the container but not its elements. Do not invent a validator API that the installed package does not export. If the project lacks a trustworthy validator, use Option A or C. + +**Option C — Report the gap and stop.** + +If neither option fits, report the installed versions and the `InputsUnion` versus `Item[]` diagnostic, and do not claim the converter migration succeeded. A partial migration that retains this path on `@openrouter/sdk` is better than a complete migration with hidden type suppressions. + +--- + +## Step 4: Updating Tests + +When `mock.module` calls reference old `@openrouter/sdk` paths, update them to match the new import paths used in the source files. The mock path must be identical to the import path in the source, not the old path. + +```diff +- mock.module('@openrouter/sdk', () => ({ ... })) ++ mock.module('@openrouter/agent', () => ({ ... })) + +- mock.module('@openrouter/sdk/lib/stop-conditions', () => ({ ... })) ++ mock.module('@openrouter/agent/stop-conditions', () => ({ ... })) + +- mock.module('@openrouter/sdk/lib/tool', () => ({ ... })) ++ mock.module('@openrouter/agent/tool', () => ({ ... })) + +- mock.module('@openrouter/sdk/lib/tool-types', () => ({ ... })) ++ mock.module('@openrouter/agent/tool-types', () => ({ ... })) + +- mock.module('@openrouter/sdk/lib/async-params', () => ({ ... })) ++ mock.module('@openrouter/agent/async-params', () => ({ ... })) + +- mock.module('@openrouter/sdk/lib/model-result', () => ({ ... })) ++ mock.module('@openrouter/agent/model-result', () => ({ ... })) + +- mock.module('@openrouter/sdk/lib/anthropic-compat', () => ({ ... })) ++ mock.module('@openrouter/agent', () => ({ ... })) // converters are now in the barrel + +- mock.module('@openrouter/sdk/lib/chat-compat', () => ({ ... })) ++ mock.module('@openrouter/agent', () => ({ ... })) // merge into barrel mock +``` + +When merging mocks, combine all named exports into one `mock.module('@openrouter/agent', ...)` factory rather than registering duplicate mocks for the same path. + +In **mixed-project** tests (platform + agent), keep the `@openrouter/sdk` mock for platform paths (`models`, `credits`, `chat`) and add a separate `@openrouter/agent` mock for agent paths. Do not merge them. + +--- + +## Step 5: Stale Import Check and Verification + +After all edits, verify no stale references remain and the project still type-checks and passes tests: + +```bash +# 1. Confirm no @openrouter/sdk imports remain in migrated files +grep -rn "@openrouter/sdk" src/ --include="*.ts" +# For mixed projects, verify only platform files retain @openrouter/sdk imports + +# 2. Confirm no type suppressions were introduced +grep -rn "as any\|as unknown as\|@ts-ignore\|@ts-expect-error\|biome-ignore" src/ --include="*.ts" +# Any hits on lines not present in the original file are a regression — remove them + +# 3. Run typecheck +tsc --noEmit + +# 4. Run tests +bun test # or npm test / yarn test + +# 5. Review the diff +git diff src/ test/ package.json +# Confirm: only import paths changed. No logic, no new casts, no new comments +# that suppress errors. Zod upgrade only if §Zod decision procedure required it. +``` + +If `tsc` reports an error on a `fromClaudeMessages`/`fromChatMessages` call site after migration, do not suppress it — apply §Converter Type Mismatch Option A or B instead. + +--- + ## Before & After Example ### Before (using @openrouter/sdk) @@ -127,32 +305,19 @@ For most use cases, prefer the `client.callModel()` method shown above. import OpenRouter, { tool, stepCountIs, hasToolCall } from '@openrouter/sdk'; import { z } from 'zod'; -const client = new OpenRouter({ - apiKey: process.env.OPENROUTER_API_KEY, -}); +const client = new OpenRouter({ apiKey: process.env.OPENROUTER_API_KEY }); const searchTool = tool({ name: 'web_search', description: 'Search the web', inputSchema: z.object({ query: z.string() }), - outputSchema: z.object({ results: z.array(z.string()) }), - execute: async ({ query }) => { - return { results: ['Result 1', 'Result 2'] }; - }, -}); - -const finishTool = tool({ - name: 'finish', - description: 'Complete the task', - inputSchema: z.object({ answer: z.string() }), - execute: async ({ answer }) => ({ answer }), + execute: async ({ query }) => ({ results: ['Result 1', 'Result 2'] }), }); const result = client.callModel({ model: 'openai/gpt-5-nano', - instructions: 'You are a research assistant.', input: 'What are the latest AI developments?', - tools: [searchTool, finishTool], + tools: [searchTool], stopWhen: [stepCountIs(10), hasToolCall('finish')], }); @@ -167,32 +332,19 @@ import { tool } from '@openrouter/agent/tool'; import { stepCountIs, hasToolCall } from '@openrouter/agent/stop-conditions'; import { z } from 'zod'; -const client = new OpenRouter({ - apiKey: process.env.OPENROUTER_API_KEY, -}); +const client = new OpenRouter({ apiKey: process.env.OPENROUTER_API_KEY }); const searchTool = tool({ name: 'web_search', description: 'Search the web', inputSchema: z.object({ query: z.string() }), - outputSchema: z.object({ results: z.array(z.string()) }), - execute: async ({ query }) => { - return { results: ['Result 1', 'Result 2'] }; - }, -}); - -const finishTool = tool({ - name: 'finish', - description: 'Complete the task', - inputSchema: z.object({ answer: z.string() }), - execute: async ({ answer }) => ({ answer }), + execute: async ({ query }) => ({ results: ['Result 1', 'Result 2'] }), }); const result = client.callModel({ model: 'openai/gpt-5-nano', - instructions: 'You are a research assistant.', input: 'What are the latest AI developments?', - tools: [searchTool, finishTool], + tools: [searchTool], stopWhen: [stepCountIs(10), hasToolCall('finish')], }); @@ -211,7 +363,6 @@ Keep `@openrouter/sdk` installed if you use any of these non-agent features: |---------|--------| | Model listing | `client.models.list()` | | Chat completions | `client.chat.send()` | -| Legacy completions | `client.completions.generate()` | | Usage analytics | `client.analytics.getUserActivity()` | | Credit balance | `client.credits.getCredits()` | | API key management | `client.apiKeys.list()`, `.create()`, etc. | @@ -225,12 +376,10 @@ import { OpenRouter as Agent } from '@openrouter/agent'; // Agent client for cal import { tool } from '@openrouter/agent/tool'; import { stepCountIs } from '@openrouter/agent/stop-conditions'; -// Use SDK client for non-agent features const sdkClient = new OpenRouter({ apiKey: process.env.OPENROUTER_API_KEY }); const models = await sdkClient.models.list(); const credits = await sdkClient.credits.getCredits(); -// Use Agent client for callModel const agent = new Agent({ apiKey: process.env.OPENROUTER_API_KEY }); const result = agent.callModel({ model: 'openai/gpt-5-nano', @@ -242,97 +391,13 @@ const result = agent.callModel({ --- -## New Features in @openrouter/agent - -These features are only available in `@openrouter/agent`, not in `@openrouter/sdk`: - -### Shared Context Schema - -Type-safe shared state across all tools in a conversation: - -```typescript -import { OpenRouter } from '@openrouter/agent'; -import { z } from 'zod'; - -const client = new OpenRouter({ apiKey: '...' }); - -const result = client.callModel({ - model: 'openai/gpt-5-nano', - input: 'Process this data', - sharedContextSchema: z.object({ - userId: z.string(), - sessionData: z.record(z.unknown()), - }), - context: { - shared: { userId: '123', sessionData: {} }, - }, - tools: [myTool], -}); -``` - -### Tool Context - -Tools can declare their own typed context and access shared context: - -```typescript -import { tool } from '@openrouter/agent/tool'; -import { z } from 'zod'; - -const myTool = tool({ - name: 'my_tool', - description: 'A tool with context', - inputSchema: z.object({ query: z.string() }), - contextSchema: z.object({ apiKey: z.string() }), - execute: async (params, context) => { - // context.local — this tool's own context - // context.shared — shared context across all tools - // context.setContext({ ... }) — update this tool's context - // context.setSharedContext({ ... }) — update shared context - return { result: 'done' }; - }, -}); -``` - -### Tool Approval Flow - -Require user approval before tool execution: - -```typescript -const dangerousTool = tool({ - name: 'delete_file', - description: 'Delete a file', - inputSchema: z.object({ path: z.string() }), - requireApproval: true, // or a function: (toolCall, context) => boolean - execute: async ({ path }) => { /* ... */ }, -}); -``` - -### Turn Lifecycle Callbacks - -```typescript -const result = client.callModel({ - model: 'openai/gpt-5-nano', - input: 'Complex task', - tools: [myTool], - onTurnStart: async (context) => { - console.log(`Starting turn ${context.numberOfTurns}`); - }, - onTurnEnd: async (context, response) => { - console.log(`Turn ${context.numberOfTurns} complete`); - }, -}); -``` - ---- - ## All Subpath Exports -`@openrouter/agent` provides granular subpath imports: +`@openrouter/agent` provides granular subpath imports (verified against v0.7.2): | Subpath | Exports | |---------|---------| -| `@openrouter/agent` | Barrel: all exports below | -| `@openrouter/agent/client` | `OpenRouter` class | +| `@openrouter/agent` | Barrel: all exports below including `OpenRouter` class | | `@openrouter/agent/call-model` | `callModel` standalone function | | `@openrouter/agent/tool` | `tool()` factory function | | `@openrouter/agent/tool-types` | `Tool`, `ToolWithExecute`, `ToolWithGenerator`, `ManualTool`, type guards | @@ -342,9 +407,6 @@ const result = client.callModel({ | `@openrouter/agent/anthropic-compat` | `fromClaudeMessages`, `toClaudeMessage` | | `@openrouter/agent/chat-compat` | `fromChatMessages`, `toChatMessage` | | `@openrouter/agent/conversation-state` | `createInitialState`, `updateState`, `appendToMessages` | -| `@openrouter/agent/next-turn-params` | `nextTurnParams` utilities | | `@openrouter/agent/stream-transformers` | `extractUnsupportedContent`, `getUnsupportedContentSummary` | -| `@openrouter/agent/tool-context` | `buildToolExecuteContext`, `ToolContextStore` | -| `@openrouter/agent/tool-event-broadcaster` | `ToolEventBroadcaster` | -| `@openrouter/agent/turn-context` | `buildTurnContext` | +The barrel export (`@openrouter/agent`) re-exports everything in the table above, including converters — use it when you need multiple imports from the same barrel. diff --git a/skills/openrouter-agent-migration/evals/evals.json b/skills/openrouter-agent-migration/evals/evals.json new file mode 100644 index 0000000..524dac8 --- /dev/null +++ b/skills/openrouter-agent-migration/evals/evals.json @@ -0,0 +1,63 @@ +{ + "skill_name": "openrouter-agent-migration", + "evals": [ + { + "id": 1, + "prompt": "Migrate the TypeScript project at evals/fixtures/agent-only from @openrouter/sdk to @openrouter/agent. The project imports OpenRouter from '@openrouter/sdk', tool from '@openrouter/sdk/lib/tool', stop conditions (stepCountIs, hasToolCall, maxCost) from '@openrouter/sdk/lib/stop-conditions', and types (CallModelInput, ModelResult, ToolWithGenerator, Tool, ToolWithExecute, ManualTool) from '@openrouter/sdk/lib/*' subpaths. No platform API features are used. Remove @openrouter/sdk entirely and update all imports to use @openrouter/agent and its subpaths.", + "expected_output": "The project's package.json replaces @openrouter/sdk with @openrouter/agent. All imports in src/ are updated: OpenRouter from '@openrouter/agent', tool from '@openrouter/agent/tool', stop conditions from '@openrouter/agent/stop-conditions', type imports (Tool, ToolWithExecute, ManualTool, ToolWithGenerator) from '@openrouter/agent/tool-types', CallModelInput from '@openrouter/agent/async-params', ModelResult from '@openrouter/agent/model-result'. No @openrouter/sdk imports remain anywhere in src/. The bun test and tsc --noEmit commands pass.", + "expectations": [ + "@openrouter/sdk is removed from package.json dependencies", + "@openrouter/agent is added to package.json dependencies", + "src/agent.ts imports OpenRouter from '@openrouter/agent' as a named import", + "src/agent.ts imports stepCountIs, hasToolCall, and maxCost from '@openrouter/agent/stop-conditions'", + "src/tools.ts imports tool from '@openrouter/agent/tool'", + "src/tools.ts imports Tool, ToolWithExecute, and ManualTool from '@openrouter/agent/tool-types'", + "src/types.ts imports CallModelInput from '@openrouter/agent/async-params'", + "src/types.ts imports ModelResult from '@openrouter/agent/model-result'", + "src/types.ts imports ToolWithGenerator from '@openrouter/agent/tool-types'", + "No file in src/ contains any import from '@openrouter/sdk'", + "The project's test suite passes (bun test exits 0)", + "TypeScript type check passes (tsc --noEmit exits 0)" + ] + }, + { + "id": 2, + "prompt": "Migrate the TypeScript project at evals/fixtures/mixed-platform-agent. The project uses @openrouter/sdk for platform features in src/platform.ts (OpenRouter client for models.list, credits.getCredits, chat.send). The src/agent.ts file uses @openrouter/sdk for agent features (OpenRouter client, tool from '@openrouter/sdk/lib/tool', stop conditions from '@openrouter/sdk/lib/stop-conditions'). Keep @openrouter/sdk for platform.ts but migrate agent.ts to use @openrouter/agent and its subpaths for all agent functionality.", + "expected_output": "package.json retains @openrouter/sdk (for platform features) and adds @openrouter/agent (for agent features). src/platform.ts is unchanged and still imports from '@openrouter/sdk'. src/agent.ts is updated: OpenRouter (or aliased as Agent) from '@openrouter/agent', tool from '@openrouter/agent/tool', stepCountIs/maxTokensUsed/finishReasonIs from '@openrouter/agent/stop-conditions'. src/agent.ts no longer imports from '@openrouter/sdk'. The bun test and tsc --noEmit commands pass.", + "expectations": [ + "@openrouter/sdk is retained in package.json dependencies", + "@openrouter/agent is added to package.json dependencies", + "src/platform.ts still imports from '@openrouter/sdk' (file unchanged)", + "src/agent.ts no longer imports from '@openrouter/sdk'", + "src/agent.ts imports OpenRouter (or an alias like Agent) from '@openrouter/agent'", + "src/agent.ts imports tool from '@openrouter/agent/tool'", + "src/agent.ts imports stepCountIs from '@openrouter/agent/stop-conditions'", + "src/agent.ts imports maxTokensUsed from '@openrouter/agent/stop-conditions'", + "src/agent.ts imports finishReasonIs from '@openrouter/agent/stop-conditions'", + "The project's test suite passes (bun test exits 0)", + "TypeScript type check passes (tsc --noEmit exits 0)" + ] + }, + { + "id": 3, + "prompt": "Migrate the TypeScript project at evals/fixtures/streaming-converters from @openrouter/sdk to @openrouter/agent. The project uses format converters in src/converters.ts (fromClaudeMessages, toClaudeMessage from '@openrouter/sdk/lib/anthropic-compat'; fromChatMessages, toChatMessage from '@openrouter/sdk/lib/chat-compat'), type guards and type imports in src/type-guards.ts (hasExecuteFunction, isGeneratorTool, isRegularExecuteTool, Tool, ToolWithExecute, ToolWithGenerator from '@openrouter/sdk/lib/tool-types'; tool from '@openrouter/sdk/lib/tool'), and streaming in src/stream.ts (OpenRouter from '@openrouter/sdk'; CallModelInput from '@openrouter/sdk/lib/async-params'). Update all imports and remove @openrouter/sdk entirely.", + "expected_output": "package.json replaces @openrouter/sdk with @openrouter/agent. src/converters.ts imports fromClaudeMessages and toClaudeMessage from '@openrouter/agent' (barrel) and fromChatMessages and toChatMessage from '@openrouter/agent' (barrel). src/type-guards.ts imports hasExecuteFunction, isGeneratorTool, isRegularExecuteTool, Tool, ToolWithExecute, ToolWithGenerator from '@openrouter/agent/tool-types' and tool from '@openrouter/agent/tool'. src/stream.ts imports OpenRouter from '@openrouter/agent' and CallModelInput from '@openrouter/agent/async-params'. No @openrouter/sdk imports remain. The bun test and tsc --noEmit commands pass.", + "expectations": [ + "@openrouter/sdk is removed from package.json dependencies", + "@openrouter/agent is added to package.json dependencies", + "src/converters.ts imports fromClaudeMessages from '@openrouter/agent'", + "src/converters.ts imports toClaudeMessage from '@openrouter/agent'", + "src/converters.ts imports fromChatMessages from '@openrouter/agent'", + "src/converters.ts imports toChatMessage from '@openrouter/agent'", + "src/type-guards.ts imports hasExecuteFunction from '@openrouter/agent/tool-types'", + "src/type-guards.ts imports isGeneratorTool from '@openrouter/agent/tool-types'", + "src/type-guards.ts imports isRegularExecuteTool from '@openrouter/agent/tool-types'", + "src/type-guards.ts imports tool from '@openrouter/agent/tool'", + "src/stream.ts imports CallModelInput from '@openrouter/agent/async-params'", + "No file in src/ contains any import from '@openrouter/sdk'", + "The project's test suite passes (bun test exits 0)", + "TypeScript type check passes (tsc --noEmit exits 0)" + ] + } + ] +} diff --git a/skills/openrouter-agent-migration/evals/fixtures/agent-only/bun.lock b/skills/openrouter-agent-migration/evals/fixtures/agent-only/bun.lock new file mode 100644 index 0000000..68aeecf --- /dev/null +++ b/skills/openrouter-agent-migration/evals/fixtures/agent-only/bun.lock @@ -0,0 +1,35 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "agent-only-fixture", + "dependencies": { + "@openrouter/sdk": "0.13.43", + "zod": "3.25.76", + }, + "devDependencies": { + "@types/bun": "latest", + "typescript": "^5.4.0", + }, + }, + }, + "overrides": { + "zod": "3.25.76", + }, + "packages": { + "@openrouter/sdk": ["@openrouter/sdk@0.13.43", "", { "dependencies": { "zod": "^3.25.0 || ^4.0.0" } }, "sha512-wD4SMxNfx/MYI4X7whWgft1IFr8mZJeB+MSod+oX8lNAk4MaJ6VoPQXVhaVU7ALttFpyvbLrYcIjsCXGnnkdlA=="], + + "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], + + "@types/node": ["@types/node@26.1.1", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw=="], + + "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], + + "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], + } +} diff --git a/skills/openrouter-agent-migration/evals/fixtures/agent-only/package.json b/skills/openrouter-agent-migration/evals/fixtures/agent-only/package.json new file mode 100644 index 0000000..2bd5d9a --- /dev/null +++ b/skills/openrouter-agent-migration/evals/fixtures/agent-only/package.json @@ -0,0 +1,19 @@ +{ + "name": "agent-only-fixture", + "type": "module", + "scripts": { + "test": "bun test", + "build": "tsc --noEmit" + }, + "dependencies": { + "@openrouter/sdk": "0.13.43", + "zod": "3.25.76" + }, + "devDependencies": { + "@types/bun": "latest", + "typescript": "^5.4.0" + }, + "resolutions": { + "zod": "3.25.76" + } +} diff --git a/skills/openrouter-agent-migration/evals/fixtures/agent-only/src/agent.ts b/skills/openrouter-agent-migration/evals/fixtures/agent-only/src/agent.ts new file mode 100644 index 0000000..8e2f358 --- /dev/null +++ b/skills/openrouter-agent-migration/evals/fixtures/agent-only/src/agent.ts @@ -0,0 +1,27 @@ +import { OpenRouter } from '@openrouter/sdk'; +import { hasToolCall, maxCost, stepCountIs } from '@openrouter/sdk/lib/stop-conditions'; +import { finishTool, searchTool } from './tools.js'; + +const client = new OpenRouter({ + apiKey: process.env.OPENROUTER_API_KEY ?? 'sk-or-test', +}); + +export async function runResearchAgent(task: string): Promise { + const result = client.callModel({ + model: 'openai/gpt-5-nano', + instructions: 'You are a research assistant. Search for information and finish when done.', + input: task, + tools: [searchTool, finishTool], + stopWhen: [stepCountIs(10), hasToolCall('finish'), maxCost(0.5)], + }); + + return await result.getText(); +} + +export async function runSimpleQuery(prompt: string): Promise { + const result = client.callModel({ + model: 'openai/gpt-5-nano', + input: prompt, + }); + return await result.getText(); +} diff --git a/skills/openrouter-agent-migration/evals/fixtures/agent-only/src/tools.ts b/skills/openrouter-agent-migration/evals/fixtures/agent-only/src/tools.ts new file mode 100644 index 0000000..3940ae3 --- /dev/null +++ b/skills/openrouter-agent-migration/evals/fixtures/agent-only/src/tools.ts @@ -0,0 +1,34 @@ +import { tool } from '@openrouter/sdk/lib/tool'; +import type { ManualTool, Tool, ToolWithExecute } from '@openrouter/sdk/lib/tool-types'; +import { z } from 'zod/v4'; + +export const searchTool = tool({ + name: 'web_search', + description: 'Search the web for information', + inputSchema: z.object({ + query: z.string().describe('Search query'), + }), + outputSchema: z.object({ + results: z.array(z.string()), + }), + execute: async ({ query }) => { + return { results: [`Result for: ${query}`] }; + }, +}); + +export const finishTool = tool({ + name: 'finish', + description: 'Signal task completion with final answer', + inputSchema: z.object({ answer: z.string() }), + execute: async ({ answer }) => ({ answer }), +}); + +export const confirmTool: ManualTool = tool({ + name: 'confirm', + description: 'Request manual confirmation', + inputSchema: z.object({ message: z.string() }), + execute: false, +}); + +export type SearchToolFn = ToolWithExecute; +export type MyTools = Tool[]; diff --git a/skills/openrouter-agent-migration/evals/fixtures/agent-only/src/types.ts b/skills/openrouter-agent-migration/evals/fixtures/agent-only/src/types.ts new file mode 100644 index 0000000..a205236 --- /dev/null +++ b/skills/openrouter-agent-migration/evals/fixtures/agent-only/src/types.ts @@ -0,0 +1,12 @@ +import type { CallModelInput } from '@openrouter/sdk/lib/async-params'; +import { ModelResult } from '@openrouter/sdk/lib/model-result'; +import type { Tool, ToolWithGenerator } from '@openrouter/sdk/lib/tool-types'; + +export type AgentInput = CallModelInput; + +export function isModelResult(value: unknown): value is ModelResult { + return value instanceof ModelResult; +} + +// NOTE: ToolWithGenerator uses the SDK's internal zod v4 types via @openrouter/sdk/lib/tool-types +export type StreamingTool = ToolWithGenerator; diff --git a/skills/openrouter-agent-migration/evals/fixtures/agent-only/test/agent.test.ts b/skills/openrouter-agent-migration/evals/fixtures/agent-only/test/agent.test.ts new file mode 100644 index 0000000..fe751d4 --- /dev/null +++ b/skills/openrouter-agent-migration/evals/fixtures/agent-only/test/agent.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, mock, test } from 'bun:test'; + +const mockGetText = mock(async () => 'mocked response'); +const mockCallModel = mock(() => ({ + getText: mockGetText, + getResponse: mock(async () => ({ text: 'mocked', usage: null })), +})); + +mock.module('@openrouter/sdk', () => ({ + OpenRouter: class MockOpenRouter { + callModel = mockCallModel; + }, +})); + +mock.module('@openrouter/sdk/lib/stop-conditions', () => ({ + stepCountIs: (n: number) => ({ type: 'stepCount', n }), + hasToolCall: (name: string) => ({ type: 'hasToolCall', name }), + maxCost: (amount: number) => ({ type: 'maxCost', amount }), +})); + +mock.module('@openrouter/sdk/lib/tool', () => ({ + tool: (config: unknown) => ({ type: 'function', function: config }), +})); + +mock.module('@openrouter/sdk/lib/tool-types', () => ({})); +mock.module('@openrouter/sdk/lib/async-params', () => ({})); +mock.module('@openrouter/sdk/lib/model-result', () => ({ + ModelResult: class MockModelResult {}, +})); + +describe('agent-only fixture', () => { + test('searchTool has expected name and execute function', async () => { + const { searchTool } = await import('../src/tools.js'); + const fn = (searchTool as unknown as { function: { name: string; execute: unknown } }).function; + expect(fn.name).toBe('web_search'); + expect(typeof fn.execute).toBe('function'); + }); + + test('finishTool execute returns answer', async () => { + const { finishTool } = await import('../src/tools.js'); + const fn = (finishTool as unknown as { + function: { execute: (a: { answer: string }) => Promise<{ answer: string }> }; + }).function; + const result = await fn.execute({ answer: 'test answer' }); + expect(result).toEqual({ answer: 'test answer' }); + }); + + test('confirmTool has execute: false (manual tool)', async () => { + const { confirmTool } = await import('../src/tools.js'); + const fn = (confirmTool as unknown as { function: { execute: unknown } }).function; + expect(fn.execute).toBe(false); + }); + + test('searchTool execute returns results array for query', async () => { + const { searchTool } = await import('../src/tools.js'); + const fn = (searchTool as unknown as { + function: { execute: (a: { query: string }) => Promise<{ results: string[] }> }; + }).function; + const result = await fn.execute({ query: 'quantum computing' }); + expect(Array.isArray(result.results)).toBe(true); + expect(result.results[0]).toContain('quantum computing'); + }); + + test('runResearchAgent calls callModel with three stop conditions', async () => { + mockCallModel.mockClear(); + mockGetText.mockClear(); + const { runResearchAgent } = await import('../src/agent.js'); + const text = await runResearchAgent('test task'); + expect(mockCallModel).toHaveBeenCalled(); + const callArgs = (mockCallModel.mock.calls as unknown as Array<[{ + stopWhen: unknown[]; + input: string; + }]>)[0][0]; + expect(callArgs.stopWhen).toHaveLength(3); + expect(callArgs.input).toBe('test task'); + expect(text).toBe('mocked response'); + }); + + test('runSimpleQuery calls callModel without stop conditions', async () => { + mockCallModel.mockClear(); + const { runSimpleQuery } = await import('../src/agent.js'); + await runSimpleQuery('hello'); + expect(mockCallModel).toHaveBeenCalled(); + const callArgs = (mockCallModel.mock.calls as unknown as Array<[{ + input: string; + stopWhen?: unknown[]; + }]>)[0][0]; + expect(callArgs.input).toBe('hello'); + expect(callArgs.stopWhen).toBeUndefined(); + }); +}); diff --git a/skills/openrouter-agent-migration/evals/fixtures/agent-only/tsconfig.json b/skills/openrouter-agent-migration/evals/fixtures/agent-only/tsconfig.json new file mode 100644 index 0000000..bc75a64 --- /dev/null +++ b/skills/openrouter-agent-migration/evals/fixtures/agent-only/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "types": ["bun-types"] + }, + "include": ["src", "test"] +} diff --git a/skills/openrouter-agent-migration/evals/fixtures/mixed-platform-agent/bun.lock b/skills/openrouter-agent-migration/evals/fixtures/mixed-platform-agent/bun.lock new file mode 100644 index 0000000..4d093f4 --- /dev/null +++ b/skills/openrouter-agent-migration/evals/fixtures/mixed-platform-agent/bun.lock @@ -0,0 +1,32 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "mixed-platform-agent-fixture", + "dependencies": { + "@openrouter/sdk": "0.13.43", + "zod": "3.25.76", + }, + "devDependencies": { + "@types/bun": "latest", + "typescript": "^5.4.0", + }, + }, + }, + "packages": { + "@openrouter/sdk": ["@openrouter/sdk@0.13.43", "", { "dependencies": { "zod": "^3.25.0 || ^4.0.0" } }, "sha512-wD4SMxNfx/MYI4X7whWgft1IFr8mZJeB+MSod+oX8lNAk4MaJ6VoPQXVhaVU7ALttFpyvbLrYcIjsCXGnnkdlA=="], + + "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], + + "@types/node": ["@types/node@26.1.1", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw=="], + + "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], + + "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], + } +} diff --git a/skills/openrouter-agent-migration/evals/fixtures/mixed-platform-agent/package.json b/skills/openrouter-agent-migration/evals/fixtures/mixed-platform-agent/package.json new file mode 100644 index 0000000..5e52e68 --- /dev/null +++ b/skills/openrouter-agent-migration/evals/fixtures/mixed-platform-agent/package.json @@ -0,0 +1,16 @@ +{ + "name": "mixed-platform-agent-fixture", + "type": "module", + "scripts": { + "test": "bun test", + "build": "tsc --noEmit" + }, + "dependencies": { + "@openrouter/sdk": "0.13.43", + "zod": "3.25.76" + }, + "devDependencies": { + "@types/bun": "latest", + "typescript": "^5.4.0" + } +} diff --git a/skills/openrouter-agent-migration/evals/fixtures/mixed-platform-agent/src/agent.ts b/skills/openrouter-agent-migration/evals/fixtures/mixed-platform-agent/src/agent.ts new file mode 100644 index 0000000..67517a3 --- /dev/null +++ b/skills/openrouter-agent-migration/evals/fixtures/mixed-platform-agent/src/agent.ts @@ -0,0 +1,41 @@ +import { OpenRouter } from '@openrouter/sdk'; +import { finishReasonIs, maxTokensUsed, stepCountIs } from '@openrouter/sdk/lib/stop-conditions'; +import { tool } from '@openrouter/sdk/lib/tool'; +import { z } from 'zod/v4'; + +const client = new OpenRouter({ apiKey: process.env.OPENROUTER_API_KEY ?? 'sk-or-test' }); + +const summarizeTool = tool({ + name: 'summarize', + description: 'Summarize content into bullet points', + inputSchema: z.object({ + content: z.string(), + maxBullets: z.number().optional().default(5), + }), + outputSchema: z.object({ + bullets: z.array(z.string()), + }), + execute: async ({ content, maxBullets }) => { + const bullets = content.split('. ').slice(0, maxBullets).map((s) => `• ${s.trim()}`); + return { bullets }; + }, +}); + +const doneTool = tool({ + name: 'done', + description: 'Mark the task as complete', + inputSchema: z.object({ summary: z.string() }), + execute: async ({ summary }) => ({ summary, completed: true }), +}); + +export async function runSummaryAgent(content: string): Promise { + const result = client.callModel({ + model: 'openai/gpt-5-nano', + instructions: 'Summarize the provided content using the summarize tool, then call done.', + input: content, + tools: [summarizeTool, doneTool], + stopWhen: [stepCountIs(5), maxTokensUsed(2000), finishReasonIs('stop')], + }); + + return await result.getText(); +} diff --git a/skills/openrouter-agent-migration/evals/fixtures/mixed-platform-agent/src/platform.ts b/skills/openrouter-agent-migration/evals/fixtures/mixed-platform-agent/src/platform.ts new file mode 100644 index 0000000..ba4d1a5 --- /dev/null +++ b/skills/openrouter-agent-migration/evals/fixtures/mixed-platform-agent/src/platform.ts @@ -0,0 +1,24 @@ +import { OpenRouter } from '@openrouter/sdk'; + +export async function listAvailableModels(apiKey: string): Promise { + const client = new OpenRouter({ apiKey }); + const models = await client.models.list(); + return models.data.map((m: { id: string }) => m.id); +} + +export async function checkCredits(apiKey: string): Promise { + const client = new OpenRouter({ apiKey }); + const credits = await client.credits.getCredits(); + return (credits.data as { total_credits?: number } | undefined)?.total_credits ?? 0; +} + +export async function chatCompletion(apiKey: string, message: string): Promise { + const client = new OpenRouter({ apiKey }); + const response = await client.chat.send({ + chatRequest: { + model: 'openai/gpt-5-nano', + messages: [{ role: 'user', content: message }], + }, + }); + return (response as { choices?: Array<{ message?: { content?: string } }> }).choices?.[0]?.message?.content ?? ''; +} diff --git a/skills/openrouter-agent-migration/evals/fixtures/mixed-platform-agent/test/mixed.test.ts b/skills/openrouter-agent-migration/evals/fixtures/mixed-platform-agent/test/mixed.test.ts new file mode 100644 index 0000000..bfe7149 --- /dev/null +++ b/skills/openrouter-agent-migration/evals/fixtures/mixed-platform-agent/test/mixed.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, mock, test } from 'bun:test'; + +const mockCallModel = mock(() => ({ + getText: mock(async () => 'agent response'), + getResponse: mock(async () => ({ text: 'agent response', usage: null })), +})); + +const mockModelsList = mock(async () => ({ + data: [{ id: 'openai/gpt-5-nano' }, { id: 'anthropic/claude-3-opus' }], +})); + +const mockGetCredits = mock(async () => ({ data: { total_credits: 10.5 } })); + +const mockChatCreate = mock(async () => ({ + choices: [{ message: { content: 'chat response' } }], +})); + +mock.module('@openrouter/sdk', () => ({ + OpenRouter: class MockOpenRouter { + callModel = mockCallModel; + models = { list: mockModelsList }; + credits = { getCredits: mockGetCredits }; + chat = { send: mockChatCreate }; + }, +})); + +mock.module('@openrouter/sdk/lib/stop-conditions', () => ({ + stepCountIs: (n: number) => ({ type: 'stepCount', n }), + maxTokensUsed: (n: number) => ({ type: 'maxTokens', n }), + finishReasonIs: (r: string) => ({ type: 'finishReason', r }), +})); + +mock.module('@openrouter/sdk/lib/tool', () => ({ + tool: (config: unknown) => ({ type: 'function', function: config }), +})); + +describe('mixed-platform-agent fixture', () => { + test('listAvailableModels returns model id array', async () => { + const { listAvailableModels } = await import('../src/platform.js'); + const models = await listAvailableModels('sk-or-test'); + expect(Array.isArray(models)).toBe(true); + expect(models[0]).toBe('openai/gpt-5-nano'); + }); + + test('checkCredits returns numeric total', async () => { + const { checkCredits } = await import('../src/platform.js'); + const credits = await checkCredits('sk-or-test'); + expect(typeof credits).toBe('number'); + expect(credits).toBe(10.5); + }); + + test('chatCompletion returns content string', async () => { + const { chatCompletion } = await import('../src/platform.js'); + const result = await chatCompletion('sk-or-test', 'hello'); + expect(result).toBe('chat response'); + }); + + test('summarizeTool execute produces bullet list from sentences', async () => { + const content = 'First sentence. Second sentence. Third sentence'; + const bullets = content.split('. ').slice(0, 5).map((s) => `• ${s.trim()}`); + expect(bullets).toHaveLength(3); + expect(bullets[0]).toBe('• First sentence'); + }); + + test('runSummaryAgent calls callModel with three stop conditions', async () => { + mockCallModel.mockClear(); + const { runSummaryAgent } = await import('../src/agent.js'); + const text = await runSummaryAgent('Content to summarize'); + expect(mockCallModel).toHaveBeenCalled(); + const callArgs = (mockCallModel.mock.calls as unknown as Array<[{ + stopWhen: unknown[]; + }]>)[0][0]; + expect(callArgs.stopWhen).toHaveLength(3); + expect(text).toBe('agent response'); + }); +}); diff --git a/skills/openrouter-agent-migration/evals/fixtures/mixed-platform-agent/tsconfig.json b/skills/openrouter-agent-migration/evals/fixtures/mixed-platform-agent/tsconfig.json new file mode 100644 index 0000000..bc75a64 --- /dev/null +++ b/skills/openrouter-agent-migration/evals/fixtures/mixed-platform-agent/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "types": ["bun-types"] + }, + "include": ["src", "test"] +} diff --git a/skills/openrouter-agent-migration/evals/fixtures/streaming-converters/bun.lock b/skills/openrouter-agent-migration/evals/fixtures/streaming-converters/bun.lock new file mode 100644 index 0000000..604c769 --- /dev/null +++ b/skills/openrouter-agent-migration/evals/fixtures/streaming-converters/bun.lock @@ -0,0 +1,32 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "streaming-converters-fixture", + "dependencies": { + "@openrouter/sdk": "0.13.43", + "zod": "3.25.76", + }, + "devDependencies": { + "@types/bun": "latest", + "typescript": "^5.4.0", + }, + }, + }, + "packages": { + "@openrouter/sdk": ["@openrouter/sdk@0.13.43", "", { "dependencies": { "zod": "^3.25.0 || ^4.0.0" } }, "sha512-wD4SMxNfx/MYI4X7whWgft1IFr8mZJeB+MSod+oX8lNAk4MaJ6VoPQXVhaVU7ALttFpyvbLrYcIjsCXGnnkdlA=="], + + "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], + + "@types/node": ["@types/node@26.1.1", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw=="], + + "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], + + "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], + } +} diff --git a/skills/openrouter-agent-migration/evals/fixtures/streaming-converters/package.json b/skills/openrouter-agent-migration/evals/fixtures/streaming-converters/package.json new file mode 100644 index 0000000..bb4ce87 --- /dev/null +++ b/skills/openrouter-agent-migration/evals/fixtures/streaming-converters/package.json @@ -0,0 +1,16 @@ +{ + "name": "streaming-converters-fixture", + "type": "module", + "scripts": { + "test": "bun test", + "build": "tsc --noEmit" + }, + "dependencies": { + "@openrouter/sdk": "0.13.43", + "zod": "3.25.76" + }, + "devDependencies": { + "@types/bun": "latest", + "typescript": "^5.4.0" + } +} diff --git a/skills/openrouter-agent-migration/evals/fixtures/streaming-converters/src/converters.ts b/skills/openrouter-agent-migration/evals/fixtures/streaming-converters/src/converters.ts new file mode 100644 index 0000000..84d3853 --- /dev/null +++ b/skills/openrouter-agent-migration/evals/fixtures/streaming-converters/src/converters.ts @@ -0,0 +1,39 @@ +import { OpenRouter } from '@openrouter/sdk'; +import { fromClaudeMessages, toClaudeMessage } from '@openrouter/sdk/lib/anthropic-compat'; +import { fromChatMessages, toChatMessage } from '@openrouter/sdk/lib/chat-compat'; + +export type ClaudeMessage = { + role: 'user' | 'assistant'; + content: string | Array<{ type: string; text?: string }>; +}; + +export type ChatMessage = { + role: 'user' | 'assistant' | 'system'; + content: string; +}; + +const client = new OpenRouter({ + apiKey: process.env.OPENROUTER_API_KEY ?? 'sk-or-test', +}); + +export async function callWithClaudeMessages(messages: ClaudeMessage[]): Promise { + const result = client.callModel({ + model: 'anthropic/claude-3-opus', + input: fromClaudeMessages(messages as Parameters[0]), + }); + const response = await result.getResponse(); + const claudeMsg = toClaudeMessage(response); + return typeof claudeMsg.content === 'string' + ? claudeMsg.content + : JSON.stringify(claudeMsg.content); +} + +export async function callWithChatMessages(messages: ChatMessage[]): Promise { + const result = client.callModel({ + model: 'openai/gpt-5-nano', + input: fromChatMessages(messages as Parameters[0]), + }); + const response = await result.getResponse(); + const chatMsg = toChatMessage(response); + return (chatMsg as { content?: string }).content ?? ''; +} diff --git a/skills/openrouter-agent-migration/evals/fixtures/streaming-converters/src/stream.ts b/skills/openrouter-agent-migration/evals/fixtures/streaming-converters/src/stream.ts new file mode 100644 index 0000000..0f4d819 --- /dev/null +++ b/skills/openrouter-agent-migration/evals/fixtures/streaming-converters/src/stream.ts @@ -0,0 +1,34 @@ +import { OpenRouter } from '@openrouter/sdk'; +import type { CallModelInput } from '@openrouter/sdk/lib/async-params'; + +const client = new OpenRouter({ + apiKey: process.env.OPENROUTER_API_KEY ?? 'sk-or-test', +}); + +export async function streamText(input: string): Promise { + const result = client.callModel({ + model: 'openai/gpt-5-nano', + input, + }); + + const chunks: string[] = []; + for await (const delta of result.getTextStream()) { + chunks.push(delta); + process.stdout.write(delta); + } + process.stdout.write('\n'); + return chunks.join(''); +} + +export async function streamWithCallback( + params: CallModelInput, + onDelta: (delta: string) => void, +): Promise { + const result = client.callModel(params); + const chunks: string[] = []; + for await (const delta of result.getTextStream()) { + chunks.push(delta); + onDelta(delta); + } + return chunks.join(''); +} diff --git a/skills/openrouter-agent-migration/evals/fixtures/streaming-converters/src/type-guards.ts b/skills/openrouter-agent-migration/evals/fixtures/streaming-converters/src/type-guards.ts new file mode 100644 index 0000000..d4f3023 --- /dev/null +++ b/skills/openrouter-agent-migration/evals/fixtures/streaming-converters/src/type-guards.ts @@ -0,0 +1,48 @@ +import { + hasExecuteFunction, + isGeneratorTool, + isRegularExecuteTool, + type Tool, + type ToolWithExecute, + type ToolWithGenerator, +} from '@openrouter/sdk/lib/tool-types'; +import { tool } from '@openrouter/sdk/lib/tool'; +import { z } from 'zod/v4'; + +const regularTool = tool({ + name: 'regular', + description: 'A regular tool', + inputSchema: z.object({ input: z.string() }), + execute: async ({ input }) => ({ output: input }), +}); + +const generatorTool = tool({ + name: 'generator', + description: 'A generator tool', + inputSchema: z.object({ query: z.string() }), + eventSchema: z.object({ progress: z.string() }), + outputSchema: z.object({ result: z.string() }), + execute: async function* ({ query }: { query: string }) { + yield { progress: 'working...' }; + return { result: query.toUpperCase() }; + }, +}); + +const manualTool = tool({ + name: 'manual', + description: 'A manual tool', + inputSchema: z.object({ data: z.string() }), + execute: false, +}); + +export function classifyTool(t: Tool): 'regular' | 'generator' | 'manual' { + if (isRegularExecuteTool(t)) return 'regular'; + if (isGeneratorTool(t)) return 'generator'; + return 'manual'; +} + +export function hasExecute(t: Tool): t is ToolWithExecute | ToolWithGenerator { + return hasExecuteFunction(t); +} + +export const tools = { regularTool, generatorTool, manualTool }; diff --git a/skills/openrouter-agent-migration/evals/fixtures/streaming-converters/test/converters.test.ts b/skills/openrouter-agent-migration/evals/fixtures/streaming-converters/test/converters.test.ts new file mode 100644 index 0000000..0292faf --- /dev/null +++ b/skills/openrouter-agent-migration/evals/fixtures/streaming-converters/test/converters.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, mock, test } from 'bun:test'; + +const mockGetText = mock(async () => 'mocked text'); +const mockGetResponse = mock(async () => ({ text: 'mocked response', usage: null })); +const mockGetTextStream = mock(async function* () { + yield 'chunk1'; + yield 'chunk2'; +}); +const mockCallModel = mock(() => ({ + getText: mockGetText, + getResponse: mockGetResponse, + getTextStream: mockGetTextStream, +})); + +const mockFromClaudeMessages = mock((msgs: Array<{ role: string; content: unknown }>) => + msgs.map((m) => ({ + role: m.role, + content: typeof m.content === 'string' ? m.content : JSON.stringify(m.content), + })), +); +const mockToClaudeMessage = mock((_response: unknown) => ({ + role: 'assistant' as const, + content: 'claude response', +})); +const mockFromChatMessages = mock((msgs: Array<{ role: string; content: string }>) => + msgs.map((m) => ({ role: m.role, content: m.content })), +); +const mockToChatMessage = mock((_response: unknown) => ({ + role: 'assistant' as const, + content: 'chat response', +})); + +mock.module('@openrouter/sdk', () => ({ + OpenRouter: class MockOpenRouter { + callModel = mockCallModel; + }, +})); + +mock.module('@openrouter/sdk/lib/anthropic-compat', () => ({ + fromClaudeMessages: mockFromClaudeMessages, + toClaudeMessage: mockToClaudeMessage, +})); + +mock.module('@openrouter/sdk/lib/chat-compat', () => ({ + fromChatMessages: mockFromChatMessages, + toChatMessage: mockToChatMessage, +})); + +const mockHasExecuteFunction = mock((t: { function?: { execute?: unknown } }) => + !!(t.function?.execute && t.function.execute !== false), +); +const mockIsGeneratorTool = mock((_t: unknown) => false); +const mockIsRegularExecuteTool = mock( + (t: { function?: { execute?: unknown } }) => + !!(t.function?.execute && typeof t.function.execute === 'function'), +); + +mock.module('@openrouter/sdk/lib/tool-types', () => ({ + hasExecuteFunction: mockHasExecuteFunction, + isGeneratorTool: mockIsGeneratorTool, + isRegularExecuteTool: mockIsRegularExecuteTool, +})); + +mock.module('@openrouter/sdk/lib/tool', () => ({ + tool: (config: unknown) => ({ type: 'function', function: config }), +})); + +mock.module('@openrouter/sdk/lib/async-params', () => ({})); + +describe('streaming-converters fixture', () => { + test('callWithClaudeMessages converts and returns string', async () => { + const { callWithClaudeMessages } = await import('../src/converters.js'); + const messages = [{ role: 'user' as const, content: 'Hello from Claude format' }]; + const result = await callWithClaudeMessages(messages); + expect(typeof result).toBe('string'); + expect(mockFromClaudeMessages).toHaveBeenCalled(); + expect(mockToClaudeMessage).toHaveBeenCalled(); + }); + + test('callWithChatMessages converts and returns string', async () => { + const { callWithChatMessages } = await import('../src/converters.js'); + const messages = [ + { role: 'system' as const, content: 'You are helpful' }, + { role: 'user' as const, content: 'Hello' }, + ]; + const result = await callWithChatMessages(messages); + expect(typeof result).toBe('string'); + expect(mockFromChatMessages).toHaveBeenCalled(); + expect(mockToChatMessage).toHaveBeenCalled(); + }); + + test('classifyTool returns regular for tool with sync execute', async () => { + const { classifyTool, tools } = await import('../src/type-guards.js'); + const kind = classifyTool(tools.regularTool); + expect(kind).toBe('regular'); + }); + + test('classifyTool returns manual for tool with execute: false', async () => { + const { classifyTool, tools } = await import('../src/type-guards.js'); + const kind = classifyTool(tools.manualTool); + expect(kind).toBe('manual'); + }); + + test('hasExecute returns true for regular tool, false for manual', async () => { + const { hasExecute, tools } = await import('../src/type-guards.js'); + expect(hasExecute(tools.regularTool)).toBe(true); + expect(hasExecute(tools.manualTool)).toBe(false); + }); + + test('streamText accumulates and returns joined chunks', async () => { + const { streamText } = await import('../src/stream.js'); + const result = await streamText('test prompt'); + expect(typeof result).toBe('string'); + expect(result).toBe('chunk1chunk2'); + }); + + test('streamWithCallback invokes callback for each delta', async () => { + const { streamWithCallback } = await import('../src/stream.js'); + const deltas: string[] = []; + const result = await streamWithCallback( + { model: 'openai/gpt-5-nano', input: 'test' }, + (delta) => deltas.push(delta), + ); + expect(deltas).toEqual(['chunk1', 'chunk2']); + expect(result).toBe('chunk1chunk2'); + }); +}); diff --git a/skills/openrouter-agent-migration/evals/fixtures/streaming-converters/tsconfig.json b/skills/openrouter-agent-migration/evals/fixtures/streaming-converters/tsconfig.json new file mode 100644 index 0000000..bc75a64 --- /dev/null +++ b/skills/openrouter-agent-migration/evals/fixtures/streaming-converters/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "types": ["bun-types"] + }, + "include": ["src", "test"] +} diff --git a/skills/openrouter-agent-migration/evals/validate.ts b/skills/openrouter-agent-migration/evals/validate.ts new file mode 100644 index 0000000..1ab6c60 --- /dev/null +++ b/skills/openrouter-agent-migration/evals/validate.ts @@ -0,0 +1,397 @@ +#!/usr/bin/env bun +import { existsSync, readFileSync, readdirSync } from 'node:fs'; +import { join, resolve } from 'node:path'; + +interface Check { + name: string; + passed: boolean; + detail: string; +} + +interface RunResult { + exit_code: number; + stdout: string; + stderr: string; +} + +interface ValidateOutput { + fixture: string; + project: string; + passed: boolean; + checks: Check[]; + run_results?: { + typecheck: RunResult; + tests: RunResult; + }; + summary: { total: number; passed: number; failed: number }; +} + +const args = process.argv.slice(2); +const flags: Record = {}; +for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg === '--skip-run') { + flags['skip-run'] = true; + continue; + } + if (arg.startsWith('--')) { + const key = arg.slice(2); + const next = args[i + 1]; + if (next !== undefined && !next.startsWith('--')) { + flags[key] = next; + i++; + } else { + flags[key] = true; + } + } +} + +const fixture = flags['fixture'] as string | undefined; +const rawProject = flags['project'] as string | undefined; +const skipRun = !!flags['skip-run']; + +if (!fixture || !rawProject) { + process.stderr.write( + 'Usage: bun evals/validate.ts --fixture --project [--skip-run]\n', + ); + process.exit(1); +} + +const projectPath = resolve(rawProject); + +function check(name: string, condition: boolean, detail: string): Check { + return { name, passed: condition, detail }; +} + +function readPkg(dir: string): Record> { + const pkgPath = join(dir, 'package.json'); + if (!existsSync(pkgPath)) return {}; + return JSON.parse(readFileSync(pkgPath, 'utf-8')) as Record< + string, + Record + >; +} + +function collectSrcFiles(dir: string): Map { + const srcDir = join(dir, 'src'); + const files = new Map(); + if (!existsSync(srcDir)) return files; + for (const entry of readdirSync(srcDir)) { + if (!entry.endsWith('.ts')) continue; + files.set(entry, readFileSync(join(srcDir, entry), 'utf-8')); + } + return files; +} + +function hasImportFrom(content: string, from: string): boolean { + const escaped = from.replace(/[/\\@]/g, (c) => `\\${c}`); + return new RegExp(`from\\s+['"]${escaped}['"]`).test(content); +} + +function hasNamedImport(content: string, name: string, from: string): boolean { + const escapedFrom = from.replace(/[/\\@]/g, (c) => `\\${c}`); + return new RegExp( + `import[^;'"]*\\b${name}\\b[^;'"]*from\\s+['"]${escapedFrom}['"]`, + ).test(content); +} + +function anyFileHasImportFrom(files: Map, from: string): boolean { + for (const content of files.values()) { + if (hasImportFrom(content, from)) return true; + } + return false; +} + +async function runCmd(cmd: string[], cwd: string): Promise { + const proc = Bun.spawn(cmd, { cwd, stdout: 'pipe', stderr: 'pipe' }); + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + return { exit_code: exitCode, stdout, stderr }; +} + +function checksForAgentOnly( + pkg: Record>, + files: Map, +): Check[] { + const deps = { ...pkg.dependencies, ...pkg.devDependencies }; + const agentTs = files.get('agent.ts') ?? ''; + const toolsTs = files.get('tools.ts') ?? ''; + const typesTs = files.get('types.ts') ?? ''; + + return [ + check( + '@openrouter/sdk removed from dependencies', + !deps['@openrouter/sdk'], + deps['@openrouter/sdk'] + ? `Found @openrouter/sdk@${deps['@openrouter/sdk']}` + : 'Not present', + ), + check( + '@openrouter/agent added to dependencies', + !!deps['@openrouter/agent'], + deps['@openrouter/agent'] + ? `Found @openrouter/agent@${deps['@openrouter/agent']}` + : 'Not found in dependencies', + ), + check( + 'No stale @openrouter/sdk imports in any src file', + !anyFileHasImportFrom(files, '@openrouter/sdk'), + anyFileHasImportFrom(files, '@openrouter/sdk') + ? 'Found @openrouter/sdk import in src/' + : 'Clean — no @openrouter/sdk imports', + ), + check( + "src/agent.ts imports OpenRouter from '@openrouter/agent' (named)", + hasNamedImport(agentTs, 'OpenRouter', '@openrouter/agent'), + agentTs ? 'Check for named import { OpenRouter }' : 'agent.ts not found', + ), + check( + "src/agent.ts imports stepCountIs from '@openrouter/agent/stop-conditions'", + hasNamedImport(agentTs, 'stepCountIs', '@openrouter/agent/stop-conditions'), + 'Expects stop condition from @openrouter/agent/stop-conditions subpath', + ), + check( + "src/agent.ts imports hasToolCall from '@openrouter/agent/stop-conditions'", + hasNamedImport(agentTs, 'hasToolCall', '@openrouter/agent/stop-conditions'), + 'Expects hasToolCall from @openrouter/agent/stop-conditions subpath', + ), + check( + "src/agent.ts imports maxCost from '@openrouter/agent/stop-conditions'", + hasNamedImport(agentTs, 'maxCost', '@openrouter/agent/stop-conditions'), + 'Expects maxCost from @openrouter/agent/stop-conditions subpath', + ), + check( + "src/tools.ts imports tool from '@openrouter/agent/tool'", + hasNamedImport(toolsTs, 'tool', '@openrouter/agent/tool'), + 'Expects tool factory from @openrouter/agent/tool', + ), + check( + "src/tools.ts imports Tool type from '@openrouter/agent/tool-types'", + hasNamedImport(toolsTs, 'Tool', '@openrouter/agent/tool-types'), + 'Expects Tool type from @openrouter/agent/tool-types', + ), + check( + "src/types.ts imports CallModelInput from '@openrouter/agent/async-params'", + hasNamedImport(typesTs, 'CallModelInput', '@openrouter/agent/async-params'), + 'Expects CallModelInput from @openrouter/agent/async-params', + ), + check( + "src/types.ts imports ModelResult from '@openrouter/agent/model-result'", + hasImportFrom(typesTs, '@openrouter/agent/model-result'), + 'Expects ModelResult from @openrouter/agent/model-result', + ), + check( + "src/types.ts imports ToolWithGenerator from '@openrouter/agent/tool-types'", + hasNamedImport(typesTs, 'ToolWithGenerator', '@openrouter/agent/tool-types'), + 'Expects ToolWithGenerator from @openrouter/agent/tool-types', + ), + ]; +} + +function checksForMixedPlatformAgent( + pkg: Record>, + files: Map, +): Check[] { + const deps = { ...pkg.dependencies, ...pkg.devDependencies }; + const platformTs = files.get('platform.ts') ?? ''; + const agentTs = files.get('agent.ts') ?? ''; + + return [ + check( + '@openrouter/sdk retained in dependencies', + !!deps['@openrouter/sdk'], + deps['@openrouter/sdk'] + ? `Found @openrouter/sdk@${deps['@openrouter/sdk']}` + : 'Missing — platform features require @openrouter/sdk', + ), + check( + '@openrouter/agent added to dependencies', + !!deps['@openrouter/agent'], + deps['@openrouter/agent'] + ? `Found @openrouter/agent@${deps['@openrouter/agent']}` + : 'Not found in dependencies', + ), + check( + "src/platform.ts retains import from '@openrouter/sdk'", + hasImportFrom(platformTs, '@openrouter/sdk'), + platformTs + ? 'platform.ts must keep @openrouter/sdk for models/credits/chat' + : 'platform.ts not found', + ), + check( + "src/agent.ts no longer imports from '@openrouter/sdk'", + !hasImportFrom(agentTs, '@openrouter/sdk'), + hasImportFrom(agentTs, '@openrouter/sdk') + ? 'Still contains @openrouter/sdk import' + : 'Clean', + ), + check( + "src/agent.ts imports OpenRouter from '@openrouter/agent'", + hasNamedImport(agentTs, 'OpenRouter', '@openrouter/agent'), + 'Expects OpenRouter (possibly aliased as Agent) from @openrouter/agent', + ), + check( + "src/agent.ts imports tool from '@openrouter/agent/tool'", + hasNamedImport(agentTs, 'tool', '@openrouter/agent/tool'), + 'Expects tool factory from @openrouter/agent/tool', + ), + check( + "src/agent.ts imports stepCountIs from '@openrouter/agent/stop-conditions'", + hasNamedImport(agentTs, 'stepCountIs', '@openrouter/agent/stop-conditions'), + 'Expects stepCountIs from @openrouter/agent/stop-conditions', + ), + check( + "src/agent.ts imports maxTokensUsed from '@openrouter/agent/stop-conditions'", + hasNamedImport(agentTs, 'maxTokensUsed', '@openrouter/agent/stop-conditions'), + 'Expects maxTokensUsed from @openrouter/agent/stop-conditions', + ), + check( + "src/agent.ts imports finishReasonIs from '@openrouter/agent/stop-conditions'", + hasNamedImport(agentTs, 'finishReasonIs', '@openrouter/agent/stop-conditions'), + 'Expects finishReasonIs from @openrouter/agent/stop-conditions', + ), + ]; +} + +function checksForStreamingConverters( + pkg: Record>, + files: Map, +): Check[] { + const deps = { ...pkg.dependencies, ...pkg.devDependencies }; + const convertersTs = files.get('converters.ts') ?? ''; + const streamTs = files.get('stream.ts') ?? ''; + const typeGuardsTs = files.get('type-guards.ts') ?? ''; + + return [ + check( + '@openrouter/sdk removed from dependencies', + !deps['@openrouter/sdk'], + deps['@openrouter/sdk'] + ? `Found @openrouter/sdk@${deps['@openrouter/sdk']}` + : 'Not present', + ), + check( + '@openrouter/agent added to dependencies', + !!deps['@openrouter/agent'], + deps['@openrouter/agent'] + ? `Found @openrouter/agent@${deps['@openrouter/agent']}` + : 'Not found in dependencies', + ), + check( + 'No stale @openrouter/sdk imports in any src file', + !anyFileHasImportFrom(files, '@openrouter/sdk'), + anyFileHasImportFrom(files, '@openrouter/sdk') + ? 'Found @openrouter/sdk import in src/' + : 'Clean — no @openrouter/sdk imports', + ), + check( + "src/converters.ts imports fromClaudeMessages from '@openrouter/agent'", + hasNamedImport(convertersTs, 'fromClaudeMessages', '@openrouter/agent'), + 'Expects fromClaudeMessages from @openrouter/agent barrel', + ), + check( + "src/converters.ts imports toClaudeMessage from '@openrouter/agent'", + hasNamedImport(convertersTs, 'toClaudeMessage', '@openrouter/agent'), + 'Expects toClaudeMessage from @openrouter/agent barrel', + ), + check( + "src/converters.ts imports fromChatMessages from '@openrouter/agent'", + hasNamedImport(convertersTs, 'fromChatMessages', '@openrouter/agent'), + 'Expects fromChatMessages from @openrouter/agent barrel', + ), + check( + "src/converters.ts imports toChatMessage from '@openrouter/agent'", + hasNamedImport(convertersTs, 'toChatMessage', '@openrouter/agent'), + 'Expects toChatMessage from @openrouter/agent barrel', + ), + check( + "src/type-guards.ts imports hasExecuteFunction from '@openrouter/agent/tool-types'", + hasNamedImport(typeGuardsTs, 'hasExecuteFunction', '@openrouter/agent/tool-types'), + 'Expects hasExecuteFunction from @openrouter/agent/tool-types', + ), + check( + "src/type-guards.ts imports isGeneratorTool from '@openrouter/agent/tool-types'", + hasNamedImport(typeGuardsTs, 'isGeneratorTool', '@openrouter/agent/tool-types'), + 'Expects isGeneratorTool from @openrouter/agent/tool-types', + ), + check( + "src/type-guards.ts imports isRegularExecuteTool from '@openrouter/agent/tool-types'", + hasNamedImport(typeGuardsTs, 'isRegularExecuteTool', '@openrouter/agent/tool-types'), + 'Expects isRegularExecuteTool from @openrouter/agent/tool-types', + ), + check( + "src/type-guards.ts imports tool from '@openrouter/agent/tool'", + hasNamedImport(typeGuardsTs, 'tool', '@openrouter/agent/tool'), + 'Expects tool factory from @openrouter/agent/tool', + ), + check( + "src/stream.ts imports CallModelInput from '@openrouter/agent/async-params'", + hasNamedImport(streamTs, 'CallModelInput', '@openrouter/agent/async-params'), + 'Expects CallModelInput from @openrouter/agent/async-params', + ), + ]; +} + +const pkg = readPkg(projectPath); +const files = collectSrcFiles(projectPath); + +let fixtureChecks: Check[]; +if (fixture === 'agent-only') { + fixtureChecks = checksForAgentOnly(pkg, files); +} else if (fixture === 'mixed-platform-agent') { + fixtureChecks = checksForMixedPlatformAgent(pkg, files); +} else if (fixture === 'streaming-converters') { + fixtureChecks = checksForStreamingConverters(pkg, files); +} else { + process.stderr.write( + `Unknown fixture: ${fixture}. Valid values: agent-only, mixed-platform-agent, streaming-converters\n`, + ); + process.exit(1); +} + +let runResults: ValidateOutput['run_results'] | undefined; +if (!skipRun) { + const [typecheck, tests] = await Promise.all([ + runCmd(['bun', 'run', 'build'], projectPath), + runCmd(['bun', 'test'], projectPath), + ]); + + fixtureChecks.push( + check( + 'TypeScript type check passes (tsc --noEmit)', + typecheck.exit_code === 0, + typecheck.exit_code === 0 + ? 'Exited 0' + : `Exit ${typecheck.exit_code}: ${typecheck.stderr.slice(0, 400)}`, + ), + check( + 'Test suite passes (bun test)', + tests.exit_code === 0, + tests.exit_code === 0 + ? 'Exited 0' + : `Exit ${tests.exit_code}: ${tests.stderr.slice(0, 400)}`, + ), + ); + + runResults = { typecheck, tests }; +} + +const passedCount = fixtureChecks.filter((c) => c.passed).length; +const output: ValidateOutput = { + fixture, + project: projectPath, + passed: fixtureChecks.every((c) => c.passed), + checks: fixtureChecks, + run_results: runResults, + summary: { + total: fixtureChecks.length, + passed: passedCount, + failed: fixtureChecks.length - passedCount, + }, +}; + +process.stdout.write(JSON.stringify(output, null, 2) + '\n'); +process.exit(output.passed ? 0 : 1);