From bdddb8200c689b355302e2955962c6e165e5e256 Mon Sep 17 00:00:00 2001 From: arc <224894192+arc0btc@users.noreply.github.com> Date: Fri, 3 Jul 2026 10:47:13 -0600 Subject: [PATCH 1/4] feat: add openrouter-integration-doctor skill Adds a cross-cutting integration doctor skill: symptom -> diagnosis -> OpenRouter-native fix for a whole integration (key/credit health, routing & fallback config, live end-to-end smoke test), plus four runnable probes. Complements openrouter-generations (single-request debugging) rather than duplicating it. --- README.md | 1 + .../openrouter-integration-doctor/README.md | 85 +++ skills/openrouter-integration-doctor/SKILL.md | 164 ++++++ .../references/error-taxonomy.md | 95 ++++ .../references/fallback-and-zdr.md | 79 +++ .../references/onboarding-checklist.md | 57 ++ .../references/provider-preferences.md | 60 ++ .../scripts/check-key-credits.ts | 159 ++++++ .../scripts/inspect-generation.ts | 192 +++++++ .../scripts/lib.ts | 304 ++++++++++ .../scripts/package-lock.json | 531 ++++++++++++++++++ .../scripts/package.json | 8 + .../scripts/smoke-test.ts | 85 +++ .../scripts/validate-fallback-config.ts | 209 +++++++ 14 files changed, 2029 insertions(+) create mode 100644 skills/openrouter-integration-doctor/README.md create mode 100644 skills/openrouter-integration-doctor/SKILL.md create mode 100644 skills/openrouter-integration-doctor/references/error-taxonomy.md create mode 100644 skills/openrouter-integration-doctor/references/fallback-and-zdr.md create mode 100644 skills/openrouter-integration-doctor/references/onboarding-checklist.md create mode 100644 skills/openrouter-integration-doctor/references/provider-preferences.md create mode 100644 skills/openrouter-integration-doctor/scripts/check-key-credits.ts create mode 100644 skills/openrouter-integration-doctor/scripts/inspect-generation.ts create mode 100644 skills/openrouter-integration-doctor/scripts/lib.ts create mode 100644 skills/openrouter-integration-doctor/scripts/package-lock.json create mode 100644 skills/openrouter-integration-doctor/scripts/package.json create mode 100644 skills/openrouter-integration-doctor/scripts/smoke-test.ts create mode 100644 skills/openrouter-integration-doctor/scripts/validate-fallback-config.ts diff --git a/README.md b/README.md index 25da4f0..a2bbfc4 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ Skills are contextual and auto-loaded based on your conversation. When a request | [openrouter-analytics-schema](skills/openrouter-analytics-schema/README.md) | Discovering the OpenRouter analytics schema — available metrics, dimensions, filter operators, and granularities, and mapping natural-language questions to query parameters | | [openrouter-analytics-query](skills/openrouter-analytics-query/README.md) | Constructing and executing analytics queries against the OpenRouter API — full parameter reference for metrics, dimensions, filters, time ranges, ordering, and pagination | | [openrouter-generations](skills/openrouter-generations/README.md) | Inspecting individual OpenRouter generations — request metadata (cost, latency, tokens, model, provider routing) and stored prompt/completion content | +| [openrouter-integration-doctor](skills/openrouter-integration-doctor/README.md) | Diagnosing a whole OpenRouter integration — key/credit health, routing & fallback config, and a live end-to-end smoke test — with a symptom→diagnosis→native-fix playbook and runnable probes (complements openrouter-generations, which debugs a single request) | ## Environment diff --git a/skills/openrouter-integration-doctor/README.md b/skills/openrouter-integration-doctor/README.md new file mode 100644 index 0000000..3042e3c --- /dev/null +++ b/skills/openrouter-integration-doctor/README.md @@ -0,0 +1,85 @@ +# openrouter-integration-doctor + +Diagnose and fix a whole OpenRouter **integration** — key/credit health, routing and +fallback config, and a live end-to-end smoke test — with a symptom→diagnosis→native-fix +playbook and runnable probes. The cross-cutting complement to `openrouter-generations`, +which debugs a single request. + +## Install + +With the [GitHub CLI](https://cli.github.com/) (v2.92.0+): + +```bash +gh skill install OpenRouterTeam/skills openrouter-integration-doctor +``` + +Works with Claude Code, Cursor, Codex, OpenCode, Gemini CLI, Windsurf, and [many more agents](https://cli.github.com/manual/gh_skill_install). Add `--scope user` to install across every project for your current agent, or `--agent claude-code` to target a specific agent. + +For other install methods (Claude Code plugin marketplace, Cursor Rules, etc.) see the [root README](../../README.md#installing). + +### Manual install + +If `gh skill install` fails (e.g. the `gh` CLI is older than v2.92.0, or you hit an auth/network issue), copy the skill directory in manually: + +```bash +git clone https://github.com/OpenRouterTeam/skills.git /tmp/or-skills +mkdir -p .github/skills +cp -r /tmp/or-skills/skills/openrouter-integration-doctor .github/skills/ +rm -rf /tmp/or-skills +``` + +## Prerequisites + +`OPENROUTER_API_KEY` must be set to a valid OpenRouter API key for the live probes (a +management/provisioning key surfaces the richest `/key` data). Get one at +[openrouter.ai/settings/keys](https://openrouter.ai/settings/keys). The config validator +needs no key and no network. + +## What it covers + +See [SKILL.md](SKILL.md) for the full reference, including: + +- The symptom → diagnosis → OpenRouter-native-fix playbook (402/403/404/429/503/529, + "No endpoints found", streaming cut-offs, structured-output degrade, ZDR conflicts). +- The canonical 27-value `ApiErrorType` enum and the typed HTTP ladder. +- Splitting a 402 into account-depletion vs key-spend-cap. +- Diagnosing the "overloaded-404" from over-constrained provider preferences. +- Configuring model fallbacks and understanding ZDR union semantics. +- A first-two-days integration-health onboarding checklist. + +## Scripts + +| Script | Network? | Purpose | +|--------|----------|---------| +| `check-key-credits.ts` | yes | `GET /key` (+ `GET /credits` with a management key) — balance, key spend cap; flags 402 risk. `/credits` is management-key-only; the probe auto-detects key type | +| `validate-fallback-config.ts` | **no** | Offline lint of a request body for routing/fallback/ZDR misconfigurations | +| `smoke-test.ts` | yes | Minimal live `POST /chat/completions` — proves the pipe is open end-to-end | +| `inspect-generation.ts` | yes | `GET /generation` (+`/content`) — cost/routing/streaming diagnosis; bridge to `openrouter-generations`. Retries the brief post-request 404 while the generation indexes | + +## Quick start + +```bash +# First-time setup (once per installation): +# cd /scripts && npm install + +# 1. Is the key valid and funded? (first move on any 402) +npx tsx check-key-credits.ts + +# 2. Is the routing/fallback config sound? (offline — no key needed) +npx tsx validate-fallback-config.ts ./request-body.json + +# 3. Does a real call complete end-to-end? +npx tsx smoke-test.ts + +# 4. Localize a specific failing call, then hand the id to openrouter-generations +npx tsx inspect-generation.ts gen-1234567890 --content +``` + +## Related Skills + +- `openrouter-generations` — inspect a **single** generation ID in exhaustive detail + (this skill's `inspect-generation.ts` is a thin bridge; use `openrouter-generations` for + the full per-field dump). +- `openrouter-models` — resolve model slugs and compare context lengths / pricing when a + 404 turns out to be a slug typo. +- `openrouter-analytics` — aggregate spend/volume/latency trends across all generations. diff --git a/skills/openrouter-integration-doctor/SKILL.md b/skills/openrouter-integration-doctor/SKILL.md new file mode 100644 index 0000000..18866f3 --- /dev/null +++ b/skills/openrouter-integration-doctor/SKILL.md @@ -0,0 +1,164 @@ +--- +name: openrouter-integration-doctor +description: Diagnose and fix a whole OpenRouter integration — not one request. Use when a customer says requests "suddenly started failing", the bill jumped, they see 402/403/404/429/503/529, "No endpoints found", streaming truncates, structured outputs degrade, or a ZDR/compliance setup broke routing — and you need a symptom→diagnosis→native-fix playbook plus runnable probes (key/credit health, /generation introspection, fallback-config linting, live smoke test) to prove where the integration is broken. +version: 0.1.0 +--- + +# openrouter-integration-doctor + +The cross-cutting integration doctor for OpenRouter. Where the `openrouter-generations` +skill inspects **one** generation ID in depth, this skill diagnoses the **whole +integration**: is the key valid, does the account have credit, is the routing/fallback +config sane, and does a live call actually complete? It turns a vague "it's broken" +report into a specific, OpenRouter-native fix. + +Every OpenRouter error normalizes to a canonical `ApiErrorType` (27 values) under +`error.metadata.error_type`, stable across `/chat/completions`, `/messages`, and +`/responses`. The whole skill is organized around switching on that enum. + +## How this differs from `openrouter-generations` + +| | `openrouter-generations` | **`openrouter-integration-doctor`** | +|---|---|---| +| Scope | One generation ID | The whole integration | +| Question | "What happened in *this* request?" | "Why is *the integration* misbehaving?" | +| Inputs | A `gen-...` ID | A key, a request-body config, or just a symptom | +| Probes | `get-generation`, `get-generation-content` | key/credit health, config lint, live smoke test, + a `/generation` bridge | + +They compose: use this skill to localize the fault, then hand a specific `gen-...` ID to +`openrouter-generations` for the exhaustive per-field dump. `inspect-generation.ts` here +is a deliberate thin bridge, not a duplicate. + +## Prerequisites + +- An OpenRouter API key for the live probes (a management/provisioning key surfaces the + richest `/key` data). Get one at [openrouter.ai/settings/keys](https://openrouter.ai/settings/keys). +- Pass it via `--api-key ` or set `OPENROUTER_API_KEY`. +- `validate-fallback-config.ts` needs **no key and no network** — it lints a request body + offline, so it is safe to run on a customer's config in a screen-share. + +## First-Time Setup + +```bash +cd /scripts && npm install +``` + +## The four probes + +| Script | Network? | Purpose | +|--------|----------|---------| +| `check-key-credits.ts` | yes | `GET /key` (+ `GET /credits` when a management key is available) — balance, key spend cap; flags 402 risk. Auto-detects key type: `/credits` is management-key-only (runtime keys get 403) | +| `inspect-generation.ts` | yes | `GET /generation` (+`/content`) — cost/routing/streaming diagnosis for one call (bridge to `openrouter-generations`). Retries the brief post-request 404 while the generation is still indexing | +| `validate-fallback-config.ts` | **no** | Offline lint of a request body for the overloaded-404, silent-degrade, ZDR-narrowing, no-fallback pitfalls | +| `smoke-test.ts` | yes | Minimal live `POST /chat/completions` — proves the pipe is open end-to-end, maps any error via the taxonomy | + +```bash +cd /scripts + +# 1. Is the key valid and funded? (first move on any 402 / "insufficient credits") +npx tsx check-key-credits.ts + +# 2. Is the routing/fallback config sound? (offline — no key needed) +npx tsx validate-fallback-config.ts ./their-request-body.json + +# 3. Does a real call complete end-to-end? +npx tsx smoke-test.ts + +# 4. Localize a specific failing call, then hand the id to openrouter-generations +npx tsx inspect-generation.ts gen-1234567890 --content +``` + +## Diagnostic workflow + +Work outside-in — cheapest/offline checks first, live calls last: + +1. **Config first (offline).** Run `validate-fallback-config.ts` on the exact body the + customer sends. Most "No endpoints found" 404s and silent structured-output failures + are visible here without touching the network. See + [references/provider-preferences.md](references/provider-preferences.md). +2. **Account/key health.** Run `check-key-credits.ts`. A 402 is either a depleted account + *or* a key that hit its own spend cap even though the account has balance — this probe + distinguishes them. Note the key-type split (live-verified): `GET /credits` only + answers to a **management key** — a runtime key gets 403 `permission_denied` + ("Only management keys can fetch credits for an account"). With a runtime key the + probe reports the key-level view and tells you to pass `--management-key` (or set + `OPENROUTER_MANAGEMENT_KEY`) for the account view. +3. **Prove the pipe.** Run `smoke-test.ts`. If it passes, the customer's own request + shape or model choice is the fault, not the integration wiring. +4. **Localize.** For a specific failing call, `inspect-generation.ts ` surfaces the + provider-fallback chain, the real vs upstream cost, and the finish state — the usual + explanations for "the bill 10x'd" and "streaming cut off". + +## Symptom → diagnosis → native fix + +The playbook. Each row maps a customer complaint to the canonical error type and the +OpenRouter-native move. This mirrors the error-taxonomy field guide 1:1 so the skill and +the guide never drift. + +| Symptom (what the customer says) | Root cause | ApiErrorType / HTTP | Native fix | +|---|---|---|---| +| "Requests suddenly 402 / 'insufficient credits'" | Out of credits, or key spend-cap hit; agentic burn | `payment_required` / **402** | `check-key-credits.ts` to split account-vs-key; top up or raise the per-key limit via the Management API; bound agent loops with `stop_server_tools_when: {max_cost}` | +| "Bill 10x'd overnight, nothing changed" | Silent BYOK/paid fallback (+5%), reasoning-token blowup, or uncontrolled agent loop | (billing, not an error type) | `inspect-generation.ts ` for authoritative cost + provider chain; group by `session_id`; reconcile `usage.cost` vs `/generation` | +| "429s / rate limited" | **Two distinct 429s:** account daily cap vs upstream provider overload | `rate_limit_exceeded` / **429** | Honor `Retry-After` + jitter. Cap → raise limit / spread keys; upstream → add a `models[]` fallback array so it reroutes | +| "Model 'overloaded' / 503 / 529" | Provider brownout inside the 30s health window | `provider_overloaded` / **529**; `provider_unavailable` / **503** | Reliability is architectural: `models[]` fallback, `allow_fallbacks:true`, `sort:throughput`/`:nitro`. Provider-layer failover is default-on | +| **"No endpoints found" / cryptic 404** | Four causes, one symptom: (1) model-slug typo, (2) provider prefs too tight, (3) data-policy narrowing, (4) unsupported param | `not_found` / **404** | `validate-fallback-config.ts` catches (2)-(4) offline; `debug:{echo_upstream_body}` to see exactly what was sent; loosen `order`/`max_price`; check `data_collection`/`zdr` | +| "Streaming works then randomly cuts off" | Mid-stream failure delivered as an SSE chunk with `finish_reason:"error"` under HTTP 200 | (in-band, not an HTTP code) | Parse the final SSE chunk + `usage`; treat `finish_reason:"error"` as failure and retry/fallback | +| "Structured outputs sometimes return garbage" | Fell back to a provider that doesn't support strict `json_schema` | `unprocessable` / **422** or silent degrade | `provider:{require_parameters:true}` + `response_format` `json_schema` `strict:true` so only capable providers route | +| "My compliant/ZDR setup started 404-ing" | ZDR / no-training is the **union** of OpenRouter + downstream retention → narrows providers to zero | `not_found` / **404** (data-policy) | `GET /endpoints/zdr` to preview impact; relax `data_collection`/`zdr` or widen the model set; explain retention is a union, not a switch | +| "Content blocked / model refused" | Moderation, guardrail stage, or model self-refusal | `content_policy_violation`, `refusal` / **403** | With `X-OpenRouter-Metadata: enabled`, inspect `openrouter_metadata.pipeline[]` for the guardrail stage that fired; adjust prompt/guardrail | +| "403 on an API call that isn't content" | **The other 403:** key-permission failure, not moderation — e.g. a runtime key calling a management-only endpoint (`GET /credits`) | `permission_denied` / **403** | Read the error message: permission-flavored wording (keys/scopes/"only management") means fix the key type or scope, not the prompt. `check-key-credits.ts` auto-detects this split | +| "400 / context too long" or image errors | Prompt + max_tokens exceed the window; or image constraints | `context_length_exceeded`, `max_tokens_exceeded`, `token_limit_exceeded`, `image_too_large`, `unsupported_image_format` | Trim/compress context ("middle-out"); pick a longer-window model via `GET /models`; validate images pre-send | +| "401 'User not found' — is it my key?" | **Feb 2026 post-mortem:** infra outages once surfaced as a misleading 401; now shipped as **503** so infra ≠ auth | `authentication` / **401** (real); infra now **503** | If the key is valid and it's 503, it's OpenRouter-side. Check status.openrouter.ai; reassure + escalate rather than send them key-hunting | +| "Edge timeout / slow" | Edge network timeout or request timeout | **524** `EdgeNetworkTimeoutResponse`; `timeout` / **408** | `sort:latency`/`:nitro`, `preferred_max_latency`; check `/generation` latency | + +Full enum + HTTP ladder: [references/error-taxonomy.md](references/error-taxonomy.md). + +## Direct API usage (curl) + +```bash +# Per-key usage/cap (works with any key; response includes is_management_key) +curl https://openrouter.ai/api/v1/key -H "Authorization: Bearer $OPENROUTER_API_KEY" + +# Account balance — MANAGEMENT key only (a runtime key gets 403 permission_denied) +curl https://openrouter.ai/api/v1/credits -H "Authorization: Bearer $OPENROUTER_MANAGEMENT_KEY" + +# See exactly what OpenRouter forwarded upstream (best single live-debug field) +curl https://openrouter.ai/api/v1/chat/completions \ + -H "Authorization: Bearer $OPENROUTER_API_KEY" -H "Content-Type: application/json" \ + -d '{"model":"openai/gpt-4o-mini","messages":[{"role":"user","content":"hi"}],"debug":{"echo_upstream_body":true}}' + +# Preview which providers a ZDR requirement leaves eligible +curl https://openrouter.ai/api/v1/endpoints/zdr -H "Authorization: Bearer $OPENROUTER_API_KEY" +``` + +## The native introspection moves + +- **`debug:{echo_upstream_body}`** — returns the transformed upstream body in a debug + chunk. The single best "show the customer what OpenRouter actually sent" field. +- **`GET /generation?id=…`** (+ `/generation/content`) — authoritative per-request + cost/latency/tokens/routing; reconcile against in-band `usage.cost`. (`inspect-generation.ts`.) +- **the `provider` object** — `order`, `only`/`ignore`, `allow_fallbacks`, + `require_parameters`, `data_collection`, `zdr`, `max_price`, `sort`. Over-constraint + lives here; this is what `validate-fallback-config.ts` lints. See + [references/provider-preferences.md](references/provider-preferences.md). +- **`session_id`** groups related requests for cost/trace attribution; + **`GET /endpoints/zdr`** previews the privacy-filter impact before it bites. + +## References (progressive disclosure) + +- [error-taxonomy.md](references/error-taxonomy.md) — the full 27-value `ApiErrorType` + enum, the HTTP ladder, and the in-band streaming-error case. +- [provider-preferences.md](references/provider-preferences.md) — every `provider` field + and the over-constraint patterns that produce the overloaded-404. +- [fallback-and-zdr.md](references/fallback-and-zdr.md) — how to configure model + fallbacks correctly and the union semantics of ZDR / data-collection policy. +- [onboarding-checklist.md](references/onboarding-checklist.md) — the integration-health + checklist an FDE runs in a customer's first two days. + +## Honest unknowns + +Flag, don't bluff. The exact semantics of the `exacto` sort, the `python`/`grammar` +response formats, and the `response-healing` / `context-compression` internals are not +fully specified by the public OpenAPI spec. "I'd confirm the exact behavior" is the +correct answer when a customer asks. This skill is v0.1, built from public evidence and +designed to be corrected. diff --git a/skills/openrouter-integration-doctor/references/error-taxonomy.md b/skills/openrouter-integration-doctor/references/error-taxonomy.md new file mode 100644 index 0000000..7b8fa52 --- /dev/null +++ b/skills/openrouter-integration-doctor/references/error-taxonomy.md @@ -0,0 +1,95 @@ +# Error taxonomy — the full `ApiErrorType` enum + HTTP ladder + +OpenRouter normalizes every failure — its own and each upstream provider's — into a +single error envelope with a canonical `error_type`. Switch on `error_type`, not the raw +provider string, and the same handler works across `/chat/completions`, `/messages`, and +`/responses`. + +## Error envelope + +```json +{ "error": { "code": 402, "message": "Insufficient credits", "metadata": { "error_type": "payment_required" } } } +``` + +- `code` mirrors the HTTP status. +- `metadata.error_type` is the canonical `ApiErrorType`. +- Provider-specific detail (when present) also rides in `metadata`. + +## The full 27-value `ApiErrorType` enum + +Verbatim from the live OpenAPI spec (3.1.0, "OpenRouter API" v1.0.0), grouped for recall: + +**Limits / context** +`context_length_exceeded` · `max_tokens_exceeded` · `token_limit_exceeded` · +`string_too_long` · `payload_too_large` + +**Auth / billing** +`authentication` · `permission_denied` · `payment_required` · `rate_limit_exceeded` + +**Routing / provider** +`provider_overloaded` · `provider_unavailable` · `not_found` · `precondition_failed` · +`server` · `timeout` · `unmapped` + +**Request validity** +`invalid_request` · `invalid_prompt` · `unprocessable` + +**Policy** +`content_policy_violation` · `refusal` + +**Images** +`invalid_image` · `image_too_large` · `image_too_small` · `unsupported_image_format` · +`image_not_found` · `image_download_failed` + +> The probes in this skill carry this exact enum plus a one-line native-fix hint per value +> (`ERROR_FIX_HINTS` in `scripts/lib.ts`), so a failed call already prints its own +> diagnosis. + +## HTTP ladder (each typed in the spec) + +| HTTP | Meaning | Typed response schema | +|------|---------|------------------------| +| `400` | Bad params / policy | — | +| `401` | Bad key | `Unauthorized` | +| **`402`** | Out of credits / key cap | `PaymentRequiredResponse` | +| `403` | **Two flavors:** moderation/guardrail (`content_policy_violation`) OR key permission (`permission_denied` — e.g. a runtime key on the management-only `GET /credits`; live-verified). Disambiguate on the message | `Forbidden` | +| `404` | Not found / over-constrained routing | `NotFound` | +| **`408`** | Request timeout | `RequestTimeoutResponse` | +| `413` | Payload too large | `PayloadTooLarge` | +| `422` | Unprocessable | `UnprocessableEntity` | +| `429` | Rate limited (honor `Retry-After`) | `TooManyRequests` | +| `500` | Server error | `InternalServer` | +| `502` | Bad gateway | `BadGateway` | +| **`503`** | No provider meets constraints, or infra | `ServiceUnavailableResponse` | +| **`524`** | Edge network timeout | `EdgeNetworkTimeoutResponse` | +| **`529`** | Provider overloaded | `ProviderOverloadedResponse` | + +## The two 429s + +A 429 is ambiguous and the fix differs: + +- **Account daily cap** — you are sending more than your account tier allows. Fix: raise + the limit, or spread traffic across keys. +- **Upstream provider overload** — the chosen provider is rate-limiting OpenRouter. Fix: + add a `models[]` fallback array so the request reroutes instead of failing. + +Honor `Retry-After` and back off with jitter in both cases. + +## The in-band streaming error (not an HTTP code) + +A streamed request that fails mid-flight returns **HTTP 200** and delivers the failure as +an SSE chunk whose `finish_reason` is `"error"`. Teams that only check the HTTP status +read these as short successes. Always: + +1. Parse the final SSE chunk. +2. Treat `finish_reason:"error"` as a failure and retry/fallback. +3. `stream_options.include_usage` is a no-op — usage is always included. + +`inspect-generation.ts` flags `finish_reason == "error"` for exactly this reason. + +## The 401→503 post-mortem (Feb 2026) + +An infra outage once surfaced as a misleading **401 "User not found"**, sending customers +to hunt for a key problem that didn't exist. OpenRouter shipped a fix so infra failures +now return **503**, not 401. Operational consequence: if the key is known-good and you see +503, it is OpenRouter-side — check status.openrouter.ai and escalate, don't send the +customer key-hunting. diff --git a/skills/openrouter-integration-doctor/references/fallback-and-zdr.md b/skills/openrouter-integration-doctor/references/fallback-and-zdr.md new file mode 100644 index 0000000..6be4108 --- /dev/null +++ b/skills/openrouter-integration-doctor/references/fallback-and-zdr.md @@ -0,0 +1,79 @@ +# Fallback configuration and ZDR / data-policy semantics + +Two of the most common "it was working, then it broke" integration failures come from +this pair: no fallback array (brittle under provider brownouts) and ZDR/no-training policy +that quietly narrows routing to zero. Both are lintable by +`validate-fallback-config.ts`. + +## Model fallback: `models[]` + +Reliability on OpenRouter is architectural, not incidental. There are two layers: + +1. **Provider-level failover** — default-on (`allow_fallbacks:true`). Within a single + model, OpenRouter reroutes across providers on upstream failure. +2. **Model-level fallback** — opt-in. Supply a `models` array of acceptable alternates; + if the primary model's providers all fail, OpenRouter tries the next model. + +```json +{ + "model": "anthropic/claude-3.5-sonnet", + "models": ["openai/gpt-4o", "google/gemini-1.5-pro"], + "provider": { "allow_fallbacks": true } +} +``` + +Without `models[]`, a single provider brownout (429 / 503 / 529) fails the whole request +instead of rerouting to an equivalent model. For any production path, supply a fallback +array of models you have validated as acceptable substitutes. + +### Fallback tuning knobs + +- `sort: "throughput"` / `:nitro` — bias toward providers that stay up under load. +- `allow_fallbacks: true` — keep provider-level failover on (don't disable it to "pin" + unless compliance requires it; that reintroduces the overloaded-404). +- Bound cost while you add resilience: `max_price` (careful — too low strands routing) and, + for agent loops, `stop_server_tools_when: { max_cost }`. + +## ZDR / data-collection is a **union**, not a switch + +Requiring Zero Data Retention or denying training data is the *intersection of eligible +providers* under **both** OpenRouter's policy **and** each downstream provider's +retention policy. Tightening it removes providers: + +- `provider.data_collection: "deny"` — exclude providers that may retain data for training. +- `provider.zdr: true` — require end-to-end Zero Data Retention. + +Push either far enough and the eligible set empties → **404 `not_found`**, which reads as +"the model disappeared" but is actually "your policy left no provider standing." + +### Preview before it bites + +```bash +curl https://openrouter.ai/api/v1/endpoints/zdr -H "Authorization: Bearer $OPENROUTER_API_KEY" +``` + +`GET /endpoints/zdr` shows which endpoints survive a ZDR requirement. Run it *before* +turning ZDR on in production so you know whether your model set still has coverage. + +### Fixing a ZDR-induced 404 + +1. Confirm the cause: does relaxing `zdr`/`data_collection` make the 404 go away? (Do this + in a test key, not prod.) +2. If yes and compliance allows, widen the model set so more ZDR-eligible providers + qualify — don't just drop the policy. +3. If compliance requires ZDR and coverage is thin, that is a real capacity constraint to + surface to the customer, not a bug to route around. + +## The consequence to teach + +Reliability (fallbacks) and compliance (ZDR) pull in opposite directions on the same +`provider` object: fallbacks *widen* the provider set, ZDR *narrows* it. An FDE's job is to +find the customer's actual floor on both and configure to it explicitly, rather than +leaving one implicit until it 404s in production. + +## The interaction with structured outputs + +Fallback plus strict `json_schema` is a trap: if a fallback provider doesn't support +strict schema, output silently degrades (or 422s). Always pair strict schema with +`provider.require_parameters:true` so only schema-capable providers are eligible — that +way fallback and structured output coexist safely. diff --git a/skills/openrouter-integration-doctor/references/onboarding-checklist.md b/skills/openrouter-integration-doctor/references/onboarding-checklist.md new file mode 100644 index 0000000..0ea3770 --- /dev/null +++ b/skills/openrouter-integration-doctor/references/onboarding-checklist.md @@ -0,0 +1,57 @@ +# Integration-health onboarding checklist + +The pass an FDE runs in a customer's first two days to catch the failures this skill +diagnoses *before* they page someone. Ordered outside-in: offline/config checks first, +live calls last. Each item names the probe or endpoint that verifies it. + +## Day 1 — wiring and account + +- [ ] **Key valid and funded.** `check-key-credits.ts` — account balance > 0 and this + key's spend cap (if any) has headroom. Splits account-depletion vs key-cap so a + future 402 is unambiguous. Account balance needs a management key + (`--management-key` / `OPENROUTER_MANAGEMENT_KEY`); the key-level view works with + the runtime key itself. +- [ ] **Spend guardrails exist.** Per-key limits set via the Management API; agent loops + bounded with `stop_server_tools_when: {max_cost}`. Prevents the "bill 10x'd + overnight" surprise. +- [ ] **Config lints clean.** `validate-fallback-config.ts ./their-request-body.json` + (offline) — no `error`-severity findings. Catches the overloaded-404, silent + structured-output degrade, and ZDR narrowing before the first real failure. +- [ ] **Pipe is open.** `smoke-test.ts` — a live call completes and returns the model / + provider / cost that served it. If this passes but the customer's own call fails, + the fault is their request shape, not the wiring. + +## Day 2 — resilience and observability + +- [ ] **Model fallback in place.** A `models[]` array of validated substitutes on every + production path; `allow_fallbacks:true` unless compliance forbids the alternates. + See [fallback-and-zdr.md](fallback-and-zdr.md). +- [ ] **Feature routing pinned.** Any dependence on strict `json_schema`, tools, or + reasoning is guarded with `provider.require_parameters:true` so fallback can't + silently pick a provider that ignores it. +- [ ] **Compliance floor known.** If ZDR / no-training is required, `GET /endpoints/zdr` + confirms the model set still has provider coverage. Thin coverage is surfaced as a + capacity constraint, not routed around. +- [ ] **Streaming handled correctly.** If the customer streams, they parse the final SSE + chunk and treat `finish_reason:"error"` as a failure. Confirm — this is the single + most common silent-failure mode. +- [ ] **Cost observability.** They can pull authoritative per-request cost/routing via + `inspect-generation.ts ` (or the `openrouter-generations` skill) and group by + `session_id` for attribution. Reconcile against in-band `usage.cost`. +- [ ] **Error handling switches on `ApiErrorType`**, not raw provider strings or bare HTTP + codes. See [error-taxonomy.md](error-taxonomy.md) for the 27-value enum. This is the + difference between an integration that degrades gracefully and one that pages on + every provider hiccup. + +## Escalation reflex + +- A known-good key returning **503** is OpenRouter-side (the 401→503 post-mortem). Check + status.openrouter.ai and escalate; do not send the customer key-hunting. +- Novel provider errors surface as `unmapped` — capture the raw message + + `echo_upstream_body` and escalate rather than guess. + +## Framing + +This checklist is v0.1, built from public evidence and designed to be corrected. It is the +integration-health floor, not a substitute for the customer's own SLAs — confidence about +shape, humility about a given customer's internals. diff --git a/skills/openrouter-integration-doctor/references/provider-preferences.md b/skills/openrouter-integration-doctor/references/provider-preferences.md new file mode 100644 index 0000000..644f3ed --- /dev/null +++ b/skills/openrouter-integration-doctor/references/provider-preferences.md @@ -0,0 +1,60 @@ +# Provider preferences — the `provider` object and the overloaded-404 + +Most "No endpoints found" 404s are not real absence — they are a request over-constrained +until zero providers qualify. The over-constraint lives in the `provider` object. This is +what `validate-fallback-config.ts` lints, offline. + +## The fields + +| Field | Type | What it does | Over-constraint risk | +|-------|------|--------------|----------------------| +| `order` | string[] | Preferred provider order to try | Short list + `allow_fallbacks:false` → 404 when those are down | +| `only` | string[] | Restrict to these providers | Narrows the set; combined with other filters can empty it | +| `ignore` | string[] | Exclude these providers | Overlap with `only` is contradictory → provider excluded | +| `allow_fallbacks` | bool | Allow provider-level failover (default true) | `false` removes the safety net | +| `require_parameters` | bool | Only route to providers supporting every requested param | Off → silent degrade on strict features | +| `data_collection` | "allow"/"deny" | Whether providers may retain data for training | `deny` narrows the eligible set | +| `zdr` | bool | Require Zero Data Retention end-to-end | Union with downstream policy can empty the set | +| `max_price` | object | Price ceiling per token type | Too low → strands routing | +| `sort` | string | `price` / `throughput` / `latency` (`:nitro`) | Wrong axis for the customer's SLA | + +## The four causes of "No endpoints found" (404 / `not_found`) + +One symptom, four unrelated roots — diagnose which: + +1. **Model-slug typo.** The slug doesn't exist. Verify against `GET /models` + (`openrouter-models` skill). +2. **Provider prefs too tight.** `order`/`only` restricted + `allow_fallbacks:false`, or + `max_price` too low. Loosen. → `validate-fallback-config.ts` flags this. +3. **Data-policy narrowing.** `zdr:true` or `data_collection:"deny"` filtered every + provider out. See [fallback-and-zdr.md](fallback-and-zdr.md). → flagged. +4. **Unsupported parameter.** A requested param (e.g. strict `json_schema`, a tool mode) + isn't supported by any eligible provider. → surfaced via `require_parameters` + + `echo_upstream_body`. + +## The single best live-debug move + +```json +{ "debug": { "echo_upstream_body": true } } +``` + +Returns the transformed body OpenRouter actually forwarded upstream, in a debug chunk. +Show the customer exactly what was sent — it collapses most "but my request is correct" +disputes instantly. + +## Healthy patterns + +- Pin nothing you don't have to. Prefer `sort` + a wide model set over a hard `order`. +- If you must pin, keep `allow_fallbacks:true` unless a compliance rule forbids the + alternates. +- Turn on `require_parameters:true` whenever you depend on a feature (strict schema, tools, + reasoning) so routing can't silently pick a provider that ignores it. +- Reserve `only`/`ignore` for genuine hard constraints; never let them overlap. + +## Sort axes + +- `price` (default-ish) — cheapest eligible, with inverse-square-of-price weighting; note + the default is *not* strictly "always cheapest". +- `throughput` / `:nitro` — favor high-throughput providers; good under load / for + overload avoidance. +- `latency` — favor low-latency providers; pair with `preferred_max_latency` for SLAs. diff --git a/skills/openrouter-integration-doctor/scripts/check-key-credits.ts b/skills/openrouter-integration-doctor/scripts/check-key-credits.ts new file mode 100644 index 0000000..10e4f1a --- /dev/null +++ b/skills/openrouter-integration-doctor/scripts/check-key-credits.ts @@ -0,0 +1,159 @@ +/** + * Key + credit health check — the first probe an FDE runs on a struggling integration. + * Reconciles two views so a 402 / "insufficient credits" report can be diagnosed in one + * shot: this key's usage/limit/rate-limit (GET /key) and, when a management key is + * available, account-level balance (GET /credits). + * + * Key-type behavior (live-verified 2026-07-03): GET /credits requires a MANAGEMENT key — + * a regular runtime key gets 403 "Only management keys can fetch credits for an + * account". GET /key works with both and reports `is_management_key`, so this probe + * auto-detects what it was given: a management key gets the full account + key view; a + * runtime key gets the key view, plus the account view if a management key is supplied + * via --management-key or OPENROUTER_MANAGEMENT_KEY. + * + * Usage: + * npx tsx check-key-credits.ts # key view (+ account view if key allows) + * npx tsx check-key-credits.ts --management-key sk-or-... # runtime key view + account view + * npx tsx check-key-credits.ts --json # raw combined JSON + */ +import { requireApiKey, fetchApi, tryFetchApi, parseArgs } from "./lib.js"; + +interface CreditsResponse { + data?: { total_credits?: number; total_usage?: number }; +} + +interface KeyResponse { + data?: { + label?: string; + usage?: number; + limit?: number | null; + limit_remaining?: number | null; + is_free_tier?: boolean; + is_management_key?: boolean; + is_provisioning_key?: boolean; + rate_limit?: { requests?: number; interval?: string } | null; + }; +} + +const args = parseArgs(process.argv.slice(2), ["json"]); +const apiKey = requireApiKey(args); +const json = args.has("json"); +const managementKey = + args.get("management-key") ?? process.env.OPENROUTER_MANAGEMENT_KEY; + +// GET /key first — it works with every key type and tells us what we were given. +const key = await fetchApi("/key", { apiKey }); +const k = key.data ?? {}; + +// Account-level credits need a management key. Prefer the main key if it IS one, +// else fall back to an explicitly supplied management key, else skip the account view. +const creditsKey = k.is_management_key ? apiKey : managementKey; +let credits: CreditsResponse | null = null; +let creditsSkipReason: string | null = null; +if (creditsKey) { + const res = await tryFetchApi("/credits", { apiKey: creditsKey }); + if (res.ok) { + credits = res.data; + } else { + creditsSkipReason = `GET /credits failed (${res.status}) with the supplied management key — check that it is a management key with access to this account.`; + } +} else { + creditsSkipReason = + "This is a runtime key: GET /credits requires a management key (403 otherwise). " + + "Pass --management-key or set OPENROUTER_MANAGEMENT_KEY to include the account view."; +} + +if (json) { + console.log( + JSON.stringify( + { + credits: credits?.data ?? null, + credits_skipped: creditsSkipReason, + key: key.data ?? null, + }, + null, + 2 + ) + ); + process.exit(0); +} + +const c = credits?.data; +const totalCredits = c?.total_credits ?? 0; +const totalUsage = c?.total_usage ?? 0; +const accountRemaining = totalCredits - totalUsage; + +console.log("=== Account credits (GET /credits) ==="); +if (c) { + console.log("Total credits purchased:", fmtUsd(totalCredits)); + console.log("Total usage: ", fmtUsd(totalUsage)); + console.log("Remaining balance: ", fmtUsd(accountRemaining)); +} else { + console.log("Skipped —", creditsSkipReason); +} +console.log(""); + +console.log("=== This key (GET /key) ==="); +console.log("Label: ", k.label ?? "(unnamed)"); +console.log("Key type: ", k.is_management_key ? "management" : "runtime"); +console.log("Free tier: ", k.is_free_tier ?? false); +console.log("Provisioning key: ", k.is_provisioning_key ?? false); +console.log("Key usage: ", fmtUsd(k.usage ?? 0)); +if (k.limit == null) { + console.log("Key spend limit: none (unlimited within account balance)"); +} else { + console.log("Key spend limit: ", fmtUsd(k.limit)); + console.log("Key remaining: ", fmtUsd(k.limit_remaining ?? Math.max(0, k.limit - (k.usage ?? 0)))); +} +// The /key rate_limit field is deprecated (live responses say "safe to ignore" and +// return requests: -1); only print it when it still carries a real value. +if (k.rate_limit && typeof k.rate_limit.requests === "number" && k.rate_limit.requests > 0) { + console.log("Rate limit: ", `${k.rate_limit.requests} req / ${k.rate_limit.interval ?? "?"}`); +} +console.log(""); + +// --- Diagnosis --------------------------------------------------------------- +const warnings: string[] = []; + +if (c) { + if (accountRemaining <= 0) { + warnings.push( + "Account balance is depleted → every request will 402 (payment_required). Top up credits." + ); + } else if (accountRemaining < 1) { + warnings.push( + `Account balance is low (${fmtUsd(accountRemaining)}) → agentic burn can 402 mid-run. Top up or set alerts.` + ); + } +} + +if (k.limit != null) { + const remaining = k.limit_remaining ?? Math.max(0, k.limit - (k.usage ?? 0)); + const pctUsed = k.limit > 0 ? ((k.usage ?? 0) / k.limit) * 100 : 100; + if (remaining <= 0) { + warnings.push( + "This key has hit its spend cap → it 402s even though the account has balance. Raise the per-key limit via the Management API." + ); + } else if (pctUsed >= 90) { + warnings.push( + `This key is at ${pctUsed.toFixed(0)}% of its spend cap (${fmtUsd(remaining)} left) → near-term 402 risk.` + ); + } +} + +if (warnings.length === 0) { + console.log( + c + ? "Diagnosis: healthy — account has balance and this key is within its cap." + : "Diagnosis: this key is within its cap. (Account balance not checked — see above.)" + ); +} else { + console.log("Diagnosis:"); + for (const w of warnings) console.log(" ! " + w); +} + +function fmtUsd(v: number): string { + if (!Number.isFinite(v)) return String(v); + const sign = v < 0 ? "-" : ""; + return `${sign}$${Math.abs(v).toFixed(v !== 0 && Math.abs(v) < 0.01 ? 6 : 2)}`; +} diff --git a/skills/openrouter-integration-doctor/scripts/inspect-generation.ts b/skills/openrouter-integration-doctor/scripts/inspect-generation.ts new file mode 100644 index 0000000..93d68af --- /dev/null +++ b/skills/openrouter-integration-doctor/scripts/inspect-generation.ts @@ -0,0 +1,192 @@ +/** + * /generation introspection probe — the doctor's cross-request bridge to the + * `openrouter-generations` skill. Given a generation ID, fetches authoritative metadata + * (GET /generation) and surfaces the fields that most often explain a customer's + * complaint: real cost vs upstream cost (billing surprises), the provider fallback + * chain (silent BYOK/paid fallback), streaming/finish state (mid-stream truncation), + * and routing. Optionally pulls stored prompt/completion (GET /generation/content). + * + * This is deliberately diagnosis-shaped, not a full field dump — for the exhaustive + * per-field reference use the `openrouter-generations` skill directly. + * + * Usage: + * npx tsx inspect-generation.ts gen-1234567890 + * npx tsx inspect-generation.ts --id gen-1234567890 --content # include prompt/completion + * npx tsx inspect-generation.ts gen-1234567890 --json # raw metadata JSON + */ +import { requireApiKey, fetchApi, tryFetchApi, reportHttpError, parseArgs } from "./lib.js"; + +interface GenerationResponse { + data?: Record | null; +} + +const args = parseArgs(process.argv.slice(2), ["json", "content"]); +const apiKey = requireApiKey(args); + +const generationId = args.get("id") ?? args.get("_0"); +if (!generationId) { + console.error( + ` +Usage: npx tsx inspect-generation.ts [--content] [--json] + npx tsx inspect-generation.ts --id gen-1234567890 [--content] [--json] + +Fetches authoritative cost / routing / status for a generation and flags the +fields that usually explain a support ticket. Add --content to also pull the +stored prompt & completion (unavailable if the request used Zero Data Retention). + +For the full per-field reference, use the openrouter-generations skill. +`.trim() + ); + process.exit(1); +} + +// Generation metadata is indexed asynchronously — a freshly returned id can 404 for a +// few seconds (live-verified: 404 immediately after a request, 200 shortly after). Retry +// a 404 with backoff before treating it as real. +const RETRY_DELAYS_MS = [2000, 4000, 6000]; +let meta!: GenerationResponse; +for (let attempt = 0; ; attempt++) { + const res = await tryFetchApi("/generation", { + apiKey, + params: { id: generationId }, + }); + if (res.ok) { + meta = res.data; + break; + } + if (res.status === 404 && attempt < RETRY_DELAYS_MS.length) { + console.error( + `Generation not indexed yet (404) — retrying in ${RETRY_DELAYS_MS[attempt] / 1000}s (${attempt + 1}/${RETRY_DELAYS_MS.length})...` + ); + await new Promise((r) => setTimeout(r, RETRY_DELAYS_MS[attempt])); + continue; + } + reportHttpError(res.status, res.rawBody); + if (res.status === 404) { + console.error( + " note: generation ids are indexed asynchronously and can 404 briefly after the request\n" + + " (already retried). If this id is old, confirm you are querying with the same key/org\n" + + " that created it — generations are only visible to their owning account." + ); + } + process.exit(1); +} + +if (args.has("json")) { + console.log(JSON.stringify(meta, null, 2)); +} else { + const d = meta.data; + if (!d) { + console.log(`No metadata available for ${generationId}.`); + process.exit(0); + } + console.log("Generation:", d.id ?? generationId); + console.log("Model: ", d.model ?? "unknown"); + console.log("Provider: ", d.provider_name ?? "unknown"); + console.log("Router: ", d.router ?? "n/a"); + console.log(""); + console.log("--- Cost (reconcile against in-band usage.cost) ---"); + console.log("total_cost: ", fmtCost(d.total_cost)); + console.log("upstream_inference_cost:", fmtCost(d.upstream_inference_cost)); + if (d.cache_discount != null) console.log("cache_discount: ", d.cache_discount); + console.log("is_byok: ", d.is_byok ?? false); + console.log(""); + console.log("--- Status ---"); + console.log("finish_reason:", d.finish_reason ?? "n/a"); + console.log("streamed: ", d.streamed ?? "n/a"); + console.log("cancelled: ", d.cancelled ?? "n/a"); + console.log("latency: ", d.latency != null ? `${d.latency} ms` : "n/a"); + console.log(""); + + // --- Doctor diagnosis ------------------------------------------------------ + const notes: string[] = []; + const providerResponses = d.provider_responses; + if (Array.isArray(providerResponses) && providerResponses.length > 1) { + // Multiple entries means earlier attempts failed. Distinguish a same-provider retry + // (e.g. 429 then 200 on the same provider) from a true cross-provider fallback — + // only the latter can silently change billing/BYOK. Live-verified: both shapes + // occur in production (OpenAI 429→OpenAI 200, and OpenAI 429→Azure 200). + const entries = providerResponses as Array<{ provider_name?: string; status?: number }>; + const providers = [...new Set(entries.map((p) => p.provider_name ?? "?"))]; + const attempts = entries + .map((p) => `${p.provider_name ?? "?"}:${p.status ?? "?"}`) + .join(" → "); + if (providers.length > 1) { + notes.push( + `provider_responses shows a cross-provider fallback (${attempts}). If billing surprised the customer, a silent paid/BYOK fallback is a prime suspect.` + ); + } else { + notes.push( + `provider_responses shows ${entries.length} attempts on ${providers[0]} (${attempts}) → upstream retried after a failure. Same provider, so billing is unchanged, but repeated upstream 429s here mean provider-side pressure — consider a models[]/provider fallback.` + ); + } + } + if (d.finish_reason === "error") { + notes.push( + "finish_reason == 'error' → this generation failed mid-flight. If it was streamed, the client likely read HTTP 200 and missed the in-band failure. Treat as a failure and retry/fallback." + ); + } + if (d.finish_reason === "length") { + notes.push("finish_reason == 'length' → hit max tokens. Raise max_tokens or expect truncation."); + } + if (d.finish_reason === "content_filter") { + notes.push("finish_reason == 'content_filter' → moderation/guardrail trimmed output (content_policy_violation)."); + } + if (d.is_byok === true) { + notes.push("is_byok == true → served on the customer's own provider key (+5% fee). Confirm this was intended."); + } + const total = toNum(d.total_cost); + const upstream = toNum(d.upstream_inference_cost); + if (total != null && upstream != null && upstream > 0 && total > upstream * 3) { + notes.push( + `total_cost ($${total}) is >3x upstream_inference_cost ($${upstream}) → verify the routing/markup is expected.` + ); + } + + if (notes.length === 0) { + console.log("Diagnosis: nothing anomalous in the routing/cost/status fields."); + } else { + console.log("Diagnosis:"); + for (const n of notes) console.log(" ! " + n); + } +} + +if (args.has("content")) { + console.log(""); + console.log("--- Stored content (GET /generation/content) ---"); + const content = await fetchApi("/generation/content", { + apiKey, + params: { id: generationId }, + }); + const cd = content.data as + | { input?: { prompt?: unknown; messages?: unknown }; output?: { completion?: unknown; reasoning?: unknown } } + | null + | undefined; + if (!cd || (cd.input == null && cd.output == null)) { + console.log("No stored content — likely Zero Data Retention (ZDR) was enabled for this request."); + } else { + console.log("prompt: ", trunc(cd.input?.prompt)); + console.log("completion:", trunc(cd.output?.completion)); + if (cd.output?.reasoning) console.log("reasoning: ", trunc(cd.output.reasoning)); + } +} + +function toNum(v: unknown): number | null { + const n = typeof v === "number" ? v : Number(v); + return Number.isFinite(n) ? n : null; +} + +function fmtCost(v: unknown): string { + const n = toNum(v); + if (n == null) return "n/a"; + if (n === 0) return "$0.00"; + const sign = n < 0 ? "-" : ""; + const abs = Math.abs(n); + return abs >= 0.000001 ? `${sign}$${abs.toFixed(6).replace(/0+$/, "").replace(/\.$/, ".00")}` : `${sign}$${abs.toExponential(4)}`; +} + +function trunc(v: unknown, max = 400): string { + if (v == null) return "(none)"; + const s = typeof v === "string" ? v : JSON.stringify(v); + return s.length > max ? s.slice(0, max) + `… [${s.length} chars]` : s; +} diff --git a/skills/openrouter-integration-doctor/scripts/lib.ts b/skills/openrouter-integration-doctor/scripts/lib.ts new file mode 100644 index 0000000..8d1d872 --- /dev/null +++ b/skills/openrouter-integration-doctor/scripts/lib.ts @@ -0,0 +1,304 @@ +const BASE_URL = process.env.OPENROUTER_BASE_URL ?? "https://openrouter.ai"; + +/** + * Canonical OpenRouter `ApiErrorType` enum — all 27 values, verbatim from the live + * OpenAPI spec (3.1.0, "OpenRouter API" v1.0.0). Error responses normalize provider + * failures into one of these under `error.metadata.error_type`, stable across + * `/chat/completions`, `/messages`, and `/responses`. The probes switch on these so + * a diagnosis maps to a single canonical cause rather than a raw HTTP number. + */ +export const API_ERROR_TYPES = [ + "context_length_exceeded", + "max_tokens_exceeded", + "token_limit_exceeded", + "string_too_long", + "authentication", + "permission_denied", + "payment_required", + "rate_limit_exceeded", + "provider_overloaded", + "provider_unavailable", + "invalid_request", + "invalid_prompt", + "not_found", + "precondition_failed", + "payload_too_large", + "unprocessable", + "content_policy_violation", + "refusal", + "invalid_image", + "image_too_large", + "image_too_small", + "unsupported_image_format", + "image_not_found", + "image_download_failed", + "server", + "timeout", + "unmapped", +] as const; + +export type ApiErrorType = (typeof API_ERROR_TYPES)[number]; + +/** + * One-line native-fix hint per canonical error type. Keyed by `ApiErrorType` so the + * live probes (and a customer's own error handler) can turn a normalized error into an + * actionable OpenRouter-native next step instead of a bare status code. + */ +export const ERROR_FIX_HINTS: Record = { + context_length_exceeded: + "Prompt + max_tokens exceed the model window. Trim/compress context or pick a longer-window model (GET /models).", + max_tokens_exceeded: + "Requested max_tokens is above what the model/provider allows. Lower max_tokens or switch model.", + token_limit_exceeded: + "Combined token budget exceeded. Reduce input or completion budget, or route to a larger-window model.", + string_too_long: + "A field value exceeds the allowed length. Shorten the offending field (often a system prompt or tool schema).", + authentication: + "Invalid or missing API key (real 401). Verify OPENROUTER_API_KEY. NOTE: infra outages now surface as 503, not 401.", + permission_denied: + "Key lacks access to this resource/model. Check key scopes and model allow-list in the OpenRouter dashboard.", + payment_required: + "Out of credits or key spend-cap hit (402). Run check-key-credits.ts; top up or raise the per-key limit via the Management API.", + rate_limit_exceeded: + "429 — either account daily cap or upstream provider overload. Honor Retry-After + jitter; add a models[] fallback array so it reroutes.", + provider_overloaded: + "529 — provider brownout inside the 30s health window. Add model fallbacks, allow_fallbacks:true, sort:throughput/:nitro.", + provider_unavailable: + "503 — no provider currently meets constraints (or infra). Loosen provider prefs or widen the model set; check status.openrouter.ai.", + invalid_request: + "Malformed request. Use debug:{echo_upstream_body} to see the transformed upstream body and diff against provider expectations.", + invalid_prompt: + "Prompt structure rejected. Check message roles/ordering and any provider-specific prompt constraints.", + not_found: + "404 / 'No endpoints found' — model-slug typo, over-tight provider prefs, data-policy narrowing, or an unsupported param. See references/provider-preferences.md.", + precondition_failed: + "A required precondition failed (e.g. an unmet header/param dependency). Inspect echo_upstream_body and required_parameters.", + payload_too_large: + "413 — request body too big. Reduce attachments/context or chunk the request.", + unprocessable: + "422 — semantically invalid (often strict json_schema a provider can't honor). Set provider.require_parameters:true so only capable providers route.", + content_policy_violation: + "Moderation/guardrail stage blocked the request. With X-OpenRouter-Metadata:enabled, inspect openrouter_metadata.pipeline[] to see which stage fired.", + refusal: + "Model self-refused. Adjust the prompt or reassign the guardrail; this is model behavior, not an integration bug.", + invalid_image: + "Image payload invalid. Validate encoding/URL before send.", + image_too_large: "Image exceeds size limits. Downscale before send.", + image_too_small: "Image below the model's minimum dimensions. Upscale or pick a different model.", + unsupported_image_format: "Image format not supported by the routed provider. Convert to a supported format (png/jpeg/webp).", + image_not_found: "Referenced image URL could not be fetched. Verify the URL is reachable and public.", + image_download_failed: "OpenRouter could not download the referenced image. Check host availability/timeouts or inline the image as base64.", + server: "500 — OpenRouter-side error. Retry with backoff; if persistent, check status.openrouter.ai.", + timeout: "408/524 — request or edge timeout. Use sort:latency/:nitro, set preferred_max_latency, check /generation latency.", + unmapped: "Provider error that did not map to a known type. Inspect the raw message + echo_upstream_body and escalate if novel.", +}; + +export function requireApiKey(args?: Map): string { + const apiKey = args?.get("api-key") ?? process.env.OPENROUTER_API_KEY; + if (!apiKey) { + console.error( + "Error: No API key provided.\n" + + "Pass --api-key or set the OPENROUTER_API_KEY environment variable.\n" + + "Get one at https://openrouter.ai/settings/keys" + ); + process.exit(1); + } + return apiKey; +} + +export interface FetchOptions { + apiKey: string; + method?: "GET" | "POST"; + params?: Record; + body?: unknown; + /** Extra headers (e.g. HTTP-Referer / X-Title for attribution). */ + headers?: Record; +} + +export type TryFetchResult = + | { ok: true; status: number; data: T } + | { ok: false; status: number; rawBody: string }; + +/** + * Non-exiting variant of `fetchApi` for probes that branch on specific failures + * instead of hard-stopping — e.g. check-key-credits.ts treats a 403 on GET /credits + * as "runtime key, skip account view" and inspect-generation.ts retries a 404 while + * generation metadata is still being indexed. Network errors still hard-stop: no + * probe can do anything useful without connectivity. + */ +export async function tryFetchApi( + path: string, + opts: FetchOptions +): Promise> { + const headers: Record = { + Authorization: `Bearer ${opts.apiKey}`, + ...(opts.headers ?? {}), + }; + const method = opts.method ?? "GET"; + if (opts.body !== undefined) { + headers["Content-Type"] = "application/json"; + } + + let res: Response; + try { + const url = new URL(`${BASE_URL}/api/v1${path}`); + if (opts.params) { + for (const [key, value] of Object.entries(opts.params)) { + url.searchParams.set(key, value); + } + } + res = await fetch(url.toString(), { + method, + headers, + body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined, + }); + } catch (err) { + console.error(`Network error: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } + + if (!res.ok) { + const raw = await res.text().catch(() => ""); + return { ok: false, status: res.status, rawBody: raw }; + } + + try { + return { ok: true, status: res.status, data: (await res.json()) as T }; + } catch (err) { + console.error(`Invalid JSON in response: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } +} + +/** + * Thin fetch wrapper against `https://openrouter.ai/api/v1`. Returns the parsed JSON + * body on success. On an HTTP error it prints a taxonomy-aware diagnosis (status + + * canonical error_type + native-fix hint) and exits non-zero — the probes are CLIs, so + * a failed request is a hard stop with an actionable message, not a thrown value the + * caller must re-handle. + */ +export async function fetchApi(path: string, opts: FetchOptions): Promise { + const result = await tryFetchApi(path, opts); + if (!result.ok) { + reportHttpError(result.status, result.rawBody); + process.exit(1); + } + return result.data; +} + +/** + * Print a taxonomy-aware diagnosis for a failed HTTP call. Pulls the canonical + * `error_type` out of the OpenRouter error envelope when present + * (`{ error: { code, message, metadata: { error_type } } }`) and appends the matching + * native-fix hint. Falls back to an HTTP-status-only diagnosis when the body is empty + * or unparseable. + */ +export function reportHttpError(status: number, rawBody: string): void { + let errorType: string | undefined; + let message: string | undefined; + if (rawBody) { + try { + const parsed = JSON.parse(rawBody) as { + error?: { message?: string; metadata?: { error_type?: string } }; + }; + errorType = parsed.error?.metadata?.error_type; + message = parsed.error?.message; + } catch { + // Non-JSON error body (e.g. an HTML gateway page); fall through to status-only. + } + } + + console.error(`Error ${status}${message ? `: ${message}` : `: ${statusHint(status)}`}`); + if (errorType && errorType in ERROR_FIX_HINTS) { + console.error(` error_type: ${errorType}`); + console.error(` fix: ${ERROR_FIX_HINTS[errorType as ApiErrorType]}`); + } else { + const guessed = statusToErrorType(status, message); + if (guessed) { + console.error(` likely error_type: ${guessed}`); + console.error(` fix: ${ERROR_FIX_HINTS[guessed]}`); + } + } +} + +/** Short human hint for a bare HTTP status when no error envelope is present. */ +function statusHint(status: number): string { + const hints: Record = { + 400: "Bad request — malformed params or policy violation", + 401: "Invalid or missing API key", + 402: "Out of credits or key spend-cap hit", + 403: "Forbidden — moderation/guardrail or no access", + 404: "Not found — model-slug typo or over-constrained routing", + 408: "Request timeout", + 413: "Payload too large", + 422: "Unprocessable — semantically invalid request", + 429: "Rate limited — honor Retry-After and back off", + 500: "Server error — retry with backoff", + 502: "Bad gateway — upstream failure, retry", + 503: "No provider meets constraints, or infra issue", + 524: "Edge network timeout", + 529: "Provider overloaded", + }; + return hints[status] ?? "Unexpected error"; +} + +/** + * Best-effort mapping from HTTP status to a canonical error_type when the body omits + * one. A 403 is genuinely ambiguous — it is EITHER a moderation/guardrail block OR a + * key-permission failure (live-verified: GET /credits with a non-management key returns + * 403 "Only management keys can fetch credits for an account"). When the error message + * is available we disambiguate on it; permission-flavored wording wins over the + * moderation default. + */ +export function statusToErrorType(status: number, message?: string): ApiErrorType | undefined { + if (status === 403 && message && /\b(key|keys|permission|scope|not allowed|access|only management)\b/i.test(message)) { + return "permission_denied"; + } + const map: Partial> = { + 401: "authentication", + 402: "payment_required", + 403: "content_policy_violation", + 404: "not_found", + 408: "timeout", + 413: "payload_too_large", + 422: "unprocessable", + 429: "rate_limit_exceeded", + 500: "server", + 503: "provider_unavailable", + 524: "timeout", + 529: "provider_overloaded", + }; + return map[status]; +} + +/** + * Minimal `--flag value` / `--flag` (boolean) argument parser. Flags listed in + * `booleanFlags` never consume the following token, so `--json ./body.json` parses + * `json` as boolean and `./body.json` as a positional. Mirrors the exemplar + * openrouter-generations parser so the whole skill family behaves identically. + */ +export function parseArgs(argv: string[], booleanFlags: readonly string[] = []): Map { + const booleans = new Set(booleanFlags); + const result = new Map(); + const positional: string[] = []; + + for (let i = 0; i < argv.length; i++) { + if ( + argv[i].startsWith("--") && + !booleans.has(argv[i].slice(2)) && + argv[i + 1] && + !argv[i + 1].startsWith("--") + ) { + result.set(argv[i].slice(2), argv[i + 1]); + i++; + } else if (argv[i].startsWith("--")) { + result.set(argv[i].slice(2), "true"); + } else { + positional.push(argv[i]); + } + } + + positional.forEach((v, i) => result.set(`_${i}`, v)); + result.set("_count", String(positional.length)); + return result; +} diff --git a/skills/openrouter-integration-doctor/scripts/package-lock.json b/skills/openrouter-integration-doctor/scripts/package-lock.json new file mode 100644 index 0000000..73e2fc9 --- /dev/null +++ b/skills/openrouter-integration-doctor/scripts/package-lock.json @@ -0,0 +1,531 @@ +{ + "name": "openrouter-integration-doctor-scripts", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "openrouter-integration-doctor-scripts", + "devDependencies": { + "tsx": "^4.8.1" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/esbuild": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/tsx": { + "version": "4.22.5", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.22.5.tgz", + "integrity": "sha512-F7JnSfPl5ASt6LqwWyUQ3T8BwN3q0eQEbFMYa2iRWaVQmmudo0d7fRmwM4O002gsvW1bs0yBYioutsAjqLJMvQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + } + } +} diff --git a/skills/openrouter-integration-doctor/scripts/package.json b/skills/openrouter-integration-doctor/scripts/package.json new file mode 100644 index 0000000..c3975ad --- /dev/null +++ b/skills/openrouter-integration-doctor/scripts/package.json @@ -0,0 +1,8 @@ +{ + "name": "openrouter-integration-doctor-scripts", + "type": "module", + "private": true, + "devDependencies": { + "tsx": "^4.8.1" + } +} diff --git a/skills/openrouter-integration-doctor/scripts/smoke-test.ts b/skills/openrouter-integration-doctor/scripts/smoke-test.ts new file mode 100644 index 0000000..ca6d690 --- /dev/null +++ b/skills/openrouter-integration-doctor/scripts/smoke-test.ts @@ -0,0 +1,85 @@ +/** + * Minimal live-call smoke test — proves the integration end-to-end: key is valid, the + * account has credit, a request routes to a provider, and a completion comes back. This + * is the "is the pipe open at all?" probe. On success it prints the model/provider/cost + * that actually served the request; on failure it routes the error through the shared + * taxonomy so the very first live call is already a diagnosis. + * + * Uses a cheap default model and a tiny prompt to keep cost negligible (fractions of a + * cent). Sends the OpenRouter attribution headers (HTTP-Referer / X-Title) so the call + * is clearly identifiable as the integration-doctor smoke test. + * + * Usage: + * npx tsx smoke-test.ts + * npx tsx smoke-test.ts --model openai/gpt-4o-mini + * npx tsx smoke-test.ts --prompt "reply with the word OK" --json + */ +import { requireApiKey, fetchApi, parseArgs } from "./lib.js"; + +interface ChatCompletionResponse { + id?: string; + model?: string; + provider?: string; + choices?: Array<{ message?: { content?: string }; finish_reason?: string }>; + usage?: { prompt_tokens?: number; completion_tokens?: number; total_tokens?: number; cost?: number }; +} + +const args = parseArgs(process.argv.slice(2), ["json"]); +const apiKey = requireApiKey(args); + +const model = args.get("model") ?? "openai/gpt-4o-mini"; +const prompt = args.get("prompt") ?? "Reply with exactly: OK"; +const maxTokens = Number(args.get("max-tokens") ?? "16"); + +const requestBody = { + model, + messages: [{ role: "user", content: prompt }], + max_tokens: Number.isFinite(maxTokens) ? maxTokens : 16, +}; + +const started = Date.now(); +const res = await fetchApi("/chat/completions", { + apiKey, + method: "POST", + body: requestBody, + headers: { + "HTTP-Referer": "https://github.com/OpenRouterTeam/skills", + "X-Title": "openrouter-integration-doctor smoke-test", + }, +}); +const elapsed = Date.now() - started; + +if (args.has("json")) { + console.log(JSON.stringify(res, null, 2)); + process.exit(0); +} + +const content = res.choices?.[0]?.message?.content ?? ""; +const finish = res.choices?.[0]?.finish_reason ?? "n/a"; + +console.log("Smoke test: PASS — the integration is live end-to-end."); +console.log(""); +console.log("Requested model:", model); +console.log("Served model: ", res.model ?? "unknown"); +console.log("Provider: ", res.provider ?? "unknown"); +console.log("Round-trip: ", `${elapsed} ms`); +console.log("Finish reason: ", finish); +if (res.usage) { + console.log( + "Tokens: ", + `${res.usage.prompt_tokens ?? "?"} prompt + ${res.usage.completion_tokens ?? "?"} completion` + ); + if (res.usage.cost != null) console.log("In-band cost: ", `$${res.usage.cost}`); +} +console.log(""); +console.log("Completion:", JSON.stringify(content)); +console.log(""); +if (res.id) { + console.log(`Tip: inspect this call with npx tsx inspect-generation.ts ${res.id}`); +} +if (finish === "error") { + console.log(""); + console.log("! finish_reason == 'error': the pipe opened but this generation failed mid-flight."); + console.log(" Treat as a failure and retry/fallback — see references/error-taxonomy.md."); + process.exit(1); +} diff --git a/skills/openrouter-integration-doctor/scripts/validate-fallback-config.ts b/skills/openrouter-integration-doctor/scripts/validate-fallback-config.ts new file mode 100644 index 0000000..1394849 --- /dev/null +++ b/skills/openrouter-integration-doctor/scripts/validate-fallback-config.ts @@ -0,0 +1,209 @@ +/** + * Fallback / provider-preference config validator — the offline probe. Takes a request + * body (the JSON a customer POSTs to /chat/completions) from a file or stdin and + * statically lints the routing configuration for the misconfigurations that produce the + * "overloaded-404", silent structured-output degrade, unnecessary ZDR narrowing, and + * no-fallback brittleness the field guide catalogs. Runs entirely offline — no API key, + * no network — so it can be pointed at a customer's config in a screen-share. + * + * Usage: + * npx tsx validate-fallback-config.ts ./request-body.json + * cat request-body.json | npx tsx validate-fallback-config.ts + * npx tsx validate-fallback-config.ts --json ./request-body.json # machine-readable findings + */ +import { readFileSync } from "node:fs"; +import { parseArgs } from "./lib.js"; + +type Severity = "error" | "warn" | "info"; +interface Finding { + severity: Severity; + field: string; + message: string; + fix: string; +} + +interface RequestBody { + model?: string; + models?: unknown; + stream?: boolean; + response_format?: { type?: string; json_schema?: { strict?: boolean } }; + provider?: { + order?: unknown; + only?: unknown; + ignore?: unknown; + allow_fallbacks?: boolean; + require_parameters?: boolean; + data_collection?: string; + zdr?: boolean; + max_price?: Record; + sort?: string; + }; + [k: string]: unknown; +} + +const args = parseArgs(process.argv.slice(2), ["json"]); +const jsonOut = args.has("json"); +const file = args.get("_0"); + +let rawText: string; +try { + rawText = file ? readFileSync(file, "utf8") : readFileSync(0, "utf8"); +} catch (err) { + console.error(`Could not read input: ${err instanceof Error ? err.message : String(err)}`); + console.error("Pass a file path or pipe JSON on stdin."); + process.exit(1); +} + +if (!rawText.trim()) { + console.error( + ` +Usage: npx tsx validate-fallback-config.ts + cat request-body.json | npx tsx validate-fallback-config.ts + +Statically lints an OpenRouter /chat/completions request body for routing / fallback / +provider-preference misconfigurations (offline — no API key required). +`.trim() + ); + process.exit(1); +} + +let body: RequestBody; +try { + body = JSON.parse(rawText) as RequestBody; +} catch (err) { + console.error(`Input is not valid JSON: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); +} + +const findings: Finding[] = []; +const p = body.provider ?? {}; + +// 1. No model fallback array at all -> single point of failure on provider overload. +const hasModelsArray = Array.isArray(body.models) && body.models.length > 0; +if (!hasModelsArray) { + findings.push({ + severity: "warn", + field: "models", + message: + "No `models` fallback array. A single provider brownout (429/503/529) fails the request instead of rerouting.", + fix: "Add a `models: [...]` array of acceptable alternates so OpenRouter reroutes on upstream failure.", + }); +} + +// 2. allow_fallbacks:false + a tight order/only -> the classic overloaded-404. +const orderList = asArray(p.order); +const onlyList = asArray(p.only); +if (p.allow_fallbacks === false) { + if (orderList.length > 0 || onlyList.length > 0) { + findings.push({ + severity: "error", + field: "provider.allow_fallbacks", + message: + "`allow_fallbacks:false` combined with a restricted `order`/`only` means if those exact providers are unavailable the request 404s ('No endpoints found') instead of falling back.", + fix: "Set `allow_fallbacks:true`, or widen `order`/`only`, unless hard provider pinning is a deliberate compliance requirement.", + }); + } else { + findings.push({ + severity: "warn", + field: "provider.allow_fallbacks", + message: "`allow_fallbacks:false` disables provider-level failover for this request.", + fix: "Confirm this is intentional; otherwise remove it to restore default failover.", + }); + } +} + +// 3. Strict json_schema without require_parameters -> silent degrade / 422 on fallback. +const wantsStrictSchema = + body.response_format?.type === "json_schema" && body.response_format?.json_schema?.strict === true; +if (wantsStrictSchema && p.require_parameters !== true) { + findings.push({ + severity: "error", + field: "provider.require_parameters", + message: + "Strict `json_schema` requested but `require_parameters` is not true. A fallback to a provider that ignores strict schema returns garbage (silent degrade) or 422.", + fix: "Set `provider.require_parameters:true` so only providers that honor the schema are eligible.", + }); +} + +// 4. ZDR / no-training narrowing -> union semantics can empty the provider set. +if (p.zdr === true || p.data_collection === "deny") { + findings.push({ + severity: "info", + field: p.zdr === true ? "provider.zdr" : "provider.data_collection", + message: + "Requiring ZDR / denying data collection is the UNION of OpenRouter + downstream retention policy — it can narrow the eligible provider set until none qualify (404).", + fix: "Preview impact with `GET /endpoints/zdr` and keep the model set wide; relax only if a 404 appears and compliance allows.", + }); +} + +// 5. max_price set with a tight order -> can strand routing. +if (p.max_price && Object.keys(p.max_price).length > 0 && orderList.length > 0 && p.allow_fallbacks === false) { + findings.push({ + severity: "warn", + field: "provider.max_price", + message: + "`max_price` plus a pinned `order` plus `allow_fallbacks:false` can leave zero eligible endpoints when the pinned providers exceed the price ceiling → 404.", + fix: "Loosen the price ceiling, widen `order`, or allow fallbacks.", + }); +} + +// 6. Streaming without a stated cut-off handling reminder. +if (body.stream === true) { + findings.push({ + severity: "info", + field: "stream", + message: + "Streaming is on. Mid-stream failures arrive as an SSE chunk with `finish_reason:\"error\"` under an HTTP 200 — easy to miss.", + fix: "Parse the final SSE chunk; treat `finish_reason:\"error\"` as a failure and retry/fallback.", + }); +} + +// 7. Contradictory only/ignore. +const ignoreList = asArray(p.ignore); +const overlap = onlyList.filter((x) => ignoreList.includes(x)); +if (overlap.length > 0) { + findings.push({ + severity: "error", + field: "provider.only/ignore", + message: `Provider(s) [${overlap.join(", ")}] appear in BOTH \`only\` and \`ignore\` — contradictory, they will be excluded.`, + fix: "Remove the overlap; a provider cannot be both required and ignored.", + }); +} + +// --- Output ------------------------------------------------------------------ +if (jsonOut) { + const counts = tally(findings); + console.log(JSON.stringify({ ok: counts.error === 0, counts, findings }, null, 2)); + process.exit(counts.error > 0 ? 2 : 0); +} + +if (findings.length === 0) { + console.log("No routing/fallback misconfigurations detected. Config looks healthy."); + process.exit(0); +} + +const order: Severity[] = ["error", "warn", "info"]; +const glyph: Record = { error: "✗", warn: "!", info: "·" }; +for (const sev of order) { + for (const f of findings.filter((x) => x.severity === sev)) { + console.log(`${glyph[sev]} [${sev}] ${f.field}`); + console.log(` ${f.message}`); + console.log(` fix: ${f.fix}`); + console.log(""); + } +} +const counts = tally(findings); +console.log(`Summary: ${counts.error} error, ${counts.warn} warn, ${counts.info} info.`); +process.exit(counts.error > 0 ? 2 : 0); + +function asArray(v: unknown): string[] { + return Array.isArray(v) ? v.map((x) => String(x)) : []; +} + +function tally(fs: Finding[]): { error: number; warn: number; info: number } { + return { + error: fs.filter((f) => f.severity === "error").length, + warn: fs.filter((f) => f.severity === "warn").length, + info: fs.filter((f) => f.severity === "info").length, + }; +} From b0f882e07e98e01da0ed8a988690b7872c067066 Mon Sep 17 00:00:00 2001 From: arc <224894192+arc0btc@users.noreply.github.com> Date: Fri, 3 Jul 2026 11:17:23 -0600 Subject: [PATCH 2/4] =?UTF-8?q?fix:=20address=20Perry=20review=20=E2=80=94?= =?UTF-8?q?=20null-input=20guard,=20--json=20exit=20code,=20live=20slugs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Applies the two non-blocking suggestions and both nits from Perry's review of PR #69: - validate-fallback-config.ts: guard non-object JSON (null / array / scalar) with a clean validation message + exit 1, instead of a TypeError on body.provider. The probe is meant to be pointed at a customer's arbitrary body in a screen-share, so it must fail gracefully. - smoke-test.ts: compute finish_reason before the --json branch and exit(failed ? 1 : 0) in BOTH output paths, so a scripted --json caller can no longer read success on an HTTP-200 + finish_reason:"error" generation — the exact silent-failure mode this skill teaches. Also defer the PASS/FAIL headline until after the finish-reason check so the first line matches the exit code. - references/fallback-and-zdr.md: refresh the illustrative fallback example off stale slugs (claude-3.5-sonnet, gemini-1.5-pro) to current live catalogue slugs (claude-sonnet-4.6, gemini-2.5-pro). Verified: validator offline against null/array/number/healthy/broken/empty-stdin cases (exit codes 1/1/1/0/2/1); smoke-test live against a scoped ephemeral key in both human and --json modes (PASS, exit 0), then key revoked; tsc --strict --module nodenext clean. --- .../references/fallback-and-zdr.md | 4 ++-- .../scripts/smoke-test.ts | 20 +++++++++++++------ .../scripts/validate-fallback-config.ts | 18 +++++++++++++++-- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/skills/openrouter-integration-doctor/references/fallback-and-zdr.md b/skills/openrouter-integration-doctor/references/fallback-and-zdr.md index 6be4108..654b9cb 100644 --- a/skills/openrouter-integration-doctor/references/fallback-and-zdr.md +++ b/skills/openrouter-integration-doctor/references/fallback-and-zdr.md @@ -16,8 +16,8 @@ Reliability on OpenRouter is architectural, not incidental. There are two layers ```json { - "model": "anthropic/claude-3.5-sonnet", - "models": ["openai/gpt-4o", "google/gemini-1.5-pro"], + "model": "anthropic/claude-sonnet-4.6", + "models": ["openai/gpt-4o", "google/gemini-2.5-pro"], "provider": { "allow_fallbacks": true } } ``` diff --git a/skills/openrouter-integration-doctor/scripts/smoke-test.ts b/skills/openrouter-integration-doctor/scripts/smoke-test.ts index ca6d690..847b588 100644 --- a/skills/openrouter-integration-doctor/scripts/smoke-test.ts +++ b/skills/openrouter-integration-doctor/scripts/smoke-test.ts @@ -49,15 +49,23 @@ const res = await fetchApi("/chat/completions", { }); const elapsed = Date.now() - started; +const content = res.choices?.[0]?.message?.content ?? ""; +const finish = res.choices?.[0]?.finish_reason ?? "n/a"; +// An HTTP 200 with finish_reason:"error" is an in-band failure — the exact silent-failure +// mode this skill teaches. Fail the exit code for it in BOTH output paths so a scripted +// `--json` caller can't read success on a failed generation. +const failed = finish === "error"; + if (args.has("json")) { console.log(JSON.stringify(res, null, 2)); - process.exit(0); + process.exit(failed ? 1 : 0); } -const content = res.choices?.[0]?.message?.content ?? ""; -const finish = res.choices?.[0]?.finish_reason ?? "n/a"; - -console.log("Smoke test: PASS — the integration is live end-to-end."); +console.log( + failed + ? "Smoke test: FAIL — the pipe opened but this generation failed mid-flight." + : "Smoke test: PASS — the integration is live end-to-end." +); console.log(""); console.log("Requested model:", model); console.log("Served model: ", res.model ?? "unknown"); @@ -77,7 +85,7 @@ console.log(""); if (res.id) { console.log(`Tip: inspect this call with npx tsx inspect-generation.ts ${res.id}`); } -if (finish === "error") { +if (failed) { console.log(""); console.log("! finish_reason == 'error': the pipe opened but this generation failed mid-flight."); console.log(" Treat as a failure and retry/fallback — see references/error-taxonomy.md."); diff --git a/skills/openrouter-integration-doctor/scripts/validate-fallback-config.ts b/skills/openrouter-integration-doctor/scripts/validate-fallback-config.ts index 1394849..ba2d614 100644 --- a/skills/openrouter-integration-doctor/scripts/validate-fallback-config.ts +++ b/skills/openrouter-integration-doctor/scripts/validate-fallback-config.ts @@ -67,14 +67,28 @@ provider-preference misconfigurations (offline — no API key required). process.exit(1); } -let body: RequestBody; +let parsed: unknown; try { - body = JSON.parse(rawText) as RequestBody; + parsed = JSON.parse(rawText); } catch (err) { console.error(`Input is not valid JSON: ${err instanceof Error ? err.message : String(err)}`); process.exit(1); } +// JSON.parse accepts `null`, arrays, numbers, strings, and booleans — none of which are a +// request body. Guard here so pointing the probe at arbitrary input in a screen-share +// yields a clean validation message instead of a `body.provider` TypeError. +if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) { + const kind = parsed === null ? "null" : Array.isArray(parsed) ? "an array" : `a ${typeof parsed}`; + console.error( + `Input is valid JSON but not a request body — expected a JSON object, got ${kind}.` + ); + console.error("Pass the object you POST to /chat/completions (the one with `model`, `messages`, `provider`, …)."); + process.exit(1); +} + +const body = parsed as RequestBody; + const findings: Finding[] = []; const p = body.provider ?? {}; From 886ea8303e9a3e9ef49676076d1a325825a711c8 Mon Sep 17 00:00:00 2001 From: arc <224894192+arc0btc@users.noreply.github.com> Date: Fri, 3 Jul 2026 11:31:34 -0600 Subject: [PATCH 3/4] fix: widen smoke-test success predicate to non-error 200 shapes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses Perry's incremental re-review finding on b0f882e (Codex, non-blocking): the smoke test only failed on finish_reason:"error", but the skill's thesis is "don't trust HTTP 200 alone" — so its own probe should be strict about the adjacent silent-success shapes too. - smoke-test.ts: `failed` now also trips on empty/absent `choices` and empty `message.content` (a 200 that generated nothing), in both the --json and human paths, so a scripted caller can't read success on them. The failure detail line names the actual shape (error / no-choices / empty-content). - smoke-test.ts: surface finish_reason:"length" as a non-failing truncation warning so a PASS with clipped output isn't mistaken for a clean one. Verified: tsc --strict --module nodenext clean; live PASS path re-run against a scoped ephemeral key in both human and --json modes (finish_reason:"stop", exit 0 — no false failure from the widened predicate), key revoked. --- .../scripts/smoke-test.ts | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/skills/openrouter-integration-doctor/scripts/smoke-test.ts b/skills/openrouter-integration-doctor/scripts/smoke-test.ts index 847b588..02768bc 100644 --- a/skills/openrouter-integration-doctor/scripts/smoke-test.ts +++ b/skills/openrouter-integration-doctor/scripts/smoke-test.ts @@ -51,10 +51,19 @@ const elapsed = Date.now() - started; const content = res.choices?.[0]?.message?.content ?? ""; const finish = res.choices?.[0]?.finish_reason ?? "n/a"; -// An HTTP 200 with finish_reason:"error" is an in-band failure — the exact silent-failure -// mode this skill teaches. Fail the exit code for it in BOTH output paths so a scripted -// `--json` caller can't read success on a failed generation. -const failed = finish === "error"; +// The skill's own thesis is "don't trust HTTP 200 alone." A 200 can still be a +// non-success in several in-band shapes, so the smoke test — of all things — must be +// strict about them and fail the exit code in BOTH output paths so a scripted `--json` +// caller can't read success on a failed generation: +// - finish_reason:"error" — the generation errored mid-flight. +// - empty/absent choices — 200 with nothing generated. +// - empty message.content — a choice came back but produced no text. +const noChoices = !res.choices || res.choices.length === 0; +const failed = finish === "error" || noChoices || content.length === 0; +// finish_reason:"length" is not a hard failure (the call worked) but the completion was +// truncated at max_tokens — surface it so a PASS with clipped output isn't mistaken for +// a clean one. +const truncated = finish === "length"; if (args.has("json")) { console.log(JSON.stringify(res, null, 2)); @@ -85,9 +94,20 @@ console.log(""); if (res.id) { console.log(`Tip: inspect this call with npx tsx inspect-generation.ts ${res.id}`); } +if (truncated && !failed) { + console.log(""); + console.log("! finish_reason == 'length': the call succeeded but the completion was truncated"); + console.log(" at max_tokens. Raise --max-tokens if you expected the full output."); +} if (failed) { + const reason = + finish === "error" + ? "finish_reason == 'error': the pipe opened but this generation failed mid-flight." + : noChoices + ? "HTTP 200 but no choices were returned — nothing was generated." + : "HTTP 200 but the completion is empty — the model returned no text."; console.log(""); - console.log("! finish_reason == 'error': the pipe opened but this generation failed mid-flight."); + console.log(`! ${reason}`); console.log(" Treat as a failure and retry/fallback — see references/error-taxonomy.md."); process.exit(1); } From c56c53ea3c42855e3c35a891f8f526a51b93ca3a Mon Sep 17 00:00:00 2001 From: arc <224894192+arc0btc@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:31:36 -0600 Subject: [PATCH 4/4] fix: derive smoke-test FAIL headline from the tripped condition The failure predicate now fires on three 200-but-not-success shapes (finish_reason:"error", empty/absent choices, empty message.content), but the human-output headline was fixed at the mid-flight phrasing that only fits the error case. Derive the headline from the same condition the predicate keyed on so the first line matches both the exit and the per-shape detail already printed lower down. --- .../scripts/smoke-test.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/skills/openrouter-integration-doctor/scripts/smoke-test.ts b/skills/openrouter-integration-doctor/scripts/smoke-test.ts index 02768bc..c7ddc4e 100644 --- a/skills/openrouter-integration-doctor/scripts/smoke-test.ts +++ b/skills/openrouter-integration-doctor/scripts/smoke-test.ts @@ -65,16 +65,22 @@ const failed = finish === "error" || noChoices || content.length === 0; // a clean one. const truncated = finish === "length"; +// Derive the FAIL headline from the specific shape that tripped `failed`, so the first +// line names the same cause the predicate keyed on (and the detail line below) instead of +// always describing the finish_reason:"error" case. +const failHeadline = + finish === "error" + ? "Smoke test: FAIL — the pipe opened but this generation failed mid-flight." + : noChoices + ? "Smoke test: FAIL — HTTP 200 but no choices were returned; nothing was generated." + : "Smoke test: FAIL — HTTP 200 but the completion is empty; the model returned no text."; + if (args.has("json")) { console.log(JSON.stringify(res, null, 2)); process.exit(failed ? 1 : 0); } -console.log( - failed - ? "Smoke test: FAIL — the pipe opened but this generation failed mid-flight." - : "Smoke test: PASS — the integration is live end-to-end." -); +console.log(failed ? failHeadline : "Smoke test: PASS — the integration is live end-to-end."); console.log(""); console.log("Requested model:", model); console.log("Served model: ", res.model ?? "unknown");