diff --git a/e2e/README.md b/e2e/README.md index c39d3af31..88820c3c3 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -39,8 +39,9 @@ comes from the deployment URL returned by `vc deploy --prebuilt`. One-time project setup: - Configure the shared Vercel project for Node.js 24. -- Provide the model-provider credentials the fixtures need (the agents and - judges run against `openai/gpt-5.5`) in the project's Preview environment. +- Provide the model-provider credentials the fixtures need (the gateway-backed + agents and judges run against `openai/gpt-5.6-sol`) in the project's Preview + environment. - Provide `VERCEL_TOKEN`, `VERCEL_ORG_ID`, and `VERCEL_PROJECT_ID` in CI. Run a fixture against Vercel from its directory: @@ -80,17 +81,16 @@ must run against the alias — the `e2e-vercel` workflow sets evals as a second `eve eval` invocation after the main suite. Without the alias env (local matrix, plain `eve eval --strict`) the eval skips. -Most fixture agents run against `anthropic/claude-sonnet-5` and judges run -against `openai/gpt-5.5`. Both are real models, so the environment must provide -the corresponding model-provider credentials. `agent-prompt-cache` is the one -fixture that authors a direct `@ai-sdk/anthropic` model instance instead of a -gateway model id: its eval asserts the harness's Anthropic cache-breakpoint -placement, which only runs on that path. The instance points at the AI -Gateway's Anthropic-compatible Messages endpoint so it uses the same -`AI_GATEWAY_API_KEY` credential as every other fixture. `agent-workflow-stress` uses eve's -`mockModel` fixture helper so its 100-turn runs stay fast and deterministic. Its -concurrent and sequential evals cover high-volume session execution and repeated -session resumption respectively. +Gateway-backed fixture agents and judges run against `openai/gpt-5.6-sol`. +`agent-prompt-cache` is the one fixture that authors a direct +`@ai-sdk/anthropic` model instance instead of a gateway model id: its eval +asserts the harness's Anthropic cache-breakpoint placement, which only runs on +that path. The instance points at the AI Gateway's Anthropic-compatible +Messages endpoint so it uses the same `AI_GATEWAY_API_KEY` credential as every +other fixture. `agent-workflow-stress` uses eve's `mockModel` fixture helper so +its 100-turn runs stay fast and deterministic. Its concurrent and sequential +evals cover high-volume session execution and repeated session resumption +respectively. ## Fixtures diff --git a/e2e/fixtures/agent-basic-runtime/agent/agent.ts b/e2e/fixtures/agent-basic-runtime/agent/agent.ts index bea278ea3..7c4adca2c 100644 --- a/e2e/fixtures/agent-basic-runtime/agent/agent.ts +++ b/e2e/fixtures/agent-basic-runtime/agent/agent.ts @@ -1,6 +1,6 @@ import { defineAgent } from "eve"; export default defineAgent({ - model: "openai/gpt-5.5", + model: "openai/gpt-5.6-sol", reasoning: "high", }); diff --git a/e2e/fixtures/agent-basic-runtime/evals/evals.config.ts b/e2e/fixtures/agent-basic-runtime/evals/evals.config.ts index 6864dd176..912e6ec89 100644 --- a/e2e/fixtures/agent-basic-runtime/evals/evals.config.ts +++ b/e2e/fixtures/agent-basic-runtime/evals/evals.config.ts @@ -2,5 +2,5 @@ import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: "openai/gpt-5.5" }, + judge: { model: "openai/gpt-5.6-sol" }, }); diff --git a/e2e/fixtures/agent-channels/agent/agent.ts b/e2e/fixtures/agent-channels/agent/agent.ts index dc6d957f5..7c4adca2c 100644 --- a/e2e/fixtures/agent-channels/agent/agent.ts +++ b/e2e/fixtures/agent-channels/agent/agent.ts @@ -1,6 +1,6 @@ import { defineAgent } from "eve"; export default defineAgent({ - model: "anthropic/claude-sonnet-5", + model: "openai/gpt-5.6-sol", reasoning: "high", }); diff --git a/e2e/fixtures/agent-channels/evals/evals.config.ts b/e2e/fixtures/agent-channels/evals/evals.config.ts index 6864dd176..912e6ec89 100644 --- a/e2e/fixtures/agent-channels/evals/evals.config.ts +++ b/e2e/fixtures/agent-channels/evals/evals.config.ts @@ -2,5 +2,5 @@ import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: "openai/gpt-5.5" }, + judge: { model: "openai/gpt-5.6-sol" }, }); diff --git a/e2e/fixtures/agent-model/agent/agent.ts b/e2e/fixtures/agent-model/agent/agent.ts index 7400bd22a..fa66a26bc 100644 --- a/e2e/fixtures/agent-model/agent/agent.ts +++ b/e2e/fixtures/agent-model/agent/agent.ts @@ -7,7 +7,7 @@ import { defineAgent, defineDynamic, type DynamicResolveContext } from "eve"; */ export default defineAgent({ model: defineDynamic({ - fallback: "openai/gpt-5.5", + fallback: "openai/gpt-5.6-sol", events: { "turn.started": (_event, ctx) => { const text = lastUserText(ctx.messages); @@ -18,7 +18,7 @@ export default defineAgent({ if (text.includes("[model: mini]")) { return { - model: "openai/gpt-5.5-mini", + model: "openai/gpt-5.6-sol", modelContextWindowTokens: 128_000, }; } diff --git a/e2e/fixtures/agent-model/evals/dynamic-model/fallback.eval.ts b/e2e/fixtures/agent-model/evals/dynamic-model/fallback.eval.ts index 09a44c505..2644c8596 100644 --- a/e2e/fixtures/agent-model/evals/dynamic-model/fallback.eval.ts +++ b/e2e/fixtures/agent-model/evals/dynamic-model/fallback.eval.ts @@ -16,7 +16,7 @@ export default defineEval({ events.some( (event) => event.type === "session.started" && - event.data.runtime?.modelId === "dynamic:openai/gpt-5.5", + event.data.runtime?.modelId === "dynamic:openai/gpt-5.6-sol", ), ); }, diff --git a/e2e/fixtures/agent-model/evals/dynamic-model/turn-selection.eval.ts b/e2e/fixtures/agent-model/evals/dynamic-model/turn-selection.eval.ts index acd09bada..52b43b9b1 100644 --- a/e2e/fixtures/agent-model/evals/dynamic-model/turn-selection.eval.ts +++ b/e2e/fixtures/agent-model/evals/dynamic-model/turn-selection.eval.ts @@ -1,8 +1,8 @@ import { defineEval } from "eve/evals"; /** - * A marked turn selects `openai/gpt-5.5-mini`; the next unmarked turn falls - * back. Both completing proves each reference serves a real model call. + * A marked turn resolves an explicit per-turn model configuration; the next + * unmarked turn falls back. Both paths use `openai/gpt-5.6-sol`. */ export default defineEval({ description: "Dynamic model smoke: per-turn selection and null fallback in one session.", diff --git a/e2e/fixtures/agent-model/evals/evals.config.ts b/e2e/fixtures/agent-model/evals/evals.config.ts index 6864dd176..912e6ec89 100644 --- a/e2e/fixtures/agent-model/evals/evals.config.ts +++ b/e2e/fixtures/agent-model/evals/evals.config.ts @@ -2,5 +2,5 @@ import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: "openai/gpt-5.5" }, + judge: { model: "openai/gpt-5.6-sol" }, }); diff --git a/e2e/fixtures/agent-openapi-swagger/agent/agent.ts b/e2e/fixtures/agent-openapi-swagger/agent/agent.ts index dc6d957f5..7c4adca2c 100644 --- a/e2e/fixtures/agent-openapi-swagger/agent/agent.ts +++ b/e2e/fixtures/agent-openapi-swagger/agent/agent.ts @@ -1,6 +1,6 @@ import { defineAgent } from "eve"; export default defineAgent({ - model: "anthropic/claude-sonnet-5", + model: "openai/gpt-5.6-sol", reasoning: "high", }); diff --git a/e2e/fixtures/agent-openapi-swagger/evals/evals.config.ts b/e2e/fixtures/agent-openapi-swagger/evals/evals.config.ts index 6864dd176..912e6ec89 100644 --- a/e2e/fixtures/agent-openapi-swagger/evals/evals.config.ts +++ b/e2e/fixtures/agent-openapi-swagger/evals/evals.config.ts @@ -2,5 +2,5 @@ import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: "openai/gpt-5.5" }, + judge: { model: "openai/gpt-5.6-sol" }, }); diff --git a/e2e/fixtures/agent-prompt-cache/evals/evals.config.ts b/e2e/fixtures/agent-prompt-cache/evals/evals.config.ts index e9b6c4d05..e05d9f344 100644 --- a/e2e/fixtures/agent-prompt-cache/evals/evals.config.ts +++ b/e2e/fixtures/agent-prompt-cache/evals/evals.config.ts @@ -2,5 +2,5 @@ import { defineEvalConfig } from "eve/evals"; /** Judge is unused here; assertions are numeric. Kept for config parity. */ export default defineEvalConfig({ - judge: { model: "openai/gpt-5.5" }, + judge: { model: "openai/gpt-5.6-sol" }, }); diff --git a/e2e/fixtures/agent-schedules/agent/agent.ts b/e2e/fixtures/agent-schedules/agent/agent.ts index dc6d957f5..7c4adca2c 100644 --- a/e2e/fixtures/agent-schedules/agent/agent.ts +++ b/e2e/fixtures/agent-schedules/agent/agent.ts @@ -1,6 +1,6 @@ import { defineAgent } from "eve"; export default defineAgent({ - model: "anthropic/claude-sonnet-5", + model: "openai/gpt-5.6-sol", reasoning: "high", }); diff --git a/e2e/fixtures/agent-schedules/evals/evals.config.ts b/e2e/fixtures/agent-schedules/evals/evals.config.ts index 6864dd176..912e6ec89 100644 --- a/e2e/fixtures/agent-schedules/evals/evals.config.ts +++ b/e2e/fixtures/agent-schedules/evals/evals.config.ts @@ -2,5 +2,5 @@ import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: "openai/gpt-5.5" }, + judge: { model: "openai/gpt-5.6-sol" }, }); diff --git a/e2e/fixtures/agent-session-limits/agent/agent.ts b/e2e/fixtures/agent-session-limits/agent/agent.ts index 1270e5255..c8b87b74d 100644 --- a/e2e/fixtures/agent-session-limits/agent/agent.ts +++ b/e2e/fixtures/agent-session-limits/agent/agent.ts @@ -1,7 +1,7 @@ import { defineAgent } from "eve"; export default defineAgent({ - model: "anthropic/claude-sonnet-5", + model: "openai/gpt-5.6-sol", // A one-token input budget guarantees every completed model call crosses // the limit, so the next conversation turn deterministically parks on the // session-limit continuation prompt. diff --git a/e2e/fixtures/agent-session-limits/evals/evals.config.ts b/e2e/fixtures/agent-session-limits/evals/evals.config.ts index 6864dd176..912e6ec89 100644 --- a/e2e/fixtures/agent-session-limits/evals/evals.config.ts +++ b/e2e/fixtures/agent-session-limits/evals/evals.config.ts @@ -2,5 +2,5 @@ import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: "openai/gpt-5.5" }, + judge: { model: "openai/gpt-5.6-sol" }, }); diff --git a/e2e/fixtures/agent-skills/agent/agent.ts b/e2e/fixtures/agent-skills/agent/agent.ts index dc6d957f5..7c4adca2c 100644 --- a/e2e/fixtures/agent-skills/agent/agent.ts +++ b/e2e/fixtures/agent-skills/agent/agent.ts @@ -1,6 +1,6 @@ import { defineAgent } from "eve"; export default defineAgent({ - model: "anthropic/claude-sonnet-5", + model: "openai/gpt-5.6-sol", reasoning: "high", }); diff --git a/e2e/fixtures/agent-skills/evals/evals.config.ts b/e2e/fixtures/agent-skills/evals/evals.config.ts index 6864dd176..912e6ec89 100644 --- a/e2e/fixtures/agent-skills/evals/evals.config.ts +++ b/e2e/fixtures/agent-skills/evals/evals.config.ts @@ -2,5 +2,5 @@ import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: "openai/gpt-5.5" }, + judge: { model: "openai/gpt-5.6-sol" }, }); diff --git a/e2e/fixtures/agent-subagents-hitl/agent/agent.ts b/e2e/fixtures/agent-subagents-hitl/agent/agent.ts index dc6d957f5..7c4adca2c 100644 --- a/e2e/fixtures/agent-subagents-hitl/agent/agent.ts +++ b/e2e/fixtures/agent-subagents-hitl/agent/agent.ts @@ -1,6 +1,6 @@ import { defineAgent } from "eve"; export default defineAgent({ - model: "anthropic/claude-sonnet-5", + model: "openai/gpt-5.6-sol", reasoning: "high", }); diff --git a/e2e/fixtures/agent-subagents-hitl/agent/subagents/stock-price/agent.ts b/e2e/fixtures/agent-subagents-hitl/agent/subagents/stock-price/agent.ts index 10007be3a..0a1a56d1f 100644 --- a/e2e/fixtures/agent-subagents-hitl/agent/subagents/stock-price/agent.ts +++ b/e2e/fixtures/agent-subagents-hitl/agent/subagents/stock-price/agent.ts @@ -3,6 +3,6 @@ import { defineAgent } from "eve"; export default defineAgent({ description: 'Look up the current stock price for a given ticker symbol. Pass the ticker symbol you want to look up in the message (e.g. "AAPL", "GOOG", or "TSLA").', - model: "anthropic/claude-sonnet-5", + model: "openai/gpt-5.6-sol", reasoning: "high", }); diff --git a/e2e/fixtures/agent-subagents-hitl/evals/evals.config.ts b/e2e/fixtures/agent-subagents-hitl/evals/evals.config.ts index 6864dd176..912e6ec89 100644 --- a/e2e/fixtures/agent-subagents-hitl/evals/evals.config.ts +++ b/e2e/fixtures/agent-subagents-hitl/evals/evals.config.ts @@ -2,5 +2,5 @@ import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: "openai/gpt-5.5" }, + judge: { model: "openai/gpt-5.6-sol" }, }); diff --git a/e2e/fixtures/agent-subagents/agent/agent.ts b/e2e/fixtures/agent-subagents/agent/agent.ts index dc6d957f5..7c4adca2c 100644 --- a/e2e/fixtures/agent-subagents/agent/agent.ts +++ b/e2e/fixtures/agent-subagents/agent/agent.ts @@ -1,6 +1,6 @@ import { defineAgent } from "eve"; export default defineAgent({ - model: "anthropic/claude-sonnet-5", + model: "openai/gpt-5.6-sol", reasoning: "high", }); diff --git a/e2e/fixtures/agent-subagents/agent/subagents/echo-marker/agent.ts b/e2e/fixtures/agent-subagents/agent/subagents/echo-marker/agent.ts index 1e5c462e5..3084a65fb 100644 --- a/e2e/fixtures/agent-subagents/agent/subagents/echo-marker/agent.ts +++ b/e2e/fixtures/agent-subagents/agent/subagents/echo-marker/agent.ts @@ -12,6 +12,6 @@ import { defineAgent } from "eve"; export default defineAgent({ description: "Smoke-test echo subagent. Call this whenever the user mentions the phrase 'echo marker subagent'. Pass any short text as the input; the subagent replies with a fixed marker token.", - model: "anthropic/claude-sonnet-5", + model: "openai/gpt-5.6-sol", reasoning: "high", }); diff --git a/e2e/fixtures/agent-subagents/evals/evals.config.ts b/e2e/fixtures/agent-subagents/evals/evals.config.ts index 6864dd176..912e6ec89 100644 --- a/e2e/fixtures/agent-subagents/evals/evals.config.ts +++ b/e2e/fixtures/agent-subagents/evals/evals.config.ts @@ -2,5 +2,5 @@ import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: "openai/gpt-5.5" }, + judge: { model: "openai/gpt-5.6-sol" }, }); diff --git a/e2e/fixtures/agent-tools-hitl-openai/agent/agent.ts b/e2e/fixtures/agent-tools-hitl-openai/agent/agent.ts index a780c41c4..99a9c1f72 100644 --- a/e2e/fixtures/agent-tools-hitl-openai/agent/agent.ts +++ b/e2e/fixtures/agent-tools-hitl-openai/agent/agent.ts @@ -7,5 +7,5 @@ import { defineAgent } from "eve"; * validated by OpenAI's `function_call` / `function_call_output` pairing. */ export default defineAgent({ - model: "openai/gpt-5.5", + model: "openai/gpt-5.6-sol", }); diff --git a/e2e/fixtures/agent-tools-hitl-openai/evals/evals.config.ts b/e2e/fixtures/agent-tools-hitl-openai/evals/evals.config.ts index 6864dd176..912e6ec89 100644 --- a/e2e/fixtures/agent-tools-hitl-openai/evals/evals.config.ts +++ b/e2e/fixtures/agent-tools-hitl-openai/evals/evals.config.ts @@ -2,5 +2,5 @@ import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: "openai/gpt-5.5" }, + judge: { model: "openai/gpt-5.6-sol" }, }); diff --git a/e2e/fixtures/agent-tools-hitl/agent/agent.ts b/e2e/fixtures/agent-tools-hitl/agent/agent.ts index dc6d957f5..7c4adca2c 100644 --- a/e2e/fixtures/agent-tools-hitl/agent/agent.ts +++ b/e2e/fixtures/agent-tools-hitl/agent/agent.ts @@ -1,6 +1,6 @@ import { defineAgent } from "eve"; export default defineAgent({ - model: "anthropic/claude-sonnet-5", + model: "openai/gpt-5.6-sol", reasoning: "high", }); diff --git a/e2e/fixtures/agent-tools-hitl/evals/evals.config.ts b/e2e/fixtures/agent-tools-hitl/evals/evals.config.ts index 6864dd176..912e6ec89 100644 --- a/e2e/fixtures/agent-tools-hitl/evals/evals.config.ts +++ b/e2e/fixtures/agent-tools-hitl/evals/evals.config.ts @@ -2,5 +2,5 @@ import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: "openai/gpt-5.5" }, + judge: { model: "openai/gpt-5.6-sol" }, }); diff --git a/e2e/fixtures/agent-tools-sandbox/agent/agent.ts b/e2e/fixtures/agent-tools-sandbox/agent/agent.ts index dc6d957f5..7c4adca2c 100644 --- a/e2e/fixtures/agent-tools-sandbox/agent/agent.ts +++ b/e2e/fixtures/agent-tools-sandbox/agent/agent.ts @@ -1,6 +1,6 @@ import { defineAgent } from "eve"; export default defineAgent({ - model: "anthropic/claude-sonnet-5", + model: "openai/gpt-5.6-sol", reasoning: "high", }); diff --git a/e2e/fixtures/agent-tools-sandbox/evals/evals.config.ts b/e2e/fixtures/agent-tools-sandbox/evals/evals.config.ts index 6864dd176..912e6ec89 100644 --- a/e2e/fixtures/agent-tools-sandbox/evals/evals.config.ts +++ b/e2e/fixtures/agent-tools-sandbox/evals/evals.config.ts @@ -2,5 +2,5 @@ import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: "openai/gpt-5.5" }, + judge: { model: "openai/gpt-5.6-sol" }, }); diff --git a/e2e/fixtures/agent-tools/agent/agent.ts b/e2e/fixtures/agent-tools/agent/agent.ts index dc6d957f5..7c4adca2c 100644 --- a/e2e/fixtures/agent-tools/agent/agent.ts +++ b/e2e/fixtures/agent-tools/agent/agent.ts @@ -1,6 +1,6 @@ import { defineAgent } from "eve"; export default defineAgent({ - model: "anthropic/claude-sonnet-5", + model: "openai/gpt-5.6-sol", reasoning: "high", }); diff --git a/e2e/fixtures/agent-tools/evals/evals.config.ts b/e2e/fixtures/agent-tools/evals/evals.config.ts index 6864dd176..912e6ec89 100644 --- a/e2e/fixtures/agent-tools/evals/evals.config.ts +++ b/e2e/fixtures/agent-tools/evals/evals.config.ts @@ -2,5 +2,5 @@ import { defineEvalConfig } from "eve/evals"; /** Default judge model for any `t.judge.*` assertion in this fixture. */ export default defineEvalConfig({ - judge: { model: "openai/gpt-5.5" }, + judge: { model: "openai/gpt-5.6-sol" }, }); diff --git a/e2e/fixtures/extensions/agent/agent.ts b/e2e/fixtures/extensions/agent/agent.ts index dc6d957f5..7c4adca2c 100644 --- a/e2e/fixtures/extensions/agent/agent.ts +++ b/e2e/fixtures/extensions/agent/agent.ts @@ -1,6 +1,6 @@ import { defineAgent } from "eve"; export default defineAgent({ - model: "anthropic/claude-sonnet-5", + model: "openai/gpt-5.6-sol", reasoning: "high", }); diff --git a/e2e/fixtures/extensions/evals/evals.config.ts b/e2e/fixtures/extensions/evals/evals.config.ts index 8102d33af..292229fa1 100644 --- a/e2e/fixtures/extensions/evals/evals.config.ts +++ b/e2e/fixtures/extensions/evals/evals.config.ts @@ -1,5 +1,5 @@ import { defineEvalConfig } from "eve/evals"; export default defineEvalConfig({ - judge: { model: "openai/gpt-5.5" }, + judge: { model: "openai/gpt-5.6-sol" }, });