From 9991e074afd51478268e346bc7ac882ec5b7af84 Mon Sep 17 00:00:00 2001 From: "kenny.rogers" Date: Wed, 15 Jul 2026 05:52:44 +0000 Subject: [PATCH] test(openrouter-images): add waza eval suite for image generation skill Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- evals/openrouter-images/README.md | 44 ++++++++++++++++ evals/openrouter-images/eval-trigger.yaml | 24 +++++++++ evals/openrouter-images/eval.yaml | 47 ++++++++++++++++++ evals/openrouter-images/fixtures/avatar.jpg | Bin 0 -> 664 bytes evals/openrouter-images/fixtures/photo.png | Bin 0 -> 140 bytes evals/openrouter-images/fixtures/sprite.webp | Bin 0 -> 88 bytes .../tasks/discovery/list-models.yaml | 24 +++++++++ .../tasks/discovery/model-params.yaml | 24 +++++++++ .../tasks/discovery/provider-passthrough.yaml | 25 ++++++++++ .../tasks/edge-cases/cost-reporting.yaml | 25 ++++++++++ .../tasks/edge-cases/invalid-model.yaml | 23 +++++++++ .../edge-cases/missing-source-image.yaml | 24 +++++++++ .../tasks/edge-cases/unsupported-format.yaml | 24 +++++++++ .../edge-cases/unsupported-parameter.yaml | 24 +++++++++ .../tasks/editing/basic-edit.yaml | 29 +++++++++++ .../tasks/editing/edit-model-capability.yaml | 27 ++++++++++ .../tasks/editing/edit-with-output.yaml | 23 +++++++++ .../tasks/editing/generate-then-edit.yaml | 27 ++++++++++ .../tasks/generation/aspect-ratio.yaml | 24 +++++++++ .../tasks/generation/basic-generate.yaml | 29 +++++++++++ .../tasks/generation/model-override.yaml | 23 +++++++++ .../tasks/generation/multiple-images.yaml | 25 ++++++++++ .../tasks/generation/output-path.yaml | 21 ++++++++ .../generation/seed-reproducibility.yaml | 23 +++++++++ .../generation/transparent-background.yaml | 24 +++++++++ .../trigger/should-not-trigger-text.yaml | 19 +++++++ .../trigger/should-not-trigger-video.yaml | 16 ++++++ .../trigger/should-not-trigger-vision.yaml | 21 ++++++++ .../tasks/trigger/should-trigger-edit.yaml | 20 ++++++++ .../trigger/should-trigger-generate.yaml | 18 +++++++ .../trigger/should-trigger-indirect.yaml | 18 +++++++ 31 files changed, 695 insertions(+) create mode 100644 evals/openrouter-images/README.md create mode 100644 evals/openrouter-images/eval-trigger.yaml create mode 100644 evals/openrouter-images/eval.yaml create mode 100644 evals/openrouter-images/fixtures/avatar.jpg create mode 100644 evals/openrouter-images/fixtures/photo.png create mode 100644 evals/openrouter-images/fixtures/sprite.webp create mode 100644 evals/openrouter-images/tasks/discovery/list-models.yaml create mode 100644 evals/openrouter-images/tasks/discovery/model-params.yaml create mode 100644 evals/openrouter-images/tasks/discovery/provider-passthrough.yaml create mode 100644 evals/openrouter-images/tasks/edge-cases/cost-reporting.yaml create mode 100644 evals/openrouter-images/tasks/edge-cases/invalid-model.yaml create mode 100644 evals/openrouter-images/tasks/edge-cases/missing-source-image.yaml create mode 100644 evals/openrouter-images/tasks/edge-cases/unsupported-format.yaml create mode 100644 evals/openrouter-images/tasks/edge-cases/unsupported-parameter.yaml create mode 100644 evals/openrouter-images/tasks/editing/basic-edit.yaml create mode 100644 evals/openrouter-images/tasks/editing/edit-model-capability.yaml create mode 100644 evals/openrouter-images/tasks/editing/edit-with-output.yaml create mode 100644 evals/openrouter-images/tasks/editing/generate-then-edit.yaml create mode 100644 evals/openrouter-images/tasks/generation/aspect-ratio.yaml create mode 100644 evals/openrouter-images/tasks/generation/basic-generate.yaml create mode 100644 evals/openrouter-images/tasks/generation/model-override.yaml create mode 100644 evals/openrouter-images/tasks/generation/multiple-images.yaml create mode 100644 evals/openrouter-images/tasks/generation/output-path.yaml create mode 100644 evals/openrouter-images/tasks/generation/seed-reproducibility.yaml create mode 100644 evals/openrouter-images/tasks/generation/transparent-background.yaml create mode 100644 evals/openrouter-images/tasks/trigger/should-not-trigger-text.yaml create mode 100644 evals/openrouter-images/tasks/trigger/should-not-trigger-video.yaml create mode 100644 evals/openrouter-images/tasks/trigger/should-not-trigger-vision.yaml create mode 100644 evals/openrouter-images/tasks/trigger/should-trigger-edit.yaml create mode 100644 evals/openrouter-images/tasks/trigger/should-trigger-generate.yaml create mode 100644 evals/openrouter-images/tasks/trigger/should-trigger-indirect.yaml diff --git a/evals/openrouter-images/README.md b/evals/openrouter-images/README.md new file mode 100644 index 0000000..cbe0915 --- /dev/null +++ b/evals/openrouter-images/README.md @@ -0,0 +1,44 @@ +# openrouter-images Eval Suite + +Waza evaluation suite for the [`openrouter-images`](../../skills/openrouter-images) skill, following the [waza eval spec guide](https://microsoft.github.io/waza/guides/eval-yaml/). + +## Structure + +``` +evals/openrouter-images/ +├── eval.yaml # Capability suite (skill body injected) +├── eval-trigger.yaml # Trigger-precision suite (inject_skill_body: false) +├── tasks/ +│ ├── discovery/ # discover.ts: list models, per-model params, passthrough options +│ ├── generation/ # generate.ts: basic, aspect ratio, model override, output path, +│ │ # multiple images, transparency, seeds +│ ├── editing/ # edit.ts: basic edit, output path, capability check, generate→edit multi-turn +│ ├── edge-cases/ # unsupported params/formats, missing files, invalid models, cost reporting +│ └── trigger/ # should/should-not invoke the skill +└── fixtures/ # small sample images copied into each task workspace +``` + +## What it measures + +- **Discovery-first behavior** — the skill instructs agents to check `supported_parameters` via `discover.ts` before sending flags; several tasks verify this instead of blind flag guessing. +- **Correct script selection** — generate vs edit vs discover (enforced by the `correct_script_usage` LLM judge in `eval.yaml`). +- **Option translation** — natural-language intent ("wide", "3 variations", "transparent", "reproducible") mapped to the right CLI flags. +- **Error handling** — missing source files, unsupported formats/parameters, nonexistent models. +- **Presentation** — reporting saved paths, model used, and generation cost per the skill's guidance. +- **Trigger precision** — `eval-trigger.yaml` runs with `inject_skill_body: false` so positive tasks check the skill fires on image requests and negative tasks check it stays quiet for vision analysis, video, and text-only requests. Basic generate/edit tasks are marked `golden` for use with `waza gate`. + +## Running + +Requires `OPENROUTER_API_KEY` in the environment (generation/editing tasks make real API calls; discovery is keyless). + +```bash +waza run evals/openrouter-images/eval.yaml --context-dir evals/openrouter-images/fixtures -o results.json +waza run evals/openrouter-images/eval-trigger.yaml -o trigger-results.json + +# Filter subsets +waza run evals/openrouter-images/eval.yaml --tags happy-path +waza run evals/openrouter-images/eval.yaml --task "edge*" + +# Enforce golden tasks in CI +waza gate results.json +``` diff --git a/evals/openrouter-images/eval-trigger.yaml b/evals/openrouter-images/eval-trigger.yaml new file mode 100644 index 0000000..60877c8 --- /dev/null +++ b/evals/openrouter-images/eval-trigger.yaml @@ -0,0 +1,24 @@ +name: openrouter-images-trigger +description: > + Trigger-precision suite for the openrouter-images skill. Measures whether the + agent invokes the skill when it should (image generation/editing requests) and + leaves it alone when it should not (text, video, audio, or vision-analysis requests). +skill: openrouter-images +schemaVersion: "1.0" +version: "1.0" + +config: + trials_per_task: 1 + timeout_seconds: 120 + parallel: false + executor: copilot-sdk + model: claude-sonnet-4.6 + inject_skill_body: false + +metrics: + - name: trigger_precision + weight: 1.0 + threshold: 0.8 + +tasks: + - "tasks/trigger/*.yaml" diff --git a/evals/openrouter-images/eval.yaml b/evals/openrouter-images/eval.yaml new file mode 100644 index 0000000..80cd935 --- /dev/null +++ b/evals/openrouter-images/eval.yaml @@ -0,0 +1,47 @@ +name: openrouter-images-eval +description: > + Evaluation suite for the openrouter-images skill. Tests capability discovery, + image generation, image editing, option handling, error recovery, and result + presentation using the dedicated Image API scripts (discover.ts, generate.ts, edit.ts). +skill: openrouter-images +schemaVersion: "1.0" +version: "1.0" + +config: + trials_per_task: 1 + timeout_seconds: 300 + parallel: false + executor: copilot-sdk + model: claude-sonnet-4.6 + judge_model: gpt-4o + inject_skill_body: true + +metrics: + - name: accuracy + weight: 1.0 + threshold: 0.8 + +graders: + - type: prompt + name: correct_script_usage + config: + prompt: | + You are grading an agent that has access to the openrouter-images skill, + which provides three scripts: + - discover.ts: lists image models and their supported parameters + - generate.ts "prompt" [flags]: generates an image from text + - edit.ts "prompt" [flags]: edits an existing image + + Call set_waza_grade_pass if the agent: + 1. Picked the script that matches the user's request (discover vs generate vs edit), + 2. Only passed flags that are plausibly supported by the target model + (or verified support with discover.ts first), + 3. Did not hand-roll raw HTTP calls to the OpenRouter API when the + skill scripts could do the job. + Otherwise call set_waza_grade_fail with a short reason. + +tasks: + - "tasks/discovery/*.yaml" + - "tasks/generation/*.yaml" + - "tasks/editing/*.yaml" + - "tasks/edge-cases/*.yaml" diff --git a/evals/openrouter-images/fixtures/avatar.jpg b/evals/openrouter-images/fixtures/avatar.jpg new file mode 100644 index 0000000000000000000000000000000000000000..030c981fc098daa2fadb07c0bdd42a65f55b7637 GIT binary patch literal 664 zcmex=^(PF6}rMnOeST|r4lSw=>~TvNxu(8R<c1}I=;VrF4wW9Q)H;sz?% zD!{d!pzFb!U9xX3zTPI5o8roG<0MW4oqZMDikqloVbuf*=gfJ(V&YTRE(2~ znmD<{#3dx9RMpfqG__1j&CD$#!~I2F+J|fxEb7u=z=N6p-vj{MMA_c} literal 0 HcmV?d00001 diff --git a/evals/openrouter-images/fixtures/photo.png b/evals/openrouter-images/fixtures/photo.png new file mode 100644 index 0000000000000000000000000000000000000000..dbe2b4f07f1abcfca61d10e1b694e96e89f4c4b6 GIT binary patch literal 140 zcmeAS@N?(olHy`uVBq!ia0vp^4nSFSZ_hjO9#CL7;NT#0 x=t9eZ&Ig5&mx3>xt9|!l^HSzG(*>l!+9iFAcV1Xo9tRrF;OXk;vd$@?2>>!DGF$)v literal 0 HcmV?d00001 diff --git a/evals/openrouter-images/fixtures/sprite.webp b/evals/openrouter-images/fixtures/sprite.webp new file mode 100644 index 0000000000000000000000000000000000000000..30bd36254fcb138f5a97d33a6df9eab644034ac9 GIT binary patch literal 88 zcmWIYbaM+}U|9`QGuE2AAc literal 0 HcmV?d00001 diff --git a/evals/openrouter-images/tasks/discovery/list-models.yaml b/evals/openrouter-images/tasks/discovery/list-models.yaml new file mode 100644 index 0000000..568b62a --- /dev/null +++ b/evals/openrouter-images/tasks/discovery/list-models.yaml @@ -0,0 +1,24 @@ +id: discovery-001 +name: List available image models +description: > + User asks which image models are available. The agent should run discover.ts + (no arguments) and summarize model IDs and capabilities rather than guessing. + +tags: + - discovery + - basic + - happy-path + +inputs: + prompt: "What image generation models can I use through OpenRouter, and which of them support image editing?" + +expected: + outcomes: + - type: task_completed + behavior: + required_tools: + - bash + output_contains_any: + - "discover" + - "input_modalities" + - "image models" diff --git a/evals/openrouter-images/tasks/discovery/model-params.yaml b/evals/openrouter-images/tasks/discovery/model-params.yaml new file mode 100644 index 0000000..216f0d8 --- /dev/null +++ b/evals/openrouter-images/tasks/discovery/model-params.yaml @@ -0,0 +1,24 @@ +id: discovery-002 +name: Check a specific model's parameters +description: > + User asks whether a specific model supports a capability (4K resolution). + The agent should run discover.ts with the model ID and answer from the + per-endpoint supported_parameters, not from memory. + +tags: + - discovery + - happy-path + +inputs: + prompt: "Does bytedance-seed/seedream-4.5 support 4K output? What aspect ratios does it accept?" + +expected: + outcomes: + - type: task_completed + behavior: + required_tools: + - bash + output_contains_any: + - "4K" + - "supported_parameters" + - "aspect" diff --git a/evals/openrouter-images/tasks/discovery/provider-passthrough.yaml b/evals/openrouter-images/tasks/discovery/provider-passthrough.yaml new file mode 100644 index 0000000..974c1d5 --- /dev/null +++ b/evals/openrouter-images/tasks/discovery/provider-passthrough.yaml @@ -0,0 +1,25 @@ +id: discovery-003 +name: Discover provider passthrough options +description: > + User asks how to control provider-specific knobs (steps/guidance) for a FLUX + model. The agent should inspect the model's endpoints with discover.ts and + explain allowed_passthrough_parameters and the --provider-options flag keyed + by provider_slug. + +tags: + - discovery + - advanced + +inputs: + prompt: "I want to tune the number of diffusion steps and guidance for black-forest-labs/flux.2-pro image generation. How do I do that and what values are allowed?" + +expected: + outcomes: + - type: task_completed + behavior: + required_tools: + - bash + output_contains_any: + - "provider-options" + - "provider_options" + - "passthrough" diff --git a/evals/openrouter-images/tasks/edge-cases/cost-reporting.yaml b/evals/openrouter-images/tasks/edge-cases/cost-reporting.yaml new file mode 100644 index 0000000..dcf4bac --- /dev/null +++ b/evals/openrouter-images/tasks/edge-cases/cost-reporting.yaml @@ -0,0 +1,25 @@ +id: edge-005 +name: Cost awareness and reporting +description: > + User asks about cost before generating. The agent should look up per-image + pricing via discover.ts endpoint data and report the cost printed to stderr + after generation, per the skill's presentation guidance. + +tags: + - edge-case + - presentation + - pricing + +inputs: + prompt: "How much will it cost to generate one image with the default model? Check the pricing, then generate a picture of a hot air balloon and tell me what it actually cost." + +expected: + outcomes: + - type: task_completed + behavior: + required_tools: + - bash + output_contains_any: + - "$" + - "cost" + - "pricing" diff --git a/evals/openrouter-images/tasks/edge-cases/invalid-model.yaml b/evals/openrouter-images/tasks/edge-cases/invalid-model.yaml new file mode 100644 index 0000000..695488d --- /dev/null +++ b/evals/openrouter-images/tasks/edge-cases/invalid-model.yaml @@ -0,0 +1,23 @@ +id: edge-004 +name: Nonexistent model requested +description: > + User names a model that does not exist. The agent should check available + image models with discover.ts and suggest a close valid alternative instead + of passing the bogus ID through and failing. + +tags: + - edge-case + - negative + - model-selection + +inputs: + prompt: "Generate a photo of a beach at sunset using the dall-e-9-ultra model." + +expected: + outcomes: + - type: task_completed + behavior: + required_tools: + - bash + output_not_contains: + - "dall-e-9-ultra generated" diff --git a/evals/openrouter-images/tasks/edge-cases/missing-source-image.yaml b/evals/openrouter-images/tasks/edge-cases/missing-source-image.yaml new file mode 100644 index 0000000..cad0842 --- /dev/null +++ b/evals/openrouter-images/tasks/edge-cases/missing-source-image.yaml @@ -0,0 +1,24 @@ +id: edge-002 +name: Missing source image +description: > + User asks to edit a file that doesn't exist in the workspace. The agent + should notice the file is missing and ask for it or report the problem + clearly, not fabricate a result or generate a new image instead. + +tags: + - edge-case + - negative + +inputs: + prompt: "Remove the background from holiday-photo.png." + +expected: + outcomes: + - type: task_completed + output_contains_any: + - "not found" + - "doesn't exist" + - "does not exist" + - "couldn't find" + - "could not find" + - "no file" diff --git a/evals/openrouter-images/tasks/edge-cases/unsupported-format.yaml b/evals/openrouter-images/tasks/edge-cases/unsupported-format.yaml new file mode 100644 index 0000000..0f14072 --- /dev/null +++ b/evals/openrouter-images/tasks/edge-cases/unsupported-format.yaml @@ -0,0 +1,24 @@ +id: edge-003 +name: Unsupported input format +description: > + User asks to edit a file type edit.ts does not accept (.tiff — supported + formats are png/jpg/jpeg/webp/gif). The agent should surface the format + limitation, ideally offering to convert first, rather than failing opaquely. + +tags: + - edge-case + - negative + +inputs: + prompt: "Take scan.tiff and make it look like an oil painting." + files: + - path: scan.tiff + content: "placeholder tiff bytes" + +expected: + outcomes: + - type: task_completed + output_contains_any: + - "tiff" + - "format" + - "convert" diff --git a/evals/openrouter-images/tasks/edge-cases/unsupported-parameter.yaml b/evals/openrouter-images/tasks/edge-cases/unsupported-parameter.yaml new file mode 100644 index 0000000..11a002c --- /dev/null +++ b/evals/openrouter-images/tasks/edge-cases/unsupported-parameter.yaml @@ -0,0 +1,24 @@ +id: edge-001 +name: Unsupported parameter handling +description: > + User requests an option the default model likely doesn't support. Instead of + blindly sending the flag and hitting a 400, the agent should discover the + model's supported_parameters and either pick a capable model or explain the + limitation. + +tags: + - edge-case + - capability-check + +inputs: + prompt: "Generate an 8K SVG vector illustration of a mountain range using the default model." + +expected: + outcomes: + - type: task_completed + behavior: + required_tools: + - bash + output_not_contains: + - "traceback" + - "unhandled" diff --git a/evals/openrouter-images/tasks/editing/basic-edit.yaml b/evals/openrouter-images/tasks/editing/basic-edit.yaml new file mode 100644 index 0000000..6fced0b --- /dev/null +++ b/evals/openrouter-images/tasks/editing/basic-edit.yaml @@ -0,0 +1,29 @@ +id: editing-001 +name: Basic image edit +description: > + User asks to modify an existing image. The agent should call edit.ts with + the source path and prompt, and must NOT use generate.ts (which would + discard the source image). + +tags: + - editing + - basic + - happy-path + - golden + +golden: true + +inputs: + prompt: "Make the sky purple in photo.png." + files: + - path: photo.png + +expected: + outcomes: + - type: task_completed + behavior: + required_tools: + - bash + output_contains_any: + - "edit" + - "photo.png" diff --git a/evals/openrouter-images/tasks/editing/edit-model-capability.yaml b/evals/openrouter-images/tasks/editing/edit-model-capability.yaml new file mode 100644 index 0000000..b935afc --- /dev/null +++ b/evals/openrouter-images/tasks/editing/edit-model-capability.yaml @@ -0,0 +1,27 @@ +id: editing-003 +name: Edit requires an image-input model +description: > + User asks to edit with a specific model. The agent should verify the model's + input_modalities include image (via discover.ts) before editing, and switch + to or recommend a capable model if it does not. + +tags: + - editing + - capability-check + - advanced + +inputs: + prompt: "Convert sprite.webp to a watercolor style. I'd like to use the cheapest model that can actually do image-to-image editing — figure out which models support that first." + files: + - path: sprite.webp + +expected: + outcomes: + - type: task_completed + behavior: + required_tools: + - bash + output_contains_any: + - "input_modalities" + - "image-to-image" + - "image input" diff --git a/evals/openrouter-images/tasks/editing/edit-with-output.yaml b/evals/openrouter-images/tasks/editing/edit-with-output.yaml new file mode 100644 index 0000000..f8d5afe --- /dev/null +++ b/evals/openrouter-images/tasks/editing/edit-with-output.yaml @@ -0,0 +1,23 @@ +id: editing-002 +name: Edit preserving the original file +description: > + User wants the edit saved as a new file so the original is untouched. The + agent should use edit.ts with --output and confirm both files exist. + +tags: + - editing + - options + +inputs: + prompt: "Add a party hat to avatar.jpg, but don't overwrite my original — save the result as avatar-hat.png." + files: + - path: avatar.jpg + +expected: + outcomes: + - type: task_completed + behavior: + required_tools: + - bash + output_contains: + - "avatar-hat.png" diff --git a/evals/openrouter-images/tasks/editing/generate-then-edit.yaml b/evals/openrouter-images/tasks/editing/generate-then-edit.yaml new file mode 100644 index 0000000..13b6a52 --- /dev/null +++ b/evals/openrouter-images/tasks/editing/generate-then-edit.yaml @@ -0,0 +1,27 @@ +id: editing-004 +name: Multi-turn generate then edit +description: > + Multi-turn workflow. The agent first generates an image, then edits the file + it just created in a follow-up turn. Tests that it tracks the saved path + from generate.ts output and feeds it to edit.ts. + +tags: + - editing + - generation + - multi-turn + - advanced + +inputs: + prompt: "Generate an image of a corgi sitting in a park." + follow_up_prompts: + - "Nice — now add a rainbow in the background of that image you just made." + +expected: + outcomes: + - type: task_completed + behavior: + required_tools: + - bash + output_contains_any: + - "rainbow" + - "edit" diff --git a/evals/openrouter-images/tasks/generation/aspect-ratio.yaml b/evals/openrouter-images/tasks/generation/aspect-ratio.yaml new file mode 100644 index 0000000..2432a97 --- /dev/null +++ b/evals/openrouter-images/tasks/generation/aspect-ratio.yaml @@ -0,0 +1,24 @@ +id: generation-002 +name: Generation with aspect ratio +description: > + User asks for a wide/landscape image. The agent should translate the intent + into --aspect-ratio 16:9 (or similar wide ratio) rather than ignoring it or + inventing an unsupported flag. + +tags: + - generation + - options + - happy-path + +inputs: + prompt: "Make me a wide landscape wallpaper of a misty pine forest at dawn. It should be in 16:9 format." + +expected: + outcomes: + - type: task_completed + behavior: + required_tools: + - bash + output_contains_any: + - "16:9" + - "aspect" diff --git a/evals/openrouter-images/tasks/generation/basic-generate.yaml b/evals/openrouter-images/tasks/generation/basic-generate.yaml new file mode 100644 index 0000000..6b37659 --- /dev/null +++ b/evals/openrouter-images/tasks/generation/basic-generate.yaml @@ -0,0 +1,29 @@ +id: generation-001 +name: Basic text-to-image generation +description: > + Simple happy-path generation request with no options. The agent should call + generate.ts with the prompt, use the default model, and report where the + image was saved. + +tags: + - generation + - basic + - happy-path + - golden + +golden: true + +inputs: + prompt: "Create an image of a red panda wearing sunglasses." + +expected: + outcomes: + - type: task_completed + behavior: + required_tools: + - bash + max_tool_calls: 6 + output_contains_any: + - "saved" + - ".png" + - "images_saved" diff --git a/evals/openrouter-images/tasks/generation/model-override.yaml b/evals/openrouter-images/tasks/generation/model-override.yaml new file mode 100644 index 0000000..8270755 --- /dev/null +++ b/evals/openrouter-images/tasks/generation/model-override.yaml @@ -0,0 +1,23 @@ +id: generation-003 +name: Generation with a non-default model +description: > + User names a specific model informally. The agent should resolve it to a + valid model ID (verifying with discover.ts if unsure) and pass it via + --model instead of using the default. + +tags: + - generation + - options + - model-selection + +inputs: + prompt: "Generate a watercolor painting of a lighthouse using gemini 3.1 flash lite image, not the default model." + +expected: + outcomes: + - type: task_completed + behavior: + required_tools: + - bash + output_contains: + - "gemini-3.1-flash-lite-image" diff --git a/evals/openrouter-images/tasks/generation/multiple-images.yaml b/evals/openrouter-images/tasks/generation/multiple-images.yaml new file mode 100644 index 0000000..6a375f0 --- /dev/null +++ b/evals/openrouter-images/tasks/generation/multiple-images.yaml @@ -0,0 +1,25 @@ +id: generation-005 +name: Generate multiple variations +description: > + User asks for several variations. The agent should use --n rather than + looping the script, and report every saved file (suffixes -1, -2, ...). + +tags: + - generation + - options + - advanced + +inputs: + prompt: "Give me 3 different variations of a minimalist logo for a coffee shop called 'Ember'." + +expected: + outcomes: + - type: task_completed + behavior: + required_tools: + - bash + output_contains_any: + - "--n 3" + - "-1" + - "3 images" + - "three" diff --git a/evals/openrouter-images/tasks/generation/output-path.yaml b/evals/openrouter-images/tasks/generation/output-path.yaml new file mode 100644 index 0000000..2c4facd --- /dev/null +++ b/evals/openrouter-images/tasks/generation/output-path.yaml @@ -0,0 +1,21 @@ +id: generation-004 +name: Generation with explicit output path +description: > + User specifies a file name for the result. The agent should pass --output + and confirm the file was written to the requested path. + +tags: + - generation + - options + +inputs: + prompt: "Generate pixel art of a small green dragon and save it as dragon.png in the current directory." + +expected: + outcomes: + - type: task_completed + behavior: + required_tools: + - bash + output_contains: + - "dragon.png" diff --git a/evals/openrouter-images/tasks/generation/seed-reproducibility.yaml b/evals/openrouter-images/tasks/generation/seed-reproducibility.yaml new file mode 100644 index 0000000..636b269 --- /dev/null +++ b/evals/openrouter-images/tasks/generation/seed-reproducibility.yaml @@ -0,0 +1,23 @@ +id: generation-007 +name: Deterministic generation with a seed +description: > + User wants reproducible output. The agent should check that the chosen model + supports seeds and pass --seed, explaining that the same seed + prompt + + model yields the same image where supported. + +tags: + - generation + - options + - advanced + +inputs: + prompt: "Generate an isometric illustration of a tiny island, and make it reproducible so I can regenerate the exact same image later." + +expected: + outcomes: + - type: task_completed + behavior: + required_tools: + - bash + output_contains: + - "seed" diff --git a/evals/openrouter-images/tasks/generation/transparent-background.yaml b/evals/openrouter-images/tasks/generation/transparent-background.yaml new file mode 100644 index 0000000..6816485 --- /dev/null +++ b/evals/openrouter-images/tasks/generation/transparent-background.yaml @@ -0,0 +1,24 @@ +id: generation-006 +name: Transparent background with capability check +description: > + User wants a transparent PNG. Not all models support the background + parameter, so the agent should verify support with discover.ts (or pick a + model that supports it) before passing --background transparent. + +tags: + - generation + - options + - capability-check + +inputs: + prompt: "I need a sticker-style image of a cartoon cactus with a transparent background as a PNG. Make sure the model you use actually supports transparency." + +expected: + outcomes: + - type: task_completed + behavior: + required_tools: + - bash + output_contains_any: + - "transparent" + - "background" diff --git a/evals/openrouter-images/tasks/trigger/should-not-trigger-text.yaml b/evals/openrouter-images/tasks/trigger/should-not-trigger-text.yaml new file mode 100644 index 0000000..1dce960 --- /dev/null +++ b/evals/openrouter-images/tasks/trigger/should-not-trigger-text.yaml @@ -0,0 +1,19 @@ +id: trigger-006 +name: Should NOT trigger on text-only request +description: > + A pure text task that merely mentions imagery should not invoke the images + skill. + +tags: + - trigger + - negative + +inputs: + prompt: "Write a short poem that paints a vivid picture of a stormy sea." + +expected: + outcomes: + - type: task_completed + behavior: + forbidden_tools: + - skill diff --git a/evals/openrouter-images/tasks/trigger/should-not-trigger-video.yaml b/evals/openrouter-images/tasks/trigger/should-not-trigger-video.yaml new file mode 100644 index 0000000..42bcc83 --- /dev/null +++ b/evals/openrouter-images/tasks/trigger/should-not-trigger-video.yaml @@ -0,0 +1,16 @@ +id: trigger-005 +name: Should NOT trigger on video request +description: > + Video generation is handled by a different skill (openrouter-video); the + images skill should not fire for video requests. + +tags: + - trigger + - negative + +inputs: + prompt: "Generate a 5-second video clip of waves crashing on a beach." + +expected: + outcomes: + - type: task_completed diff --git a/evals/openrouter-images/tasks/trigger/should-not-trigger-vision.yaml b/evals/openrouter-images/tasks/trigger/should-not-trigger-vision.yaml new file mode 100644 index 0000000..51b1504 --- /dev/null +++ b/evals/openrouter-images/tasks/trigger/should-not-trigger-vision.yaml @@ -0,0 +1,21 @@ +id: trigger-004 +name: Should NOT trigger on image analysis +description: > + Describing/analyzing an existing image (vision) is not generation or + editing; the openrouter-images skill should not be invoked. + +tags: + - trigger + - negative + +inputs: + prompt: "Look at photo.png and tell me what's in it." + files: + - path: photo.png + +expected: + outcomes: + - type: task_completed + behavior: + forbidden_tools: + - skill diff --git a/evals/openrouter-images/tasks/trigger/should-trigger-edit.yaml b/evals/openrouter-images/tasks/trigger/should-trigger-edit.yaml new file mode 100644 index 0000000..e71bf4b --- /dev/null +++ b/evals/openrouter-images/tasks/trigger/should-trigger-edit.yaml @@ -0,0 +1,20 @@ +id: trigger-002 +name: Should trigger on edit request +description: > + An image-editing request phrased without the words "generate" or "image API" + should still trigger the skill. + +tags: + - trigger + - positive + +inputs: + prompt: "Can you touch up photo.png so it looks like it was taken at golden hour?" + files: + - path: photo.png + +expected: + outcomes: + - type: task_completed + - type: tool_called + tool_name: skill diff --git a/evals/openrouter-images/tasks/trigger/should-trigger-generate.yaml b/evals/openrouter-images/tasks/trigger/should-trigger-generate.yaml new file mode 100644 index 0000000..b9b811f --- /dev/null +++ b/evals/openrouter-images/tasks/trigger/should-trigger-generate.yaml @@ -0,0 +1,18 @@ +id: trigger-001 +name: Should trigger on generation request +description: > + A plain image-creation request should cause the agent to invoke the + openrouter-images skill without the skill body being pre-injected. + +tags: + - trigger + - positive + +inputs: + prompt: "Create an illustration of a fox reading a book under a tree." + +expected: + outcomes: + - type: task_completed + - type: tool_called + tool_name: skill diff --git a/evals/openrouter-images/tasks/trigger/should-trigger-indirect.yaml b/evals/openrouter-images/tasks/trigger/should-trigger-indirect.yaml new file mode 100644 index 0000000..b4e5328 --- /dev/null +++ b/evals/openrouter-images/tasks/trigger/should-trigger-indirect.yaml @@ -0,0 +1,18 @@ +id: trigger-003 +name: Should trigger on indirect phrasing +description: > + Requests that imply image creation without saying "image" (e.g. logo, + wallpaper, sticker) should still trigger the skill. + +tags: + - trigger + - positive + +inputs: + prompt: "I need a logo for my new bakery, something warm with a croissant motif." + +expected: + outcomes: + - type: task_completed + - type: tool_called + tool_name: skill