From 1073027239eeffd5004d609769dc9ed807ff98b0 Mon Sep 17 00:00:00 2001 From: "kiloloop-release[bot]" <269344367+kiloloop-release[bot]@users.noreply.github.com> Date: Wed, 20 May 2026 19:41:39 -0700 Subject: [PATCH] feat: v0.7.0 New task categories (frontend/UI, app development), 3-round review mode, Opus 4.7 / GPT-5.5 METR thresholds, opt-in audit logging, refreshed estimate skill, and repository tooling (Makefile, preflight, multi-runtime skills layout). See CHANGELOG for details. Co-Authored-By: Claude Opus 4.7 (1M context) --- .agent/skills/estimate/SKILL.md | 90 ------ .claude-plugin/plugin.json | 2 +- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/ci.yml | 6 +- .github/workflows/test-action.yml | 1 + .gitignore | 1 + CHANGELOG.md | 20 ++ Makefile | 21 ++ README.md | 248 +++++++------- SECURITY.md | 4 +- action.yml | 6 +- examples/README.md | 2 +- metr_thresholds.yaml | 9 + pyproject.toml | 4 +- scripts/preflight.py | 319 +++++++++++++++++++ skills/estimate/README.md | 46 +++ skills/estimate/{ => claude}/SKILL.md | 19 +- skills/estimate/codex/SKILL.md | 126 ++++++++ skills/estimate/shared/INTENT.md | 66 ++++ skills/estimate/skill.yaml | 12 + src/agent_estimate/adapters/github_ghcli.py | 104 ++++-- src/agent_estimate/adapters/github_rest.py | 53 +++ src/agent_estimate/audit.py | 213 +++++++++++++ src/agent_estimate/cli/app.py | 2 + src/agent_estimate/cli/commands/_pipeline.py | 39 ++- src/agent_estimate/cli/commands/estimate.py | 94 +++++- src/agent_estimate/cli/commands/session.py | 20 ++ src/agent_estimate/core/__init__.py | 4 + src/agent_estimate/core/human_comparison.py | 2 + src/agent_estimate/core/models.py | 23 +- src/agent_estimate/core/modifiers.py | 3 +- src/agent_estimate/core/pert.py | 16 +- src/agent_estimate/core/task_type_models.py | 199 +++++++++++- src/agent_estimate/metr_thresholds.yaml | 9 + src/agent_estimate/version.py | 2 +- tests/integration/test_cli_e2e.py | 54 ++++ tests/unit/test_audit.py | 41 +++ tests/unit/test_github_adapters.py | 106 ++++++ tests/unit/test_human_comparison.py | 12 + tests/unit/test_modifiers.py | 10 +- tests/unit/test_pert_engine.py | 18 +- tests/unit/test_plugin_structure.py | 88 +++-- tests/unit/test_task_type_models.py | 107 +++++++ tests/unit/test_version.py | 2 +- 44 files changed, 1898 insertions(+), 327 deletions(-) delete mode 100644 .agent/skills/estimate/SKILL.md create mode 100644 Makefile create mode 100644 scripts/preflight.py create mode 100644 skills/estimate/README.md rename skills/estimate/{ => claude}/SKILL.md (72%) create mode 100644 skills/estimate/codex/SKILL.md create mode 100644 skills/estimate/shared/INTENT.md create mode 100644 skills/estimate/skill.yaml create mode 100644 src/agent_estimate/audit.py create mode 100644 tests/unit/test_audit.py diff --git a/.agent/skills/estimate/SKILL.md b/.agent/skills/estimate/SKILL.md deleted file mode 100644 index aaf6d82..0000000 --- a/.agent/skills/estimate/SKILL.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -name: estimate -description: Codex skill for running agent-estimate CLI commands (estimate, validate, calibrate). ---- - -# estimate (Codex) - -Use this skill when a user asks to estimate delivery effort for coding work, validate an estimate with observed results, or recalibrate model factors. - -The skill is command-first: execute the `agent-estimate` CLI and return its output. - -## Intent Mapping - -| User intent | Command | -| --- | --- | -| Estimate one task or multiple tasks | `agent-estimate estimate ...` | -| Validate estimate vs actuals | `agent-estimate validate ...` | -| Recompute calibration summary | `agent-estimate calibrate ...` | - -## Build Rules - -### For `estimate` - -- Accept exactly one input source: - - task description argument - - `--file ` - - `--issues ` (requires `--repo `) -- Supported flags: - - `--config ` - - `--format markdown|json` - - `--review-mode none|self|2x-lgtm` - - `--title ` - - `--verbose` - -If input source is missing, ask for one. - -### For `validate` - -- Required: observation YAML path. -- Optional: `--db `. - -### For `calibrate` - -- Optional: `--db `. -- Default DB: `~/.agent-estimate/calibration.db`. - -## Execution Rules - -1. Execute commands from the repository root. -2. Prefer `agent-estimate` binary; fallback to `python -m agent_estimate.cli.app` if needed. -3. Capture stdout/stderr and exit code. -4. If command fails, return the error concisely and include the attempted command. -5. If command succeeds, return CLI output directly. -6. `--format json` is supported and should be treated as a normal success path. - -## Command Examples - -```bash -agent-estimate estimate "Add login button with OAuth" -agent-estimate estimate --file tasks.md -agent-estimate estimate --issues 1,2,3 --repo org/name -agent-estimate estimate --config agents.yaml --format json "Refactor auth module" -agent-estimate validate observation.yaml --db ~/.agent-estimate/calibration.db -agent-estimate calibrate --db ~/.agent-estimate/calibration.db -``` - -## Observation YAML Example - -```yaml -task_type: feature -estimated_minutes: 45.0 -actual_work_minutes: 52.0 -actual_total_minutes: 60.0 -file_count: 3 -line_count: 120 -test_count: 5 -execution_mode: single -review_mode: 2x-lgtm -review_overhead_minutes: 8.0 -modifiers: - spec_clarity: 1.0 - warm_context: 0.9 -``` - -## Notes - -- Requires `agent-estimate` installed: `pip install agent-estimate` or `pip install -e '.[dev]'` in this repo. -- Default config uses bundled `default_agents.yaml`. Pass `--config` to override agent definitions. -- `--review-mode` defaults to `2x-lgtm`. -- Keep command output as source of truth; do not invent computed values. diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index e59f52a..7ff7240 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "agent-estimate", "description": "Effort estimation for AI coding agents — PERT three-point estimation with METR reliability thresholds and wave planning", - "version": "0.6.1", + "version": "0.7.0", "author": { "name": "Kiloloop" }, diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 83c0dec..8e2ceca 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -55,7 +55,7 @@ body: id: version attributes: label: agent-estimate version - placeholder: "0.6.1" + placeholder: "0.7.0" validations: required: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2294264..73c8902 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,5 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: pip install -e .[dev] - - name: Lint with ruff - run: ruff check . - - name: Run tests - run: python -m pytest tests/ -x -q + - name: Run preflight (full) + run: make preflight ARGS="--full" diff --git a/.github/workflows/test-action.yml b/.github/workflows/test-action.yml index ea71406..0088b08 100644 --- a/.github/workflows/test-action.yml +++ b/.github/workflows/test-action.yml @@ -14,6 +14,7 @@ on: permissions: contents: read + issues: read jobs: test-action-summary: diff --git a/.gitignore b/.gitignore index 13c0659..ce61858 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ AGENTS.md CLAUDE.md .claude/ +.oacp reports/ # Python diff --git a/CHANGELOG.md b/CHANGELOG.md index dc24a16..1e9327c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.7.0] - 2026-05-20 + +### Added +- Frontend/UI task category with separate content-patch (15/25/40) and page-build (40/60/90) bands. +- App-development task category with a generic cold L-style prior and app/UI human-comparison multiplier. +- `3-round` review mode with a 35 minute additive review tier. +- METR threshold entries for Opus 4.7 (current) and GPT-5.5; `opus_4_x` retained as a forward-compatible alias. +- Opt-in structured audit logging via `AGENT_ESTIMATE_AUDIT_*` environment variables, emitting secret-scrubbed JSON events to stdout, stderr, or a file. + +### Changed +- Research-grounded brainstorms now route to the research band instead of the flat brainstorm band. +- Codex model-key alias now resolves to the GPT-5.5 METR threshold; GPT-5.4 remains available. +- Corrected the Codex skill install path in `skills/estimate/README.md` to `.codex/skills/...`. +- Version bumped to v0.7.0 across package, plugin, action, issue template, and tests. +- Claude runtime `/estimate` skill refreshed to v0.7.0 parity with the Codex slice (frontend/app_dev types, `3-round` review mode, refreshed METR keys). +- `claude`/`claude_opus` model-key aliases now resolve to `opus_4_7` (Opus 4.7); `opus_4_6` retained for backward compatibility. + ## [0.6.1] - 2026-03-20 ### Fixed @@ -77,6 +94,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Modifier flags: `--warm-context`, `--spec-clarity`, `--issues` - PyPI package: `pip install agent-estimate` +[0.7.0]: https://github.com/kiloloop/agent-estimate/releases/tag/v0.7.0 +[0.6.1]: https://github.com/kiloloop/agent-estimate/releases/tag/v0.6.1 +[0.6.0]: https://github.com/kiloloop/agent-estimate/releases/tag/v0.6.0 [0.5.0]: https://github.com/kiloloop/agent-estimate/releases/tag/v0.5.0 [0.4.0]: https://github.com/kiloloop/agent-estimate/releases/tag/v0.4.0 [0.3.0]: https://github.com/kiloloop/agent-estimate/releases/tag/v0.3.0 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5488b81 --- /dev/null +++ b/Makefile @@ -0,0 +1,21 @@ +# Self-documenting Makefile — run `make help` for available targets. +.DEFAULT_GOAL := help + +ARGS ?= + +# ── Quality ─────────────────────────────────────────────────────────── +.PHONY: preflight test lint help + +preflight: ## Run fast preflight checks (conflict markers, YAML, ruff) + python3 scripts/preflight.py $(ARGS) + +test: ## Run pytest suite + python3 -m pytest tests/ -x -q + +lint: ## Run ruff linter + ruff check . + +# ── Utilities ───────────────────────────────────────────────────────── +help: ## Show this help + @grep -E '^[a-zA-Z_-]+:.*##' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = ":.*## "}; {printf " \033[36m%-20s\033[0m %s\n", $$1, $$2}' diff --git a/README.md b/README.md index 887ee3c..33f5c69 100644 --- a/README.md +++ b/README.md @@ -5,139 +5,147 @@ [![License](https://img.shields.io/pypi/l/agent-estimate)](https://github.com/kiloloop/agent-estimate/blob/main/LICENSE) [![CI](https://github.com/kiloloop/agent-estimate/actions/workflows/ci.yml/badge.svg)](https://github.com/kiloloop/agent-estimate/actions/workflows/ci.yml) -**Know what an AI task will cost before you run it.** +**Know before you build.** -`agent-estimate` tells you how long an AI agent will take — and how that compares to doing it yourself. +PERT estimates for AI-agent tasks — how long, which model's reliable enough, and the human-equivalent cost. In one command. -``` -$ agent-estimate estimate "Implement OAuth 2.0 flow (Google + GitHub)" -``` +**[Website](https://kiloloop.com/agent-estimate/)** · [Compare](https://kiloloop.com/agent-estimate/compare/) · [PyPI](https://pypi.org/project/agent-estimate/) -| Metric | Value | -| --- | --- | -| Expected case | 75.4m | -| Human-speed equivalent | 190.9m | -| **Compression ratio** | **2.53x** | +## Why -One command. Three numbers. Now you know whether to dispatch an agent or do it yourself. +AI agents can write the code — but *how long will the task actually take?* Manual estimation is slow and biased toward optimism; no estimate means scope creep and missed deadlines. The gap between "agents can do it" and "we know when it'll be done" is where projects break down. -## See it in action +`agent-estimate` closes that gap in one command: a three-point PERT timeline calibrated on real agent runs, plus a human-speed comparison so you see the compression before you spend the compute. It sizes the task, picks a tier, routes it to a model, and flags when the work runs past that model's reliability horizon — calibrated forecasts in seconds, not meetings. -### Single task — coding +Multi-model matters because the models aren't interchangeable. Opus 4.7, GPT-5.5, and Gemini 3.1 have different reliability horizons ([METR p80](https://metr.org/)) and different costs per turn. A safe 40-minute job for one model is a coin flip for another. agent-estimate models the whole fleet, not a single agent — so the number reflects who actually runs the work. -```bash -$ agent-estimate estimate "Implement OAuth 2.0 flow (Google + GitHub)" -``` +## Quick Start -| Metric | Value | -| --- | --- | -| Task | Implement OAuth 2.0 flow (Google + GitHub) | -| Tier / Agent | M / Claude | -| Base PERT (O/M/P) | 25m / 50m / 90m (E=52.5m) | -| Best case | 44.7m | -| Expected case | 75.4m | -| Worst case | 117.2m | -| Human-speed equivalent | 190.9m | -| **Compression ratio** | **2.53x** | -| Review overhead | +15m (standard) | -| Estimated cost | $1.45 | +> First estimate: 30 seconds to install. Every one after: instant. -A medium coding task. Your agent finishes in ~75 minutes. Doing it yourself? ~3 hours. ([Full output](./examples/coding-m.md)) +### With your agent (recommended) -### Single task — research +Paste this into your Claude Code or Codex session: -```bash -$ agent-estimate estimate "Audit dependencies for known CVEs" --type research -``` +~~~ +Install the agent-estimate plugin (https://github.com/kiloloop/agent-estimate) and +estimate this task for me: "Implement OAuth 2.0 flow (Google + GitHub)". Tell me the +expected time, the human-speed equivalent, and the compression ratio. +~~~ -| Metric | Value | -| --- | --- | -| Task | Audit dependencies for known CVEs | -| Tier / Agent | S / Claude | -| Base PERT (O/M/P) | 10m / 20m / 30m (E=20m) | -| Expected case | 38m | -| Human-speed equivalent | 99m | -| **Compression ratio** | **2.61x** | -| Estimated cost | $0.55 | +Your agent installs the tool, runs the estimate, and reads back the numbers. Nothing to memorize — describe the task in plain English and let the agent translate to flags. + +For a whole backlog: -Research tasks have high human-multipliers — pattern matching across hundreds of dependencies is exactly where agents shine. ([Full output](./examples/research.md)) +~~~ +Estimate every open issue in this repo with agent-estimate, group them into parallel +waves, and tell me the total wall-clock time for a 3-agent fleet versus doing them +sequentially myself. +~~~ -### Multi-agent session — 3 tasks in parallel +### Manual ```bash -$ agent-estimate estimate --file tasks.txt +pip install agent-estimate +agent-estimate estimate "your task description here" ``` -Where `tasks.txt` contains: -``` -Implement OAuth 2.0 flow (Google + GitHub) -Write unit tests for OAuth flow -Write API reference for auth module +No config required — sensible defaults for a 3-agent fleet (Claude, Codex, Gemini). Point it at a file or GitHub issues when you're ready: + +```bash +agent-estimate estimate --file tasks.txt +agent-estimate estimate --repo myorg/myrepo --issues 11,12,14 ``` -| Task | Tier | Agent | Expected | Human Equiv | -| --- | --- | --- | --- | --- | -| Implement OAuth 2.0 flow | M | Codex | 52.5m | 190.9m | -| Write unit tests for OAuth flow | M | Gemini | 52.5m | 190.9m | -| Write API reference for auth module | L | Claude | 100.8m | 327.6m | +## How It Works -| Metric | Value | -| --- | --- | -| Wave 0 | All 3 tasks in parallel (Claude + Codex + Gemini) | -| Expected case | 131m | -| Human-speed equivalent | 709.5m | -| **Compression ratio** | **5.42x** | -| Estimated cost | $4.84 | +agent-estimate produces three-point [PERT](https://en.wikipedia.org/wiki/Program_evaluation_and_review_technique) estimates calibrated for agents, not humans: -Three agents working in parallel. ~2 hours wall clock vs ~12 hours sequential human work. That's the power of fleet estimation — you see the compression before you commit the compute. ([Full output](./examples/multi-agent.md)) +- **Tier classification** — auto-sizes tasks XS→XL from complexity signals +- **PERT math** — optimistic / most-likely / pessimistic, weighted to an expected value +- **Human comparison** — a per-task-type multiplier, so you see the compression +- **METR thresholds** — warns when an estimate exceeds a model's p80 reliability horizon +- **Wave planning** — schedules independent tasks in parallel across the fleet +- **Review overhead** — models review cycles as additive cost (`standard`, `complex`, `3-round`) +- **Modifiers** — `--spec-clarity`, `--warm-context`, `--agent-fit` tune the estimate -> More examples in [`examples/`](./examples/) — coding S/M, research, documentation, and multi-agent sessions. +### Task types -## Try it now +| Type | Flag | Models | +|------|------|--------| +| Coding | (default) | Feature work, fixes, refactors | +| Research | `--type research` | Audits, investigations, analysis | +| Documentation | `--type documentation` | API docs, guides, changelogs | +| Brainstorm | `--type brainstorm` | Ideation, spikes, design exploration | +| Config/SRE | `--type config` | Deploys, infra, CI/CD | +| Frontend/UI | `--type frontend` | Content patches vs. component builds | +| App dev | `--type app_dev` | App shells, desktop/mobile builds | -```bash -pip install agent-estimate -``` +### METR thresholds (defaults) -```bash -agent-estimate estimate "your task description here" -``` +| Model | p80 threshold | +|-------|---------------| +| Opus 4.7 | 90 min | +| GPT-5.5 | 90 min | +| GPT-5.4 | 60 min | +| Gemini 3.1 Pro | 45 min | +| Sonnet 4.6 | 30 min | +| Haiku 4.5 | 15 min | -That's it. No config needed — sensible defaults for a 3-agent fleet (Claude, Codex, Gemini). +`opus_4_x` is a forward-compatible alias that resolves to the current Opus threshold. Legacy keys (`opus_4_6`, GPT-5/5.2/5.3, Gemini 3 Pro, Sonnet) stay supported. Estimates are calibrated against Claude Code (Opus 4.7, high thinking) and Codex (GPT-5.4/5.5, extra-high) — shift with `--spec-clarity` and `--warm-context` for other setups. -## What it does +## Examples -`agent-estimate` produces three-point [PERT](https://en.wikipedia.org/wiki/Program_evaluation_and_review_technique) estimates calibrated for AI agents, not humans: +Real estimates from production use — including the misses. -- **Tier classification** — auto-sizes tasks as XS/S/M/L/XL based on complexity signals -- **PERT math** — optimistic, most-likely, pessimistic with weighted expected value -- **Human comparison** — multiplier per task type (coding, research, docs) so you see the compression -- **METR thresholds** — warns when estimates exceed model reliability limits ([METR p80 benchmarks](https://metr.org/)) -- **Wave planning** — dependency-aware scheduling across multiple agents with parallelism -- **Review overhead** — models review cycles as flat additive cost, amortized per agent per wave -- **Modifiers** — tune estimates with `--spec-clarity`, `--warm-context`, `--agent-fit` +**The tool, estimating its own docs.** We sized this v0.7.0 skill-and-README refresh at ~30 minutes. It took 28. -### Supported task types +**An honest over-estimate.** We pre-registered a UI mockup build at ~95 minutes with no prior app-dev data. Two agents did it in parallel in 12 and 25 minutes — a 4–8x over-estimate. agent-estimate now ships an `app_dev` prior shaped by that result. The miss stays in the README because calibration means showing where you were wrong. -| Type | Flag | What it models | -|------|------|---------------| -| Coding | (default) | Feature work, bug fixes, refactors — tier-based PERT | -| Research | `--type research` | Audits, investigations, analysis — flat PERT with depth scaling | -| Documentation | `--type documentation` | API docs, guides, changelogs | -| Brainstorm | `--type brainstorm` | Ideation, spikes, design exploration | -| Config/SRE | `--type config` | Deploys, infra changes, CI/CD work | +**Two tasks, one model** — what the tool prints, including the METR reliability flag: + +```text +$ agent-estimate estimate "Implement auth" "Add tests" --model opus + +Task Tier PERT (O/M/P) Expected Human-eq +─────────────────────────────────────────────────────────── +Implement auth M 25/50/90m 57.8m 160m +Add tests S 12/23/40m 24.0m 75m + +Timeline ────────────────────────────── + best 37m · expected 81.8m · worst 130m + human-equivalent: 235m → 2.87× compression + + ⚠ METR warning: "Implement auth" exceeds Opus p80 +``` + +~82 minutes expected versus ~4 hours by hand — plus a flag that the auth task runs past Opus's p80 reliability horizon, so you split it or add a checkpoint before dispatching. + +**Three tasks, three agents, in parallel:** + +```bash +$ agent-estimate estimate --file tasks.txt +``` + +| Metric | Value | +| --- | --- | +| Wave 0 | All 3 tasks in parallel (Claude + Codex + Gemini) | +| Expected case | 131m | +| Human-speed equivalent | 709.5m | +| **Compression ratio** | **5.42x** | +| Estimated cost | $4.84 | + +~2 hours wall-clock versus ~12 hours sequential. You see the compression before you commit the compute. More in [`examples/`](./examples/) — coding S/M, research, documentation, multi-agent. ## Integrations -### Claude Code Plugin +### Claude Code plugin ``` /plugin marketplace add kiloloop/agent-estimate /plugin install agent-estimate@agent-estimate-marketplace ``` -Then use directly in Claude Code sessions: - ``` /estimate Add a login page with OAuth /estimate --file spec.md @@ -191,11 +199,11 @@ jobs: | `output-mode` | no | `summary` | `summary`, `pr-comment`, `step-output`, `summary+pr-comment` | | `config` | no | — | Path to agent config YAML | | `title` | no | `Agent Estimate Report` | Report title | -| `review-mode` | no | `standard` | Review tier: `none`, `standard`, `complex` | -| `spec-clarity` | no | `1.0` | Spec clarity modifier (0.3-1.3) | -| `warm-context` | no | `1.0` | Warm context modifier (0.3-1.15) | -| `agent-fit` | no | `1.0` | Agent fit modifier (0.9-1.2) | -| `task-type` | no | — | Task category: `coding`, `brainstorm`, `research`, `config`, `documentation` | +| `review-mode` | no | `standard` | Review tier: `none`, `standard`, `complex`, `3-round` | +| `spec-clarity` | no | `1.0` | Spec clarity modifier (0.3–1.3) | +| `warm-context` | no | `1.0` | Warm context modifier (0.3–1.15) | +| `agent-fit` | no | `1.0` | Agent fit modifier (0.9–1.2) | +| `task-type` | no | — | Category: `coding`, `brainstorm`, `research`, `config`, `documentation`, `frontend`, `app_dev` | | `python-version` | no | `3.12` | Python version to use | | `version` | no | latest | `agent-estimate` version to install | | `token` | no | `${{ github.token }}` | GitHub token | @@ -207,15 +215,26 @@ jobs: -### Codex Skill +### Skill layout -Codex-specific skill at `.agent/skills/estimate/SKILL.md`. Claude plugin skill at `skills/estimate/SKILL.md`. +Skills follow the [oacp-skills](https://github.com/kiloloop/oacp-skills) convention: + +``` +skills/estimate/ + skill.yaml # machine-readable metadata + README.md # human-readable docs + shared/INTENT.md # shared intent across runtimes + claude/SKILL.md # Claude Code skill definition + codex/SKILL.md # Codex skill definition +``` + +Both runtime slices cover the same CLI (`estimate`, `validate`, `calibrate`), phrased for their respective ecosystems. ## Configuration ### Agent fleet -Pass a custom config to model your own agent fleet: +Pass a config to model your own fleet: ```yaml agents: @@ -240,26 +259,12 @@ settings: agent-estimate estimate "Ship packaging flow" --config ./my_agents.yaml ``` -### Default METR thresholds - -| Model | p80 threshold | -| -------------- | ------------- | -| Opus 4.6 | 90 minutes | -| GPT-5.4 | 60 minutes | -| Gemini 3.1 Pro | 45 minutes | -| Sonnet 4.6 | 30 minutes | -| Haiku 4.5 | 15 minutes | - -Legacy model keys (Opus, GPT-5/5.2/5.3, Gemini 3 Pro, Sonnet) are still supported for backward compatibility. - -> **Note:** Estimates are calibrated against Claude Code (Opus 4.6, High thinking) and Codex (GPT-5.4, Extra High thinking). Different thinking levels or model versions may shift estimates — adjust with `--spec-clarity` and `--warm-context` modifiers as needed. - ### Output formats ```bash -agent-estimate estimate "Refactor auth pipeline" --format json # machine-readable -agent-estimate estimate --repo myorg/myrepo --issues 11,12,14 # from GitHub issues -agent-estimate estimate --file tasks.txt # from file +agent-estimate estimate "Refactor auth pipeline" --format json # machine-readable +agent-estimate estimate --repo myorg/myrepo --issues 11,12,14 # from GitHub issues +agent-estimate estimate --file tasks.txt # from file ``` ### Calibration @@ -270,13 +275,16 @@ Validate estimates against observed outcomes and build a calibration database: agent-estimate validate observation.yaml --db ~/.agent-estimate/calibration.db ``` -## Related +## Project -**[OACP](https://github.com/kiloloop/oacp)** — Coordinate the agents you just estimated. Open Agent Coordination Protocol for multi-agent async workflows. +- **[Website](https://kiloloop.com/agent-estimate/)** — landing page, live demo, and the [estimate comparison view](https://kiloloop.com/agent-estimate/compare/). +- **[OACP](https://github.com/kiloloop/oacp)** — coordinate the agents you just estimated. Open Agent Coordination Protocol for multi-agent async workflows. +- **[oacp-skills](https://github.com/kiloloop/oacp-skills)** — the skill bundle agent-estimate's `/estimate` ships in. +- **[kiloloop](https://github.com/kiloloop)** — the rest of the ecosystem. ## Contributing -See [CONTRIBUTING.md](./CONTRIBUTING.md) for full workflow and expectations. +See [CONTRIBUTING.md](./CONTRIBUTING.md) for the full workflow. ```bash pip install -e '.[dev]' diff --git a/SECURITY.md b/SECURITY.md index 8b485e0..43476a3 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -6,8 +6,8 @@ The project currently provides security fixes for the latest released minor line | Version | Supported | | --- | --- | -| 0.5.x | Yes | -| < 0.5.0 | No | +| 0.4.x | Yes | +| < 0.4.0 | No | ## Reporting a Vulnerability diff --git a/action.yml b/action.yml index 0e358b5..a387999 100644 --- a/action.yml +++ b/action.yml @@ -30,7 +30,7 @@ inputs: required: false default: 'Agent Estimate Report' review-mode: - description: 'Review overhead tier: none, standard, complex' + description: 'Review overhead tier: none, standard, complex, 3-round' required: false default: 'standard' spec-clarity: @@ -46,14 +46,14 @@ inputs: required: false default: '1.0' task-type: - description: 'Task category: coding, brainstorm, research, config, documentation' + description: 'Task category: coding, brainstorm, research, config, documentation, frontend, app_dev' required: false python-version: description: 'Python version to use' required: false default: '3.12' version: - description: 'agent-estimate version to install (e.g. "0.6.1"). Omit for latest.' + description: 'agent-estimate version to install (e.g. "0.7.0"). Omit for latest.' required: false token: description: 'GitHub token for issue fetching and PR comments' diff --git a/examples/README.md b/examples/README.md index 53d0aee..8da19e5 100644 --- a/examples/README.md +++ b/examples/README.md @@ -30,6 +30,6 @@ These examples are drawn from a production multi-agent fleet (Claude, Codex, Gem | Total dispatches tracked | 190+ | | M-tier accuracy | Expected case within ±20% | | Task types covered | Coding, research, documentation, brainstorm, config | -| Agents calibrated against | Claude Code (Opus 4.6, High thinking), Codex (GPT-5.4, Extra High thinking) | +| Agents calibrated against | Claude Code (Opus 4.7, High thinking), Codex (GPT-5.4, Extra High thinking) | Estimates improve with calibration data. Use `agent-estimate validate` to feed your own dispatch outcomes back into the model. diff --git a/metr_thresholds.yaml b/metr_thresholds.yaml index 795eceb..d93d77d 100644 --- a/metr_thresholds.yaml +++ b/metr_thresholds.yaml @@ -1,7 +1,16 @@ models: + opus_4_x: + display_name: "Opus 4.x" + p80_minutes: 90 + opus_4_7: + display_name: "Opus 4.7" + p80_minutes: 90 opus_4_6: display_name: "Opus 4.6" p80_minutes: 90 + gpt_5_5: + display_name: "GPT-5.5" + p80_minutes: 90 gpt_5_4: display_name: "GPT-5.4" p80_minutes: 60 diff --git a/pyproject.toml b/pyproject.toml index ecd5bf4..b6acf73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-estimate" -version = "0.6.1" +version = "0.7.0" description = "Know what an AI task will cost before you run it" readme = "README.md" license = "Apache-2.0" @@ -24,7 +24,7 @@ classifiers = [ dependencies = [ "typer>=0.12,<1.0", "pyyaml>=6.0,<7.0", - "pydantic>=2.0,<3.0", + "pydantic>=2.4,<3.0", "networkx>=3.0,<4.0", ] diff --git a/scripts/preflight.py b/scripts/preflight.py new file mode 100644 index 0000000..3d3efd1 --- /dev/null +++ b/scripts/preflight.py @@ -0,0 +1,319 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Kiloloop +# SPDX-License-Identifier: Apache-2.0 +"""Unified preflight checks for agent-estimate. + +Fast mode (default): +- Merge conflict marker scan +- YAML syntax validation for src/ and tests/fixtures/ +- `ruff` on all tracked Python files + +Extended mode (`--full`): fast mode + `pytest`. +""" + +from __future__ import annotations + +import argparse +import shutil +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, List, Optional, Sequence, Tuple + +Runner = Callable[[Sequence[str], Path], Tuple[int, str]] +YamlLoader = Callable[[str], object] + +MARKER_PREFIXES = ("<<<<<<<", "=======", ">>>>>>>") +YAML_EXTENSIONS = {".yaml", ".yml"} +SKIP_DIRS = {".git", ".venv", "venv", "__pycache__", ".pytest_cache", ".mypy_cache", "dist"} + + +@dataclass +class CheckResult: + name: str + passed: bool + details: str + duration_s: float + + @property + def status(self) -> str: + return "PASS" if self.passed else "FAIL" + + +def run_command(command: Sequence[str], cwd: Path) -> Tuple[int, str]: + """Run a command and return (exit_code, combined_output).""" + try: + completed = subprocess.run( + list(command), + cwd=str(cwd), + capture_output=True, + text=True, + check=False, + ) + except FileNotFoundError: + return 127, f"Command not found: {command[0]}" + + combined = "\n".join(part for part in (completed.stdout, completed.stderr) if part) + return completed.returncode, combined.strip() + + +def _discover_repo_files(repo_root: Path, runner: Runner) -> List[Path]: + rc, output = runner(["git", "ls-files"], repo_root) + if rc == 0: + files: List[Path] = [] + for rel in output.splitlines(): + rel = rel.strip() + if not rel: + continue + path = repo_root / rel + if path.is_file(): + files.append(path) + return files + + # Fallback for non-git environments. + files = [] + for path in repo_root.rglob("*"): + if not path.is_file(): + continue + if any(part in SKIP_DIRS for part in path.relative_to(repo_root).parts): + continue + files.append(path) + return files + + +def check_conflict_markers(repo_root: Path, runner: Runner = run_command) -> CheckResult: + start = time.monotonic() + hits: List[str] = [] + + for file_path in _discover_repo_files(repo_root, runner): + rel = file_path.relative_to(repo_root) + if any(part in SKIP_DIRS for part in rel.parts): + continue + + try: + raw = file_path.read_bytes() + except OSError: + continue + + if b"\x00" in raw: + continue + + for lineno, line in enumerate(raw.decode("utf-8", errors="replace").splitlines(), start=1): + stripped = line.strip() + if ( + stripped.startswith(MARKER_PREFIXES[0]) + or stripped == MARKER_PREFIXES[1] + or stripped.startswith(MARKER_PREFIXES[2]) + ): + preview = stripped[:60] + hits.append(f"{rel}:{lineno}: {preview}") + if len(hits) >= 20: + break + if len(hits) >= 20: + break + + if hits: + shown = "\n".join(hits[:10]) + extra = "" if len(hits) <= 10 else f"\n... and {len(hits) - 10} more" + return CheckResult( + name="conflict-markers", + passed=False, + details=f"merge conflict markers found:\n{shown}{extra}", + duration_s=time.monotonic() - start, + ) + + return CheckResult( + name="conflict-markers", + passed=True, + details="no merge conflict markers detected", + duration_s=time.monotonic() - start, + ) + + +def default_yaml_loader() -> Optional[YamlLoader]: + try: + import yaml # type: ignore + except Exception: + return None + return yaml.safe_load + + +def check_yaml_syntax(repo_root: Path, loader: Optional[YamlLoader] = None) -> CheckResult: + start = time.monotonic() + yaml_files: List[Path] = [] + for rel_root in ("src", "tests/fixtures"): + root = repo_root / rel_root + if not root.is_dir(): + continue + for path in sorted(root.rglob("*")): + if path.is_file() and path.suffix.lower() in YAML_EXTENSIONS: + yaml_files.append(path) + # Also check root-level YAML files. + for path in sorted(repo_root.glob("*.yaml")): + if path.is_file(): + yaml_files.append(path) + + if not yaml_files: + return CheckResult( + name="yaml-validate", + passed=True, + details="no YAML files found", + duration_s=time.monotonic() - start, + ) + + yaml_loader = loader or default_yaml_loader() + if yaml_loader is None: + return CheckResult( + name="yaml-validate", + passed=False, + details="PyYAML is required for YAML validation (install with `pip install pyyaml`)", + duration_s=time.monotonic() - start, + ) + + errors: List[str] = [] + for path in yaml_files: + rel = path.relative_to(repo_root) + try: + text = path.read_text(encoding="utf-8") + yaml_loader(text) + except Exception as exc: + errors.append(f"{rel}: {exc}") + + if errors: + shown = "\n".join(errors[:10]) + extra = "" if len(errors) <= 10 else f"\n... and {len(errors) - 10} more" + return CheckResult( + name="yaml-validate", + passed=False, + details=f"invalid YAML detected:\n{shown}{extra}", + duration_s=time.monotonic() - start, + ) + + return CheckResult( + name="yaml-validate", + passed=True, + details=f"validated {len(yaml_files)} YAML files", + duration_s=time.monotonic() - start, + ) + + +def _run_external_check( + *, + name: str, + command: Sequence[str], + repo_root: Path, + runner: Runner, +) -> CheckResult: + start = time.monotonic() + tool = command[0] + if shutil.which(tool) is None: + return CheckResult( + name=name, + passed=False, + details=f"{tool} is not on PATH", + duration_s=time.monotonic() - start, + ) + + rc, output = runner(command, repo_root) + if rc == 0: + return CheckResult( + name=name, + passed=True, + details="ok", + duration_s=time.monotonic() - start, + ) + + details = output or f"{tool} exited with code {rc}" + return CheckResult( + name=name, + passed=False, + details=details, + duration_s=time.monotonic() - start, + ) + + +def check_ruff(repo_root: Path, runner: Runner = run_command) -> CheckResult: + return _run_external_check( + name="ruff", + command=["ruff", "check", "."], + repo_root=repo_root, + runner=runner, + ) + + +def check_tests(repo_root: Path, runner: Runner = run_command) -> CheckResult: + return _run_external_check( + name="tests", + command=["python3", "-m", "pytest", "tests/", "-x", "-q"], + repo_root=repo_root, + runner=runner, + ) + + +def run_preflight( + repo_root: Path, + *, + full: bool, + runner: Runner = run_command, + yaml_loader: Optional[YamlLoader] = None, +) -> List[CheckResult]: + results = [ + check_conflict_markers(repo_root, runner=runner), + check_yaml_syntax(repo_root, loader=yaml_loader), + check_ruff(repo_root, runner=runner), + ] + + if full: + results.append(check_tests(repo_root, runner=runner)) + + return results + + +def print_report(results: Sequence[CheckResult], *, full: bool) -> None: + mode = "extended" if full else "fast" + print(f"Preflight mode: {mode}") + + for result in results: + print(f"[{result.status}] {result.name} ({result.duration_s:.2f}s)") + if result.details and result.details != "ok": + for line in result.details.splitlines(): + print(f" {line}") + + failures = [result for result in results if not result.passed] + if failures: + print(f"\nPreflight FAILED ({len(failures)} check(s) failed).") + else: + print("\nPreflight PASSED.") + + +def parse_args(argv: Sequence[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run unified preflight checks.") + parser.add_argument( + "--repo-root", + default=str(Path(__file__).resolve().parent.parent), + help="Repository root to validate (default: repo containing this script).", + ) + parser.add_argument( + "--full", + action="store_true", + help="Run extended mode (includes pytest).", + ) + return parser.parse_args(list(argv)) + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv or sys.argv[1:]) + repo_root = Path(args.repo_root).resolve() + if not repo_root.is_dir(): + print(f"Repository root not found: {repo_root}", file=sys.stderr) + return 2 + + results = run_preflight(repo_root, full=bool(args.full)) + print_report(results, full=bool(args.full)) + return 0 if all(result.passed for result in results) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/estimate/README.md b/skills/estimate/README.md new file mode 100644 index 0000000..41c9d00 --- /dev/null +++ b/skills/estimate/README.md @@ -0,0 +1,46 @@ +# estimate + +PERT three-point estimation with METR reliability thresholds and wave planning for AI agent tasks. + +## Overview + +Wraps the `agent-estimate` CLI to provide effort estimation, observation validation, and calibration from within an agent runtime. The skill maps user intent to CLI subcommands (`estimate`, `validate`, `calibrate`) and returns output directly. + +## Prerequisites + +- `agent-estimate` installed: `pip install agent-estimate` or `pip install -e '.[dev]'` in the repo +- Default config uses bundled `default_agents.yaml`; pass `--config` to override + +## Runtimes + +- **Claude Code**: Install to `.claude/skills/estimate/SKILL.md` +- **Codex**: Install to `.codex/skills/estimate/SKILL.md` + +## Install + +```bash +# Claude Code +mkdir -p .claude/skills/estimate +cp skills/estimate/claude/SKILL.md .claude/skills/estimate/SKILL.md + +# Codex +mkdir -p .codex/skills/estimate +cp skills/estimate/codex/SKILL.md .codex/skills/estimate/SKILL.md +``` + +## Usage + +```bash +# Claude Code +/estimate Add a login page with OAuth +/estimate --file tasks.md +/estimate --issues 1,2,3 --repo myorg/myrepo +/validate-estimate observation.yaml +/calibrate + +# Codex +# Invoke via AGENTS.md task dispatch or direct skill reference +agent-estimate estimate "Add a login page with OAuth" +agent-estimate validate observation.yaml +agent-estimate calibrate +``` diff --git a/skills/estimate/SKILL.md b/skills/estimate/claude/SKILL.md similarity index 72% rename from skills/estimate/SKILL.md rename to skills/estimate/claude/SKILL.md index 3d3f20c..93f2afe 100644 --- a/skills/estimate/SKILL.md +++ b/skills/estimate/claude/SKILL.md @@ -53,7 +53,11 @@ Parse these optional flags from user input and pass them through verbatim: | `--file ` | `-f` | Path to task file (one task per line) | | `--config ` | `-c` | Path to config YAML with agent definitions | | `--format ` | | Output format: `markdown` (default) or `json` | -| `--review-mode ` | | Review overhead tier: `none` (0 m), `standard` (15 m, default), `complex` (25 m) | +| `--type ` | | Task category: `coding`, `brainstorm`, `research`, `config`, `documentation`, `frontend`, `app_dev`. Omit to auto-detect. | +| `--review-mode ` | | Review overhead tier: `none` (0 m), `standard` (15 m, default), `complex` (25 m), `3-round` (35 m) | +| `--spec-clarity ` | | Spec-clarity modifier (`0.3`–`1.3`) | +| `--warm-context ` | | Warm-context modifier (`0.3`–`1.15`) | +| `--agent-fit ` | | Agent-fit modifier (`0.9`–`1.2`) | | `--issues ` | `-i` | Comma-separated GitHub issue numbers | | `--repo ` | `-r` | GitHub repo (required with `--issues`) | | `--title ` | `-t` | Report title | @@ -61,6 +65,16 @@ Parse these optional flags from user input and pass them through verbatim: If none of `--file`, `--issues`, or a task description is provided, prompt the user: > Please provide a task description, `--file `, or `--issues --repo `. +**Task categories** (`--type`, or auto-detected when omitted): + +- `coding` — feature work, bug fixes, tests, refactors (default tiered PERT model). +- `brainstorm` — pure ideation and design exploration. +- `research` — audits, investigations, OSS comparisons, citation/source-grounded work. Research-grounded brainstorms (citation, OSS, benchmark, source, or landscape signals) route here instead of the flat brainstorm band. +- `config` — deploys, infra, CI/CD, runbooks, monitoring, SRE. +- `documentation` — API docs, guides, README changes, changelogs. +- `frontend` — UI/page work (content patches use 15/25/40; page builds use 40/60/90). +- `app_dev` — app shells and desktop/mobile builds (cold generic L-style prior; use modifiers for warm or highly specified work). + #### For `/validate-estimate` The argument after `/validate-estimate` is the observation YAML file path. Optionally pass `--db ` if provided. @@ -138,5 +152,6 @@ modifiers: - Requires `agent-estimate` installed: `pip install agent-estimate` or `pip install -e .[dev]` in the repo. - Default config uses bundled `default_agents.yaml`. Pass `--config` to override agent definitions. -- `--review-mode` defaults to `standard` (15 m additive; clean 2x-LGTM). Use `complex` for 3+ review rounds. Use `none` for self-merge workflows. +- `--review-mode` defaults to `standard` (15 m additive; clean 2x-LGTM). Use `complex` (25 m) for involved or security-sensitive review, `3-round` (35 m) for explicit 3-round cross-agent review, or `none` for self-merge workflows. - JSON output is available via `--format json`. +- METR p80 reliability thresholds back the over-threshold warnings. Current model keys: `opus_4_7` (Opus 4.7, 90 m), `gpt_5_5` (90 m), `gpt_5_4` (60 m), `gemini_3_1_pro` (45 m), `sonnet_4_6` (30 m), `haiku_4_5` (15 m). `opus_4_x` is a forward-compatible Opus alias; legacy keys (`opus_4_6`, `opus`, `gpt_5`/`5.2`/`5.3`, `gemini_3_pro`, `sonnet`) remain accepted. diff --git a/skills/estimate/codex/SKILL.md b/skills/estimate/codex/SKILL.md new file mode 100644 index 0000000..22f2359 --- /dev/null +++ b/skills/estimate/codex/SKILL.md @@ -0,0 +1,126 @@ +--- +name: estimate +description: Codex skill for running agent-estimate CLI commands (estimate, validate, calibrate). +--- + +# estimate (Codex) + +Use this skill when a user asks to estimate AI-agent effort, compare agent time +with human time, validate an estimate against observed work, or recalibrate +local model factors. + +The skill is command-first: execute the `agent-estimate` CLI and return its +output. Do not invent computed values. + +## Intent Mapping + +| User intent | Command | +| --- | --- | +| Estimate one task | `agent-estimate estimate ""` | +| Estimate tasks from a file | `agent-estimate estimate --file ` | +| Estimate GitHub issues | `agent-estimate estimate --issues --repo ` | +| Validate estimate vs actuals | `agent-estimate validate ` | +| Recompute calibration summary | `agent-estimate calibrate` | + +## Estimate Inputs + +Accept exactly one input source: + +- task description argument +- `--file ` +- `--issues ` with `--repo ` + +If the input source is missing or ambiguous, ask for the missing piece. + +## Estimate Flags + +- `--config ` - custom agent fleet config. +- `--format markdown|json` - output format. +- `--review-mode none|standard|complex|3-round` - additive review tier: + - `none`: +0m + - `standard`: +15m + - `complex`: +25m + - `3-round`: +35m +- `--type coding|brainstorm|research|config|documentation|frontend|app_dev`. +- `--spec-clarity <0.3..1.3>`. +- `--warm-context <0.3..1.15>`. +- `--agent-fit <0.9..1.2>`. +- `--title `. +- `--verbose`. + +When `--type` is omitted, the CLI auto-detects the category. Research-grounded +brainstorms with citation, OSS, benchmark, source, or landscape signals route to +the research band instead of the flat brainstorm band. + +## Type Guidance + +- `coding`: default tiered PERT model for feature work, bug fixes, tests, and refactors. +- `brainstorm`: pure ideation and design exploration. +- `research`: audits, investigations, OSS comparisons, citation/source-grounded work. +- `config`: deploys, infra, CI/CD, runbooks, monitoring, and SRE changes. +- `documentation`: API docs, guides, README changes, changelogs. +- `frontend`: UI/page work. Content patches use 15/25/40; page builds use 40/60/90. +- `app_dev`: app shells and desktop/mobile builds. Uses a cold generic L-style prior; use modifiers for warm or highly specified work. + +## METR Model Keys + +Current threshold keys include: + +- `opus_4_x`, `opus_4_7`, `opus_4_6` +- `gpt_5_5`, `gpt_5_4` +- `gemini_3_1_pro` +- `sonnet_4_6` +- `haiku_4_5` + +Legacy keys such as `opus`, `gpt_5`, `gpt_5_2`, `gpt_5_3`, +`gemini_3_pro`, and `sonnet` remain accepted. + +## Execution Rules + +1. Execute commands from the repository root. +2. Prefer the installed `agent-estimate` binary. +3. If the binary is absent, use `python -m agent_estimate.cli.app`. +4. Capture stdout, stderr, and exit code. +5. If the command fails, return the error concisely and include the attempted command. +6. If the command succeeds, return the CLI output directly. +7. Treat `--format json` as a normal success path. + +## Examples + +```bash +agent-estimate estimate "Add login button with OAuth" +agent-estimate estimate "Audit dependencies for known CVEs" --type research +agent-estimate estimate "Build a landing page" --type frontend +agent-estimate estimate "Build an Electron app shell" --type app_dev --spec-clarity 0.3 --warm-context 0.3 +agent-estimate estimate --file tasks.md +agent-estimate estimate --issues 1,2,3 --repo org/name +agent-estimate estimate --review-mode 3-round "Refactor auth module" +agent-estimate validate observation.yaml --db ~/.agent-estimate/calibration.db +agent-estimate calibrate --db ~/.agent-estimate/calibration.db +``` + +## Observation YAML Example + +```yaml +task_type: frontend +estimated_minutes: 60.0 +actual_work_minutes: 52.0 +actual_total_minutes: 87.0 +file_count: 4 +line_count: 180 +test_count: 3 +execution_mode: single +review_mode: 3-round +review_overhead_minutes: 35.0 +modifiers: + spec_clarity: 1.0 + warm_context: 0.9 +``` + +## Notes + +- Requires `agent-estimate` installed: `pip install agent-estimate` or + `pip install -e '.[dev]'` in this repo. +- Default config uses bundled `default_agents.yaml`. +- Estimates are generic priors. User-local `validate` and `calibrate` should tune + them against local SQLite history over time. diff --git a/skills/estimate/shared/INTENT.md b/skills/estimate/shared/INTENT.md new file mode 100644 index 0000000..65c2fcc --- /dev/null +++ b/skills/estimate/shared/INTENT.md @@ -0,0 +1,66 @@ +# estimate — Shared Intent + +## What it does + +Wraps the `agent-estimate` CLI to provide PERT three-point effort estimation with METR reliability thresholds and wave planning for AI agent tasks. The skill is command-first: it maps user intent to a CLI subcommand, executes it, and returns the output directly. + +## Subcommands + +| Intent | Command | +|--------|---------| +| Estimate one or more tasks | `agent-estimate estimate ...` | +| Validate estimate vs actuals | `agent-estimate validate ...` | +| Recompute calibration summary | `agent-estimate calibrate ...` | + +## CLI flags + +### `estimate` + +Accepts exactly one input source: +- Task description as a positional argument +- `--file ` — path to task file (one task per line) +- `--issues ` — comma-separated GitHub issue numbers (requires `--repo `) + +Optional flags: +- `--config ` — path to config YAML with agent definitions +- `--format markdown|json` — output format (default: `markdown`) +- `--review-mode none|standard|complex` — review overhead tier +- `--title ` — report title +- `--verbose` — enable debug logging + +If no input source is provided, prompt the user. + +### `validate` + +- Required: observation YAML path +- Optional: `--db ` + +### `calibrate` + +- Optional: `--db ` (default: `~/.agent-estimate/calibration.db`) + +## Observation YAML format + +```yaml +task_type: feature +estimated_minutes: 45.0 +actual_work_minutes: 52.0 +actual_total_minutes: 60.0 +file_count: 3 +line_count: 120 +test_count: 5 +execution_mode: single +review_mode: standard +review_overhead_minutes: 8.0 +modifiers: + spec_clarity: 1.0 + warm_context: 0.9 +``` + +## Acceptance criteria + +- CLI command is constructed correctly from user input +- Command is executed and stdout/stderr captured +- On success, CLI output is returned directly (no post-processing) +- On failure, error message and attempted command are shown +- JSON output (`--format json`) is a normal success path diff --git a/skills/estimate/skill.yaml b/skills/estimate/skill.yaml new file mode 100644 index 0000000..6b70bc7 --- /dev/null +++ b/skills/estimate/skill.yaml @@ -0,0 +1,12 @@ +id: estimate +name: Estimate +category: estimation +summary: PERT three-point estimation with METR reliability thresholds and wave planning for AI agent tasks. +license: Apache-2.0 +compatible_runtimes: + - claude-code + - codex +requires_oacp: ">=0.1.0" +public_status: staging +owners: + - kiloloop diff --git a/src/agent_estimate/adapters/github_ghcli.py b/src/agent_estimate/adapters/github_ghcli.py index fa2d6f2..ab34e75 100644 --- a/src/agent_estimate/adapters/github_ghcli.py +++ b/src/agent_estimate/adapters/github_ghcli.py @@ -4,6 +4,7 @@ import json import subprocess +import time from typing import Callable, Sequence from agent_estimate.adapters.github_adapter import ( @@ -11,6 +12,7 @@ GitHubIssue, build_task_description, ) +from agent_estimate.audit import emit_audit_event class GitHubGhCliAdapter: @@ -23,17 +25,40 @@ def fetch_issues_by_numbers(self, repo: str, issue_numbers: Sequence[int]) -> li """Fetch specific issues by number.""" issues: list[GitHubIssue] = [] for issue_number in issue_numbers: - output = self._runner( - [ - "gh", - "issue", - "view", - str(issue_number), - "--repo", - repo, - "--json", - "number,title,body", - ], + args = [ + "gh", + "issue", + "view", + str(issue_number), + "--repo", + repo, + "--json", + "number,title,body", + ] + started_at = time.perf_counter() + try: + output = self._runner(args) + except Exception as exc: + emit_audit_event( + "api_call", + action="github_issue_view", + outcome="error", + duration_ms=(time.perf_counter() - started_at) * 1000.0, + client="gh", + endpoint="gh issue view", + repo=repo, + issue_number=issue_number, + error_type=type(exc).__name__, + ) + raise + emit_audit_event( + "api_call", + action="github_issue_view", + duration_ms=(time.perf_counter() - started_at) * 1000.0, + client="gh", + endpoint="gh issue view", + repo=repo, + issue_number=issue_number, ) payload = json.loads(output) issues.append(_parse_issue(payload)) @@ -47,22 +72,47 @@ def fetch_issues_by_label( state: str = "open", ) -> list[GitHubIssue]: """Fetch issues by label with a high limit for CLI pagination fallback.""" - output = self._runner( - [ - "gh", - "issue", - "list", - "--repo", - repo, - "--label", - label, - "--state", - state, - "--limit", - "1000", - "--json", - "number,title,body", - ], + args = [ + "gh", + "issue", + "list", + "--repo", + repo, + "--label", + label, + "--state", + state, + "--limit", + "1000", + "--json", + "number,title,body", + ] + started_at = time.perf_counter() + try: + output = self._runner(args) + except Exception as exc: + emit_audit_event( + "api_call", + action="github_issue_list", + outcome="error", + duration_ms=(time.perf_counter() - started_at) * 1000.0, + client="gh", + endpoint="gh issue list", + repo=repo, + label=label, + state=state, + error_type=type(exc).__name__, + ) + raise + emit_audit_event( + "api_call", + action="github_issue_list", + duration_ms=(time.perf_counter() - started_at) * 1000.0, + client="gh", + endpoint="gh issue list", + repo=repo, + label=label, + state=state, ) payload = json.loads(output) if not isinstance(payload, list): diff --git a/src/agent_estimate/adapters/github_rest.py b/src/agent_estimate/adapters/github_rest.py index e006cfb..4a01afa 100644 --- a/src/agent_estimate/adapters/github_rest.py +++ b/src/agent_estimate/adapters/github_rest.py @@ -16,6 +16,7 @@ GitHubIssue, build_task_description, ) +from agent_estimate.audit import emit_audit_event BASE_URL = "https://api.github.com" @@ -111,13 +112,34 @@ def _request_json(self, url: str) -> tuple[object, dict[str, str]]: } for attempt in range(self._max_retries + 1): + started_at = time.perf_counter() status, response_headers, body = self._request_fn(url, headers) + duration_ms = (time.perf_counter() - started_at) * 1000.0 normalized_headers = {key.lower(): value for key, value in response_headers.items()} if status < 400: + emit_audit_event( + "api_call", + action="github_rest_request", + duration_ms=duration_ms, + client="github_rest", + endpoint=url, + status_code=status, + ) return json.loads(body), normalized_headers if _is_rate_limited(status, normalized_headers) and attempt < self._max_retries: + emit_audit_event( + "api_call", + action="github_rest_request", + outcome="retry", + level="WARNING", + duration_ms=duration_ms, + client="github_rest", + endpoint=url, + status_code=status, + retry_attempt=attempt + 1, + ) self._sleep_fn( _compute_retry_delay( headers=normalized_headers, @@ -128,6 +150,16 @@ def _request_json(self, url: str) -> tuple[object, dict[str, str]]: ) continue + emit_audit_event( + "api_call", + action="github_rest_request", + outcome="error", + level="ERROR", + duration_ms=duration_ms, + client="github_rest", + endpoint=url, + status_code=status, + ) raise GitHubAdapterError( f"GitHub API request failed with status {status} for {url}: {body[:200]}", ) @@ -147,6 +179,12 @@ def _default_request(self, url: str, headers: Mapping[str, str]) -> tuple[int, d def _resolve_github_token() -> str: token = os.getenv("GITHUB_TOKEN") if token: + emit_audit_event( + "authentication_event", + action="resolve_github_token", + provider="env", + auth_target="github", + ) return token result = subprocess.run( @@ -157,10 +195,25 @@ def _resolve_github_token() -> str: ) if result.returncode != 0 or not result.stdout.strip(): message = result.stderr.strip() or "gh auth token returned no token" + emit_audit_event( + "authentication_event", + action="resolve_github_token", + outcome="error", + level="ERROR", + provider="gh", + auth_target="github", + error_type="gh_auth_failure", + ) raise GitHubAdapterError( "GitHub authentication required. Set GITHUB_TOKEN or login with gh CLI. " f"Details: {message}", ) + emit_audit_event( + "authentication_event", + action="resolve_github_token", + provider="gh", + auth_target="github", + ) return result.stdout.strip() diff --git a/src/agent_estimate/audit.py b/src/agent_estimate/audit.py new file mode 100644 index 0000000..94ed691 --- /dev/null +++ b/src/agent_estimate/audit.py @@ -0,0 +1,213 @@ +"""Structured audit logging helpers.""" + +from __future__ import annotations + +import json +import os +import re +import sys +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from threading import RLock +from typing import Any, Mapping +from uuid import uuid4 + +_LEVEL_ORDER = { + "DEBUG": 10, + "INFO": 20, + "WARNING": 30, + "ERROR": 40, +} +_SENSITIVE_KEY_SEGMENTS = ( + "authorization", + "cookie", + "email", + "password", + "secret", + "token", +) +_SENSITIVE_KEY_PAIRS = { + ("access", "key"), + ("api", "key"), + ("private", "key"), + ("refresh", "token"), +} +_SENSITIVE_VALUE_PATTERNS = ( + re.compile(r"Bearer\s+[A-Za-z0-9._-]+", re.IGNORECASE), + re.compile(r"github_pat_[A-Za-z0-9_]+"), + re.compile(r"gh[pousr]_[A-Za-z0-9]+"), +) +_EMAIL_PATTERN = re.compile(r"\b[^@\s]+@[^@\s]+\.[^@\s]+\b") +_MAX_STRING_LENGTH = 240 + +_audit_logger: "AuditLogger | None" = None +# Guards swaps of the process-global logger instance. +_audit_lock = RLock() + + +@dataclass(frozen=True) +class AuditConfig: + """Runtime audit logging configuration.""" + + enabled: bool + level: str + destination: str + actor: str + environment: str + + @classmethod + def from_env(cls) -> "AuditConfig": + enabled_raw = os.getenv("AGENT_ESTIMATE_AUDIT_ENABLED") + destination = os.getenv("AGENT_ESTIMATE_AUDIT_DESTINATION", "").strip() + enabled = _parse_bool(enabled_raw) if enabled_raw is not None else bool(destination) + level = os.getenv("AGENT_ESTIMATE_AUDIT_LEVEL", "INFO").strip().upper() or "INFO" + if level not in _LEVEL_ORDER: + level = "INFO" + if not destination: + destination = "stderr" + actor = os.getenv("AGENT_ESTIMATE_AUDIT_ACTOR", "agent-estimate").strip() or "agent-estimate" + environment = os.getenv("AGENT_ESTIMATE_ENVIRONMENT", "local").strip() or "local" + return cls( + enabled=enabled, + level=level, + destination=destination, + actor=actor, + environment=environment, + ) + + +class AuditLogger: + """Write scrubbed JSON events to a configured sink.""" + + def __init__(self, config: AuditConfig) -> None: + self._config = config + # Serializes writes for this logger instance. + self._lock = RLock() + + def emit( + self, + event_type: str, + *, + level: str = "INFO", + outcome: str = "success", + duration_ms: float | None = None, + **details: Any, + ) -> None: + normalized_level = level.upper() + if not self._config.enabled: + return + if _LEVEL_ORDER.get(normalized_level, 100) < _LEVEL_ORDER[self._config.level]: + return + + event: dict[str, Any] = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "event_id": uuid4().hex, + "level": normalized_level, + "event_type": event_type, + "actor": self._config.actor, + "environment": self._config.environment, + "outcome": outcome, + "details": _scrub(details), + } + if duration_ms is not None: + event["duration_ms"] = round(duration_ms, 3) + + self._write_line(json.dumps(event, sort_keys=True)) + + def _write_line(self, line: str) -> None: + with self._lock: + if self._config.destination == "stdout": + print(line, file=sys.stdout) + return + if self._config.destination == "stderr": + print(line, file=sys.stderr) + return + + destination = Path(self._config.destination) + destination.parent.mkdir(parents=True, exist_ok=True) + # Reopen per write to keep the CLI logger stateless; revisit if a + # long-lived service needs a persistent buffered handle. + destination_fd = os.open( + destination, + os.O_APPEND | os.O_CREAT | os.O_WRONLY, + 0o600, + ) + with os.fdopen(destination_fd, "a", encoding="utf-8") as handle: + handle.write(f"{line}\n") + + +def configure_audit_logger(config: AuditConfig | None = None) -> AuditLogger: + """Configure the global audit logger.""" + global _audit_logger + with _audit_lock: + _audit_logger = AuditLogger(config or AuditConfig.from_env()) + return _audit_logger + + +def reset_audit_logger() -> None: + """Reset the global audit logger so tests can reconfigure it.""" + global _audit_logger + with _audit_lock: + _audit_logger = None + + +def emit_audit_event( + event_type: str, + *, + level: str = "INFO", + outcome: str = "success", + duration_ms: float | None = None, + **details: Any, +) -> None: + """Emit one structured audit event.""" + get_audit_logger().emit( + event_type, + level=level, + outcome=outcome, + duration_ms=duration_ms, + **details, + ) + + +def get_audit_logger() -> AuditLogger: + """Return the configured global audit logger.""" + global _audit_logger + with _audit_lock: + if _audit_logger is None: + _audit_logger = AuditLogger(AuditConfig.from_env()) + return _audit_logger + + +def _parse_bool(value: str | None) -> bool: + if value is None: + return False + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _scrub(value: Any, *, key: str | None = None) -> Any: + if key and _is_sensitive_key(key): + return "[REDACTED]" + if isinstance(value, Mapping): + return {str(k): _scrub(v, key=str(k)) for k, v in value.items()} + if isinstance(value, (list, tuple, set)): + return [_scrub(item) for item in value] + if isinstance(value, Path): + return value.name + if isinstance(value, str): + scrubbed = value + for pattern in _SENSITIVE_VALUE_PATTERNS: + scrubbed = pattern.sub("[REDACTED]", scrubbed) + scrubbed = _EMAIL_PATTERN.sub("[REDACTED]", scrubbed) + if len(scrubbed) > _MAX_STRING_LENGTH: + scrubbed = f"{scrubbed[:_MAX_STRING_LENGTH]}..." + return scrubbed + return value + + +def _is_sensitive_key(key: str) -> bool: + normalized = re.sub(r"(? None: """Global options for agent-estimate.""" + configure_audit_logger() if verbose: logging.basicConfig(level=logging.DEBUG, format="%(levelname)s: %(message)s", force=True) diff --git a/src/agent_estimate/cli/commands/_pipeline.py b/src/agent_estimate/cli/commands/_pipeline.py index c6d0239..9f8e917 100644 --- a/src/agent_estimate/cli/commands/_pipeline.py +++ b/src/agent_estimate/cli/commands/_pipeline.py @@ -12,7 +12,6 @@ ReviewMode, TaskEstimate, TaskNode, - TaskType, WavePlan, auto_correct_tier, classify_task, @@ -20,9 +19,11 @@ check_metr_threshold, compute_human_equivalent, detect_estimation_category, + estimate_app_dev, estimate_brainstorm, estimate_config_sre, estimate_documentation, + estimate_frontend, estimate_research, estimate_task, load_metr_thresholds, @@ -89,7 +90,7 @@ def _estimate_by_category( fallback_threshold=fallback, agent_name=agent_name, ) - human_eq = compute_human_equivalent(est.total_expected_minutes, TaskType.UNKNOWN) + human_eq = compute_human_equivalent(est.total_expected_minutes, est.sizing.task_type) est = replace(est, human_equivalent_minutes=human_eq) return est, task_tier_warnings @@ -103,7 +104,7 @@ def _estimate_by_category( fallback_threshold=fallback, agent_name=agent_name, ) - human_eq = compute_human_equivalent(est.total_expected_minutes, TaskType.UNKNOWN) + human_eq = compute_human_equivalent(est.total_expected_minutes, est.sizing.task_type) est = replace(est, human_equivalent_minutes=human_eq) return est, task_tier_warnings @@ -117,7 +118,7 @@ def _estimate_by_category( fallback_threshold=fallback, agent_name=agent_name, ) - human_eq = compute_human_equivalent(est.total_expected_minutes, TaskType.UNKNOWN) + human_eq = compute_human_equivalent(est.total_expected_minutes, est.sizing.task_type) est = replace(est, human_equivalent_minutes=human_eq) return est, task_tier_warnings @@ -131,7 +132,35 @@ def _estimate_by_category( fallback_threshold=fallback, agent_name=agent_name, ) - human_eq = compute_human_equivalent(est.total_expected_minutes, TaskType.UNKNOWN) + human_eq = compute_human_equivalent(est.total_expected_minutes, est.sizing.task_type) + est = replace(est, human_equivalent_minutes=human_eq) + return est, task_tier_warnings + + if category == EstimationCategory.FRONTEND: + est = estimate_frontend( + desc, + modifiers, + review_mode=review_mode, + model_key=model_key, + thresholds=thresholds, + fallback_threshold=fallback, + agent_name=agent_name, + ) + human_eq = compute_human_equivalent(est.total_expected_minutes, est.sizing.task_type) + est = replace(est, human_equivalent_minutes=human_eq) + return est, task_tier_warnings + + if category == EstimationCategory.APP_DEV: + est = estimate_app_dev( + desc, + modifiers, + review_mode=review_mode, + model_key=model_key, + thresholds=thresholds, + fallback_threshold=fallback, + agent_name=agent_name, + ) + human_eq = compute_human_equivalent(est.total_expected_minutes, est.sizing.task_type) est = replace(est, human_equivalent_minutes=human_eq) return est, task_tier_warnings diff --git a/src/agent_estimate/cli/commands/estimate.py b/src/agent_estimate/cli/commands/estimate.py index 9dff07d..d2f0a27 100644 --- a/src/agent_estimate/cli/commands/estimate.py +++ b/src/agent_estimate/cli/commands/estimate.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +import time from pathlib import Path from typing import NoReturn, Optional @@ -11,9 +12,10 @@ from agent_estimate.adapters.config_loader import load_config, load_default_config from agent_estimate.adapters.github_adapter import GitHubAdapterError from agent_estimate.adapters.github_ghcli import GitHubGhCliAdapter +from agent_estimate.audit import emit_audit_event from agent_estimate.cli.commands._pipeline import run_estimate_pipeline from agent_estimate.cli.commands.github import parse_issue_selection -from agent_estimate.core import EstimationCategory, ReviewMode +from agent_estimate.core import EstimationCategory, EstimationConfig, ReviewMode from agent_estimate.core.history import infer_warm_context from agent_estimate.render import render_json_report, render_markdown_report @@ -34,7 +36,10 @@ def run( review_mode: str = typer.Option( "standard", "--review-mode", - help="Review overhead tier: none (0 m, self-merge), standard (15 m, 2x-LGTM), complex (25 m, 3+ rounds).", + help=( + "Review overhead tier: none (0 m), standard (15 m), complex (25 m), " + "3-round (35 m)." + ), ), issues: Optional[str] = typer.Option( None, @@ -102,12 +107,17 @@ def run( None, "--type", help=( - "Task category: coding (default), brainstorm, research, config, documentation. " - "Auto-detected from description when not provided." + "Task category: coding (default), brainstorm, research, config, " + "documentation, frontend, app_dev. Auto-detected from description " + "when not provided." ), ), ) -> None: """Estimate effort for one or more task descriptions.""" + started_at = time.perf_counter() + config_path = config + input_source = "task" if task is not None else "file" if file is not None else "issues" + # --- Resolve input source (exactly one) --- sources = sum([task is not None, file is not None, issues is not None]) if sources == 0: @@ -149,17 +159,34 @@ def run( mode = ReviewMode(review_mode) except ValueError: _error( - f"Invalid review mode: {review_mode!r}. Use none, standard, or complex.", 2 + f"Invalid review mode: {review_mode!r}. " + "Use none, standard, complex, or 3-round.", + 2, ) # --- Load config --- try: - cfg = load_config(config) if config else load_default_config() + cfg = load_config(config_path) if config_path else load_default_config() except FileNotFoundError: - _error(f"Config file not found: {config}", 2) + _error(f"Config file not found: {config_path}", 2) except ValueError as exc: _error(f"Config validation error: {exc}", 2) + baseline_cfg = cfg + if config_path: + try: + baseline_cfg = load_default_config() + except (FileNotFoundError, ValueError): + baseline_cfg = cfg + emit_audit_event( + "configuration_change", + action="config_load", + trigger="cli --config" if config_path else "packaged-default", + source=config_path.name if config_path else "default_agents.yaml", + changed_fields=_summarize_config_changes(cfg, baseline_cfg), + agent_names=[agent.name for agent in cfg.agents], + ) + # --- Infer warm context from dispatch history --- history_path = history_file if history_path is None: @@ -211,6 +238,28 @@ def run( except RuntimeError as exc: _error(f"Runtime error: {exc}", 1) + emit_audit_event( + "estimation_request", + action="estimate", + duration_ms=(time.perf_counter() - started_at) * 1000.0, + request={ + "input_source": input_source, + "description_count": len(descriptions), + "format": format, + "review_mode": mode.value, + "task_type": estimation_category.value if estimation_category is not None else "auto", + "config_source": config_path.name if config_path else "default_agents.yaml", + }, + result={ + "task_count": len(report.tasks), + "critical_path_length": len(report.critical_path), + "best_case_minutes": report.timeline.best_case_minutes, + "expected_case_minutes": report.timeline.expected_case_minutes, + "worst_case_minutes": report.timeline.worst_case_minutes, + "review_overhead_minutes": report.review_overhead_minutes, + }, + ) + # --- Output --- if format == "markdown": typer.echo(render_markdown_report(report)) @@ -224,3 +273,34 @@ def _error(message: str, exit_code: int) -> NoReturn: """Print error to stderr and exit.""" typer.echo(f"Error: {message}", err=True) raise typer.Exit(code=exit_code) + + +def _summarize_config_changes( + current: EstimationConfig, + baseline: EstimationConfig, +) -> list[str]: + current_dump = current.model_dump() + baseline_dump = baseline.model_dump() + changed_fields: list[str] = [] + + current_settings = current_dump["settings"] + baseline_settings = baseline_dump["settings"] + for key, value in current_settings.items(): + if baseline_settings.get(key) != value: + changed_fields.append(f"settings.{key}") + + current_agents = {agent["name"]: agent for agent in current_dump["agents"]} + baseline_agents = {agent["name"]: agent for agent in baseline_dump["agents"]} + if set(current_agents) != set(baseline_agents): + changed_fields.append("agents") + else: + for agent_name in sorted(current_agents): + current_agent = current_agents[agent_name] + baseline_agent = baseline_agents[agent_name] + for field, value in current_agent.items(): + if field == "name": + continue + if baseline_agent.get(field) != value: + changed_fields.append(f"agents.{agent_name}.{field}") + + return sorted(set(changed_fields)) diff --git a/src/agent_estimate/cli/commands/session.py b/src/agent_estimate/cli/commands/session.py index 07a7de7..abee2d5 100644 --- a/src/agent_estimate/cli/commands/session.py +++ b/src/agent_estimate/cli/commands/session.py @@ -2,10 +2,12 @@ from __future__ import annotations +import time from typing import NoReturn, Optional import typer +from agent_estimate.audit import emit_audit_event from agent_estimate.core.session import ( DEFAULT_COORDINATION_OVERHEAD_MINUTES, SESSION_TYPE_DURATIONS, @@ -59,6 +61,7 @@ def run( ), ) -> None: """Estimate wall-clock and agent-minutes for a multi-agent session.""" + started_at = time.perf_counter() overhead = ( coordination_overhead if coordination_overhead is not None @@ -76,6 +79,23 @@ def run( except ValueError as exc: _error(str(exc), 2) + emit_audit_event( + "estimation_request", + action="session_estimate", + duration_ms=(time.perf_counter() - started_at) * 1000.0, + request={ + "agents": agents, + "rounds": rounds, + "task_type": type, + "format": format, + }, + result={ + "wall_clock_minutes": result.wall_clock_minutes, + "agent_minutes": result.agent_minutes, + "coordination_overhead_minutes": result.coordination_overhead_minutes, + }, + ) + if format == "json": import json diff --git a/src/agent_estimate/core/__init__.py b/src/agent_estimate/core/__init__.py index cf4cfeb..8c45d4b 100644 --- a/src/agent_estimate/core/__init__.py +++ b/src/agent_estimate/core/__init__.py @@ -35,9 +35,11 @@ from agent_estimate.core.sizing import TierCorrection, auto_correct_tier, classify_task from agent_estimate.core.task_type_models import ( detect_estimation_category, + estimate_app_dev, estimate_brainstorm, estimate_config_sre, estimate_documentation, + estimate_frontend, estimate_research, ) from agent_estimate.core.wave_planner import plan_waves @@ -71,9 +73,11 @@ "compute_pert", "compute_review_overhead", "detect_estimation_category", + "estimate_app_dev", "estimate_brainstorm", "estimate_config_sre", "estimate_documentation", + "estimate_frontend", "estimate_research", "estimate_task", "get_human_multiplier", diff --git a/src/agent_estimate/core/human_comparison.py b/src/agent_estimate/core/human_comparison.py index e6847cf..ab35cbe 100644 --- a/src/agent_estimate/core/human_comparison.py +++ b/src/agent_estimate/core/human_comparison.py @@ -15,6 +15,8 @@ TaskType.REFACTOR: (2.0, 3.5), TaskType.TEST: (2.5, 4.5), TaskType.DOCS: (3.0, 6.0), + TaskType.FRONTEND: (2.5, 3.5), + TaskType.APP_DEV: (2.5, 3.5), TaskType.UNKNOWN: (2.0, 4.0), } diff --git a/src/agent_estimate/core/models.py b/src/agent_estimate/core/models.py index 351b642..10bf2d5 100644 --- a/src/agent_estimate/core/models.py +++ b/src/agent_estimate/core/models.py @@ -94,6 +94,8 @@ class TaskType(enum.Enum): REFACTOR = "refactor" TEST = "test" DOCS = "docs" + FRONTEND = "frontend" + APP_DEV = "app_dev" UNKNOWN = "unknown" @@ -105,11 +107,15 @@ class EstimationCategory(enum.Enum): RESEARCH — time-boxed; 15-45m depending on depth CONFIG_SRE — flat + verification; ~15-30m DOCUMENTATION — line-count based; similar to coding but lower floor + FRONTEND — bimodal UI/page model; content patch vs page build + APP_DEV — app-shell/build model; generic cold L prior Aliases accepted via ``_missing_``: "sre" → CONFIG_SRE "config_sre" → CONFIG_SRE "docs" → DOCUMENTATION + "ui" → FRONTEND + "app-dev" → APP_DEV """ CODING = "coding" @@ -117,6 +123,8 @@ class EstimationCategory(enum.Enum): RESEARCH = "research" CONFIG_SRE = "config" DOCUMENTATION = "documentation" + FRONTEND = "frontend" + APP_DEV = "app_dev" @classmethod def _missing_(cls, value: object) -> "EstimationCategory | None": @@ -125,6 +133,13 @@ def _missing_(cls, value: object) -> "EstimationCategory | None": "sre": cls.CONFIG_SRE, "config_sre": cls.CONFIG_SRE, "docs": cls.DOCUMENTATION, + "ui": cls.FRONTEND, + "front_end": cls.FRONTEND, + "front-end": cls.FRONTEND, + "web": cls.FRONTEND, + "app": cls.APP_DEV, + "app-dev": cls.APP_DEV, + "appdev": cls.APP_DEV, } if isinstance(value, str): return _aliases.get(value.lower()) @@ -136,7 +151,8 @@ class ReviewMode(enum.Enum): NONE — self-merge, no cross-agent review (0 m) STANDARD — clean 2x-LGTM, 1-2 rounds (15 m) - COMPLEX — 3+ rounds, security-sensitive (25 m) + COMPLEX — involved/security-sensitive review (25 m) + THREE_ROUND — explicit 3-round cross-agent review (35 m) Legacy aliases kept for backwards compatibility: "self" → NONE (maps to 0 m; was previously 7.5 m) @@ -146,6 +162,7 @@ class ReviewMode(enum.Enum): NONE = "none" STANDARD = "standard" COMPLEX = "complex" + THREE_ROUND = "3-round" @classmethod def _missing_(cls, value: object) -> "ReviewMode | None": @@ -153,6 +170,10 @@ def _missing_(cls, value: object) -> "ReviewMode | None": _legacy: dict[str, "ReviewMode"] = { "self": cls.NONE, "2x-lgtm": cls.STANDARD, + "three-round": cls.THREE_ROUND, + "three_round": cls.THREE_ROUND, + "3_round": cls.THREE_ROUND, + "3round": cls.THREE_ROUND, } if isinstance(value, str): return _legacy.get(value) diff --git a/src/agent_estimate/core/modifiers.py b/src/agent_estimate/core/modifiers.py index 47abbaa..542724a 100644 --- a/src/agent_estimate/core/modifiers.py +++ b/src/agent_estimate/core/modifiers.py @@ -15,7 +15,8 @@ _REVIEW_OVERHEAD: dict[ReviewMode, float] = { ReviewMode.NONE: 0.0, # self-merge, no cross-agent review ReviewMode.STANDARD: 15.0, # clean 2x-LGTM, 1-2 rounds - ReviewMode.COMPLEX: 25.0, # 3+ rounds, security-sensitive, new algorithms + ReviewMode.COMPLEX: 25.0, # involved review, security-sensitive, new algorithms + ReviewMode.THREE_ROUND: 35.0, # explicit 3-round cross-agent review } diff --git a/src/agent_estimate/core/pert.py b/src/agent_estimate/core/pert.py index 6703e52..fea6963 100644 --- a/src/agent_estimate/core/pert.py +++ b/src/agent_estimate/core/pert.py @@ -23,12 +23,16 @@ logger = logging.getLogger("agent_estimate") _MODEL_KEY_ALIASES: dict[str, str] = { - # Current fleet (2026-03) + # Current fleet (2026-05) + "opus_4_x": "opus_4_x", + "opus_4_7": "opus_4_7", "opus_4_6": "opus_4_6", - "claude": "opus_4_6", - "claude_opus": "opus_4_6", + "claude": "opus_4_7", + "claude_opus": "opus_4_7", + "gpt_5_5": "gpt_5_5", + "codex": "gpt_5_5", + "codex_latest": "gpt_5_5", "gpt_5_4": "gpt_5_4", - "codex": "gpt_5_4", "production": "gpt_5_4", "gemini_3_1_pro": "gemini_3_1_pro", "gemini": "gemini_3_1_pro", @@ -59,9 +63,9 @@ def _resolve_threshold_model_key(model_key: str, *, agent_name: str | None = Non if normalized_model == "frontier" and agent_name: normalized_agent = _normalize_model_token(agent_name) if "claude" in normalized_agent: - return "opus_4_6" + return "opus_4_7" if "codex" in normalized_agent: - return "gpt_5_4" + return "gpt_5_5" if "gemini" in normalized_agent: return "gemini_3_1_pro" diff --git a/src/agent_estimate/core/task_type_models.py b/src/agent_estimate/core/task_type_models.py index 1490521..b5bc85c 100644 --- a/src/agent_estimate/core/task_type_models.py +++ b/src/agent_estimate/core/task_type_models.py @@ -28,20 +28,13 @@ # --------------------------------------------------------------------------- _CATEGORY_PATTERNS: list[tuple[re.Pattern[str], EstimationCategory]] = [ - # Brainstorm — ideation, design, discussion + # Research-grounded brainstorms must route to research before flat brainstorm. ( re.compile( - r"\b(brainstorm|ideate|explore ideas?|design session|whiteboard|discuss|" - r"spike|discovery|kickoff|alignment)\b", - re.I, - ), - EstimationCategory.BRAINSTORM, - ), - # Research — investigation, analysis, evaluation - ( - re.compile( - r"\b(research|investigate|analyze|analyse|survey|evaluate|" - r"feasibility|benchmarks?|compare|assessment|audit)\b", + r"(?=.*\b(brainstorm|ideate|explore ideas?|spike|discovery)\b)" + r"(?=.*\b(research|citation|citations|sources?|primary[- ]source|" + r"evidence|oss|open[- ]source|github|compare|survey|benchmark|" + r"landscape)\b)", re.I, ), EstimationCategory.RESEARCH, @@ -58,6 +51,43 @@ ), EstimationCategory.CONFIG_SRE, ), + # Frontend/UI — page content patches and page/component builds + ( + re.compile( + r"\b(front[- ]?end|ui|ux|landing page|web page|page build|" + r"component page|design system|mdx|seo snippet|structured data|" + r"copy update|single[- ]section|hero section)\b", + re.I, + ), + EstimationCategory.FRONTEND, + ), + # App development — app shell, desktop/mobile apps, Electron/native shells + ( + re.compile( + r"\b(app[- ]?dev|app shell|desktop app|mobile app|electron|tauri|" + r"native app|mac app|ios app|android app|application shell)\b", + re.I, + ), + EstimationCategory.APP_DEV, + ), + # Brainstorm — ideation, design, discussion + ( + re.compile( + r"\b(brainstorm|ideate|explore ideas?|design session|whiteboard|discuss|" + r"spike|discovery|kickoff|alignment)\b", + re.I, + ), + EstimationCategory.BRAINSTORM, + ), + # Research — investigation, analysis, evaluation + ( + re.compile( + r"\b(research|investigate|analyze|analyse|survey|evaluate|" + r"feasibility|benchmarks?|compare|assessment|audit)\b", + re.I, + ), + EstimationCategory.RESEARCH, + ), # Documentation — writing, docs, readme ( re.compile( @@ -87,6 +117,13 @@ # Documentation: line-count based — lower floor than coding _DOCUMENTATION_BASELINES = (10.0, 25.0, 45.0) +# Frontend/UI: bimodal content-patch vs page-build regimes +_FRONTEND_CONTENT_BASELINES = (15.0, 25.0, 40.0) +_FRONTEND_BUILD_BASELINES = (40.0, 60.0, 90.0) + +# App development: generic cold L-style prior; modifiers collapse warm/specified work +_APP_DEV_BASELINES = (45.0, 95.0, 180.0) + # Depth keywords that push research to the "deep" band _RESEARCH_DEEP_PATTERNS = re.compile( r"\b(deep|thorough|comprehensive|in[-\s]?depth|extensive|detailed|" @@ -94,6 +131,12 @@ re.I, ) +_FRONTEND_CONTENT_PATTERNS = re.compile( + r"\b(content|copy|seo|snippet|structured data|metadata|mdx|markdown|" + r"single[- ]section|text update|copy update|small patch|minor patch)\b", + re.I, +) + def detect_estimation_category(text: str) -> EstimationCategory: """Infer EstimationCategory from task title / description text. @@ -109,15 +152,21 @@ def detect_estimation_category(text: str) -> EstimationCategory: def _make_non_coding_sizing( - o: float, m: float, p: float, label: str + o: float, + m: float, + p: float, + label: str, + *, + task_type: TaskType = TaskType.UNKNOWN, + tier: SizeTier = SizeTier.S, ) -> SizingResult: """Build a synthetic SizingResult for non-coding tasks.""" return SizingResult( - tier=SizeTier.S, # tier is not meaningful for non-coding; use S as placeholder + tier=tier, baseline_optimistic=o, baseline_most_likely=m, baseline_pessimistic=p, - task_type=TaskType.UNKNOWN, + task_type=task_type, signals=(label,), ) @@ -329,3 +378,123 @@ def estimate_documentation( metr_warning=metr_warning, estimation_category=EstimationCategory.DOCUMENTATION, ) + + +def estimate_frontend( + description: str, + modifiers: ModifierSet, + *, + review_mode=None, + model_key: str = "opus", + thresholds=None, + fallback_threshold: float = 40.0, + agent_name: str | None = None, + human_equivalent_minutes: float | None = None, +) -> TaskEstimate: + """Estimate a frontend/UI task. + + Frontend work is bimodal: content/patch tasks use a smaller band, while + page/component builds use a larger page-build band. + """ + if review_mode is None: + review_mode = ReviewMode.NONE + + if _FRONTEND_CONTENT_PATTERNS.search(description or ""): + o, m, p = _FRONTEND_CONTENT_BASELINES + label = "frontend-content-model" + tier = SizeTier.S + else: + o, m, p = _FRONTEND_BUILD_BASELINES + label = "frontend-build-model" + tier = SizeTier.M + + sizing = _make_non_coding_sizing( + o, + m, + p, + label, + task_type=TaskType.FRONTEND, + tier=tier, + ) + + adjusted_o = o * modifiers.combined + adjusted_m = m * modifiers.combined + adjusted_p = p * modifiers.combined + + pert = compute_pert(adjusted_o, adjusted_m, adjusted_p) + review_minutes = compute_review_overhead(review_mode) + total = pert.expected + review_minutes + + metr_warning = check_metr_threshold( + model_key, + total, + thresholds=thresholds, + fallback_threshold=fallback_threshold, + agent_name=agent_name, + ) + + return TaskEstimate( + sizing=sizing, + pert=pert, + modifiers=modifiers, + review_minutes=review_minutes, + total_expected_minutes=total, + human_equivalent_minutes=human_equivalent_minutes, + metr_warning=metr_warning, + estimation_category=EstimationCategory.FRONTEND, + ) + + +def estimate_app_dev( + description: str, + modifiers: ModifierSet, + *, + review_mode=None, + model_key: str = "opus", + thresholds=None, + fallback_threshold: float = 40.0, + agent_name: str | None = None, + human_equivalent_minutes: float | None = None, +) -> TaskEstimate: + """Estimate an app-development task using a generic cold L-style prior.""" + _ = description # modifiers handle warm/specified app-dev collapse + + if review_mode is None: + review_mode = ReviewMode.NONE + + o, m, p = _APP_DEV_BASELINES + sizing = _make_non_coding_sizing( + o, + m, + p, + "app-dev-generic-l-model", + task_type=TaskType.APP_DEV, + tier=SizeTier.L, + ) + + adjusted_o = o * modifiers.combined + adjusted_m = m * modifiers.combined + adjusted_p = p * modifiers.combined + + pert = compute_pert(adjusted_o, adjusted_m, adjusted_p) + review_minutes = compute_review_overhead(review_mode) + total = pert.expected + review_minutes + + metr_warning = check_metr_threshold( + model_key, + total, + thresholds=thresholds, + fallback_threshold=fallback_threshold, + agent_name=agent_name, + ) + + return TaskEstimate( + sizing=sizing, + pert=pert, + modifiers=modifiers, + review_minutes=review_minutes, + total_expected_minutes=total, + human_equivalent_minutes=human_equivalent_minutes, + metr_warning=metr_warning, + estimation_category=EstimationCategory.APP_DEV, + ) diff --git a/src/agent_estimate/metr_thresholds.yaml b/src/agent_estimate/metr_thresholds.yaml index 795eceb..d93d77d 100644 --- a/src/agent_estimate/metr_thresholds.yaml +++ b/src/agent_estimate/metr_thresholds.yaml @@ -1,7 +1,16 @@ models: + opus_4_x: + display_name: "Opus 4.x" + p80_minutes: 90 + opus_4_7: + display_name: "Opus 4.7" + p80_minutes: 90 opus_4_6: display_name: "Opus 4.6" p80_minutes: 90 + gpt_5_5: + display_name: "GPT-5.5" + p80_minutes: 90 gpt_5_4: display_name: "GPT-5.4" p80_minutes: 60 diff --git a/src/agent_estimate/version.py b/src/agent_estimate/version.py index 42eff7b..38d6476 100644 --- a/src/agent_estimate/version.py +++ b/src/agent_estimate/version.py @@ -1,3 +1,3 @@ """Version constants for agent-estimate.""" -__version__ = "0.6.1" +__version__ = "0.7.0" diff --git a/tests/integration/test_cli_e2e.py b/tests/integration/test_cli_e2e.py index 4d3f378..9dd3c5a 100644 --- a/tests/integration/test_cli_e2e.py +++ b/tests/integration/test_cli_e2e.py @@ -2,11 +2,13 @@ from __future__ import annotations +import json import re from pathlib import Path from typer.testing import CliRunner +from agent_estimate.audit import reset_audit_logger from agent_estimate.cli.app import app from agent_estimate.cli.commands import estimate as estimate_command @@ -69,6 +71,54 @@ def test_estimate_custom_title(self) -> None: assert result.exit_code == 0 assert "My Custom Report" in result.output + def test_estimate_emits_audit_events(self, monkeypatch, tmp_path: Path) -> None: + audit_log = tmp_path / "audit.jsonl" + monkeypatch.setenv("AGENT_ESTIMATE_AUDIT_ENABLED", "1") + monkeypatch.setenv("AGENT_ESTIMATE_AUDIT_DESTINATION", str(audit_log)) + monkeypatch.setenv("AGENT_ESTIMATE_AUDIT_LEVEL", "INFO") + reset_audit_logger() + + config = str(FIXTURES / "minimal_agents.yaml") + result = runner.invoke(app, ["estimate", "--config", config, "Add login button"]) + + reset_audit_logger() + assert result.exit_code == 0 + events = [ + json.loads(line) + for line in audit_log.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + event_types = {event["event_type"] for event in events} + assert "configuration_change" in event_types + assert "estimation_request" in event_types + + config_event = next(event for event in events if event["event_type"] == "configuration_change") + assert config_event["details"]["trigger"] == "cli --config" + assert config_event["details"]["source"] == "minimal_agents.yaml" + assert config_event["details"]["changed_fields"] + + estimation_event = next(event for event in events if event["event_type"] == "estimation_request") + assert estimation_event["details"]["request"]["input_source"] == "task" + assert estimation_event["details"]["result"]["expected_case_minutes"] > 0 + + def test_estimate_with_custom_config_tolerates_missing_packaged_baseline( + self, + monkeypatch, + ) -> None: + config = str(FIXTURES / "minimal_agents.yaml") + calls = {"count": 0} + + def fake_load_default_config(): + calls["count"] += 1 + raise FileNotFoundError("default_agents.yaml missing") + + monkeypatch.setattr(estimate_command, "load_default_config", fake_load_default_config) + result = runner.invoke(app, ["estimate", "--config", config, "Add login button"]) + + assert calls["count"] == 1 + assert result.exit_code == 0 + assert "Agent Estimate Report" in result.output + class TestEstimateModifierFlags: def test_modifier_flags_affect_report_for_text_input(self) -> None: @@ -231,6 +281,10 @@ def test_review_mode_complex(self) -> None: result = runner.invoke(app, ["estimate", "--review-mode", "complex", "Add button"]) assert result.exit_code == 0 + def test_review_mode_three_round(self) -> None: + result = runner.invoke(app, ["estimate", "--review-mode", "3-round", "Add button"]) + assert result.exit_code == 0 + def test_review_mode_invalid(self) -> None: result = runner.invoke(app, ["estimate", "--review-mode", "bogus", "Add button"]) assert result.exit_code != 0 diff --git a/tests/unit/test_audit.py b/tests/unit/test_audit.py new file mode 100644 index 0000000..21b11dc --- /dev/null +++ b/tests/unit/test_audit.py @@ -0,0 +1,41 @@ +"""Unit tests for audit logger hardening.""" + +from __future__ import annotations + +from pathlib import Path + +from agent_estimate.audit import AuditConfig, AuditLogger, _scrub + + +def test_audit_logger_creates_owner_only_log_file(tmp_path: Path) -> None: + audit_log = tmp_path / "audit.jsonl" + logger = AuditLogger( + AuditConfig( + enabled=True, + level="INFO", + destination=str(audit_log), + actor="test-agent", + environment="test", + ), + ) + + logger.emit("authentication_event", provider="gh") + + assert audit_log.exists() + assert audit_log.stat().st_mode & 0o777 == 0o600 + + +def test_scrub_uses_segment_aware_sensitive_key_detection() -> None: + scrubbed = _scrub( + { + "monkey_key_phase": 42, + "api_key": "secret", + "privateKey": "secret", + "plain_token": "secret", + }, + ) + + assert scrubbed["monkey_key_phase"] == 42 + assert scrubbed["api_key"] == "[REDACTED]" + assert scrubbed["privateKey"] == "[REDACTED]" + assert scrubbed["plain_token"] == "[REDACTED]" diff --git a/tests/unit/test_github_adapters.py b/tests/unit/test_github_adapters.py index e737ec8..0954891 100644 --- a/tests/unit/test_github_adapters.py +++ b/tests/unit/test_github_adapters.py @@ -3,9 +3,16 @@ from __future__ import annotations import json +import subprocess from collections import defaultdict from collections.abc import Mapping +from pathlib import Path +import pytest + +from agent_estimate.audit import reset_audit_logger +from agent_estimate.adapters import github_rest +from agent_estimate.adapters.github_adapter import GitHubAdapterError from agent_estimate.adapters.github_ghcli import GitHubGhCliAdapter from agent_estimate.adapters.github_rest import GitHubRestAdapter @@ -113,3 +120,102 @@ def runner(args: list[str]) -> str: assert by_number == ["CLI title\n\nCLI body"] assert by_label == ["Label title\n\nBody", "Second title"] + + +def test_rest_adapter_emits_auth_and_api_audit_events( + tmp_path: Path, + monkeypatch, +) -> None: + audit_log = tmp_path / "audit.jsonl" + monkeypatch.setenv("AGENT_ESTIMATE_AUDIT_ENABLED", "1") + monkeypatch.setenv("AGENT_ESTIMATE_AUDIT_DESTINATION", str(audit_log)) + monkeypatch.setenv("AGENT_ESTIMATE_AUDIT_LEVEL", "INFO") + monkeypatch.setenv("GITHUB_TOKEN", "test-token") + reset_audit_logger() + + def request_fn(url: str, headers: Mapping[str, str]) -> tuple[int, dict[str, str], str]: + assert headers["Authorization"] == "Bearer test-token" + return ( + 200, + {}, + json.dumps({"number": 5, "title": "Audit this", "body": "No secrets"}), + ) + + adapter = GitHubRestAdapter(request_fn=request_fn) + issues = adapter.fetch_issues_by_numbers("acme/repo", [5]) + + reset_audit_logger() + assert issues[0].number == 5 + raw_log = audit_log.read_text(encoding="utf-8") + assert "test-token" not in raw_log + events = [json.loads(line) for line in raw_log.splitlines() if line.strip()] + + auth_event = next(event for event in events if event["event_type"] == "authentication_event") + assert auth_event["details"]["provider"] == "env" + + api_event = next(event for event in events if event["event_type"] == "api_call") + assert api_event["details"]["client"] == "github_rest" + assert api_event["details"]["status_code"] == 200 + assert api_event["details"]["endpoint"].endswith("/repos/acme/repo/issues/5") + + +def test_gh_cli_adapter_emits_api_audit_event(tmp_path: Path, monkeypatch) -> None: + audit_log = tmp_path / "audit.jsonl" + monkeypatch.setenv("AGENT_ESTIMATE_AUDIT_ENABLED", "1") + monkeypatch.setenv("AGENT_ESTIMATE_AUDIT_DESTINATION", str(audit_log)) + monkeypatch.setenv("AGENT_ESTIMATE_AUDIT_LEVEL", "INFO") + reset_audit_logger() + + def runner(args: list[str]) -> str: + assert args[:3] == ["gh", "issue", "view"] + return json.dumps({"number": 9, "title": "CLI title", "body": "CLI body"}) + + adapter = GitHubGhCliAdapter(runner=runner) + issues = adapter.fetch_issues_by_numbers("acme/repo", [9]) + + reset_audit_logger() + assert issues[0].number == 9 + events = [ + json.loads(line) + for line in audit_log.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + api_event = next(event for event in events if event["event_type"] == "api_call") + assert api_event["details"]["client"] == "gh" + assert api_event["details"]["endpoint"] == "gh issue view" + assert api_event["details"]["issue_number"] == 9 + assert "status" not in api_event["details"] + + +def test_rest_adapter_auth_failure_audit_event_redacts_raw_gh_output( + tmp_path: Path, + monkeypatch, +) -> None: + audit_log = tmp_path / "audit.jsonl" + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + monkeypatch.setenv("AGENT_ESTIMATE_AUDIT_ENABLED", "1") + monkeypatch.setenv("AGENT_ESTIMATE_AUDIT_DESTINATION", str(audit_log)) + monkeypatch.setenv("AGENT_ESTIMATE_AUDIT_LEVEL", "INFO") + reset_audit_logger() + + def fake_run(*args, **kwargs) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess( + args[0], + 1, + stdout="", + stderr="gh auth token failed for /Users/testuser/.config/gh/hosts.yml", + ) + + monkeypatch.setattr(github_rest.subprocess, "run", fake_run) + + with pytest.raises(GitHubAdapterError): + github_rest._resolve_github_token() + + reset_audit_logger() + raw_log = audit_log.read_text(encoding="utf-8") + assert "/Users/testuser" not in raw_log + events = [json.loads(line) for line in raw_log.splitlines() if line.strip()] + auth_event = next(event for event in events if event["event_type"] == "authentication_event") + assert auth_event["outcome"] == "error" + assert auth_event["details"]["error_type"] == "gh_auth_failure" + assert "error" not in auth_event["details"] diff --git a/tests/unit/test_human_comparison.py b/tests/unit/test_human_comparison.py index b7cf068..5e8b34a 100644 --- a/tests/unit/test_human_comparison.py +++ b/tests/unit/test_human_comparison.py @@ -18,6 +18,8 @@ TaskType.REFACTOR: (2.0, 3.5), TaskType.TEST: (2.5, 4.5), TaskType.DOCS: (3.0, 6.0), + TaskType.FRONTEND: (2.5, 3.5), + TaskType.APP_DEV: (2.5, 3.5), TaskType.UNKNOWN: (2.0, 4.0), } @@ -91,6 +93,16 @@ def test_docs_type(self) -> None: result = compute_human_equivalent(agent_min, TaskType.DOCS) assert result == pytest.approx(15.0 * math.sqrt(18.0)) + def test_frontend_type(self) -> None: + agent_min = 30.0 + result = compute_human_equivalent(agent_min, TaskType.FRONTEND) + assert result == pytest.approx(30.0 * math.sqrt(8.75)) + + def test_app_dev_type(self) -> None: + agent_min = 30.0 + result = compute_human_equivalent(agent_min, TaskType.APP_DEV) + assert result == pytest.approx(30.0 * math.sqrt(8.75)) + def test_unknown_type(self) -> None: agent_min = 25.0 result = compute_human_equivalent(agent_min, TaskType.UNKNOWN) diff --git a/tests/unit/test_modifiers.py b/tests/unit/test_modifiers.py index 8f3810b..25c0bd3 100644 --- a/tests/unit/test_modifiers.py +++ b/tests/unit/test_modifiers.py @@ -210,9 +210,13 @@ def test_standard_mode_is_fifteen_minutes(self) -> None: assert compute_review_overhead(ReviewMode.STANDARD) == pytest.approx(15.0) def test_complex_mode_is_twenty_five_minutes(self) -> None: - """3+ rounds, security-sensitive, new algorithms: 25 m overhead.""" + """Involved/security-sensitive review: 25 m overhead.""" assert compute_review_overhead(ReviewMode.COMPLEX) == pytest.approx(25.0) + def test_three_round_mode_is_thirty_five_minutes(self) -> None: + """Explicit 3-round review: 35 m overhead.""" + assert compute_review_overhead(ReviewMode.THREE_ROUND) == pytest.approx(35.0) + def test_all_review_modes_covered(self) -> None: for mode in ReviewMode: overhead = compute_review_overhead(mode) @@ -246,6 +250,10 @@ def test_legacy_2x_lgtm_maps_to_standard(self) -> None: assert mode is ReviewMode.STANDARD assert compute_review_overhead(mode) == pytest.approx(15.0) + def test_three_round_aliases_map_to_three_round(self) -> None: + assert ReviewMode("3-round") is ReviewMode.THREE_ROUND + assert ReviewMode("three_round") is ReviewMode.THREE_ROUND + def test_unknown_mode_string_raises(self) -> None: with pytest.raises(ValueError): ReviewMode("bogus") diff --git a/tests/unit/test_pert_engine.py b/tests/unit/test_pert_engine.py index 9a5ff8b..1472a18 100644 --- a/tests/unit/test_pert_engine.py +++ b/tests/unit/test_pert_engine.py @@ -180,6 +180,9 @@ def test_standard_is_fifteen(self) -> None: def test_complex_is_twenty_five(self) -> None: assert compute_review_overhead(ReviewMode.COMPLEX) == pytest.approx(25.0) + def test_three_round_is_thirty_five(self) -> None: + assert compute_review_overhead(ReviewMode.THREE_ROUND) == pytest.approx(35.0) + # --------------------------------------------------------------------------- # human_comparison @@ -236,8 +239,9 @@ def test_check_exceeds_threshold_returns_warning(self) -> None: @pytest.mark.parametrize( ("model_key", "expected_model_key", "expected_threshold"), [ - ("claude", "opus_4_6", 90.0), - ("codex", "gpt_5_4", 60.0), + ("claude", "opus_4_7", 90.0), + ("opus_4_6", "opus_4_6", 90.0), + ("codex", "gpt_5_5", 90.0), ("gemini", "gemini_3_1_pro", 45.0), ("gpt-5.3", "gpt_5_3", 60.0), ], @@ -246,7 +250,9 @@ def test_alias_model_key_resolves_to_threshold_key( self, model_key: str, expected_model_key: str, expected_threshold: float ) -> None: thresholds = { + "opus_4_7": 90.0, "opus_4_6": 90.0, + "gpt_5_5": 90.0, "gpt_5_4": 60.0, "gemini_3_1_pro": 45.0, "gpt_5_3": 60.0, @@ -260,7 +266,9 @@ def test_alias_model_key_resolves_to_threshold_key( def test_frontier_model_tier_resolves_by_assigned_agent(self) -> None: thresholds = { + "opus_4_7": 90.0, "opus_4_6": 90.0, + "gpt_5_5": 90.0, "gpt_5_4": 60.0, "gemini_3_1_pro": 45.0, } @@ -280,15 +288,15 @@ def test_frontier_model_tier_resolves_by_assigned_agent(self) -> None: ) codex_result = check_metr_threshold( "frontier", - 70.0, + 95.0, thresholds=thresholds, fallback_threshold=45.0, agent_name="Codex", ) assert claude_result is None # 70 < 90 opus threshold assert codex_result is not None - assert codex_result.model_key == "gpt_5_4" - assert codex_result.threshold_minutes == pytest.approx(60.0) + assert codex_result.model_key == "gpt_5_5" + assert codex_result.threshold_minutes == pytest.approx(90.0) assert gemini_result is not None assert gemini_result.model_key == "gemini_3_1_pro" assert gemini_result.threshold_minutes == pytest.approx(45.0) diff --git a/tests/unit/test_plugin_structure.py b/tests/unit/test_plugin_structure.py index de8bfda..9c9fc63 100644 --- a/tests/unit/test_plugin_structure.py +++ b/tests/unit/test_plugin_structure.py @@ -4,6 +4,7 @@ from pathlib import Path import pytest +import yaml ROOT = Path(__file__).resolve().parents[2] @@ -40,26 +41,46 @@ def test_plugin_version_matches_package(self): ) -class TestSkillLocation: - """Tests for skills/estimate/SKILL.md placement.""" +class TestSkillStructure: + """Tests for oacp-skills convention layout.""" - skill_md = ROOT / "skills" / "estimate" / "SKILL.md" - old_skill_md = ROOT / "src" / "agent_estimate" / "skill" / "SKILL.md" + skill_dir = ROOT / "skills" / "estimate" + skill_yaml = skill_dir / "skill.yaml" + skill_readme = skill_dir / "README.md" + intent_md = skill_dir / "shared" / "INTENT.md" + claude_skill_md = skill_dir / "claude" / "SKILL.md" + codex_skill_md = skill_dir / "codex" / "SKILL.md" + old_flat_skill_md = skill_dir / "SKILL.md" + old_src_skill_md = ROOT / "src" / "agent_estimate" / "skill" / "SKILL.md" + old_agent_skill_md = ROOT / ".agent" / "skills" / "estimate" / "SKILL.md" - def test_skill_md_exists(self): - assert self.skill_md.exists(), "skills/estimate/SKILL.md must exist" + def test_skill_yaml_exists(self): + assert self.skill_yaml.exists(), "skills/estimate/skill.yaml must exist" - def test_skill_md_has_yaml_frontmatter(self): - content = self.skill_md.read_text() - assert content.startswith("---"), "SKILL.md must start with YAML frontmatter" - # Find the closing --- + def test_skill_yaml_is_valid(self): + data = yaml.safe_load(self.skill_yaml.read_text()) + assert data["id"] == "estimate" + assert "compatible_runtimes" in data + + def test_skill_readme_exists(self): + assert self.skill_readme.exists(), "skills/estimate/README.md must exist" + + def test_shared_intent_exists(self): + assert self.intent_md.exists(), "skills/estimate/shared/INTENT.md must exist" + + def test_claude_skill_md_exists(self): + assert self.claude_skill_md.exists(), "skills/estimate/claude/SKILL.md must exist" + + def test_claude_skill_md_has_yaml_frontmatter(self): + content = self.claude_skill_md.read_text() + assert content.startswith("---"), "Claude SKILL.md must start with YAML frontmatter" second_fence = content.index("---", 3) frontmatter = content[3:second_fence].strip() assert "name:" in frontmatter, "frontmatter must contain 'name:'" assert "description:" in frontmatter, "frontmatter must contain 'description:'" - def test_skill_frontmatter_name_is_estimate(self): - content = self.skill_md.read_text() + def test_claude_skill_frontmatter_name_is_estimate(self): + content = self.claude_skill_md.read_text() second_fence = content.index("---", 3) frontmatter = content[3:second_fence].strip() for line in frontmatter.splitlines(): @@ -69,23 +90,10 @@ def test_skill_frontmatter_name_is_estimate(self): return pytest.fail("'name:' not found in frontmatter") - def test_old_skill_md_removed(self): - assert not self.old_skill_md.exists(), ( - "src/agent_estimate/skill/SKILL.md must be removed — " - "canonical location is skills/estimate/SKILL.md" - ) - - -class TestCodexSkillMirror: - """Tests for Codex-compatible .agent skill.""" - - canonical_skill_md = ROOT / "skills" / "estimate" / "SKILL.md" - codex_skill_md = ROOT / ".agent" / "skills" / "estimate" / "SKILL.md" + def test_codex_skill_md_exists(self): + assert self.codex_skill_md.exists(), "skills/estimate/codex/SKILL.md must exist" - def test_codex_skill_mirror_exists(self): - assert self.codex_skill_md.exists(), ".agent/skills/estimate/SKILL.md must exist" - - def test_codex_skill_mirror_has_yaml_frontmatter(self): + def test_codex_skill_md_has_yaml_frontmatter(self): content = self.codex_skill_md.read_text() assert content.startswith("---"), "Codex SKILL.md must start with YAML frontmatter" second_fence = content.index("---", 3) @@ -115,8 +123,26 @@ def test_codex_skill_documents_json_as_supported(self): assert "--format json" in content assert "NOT YET IMPLEMENTED" not in content - def test_codex_and_canonical_skills_share_skill_name(self): - canonical = self.canonical_skill_md.read_text() + def test_both_skills_share_skill_name(self): + claude = self.claude_skill_md.read_text() codex = self.codex_skill_md.read_text() - assert "name: estimate" in canonical + assert "name: estimate" in claude assert "name: estimate" in codex + + def test_old_flat_skill_md_removed(self): + assert not self.old_flat_skill_md.exists(), ( + "skills/estimate/SKILL.md must be removed — " + "canonical location is skills/estimate/claude/SKILL.md" + ) + + def test_old_src_skill_md_removed(self): + assert not self.old_src_skill_md.exists(), ( + "src/agent_estimate/skill/SKILL.md must be removed — " + "canonical location is skills/estimate/claude/SKILL.md" + ) + + def test_old_agent_skill_md_removed(self): + assert not self.old_agent_skill_md.exists(), ( + ".agent/skills/estimate/SKILL.md must be removed — " + "canonical location is skills/estimate/codex/SKILL.md" + ) diff --git a/tests/unit/test_task_type_models.py b/tests/unit/test_task_type_models.py index e94d525..ec66b0c 100644 --- a/tests/unit/test_task_type_models.py +++ b/tests/unit/test_task_type_models.py @@ -5,15 +5,20 @@ from agent_estimate.core.models import EstimationCategory, ReviewMode from agent_estimate.core.modifiers import build_modifier_set from agent_estimate.core.task_type_models import ( + _APP_DEV_BASELINES, _BRAINSTORM_BASELINES, _CONFIG_SRE_BASELINES, _DOCUMENTATION_BASELINES, + _FRONTEND_BUILD_BASELINES, + _FRONTEND_CONTENT_BASELINES, _RESEARCH_BASELINES_DEEP, _RESEARCH_BASELINES_SHALLOW, detect_estimation_category, + estimate_app_dev, estimate_brainstorm, estimate_config_sre, estimate_documentation, + estimate_frontend, estimate_research, ) @@ -37,6 +42,12 @@ def test_whitespace_is_coding(self) -> None: def test_brainstorm_keyword(self) -> None: assert detect_estimation_category("Brainstorm ideas for the new dashboard") == EstimationCategory.BRAINSTORM + def test_research_grounded_brainstorm_routes_to_research(self) -> None: + assert ( + detect_estimation_category("Brainstorm OSS options with citations") + == EstimationCategory.RESEARCH + ) + def test_spike_keyword(self) -> None: assert detect_estimation_category("Spike: explore auth options") == EstimationCategory.BRAINSTORM @@ -87,6 +98,16 @@ def test_monitoring_keyword(self) -> None: def test_cicd_keyword(self) -> None: assert detect_estimation_category("CI/CD pipeline for frontend") == EstimationCategory.CONFIG_SRE + # Frontend / app-dev + def test_frontend_keyword(self) -> None: + assert detect_estimation_category("Build a new frontend dashboard page") == EstimationCategory.FRONTEND + + def test_frontend_content_keyword(self) -> None: + assert detect_estimation_category("Update MDX SEO snippet") == EstimationCategory.FRONTEND + + def test_app_dev_keyword(self) -> None: + assert detect_estimation_category("Build an Electron desktop app shell") == EstimationCategory.APP_DEV + # Documentation def test_documentation_keyword(self) -> None: assert detect_estimation_category("Write documentation for new API") == EstimationCategory.DOCUMENTATION @@ -287,6 +308,66 @@ def test_default_review_is_none(self) -> None: assert est.review_minutes == 0.0 +# --------------------------------------------------------------------------- +# estimate_frontend +# --------------------------------------------------------------------------- + + +class TestEstimateFrontend: + def setup_method(self) -> None: + self.modifiers = build_modifier_set() + + def test_returns_frontend_category(self) -> None: + est = estimate_frontend("Build a landing page", self.modifiers) + assert est.estimation_category == EstimationCategory.FRONTEND + + def test_content_work_uses_content_baselines(self) -> None: + est = estimate_frontend("Update MDX content and SEO snippet", self.modifiers) + o, m, p = _FRONTEND_CONTENT_BASELINES + assert est.sizing.baseline_optimistic == o + assert est.sizing.baseline_most_likely == m + assert est.sizing.baseline_pessimistic == p + assert "frontend-content-model" in est.sizing.signals + + def test_build_work_uses_build_baselines(self) -> None: + est = estimate_frontend("Build a new landing page", self.modifiers) + o, m, p = _FRONTEND_BUILD_BASELINES + assert est.sizing.baseline_optimistic == o + assert est.sizing.baseline_most_likely == m + assert est.sizing.baseline_pessimistic == p + assert "frontend-build-model" in est.sizing.signals + + def test_review_mode_applied(self) -> None: + est = estimate_frontend( + "Build a landing page", + self.modifiers, + review_mode=ReviewMode.THREE_ROUND, + ) + assert est.review_minutes == 35.0 + + +# --------------------------------------------------------------------------- +# estimate_app_dev +# --------------------------------------------------------------------------- + + +class TestEstimateAppDev: + def setup_method(self) -> None: + self.modifiers = build_modifier_set() + + def test_returns_app_dev_category(self) -> None: + est = estimate_app_dev("Build a mac app shell", self.modifiers) + assert est.estimation_category == EstimationCategory.APP_DEV + + def test_uses_generic_l_baselines(self) -> None: + est = estimate_app_dev("Build an Electron app", self.modifiers) + o, m, p = _APP_DEV_BASELINES + assert est.sizing.baseline_optimistic == o + assert est.sizing.baseline_most_likely == m + assert est.sizing.baseline_pessimistic == p + assert "app-dev-generic-l-model" in est.sizing.signals + + # --------------------------------------------------------------------------- # Pipeline integration: --type flag routes correctly # --------------------------------------------------------------------------- @@ -298,8 +379,10 @@ class TestPipelineRouting: def setup_method(self) -> None: from agent_estimate.core.task_type_models import ( _BRAINSTORM_BASELINES, + _APP_DEV_BASELINES, _CONFIG_SRE_BASELINES, _DOCUMENTATION_BASELINES, + _FRONTEND_BUILD_BASELINES, _RESEARCH_BASELINES_SHALLOW, ) from agent_estimate.core.sizing import TIER_BASELINES, SizeTier @@ -308,6 +391,8 @@ def setup_method(self) -> None: self.research_m = _RESEARCH_BASELINES_SHALLOW[1] self.config_m = _CONFIG_SRE_BASELINES[1] self.doc_m = _DOCUMENTATION_BASELINES[1] + self.frontend_build_m = _FRONTEND_BUILD_BASELINES[1] + self.app_dev_m = _APP_DEV_BASELINES[1] # Coding M baseline self.coding_m = TIER_BASELINES[SizeTier.M][1] @@ -364,6 +449,16 @@ def test_documentation_category_produces_doc_model(self) -> None: assert report.tasks[0].estimation_category == EstimationCategory.DOCUMENTATION assert report.tasks[0].base_pert_most_likely_minutes == self.doc_m + def test_frontend_category_produces_frontend_model(self) -> None: + report = self._run_pipeline("Build a landing page", EstimationCategory.FRONTEND) + assert report.tasks[0].estimation_category == EstimationCategory.FRONTEND + assert report.tasks[0].base_pert_most_likely_minutes == self.frontend_build_m + + def test_app_dev_category_produces_app_dev_model(self) -> None: + report = self._run_pipeline("Build a desktop app", EstimationCategory.APP_DEV) + assert report.tasks[0].estimation_category == EstimationCategory.APP_DEV + assert report.tasks[0].base_pert_most_likely_minutes == self.app_dev_m + def test_coding_category_uses_pert_tier_model(self) -> None: report = self._run_pipeline("Do some work", EstimationCategory.CODING) assert report.tasks[0].estimation_category == EstimationCategory.CODING @@ -378,6 +473,18 @@ def test_auto_detection_research(self) -> None: report = self._run_pipeline("Research caching solutions", None) assert report.tasks[0].estimation_category == EstimationCategory.RESEARCH + def test_auto_detection_research_grounded_brainstorm(self) -> None: + report = self._run_pipeline("Brainstorm OSS options with citations", None) + assert report.tasks[0].estimation_category == EstimationCategory.RESEARCH + + def test_auto_detection_frontend(self) -> None: + report = self._run_pipeline("Build a frontend page", None) + assert report.tasks[0].estimation_category == EstimationCategory.FRONTEND + + def test_auto_detection_app_dev(self) -> None: + report = self._run_pipeline("Build an Electron app shell", None) + assert report.tasks[0].estimation_category == EstimationCategory.APP_DEV + def test_auto_detection_coding_default(self) -> None: report = self._run_pipeline("Fix the authentication bug", None) assert report.tasks[0].estimation_category == EstimationCategory.CODING diff --git a/tests/unit/test_version.py b/tests/unit/test_version.py index 65cfd5d..4ead920 100644 --- a/tests/unit/test_version.py +++ b/tests/unit/test_version.py @@ -4,4 +4,4 @@ def test_version_string_present() -> None: - assert __version__ == "0.6.1" + assert __version__ == "0.7.0"