diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index b038bb4..3b81396 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "agent-estimate", "description": "Effort estimation for AI coding agents — PERT three-point estimation with METR reliability thresholds and wave planning", - "version": "0.7.2", + "version": "0.7.3", "skills": ["./skills/estimate/claude"], "author": { "name": "Kiloloop" diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 407ff22..1a4cc9c 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -55,7 +55,7 @@ body: id: version attributes: label: agent-estimate version - placeholder: "0.7.2" + placeholder: "0.7.3" validations: required: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ed5dac..123c8c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.7.3] - 2026-07-10 + +### Fixed +- `agent-estimate validate` now reports a clean user-facing error instead of a traceback when calibration DB observation fields have invalid types, with end-to-end tests covering the calibration write path. +- Refreshed shipped example outputs for current engine numbers and expanded docs freshness tests to cover every example capture. + ## [0.7.2] - 2026-06-14 ### Fixed @@ -110,6 +116,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Modifier flags: `--warm-context`, `--spec-clarity`, `--issues` - PyPI package: `pip install agent-estimate` +[0.7.3]: https://github.com/kiloloop/agent-estimate/releases/tag/v0.7.3 [0.7.2]: https://github.com/kiloloop/agent-estimate/releases/tag/v0.7.2 [0.7.1]: https://github.com/kiloloop/agent-estimate/releases/tag/v0.7.1 [0.7.0]: https://github.com/kiloloop/agent-estimate/releases/tag/v0.7.0 diff --git a/action.yml b/action.yml index 733eace..837685a 100644 --- a/action.yml +++ b/action.yml @@ -53,7 +53,7 @@ inputs: required: false default: '3.12' version: - description: 'agent-estimate version to install (e.g. "0.7.2"). Omit for latest.' + description: 'agent-estimate version to install (e.g. "0.7.3"). Omit for latest.' required: false token: description: 'GitHub token for issue fetching and PR comments' diff --git a/examples/README.md b/examples/README.md index 76026aa..0bb4de7 100644 --- a/examples/README.md +++ b/examples/README.md @@ -5,9 +5,9 @@ Real input/output examples from production agent dispatches. Every command below | Example | Task type | Tier | Compression | File | |---------|-----------|------|-------------|------| | Fix pyproject URLs after org rename | Coding | XS | 2.00x | [coding-s.md](./coding-s.md) | -| Implement CLI command with code generation | Coding | M | 2.53x | [coding-m.md](./coding-m.md) | +| Implement CLI command with code generation | Coding | M | 3.00x | [coding-m.md](./coding-m.md) | | Audit cloud infrastructure providers | Research | S | 2.61x | [research.md](./research.md) | -| Write quickstart guide + README | Documentation | S | 2.58x | [documentation.md](./documentation.md) | +| Write quickstart guide + README | Documentation | S | 3.87x | [documentation.md](./documentation.md) | | 3-agent parallel session (3 features) | Multi-agent | M×3 | 8.07x | [multi-agent.md](./multi-agent.md) | ## How to read the output diff --git a/examples/coding-m.md b/examples/coding-m.md index f10c977..6ad18c9 100644 --- a/examples/coding-m.md +++ b/examples/coding-m.md @@ -14,7 +14,13 @@ agent-estimate estimate "Implement add-agent CLI command with SPEC.md generation | Task | Model | Tier | Agent | Base PERT (O/M/P) | Modifiers | Effective Duration | Human Equivalent | | --- | --- | --- | --- | --- | --- | --- | --- | -| **Implement add-agent CLI command with SPEC.md generation** | coding | M | Claude | 25m / 50m / 90m (E=52.5m) | spec 1.00 x warm 1.00 x fit 1.00 = 1.00 | 52.5m | 190.9m | +| **Implement add-agent CLI command with SPEC.md generation** | coding | M | Claude | 25m / 50m / 90m (E=52.5m) | spec 1.00 x warm 1.00 x fit 1.00 = 1.00 | 52.5m | 226.4m | + +### Wave Plan + +| Wave | Tasks | Duration | Agent Assignments (amortized review) | +| --- | --- | --- | --- | +| 0 | Implement add-agent CLI command with SPEC.md generation | 75.4m | Claude: Implement add-agent CLI command with SPEC.md generation +15m review | ### Timeline Summary @@ -23,15 +29,37 @@ agent-estimate estimate "Implement add-agent CLI command with SPEC.md generation | Best case | 44.7m | | Expected case | 75.4m | | Worst case | 117.2m | -| Human-speed equivalent | 190.9m | -| Compression ratio | 2.53x | +| Human-speed equivalent | 226.4m | +| Compression ratio | 3.00x | | Review overhead (per-task, pre-amortization) | 15m | +### Review Overhead + +Review is amortized per agent per wave: one review cycle covers all PRs from that +agent in the wave. Per-task values below are the naive (pre-amortization) figures. + +| Task | Review Overhead | +| --- | --- | +| Implement add-agent CLI command with SPEC.md generation | 15m | +| **Total (naive)** | **15m** | + ### Agent Load Summary | Agent | Task Count | Total Work | Estimated Cost | | --- | --- | --- | --- | | Claude | 1 | 60.4m | $1.45 | +| Codex | 0 | 0m | $0.00 | +| Gemini | 0 | 0m | $0.00 | + +### Critical Path + +**Implement add-agent CLI command with SPEC.md generation** + +### Tier Corrections + +No tier corrections. + +### METR Warnings No METR threshold warnings. @@ -66,4 +94,4 @@ The task ran slightly over the expected case due to a second review round. This ## Key takeaway -M-tier tasks are the most common dispatch target — complex enough to justify agent overhead, predictable enough for reliable estimation. The 2.53x compression means a feature that takes a human developer ~3 hours lands in ~75 minutes of wall clock time. +M-tier tasks are the most common dispatch target — complex enough to justify agent overhead, predictable enough for reliable estimation. The 3.00x compression means a feature that takes a human developer ~3.8 hours lands in ~75 minutes of wall clock time. diff --git a/examples/coding-s.md b/examples/coding-s.md index f7575f2..154bb37 100644 --- a/examples/coding-s.md +++ b/examples/coding-s.md @@ -16,6 +16,12 @@ agent-estimate estimate "Fix pyproject.toml URLs after org rename" | --- | --- | --- | --- | --- | --- | --- | --- | | **Fix pyproject.toml URLs after org rename** | coding | XS | Claude | 5m / 10m / 20m (E=10.8m) | spec 1.00 x warm 1.00 x fit 1.00 = 1.00 | 10.8m | 54.8m | +### Wave Plan + +| Wave | Tasks | Duration | Agent Assignments (amortized review) | +| --- | --- | --- | --- | +| 0 | Fix pyproject.toml URLs after org rename | 27.5m | Claude: Fix pyproject.toml URLs after org rename +15m review | + ### Timeline Summary | Metric | Value | @@ -27,14 +33,37 @@ agent-estimate estimate "Fix pyproject.toml URLs after org rename" | Compression ratio | 2.00x | | Review overhead (per-task, pre-amortization) | 15m | +### Review Overhead + +Review is amortized per agent per wave: one review cycle covers all PRs from that +agent in the wave. Per-task values below are the naive (pre-amortization) figures. + +| Task | Review Overhead | +| --- | --- | +| Fix pyproject.toml URLs after org rename | 15m | +| **Total (naive)** | **15m** | + ### Agent Load Summary | Agent | Task Count | Total Work | Estimated Cost | | --- | --- | --- | --- | | Claude | 1 | 12.5m | $0.30 | +| Codex | 0 | 0m | $0.00 | +| Gemini | 0 | 0m | $0.00 | + +### Critical Path + +**Fix pyproject.toml URLs after org rename** + +### Tier Corrections + +No tier corrections. + +### METR Warnings No METR threshold warnings. + ## What actually happened | Metric | Estimated | Actual | @@ -49,4 +78,4 @@ The task was dispatched to Codex (better fit for mechanical find-and-replace acr ## Key takeaway -XS tasks have a lower compression ratio (2.00x) because the fixed 15m review overhead dominates. For self-merged work where you trust the agent, use `--review-mode none` to see the raw speed advantage — the effective duration drops to ~11m vs ~55m human time (5x compression). +XS tasks have a lower compression ratio (2.00x) because the fixed 15m review overhead dominates. For self-merged work where you trust the agent, use `--review-mode none` to remove the review overhead: an executed none-mode run keeps effective duration at 10.8m, drops expected case to 12.5m, and scales human-equivalent time down to 23m human time, for 1.84x compression. Human-equivalent time scales with the agent total, so removing review lowers both sides of the comparison and keeps roughly the same ~2x compression ratio instead of creating a 5x raw-speed claim. diff --git a/examples/documentation.md b/examples/documentation.md index 6a8dc35..fe418bf 100644 --- a/examples/documentation.md +++ b/examples/documentation.md @@ -16,7 +16,13 @@ agent-estimate estimate \ | Task | Model | Tier | Agent | Base PERT (O/M/P) | Modifiers | Effective Duration | Human Equivalent | | --- | --- | --- | --- | --- | --- | --- | --- | -| **Write quickstart guide and README with protocol comparison table** | documentation | S | Claude | 10m / 25m / 45m (E=25.8m) | spec 1.00 x warm 1.00 x fit 1.00 = 1.00 | 25.8m | 115.5m | +| **Write quickstart guide and README with protocol comparison …** | documentation | S | Claude | 10m / 25m / 45m (E=25.8m) | spec 1.00 x warm 1.00 x fit 1.00 = 1.00 | 25.8m | 173.2m | + +### Wave Plan + +| Wave | Tasks | Duration | Agent Assignments (amortized review) | +| --- | --- | --- | --- | +| 0 | Write quickstart guide and README with protocol comparison … | 44.7m | Claude: Write quickstart guide and README with protocol comparison … +15m review | ### Timeline Summary @@ -25,18 +31,41 @@ agent-estimate estimate \ | Best case | 27.4m | | Expected case | 44.7m | | Worst case | 65.7m | -| Human-speed equivalent | 115.5m | -| Compression ratio | 2.58x | +| Human-speed equivalent | 173.2m | +| Compression ratio | 3.87x | | Review overhead (per-task, pre-amortization) | 15m | +### Review Overhead + +Review is amortized per agent per wave: one review cycle covers all PRs from that +agent in the wave. Per-task values below are the naive (pre-amortization) figures. + +| Task | Review Overhead | +| --- | --- | +| Write quickstart guide and README with protocol comparison … | 15m | +| **Total (naive)** | **15m** | + ### Agent Load Summary | Agent | Task Count | Total Work | Estimated Cost | | --- | --- | --- | --- | | Claude | 1 | 29.7m | $0.71 | +| Codex | 0 | 0m | $0.00 | +| Gemini | 0 | 0m | $0.00 | + +### Critical Path + +**Write quickstart guide and README with protocol comparison …** + +### Tier Corrections + +No tier corrections. + +### METR Warnings No METR threshold warnings. + ## What actually happened | Metric | Estimated | Actual | @@ -51,4 +80,4 @@ Almost exactly on estimate. Documentation tasks are the most predictable categor ## Key takeaway -Documentation has the highest human-multiplier range (3.0-6.0x) because humans find technical writing tedious and slow. Agents produce consistent, structured output quickly. The 2.58x wall-clock compression accounts for the review cycle — without review (`--review-mode none`), raw agent time is ~26m vs ~116m human time, a 4.5x compression. +Documentation has the highest human-multiplier range (3.0-6.0x) because humans find technical writing tedious and slow. Agents produce consistent, structured output quickly. The 3.87x wall-clock compression accounts for the review cycle — without review (`--review-mode none`), an executed none-mode run reports 25.8m effective duration, 29.7m expected case, 109.6m human-equivalent time, and 3.69x compression. diff --git a/examples/multi-agent.md b/examples/multi-agent.md index 49092d7..80f83fa 100644 --- a/examples/multi-agent.md +++ b/examples/multi-agent.md @@ -32,7 +32,7 @@ agent-estimate estimate --file tasks.txt | Wave | Tasks | Duration | Agent Assignments (amortized review) | | --- | --- | --- | --- | -| 0 | All 3 tasks | 75.4m | Claude: add-agent CLI +15m review; Codex: known_debt.md +15m review; Gemini: quickstart guide +15m review | +| 0 | Implement add-agent CLI command with SPEC.md generation, Add known_debt.md as standard protocol memory file, Write quickstart guide with protocol comparison table | 75.4m | Claude: Implement add-agent CLI command with SPEC.md generation +15m review; Codex: Add known_debt.md as standard protocol memory file +15m review; Gemini: Write quickstart guide with protocol comparison table +15m review | ### Timeline Summary @@ -45,6 +45,18 @@ agent-estimate estimate --file tasks.txt | Compression ratio | 8.07x | | Review overhead (per-task, pre-amortization) | 45m | +### Review Overhead + +Review is amortized per agent per wave: one review cycle covers all PRs from that +agent in the wave. Per-task values below are the naive (pre-amortization) figures. + +| Task | Review Overhead | +| --- | --- | +| Implement add-agent CLI command with SPEC.md generation | 15m | +| Add known_debt.md as standard protocol memory file | 15m | +| Write quickstart guide with protocol comparison table | 15m | +| **Total (naive)** | **45m** | + ### Agent Load Summary | Agent | Task Count | Total Work | Estimated Cost | @@ -57,11 +69,16 @@ agent-estimate estimate --file tasks.txt **Implement add-agent CLI command with SPEC.md generation** +### Tier Corrections + +No tier corrections. + ### METR Warnings - **Add known_debt.md as standard protocol memory file**: Estimate (75m) exceeds gpt_5_4 p80 threshold (60m). Consider splitting the task. - **Write quickstart guide with protocol comparison table**: Estimate (75m) exceeds gemini_3_1_pro p80 threshold (45m). Consider splitting the task. + ## What actually happened These three tasks were dispatched to real agents in a production multi-agent workflow: diff --git a/examples/research.md b/examples/research.md index 040dffe..d6c8901 100644 --- a/examples/research.md +++ b/examples/research.md @@ -16,7 +16,13 @@ agent-estimate estimate \ | Task | Model | Tier | Agent | Base PERT (O/M/P) | Modifiers | Effective Duration | Human Equivalent | | --- | --- | --- | --- | --- | --- | --- | --- | -| **Audit cloud infrastructure providers for production deployment** | research | S | Claude | 10m / 20m / 30m (E=20m) | spec 1.00 x warm 1.00 x fit 1.00 = 1.00 | 20m | 99m | +| **Audit cloud infrastructure providers for production deploym…** | research | S | Claude | 10m / 20m / 30m (E=20m) | spec 1.00 x warm 1.00 x fit 1.00 = 1.00 | 20m | 99m | + +### Wave Plan + +| Wave | Tasks | Duration | Agent Assignments (amortized review) | +| --- | --- | --- | --- | +| 0 | Audit cloud infrastructure providers for production deploym… | 38m | Claude: Audit cloud infrastructure providers for production deploym… +15m review | ### Timeline Summary @@ -29,14 +35,37 @@ agent-estimate estimate \ | Compression ratio | 2.61x | | Review overhead (per-task, pre-amortization) | 15m | +### Review Overhead + +Review is amortized per agent per wave: one review cycle covers all PRs from that +agent in the wave. Per-task values below are the naive (pre-amortization) figures. + +| Task | Review Overhead | +| --- | --- | +| Audit cloud infrastructure providers for production deploym… | 15m | +| **Total (naive)** | **15m** | + ### Agent Load Summary | Agent | Task Count | Total Work | Estimated Cost | | --- | --- | --- | --- | | Claude | 1 | 23m | $0.55 | +| Codex | 0 | 0m | $0.00 | +| Gemini | 0 | 0m | $0.00 | + +### Critical Path + +**Audit cloud infrastructure providers for production deploym…** + +### Tier Corrections + +No tier corrections. + +### METR Warnings No METR threshold warnings. + ## What actually happened | Metric | Estimated | Actual | @@ -50,4 +79,4 @@ The agent finished well under the estimate — research tasks with web access te ## Key takeaway -Research tasks have the highest human-multiplier range (3.0-6.0x) because pattern matching across large datasets is tedious for humans but routine for agents. Scanning 7 cloud providers' pricing, compliance, and feature matrices would take a human most of a workday. Use `--type research` to select the research estimation model — it uses a flat PERT curve with depth scaling instead of the tier-based coding model. +Research tasks use the generic human-multiplier range (2.0-4.0x) while the research model handles the agent-side time box. Pattern matching across large datasets is still tedious for humans but routine for agents: scanning 7 cloud providers' pricing, compliance, and feature matrices would take a human most of a workday. Use `--type research` to select the research estimation model — it uses a flat PERT curve with depth scaling instead of the tier-based coding model. diff --git a/pyproject.toml b/pyproject.toml index 646b5b1..1282fff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-estimate" -version = "0.7.2" +version = "0.7.3" description = "Know what an AI task will cost before you run it" readme = "README.md" license = "Apache-2.0" diff --git a/src/agent_estimate/cli/commands/validate.py b/src/agent_estimate/cli/commands/validate.py index 6810d53..80aa57b 100644 --- a/src/agent_estimate/cli/commands/validate.py +++ b/src/agent_estimate/cli/commands/validate.py @@ -70,7 +70,7 @@ def run( if db is not None: try: obs = _build_observation(raw, estimated, actual_work, actual_total, error_ratio, verdict) - except ValueError as exc: + except (TypeError, ValueError) as exc: typer.echo(f"Error: Invalid observation field: {exc}", err=True) raise typer.Exit(code=2) diff --git a/src/agent_estimate/version.py b/src/agent_estimate/version.py index 04c5f63..a0bca4b 100644 --- a/src/agent_estimate/version.py +++ b/src/agent_estimate/version.py @@ -1,3 +1,3 @@ """Version constants for agent-estimate.""" -__version__ = "0.7.2" +__version__ = "0.7.3" diff --git a/tests/integration/test_cli_e2e.py b/tests/integration/test_cli_e2e.py index 9c4f9bf..cc57150 100644 --- a/tests/integration/test_cli_e2e.py +++ b/tests/integration/test_cli_e2e.py @@ -4,10 +4,13 @@ import json import re +from datetime import datetime, timezone from pathlib import Path +import pytest from typer.testing import CliRunner +from agent_estimate.adapters.sqlite_store import SQLiteCalibrationStore from agent_estimate.audit import reset_audit_logger from agent_estimate.cli.app import app from agent_estimate.cli.commands import estimate as estimate_command @@ -531,6 +534,146 @@ def test_validate_bad_optional_field_is_input_error(self, tmp_path: Path) -> Non assert "Invalid observation field" in result.output assert "Error storing observation" not in result.output + def test_validate_with_db_stores_observation_round_trip(self, tmp_path: Path) -> None: + observation = tmp_path / "obs.yaml" + observation.write_text( + "\n".join( + [ + "task_type: feature", + "estimated_minutes: 10", + "actual_work_minutes: 12", + "actual_total_minutes: 15", + "file_count: 2", + "line_count: 80", + "test_count: 3", + "project_hash: proj-abc", + "execution_mode: batch", + "review_mode: standard", + "review_overhead_minutes: 3", + "modifiers:", + " spec_clarity: 0.8", + " warm_context: 0.7", + "modifiers_should_have_been:", + " spec_clarity: 0.9", + ] + ), + encoding="utf-8", + ) + db_path = tmp_path / "calibration.db" + + before = datetime.now(timezone.utc) + result = runner.invoke(app, ["validate", str(observation), "--db", str(db_path)]) + after = datetime.now(timezone.utc) + + assert result.exit_code == 0 + assert "Observation stored" in result.output + + with SQLiteCalibrationStore(db_path) as store: + rows = store._query_observations(task_type="feature") + + assert len(rows) == 1 + row = rows[0] + assert row["estimated_secs"] == 600.0 + assert row["actual_work_secs"] == 720.0 + assert row["actual_total_secs"] == 900.0 + assert row["file_count"] == 2 + assert row["line_count"] == 80 + assert row["test_count"] == 3 + assert row["project_hash"] == "proj-abc" + assert row["execution_mode"] == "batch" + assert row["review_mode"] == "standard" + assert row["spec_clarity_modifier"] == 0.8 + assert row["warm_context_modifier"] == 0.7 + assert row["review_overhead_secs"] == 180.0 + assert json.loads(row["modifiers_should_have_been"]) == {"spec_clarity": 0.9} + + observed_at = datetime.fromisoformat(row["observed_at"]) + assert before.replace(microsecond=0) <= observed_at <= after.replace(microsecond=0) + + def test_validate_with_db_stores_default_observation_fields( + self, tmp_path: Path + ) -> None: + observation = tmp_path / "obs.yaml" + observation.write_text( + "\n".join( + [ + "task_type: bugfix", + "estimated_minutes: 10", + "actual_work_minutes: 12", + ] + ), + encoding="utf-8", + ) + db_path = tmp_path / "calibration.db" + + result = runner.invoke(app, ["validate", str(observation), "--db", str(db_path)]) + + assert result.exit_code == 0 + with SQLiteCalibrationStore(db_path) as store: + rows = store._query_observations(task_type="bugfix") + + assert len(rows) == 1 + row = rows[0] + assert row["file_count"] == 0 + assert row["line_count"] == 0 + assert row["test_count"] == 0 + assert row["project_hash"] == "unknown" + assert row["execution_mode"] == "single" + assert row["review_mode"] == "none" + assert row["spec_clarity_modifier"] == 1.0 + assert row["warm_context_modifier"] == 1.0 + assert row["review_overhead_secs"] == 0.0 + assert json.loads(row["modifiers_should_have_been"]) == {} + + def test_validate_db_null_field_is_input_error(self, tmp_path: Path) -> None: + observation = tmp_path / "obs.yaml" + observation.write_text( + "\n".join( + [ + "estimated_minutes: 10", + "actual_work_minutes: 12", + "file_count:", + ] + ), + encoding="utf-8", + ) + + result = runner.invoke(app, ["validate", str(observation), "--db", str(tmp_path / "c.db")]) + + assert result.exit_code == 2 + assert "Invalid observation field" in result.output + assert "Error storing observation" not in result.output + assert "Traceback" not in result.output + + @pytest.mark.parametrize( + "field_body", + [ + "file_count:\n - 1", + "file_count:\n value: 1", + ], + ) + def test_validate_db_container_field_is_input_error( + self, tmp_path: Path, field_body: str + ) -> None: + observation = tmp_path / "obs.yaml" + observation.write_text( + "\n".join( + [ + "estimated_minutes: 10", + "actual_work_minutes: 12", + field_body, + ] + ), + encoding="utf-8", + ) + + result = runner.invoke(app, ["validate", str(observation), "--db", str(tmp_path / "c.db")]) + + assert result.exit_code == 2 + assert "Invalid observation field" in result.output + assert "Error storing observation" not in result.output + assert "Traceback" not in result.output + # --------------------------------------------------------------------------- # No args / help diff --git a/tests/unit/test_docs_freshness.py b/tests/unit/test_docs_freshness.py index e625b82..555105c 100644 --- a/tests/unit/test_docs_freshness.py +++ b/tests/unit/test_docs_freshness.py @@ -1,13 +1,17 @@ -"""Docs freshness guards (#28). +"""Docs freshness guards (#28, #53). -README's hero example and examples/multi-agent.md embed captured CLI output. -These tests re-run the same estimate and assert the embedded headline lines -still match what the tool actually prints, so a prior recalibration or METR -model-key bump fails CI instead of silently rotting the docs. +README and examples/*.md embed captured CLI output. These tests re-run the same +estimates and assert the embedded examples still match what the tool actually +prints, so a recalibration, multiplier, or METR model-key bump fails CI instead +of silently rotting the docs. """ +from dataclasses import dataclass +from functools import cache from pathlib import Path +from tempfile import TemporaryDirectory +import pytest from typer.testing import CliRunner from agent_estimate.cli.app import app @@ -34,22 +38,136 @@ runner = CliRunner() -def _real_output(tmp_path: Path) -> str: - task_file = tmp_path / "tasks.txt" - task_file.write_text("\n".join(MULTI_AGENT_TASKS) + "\n", encoding="utf-8") - result = runner.invoke(app, ["estimate", "--file", str(task_file)]) +@dataclass(frozen=True) +class ExampleCase: + path: str + args: tuple[str, ...] + index_label: str + index_type: str + index_tier: str + tasks: tuple[str, ...] = () + + +EXAMPLE_CASES = [ + ExampleCase( + path="examples/coding-s.md", + args=("estimate", "Fix pyproject.toml URLs after org rename"), + index_label="Fix pyproject URLs after org rename", + index_type="Coding", + index_tier="XS", + ), + ExampleCase( + path="examples/coding-m.md", + args=("estimate", "Implement add-agent CLI command with SPEC.md generation"), + index_label="Implement CLI command with code generation", + index_type="Coding", + index_tier="M", + ), + ExampleCase( + path="examples/research.md", + args=( + "estimate", + "Audit cloud infrastructure providers for production deployment", + "--type", + "research", + ), + index_label="Audit cloud infrastructure providers", + index_type="Research", + index_tier="S", + ), + ExampleCase( + path="examples/documentation.md", + args=( + "estimate", + "Write quickstart guide and README with protocol comparison table", + "--type", + "documentation", + ), + index_label="Write quickstart guide + README", + index_type="Documentation", + index_tier="S", + ), + ExampleCase( + path="examples/multi-agent.md", + args=("estimate", "--file", "{task_file}"), + index_label="3-agent parallel session (3 features)", + index_type="Multi-agent", + index_tier="M×3", + tasks=tuple(MULTI_AGENT_TASKS), + ), +] + + +@cache +def _invoke_example(case: ExampleCase) -> str: + args = list(case.args) + if case.tasks: + with TemporaryDirectory() as tmp: + task_file = Path(tmp) / f"{Path(case.path).stem}-tasks.txt" + task_file.write_text("\n".join(case.tasks) + "\n", encoding="utf-8") + args = [str(task_file) if arg == "{task_file}" else arg for arg in args] + result = runner.invoke(app, args) + else: + result = runner.invoke(app, args) assert result.exit_code == 0, result.output return result.output +def _invoke_args(args: tuple[str, ...]) -> str: + result = runner.invoke(app, list(args)) + assert result.exit_code == 0, result.output + return result.output + + +def _captured_output(output: str) -> str: + lines = output.strip().splitlines() + if lines and lines[0] == "# Agent Estimate Report": + lines = lines[1:] + if lines and not lines[0]: + lines = lines[1:] + return "\n".join("#" + line if line.startswith("## ") else line for line in lines) + + +def _compression_ratio(output: str) -> str: + for line in output.splitlines(): + if line.startswith("| Compression ratio |"): + return line.split("|")[2].strip() + raise AssertionError("output did not include a Compression ratio row") + + +def _timeline_value(output: str, metric: str) -> str: + prefix = f"| {metric} |" + for line in output.splitlines(): + if line.startswith(prefix): + return line.split("|")[2].strip() + raise AssertionError(f"output did not include {metric!r}") + + +def _single_task_values(output: str) -> tuple[str, str]: + for line in output.splitlines(): + if line.startswith("| **"): + cells = [cell.strip() for cell in line.strip("|").split("|")] + return cells[6], cells[7] + raise AssertionError("output did not include a single-task estimate row") + + +def _key_takeaway(text: str) -> str: + start = text.index("## Key takeaway") + return text[start:] + + +def _real_output() -> str: + return _invoke_example(EXAMPLE_CASES[-1]) + + class TestDocsFreshness: def test_example_input_block_lists_the_tasks(self): example = (ROOT / "examples" / "multi-agent.md").read_text() for task in MULTI_AGENT_TASKS: assert task in example, f"examples/multi-agent.md lost input task {task!r}" - def test_readme_hero_matches_real_run(self, tmp_path): - output = _real_output(tmp_path) + def test_readme_hero_matches_real_run(self): + output = _real_output() readme = (ROOT / "README.md").read_text() for line in HEADLINE_LINES: assert line in output, ( @@ -58,12 +176,63 @@ def test_readme_hero_matches_real_run(self, tmp_path): ) assert line in readme, f"README hero block lost {line!r}" - def test_example_output_matches_real_run(self, tmp_path): - output = _real_output(tmp_path) - example = (ROOT / "examples" / "multi-agent.md").read_text() - for line in HEADLINE_LINES: - assert line in output - assert line in example, f"examples/multi-agent.md output block lost {line!r}" + @pytest.mark.parametrize("case", EXAMPLE_CASES, ids=[case.path for case in EXAMPLE_CASES]) + def test_example_output_matches_real_run(self, case: ExampleCase): + output = _captured_output(_invoke_example(case)) + example = (ROOT / case.path).read_text(encoding="utf-8") + assert output in example, ( + f"{case.path} output block is stale — re-capture it from the current CLI" + ) + + def test_examples_index_compression_matches_real_runs(self): + index = (ROOT / "examples" / "README.md").read_text(encoding="utf-8") + for case in EXAMPLE_CASES: + output = _invoke_example(case) + compression = _compression_ratio(output) + expected_row = ( + f"| {case.index_label} | {case.index_type} | {case.index_tier} | " + f"{compression} | [{Path(case.path).name}](./{Path(case.path).name}) |" + ) + assert expected_row in index + + def test_example_case_list_covers_all_shipped_examples(self): + example_files = { + path.name for path in (ROOT / "examples").glob("*.md") if path.name != "README.md" + } + covered_files = {Path(case.path).name for case in EXAMPLE_CASES} + assert covered_files == example_files + + @pytest.mark.parametrize( + ("path", "args"), + [ + ( + "examples/coding-s.md", + ("estimate", "Fix pyproject.toml URLs after org rename", "--review-mode", "none"), + ), + ( + "examples/documentation.md", + ( + "estimate", + "Write quickstart guide and README with protocol comparison table", + "--type", + "documentation", + "--review-mode", + "none", + ), + ), + ], + ) + def test_review_mode_none_prose_matches_real_run( + self, path: str, args: tuple[str, ...] + ): + output = _invoke_args(args) + effective_duration, human_equivalent = _single_task_values(output) + expected_case = _timeline_value(output, "Expected case") + compression = _timeline_value(output, "Compression ratio") + example = _key_takeaway((ROOT / path).read_text(encoding="utf-8")) + + for value in (effective_duration, human_equivalent, expected_case, compression): + assert value in example def test_session_command_is_documented_outside_changelog(self): readme = (ROOT / "README.md").read_text(encoding="utf-8") diff --git a/tests/unit/test_version.py b/tests/unit/test_version.py index a4533fe..7aa5157 100644 --- a/tests/unit/test_version.py +++ b/tests/unit/test_version.py @@ -4,4 +4,4 @@ def test_version_string_present() -> None: - assert __version__ == "0.7.2" + assert __version__ == "0.7.3"