diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 7ff7240..aeacd44 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,8 @@ { "name": "agent-estimate", "description": "Effort estimation for AI coding agents — PERT three-point estimation with METR reliability thresholds and wave planning", - "version": "0.7.0", + "version": "0.7.1", + "skills": ["./skills/estimate/claude"], "author": { "name": "Kiloloop" }, diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 8e2ceca..a2144f1 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -55,7 +55,7 @@ body: id: version attributes: label: agent-estimate version - placeholder: "0.7.0" + placeholder: "0.7.1" validations: required: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e9327c..3a8f308 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.7.1] - 2026-06-11 + +### Fixed +- Fixed six engine correctness gaps: early `--format` validation before side effects, explicit `--warm-context 1.0` handling, friction-aware assigned-agent METR re-checks, Python 3.10-compatible Z timestamp parsing, UTC-normalized calibration cohorts/order, and `plan_waves` duplicate/negative input validation (#31, #33, #36, #37, #38, #39). +- README hero example now uses the real CLI surface with real captured output — the previous block invoked a nonexistent `--model` flag, passed an unsupported second positional task, and showed fabricated numbers (#28). +- README Action workflow example now grants `issues: read`, required for issue fetching on private repos where an explicit `permissions:` block zeroes unlisted scopes (#40). +- Claude Code plugin exposes the `/estimate` skill again: `plugin.json` now points skill discovery at `skills/estimate/claude/`, which the v0.7.0 multi-runtime restructure had moved out of the default `skills//SKILL.md` scan path (#29). +- Dropped the phantom `/validate-estimate` and `/calibrate` slash commands (documented but never installable); validation and calibration now route through the installed skill as `/estimate validate` and `/estimate calibrate` (#29). +- Refreshed stale METR model keys in `examples/multi-agent.md` output (`gpt_5_3` → `gpt_5_4`, `gemini_3_pro` → `gemini_3_1_pro`) to match current output (#28). + ## [0.7.0] - 2026-05-20 ### Added @@ -94,6 +104,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Modifier flags: `--warm-context`, `--spec-clarity`, `--issues` - PyPI package: `pip install agent-estimate` +[0.7.1]: https://github.com/kiloloop/agent-estimate/releases/tag/v0.7.1 [0.7.0]: https://github.com/kiloloop/agent-estimate/releases/tag/v0.7.0 [0.6.1]: https://github.com/kiloloop/agent-estimate/releases/tag/v0.6.1 [0.6.0]: https://github.com/kiloloop/agent-estimate/releases/tag/v0.6.0 diff --git a/README.md b/README.md index 33f5c69..f23ae94 100644 --- a/README.md +++ b/README.md @@ -102,40 +102,29 @@ Real estimates from production use — including the misses. **An honest over-estimate.** We pre-registered a UI mockup build at ~95 minutes with no prior app-dev data. Two agents did it in parallel in 12 and 25 minutes — a 4–8x over-estimate. agent-estimate now ships an `app_dev` prior shaped by that result. The miss stays in the README because calibration means showing where you were wrong. -**Two tasks, one model** — what the tool prints, including the METR reliability flag: +**Three tasks, three agents, in parallel** — what the tool prints, including the METR reliability flags. Input is the three-task `tasks.txt` from [`examples/multi-agent.md`](./examples/multi-agent.md); the output below is captured from a real run, trimmed to the timeline and warnings (the full report — per-task PERT table, wave plan, agent loads — is in that example): ```text -$ agent-estimate estimate "Implement auth" "Add tests" --model opus - -Task Tier PERT (O/M/P) Expected Human-eq -─────────────────────────────────────────────────────────── -Implement auth M 25/50/90m 57.8m 160m -Add tests S 12/23/40m 24.0m 75m - -Timeline ────────────────────────────── - best 37m · expected 81.8m · worst 130m - human-equivalent: 235m → 2.87× compression +$ agent-estimate estimate --file tasks.txt - ⚠ METR warning: "Implement auth" exceeds Opus p80 -``` +## Timeline Summary -~82 minutes expected versus ~4 hours by hand — plus a flag that the auth task runs past Opus's p80 reliability horizon, so you split it or add a checkpoint before dispatching. +| Metric | Value | +| --- | --- | +| Best case | 44.7m | +| Expected case | 75.4m | +| Worst case | 117.2m | +| Human-speed equivalent | 572.8m | +| Compression ratio | 7.60x | +| Review overhead (per-task, pre-amortization) | 45m | -**Three tasks, three agents, in parallel:** +## METR Warnings -```bash -$ agent-estimate estimate --file tasks.txt +- **Add known_debt.md as standard protocol memory file**: Estimate (68m) exceeds gpt_5_4 p80 threshold (60m). Consider splitting the task. +- **Write quickstart guide with protocol comparison table**: Estimate (68m) exceeds gemini_3_1_pro p80 threshold (45m). Consider splitting the task. ``` -| Metric | Value | -| --- | --- | -| Wave 0 | All 3 tasks in parallel (Claude + Codex + Gemini) | -| Expected case | 131m | -| Human-speed equivalent | 709.5m | -| **Compression ratio** | **5.42x** | -| Estimated cost | $4.84 | - -~2 hours wall-clock versus ~12 hours sequential. You see the compression before you commit the compute. More in [`examples/`](./examples/) — coding S/M, research, documentation, multi-agent. +~75 minutes wall-clock versus ~9.5 hours of sequential human work, at an estimated $3.51 fleet cost — plus two flags that the Codex- and Gemini-assigned tasks run past their models' p80 reliability horizons, so you split them or add a checkpoint before dispatching. The same three tasks were later run by real agents; the retro is in the example file. More in [`examples/`](./examples/) — coding S/M, research, documentation, multi-agent. ## Integrations @@ -150,8 +139,8 @@ $ agent-estimate estimate --file tasks.txt /estimate Add a login page with OAuth /estimate --file spec.md /estimate --issues 1,2,3 --repo myorg/myrepo -/validate-estimate observation.yaml -/calibrate +/estimate validate observation.yaml +/estimate calibrate ``` ### GitHub Action @@ -173,6 +162,7 @@ on: permissions: contents: read + issues: read pull-requests: write jobs: @@ -265,8 +255,13 @@ agent-estimate estimate "Ship packaging flow" --config ./my_agents.yaml agent-estimate estimate "Refactor auth pipeline" --format json # machine-readable agent-estimate estimate --repo myorg/myrepo --issues 11,12,14 # from GitHub issues agent-estimate estimate --file tasks.txt # from file +agent-estimate estimate "Follow-up fix" --history-file data.json # auto warm-context ``` +When `--warm-context` is omitted, the CLI can auto-infer it from `--history-file`; +if no history file is passed and `./data.json` exists, that file is used as the +default dispatch history source. + ### Calibration Validate estimates against observed outcomes and build a calibration database: diff --git a/action.yml b/action.yml index a387999..b04c12d 100644 --- a/action.yml +++ b/action.yml @@ -53,7 +53,7 @@ inputs: required: false default: '3.12' version: - description: 'agent-estimate version to install (e.g. "0.7.0"). Omit for latest.' + description: 'agent-estimate version to install (e.g. "0.7.1"). Omit for latest.' required: false token: description: 'GitHub token for issue fetching and PR comments' diff --git a/examples/multi-agent.md b/examples/multi-agent.md index a9272fb..fd9fbfe 100644 --- a/examples/multi-agent.md +++ b/examples/multi-agent.md @@ -59,8 +59,8 @@ agent-estimate estimate --file tasks.txt ### METR Warnings -- **Add known_debt.md as standard protocol memory file**: Estimate (68m) exceeds gpt_5_3 p80 threshold (60m). Consider splitting the task. -- **Write quickstart guide with protocol comparison table**: Estimate (68m) exceeds gemini_3_pro p80 threshold (45m). Consider splitting the task. +- **Add known_debt.md as standard protocol memory file**: Estimate (68m) exceeds gpt_5_4 p80 threshold (60m). Consider splitting the task. +- **Write quickstart guide with protocol comparison table**: Estimate (68m) exceeds gemini_3_1_pro p80 threshold (45m). Consider splitting the task. ## What actually happened diff --git a/pyproject.toml b/pyproject.toml index b6acf73..2f8d575 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-estimate" -version = "0.7.0" +version = "0.7.1" description = "Know what an AI task will cost before you run it" readme = "README.md" license = "Apache-2.0" diff --git a/skills/estimate/README.md b/skills/estimate/README.md index 41c9d00..ae8f8aa 100644 --- a/skills/estimate/README.md +++ b/skills/estimate/README.md @@ -35,8 +35,8 @@ cp skills/estimate/codex/SKILL.md .codex/skills/estimate/SKILL.md /estimate Add a login page with OAuth /estimate --file tasks.md /estimate --issues 1,2,3 --repo myorg/myrepo -/validate-estimate observation.yaml -/calibrate +/estimate validate observation.yaml +/estimate calibrate # Codex # Invoke via AGENTS.md task dispatch or direct skill reference diff --git a/skills/estimate/claude/SKILL.md b/skills/estimate/claude/SKILL.md index 93f2afe..5f71c74 100644 --- a/skills/estimate/claude/SKILL.md +++ b/skills/estimate/claude/SKILL.md @@ -18,8 +18,8 @@ Run PERT three-point estimation with METR reliability thresholds and wave planni /estimate --format json /estimate --review-mode none /estimate --title "My Report" -/validate-estimate -/calibrate +/estimate validate +/estimate calibrate ``` ## Instructions @@ -28,13 +28,15 @@ When the user invokes this skill, follow these steps: ### 1. Parse the invocation -Determine which subcommand to run based on context: +Determine which subcommand to run from the shape of the arguments after `/estimate`: -| Invocation pattern | Subcommand | -| ----------------------------- | -------------------------- | -| `/estimate ...` with any args | `agent-estimate estimate` | -| `/validate-estimate ` | `agent-estimate validate` | -| `/calibrate` | `agent-estimate calibrate` | +| Invocation pattern | Subcommand | +| ------------------------------- | -------------------------- | +| `validate ` — first token is `validate`, rest is one `.yaml`/`.yml` path plus optional `--db ` | `agent-estimate validate` | +| `calibrate` — first token is `calibrate`, rest is empty or only `--db ` | `agent-estimate calibrate` | +| anything else | `agent-estimate estimate` | + +Shape-match strictly: if the arguments start with `validate` or `calibrate` but the rest does not fit the shapes above, treat the whole argument string as a task description for `agent-estimate estimate` — e.g. `/estimate validate the login flow` is an estimation request, not a malformed validate call. ### 2. Build the CLI command @@ -75,11 +77,11 @@ If none of `--file`, `--issues`, or a task description is provided, prompt the u - `frontend` — UI/page work (content patches use 15/25/40; page builds use 40/60/90). - `app_dev` — app shells and desktop/mobile builds (cold generic L-style prior; use modifiers for warm or highly specified work). -#### For `/validate-estimate` +#### For `/estimate validate` -The argument after `/validate-estimate` is the observation YAML file path. Optionally pass `--db ` if provided. +The argument after `validate` is the observation YAML file path. Optionally pass `--db ` if provided. -#### For `/calibrate` +#### For `/estimate calibrate` Optionally pass `--db ` if provided. Default: `~/.agent-estimate/calibration.db`. @@ -120,17 +122,17 @@ Display the CLI output directly to the user. No post-processing — the CLI hand /estimate --config agents.yaml --format json "Refactor auth module" → agent-estimate estimate --config agents.yaml --format json "Refactor auth module" -/validate-estimate results/sprint1.yaml +/estimate validate results/sprint1.yaml → agent-estimate validate results/sprint1.yaml -/validate-estimate results/sprint1.yaml --db ~/.agent-estimate/calibration.db +/estimate validate results/sprint1.yaml --db ~/.agent-estimate/calibration.db → agent-estimate validate results/sprint1.yaml --db ~/.agent-estimate/calibration.db -/calibrate +/estimate calibrate → agent-estimate calibrate ``` -## Observation YAML format (for `/validate-estimate`) +## Observation YAML format (for `/estimate validate`) ```yaml task_type: feature diff --git a/src/agent_estimate/adapters/sqlite_store.py b/src/agent_estimate/adapters/sqlite_store.py index c546844..f92b7a7 100644 --- a/src/agent_estimate/adapters/sqlite_store.py +++ b/src/agent_estimate/adapters/sqlite_store.py @@ -406,17 +406,26 @@ def _validate_observation(observation: ObservationInput) -> None: def _normalize_timestamp(value: str | None) -> str: if value is None: - return datetime.now(timezone.utc).isoformat(timespec="seconds") - return value + dt = datetime.now(timezone.utc) + else: + dt = _parse_iso_timestamp(value) + return dt.isoformat(timespec="seconds") def _week_start(timestamp: str) -> str: - normalized = timestamp.replace("Z", "+00:00") - dt = datetime.fromisoformat(normalized) + dt = _parse_iso_timestamp(timestamp) monday = (dt - timedelta(days=dt.weekday())).date() return monday.isoformat() +def _parse_iso_timestamp(value: str) -> datetime: + normalized = value[:-1] + "+00:00" if value.endswith("Z") else value + dt = datetime.fromisoformat(normalized) + if dt.tzinfo is None: + return dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc) + + def _percentile(values: list[float], percent: float) -> float: if not values: raise ValueError("percentile requires at least one value") diff --git a/src/agent_estimate/cli/commands/_pipeline.py b/src/agent_estimate/cli/commands/_pipeline.py index 9f8e917..b84a10e 100644 --- a/src/agent_estimate/cli/commands/_pipeline.py +++ b/src/agent_estimate/cli/commands/_pipeline.py @@ -330,10 +330,12 @@ def _build_report( assigned_agent = assignment_map.get(str(i), default_agent) model_tier = agent_model_tier.get(assigned_agent, default_tier) - # Re-check METR threshold with the assigned agent's model tier + # Re-check METR threshold with the assigned agent's model tier and the + # same frictioned work duration the wave planner schedules. + metr_minutes = (est.pert.expected * config.settings.friction_multiplier) + est.review_minutes corrected_warning = check_metr_threshold( model_tier, - est.total_expected_minutes, + metr_minutes, thresholds=thresholds, fallback_threshold=fallback, agent_name=assigned_agent, diff --git a/src/agent_estimate/cli/commands/_utils.py b/src/agent_estimate/cli/commands/_utils.py new file mode 100644 index 0000000..8f7f6c2 --- /dev/null +++ b/src/agent_estimate/cli/commands/_utils.py @@ -0,0 +1,12 @@ +"""Shared CLI command helpers.""" + +from __future__ import annotations + +import typer + + +def validate_output_format(value: str) -> None: + """Validate a command's output format before side-effectful work runs.""" + if value not in {"markdown", "json"}: + typer.echo(f"Error: Unknown format: {value!r}. Use markdown or json.", err=True) + raise typer.Exit(code=2) diff --git a/src/agent_estimate/cli/commands/estimate.py b/src/agent_estimate/cli/commands/estimate.py index d2f0a27..796fe8b 100644 --- a/src/agent_estimate/cli/commands/estimate.py +++ b/src/agent_estimate/cli/commands/estimate.py @@ -14,6 +14,7 @@ from agent_estimate.adapters.github_ghcli import GitHubGhCliAdapter from agent_estimate.audit import emit_audit_event from agent_estimate.cli.commands._pipeline import run_estimate_pipeline +from agent_estimate.cli.commands._utils import validate_output_format from agent_estimate.cli.commands.github import parse_issue_selection from agent_estimate.core import EstimationCategory, EstimationConfig, ReviewMode from agent_estimate.core.history import infer_warm_context @@ -58,10 +59,13 @@ def run( "--spec-clarity", help="Spec clarity modifier (range: 0.3 to 1.3; lower means clearer spec).", ), - warm_context: float = typer.Option( - 1.0, + warm_context: Optional[float] = typer.Option( + None, "--warm-context", - help="Warm context modifier (range: 0.3 to 1.15; lower means warmer context).", + help=( + "Warm context modifier (range: 0.3 to 1.15; lower means warmer context). " + "When omitted, auto-infers from --history-file or ./data.json when present." + ), ), agent_fit: float = typer.Option( 1.0, @@ -124,6 +128,7 @@ def run( _error("Provide a task description, --file, or --issues.", 2) if sources > 1: _error("Provide only one input source: task argument, --file, or --issues.", 2) + validate_output_format(format) descriptions: list[str] = [] @@ -197,10 +202,10 @@ def run( warm_ctx = infer_warm_context( history_path, agent=history_agent, project=history_project ) - # Auto-inferred warm_context applies when --warm-context wasn't explicitly set - effective_warm_context = warm_context + # Auto-inferred warm_context applies only when --warm-context was omitted. + effective_warm_context = 1.0 if warm_context is None else warm_context effective_detail: str | None = None - if warm_ctx.value != 1.0 and warm_context == 1.0: + if warm_ctx.value != 1.0 and warm_context is None: effective_warm_context = warm_ctx.value effective_detail = warm_ctx.detail logger.info( @@ -263,10 +268,8 @@ def run( # --- Output --- if format == "markdown": typer.echo(render_markdown_report(report)) - elif format == "json": - typer.echo(render_json_report(report), nl=False) else: - _error(f"Unknown format: {format!r}. Use markdown or json.", 2) + typer.echo(render_json_report(report), nl=False) def _error(message: str, exit_code: int) -> NoReturn: diff --git a/src/agent_estimate/cli/commands/session.py b/src/agent_estimate/cli/commands/session.py index abee2d5..709415c 100644 --- a/src/agent_estimate/cli/commands/session.py +++ b/src/agent_estimate/cli/commands/session.py @@ -8,6 +8,7 @@ import typer from agent_estimate.audit import emit_audit_event +from agent_estimate.cli.commands._utils import validate_output_format from agent_estimate.core.session import ( DEFAULT_COORDINATION_OVERHEAD_MINUTES, SESSION_TYPE_DURATIONS, @@ -62,6 +63,7 @@ def run( ) -> None: """Estimate wall-clock and agent-minutes for a multi-agent session.""" started_at = time.perf_counter() + validate_output_format(format) overhead = ( coordination_overhead if coordination_overhead is not None @@ -110,10 +112,8 @@ def run( "rounds_breakdown": list(result.rounds_breakdown), } typer.echo(json.dumps(data, indent=2), nl=False) - elif format == "markdown": - _render_markdown(result) else: - _error(f"Unknown format: {format!r}. Use markdown or json.", 2) + _render_markdown(result) # --------------------------------------------------------------------------- @@ -156,3 +156,4 @@ def _render_markdown(result: SessionEstimate) -> None: def _error(message: str, exit_code: int) -> NoReturn: typer.echo(f"Error: {message}", err=True) raise typer.Exit(code=exit_code) + diff --git a/src/agent_estimate/core/history.py b/src/agent_estimate/core/history.py index 557874b..8ec364c 100644 --- a/src/agent_estimate/core/history.py +++ b/src/agent_estimate/core/history.py @@ -47,6 +47,10 @@ def infer_warm_context( if reference_time is None: reference_time = datetime.now(timezone.utc) + elif reference_time.tzinfo is None: + reference_time = reference_time.replace(tzinfo=timezone.utc) + else: + reference_time = reference_time.astimezone(timezone.utc) # Filter by agent and project if specified filtered = dispatches @@ -66,9 +70,7 @@ def infer_warm_context( if completed_at is None: continue try: - dt = datetime.fromisoformat(completed_at) - if dt.tzinfo is None: - dt = dt.replace(tzinfo=timezone.utc) + dt = _parse_completed_at(completed_at) except (ValueError, TypeError): continue if most_recent_time is None or dt > most_recent_time: @@ -78,7 +80,7 @@ def infer_warm_context( if most_recent is None or most_recent_time is None: return WarmContextResult(value=1.0, source="default") - hours_ago = (reference_time - most_recent_time).total_seconds() / 3600.0 + hours_ago = max(0.0, (reference_time - most_recent_time).total_seconds() / 3600.0) value = _decay_to_warm_context(hours_ago) @@ -112,6 +114,16 @@ def _decay_to_warm_context(hours_ago: float) -> float: return 1.0 +def _parse_completed_at(value: object) -> datetime: + if not isinstance(value, str): + raise TypeError("completed_at must be a string") + normalized = value[:-1] + "+00:00" if value.endswith("Z") else value + dt = datetime.fromisoformat(normalized) + if dt.tzinfo is None: + return dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc) + + def _load_dispatches(path: Path) -> list[dict]: """Load dispatch records from a JSON file. diff --git a/src/agent_estimate/core/wave_planner.py b/src/agent_estimate/core/wave_planner.py index 786a74e..bf6b767 100644 --- a/src/agent_estimate/core/wave_planner.py +++ b/src/agent_estimate/core/wave_planner.py @@ -34,7 +34,9 @@ def plan_waves( Raises: ValueError: If the dependency graph contains a cycle, a task requires capabilities that no agent provides, a dependency references an - unknown task, or no agents are provided. + unknown task, no agents are provided, a duplicate task ID is + supplied, or any task has negative ``duration_minutes`` or + ``review_minutes``. """ if inter_wave_overhead_hours < 0: raise ValueError( @@ -58,6 +60,16 @@ def plan_waves( G = nx.DiGraph() task_map: dict[str, TaskNode] = {} for t in tasks: + if t.task_id in task_map: + raise ValueError(f"Duplicate task_id {t.task_id!r} is not allowed") + if t.duration_minutes < 0: + raise ValueError( + f"Task {t.task_id!r} duration_minutes must be >= 0, got {t.duration_minutes}" + ) + if t.review_minutes < 0: + raise ValueError( + f"Task {t.task_id!r} review_minutes must be >= 0, got {t.review_minutes}" + ) task_map[t.task_id] = t G.add_node(t.task_id, duration_minutes=t.duration_minutes) for t in tasks: diff --git a/src/agent_estimate/version.py b/src/agent_estimate/version.py index 38d6476..1a8560e 100644 --- a/src/agent_estimate/version.py +++ b/src/agent_estimate/version.py @@ -1,3 +1,3 @@ """Version constants for agent-estimate.""" -__version__ = "0.7.0" +__version__ = "0.7.1" diff --git a/tests/integration/test_cli_e2e.py b/tests/integration/test_cli_e2e.py index 9dd3c5a..23adab0 100644 --- a/tests/integration/test_cli_e2e.py +++ b/tests/integration/test_cli_e2e.py @@ -328,6 +328,47 @@ def test_format_unknown(self) -> None: assert result.exit_code != 0 assert "Unknown format" in result.output + def test_format_unknown_does_not_emit_estimation_audit( + self, + monkeypatch, + tmp_path: Path, + ) -> None: + audit_log = tmp_path / "audit.jsonl" + monkeypatch.setenv("AGENT_ESTIMATE_AUDIT_ENABLED", "1") + monkeypatch.setenv("AGENT_ESTIMATE_AUDIT_DESTINATION", str(audit_log)) + monkeypatch.setenv("AGENT_ESTIMATE_AUDIT_LEVEL", "INFO") + reset_audit_logger() + + result = runner.invoke(app, ["estimate", "--format", "xml", "Add button"]) + + reset_audit_logger() + assert result.exit_code != 0 + assert not audit_log.exists() + + def test_format_unknown_with_issues_does_not_fetch_github(self, monkeypatch) -> None: + class _FailingGitHubAdapter: + def fetch_task_descriptions_by_numbers( + self, repo: str, issue_numbers: list[int] + ) -> list[str]: + raise AssertionError("GitHub fetch should not run for invalid format") + + monkeypatch.setattr(estimate_command, "GitHubGhCliAdapter", _FailingGitHubAdapter) + result = runner.invoke( + app, + [ + "estimate", + "--issues", + "11", + "--repo", + "kiloloop/agent-estimate", + "--format", + "xml", + ], + ) + + assert result.exit_code != 0 + assert "Unknown format" in result.output + # --------------------------------------------------------------------------- # Estimate — error cases @@ -504,3 +545,34 @@ def test_history_agent_filter_scopes_inference(self, tmp_path: Path) -> None: data = json.loads(result.output) task = data["tasks"][0] assert task["modifiers"]["warm_context"] == 1.0 + + def test_explicit_warm_context_10_overrides_default_data_json( + self, + tmp_path: Path, + monkeypatch, + ) -> None: + import json + from datetime import datetime, timedelta, timezone + + monkeypatch.chdir(tmp_path) + recent = (datetime.now(timezone.utc) - timedelta(minutes=10)).isoformat() + Path("data.json").write_text( + json.dumps( + { + "dispatches": [ + {"agent": "codex", "project": "proj", "completed_at": recent} + ] + } + ), + encoding="utf-8", + ) + + result = runner.invoke( + app, + ["estimate", "--warm-context", "1.0", "--format", "json", "Add a button"], + ) + + assert result.exit_code == 0 + data = json.loads(result.output) + task = data["tasks"][0] + assert task["modifiers"]["warm_context"] == 1.0 diff --git a/tests/unit/test_docs_freshness.py b/tests/unit/test_docs_freshness.py new file mode 100644 index 0000000..c271048 --- /dev/null +++ b/tests/unit/test_docs_freshness.py @@ -0,0 +1,66 @@ +"""Docs freshness guards (#28). + +README's hero example and examples/multi-agent.md embed captured CLI output. +These tests re-run the same estimate and assert the embedded headline lines +still match what the tool actually prints, so a prior recalibration or METR +model-key bump fails CI instead of silently rotting the docs. +""" + +from pathlib import Path + +from typer.testing import CliRunner + +from agent_estimate.cli.app import app + +ROOT = Path(__file__).resolve().parents[2] + +# The documented input from examples/multi-agent.md "Input file" block. +MULTI_AGENT_TASKS = [ + "Implement add-agent CLI command with SPEC.md generation", + "Add known_debt.md as standard protocol memory file", + "Write quickstart guide with protocol comparison table", +] + +# Headline output lines embedded verbatim in README.md and +# examples/multi-agent.md. If the CLI stops producing them, both docs +# need re-capturing. +HEADLINE_LINES = [ + "| Expected case | 75.4m |", + "| Compression ratio | 7.60x |", + "exceeds gpt_5_4 p80 threshold (60m)", + "exceeds gemini_3_1_pro p80 threshold (45m)", +] + +runner = CliRunner() + + +def _real_output(tmp_path: Path) -> str: + task_file = tmp_path / "tasks.txt" + task_file.write_text("\n".join(MULTI_AGENT_TASKS) + "\n", encoding="utf-8") + result = runner.invoke(app, ["estimate", "--file", str(task_file)]) + assert result.exit_code == 0, result.output + return result.output + + +class TestDocsFreshness: + def test_example_input_block_lists_the_tasks(self): + example = (ROOT / "examples" / "multi-agent.md").read_text() + for task in MULTI_AGENT_TASKS: + assert task in example, f"examples/multi-agent.md lost input task {task!r}" + + def test_readme_hero_matches_real_run(self, tmp_path): + output = _real_output(tmp_path) + readme = (ROOT / "README.md").read_text() + for line in HEADLINE_LINES: + assert line in output, ( + f"CLI no longer prints {line!r} for the documented tasks — " + "re-capture the README hero block and examples/multi-agent.md" + ) + assert line in readme, f"README hero block lost {line!r}" + + def test_example_output_matches_real_run(self, tmp_path): + output = _real_output(tmp_path) + example = (ROOT / "examples" / "multi-agent.md").read_text() + for line in HEADLINE_LINES: + assert line in output + assert line in example, f"examples/multi-agent.md output block lost {line!r}" diff --git a/tests/unit/test_history.py b/tests/unit/test_history.py index 20031aa..657cc52 100644 --- a/tests/unit/test_history.py +++ b/tests/unit/test_history.py @@ -197,6 +197,47 @@ def test_multiple_dispatches_uses_most_recent( assert result.source == "auto" +def test_z_suffix_completed_at_is_parsed_python310_compatible( + tmp_path: Path, ref_time: datetime +) -> None: + path = _write_history( + tmp_path, + [ + { + "agent": "codex", + "project": "agent-estimate", + "completed_at": "2026-02-19T08:52:00Z", + } + ], + ) + + result = infer_warm_context(path, reference_time=ref_time) + + assert result.value == 0.3 + assert result.source == "auto" + assert "8m ago" in result.detail + + +def test_future_dispatch_clamps_elapsed_to_now( + tmp_path: Path, ref_time: datetime +) -> None: + path = _write_history( + tmp_path, + [ + { + "agent": "codex", + "project": "agent-estimate", + "completed_at": (ref_time + timedelta(minutes=30)).isoformat(), + } + ], + ) + + result = infer_warm_context(path, reference_time=ref_time) + + assert result.value == 0.3 + assert "0m ago" in result.detail + + # --- None path test --- diff --git a/tests/unit/test_pipeline_metr_mapping.py b/tests/unit/test_pipeline_metr_mapping.py index 2c29a6a..454e176 100644 --- a/tests/unit/test_pipeline_metr_mapping.py +++ b/tests/unit/test_pipeline_metr_mapping.py @@ -35,6 +35,19 @@ def _claude_frontier_config() -> EstimationConfig: ) +def _claude_frontier_config_with_friction(friction: float) -> EstimationConfig: + config = _claude_frontier_config() + return EstimationConfig( + agents=config.agents, + settings=ProjectSettings( + friction_multiplier=friction, + inter_wave_overhead=0.0, + review_overhead=0.0, + metr_fallback_threshold=45.0, + ), + ) + + class TestPipelineMetrMapping: def test_claude_assigned_task_uses_opus_threshold(self, monkeypatch) -> None: monkeypatch.setattr( @@ -81,3 +94,27 @@ def test_no_false_positive_for_claude_task_at_or_below_90m(self, monkeypatch) -> task = report.tasks[0] assert task.agent == "Claude" assert task.metr_warning is None + + def test_assigned_task_metr_recheck_includes_friction(self, monkeypatch) -> None: + monkeypatch.setattr( + _pipeline, + "classify_task", + lambda _description: SizingResult( + tier=SizeTier.M, + baseline_optimistic=30.0, + baseline_most_likely=50.0, + baseline_pessimistic=85.0, + task_type=TaskType.FEATURE, + signals=("test",), + ), + ) + + report = run_estimate_pipeline( + ["deterministic"], + _claude_frontier_config_with_friction(2.0), + review_mode=ReviewMode.STANDARD, + ) + + task = report.tasks[0] + assert task.metr_warning is not None + assert "Estimate (120m) exceeds opus_4_7 p80 threshold (90m)" in task.metr_warning diff --git a/tests/unit/test_plugin_structure.py b/tests/unit/test_plugin_structure.py index 9c9fc63..fe1a918 100644 --- a/tests/unit/test_plugin_structure.py +++ b/tests/unit/test_plugin_structure.py @@ -40,6 +40,27 @@ def test_plugin_version_matches_package(self): f"version.py ({__version__})" ) + def test_plugin_json_skills_paths_resolve(self): + """Every plugin.json 'skills' entry must contain a SKILL.md (#29). + + The v0.7.0 restructure moved SKILL.md out of the default + skills//SKILL.md scan path with no test failing; the 'skills' + field is now the only thing making the plugin's skill discoverable. + """ + data = json.loads(self.plugin_json.read_text()) + skills = data.get("skills") + assert skills, ( + "plugin.json must declare 'skills' — the multi-runtime layout has " + "no skills//SKILL.md for the default plugin scan to find" + ) + entries = [skills] if isinstance(skills, str) else skills + for entry in entries: + skill_md = ROOT / entry / "SKILL.md" + assert skill_md.is_file(), ( + f"plugin.json skills entry {entry!r} must resolve to a " + f"directory containing SKILL.md (checked {skill_md})" + ) + class TestSkillStructure: """Tests for oacp-skills convention layout.""" diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index f5ee4f3..c73647b 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -7,6 +7,7 @@ import pytest from typer.testing import CliRunner +from agent_estimate.audit import reset_audit_logger from agent_estimate.cli.app import app from agent_estimate.core.session import ( DEFAULT_COORDINATION_OVERHEAD_MINUTES, @@ -268,6 +269,23 @@ def test_unknown_format(self) -> None: result = runner.invoke(app, ["session", "--format", "xml"]) assert result.exit_code != 0 + def test_unknown_format_does_not_emit_estimation_audit( + self, + monkeypatch, + tmp_path, + ) -> None: + audit_log = tmp_path / "audit.jsonl" + monkeypatch.setenv("AGENT_ESTIMATE_AUDIT_ENABLED", "1") + monkeypatch.setenv("AGENT_ESTIMATE_AUDIT_DESTINATION", str(audit_log)) + monkeypatch.setenv("AGENT_ESTIMATE_AUDIT_LEVEL", "INFO") + reset_audit_logger() + + result = runner.invoke(app, ["session", "--format", "xml"]) + + reset_audit_logger() + assert result.exit_code != 0 + assert not audit_log.exists() + def test_unknown_task_type(self) -> None: result = runner.invoke(app, ["session", "--type", "unknown_xyz"]) assert result.exit_code != 0 diff --git a/tests/unit/test_sqlite_gaps.py b/tests/unit/test_sqlite_gaps.py index 3226048..54b4bda 100644 --- a/tests/unit/test_sqlite_gaps.py +++ b/tests/unit/test_sqlite_gaps.py @@ -110,6 +110,43 @@ def test_zero_values_accepted(self, store: SQLiteCalibrationStore) -> None: assert obs_id > 0 +class TestTimestampNormalization: + def test_equivalent_instants_share_utc_week_start( + self, + store: SQLiteCalibrationStore, + ) -> None: + store.insert_observation( + _observation(observed_at="2026-03-01T22:30:00+00:00") + ) + store.insert_observation( + _observation(observed_at="2026-03-02T00:30:00+02:00") + ) + + rows = store._query_observations() + + assert {row["observed_at"] for row in rows} == {"2026-03-01T22:30:00+00:00"} + assert {row["week_start"] for row in rows} == {"2026-02-23"} + + def test_observations_order_chronologically_after_utc_normalization( + self, + store: SQLiteCalibrationStore, + ) -> None: + later_id = store.insert_observation( + _observation(observed_at="2026-03-02T20:00:00+00:00") + ) + earlier_id = store.insert_observation( + _observation(observed_at="2026-03-02T23:00:00+05:00") + ) + + rows = store._query_observations() + + assert [row["id"] for row in rows] == [earlier_id, later_id] + assert [row["observed_at"] for row in rows] == [ + "2026-03-02T18:00:00+00:00", + "2026-03-02T20:00:00+00:00", + ] + + # --------------------------------------------------------------------------- # _percentile edge cases # --------------------------------------------------------------------------- diff --git a/tests/unit/test_version.py b/tests/unit/test_version.py index 4ead920..f471d30 100644 --- a/tests/unit/test_version.py +++ b/tests/unit/test_version.py @@ -4,4 +4,4 @@ def test_version_string_present() -> None: - assert __version__ == "0.7.0" + assert __version__ == "0.7.1" diff --git a/tests/unit/test_wave_planner.py b/tests/unit/test_wave_planner.py index f3ddbef..9531736 100644 --- a/tests/unit/test_wave_planner.py +++ b/tests/unit/test_wave_planner.py @@ -262,6 +262,20 @@ def test_negative_overhead_raises(self) -> None: plan_waves([_node("A", 10)], [_agent()], inter_wave_overhead_hours=-0.5) +class TestTaskInputValidation: + def test_duplicate_task_ids_raise(self) -> None: + with pytest.raises(ValueError, match="Duplicate task_id 'A'"): + plan_waves([_node("A", 30), _node("A", 40)], [_agent()]) + + def test_negative_duration_raises(self) -> None: + with pytest.raises(ValueError, match="duration_minutes"): + plan_waves([_node("A", -1)], [_agent()]) + + def test_negative_review_minutes_raises(self) -> None: + with pytest.raises(ValueError, match="review_minutes"): + plan_waves([_rnode("A", 10, -1)], [_agent()]) + + class TestEmptyInput: """Empty task list returns a zero-valued plan."""