From 475bce7d5b8caefa0c6367ae01fc840b7c76b660 Mon Sep 17 00:00:00 2001 From: spbkgw-beep Date: Sun, 14 Jun 2026 23:09:37 -0700 Subject: [PATCH] feat: v0.7.2 Engine and documentation correctness fixes (#30, #32, #34, #35, #41-#45) plus the AGENT_ESTIMATE_AUDIT_DESTINATION=stdout deprecation. Adds the session estimate command surface and refreshes docs/examples. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude-plugin/plugin.json | 2 +- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- CHANGELOG.md | 7 ++ CONTRIBUTING.md | 8 ++ README.md | 24 ++++-- action.yml | 2 +- examples/README.md | 2 +- examples/multi-agent.md | 16 ++-- pyproject.toml | 2 +- skills/estimate/shared/INTENT.md | 16 +++- skills/estimate/skill.yaml | 2 +- src/agent_estimate/adapters/config_loader.py | 11 ++- src/agent_estimate/adapters/github_ghcli.py | 44 ++++++++++- src/agent_estimate/adapters/github_rest.py | 61 +++++++++++++--- src/agent_estimate/adapters/sqlite_store.py | 29 ++++++-- src/agent_estimate/audit.py | 36 ++++++--- src/agent_estimate/cli/commands/calibrate.py | 2 +- src/agent_estimate/cli/commands/estimate.py | 37 +++++++++- src/agent_estimate/cli/commands/session.py | 7 +- src/agent_estimate/cli/commands/validate.py | 74 +++++++++++-------- src/agent_estimate/core/models.py | 15 ++++ src/agent_estimate/core/modifiers.py | 2 +- src/agent_estimate/core/pert.py | 4 + src/agent_estimate/core/sizing.py | 7 +- src/agent_estimate/core/task_type_models.py | 8 +- src/agent_estimate/core/wave_planner.py | 12 +-- src/agent_estimate/render/json_report.py | 10 +++ src/agent_estimate/render/markdown_report.py | 26 ++++++- src/agent_estimate/skill/claude_wrapper.py | 4 +- src/agent_estimate/version.py | 2 +- tests/fixtures/json_report_golden.json | 20 ++++- tests/integration/test_cli_e2e.py | 77 ++++++++++++++++++++ tests/unit/test_audit.py | 40 ++++++++++ tests/unit/test_claude_wrapper.py | 47 ++++++++++++ tests/unit/test_config_loader.py | 20 +++++ tests/unit/test_docs_freshness.py | 11 ++- tests/unit/test_github_adapters.py | 53 ++++++++++++++ tests/unit/test_json_report.py | 7 ++ tests/unit/test_markdown_report.py | 42 +++++++++++ tests/unit/test_models.py | 27 +++++++ tests/unit/test_pert_engine.py | 6 ++ tests/unit/test_pert_gaps.py | 9 +++ tests/unit/test_plugin_structure.py | 11 +++ tests/unit/test_session.py | 16 ++++ tests/unit/test_sizing.py | 17 ++++- tests/unit/test_sqlite_gaps.py | 43 +++++++++++ tests/unit/test_sqlite_store.py | 19 +++++ tests/unit/test_task_type_models.py | 6 +- tests/unit/test_version.py | 2 +- tests/unit/test_wave_planner.py | 11 +-- 50 files changed, 835 insertions(+), 123 deletions(-) create mode 100644 tests/unit/test_claude_wrapper.py diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index aeacd44..b038bb4 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "agent-estimate", "description": "Effort estimation for AI coding agents — PERT three-point estimation with METR reliability thresholds and wave planning", - "version": "0.7.1", + "version": "0.7.2", "skills": ["./skills/estimate/claude"], "author": { "name": "Kiloloop" diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index a2144f1..407ff22 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -55,7 +55,7 @@ body: id: version attributes: label: agent-estimate version - placeholder: "0.7.1" + placeholder: "0.7.2" validations: required: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a8f308..5ed5dac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.7.2] - 2026-06-14 + +### Fixed +- Fixed engine and documentation correctness gaps: documentation estimates now use the docs human multiplier, CLI file/config and GitHub adapter failures stay user-facing, REST issue ingestion is reachable in token-only environments, renderer warning/detail fields are escaped and serialized, sizing heuristics prefer explicit test/docs matches, store/audit/validation edge cases are hardened, and stale skill/session/release docs are refreshed (#30, #32, #34, #35, #41, #42, #43, #44, #45). +- `AGENT_ESTIMATE_AUDIT_DESTINATION=stdout` now emits a deprecation warning and routes audit events to stderr so report stdout remains parseable for JSON consumers (#43). + ## [0.7.1] - 2026-06-11 ### Fixed @@ -104,6 +110,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Modifier flags: `--warm-context`, `--spec-clarity`, `--issues` - PyPI package: `pip install agent-estimate` +[0.7.2]: https://github.com/kiloloop/agent-estimate/releases/tag/v0.7.2 [0.7.1]: https://github.com/kiloloop/agent-estimate/releases/tag/v0.7.1 [0.7.0]: https://github.com/kiloloop/agent-estimate/releases/tag/v0.7.0 [0.6.1]: https://github.com/kiloloop/agent-estimate/releases/tag/v0.6.1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3153ac9..913a021 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,6 +48,14 @@ If your change touches CLI behavior, include at least one integration-style test 4. Include a short test plan and validation output. 5. Update docs/changelog when behavior changes. +## Release Checklist + +1. Bump all versioned artifacts together: `pyproject.toml`, `src/agent_estimate/version.py`, `.claude-plugin/plugin.json`, `action.yml`, issue templates, and version tests. +2. Run `ruff check .`, `pytest -q`, `python -m build`, and any release-specific smoke tests. +3. Publish the GitHub release and PyPI artifacts. +4. From the `kiloloop/agent-estimate` public clone, move the floating public Action tag, for example `git tag -f v0 vX.Y.Z && git push origin v0 --force`, so `uses: kiloloop/agent-estimate@v0` tracks the latest compatible release. Do not perform this step from the private staging clone. +5. Confirm README, examples, and skill docs match the released CLI output and model keys. + ## Reporting Bugs Use the bug report issue template and include: diff --git a/README.md b/README.md index f23ae94..5020b1f 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ No config required — sensible defaults for a 3-agent fleet (Claude, Codex, Gem ```bash agent-estimate estimate --file tasks.txt agent-estimate estimate --repo myorg/myrepo --issues 11,12,14 +agent-estimate session --agents 3 --rounds 2 --type review ``` ## How It Works @@ -114,17 +115,17 @@ $ agent-estimate estimate --file tasks.txt | Best case | 44.7m | | Expected case | 75.4m | | Worst case | 117.2m | -| Human-speed equivalent | 572.8m | -| Compression ratio | 7.60x | +| Human-speed equivalent | 608.2m | +| Compression ratio | 8.07x | | Review overhead (per-task, pre-amortization) | 45m | ## METR Warnings -- **Add known_debt.md as standard protocol memory file**: Estimate (68m) exceeds gpt_5_4 p80 threshold (60m). Consider splitting the task. -- **Write quickstart guide with protocol comparison table**: Estimate (68m) exceeds gemini_3_1_pro p80 threshold (45m). Consider splitting the task. +- **Add known_debt.md as standard protocol memory file**: Estimate (75m) exceeds gpt_5_4 p80 threshold (60m). Consider splitting the task. +- **Write quickstart guide with protocol comparison table**: Estimate (75m) exceeds gemini_3_1_pro p80 threshold (45m). Consider splitting the task. ``` -~75 minutes wall-clock versus ~9.5 hours of sequential human work, at an estimated $3.51 fleet cost — plus two flags that the Codex- and Gemini-assigned tasks run past their models' p80 reliability horizons, so you split them or add a checkpoint before dispatching. The same three tasks were later run by real agents; the retro is in the example file. More in [`examples/`](./examples/) — coding S/M, research, documentation, multi-agent. +~75 minutes wall-clock versus ~10.1 hours of sequential human work, at an estimated $3.51 fleet cost — plus two flags that the Codex- and Gemini-assigned tasks run past their models' p80 reliability horizons, so you split them or add a checkpoint before dispatching. The same three tasks were later run by real agents; the retro is in the example file. More in [`examples/`](./examples/) — coding S/M, research, documentation, multi-agent. ## Integrations @@ -262,6 +263,19 @@ When `--warm-context` is omitted, the CLI can auto-infer it from `--history-file if no history file is passed and `./data.json` exists, that file is used as the default dispatch history source. +### Session estimates + +Use `agent-estimate session` for coordinated workflows where multiple agents run +rounds of brainstorm, review, research, documentation, config, or coding work: + +```bash +agent-estimate session --agents 3 --rounds 2 --type review +agent-estimate session --agents 4 --rounds 1 --per-round-minutes 25 --format json +``` + +The command reports wall-clock time, total agent-minutes, coordination overhead, +and per-round breakdowns. + ### Calibration Validate estimates against observed outcomes and build a calibration database: diff --git a/action.yml b/action.yml index b04c12d..733eace 100644 --- a/action.yml +++ b/action.yml @@ -53,7 +53,7 @@ inputs: required: false default: '3.12' version: - description: 'agent-estimate version to install (e.g. "0.7.1"). Omit for latest.' + description: 'agent-estimate version to install (e.g. "0.7.2"). Omit for latest.' required: false token: description: 'GitHub token for issue fetching and PR comments' diff --git a/examples/README.md b/examples/README.md index 8da19e5..76026aa 100644 --- a/examples/README.md +++ b/examples/README.md @@ -8,7 +8,7 @@ Real input/output examples from production agent dispatches. Every command below | Implement CLI command with code generation | Coding | M | 2.53x | [coding-m.md](./coding-m.md) | | Audit cloud infrastructure providers | Research | S | 2.61x | [research.md](./research.md) | | Write quickstart guide + README | Documentation | S | 2.58x | [documentation.md](./documentation.md) | -| 3-agent parallel session (3 features) | Multi-agent | M×3 | 7.60x | [multi-agent.md](./multi-agent.md) | +| 3-agent parallel session (3 features) | Multi-agent | M×3 | 8.07x | [multi-agent.md](./multi-agent.md) | ## How to read the output diff --git a/examples/multi-agent.md b/examples/multi-agent.md index fd9fbfe..49092d7 100644 --- a/examples/multi-agent.md +++ b/examples/multi-agent.md @@ -24,7 +24,7 @@ agent-estimate estimate --file tasks.txt | Task | Model | Tier | Agent | Base PERT (O/M/P) | Modifiers | Effective Duration | Human Equivalent | | --- | --- | --- | --- | --- | --- | --- | --- | -| **Implement add-agent CLI command with SPEC.md generation** | coding | M | Claude | 25m / 50m / 90m (E=52.5m) | spec 1.00 x warm 1.00 x fit 1.00 = 1.00 | 52.5m | 190.9m | +| **Implement add-agent CLI command with SPEC.md generation** | coding | M | Claude | 25m / 50m / 90m (E=52.5m) | spec 1.00 x warm 1.00 x fit 1.00 = 1.00 | 52.5m | 226.4m | | Add known_debt.md as standard protocol memory file | coding | M | Codex | 25m / 50m / 90m (E=52.5m) | spec 1.00 x warm 1.00 x fit 1.00 = 1.00 | 52.5m | 190.9m | | Write quickstart guide with protocol comparison table | coding | M | Gemini | 25m / 50m / 90m (E=52.5m) | spec 1.00 x warm 1.00 x fit 1.00 = 1.00 | 52.5m | 190.9m | @@ -41,8 +41,8 @@ agent-estimate estimate --file tasks.txt | Best case | 44.7m | | Expected case | 75.4m | | Worst case | 117.2m | -| Human-speed equivalent | 572.8m | -| Compression ratio | 7.60x | +| Human-speed equivalent | 608.2m | +| Compression ratio | 8.07x | | Review overhead (per-task, pre-amortization) | 45m | ### Agent Load Summary @@ -59,8 +59,8 @@ agent-estimate estimate --file tasks.txt ### METR Warnings -- **Add known_debt.md as standard protocol memory file**: Estimate (68m) exceeds gpt_5_4 p80 threshold (60m). Consider splitting the task. -- **Write quickstart guide with protocol comparison table**: Estimate (68m) exceeds gemini_3_1_pro p80 threshold (45m). Consider splitting the task. +- **Add known_debt.md as standard protocol memory file**: Estimate (75m) exceeds gpt_5_4 p80 threshold (60m). Consider splitting the task. +- **Write quickstart guide with protocol comparison table**: Estimate (75m) exceeds gemini_3_1_pro p80 threshold (45m). Consider splitting the task. ## What actually happened @@ -72,12 +72,12 @@ These three tasks were dispatched to real agents in a production multi-agent wor | Protocol memory file | Codex | 75.4m | ~20m | Clean merge | Merged | | Quickstart + README | Claude | 75.4m | ~45m | 2 rounds | Merged | -**Wall clock**: All three ran in parallel. Total elapsed ~90m (bounded by the slowest task). Sequential human equivalent: ~573m (~9.5 hours). +**Wall clock**: All three ran in parallel. Total elapsed ~90m (bounded by the slowest task). Sequential human equivalent: ~608m (~10.1 hours). -**Actual compression**: 6.4x (90m wall clock / 573m human equivalent). The estimate predicted 7.6x — close, because one task (add-agent CLI) needed an extra review round that pushed it past the expected case. +**Actual compression**: 6.8x (90m wall clock / 608m human equivalent). The estimate predicted 8.1x — close, because one task (add-agent CLI) needed an extra review round that pushed it past the expected case. The METR warnings were useful: Codex's task landed well under the threshold (20m actual vs 60m warning), but the warning correctly flagged that the Gemini-assigned task was near its reliability limit. ## Key takeaway -Multi-agent sessions are where `agent-estimate` delivers the most value. Three agents working in parallel produce **7.6x compression** — ~75 minutes wall clock vs ~9.5 hours of sequential human work. The wave planner automatically assigns tasks to agents, schedules them in parallel, and flags reliability risks via METR warnings. You see the total cost ($3.51), parallelism benefit, and risk before committing compute. +Multi-agent sessions are where `agent-estimate` delivers the most value. Three agents working in parallel produce **8.1x compression** — ~75 minutes wall clock vs ~10.1 hours of sequential human work. The wave planner automatically assigns tasks to agents, schedules them in parallel, and flags reliability risks via METR warnings. You see the total cost ($3.51), parallelism benefit, and risk before committing compute. diff --git a/pyproject.toml b/pyproject.toml index 2f8d575..646b5b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-estimate" -version = "0.7.1" +version = "0.7.2" description = "Know what an AI task will cost before you run it" readme = "README.md" license = "Apache-2.0" diff --git a/skills/estimate/shared/INTENT.md b/skills/estimate/shared/INTENT.md index 65c2fcc..6acfd78 100644 --- a/skills/estimate/shared/INTENT.md +++ b/skills/estimate/shared/INTENT.md @@ -9,6 +9,7 @@ Wraps the `agent-estimate` CLI to provide PERT three-point effort estimation wit | Intent | Command | |--------|---------| | Estimate one or more tasks | `agent-estimate estimate ...` | +| Estimate a multi-agent session | `agent-estimate session ...` | | Validate estimate vs actuals | `agent-estimate validate ...` | | Recompute calibration summary | `agent-estimate calibrate ...` | @@ -24,12 +25,25 @@ Accepts exactly one input source: Optional flags: - `--config ` — path to config YAML with agent definitions - `--format markdown|json` — output format (default: `markdown`) -- `--review-mode none|standard|complex` — review overhead tier +- `--review-mode none|standard|complex|3-round` — review overhead tier +- `--type coding|brainstorm|research|config|documentation|frontend|app_dev` — task category; omit to auto-detect +- `--spec-clarity <0.3..1.3>` — spec clarity modifier +- `--warm-context <0.3..1.15>` — warm context modifier +- `--agent-fit <0.9..1.2>` — agent fit modifier - `--title ` — report title - `--verbose` — enable debug logging If no input source is provided, prompt the user. +### `session` + +- `--agents ` — number of parallel agents +- `--rounds ` — number of sequential rounds +- `--type brainstorm|review|research|documentation|config|coding` — session task type +- `--coordination-overhead ` — per-round coordination overhead +- `--per-round-minutes ` — explicit per-agent round duration +- `--format markdown|json` — output format + ### `validate` - Required: observation YAML path diff --git a/skills/estimate/skill.yaml b/skills/estimate/skill.yaml index 6b70bc7..93e59c6 100644 --- a/skills/estimate/skill.yaml +++ b/skills/estimate/skill.yaml @@ -7,6 +7,6 @@ compatible_runtimes: - claude-code - codex requires_oacp: ">=0.1.0" -public_status: staging +public_status: public owners: - kiloloop diff --git a/src/agent_estimate/adapters/config_loader.py b/src/agent_estimate/adapters/config_loader.py index 5f36226..d7e429c 100644 --- a/src/agent_estimate/adapters/config_loader.py +++ b/src/agent_estimate/adapters/config_loader.py @@ -7,6 +7,7 @@ from importlib.resources import as_file, files from pathlib import Path from typing import Any +import warnings import yaml from pydantic import ValidationError @@ -72,7 +73,15 @@ def discover_plugin_profiles() -> list[AgentProfile]: """Discover and validate agent profiles from entry points.""" discovered: dict[str, AgentProfile] = {} for ep in _iter_agent_entry_points(): - profile = _load_entry_point_profile(ep) + try: + profile = _load_entry_point_profile(ep) + except ValueError as exc: + warnings.warn( + f"Skipping agent profile entry point {ep.name!r}: {exc}", + RuntimeWarning, + stacklevel=2, + ) + continue discovered[profile.name] = profile return list(discovered.values()) diff --git a/src/agent_estimate/adapters/github_ghcli.py b/src/agent_estimate/adapters/github_ghcli.py index ab34e75..edc3c4a 100644 --- a/src/agent_estimate/adapters/github_ghcli.py +++ b/src/agent_estimate/adapters/github_ghcli.py @@ -5,6 +5,7 @@ import json import subprocess import time +import warnings from typing import Callable, Sequence from agent_estimate.adapters.github_adapter import ( @@ -14,6 +15,8 @@ ) from agent_estimate.audit import emit_audit_event +_ISSUE_LIST_LIMIT = 1000 + class GitHubGhCliAdapter: """Fetch GitHub issues through gh CLI commands.""" @@ -60,7 +63,9 @@ def fetch_issues_by_numbers(self, repo: str, issue_numbers: Sequence[int]) -> li repo=repo, issue_number=issue_number, ) - payload = json.loads(output) + payload = _load_json(output, "gh issue view") + if not isinstance(payload, dict): + raise GitHubAdapterError(f"Unexpected gh issue view output: {payload!r}") issues.append(_parse_issue(payload)) return issues @@ -83,7 +88,7 @@ def fetch_issues_by_label( "--state", state, "--limit", - "1000", + str(_ISSUE_LIST_LIMIT), "--json", "number,title,body", ] @@ -114,9 +119,28 @@ def fetch_issues_by_label( label=label, state=state, ) - payload = json.loads(output) + payload = _load_json(output, "gh issue list") if not isinstance(payload, list): raise GitHubAdapterError(f"Unexpected gh issue list output: {payload!r}") + if len(payload) >= _ISSUE_LIST_LIMIT: + warnings.warn( + "gh issue list returned the 1000-item limit; results may be truncated", + RuntimeWarning, + stacklevel=2, + ) + emit_audit_event( + "api_call", + action="github_issue_list", + outcome="warning", + level="WARNING", + client="gh", + endpoint="gh issue list", + repo=repo, + label=label, + state=state, + result_count=len(payload), + limit=_ISSUE_LIST_LIMIT, + ) return [_parse_issue(raw_issue) for raw_issue in payload] def fetch_task_descriptions_by_numbers( @@ -139,7 +163,12 @@ def fetch_task_descriptions_by_label( def _run_gh(args: list[str]) -> str: - result = subprocess.run(args, check=False, capture_output=True, text=True) + try: + result = subprocess.run(args, check=False, capture_output=True, text=True) + except OSError as exc: + raise GitHubAdapterError( + "gh CLI not found; install gh or set GITHUB_TOKEN", + ) from exc if result.returncode != 0: raise GitHubAdapterError( f"gh command failed ({' '.join(args)}): {result.stderr.strip() or result.stdout.strip()}", @@ -147,6 +176,13 @@ def _run_gh(args: list[str]) -> str: return result.stdout +def _load_json(output: str, command: str) -> object: + try: + return json.loads(output) + except json.JSONDecodeError as exc: + raise GitHubAdapterError(f"{command} returned invalid JSON: {exc}") from exc + + def _parse_issue(payload: dict[str, object]) -> GitHubIssue: number = int(payload["number"]) title = str(payload.get("title", "")) diff --git a/src/agent_estimate/adapters/github_rest.py b/src/agent_estimate/adapters/github_rest.py index 4a01afa..904612e 100644 --- a/src/agent_estimate/adapters/github_rest.py +++ b/src/agent_estimate/adapters/github_rest.py @@ -7,7 +7,7 @@ import subprocess import time from typing import Callable, Mapping, Sequence -from urllib.error import HTTPError +from urllib.error import HTTPError, URLError from urllib.parse import urlencode from urllib.request import Request, urlopen @@ -19,6 +19,7 @@ from agent_estimate.audit import emit_audit_event BASE_URL = "https://api.github.com" +MAX_RETRY_DELAY_SECONDS = 60.0 class GitHubRestAdapter: @@ -126,7 +127,23 @@ def _request_json(self, url: str) -> tuple[object, dict[str, str]]: endpoint=url, status_code=status, ) - return json.loads(body), normalized_headers + try: + return json.loads(body), normalized_headers + except json.JSONDecodeError as exc: + emit_audit_event( + "api_call", + action="github_rest_request", + outcome="error", + level="ERROR", + duration_ms=duration_ms, + client="github_rest", + endpoint=url, + status_code=status, + error_type="json_decode_error", + ) + raise GitHubAdapterError( + f"GitHub API returned invalid JSON for {url}: {exc}", + ) from exc if _is_rate_limited(status, normalized_headers) and attempt < self._max_retries: emit_audit_event( @@ -174,6 +191,8 @@ def _default_request(self, url: str, headers: Mapping[str, str]) -> tuple[int, d except HTTPError as exc: body = exc.read().decode("utf-8", errors="replace") return exc.code, dict(exc.headers.items()) if exc.headers else {}, body + except (OSError, URLError) as exc: + raise GitHubAdapterError(f"GitHub API request failed for {url}: {exc}") from exc def _resolve_github_token() -> str: @@ -187,12 +206,26 @@ def _resolve_github_token() -> str: ) return token - result = subprocess.run( - ["gh", "auth", "token"], - check=False, - capture_output=True, - text=True, - ) + try: + result = subprocess.run( + ["gh", "auth", "token"], + check=False, + capture_output=True, + text=True, + ) + except OSError as exc: + emit_audit_event( + "authentication_event", + action="resolve_github_token", + outcome="error", + level="ERROR", + provider="gh", + auth_target="github", + error_type="gh_cli_missing", + ) + raise GitHubAdapterError( + "gh CLI not found; install gh or set GITHUB_TOKEN", + ) from exc if result.returncode != 0 or not result.stdout.strip(): message = result.stderr.strip() or "gh auth token returned no token" emit_audit_event( @@ -245,15 +278,21 @@ def _compute_retry_delay( retry_after = headers.get("retry-after") if retry_after: try: - return max(float(retry_after), 1.0) + return min(max(float(retry_after), 1.0), MAX_RETRY_DELAY_SECONDS) except ValueError: pass reset_epoch = headers.get("x-ratelimit-reset") if reset_epoch: try: - return max(float(reset_epoch) - now_seconds, 1.0) + return min( + max(float(reset_epoch) - now_seconds, 1.0), + MAX_RETRY_DELAY_SECONDS, + ) except ValueError: pass - return max(initial_backoff_seconds * (2**attempt), 1.0) + return min( + max(initial_backoff_seconds * (2**attempt), 1.0), + MAX_RETRY_DELAY_SECONDS, + ) diff --git a/src/agent_estimate/adapters/sqlite_store.py b/src/agent_estimate/adapters/sqlite_store.py index f92b7a7..0e95768 100644 --- a/src/agent_estimate/adapters/sqlite_store.py +++ b/src/agent_estimate/adapters/sqlite_store.py @@ -10,6 +10,8 @@ from threading import RLock from typing import Any +SCHEMA_VERSION = 1 + @dataclass(frozen=True) class ObservationInput: @@ -51,6 +53,11 @@ def __init__( self._connection.row_factory = sqlite3.Row self._enable_pragmas() self._create_schema() + try: + self._validate_schema_version() + except Exception: + self._connection.close() + raise def __enter__(self) -> SQLiteCalibrationStore: """Allow `with SQLiteCalibrationStore(...) as store:` usage.""" @@ -180,8 +187,8 @@ def _query_observations( def calibrate(self) -> None: """Recompute calibration_summary from the raw observations table.""" - grouped: dict[tuple[str, int], list[float]] = {} with self._lock: + grouped: dict[tuple[str, int], list[float]] = {} rows = self._connection.execute( """ SELECT week_start, task_type_id, error_ratio @@ -190,11 +197,10 @@ def calibrate(self) -> None: """, ).fetchall() - for row in rows: - key = (str(row["week_start"]), int(row["task_type_id"])) - grouped.setdefault(key, []).append(float(row["error_ratio"]) * 100.0) + for row in rows: + key = (str(row["week_start"]), int(row["task_type_id"])) + grouped.setdefault(key, []).append(float(row["error_ratio"]) * 100.0) - with self._lock: with self._connection: self._connection.execute("DELETE FROM calibration_summary") for (week_start, task_type_id), values in grouped.items(): @@ -334,10 +340,21 @@ def _create_schema(self) -> None: ) self._connection.execute( """ - INSERT OR IGNORE INTO schema_version (version) VALUES (1) + INSERT OR IGNORE INTO schema_version (version) VALUES (?) """, + (SCHEMA_VERSION,), ) + def _validate_schema_version(self) -> None: + with self._lock: + row = self._connection.execute("SELECT MAX(version) FROM schema_version").fetchone() + version = int(row[0]) if row is not None and row[0] is not None else 0 + if version > SCHEMA_VERSION: + raise RuntimeError( + "Unsupported calibration DB schema version " + f"{version}; this agent-estimate supports version {SCHEMA_VERSION}", + ) + def _upsert_task_type(self, task_type: str) -> int: if not task_type: raise ValueError("task_type must be non-empty") diff --git a/src/agent_estimate/audit.py b/src/agent_estimate/audit.py index 94ed691..48a5fae 100644 --- a/src/agent_estimate/audit.py +++ b/src/agent_estimate/audit.py @@ -84,6 +84,7 @@ def __init__(self, config: AuditConfig) -> None: self._config = config # Serializes writes for this logger instance. self._lock = RLock() + self._warned_stdout_redirect = False def emit( self, @@ -118,23 +119,36 @@ def emit( def _write_line(self, line: str) -> None: with self._lock: if self._config.destination == "stdout": - print(line, file=sys.stdout) + if not self._warned_stdout_redirect: + print( + "Warning: AGENT_ESTIMATE_AUDIT_DESTINATION=stdout is deprecated; " + "writing audit events to stderr to preserve report stdout.", + file=sys.stderr, + ) + self._warned_stdout_redirect = True + print(line, file=sys.stderr) return if self._config.destination == "stderr": print(line, file=sys.stderr) return destination = Path(self._config.destination) - destination.parent.mkdir(parents=True, exist_ok=True) - # Reopen per write to keep the CLI logger stateless; revisit if a - # long-lived service needs a persistent buffered handle. - destination_fd = os.open( - destination, - os.O_APPEND | os.O_CREAT | os.O_WRONLY, - 0o600, - ) - with os.fdopen(destination_fd, "a", encoding="utf-8") as handle: - handle.write(f"{line}\n") + try: + destination.parent.mkdir(parents=True, exist_ok=True) + # Reopen per write to keep the CLI logger stateless; revisit if a + # long-lived service needs a persistent buffered handle. + destination_fd = os.open( + destination, + os.O_APPEND | os.O_CREAT | os.O_WRONLY, + 0o600, + ) + with os.fdopen(destination_fd, "a", encoding="utf-8") as handle: + handle.write(f"{line}\n") + except OSError as exc: + print( + f"Warning: failed to write audit log to {destination.name}: {exc}", + file=sys.stderr, + ) def configure_audit_logger(config: AuditConfig | None = None) -> AuditLogger: diff --git a/src/agent_estimate/cli/commands/calibrate.py b/src/agent_estimate/cli/commands/calibrate.py index e13d996..c468858 100644 --- a/src/agent_estimate/cli/commands/calibrate.py +++ b/src/agent_estimate/cli/commands/calibrate.py @@ -20,7 +20,7 @@ def run( if not db.exists(): typer.echo("Error: No calibration database found.", err=True) typer.echo(f"Expected: {db}", err=True) - raise typer.Exit(code=1) + raise typer.Exit(code=2) try: with SQLiteCalibrationStore(db) as store: diff --git a/src/agent_estimate/cli/commands/estimate.py b/src/agent_estimate/cli/commands/estimate.py index 796fe8b..177d2b1 100644 --- a/src/agent_estimate/cli/commands/estimate.py +++ b/src/agent_estimate/cli/commands/estimate.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +import os import time from pathlib import Path from typing import NoReturn, Optional @@ -12,6 +13,7 @@ from agent_estimate.adapters.config_loader import load_config, load_default_config from agent_estimate.adapters.github_adapter import GitHubAdapterError from agent_estimate.adapters.github_ghcli import GitHubGhCliAdapter +from agent_estimate.adapters.github_rest import GitHubRestAdapter from agent_estimate.audit import emit_audit_event from agent_estimate.cli.commands._pipeline import run_estimate_pipeline from agent_estimate.cli.commands._utils import validate_output_format @@ -140,6 +142,10 @@ def run( descriptions = [ln.strip() for ln in lines if ln.strip()] except FileNotFoundError: _error(f"File not found: {file}", 2) + except UnicodeDecodeError as exc: + _error(f"Failed to decode task file {file}: {exc}", 2) + except OSError as exc: + _error(f"Failed to read task file {file}: {exc}", 2) if not descriptions: _error(f"No task descriptions found in {file}.", 2) elif issues is not None: @@ -152,10 +158,7 @@ def run( if not issue_numbers: _error("No issue numbers provided.", 2) try: - adapter = GitHubGhCliAdapter() - descriptions = adapter.fetch_task_descriptions_by_numbers( - repo, issue_numbers - ) + descriptions = _fetch_github_task_descriptions(repo, issue_numbers) except GitHubAdapterError as exc: _error(f"GitHub error: {exc}", 1) @@ -174,6 +177,8 @@ def run( cfg = load_config(config_path) if config_path else load_default_config() except FileNotFoundError: _error(f"Config file not found: {config_path}", 2) + except OSError as exc: + _error(f"Failed to read config file {config_path}: {exc}", 2) except ValueError as exc: _error(f"Config validation error: {exc}", 2) @@ -278,6 +283,30 @@ def _error(message: str, exit_code: int) -> NoReturn: raise typer.Exit(code=exit_code) +def _fetch_github_task_descriptions(repo: str, issue_numbers: list[int]) -> list[str]: + """Fetch GitHub issues using REST in token-only environments, otherwise gh CLI.""" + errors: list[str] = [] + if os.getenv("GITHUB_TOKEN"): + try: + return GitHubRestAdapter().fetch_task_descriptions_by_numbers( + repo, + issue_numbers, + ) + except GitHubAdapterError as exc: + errors.append(f"REST API: {exc}") + + try: + return GitHubGhCliAdapter().fetch_task_descriptions_by_numbers( + repo, + issue_numbers, + ) + except GitHubAdapterError as exc: + if errors: + errors.append(f"gh CLI: {exc}") + raise GitHubAdapterError("; ".join(errors)) from exc + raise + + def _summarize_config_changes( current: EstimationConfig, baseline: EstimationConfig, diff --git a/src/agent_estimate/cli/commands/session.py b/src/agent_estimate/cli/commands/session.py index 709415c..05f881b 100644 --- a/src/agent_estimate/cli/commands/session.py +++ b/src/agent_estimate/cli/commands/session.py @@ -122,8 +122,6 @@ def run( def _render_markdown(result: SessionEstimate) -> None: - assert isinstance(result, SessionEstimate) - wall_h, wall_m = divmod(round(result.wall_clock_minutes), 60) agent_h, agent_m = divmod(round(result.agent_minutes), 60) @@ -142,8 +140,8 @@ def _render_markdown(result: SessionEstimate) -> None: typer.echo( f"| Coordination overhead | {result.coordination_overhead_minutes:.0f}m / round{'':<15} |" ) - typer.echo(f"| **Wall-clock** | **{wall_str}**{'':<{22 - len(wall_str)}} |") - typer.echo(f"| **Agent-minutes** | **{agent_str}**{'':<{22 - len(agent_str)}} |") + typer.echo(f"| **Wall-clock** | **{wall_str}**{'':<{max(0, 22 - len(wall_str))}} |") + typer.echo(f"| **Agent-minutes** | **{agent_str}**{'':<{max(0, 22 - len(agent_str))}} |") if len(result.rounds_breakdown) > 1: typer.echo("\n### Round breakdown\n") @@ -156,4 +154,3 @@ def _render_markdown(result: SessionEstimate) -> None: def _error(message: str, exit_code: int) -> NoReturn: typer.echo(f"Error: {message}", err=True) raise typer.Exit(code=exit_code) - diff --git a/src/agent_estimate/cli/commands/validate.py b/src/agent_estimate/cli/commands/validate.py index db708a6..6810d53 100644 --- a/src/agent_estimate/cli/commands/validate.py +++ b/src/agent_estimate/cli/commands/validate.py @@ -68,42 +68,56 @@ def run( # Optionally store in calibration DB if db is not None: - modifiers_raw = raw.get("modifiers") or {} - if not isinstance(modifiers_raw, dict): - typer.echo("Error: 'modifiers' must be a YAML mapping.", err=True) + try: + obs = _build_observation(raw, estimated, actual_work, actual_total, error_ratio, verdict) + except ValueError as exc: + typer.echo(f"Error: Invalid observation field: {exc}", err=True) raise typer.Exit(code=2) try: - obs = ObservationInput( - task_type=str(raw.get("task_type", "unknown")), - estimated_secs=estimated * 60, - actual_work_secs=actual_work * 60, - actual_total_secs=actual_total * 60, - error_ratio=error_ratio, - file_count=int(raw.get("file_count", 0)), - line_count=int(raw.get("line_count", 0)), - test_count=int(raw.get("test_count", 0)), - project_hash=str(raw.get("project_hash") or "unknown"), - spec_clarity_modifier=float( - modifiers_raw.get("spec_clarity", 1.0) - ), - warm_context_modifier=float( - modifiers_raw.get("warm_context", 1.0) - ), - execution_mode=str(raw.get("execution_mode", "single")), - review_mode=str(raw.get("review_mode", "none")), - review_overhead_secs=float( - raw.get("review_overhead_minutes", 0) - ) - * 60, - verdict=verdict, - modifiers_should_have_been=raw.get( - "modifiers_should_have_been", {} - ), - ) with SQLiteCalibrationStore(db) as store: row_id = store.insert_observation(obs) typer.echo(f"\nObservation stored (id={row_id}) in {db}") + except ValueError as exc: + typer.echo(f"Error: Invalid observation field: {exc}", err=True) + raise typer.Exit(code=2) except Exception as exc: typer.echo(f"Error storing observation: {exc}", err=True) raise typer.Exit(code=1) + + +def _build_observation( + raw: dict[object, object], + estimated: float, + actual_work: float, + actual_total: float, + error_ratio: float, + verdict: str, +) -> ObservationInput: + modifiers_raw = raw.get("modifiers") or {} + if not isinstance(modifiers_raw, dict): + raise ValueError("'modifiers' must be a YAML mapping") + modifiers_should_have_been = raw.get("modifiers_should_have_been", {}) + if not isinstance(modifiers_should_have_been, dict): + raise ValueError("'modifiers_should_have_been' must be a YAML mapping") + + return ObservationInput( + task_type=str(raw.get("task_type", "unknown")), + estimated_secs=estimated * 60, + actual_work_secs=actual_work * 60, + actual_total_secs=actual_total * 60, + error_ratio=error_ratio, + file_count=int(raw.get("file_count", 0)), + line_count=int(raw.get("line_count", 0)), + test_count=int(raw.get("test_count", 0)), + project_hash=str(raw.get("project_hash") or "unknown"), + spec_clarity_modifier=float(modifiers_raw.get("spec_clarity", 1.0)), + warm_context_modifier=float(modifiers_raw.get("warm_context", 1.0)), + execution_mode=str(raw.get("execution_mode", "single")), + review_mode=str(raw.get("review_mode", "none")), + review_overhead_secs=float(raw.get("review_overhead_minutes", 0)) * 60, + verdict=verdict, + modifiers_should_have_been={ + str(key): float(value) for key, value in modifiers_should_have_been.items() + }, + ) diff --git a/src/agent_estimate/core/models.py b/src/agent_estimate/core/models.py index 10bf2d5..7dcb39d 100644 --- a/src/agent_estimate/core/models.py +++ b/src/agent_estimate/core/models.py @@ -6,6 +6,7 @@ from collections.abc import Mapping, Sequence import dataclasses from dataclasses import dataclass +from types import MappingProxyType from typing import Annotated, Protocol, runtime_checkable from pydantic import BaseModel, ConfigDict, Field, StringConstraints @@ -299,6 +300,13 @@ class Wave: assignments: tuple[WaveAssignment, ...] agent_review_minutes: Mapping[str, float] = dataclasses.field(default_factory=dict) # type: ignore[assignment] + def __post_init__(self) -> None: + object.__setattr__( + self, + "agent_review_minutes", + MappingProxyType(dict(self.agent_review_minutes)), + ) + @dataclass(frozen=True) class WavePlan: @@ -311,3 +319,10 @@ class WavePlan: parallel_efficiency: float total_wall_clock_minutes: float total_sequential_minutes: float + + def __post_init__(self) -> None: + object.__setattr__( + self, + "agent_utilization", + MappingProxyType(dict(self.agent_utilization)), + ) diff --git a/src/agent_estimate/core/modifiers.py b/src/agent_estimate/core/modifiers.py index 542724a..1413cf5 100644 --- a/src/agent_estimate/core/modifiers.py +++ b/src/agent_estimate/core/modifiers.py @@ -48,7 +48,7 @@ def build_modifier_set( if clamped: combined = _MODIFIER_FLOOR logger.warning( - "Modifier product %.4f clamped to %.2f (prevents sub-10m pathology)", + "Modifier product %.4f clamped to %.2f (modifier floor)", raw_combined, _MODIFIER_FLOOR, ) diff --git a/src/agent_estimate/core/pert.py b/src/agent_estimate/core/pert.py index fea6963..7ca491e 100644 --- a/src/agent_estimate/core/pert.py +++ b/src/agent_estimate/core/pert.py @@ -10,6 +10,7 @@ import yaml from agent_estimate.core.models import ( + EstimationCategory, MetrWarning, ModifierSet, PertResult, @@ -77,6 +78,8 @@ def compute_pert(optimistic: float, most_likely: float, pessimistic: float) -> P Formula: E = (O + 4M + P) / 6, sigma = (P - O) / 6 """ + if optimistic < 0: + raise ValueError(f"PERT requires O >= 0, got O={optimistic}") if not (optimistic <= most_likely <= pessimistic): raise ValueError( f"PERT requires O <= M <= P, got O={optimistic}, M={most_likely}, P={pessimistic}" @@ -213,4 +216,5 @@ def estimate_task( total_expected_minutes=total, human_equivalent_minutes=human_equivalent_minutes, metr_warning=metr_warning, + estimation_category=EstimationCategory.CODING, ) diff --git a/src/agent_estimate/core/sizing.py b/src/agent_estimate/core/sizing.py index fa1ecfa..23d099a 100644 --- a/src/agent_estimate/core/sizing.py +++ b/src/agent_estimate/core/sizing.py @@ -38,10 +38,10 @@ _TYPE_PATTERNS: list[tuple[re.Pattern[str], TaskType]] = [ (re.compile(r"\b(boilerplate|scaffold|template|stub|generate)\b", re.I), TaskType.BOILERPLATE), (re.compile(r"\b(bug|fix|patch|hotfix|regression|broken)\b", re.I), TaskType.BUG_FIX), + (re.compile(r"\b(tests?|specs?|coverage|assertions?)\b", re.I), TaskType.TEST), + (re.compile(r"\b(docs?|readme|comments?|changelog)\b", re.I), TaskType.DOCS), (re.compile(r"\b(feature|implement|add|create|build|new)\b", re.I), TaskType.FEATURE), (re.compile(r"\b(refactor|restructure|clean|rewrite|simplify)\b", re.I), TaskType.REFACTOR), - (re.compile(r"\b(test|spec|coverage|assertion)\b", re.I), TaskType.TEST), - (re.compile(r"\b(doc|readme|comment|changelog)\b", re.I), TaskType.DOCS), ] _TIER_ORDER = [SizeTier.XS, SizeTier.S, SizeTier.M, SizeTier.L, SizeTier.XL] @@ -183,7 +183,8 @@ def classify_task(description: str) -> SizingResult: tier_votes.append(tier) signals.append(signal_name) - # Base tier: median of votes, or M as default + # Base tier: conservative upper-middle vote, or M as default. For an even + # vote count this intentionally picks the higher tier (e.g. [S, L] -> L). if tier_votes: sorted_votes = sorted(tier_votes, key=lambda t: _TIER_ORDER.index(t)) base_tier = sorted_votes[len(sorted_votes) // 2] diff --git a/src/agent_estimate/core/task_type_models.py b/src/agent_estimate/core/task_type_models.py index b5bc85c..dd28d0e 100644 --- a/src/agent_estimate/core/task_type_models.py +++ b/src/agent_estimate/core/task_type_models.py @@ -350,7 +350,13 @@ def estimate_documentation( review_mode = ReviewMode.NONE o, m, p = _DOCUMENTATION_BASELINES - sizing = _make_non_coding_sizing(o, m, p, "documentation-model") + sizing = _make_non_coding_sizing( + o, + m, + p, + "documentation-model", + task_type=TaskType.DOCS, + ) adjusted_o = o * modifiers.combined adjusted_m = m * modifiers.combined diff --git a/src/agent_estimate/core/wave_planner.py b/src/agent_estimate/core/wave_planner.py index bf6b767..f41170d 100644 --- a/src/agent_estimate/core/wave_planner.py +++ b/src/agent_estimate/core/wave_planner.py @@ -125,6 +125,7 @@ def plan_waves( # Track which tasks land on each agent in this wave (for co-dispatch detection) agent_wave_tasks: dict[str, list[str]] = defaultdict(list) + slot_wave_tasks: dict[tuple[str, int], list[str]] = defaultdict(list) for tid in sorted_tasks: node = task_map[tid] @@ -148,6 +149,7 @@ def plan_waves( wave_bin_load[best] += node.duration_minutes slot_load[best] += node.duration_minutes agent_wave_tasks[best[0]].append(tid) + slot_wave_tasks[best].append(tid) assignments.append( WaveAssignment( @@ -159,15 +161,15 @@ def plan_waves( ) # ------------------------------------------------------------------ - # Co-dispatch: for each agent with 2+ tasks in this wave, apply 0.5x - # warm-context reduction to all tasks beyond the first. + # Co-dispatch: for each slot with 2+ tasks in this wave, apply 0.5x + # warm-context reduction to all tasks in that slot beyond the first. # ------------------------------------------------------------------ - # Build a mapping from task_id → co_dispatch_group for agents with - # multiple tasks. Then rebuild assignments with adjusted durations + # Build a mapping from task_id → co_dispatch_group for slots with + # multiple tasks. Then rebuild assignments with adjusted durations # and co_dispatch_group populated. co_dispatch_group_map: dict[str, tuple[str, ...]] = {} adjusted_duration_map: dict[str, float] = {} - for agent_name, tids in agent_wave_tasks.items(): + for tids in slot_wave_tasks.values(): if len(tids) < 2: continue group = tuple(tids) diff --git a/src/agent_estimate/render/json_report.py b/src/agent_estimate/render/json_report.py index d176a8e..b9f4899 100644 --- a/src/agent_estimate/render/json_report.py +++ b/src/agent_estimate/render/json_report.py @@ -41,6 +41,7 @@ def _build_payload(report: EstimationReport) -> dict[str, Any]: "modifiers": { "spec_clarity": task.modifier_spec_clarity, "warm_context": task.modifier_warm_context, + "warm_context_detail": task.warm_context_detail, "agent_fit": task.modifier_agent_fit, "combined": task.modifier_combined, "raw_combined": task.modifier_raw_combined, @@ -51,6 +52,7 @@ def _build_payload(report: EstimationReport) -> dict[str, Any]: "human_equivalent_minutes": task.human_equivalent_minutes, "review_overhead_minutes": task.review_overhead_minutes, "metr_warning": task.metr_warning, + "tier_correction_warnings": list(task.tier_correction_warnings), "is_critical_path": task.name in critical_tasks, } for task in report.tasks @@ -75,6 +77,14 @@ def _build_payload(report: EstimationReport) -> dict[str, Any]: ], "critical_path": list(report.critical_path), "metr_warnings": warnings, + "tier_correction_warnings": [ + { + "task": task.name, + "warning": warning, + } + for task in report.tasks + for warning in task.tier_correction_warnings + ], } diff --git a/src/agent_estimate/render/markdown_report.py b/src/agent_estimate/render/markdown_report.py index 8231641..b132672 100644 --- a/src/agent_estimate/render/markdown_report.py +++ b/src/agent_estimate/render/markdown_report.py @@ -8,7 +8,7 @@ def render_markdown_report(report: EstimationReport) -> str: """Render an estimation report as GitHub-compatible Markdown.""" lines: list[str] = [ - f"# {report.title}", + f"# {_normalize_inline(report.title)}", "", ] lines.extend(_render_task_table(report)) @@ -23,6 +23,8 @@ def render_markdown_report(report: EstimationReport) -> str: lines.extend([""]) lines.extend(_render_critical_path(report)) lines.extend([""]) + lines.extend(_render_tier_correction_warnings(report)) + lines.extend([""]) lines.extend(_render_metr_warnings(report)) lines.append("") return "\n".join(lines) @@ -49,7 +51,7 @@ def _render_task_table(report: EstimationReport) -> list[str]: ) warm_str = f"warm {task.modifier_warm_context:.2f}" if task.warm_context_detail: - warm_str += f" (auto: {task.warm_context_detail})" + warm_str += f" (auto: {_escape_cell(task.warm_context_detail)})" modifiers = ( f"spec {task.modifier_spec_clarity:.2f} x " f"{warm_str} x " @@ -147,6 +149,22 @@ def _render_critical_path(report: EstimationReport) -> list[str]: return lines +def _render_tier_correction_warnings(report: EstimationReport) -> list[str]: + lines = ["## Tier Corrections", ""] + warnings = [ + (task.name, task.tier_correction_warnings) + for task in report.tasks + if task.tier_correction_warnings + ] + if not warnings: + lines.append("No tier corrections.") + return lines + for task_name, task_warnings in warnings: + for warning in task_warnings: + lines.append(f"- **{_escape_cell(task_name)}**: {_escape_cell(warning)}") + return lines + + def _render_metr_warnings(report: EstimationReport) -> list[str]: lines = ["## METR Warnings", ""] warnings = [(task.name, task.metr_warning) for task in report.tasks if task.metr_warning] @@ -168,3 +186,7 @@ def _format_minutes(value: float) -> str: def _escape_cell(value: str) -> str: normalized = value.replace("\r\n", "\n").replace("\r", "\n").replace("\n", "
") return normalized.replace("|", "\\|") + + +def _normalize_inline(value: str) -> str: + return " ".join(value.replace("\r\n", "\n").replace("\r", "\n").splitlines()).strip() diff --git a/src/agent_estimate/skill/claude_wrapper.py b/src/agent_estimate/skill/claude_wrapper.py index f89f11c..268cb5e 100644 --- a/src/agent_estimate/skill/claude_wrapper.py +++ b/src/agent_estimate/skill/claude_wrapper.py @@ -21,7 +21,7 @@ def run_estimate( file: Path | None = None, config: Path | None = None, format: str = "markdown", - review_mode: str = "2x-lgtm", + review_mode: str = "standard", issues: str | None = None, repo: str | None = None, title: str = "Agent Estimate Report", @@ -47,7 +47,7 @@ def run_estimate( cmd += ["--config", str(config)] if format != "markdown": cmd += ["--format", format] - if review_mode != "2x-lgtm": + if review_mode != "standard": cmd += ["--review-mode", review_mode] if issues is not None: cmd += ["--issues", issues] diff --git a/src/agent_estimate/version.py b/src/agent_estimate/version.py index 1a8560e..04c5f63 100644 --- a/src/agent_estimate/version.py +++ b/src/agent_estimate/version.py @@ -1,3 +1,3 @@ """Version constants for agent-estimate.""" -__version__ = "0.7.1" +__version__ = "0.7.2" diff --git a/tests/fixtures/json_report_golden.json b/tests/fixtures/json_report_golden.json index 9f83e68..3158360 100644 --- a/tests/fixtures/json_report_golden.json +++ b/tests/fixtures/json_report_golden.json @@ -43,11 +43,15 @@ "combined": 1.1, "raw_combined": 1.1, "spec_clarity": 1.1, - "warm_context": 1.0 + "warm_context": 1.0, + "warm_context_detail": "auto warm match" }, "name": "Implement auth", "review_overhead_minutes": 17.5, - "tier": "M" + "tier": "M", + "tier_correction_warnings": [ + "Upgraded M\u2192L: 4 concerns" + ] }, { "agent": "Claude", @@ -68,11 +72,19 @@ "combined": 1.0, "raw_combined": 1.0, "spec_clarity": 1.0, - "warm_context": 1.0 + "warm_context": 1.0, + "warm_context_detail": null }, "name": "Add tests", "review_overhead_minutes": 7.5, - "tier": "S" + "tier": "S", + "tier_correction_warnings": [] + } + ], + "tier_correction_warnings": [ + { + "task": "Implement auth", + "warning": "Upgraded M\u2192L: 4 concerns" } ], "timeline": { diff --git a/tests/integration/test_cli_e2e.py b/tests/integration/test_cli_e2e.py index 23adab0..9c4f9bf 100644 --- a/tests/integration/test_cli_e2e.py +++ b/tests/integration/test_cli_e2e.py @@ -156,6 +156,8 @@ def test_modifier_flags_work_with_file_input(self) -> None: assert "spec 0.60 x warm 0.50 x fit 1.10 = 0.33" in result.output def test_modifier_flags_work_with_issues_input(self, monkeypatch) -> None: + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + class _FakeGitHubAdapter: def fetch_task_descriptions_by_numbers( self, repo: str, issue_numbers: list[int] @@ -182,9 +184,50 @@ def fetch_task_descriptions_by_numbers( assert result.exit_code == 0 assert result.output.count("spec 0.70 x warm 0.60 x fit 1.00 = 0.42") == 2 + def test_issues_input_uses_rest_adapter_when_token_is_set( + self, + monkeypatch, + ) -> None: + calls: dict[str, bool] = {} + + class _FakeRestAdapter: + def fetch_task_descriptions_by_numbers( + self, repo: str, issue_numbers: list[int] + ) -> list[str]: + assert repo == "kiloloop/agent-estimate" + assert issue_numbers == [11] + calls["rest"] = True + return ["Add tests"] + + class _UnexpectedGhAdapter: + def fetch_task_descriptions_by_numbers( + self, repo: str, issue_numbers: list[int] + ) -> list[str]: + raise AssertionError("gh CLI adapter should not be used") + + monkeypatch.setenv("GITHUB_TOKEN", "test-token") + monkeypatch.setattr(estimate_command, "GitHubRestAdapter", _FakeRestAdapter) + monkeypatch.setattr(estimate_command, "GitHubGhCliAdapter", _UnexpectedGhAdapter) + + result = runner.invoke( + app, + [ + "estimate", + "--issues", + "11", + "--repo", + "kiloloop/agent-estimate", + ], + ) + + assert result.exit_code == 0 + assert calls == {"rest": True} + def test_modifier_out_of_range_with_issues_is_user_facing_error( self, monkeypatch ) -> None: + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + class _FakeGitHubAdapter: def fetch_task_descriptions_by_numbers( self, repo: str, issue_numbers: list[int] @@ -262,6 +305,18 @@ def test_estimate_file_not_found(self, tmp_path: Path) -> None: assert result.exit_code != 0 assert "File not found" in result.output + def test_estimate_file_directory_is_user_facing_error(self, tmp_path: Path) -> None: + result = runner.invoke(app, ["estimate", "--file", str(tmp_path)]) + assert result.exit_code == 2 + assert "Failed to read task file" in result.output + + def test_estimate_file_non_utf8_is_user_facing_error(self, tmp_path: Path) -> None: + task_file = tmp_path / "tasks.txt" + task_file.write_bytes(b"\xff\xfe\x00") + result = runner.invoke(app, ["estimate", "--file", str(task_file)]) + assert result.exit_code == 2 + assert "Failed to decode task file" in result.output + # --------------------------------------------------------------------------- # Estimate — review modes @@ -395,6 +450,11 @@ def test_config_file_not_found(self, tmp_path: Path) -> None: assert result.exit_code != 0 assert "not found" in result.output.lower() + def test_config_directory_is_user_facing_error(self, tmp_path: Path) -> None: + result = runner.invoke(app, ["estimate", "--config", str(tmp_path), "task"]) + assert result.exit_code == 2 + assert "Failed to read config file" in result.output + def test_config_invalid_validation(self) -> None: config = str(FIXTURES / "cycle_invalid.yaml") result = runner.invoke(app, ["estimate", "--config", config, "task"]) @@ -454,6 +514,23 @@ def test_validate_missing_required_fields(self, tmp_path: Path) -> None: result = runner.invoke(app, ["validate", str(incomplete)]) assert result.exit_code != 0 + def test_validate_bad_optional_field_is_input_error(self, tmp_path: Path) -> None: + observation = tmp_path / "obs.yaml" + observation.write_text( + "\n".join( + [ + "estimated_minutes: 10", + "actual_work_minutes: 12", + "review_overhead_minutes: abc", + ] + ), + encoding="utf-8", + ) + result = runner.invoke(app, ["validate", str(observation), "--db", str(tmp_path / "c.db")]) + assert result.exit_code == 2 + assert "Invalid observation field" in result.output + assert "Error storing observation" not in result.output + # --------------------------------------------------------------------------- # No args / help diff --git a/tests/unit/test_audit.py b/tests/unit/test_audit.py index 21b11dc..283b1e4 100644 --- a/tests/unit/test_audit.py +++ b/tests/unit/test_audit.py @@ -25,6 +25,46 @@ def test_audit_logger_creates_owner_only_log_file(tmp_path: Path) -> None: assert audit_log.stat().st_mode & 0o777 == 0o600 +def test_audit_logger_stdout_destination_writes_to_stderr(capsys) -> None: + logger = AuditLogger( + AuditConfig( + enabled=True, + level="INFO", + destination="stdout", + actor="test-agent", + environment="test", + ), + ) + + logger.emit("estimation_request", action="estimate") + + captured = capsys.readouterr() + assert captured.out == "" + assert "AGENT_ESTIMATE_AUDIT_DESTINATION=stdout is deprecated" in captured.err + assert '"event_type": "estimation_request"' in captured.err + + +def test_audit_logger_degrades_when_file_sink_fails( + monkeypatch, + tmp_path: Path, + capsys, +) -> None: + logger = AuditLogger( + AuditConfig( + enabled=True, + level="INFO", + destination=str(tmp_path / "audit.jsonl"), + actor="test-agent", + environment="test", + ), + ) + monkeypatch.setattr("agent_estimate.audit.os.open", lambda *_, **__: (_ for _ in ()).throw(PermissionError("denied"))) + + logger.emit("estimation_request", action="estimate") + + assert "failed to write audit log" in capsys.readouterr().err + + def test_scrub_uses_segment_aware_sensitive_key_detection() -> None: scrubbed = _scrub( { diff --git a/tests/unit/test_claude_wrapper.py b/tests/unit/test_claude_wrapper.py new file mode 100644 index 0000000..46d44ef --- /dev/null +++ b/tests/unit/test_claude_wrapper.py @@ -0,0 +1,47 @@ +"""Tests for the Claude skill subprocess wrapper.""" + +from __future__ import annotations + +import subprocess + +from agent_estimate.skill import claude_wrapper + + +def test_run_estimate_default_review_mode_is_standard( + monkeypatch, +) -> None: + captured: dict[str, list[str]] = {} + monkeypatch.setattr(claude_wrapper.shutil, "which", lambda _: "agent-estimate") + + def fake_run(cmd: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + captured["cmd"] = cmd + return subprocess.CompletedProcess(cmd, 0, "", "") + + monkeypatch.setattr(claude_wrapper.subprocess, "run", fake_run) + + claude_wrapper.run_estimate(task="Add tests") + + assert captured["cmd"] == ["agent-estimate", "estimate", "Add tests"] + + +def test_run_estimate_custom_review_mode_is_forwarded( + monkeypatch, +) -> None: + captured: dict[str, list[str]] = {} + monkeypatch.setattr(claude_wrapper.shutil, "which", lambda _: "agent-estimate") + + def fake_run(cmd: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + captured["cmd"] = cmd + return subprocess.CompletedProcess(cmd, 0, "", "") + + monkeypatch.setattr(claude_wrapper.subprocess, "run", fake_run) + + claude_wrapper.run_estimate(task="Add tests", review_mode="complex") + + assert captured["cmd"] == [ + "agent-estimate", + "estimate", + "Add tests", + "--review-mode", + "complex", + ] diff --git a/tests/unit/test_config_loader.py b/tests/unit/test_config_loader.py index 74fb232..d1e0801 100644 --- a/tests/unit/test_config_loader.py +++ b/tests/unit/test_config_loader.py @@ -213,3 +213,23 @@ def _gemini_profile_factory() -> dict[str, object]: assert [agent.name for agent in config.agents] == ["Claude", "Gemini"] assert config.agents[1].model_tier == "gemini-3-pro" + + +def test_load_config_skips_broken_entry_point_profile( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def _broken_profile_factory() -> dict[str, object]: + raise RuntimeError("plugin exploded") + + config_path = _write(tmp_path, "config.yaml", VALID_CONFIG) + monkeypatch.setattr( + config_loader, + "_iter_agent_entry_points", + lambda: [_FakeEntryPoint("broken_plugin", _broken_profile_factory)], + ) + + with pytest.warns(RuntimeWarning, match="broken_plugin"): + config = load_config(config_path) + + assert [agent.name for agent in config.agents] == ["Claude"] diff --git a/tests/unit/test_docs_freshness.py b/tests/unit/test_docs_freshness.py index c271048..e625b82 100644 --- a/tests/unit/test_docs_freshness.py +++ b/tests/unit/test_docs_freshness.py @@ -26,7 +26,7 @@ # need re-capturing. HEADLINE_LINES = [ "| Expected case | 75.4m |", - "| Compression ratio | 7.60x |", + "| Compression ratio | 8.07x |", "exceeds gpt_5_4 p80 threshold (60m)", "exceeds gemini_3_1_pro p80 threshold (45m)", ] @@ -64,3 +64,12 @@ def test_example_output_matches_real_run(self, tmp_path): for line in HEADLINE_LINES: assert line in output assert line in example, f"examples/multi-agent.md output block lost {line!r}" + + def test_session_command_is_documented_outside_changelog(self): + readme = (ROOT / "README.md").read_text(encoding="utf-8") + assert "agent-estimate session" in readme + + def test_release_checklist_mentions_floating_v0_action_tag(self): + contributing = (ROOT / "CONTRIBUTING.md").read_text(encoding="utf-8") + assert "floating public Action tag" in contributing + assert "git tag -f v0" in contributing diff --git a/tests/unit/test_github_adapters.py b/tests/unit/test_github_adapters.py index 0954891..d6a5439 100644 --- a/tests/unit/test_github_adapters.py +++ b/tests/unit/test_github_adapters.py @@ -101,6 +101,17 @@ def request_fn(url: str, _: Mapping[str, str]) -> tuple[int, dict[str, str], str assert sleep_calls == [2.0] +def test_rest_adapter_caps_rate_limit_reset_delay() -> None: + delay = github_rest._compute_retry_delay( + headers={"x-ratelimit-reset": "10000"}, + attempt=0, + initial_backoff_seconds=1.0, + now_seconds=100.0, + ) + + assert delay == github_rest.MAX_RETRY_DELAY_SECONDS + + def test_gh_cli_adapter_fetches_issues_by_number_and_label() -> None: def runner(args: list[str]) -> str: if args[:3] == ["gh", "issue", "view"]: @@ -122,6 +133,34 @@ def runner(args: list[str]) -> str: assert by_label == ["Label title\n\nBody", "Second title"] +def test_gh_cli_adapter_wraps_invalid_json() -> None: + adapter = GitHubGhCliAdapter(runner=lambda _: "not-json") + + with pytest.raises(GitHubAdapterError, match="invalid JSON"): + adapter.fetch_issues_by_numbers("acme/repo", [9]) + + +def test_gh_cli_adapter_wraps_missing_binary(monkeypatch: pytest.MonkeyPatch) -> None: + def fake_run(*args, **kwargs): + raise FileNotFoundError("gh") + + monkeypatch.setattr("agent_estimate.adapters.github_ghcli.subprocess.run", fake_run) + + adapter = GitHubGhCliAdapter() + with pytest.raises(GitHubAdapterError, match="gh CLI not found"): + adapter.fetch_issues_by_numbers("acme/repo", [9]) + + +def test_rest_adapter_wraps_invalid_json_response() -> None: + adapter = GitHubRestAdapter( + token_provider=lambda: "test-token", + request_fn=lambda _url, _headers: (200, {}, "not-json"), + ) + + with pytest.raises(GitHubAdapterError, match="invalid JSON"): + adapter.fetch_issues_by_numbers("acme/repo", [9]) + + def test_rest_adapter_emits_auth_and_api_audit_events( tmp_path: Path, monkeypatch, @@ -219,3 +258,17 @@ def fake_run(*args, **kwargs) -> subprocess.CompletedProcess[str]: assert auth_event["outcome"] == "error" assert auth_event["details"]["error_type"] == "gh_auth_failure" assert "error" not in auth_event["details"] + + +def test_rest_adapter_wraps_missing_gh_token_binary( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + + def fake_run(*args, **kwargs): + raise FileNotFoundError("gh") + + monkeypatch.setattr(github_rest.subprocess, "run", fake_run) + + with pytest.raises(GitHubAdapterError, match="gh CLI not found"): + github_rest._resolve_github_token() diff --git a/tests/unit/test_json_report.py b/tests/unit/test_json_report.py index 5d02552..b2f0903 100644 --- a/tests/unit/test_json_report.py +++ b/tests/unit/test_json_report.py @@ -41,6 +41,8 @@ def _build_report() -> EstimationReport: human_equivalent_minutes=160.0, review_overhead_minutes=17.5, metr_warning="Estimate exceeds threshold", + warm_context_detail="auto warm match", + tier_correction_warnings=("Upgraded M→L: 4 concerns",), ), ReportTask( name="Add tests", @@ -117,6 +119,11 @@ def test_render_json_report_is_canonical_and_round_trips() -> None: assert {"tasks", "waves", "timeline", "agent_load", "critical_path", "metr_warnings"} <= set( payload ) + assert "warm_context_detail" in payload["tasks"][0]["modifiers"] + assert payload["tasks"][0]["tier_correction_warnings"] == ["Upgraded M→L: 4 concerns"] + assert payload["tier_correction_warnings"] == [ + {"task": "Implement auth", "warning": "Upgraded M→L: 4 concerns"} + ] def test_estimate_command_json_format_outputs_json() -> None: diff --git a/tests/unit/test_markdown_report.py b/tests/unit/test_markdown_report.py index 413985c..2cd5c70 100644 --- a/tests/unit/test_markdown_report.py +++ b/tests/unit/test_markdown_report.py @@ -278,3 +278,45 @@ def test_render_markdown_report_normalizes_multiline_cells() -> None: assert "Task \\| Alpha
line two" in rendered assert "Code\\|x" in rendered assert "Warning with pipe \\| and
newline" in rendered + + +def test_render_markdown_report_sanitizes_title_and_warm_detail() -> None: + task = ReportTask( + name="Task", + tier="S", + agent="Codex", + base_pert_optimistic_minutes=12.0, + base_pert_most_likely_minutes=23.0, + base_pert_pessimistic_minutes=40.0, + modifier_spec_clarity=1.0, + modifier_warm_context=0.5, + modifier_agent_fit=1.0, + modifier_combined=0.5, + modifier_raw_combined=0.5, + modifier_clamped=False, + effective_duration_minutes=12.0, + human_equivalent_minutes=60.0, + review_overhead_minutes=0.0, + warm_context_detail="agent|name\nproject", + tier_correction_warnings=("Upgraded S→L: 4 concerns",), + ) + report = EstimationReport( + title="Title\nInjected Heading", + tasks=(task,), + waves=(), + timeline=ReportTimeline( + best_case_minutes=10.0, + expected_case_minutes=20.0, + worst_case_minutes=30.0, + human_equivalent_minutes=40.0, + ), + agent_load=(), + critical_path=(), + ) + + rendered = render_markdown_report(report) + + assert "# Title Injected Heading" in rendered + assert "agent\\|name
project" in rendered + assert "## Tier Corrections" in rendered + assert "Upgraded S→L: 4 concerns" in rendered diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py index 3c405d5..58bad4a 100644 --- a/tests/unit/test_models.py +++ b/tests/unit/test_models.py @@ -204,3 +204,30 @@ def test_sizing_result_is_frozen(self) -> None: ) with pytest.raises(AttributeError): sizing.tier = SizeTier.XL # type: ignore[misc] + + def test_wave_mapping_defaults_are_immutable(self) -> None: + from agent_estimate.core.models import Wave + + wave = Wave( + wave_number=0, + start_minutes=0.0, + end_minutes=1.0, + assignments=(), + ) + with pytest.raises(TypeError): + wave.agent_review_minutes["codex"] = 1.0 # type: ignore[index] + + def test_wave_plan_agent_utilization_is_immutable(self) -> None: + from agent_estimate.core.models import WavePlan + + plan = WavePlan( + waves=(), + critical_path=(), + critical_path_minutes=0.0, + agent_utilization={"codex": 0.5}, + parallel_efficiency=1.0, + total_wall_clock_minutes=1.0, + total_sequential_minutes=1.0, + ) + with pytest.raises(TypeError): + plan.agent_utilization["codex"] = 1.0 # type: ignore[index] diff --git a/tests/unit/test_pert_engine.py b/tests/unit/test_pert_engine.py index 1472a18..13f30b3 100644 --- a/tests/unit/test_pert_engine.py +++ b/tests/unit/test_pert_engine.py @@ -8,6 +8,7 @@ from agent_estimate.core.human_comparison import compute_human_equivalent, get_human_multiplier from agent_estimate.core.models import ( + EstimationCategory, MetrWarning, ReviewMode, SizeTier, @@ -59,6 +60,10 @@ def test_optimistic_exceeds_most_likely_raises(self) -> None: with pytest.raises(ValueError, match="O <= M <= P"): compute_pert(25, 20, 30) + def test_negative_optimistic_raises(self) -> None: + with pytest.raises(ValueError, match="O >= 0"): + compute_pert(-10, 5, 10) + def test_result_is_frozen(self) -> None: result = compute_pert(10, 20, 30) with pytest.raises(AttributeError): @@ -356,6 +361,7 @@ def test_basic_pipeline(self) -> None: ) assert isinstance(result, TaskEstimate) + assert result.estimation_category == EstimationCategory.CODING # PERT E = (O + 4M + P) / 6 = (12 + 4*23 + 40) / 6 = 144 / 6 = 24.0 o, m, p = TIER_BASELINES[SizeTier.S] expected_pert = (o + 4 * m + p) / 6 diff --git a/tests/unit/test_pert_gaps.py b/tests/unit/test_pert_gaps.py index b1284bb..29261d2 100644 --- a/tests/unit/test_pert_gaps.py +++ b/tests/unit/test_pert_gaps.py @@ -2,6 +2,7 @@ from __future__ import annotations +import filecmp from pathlib import Path from unittest.mock import patch @@ -138,3 +139,11 @@ def test_none_thresholds_warn_for_large_estimate(self) -> None: result = check_metr_threshold("opus", 99999.0, thresholds=None) assert result is not None assert result.model_key == "opus" + + def test_root_and_packaged_metr_thresholds_stay_in_sync(self) -> None: + repo_root = Path(__file__).resolve().parents[2] + assert filecmp.cmp( + repo_root / "metr_thresholds.yaml", + repo_root / "src" / "agent_estimate" / "metr_thresholds.yaml", + shallow=False, + ) diff --git a/tests/unit/test_plugin_structure.py b/tests/unit/test_plugin_structure.py index fe1a918..69382ca 100644 --- a/tests/unit/test_plugin_structure.py +++ b/tests/unit/test_plugin_structure.py @@ -83,6 +83,10 @@ def test_skill_yaml_is_valid(self): assert data["id"] == "estimate" assert "compatible_runtimes" in data + def test_skill_yaml_is_not_marked_staging(self): + data = yaml.safe_load(self.skill_yaml.read_text()) + assert data["public_status"] == "public" + def test_skill_readme_exists(self): assert self.skill_readme.exists(), "skills/estimate/README.md must exist" @@ -144,6 +148,13 @@ def test_codex_skill_documents_json_as_supported(self): assert "--format json" in content assert "NOT YET IMPLEMENTED" not in content + def test_shared_intent_documents_current_cli_surface(self): + content = self.intent_md.read_text() + assert "agent-estimate session" in content + assert "--review-mode none|standard|complex|3-round" in content + for flag in ("--type", "--spec-clarity", "--warm-context", "--agent-fit"): + assert flag in content + def test_both_skills_share_skill_name(self): claude = self.claude_skill_md.read_text() codex = self.codex_skill_md.read_text() diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index c73647b..ded9cef 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -230,6 +230,22 @@ def test_per_round_minutes_flag(self) -> None: ) assert result.exit_code == 0 + def test_large_duration_does_not_crash_markdown_padding(self) -> None: + result = runner.invoke( + app, + [ + "session", + "--agents", + "2", + "--rounds", + "1", + "--per-round-minutes", + "99999999999999999999999", + ], + ) + assert result.exit_code == 0 + assert "Wall-clock" in result.output + class TestSessionCLIJson: """CLI session subcommand — JSON output.""" diff --git a/tests/unit/test_sizing.py b/tests/unit/test_sizing.py index 70fb1a3..a1725ed 100644 --- a/tests/unit/test_sizing.py +++ b/tests/unit/test_sizing.py @@ -32,10 +32,18 @@ def test_test_detection(self) -> None: result = classify_task("Write test coverage for the auth module") assert result.task_type == TaskType.TEST + def test_add_tests_prefers_test_over_feature(self) -> None: + result = classify_task("Add tests for the parser") + assert result.task_type == TaskType.TEST + def test_docs_detection(self) -> None: result = classify_task("Update the README with setup instructions") assert result.task_type == TaskType.DOCS + def test_add_docs_prefers_docs_over_feature(self) -> None: + result = classify_task("Add docs for the parser") + assert result.task_type == TaskType.DOCS + def test_unknown_type_when_no_keyword(self) -> None: result = classify_task("Do the thing with the stuff") assert result.task_type == TaskType.UNKNOWN @@ -92,14 +100,17 @@ def test_complexity_signals_recorded(self) -> None: class TestTierVoteMedian: - def test_conflicting_signals_picks_median(self) -> None: - # "simple" → S vote, "large" → L vote → median of [S, L] = L (index 1 of 2) + def test_conflicting_signals_pick_conservative_upper_middle(self) -> None: + # "simple" → S vote, "large" → L vote → upper-middle of [S, L] = L. result = classify_task("Simple but large refactoring task") - # Sorted: [S, L], median index = 2//2 = 1 → L # But "refactoring" triggers structural-change complexity signal → may bump further # The structural-change is 1 complexity signal → no bump yet assert result.tier in (SizeTier.L, SizeTier.XL) + def test_even_vote_count_biases_to_upper_middle(self) -> None: + result = classify_task("a small but complex change") + assert result.tier == SizeTier.L + def test_all_same_tier_votes_return_that_tier(self) -> None: result = classify_task("trivial rename one-liner typo") assert result.tier == SizeTier.XS diff --git a/tests/unit/test_sqlite_gaps.py b/tests/unit/test_sqlite_gaps.py index 54b4bda..c85abb0 100644 --- a/tests/unit/test_sqlite_gaps.py +++ b/tests/unit/test_sqlite_gaps.py @@ -4,9 +4,11 @@ from collections.abc import Generator from pathlib import Path +import threading import pytest +from agent_estimate.adapters import sqlite_store from agent_estimate.adapters.sqlite_store import ( ObservationInput, SQLiteCalibrationStore, @@ -207,6 +209,47 @@ def test_each_task_type_has_correct_count( assert summaries["alpha"]["sample_count"] == 3 assert summaries["beta"]["sample_count"] == 7 + def test_calibrate_holds_store_lock_for_full_recompute( + self, + store: SQLiteCalibrationStore, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + percentile_started = threading.Event() + release_percentile = threading.Event() + insert_finished = threading.Event() + original_percentile = sqlite_store._percentile + + def blocking_percentile(values: list[float], percent: float) -> float: + if percent == 50.0 and not percentile_started.is_set(): + percentile_started.set() + release_percentile.wait(timeout=2) + return original_percentile(values, percent) + + for i in range(3): + store.insert_observation(_observation(task_type="alpha", error_ratio=0.1 * (i + 1))) + + monkeypatch.setattr(sqlite_store, "_percentile", blocking_percentile) + + calibrate_thread = threading.Thread(target=store.calibrate) + calibrate_thread.start() + assert percentile_started.wait(timeout=2) + + insert_thread = threading.Thread( + target=lambda: ( + store.insert_observation(_observation(task_type="alpha", error_ratio=0.4)), + insert_finished.set(), + ), + ) + insert_thread.start() + + assert insert_finished.wait(timeout=0.1) is False + release_percentile.set() + calibrate_thread.join(timeout=2) + insert_thread.join(timeout=2) + + assert not calibrate_thread.is_alive() + assert insert_finished.is_set() + # --------------------------------------------------------------------------- # Custom k_anonymity_floor diff --git a/tests/unit/test_sqlite_store.py b/tests/unit/test_sqlite_store.py index bc09606..0b76e96 100644 --- a/tests/unit/test_sqlite_store.py +++ b/tests/unit/test_sqlite_store.py @@ -5,6 +5,7 @@ import json from collections.abc import Generator from pathlib import Path +import sqlite3 import pytest @@ -136,6 +137,24 @@ def test_schema_version_table_exists(store: SQLiteCalibrationStore) -> None: assert row["version"] == 1 +def test_future_schema_version_is_rejected(tmp_path: Path) -> None: + db_path = tmp_path / "future.db" + connection = sqlite3.connect(db_path) + try: + connection.execute( + "CREATE TABLE schema_version (version INTEGER PRIMARY KEY, applied_at TEXT)" + ) + connection.execute( + "INSERT INTO schema_version (version, applied_at) VALUES (999, CURRENT_TIMESTAMP)" + ) + connection.commit() + finally: + connection.close() + + with pytest.raises(RuntimeError, match="Unsupported calibration DB schema version 999"): + SQLiteCalibrationStore(db_path) + + def test_insert_rejects_invalid_negative_values(store: SQLiteCalibrationStore) -> None: invalid = _observation() invalid = ObservationInput( diff --git a/tests/unit/test_task_type_models.py b/tests/unit/test_task_type_models.py index ec66b0c..9b53c3e 100644 --- a/tests/unit/test_task_type_models.py +++ b/tests/unit/test_task_type_models.py @@ -2,7 +2,7 @@ from __future__ import annotations -from agent_estimate.core.models import EstimationCategory, ReviewMode +from agent_estimate.core.models import EstimationCategory, ReviewMode, TaskType from agent_estimate.core.modifiers import build_modifier_set from agent_estimate.core.task_type_models import ( _APP_DEV_BASELINES, @@ -303,6 +303,10 @@ def test_signal_label_in_sizing(self) -> None: est = estimate_documentation("Write API docs", self.modifiers) assert "documentation-model" in est.sizing.signals + def test_uses_docs_task_type_for_human_multiplier(self) -> None: + est = estimate_documentation("Write API docs", self.modifiers) + assert est.sizing.task_type == TaskType.DOCS + def test_default_review_is_none(self) -> None: est = estimate_documentation("Write API docs", self.modifiers) assert est.review_minutes == 0.0 diff --git a/tests/unit/test_version.py b/tests/unit/test_version.py index f471d30..a4533fe 100644 --- a/tests/unit/test_version.py +++ b/tests/unit/test_version.py @@ -4,4 +4,4 @@ def test_version_string_present() -> None: - assert __version__ == "0.7.1" + assert __version__ == "0.7.2" diff --git a/tests/unit/test_wave_planner.py b/tests/unit/test_wave_planner.py index 9531736..da072cd 100644 --- a/tests/unit/test_wave_planner.py +++ b/tests/unit/test_wave_planner.py @@ -113,11 +113,10 @@ def test_single_agent(self) -> None: class TestUnbalancedLoad: """LPT should produce better balance than naive assignment. - With co-dispatch: all 4 tasks land on the same agent across 2 slots. + With co-dispatch: only tasks in the same agent slot get warm-context reduction. LPT assigns A(40)→slot0, B(30)→slot1, C(20)→slot1, D(10)→slot0. - agent_wave_tasks['claude'] = ['A', 'B', 'C', 'D'] — A is first (no reduction), - B, C, D get 0.5x: B=15, C=10, D=5. - Revised slot loads: slot0=A(40)+D(5)=45, slot1=B(15)+C(10)=25 → makespan=45. + Slot groups are [A, D] and [B, C], so only D and C get 0.5x. + Revised slot loads: slot0=A(40)+D(5)=45, slot1=B(30)+C(10)=40 → makespan=45. """ def test_unbalanced_load(self) -> None: @@ -131,8 +130,10 @@ def test_unbalanced_load(self) -> None: plan = plan_waves(tasks, [_agent(parallelism=2)]) assert len(plan.waves) == 1 - # Co-dispatch reduces B, C, D by 0.5x; makespan = slot0 = 40+5 = 45 + # Co-dispatch reduces C and D by 0.5x; makespan = slot0 = 40+5 = 45 assert plan.waves[0].end_minutes == pytest.approx(45.0) + durations = {a.task_id: a.duration_minutes for a in plan.waves[0].assignments} + assert durations == {"A": 40, "B": 30, "C": 10, "D": 5} class TestCapabilityFiltering: