From 745e40b5fd9385612839f8dd96b088c38356caaf Mon Sep 17 00:00:00 2001 From: Sima Bagheri <37793675+simaba@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:20:28 +0800 Subject: [PATCH 1/4] fix: soften unsupported certainty in fallback summaries --- src/assessment_service.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/assessment_service.py b/src/assessment_service.py index 288267a..b45cd53 100644 --- a/src/assessment_service.py +++ b/src/assessment_service.py @@ -32,6 +32,31 @@ def _generation_note(mode: str, reason: str | None = None) -> str: return f"Generation note: deterministic fallback. {suffix}" +def _soften_fallback_certainty(result: AssessmentResult) -> AssessmentResult: + """Keep deterministic starter output from overstating unvalidated inputs.""" + define_items = [ + replace( + item, + statement=item.statement.replace( + "Problem confirmed:", "Reported problem statement:" + ), + ) + for item in result.dmaic_structure.get("define", []) + ] + dmaic_structure = {**result.dmaic_structure, "define": define_items} + return replace( + result, + cleaned_problem_statement=result.cleaned_problem_statement.replace( + "has a measurable performance gap:", + "has a reported performance concern:", + ), + role_summary=result.role_summary.replace( + "has a confirmed performance gap", "has a reported performance concern" + ), + dmaic_structure=dmaic_structure, + ) + + def _fallback( project: ProjectInput, mode: str, @@ -39,7 +64,7 @@ def _fallback( reason: str, ) -> AssessmentResult: """Return a deterministic result that records why it was used.""" - result = _deterministic_fallback(project, mode, audience) + result = _soften_fallback_certainty(_deterministic_fallback(project, mode, audience)) return replace( result, role_summary=f"{_generation_note('deterministic_fallback', reason)}\n\n{result.role_summary}", From 465277db8db1d9349fe21c46047caf304b9f11d5 Mon Sep 17 00:00:00 2001 From: Sima Bagheri <37793675+simaba@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:20:45 +0800 Subject: [PATCH 2/4] test: keep fallback language evidence-aware --- tests/test_assessment_provenance.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_assessment_provenance.py b/tests/test_assessment_provenance.py index fc9edaf..6e7629a 100644 --- a/tests/test_assessment_provenance.py +++ b/tests/test_assessment_provenance.py @@ -30,6 +30,20 @@ def test_missing_api_key_is_disclosed_as_deterministic_fallback(monkeypatch) -> assert result.role_summary.startswith("Generation note: deterministic fallback.") +def test_fallback_does_not_present_unvalidated_input_as_confirmed(monkeypatch) -> None: + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + + result = run_assessment(project(), mode="dmaic", audience="executive") + + assert "confirmed performance gap" not in result.role_summary + assert "reported performance concern" in result.role_summary + assert "measurable performance gap" not in result.cleaned_problem_statement + assert "reported performance concern" in result.cleaned_problem_statement + assert result.dmaic_structure["define"][0].statement.startswith( + "Reported problem statement:" + ) + + def test_require_llm_fails_instead_of_silently_falling_back(monkeypatch) -> None: monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) From b4043d866925bc20c10251873fdf15b242febc7e Mon Sep 17 00:00:00 2001 From: Sima Bagheri <37793675+simaba@users.noreply.github.com> Date: Tue, 7 Jul 2026 15:33:53 +0800 Subject: [PATCH 3/4] docs: define the supported assessment generation boundary --- docs/assessment-architecture.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 docs/assessment-architecture.md diff --git a/docs/assessment-architecture.md b/docs/assessment-architecture.md new file mode 100644 index 0000000..4420015 --- /dev/null +++ b/docs/assessment-architecture.md @@ -0,0 +1,26 @@ +# Assessment Generation Boundary + +## Supported entry point + +Use `src.engine.run_assessment()` for every application, CLI, test, or integration call that produces an assessment. + +```python +from src.engine import run_assessment + +result = run_assessment(project, mode="dmaic", audience="pm") +``` + +The engine delegates to `src.assessment_service.run_assessment_with_provenance()`. Every returned `AssessmentResult` records one of two modes: + +- `llm`: a live model response was parsed into the required assessment structure. +- `deterministic_fallback`: a predictable local starter assessment was used, with a safe reason explaining why. + +Set `require_llm=True` when deterministic fallback would be misleading; the call then raises a concise `AssessmentGenerationError` instead of returning fallback output. + +## Boundary rule + +The phase module contains low-level prompt, parsing, and deterministic-fallback helpers. It is not an application-facing generation API. New code must not call phase-level model-generation helpers directly, because that would bypass provenance and strict-mode behavior. + +## Review rule + +A regression test verifies that the supported engine is wired to the provenance-aware service and does not import or call the legacy phase-level generator. This preserves the intended public behavior while allowing the lower-level helper module to be refactored separately. From 882389c86d25aad74eecb2b2e2901f609f746ba8 Mon Sep 17 00:00:00 2001 From: Sima Bagheri <37793675+simaba@users.noreply.github.com> Date: Tue, 7 Jul 2026 15:34:23 +0800 Subject: [PATCH 4/4] test: keep public assessment calls provenance-aware --- tests/test_assessment_api_boundary.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 tests/test_assessment_api_boundary.py diff --git a/tests/test_assessment_api_boundary.py b/tests/test_assessment_api_boundary.py new file mode 100644 index 0000000..101b061 --- /dev/null +++ b/tests/test_assessment_api_boundary.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import inspect + +import src.engine as engine +from src.assessment_service import run_assessment_with_provenance + + +def test_engine_delegates_to_provenance_aware_generation_service() -> None: + source = inspect.getsource(engine) + + assert engine.run_assessment.__module__ == "src.engine" + assert "run_assessment_with_provenance" in source + assert "run_llm_assessment" not in source + + +def test_engine_uses_the_supported_generation_service() -> None: + result = engine.run_assessment + + assert callable(result) + assert callable(run_assessment_with_provenance)