Make live-model use and deterministic fallback explicit

.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+updates:
+  - package-ecosystem: pip
+    directory: "/"
+    schedule:
+      interval: monthly
+    open-pull-requests-limit: 3
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: monthly
+    open-pull-requests-limit: 3

docs/assessment-generation.md

@@ -0,0 +1,28 @@
+# Assessment Generation Modes
+
+Lean AI Ops can produce a structured assessment in two ways:
+
+| Mode | When it is used | What it means |
+|---|---|---|
+| `llm` | A configured Anthropic client returns a response that matches the required structure | The assessment was generated from the live model request. Its recommendations still require evidence review and domain validation. |
+| `deterministic_fallback` | No API key is configured, the client is unavailable, the live request fails, or the response cannot be parsed into the required structure | The app generated a predictable, evidence-tagged starter package from the project inputs. It is not a live-model result. |
+
+The CLI prints the generation mode before the report. `AssessmentResult` also records `generation_mode`, `model_name`, and `fallback_reason` for callers that want to render provenance in a UI or export.
+
+## Strict live-model mode
+
+Use `--require-llm` when a deterministic fallback would be misleading for your workflow:
+
+```bash
+python run_demo.py \
+  --input templates/sample_project.json \
+  --mode dmaic \
+  --audience pm \
+  --require-llm
+```
+
+In strict mode, the command exits with a concise error when the live model result is unavailable instead of silently substituting fallback output.
+
+## Important limit
+
+A live-model result is still a structured draft. The app's evidence tags distinguish input-supported statements, hypotheses, and missing evidence; they do not validate root causes, statistical conclusions, implementation feasibility, or release decisions.

run_demo.py

@@ -4,6 +4,7 @@
 import json
 from pathlib import Path
 
+from src.assessment_service import AssessmentGenerationError
 from src.engine import run_assessment
 from src.models import ProjectInput
 from src.renderers import render_markdown_summary
@@ -14,6 +15,13 @@ def load_input(path: Path) -> ProjectInput:
     return ProjectInput(**data)
 
 
+def _provenance_line(result) -> str:
+    if result.generation_mode == "llm":
+        return f"Generation: live model ({result.model_name})"
+    reason = result.fallback_reason or "Deterministic fallback was used."
+    return f"Generation: deterministic fallback — {reason}"
+
+
 def main() -> None:
     parser = argparse.ArgumentParser(description="Run the Lean Six Sigma AI copilot demo.")
     parser.add_argument("--input", type=Path, required=True, help="Path to project input JSON")
@@ -29,10 +37,26 @@ def main() -> None:
         default="pm",
         help="Audience for summary emphasis",
     )
+    parser.add_argument(
+        "--require-llm",
+        action="store_true",
+        help="Fail instead of using deterministic fallback when a live model result is unavailable.",
+    )
     args = parser.parse_args()
 
     project = load_input(args.input)
-    result = run_assessment(project, mode=args.mode, audience=args.audience)
+    try:
+        result = run_assessment(
+            project,
+            mode=args.mode,
+            audience=args.audience,
+            require_llm=args.require_llm,
+        )
+    except AssessmentGenerationError as exc:
+        parser.error(str(exc))
+
+    print(_provenance_line(result))
+    print()
     print(render_markdown_summary(result))
 
 

src/assessment_service.py

@@ -0,0 +1,123 @@
+"""Assessment orchestration with explicit model/fallback provenance."""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import replace
+from typing import Any, Dict
+
+from src.models import AssessmentResult, ProjectInput
+from src.phases import (
+    _SYSTEM_PROMPT,
+    _build_user_message,
+    _deterministic_fallback,
+    _parse_dmaic,
+    _parse_items,
+)
+
+
+MODEL_NAME = "claude-sonnet-4-6"
+
+
+class AssessmentGenerationError(RuntimeError):
+    """Raised when an LLM result is required but cannot be generated."""
+
+
+def _generation_note(mode: str, reason: str | None = None) -> str:
+    """Return a concise provenance note suitable for user-visible summaries."""
+    if mode == "llm":
+        return f"Generation note: live model output ({MODEL_NAME})."
+    suffix = reason or "Deterministic fallback was used."
+    return f"Generation note: deterministic fallback. {suffix}"
+
+
+def _fallback(
+    project: ProjectInput,
+    mode: str,
+    audience: str,
+    reason: str,
+) -> AssessmentResult:
+    """Return a deterministic result that records why it was used."""
+    result = _deterministic_fallback(project, mode, audience)
+    return replace(
+        result,
+        role_summary=f"{_generation_note('deterministic_fallback', reason)}\n\n{result.role_summary}",
+        generation_mode="deterministic_fallback",
+        model_name=None,
+        fallback_reason=reason,
+    )
+
+
+def _reason_from_exception(exc: Exception) -> str:
+    """Provide an actionable category without exposing credentials or payloads."""
+    if isinstance(exc, json.JSONDecodeError):
+        return "The model response was not valid JSON for the required assessment structure."
+    if isinstance(exc, (KeyError, TypeError, ValueError, IndexError)):
+        return "The model response did not match the required assessment structure."
+    if isinstance(exc, ImportError):
+        return "The optional Anthropic client package is unavailable in this environment."
+    return "The live model request could not be completed."
+
+
+def run_assessment_with_provenance(
+    project: ProjectInput,
+    mode: str,
+    audience: str,
+    *,
+    require_llm: bool = False,
+) -> AssessmentResult:
+    """Generate an assessment and label whether it came from the LLM or fallback.
+
+    The fallback remains useful for demonstrations and offline work. When
+    ``require_llm`` is true, an unavailable or malformed live response raises a
+    concise error instead of silently substituting deterministic output.
+    """
+    api_key = os.environ.get("ANTHROPIC_API_KEY", "").strip()
+    if not api_key:
+        reason = "No ANTHROPIC_API_KEY is configured; deterministic fallback was used."
+        if require_llm:
+            raise AssessmentGenerationError(reason)
+        return _fallback(project, mode, audience, reason)
+
+    try:
+        import anthropic
+
+        client = anthropic.Anthropic(api_key=api_key)
+        message = client.messages.create(
+            model=MODEL_NAME,
+            max_tokens=4096,
+            system=_SYSTEM_PROMPT,
+            messages=[{"role": "user", "content": _build_user_message(project, mode, audience)}],
+        )
+        raw = message.content[0].text.strip()
+        if raw.startswith("```"):
+            raw = raw.split("\n", 1)[1]
+            if raw.endswith("```"):
+                raw = raw[: raw.rfind("```")]
+            raw = raw.strip()
+        data: Dict[str, Any] = json.loads(raw)
+        return AssessmentResult(
+            project_name=project.project_name,
+            mode=mode,
+            audience=audience,
+            cleaned_problem_statement=data["cleaned_problem_statement"],
+            ctqs=_parse_items(data["ctqs"]),
+            sipoc=data["sipoc"],
+            dmaic_structure=_parse_dmaic(data["dmaic_structure"]),
+            root_causes=_parse_items(data["root_causes"]),
+            suggested_metrics=_parse_items(data["suggested_metrics"]),
+            improvement_actions=_parse_items(data["improvement_actions"]),
+            control_plan=_parse_items(data["control_plan"]),
+            action_tracker=data["action_tracker"],
+            project_memory=data["project_memory"],
+            role_summary=f"{_generation_note('llm')}\n\n{data['role_summary']}",
+            generation_mode="llm",
+            model_name=MODEL_NAME,
+            fallback_reason=None,
+        )
+    except Exception as exc:
+        reason = _reason_from_exception(exc)
+        if require_llm:
+            raise AssessmentGenerationError(reason) from exc
+        return _fallback(project, mode, audience, reason)

src/engine.py

@@ -1,8 +1,20 @@
 from __future__ import annotations
 
+from src.assessment_service import run_assessment_with_provenance
 from src.models import AssessmentResult, ProjectInput
-from src.phases import run_llm_assessment
 
 
-def run_assessment(project: ProjectInput, mode: str, audience: str) -> AssessmentResult:
-    return run_llm_assessment(project, mode, audience)
+def run_assessment(
+    project: ProjectInput,
+    mode: str,
+    audience: str,
+    *,
+    require_llm: bool = False,
+) -> AssessmentResult:
+    """Generate an assessment with explicit model/fallback provenance."""
+    return run_assessment_with_provenance(
+        project,
+        mode,
+        audience,
+        require_llm=require_llm,
+    )

src/models.py

@@ -42,3 +42,6 @@ class AssessmentResult:
     action_tracker: List[Dict[str, str]]
     project_memory: Dict[str, List[str]]
     role_summary: str
+    generation_mode: str = "unknown"
+    model_name: str | None = None
+    fallback_reason: str | None = None

tests/test_assessment_provenance.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+import pytest
+
+from src.assessment_service import AssessmentGenerationError
+from src.engine import run_assessment
+from src.models import ProjectInput
+
+
+def project() -> ProjectInput:
+    return ProjectInput(
+        project_name="Fictional intake delay",
+        problem_statement="A fictional request-intake process has avoidable delay and rework.",
+        current_symptoms=["requests wait for review", "some requests are reworked"],
+        current_metrics={"cycle_time_days": "12"},
+        constraints=["No production change during the fictional pilot"],
+        stakeholder_concerns=["Operations: reduce waiting time"],
+    )
+
+
+def test_missing_api_key_is_disclosed_as_deterministic_fallback(monkeypatch) -> None:
+    monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
+
+    result = run_assessment(project(), mode="dmaic", audience="pm")
+
+    assert result.generation_mode == "deterministic_fallback"
+    assert result.model_name is None
+    assert result.fallback_reason
+    assert "No ANTHROPIC_API_KEY" in result.fallback_reason
+    assert result.role_summary.startswith("Generation note: deterministic fallback.")
+
+
+def test_require_llm_fails_instead_of_silently_falling_back(monkeypatch) -> None:
+    monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
+
+    with pytest.raises(AssessmentGenerationError, match="No ANTHROPIC_API_KEY"):
+        run_assessment(project(), mode="dmaic", audience="pm", require_llm=True)

tests/test_tool_recommender.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from ui.tool_recommender import _recommendation
+
+
+DEFAULTS = {
+    "problem_type": "Not sure — I just know something is wrong",
+    "data_availability": "Some data — a few weeks / small sample",
+    "scope": "End-to-end process (multiple steps or departments)",
+    "urgency": "Medium-term — 1-3 months project",
+    "measurement_confidence": "We measure but haven't validated the measurement system",
+    "root_cause_status": "Have some hunches but not confirmed",
+    "experience": "Yellow Belt — familiar with basics",
+}
+
+
+def recommend(**overrides):
+    values = {**DEFAULTS, **overrides}
+    return _recommendation(**values)
+
+
+def test_known_solution_routes_to_control_plan() -> None:
+    recommendation = recommend(
+        root_cause_status="Root cause and solution are known — need to implement and sustain"
+    )
+
+    assert recommendation["tool"] == "Control Plan"
+
+
+def test_data_rich_defect_problem_requires_msa_before_capability() -> None:
+    recommendation = recommend(
+        problem_type="Too many defects or errors",
+        data_availability="Good data — months of history, 30+ data points",
+        measurement_confidence="We measure but haven't validated the measurement system",
+    )
+
+    assert recommendation["tool"] == "MSA / Gauge R&R"
+
+
+def test_validated_measurement_routes_data_rich_defects_to_capability() -> None:
+    recommendation = recommend(
+        problem_type="Too many defects or errors",
+        data_availability="Lots of data — automated / ongoing process data",
+        measurement_confidence="Measurement system is validated (MSA / Gauge R&R done)",
+    )
+
+    assert recommendation["tool"] == "Process Capability"
+
+
+def test_limited_data_variation_routes_to_root_cause_hypotheses() -> None:
+    recommendation = recommend(
+        problem_type="Results are inconsistent / too much variation",
+        data_availability="None yet — haven't started measuring",
+    )
+
+    assert recommendation["tool"] == "Root Cause Analysis"
+
+
+def test_urgent_unspecified_problem_routes_to_kaizen() -> None:
+    recommendation = recommend(
+        urgency="Immediate — something needs fixing this week"
+    )
+
+    assert recommendation["tool"] == "Kaizen / Rapid Improvement"