feat(agents,api): route LLM tasks through gateway aliases; drop default model (#478)

.env.example

@@ -83,25 +83,33 @@ QDRANT_URL=http://localhost:6333
 
 # ─── AI / LLM ─────────────────────────────────────────────────────────────────
 OPENAI_API_KEY=sk-your-openai-api-key-here
+# OPENAI_MODEL now only affects the "explain this alert" / BYOK path. Task
+# workloads (triage, recon, investigation, copilot, summary, report, NL) request
+# a logical alias instead — see the LLM gateway block below.
 OPENAI_MODEL=gpt-4-turbo-preview
 
 # ─── LLM gateway (LiteLLM) — issue #478 ───────────────────────────────────────
-# LiteLLM is the single entry point for live LLM calls. AiSOC asks for a logical
-# task alias (aisoc-triage, aisoc-investigation, …); the alias → real-model map
-# lives in infra/litellm/config.yaml, so you can swap local/hosted models there
-# without changing AiSOC. The service is defined in docker-compose.yml.
+# Since #478 AiSOC asks for a logical task alias (aisoc-triage,
+# aisoc-investigation, …) for every live LLM call — there is no hardcoded default
+# model. The alias → real-model map lives in infra/litellm/config.yaml, so you
+# swap local/hosted models there without changing AiSOC. The gateway service is
+# defined in docker-compose.yml.
 #
 # Key that AiSOC uses to authenticate to the gateway (also LiteLLM's admin key):
 LITELLM_MASTER_KEY=sk-aisoc-local
 # In-network URL of the gateway (already set per-service in docker-compose.yml):
 LLM_GATEWAY_URL=http://litellm:4000/v1
 #
-# To ROUTE AiSOC through the gateway, uncomment the two lines below — they make
-# every service send its calls to LiteLLM instead of straight to the provider.
-# Put your real provider key in OPENAI_API_KEY above (LiteLLM reads it), then:
-#   OPENAI_BASE_URL=http://litellm:4000/v1
-#   OPENAI_API_KEY=${LITELLM_MASTER_KEY}
-# Leave them unset to keep calling the provider directly (unchanged behaviour).
+# ENABLE LIVE LLM — pick one:
+#  (a) Route through the gateway (recommended). Put your real provider key in
+#      OPENAI_API_KEY above (LiteLLM reads it), then uncomment:
+#        OPENAI_BASE_URL=http://litellm:4000/v1
+#        OPENAI_API_KEY=${LITELLM_MASTER_KEY}
+#  (b) Call a provider directly without the gateway — pin each task role to a
+#      concrete model (escape hatch), e.g.:
+#        AISOC_MODEL_PIN_TRIAGE=gpt-4o-mini
+#        AISOC_MODEL_PIN_INVESTIGATION=gpt-4o
+# With neither configured, AiSOC uses its deterministic offline path.
 
 # ─── Threat Intelligence ──────────────────────────────────────────────────────
 # Open-source / community feeds (work out of the box, no key required for some)

CHANGELOG.md

@@ -27,6 +27,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   `services/agents/tests/test_litellm_config.py`. (A follow-up PR wires the ~10
   in-code callsites to request these aliases via `model_pins` and removes the
   hardcoded `gpt-4o-mini` default, closing #478.)
+- **LLM task-alias routing — no more shipped default model
+  ([#478](https://github.com/beenuar/AiSOC/issues/478), PR2).** Every live LLM
+  call now asks for a **logical task alias** instead of a hardcoded model. New
+  `services/agents/app/llm/factory.py` (`make_chat_model` / `resolve_model_alias`)
+  resolves a task role to its `aisoc-<role>` alias + the gateway base URL;
+  `model_pins.py` now pins all seven roles (triage, recon, investigation, copilot,
+  summary, report, nl) to aliases with a `deterministic` floor. The ~10 scattered
+  `os.getenv("AISOC_LLM_MODEL"/"OPENAI_MODEL","gpt-4o-mini")` + `ChatOpenAI(...)` /
+  raw-HTTP callsites across the agents (auto-triage, cloud/identity/insider/
+  phishing, recon/forensic/responder/report-writer, copilot, contextual, NL
+  translator) and the API endpoints (translation, hunts, knowledge base, phishing;
+  via new `services/api/app/services/model_aliases.py`) now request aliases; the
+  hardcoded `gpt-4o-mini` default is gone. **Behaviour change:** live LLM now runs
+  through the gateway (`OPENAI_BASE_URL`), or pin a concrete model per role via
+  `AISOC_MODEL_PIN_<ROLE>` (escape hatch; the slim demo does this so keyed demos
+  keep working); with neither, the deterministic offline path is used. Tests:
+  `services/agents/tests/test_llm_factory.py` + tightened `test_litellm_config.py`.
+  Closes #478.
 - **v8 P4 — Compounding Memory (verdicts that measurably improve).** New
   `services/fusion/app/memory/`: a nightly-distillable institutional memory that
   makes verdicts more accurate the longer an instance runs. **Distillation**

apps/docs/docs/operations/llm-gateway.md

@@ -62,9 +62,19 @@ OPENAI_BASE_URL=http://litellm:4000/v1     # send AiSOC's calls to the gateway
 # OPENAI_API_KEY=${LITELLM_MASTER_KEY}     # (in the AiSOC services' environment)
 ```
 
-Leave `OPENAI_BASE_URL` unset to keep calling the provider directly — the
-gateway then runs idle (still serving `/health` and `/metrics`) and behaviour is
-unchanged.
+AiSOC now requests a task **alias** for every live call, so an alias only
+resolves when it reaches the gateway. If you don't run the gateway, pin each
+role to a concrete provider model instead (the **escape hatch**):
+
+```bash
+AISOC_MODEL_PIN_TRIAGE=gpt-4o-mini
+AISOC_MODEL_PIN_INVESTIGATION=gpt-4o
+# … one per role: triage, recon, investigation, copilot, summary, report, nl
+```
+
+With neither the gateway nor pin overrides configured, AiSOC uses its
+deterministic offline path. (`OPENAI_MODEL` still applies to the separate
+"explain this alert" / BYOK path.)
 
 ## Re-point a task to a local model
 

infra/compose/docker-compose.demo.yml

@@ -191,6 +191,17 @@ services:
       OPENAI_API_KEY: ${OPENAI_API_KEY:-}
       ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
       OPENAI_MODEL: ${OPENAI_MODEL:-gpt-4o-mini}
+      # Issue #478 — the slim demo runs without the LiteLLM gateway, so pin every
+      # task role to a concrete provider model (AiSOC calls the provider directly
+      # with OPENAI_API_KEY; deterministic fallback when no key is set). The full
+      # stack (root docker-compose.yml) routes these aliases through the gateway.
+      AISOC_MODEL_PIN_TRIAGE: ${OPENAI_MODEL:-gpt-4o-mini}
+      AISOC_MODEL_PIN_RECON: ${OPENAI_MODEL:-gpt-4o-mini}
+      AISOC_MODEL_PIN_INVESTIGATION: ${OPENAI_MODEL:-gpt-4o-mini}
+      AISOC_MODEL_PIN_COPILOT: ${OPENAI_MODEL:-gpt-4o-mini}
+      AISOC_MODEL_PIN_SUMMARY: ${OPENAI_MODEL:-gpt-4o-mini}
+      AISOC_MODEL_PIN_REPORT: ${OPENAI_MODEL:-gpt-4o-mini}
+      AISOC_MODEL_PIN_NL: ${OPENAI_MODEL:-gpt-4o-mini}
       ENV: development
     networks:
       - aisoc-demo

services/agents/app/agents/auto_triage_agent.py

@@ -19,10 +19,10 @@
 
 import structlog
 from langchain_core.messages import HumanMessage, SystemMessage
-from langchain_openai import ChatOpenAI
 
 from app.investigator.prompt_sanitizer import sanitize_text, wrap_untrusted
 from app.llm import safe_ainvoke
+from app.llm.factory import make_chat_model
 from app.models.state import AgentStatus, InvestigationState
 from app.prompt_serialization import format_extra_fields_for_llm
 
@@ -176,8 +176,7 @@ async def run_auto_triage(state: InvestigationState) -> InvestigationState:
 
     alert_context = _build_alert_context(state)
 
-    model_name = os.getenv("AISOC_LLM_MODEL", "gpt-4o-mini")
-    llm = ChatOpenAI(model=model_name, temperature=0.0, max_tokens=512)
+    llm = make_chat_model("triage", temperature=0.0, max_tokens=512)
 
     t0 = time.monotonic()
     try:

services/agents/app/agents/cloud_agent.py

@@ -10,17 +10,16 @@
 from __future__ import annotations
 
 import json
-import os
 import time
 from typing import Any
 
 import structlog
 from langchain_core.messages import HumanMessage, SystemMessage
-from langchain_openai import ChatOpenAI
 
 from app.context import ContextBundle
 from app.investigator.prompt_sanitizer import sanitize_text, wrap_untrusted
 from app.llm import safe_ainvoke
+from app.llm.factory import make_chat_model
 from app.models.state import AgentStatus, InvestigationState
 from app.prompt_serialization import format_extra_fields_for_llm, summarize_structure_for_llm
 
@@ -200,8 +199,7 @@ async def run_cloud(
     bundle_lines = bundle.prompt_context_lines() if bundle is not None else []
     prompt_context = base_context + (("\n" + "\n".join(bundle_lines)) if bundle_lines else "")
 
-    model_name = os.getenv("AISOC_LLM_MODEL", "gpt-4o-mini")
-    llm = ChatOpenAI(model=model_name, temperature=0.0, max_tokens=768)
+    llm = make_chat_model("investigation", temperature=0.0, max_tokens=768)
 
     t0 = time.monotonic()
     try:

services/agents/app/agents/identity_agent.py

@@ -9,17 +9,16 @@
 from __future__ import annotations
 
 import json
-import os
 import time
 from typing import Any
 
 import structlog
 from langchain_core.messages import HumanMessage, SystemMessage
-from langchain_openai import ChatOpenAI
 
 from app.context import ContextBundle
 from app.investigator.prompt_sanitizer import sanitize_text, wrap_untrusted
 from app.llm import safe_ainvoke
+from app.llm.factory import make_chat_model
 from app.models.state import AgentStatus, InvestigationState
 from app.prompt_serialization import format_extra_fields_for_llm, summarize_structure_for_llm
 
@@ -195,8 +194,7 @@ async def run_identity(
     bundle_lines = bundle.prompt_context_lines() if bundle is not None else []
     prompt_context = base_context + (("\n" + "\n".join(bundle_lines)) if bundle_lines else "")
 
-    model_name = os.getenv("AISOC_LLM_MODEL", "gpt-4o-mini")
-    llm = ChatOpenAI(model=model_name, temperature=0.0, max_tokens=768)
+    llm = make_chat_model("investigation", temperature=0.0, max_tokens=768)
 
     t0 = time.monotonic()
     try:

services/agents/app/agents/insider_threat_agent.py

@@ -10,17 +10,16 @@
 from __future__ import annotations
 
 import json
-import os
 import time
 from typing import Any
 
 import structlog
 from langchain_core.messages import HumanMessage, SystemMessage
-from langchain_openai import ChatOpenAI
 
 from app.context import ContextBundle
 from app.investigator.prompt_sanitizer import sanitize_text, wrap_untrusted
 from app.llm import safe_ainvoke
+from app.llm.factory import make_chat_model
 from app.models.state import AgentStatus, InvestigationState
 from app.prompt_serialization import format_extra_fields_for_llm, summarize_structure_for_llm
 
@@ -218,8 +217,7 @@ async def run_insider_threat(
     bundle_lines = bundle.prompt_context_lines() if bundle is not None else []
     prompt_context = base_context + (("\n" + "\n".join(bundle_lines)) if bundle_lines else "")
 
-    model_name = os.getenv("AISOC_LLM_MODEL", "gpt-4o-mini")
-    llm = ChatOpenAI(model=model_name, temperature=0.0, max_tokens=768)
+    llm = make_chat_model("investigation", temperature=0.0, max_tokens=768)
 
     t0 = time.monotonic()
     try:

services/agents/app/agents/phishing_agent.py

@@ -10,17 +10,16 @@
 from __future__ import annotations
 
 import json
-import os
 import time
 from typing import Any
 
 import structlog
 from langchain_core.messages import HumanMessage, SystemMessage
-from langchain_openai import ChatOpenAI
 
 from app.context import ContextBundle
 from app.investigator.prompt_sanitizer import sanitize_text, wrap_untrusted
 from app.llm import safe_ainvoke
+from app.llm.factory import make_chat_model
 from app.models.state import AgentStatus, InvestigationState
 from app.prompt_serialization import format_extra_fields_for_llm, summarize_structure_for_llm
 
@@ -187,8 +186,7 @@ async def run_phishing(
     bundle_lines = bundle.prompt_context_lines() if bundle is not None else []
     prompt_context = base_context + (("\n" + "\n".join(bundle_lines)) if bundle_lines else "")
 
-    model_name = os.getenv("AISOC_LLM_MODEL", "gpt-4o-mini")
-    llm = ChatOpenAI(model=model_name, temperature=0.0, max_tokens=768)
+    llm = make_chat_model("investigation", temperature=0.0, max_tokens=768)
 
     t0 = time.monotonic()
     try:

services/agents/app/api/contextual.py

@@ -46,6 +46,7 @@
 from pydantic import BaseModel, Field
 
 from app.llm import safe_ainvoke, safe_astream
+from app.llm.factory import resolve_model_alias
 from app.prompt_serialization import summarize_structure_for_llm
 
 logger = structlog.get_logger()
@@ -418,10 +419,10 @@ def _fallback_response(system: str, user: str) -> str:
         "You're seeing a deterministic placeholder response.\n\n"
         "**To enable real contextual answers:**\n\n"
         "1. Set `OPENAI_API_KEY` in your `.env` file\n"
-        "2. Optionally set `OPENAI_MODEL` (default: `gpt-4o-mini`)\n"
+        "2. Point AiSOC at the LiteLLM gateway (`OPENAI_BASE_URL=http://litellm:4000/v1`), "
+        "which maps each task alias to a real model\n"
         "3. Restart the `aisoc-agents` service\n\n"
-        "Refer to [the docs](https://example.com/docs/copilot) for self-hosted "
-        "model alternatives (Ollama, vLLM, Together)."
+        "See the LLM-gateway docs for self-hosted model alternatives (Ollama, vLLM, Together)."
     )
 
 
@@ -445,7 +446,7 @@ async def list_actions() -> ContextualActionsCatalogue:
 async def run_action(req: ContextualActionRequest) -> ContextualActionResponse:
     started = time.monotonic()
     system, user = _build_messages(req)
-    model = os.getenv("OPENAI_MODEL", "gpt-4o-mini")
+    model = resolve_model_alias("copilot")
 
     fallback = not bool(os.getenv("OPENAI_API_KEY"))
     try:
@@ -503,7 +504,7 @@ async def run_action(req: ContextualActionRequest) -> ContextualActionResponse:
 )
 async def run_action_stream(req: ContextualActionRequest) -> StreamingResponse:
     system, user = _build_messages(req)
-    model = os.getenv("OPENAI_MODEL", "gpt-4o-mini")
+    model = resolve_model_alias("copilot")
     title = _TITLES.get((req.page, req.action), f"{req.page} · {req.action}")
     suggestions = _FOLLOW_UPS.get((req.page, req.action), [])
     fallback = not bool(os.getenv("OPENAI_API_KEY"))

services/agents/app/api/copilot.py

@@ -122,6 +122,7 @@ async def _get_openai_reply(
 
     try:
         from app.llm.contract import safe_chat_completions_request
+        from app.llm.factory import chat_completions_url, resolve_model_alias
 
         messages: list[dict[str, str]] = [
             {
@@ -140,8 +141,9 @@ async def _get_openai_reply(
 
         body = await safe_chat_completions_request(
             api_key=api_key,
-            model="gpt-4o-mini",
+            model=resolve_model_alias("copilot"),
             messages=messages,
+            url=chat_completions_url(),
             max_tokens=512,
         )
         return body["choices"][0]["message"]["content"]

services/agents/app/investigator/forensic_agent.py

@@ -17,10 +17,10 @@
 
 import structlog
 from langchain_core.messages import HumanMessage, SystemMessage
-from langchain_openai import ChatOpenAI
 
 from app.core.cost_telemetry import record_llm_call
 from app.llm import safe_ainvoke
+from app.llm.factory import make_chat_model, resolve_model_alias
 from app.prompt_serialization import summarize_structure_for_llm
 
 from .bundle_prompt import format_bundle_prompt_append
@@ -54,10 +54,8 @@
 
 
 async def _llm_forensic(state: InvestigatorState) -> dict[str, Any]:
-    import os
-
-    model = os.getenv("OPENAI_MODEL", "gpt-4o-mini")
-    llm = ChatOpenAI(model=model, temperature=0)
+    model = resolve_model_alias("investigation")
+    llm = make_chat_model("investigation", temperature=0)
 
     # Defence-in-depth: alert_summary, recon.summary, and the enrichment cache
     # can all carry attacker-controlled strings (banners, dark-web excerpts,

services/agents/app/investigator/recon_agent.py

@@ -16,10 +16,10 @@
 
 import structlog
 from langchain_core.messages import HumanMessage, SystemMessage
-from langchain_openai import ChatOpenAI
 
 from app.core.cost_telemetry import record_llm_call
 from app.llm import safe_ainvoke
+from app.llm.factory import make_chat_model, resolve_model_alias
 from app.prompt_serialization import summarize_structure_for_llm
 
 from .bundle_prompt import format_bundle_prompt_append
@@ -54,10 +54,9 @@ async def _llm_recon(state: InvestigatorState) -> dict[str, Any]:
     reasoning trace is replayable.
     """
     import json
-    import os
 
-    model = os.getenv("OPENAI_MODEL", "gpt-4o-mini")
-    llm = ChatOpenAI(model=model, temperature=0)
+    model = resolve_model_alias("recon")
+    llm = make_chat_model("recon", temperature=0)
 
     # Defence-in-depth: every field surfaced here can be attacker-influenced
     # (alert_summary often echoes log lines; raw_alert is verbatim event data).

services/agents/app/investigator/report_writer_agent.py

@@ -15,10 +15,10 @@
 
 import structlog
 from langchain_core.messages import HumanMessage, SystemMessage
-from langchain_openai import ChatOpenAI
 
 from app.core.cost_telemetry import record_llm_call
 from app.llm import safe_ainvoke
+from app.llm.factory import make_chat_model, resolve_model_alias
 from app.prompt_serialization import summarize_structure_for_llm
 
 from .bundle_prompt import format_bundle_prompt_append
@@ -158,15 +158,13 @@ def _md_to_html(md: str, case_id: str) -> str:
 
 async def run_report_writer(state_dict: dict[str, Any]) -> dict[str, Any]:
     """LangGraph node."""
-    import os
-
     state = InvestigatorState.from_dict(state_dict)
     t0 = time.monotonic()
 
     logger.info("report_writer.start", case_id=state.case_id)
 
-    model = os.getenv("OPENAI_MODEL", "gpt-4o-mini")
-    llm = ChatOpenAI(model=model, temperature=0)
+    model = resolve_model_alias("report")
+    llm = make_chat_model("report", temperature=0)
 
     context = _build_context(state)
     bundle_append = format_bundle_prompt_append(state.context_bundle)