feat(escalation): auto-remediation action steps (webhook runbooks)

app/api/routes/escalations.py

@@ -3,6 +3,7 @@
 from fastapi import APIRouter, HTTPException, Response, status
 
 from app.api.deps import CurrentUser, DBSession
+from app.core.models.notification_channel import ChannelType
 from app.core.schemas.escalation import EscalationPolicyCreate, EscalationPolicyRead
 from app.core.services.escalation_service import EscalationService
 
@@ -20,6 +21,16 @@ async def _validate_channels(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
             detail=f"unknown or unowned channel(s): {', '.join(str(m) for m in missing)}",
         )
+    # Auto-remediation steps must target a webhook (a machine endpoint).
+    remediation_channels = {s.channel_id for s in payload.steps if s.action_command}
+    if remediation_channels:
+        types = await service.channel_types(owner_id, remediation_channels)
+        non_webhook = [cid for cid in remediation_channels if types.get(cid) != ChannelType.WEBHOOK]
+        if non_webhook:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail="auto-remediation steps must target a webhook channel",
+            )
 
 
 @router.get("", response_model=list[EscalationPolicyRead], summary="List escalation policies")

app/api/routes/web/escalation.py

@@ -1,12 +1,14 @@
 from __future__ import annotations
 
 import uuid
+from itertools import zip_longest
 
 from fastapi import APIRouter, Form, HTTPException, Request, status
 from fastapi.responses import HTMLResponse, RedirectResponse
 
 from app.api.deps.db import DBSession
 from app.api.routes.web._shared import login_redirect, resolve_current_user, templates
+from app.core.models.notification_channel import ChannelType
 from app.core.models.user import User
 from app.core.schemas.escalation import EscalationPolicyCreate, EscalationStepCreate
 from app.core.services.escalation_service import EscalationService
@@ -57,9 +59,12 @@ async def create_escalation_form(
     form = await request.form()
     delay_minutes = [str(v) for v in form.getlist("delay_minutes")]
     channel_id = [str(v) for v in form.getlist("channel_id")]
+    action_command = [str(v) for v in form.getlist("action_command")]
     steps: list[EscalationStepCreate] = []
     order = 0
-    for delay, channel in zip(delay_minutes, channel_id, strict=False):
+    for delay, channel, command in zip_longest(
+        delay_minutes, channel_id, action_command, fillvalue=""
+    ):
         if not channel:
             continue
         order += 1
@@ -69,6 +74,7 @@ async def create_escalation_form(
                     step_order=order,
                     delay_minutes=int(delay or 0),
                     channel_id=uuid.UUID(channel),
+                    action_command=command.strip() or None,
                 )
             )
         except (ValueError, TypeError):
@@ -84,9 +90,14 @@ async def _error(message: str) -> HTMLResponse:
 
     if not steps:
         return await _error("Add at least one step with a channel.")
-    owned = await service.channels_owned_by(user.id, {s.channel_id for s in steps})
-    if owned != {s.channel_id for s in steps}:
+    wanted = {s.channel_id for s in steps}
+    if await service.channels_owned_by(user.id, wanted) != wanted:
         return await _error("One or more selected channels are not yours.")
+    remediation_channels = {s.channel_id for s in steps if s.action_command}
+    if remediation_channels:
+        types = await service.channel_types(user.id, remediation_channels)
+        if any(types.get(cid) != ChannelType.WEBHOOK for cid in remediation_channels):
+            return await _error("Auto-remediation steps must target a webhook channel.")
     try:
         payload = EscalationPolicyCreate(name=name, is_enabled=(is_enabled == "on"), steps=steps)
     except (ValueError, TypeError) as exc:

app/core/models/escalation.py

@@ -58,6 +58,11 @@ class EscalationStep(UUIDPrimaryKey, Base):
         ForeignKey("notification_channels.id", ondelete="CASCADE"),
         nullable=False,
     )
+    # When set, this is an auto-remediation step: instead of a plain notification it
+    # POSTs a structured remediation intent (this command + the problem context) to
+    # its webhook channel, for an external runbook to act on. The server never runs
+    # commands itself.
+    action_command: Mapped[str | None] = mapped_column(String(500), nullable=True)
 
     policy: Mapped[EscalationPolicy] = relationship(back_populates="steps")
     channel: Mapped[NotificationChannel] = relationship(lazy="noload")

app/core/schemas/escalation.py

@@ -10,6 +10,8 @@ class EscalationStepCreate(BaseModel):
     step_order: int = Field(ge=1, le=100)
     delay_minutes: int = Field(ge=0, le=10080)  # up to a week
     channel_id: uuid.UUID
+    # When set, an auto-remediation step: POSTs this command (+ context) to a webhook.
+    action_command: str | None = Field(default=None, max_length=500)
 
 
 class EscalationStepRead(BaseModel):
@@ -19,6 +21,7 @@ class EscalationStepRead(BaseModel):
     step_order: int
     delay_minutes: int
     channel_id: uuid.UUID
+    action_command: str | None
 
 
 class EscalationPolicyCreate(BaseModel):

app/core/services/escalation_service.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import logging
 import uuid
 from collections.abc import Sequence
 from datetime import datetime
@@ -9,11 +10,13 @@
 from sqlalchemy.orm import selectinload
 
 from app.core.models.escalation import EscalationPolicy, EscalationStep
-from app.core.models.notification_channel import NotificationChannel
+from app.core.models.notification_channel import ChannelType, NotificationChannel
 from app.core.models.problem_event import ProblemEvent
 from app.core.schemas.escalation import EscalationPolicyCreate
 from app.tasks.notifications.events import EscalationAlertEvent
 
+logger = logging.getLogger(__name__)
+
 
 class EscalationService:
     def __init__(self, session: AsyncSession) -> None:
@@ -44,6 +47,7 @@ async def create(self, owner_id: uuid.UUID, data: EscalationPolicyCreate) -> Esc
                     step_order=step.step_order,
                     delay_minutes=step.delay_minutes,
                     channel_id=step.channel_id,
+                    action_command=step.action_command,
                 )
             )
         self._session.add(policy)
@@ -62,6 +66,7 @@ async def replace(
                     step_order=step.step_order,
                     delay_minutes=step.delay_minutes,
                     channel_id=step.channel_id,
+                    action_command=step.action_command,
                 )
             )
         await self._session.commit()
@@ -88,6 +93,18 @@ async def channels_owned_by(
         )
         return set((await self._session.execute(stmt)).scalars().all())
 
+    async def channel_types(
+        self, owner_id: uuid.UUID, channel_ids: set[uuid.UUID]
+    ) -> dict[uuid.UUID, ChannelType]:
+        """Map the owner's channels (among `channel_ids`) to their type."""
+        if not channel_ids:
+            return {}
+        stmt = select(NotificationChannel.id, NotificationChannel.type).where(
+            NotificationChannel.owner_id == owner_id,
+            NotificationChannel.id.in_(channel_ids),
+        )
+        return {cid: ctype for cid, ctype in (await self._session.execute(stmt))}
+
     async def _enabled_policy(self, owner_id: uuid.UUID) -> EscalationPolicy | None:
         stmt = (
             select(EscalationPolicy)
@@ -132,21 +149,28 @@ async def due_escalations(
                     break  # later steps have larger delays
                 channel = await self._session.get(NotificationChannel, step.channel_id)
                 if channel is not None and channel.is_enabled:
-                    deliveries.append(
-                        (
-                            EscalationAlertEvent(
-                                problem_id=problem.id,
-                                subject=problem.subject,
-                                trigger_name=problem.trigger_name,
-                                severity=problem.severity,
-                                step_order=step.step_order,
-                                value=problem.value,
-                                started_at=problem.started_at,
-                                timestamp=now,
-                            ),
-                            channel,
+                    # Auto-remediation must reach a machine endpoint, never an inbox.
+                    if step.action_command is not None and channel.type != ChannelType.WEBHOOK:
+                        logger.warning(
+                            "remediation step %s targets a non-webhook channel; skipping", step.id
+                        )
+                    else:
+                        deliveries.append(
+                            (
+                                EscalationAlertEvent(
+                                    problem_id=problem.id,
+                                    subject=problem.subject,
+                                    trigger_name=problem.trigger_name,
+                                    severity=problem.severity,
+                                    step_order=step.step_order,
+                                    value=problem.value,
+                                    started_at=problem.started_at,
+                                    timestamp=now,
+                                    action_command=step.action_command,
+                                ),
+                                channel,
+                            )
                         )
-                    )
                 problem.escalated_step = step.step_order
         await self._session.commit()
         return deliveries

app/tasks/notifications/events.py

@@ -147,14 +147,20 @@ class EscalationAlertEvent:
     value: float | None
     started_at: datetime
     timestamp: datetime
+    # When set, this step is an auto-remediation action (not a plain notification).
+    action_command: str | None = None
 
     @property
     def is_recovery(self) -> bool:
         return False
 
+    @property
+    def is_remediation(self) -> bool:
+        return self.action_command is not None
+
     def payload(self) -> dict[str, Any]:
-        return {
-            "event": "escalation",
+        body: dict[str, Any] = {
+            "event": "remediation" if self.is_remediation else "escalation",
             "severity": self.severity.value,
             "step": self.step_order,
             "subject": self.subject,
@@ -163,3 +169,6 @@ def payload(self) -> dict[str, Any]:
             "started_at": self.started_at.isoformat(),
             "timestamp": self.timestamp.isoformat(),
         }
+        if self.action_command is not None:
+            body["action"] = {"command": self.action_command}
+        return body

docs/roadmap.md

@@ -114,8 +114,11 @@ GhostMonitor's reason to exist over a plain Zabbix clone is privacy (ghost-suite
 - ✅ **Hosts overview**: the host list shows each host's health (item count, ongoing
   problem count and worst severity), problem hosts sorted first — aggregated in two
   grouped queries (no N+1).
-- *(next)* long-range charts backed by trends; auto-remediation hooks as an
-  escalation action.
+- ✅ **Auto-remediation actions**: an escalation step can carry an `action_command`,
+  making it POST a structured remediation intent (command + problem context) to a
+  **webhook** channel for an external runbook — the server never runs commands itself
+  (webhook-only, validated on create and at fire time).
+- *(next)* long-range charts backed by trends.
 
 ## Phase 5 — Discovery & scale
 

migrations/versions/e2a6636ea1da_remediation_step_command.py

@@ -0,0 +1,31 @@
+"""remediation step command
+
+Revision ID: e2a6636ea1da
+Revises: 0414a77d051d
+Create Date: 2026-06-30 21:44:14.940544
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "e2a6636ea1da"
+down_revision: str | Sequence[str] | None = "0414a77d051d"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "escalation_steps", sa.Column("action_command", sa.String(length=500), nullable=True)
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("escalation_steps", "action_command")
+    # ### end Alembic commands ###

templates/escalation/list.html

@@ -28,7 +28,7 @@ <h2>Policies <span class="count">{{ policies|length }}</span></h2>
             <td>{{ p.name }}</td>
             <td class="muted">{{ 'yes' if p.is_enabled else 'no' }}</td>
             <td class="muted">
-              {% for s in p.steps %}{{ s.delay_minutes }}m → {{ channel_names.get(s.channel_id, '?') }}{% if not loop.last %} · {% endif %}{% endfor %}
+              {% for s in p.steps %}{{ s.delay_minutes }}m → {{ channel_names.get(s.channel_id, '?') }}{% if s.action_command %} <span title="auto-remediation: {{ s.action_command }}">⚙</span>{% endif %}{% if not loop.last %} · {% endif %}{% endfor %}
             </td>
             <td class="row-actions">
               <form method="post" action="/escalation/{{ p.id }}/delete" class="form-inline"
@@ -59,7 +59,7 @@ <h2>New policy</h2>
             <span>Enabled</span>
           </label>
         </div>
-        <p class="muted">Each step notifies a channel after the delay (in minutes) from when the problem opened. Leave a channel empty to skip the row.</p>
+        <p class="muted">Each step notifies a channel after the delay (in minutes) from when the problem opened. Leave a channel empty to skip the row. Set an <strong>action command</strong> to make a step auto-remediation: it POSTs the command + problem context to a <strong>webhook</strong> channel for an external runbook (the server never runs commands itself).</p>
         {% for i in step_rows %}
         <div class="field-row">
           <label class="field">
@@ -73,6 +73,10 @@ <h2>New policy</h2>
               {% for c in channels %}<option value="{{ c.id }}">{{ c.name }} ({{ c.type.value }})</option>{% endfor %}
             </select>
           </label>
+          <label class="field">
+            <span>Action command (optional — webhook only)</span>
+            <input type="text" name="action_command" maxlength="500" placeholder="e.g. restart nginx">
+          </label>
         </div>
         {% endfor %}
         <div class="form-actions">