Feat/obs

.env.dist

@@ -17,7 +17,7 @@ MAILNAME=mail.example.com
 [email protected]
 
 # Relay mails to another SMTP server
-# @see docs/RELAYHOST.md
+# @see docs/configuration/relayhost.md
 RELAYHOST=false
 RELAY_PASSWD_FILE=false
 
@@ -27,11 +27,14 @@ FILTER_MIME=false
 # Password to access the rspamd web-interface
 CONTROLLER_PASSWORD=changeme2
 
+# Password to access the dovecot doveadm api
+DOVEADM_API_KEY=changeme2
+
 # How long to wait for services to start
 WAITSTART_TIMEOUT=2m
 
 # Configure local address extension
-# @see docs/LOCAL_ADDRESS_EXTENSION.md
+# @see docs/configuration/local-address-extension.md
 RECIPIENT_DELIMITER=-
 
 # Interval in seconds for fetchmail to check for new mails

.mkdocs.yaml

@@ -40,6 +40,8 @@ nav:
       - administration/dns-validation-wizard.md
       - administration/user-roles.md
       - administration/ios-macos-profile.md
+  - Observability:
+      - observability/intro.md
   - Recipes:
       - Docker:
           - Traefik Reverse Proxy: https://github.com/jeboehm/docker-mailserver/tree/main/docs/example-configs/compose/traefik-reverse-proxy

deploy/kustomize/common/configmap.yaml

@@ -6,6 +6,7 @@ data:
   FILTER_MILTER_ADDRESS: filter:11332
   FILTER_WEB_ADDRESS: filter:11334
   MDA_AUTH_ADDRESS: mda:2004
+  MDA_DOVEADM_ADDRESS: mda:8080
   MDA_IMAP_ADDRESS: mda:143
   MDA_IMAPS_ADDRESS: mda:993
   MDA_LMTP_ADDRESS: mda:2003

deploy/kustomize/mda/network-policy.yaml

@@ -33,6 +33,8 @@ spec:
           port: 2004 # Auth
         - protocol: TCP
           port: 4190 # ManageSieve
+        - protocol: TCP
+          port: 8080 # Doveadm
     # Allow external access (from internet)
     - from:
         - ipBlock:

deploy/kustomize/mda/service.yaml

@@ -37,6 +37,10 @@ spec:
       port: 4190
       protocol: TCP
       targetPort: managesieve
+    - name: doveadm
+      port: 8080
+      protocol: TCP
+      targetPort: doveadm
   selector:
     app.kubernetes.io/name: mda
     app.kubernetes.io/component: mail

deploy/kustomize/mda/statefulset.yaml

@@ -66,6 +66,9 @@ spec:
             - name: managesieve
               containerPort: 4190
               protocol: TCP
+            - name: doveadm
+              containerPort: 8080
+              protocol: TCP
           livenessProbe:
             exec:
               command:

deploy/kustomize/web/deployment.yaml

@@ -72,7 +72,7 @@ spec:
           resources:
             requests:
               cpu: 50m
-              memory: 128Mi
+              memory: 196Mi
           securityContext:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true

docs/configuration/environment-variables.md

@@ -40,11 +40,10 @@ It is required to **always set** `REDIS_PASSWORD`.
 
 ### Authentication
 
-This is the password for the RSpamd controller access. It is required to **always set** `CONTROLLER_PASSWORD`.
-
 | Variable              | Default      | Description                           |
 | --------------------- | ------------ | ------------------------------------- |
 | `CONTROLLER_PASSWORD` | _(required)_ | Password for RSpamd controller access |
+| `DOVEADM_API_KEY`     | _(required)_ | Password for Dovecot API access       |
 
 ### Relay Configuration
 

docs/observability/intro.md

@@ -0,0 +1,45 @@
+# Observability
+
+Observability is the ability to understand a system's internal state by examining its external outputs, crucial for complex, modern software systems.
+
+`mailserver-admin` provides observability features for monitoring the mailserver components, enabling administrators to monitor service health, message processing statistics, and operational metrics through web-based dashboards.
+
+## Overview
+
+The observability features integrate with the Rspamd filter service and Dovecot mail delivery agent to collect and display real-time statistics and historical trends. These dashboards provide visibility into message processing, spam filtering effectiveness, authentication patterns, and mail delivery rates.
+
+## Rspamd Statistics
+
+The Rspamd Statistics dashboard displays metrics from the Rspamd controller service, providing insights into spam filtering operations and message processing.
+
+![Rspamd Statistics](../images/admin/obs_rspamd.png)
+
+The dashboard includes:
+
+- **Service Status**: Health check indicator showing Rspamd controller availability and response time
+- **Summary Metrics**: Key performance indicators including:
+  - Total messages scanned
+  - Spam messages detected
+  - Ham (clean) messages identified
+  - Messages used for machine learning
+  - Active connections to the service
+- **Throughput Chart**: Time-series visualization of message processing rates per minute, showing action distributions including rejections, soft rejections, subject rewrites, header additions, greylisting, and no-action messages. Supports time aggregation for day, week, month, and year views
+- **Action Distribution**: Donut chart showing the proportion of messages categorised by Rspamd actions (clean, rejected, temporarily rejected, probable spam, greylisted)
+- **Configuration Details**: Displays action thresholds and top symbols used in spam scoring
+
+## Dovecot Statistics
+
+The Dovecot Statistics dashboard presents metrics from the Dovecot mail delivery agent via the Doveadm HTTP API, focusing on authentication and mail delivery operations.
+
+![Dovecot Statistics](../images/admin/obs_dovecot.png)
+
+The dashboard provides:
+
+- **API Status**: Connection status and authentication verification for the Doveadm HTTP API, including last update timestamp
+- **Summary Metrics**: Overview cards showing:
+  - Authentication successes
+  - Authentication failures
+  - Index operations count
+- **Authentication Rate Chart**: Time-series graph displaying authentication success and failure rates per minute, enabling identification of authentication patterns and potential security issues
+- **Mail Deliveries Chart**: Time-series visualization of mail delivery rates per minute, showing message throughput to user mailboxes
+- **Data Collection**: Charts are calculated from consecutive samples, with rates computed as deltas between measurements

docs/upgrade.md

@@ -4,6 +4,16 @@ Upgrade guide for docker-mailserver.
 
 When upgrading, ensure that container configuration files are updated to match the requirements of the new version. Review the manifests in `deploy/compose` and `deploy/kustomize` for any changes to persistent volumes or configuration, and update them as necessary. If new environment variables have been introduced, update the `.env` file accordingly.
 
+## v7.3
+
+`mailserver-admin` now provides observability features for the mailserver. To make sure the service to service communication works, the
+following environment variables have been added:
+
+- `MDA_DOVEADM_ADDRESS`: The address of the MDA service for Dovecot API access. (default: `mda:8080`)
+- `DOVEADM_API_KEY`: The API key for Dovecot API access.
+
+Please make sure to change at least `DOVEADM_API_KEY`.
+
 ## v7.1
 
 - **web**: mailserver-admin now includes a feature to generate mobileconfig files for iOS and macOS. These files are signed with the same TLS certificate that the mailserver displays to connecting clients. To generate these files, it’s necessary to mount the certificate to the web container.

target/mda/rootfs/etc/dovecot/conf.d/10-common.conf

@@ -2,9 +2,11 @@ import_environment {
   POSTMASTER = %{env:POSTMASTER}
   MAILNAME = %{env:MAILNAME}
   MTA_SMTP_ADDRESS = %{env:MTA_SMTP_ADDRESS}
+  DOVEADM_API_KEY = %{env:DOVEADM_API_KEY}
 }
 
 protocols = imap lmtp sieve pop3
 postmaster_address = $ENV:POSTMASTER
 hostname = $ENV:MAILNAME
 submission_host = $ENV:MTA_SMTP_ADDRESS
+doveadm_api_key = $ENV:DOVEADM_API_KEY

target/web/Dockerfile

@@ -10,15 +10,20 @@ ENV MYSQL_HOST=db \
     MYSQL_USER=mailserver \
     REDIS_HOST=redis \
     REDIS_PORT=6379 \
+    APP_SHARE_DIR="var/cache" \
+    DATABASE_URL="mysql://%env(MYSQL_USER)%:%env(MYSQL_PASSWORD)%@%env(MYSQL_HOST)%:%env(MYSQL_PORT)%/%env(MYSQL_DATABASE)%?serverVersion=8.4" \
+    DOVEADM_HTTP_URL="https://%env(MDA_DOVEADM_ADDRESS)%" \
     FILTER_WEB_ADDRESS=filter:11334 \
+    MAILNAME=mail.example.com \
+    MDA_DOVEADM_ADDRESS=mda:8080 \
     MDA_IMAP_ADDRESS=mda:31143 \
     MDA_MANAGESIEVE_ADDRESS=mda:4190 \
     MTA_SMTP_SUBMISSION_ADDRESS=mta:587 \
+    RSPAMD_CONTROLLER_URL="http://%env(FILTER_WEB_ADDRESS)%" \
+    RSPAMD_PASSWORD="%env(CONTROLLER_PASSWORD)%" \
+    SERVER_ROOT=/var/www/html \
     SUPPORT_URL=https://github.com/jeboehm/docker-mailserver \
-    MAILNAME=mail.example.com \
-    WAITSTART_TIMEOUT=1m \
-    DATABASE_URL="mysql://%env(MYSQL_USER)%:%env(MYSQL_PASSWORD)%@%env(MYSQL_HOST)%:%env(MYSQL_PORT)%/%env(MYSQL_DATABASE)%?serverVersion=8.4" \
-    SERVER_ROOT=/var/www/html
+    WAITSTART_TIMEOUT=1m
 
 ARG FRANKENPHP_VER=v1.11.1 # renovate: depName=php/frankenphp
 RUN apk add --no-cache \
@@ -74,17 +79,15 @@ RUN --mount=type=cache,target=/root/.composer \
 
 FROM composer AS admin-builder
 
-ARG ADMIN_VER=5.13.4 # renovate: depName=jeboehm/mailserver-admin
+ARG ADMIN_VER=5.14.1 # renovate: depName=jeboehm/mailserver-admin
 WORKDIR /opt/admin
 RUN curl -sSLf \
         -o /tmp/admin.tar.gz \
         https://github.com/jeboehm/mailserver-admin/releases/download/${ADMIN_VER}/release-${ADMIN_VER}.tar.gz && \
-    tar -oxf /tmp/admin.tar.gz --strip=1 && \
+    tar -oxf /tmp/admin.tar.gz && \
     rm /tmp/admin.tar.gz && \
-    composer symfony:dump-env prod && \
     chmod 777 \
         /opt/admin/var/cache \
-        /opt/admin/var/cache/prod \
         /opt/admin/var/log
 
 FROM base AS prod

test/bats/Dockerfile

@@ -1,16 +1,16 @@
 FROM ghcr.io/jeboehm/dockerize:0.9.3@sha256:d4e824aa120670658d7012421d2fdf1b2437be34a6acbb7a4ad92ed52edec8eb AS dockerize
-FROM alpine:3.22@sha256:4b7ce07002c69e8f3d704a9c5d6fd3053be500b7f1c69fc0d80990c2ad8dd412
+FROM alpine:3.23
 
 LABEL maintainer="https://github.com/jeboehm/docker-mailserver"
 LABEL vendor="https://github.com/jeboehm/docker-mailserver"
 
 ENV MYSQL_HOST=db \
     MYSQL_PORT=3306 \
     MYSQL_USER=root \
-    MYSQL_PASSWORD= \
     MYSQL_DATABASE=mailserver \
     FILTER_WEB_ADDRESS=filter:11334 \
     IS_KUBERNETES=0 \
+    KUBECTL_CONFIG=/tmp/kubeconfig \
     MDA_POP3_ADDRESS=mda:31110 \
     MDA_IMAP_ADDRESS=mda:31143 \
     MDA_POP3S_ADDRESS=mda:31995 \
@@ -33,31 +33,31 @@ RUN apk --no-cache add \
       curl \
       docker \
       jq \
+      kubectl \
       mariadb-client \
       mariadb-connector-c \
       openssl \
       perl \
       perl-net-ssleay \
-      php82 \
-      php82-dom \
-      php82-fileinfo \
-      php82-mbstring \
-      php82-iconv \
-      php82-phar \
-      php82-iconv \
-      php82-openssl \
-      php82-tokenizer \
+      php84 \
+      php84-dom \
+      php84-fileinfo \
+      php84-mbstring \
+      php84-iconv \
+      php84-phar \
+      php84-iconv \
+      php84-openssl \
+      php84-tokenizer \
       redis \
-    && ln -sf /usr/bin/php82 /usr/bin/php \
+    && ln -sf /usr/bin/php84 /usr/bin/php \
     && wget -q -O /usr/local/bin/imap-tester https://github.com/jeboehm/imap-tester/releases/download/${IMAPTESTER_VER}/imap-tester.phar \
     && chmod +x /usr/local/bin/imap-tester \
     && mkdir -p /usr/share/fixtures \
     && wget -q -O /usr/share/fixtures/gtube.txt https://spamassassin.apache.org/gtube/gtube.txt
 RUN apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ swaks=20240103.0-r0
 
 COPY --from=dockerize /bin/dockerize /usr/local/bin/dockerize
-COPY _helpers/dockerlogs.sh /usr/local/bin/dockerlogs.sh
-COPY _helpers/entrypoint.sh /entrypoint.sh
+COPY rootfs/ /
 
 WORKDIR /app/tests
 COPY integration/ /app/tests/

test/bats/integration/070_docker.bats

@@ -16,3 +16,9 @@ setup() {
 
 	[ "$status" -eq 0 ]
 }
+
+@test "system:check command succeeds" {
+	run docker exec docker-mailserver-web-1 /opt/admin/bin/console system:check --all
+
+	[ "$status" -eq 0 ]
+}

test/bats/integration/075_kubernetes.bats

@@ -0,0 +1,18 @@
+#!/usr/bin/env bats
+
+setup() {
+	load '_helper'
+	skip_in_non_kubernetes
+}
+
+@test "Moved mail was sent to rspamd and learned successfully" {
+	run bash -c "kubectl logs statefulset/filter | grep 'learned message as spam:' | grep rspamd_controller_learn_fin_task"
+
+	[ "$status" -eq 0 ]
+}
+
+@test "system:check command succeeds" {
+	run kubectl exec deploy/web -c web -- /opt/admin/bin/console system:check --all
+
+	[ "$status" -eq 0 ]
+}

test/bats/integration/_helper.bash

@@ -6,6 +6,12 @@ skip_in_kubernetes() {
 	fi
 }
 
+skip_in_non_kubernetes() {
+	if [ "${IS_KUBERNETES}" -ne "1" ]; then
+		skip "Skipping test in non-Kubernetes"
+	fi
+}
+
 # Split a string by colon (:) character and return both parts
 # Usage: split_by_colon "hostname:8080"
 # Returns: Two lines - first part on line 1, second part on line 2

test/bats/_helpers/entrypoint.sh → test/bats/rootfs/entrypoint.sh

@@ -1,5 +1,9 @@
 #!/bin/sh
 
+if [ "${IS_KUBERNETES}" -eq "1" ]; then
+	/usr/local/bin/kubectl.sh
+fi
+
 exec dockerize \
 	-wait "tcp://${MYSQL_HOST}:${MYSQL_PORT}" \
 	-wait "tcp://${FILTER_WEB_ADDRESS}" \

test/bats/rootfs/usr/local/bin/kubectl.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+SA_DIR="/var/run/secrets/kubernetes.io/serviceaccount"
+APISERVER="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"
+TOKEN="$(cat ${SA_DIR}/token)"
+CACERT="${SA_DIR}/ca.crt"
+NAMESPACE="$(cat ${SA_DIR}/namespace)"
+
+cat >/tmp/kubeconfig <<EOF
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    certificate-authority: ${CACERT}
+    server: ${APISERVER}
+  name: in-cluster
+contexts:
+- context:
+    cluster: in-cluster
+    namespace: ${NAMESPACE}
+    user: sa
+  name: sa@in-cluster
+current-context: sa@in-cluster
+users:
+- name: sa
+  user:
+    token: ${TOKEN}
+EOF