test(k8s): add container tests for kubernetes

Author: jeboehm

deploy/kustomize/mda/service.yaml

@@ -37,6 +37,10 @@ spec:
       port: 4190
       protocol: TCP
       targetPort: managesieve
+    - name: doveadm
+      port: 8080
+      protocol: TCP
+      targetPort: doveadm
   selector:
     app.kubernetes.io/name: mda
     app.kubernetes.io/component: mail

deploy/kustomize/mda/statefulset.yaml

@@ -66,6 +66,9 @@ spec:
             - name: managesieve
               containerPort: 4190
               protocol: TCP
+            - name: doveadm
+              containerPort: 8080
+              protocol: TCP
           livenessProbe:
             exec:
               command:

test/bats/Dockerfile

@@ -10,6 +10,7 @@ ENV MYSQL_HOST=db \
     MYSQL_DATABASE=mailserver \
     FILTER_WEB_ADDRESS=filter:11334 \
     IS_KUBERNETES=0 \
+    KUBECTL_CONFIG=/tmp/kubeconfig \
     MDA_POP3_ADDRESS=mda:31110 \
     MDA_IMAP_ADDRESS=mda:31143 \
     MDA_POP3S_ADDRESS=mda:31995 \
@@ -56,8 +57,7 @@ RUN apk --no-cache add \
 RUN apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ swaks=20240103.0-r0
 
 COPY --from=dockerize /bin/dockerize /usr/local/bin/dockerize
-COPY _helpers/dockerlogs.sh /usr/local/bin/dockerlogs.sh
-COPY _helpers/entrypoint.sh /entrypoint.sh
+COPY rootfs/ /
 
 WORKDIR /app/tests
 COPY integration/ /app/tests/

test/bats/integration/075_kubernetes.bats

@@ -0,0 +1,18 @@
+#!/usr/bin/env bats
+
+setup() {
+	load '_helper'
+	skip_in_non_kubernetes
+}
+
+@test "Moved mail was sent to rspamd and learned successfully" {
+	run bash -c "kubectl logs statefulset/filter | grep 'learned message as spam:' | grep rspamd_controller_learn_fin_task"
+
+	[ "$status" -eq 0 ]
+}
+
+@test "system:check command succeeds" {
+	run kubectl exec deploy/web -c web -- /opt/admin/bin/console system:check --all
+
+	[ "$status" -eq 0 ]
+}

test/bats/integration/_helper.bash

@@ -6,6 +6,12 @@ skip_in_kubernetes() {
 	fi
 }
 
+skip_in_non_kubernetes() {
+	if [ "${IS_KUBERNETES}" -ne "1" ]; then
+		skip "Skipping test in non-Kubernetes"
+	fi
+}
+
 # Split a string by colon (:) character and return both parts
 # Usage: split_by_colon "hostname:8080"
 # Returns: Two lines - first part on line 1, second part on line 2

test/bats/_helpers/entrypoint.sh test/bats/rootfs/entrypoint.shtest/bats/_helpers/entrypoint.sh renamed to test/bats/rootfs/entrypoint.sh

@@ -1,5 +1,9 @@
 #!/bin/sh
 
+if [ "${IS_KUBERNETES}" -eq "1" ]; then
+	/usr/local/bin/kubectl.sh
+fi
+
 exec dockerize \
 	-wait "tcp://${MYSQL_HOST}:${MYSQL_PORT}" \
 	-wait "tcp://${FILTER_WEB_ADDRESS}" \

test/bats/_helpers/dockerlogs.sh …/bats/rootfs/usr/local/bin/dockerlogs.shtest/bats/_helpers/dockerlogs.sh renamed to test/bats/rootfs/usr/local/bin/dockerlogs.sh test/bats/_helpers/dockerlogs.sh renamed to test/bats/rootfs/usr/local/bin/dockerlogs.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+SA_DIR="/var/run/secrets/kubernetes.io/serviceaccount"
+APISERVER="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"
+TOKEN="$(cat ${SA_DIR}/token)"
+CACERT="${SA_DIR}/ca.crt"
+NAMESPACE="$(cat ${SA_DIR}/namespace)"
+
+cat >/tmp/kubeconfig <<EOF
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    certificate-authority: ${CACERT}
+    server: ${APISERVER}
+  name: in-cluster
+contexts:
+- context:
+    cluster: in-cluster
+    namespace: ${NAMESPACE}
+    user: sa
+  name: sa@in-cluster
+current-context: sa@in-cluster
+users:
+- name: sa
+  user:
+    token: ${TOKEN}
+EOF

test/bats/rootfs/usr/local/bin/kubectl.sh

@@ -16,7 +16,8 @@ spec:
         app.kubernetes.io/component: test
         app.kubernetes.io/part-of: docker-mailserver
     spec:
-      automountServiceAccountToken: false
+      automountServiceAccountToken: true
+      serviceAccountName: test-runner-sa
       restartPolicy: Never
       initContainers:
         - name: load-fixtures
@@ -28,7 +29,7 @@ spec:
           args:
             - |
               /opt/admin/bin/console system:check --wait
-              /opt/admin/bin/console domain:add example.com
+              /opt/admin/bin/console domain:add example.com || exit 0
               /opt/admin/bin/console domain:add example.org
               /opt/admin/bin/console user:add --admin --password=changeme --enable admin example.com
               /opt/admin/bin/console user:add --password=test1234 --enable --sendonly sendonly example.com
@@ -109,3 +110,29 @@ spec:
         - name: vmail
           persistentVolumeClaim:
             claimName: mda-data-mda-0
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: test-runner-sa
+  labels:
+    app.kubernetes.io/name: test-runner-sa
+    app.kubernetes.io/component: test
+    app.kubernetes.io/part-of: docker-mailserver
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: test-runner-cluster-admin
+  labels:
+    app.kubernetes.io/name: test-runner-cluster-admin
+    app.kubernetes.io/component: test
+    app.kubernetes.io/part-of: docker-mailserver
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - kind: ServiceAccount
+    name: test-runner-sa
+    namespace: default