WPB-26489 backend adminless scheduled jobs model for deletion and reminder

cabal.project

@@ -1,6 +1,7 @@
 repository hackage.haskell.org
   url: https://hackage.haskell.org/
 index-state: 2023-10-03T15:17:00Z
+
 packages:
     integration
   , libs/bilge/
@@ -96,3 +97,14 @@ package wire-message-proto-lens
   flags: +nix-dev-env
 package types-common-journal
   flags: +nix-dev-env
+
+source-repository-package
+  type: git
+  location: https://github.com/velveteer/arbiter.git
+  tag: 96097411a0480e182d0cbce819c45023fe8aeec7
+  subdir:
+    arbiter-core
+    arbiter-hasql
+    arbiter-migrations
+    arbiter-worker
+    arbiter-test-common

changelog.d/0-release-notes/WPB-26489

@@ -0,0 +1 @@
+Background-worker scheduled jobs now run through Arbiter. The background-worker configuration includes `scheduledJobs.pollInterval`, which controls how often the scheduler wakes up to look for due jobs.

changelog.d/2-features/WPB-26489

@@ -0,0 +1 @@
+Introduce Arbiter-backed schedulable background jobs, migrate meetings cleanup to the new job runner, and add the initial adminless reminder and deletion jobs.

charts/wire-server/templates/background-worker/configmap.yaml

@@ -93,6 +93,10 @@ data:
     backgroundJobs:
 {{ toYaml . | indent 6 }}
     {{- end }}
+    scheduledJobs:
+      # Arbiter dispatcher poll interval for all scheduled jobs.
+      # Lower values reduce discovery latency, but increase DB polling.
+      pollInterval: {{ .scheduledJobs.pollInterval }}
     {{- with .meetingsCleanup }}
     meetingsCleanup:
 {{ toYaml . | indent 6 }}

charts/wire-server/values.yaml

@@ -1002,6 +1002,12 @@ background-worker:
       # Total attempts, including the first try
       maxAttempts: 3
 
+    # Scheduled-jobs dispatcher configuration.
+    scheduledJobs:
+      # Arbiter dispatcher poll interval for all scheduled jobs.
+      # Lower values reduce discovery latency, but increase DB polling.
+      pollInterval: 5s
+
     # Meetings cleanup configuration
     meetingsCleanup:
       # Delete meetings older than this many hours (48 hours = 2 days)

docs/src/developer/reference/config-options.md

@@ -2235,3 +2235,4 @@ Notes
 - The `migrate...` flags control the corresponding PostgreSQL backfill jobs for the current migration settings; leave them `false` for new installs and after migration.
 - `concurrency`, `jobTimeout`, and `maxAttempts` control parallelism and retry behavior of the consumer.
 - `brig` and `gundeck` endpoints default to in-cluster services; override via `background-worker.config.brig` and `.gundeck` if your service DNS/ports differ.
+- `scheduledJobs.pollInterval` controls how often the background worker wakes up to check Arbiter for due jobs.

flake.nix

@@ -108,6 +108,11 @@
       url = "github:wireapp/hasql-resource-pool?rev=5b5d3df0fff81801986a0110acae5420215f01c5";
       flake = false;
     };
+
+    arbiter = {
+      url = "github:velveteer/arbiter?rev=96097411a0480e182d0cbce819c45023fe8aeec7";
+      flake = false;
+    };
   };
 
   outputs = inputs@{ nixpkgs, nixpkgs_24_11, flake-utils, tom-bombadil, sbomnix, ... }:

integration/test/Notifications.hs

@@ -205,6 +205,9 @@ isConvCreateNotifNotSelf n =
 isConvDeleteNotif :: (HasCallStack, MakesValue a) => a -> App Bool
 isConvDeleteNotif n = fieldEquals n "payload.0.type" "conversation.delete"
 
+isConvAdminlessReminderNotif :: (HasCallStack, MakesValue a) => a -> App Bool
+isConvAdminlessReminderNotif n = fieldEquals n "payload.0.type" "conversation.adminless-reminder"
+
 notifTypeIsEqual :: (HasCallStack, MakesValue a) => String -> a -> App Bool
 notifTypeIsEqual typ n = nPayload n %. "type" `isEqual` typ
 

integration/test/Test/AdminlessGroups.hs

@@ -20,7 +20,9 @@ module Test.AdminlessGroups where
 import API.Brig
 import API.Galley
 import API.GalleyInternal hiding (getConversation)
+import qualified API.GalleyInternal as GalleyI
 import MLS.Util
+import Notifications
 import SetupHelpers hiding (deleteUser)
 import Testlib.Prelude
 
@@ -100,19 +102,50 @@ testOnLastAdminLeaveNoEligibleMembersExist = do
   (alice, tid, _) <- createTeam OwnDomain 1
 
   setTeamFeatureLockStatus alice tid "preventAdminlessGroups" "unlocked"
-  patchTeamFeature OwnDomain tid "preventAdminlessGroups" (object ["status" .= "enabled"]) >>= assertSuccess
+  patchTeamFeature
+    OwnDomain
+    tid
+    "preventAdminlessGroups"
+    ( object
+        [ "status" .= "enabled",
+          "config"
+            .= object
+              -- The reminders are due early (+1s and +2s), while deletion is
+              -- later (+10s). This gives Arbiter's 1s polling and serial
+              -- grouped-job processing enough room to emit both reminders
+              -- before the conversation is deleted.
+              [ "deletionTimeoutDuration" .= "10s",
+                "reminderTimeoutDurations" .= ["9s", "8s"],
+                "promotionStrategy" .= "random"
+              ]
+        ]
+    )
+    >>= assertSuccess
+
+  let newApp :: NewApp
+      newApp = def {name = "adminless-reminder-app", description = "non-eligible reminder recipient"}
+  app <- bindResponse (createApp alice tid newApp) $ \resp -> do
+    resp.status `shouldMatchInt` 200
+    resp.json %. "user"
 
-  alice1 <- createMLSClient def alice
-  void $ uploadNewKeyPackage def alice1
+  [alice1, app1] <- traverse (createMLSClient def) [alice, app]
+  traverse_ (uploadNewKeyPackage def) [alice1, app1]
 
   conv <- postConversation alice defMLS {team = Just tid} >>= getJSON 201
   convId <- objConvId conv
   createGroup def alice1 convId
-  void $ createAddCommit alice1 convId [] >>= sendAndConsumeCommitBundle
+  void $ createAddCommit alice1 convId [app] >>= sendAndConsumeCommitBundle
 
-  -- alice leaves the conversation, no error, group will be marked for deletion
-  bindResponse (removeMember alice conv alice) $ \resp -> do
-    resp.status `shouldMatchInt` 200
+  withWebSockets [app] $ \[wsApp] -> do
+    -- alice leaves the conversation, no error, group will be marked for deletion
+    bindResponse (removeMember alice conv alice) $ \resp -> do
+      resp.status `shouldMatchInt` 200
+
+    void $ awaitNMatches 2 isConvAdminlessReminderNotif wsApp
+
+    retryT $ do
+      bindResponse (GalleyI.getConversation conv) $ \resp -> do
+        resp.status `shouldMatchInt` 404
 
 testOnLastAdminLeaveFeatureDisabled :: (HasCallStack) => App ()
 testOnLastAdminLeaveFeatureDisabled = do

libs/extended/src/Hasql/Pool/Extended.hs

@@ -18,8 +18,9 @@
 module Hasql.Pool.Extended where
 
 import Data.Aeson
-import Data.Map as Map
+import Data.Map qualified as Map
 import Data.Misc
+import Data.Secret (SecretText, secretText)
 import Hasql.Connection qualified
 import Hasql.Connection.Settings qualified as HasqlConnSettings
 import Hasql.Pool qualified as HasqlPool
@@ -47,6 +48,17 @@ instance FromJSON PoolConfig where
       <*> o .: "acquisitionTimeout"
       <*> o .: "idlenessTimeout"
 
+-- | Render a PostgreSQL connection string in libpq key-value format.
+--
+-- Passwords from the optional secret file are inserted into the key-value map
+-- before rendering. The result is wrapped because it may contain the password.
+postgresqlConnectionStringWithPassword :: Map Text Text -> Maybe FilePathSecrets -> IO SecretText
+postgresqlConnectionStringWithPassword pgConfig mFpSecrets = do
+  mPw <- for mFpSecrets initCredentials
+  let pgConfig' = maybe pgConfig (\pw -> Map.insert "password" pw pgConfig) mPw
+  pure . secretText . PostgresqlConnectionString.toKeyValueString $
+    PostgresqlConnectionString.fromKeyValueParams pgConfig'
+
 data HasqlPoolMetrics = HasqlPoolMetrics
   { readyForUseGauge :: Gauge,
     inUseGauge :: Gauge,

libs/types-common/src/Data/Secret.hs

@@ -0,0 +1,43 @@
+-- This file is part of the Wire Server implementation.
+--
+-- Copyright (C) 2026 Wire Swiss GmbH <[email protected]>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option)
+-- any later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
+module Data.Secret
+  ( SecretText,
+    secretText,
+    revealSecretText,
+  )
+where
+
+import Imports
+
+-- | Text that may contain credentials or other sensitive material.
+--
+-- The constructor is intentionally opaque. 'revealSecretText' should only be
+-- used at the narrow boundary where an external API requires the plaintext
+-- representation.
+newtype SecretText = SecretText Text
+
+instance Show SecretText where
+  show _ = "<redacted>"
+
+-- | Wrap sensitive text without exposing it through the public constructor.
+secretText :: Text -> SecretText
+secretText = SecretText
+
+-- | Reveal sensitive text for an API that explicitly requires plaintext.
+revealSecretText :: SecretText -> Text
+revealSecretText (SecretText value) = value

libs/types-common/test/Main.hs

@@ -23,6 +23,7 @@ where
 import Imports
 import Test.Data.Mailbox qualified as Mailbox
 import Test.Data.PEMKeys qualified as PEMKeys
+import Test.Data.Secret qualified as Secret
 import Test.Domain qualified as Domain
 import Test.Handle qualified as Handle
 import Test.Properties qualified as Properties
@@ -41,5 +42,6 @@ main =
         Handle.tests,
         Qualified.tests,
         PEMKeys.tests,
-        Mailbox.tests
+        Mailbox.tests,
+        Secret.tests
       ]

libs/types-common/test/Test/Data/Secret.hs

@@ -0,0 +1,31 @@
+-- This file is part of the Wire Server implementation.
+--
+-- Copyright (C) 2026 Wire Swiss GmbH <[email protected]>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option)
+-- any later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
+module Test.Data.Secret (tests) where
+
+import Data.Secret
+import Imports
+import Test.Tasty
+import Test.Tasty.HUnit
+
+tests :: TestTree
+tests =
+  testGroup
+    "SecretText"
+    [ testCase "does not expose its value through Show" $
+        show (secretText "database-password") @?= "<redacted>"
+    ]

libs/types-common/types-common.cabal

@@ -31,6 +31,7 @@ library
     Data.Qualified
     Data.Range
     Data.RetryAfter
+    Data.Secret
     Data.SizedHashMap
     Data.Text.Ascii
     Data.UUID.Tagged
@@ -160,6 +161,7 @@ test-suite types-common-tests
     Paths_types_common
     Test.Data.Mailbox
     Test.Data.PEMKeys
+    Test.Data.Secret
     Test.Domain
     Test.Handle
     Test.Properties