feat: replace hourly cleanup cron with S3 lifecycle policy set at startup

Author: bendersej

agents/src/main.rs

@@ -30,17 +30,7 @@ async fn main() {
         config,
     });
 
-    tokio::spawn({
-        let state = Arc::clone(&state);
-        async move {
-            let mut interval = tokio::time::interval(std::time::Duration::from_secs(60 * 60));
-            interval.tick().await;
-            loop {
-                interval.tick().await;
-                state.storage.cleanup_expired().await;
-            }
-        }
-    });
+    state.storage.ensure_lifecycle_policy().await;
 
     let app = Router::new()
         .merge(routes::router())

agents/src/storage.rs

@@ -8,7 +8,7 @@ use crate::config::Config;
 use crate::error::AppError;
 
 const PRESIGN_EXPIRY: Duration = Duration::from_secs(24 * 60 * 60);
-const CLEANUP_MAX_AGE: Duration = Duration::from_secs(24 * 60 * 60);
+const LIFECYCLE_EXPIRY_DAYS: i32 = 1;
 
 pub struct Storage {
     client: Client,
@@ -83,73 +83,41 @@ impl Storage {
         Ok(UploadResult { presigned_url })
     }
 
-    pub async fn cleanup_expired(&self) {
-        let cutoff = aws_sdk_s3::primitives::DateTime::from_secs_f64(
-            std::time::SystemTime::now()
-                .duration_since(std::time::UNIX_EPOCH)
-                .unwrap_or_default()
-                .as_secs_f64()
-                - CLEANUP_MAX_AGE.as_secs_f64(),
-        );
-
-        let mut continuation_token: Option<String> = None;
-
-        loop {
-            let mut request = self
-                .client
-                .list_objects_v2()
-                .bucket(&self.bucket)
-                .prefix("uploads/");
-
-            if let Some(token) = &continuation_token {
-                request = request.continuation_token(token);
-            }
-
-            let response = match request.send().await {
-                Ok(response) => response,
-                Err(e) => {
-                    tracing::error!("cleanup: failed to list objects: {e}");
-                    return;
-                }
-            };
-
-            let mut deleted = 0;
-            for object in response.contents() {
-                let is_expired = object
-                    .last_modified()
-                    .map(|modified| *modified < cutoff)
-                    .unwrap_or(false);
-
-                if !is_expired {
-                    continue;
-                }
-
-                let Some(key) = object.key() else {
-                    continue;
-                };
-
-                if let Err(e) = self
-                    .client
-                    .delete_object()
-                    .bucket(&self.bucket)
-                    .key(key)
-                    .send()
-                    .await
-                {
-                    tracing::error!("cleanup: failed to delete {key}: {e}");
-                } else {
-                    deleted += 1;
-                }
-            }
-
-            if deleted > 0 {
-                tracing::info!("cleanup: deleted {deleted} expired files");
-            }
-
-            match response.next_continuation_token() {
-                Some(token) => continuation_token = Some(token.to_string()),
-                None => break,
-            }
+    pub async fn ensure_lifecycle_policy(&self) {
+        use aws_sdk_s3::types::{
+            BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule,
+            LifecycleRuleFilter,
+        };
+
+        let rule = LifecycleRule::builder()
+            .id("expire-uploads-24h")
+            .status(ExpirationStatus::Enabled)
+            .filter(LifecycleRuleFilter::builder().prefix("uploads/").build())
+            .expiration(
+                LifecycleExpiration::builder()
+                    .days(LIFECYCLE_EXPIRY_DAYS)
+                    .build(),
+            )
+            .build()
+            .expect("valid lifecycle rule");
+
+        let config = BucketLifecycleConfiguration::builder()
+            .rules(rule)
+            .build()
+            .expect("valid lifecycle config");
+
+        match self
+            .client
+            .put_bucket_lifecycle_configuration()
+            .bucket(&self.bucket)
+            .lifecycle_configuration(config)
+            .send()
+            .await
+        {
+            Ok(_) => tracing::info!(
+                "lifecycle policy set: uploads/ expire after {LIFECYCLE_EXPIRY_DAYS} day(s)"
+            ),
+            Err(e) => tracing::error!("failed to set lifecycle policy: {e}"),
         }
     }
 }