fix(logs): automated log retention and API page_size cap

docs/development.md

@@ -457,10 +457,34 @@ task docker:purge && EDA_SECRET_KEY=insecure task docker:up
 |----------|---------|
 | `EDA_SECRET_KEY` | Sets `settings.SECRET_KEY` via dynaconf. Used by Django for encryption at rest. |
 | `EDA_DB_ROTATION_KEY` | Provides the new key for `rotate_db_encryption_key --use-custom-key`. Operator-set, ad-hoc only. |
+| `EDA_LOG_RETENTION_DAYS` | Days of rulebook process logs to retain. An automated task purges older logs every hour. Default: `30`. Set to `0` to disable. |
+| `EDA_MAX_LOG_LINES_PER_INSTANCE` | Maximum log lines stored per activation instance. Oldest lines are trimmed when exceeded. Default: `500000`. Set to `0` to disable. |
 
 Do **not** confuse the two — see the command docstring in
 `src/aap_eda/core/management/commands/rotate_db_encryption_key.py`.
 
+### Log retention
+
+Rulebook process logs are automatically purged every hour based on `EDA_LOG_RETENTION_DAYS`.
+A per-instance safety valve (`EDA_MAX_LOG_LINES_PER_INSTANCE`) trims the oldest log lines
+when an individual activation instance exceeds the cap.
+
+For ad-hoc purging, use the management command:
+
+```shell
+# Purge logs older than LOG_RETENTION_DAYS (default 30)
+task manage -- purge_log_records
+
+# Purge logs older than a specific date
+task manage -- purge_log_records --date 2024-06-01
+
+# Include audit trail entries recording the purge
+task manage -- purge_log_records --date 2024-06-01 --audit-trail
+
+# Scope the audit trail to specific activations
+task manage -- purge_log_records --date 2024-06-01 --audit-trail --activation-ids 1 2 3
+```
+
 ### Running linters
 
 The project uses `flake8` with a set of plugins, `black`, `isort` and `ruff` (experimental),

src/aap_eda/api/pagination.py

@@ -20,6 +20,12 @@
 class DefaultPagination(pagination.PageNumberPagination):
     page_size_query_param = "page_size"
 
+
+class LogPagination(DefaultPagination):
+    """Pagination for log endpoints, capped at 1000 results per page."""
+
+    max_page_size = 1000
+
     def get_next_link(self):
         if not self.page.has_next():
             return None

src/aap_eda/api/views/activation.py

@@ -31,6 +31,7 @@
 from rest_framework.response import Response
 
 from aap_eda.api import exceptions as api_exc, filters, serializers
+from aap_eda.api.pagination import LogPagination
 from aap_eda.api.serializers.activation import is_activation_valid
 from aap_eda.core import models
 from aap_eda.core.enums import Action, ActivationStatus, ProcessParentType
@@ -815,7 +816,10 @@ def filter_queryset(self, queryset):
         return super().filter_queryset(queryset)
 
     @extend_schema(
-        description="List Activation instance logs",
+        description=(
+            "List Activation instance logs. "
+            "Results are paginated with a maximum page_size of 1000."
+        ),
         request=None,
         responses={
             status.HTTP_200_OK: serializers.ActivationInstanceLogSerializer(
@@ -828,13 +832,20 @@ def filter_queryset(self, queryset):
                 type=int,
                 location=OpenApiParameter.PATH,
                 description="A unique integer value identifying this Activation Instance.",  # noqa: E501
-            )
+            ),
+            OpenApiParameter(
+                name="page_size",
+                type=int,
+                location=OpenApiParameter.QUERY,
+                description="Number of results per page (default: 20, max: 1000).",  # noqa: E501
+            ),
         ],
         extensions={
             "x-ai-description": (
                 "List logs for an activation instance by ID. "
                 "Returns log records with timestamps and levels. "
-                "Supports filtering and pagination."
+                "Supports filtering and pagination. "
+                "Maximum page_size is 1000."
             )
         },
     )
@@ -846,6 +857,7 @@ def filter_queryset(self, queryset):
         url_path="(?P<id>[^/.]+)/logs",
     )
     def logs(self, request, id):
+        self.pagination_class = LogPagination
         instance_exists = (
             models.RulebookProcess.access_qs(request.user)
             .filter(pk=id)

src/aap_eda/core/management/commands/purge_log_records.py

@@ -12,27 +12,31 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from datetime import datetime
+from datetime import datetime, timedelta
 
+from django.conf import settings
 from django.core.management.base import (
     BaseCommand,
     CommandError,
     CommandParser,
 )
 from django.db import transaction
-from django.db.models import Q
+from django.utils import timezone
 
-from aap_eda.core import models
+from aap_eda.core.utils.delete_log_util import (
+    create_audit_trail,
+    delete_logs_older_than,
+)
 
 
 class Command(BaseCommand):
     """Purge the logs from a rulebook process."""
 
     help = (
         "Purge log records from rulebook processes. "
-        "Always purges ALL log records older than the cutoff date globally. "
-        "If activation ids or names are specified, detailed reporting is "
-        "provided for those activations and orphaned records."
+        "Uses --date for a specific cutoff or defaults to "
+        "LOG_RETENTION_DAYS. Use --audit-trail to record "
+        "the purge in activation logs."
     )
 
     def add_arguments(self, parser: CommandParser) -> None:
@@ -42,8 +46,8 @@ def add_arguments(self, parser: CommandParser) -> None:
             type=int,
             dest="activation-ids",
             help=(
-                "Specify the activation ids which you want to clear their "
-                "records (e.g., ActivationID1 ActivationID2)"
+                "Scope audit trail to these activation ids "
+                "(e.g., 1 2 3). Only used with --audit-trail."
             ),
         )
         parser.add_argument(
@@ -52,108 +56,67 @@ def add_arguments(self, parser: CommandParser) -> None:
             type=str,
             dest="activation-names",
             help=(
-                "Specify the activation names which you want to clear their "
-                "records (e.g., ActivationName1 ActivationName2)"
+                "Scope audit trail to these activation names "
+                "(e.g., name1 name2). Only used with --audit-trail."
             ),
         )
         parser.add_argument(
             "--date",
             dest="date",
             action="store",
-            required=True,
             help=(
-                "Purge records older than this date from the database. "
-                "The cutoff date in YYYY-MM-DD format"
+                "Purge records older than this date (YYYY-MM-DD). "
+                "Defaults to LOG_RETENTION_DAYS if omitted."
             ),
         )
-
-    def purge_log_records(
-        self, ids: list[int], names: list[str], cutoff_timestamp: datetime
-    ) -> None:
-        # Global purge of all old logs (regardless of activation)
-        cutoff_ts = int(cutoff_timestamp.timestamp())
-        old_logs = models.RulebookProcessLog.objects.filter(
-            log_timestamp__lt=cutoff_ts
+        parser.add_argument(
+            "--audit-trail",
+            dest="audit_trail",
+            action="store_true",
+            default=False,
+            help="Create audit trail log entries recording the purge.",
         )
-        deleted_count = old_logs.count()
 
-        if deleted_count == 0:
+    @transaction.atomic
+    def handle(self, *args, **options):
+        cutoff_date = options.get("date")
+
+        if cutoff_date:
+            try:
+                cutoff = datetime.strptime(cutoff_date, "%Y-%m-%d")
+            except ValueError as e:
+                raise CommandError(f"{e}") from e
+        else:
+            retention_days = settings.LOG_RETENTION_DAYS
+            if retention_days <= 0:
+                self.stdout.write("LOG_RETENTION_DAYS is 0; nothing to purge.")
+                return
+            cutoff = timezone.now() - timedelta(days=retention_days)
+
+        deleted = delete_logs_older_than(cutoff)
+
+        if deleted == 0:
             self.stdout.write(
                 self.style.SUCCESS(
                     f"No log records found older than "
-                    f"{cutoff_timestamp.strftime('%Y-%m-%d')}."
+                    f"{cutoff.strftime('%Y-%m-%d')}."
                 )
             )
             return
 
-        # Delete all old logs globally
-        old_logs.delete()
-
         self.stdout.write(
             self.style.SUCCESS(
-                f"Purged {deleted_count} log records older than "
-                f"{cutoff_timestamp.strftime('%Y-%m-%d')} globally."
+                f"Purged {deleted} log records older than "
+                f"{cutoff.strftime('%Y-%m-%d')} globally."
             )
         )
 
-        # Create audit trail logs for reporting
-        audit_logs = []
-
-        if not bool(ids) and not bool(names):
-            # No filters: Create audit logs for all instances that might
-            # have been affected
-            instances = models.RulebookProcess.objects.all()
-        else:
-            # Filters provided: Create audit logs only for specified
-            # activations and orphaned records
-            instances = models.RulebookProcess.objects.filter(
-                Q(activation__id__in=ids)
-                | Q(activation__name__in=names)
-                | Q(activation__isnull=True),
-            )
-
-            # Report on orphaned records for user visibility
-            orphaned_count = models.RulebookProcess.objects.filter(
-                activation__isnull=True
-            ).count()
-            if orphaned_count > 0:
-                self.stdout.write(
-                    f"Audit trail will include {orphaned_count} orphaned "
-                    f"RulebookProcess records (with NULL activation)."
-                )
-
-        # Create audit trail logs for instances
-        dt = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        for instance in instances:
-            audit_logs.append(
-                models.RulebookProcessLog(
-                    log=(
-                        f"All log records older than "
-                        f"{cutoff_timestamp.strftime('%Y-%m-%d')} "
-                        f"were purged at {dt}."
-                    ),
-                    activation_instance_id=instance.id,
-                    log_timestamp=cutoff_ts,
-                )
-            )
-
-        if audit_logs:
-            models.RulebookProcessLog.objects.bulk_create(audit_logs)
-            self.stdout.write(
-                f"Created {len(audit_logs)} audit trail log entries."
+        if options.get("audit_trail"):
+            ids = options.get("activation-ids") or []
+            names = options.get("activation-names") or []
+            count = create_audit_trail(
+                cutoff,
+                activation_ids=ids,
+                activation_names=names,
             )
-
-    @transaction.atomic
-    def handle(self, *args, **options):
-        input_ids = options.get("activation-ids") or []
-        input_names = options.get("activation-names") or []
-        cutoff_date = options.get("date")
-
-        try:
-            ts = datetime.strptime(cutoff_date, "%Y-%m-%d")
-        except ValueError as e:
-            raise CommandError(f"{e}") from e
-
-        self.purge_log_records(
-            ids=input_ids, names=input_names, cutoff_timestamp=ts
-        )
+            self.stdout.write(f"Created {count} audit trail log entries.")

src/aap_eda/core/utils/delete_log_util.py

@@ -0,0 +1,79 @@
+#  Copyright 2025 Red Hat, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from datetime import datetime
+
+from django.db.models import Q
+from django.utils import timezone
+
+from aap_eda.core import models
+
+
+def delete_logs_older_than(cutoff: datetime) -> int:
+    """Delete all RulebookProcessLog records older than the cutoff.
+
+    Returns the number of records deleted.
+    """
+    cutoff_ts = int(cutoff.timestamp())
+    deleted, _ = models.RulebookProcessLog.objects.filter(
+        log_timestamp__lt=cutoff_ts,
+    ).delete()
+    return deleted
+
+
+def create_audit_trail(
+    cutoff: datetime,
+    activation_ids: list[int] | None = None,
+    activation_names: list[str] | None = None,
+) -> int:
+    """Create audit trail log entries recording a purge operation.
+
+    If activation_ids or activation_names are provided, audit entries
+    are scoped to those activations plus orphaned records. Otherwise,
+    audit entries are created for all RulebookProcess instances.
+
+    Returns the number of audit entries created.
+    """
+    ids = activation_ids or []
+    names = activation_names or []
+
+    if not ids and not names:
+        instances = models.RulebookProcess.objects.all()
+    else:
+        instances = models.RulebookProcess.objects.filter(
+            Q(activation__id__in=ids)
+            | Q(activation__name__in=names)
+            | Q(activation__isnull=True),
+        )
+
+    dt = timezone.now().strftime("%Y-%m-%d %H:%M:%S")
+    cutoff_ts = int(cutoff.timestamp())
+
+    audit_logs = [
+        models.RulebookProcessLog(
+            log=(
+                f"All log records older than "
+                f"{cutoff.strftime('%Y-%m-%d')} "
+                f"were purged at {dt}."
+            ),
+            activation_instance_id=instance.id,
+            log_timestamp=cutoff_ts,
+        )
+        for instance in instances
+    ]
+
+    if audit_logs:
+        models.RulebookProcessLog.objects.bulk_create(audit_logs)
+
+    return len(audit_logs)