Status Aggregator - Better detect noisy incidents and do not post messages for them (#549)

Author: Scott Bommarito

src/StatusAggregator/EventUpdater.cs

@@ -66,11 +66,21 @@ public async Task<bool> UpdateEvent(EventEntity eventEntity, DateTime cursor)
                     return false;
                 }
 
-                var shouldDeactivate = !incidentsLinkedToEventQuery
-                        .Where(i => i.IsActive || i.MitigationTime > cursor - _eventEndDelay)
-                        .ToList()
-                        .Any();
+                // We are querying twice here because table storage ignores rows where a column specified by a query is null.
+                // MitigationTime is null when IsActive is true.
+                // If we do not query separately here, rows where IsActive is true will be ignored in query results.
 
+                var hasActiveIncidents = incidentsLinkedToEventQuery
+                    .Where(i => i.IsActive)
+                    .ToList()
+                    .Any();
+
+                var hasRecentIncidents = incidentsLinkedToEventQuery
+                    .Where(i => i.MitigationTime > cursor - _eventEndDelay)
+                    .ToList()
+                    .Any();
+
+                var shouldDeactivate = !(hasActiveIncidents || hasRecentIncidents);
                 if (shouldDeactivate)
                 {
                     _logger.LogInformation("Deactivating event because its incidents are inactive and too old.");

src/StatusAggregator/IncidentUpdater.cs

@@ -16,7 +16,6 @@ namespace StatusAggregator
     public class IncidentUpdater : IIncidentUpdater
     {
         private readonly ITableWrapper _table;
-        private readonly IEventUpdater _eventUpdater;
         private readonly IAggregateIncidentParser _aggregateIncidentParser;
         private readonly IIncidentApiClient _incidentApiClient;
         private readonly IIncidentFactory _incidentFactory;
@@ -26,15 +25,13 @@ public class IncidentUpdater : IIncidentUpdater
 
         public IncidentUpdater(
             ITableWrapper table,
-            IEventUpdater eventUpdater,
             IIncidentApiClient incidentApiClient,
             IAggregateIncidentParser aggregateIncidentParser,
             IIncidentFactory incidentFactory,
             StatusAggregatorConfiguration configuration,
             ILogger<IncidentUpdater> logger)
         {
             _table = table ?? throw new ArgumentNullException(nameof(table));
-            _eventUpdater = eventUpdater ?? throw new ArgumentNullException(nameof(eventUpdater));
             _incidentApiClient = incidentApiClient ?? throw new ArgumentNullException(nameof(incidentApiClient));
             _aggregateIncidentParser = aggregateIncidentParser ?? throw new ArgumentNullException(nameof(aggregateIncidentParser));
             _incidentFactory = incidentFactory ?? throw new ArgumentNullException(nameof(incidentFactory));

src/StatusAggregator/Job.cs

@@ -164,7 +164,7 @@ private static CloudBlobContainer GetCloudBlobContainer(IComponentContext ctx, C
         }
 
         private const int _defaultEventStartMessageDelayMinutes = 15;
-        private const int _defaultEventEndDelayMinutes = 10;
+        private const int _defaultEventEndDelayMinutes = 15;
         private const int _defaultEventVisibilityPeriod = 10;
 
         private static void AddConfiguration(IServiceCollection serviceCollection, IDictionary<string, string> jobArgsDictionary)

src/StatusAggregator/MessageUpdater.cs

@@ -49,6 +49,32 @@ public async Task CreateMessageForEventStart(EventEntity eventEntity, DateTime c
                     return;
                 }
 
+                var incidentsLinkedToEventQuery = _table.GetIncidentsLinkedToEvent(eventEntity);
+
+                // We are querying twice here because table storage ignores rows where a column specified by a query is null.
+                // MitigationTime is null when IsActive is true.
+                // If we do not query separately here, rows where IsActive is true will be ignored in query results.
+
+                var hasCurrentlyActiveIncidents = incidentsLinkedToEventQuery
+                    .Where(i => i.IsActive)
+                    .ToList()
+                    .Any();
+
+                var hasIncidentsActiveAtCursorTime = incidentsLinkedToEventQuery
+                    .Where(i => i.MitigationTime >= cursor)
+                    .ToList()
+                    .Any();
+
+                var hasActiveIncidents = hasCurrentlyActiveIncidents || hasIncidentsActiveAtCursorTime;
+                if (!hasActiveIncidents)
+                {
+                    // If we haven't told customers about an event and it doesn't have any active incidents, it must have be linked to an incident that fired and was quickly mitigated.
+                    // This event will be deactivated if it is not linked to an active incident soon.
+                    // We will create a message when it is linked to another active incident.
+                    _logger.LogInformation("Event has no active incidents, cannot create message for its start.");
+                    return;
+                }
+
                 if (TryGetContentsForMessageForEventStart(eventEntity, out var contents))
                 {
                     await CreateMessage(eventEntity, eventEntity.StartTime, contents);

src/StatusAggregator/StatusUpdater.cs

@@ -45,22 +45,26 @@ public async Task Update()
                     await ProcessCursor($"{ManualCursorBaseName}{manualStatusChangeUpdater.Name}", manualStatusChangeUpdater.ProcessNewManualChanges);
                 }
 
-                await ProcessCursor(IncidentCursorName, async (value) =>
+                var incidentCursor = await ProcessCursor(IncidentCursorName, async (value) =>
                 {
                     await _incidentUpdater.RefreshActiveIncidents();
                     return await _incidentUpdater.FetchNewIncidents(value);
                 });
+
+                await _eventUpdater.UpdateActiveEvents(incidentCursor);
             }
         }
 
-        private async Task ProcessCursor(string name, Func<DateTime, Task<DateTime?>> processCursor)
+        private async Task<DateTime> ProcessCursor(string name, Func<DateTime, Task<DateTime?>> processCursor)
         {
             var lastCursor = await _cursor.Get(name);
             var nextCursor = await processCursor(lastCursor);
             if (nextCursor.HasValue)
             {
                 await _cursor.Set(name, nextCursor.Value);
             }
+
+            return nextCursor ?? lastCursor;
         }
     }
 }