Status aggregator job (#494)

Author: Scott Bommarito

NuGet.Jobs.sln

@@ -107,6 +107,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Common.Job", "sr
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PackageHash", "src\PackageHash\PackageHash.csproj", "{40843020-6F0A-48F0-AC28-42FFE3A5C21E}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StatusAggregator", "src\StatusAggregator\StatusAggregator.csproj", "{D357FDB5-BF19-41A5-82B0-14C8CEC2A5EB}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Common.Job.Tests", "tests\Validation.Common.Job.Tests\Validation.Common.Job.Tests.csproj", "{430F63C7-20C2-4872-AC3E-DDE846E50AA4}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.PackageSigning.ProcessSignature", "src\Validation.PackageSigning.ProcessSignature\Validation.PackageSigning.ProcessSignature.csproj", "{DD043977-6BCD-475A-BEE2-8C34309EC622}"
@@ -135,6 +137,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.Revalidate",
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.Revalidate.Tests", "tests\NuGet.Services.Revalidate.Tests\NuGet.Services.Revalidate.Tests.csproj", "{19780DCB-B307-4254-B10C-4335FC784DEA}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StatusAggregator.Tests", "tests\StatusAggregator.Tests\StatusAggregator.Tests.csproj", "{784F938D-4142-4C1C-B654-0978FEAD1731}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Symbols.Core", "src\Validation.Symbols.Core\Validation.Symbols.Core.csproj", "{17510A22-176F-4E96-A867-E79F1B54F54F}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Monitoring.RebootSearchInstance", "src\Monitoring.RebootSearchInstance\Monitoring.RebootSearchInstance.csproj", "{ECD8DFCE-8E3C-4510-AFE3-D7EC168E8D66}"
@@ -309,6 +313,10 @@ Global
 		{40843020-6F0A-48F0-AC28-42FFE3A5C21E}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{40843020-6F0A-48F0-AC28-42FFE3A5C21E}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{40843020-6F0A-48F0-AC28-42FFE3A5C21E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D357FDB5-BF19-41A5-82B0-14C8CEC2A5EB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D357FDB5-BF19-41A5-82B0-14C8CEC2A5EB}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D357FDB5-BF19-41A5-82B0-14C8CEC2A5EB}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D357FDB5-BF19-41A5-82B0-14C8CEC2A5EB}.Release|Any CPU.Build.0 = Release|Any CPU
 		{430F63C7-20C2-4872-AC3E-DDE846E50AA4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{430F63C7-20C2-4872-AC3E-DDE846E50AA4}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{430F63C7-20C2-4872-AC3E-DDE846E50AA4}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -361,6 +369,10 @@ Global
 		{19780DCB-B307-4254-B10C-4335FC784DEA}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{19780DCB-B307-4254-B10C-4335FC784DEA}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{19780DCB-B307-4254-B10C-4335FC784DEA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{784F938D-4142-4C1C-B654-0978FEAD1731}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{784F938D-4142-4C1C-B654-0978FEAD1731}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{784F938D-4142-4C1C-B654-0978FEAD1731}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{784F938D-4142-4C1C-B654-0978FEAD1731}.Release|Any CPU.Build.0 = Release|Any CPU
 		{17510A22-176F-4E96-A867-E79F1B54F54F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{17510A22-176F-4E96-A867-E79F1B54F54F}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{17510A22-176F-4E96-A867-E79F1B54F54F}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -425,6 +437,7 @@ Global
 		{B4B7564A-965B-447B-927F-6749E2C08880} = {6A776396-02B1-475D-A104-26940ADB04AB}
 		{FA87D075-A934-4443-8D0B-5DB32640B6D7} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
 		{40843020-6F0A-48F0-AC28-42FFE3A5C21E} = {FA5644B5-4F08-43F6-86B3-039374312A47}
+		{D357FDB5-BF19-41A5-82B0-14C8CEC2A5EB} = {FA5644B5-4F08-43F6-86B3-039374312A47}
 		{430F63C7-20C2-4872-AC3E-DDE846E50AA4} = {6A776396-02B1-475D-A104-26940ADB04AB}
 		{DD043977-6BCD-475A-BEE2-8C34309EC622} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
 		{ED2D370C-D921-433A-A0B9-A601F936EDD3} = {FA5644B5-4F08-43F6-86B3-039374312A47}
@@ -438,6 +451,7 @@ Global
 		{60152AB1-2EB4-4D44-B6D6-EEE24209A1F7} = {6A776396-02B1-475D-A104-26940ADB04AB}
 		{1963909D-8BE3-4CB8-B57E-AB6A8CB22FED} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
 		{19780DCB-B307-4254-B10C-4335FC784DEA} = {6A776396-02B1-475D-A104-26940ADB04AB}
+		{784F938D-4142-4C1C-B654-0978FEAD1731} = {6A776396-02B1-475D-A104-26940ADB04AB}
 		{17510A22-176F-4E96-A867-E79F1B54F54F} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
 		{ECD8DFCE-8E3C-4510-AFE3-D7EC168E8D66} = {814F9B31-4AF3-46CC-AD61-CEB40F47083A}
 		{21C0A0EE-8696-4013-950F-D6495D0C6E40} = {6A776396-02B1-475D-A104-26940ADB04AB}

build.ps1

@@ -115,6 +115,7 @@ Invoke-BuildStep 'Set version metadata in AssemblyInfo.cs' { `
             "$PSScriptRoot\src\Validation.Common.Job\Properties\AssemblyInfo.g.cs",
             "$PSScriptRoot\src\Validation.ScanAndSign.Core\Properties\AssemblyInfo.g.cs",
             "$PSScriptRoot\src\PackageLagMonitor\Properties\AssemblyInfo.g.cs",
+            "$PSScriptRoot\src\StatusAggregator\Properties\AssemblyInfo.g.cs",
             "$PSScriptRoot\src\Validation.Symbols.Core\Properties\AssemblyInfo.g.cs",
             "$PSScriptRoot\src\Monitoring.RebootSearchInstance\Properties\AssemblyInfo.g.cs"
 
@@ -177,7 +178,8 @@ Invoke-BuildStep 'Creating artifacts' {
             "src/Validation.PackageSigning.ValidateCertificate/Validation.PackageSigning.ValidateCertificate.csproj", `
             "src/Validation.PackageSigning.RevalidateCertificate/Validation.PackageSigning.RevalidateCertificate.csproj", `
             "src/PackageLagMonitor/Monitoring.PackageLag.csproj", `
-            "src/Monitoring.RebootSearchInstance/Monitoring.RebootSearchInstance.csproj", `
+            "src/StatusAggregator/StatusAggregator.csproj", `
+            "src/Validation.Symbols.Core/Validation.Symbols.Core.csproj", `
             "src/Validation.Symbols/Validation.Symbols.csproj" `
             + $ProjectsWithSymbols
 

src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs

@@ -118,6 +118,19 @@ public static class JobArgumentNames
         public const string MailFrom = "MailFrom";
         public const string SmtpUri = "SmtpUri";
 
+        // Arguments specific to StatusAggregator
+        public const string StatusStorageAccount = "StatusStorageAccount";
+        public const string StatusContainerName = "StatusContainerName";
+        public const string StatusTableName = "StatusTableName";
+        public const string StatusEnvironment = "StatusEnvironment";
+        public const string StatusMaximumSeverity = "StatusMaximumSeverity";
+        public const string StatusIncidentApiBaseUri = "StatusIncidentApiBaseUri";
+        public const string StatusIncidentApiCertificate = "StatusIncidentApiCertificate";
+        public const string StatusIncidentApiTeamId = "StatusIncidentApiTeamId";
+        public const string StatusEventStartMessageDelayMinutes = "StatusEventStartMessageDelayMinutes";
+        public const string StatusEventEndDelayMinutes = "StatusEventEndDelayMinutes";
+        public const string StatusEventVisibilityPeriodDays = "StatusEventVisibilityPeriodDays";
+
         // Arguments specific to Stats.AggregateCdnDownloadsInGallery
         public static string BatchSleepSeconds = "BatchSleepSeconds";
     }

src/NuGet.Jobs.Common/Configuration/JobConfigurationManager.cs

@@ -5,7 +5,9 @@
 using System.Collections.Generic;
 using System.ComponentModel.Design;
 using System.Linq;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using NuGet.Jobs.Extensions;
 using NuGet.Services.Configuration;
 using NuGet.Services.KeyVault;
 
@@ -190,7 +192,7 @@ private static Dictionary<string, string> ReadCommandLineArguments(ILogger logge
 
         private static IDictionary<string, string> InjectSecrets(IServiceContainer serviceContainer, Dictionary<string, string> argsDictionary)
         {
-            var secretReaderFactory = (ISecretReaderFactory)serviceContainer.GetService(typeof(ISecretReaderFactory));
+            var secretReaderFactory = serviceContainer.GetRequiredService<ISecretReaderFactory>();
 
             var secretReader = secretReaderFactory.CreateSecretReader(argsDictionary);
             if (secretReader == null)

src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs

@@ -0,0 +1,54 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using Microsoft.Extensions.Logging;
+using System;
+
+namespace NuGet.Jobs.Extensions
+{
+    public static class LoggerExtensions
+    {
+        /// <summary>
+        /// Calls <see cref="ILogger.BeginScope{TState}(TState)"/> and logs a message when entering and leaving the scope.
+        /// </summary>
+        public static IDisposable Scope(
+            this ILogger logger, 
+            string message, 
+            params object[] args)
+        {
+            return new LoggerScopeHelper(logger, message, args);
+        }
+
+        private class LoggerScopeHelper : IDisposable
+        {
+            private readonly ILogger _logger;
+            private readonly IDisposable _scope;
+
+            private readonly string _message;
+            private readonly object[] _args;
+
+            private bool _isDisposed = false;
+
+            public LoggerScopeHelper(
+                ILogger logger, string message, object[] args)
+            {
+                _logger = logger;
+                _message = message;
+                _args = args;
+
+                _scope = logger.BeginScope(_message, _args);
+                _logger.LogInformation("Entering scope: " + _message, _args);
+            }
+
+            public void Dispose()
+            {
+                if (!_isDisposed)
+                {
+                    _logger.LogInformation("Leaving scope: " + _message, _args);
+                    _scope?.Dispose(); // ILogger can return a null scope (most notably during testing with a Mock<ILogger>)
+                    _isDisposed = true;
+                }
+            }
+        }
+    }
+}

src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj

@@ -49,6 +49,7 @@
     <Compile Include="Configuration\ServiceBusConfiguration.cs" />
     <Compile Include="Configuration\ValidationDbConfiguration.cs" />
     <Compile Include="Configuration\ValidationStorageConfiguration.cs" />
+    <Compile Include="Extensions\LoggerExtensions.cs" />
     <Compile Include="Extensions\SqlConnectionStringBuilderExtensions.cs" />
     <Compile Include="Extensions\XElementExtensions.cs" />
     <Compile Include="SecretReader\ISecretReaderFactory.cs" />

src/StatusAggregator/Cursor.cs

@@ -0,0 +1,60 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using NuGet.Jobs.Extensions;
+using NuGet.Services.Status.Table;
+using StatusAggregator.Table;
+
+namespace StatusAggregator
+{
+    public class Cursor : ICursor
+    {
+        public Cursor(
+            ITableWrapper table,
+            ILogger<Cursor> logger)
+        {
+            _table = table ?? throw new ArgumentNullException(nameof(table));
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        }
+
+        private readonly ITableWrapper _table;
+
+        private readonly ILogger<Cursor> _logger;
+
+        public async Task<DateTime> Get()
+        {
+            using (_logger.Scope("Fetching cursor."))
+            {
+                var cursor = await _table.Retrieve<CursorEntity>(
+                    CursorEntity.DefaultPartitionKey, CursorEntity.DefaultRowKey);
+
+                DateTime value;
+                if (cursor == null)
+                {
+                    // If we can't find a cursor, the job is likely uninitialized, so start at the beginning of time.
+                    value = DateTime.MinValue;
+                    _logger.LogInformation("Could not fetch cursor, reinitializing cursor at {Cursor}.", value);
+                }
+                else
+                {
+                    value = cursor.Value;
+                    _logger.LogInformation("Fetched cursor with value {Cursor}.", value);
+                }
+
+                return value;
+            }
+        }
+
+        public Task Set(DateTime value)
+        {
+            using (_logger.Scope("Updating cursor to {Cursor}.", value))
+            {
+                var cursorEntity = new CursorEntity(value);
+                return _table.InsertOrReplaceAsync(cursorEntity);
+            }
+        }
+    }
+}

src/StatusAggregator/EventUpdater.cs

@@ -0,0 +1,97 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using NuGet.Jobs.Extensions;
+using NuGet.Services.Status.Table;
+using StatusAggregator.Table;
+
+namespace StatusAggregator
+{
+    public class EventUpdater : IEventUpdater
+    {
+        public readonly TimeSpan _eventEndDelay;
+
+        private readonly ITableWrapper _table;
+        private readonly IMessageUpdater _messageUpdater;
+
+        private readonly ILogger<EventUpdater> _logger;
+
+        public EventUpdater(
+            ITableWrapper table, 
+            IMessageUpdater messageUpdater, 
+            StatusAggregatorConfiguration configuration,
+            ILogger<EventUpdater> logger)
+        {
+            _table = table ?? throw new ArgumentNullException(nameof(table));
+            _messageUpdater = messageUpdater ?? throw new ArgumentNullException(nameof(messageUpdater));
+            _eventEndDelay = TimeSpan.FromMinutes(configuration?.EventEndDelayMinutes ?? throw new ArgumentNullException(nameof(configuration)));
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        }
+
+        public async Task UpdateActiveEvents(DateTime cursor)
+        {
+            using (_logger.Scope("Updating active events."))
+            {
+                var activeEvents = _table.GetActiveEvents().ToList();
+                _logger.LogInformation("Updating {ActiveEventsCount} active events.", activeEvents.Count());
+                foreach (var activeEvent in activeEvents)
+                {
+                    await UpdateEvent(activeEvent, cursor);
+                }
+            }
+        }
+
+        public async Task<bool> UpdateEvent(EventEntity eventEntity, DateTime cursor)
+        {
+            eventEntity = eventEntity ?? throw new ArgumentNullException(nameof(eventEntity));
+
+            using (_logger.Scope("Updating event '{EventRowKey}' given cursor {Cursor}.", eventEntity.RowKey, cursor))
+            {
+                if (!eventEntity.IsActive)
+                {
+                    _logger.LogInformation("Event is inactive, cannot update.");
+                    return false;
+                }
+
+                var incidentsLinkedToEventQuery = _table.GetIncidentsLinkedToEvent(eventEntity);
+
+                var incidentsLinkedToEvent = incidentsLinkedToEventQuery.ToList();
+                if (!incidentsLinkedToEvent.Any())
+                {
+                    _logger.LogInformation("Event has no linked incidents and must have been created manually, cannot update.");
+                    return false;
+                }
+
+                var shouldDeactivate = !incidentsLinkedToEventQuery
+                        .Where(i => i.IsActive || i.MitigationTime > cursor - _eventEndDelay)
+                        .ToList()
+                        .Any();
+
+                if (shouldDeactivate)
+                {
+                    _logger.LogInformation("Deactivating event because its incidents are inactive and too old.");
+                    var mitigationTime = incidentsLinkedToEvent
+                        .Max(i => i.MitigationTime ?? DateTime.MinValue);
+                    eventEntity.EndTime = mitigationTime;
+
+                    await _messageUpdater.CreateMessageForEventStart(eventEntity, mitigationTime);
+                    await _messageUpdater.CreateMessageForEventEnd(eventEntity);
+
+                    // Update the event
+                    await _table.InsertOrReplaceAsync(eventEntity);
+                }
+                else
+                {
+                    _logger.LogInformation("Event has active or recent incidents so it will not be deactivated.");
+                    await _messageUpdater.CreateMessageForEventStart(eventEntity, cursor);
+                }
+
+                return shouldDeactivate;
+            }
+        }
+    }
+}

src/StatusAggregator/ICursor.cs

@@ -0,0 +1,17 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Threading.Tasks;
+
+namespace StatusAggregator
+{
+    /// <summary>
+    /// Maintains the current progress of the job.
+    /// </summary>
+    public interface ICursor
+    {
+        Task<DateTime> Get();
+        Task Set(DateTime value);
+    }
+}

src/StatusAggregator/IEventUpdater.cs

@@ -0,0 +1,29 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Threading.Tasks;
+using NuGet.Services.Status.Table;
+
+namespace StatusAggregator
+{
+    /// <summary>
+    /// Handles updating any active <see cref="EventEntity"/>s.
+    /// </summary>
+    public interface IEventUpdater
+    {
+        /// <summary>
+        /// Updates all active <see cref="EventEntity"/>s.
+        /// </summary>
+        /// <param name="cursor">The current timestamp processed by the job.</param>
+        Task UpdateActiveEvents(DateTime cursor);
+
+        /// <summary>
+        /// Update <paramref name="eventEntity"/> given <paramref name="cursor"/>.
+        /// Determines whether or not to deactivate <paramref name="eventEntity"/> and updates any messages associated with the event.
+        /// </summary>
+        /// <param name="cursor">The current timestamp processed by the job.</param>
+        /// <returns>Whether or not <paramref name="eventEntity"/> was deactivated.</returns>
+        Task<bool> UpdateEvent(EventEntity eventEntity, DateTime cursor);
+    }
+}