StatusAggregator should handle incident ingestion failures gracefully (#671)

Scott Bommarito · web-flow · commit 5f99bbd376c6 · 2018-11-15T17:27:17.000-08:00
diff --git a/src/StatusAggregator/Export/IStatusSerializer.cs b/src/StatusAggregator/Export/IStatusSerializer.cs
@@ -11,8 +11,8 @@ namespace StatusAggregator.Export
     public interface IStatusSerializer
     {
         /// <summary>
-        /// Serializes <paramref name="rootComponent"/> and <paramref name="recentEvents"/> and saves to storage with a time of <paramref name="cursor"/>.
+        /// Serializes <paramref name="rootComponent"/> and <paramref name="recentEvents"/> and saves to storage with a last built time of <paramref name="lastBuilt"/> and a last updated time of <paramref name="lastUpdated"/>.
         /// </summary>
-        Task Serialize(DateTime cursor, IComponent rootComponent, IEnumerable<Event> recentEvents);
+        Task Serialize(DateTime lastBuilt, DateTime lastUpdated, IComponent rootComponent, IEnumerable<Event> recentEvents);
     }
 }
diff --git a/src/StatusAggregator/Export/StatusExporter.cs b/src/StatusAggregator/Export/StatusExporter.cs
@@ -5,36 +5,43 @@
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using NuGet.Jobs.Extensions;
+using StatusAggregator.Collector;
+using StatusAggregator.Update;
 
 namespace StatusAggregator.Export
 {
     public class StatusExporter : IStatusExporter
     {
+        private readonly ICursor _cursor;
         private readonly IComponentExporter _componentExporter;
         private readonly IEventsExporter _eventExporter;
         private readonly IStatusSerializer _serializer;
 
         private readonly ILogger<StatusExporter> _logger;
 
         public StatusExporter(
+            ICursor cursor,
             IComponentExporter componentExporter,
             IEventsExporter eventExporter,
             IStatusSerializer serializer,
             ILogger<StatusExporter> logger)
         {
+            _cursor = cursor ?? throw new ArgumentNullException(nameof(cursor));
             _componentExporter = componentExporter ?? throw new ArgumentNullException(nameof(componentExporter));
             _eventExporter = eventExporter ?? throw new ArgumentNullException(nameof(eventExporter));
             _serializer = serializer ?? throw new ArgumentNullException(nameof(serializer));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
 
-        public Task Export(DateTime cursor)
+        public async Task Export(DateTime cursor)
         {
             using (_logger.Scope("Exporting service status."))
             {
                 var rootComponent = _componentExporter.Export();
                 var recentEvents = _eventExporter.Export(cursor);
-                return _serializer.Serialize(cursor, rootComponent, recentEvents);
+
+                var lastUpdated = await _cursor.Get(StatusUpdater.LastUpdatedCursorName);
+                await _serializer.Serialize(cursor, lastUpdated, rootComponent, recentEvents);
             }
         }
     }
diff --git a/src/StatusAggregator/Export/StatusSerializer.cs b/src/StatusAggregator/Export/StatusSerializer.cs
@@ -36,13 +36,13 @@ public StatusSerializer(
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
 
-        public async Task Serialize(DateTime cursor, IComponent rootComponent, IEnumerable<Event> recentEvents)
+        public async Task Serialize(DateTime lastBuilt, DateTime lastUpdated, IComponent rootComponent, IEnumerable<Event> recentEvents)
         {
             ServiceStatus status;
             string statusJson;
             using (_logger.Scope("Serializing service status."))
             {
-                status = new ServiceStatus(cursor, rootComponent, recentEvents);
+                status = new ServiceStatus(lastBuilt, lastUpdated, rootComponent, recentEvents);
                 statusJson = JsonConvert.SerializeObject(status, Settings);
             }
 
diff --git a/src/StatusAggregator/LogEvents.cs b/src/StatusAggregator/LogEvents.cs
@@ -6,5 +6,6 @@ public static class LogEvents
     {
         public static EventId RegexFailure = new EventId(400, "Failed to parse incident using Regex.");
         public static EventId ManualChangeFailure = new EventId(401, "Failed to apply a manual change.");
+        public static EventId IncidentIngestionFailure = new EventId(402, "Failed to update incident API data.");
     }
 }
diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj
@@ -160,10 +160,10 @@
       <Version>2.33.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Status">
-      <Version>2.33.0</Version>
+      <Version>2.37.0-sb-statusfb-2205681</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Status.Table">
-      <Version>2.33.0</Version>
+      <Version>2.37.0-sb-statusfb-2205681</Version>
     </PackageReference>
     <PackageReference Include="WindowsAzure.Storage">
       <Version>9.2.0</Version>
diff --git a/src/StatusAggregator/Update/StatusUpdater.cs b/src/StatusAggregator/Update/StatusUpdater.cs
@@ -13,8 +13,7 @@ namespace StatusAggregator.Update
 {
     public class StatusUpdater : IStatusUpdater
     {
-        private const string ManualCursorBaseName = "manual";
-        private const string IncidentCursorName = "incident";
+        public const string LastUpdatedCursorName = "updated";
 
         private readonly ICursor _cursor;
         private readonly IEntityCollector _incidentCollector;
@@ -50,8 +49,17 @@ public async Task Update(DateTime cursor)
                     await manualStatusChangeCollector.FetchLatest();
                 }
 
-                await _incidentCollector.FetchLatest();
-                await _activeEventUpdater.UpdateAllAsync(cursor);
+                try
+                {
+                    await _incidentCollector.FetchLatest();
+                    await _activeEventUpdater.UpdateAllAsync(cursor);
+
+                    await _cursor.Set(LastUpdatedCursorName, cursor);
+                }
+                catch (Exception e)
+                {
+                    _logger.LogError(LogEvents.IncidentIngestionFailure, e, "Failed to update incident API data.");
+                }
             }
         }
 
diff --git a/tests/StatusAggregator.Tests/Export/StatusExporterTests.cs b/tests/StatusAggregator.Tests/Export/StatusExporterTests.cs
@@ -6,7 +6,9 @@
 using Microsoft.Extensions.Logging;
 using Moq;
 using NuGet.Services.Status;
+using StatusAggregator.Collector;
 using StatusAggregator.Export;
+using StatusAggregator.Update;
 using Xunit;
 
 namespace StatusAggregator.Tests.Export
@@ -29,30 +31,38 @@ public async Task ExportsAtCursor()
                 EventExporter
                     .Setup(x => x.Export(cursor))
                     .Returns(events);
-                
+
+                var lastUpdated = new DateTime(2018, 9, 12);
+                Cursor
+                    .Setup(x => x.Get(StatusUpdater.LastUpdatedCursorName))
+                    .ReturnsAsync(lastUpdated);
+
                 await Exporter.Export(cursor);
 
                 Serializer
                     .Verify(
-                        x => x.Serialize(cursor, component, events),
+                        x => x.Serialize(cursor, lastUpdated, component, events),
                         Times.Once());
             }
         }
 
         public class StatusExporterTest
         {
+            public Mock<ICursor> Cursor { get; }
             public Mock<IComponentExporter> ComponentExporter { get; }
             public Mock<IEventsExporter> EventExporter { get; }
             public Mock<IStatusSerializer> Serializer { get; }
             public StatusExporter Exporter { get; }
 
             public StatusExporterTest()
             {
+                Cursor = new Mock<ICursor>();
                 ComponentExporter = new Mock<IComponentExporter>();
                 EventExporter = new Mock<IEventsExporter>();
                 Serializer = new Mock<IStatusSerializer>();
 
                 Exporter = new StatusExporter(
+                    Cursor.Object,
                     ComponentExporter.Object,
                     EventExporter.Object,
                     Serializer.Object,
diff --git a/tests/StatusAggregator.Tests/Export/StatusSerializerTests.cs b/tests/StatusAggregator.Tests/Export/StatusSerializerTests.cs
@@ -20,14 +20,15 @@ public class TheSerializeMethod : StatusSerializerTest
             [Fact]
             public async Task SerializesStatus()
             {
-                var cursor = new DateTime(2018, 9, 13);
+                var lastBuilt = new DateTime(2018, 11, 13);
+                var lastUpdated = new DateTime(2018, 9, 13);
                 var component = new TestComponent("hi", new[] { new TestComponent("yo"), new TestComponent("what's up") });
-                var events = new[] { new Event("", cursor, cursor, new[] { new Message(cursor, "howdy") }) };
+                var events = new[] { new Event("", lastUpdated, lastUpdated, new[] { new Message(lastUpdated, "howdy") }) };
 
-                var expectedStatus = new ServiceStatus(cursor, component, events);
+                var expectedStatus = new ServiceStatus(lastBuilt, lastUpdated, component, events);
                 var expectedJson = JsonConvert.SerializeObject(expectedStatus, StatusSerializer.Settings);
 
-                await Serializer.Serialize(cursor, component, events);
+                await Serializer.Serialize(lastBuilt, lastUpdated, component, events);
 
                 Container
                     .Verify(
diff --git a/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj b/tests/StatusAggregator.Tests/StatusAggregator.Tests.csproj
@@ -91,6 +91,7 @@
     <Compile Include="Update\AggregationEntityUpdaterTests.cs" />
     <Compile Include="Update\EventMessagingUpdaterTests.cs" />
     <Compile Include="Update\IncidentUpdaterTests.cs" />
+    <Compile Include="Update\StatusUpdaterTests.cs" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\..\src\StatusAggregator\StatusAggregator.csproj">
@@ -111,9 +112,6 @@
     <PackageReference Include="System.ValueTuple">
       <Version>4.4.0</Version>
     </PackageReference>
-    <PackageReference Include="WindowsAzure.Storage">
-      <Version>9.2.0</Version>
-    </PackageReference>
     <PackageReference Include="xunit">
       <Version>2.4.0</Version>
     </PackageReference>
diff --git a/tests/StatusAggregator.Tests/Update/StatusUpdaterTests.cs b/tests/StatusAggregator.Tests/Update/StatusUpdaterTests.cs

Original file line number	Diff line number	Diff line change
`@@ -11,8 +11,8 @@ namespace StatusAggregator.Export`
`11`	`11`	`public interface IStatusSerializer`
`12`	`12`	`{`
`13`	`13`	`/// <summary>`
`14`		`- /// Serializes <paramref name="rootComponent"/> and <paramref name="recentEvents"/> and saves to storage with a time of <paramref name="cursor"/>.`
	`14`	`+ /// Serializes <paramref name="rootComponent"/> and <paramref name="recentEvents"/> and saves to storage with a last built time of <paramref name="lastBuilt"/> and a last updated time of <paramref name="lastUpdated"/>.`
`15`	`15`	`/// </summary>`
`16`		`- Task Serialize(DateTime cursor, IComponent rootComponent, IEnumerable<Event> recentEvents);`
	`16`	`+ Task Serialize(DateTime lastBuilt, DateTime lastUpdated, IComponent rootComponent, IEnumerable<Event> recentEvents);`
`17`	`17`	`}`
`18`	`18`	`}`
Original file line number	Diff line number	Diff line change
`@@ -5,36 +5,43 @@`
`5`	`5`	`using System.Threading.Tasks;`
`6`	`6`	`using Microsoft.Extensions.Logging;`
`7`	`7`	`using NuGet.Jobs.Extensions;`
	`8`	`+using StatusAggregator.Collector;`
	`9`	`+using StatusAggregator.Update;`
`8`	`10`
`9`	`11`	`namespace StatusAggregator.Export`
`10`	`12`	`{`
`11`	`13`	`public class StatusExporter : IStatusExporter`
`12`	`14`	`{`
	`15`	`+ private readonly ICursor _cursor;`
`13`	`16`	`private readonly IComponentExporter _componentExporter;`
`14`	`17`	`private readonly IEventsExporter _eventExporter;`
`15`	`18`	`private readonly IStatusSerializer _serializer;`
`16`	`19`
`17`	`20`	`private readonly ILogger<StatusExporter> _logger;`
`18`	`21`
`19`	`22`	`public StatusExporter(`
	`23`	`+ ICursor cursor,`
`20`	`24`	`IComponentExporter componentExporter,`
`21`	`25`	`IEventsExporter eventExporter,`
`22`	`26`	`IStatusSerializer serializer,`
`23`	`27`	`ILogger<StatusExporter> logger)`
`24`	`28`	`{`
	`29`	`+ _cursor = cursor ?? throw new ArgumentNullException(nameof(cursor));`
`25`	`30`	`_componentExporter = componentExporter ?? throw new ArgumentNullException(nameof(componentExporter));`
`26`	`31`	`_eventExporter = eventExporter ?? throw new ArgumentNullException(nameof(eventExporter));`
`27`	`32`	`_serializer = serializer ?? throw new ArgumentNullException(nameof(serializer));`
`28`	`33`	`_logger = logger ?? throw new ArgumentNullException(nameof(logger));`
`29`	`34`	`}`
`30`	`35`
`31`		`- public Task Export(DateTime cursor)`
	`36`	`+ public async Task Export(DateTime cursor)`
`32`	`37`	`{`
`33`	`38`	`using (_logger.Scope("Exporting service status."))`
`34`	`39`	`{`
`35`	`40`	`var rootComponent = _componentExporter.Export();`
`36`	`41`	`var recentEvents = _eventExporter.Export(cursor);`
`37`		`- return _serializer.Serialize(cursor, rootComponent, recentEvents);`
	`42`	`+`
	`43`	`+ var lastUpdated = await _cursor.Get(StatusUpdater.LastUpdatedCursorName);`
	`44`	`+ await _serializer.Serialize(cursor, lastUpdated, rootComponent, recentEvents);`
`38`	`45`	`}`
`39`	`46`	`}`
`40`	`47`	`}`
Original file line number	Diff line number	Diff line change
`@@ -36,13 +36,13 @@ public StatusSerializer(`
`36`	`36`	`_logger = logger ?? throw new ArgumentNullException(nameof(logger));`
`37`	`37`	`}`
`38`	`38`
`39`		`- public async Task Serialize(DateTime cursor, IComponent rootComponent, IEnumerable<Event> recentEvents)`
	`39`	`+ public async Task Serialize(DateTime lastBuilt, DateTime lastUpdated, IComponent rootComponent, IEnumerable<Event> recentEvents)`
`40`	`40`	`{`
`41`	`41`	`ServiceStatus status;`
`42`	`42`	`string statusJson;`
`43`	`43`	`using (_logger.Scope("Serializing service status."))`
`44`	`44`	`{`
`45`		`- status = new ServiceStatus(cursor, rootComponent, recentEvents);`
	`45`	`+ status = new ServiceStatus(lastBuilt, lastUpdated, rootComponent, recentEvents);`
`46`	`46`	`statusJson = JsonConvert.SerializeObject(status, Settings);`
`47`	`47`	`}`
`48`	`48`
Original file line number	Diff line number	Diff line change
`@@ -6,5 +6,6 @@ public static class LogEvents`
`6`	`6`	`{`
`7`	`7`	`public static EventId RegexFailure = new EventId(400, "Failed to parse incident using Regex.");`
`8`	`8`	`public static EventId ManualChangeFailure = new EventId(401, "Failed to apply a manual change.");`
	`9`	`+ public static EventId IncidentIngestionFailure = new EventId(402, "Failed to update incident API data.");`
`9`	`10`	`}`
`10`	`11`	`}`
Original file line number	Diff line number	Diff line change
`@@ -13,8 +13,7 @@ namespace StatusAggregator.Update`
`13`	`13`	`{`
`14`	`14`	`public class StatusUpdater : IStatusUpdater`
`15`	`15`	`{`
`16`		`- private const string ManualCursorBaseName = "manual";`
`17`		`- private const string IncidentCursorName = "incident";`
	`16`	`+ public const string LastUpdatedCursorName = "updated";`
`18`	`17`
`19`	`18`	`private readonly ICursor _cursor;`
`20`	`19`	`private readonly IEntityCollector _incidentCollector;`
`@@ -50,8 +49,17 @@ public async Task Update(DateTime cursor)`
`50`	`49`	`await manualStatusChangeCollector.FetchLatest();`
`51`	`50`	`}`
`52`	`51`
`53`		`- await _incidentCollector.FetchLatest();`
`54`		`- await _activeEventUpdater.UpdateAllAsync(cursor);`
	`52`	`+ try`
	`53`	`+ {`
	`54`	`+ await _incidentCollector.FetchLatest();`
	`55`	`+ await _activeEventUpdater.UpdateAllAsync(cursor);`
	`56`	`+`
	`57`	`+ await _cursor.Set(LastUpdatedCursorName, cursor);`
	`58`	`+ }`
	`59`	`+ catch (Exception e)`
	`60`	`+ {`
	`61`	`+ _logger.LogError(LogEvents.IncidentIngestionFailure, e, "Failed to update incident API data.");`
	`62`	`+ }`
`55`	`63`	`}`
`56`	`64`	`}`
`57`	`65`