Merge pull request #676 from NuGet/dev

[ReleasePrep][2018.11.20]RI of dev into master

Author: loic-sharma

src/NuGet.Services.Revalidate/Services/RevalidationJobStateService.cs

@@ -104,7 +104,8 @@ await _state.MaybeUpdateStateAsync(state =>
                     return false;
                 }
 
-                var nextRate = Math.Min(_config.MaxPackageEventRate, state.DesiredPackageEventRate + 1);
+                var nextRate = state.DesiredPackageEventRate + _config.Queue.MaxBatchSize;
+                nextRate = Math.Min(_config.MaxPackageEventRate, nextRate);
 
                 _logger.LogInformation(
                     "Increasing desired package event rate to {ToRate} from {FromRate}",

src/StatusAggregator/Export/ComponentExporter.cs

@@ -71,7 +71,7 @@ public IComponent Export()
                 foreach (var activeEntity in activeEntities)
                 {
                     using (_logger.Scope("Applying active entity affecting {AffectedComponentPath} of severity {AffectedComponentStatus} at {StartTime} to root component",
-                        activeEntity.AffectedComponentPath, activeEntity.AffectedComponentStatus, activeEntity.StartTime))
+                        activeEntity.AffectedComponentPath, (ComponentStatus)activeEntity.AffectedComponentStatus, activeEntity.StartTime))
                     {
                         var currentComponent = rootComponent.GetByPath(activeEntity.AffectedComponentPath);
 

src/StatusAggregator/Export/IStatusSerializer.cs

@@ -11,8 +11,8 @@ namespace StatusAggregator.Export
     public interface IStatusSerializer
     {
         /// <summary>
-        /// Serializes <paramref name="rootComponent"/> and <paramref name="recentEvents"/> and saves to storage with a time of <paramref name="cursor"/>.
+        /// Serializes <paramref name="rootComponent"/> and <paramref name="recentEvents"/> and saves to storage with a last built time of <paramref name="lastBuilt"/> and a last updated time of <paramref name="lastUpdated"/>.
         /// </summary>
-        Task Serialize(DateTime cursor, IComponent rootComponent, IEnumerable<Event> recentEvents);
+        Task Serialize(DateTime lastBuilt, DateTime lastUpdated, IComponent rootComponent, IEnumerable<Event> recentEvents);
     }
 }

src/StatusAggregator/Export/StatusExporter.cs

@@ -5,36 +5,43 @@
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using NuGet.Jobs.Extensions;
+using StatusAggregator.Collector;
+using StatusAggregator.Update;
 
 namespace StatusAggregator.Export
 {
     public class StatusExporter : IStatusExporter
     {
+        private readonly ICursor _cursor;
         private readonly IComponentExporter _componentExporter;
         private readonly IEventsExporter _eventExporter;
         private readonly IStatusSerializer _serializer;
 
         private readonly ILogger<StatusExporter> _logger;
 
         public StatusExporter(
+            ICursor cursor,
             IComponentExporter componentExporter,
             IEventsExporter eventExporter,
             IStatusSerializer serializer,
             ILogger<StatusExporter> logger)
         {
+            _cursor = cursor ?? throw new ArgumentNullException(nameof(cursor));
             _componentExporter = componentExporter ?? throw new ArgumentNullException(nameof(componentExporter));
             _eventExporter = eventExporter ?? throw new ArgumentNullException(nameof(eventExporter));
             _serializer = serializer ?? throw new ArgumentNullException(nameof(serializer));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
 
-        public Task Export(DateTime cursor)
+        public async Task Export(DateTime cursor)
         {
             using (_logger.Scope("Exporting service status."))
             {
                 var rootComponent = _componentExporter.Export();
                 var recentEvents = _eventExporter.Export(cursor);
-                return _serializer.Serialize(cursor, rootComponent, recentEvents);
+
+                var lastUpdated = await _cursor.Get(StatusUpdater.LastUpdatedCursorName);
+                await _serializer.Serialize(cursor, lastUpdated, rootComponent, recentEvents);
             }
         }
     }

src/StatusAggregator/Export/StatusSerializer.cs

@@ -36,13 +36,13 @@ public StatusSerializer(
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
 
-        public async Task Serialize(DateTime cursor, IComponent rootComponent, IEnumerable<Event> recentEvents)
+        public async Task Serialize(DateTime lastBuilt, DateTime lastUpdated, IComponent rootComponent, IEnumerable<Event> recentEvents)
         {
             ServiceStatus status;
             string statusJson;
             using (_logger.Scope("Serializing service status."))
             {
-                status = new ServiceStatus(cursor, rootComponent, recentEvents);
+                status = new ServiceStatus(lastBuilt, lastUpdated, rootComponent, recentEvents);
                 statusJson = JsonConvert.SerializeObject(status, Settings);
             }
 

src/StatusAggregator/Factory/NuGetServiceComponentFactory.cs

@@ -34,7 +34,7 @@ public IComponent Create()
                 "",
                 new IComponent[]
                 {
-                    new PrimarySecondaryComponent(
+                    new ActivePassiveComponent(
                         GalleryName,
                         "Browsing the Gallery website",
                         new[]
@@ -55,7 +55,7 @@ public IComponent Create()
                                     new LeafComponent(GlobalRegionName, "V3 restore for users outside of China"),
                                     new LeafComponent(ChinaRegionName, "V3 restore for users inside China")
                                 }),
-                            new PrimarySecondaryComponent(
+                            new ActiveActiveComponent(
                                 V2ProtocolName,
                                 "Restore using the V2 API",
                                 new[]
@@ -69,15 +69,15 @@ public IComponent Create()
                         "Searching for new and existing packages in Visual Studio or the Gallery website",
                         new[]
                         {
-                            new PrimarySecondaryComponent(
+                            new ActiveActiveComponent(
                                 GlobalRegionName,
                                 "Search for packages outside China",
                                 new[]
                                 {
                                     new LeafComponent(UsncInstanceName, "Primary region"),
                                     new LeafComponent(UsscInstanceName, "Backup region")
                                 }),
-                            new PrimarySecondaryComponent(
+                            new ActiveActiveComponent(
                                 ChinaRegionName,
                                 "Search for packages inside China",
                                 new[]

src/StatusAggregator/LogEvents.cs

@@ -6,5 +6,6 @@ public static class LogEvents
     {
         public static EventId RegexFailure = new EventId(400, "Failed to parse incident using Regex.");
         public static EventId ManualChangeFailure = new EventId(401, "Failed to apply a manual change.");
+        public static EventId IncidentIngestionFailure = new EventId(402, "Failed to update incident API data.");
     }
 }

src/StatusAggregator/Parse/IncidentRegexParser.cs

@@ -56,7 +56,7 @@ public bool TryParseIncident(Incident incident, out ParsedIncident parsedInciden
                     return false;
                 }
 
-                _logger.LogInformation("RegEx match result: {MatchResult}", title, match.Success);
+                _logger.LogInformation("RegEx match result: {MatchResult}", match.Success);
                 return match.Success && TryParseIncident(incident, match.Groups, out parsedIncident);
             }
         }

src/StatusAggregator/StatusAggregator.csproj

@@ -157,13 +157,13 @@
       <Version>1.1.1</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Incidents">
-      <Version>2.33.0</Version>
+      <Version>2.38.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Status">
-      <Version>2.33.0</Version>
+      <Version>2.38.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Status.Table">
-      <Version>2.33.0</Version>
+      <Version>2.38.0</Version>
     </PackageReference>
     <PackageReference Include="WindowsAzure.Storage">
       <Version>9.2.0</Version>

src/StatusAggregator/Update/StatusUpdater.cs

@@ -13,8 +13,7 @@ namespace StatusAggregator.Update
 {
     public class StatusUpdater : IStatusUpdater
     {
-        private const string ManualCursorBaseName = "manual";
-        private const string IncidentCursorName = "incident";
+        public const string LastUpdatedCursorName = "updated";
 
         private readonly ICursor _cursor;
         private readonly IEntityCollector _incidentCollector;
@@ -50,8 +49,17 @@ public async Task Update(DateTime cursor)
                     await manualStatusChangeCollector.FetchLatest();
                 }
 
-                await _incidentCollector.FetchLatest();
-                await _activeEventUpdater.UpdateAllAsync(cursor);
+                try
+                {
+                    await _incidentCollector.FetchLatest();
+                    await _activeEventUpdater.UpdateAllAsync(cursor);
+
+                    await _cursor.Set(LastUpdatedCursorName, cursor);
+                }
+                catch (Exception e)
+                {
+                    _logger.LogError(LogEvents.IncidentIngestionFailure, e, "Failed to update incident API data.");
+                }
             }
         }