[Revalidation] Add Package Revalidation State (#481)

This change adds the state used to track whether the job has been initialized or killswitched. The state is also used to track the desired event rate, which is used to speed up the job up over time.

This change does not include health checks and will not slow down revalidations if the ingestion pipeline is unhealthy. This will be addressed in follow-up changes.

Addresses https://github.com/NuGet/Engineering/issues/1440
Partially addresses https://github.com/nuget/engineering/issues/1441

Author: loic-sharma

src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj

@@ -74,13 +74,13 @@
       <Version>1.50.2</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Configuration">
-      <Version>2.25.0</Version>
+      <Version>2.27.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.KeyVault">
-      <Version>2.26.0-master-33196</Version>
+      <Version>2.27.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Logging">
-      <Version>2.25.0</Version>
+      <Version>2.27.0</Version>
     </PackageReference>
     <PackageReference Include="System.Net.Http">
       <Version>4.3.3</Version>

src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs

@@ -7,6 +7,19 @@ namespace NuGet.Services.Revalidate
 {
     public class RevalidationConfiguration
     {
+        /// <summary>
+        /// The lower limit for the desired package event rate (includes package pushes, lists, unlists, and revalidations).
+        /// If the ingestion pipeline remains healthy, the job will increase its rate over time. If the ingestion pipeline becomes
+        /// unhealthy, the job will reset its rate to this value.
+        /// </summary>
+        public int MinPackageEventRate { get; set; }
+
+        /// <summary>
+        /// The revalidation job will speed up over time. This is the upper limit for the desired package event
+        /// rate (includes package pushes, lists, unlists, and revalidations).
+        /// </summary>
+        public int MaxPackageEventRate { get; set; }
+
         /// <summary>
         /// The time before the revalidation job restarts itself.
         /// </summary>

src/NuGet.Services.Revalidate/Initialization/InitializationManager.cs

@@ -15,29 +15,35 @@ public class InitializationManager
     {
         private static int BatchSize = 1000;
 
-        private readonly IRevalidationStateService _revalidationState;
+        private readonly IRevalidationJobStateService _jobState;
+        private readonly IPackageRevalidationStateService _packageState;
         private readonly IPackageFinder _packageFinder;
         private readonly InitializationConfiguration _config;
         private readonly ILogger<InitializationManager> _logger;
 
         public InitializationManager(
-            IRevalidationStateService revalidationState,
+            IRevalidationJobStateService jobState,
+            IPackageRevalidationStateService packageState,
             IPackageFinder packageFinder,
             InitializationConfiguration config,
             ILogger<InitializationManager> logger)
         {
-            // TODO: Accept service for settings (IsInitialized, etc...)
-            // See: https://github.com/NuGet/Engineering/issues/1440
-            _revalidationState = revalidationState ?? throw new ArgumentNullException(nameof(revalidationState));
+            _jobState = jobState ?? throw new ArgumentNullException(nameof(jobState));
+            _packageState = packageState ?? throw new ArgumentNullException(nameof(packageState));
             _packageFinder = packageFinder ?? throw new ArgumentNullException(nameof(packageFinder));
             _config = config ?? throw new ArgumentNullException(nameof(config));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
 
         public async Task InitializeAsync()
         {
-            // TODO: Check "IsInitialized" setting. If true, error!
-            // See: https://github.com/NuGet/Engineering/issues/1440
+            if (await _jobState.IsInitializedAsync())
+            {
+                _logger.LogError("Attempted to initialize the revalidation job when it is already initialized!");
+
+                throw new InvalidOperationException("Attempted to initialize the revalidation job when it is already initialized!");
+            }
+
             await ClearPackageRevalidationStateAsync();
 
             // Find packages owned by Microsoft or preinstalled by Visual Studio.
@@ -66,16 +72,20 @@ public async Task InitializeAsync()
             await InitializePackageSetAsync(PackageFinder.DependencySetName, dependencyPackages);
             await InitializePackageSetAsync(PackageFinder.RemainingSetName, remainingPackages);
 
-            // TODO: Set "IsInitialized" setting to true
-            // See: https://github.com/NuGet/Engineering/issues/1440
+            await _jobState.MarkAsInitializedAsync();
         }
 
         public async Task VerifyInitializationAsync()
         {
-            // TODO: Check "IsInitialized" setting. If false, error!
-            // See: https://github.com/NuGet/Engineering/issues/1440
+            if (!await _jobState.IsInitializedAsync())
+            {
+                _logger.LogError("Expected revalidation state to be initialized");
+
+                throw new Exception("Expected revalidation state to be initialized");
+            }
+
             var expectedCount = _packageFinder.AppropriatePackageCount();
-            var actualCount = await _revalidationState.PackageRevalidationCountAsync();
+            var actualCount = await _packageState.PackageRevalidationCountAsync();
 
             if (actualCount != expectedCount)
             {
@@ -93,7 +103,7 @@ private async Task ClearPackageRevalidationStateAsync()
 
             do
             {
-                removedRevalidations = await _revalidationState.RemoveRevalidationsAsync(BatchSize);
+                removedRevalidations = await _packageState.RemovePackageRevalidationsAsync(BatchSize);
 
                 if (removedRevalidations > 0)
                 {
@@ -120,9 +130,19 @@ private async Task InitializePackageSetAsync(string setName, HashSet<int> packag
 
             for (var chunkIndex = 0; chunkIndex < chunks.Count; chunkIndex++)
             {
-                // TODO: Check the kill switch
-                // See https://github.com/NuGet/Engineering/issues/1440
-                _logger.LogInformation("Initializing chunk {Chunk} of {Chunks} for package set {SetName}...",
+                while (await _jobState.IsKillswitchActiveAsync())
+                {
+                    _logger.LogInformation(
+                        "Delaying initialization of chunk {Chunk} of {Chunks} for package set {SetName} due to active killswitch",
+                        chunkIndex + 1,
+                        chunks.Count,
+                        setName);
+
+                    await Task.Delay(_config.SleepDurationBetweenBatches);
+                }
+
+                _logger.LogInformation(
+                    "Initializing chunk {Chunk} of {Chunks} for package set {SetName}...",
                     chunkIndex + 1,
                     chunks.Count,
                     setName);
@@ -132,7 +152,8 @@ private async Task InitializePackageSetAsync(string setName, HashSet<int> packag
 
                 await InitializeRevalidationsAsync(chunk, versions);
 
-                _logger.LogInformation("Initialized chunk {Chunk} of {Chunks} for package set {SetName}",
+                _logger.LogInformation(
+                    "Initialized chunk {Chunk} of {Chunks} for package set {SetName}",
                     chunkIndex + 1,
                     chunks.Count,
                     setName);
@@ -184,7 +205,7 @@ private async Task InitializeRevalidationsAsync(
                 }
             }
 
-            await _revalidationState.AddPackageRevalidationsAsync(revalidations);
+            await _packageState.AddPackageRevalidationsAsync(revalidations);
         }
     }
 }

src/NuGet.Services.Revalidate/Job.cs

@@ -109,7 +109,9 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
             services.AddTransient<ITelemetryService, TelemetryService>();
             services.AddTransient<ITelemetryClient, TelemetryClientWrapper>();
 
-            services.AddTransient<IRevalidationStateService, RevalidationStateService>();
+            services.AddTransient<IPackageRevalidationStateService, PackageRevalidationStateService>();
+            services.AddTransient<IRevalidationJobStateService, RevalidationJobStateService>();
+            services.AddTransient<NuGetGallery.IRevalidationStateService, NuGetGallery.RevalidationStateService>();
 
             // Initialization
             services.AddTransient<IPackageFinder, PackageFinder>();

src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj

@@ -54,14 +54,17 @@
     <Compile Include="Services\HealthService.cs" />
     <Compile Include="Services\IHealthService.cs" />
     <Compile Include="Services\IRevalidationQueue.cs" />
-    <Compile Include="Services\IRevalidationStateService.cs" />
+    <Compile Include="Services\IRevalidationJobStateService.cs" />
+    <Compile Include="Services\IPackageRevalidationStateService.cs" />
     <Compile Include="Services\IRevalidationService.cs" />
     <Compile Include="Services\IRevalidationThrottler.cs" />
     <Compile Include="Services\ISingletonService.cs" />
+    <Compile Include="Services\RevalidationOperation.cs" />
     <Compile Include="Services\RevalidationQueue.cs" />
     <Compile Include="Services\RevalidationResult.cs" />
     <Compile Include="Services\RevalidationService.cs" />
-    <Compile Include="Services\RevalidationStateService.cs" />
+    <Compile Include="Services\RevalidationJobStateService.cs" />
+    <Compile Include="Services\PackageRevalidationStateService.cs" />
     <Compile Include="Services\RevalidationThrottler.cs" />
     <Compile Include="Services\SingletonService.cs" />
     <Compile Include="Services\TelemetryService.cs" />
@@ -99,6 +102,11 @@
       <Name>Validation.PackageSigning.Core</Name>
     </ProjectReference>
   </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="NuGet.Services.Storage">
+      <Version>2.27.0</Version>
+    </PackageReference>
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <PropertyGroup>
     <SignPath>..\..\build</SignPath>

…te/Services/IRevalidationStateService.cs …ices/IPackageRevalidationStateService.cssrc/NuGet.Services.Revalidate/Services/IRevalidationStateService.cs renamed to src/NuGet.Services.Revalidate/Services/IPackageRevalidationStateService.cs src/NuGet.Services.Revalidate/Services/IRevalidationStateService.cs renamed to src/NuGet.Services.Revalidate/Services/IPackageRevalidationStateService.cs

@@ -7,37 +7,37 @@
 
 namespace NuGet.Services.Revalidate
 {
-    public interface IRevalidationStateService
+    public interface IPackageRevalidationStateService
     {
-        /// <summary>
-        /// Check whether the killswitch has been activated. If it has, all revalidation operations should be halted.
-        /// </summary>
-        /// <returns>Whether the killswitch has been activated.</returns>
-        Task<bool> IsKillswitchActiveAsync();
-
         /// <summary>
         /// Add the new revalidations to the database.
         /// </summary>
         /// <returns>A task that completes once the revalidations have been saved.</returns>
         Task AddPackageRevalidationsAsync(IReadOnlyList<PackageRevalidation> revalidations);
 
         /// <summary>
-        /// Remove revalidations from the database.
+        /// Remove package revalidation entities from the database.
         /// </summary>
         /// <returns>A task that returns the number of revalidations that have been removed.</returns>
-        Task<int> RemoveRevalidationsAsync(int max);
+        Task<int> RemovePackageRevalidationsAsync(int max);
 
         /// <summary>
         /// Count the number of package revalidations in the database.
         /// </summary>
         /// <returns>The count of package revalidations in the database.</returns>
         Task<int> PackageRevalidationCountAsync();
 
+        /// <summary>
+        /// Count the number of package revalidations that were enqueued in the past hour.
+        /// </summary>
+        /// <returns>The number of enqueued revalidations.</returns>
+        Task<int> CountRevalidationsEnqueuedInPastHourAsync();
+
         /// <summary>
         /// Update the package revalidation and mark is as enqueued.
         /// </summary>
         /// <param name="revalidation">The revalidation to update.</param>
         /// <returns>A task that completes once the revalidation has been updated.</returns>
-        Task MarkRevalidationAsEnqueuedAsync(PackageRevalidation revalidation);
+        Task MarkPackageRevalidationAsEnqueuedAsync(PackageRevalidation revalidation);
     }
 }

src/NuGet.Services.Revalidate/Services/IRevalidationJobStateService.cs

@@ -0,0 +1,50 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Revalidate
+{
+    /// <summary>
+    /// The state shared between the Gallery and the revalidation job.
+    /// </summary>
+    public interface IRevalidationJobStateService
+    {
+        /// <summary>
+        /// Check whether the revalidation state has been initialized.
+        /// </summary>
+        /// <returns>Whether the revalidation state has been initialized.</returns>
+        Task<bool> IsInitializedAsync();
+
+        /// <summary>
+        /// Update the settings to mark the revalidation job as initialized.
+        /// </summary>
+        /// <returns>A task that completes once the settings have been updated.</returns>
+        Task MarkAsInitializedAsync();
+
+        /// <summary>
+        /// Check whether the killswitch has been activated. If it has, all revalidation operations should be halted.
+        /// </summary>
+        /// <returns>Whether the killswitch has been activated.</returns>
+        Task<bool> IsKillswitchActiveAsync();
+
+        /// <summary>
+        /// Determine the desired package event rate per hour. Package events include package pushes,
+        /// edits, and revalidations.
+        /// </summary>
+        /// <returns>The desired maximum number of package events per hour.</returns>
+        Task<int> GetDesiredPackageEventRateAsync();
+
+        /// <summary>
+        /// Reset the desired package event rate to the configured minimum.
+        /// </summary>
+        /// <returns>A task that completes once the rate has been reset.</returns>
+        Task ResetDesiredPackageEventRateAsync();
+
+        /// <summary>
+        /// Increment the desired package event rate per hour.
+        /// </summary>
+        /// <returns>A task that completes once the rate has been incremented.</returns>
+        Task IncreaseDesiredPackageEventRateAsync();
+    }
+}

src/NuGet.Services.Revalidate/Services/IRevalidationThrottler.cs

@@ -13,19 +13,6 @@ public interface IRevalidationThrottler
         /// <returns>If true, no more revalidations should be performed.</returns>
         Task<bool> IsThrottledAsync();
 
-        /// <summary>
-        /// Reset the capacity to the configured minimum value. Call this when the service's status is degraded to
-        /// throttle the revalidations.
-        /// </summary>
-        /// <returns>A task that completes once the capacity theshold has been reset.</returns>
-        Task ResetCapacityAsync();
-
-        /// <summary>
-        /// Increase the revalidation capacity by one revalidation per minute.
-        /// </summary>
-        /// <returns>A task taht completes once the capacity has been increased.</returns>
-        Task IncreaseCapacityAsync();
-
         /// <summary>
         /// Delay the current task to achieve the desired revalidation rate.
         /// </summary>

src/NuGet.Services.Revalidate/Services/ITelemetryService.cs

@@ -1,13 +1,13 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
-using System;
+using NuGet.Services.Logging;
 
 namespace NuGet.Services.Revalidate
 {
     public interface ITelemetryService
     {
-        IDisposable TrackDurationToStartNextRevalidation();
+        DurationMetric<StartNextRevalidationOperation> TrackStartNextRevalidationOperation();
 
         void TrackPackageRevalidationMarkedAsCompleted(string packageId, string normalizedVersion);
 

…ate/Services/RevalidationStateService.cs …vices/PackageRevalidationStateService.cssrc/NuGet.Services.Revalidate/Services/RevalidationStateService.cs renamed to src/NuGet.Services.Revalidate/Services/PackageRevalidationStateService.cs src/NuGet.Services.Revalidate/Services/RevalidationStateService.cs renamed to src/NuGet.Services.Revalidate/Services/PackageRevalidationStateService.cs

@@ -12,23 +12,17 @@
 
 namespace NuGet.Services.Revalidate
 {
-    public class RevalidationStateService : IRevalidationStateService
+    public class PackageRevalidationStateService : IPackageRevalidationStateService
     {
         private readonly IValidationEntitiesContext _context;
-        private readonly ILogger<RevalidationStateService> _logger;
+        private readonly ILogger<PackageRevalidationStateService> _logger;
 
-        public RevalidationStateService(IValidationEntitiesContext context, ILogger<RevalidationStateService> logger)
+        public PackageRevalidationStateService(IValidationEntitiesContext context, ILogger<PackageRevalidationStateService> logger)
         {
             _context = context ?? throw new ArgumentNullException(nameof(context));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
 
-        public Task<bool> IsKillswitchActiveAsync()
-        {
-            // TODO
-            return Task.FromResult(false);
-        }
-
         public async Task AddPackageRevalidationsAsync(IReadOnlyList<PackageRevalidation> revalidations)
         {
             var validationContext = _context as ValidationEntitiesContext;
@@ -53,7 +47,7 @@ public async Task AddPackageRevalidationsAsync(IReadOnlyList<PackageRevalidation
             }
         }
 
-        public async Task<int> RemoveRevalidationsAsync(int max)
+        public async Task<int> RemovePackageRevalidationsAsync(int max)
         {
             var revalidations = await _context.PackageRevalidations
                 .Take(max)
@@ -77,7 +71,16 @@ public async Task<int> PackageRevalidationCountAsync()
             return await _context.PackageRevalidations.CountAsync();
         }
 
-        public async Task MarkRevalidationAsEnqueuedAsync(PackageRevalidation revalidation)
+        public async Task<int> CountRevalidationsEnqueuedInPastHourAsync()
+        {
+            var lowerBound = DateTime.UtcNow.Subtract(TimeSpan.FromHours(1));
+
+            return await _context.PackageRevalidations
+                .Where(r => r.Enqueued >= lowerBound)
+                .CountAsync();
+        }
+
+        public async Task MarkPackageRevalidationAsEnqueuedAsync(PackageRevalidation revalidation)
         {
             try
             {