[Revalidation] Improve initialization (#509)

The revalidation job prioritizes packages that are installed by Visual Studio or .NET SDK. Before, the revalidation job required that Visual Studio and the .NET SDK were installed on the VM so that it could find these packages.

This change removes that requirement by embedding a list of known preinstalled packages.

Addresses https://github.com/NuGet/Engineering/issues/1545

Author: loic-sharma

src/NuGet.Services.Revalidate/Initialization/PackageFinder.cs

@@ -8,6 +8,7 @@
 using System.Linq.Expressions;
 using System.Threading;
 using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
 using NuGet.Versioning;
 using NuGetGallery;
 
@@ -22,6 +23,8 @@ public class PackageFinder : IPackageFinder
         public const string DependencySetName = "Dependency";
         public const string RemainingSetName = "Remaining";
 
+        private const string PreinstalledPackagesResource = "NuGet.Services.Revalidate.Initialization.PreinstalledPackages.json";
+
         private static int BatchSize = 1000;
         private static string MicrosoftAccountName = "Microsoft";
 
@@ -46,16 +49,13 @@ public HashSet<int> FindMicrosoftPackages()
 
         public HashSet<int> FindPreinstalledPackages(HashSet<int> except)
         {
-            var preinstalledPackagesNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+            List<string> preinstalledPackagesNames;
+            var assembly = typeof(PackageFinder).Assembly;
 
-            foreach (var path in _config.PreinstalledPaths)
+            using (var resource = assembly.GetManifestResourceStream(PreinstalledPackagesResource))
+            using (var reader = new StreamReader(resource))
             {
-                var expandedPath = Environment.ExpandEnvironmentVariables(path);
-                var packagesInPath = Directory.GetDirectories(expandedPath)
-                    .Select(d => d.Replace(expandedPath, "").Trim('\\').ToLowerInvariant())
-                    .Where(d => !d.StartsWith("."));
-
-                preinstalledPackagesNames.UnionWith(packagesInPath);
+                preinstalledPackagesNames = JsonConvert.DeserializeObject<List<string>>(reader.ReadToEnd());
             }
 
             var preinstalledPackages = FindRegistrationKeys(PreinstalledSetName, r => preinstalledPackagesNames.Contains(r.Id));

src/NuGet.Services.Revalidate/Initialization/PreinstalledPackages.json

@@ -4,12 +4,15 @@
 using System;
 using System.Collections.Generic;
 using System.ComponentModel.Design;
+using System.IO;
+using System.Linq;
 using System.Threading.Tasks;
 using Autofac;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using Newtonsoft.Json;
 using NuGet.Jobs;
 using NuGet.Jobs.Configuration;
 using NuGet.Jobs.Validation;
@@ -25,19 +28,22 @@ namespace NuGet.Services.Revalidate
 
     public class Job : JsonConfigurationJob
     {
+        private const string RebuildPreinstalledSetArgumentName = "RebuildPreinstalledSet";
         private const string InitializeArgumentName = "Initialize";
         private const string VerifyInitializationArgumentName = "VerifyInitialization";
         private const string JobConfigurationSectionName = "RevalidateJob";
 
         private static readonly TimeSpan RetryLaterSleepDuration = TimeSpan.FromMinutes(5);
 
+        private string _preinstalledSetPath;
         private bool _initialize;
         private bool _verifyInitialization;
 
         public override void Init(IServiceContainer serviceContainer, IDictionary<string, string> jobArgsDictionary)
         {
             base.Init(serviceContainer, jobArgsDictionary);
 
+            _preinstalledSetPath = JobConfigurationManager.TryGetArgument(jobArgsDictionary, RebuildPreinstalledSetArgumentName);
             _initialize = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, InitializeArgumentName);
             _verifyInitialization = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, VerifyInitializationArgumentName);
 
@@ -56,7 +62,28 @@ public override async Task Run()
         {
             using (var scope = _serviceProvider.CreateScope())
             {
-                if (_initialize || _verifyInitialization)
+                if (!string.IsNullOrEmpty(_preinstalledSetPath))
+                {
+                    Logger.LogInformation("Rebuilding the preinstalled packages set...");
+
+                    var config = scope.ServiceProvider.GetRequiredService<InitializationConfiguration>();
+                    var preinstalledPackagesNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+
+                    foreach (var path in config.PreinstalledPaths)
+                    {
+                        var expandedPath = Environment.ExpandEnvironmentVariables(path);
+                        var packagesInPath = Directory.GetDirectories(expandedPath)
+                            .Select(d => d.Replace(expandedPath, "").Trim('\\').ToLowerInvariant())
+                            .Where(d => !d.StartsWith("."));
+
+                        preinstalledPackagesNames.UnionWith(packagesInPath);
+                    }
+
+                    File.WriteAllText(_preinstalledSetPath, JsonConvert.SerializeObject(preinstalledPackagesNames));
+
+                    Logger.LogInformation("Rebuilt the preinstalled package set. Found {PreinstalledPackages} package ids", preinstalledPackagesNames.Count);
+                }
+                else if (_initialize || _verifyInitialization)
                 {
                     var initializer = scope.ServiceProvider.GetRequiredService<InitializationManager>();
 

src/NuGet.Services.Revalidate/Job.cs

@@ -89,6 +89,9 @@
   <ItemGroup>
     <Content Include="Scripts\nssm.exe" />
   </ItemGroup>
+  <ItemGroup>
+    <EmbeddedResource Include="Initialization\PreinstalledPackages.json" />
+  </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\NuGet.Jobs.Common\NuGet.Jobs.Common.csproj">
       <Project>{4B4B1EFB-8F33-42E6-B79F-54E7F3293D31}</Project>

src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj

@@ -10,7 +10,7 @@ class Program
         static void Main(string[] args)
         {
             var job = new Job();
-            JobRunner.Run(job, args).GetAwaiter().GetResult();
+            JobRunner.RunOnce(job, args).GetAwaiter().GetResult();
         }
     }
 }

src/NuGet.Services.Revalidate/Program.cs

@@ -3,11 +3,11 @@
     "Initialization": {
       "PreinstalledPaths": [
         "C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages",
-        "%USERPROFILE%\\.dotnet\\NuGetFallbackFolder",
-        "%USERPROFILE%\\.dotnet\\NuGetFallbackFolder\\.tools"
+        "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder",
+        "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder\\.tools"
       ],
       "MaxPackageCreationDate": "2021-03-01T23:52:40.7022034+00:00", // TODO: Update this when repository signing is enabled
-      "SleepDurationBetweenBatches": "00:00:01"
+      "SleepDurationBetweenBatches": "00:00:30"
     },
 
     "Health": {

src/NuGet.Services.Revalidate/Settings/dev.json

@@ -3,8 +3,8 @@
     "Initialization": {
       "PreinstalledPaths": [
         "C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages",
-        "%USERPROFILE%\\.dotnet\\NuGetFallbackFolder",
-        "%USERPROFILE%\\.dotnet\\NuGetFallbackFolder\\.tools"
+        "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder",
+        "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder\\.tools"
       ],
       "MaxPackageCreationDate": "2021-03-01T23:52:40.7022034+00:00", // TODO: Update this when repository signing is enabled
       "SleepDurationBetweenBatches": "00:00:30"

src/NuGet.Services.Revalidate/Settings/int.json

@@ -3,8 +3,8 @@
     "Initialization": {
       "PreinstalledPaths": [
         "C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages",
-        "%USERPROFILE%\\.dotnet\\NuGetFallbackFolder",
-        "%USERPROFILE%\\.dotnet\\NuGetFallbackFolder\\.tools"
+        "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder",
+        "C:\\Program Files\\dotnet\\sdk\\NuGetFallbackFolder\\.tools"
       ],
       "MaxPackageCreationDate": "2021-03-01T23:52:40.7022034+00:00", // TODO: Update this when repository signing is enabled
       "SleepDurationBetweenBatches": "00:00:30"

src/NuGet.Services.Revalidate/Settings/prod.json

@@ -3,18 +3,31 @@
 This job enqueues packages revalidation as fast as possible without affecting the
 health of NuGet's ingestion pipeline. It does so in two phases:
 
-1. Initialization phase - the job determines which packages should be revalidated.
-2. Revalidation phase - packages are enqueued for revalidations
+1. Build Preinstalled Packages phase - the job builds a JSON file of  packages that
+are installed by .NET SDK and Visual Studio
+2. Initialization phase - the job determines which packages should be revalidated.
+3. Revalidation phase - packages are enqueued for revalidations
 
-The initialization phase MUST complete before the revalidation phase is started.
+These phase MUST be completed in order.
+
+# The Build Preinstalled Packages phase
+
+This phase should run be at development time before the job is deployed:
+
+```
+NuGet.Services.Revalidate.exe ^
+    -Configuration "C:\Path\to\job\Settings\dev.json" ^
+    -RebuildPreinstalledSet "C:\Path\to\job\Initialization\PreinstalledPackages.json" ^
+    -Once
+```
 
 # The Initialization Phase
 
 To initialize the job, run:
 
 ```
 NuGet.Services.Revalidate.exe ^
-    -Configuration "C:\Path\to\config.json" ^
+    -Configuration "C:\Path\to\job\Settings\dev.json" ^
     -Initialize
     -VerifyInitialization
     -Once
@@ -36,6 +49,6 @@ To enqueue revalidations, run:
 
 ```
 NuGet.Services.Revalidate.exe ^
-    -Configuration "C:\Path\to\config.json" ^
+    -Configuration "C:\Path\to\job\Settings\dev.json" ^
     -Initialize
 ```

src/NuGet.Services.Revalidate/readme.md

@@ -74,8 +74,8 @@ public void FindsPreinstalledPackages()
             {
                 _context.Mock(packageRegistrations: new[]
                 {
-                    new PackageRegistration { Key = 1, Id = "System.Linq" },
-                    new PackageRegistration { Key = 2, Id = "Newtonsoft.Json" }
+                    new PackageRegistration { Key = 1, Id = "system.linq" },
+                    new PackageRegistration { Key = 2, Id = "newtonsoft.json" }
                 });
 
                 var actual = _target.FindPreinstalledPackages(except: new HashSet<int>());
@@ -89,8 +89,8 @@ public void SkipsPackagesInExceptSet()
             {
                 _context.Mock(packageRegistrations: new[]
                 {
-                    new PackageRegistration { Key = 1, Id = "System.Linq" },
-                    new PackageRegistration { Key = 2, Id = "Newtonsoft.Json" }
+                    new PackageRegistration { Key = 1, Id = "system.linq" },
+                    new PackageRegistration { Key = 2, Id = "newtonsoft.json" }
                 });
 
                 var actual = _target.FindPreinstalledPackages(except: new HashSet<int> { 1 });
@@ -104,7 +104,7 @@ public void SkipsPackagesWithNoRegistration()
             {
                 _context.Mock(packageRegistrations: new[]
                 {
-                    new PackageRegistration { Key = 1, Id = "System.Linq" },
+                    new PackageRegistration { Key = 1, Id = "system.linq" },
                 });
 
                 var actual = _target.FindPreinstalledPackages(except: new HashSet<int>());