Merge pull request #591 from NuGet/dev

[ReleasePrep][2018.10.22]FI of master into dev

Author: loic-sharma

src/NuGet.Services.Revalidate/Job.cs

@@ -32,6 +32,7 @@ public class Job : ValidationJobBase
         private const string RebuildPreinstalledSetArgumentName = "RebuildPreinstalledSet";
         private const string InitializeArgumentName = "Initialize";
         private const string VerifyInitializationArgumentName = "VerifyInitialization";
+
         private const string JobConfigurationSectionName = "RevalidateJob";
 
         private static readonly TimeSpan RetryLaterSleepDuration = TimeSpan.FromMinutes(5);
@@ -112,6 +113,7 @@ await scope.ServiceProvider
         protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot)
         {
             services.Configure<RevalidationConfiguration>(configurationRoot.GetSection(JobConfigurationSectionName));
+
             services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value);
             services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.Initialization);
             services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.Health);

src/NuGet.Services.Revalidate/Settings/dev.json

@@ -16,8 +16,8 @@
       "ComponentPath": "NuGet/Package Publishing"
     },
 
-    "MinPackageEventRate": 120,
-    "MaxPackageEventRate": 400,
+    "MinPackageEventRate": "#{Jobs.nuget.services.revalidation.MinPackageEventRate}",
+    "MaxPackageEventRate": "#{Jobs.nuget.services.revalidation.MaxPackageEventRate}",
 
     "RetryLaterSleep": "00:00:30",
 

src/NuGet.Services.Revalidate/Settings/int.json

@@ -16,8 +16,8 @@
       "ComponentPath": "NuGet/Package Publishing"
     },
 
-    "MinPackageEventRate": 120,
-    "MaxPackageEventRate": 400,
+    "MinPackageEventRate": "#{Jobs.nuget.services.revalidation.MinPackageEventRate}",
+    "MaxPackageEventRate": "#{Jobs.nuget.services.revalidation.MaxPackageEventRate}",
 
     "RetryLaterSleep": "00:00:30",
 

src/NuGet.Services.Revalidate/Settings/prod.json

@@ -16,8 +16,8 @@
       "ComponentPath": "NuGet/Package Publishing"
     },
 
-    "MinPackageEventRate": 120,
-    "MaxPackageEventRate": 400,
+    "MinPackageEventRate": "#{Jobs.nuget.services.revalidation.MinPackageEventRate}",
+    "MaxPackageEventRate": "#{Jobs.nuget.services.revalidation.MaxPackageEventRate}",
 
     "RetryLaterSleep": "00:00:30",
 

src/NuGet.Services.Validation.Orchestrator/Services/IEntityService.cs

@@ -17,9 +17,15 @@ public interface IEntityService<T> where T : class, IEntity
         /// </summary>
         /// <param name="id">The id .</param>
         /// <param name="version">The version.</param>
-        /// <returns></returns>
+        /// <returns>The entity.</returns>
         IValidatingEntity<T> FindPackageByIdAndVersionStrict(string id, string version);
 
+        /// <summary>
+        /// Find the entity based on the key.
+        /// </summary>
+        /// <returns>The entity.</returns>
+        IValidatingEntity<T> FindPackageByKey(int key);
+
         /// <summary>
         /// Update the status of the entity.
         /// </summary>
@@ -35,7 +41,7 @@ public interface IEntityService<T> where T : class, IEntity
         /// <param name="entity">The entity.</param>
         /// <param name="metadata">The metadata.</param>
         /// <param name="commitChanges">True if the changes will be commited to the database.</param>
-        /// <returns></returns>
+        /// <returns>A <see cref="Task"/> that can be used to await for the operation completion.</returns>
         Task UpdateMetadataAsync(T entity, object metadata, bool commitChanges = true);
     }
 }

src/NuGet.Services.Validation.Orchestrator/Services/PackageEntityService.cs

@@ -1,6 +1,9 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
+using System.Linq;
+using System.Data.Entity;
 using System.Threading.Tasks;
 using NuGetGallery;
 using NuGetGallery.Packaging;
@@ -13,10 +16,12 @@ namespace NuGet.Services.Validation.Orchestrator
     public class PackageEntityService : IEntityService<Package>
     {
         private ICorePackageService _galleryEntityService;
+        private IEntityRepository<Package> _packageRepository;
 
-        public PackageEntityService(ICorePackageService galleryEntityService)
+        public PackageEntityService(ICorePackageService galleryEntityService, IEntityRepository<Package> packageRepository)
         {
-            _galleryEntityService = galleryEntityService;
+            _galleryEntityService = galleryEntityService ?? throw new ArgumentNullException(nameof(galleryEntityService));
+            _packageRepository = packageRepository ?? throw new ArgumentNullException(nameof(packageRepository));
         }
 
         public IValidatingEntity<Package> FindPackageByIdAndVersionStrict(string id, string version)
@@ -25,6 +30,18 @@ public IValidatingEntity<Package> FindPackageByIdAndVersionStrict(string id, str
             return p == null ? null : new PackageValidatingEntity(p);
         }
 
+        public IValidatingEntity<Package> FindPackageByKey(int key)
+        {
+            var package = _packageRepository
+                   .GetAll()
+                   .Include(p => p.LicenseReports)
+                   .Include(p => p.PackageRegistration)
+                   .Include(p => p.User)
+                   .Include(p => p.SymbolPackages)
+                   .SingleOrDefault(p => p.Key == key);
+            return package == null ? null : new PackageValidatingEntity(package);
+        }
+
         public async Task UpdateStatusAsync(Package entity, PackageStatus newStatus, bool commitChanges = true)
         {
             await _galleryEntityService.UpdatePackageStatusAsync(entity, newStatus, commitChanges);

src/NuGet.Services.Validation.Orchestrator/Services/SymbolEntityService.cs

@@ -13,10 +13,13 @@ namespace NuGet.Services.Validation.Orchestrator
     /// </summary>
     public class SymbolEntityService : IEntityService<SymbolPackage>
     {
-        ICoreSymbolPackageService _galleryEntityService;
-        public SymbolEntityService(ICoreSymbolPackageService galleryEntityService)
+        private ICoreSymbolPackageService _galleryEntityService;
+        private IEntityRepository<SymbolPackage> _symbolsPackageRepository;
+
+        public SymbolEntityService(ICoreSymbolPackageService galleryEntityService, IEntityRepository<SymbolPackage> symbolsPackageRepository)
         {
             _galleryEntityService = galleryEntityService ?? throw new ArgumentNullException(nameof(galleryEntityService));
+            _symbolsPackageRepository = symbolsPackageRepository ?? throw new ArgumentNullException(nameof(symbolsPackageRepository));
         }
 
         /// <summary>
@@ -35,6 +38,14 @@ public IValidatingEntity<SymbolPackage> FindPackageByIdAndVersionStrict(string i
             return symbolPackage == null ? null : new SymbolPackageValidatingEntity(symbolPackage);
         }
 
+        public IValidatingEntity<SymbolPackage> FindPackageByKey(int key)
+        {
+            var symbolPackage = _symbolsPackageRepository
+                   .GetAll()
+                   .SingleOrDefault(p => p.Key == key);
+            return symbolPackage == null ? null : new SymbolPackageValidatingEntity(symbolPackage);
+        }
+
         public async Task UpdateStatusAsync(SymbolPackage entity, PackageStatus newStatus, bool commitChanges = true)
         {
             if(entity == null)

src/NuGet.Services.Validation.Orchestrator/SymbolValidationMessageHandler.cs

@@ -73,7 +73,11 @@ public async Task<bool> HandleAsync(PackageValidationMessageData message)
                 message.PackageNormalizedVersion,
                 message.ValidationTrackingId))
             {
-                var symbolPackageEntity = _gallerySymbolService.FindPackageByIdAndVersionStrict(message.PackageId, message.PackageNormalizedVersion);
+                // When a message is sent from the Gallery with validation of a new entity, the EntityKey will be null because the message is sent to the service bus before the entity is persisted in the DB
+                // However when a revalidation happens or when the message is re-sent by the orchestrator the message will contain the key. In this case the key is used to find the entity to validate.
+                var symbolPackageEntity = message.EntityKey.HasValue
+                    ? _gallerySymbolService.FindPackageByKey(message.EntityKey.Value)
+                    : _gallerySymbolService.FindPackageByIdAndVersionStrict(message.PackageId, message.PackageNormalizedVersion);
 
                 if (symbolPackageEntity == null)
                 {
@@ -94,9 +98,10 @@ public async Task<bool> HandleAsync(PackageValidationMessageData message)
                     }
                     else
                     {
-                        _logger.LogInformation("Could not find symbols for package {PackageId} {PackageNormalizedVersion} in DB, retrying",
+                        _logger.LogInformation("Could not find symbols for package {PackageId} {PackageNormalizedVersion} {Key} in DB, retrying",
                             message.PackageId,
-                            message.PackageNormalizedVersion);
+                            message.PackageNormalizedVersion,
+                            message.EntityKey.HasValue);
 
                         return false;
                     }
@@ -114,6 +119,7 @@ public async Task<bool> HandleAsync(PackageValidationMessageData message)
                 }
 
                 var processorStats = await _validationSetProcessor.ProcessValidationsAsync(validationSet);
+                // As part of the processing the validation outcome the orchestrator will send itself a message if validation are still being processed.
                 await _validationOutcomeProcessor.ProcessValidationOutcomeAsync(validationSet, symbolPackageEntity, processorStats);
             }
 

src/NuGet.Services.Validation.Orchestrator/Symbols/SymbolScanValidator.cs

@@ -113,7 +113,7 @@ private bool ShouldSkipScan(IValidationRequest request)
         {
             var symbolPackage = _symbolPackageService
                 .FindSymbolPackagesByIdAndVersion(request.PackageId,request.PackageVersion)
-                .FirstOrDefault(sp => sp.StatusKey == PackageStatus.Validating);
+                .FirstOrDefault(sp => sp.Key == request.PackageKey);
 
             if (symbolPackage == null)
             {

src/NuGet.Services.Validation.Orchestrator/ValidationOutcomeProcessor.cs

@@ -264,7 +264,12 @@ private async Task ScheduleCheckIfNotTimedOut(PackageValidationSet validationSet
             // Schedule another check if we haven't reached the validation set timeout yet.
             if (validationSetDuration <= _validationConfiguration.TimeoutValidationSetAfter)
             {
-                var messageData = new PackageValidationMessageData(validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId, validationSet.ValidatingType);
+                var messageData = new PackageValidationMessageData(
+                    validationSet.PackageId,
+                    validationSet.PackageNormalizedVersion,
+                    validationSet.ValidationTrackingId,
+                    validationSet.ValidatingType,
+                    entityKey: validationSet.PackageKey);
                 var postponeUntil = DateTimeOffset.UtcNow + _validationConfiguration.ValidationMessageRecheckPeriod;
 
                 await _validationEnqueuer.StartValidationAsync(messageData, postponeUntil);