Merge Dev => Master (#348)

* [Orchestrator] Add maximum retry count for missing packages (#342)

* Change parsing algorithm from download URL to package id+version (#343)

* Change parsing algorithm from download URL to package id+version

* PR comments

* bug fix

* check failing cases

* Fix id+version parsing issue for Microsoft.VisualStudio.Shell.15.0 (#347)

Author: skofman1

src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj

@@ -102,7 +102,7 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="NuGet.Services.Validation.Issues">
-      <Version>2.14.0</Version>
+      <Version>2.15.0</Version>
     </PackageReference>
   </ItemGroup>
   <ItemGroup>

src/NuGet.Services.Validation.Orchestrator/Program.cs

@@ -8,16 +8,14 @@ namespace NuGet.Services.Validation.Orchestrator
 {
     class Program
     {
-        private const string LoggingCategory = "Validation.Orchestrator";
-
         static int Main(string[] args)
         {
             if (!args.Contains(JobArgumentNames.Once))
             {
                 args = args.Concat(new[] { "-" + JobArgumentNames.Once }).ToArray();
             }
             var job = new Job();
-            JobRunner.Run(job, args).Wait();
+            JobRunner.Run(job, args).GetAwaiter().GetResult();
 
             // if configuration validation failed, return non-zero status so we can detect failures in automation
             return job.ConfigurationValidated ? 0 : 1;

src/NuGet.Services.Validation.Orchestrator/Telemetry/ITelemetryService.cs

@@ -79,6 +79,13 @@ public interface ITelemetryService
         /// <param name="clientCode">The client code.</param>
         void TrackClientValidationIssue(string validatorType, string clientCode);
 
+        /// <summary>
+        /// A counter metric emitted when the orchestrator is requested to validate a package,
+        /// but, the package does not exist in the Gallery database even after <see cref="ValidationConfiguration.MissingPackageRetryCount"/>
+        /// retries.
+        /// </summary>
+        void TrackMissingPackageForValidationMessage(string packageId, string normalizedVersion, string validationTrackingId);
+
         /// <summary>
         /// A metric for the case when orchestrator sees a package marked as available, but the blob is missing
         /// in the public container.

src/NuGet.Services.Validation.Orchestrator/Telemetry/TelemetryService.cs

@@ -21,6 +21,7 @@ public class TelemetryService : ITelemetryService
         private const string ValidatorDurationSeconds = Prefix + "ValidatorDurationSeconds";
         private const string ValidatorStarted = Prefix + "ValidatorStarted";
         private const string ClientValidationIssue = Prefix + "ClientValidationIssue";
+        private const string MissingPackageForValidationMessage = Prefix + "MissingPackageForValidationMessage";
         private const string MissingNupkgForAvailablePackage = Prefix + "MissingNupkgForAvailablePackage";
 
         private const string FromStatus = "FromStatus";
@@ -140,6 +141,17 @@ public void TrackClientValidationIssue(string validatorType, string clientCode)
                 });
         }
 
+        public void TrackMissingPackageForValidationMessage(string packageId, string normalizedVersion, string validationTrackingId)
+            => _telemetryClient.TrackMetric(
+                    MissingPackageForValidationMessage,
+                    1,
+                    new Dictionary<string, string>
+                    {
+                        { PackageId, packageId },
+                        { NormalizedVersion, normalizedVersion },
+                        { ValidationTrackingId, validationTrackingId },
+                    });
+
         public void TrackMissingNupkgForAvailablePackage(string packageId, string normalizedVersion, string validationTrackingId)
             => _telemetryClient.TrackMetric(
                     MissingNupkgForAvailablePackage,

src/NuGet.Services.Validation.Orchestrator/ValidationConfiguration.cs

@@ -21,6 +21,12 @@ public class ValidationConfiguration
         /// </summary>
         public string ValidationStorageConnectionString { get; set; }
 
+        /// <summary>
+        /// How many times the Orchestrator should retry to validate a package
+        /// that is missing from the Gallery database.
+        /// </summary>
+        public int MissingPackageRetryCount { get; set; }
+
         /// <summary>
         /// Time to wait between checking the state of a certain validation.
         /// </summary>

src/NuGet.Services.Validation.Orchestrator/ValidationMessageHandler.cs

@@ -4,30 +4,57 @@
 using System;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 using NuGet.Services.ServiceBus;
+using NuGet.Services.Validation.Orchestrator.Telemetry;
 using NuGetGallery;
 
 namespace NuGet.Services.Validation.Orchestrator
 {
     public class ValidationMessageHandler : IMessageHandler<PackageValidationMessageData>
     {
+        private readonly ValidationConfiguration _configs;
         private readonly ICorePackageService _galleryPackageService;
         private readonly IValidationSetProvider _validationSetProvider;
         private readonly IValidationSetProcessor _validationSetProcessor;
         private readonly IValidationOutcomeProcessor _validationOutcomeProcessor;
+        private readonly ITelemetryService _telemetryService;
         private readonly ILogger<ValidationMessageHandler> _logger;
 
         public ValidationMessageHandler(
+            IOptionsSnapshot<ValidationConfiguration> validationConfigsAccessor,
             ICorePackageService galleryPackageService,
             IValidationSetProvider validationSetProvider,
             IValidationSetProcessor validationSetProcessor,
             IValidationOutcomeProcessor validationOutcomeProcessor,
+            ITelemetryService telemetryService,
             ILogger<ValidationMessageHandler> logger)
         {
+            if (validationConfigsAccessor == null)
+            {
+                throw new ArgumentNullException(nameof(validationConfigsAccessor));
+            }
+
+            if (validationConfigsAccessor.Value == null)
+            {
+                throw new ArgumentException(
+                    $"The {nameof(IOptionsSnapshot<ValidationConfiguration>)}.{nameof(IOptionsSnapshot<ValidationConfiguration>.Value)} property cannot be null",
+                    nameof(validationConfigsAccessor));
+            }
+
+            if (validationConfigsAccessor.Value.MissingPackageRetryCount < 1)
+            {
+                throw new ArgumentOutOfRangeException(
+                    nameof(validationConfigsAccessor),
+                    $"{nameof(ValidationConfiguration)}.{nameof(ValidationConfiguration.MissingPackageRetryCount)} must be at least 1");
+            }
+
+            _configs = validationConfigsAccessor.Value;
             _galleryPackageService = galleryPackageService ?? throw new ArgumentNullException(nameof(galleryPackageService));
             _validationSetProvider = validationSetProvider ?? throw new ArgumentNullException(nameof(validationSetProvider));
             _validationSetProcessor = validationSetProcessor ?? throw new ArgumentNullException(nameof(validationSetProcessor));
             _validationOutcomeProcessor = validationOutcomeProcessor ?? throw new ArgumentNullException(nameof(validationOutcomeProcessor));
+            _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
 
@@ -48,10 +75,28 @@ public async Task<bool> HandleAsync(PackageValidationMessageData message)
                 if (package == null)
                 {
                     // no package in DB yet. Might have received message a bit early, need to retry later
-                    _logger.LogInformation("Did not find information in DB for package {PackageId} {PackageVersion}",
-                        message.PackageId,
-                        message.PackageVersion);
-                    return false;
+                    if (message.DeliveryCount - 1 >= _configs.MissingPackageRetryCount)
+                    {
+                        _logger.LogWarning("Could not find package {PackageId} {PackageVersion} in DB after {DeliveryCount} tries, dropping message",
+                            message.PackageId,
+                            message.PackageVersion,
+                            message.DeliveryCount);
+
+                        _telemetryService.TrackMissingPackageForValidationMessage(
+                            message.PackageId,
+                            message.PackageVersion,
+                            message.ValidationTrackingId.ToString());
+
+                        return true;
+                    }
+                    else
+                    {
+                        _logger.LogInformation("Could not find package {PackageId} {PackageVersion} in DB, retrying",
+                            message.PackageId,
+                            message.PackageVersion);
+
+                        return false;
+                    }
                 }
 
                 var validationSet = await _validationSetProvider.TryGetOrCreateValidationSetAsync(message.ValidationTrackingId, package);

src/NuGet.Services.Validation.Orchestrator/settings.json

@@ -10,8 +10,9 @@
       }
     ],
     "ValidationStorageConnectionString": "",
+    "MissingPackageRetryCount":  15,
     "ValidationMessageRecheckPeriod": "00:01:00",
-    "NewValidationRequestDeduplicationWindow": "00:10:00"
+    "NewValidationRequestDeduplicationWindow": "00:20:00"
   },
   "Vcs": {
     "ContainerName": "validation",

src/PackageHash/PackageHash.csproj

@@ -79,7 +79,7 @@
       <Version>1.1.2</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Cursor">
-      <Version>2.14.0</Version>
+      <Version>2.15.0</Version>
     </PackageReference>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />

src/Stats.AzureCdnLogs.Common/LogEvents.cs

@@ -22,6 +22,8 @@ public class LogEvents
         public static EventId FailedToGetFtpResponse = new EventId(512, "Failed to get FTP response");
         public static EventId FailedToCheckAlreadyProcessedLogFilePackageStatistics = new EventId(513, "Failed to check already processed package statistics for log file");
         public static EventId FailedToCheckAlreadyProcessedLogFileToolStatistics = new EventId(514, "Failed to check already processed tool statistics for log file");
+        public static EventId MultiplePackageIDVersionParseOptions = new EventId(515, "Multiple package id/version parse options");
+        public static EventId TranslatedPackageIdVersion = new EventId(516, "Translated package id and version");
         public static EventId JobRunFailed = new EventId(550, "Job run failed");
         public static EventId JobInitFailed = new EventId(551, "Job initialization failed");
     }