[Orchestrator] Remove validation failure on timeout (#368)

Author: loic-sharma

src/NuGet.Services.Validation.Orchestrator/Configuration/ConfigurationValidator.cs

@@ -67,9 +67,9 @@ private void CheckPropertyValues()
                     throw new ConfigurationErrorsException("Validation name cannot be empty");
                 }
 
-                if (validationConfigurationItem.FailAfter == TimeSpan.Zero)
+                if (validationConfigurationItem.TrackAfter == TimeSpan.Zero)
                 {
-                    throw new ConfigurationErrorsException($"failAfter timeout must be set for validation {validationConfigurationItem.Name}");
+                    throw new ConfigurationErrorsException($"{nameof(validationConfigurationItem.TrackAfter)} must be set for validation {validationConfigurationItem.Name}");
                 }
             }
         }

src/NuGet.Services.Validation.Orchestrator/Configuration/ValidationConfiguration.cs

@@ -39,5 +39,15 @@ public class ValidationConfiguration
         /// this window.
         /// </summary>
         public TimeSpan NewValidationRequestDeduplicationWindow { get; set; }
+
+        /// <summary>
+        /// The threshold until which an email will be sent out due to a validation set taking too long.
+        /// </summary>
+        public TimeSpan ValidationSetNotificationTimeout { get; set; }
+
+        /// <summary>
+        /// The threshold until a validation set is no longer processed.
+        /// </summary>
+        public TimeSpan TimeoutValidationSetAfter { get; set; }
     }
 }

src/NuGet.Services.Validation.Orchestrator/Configuration/ValidationConfigurationItem.cs

@@ -17,9 +17,9 @@ public class ValidationConfigurationItem
         public string Name { get; set; }
 
         /// <summary>
-        /// Timeout after which started validation is considered failed if it didn't produce any outcome
+        /// Time after which a validation's processing time will be tracked. Use this to track validations that take too long.
         /// </summary>
-        public TimeSpan FailAfter { get; set; }
+        public TimeSpan TrackAfter { get; set; }
 
         /// <summary>
         /// List of validation names that must finish before this validation can run

src/NuGet.Services.Validation.Orchestrator/IMessageService.cs

@@ -10,5 +10,6 @@ public interface IMessageService
         void SendPackagePublishedMessage(Package package);
         void SendPackageValidationFailedMessage(Package package);
         void SendPackageSignedValidationFailedMessage(Package package);
+        void SendPackageValidationTakingTooLongMessage(Package package);
     }
 }

src/NuGet.Services.Validation.Orchestrator/IValidationStorageService.cs

@@ -45,6 +45,13 @@ public interface IValidationStorageService
         /// <exception cref="ArgumentOutOfRangeException">If <paramref name="validationResult"/> has status <see cref="ValidationStatus.NotStarted"/></exception>
         Task MarkValidationStartedAsync(PackageValidation packageValidation, IValidationResult validationResult);
 
+        /// <summary>
+        /// Updates the <see cref="PackageValidationSet.Updated"/> to the current timestamp and persists changes.
+        /// </summary>
+        /// <param name="packageValidationSet">The validation set to update.</param>
+        /// <returns>Task object tracking the async operation status.</returns>
+        Task UpdateValidationSetAsync(PackageValidationSet packageValidationSet);
+
         /// <summary>
         /// Updates the passed <see cref="PackageValidation"/> object with the result's validation status,
         /// updates the <see cref="PackageValidation.ValidationStatusTimestamp"/> property to the current

src/NuGet.Services.Validation.Orchestrator/MessageService.cs

@@ -49,26 +49,39 @@ public void SendPackagePublishedMessage(Package package)
         {
             package = package ?? throw new ArgumentNullException(nameof(package));
 
-            var galleryPackageUrl = string.Format(_emailConfiguration.PackageUrlTemplate, package.PackageRegistration.Id, package.NormalizedVersion);
-            var packageSupportUrl = string.Format(_emailConfiguration.PackageSupportTemplate, package.PackageRegistration.Id, package.NormalizedVersion);
+            var galleryPackageUrl = GalleryPackageUrl(package);
+            var packageSupportUrl = PackageSupportUrl(package);
+
             _coreMessageService.SendPackageAddedNotice(package, galleryPackageUrl, packageSupportUrl, _emailConfiguration.EmailSettingsUrl);
         }
 
         public void SendPackageValidationFailedMessage(Package package)
         {
             package = package ?? throw new ArgumentNullException(nameof(package));
 
-            var galleryPackageUrl = string.Format(_emailConfiguration.PackageUrlTemplate, package.PackageRegistration.Id, package.NormalizedVersion);
-            var packageSupportUrl = string.Format(_emailConfiguration.PackageSupportTemplate, package.PackageRegistration.Id, package.NormalizedVersion);
+            var galleryPackageUrl = GalleryPackageUrl(package);
+            var packageSupportUrl = PackageSupportUrl(package);
+
             _coreMessageService.SendPackageValidationFailedNotice(package, galleryPackageUrl, packageSupportUrl);
         }
 
         public void SendPackageSignedValidationFailedMessage(Package package)
         {
             package = package ?? throw new ArgumentNullException(nameof(package));
 
-            var galleryPackageUrl = string.Format(_emailConfiguration.PackageUrlTemplate, package.PackageRegistration.Id, package.NormalizedVersion);
+            var galleryPackageUrl = GalleryPackageUrl(package);
+
             _coreMessageService.SendSignedPackageNotAllowedNotice(package, galleryPackageUrl, _emailConfiguration.AnnouncementsUrl, _emailConfiguration.TwitterUrl);
         }
+
+        public void SendPackageValidationTakingTooLongMessage(Package package)
+        {
+            package = package ?? throw new ArgumentNullException(nameof(package));
+
+            _coreMessageService.SendValidationTakingTooLongNotice(package, GalleryPackageUrl(package));
+        }
+
+        private string GalleryPackageUrl(Package package) => string.Format(_emailConfiguration.PackageUrlTemplate, package.PackageRegistration.Id, package.NormalizedVersion);
+        private string PackageSupportUrl(Package package) => string.Format(_emailConfiguration.PackageSupportTemplate, package.PackageRegistration.Id, package.NormalizedVersion);
     }
 }

src/NuGet.Services.Validation.Orchestrator/Telemetry/ITelemetryService.cs

@@ -34,7 +34,17 @@ public interface ITelemetryService
         void TrackTotalValidationDuration(TimeSpan duration, bool isSuccess);
 
         /// <summary>
-        /// A counter metric emitted when a validator fails due to the <see cref="ValidationConfigurationItem.FailAfter"/>
+        /// A counter metric emitted when a notification is sent because a validation set takes too long.
+        /// </summary>
+        void TrackSentValidationTakingTooLongMessage(string packageId, string normalizedVersion, Guid validationTrackingId);
+
+        /// <summary>
+        /// A counter metric emitted when a validation set times out.
+        /// </summary>
+        void TrackValidationSetTimeout(string packageId, string normalizedVersion, Guid validationTrackingId);
+
+        /// <summary>
+        /// A counter metric emitted when a validation is past its validator's <see cref="ValidationConfigurationItem.TrackAfter"/>
         /// configuration.
         /// </summary>
         /// <param name="validatorType">The validator type (name).</param>

src/NuGet.Services.Validation.Orchestrator/Telemetry/TelemetryService.cs

@@ -15,6 +15,8 @@ public class TelemetryService : ITelemetryService
         private const string DurationToValidationSetCreationSeconds = Prefix + "DurationToValidationSetCreationSeconds";
         private const string PackageStatusChange = Prefix + "PackageStatusChange";
         private const string TotalValidationDurationSeconds = Prefix + "TotalValidationDurationSeconds";
+        private const string SentValidationTakingTooLongMessage = Prefix + "SentValidationTakingTooLongMessage";
+        private const string ValidationSetTimeout = Prefix + "TotalValidationDurationSeconds";
         private const string ValidationIssue = Prefix + "ValidationIssue";
         private const string ValidationIssueCount = Prefix + "ValidationIssueCount";
         private const string ValidatorTimeout = Prefix + "ValidatorTimeout";
@@ -90,6 +92,28 @@ public void TrackTotalValidationDuration(TimeSpan duration, bool isSuccess)
                 });
         }
 
+        public void TrackSentValidationTakingTooLongMessage(string packageId, string normalizedVersion, Guid validationTrackingId)
+            => _telemetryClient.TrackMetric(
+                    SentValidationTakingTooLongMessage,
+                    1,
+                    new Dictionary<string, string>
+                    {
+                        { PackageId, packageId },
+                        { NormalizedVersion, normalizedVersion },
+                        { ValidationTrackingId, validationTrackingId.ToString() },
+                    });
+
+        public void TrackValidationSetTimeout(string packageId, string normalizedVersion, Guid validationTrackingId)
+            => _telemetryClient.TrackMetric(
+                    ValidationSetTimeout,
+                    1,
+                    new Dictionary<string, string>
+                    {
+                        { PackageId, packageId },
+                        { NormalizedVersion, normalizedVersion },
+                        { ValidationTrackingId, validationTrackingId.ToString() },
+                    });
+
         public void TrackValidationIssue(string validatorType, ValidationIssueCode code)
         {
             _telemetryClient.TrackMetric(