Use validator type instead of instance for configuration validation (#360)

Related to NuGet/Engineering#1190

Author: joelverhagen

src/NuGet.Services.Validation.Orchestrator/Configuration/ConfigurationValidator.cs

@@ -103,8 +103,8 @@ private void CheckUnknownValidators()
             foreach (var validatorItem in _configuration.Validations)
             {
                 // This method will throw if the validator does not exist.
-                var validator = _validatorProvider.GetValidator(validatorItem.Name);
-                if (validator == null)
+                var validatorType = _validatorProvider.GetValidatorType(validatorItem.Name);
+                if (validatorType == null)
                 {
                     throw new ConfigurationErrorsException("Validator implementation not found for " + validatorItem.Name);
                 }
@@ -142,7 +142,7 @@ private void CheckForCyclesAndParallelProcessors()
             var processorNames = _configuration
                 .Validations
                 .Select(x => x.Name)
-                .Where(x => _validatorProvider.GetValidator(x) is IProcessor)
+                .Where(x => typeof(IProcessor).IsAssignableFrom(_validatorProvider.GetValidatorType(x)))
                 .ToList();
 
             TopologicalSort.Validate(_configuration.Validations, processorNames);

src/NuGet.Services.Validation.Orchestrator/IValidatorProvider.cs

@@ -1,13 +1,16 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
+
 namespace NuGet.Services.Validation.Orchestrator
 {
     /// <summary>
     /// Interface for the class that provides validator instances by their name
     /// </summary>
     public interface IValidatorProvider
     {
+        Type GetValidatorType(string validatorName);
         IValidator GetValidator(string validatorName);
     }
 }

src/NuGet.Services.Validation.Orchestrator/ValidatorProvider.cs

@@ -35,18 +35,27 @@ public ValidatorProvider(IServiceProvider serviceProvider, ILogger<ValidatorProv
             }
         }
 
-        public IValidator GetValidator(string validatorName)
+        public Type GetValidatorType(string validatorName)
         {
             validatorName = validatorName ?? throw new ArgumentNullException(nameof(validatorName));
 
             if (_validatorTypes.TryGetValue(validatorName, out Type validatorType))
             {
-                return (IValidator)_serviceProvider.GetRequiredService(validatorType);
+                return validatorType;
             }
 
             throw new ArgumentException($"Unknown validator name: {validatorName}", nameof(validatorName));
         }
 
+        public IValidator GetValidator(string validatorName)
+        {
+            validatorName = validatorName ?? throw new ArgumentNullException(nameof(validatorName));
+
+            var validatorType = GetValidatorType(validatorName);
+
+            return (IValidator)_serviceProvider.GetRequiredService(validatorType);
+        }
+
         private static IEnumerable<Type> GetCandidateTypes(Assembly callingAssembly)
         {
             var executingAssembly = Assembly.GetExecutingAssembly();

src/NuGet.Services.Validation.Orchestrator/settings.json

@@ -2,11 +2,25 @@
   "Configuration": {
     "Validations": [
       {
-        "name": "",
+        "name": "VcsValidator",
         "failAfter": "1:00:00:00",
         "requiredValidations": [],
         "ShouldStart": true,
         "FailureBehavior": "MustSucceed"
+      },
+      {
+        "name": "PackageSigningValidator",
+        "failAfter": "1:00:00:00",
+        "requiredValidations": [],
+        "ShouldStart": true,
+        "FailureBehavior": "MustSucceed"
+      },
+      {
+        "name": "PackageCertificatesValidator",
+        "failAfter": "10:00",
+        "requiredValidations": [ "PackageSigningValidator" ],
+        "ShouldStart": true,
+        "FailureBehavior": "AllowedToFail"
       }
     ],
     "ValidationStorageConnectionString": "",

test.ps1

@@ -42,6 +42,17 @@ Function Run-Tests {
         & $xUnitExe (Join-Path $PSScriptRoot $Test) -xml "Results.$TestCount.xml"
         $TestCount++
     }
+
+    Trace-Log "Running configuration validations";
+
+    $exe = "src\NuGet.Services.Validation.Orchestrator\bin\$Configuration\NuGet.Services.Validation.Orchestrator.exe";
+    $cfg = "src\NuGet.Services.Validation.Orchestrator\settings.json";
+
+    & $exe -Configuration $cfg -Validate
+    if ( $LASTEXITCODE -gt 0 )
+    {
+        Write-Error "Validation of the $cfg failed"
+    }
 }
 
 Write-Host ("`r`n" * 3)

tests/NuGet.Services.Validation.Orchestrator.Tests/Configuration/ConfigurationFacts.cs

@@ -148,11 +148,11 @@ public void ConfigurationValidatorDetectsParallelProcessors()
 
             var validatorProvider = new Mock<IValidatorProvider>();
             validatorProvider
-                .Setup(x => x.GetValidator(It.Is<string>(n => n == validationName1 || n == validationName2)))
-                .Returns(() => new Mock<IValidator>().Object);
+                .Setup(x => x.GetValidatorType(It.Is<string>(n => n == validationName1 || n == validationName2)))
+                .Returns(() => new Mock<IValidator>().Object.GetType());
             validatorProvider
-                .Setup(x => x.GetValidator(processorName1))
-                .Returns(() => new Mock<IProcessor>().Object);
+                .Setup(x => x.GetValidatorType(processorName1))
+                .Returns(() => new Mock<IProcessor>().Object.GetType());
 
             var ex = Record.Exception(() => Validate(validatorProvider.Object, configuration));
 
@@ -195,11 +195,11 @@ public void ConfigurationValidatorAllowsNonParallelProcessors()
 
             var validatorProvider = new Mock<IValidatorProvider>();
             validatorProvider
-                .Setup(x => x.GetValidator(It.Is<string>(n => n == validationName1 || n == validationName2)))
-                .Returns(() => new Mock<IValidator>().Object);
+                .Setup(x => x.GetValidatorType(It.Is<string>(n => n == validationName1 || n == validationName2)))
+                .Returns(() => new Mock<IValidator>().Object.GetType());
             validatorProvider
-                .Setup(x => x.GetValidator(processorName1))
-                .Returns(() => new Mock<IProcessor>().Object);
+                .Setup(x => x.GetValidatorType(processorName1))
+                .Returns(() => new Mock<IProcessor>().Object.GetType());
 
             var ex = Record.Exception(() => Validate(validatorProvider.Object, configuration));
 
@@ -601,8 +601,8 @@ private static void Validate(ValidationConfiguration configuration)
         {
             var validatorProvider = new Mock<IValidatorProvider>();
             validatorProvider
-                .Setup(x => x.GetValidator(It.IsAny<string>()))
-                .Returns(() => new Mock<IValidator>().Object);
+                .Setup(x => x.GetValidatorType(It.IsAny<string>()))
+                .Returns(() => new Mock<IValidator>().Object.GetType());
 
             Validate(validatorProvider.Object, configuration);
         }