[Package Signing] Add client APIs to Package Signature Validation Job (#264)

Calls `NuGet.Client`'s APIs to determine whether a package is signed. **NOTE**: This doesn't have any integration tests yet as that would require a signed package to be checked-in to source control.

Author: loic-sharma

src/NuGet.Jobs.Common/JobRunner.cs

@@ -7,7 +7,6 @@
 using System.Diagnostics;
 using System.Linq;
 using System.Net;
-using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using NuGet.Services.Logging;
@@ -163,7 +162,7 @@ private static async Task JobLoop(JobBase job, bool runContinuously, int sleepDu
                 // Wait for <sleepDuration> milliSeconds and run the job again
                 _logger.LogInformation("Will sleep for {SleepDuration} before the next Job run", PrettyPrintTime(sleepDuration));
 
-                Thread.Sleep(sleepDuration);
+                await Task.Delay(sleepDuration);
             }
         }
     }

src/Validation.PackageSigning.Core/Error.cs

@@ -9,5 +9,7 @@ public static class Error
     {
         public static EventId ValidatorStateServiceFailedToAddStatus = new EventId(1000, "Failed to add validator's status");
         public static EventId ValidatorStateServiceFailedToUpdateStatus = new EventId(1001, "Failed to update validator's status");
+
+        public static EventId ValidateSignatureFailedToDownloadPackageStatus = new EventId(1100, "Failed to download package");
     }
 }

src/Validation.PackageSigning.ExtractAndValidateSignature/Job.cs

@@ -5,6 +5,8 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
+using System.Net;
+using System.Net.Http;
 using System.Threading.Tasks;
 using Autofac;
 using Autofac.Extensions.DependencyInjection;
@@ -26,9 +28,9 @@ namespace NuGet.Jobs.Validation.PackageSigning.ExtractAndValidateSignature
     public class Job : JobBase
     {
         /// <summary>
-        /// The configured service provider, used to instiate the services this job depends on.
+        /// The maximum number of concurrent connections that can be established to a single server.
         /// </summary>
-        private IServiceProvider _serviceProvider;
+        private const int MaximumConnectionsPerServer = 64;
 
         /// <summary>
         /// The argument this job uses to determine the configuration file's path.
@@ -37,6 +39,7 @@ public class Job : JobBase
 
         private const string ValidationDbConfigurationSectionName = "ValidationDb";
         private const string ServiceBusConfigurationSectionName = "ServiceBus";
+        private const string PackageDownloadTimeoutName = "PackageDownloadTimeout";
 
         /// <summary>
         /// The maximum time that a KeyVault secret will be cached for.
@@ -54,11 +57,18 @@ public class Job : JobBase
         /// </summary>
         private static readonly TimeSpan MaxShutdownTime = TimeSpan.FromMinutes(1);
 
+        /// <summary>
+        /// The configured service provider, used to instiate the services this job depends on.
+        /// </summary>
+        private IServiceProvider _serviceProvider;
+
         public override void Init(IDictionary<string, string> jobArgsDictionary)
         {
             var configurationFilename = JobConfigurationManager.GetArgument(jobArgsDictionary, ConfigurationArgument);
 
             _serviceProvider = GetServiceProvider(GetConfigurationRoot(configurationFilename));
+
+            ServicePointManager.DefaultConnectionLimit = MaximumConnectionsPerServer;
         }
 
         public override async Task Run()
@@ -144,6 +154,22 @@ private void ConfigureJobServices(IServiceCollection services, IConfigurationRoo
             services.AddTransient<IBrokeredMessageSerializer<SignatureValidationMessage>, SignatureValidationMessageSerializer>();
             services.AddTransient<IMessageHandler<SignatureValidationMessage>, SignatureValidationMessageHandler>();
             services.AddTransient<IPackageSigningStateService, PackageSigningStateService>();
+
+            services.AddSingleton(p =>
+            {
+                var assemblyName = typeof(SignatureValidationMessage).Assembly.GetName();
+
+                var client = new HttpClient(new WebRequestHandler
+                {
+                    AllowPipelining = true,
+                    AutomaticDecompression = (DecompressionMethods.GZip | DecompressionMethods.Deflate),
+                });
+
+                client.Timeout = configurationRoot.GetValue<TimeSpan>(PackageDownloadTimeoutName);
+                client.DefaultRequestHeaders.Add("user-agent", $"{assemblyName.Name}/{assemblyName.Version}");
+
+                return client;
+            });
         }
 
         private IServiceProvider GetServiceProvider(IConfigurationRoot configurationRoot)

src/Validation.PackageSigning.ExtractAndValidateSignature/Settings/dev.json

@@ -7,6 +7,9 @@
     "TopicPath": "validate-signature",
     "SubscriptionName": "extract-and-validate-signature"
   },
+
+  "PackageDownloadTimeout": "10:00",
+
   "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
   "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
   "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",

src/Validation.PackageSigning.ExtractAndValidateSignature/SignatureValidationMessageHandler.cs

@@ -3,14 +3,18 @@
 
 using System;
 using System.Data.Entity.Infrastructure;
+using System.Diagnostics;
+using System.IO;
+using System.Net;
 using System.Net.Http;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using NuGet.Jobs.Validation.PackageSigning.Messages;
 using NuGet.Jobs.Validation.PackageSigning.Storage;
+using NuGet.Packaging;
 using NuGet.Services.ServiceBus;
 using NuGet.Services.Validation;
-using NuGet.Versioning;
 
 namespace NuGet.Jobs.Validation.PackageSigning.ExtractAndValidateSignature
 {
@@ -23,22 +27,27 @@ namespace NuGet.Jobs.Validation.PackageSigning.ExtractAndValidateSignature
     public class SignatureValidationMessageHandler
         : IMessageHandler<SignatureValidationMessage>
     {
+        private const int BufferSize = 8192;
+
+        private readonly HttpClient _httpClient;
         private readonly IValidatorStateService _validatorStateService;
         private readonly IPackageSigningStateService _packageSigningStateService;
         private readonly ILogger<SignatureValidationMessageHandler> _logger;
 
         /// <summary>
         /// Instantiate's a new package signatures validator.
         /// </summary>
-        /// <param name="validationContext">The persisted validation context.</param>
-        /// <param name="certificateStore">The persisted certificate store.</param>
+        /// <param name="httpClient">The HTTP client used to download packages.</param>
         /// <param name="validatorStateService">The service used to retrieve and persist this validator's state.</param>
         /// <param name="packageSigningStateService">The service used to retrieve and persist package signing state.</param>
+        /// <param name="logger">The logger that should be used.</param>
         public SignatureValidationMessageHandler(
+            HttpClient httpClient,
             IValidatorStateService validatorStateService,
             IPackageSigningStateService packageSigningStateService,
             ILogger<SignatureValidationMessageHandler> logger)
         {
+            _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
             _validatorStateService = validatorStateService ?? throw new ArgumentNullException(nameof(validatorStateService));
             _packageSigningStateService = packageSigningStateService ?? throw new ArgumentNullException(nameof(packageSigningStateService));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
@@ -83,8 +92,7 @@ public async Task<bool> HandleAsync(SignatureValidationMessage message)
             }
 
             // Validate package
-            // TODO: consume actual client nupkg's containing missing signing APIs
-            if (!IsSigned(message.PackageVersion))
+            if ( ! await IsSigned(message.NupkgUri, CancellationToken.None))
             {
                 return await HandleUnsignedPackageAsync(validation, message);
             }
@@ -95,10 +103,13 @@ public async Task<bool> HandleAsync(SignatureValidationMessage message)
             }
         }
 
-        private bool IsSigned(string packageVersion)
+        private async Task<bool> IsSigned(Uri packageUri, CancellationToken cancellationToken)
         {
-            var nugetVersion = NuGetVersion.Parse(packageVersion);
-            return nugetVersion.IsPrerelease && string.Equals(nugetVersion.Release, "signed", StringComparison.OrdinalIgnoreCase);
+            using (var packageStream = await DownloadPackageAsync(packageUri, cancellationToken))
+            using (var package = new PackageArchiveReader(packageStream, leaveStreamOpen: false))
+            {
+                return await package.IsSignedAsync(cancellationToken);
+            }
         }
 
         private async Task<bool> HandleUnsignedPackageAsync(ValidatorStatus validation, SignatureValidationMessage message)
@@ -171,5 +182,67 @@ private async Task<bool> BlockSignedPackageAsync(ValidatorStatus validation, Sig
             // Consume the message if successfully saved state.
             return saveStateResult == SaveStatusResult.Success;
         }
+
+        private async Task<Stream> DownloadPackageAsync(Uri packageUri, CancellationToken cancellationToken)
+        {
+            _logger.LogInformation("Attempting to download package from {PackageUri}...", packageUri);
+
+            Stream packageStream = null;
+            var stopwatch = Stopwatch.StartNew();
+
+            try
+            {
+                // Download the package from the network to a temporary file.
+                using (var response = await _httpClient.GetAsync(packageUri, HttpCompletionOption.ResponseHeadersRead))
+                {
+                    _logger.LogInformation(
+                        "Received response {StatusCode}: {ReasonPhrase} of type {ContentType} for request {PackageUri}",
+                        response.StatusCode,
+                        response.ReasonPhrase,
+                        response.Content.Headers.ContentType,
+                        packageUri);
+
+                    if (response.StatusCode != HttpStatusCode.OK)
+                    {
+                        throw new InvalidOperationException($"Expected status code {HttpStatusCode.OK} for package download, actual: {response.StatusCode}");
+                    }
+
+                    using (var networkStream = await response.Content.ReadAsStreamAsync())
+                    {
+                        packageStream = new FileStream(
+                                            Path.GetTempFileName(),
+                                            FileMode.Create,
+                                            FileAccess.ReadWrite,
+                                            FileShare.None,
+                                            BufferSize,
+                                            FileOptions.DeleteOnClose | FileOptions.Asynchronous);
+
+                        await networkStream.CopyToAsync(packageStream, BufferSize, cancellationToken);
+                    }
+                }
+
+                packageStream.Position = 0;
+
+                _logger.LogInformation(
+                    "Downloaded {PackageSizeInBytes} bytes in {DownloadElapsedTime} seconds for request {PackageUri}",
+                    packageStream.Length,
+                    stopwatch.Elapsed.TotalSeconds,
+                    packageUri);
+
+                return packageStream;
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(
+                    Error.ValidateSignatureFailedToDownloadPackageStatus,
+                    e,
+                    "Exception thrown when trying to download package from {PackageUri}",
+                    packageUri);
+
+                packageStream?.Dispose();
+
+                throw;
+            }
+        }
     }
 }

src/Validation.PackageSigning.ExtractAndValidateSignature/Validation.PackageSigning.ExtractAndValidateSignature.csproj

@@ -96,6 +96,9 @@
     <PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions">
       <Version>1.1.2</Version>
     </PackageReference>
+    <PackageReference Include="NuGet.Packaging">
+      <Version>4.6.0-preview1-4690</Version>
+    </PackageReference>
     <PackageReference Include="NuGet.Services.Configuration">
       <Version>2.4.3</Version>
     </PackageReference>
@@ -114,9 +117,6 @@
     <PackageReference Include="NuGet.Services.Validation">
       <Version>2.4.3</Version>
     </PackageReference>
-    <PackageReference Include="NuGet.Versioning">
-      <Version>4.4.0</Version>
-    </PackageReference>
     <PackageReference Include="System.Net.Http">
       <Version>4.3.3</Version>
     </PackageReference>