Add ProcessorPackageFileService to allow processors to store .nupkgs (#388)

joelverhagen · joelverhagen · commit 20e109ee4a86 · 2018-03-26T15:48:27.000-07:00
Progress on NuGet/Engineering#1207
diff --git a/src/NuGet.Jobs.Common/Configuration/ValidationStorageConfiguration.cs b/src/NuGet.Jobs.Common/Configuration/ValidationStorageConfiguration.cs
@@ -0,0 +1,10 @@
+﻿// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace NuGet.Jobs.Configuration
+{
+    public class ValidationStorageConfiguration
+    {
+        public string ConnectionString { get; set; }
+    }
+}
diff --git a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj
@@ -191,6 +191,7 @@
     <Compile Include="Configuration\JobArgumentNames.cs" />
     <Compile Include="Configuration\ServiceBusConfiguration.cs" />
     <Compile Include="Configuration\ValidationDbConfiguration.cs" />
+    <Compile Include="Configuration\ValidationStorageConfiguration.cs" />
     <Compile Include="Extensions\SqlConnectionStringBuilderExtensions.cs" />
     <Compile Include="Extensions\XElementExtensions.cs" />
     <Compile Include="SecretReader\ISecretReaderFactory.cs" />
diff --git a/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj b/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj
@@ -108,7 +108,7 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="NuGet.Services.Validation.Issues">
-      <Version>2.19.1</Version>
+      <Version>2.22.0</Version>
     </PackageReference>
   </ItemGroup>
   <ItemGroup>
diff --git a/src/PackageHash/PackageHash.csproj b/src/PackageHash/PackageHash.csproj
@@ -79,7 +79,7 @@
       <Version>1.1.2</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Cursor">
-      <Version>2.17.0</Version>
+      <Version>2.22.0</Version>
     </PackageReference>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
diff --git a/src/PackageHash/Settings/dev.json b/src/PackageHash/Settings/dev.json
@@ -13,7 +13,7 @@
     ]
   },
 
-  "PackageDownloadTimeout": "10:00",
+  "PackageDownloadTimeout": "00:10:00",
 
   "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
   "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
diff --git a/src/PackageHash/Settings/int.json b/src/PackageHash/Settings/int.json
@@ -13,7 +13,7 @@
     ]
   },
 
-  "PackageDownloadTimeout": "10:00",
+  "PackageDownloadTimeout": "00:10:00",
 
   "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
   "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
diff --git a/src/PackageHash/Settings/prod.json b/src/PackageHash/Settings/prod.json
@@ -13,7 +13,7 @@
     ]
   },
 
-  "PackageDownloadTimeout": "10:00",
+  "PackageDownloadTimeout": "00:10:00",
 
   "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
   "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
diff --git a/src/Validation.Common.Job/FileStreamUtility.cs b/src/Validation.Common.Job/FileStreamUtility.cs
@@ -0,0 +1,23 @@
+﻿// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.IO;
+
+namespace NuGet.Jobs.Validation
+{
+    public static class FileStreamUtility
+    {
+        public const int BufferSize = 8192;
+
+        public static FileStream GetTemporaryFile()
+        {
+            return new FileStream(
+                Path.GetTempFileName(),
+                FileMode.Create,
+                FileAccess.ReadWrite,
+                FileShare.None,
+                BufferSize,
+                FileOptions.DeleteOnClose | FileOptions.Asynchronous);
+        }
+    }
+}
diff --git a/src/Validation.Common.Job/JsonConfigurationJob.cs b/src/Validation.Common.Job/JsonConfigurationJob.cs
@@ -15,6 +15,7 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using NuGet.Jobs.Configuration;
+using NuGet.Jobs.Validation.Storage;
 using NuGet.Services.Configuration;
 using NuGet.Services.KeyVault;
 using NuGet.Services.Logging;
@@ -107,13 +108,23 @@ private void ConfigureDefaultJobServices(IServiceCollection services, IConfigura
             services.Configure<GalleryDbConfiguration>(configurationRoot.GetSection(GalleryDbConfigurationSectionName));
             services.Configure<ValidationDbConfiguration>(configurationRoot.GetSection(ValidationDbConfigurationSectionName));
             services.Configure<ServiceBusConfiguration>(configurationRoot.GetSection(ServiceBusConfigurationSectionName));
+            services.Configure<ValidationStorageConfiguration>(configurationRoot.GetSection(ServiceBusConfigurationSectionName));
 
             services.AddSingleton(new TelemetryClient());
             services.AddTransient<ITelemetryClient, TelemetryClientWrapper>();
             services.AddTransient<ICommonTelemetryService, CommonTelemetryService>();
             services.AddTransient<IDiagnosticsService, LoggerDiagnosticsService>();
             services.AddTransient<IPackageDownloader, PackageDownloader>();
 
+            services.AddTransient<ICloudBlobClient>(c =>
+            {
+                var configurationAccessor = c.GetRequiredService<IOptionsSnapshot<ValidationStorageConfiguration>>();
+                return new CloudBlobClientWrapper(
+                    configurationAccessor.Value.ConnectionString,
+                    readAccessGeoRedundant: false);
+            });
+            services.AddTransient<ICoreFileStorageService, CloudBlobCoreFileStorageService>();
+
             services.AddScoped<IValidationEntitiesContext>(p =>
             {
                 var config = p.GetRequiredService<IOptionsSnapshot<ValidationDbConfiguration>>().Value;
diff --git a/src/Validation.Common.Job/PackageDownloader.cs b/src/Validation.Common.Job/PackageDownloader.cs
@@ -14,8 +14,6 @@ namespace NuGet.Jobs.Validation
 {
     public class PackageDownloader : IPackageDownloader
     {
-        private const int BufferSize = 8192;
-
         private readonly HttpClient _httpClient;
         private readonly ICommonTelemetryService _telemetryService;
         private readonly ILogger<PackageDownloader> _logger;
@@ -56,15 +54,9 @@ public async Task<Stream> DownloadAsync(Uri packageUri, CancellationToken cancel
 
                     using (var networkStream = await response.Content.ReadAsStreamAsync())
                     {
-                        packageStream = new FileStream(
-                                            Path.GetTempFileName(),
-                                            FileMode.Create,
-                                            FileAccess.ReadWrite,
-                                            FileShare.None,
-                                            BufferSize,
-                                            FileOptions.DeleteOnClose | FileOptions.Asynchronous);
-
-                        await networkStream.CopyToAsync(packageStream, BufferSize, cancellationToken);
+                        packageStream = FileStreamUtility.GetTemporaryFile();
+
+                        await networkStream.CopyToAsync(packageStream, FileStreamUtility.BufferSize, cancellationToken);
                     }
                 }
 
diff --git a/src/Validation.Common.Job/Storage/IProcessorPackageFileService.cs b/src/Validation.Common.Job/Storage/IProcessorPackageFileService.cs
@@ -0,0 +1,15 @@
+﻿// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.IO;
+using System.Threading.Tasks;
+
+namespace NuGet.Jobs.Validation.Storage
+{
+    public interface IProcessorPackageFileService
+    {
+        Task<Uri> GetReadAndDeleteUriAsync(string packageId, string packageNormalizedVersion, Guid validationId);
+        Task SaveAsync(string packageId, string packageNormalizedVersion, Guid validationId, Stream packageFile);
+    }
+}
diff --git a/src/Validation.Common.Job/Storage/ISimpleCloudBlobProvider.cs b/src/Validation.Common.Job/Storage/ISimpleCloudBlobProvider.cs
@@ -0,0 +1,18 @@
+﻿// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using NuGetGallery;
+
+namespace NuGet.Jobs.Validation.Storage
+{
+    public interface ISimpleCloudBlobProvider
+    {
+        /// <summary>
+        /// Initializes a blob from a blob URL. This blob URL can contain a SAS token as the query string to allow
+        /// read access to private blob or allow write operations.
+        /// </summary>
+        /// <param name="blobUrl">The blob URL, optionally having a SAS token query string.</param>
+        /// <returns>The blob.</returns>
+        ISimpleCloudBlob GetBlobFromUrl(string blobUrl);
+    }
+}
diff --git a/src/Validation.Common.Job/Storage/ProcessorPackageFileService.cs b/src/Validation.Common.Job/Storage/ProcessorPackageFileService.cs
@@ -0,0 +1,106 @@
+﻿// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using NuGet.Services.Validation;
+using NuGetGallery;
+
+namespace NuGet.Jobs.Validation.Storage
+{
+    public class ProcessorPackageFileService : IProcessorPackageFileService
+    {
+        /// <summary>
+        /// This meant to be significantly longer than the maximum validation set duration (currently 1 day).
+        /// </summary>
+        private static readonly TimeSpan AccessDuration = TimeSpan.FromDays(7);
+
+        private ICoreFileStorageService _fileStorageService;
+        private ILogger<ProcessorPackageFileService> _logger;
+        private readonly string _processorName;
+
+        public ProcessorPackageFileService(
+            ICoreFileStorageService fileStorageService,
+            Type processorType,
+            ILogger<ProcessorPackageFileService> logger)
+        {
+            _fileStorageService = fileStorageService ?? throw new ArgumentNullException(nameof(fileStorageService));
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+
+            if (processorType == null)
+            {
+                throw new ArgumentNullException(nameof(processorType));
+            }
+
+            if (!typeof(IProcessor).IsAssignableFrom(processorType))
+            {
+                throw new ArgumentException($"The validator type {processorType} must extend {nameof(IProcessor)}", nameof(processorType));
+            }
+
+            _processorName = processorType.Name;
+        }
+
+        public Task<Uri> GetReadAndDeleteUriAsync(
+            string packageId,
+            string packageNormalizedVersion,
+            Guid validationId)
+        {
+            var fileName = BuildFileName(packageId, packageNormalizedVersion, validationId);
+
+            return _fileStorageService.GetPriviledgedFileUriAsync(
+                CoreConstants.ValidationFolderName,
+                fileName,
+                FileUriPermissions.Read | FileUriPermissions.Delete,
+                DateTimeOffset.UtcNow + AccessDuration);
+        }
+
+        public Task SaveAsync(
+            string packageId,
+            string packageNormalizedVersion,
+            Guid validationId,
+            Stream packageFile)
+        {
+            if (packageFile == null)
+            {
+                throw new ArgumentNullException(nameof(packageFile));
+            }
+
+            var fileName = BuildFileName(packageId, packageNormalizedVersion, validationId);
+
+            _logger.LogInformation(
+                "Saving package file {PackageId} {PackageNormalizedVersion} for processor {ProcessorName} and " +
+                "validation ID {ValidationId}.",
+                packageId,
+                packageNormalizedVersion,
+                _processorName,
+                validationId);
+
+            packageFile.Position = 0;
+
+            return _fileStorageService.SaveFileAsync(
+                CoreConstants.ValidationFolderName,
+                fileName,
+                packageFile,
+                overwrite: true);
+        }
+
+        private string BuildFileName(string packageId, string packageNormalizedVersion, Guid validationId)
+        {
+            if (packageId == null)
+            {
+                throw new ArgumentNullException(nameof(packageId));
+            }
+
+            if (packageNormalizedVersion == null)
+            {
+                throw new ArgumentNullException(nameof(packageNormalizedVersion));
+            }
+
+            return $"{_processorName}/{validationId}/{packageId.ToLowerInvariant()}." +
+                packageNormalizedVersion.ToLowerInvariant() +
+                CoreConstants.NuGetPackageFileExtension;
+        }
+    }
+}
diff --git a/src/Validation.Common.Job/Storage/SimpleCloudBlobProvider.cs b/src/Validation.Common.Job/Storage/SimpleCloudBlobProvider.cs
@@ -0,0 +1,34 @@
+﻿// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using Microsoft.WindowsAzure.Storage.Auth;
+using Microsoft.WindowsAzure.Storage.Blob;
+using NuGetGallery;
+
+namespace NuGet.Jobs.Validation.Storage
+{
+    public class SimpleCloudBlobProvider : ISimpleCloudBlobProvider
+    {
+        public ISimpleCloudBlob GetBlobFromUrl(string blobUrl)
+        {
+            var blobUri = new Uri(blobUrl);
+            var sasToken = blobUri.Query;
+            var blobUriBuilder = new UriBuilder(blobUri) { Query = null };
+
+            CloudBlockBlob innerBlob;
+            if (string.IsNullOrEmpty(sasToken))
+            {
+                innerBlob = new CloudBlockBlob(blobUriBuilder.Uri);
+            }
+            else
+            {
+                innerBlob = new CloudBlockBlob(
+                    blobUriBuilder.Uri,
+                    new StorageCredentials(sasToken));
+            }
+
+            return new CloudBlobWrapper(innerBlob);
+        }
+    }
+}
diff --git a/src/Validation.Common.Job/Storage/ValidatorStateService.cs b/src/Validation.Common.Job/Storage/ValidatorStateService.cs
@@ -33,7 +33,7 @@ public ValidatorStateService(
 
             if (!typeof(IValidator).IsAssignableFrom(validatorType))
             {
-                throw new ArgumentException($"The validator type {validatorType} must extend {nameof(IValidator)}", nameof(validatorType));
+                throw new ArgumentException($"The validator type {validatorType} must implement {nameof(IValidator)}.", nameof(validatorType));
             }
 
             _validatorName = validatorType.Name;
diff --git a/src/Validation.Common.Job/Storage/ValidatorStatusExtensions.cs b/src/Validation.Common.Job/Storage/ValidatorStatusExtensions.cs
@@ -36,7 +36,7 @@ public static IValidationResult ToValidationResult(this ValidatorStatus validato
                 .Select(x => new SerializedValidationIssue(x.IssueCode, x.Data))
                 .ToList();
 
-            return new ValidationResult(validatorStatus.State, issues);
+            return new ValidationResult(validatorStatus.State, issues, validatorStatus.NupkgUrl);
         }
     }
 }
diff --git a/src/Validation.Common.Job/Validation.Common.Job.csproj b/src/Validation.Common.Job/Validation.Common.Job.csproj
@@ -39,6 +39,7 @@
     <Compile Include="CommonTelemetryService.cs" />
     <Compile Include="Error.cs" />
     <Compile Include="ExceptionExtensions.cs" />
+    <Compile Include="FileStreamUtility.cs" />
     <Compile Include="ICommonTelemetryService.cs" />
     <Compile Include="IPackageDownloader.cs" />
     <Compile Include="LoggerDiagnosticsService.cs" />
@@ -47,9 +48,13 @@
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Properties\AssemblyInfo.*.cs" />
     <Compile Include="Storage\AddStatusResult.cs" />
+    <Compile Include="Storage\IProcessorPackageFileService.cs" />
+    <Compile Include="Storage\ISimpleCloudBlobProvider.cs" />
     <Compile Include="Storage\IValidatorStateService.cs" />
+    <Compile Include="Storage\ProcessorPackageFileService.cs" />
     <Compile Include="Storage\SaveStatusResult.cs" />
     <Compile Include="Storage\SerializedValidationIssue.cs" />
+    <Compile Include="Storage\SimpleCloudBlobProvider.cs" />
     <Compile Include="Storage\ValidatorStateService.cs" />
     <Compile Include="Storage\ValidatorStatusExtensions.cs" />
     <Compile Include="SubcriptionProcessorJob.cs" />
@@ -75,19 +80,19 @@
       <Version>4.7.0-preview1.5029</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Configuration">
-      <Version>2.19.1</Version>
+      <Version>2.22.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Logging">
-      <Version>2.19.1</Version>
+      <Version>2.22.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Storage">
-      <Version>2.19.1</Version>
+      <Version>2.22.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Validation">
-      <Version>2.19.1</Version>
+      <Version>2.22.0</Version>
     </PackageReference>
     <PackageReference Include="NuGetGallery.Core">
-      <Version>4.4.4-dev-26021</Version>
+      <Version>4.4.4-dev-26726</Version>
     </PackageReference>
     <PackageReference Include="Serilog">
       <Version>2.5.0</Version>
diff --git a/src/Validation.Common.Job/Validation.Common.Job.nuspec b/src/Validation.Common.Job/Validation.Common.Job.nuspec
diff --git a/src/Validation.PackageSigning.ProcessSignature/Settings/dev.json b/src/Validation.PackageSigning.ProcessSignature/Settings/dev.json
diff --git a/src/Validation.PackageSigning.ProcessSignature/Settings/int.json b/src/Validation.PackageSigning.ProcessSignature/Settings/int.json
diff --git a/src/Validation.PackageSigning.ProcessSignature/Settings/prod.json b/src/Validation.PackageSigning.ProcessSignature/Settings/prod.json
diff --git a/tests/Validation.Common.Job.Tests/FileStreamUtilityFacts.cs b/tests/Validation.Common.Job.Tests/FileStreamUtilityFacts.cs
diff --git a/tests/Validation.Common.Job.Tests/Storage/ProcessorPackageFileServiceFacts.cs b/tests/Validation.Common.Job.Tests/Storage/ProcessorPackageFileServiceFacts.cs
diff --git a/tests/Validation.Common.Job.Tests/Validation.Common.Job.Tests.csproj b/tests/Validation.Common.Job.Tests/Validation.Common.Job.Tests.csproj
diff --git a/tests/Validation.PackageSigning.Core.Tests/Storage/ValidatorStatusExtensionsFacts.cs b/tests/Validation.PackageSigning.Core.Tests/Storage/ValidatorStatusExtensionsFacts.cs

Original file line number	Diff line number	Diff line change
`@@ -13,7 +13,7 @@`
`13`	`13`	`]`
`14`	`14`	`},`
`15`	`15`
`16`		`- "PackageDownloadTimeout": "10:00",`
	`16`	`+ "PackageDownloadTimeout": "00:10:00",`
`17`	`17`
`18`	`18`	`"KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",`
`19`	`19`	`"KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",`
Original file line number	Diff line number	Diff line change
`@@ -14,8 +14,6 @@ namespace NuGet.Jobs.Validation`
`14`	`14`	`{`
`15`	`15`	`public class PackageDownloader : IPackageDownloader`
`16`	`16`	`{`
`17`		`- private const int BufferSize = 8192;`
`18`		`-`
`19`	`17`	`private readonly HttpClient _httpClient;`
`20`	`18`	`private readonly ICommonTelemetryService _telemetryService;`
`21`	`19`	`private readonly ILogger<PackageDownloader> _logger;`
`@@ -56,15 +54,9 @@ public async Task<Stream> DownloadAsync(Uri packageUri, CancellationToken cancel`
`56`	`54`
`57`	`55`	`using (var networkStream = await response.Content.ReadAsStreamAsync())`
`58`	`56`	`{`
`59`		`- packageStream = new FileStream(`
`60`		`- Path.GetTempFileName(),`
`61`		`- FileMode.Create,`
`62`		`- FileAccess.ReadWrite,`
`63`		`- FileShare.None,`
`64`		`- BufferSize,`
`65`		`- FileOptions.DeleteOnClose \| FileOptions.Asynchronous);`
`66`		`-`
`67`		`- await networkStream.CopyToAsync(packageStream, BufferSize, cancellationToken);`
	`57`	`+ packageStream = FileStreamUtility.GetTemporaryFile();`
	`58`	`+`
	`59`	`+ await networkStream.CopyToAsync(packageStream, FileStreamUtility.BufferSize, cancellationToken);`
`68`	`60`	`}`
`69`	`61`	`}`
`70`	`62`