Back up the package if there are any processors (#374)

joelverhagen · joelverhagen · commit 3e45ecfc235b · 2018-03-14T17:07:35.000-07:00
Progress on NuGet/Engineering#1190
diff --git a/src/NuGet.Services.Validation.Orchestrator/IValidationPackageFileService.cs b/src/NuGet.Services.Validation.Orchestrator/IValidationPackageFileService.cs
@@ -17,6 +17,13 @@ public interface IValidationPackageFileService : ICorePackageFileService
         /// <returns>The package stream.</returns>
         Task<Stream> DownloadPackageFileToDiskAsync(Package package);
 
+        /// <summary>
+        /// Backs up the package file from the location specific for the validation set.
+        /// </summary>
+        /// <param name="package">The package metadata.</param>
+        /// <param name="validationSet">The validation set, containing validation set and package identifiers.</param>
+        Task BackupPackageFileFromValidationSetPackageAsync(Package package, PackageValidationSet validationSet);
+
         /// <summary>
         /// Copy a package from the validation container to a location specific for the validation set. This allows the
         /// validation set to have its own copy of the package to mutate (via <see cref="IProcessor"/>) and validate.
diff --git a/src/NuGet.Services.Validation.Orchestrator/ValidationPackageFileService.cs b/src/NuGet.Services.Validation.Orchestrator/ValidationPackageFileService.cs
@@ -13,6 +13,12 @@ namespace NuGet.Services.Validation.Orchestrator
 {
     public class ValidationPackageFileService : CorePackageFileService, IValidationPackageFileService
     {
+        /// <summary>
+        /// The value picked today is based off of the maximum duration we wait when downloading packages using the
+        /// <see cref="IPackageDownloader"/>.
+        /// </summary>
+        private static readonly TimeSpan AccessDuration = TimeSpan.FromMinutes(10);
+
         private readonly ICoreFileStorageService _fileStorageService;
         private readonly IPackageDownloader _packageDownloader;
         private readonly ILogger<ValidationPackageFileService> _logger;
@@ -50,6 +56,24 @@ public Task CopyValidationPackageForValidationSetAsync(PackageValidationSet vali
                 AccessConditionWrapper.GenerateEmptyCondition());
         }
 
+        public async Task BackupPackageFileFromValidationSetPackageAsync(Package package, PackageValidationSet validationSet)
+        {
+            _logger.LogInformation(
+                "Backing up package for validation set {ValidationTrackingId} ({PackageId} {PackageVersion}).",
+                validationSet.ValidationTrackingId,
+                validationSet.PackageId,
+                validationSet.PackageNormalizedVersion);
+
+            var packageUri = await GetPackageForValidationSetReadUriAsync(
+                validationSet,
+                DateTimeOffset.UtcNow.Add(AccessDuration));
+
+            using (var packageStream = await _packageDownloader.DownloadAsync(packageUri, CancellationToken.None))
+            {
+                await StorePackageFileInBackupLocationAsync(package, packageStream);
+            }
+        }
+
         public Task<string> CopyPackageFileForValidationSetAsync(PackageValidationSet validationSet)
         {
             var srcFileName = BuildFileName(
diff --git a/src/NuGet.Services.Validation.Orchestrator/ValidationSetProvider.cs b/src/NuGet.Services.Validation.Orchestrator/ValidationSetProvider.cs
@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
@@ -15,19 +16,22 @@ public class ValidationSetProvider : IValidationSetProvider
     {
         private readonly IValidationStorageService _validationStorageService;
         private readonly IValidationPackageFileService _packageFileService;
+        private readonly IValidatorProvider _validatorProvider;
         private readonly ValidationConfiguration _validationConfiguration;
         private readonly ITelemetryService _telemetryService;
         private readonly ILogger<ValidationSetProvider> _logger;
 
         public ValidationSetProvider(
             IValidationStorageService validationStorageService,
             IValidationPackageFileService packageFileService,
+            IValidatorProvider validatorProvider,
             IOptionsSnapshot<ValidationConfiguration> validationConfigurationAccessor,
             ITelemetryService telemetryService,
             ILogger<ValidationSetProvider> logger)
         {
             _validationStorageService = validationStorageService ?? throw new ArgumentNullException(nameof(validationStorageService));
             _packageFileService = packageFileService ?? throw new ArgumentNullException(nameof(packageFileService));
+            _validatorProvider = validatorProvider ?? throw new ArgumentNullException(nameof(validatorProvider));
             if (validationConfigurationAccessor == null)
             {
                 throw new ArgumentNullException(nameof(validationConfigurationAccessor));
@@ -70,6 +74,13 @@ public async Task<PackageValidationSet> TryGetOrCreateValidationSetAsync(Guid va
                     validationSet.PackageETag = null;
                 }
 
+                // If there are any processors in the validation set, back up the original. We back up from the
+                // validation set copy to avoid concurrency issues.
+                if (validationSet.PackageValidations.Any(x => _validatorProvider.IsProcessor(x.Type)))
+                {
+                    await _packageFileService.BackupPackageFileFromValidationSetPackageAsync(package, validationSet);
+                }
+
                 validationSet = await PersistValidationSetAsync(validationSet, package);
             }
             else
diff --git a/src/Validation.Common.Job/Validation.Common.Job.csproj b/src/Validation.Common.Job/Validation.Common.Job.csproj
@@ -84,7 +84,7 @@
       <Version>2.18.0</Version>
     </PackageReference>
     <PackageReference Include="NuGetGallery.Core">
-      <Version>4.4.4-dev-25804</Version>
+      <Version>4.4.4-dev-25969</Version>
     </PackageReference>
     <PackageReference Include="Serilog">
       <Version>2.5.0</Version>
diff --git a/tests/NuGet.Services.Validation.Orchestrator.Tests/ValidationPackageFileServiceFacts.cs b/tests/NuGet.Services.Validation.Orchestrator.Tests/ValidationPackageFileServiceFacts.cs
@@ -3,6 +3,7 @@
 
 using System;
 using System.IO;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
@@ -18,11 +19,16 @@ public class ValidationPackageFileServiceFacts
         private readonly PackageValidationSet _validationSet;
         private readonly Package _package;
         private readonly string _validationContainerName;
+        private readonly string _backupContainerName;
         private readonly string _packagesContainerName;
         private readonly string _packageFileName;
         private readonly string _validationSetPackageFileName;
+        private readonly string _backupFileName;
         private readonly Uri _testUri;
         private readonly string _etag;
+        private readonly string _packageContent;
+        private readonly MemoryStream _packageStream;
+        private readonly DateTimeOffset _endOfAccess;
         private readonly Mock<ICoreFileStorageService> _fileStorageService;
         private readonly Mock<IPackageDownloader> _packageDownloader;
         private readonly Mock<ILogger<ValidationPackageFileService>> _logger;
@@ -37,6 +43,7 @@ public ValidationPackageFileServiceFacts()
                     Id = "NuGet.Versioning",
                 },
                 NormalizedVersion = "4.5.0-ALPHA",
+                Hash = "NzMzMS1QNENLNEczSDQ1SA==",
             };
             _validationSet = new PackageValidationSet
             {
@@ -47,10 +54,15 @@ public ValidationPackageFileServiceFacts()
 
             _packagesContainerName = "packages";
             _validationContainerName = "validation";
+            _backupContainerName = "package-backups";
             _packageFileName = "nuget.versioning.4.5.0-alpha.nupkg";
             _validationSetPackageFileName = "validation-sets/0b44d53f-0689-4f82-9530-f25f26b321aa/nuget.versioning.4.5.0-alpha.nupkg";
+            _backupFileName = "nuget.versioning/4.5.0-alpha/rQw3wx1psxXzqB8TyM3nAQlK2RcluhsNwxmcqXE2YbgoDW735o8TPmIR4uWpoxUERddvFwjgRSGw7gNPCwuvJg2..nupkg";
             _testUri = new Uri("http://example.com/nupkg.nupkg");
             _etag = "\"some-etag\"";
+            _packageContent = "Hello, world.";
+            _packageStream = new MemoryStream(Encoding.ASCII.GetBytes(_packageContent));
+            _endOfAccess = new DateTimeOffset(2018, 1, 3, 8, 30, 0, TimeSpan.Zero);
 
             _fileStorageService = new Mock<ICoreFileStorageService>(MockBehavior.Strict);
             _packageDownloader = new Mock<IPackageDownloader>(MockBehavior.Strict);
@@ -62,6 +74,45 @@ public ValidationPackageFileServiceFacts()
                 _logger.Object);
         }
 
+        [Fact]
+        public async Task BackupPackageFileFromValidationSetPackageAsync()
+        {
+            DateTimeOffset? endOfAccess = null;
+            _fileStorageService
+                .Setup(x => x.GetFileReadUriAsync(
+                    _validationContainerName,
+                    _validationSetPackageFileName,
+                    It.IsAny<DateTimeOffset?>()))
+                .ReturnsAsync(_testUri)
+                .Callback<string, string, DateTimeOffset?>((_, __, a) => endOfAccess = a)
+                .Verifiable();
+
+            _packageDownloader
+                .Setup(x => x.DownloadAsync(_testUri, CancellationToken.None))
+                .ReturnsAsync(_packageStream)
+                .Verifiable();
+
+            _fileStorageService
+                .Setup(x => x.FileExistsAsync(_backupContainerName, _backupFileName))
+                .ReturnsAsync(false)
+                .Verifiable();
+
+            _fileStorageService
+                .Setup(x => x.SaveFileAsync(_backupContainerName, _backupFileName, _packageStream, true))
+                .Returns(Task.CompletedTask)
+                .Verifiable();
+
+            var before = DateTimeOffset.UtcNow;
+            await _target.BackupPackageFileFromValidationSetPackageAsync(_package, _validationSet);
+            var after = DateTimeOffset.UtcNow;
+
+            _fileStorageService.Verify();
+            _packageDownloader.Verify();
+            Assert.NotNull(endOfAccess);
+            Assert.InRange(endOfAccess.Value, before.AddMinutes(10), after.AddMinutes(10));
+            Assert.Throws<ObjectDisposedException>(() => _packageStream.Length);
+        }
+
         [Fact]
         public async Task DownloadPackageFileToDiskAsync()
         {
@@ -72,18 +123,17 @@ public async Task DownloadPackageFileToDiskAsync()
                     null))
                 .ReturnsAsync(_testUri)
                 .Verifiable();
-
-            var expected = new MemoryStream();
+            
             _packageDownloader
-                .Setup(x => x.DownloadAsync(
-                    _testUri,
-                    CancellationToken.None))
-                .ReturnsAsync(expected);
+                .Setup(x => x.DownloadAsync(_testUri, CancellationToken.None))
+                .ReturnsAsync(_packageStream)
+                .Verifiable();
 
             var actual = await _target.DownloadPackageFileToDiskAsync(_package);
 
-            Assert.Same(expected, actual);
+            Assert.Same(_packageStream, actual);
             _fileStorageService.Verify();
+            _packageDownloader.Verify();
         }
 
         [Fact]
@@ -96,7 +146,7 @@ public async Task CopyValidationPackageForValidationSetAsync()
                     _validationContainerName,
                     _validationSetPackageFileName,
                     It.Is<IAccessCondition>(y => y.IfMatchETag == null && y.IfNoneMatchETag == null)))
-                .ReturnsAsync(string.Empty)
+                .ReturnsAsync(_etag)
                 .Verifiable();
 
             await _target.CopyValidationPackageForValidationSetAsync(_validationSet);
@@ -133,7 +183,7 @@ public async Task CopyValidationPackageToPackageFileAsync()
                     _packagesContainerName,
                     _packageFileName,
                     It.Is<IAccessCondition>(y => y.IfNoneMatchETag == "*")))
-                .ReturnsAsync(string.Empty)
+                .ReturnsAsync(_etag)
                 .Verifiable();
 
             await _target.CopyValidationPackageToPackageFileAsync(_validationSet.PackageId, _validationSet.PackageNormalizedVersion);
@@ -168,7 +218,7 @@ public async Task CopyValidationSetPackageToPackageFileAsync()
                     _packagesContainerName,
                     _packageFileName,
                     accessCondition))
-                .ReturnsAsync(string.Empty)
+                .ReturnsAsync(_etag)
                 .Verifiable();
 
             await _target.CopyValidationSetPackageToPackageFileAsync(_validationSet, accessCondition);
@@ -211,16 +261,15 @@ public async Task DeletePackageForValidationSetAsync()
         [Fact]
         public async Task GetPackageForValidationSetReadUriAsync()
         {
-            var endOfAccess = new DateTimeOffset(2018, 1, 3, 8, 30, 0, TimeSpan.Zero);
             _fileStorageService
                 .Setup(x => x.GetFileReadUriAsync(
                     _validationContainerName,
                     _validationSetPackageFileName,
-                    endOfAccess))
+                    _endOfAccess))
                 .ReturnsAsync(_testUri)
                 .Verifiable();
 
-            var actual = await _target.GetPackageForValidationSetReadUriAsync(_validationSet, endOfAccess);
+            var actual = await _target.GetPackageForValidationSetReadUriAsync(_validationSet, _endOfAccess);
 
             Assert.Equal(_testUri, actual);
             _fileStorageService.Verify();
diff --git a/tests/NuGet.Services.Validation.Orchestrator.Tests/ValidationSetProviderFacts.cs b/tests/NuGet.Services.Validation.Orchestrator.Tests/ValidationSetProviderFacts.cs
