Merge pull request #366 from NuGet/dev

[ReleasePrep][2018.03.09]RI of dev into master

Author: Scott Bommarito

src/Gallery.CredentialExpiration/Strings.Designer.cs

@@ -151,7 +151,7 @@ WHERE u.[EmailAllowed] = 1
   AND u.[EmailAddress] <> ''
   AND cr.[Expires] <= DATEADD(day,@DaysBeforeExpiration,GETUTCDATE())
   AND cr.[Expires] > DATEADD(day,-1 * @DaysBeforeExpiration,GETUTCDATE())
-  AND (cr.[Type] = 'apikey.v1' or cr.[Type] = 'apikey.v2')
+  AND (cr.[Type] = 'apikey.v3' or cr.[Type] = 'apikey.v4')
 ORDER BY u.[Username]</value>
   </data>
   <data name="ExpiredEmailSubject" xml:space="preserve">

src/Gallery.CredentialExpiration/Strings.resx

@@ -5,10 +5,6 @@ namespace NuGet.Services.Validation.Orchestrator
 {
     public class SmtpConfiguration
     {
-        public string SmtpHost { get; set; }
-        public int SmtpPort { get; set; }
-        public bool EnableSsl { get; set; }
-        public string Username { get; set; }
-        public string Password { get; set; }
+        public string SmtpUri { get; set; }
     }
 }

src/NuGet.Services.Validation.Orchestrator/Configuration/SmtpConfiguration.cs

@@ -0,0 +1,60 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Threading.Tasks;
+using NuGetGallery;
+
+namespace NuGet.Services.Validation.Orchestrator
+{
+    public interface IValidationPackageFileService : ICorePackageFileService
+    {
+        /// <summary>
+        /// Copy a package from the validation container to a location specific for the validation set. This allows the
+        /// validation set to have its own copy of the package to mutate (via <see cref="IProcessor"/>) and validate.
+        /// </summary>
+        /// <param name="validationSet">The validation set, containing validation set and package identifiers.</param>
+        Task CopyValidationPackageForValidationSetAsync(PackageValidationSet validationSet);
+
+        /// <summary>
+        /// Copy a package from the packages container to a location specific for the validation set. This allows the
+        /// validation set to have its own copy of the package to mutate (via <see cref="IProcessor"/>) and validate.
+        /// </summary>
+        /// <param name="validationSet">The validation set, containing validation set and package identifiers.</param>
+        Task CopyPackageFileForValidationSetAsync(PackageValidationSet validationSet);
+
+        /// <summary>
+        /// Copy a package from a location specific for the validation set to the packages container.
+        /// </summary>
+        /// <param name="validationSet">The validation set, containing validation set and package identifiers.</param>
+        Task CopyValidationSetPackageToPackageFileAsync(PackageValidationSet validationSet);
+
+        /// <summary>
+        /// Copy a package from the validation container to the packages container.
+        /// </summary>
+        /// <param name="id">The package ID.</param>
+        /// <param name="normalizedVersion">The normalized package version.</param>
+        Task CopyValidationPackageToPackageFileAsync(string id, string normalizedVersion);
+
+        /// <summary>
+        /// Delete a package from a location specific for the validation set.
+        /// </summary>
+        /// <param name="validationSet">The validation set, containing validation set and package identifiers.</param>
+        Task DeletePackageForValidationSetAsync(PackageValidationSet validationSet);
+
+        /// <summary>
+        /// Generates the URI for the specified validating package, which can be used to download it.
+        /// </summary>
+        /// <param name="validationSet">The validation set, containing validation set and package identifiers.</param>
+        /// <param name="endOfAccess">The timestamp that limits the URI usage period.</param>
+        /// <returns>Time limited (if implementation supports) URI for the package.</returns>
+        Task<Uri> GetPackageForValidationSetReadUriAsync(PackageValidationSet validationSet, DateTimeOffset endOfAccess);
+
+        /// <summary>
+        /// Checks whether the validation set's package file exists.
+        /// </summary>
+        /// <param name="validationSet">The validation set, containing validation set and package identifiers.</param>
+        /// <returns>True if file exists, false otherwise</returns>
+        Task<bool> DoesValidationSetPackageExistAsync(PackageValidationSet validationSet);
+    }
+}

src/NuGet.Services.Validation.Orchestrator/IValidationPackageFileService.cs

@@ -190,14 +190,21 @@ private void ConfigureJobServices(IServiceCollection services, IConfigurationRoo
             {
                 var smtpConfigurationAccessor = serviceProvider.GetRequiredService<IOptionsSnapshot<SmtpConfiguration>>();
                 var smtpConfiguration = smtpConfigurationAccessor.Value;
+                if (string.IsNullOrWhiteSpace(smtpConfiguration.SmtpUri))
+                {
+                    return new MailSenderConfiguration();
+                }
+                var smtpUri = new SmtpUri(new Uri(smtpConfiguration.SmtpUri));
                 return new MailSenderConfiguration
                 {
                     DeliveryMethod = System.Net.Mail.SmtpDeliveryMethod.Network,
-                    Host = smtpConfiguration.SmtpHost,
-                    Port = smtpConfiguration.SmtpPort,
-                    EnableSsl = smtpConfiguration.EnableSsl,
+                    Host = smtpUri.Host,
+                    Port = smtpUri.Port,
+                    EnableSsl = smtpUri.Secure,
                     UseDefaultCredentials = false,
-                    Credentials = new NetworkCredential(smtpConfiguration.Username, smtpConfiguration.Password)
+                    Credentials = new NetworkCredential(
+                        smtpUri.UserName,
+                        smtpUri.Password)
                 };
             });
             services.AddTransient<IMailSender>(serviceProvider =>
@@ -280,21 +287,27 @@ private static IServiceProvider CreateProvider(IServiceCollection services)
                     parameterKey: ValidationStorageBindingKey);
 
             containerBuilder
-                .RegisterKeyedTypeWithKeyedParameter<NuGetGallery.ICorePackageFileService, NuGetGallery.CorePackageFileService, NuGetGallery.ICoreFileStorageService>(
+                .RegisterKeyedTypeWithKeyedParameter<IValidationPackageFileService, ValidationPackageFileService, NuGetGallery.ICoreFileStorageService>(
                     typeKey: ValidationStorageBindingKey,
                     parameterKey: ValidationStorageBindingKey);
 
             containerBuilder
                 .RegisterTypeWithKeyedParameter<
                     IValidationOutcomeProcessor,
                     ValidationOutcomeProcessor,
-                    NuGetGallery.ICorePackageFileService>(ValidationStorageBindingKey);
+                    IValidationPackageFileService>(ValidationStorageBindingKey);
+
+            containerBuilder
+                .RegisterTypeWithKeyedParameter<
+                    IValidationSetProvider,
+                    ValidationSetProvider,
+                    IValidationPackageFileService>(ValidationStorageBindingKey);
 
             containerBuilder
                 .RegisterTypeWithKeyedParameter<
                     IValidationSetProcessor,
                     ValidationSetProcessor,
-                    NuGetGallery.ICorePackageFileService>(ValidationStorageBindingKey);
+                    IValidationPackageFileService>(ValidationStorageBindingKey);
 
             containerBuilder
                 .RegisterType<ScopedMessageHandler<PackageValidationMessageData>>()

src/NuGet.Services.Validation.Orchestrator/Job.cs

@@ -53,6 +53,7 @@
     <Compile Include="Error.cs" />
     <Compile Include="IMessageService.cs" />
     <Compile Include="IValidationOutcomeProcessor.cs" />
+    <Compile Include="IValidationPackageFileService.cs" />
     <Compile Include="IValidationSetProcessor.cs" />
     <Compile Include="IValidationSetProvider.cs" />
     <Compile Include="IValidationStorageService.cs" />
@@ -79,6 +80,7 @@
     <Compile Include="Configuration\ValidationConfiguration.cs" />
     <Compile Include="Configuration\ValidationConfigurationItem.cs" />
     <Compile Include="ValidationFailureBehavior.cs" />
+    <Compile Include="ValidationPackageFileService.cs" />
     <Compile Include="Vcs\IPackageCriteria.cs" />
     <Compile Include="Vcs\IPackageCriteriaEvaluator.cs" />
     <Compile Include="Vcs\PackageCriteriaEvaluator.cs" />

src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj

@@ -226,8 +226,8 @@ private Task<PackageSignature> FindSignatureAsync(IValidationRequest request)
         {
             return _validationContext
                         .PackageSignatures
-                        .Include(s => s.TrustedTimestamps)
                         .Include(s => s.EndCertificate)
+                        .Include(s => s.TrustedTimestamps.Select(t => t.EndCertificate))
                         .SingleAsync(s => s.PackageKey == request.PackageKey);
         }
 

src/NuGet.Services.Validation.Orchestrator/PackageCertificates/PackageCertificatesValidator.cs

@@ -15,7 +15,7 @@ namespace NuGet.Services.Validation.Orchestrator
     public class ValidationOutcomeProcessor : IValidationOutcomeProcessor
     {
         private readonly ICorePackageService _galleryPackageService;
-        private readonly ICorePackageFileService _packageFileService;
+        private readonly IValidationPackageFileService _packageFileService;
         private readonly IPackageValidationEnqueuer _validationEnqueuer;
         private readonly ValidationConfiguration _validationConfiguration;
         private readonly IMessageService _messageService;
@@ -24,7 +24,7 @@ public class ValidationOutcomeProcessor : IValidationOutcomeProcessor
 
         public ValidationOutcomeProcessor(
             ICorePackageService galleryPackageService,
-            ICorePackageFileService packageFileService,
+            IValidationPackageFileService packageFileService,
             IPackageValidationEnqueuer validationEnqueuer,
             IOptionsSnapshot<ValidationConfiguration> validationConfigurationAccessor,
             IMessageService messageService,
@@ -106,6 +106,8 @@ ValidationConfigurationItem GetValidationConfigurationItem(string validationName
                         validationSet.ValidationTrackingId);
                 }
 
+                await _packageFileService.DeletePackageForValidationSetAsync(validationSet);
+
                 TrackTotalValidationDuration(validationSet, isSuccess: false);
             }
             else if (AllValidationsSucceeded(validationSet, GetValidationConfigurationItem))
@@ -114,6 +116,7 @@ ValidationConfigurationItem GetValidationConfigurationItem(string validationName
                     package.PackageRegistration.Id,
                     package.NormalizedVersion,
                     validationSet.ValidationTrackingId);
+
                 if (package.PackageStatusKey != PackageStatus.Available)
                 {
                     await MoveFileToPublicStorageAndMarkPackageAsAvailable(validationSet, package);
@@ -139,6 +142,9 @@ ValidationConfigurationItem GetValidationConfigurationItem(string validationName
                         TrackMissingNupkgForAvailablePackage(validationSet);
                     }
                 }
+
+                await _packageFileService.DeletePackageForValidationSetAsync(validationSet);
+
                 _logger.LogInformation("Done processing {PackageId} {PackageVersion} {ValidationSetId}",
                     package.PackageRegistration.Id,
                     package.NormalizedVersion,
@@ -176,23 +182,27 @@ private async Task MoveFileToPublicStorageAndMarkPackageAsAvailable(PackageValid
                 package.PackageRegistration.Id,
                 package.NormalizedVersion,
                 validationSet.ValidationTrackingId);
-            var packageStream = await _packageFileService.DownloadValidationPackageFileAsync(package);
 
-            try
+            if (await _packageFileService.DoesValidationSetPackageExistAsync(validationSet))
             {
-                await _packageFileService.SavePackageFileAsync(package, packageStream);
+                await CopyAsync(
+                    validationSet,
+                    package,
+                    x => _packageFileService.CopyValidationSetPackageToPackageFileAsync(x));
             }
-            catch (InvalidOperationException)
+            else
             {
-                // The package already exists in the packages container. This can happen if the DB commit below fails
-                // and this flow is retried. We assume that the package content has not changed. Today there is no way
-                // for the content to change. Hard deletes (the one way a package ID and version can get different
-                // content) delete from both the packages and validating container so this can't be a mismatch.
                 _logger.LogInformation(
-                    "Package already exists in packages container for {PackageId} {PackageVersion}, validation set {ValidationSetId}",
+                    "The package specific to the validation set does not exist. Falling back to the validation " +
+                    "container for package {PackageId} {PackageVersion}, validation set {ValidationSetId}",
                     package.PackageRegistration.Id,
                     package.NormalizedVersion,
                     validationSet.ValidationTrackingId);
+
+                await CopyAsync(
+                    validationSet,
+                    package,
+                    x => _packageFileService.CopyValidationPackageToPackageFileAsync(x.PackageId, x.PackageNormalizedVersion));
             }
 
             _logger.LogInformation("Marking package {PackageId} {PackageVersion}, validation set {ValidationSetId} as {PackageStatus} in DB",
@@ -229,6 +239,26 @@ private async Task MoveFileToPublicStorageAndMarkPackageAsAvailable(PackageValid
             await _packageFileService.DeleteValidationPackageFileAsync(package.PackageRegistration.Id, package.Version);
         }
 
+        private async Task CopyAsync(PackageValidationSet validationSet, Package package, Func<PackageValidationSet, Task> copyAsync)
+        {
+            try
+            {
+                await copyAsync(validationSet);
+            }
+            catch (InvalidOperationException)
+            {
+                // The package already exists in the packages container. This can happen if the DB commit below fails
+                // and this flow is retried. We assume that the package content has not changed. Today there is no way
+                // for the content to change. Hard deletes (the one way a package ID and version can get different
+                // content) delete from both the packages and validating container so this can't be a mismatch.
+                _logger.LogInformation(
+                    "Package already exists in packages container for {PackageId} {PackageVersion}, validation set {ValidationSetId}",
+                    package.PackageRegistration.Id,
+                    package.NormalizedVersion,
+                    validationSet.ValidationTrackingId);
+            }
+        }
+
         private bool AllValidationsSucceeded(
             PackageValidationSet packageValidationSet,
             Func<string, ValidationConfigurationItem> getValidationConfigurationItem)