Merge pull request #395 from NuGet/dev

[ReleasePrep][2018.04.02]RI of dev into master

Author: loic-sharma

NuGet.Jobs.sln

@@ -115,6 +115,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SnapshotAzureBlob", "src\Sn
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.PackageSigning.ProcessSignature.Tests", "tests\Validation.PackageSigning.ProcessSignature.Tests\Validation.PackageSigning.ProcessSignature.Tests.csproj", "{26435822-8938-48C9-96FD-0DCCF8F7CE00}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.PackageSigning.RevalidateCertificate", "src\Validation.PackageSigning.RevalidateCertificate\Validation.PackageSigning.RevalidateCertificate.csproj", "{EA32E1E5-7E7D-44E6-B496-43E1FEDE9400}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.PackageSigning.RevalidateCertificate.Tests", "tests\Validation.PackageSigning.RevalidateCertificate.Tests\Validation.PackageSigning.RevalidateCertificate.Tests.csproj", "{64095857-E9E3-4D9C-8769-7E558CD757CB}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tests.Stats.AggregateCdnDownloadsInGallery", "tests\Tests.Stats.AggregateCdnDownloadsInGallery\Tests.Stats.AggregateCdnDownloadsInGallery.csproj", "{136411AF-B9FA-438D-B790-9FB78A5F7F54}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -295,6 +301,18 @@ Global
 		{26435822-8938-48C9-96FD-0DCCF8F7CE00}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{26435822-8938-48C9-96FD-0DCCF8F7CE00}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{26435822-8938-48C9-96FD-0DCCF8F7CE00}.Release|Any CPU.Build.0 = Release|Any CPU
+		{EA32E1E5-7E7D-44E6-B496-43E1FEDE9400}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{EA32E1E5-7E7D-44E6-B496-43E1FEDE9400}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{EA32E1E5-7E7D-44E6-B496-43E1FEDE9400}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{EA32E1E5-7E7D-44E6-B496-43E1FEDE9400}.Release|Any CPU.Build.0 = Release|Any CPU
+		{64095857-E9E3-4D9C-8769-7E558CD757CB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{64095857-E9E3-4D9C-8769-7E558CD757CB}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{64095857-E9E3-4D9C-8769-7E558CD757CB}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{64095857-E9E3-4D9C-8769-7E558CD757CB}.Release|Any CPU.Build.0 = Release|Any CPU
+		{136411AF-B9FA-438D-B790-9FB78A5F7F54}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{136411AF-B9FA-438D-B790-9FB78A5F7F54}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{136411AF-B9FA-438D-B790-9FB78A5F7F54}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{136411AF-B9FA-438D-B790-9FB78A5F7F54}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -343,6 +361,9 @@ Global
 		{DD043977-6BCD-475A-BEE2-8C34309EC622} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
 		{ED2D370C-D921-433A-A0B9-A601F936EDD3} = {FA5644B5-4F08-43F6-86B3-039374312A47}
 		{26435822-8938-48C9-96FD-0DCCF8F7CE00} = {6A776396-02B1-475D-A104-26940ADB04AB}
+		{EA32E1E5-7E7D-44E6-B496-43E1FEDE9400} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
+		{64095857-E9E3-4D9C-8769-7E558CD757CB} = {6A776396-02B1-475D-A104-26940ADB04AB}
+		{136411AF-B9FA-438D-B790-9FB78A5F7F54} = {6A776396-02B1-475D-A104-26940ADB04AB}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {284A7AC3-FB43-4F1F-9C9C-2AF0E1F46C2B}

build.ps1

@@ -109,6 +109,7 @@ Invoke-BuildStep 'Set version metadata in AssemblyInfo.cs' { `
             "$PSScriptRoot\src\Stats.CollectAzureChinaCDNLogs\Properties\AssemblyInfo.g.cs",
             "$PSScriptRoot\src\Validation.PackageSigning.ProcessSignature\Properties\AssemblyInfo.g.cs",
             "$PSScriptRoot\src\Validation.PackageSigning.ValidateCertificate\Properties\AssemblyInfo.g.cs",
+            "$PSScriptRoot\src\Validation.PackageSigning.RevalidateCertificate\Properties\AssemblyInfo.g.cs",
             "$PSScriptRoot\src\NuGet.Jobs.Common\Properties\AssemblyInfo.g.cs",
             "$PSScriptRoot\src\Validation.Common.Job\Properties\AssemblyInfo.g.cs"
 
@@ -137,6 +138,13 @@ Invoke-BuildStep 'Prepare NuGetCDNRedirect Package' { Prepare-NuGetCDNRedirect }
     -ev +BuildErrors
 
 Invoke-BuildStep 'Creating artifacts' {
+        # We need a few projects to be published for sharing the common bits with other repos.
+        # We need symbols published for those, too. All other packages are deployment ones and
+        # don't need to be shared, hence no need for symbols for them
+        $ProjectsWithSymbols =
+            "src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj",
+            "src/Validation.Common.Job/Validation.Common.Job.csproj"
+
         $Projects = `
             "src/Stats.CollectAzureCdnLogs/Stats.CollectAzureCdnLogs.csproj", `
             "src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj", `
@@ -159,11 +167,12 @@ Invoke-BuildStep 'Creating artifacts' {
             "src/Stats.CollectAzureChinaCDNLogs/Stats.CollectAzureChinaCDNLogs.csproj", `
             "src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj", `
             "src/Validation.PackageSigning.ValidateCertificate/Validation.PackageSigning.ValidateCertificate.csproj", `
-            "src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj", `
-            "src/Validation.Common.Job/Validation.Common.Job.csproj"
+            "src/Validation.PackageSigning.RevalidateCertificate/Validation.PackageSigning.RevalidateCertificate.csproj" `
+            + $ProjectsWithSymbols
 
         Foreach ($Project in $Projects) {
-            New-Package (Join-Path $PSScriptRoot "$Project") -Configuration $Configuration -BuildNumber $BuildNumber -Version $SemanticVersion -Branch $Branch -MSBuildVersion "$msBuildVersion"
+            $Symbols = $ProjectsWithSymbols -contains $Project;
+            New-Package (Join-Path $PSScriptRoot "$Project") -Configuration $Configuration -BuildNumber $BuildNumber -Version $SemanticVersion -Branch $Branch -MSBuildVersion "$msBuildVersion" -Symbols:$Symbols
         }
     } `
     -ev +BuildErrors

src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs

@@ -117,5 +117,7 @@ public static class JobArgumentNames
         public const string MailFrom = "MailFrom";
         public const string SmtpUri = "SmtpUri";
 
+        // Arguments specific to Stats.AggregateCdnDownloadsInGallery
+        public static string BatchSleepSeconds = "BatchSleepSeconds";
     }
 }

src/NuGet.Jobs.Common/Configuration/ValidationStorageConfiguration.cs

@@ -0,0 +1,10 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace NuGet.Jobs.Configuration
+{
+    public class ValidationStorageConfiguration
+    {
+        public string ConnectionString { get; set; }
+    }
+}

src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj

@@ -191,6 +191,7 @@
     <Compile Include="Configuration\JobArgumentNames.cs" />
     <Compile Include="Configuration\ServiceBusConfiguration.cs" />
     <Compile Include="Configuration\ValidationDbConfiguration.cs" />
+    <Compile Include="Configuration\ValidationStorageConfiguration.cs" />
     <Compile Include="Extensions\SqlConnectionStringBuilderExtensions.cs" />
     <Compile Include="Extensions\XElementExtensions.cs" />
     <Compile Include="SecretReader\ISecretReaderFactory.cs" />

src/NuGet.Services.Validation.Orchestrator/Job.cs

@@ -23,6 +23,7 @@
 using NuGet.Jobs.Validation.Common;
 using NuGet.Jobs.Validation.PackageSigning.Messages;
 using NuGet.Jobs.Validation.PackageSigning.Storage;
+using NuGet.Jobs.Validation.Storage;
 using NuGet.Services.Configuration;
 using NuGet.Services.KeyVault;
 using NuGet.Services.Logging;
@@ -203,6 +204,7 @@ private void ConfigureJobServices(IServiceCollection services, IConfigurationRoo
             services.AddTransient<IBrokeredMessageSerializer<SignatureValidationMessage>, SignatureValidationMessageSerializer>();
             services.AddTransient<IBrokeredMessageSerializer<CertificateValidationMessage>, CertificateValidationMessageSerializer>();
             services.AddTransient<IValidatorStateService, ValidatorStateService>();
+            services.AddTransient<ISimpleCloudBlobProvider, SimpleCloudBlobProvider>();
             services.AddTransient<PackageSigningValidator>();
             services.AddTransient<MailSenderConfiguration>(serviceProvider =>
             {

src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj

@@ -108,7 +108,7 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="NuGet.Services.Validation.Issues">
-      <Version>2.19.1</Version>
+      <Version>2.22.0</Version>
     </PackageReference>
   </ItemGroup>
   <ItemGroup>
@@ -136,4 +136,4 @@
     <SignPath Condition="'$(NuGetBuildPath)' != ''">$(NuGetBuildPath)</SignPath>
   </PropertyGroup>
   <Import Project="$(SignPath)\sign.targets" Condition="Exists('$(SignPath)\sign.targets')" />
-</Project>
+</Project>

src/NuGet.Services.Validation.Orchestrator/PackageSigning/ProcessSignature/PackageSigningValidator.cs

@@ -6,26 +6,29 @@
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using NuGet.Jobs.Validation.PackageSigning.Storage;
-using NuGet.Services.Validation.Orchestrator;
+using NuGet.Jobs.Validation.Storage;
 using NuGet.Services.Validation.Orchestrator.Telemetry;
 
 namespace NuGet.Services.Validation.PackageSigning.ProcessSignature
 {
-    public class PackageSigningValidator : BaseValidator, IValidator
+    public class PackageSigningValidator : IProcessor
     {
         private readonly IValidatorStateService _validatorStateService;
         private readonly IProcessSignatureEnqueuer _signatureVerificationEnqueuer;
+        private readonly ISimpleCloudBlobProvider _blobProvider;
         private readonly ITelemetryService _telemetryService;
         private readonly ILogger<PackageSigningValidator> _logger;
 
         public PackageSigningValidator(
             IValidatorStateService validatorStateService,
             IProcessSignatureEnqueuer signatureVerificationEnqueuer,
+            ISimpleCloudBlobProvider blobProvider,
             ITelemetryService telemetryService,
             ILogger<PackageSigningValidator> logger)
         {
             _validatorStateService = validatorStateService ?? throw new ArgumentNullException(nameof(validatorStateService));
             _signatureVerificationEnqueuer = signatureVerificationEnqueuer ?? throw new ArgumentNullException(nameof(signatureVerificationEnqueuer));
+            _blobProvider = blobProvider ?? throw new ArgumentNullException(nameof(blobProvider));
             _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
@@ -44,6 +47,25 @@ public async Task<IValidationResult> StartAsync(IValidationRequest request)
             return validatorStatus.ToValidationResult();
         }
 
+        public async Task CleanUpAsync(IValidationRequest request)
+        {
+            var validatorStatus = await _validatorStateService.GetStatusAsync(request);
+
+            if (validatorStatus.NupkgUrl == null)
+            {
+                return;
+            }
+
+            _logger.LogInformation(
+                "Cleaning up the .nupkg URL for validation ID {ValidationId} ({PackageId} {PackageVersion}).",
+                request.ValidationId,
+                request.PackageId,
+                request.PackageVersion);
+
+            var blob = _blobProvider.GetBlobFromUrl(validatorStatus.NupkgUrl);
+            await blob.DeleteIfExistsAsync();
+        }
+
         private async Task<ValidatorStatus> StartInternalAsync(IValidationRequest request)
         {
             // Check that this is the first validation for this specific request.

src/NuGet.Services.Validation.Orchestrator/PackageSigning/ValidateCertificate/PackageCertificatesValidator.cs

@@ -244,7 +244,7 @@ private Task<PackageSignature> FindSignatureAsync(IValidationRequest request)
         private void PromoteSignature(IValidationRequest request, PackageSignature signature)
         {
 
-            var newSignatureStatus = (IsValidSignatureOutOfGracePeriod(request, signature))
+            var newSignatureStatus = signature.IsPromotable()
                                         ? PackageSignatureStatus.Valid
                                         : PackageSignatureStatus.InGracePeriod;
 

src/NuGet.Services.Validation.Orchestrator/settings.json

@@ -4,7 +4,7 @@
       {
         "name": "VcsValidator",
         "TrackAfter": "1:00:00:00",
-        "requiredValidations": [],
+        "requiredValidations": [ "PackageSigningValidator" ],
         "ShouldStart": true,
         "FailureBehavior": "MustSucceed"
       },