Skip to content
This repository was archived by the owner on Jul 30, 2024. It is now read-only.

Commit 29f959b

Browse files
loic-sharmaloic-sharma
authored
[Package Signing] Add Revalidate Certificate job (#385)
See https://github.com/NuGet/Engineering/issues/788
1 parent 6694d3b commit 29f959b

37 files changed

Lines changed: 1531 additions & 8 deletions

File tree

‎NuGet.Jobs.sln‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SnapshotAzureBlob", "src\Sn
115115
EndProject
116116
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.PackageSigning.ProcessSignature.Tests", "tests\Validation.PackageSigning.ProcessSignature.Tests\Validation.PackageSigning.ProcessSignature.Tests.csproj", "{26435822-8938-48C9-96FD-0DCCF8F7CE00}"
117117
EndProject
118+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.PackageSigning.RevalidateCertificate", "src\Validation.PackageSigning.RevalidateCertificate\Validation.PackageSigning.RevalidateCertificate.csproj", "{EA32E1E5-7E7D-44E6-B496-43E1FEDE9400}"
119+
EndProject
120+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.PackageSigning.RevalidateCertificate.Tests", "tests\Validation.PackageSigning.RevalidateCertificate.Tests\Validation.PackageSigning.RevalidateCertificate.Tests.csproj", "{64095857-E9E3-4D9C-8769-7E558CD757CB}"
121+
EndProject
118122
Global
119123
GlobalSection(SolutionConfigurationPlatforms) = preSolution
120124
Debug|Any CPU = Debug|Any CPU
@@ -295,6 +299,14 @@ Global
295299
{26435822-8938-48C9-96FD-0DCCF8F7CE00}.Debug|Any CPU.Build.0 = Debug|Any CPU
296300
{26435822-8938-48C9-96FD-0DCCF8F7CE00}.Release|Any CPU.ActiveCfg = Release|Any CPU
297301
{26435822-8938-48C9-96FD-0DCCF8F7CE00}.Release|Any CPU.Build.0 = Release|Any CPU
302+
{EA32E1E5-7E7D-44E6-B496-43E1FEDE9400}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
303+
{EA32E1E5-7E7D-44E6-B496-43E1FEDE9400}.Debug|Any CPU.Build.0 = Debug|Any CPU
304+
{EA32E1E5-7E7D-44E6-B496-43E1FEDE9400}.Release|Any CPU.ActiveCfg = Release|Any CPU
305+
{EA32E1E5-7E7D-44E6-B496-43E1FEDE9400}.Release|Any CPU.Build.0 = Release|Any CPU
306+
{64095857-E9E3-4D9C-8769-7E558CD757CB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
307+
{64095857-E9E3-4D9C-8769-7E558CD757CB}.Debug|Any CPU.Build.0 = Debug|Any CPU
308+
{64095857-E9E3-4D9C-8769-7E558CD757CB}.Release|Any CPU.ActiveCfg = Release|Any CPU
309+
{64095857-E9E3-4D9C-8769-7E558CD757CB}.Release|Any CPU.Build.0 = Release|Any CPU
298310
EndGlobalSection
299311
GlobalSection(SolutionProperties) = preSolution
300312
HideSolutionNode = FALSE
@@ -343,6 +355,8 @@ Global
343355
{DD043977-6BCD-475A-BEE2-8C34309EC622} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
344356
{ED2D370C-D921-433A-A0B9-A601F936EDD3} = {FA5644B5-4F08-43F6-86B3-039374312A47}
345357
{26435822-8938-48C9-96FD-0DCCF8F7CE00} = {6A776396-02B1-475D-A104-26940ADB04AB}
358+
{EA32E1E5-7E7D-44E6-B496-43E1FEDE9400} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
359+
{64095857-E9E3-4D9C-8769-7E558CD757CB} = {6A776396-02B1-475D-A104-26940ADB04AB}
346360
EndGlobalSection
347361
GlobalSection(ExtensibilityGlobals) = postSolution
348362
SolutionGuid = {284A7AC3-FB43-4F1F-9C9C-2AF0E1F46C2B}

‎build.ps1‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ Invoke-BuildStep 'Set version metadata in AssemblyInfo.cs' { `
109109
"$PSScriptRoot\src\Stats.CollectAzureChinaCDNLogs\Properties\AssemblyInfo.g.cs",
110110
"$PSScriptRoot\src\Validation.PackageSigning.ProcessSignature\Properties\AssemblyInfo.g.cs",
111111
"$PSScriptRoot\src\Validation.PackageSigning.ValidateCertificate\Properties\AssemblyInfo.g.cs",
112+
"$PSScriptRoot\src\Validation.PackageSigning.RevalidateCertificate\Properties\AssemblyInfo.g.cs",
112113
"$PSScriptRoot\src\NuGet.Jobs.Common\Properties\AssemblyInfo.g.cs",
113114
"$PSScriptRoot\src\Validation.Common.Job\Properties\AssemblyInfo.g.cs"
114115

@@ -165,7 +166,8 @@ Invoke-BuildStep 'Creating artifacts' {
165166
"src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj", `
166167
"src/Stats.CollectAzureChinaCDNLogs/Stats.CollectAzureChinaCDNLogs.csproj", `
167168
"src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj", `
168-
"src/Validation.PackageSigning.ValidateCertificate/Validation.PackageSigning.ValidateCertificate.csproj" `
169+
"src/Validation.PackageSigning.ValidateCertificate/Validation.PackageSigning.ValidateCertificate.csproj", `
170+
"src/Validation.PackageSigning.RevalidateCertificate/Validation.PackageSigning.RevalidateCertificate.csproj" `
169171
+ $ProjectsWithSymbols
170172

171173
Foreach ($Project in $Projects) {

‎src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,4 +136,4 @@
136136
<SignPath Condition="'$(NuGetBuildPath)' != ''">$(NuGetBuildPath)</SignPath>
137137
</PropertyGroup>
138138
<Import Project="$(SignPath)\sign.targets" Condition="Exists('$(SignPath)\sign.targets')" />
139-
</Project>
139+
</Project>

‎src/NuGet.Services.Validation.Orchestrator/PackageSigning/ValidateCertificate/PackageCertificatesValidator.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -244,7 +244,7 @@ private Task<PackageSignature> FindSignatureAsync(IValidationRequest request)
244244
private void PromoteSignature(IValidationRequest request, PackageSignature signature)
245245
{
246246

247-
var newSignatureStatus = (IsValidSignatureOutOfGracePeriod(request, signature))
247+
var newSignatureStatus = signature.IsPromotable()
248248
? PackageSignatureStatus.Valid
249249
: PackageSignatureStatus.InGracePeriod;
250250

‎src/Validation.Common.Job/Validation.Common.Job.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,4 +114,4 @@
114114
<SignPath Condition="'$(NuGetBuildPath)' != ''">$(NuGetBuildPath)</SignPath>
115115
</PropertyGroup>
116116
<Import Project="$(SignPath)\sign.targets" Condition="Exists('$(SignPath)\sign.targets')" />
117-
</Project>
117+
</Project>

‎src/Validation.PackageSigning.Core/Extensions/PackageSignatureExtensions.cs‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Linq;
6+
7+
namespace NuGet.Services.Validation
8+
{
9+
public static class PackageSignatureExtensions
10+
{
11+
/// <summary>
12+
/// Decide whether the valid signature should be considered "Valid" or "InGracePeriod".
13+
/// </summary>
14+
/// <param name="request">The validation request for the package whose signature should be inspected.</param>
15+
/// <param name="signature">The valid signature whose status should be decided.</param>
16+
/// <returns>True if the signature should be "Valid", false if it should be "InGracePeriod".</returns>
17+
public static bool IsPromotable(this PackageSignature signature)
18+
{
19+
if (signature == null)
20+
{
21+
throw new ArgumentNullException(nameof(signature));
22+
}
23+
24+
if (signature.Status == PackageSignatureStatus.Invalid)
25+
{
26+
throw new ArgumentException($"Package signature {signature.Key} is invalid and cannot be promoted", nameof(signature));
27+
}
28+
29+
var signingTime = signature.TrustedTimestamps.Max(t => t.Value);
30+
31+
// Ensure the timestamps' certificate statuses are fresher than the signature.
32+
foreach (var timestamp in signature.TrustedTimestamps)
33+
{
34+
// A valid signature should NEVER have a timestamp whose end certificate is revoked.
35+
// Note that it is possible for a valid signature to have an invalid certificate as
36+
// certain certificate statuses, like "NotTimeNested", do not affect signatures.
37+
if (timestamp.EndCertificate.Status == EndCertificateStatus.Revoked)
38+
{
39+
throw new ArgumentException(
40+
$"Package signature {signature.Key} is valid but has a timestamp whose end certificate is revoked",
41+
nameof(signature));
42+
}
43+
44+
if (!IsCertificateStatusPastTime(timestamp.EndCertificate, signingTime))
45+
{
46+
return false;
47+
}
48+
}
49+
50+
// A signature can be valid even if its certificate is revoked as long as the certificate
51+
// revocation date begins after the signature was created. The validation pipeline does
52+
// not revalidate revoked certificates, thus, a valid package signature with a revoked
53+
// certificate is considered out of the grace period regardless of the certificate's
54+
// status update time.
55+
if (signature.EndCertificate.Status != EndCertificateStatus.Revoked)
56+
{
57+
// Ensure the signature's certificate status is fresher than the signature.
58+
if (!IsCertificateStatusPastTime(signature.EndCertificate, signingTime))
59+
{
60+
return false;
61+
}
62+
}
63+
64+
return true;
65+
}
66+
67+
private static bool IsCertificateStatusPastTime(EndCertificate certificate, DateTime time)
68+
{
69+
return (certificate.StatusUpdateTime.HasValue && certificate.StatusUpdateTime > time);
70+
}
71+
}
72+
}

‎src/Validation.PackageSigning.Core/Validation.PackageSigning.Core.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
<ItemGroup>
4242
<Compile Include="Configuration\CertificateStoreConfiguration.cs" />
4343
<Compile Include="Extensions\CryptographicAttributeObjectCollectionExtensions.cs" />
44+
<Compile Include="Extensions\PackageSignatureExtensions.cs" />
4445
<Compile Include="Extensions\X509Certificate2Extensions.cs" />
4546
<Compile Include="Storage\CertificateStore.cs" />
4647
<Compile Include="Storage\ICertificateStore.cs" />

‎src/Validation.PackageSigning.RevalidateCertificate/App.config‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<configuration>
3+
<startup>
4+
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.1"/>
5+
</startup>
6+
</configuration>

‎src/Validation.PackageSigning.RevalidateCertificate/CertificateRevalidator.cs‎

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.Data.Entity;
7+
using System.Diagnostics;
8+
using System.Linq;
9+
using System.Threading.Tasks;
10+
using Microsoft.Extensions.Logging;
11+
using NuGet.Services.Validation;
12+
13+
namespace Validation.PackageSigning.RevalidateCertificate
14+
{
15+
public class CertificateRevalidator : ICertificateRevalidator
16+
{
17+
private readonly RevalidationConfiguration _config;
18+
private readonly IValidationEntitiesContext _context;
19+
private readonly IValidateCertificateEnqueuer _validationEnqueuer;
20+
private readonly ITelemetryService _telemetry;
21+
private readonly ILogger<CertificateRevalidator> _logger;
22+
23+
public CertificateRevalidator(
24+
RevalidationConfiguration config,
25+
IValidationEntitiesContext context,
26+
IValidateCertificateEnqueuer validationEnqueuer,
27+
ITelemetryService telemetry,
28+
ILogger<CertificateRevalidator> logger)
29+
{
30+
_config = config ?? throw new ArgumentNullException(nameof(config));
31+
_context = context ?? throw new ArgumentNullException(nameof(context));
32+
_validationEnqueuer = validationEnqueuer ?? throw new ArgumentNullException(nameof(validationEnqueuer));
33+
_telemetry = telemetry ?? throw new ArgumentNullException(nameof(telemetry));
34+
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
35+
}
36+
37+
public async Task PromoteSignaturesAsync()
38+
{
39+
using (_telemetry.TrackPromoteSignaturesDuration())
40+
{
41+
var promotableSignatures = await FindPromotableSignaturesAsync();
42+
43+
if (!promotableSignatures.Any())
44+
{
45+
return;
46+
}
47+
48+
foreach (var signature in promotableSignatures)
49+
{
50+
_logger.LogInformation(
51+
$"Promoting signature {{SignatureKey}} for package {{PackageKey}} to status {nameof(PackageSignatureStatus.Valid)}",
52+
signature.Key,
53+
signature.PackageKey);
54+
55+
signature.Status = PackageSignatureStatus.Valid;
56+
}
57+
58+
await _context.SaveChangesAsync();
59+
}
60+
}
61+
62+
private async Task<List<PackageSignature>> FindPromotableSignaturesAsync()
63+
{
64+
var promotableSignatures = new List<PackageSignature>();
65+
var signaturesScanned = 0;
66+
var scans = 0;
67+
68+
while (promotableSignatures.Count < _config.SignaturePromotionBatchSize)
69+
{
70+
var take = Math.Min(
71+
_config.SignaturePromotionScanSize,
72+
_config.SignaturePromotionBatchSize - promotableSignatures.Count);
73+
74+
var potentialSignatures = await _context.PackageSignatures
75+
.Where(s => s.Status == PackageSignatureStatus.InGracePeriod)
76+
.Include(s => s.EndCertificate)
77+
.Include(s => s.TrustedTimestamps.Select(t => t.EndCertificate))
78+
.OrderBy(s => s.CreatedAt)
79+
.Skip(signaturesScanned)
80+
.Take(take)
81+
.ToListAsync();
82+
83+
promotableSignatures.AddRange(potentialSignatures.Where(s => s.IsPromotable()));
84+
85+
signaturesScanned += potentialSignatures.Count;
86+
scans += 1;
87+
88+
// We've scanned all potential signatures if the last scan found less potential signatures
89+
// than the maximal scan size.
90+
if (potentialSignatures.Count < _config.SignaturePromotionScanSize)
91+
{
92+
break;
93+
}
94+
}
95+
96+
_logger.LogInformation(
97+
"Found {PromotableSignaturesCount} promotable signatures after {ScanCount} scans",
98+
promotableSignatures.Count,
99+
scans);
100+
101+
return promotableSignatures;
102+
}
103+
104+
public async Task RevalidateStaleCertificatesAsync()
105+
{
106+
using (_telemetry.TrackCertificateRevalidationDuration())
107+
{
108+
var certificates = await FindStaleCertificatesAsync();
109+
110+
if (!certificates.Any())
111+
{
112+
_logger.LogInformation("Did not find any stale certificates to revalidate");
113+
return;
114+
}
115+
116+
var validationId = Guid.NewGuid();
117+
var stopwatch = Stopwatch.StartNew();
118+
119+
using (_logger.BeginScope("Starting validation {ValidationId} for {CertificateCount} certificates...",
120+
validationId,
121+
certificates.Count))
122+
{
123+
await StartCertificateValidationsAsync(validationId, certificates);
124+
await WaitOnCertificateValidationsAsync(validationId, stopwatch);
125+
}
126+
}
127+
}
128+
129+
private Task<List<EndCertificate>> FindStaleCertificatesAsync()
130+
{
131+
// Find certificates that are before the stale cut off. Revoked certificates should never
132+
// be revalidated as Certificate Authorities may not drop revocation status.
133+
var staleCutoff = DateTime.UtcNow - _config.RevalidationPeriodForCertificates;
134+
135+
return _context.EndCertificates
136+
.Where(c => c.Status != EndCertificateStatus.Revoked)
137+
.Where(c => c.LastVerificationTime != null)
138+
.Where(c => c.LastVerificationTime < staleCutoff)
139+
.OrderBy(c => c.LastVerificationTime)
140+
.Take(_config.CertificateRevalidationBatchSize)
141+
.ToListAsync();
142+
}
143+
144+
private async Task StartCertificateValidationsAsync(Guid validationId, List<EndCertificate> certificates)
145+
{
146+
_logger.LogInformation("Starting {Count} certificate validations...", certificates.Count);
147+
148+
var validationTasks = new List<Task>();
149+
150+
foreach (var certificate in certificates)
151+
{
152+
var task = _validationEnqueuer.EnqueueValidationAsync(validationId, certificate);
153+
154+
validationTasks.Add(task);
155+
156+
_context.CertificateValidations.Add(new EndCertificateValidation
157+
{
158+
ValidationId = validationId,
159+
EndCertificate = certificate,
160+
Status = null,
161+
});
162+
}
163+
164+
// Wait until all revalidations have been enqueued, then, persist database changes.
165+
await Task.WhenAll(validationTasks);
166+
await _context.SaveChangesAsync();
167+
}
168+
169+
private async Task WaitOnCertificateValidationsAsync(Guid validationId, Stopwatch stopwatch)
170+
{
171+
_logger.LogInformation("Waiting until all certificate validations finish...");
172+
173+
while (stopwatch.Elapsed < _config.CertificateRevalidationTimeout)
174+
{
175+
await Task.Delay(_config.CertificateRevalidationPollTime);
176+
177+
var validationsLeft = await _context.CertificateValidations
178+
.Where(v => v.ValidationId == validationId)
179+
.Where(v => v.Status == null)
180+
.CountAsync();
181+
182+
if (validationsLeft == 0)
183+
{
184+
_logger.LogInformation("All certificate validations finished after {ElapsedTime}", stopwatch.Elapsed);
185+
186+
return;
187+
}
188+
else if (stopwatch.Elapsed >= _config.CertificateRevalidationTrackAfter)
189+
{
190+
_logger.LogWarning(
191+
"{ValidationsLeft} certificate validations left after {ElapsedTime} - this is longer than expected!",
192+
validationsLeft,
193+
stopwatch.Elapsed);
194+
195+
_telemetry.TrackCertificateRevalidationTakingTooLong();
196+
}
197+
else
198+
{
199+
_logger.LogInformation(
200+
"{ValidationsLeft} certificate validations left after {ElapsedTime}...",
201+
validationsLeft,
202+
stopwatch.Elapsed);
203+
}
204+
}
205+
206+
if (stopwatch.Elapsed >= _config.CertificateRevalidationTimeout)
207+
{
208+
_logger.LogError("Reached certificate revalidation timeout after {ElapsedTime}", stopwatch.Elapsed);
209+
_telemetry.TrackCertificateRevalidationReachedTimeout();
210+
}
211+
}
212+
}
213+
}

‎src/Validation.PackageSigning.RevalidateCertificate/ICertificateRevalidator.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System.Threading.Tasks;
5+
6+
namespace Validation.PackageSigning.RevalidateCertificate
7+
{
8+
interface ICertificateRevalidator
9+
{
10+
Task PromoteSignaturesAsync();
11+
12+
Task RevalidateStaleCertificatesAsync();
13+
}
14+
}

0 commit comments

Comments
 (0)