Skip to content
This repository was archived by the owner on Jul 30, 2024. It is now read-only.

Commit 028fef4

Browse files
joelverhagenjoelverhagen
committed
Add visibility delay option for signature and certificate Service Bus (#371)
Progress on NuGet/Engineering#1190
1 parent 876ca9d commit 028fef4

16 files changed

Lines changed: 376 additions & 41 deletions

‎src/NuGet.Services.Validation.Orchestrator/Job.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,7 @@ private void ConfigureJobServices(IServiceCollection services, IConfigurationRoo
199199
services.AddTransient<IValidationSetProvider, ValidationSetProvider>();
200200
services.AddTransient<IValidationSetProcessor, ValidationSetProcessor>();
201201
services.AddTransient<IBrokeredMessageSerializer<SignatureValidationMessage>, SignatureValidationMessageSerializer>();
202+
services.AddTransient<IBrokeredMessageSerializer<CertificateValidationMessage>, CertificateValidationMessageSerializer>();
202203
services.AddTransient<IValidatorStateService, ValidatorStateService>();
203204
services.AddTransient<PackageSigningValidator>();
204205
services.AddTransient<MailSenderConfiguration>(serviceProvider =>

‎src/NuGet.Services.Validation.Orchestrator/PackageCertificates/CertificateVerificationEnqueuer.cs‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
using System;
55
using System.Threading.Tasks;
6+
using Microsoft.Extensions.Options;
67
using NuGet.Jobs.Validation.PackageSigning.Messages;
78
using NuGet.Services.ServiceBus;
89

@@ -11,18 +12,27 @@ namespace NuGet.Services.Validation.PackageCertificates
1112
public class CertificateVerificationEnqueuer : ICertificateVerificationEnqueuer
1213
{
1314
private readonly ITopicClient _topicClient;
14-
private static readonly IBrokeredMessageSerializer<CertificateValidationMessage> _serializer = new CertificateValidationMessageSerializer();
15+
private readonly IOptionsSnapshot<PackageCertificatesConfiguration> _configuration;
16+
private readonly IBrokeredMessageSerializer<CertificateValidationMessage> _serializer;
1517

16-
public CertificateVerificationEnqueuer(ITopicClient topicClient)
18+
public CertificateVerificationEnqueuer(
19+
ITopicClient topicClient,
20+
IBrokeredMessageSerializer<CertificateValidationMessage> serializer,
21+
IOptionsSnapshot<PackageCertificatesConfiguration> configuration)
1722
{
1823
_topicClient = topicClient ?? throw new ArgumentNullException(nameof(topicClient));
24+
_serializer = serializer ?? throw new ArgumentNullException(nameof(serializer));
25+
_configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
1926
}
2027

2128
public async Task EnqueueVerificationAsync(IValidationRequest request, EndCertificate certificate)
2229
{
2330
var message = new CertificateValidationMessage(certificate.Key, request.ValidationId);
2431
var brokeredMessage = _serializer.Serialize(message);
2532

33+
var visibleAt = DateTimeOffset.UtcNow + (_configuration.Value.MessageDelay ?? TimeSpan.Zero);
34+
brokeredMessage.ScheduledEnqueueTimeUtc = visibleAt;
35+
2636
await _topicClient.SendAsync(brokeredMessage);
2737
}
2838
}

‎src/NuGet.Services.Validation.Orchestrator/PackageCertificates/PackageCertificatesConfiguration.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,11 @@ public class PackageCertificatesConfiguration
1313
/// </summary>
1414
public TimeSpan? CertificateRevalidationThreshold { get; set; }
1515

16+
/// <summary>
17+
/// The visibility delay to apply to Service Bus messages requesting a new validation.
18+
/// </summary>
19+
public TimeSpan? MessageDelay { get; set; }
20+
1621
/// <summary>
1722
/// The Service Bus configuration used to enqueue package certificate validations.
1823
/// </summary>

‎src/NuGet.Services.Validation.Orchestrator/PackageCertificates/PackageCertificatesValidator.cs‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,13 @@
44
using System;
55
using System.Collections.Generic;
66
using System.Data.Entity;
7+
using System.Diagnostics;
78
using System.Linq;
89
using System.Threading.Tasks;
910
using Microsoft.Extensions.Logging;
1011
using NuGet.Jobs.Validation.PackageSigning.Storage;
1112
using NuGet.Services.Validation.Orchestrator;
13+
using NuGet.Services.Validation.Orchestrator.Telemetry;
1214

1315
namespace NuGet.Services.Validation.PackageCertificates
1416
{
@@ -19,6 +21,7 @@ public class PackageCertificatesValidator : BaseValidator, IValidator
1921
private readonly IValidationEntitiesContext _validationContext;
2022
private readonly IValidatorStateService _validatorStateService;
2123
private readonly ICertificateVerificationEnqueuer _certificateVerificationEnqueuer;
24+
private readonly ITelemetryService _telemetryService;
2225
private readonly TimeSpan _certificateRevalidationThresholdTime;
2326
private readonly ILogger<PackageCertificatesValidator> _logger;
2427

@@ -34,12 +37,14 @@ public PackageCertificatesValidator(
3437
IValidationEntitiesContext validationContext,
3538
IValidatorStateService validatorStateService,
3639
ICertificateVerificationEnqueuer certificateVerificationEnqueuer,
40+
ITelemetryService telemetryService,
3741
ILogger<PackageCertificatesValidator> logger,
3842
TimeSpan? certificateRevalidationThreshold = null)
3943
{
4044
_validationContext = validationContext ?? throw new ArgumentNullException(nameof(validationContext));
4145
_validatorStateService = validatorStateService ?? throw new ArgumentNullException(nameof(validatorStateService));
4246
_certificateVerificationEnqueuer = certificateVerificationEnqueuer ?? throw new ArgumentNullException(nameof(certificateVerificationEnqueuer));
47+
_telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
4348
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
4449

4550
_certificateRevalidationThresholdTime = certificateRevalidationThreshold ?? DefaultCertificateRevalidationThresholdTime;
@@ -172,18 +177,18 @@ private async Task<ValidatorStatus> StartInternalAsync(IValidationRequest reques
172177

173178
if (certificates.Any())
174179
{
180+
var stopwatch = Stopwatch.StartNew();
181+
175182
await StartCertificateValidationsAsync(request, certificates);
176183

177-
return await _validatorStateService.TryAddValidatorStatusAsync(request, status, ValidationStatus.Incomplete);
184+
var result = await _validatorStateService.TryAddValidatorStatusAsync(request, status, ValidationStatus.Incomplete);
185+
186+
_telemetryService.TrackDurationToStartPackageCertificatesValidator(stopwatch.Elapsed);
187+
188+
return result;
178189
}
179190
else
180191
{
181-
_logger.LogInformation(
182-
"All certificates for package {PackageId} {PackageVersion} have already been validated, no additional validations necessary. " +
183-
"Promoting signature to status {SignatureStatus}",
184-
request.PackageId,
185-
request.PackageVersion);
186-
187192
PromoteSignature(request, signature);
188193

189194
return await _validatorStateService.TryAddValidatorStatusAsync(request, status, ValidationStatus.Succeeded);

‎src/NuGet.Services.Validation.Orchestrator/PackageSigning/PackageSignatureVerificationEnqueuer.cs‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
using System;
55
using System.Threading.Tasks;
6+
using Microsoft.Extensions.Options;
67
using NuGet.Jobs.Validation.PackageSigning.Messages;
78
using NuGet.Services.ServiceBus;
89

@@ -14,14 +15,17 @@ namespace NuGet.Services.Validation.PackageSigning
1415
public class PackageSignatureVerificationEnqueuer : IPackageSignatureVerificationEnqueuer
1516
{
1617
private readonly ITopicClient _topicClient;
17-
private readonly IBrokeredMessageSerializer<SignatureValidationMessage> _signatureValidationSerializer;
18+
private readonly IBrokeredMessageSerializer<SignatureValidationMessage> _serializer;
19+
private readonly IOptionsSnapshot<PackageSigningConfiguration> _configuration;
1820

1921
public PackageSignatureVerificationEnqueuer(
2022
ITopicClient topicClient,
21-
IBrokeredMessageSerializer<SignatureValidationMessage> signatureValidationSerializer)
23+
IBrokeredMessageSerializer<SignatureValidationMessage> serializer,
24+
IOptionsSnapshot<PackageSigningConfiguration> configuration)
2225
{
2326
_topicClient = topicClient ?? throw new ArgumentNullException(nameof(topicClient));
24-
_signatureValidationSerializer = signatureValidationSerializer ?? throw new ArgumentNullException(nameof(signatureValidationSerializer));
27+
_serializer = serializer ?? throw new ArgumentNullException(nameof(serializer));
28+
_configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
2529
}
2630

2731
/// <summary>
@@ -34,8 +38,15 @@ public PackageSignatureVerificationEnqueuer(
3438
/// <returns>A task that will complete when the verification process has been queued.</returns>
3539
public Task EnqueueVerificationAsync(IValidationRequest request)
3640
{
37-
var brokeredMessage = _signatureValidationSerializer.Serialize(
38-
new SignatureValidationMessage(request.PackageId, request.PackageVersion, new Uri(request.NupkgUrl), request.ValidationId));
41+
var message = new SignatureValidationMessage(
42+
request.PackageId,
43+
request.PackageVersion,
44+
new Uri(request.NupkgUrl),
45+
request.ValidationId);
46+
var brokeredMessage = _serializer.Serialize(message);
47+
48+
var visibleAt = DateTimeOffset.UtcNow + (_configuration.Value.MessageDelay ?? TimeSpan.Zero);
49+
brokeredMessage.ScheduledEnqueueTimeUtc = visibleAt;
3950

4051
return _topicClient.SendAsync(brokeredMessage);
4152
}

‎src/NuGet.Services.Validation.Orchestrator/PackageSigning/PackageSigningConfiguration.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

4+
using System;
45
using NuGet.Jobs.Configuration;
56

67
namespace NuGet.Services.Validation.PackageSigning
@@ -14,5 +15,10 @@ public class PackageSigningConfiguration
1415
/// The Service Bus configuration used to enqueue package signing validations.
1516
/// </summary>
1617
public ServiceBusConfiguration ServiceBus { get; set; }
18+
19+
/// <summary>
20+
/// The visibility delay to apply to Service Bus messages requesting a new validation.
21+
/// </summary>
22+
public TimeSpan? MessageDelay { get; set; }
1723
}
1824
}

‎src/NuGet.Services.Validation.Orchestrator/PackageSigning/PackageSigningValidator.cs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,26 +2,31 @@
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

44
using System;
5+
using System.Diagnostics;
56
using System.Threading.Tasks;
67
using Microsoft.Extensions.Logging;
78
using NuGet.Jobs.Validation.PackageSigning.Storage;
89
using NuGet.Services.Validation.Orchestrator;
10+
using NuGet.Services.Validation.Orchestrator.Telemetry;
911

1012
namespace NuGet.Services.Validation.PackageSigning
1113
{
1214
public class PackageSigningValidator : BaseValidator, IValidator
1315
{
1416
private readonly IValidatorStateService _validatorStateService;
1517
private readonly IPackageSignatureVerificationEnqueuer _signatureVerificationEnqueuer;
18+
private readonly ITelemetryService _telemetryService;
1619
private readonly ILogger<PackageSigningValidator> _logger;
1720

1821
public PackageSigningValidator(
1922
IValidatorStateService validatorStateService,
2023
IPackageSignatureVerificationEnqueuer signatureVerificationEnqueuer,
24+
ITelemetryService telemetryService,
2125
ILogger<PackageSigningValidator> logger)
2226
{
2327
_validatorStateService = validatorStateService ?? throw new ArgumentNullException(nameof(validatorStateService));
2428
_signatureVerificationEnqueuer = signatureVerificationEnqueuer ?? throw new ArgumentNullException(nameof(signatureVerificationEnqueuer));
29+
_telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
2530
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
2631
}
2732

@@ -57,9 +62,15 @@ private async Task<ValidatorStatus> StartInternalAsync(IValidationRequest reques
5762

5863
// Kick off the verification process. Note that the jobs will not verify the package until the
5964
// state of this validator has been persisted to the database.
65+
var stopwatch = Stopwatch.StartNew();
66+
6067
await _signatureVerificationEnqueuer.EnqueueVerificationAsync(request);
6168

62-
return await _validatorStateService.TryAddValidatorStatusAsync(request, validatorStatus, ValidationStatus.Incomplete);
69+
var result = await _validatorStateService.TryAddValidatorStatusAsync(request, validatorStatus, ValidationStatus.Incomplete);
70+
71+
_telemetryService.TrackDurationToStartPackageSigningValidator(stopwatch.Elapsed);
72+
73+
return result;
6374
}
6475
}
6576
}

‎src/NuGet.Services.Validation.Orchestrator/Telemetry/ITelemetryService.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,19 @@ public interface ITelemetryService
6565
/// <param name="validatorType">The validator type (name).</param>
6666
void TrackValidatorStarted(string validatorType);
6767

68+
/// <summary>
69+
/// The duration to start the package signing validator. This includes both the enqueue and DB commit time.
70+
/// </summary>
71+
/// <param name="duration">The duration.</param>
72+
void TrackDurationToStartPackageSigningValidator(TimeSpan duration);
73+
74+
/// <summary>
75+
/// The duration to star the package certificates validator. This includes all enqueue times and the DB commit
76+
/// time.
77+
/// </summary>
78+
/// <param name="duration">The duration.</param>
79+
void TrackDurationToStartPackageCertificatesValidator(TimeSpan duration);
80+
6881
/// <summary>
6982
/// A counter metric emmitted when a validator reaches a terminal state and potentially persists validation
7083
/// issues. A count of zero is emitted if the validator does not produce any issues. This metric is not emitted

‎src/NuGet.Services.Validation.Orchestrator/Telemetry/TelemetryService.cs‎

Lines changed: 36 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -10,22 +10,28 @@ namespace NuGet.Services.Validation.Orchestrator.Telemetry
1010
{
1111
public class TelemetryService : ITelemetryService
1212
{
13-
private const string Prefix = "Orchestrator.";
14-
15-
private const string DurationToValidationSetCreationSeconds = Prefix + "DurationToValidationSetCreationSeconds";
16-
private const string PackageStatusChange = Prefix + "PackageStatusChange";
17-
private const string TotalValidationDurationSeconds = Prefix + "TotalValidationDurationSeconds";
18-
private const string SentValidationTakingTooLongMessage = Prefix + "SentValidationTakingTooLongMessage";
19-
private const string ValidationSetTimeout = Prefix + "TotalValidationDurationSeconds";
20-
private const string ValidationIssue = Prefix + "ValidationIssue";
21-
private const string ValidationIssueCount = Prefix + "ValidationIssueCount";
22-
private const string ValidatorTimeout = Prefix + "ValidatorTimeout";
23-
private const string ValidatorDurationSeconds = Prefix + "ValidatorDurationSeconds";
24-
private const string ValidatorStarted = Prefix + "ValidatorStarted";
25-
private const string ClientValidationIssue = Prefix + "ClientValidationIssue";
26-
private const string MissingPackageForValidationMessage = Prefix + "MissingPackageForValidationMessage";
27-
private const string MissingNupkgForAvailablePackage = Prefix + "MissingNupkgForAvailablePackage";
28-
private const string DurationToHashPackageSeconds = Prefix + "DurationToHashPackageSeconds";
13+
private const string OrchestratorPrefix = "Orchestrator.";
14+
private const string PackageSigningPrefix = "PackageSigning.";
15+
private const string PackageCertificatesPrefix = "PackageCertificates.";
16+
17+
private const string DurationToValidationSetCreationSeconds = OrchestratorPrefix + "DurationToValidationSetCreationSeconds";
18+
private const string PackageStatusChange = OrchestratorPrefix + "PackageStatusChange";
19+
private const string TotalValidationDurationSeconds = OrchestratorPrefix + "TotalValidationDurationSeconds";
20+
private const string SentValidationTakingTooLongMessage = OrchestratorPrefix + "SentValidationTakingTooLongMessage";
21+
private const string ValidationSetTimeout = OrchestratorPrefix + "TotalValidationDurationSeconds";
22+
private const string ValidationIssue = OrchestratorPrefix + "ValidationIssue";
23+
private const string ValidationIssueCount = OrchestratorPrefix + "ValidationIssueCount";
24+
private const string ValidatorTimeout = OrchestratorPrefix + "ValidatorTimeout";
25+
private const string ValidatorDurationSeconds = OrchestratorPrefix + "ValidatorDurationSeconds";
26+
private const string ValidatorStarted = OrchestratorPrefix + "ValidatorStarted";
27+
private const string ClientValidationIssue = OrchestratorPrefix + "ClientValidationIssue";
28+
private const string MissingPackageForValidationMessage = OrchestratorPrefix + "MissingPackageForValidationMessage";
29+
private const string MissingNupkgForAvailablePackage = OrchestratorPrefix + "MissingNupkgForAvailablePackage";
30+
private const string DurationToHashPackageSeconds = OrchestratorPrefix + "DurationToHashPackageSeconds";
31+
32+
private const string DurationToStartPackageSigningValidatorSeconds = PackageSigningPrefix + "DurationToStartSeconds";
33+
34+
private const string DurationToStartPackageCertificatesValidatorSeconds = PackageCertificatesPrefix + "DurationToStartSeconds";
2935

3036
private const string FromStatus = "FromStatus";
3137
private const string ToStatus = "ToStatus";
@@ -205,5 +211,19 @@ public void TrackMissingNupkgForAvailablePackage(string packageId, string normal
205211
{ NormalizedVersion, normalizedVersion },
206212
{ ValidationTrackingId, validationTrackingId },
207213
});
214+
215+
public void TrackDurationToStartPackageSigningValidator(TimeSpan duration)
216+
{
217+
_telemetryClient.TrackMetric(
218+
DurationToStartPackageSigningValidatorSeconds,
219+
duration.TotalSeconds);
220+
}
221+
222+
public void TrackDurationToStartPackageCertificatesValidator(TimeSpan duration)
223+
{
224+
_telemetryClient.TrackMetric(
225+
DurationToStartPackageCertificatesValidatorSeconds,
226+
duration.TotalSeconds);
227+
}
208228
}
209229
}

‎src/NuGet.Services.Validation.Orchestrator/ValidationSetProcessor.cs‎

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -94,13 +94,31 @@ private async Task<bool> ProcessIncompleteValidations(PackageValidationSet valid
9494
var validator = _validatorProvider.GetValidator(packageValidation.Type);
9595
var validationRequest = await CreateValidationRequest(packageValidation.PackageValidationSet, packageValidation, package, validationConfiguration);
9696
var validationResult = await validator.GetResultAsync(validationRequest);
97-
_logger.LogInformation("New status for validation {ValidationType} for {PackageId} {PackageVersion} is {ValidationStatus}, validation set {ValidationSetId}, {ValidationId}",
98-
packageValidation.Type,
99-
package.PackageRegistration.Id,
100-
package.NormalizedVersion,
101-
validationResult.Status,
102-
validationSet.ValidationTrackingId,
103-
packageValidation.Key);
97+
98+
if (validationResult.Status != ValidationStatus.Incomplete)
99+
{
100+
_logger.LogInformation(
101+
"New status for validation {ValidationType} for {PackageId} {PackageVersion} is " +
102+
"{ValidationStatus}, validation set {ValidationSetId}, {ValidationId}",
103+
packageValidation.Type,
104+
package.PackageRegistration.Id,
105+
package.NormalizedVersion,
106+
validationResult.Status,
107+
validationSet.ValidationTrackingId,
108+
packageValidation.Key);
109+
}
110+
else
111+
{
112+
_logger.LogDebug(
113+
"Validation {ValidationType} for {PackageId} {PackageVersion} is still " +
114+
"{ValidationStatus}, validation set {ValidationSetId}, {ValidationId}",
115+
packageValidation.Type,
116+
package.PackageRegistration.Id,
117+
package.NormalizedVersion,
118+
validationResult.Status,
119+
validationSet.ValidationTrackingId,
120+
packageValidation.Key);
121+
}
104122

105123
switch (validationResult.Status)
106124
{

0 commit comments

Comments
 (0)