Skip to content
This repository was archived by the owner on Jul 30, 2024. It is now read-only.

Commit aac6e6e

Browse files
Support symbol revalidation (#589)
Support for symbol revalidation and add use the validation set to download the symbol packages.
1 parent 7f052ef commit aac6e6e

21 files changed

Lines changed: 170 additions & 135 deletions

src/NuGet.Services.Validation.Orchestrator/Services/IEntityService.cs

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,15 @@ public interface IEntityService<T> where T : class, IEntity
1717
/// </summary>
1818
/// <param name="id">The id .</param>
1919
/// <param name="version">The version.</param>
20-
/// <returns></returns>
20+
/// <returns>The entity.</returns>
2121
IValidatingEntity<T> FindPackageByIdAndVersionStrict(string id, string version);
2222

23+
/// <summary>
24+
/// Find the entity based on the key.
25+
/// </summary>
26+
/// <returns>The entity.</returns>
27+
IValidatingEntity<T> FindPackageByKey(int key);
28+
2329
/// <summary>
2430
/// Update the status of the entity.
2531
/// </summary>
@@ -35,7 +41,7 @@ public interface IEntityService<T> where T : class, IEntity
3541
/// <param name="entity">The entity.</param>
3642
/// <param name="metadata">The metadata.</param>
3743
/// <param name="commitChanges">True if the changes will be commited to the database.</param>
38-
/// <returns></returns>
44+
/// <returns>A <see cref="Task"/> that can be used to await for the operation completion.</returns>
3945
Task UpdateMetadataAsync(T entity, object metadata, bool commitChanges = true);
4046
}
4147
}

src/NuGet.Services.Validation.Orchestrator/Services/PackageEntityService.cs

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

4+
using System;
5+
using System.Linq;
6+
using System.Data.Entity;
47
using System.Threading.Tasks;
58
using NuGetGallery;
69
using NuGetGallery.Packaging;
@@ -13,10 +16,12 @@ namespace NuGet.Services.Validation.Orchestrator
1316
public class PackageEntityService : IEntityService<Package>
1417
{
1518
private ICorePackageService _galleryEntityService;
19+
private IEntityRepository<Package> _packageRepository;
1620

17-
public PackageEntityService(ICorePackageService galleryEntityService)
21+
public PackageEntityService(ICorePackageService galleryEntityService, IEntityRepository<Package> packageRepository)
1822
{
19-
_galleryEntityService = galleryEntityService;
23+
_galleryEntityService = galleryEntityService ?? throw new ArgumentNullException(nameof(galleryEntityService));
24+
_packageRepository = packageRepository ?? throw new ArgumentNullException(nameof(packageRepository));
2025
}
2126

2227
public IValidatingEntity<Package> FindPackageByIdAndVersionStrict(string id, string version)
@@ -25,6 +30,18 @@ public IValidatingEntity<Package> FindPackageByIdAndVersionStrict(string id, str
2530
return p == null ? null : new PackageValidatingEntity(p);
2631
}
2732

33+
public IValidatingEntity<Package> FindPackageByKey(int key)
34+
{
35+
var package = _packageRepository
36+
.GetAll()
37+
.Include(p => p.LicenseReports)
38+
.Include(p => p.PackageRegistration)
39+
.Include(p => p.User)
40+
.Include(p => p.SymbolPackages)
41+
.SingleOrDefault(p => p.Key == key);
42+
return package == null ? null : new PackageValidatingEntity(package);
43+
}
44+
2845
public async Task UpdateStatusAsync(Package entity, PackageStatus newStatus, bool commitChanges = true)
2946
{
3047
await _galleryEntityService.UpdatePackageStatusAsync(entity, newStatus, commitChanges);

src/NuGet.Services.Validation.Orchestrator/Services/SymbolEntityService.cs

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,13 @@ namespace NuGet.Services.Validation.Orchestrator
1313
/// </summary>
1414
public class SymbolEntityService : IEntityService<SymbolPackage>
1515
{
16-
ICoreSymbolPackageService _galleryEntityService;
17-
public SymbolEntityService(ICoreSymbolPackageService galleryEntityService)
16+
private ICoreSymbolPackageService _galleryEntityService;
17+
private IEntityRepository<SymbolPackage> _symbolsPackageRepository;
18+
19+
public SymbolEntityService(ICoreSymbolPackageService galleryEntityService, IEntityRepository<SymbolPackage> symbolsPackageRepository)
1820
{
1921
_galleryEntityService = galleryEntityService ?? throw new ArgumentNullException(nameof(galleryEntityService));
22+
_symbolsPackageRepository = symbolsPackageRepository ?? throw new ArgumentNullException(nameof(symbolsPackageRepository));
2023
}
2124

2225
/// <summary>
@@ -35,6 +38,14 @@ public IValidatingEntity<SymbolPackage> FindPackageByIdAndVersionStrict(string i
3538
return symbolPackage == null ? null : new SymbolPackageValidatingEntity(symbolPackage);
3639
}
3740

41+
public IValidatingEntity<SymbolPackage> FindPackageByKey(int key)
42+
{
43+
var symbolPackage = _symbolsPackageRepository
44+
.GetAll()
45+
.SingleOrDefault(p => p.Key == key);
46+
return symbolPackage == null ? null : new SymbolPackageValidatingEntity(symbolPackage);
47+
}
48+
3849
public async Task UpdateStatusAsync(SymbolPackage entity, PackageStatus newStatus, bool commitChanges = true)
3950
{
4051
if(entity == null)

src/NuGet.Services.Validation.Orchestrator/SymbolValidationMessageHandler.cs

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,11 @@ public async Task<bool> HandleAsync(PackageValidationMessageData message)
7373
message.PackageNormalizedVersion,
7474
message.ValidationTrackingId))
7575
{
76-
var symbolPackageEntity = _gallerySymbolService.FindPackageByIdAndVersionStrict(message.PackageId, message.PackageNormalizedVersion);
76+
// When a message is sent from the Gallery with validation of a new entity, the EntityKey will be null because the message is sent to the service bus before the entity is persisted in the DB
77+
// However when a revalidation happens or when the message is re-sent by the orchestrator the message will contain the key. In this case the key is used to find the entity to validate.
78+
var symbolPackageEntity = message.EntityKey.HasValue
79+
? _gallerySymbolService.FindPackageByKey(message.EntityKey.Value)
80+
: _gallerySymbolService.FindPackageByIdAndVersionStrict(message.PackageId, message.PackageNormalizedVersion);
7781

7882
if (symbolPackageEntity == null)
7983
{
@@ -94,9 +98,10 @@ public async Task<bool> HandleAsync(PackageValidationMessageData message)
9498
}
9599
else
96100
{
97-
_logger.LogInformation("Could not find symbols for package {PackageId} {PackageNormalizedVersion} in DB, retrying",
101+
_logger.LogInformation("Could not find symbols for package {PackageId} {PackageNormalizedVersion} {Key} in DB, retrying",
98102
message.PackageId,
99-
message.PackageNormalizedVersion);
103+
message.PackageNormalizedVersion,
104+
message.EntityKey.HasValue);
100105

101106
return false;
102107
}
@@ -114,6 +119,7 @@ public async Task<bool> HandleAsync(PackageValidationMessageData message)
114119
}
115120

116121
var processorStats = await _validationSetProcessor.ProcessValidationsAsync(validationSet);
122+
// As part of the processing the validation outcome the orchestrator will send itself a message if validation are still being processed.
117123
await _validationOutcomeProcessor.ProcessValidationOutcomeAsync(validationSet, symbolPackageEntity, processorStats);
118124
}
119125

src/NuGet.Services.Validation.Orchestrator/Symbols/SymbolScanValidator.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ private bool ShouldSkipScan(IValidationRequest request)
113113
{
114114
var symbolPackage = _symbolPackageService
115115
.FindSymbolPackagesByIdAndVersion(request.PackageId,request.PackageVersion)
116-
.FirstOrDefault(sp => sp.StatusKey == PackageStatus.Validating);
116+
.FirstOrDefault(sp => sp.Key == request.PackageKey);
117117

118118
if (symbolPackage == null)
119119
{

src/NuGet.Services.Validation.Orchestrator/ValidationOutcomeProcessor.cs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,12 @@ private async Task ScheduleCheckIfNotTimedOut(PackageValidationSet validationSet
264264
// Schedule another check if we haven't reached the validation set timeout yet.
265265
if (validationSetDuration <= _validationConfiguration.TimeoutValidationSetAfter)
266266
{
267-
var messageData = new PackageValidationMessageData(validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId, validationSet.ValidatingType);
267+
var messageData = new PackageValidationMessageData(
268+
validationSet.PackageId,
269+
validationSet.PackageNormalizedVersion,
270+
validationSet.ValidationTrackingId,
271+
validationSet.ValidatingType,
272+
entityKey: validationSet.PackageKey);
268273
var postponeUntil = DateTimeOffset.UtcNow + _validationConfiguration.ValidationMessageRecheckPeriod;
269274

270275
await _validationEnqueuer.StartValidationAsync(messageData, postponeUntil);

src/Validation.Symbols/ISymbolsFileService.cs

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,9 @@ public interface ISymbolsFileService
2121
/// <summary>
2222
/// Downloads the snupkg file.
2323
/// </summary>
24-
/// <param name="packageId">The package id.</param>
25-
/// <param name="packageNormalizedVersion">The package normalized version.</param>
24+
/// <param name="snupkgUri">The uri of the snupkg.</param>
2625
/// <param name="cancellationToken">Cancellation token.</param>
27-
/// <returns>The nupkg stream.</returns>
28-
Task<Stream> DownloadSnupkgFileAsync(string packageId, string packageNormalizedVersion, CancellationToken cancellationToken);
26+
/// <returns>The snupkg stream.</returns>
27+
Task<Stream> DownloadSnupkgFileAsync(string snupkgUri, CancellationToken cancellationToken);
2928
}
3029
}
Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,21 @@
11
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

4-
using System.Collections.Generic;
5-
using System.IO;
64
using System.Threading;
75
using System.Threading.Tasks;
6+
using NuGet.Jobs.Validation.Symbols.Core;
87
using NuGet.Services.Validation;
98

109
namespace Validation.Symbols
1110
{
1211
public interface ISymbolsValidatorService
1312
{
1413
/// <summary>
15-
/// Validates the symbols against the PE files.
14+
/// Validates the symbol package.
1615
/// </summary>
17-
/// <param name="packageId">The package Id.</param>
18-
/// <param name="packageNormalizedVersion">The package normalized version.</param>
19-
/// <param name="token">A cancellation token to be used for cancellation of the async execution.</param>
20-
/// <returns></returns>
21-
Task<IValidationResult> ValidateSymbolsAsync(string packageId, string packageNormalizedVersion, CancellationToken token);
16+
/// <param name="message">The <see cref="SymbolsValidatorMessage"/> regarding to the symbols pacakge to be validated..</param>
17+
/// <param name="token">Cancellation token.</param>
18+
/// <returns>The validation result.</returns>
19+
Task<IValidationResult> ValidateSymbolsAsync(SymbolsValidatorMessage message, CancellationToken token);
2220
}
2321
}

src/Validation.Symbols/Job.cs

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,7 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
4040
configurationAccessor.Value.ValidationPackageConnectionString,
4141
readAccessGeoRedundant: false), c.GetRequiredService<IDiagnosticsService>());
4242

43-
var symbolValidationStorageService = new CloudBlobCoreFileStorageService(new CloudBlobClientWrapper(
44-
configurationAccessor.Value.ValidationSymbolsConnectionString,
45-
readAccessGeoRedundant: false), c.GetRequiredService<IDiagnosticsService>());
46-
47-
return new SymbolsFileService(packageStorageService, packageValidationStorageService, symbolValidationStorageService);
43+
return new SymbolsFileService(packageStorageService, packageValidationStorageService, c.GetRequiredService<IFileDownloader>());
4844
});
4945
services.AddSingleton(new TelemetryClient());
5046
}

src/Validation.Symbols/Settings/dev.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
"PackageConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$",
1414
"ValidationSymbolsConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$"
1515
},
16+
"PackageDownloadTimeout": "00:10:00",
1617
"KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
1718
"KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
1819
"KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",

0 commit comments

Comments
 (0)