Skip to content
This repository was archived by the owner on Jul 30, 2024. It is now read-only.

Commit 2139be8

Browse files
cristinamanumshishirx34
cristinamanum
authored and
shishirx34
committed
SymbolOrchestrator + Validators - DI (#510)
1 parent f55f2be commit 2139be8

32 files changed

Lines changed: 1612 additions & 69 deletions

‎src/NuGet.Services.Validation.Orchestrator/Configuration/OrchestrationRunnerConfiguration.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,11 @@ public class OrchestrationRunnerConfiguration
1919
public TimeSpan ShutdownWaitInterval { get; set; }
2020

2121
/// <summary>
22+
23+
/// Information used by the initialization to register the correct IMessageHandler to be used by the Orchestrator.
24+
/// /// </summary>
25+
public ValidatingType ValidatingType { get; set; }
26+
2227
/// Max number of concurrent calls to be handled by the service bus library
2328
/// </summary>
2429
public int MaxConcurrentCalls { get; set; }

‎src/NuGet.Services.Validation.Orchestrator/Job.cs‎

Lines changed: 194 additions & 10 deletions
Large diffs are not rendered by default.

‎src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,15 @@
5151
<Compile Include="DiskMailSender.cs" />
5252
<Compile Include="Configuration\EmailConfiguration.cs" />
5353
<Compile Include="Error.cs" />
54+
<Compile Include="SymbolMessageService.cs" />
55+
<Compile Include="Symbols\SymbolScanOnlyConfiguration.cs" />
56+
<Compile Include="Symbols\SymbolCriteriaEvaluator.cs" />
57+
<Compile Include="Symbols\SymbolScanValidator.cs" />
5458
<Compile Include="PackageSigning\Scan\ScanValidator.cs" />
5559
<Compile Include="Services\IEntityService.cs" />
5660
<Compile Include="IMessageService.cs" />
5761
<Compile Include="IPackageStatusProcessor.cs" />
62+
<Compile Include="Services\SymbolEntityService.cs" />
5863
<Compile Include="Symbols\ISymbolsMessageEnqueuer.cs" />
5964
<Compile Include="Symbols\SymbolsMessageEnqueuer.cs" />
6065
<Compile Include="Symbols\SymbolsIngesterConfiguration.cs" />
@@ -95,8 +100,11 @@
95100
<Compile Include="Telemetry\TelemetryService.cs" />
96101
<Compile Include="Configuration\ValidationConfiguration.cs" />
97102
<Compile Include="Configuration\ValidationConfigurationItem.cs" />
103+
<Compile Include="ValidatingEntitites\SymbolPackageValidatingEntity.cs" />
98104
<Compile Include="ValidatingEntitites\PackageValidatingEntity.cs" />
99105
<Compile Include="ValidationFailureBehavior.cs" />
106+
<Compile Include="SymbolValidationMessageHandler.cs" />
107+
<Compile Include="ValidationSymbolFileService.cs" />
100108
<Compile Include="ValidationPackageFileService.cs" />
101109
<Compile Include="ValidationSetProcessorResult.cs" />
102110
<Compile Include="Vcs\IPackageCriteria.cs" />
@@ -106,7 +114,7 @@
106114
<Compile Include="Vcs\VcsConfiguration.cs" />
107115
<Compile Include="Vcs\VcsValidator.cs" />
108116
<Compile Include="Configuration\ValidationDbConfiguration.cs" />
109-
<Compile Include="ValidationMessageHandler.cs" />
117+
<Compile Include="PackageValidationMessageHandler.cs" />
110118
<Compile Include="ValidationOutcomeProcessor.cs" />
111119
<Compile Include="ValidationRequest.cs" />
112120
<Compile Include="ValidationSetProcessor.cs" />
@@ -118,6 +126,9 @@
118126
<None Include="App.config">
119127
<SubType>Designer</SubType>
120128
</None>
129+
<None Include="symbolsettings.json">
130+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
131+
</None>
121132
<None Include="settings.json">
122133
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
123134
</None>

src/NuGet.Services.Validation.Orchestrator/ValidationMessageHandler.cs renamed to src/NuGet.Services.Validation.Orchestrator/PackageValidationMessageHandler.cs

File renamed without changes.

‎src/NuGet.Services.Validation.Orchestrator/Services/IEntityService.cs‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,6 @@ public interface IEntityService<T> where T : class, IEntity
2020
/// <returns></returns>
2121
IValidatingEntity<T> FindPackageByIdAndVersionStrict(string id, string version);
2222

23-
/// <summary>
24-
/// Retrieve an entity with the specified key.
25-
/// </summary>
26-
/// <param name="key">The entity key.</param>
27-
/// <returns></returns>
28-
IValidatingEntity<T> FindByKey(int key);
29-
3023
/// <summary>
3124
/// Update the status of the entity.
3225
/// </summary>

‎src/NuGet.Services.Validation.Orchestrator/Services/PackageEntityService.cs‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

4-
using System;
54
using System.Threading.Tasks;
65
using NuGetGallery;
76
using NuGetGallery.Packaging;
@@ -26,11 +25,6 @@ public IValidatingEntity<Package> FindPackageByIdAndVersionStrict(string id, str
2625
return p == null ? null : new PackageValidatingEntity(p);
2726
}
2827

29-
public IValidatingEntity<Package> FindByKey(int key)
30-
{
31-
throw new NotImplementedException();
32-
}
33-
3428
public async Task UpdateStatusAsync(Package entity, PackageStatus newStatus, bool commitChanges = true)
3529
{
3630
await _galleryEntityService.UpdatePackageStatusAsync(entity, newStatus, commitChanges);

‎src/NuGet.Services.Validation.Orchestrator/Services/SymbolEntityService.cs‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Linq;
6+
using System.Threading.Tasks;
7+
using NuGetGallery;
8+
9+
namespace NuGet.Services.Validation.Orchestrator
10+
{
11+
/// <summary>
12+
/// The EntityService for the <see cref="NuGetGallery.SymbolPackage"/>.
13+
/// </summary>
14+
public class SymbolEntityService : IEntityService<SymbolPackage>
15+
{
16+
ICoreSymbolPackageService _galleryEntityService;
17+
public SymbolEntityService(ICoreSymbolPackageService galleryEntityService)
18+
{
19+
_galleryEntityService = galleryEntityService ?? throw new ArgumentNullException(nameof(galleryEntityService));
20+
}
21+
22+
/// <summary>
23+
/// Only the package symbols that are in validating state will be sent to the symbols validation/ingestion.
24+
/// </summary>
25+
/// <param name="id">The id of the package.</param>
26+
/// <param name="version">The version of the package.</param>
27+
/// <returns></returns>
28+
public IValidatingEntity<SymbolPackage> FindPackageByIdAndVersionStrict(string id, string version)
29+
{
30+
var symbolPackage = _galleryEntityService
31+
.FindSymbolPackagesByIdAndVersion(id, version)
32+
.Where(s => s.StatusKey == PackageStatus.Validating)
33+
.FirstOrDefault();
34+
35+
return symbolPackage == null ? null : new SymbolPackageValidatingEntity(symbolPackage);
36+
}
37+
38+
public async Task UpdateStatusAsync(SymbolPackage entity, PackageStatus newStatus, bool commitChanges = true)
39+
{
40+
if(entity == null)
41+
{
42+
throw new ArgumentException(nameof(entity));
43+
}
44+
45+
if (newStatus == entity.StatusKey)
46+
{
47+
return;
48+
}
49+
50+
if (newStatus == PackageStatus.Available)
51+
{
52+
var previousAvailableSymbolPackage = _galleryEntityService
53+
.FindSymbolPackagesByIdAndVersion(entity.Package.PackageRegistration.Id, entity.Package.NormalizedVersion)
54+
.FirstOrDefault(s => s.StatusKey == PackageStatus.Available);
55+
56+
if (previousAvailableSymbolPackage != null)
57+
{
58+
await _galleryEntityService.UpdateStatusAsync(previousAvailableSymbolPackage, PackageStatus.Deleted, commitChanges: false);
59+
}
60+
}
61+
62+
await _galleryEntityService.UpdateStatusAsync(entity, newStatus, commitChanges);
63+
}
64+
65+
public async Task UpdateMetadataAsync(SymbolPackage entity, object metadata, bool commitChanges = true)
66+
{
67+
// No action for symbols
68+
// For each new symbol a new entry will be added to db and the file symbols will be overwriten
69+
await Task.CompletedTask;
70+
}
71+
}
72+
}

‎src/NuGet.Services.Validation.Orchestrator/SymbolMessageService.cs‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using Microsoft.Extensions.Logging;
6+
using Microsoft.Extensions.Options;
7+
using NuGetGallery;
8+
using NuGetGallery.Services;
9+
10+
namespace NuGet.Services.Validation.Orchestrator
11+
{
12+
//ToDo: https://github.com/NuGet/NuGetGallery/issues/6255
13+
public class SymbolPackageMessageService : IMessageService<SymbolPackage>
14+
{
15+
private readonly ICoreMessageService _coreMessageService;
16+
private readonly EmailConfiguration _emailConfiguration;
17+
private readonly ILogger<SymbolPackageMessageService> _logger;
18+
19+
public SymbolPackageMessageService(
20+
ICoreMessageService coreMessageService,
21+
IOptionsSnapshot<EmailConfiguration> emailConfigurationAccessor,
22+
ILogger<SymbolPackageMessageService> logger)
23+
{
24+
_coreMessageService = coreMessageService ?? throw new ArgumentNullException(nameof(coreMessageService));
25+
if (emailConfigurationAccessor == null)
26+
{
27+
throw new ArgumentNullException(nameof(emailConfigurationAccessor));
28+
}
29+
_emailConfiguration = emailConfigurationAccessor.Value ?? throw new ArgumentException("Value cannot be null", nameof(emailConfigurationAccessor));
30+
if (string.IsNullOrWhiteSpace(_emailConfiguration.PackageUrlTemplate))
31+
{
32+
throw new ArgumentException($"{nameof(emailConfigurationAccessor.Value)}.{nameof(_emailConfiguration.PackageUrlTemplate)} cannot be empty", nameof(emailConfigurationAccessor));
33+
}
34+
if (string.IsNullOrWhiteSpace(_emailConfiguration.PackageSupportTemplate))
35+
{
36+
throw new ArgumentException($"{nameof(emailConfigurationAccessor.Value)}.{nameof(_emailConfiguration.PackageSupportTemplate)} cannot be empty", nameof(emailConfigurationAccessor));
37+
}
38+
if (string.IsNullOrWhiteSpace(_emailConfiguration.EmailSettingsUrl))
39+
{
40+
throw new ArgumentException($"{nameof(emailConfigurationAccessor.Value)}.{nameof(_emailConfiguration.EmailSettingsUrl)} cannot be empty", nameof(emailConfigurationAccessor));
41+
}
42+
if (!Uri.TryCreate(_emailConfiguration.EmailSettingsUrl, UriKind.Absolute, out Uri result))
43+
{
44+
throw new ArgumentException($"{nameof(emailConfigurationAccessor.Value)}.{nameof(_emailConfiguration.EmailSettingsUrl)} must be an absolute Url", nameof(emailConfigurationAccessor));
45+
}
46+
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
47+
}
48+
49+
public void SendPublishedMessage(SymbolPackage package)
50+
{
51+
//No action until the Symbol messages are defined
52+
}
53+
54+
public void SendValidationFailedMessage(SymbolPackage package, PackageValidationSet validationSet)
55+
{
56+
//No action until the Symbol messages are defined
57+
}
58+
59+
public void SendValidationTakingTooLongMessage(SymbolPackage package)
60+
{
61+
//No action until the Symbol messages are defined
62+
}
63+
}
64+
}

‎src/NuGet.Services.Validation.Orchestrator/SymbolValidationMessageHandler.cs‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Threading.Tasks;
6+
using Microsoft.Extensions.Logging;
7+
using Microsoft.Extensions.Options;
8+
using NuGet.Services.ServiceBus;
9+
using NuGet.Services.Validation.Orchestrator.Telemetry;
10+
using NuGetGallery;
11+
12+
namespace NuGet.Services.Validation.Orchestrator
13+
{
14+
/// <summary>
15+
/// The message handler for Symbols.
16+
/// </summary>
17+
public class SymbolValidationMessageHandler : IMessageHandler<PackageValidationMessageData>
18+
{
19+
private readonly ValidationConfiguration _configs;
20+
private readonly IEntityService<SymbolPackage> _gallerySymbolService;
21+
private readonly IValidationSetProvider<SymbolPackage> _validationSetProvider;
22+
private readonly IValidationSetProcessor _validationSetProcessor;
23+
private readonly IValidationOutcomeProcessor<SymbolPackage> _validationOutcomeProcessor;
24+
private readonly ITelemetryService _telemetryService;
25+
private readonly ILogger<SymbolValidationMessageHandler> _logger;
26+
27+
public SymbolValidationMessageHandler(
28+
IOptionsSnapshot<ValidationConfiguration> validationConfigsAccessor,
29+
IEntityService<SymbolPackage> gallerySymbolService,
30+
IValidationSetProvider<SymbolPackage> validationSetProvider,
31+
IValidationSetProcessor validationSetProcessor,
32+
IValidationOutcomeProcessor<SymbolPackage> validationOutcomeProcessor,
33+
ITelemetryService telemetryService,
34+
ILogger<SymbolValidationMessageHandler> logger)
35+
{
36+
if (validationConfigsAccessor == null)
37+
{
38+
throw new ArgumentNullException(nameof(validationConfigsAccessor));
39+
}
40+
41+
if (validationConfigsAccessor.Value == null)
42+
{
43+
throw new ArgumentException(
44+
$"The {nameof(IOptionsSnapshot<ValidationConfiguration>)}.{nameof(IOptionsSnapshot<ValidationConfiguration>.Value)} property cannot be null",
45+
nameof(validationConfigsAccessor));
46+
}
47+
48+
if (validationConfigsAccessor.Value.MissingPackageRetryCount < 1)
49+
{
50+
throw new ArgumentOutOfRangeException(
51+
nameof(validationConfigsAccessor),
52+
$"{nameof(ValidationConfiguration)}.{nameof(ValidationConfiguration.MissingPackageRetryCount)} must be at least 1");
53+
}
54+
55+
_configs = validationConfigsAccessor.Value;
56+
_gallerySymbolService = gallerySymbolService ?? throw new ArgumentNullException(nameof(gallerySymbolService));
57+
_validationSetProvider = validationSetProvider ?? throw new ArgumentNullException(nameof(validationSetProvider));
58+
_validationSetProcessor = validationSetProcessor ?? throw new ArgumentNullException(nameof(validationSetProcessor));
59+
_validationOutcomeProcessor = validationOutcomeProcessor ?? throw new ArgumentNullException(nameof(validationOutcomeProcessor));
60+
_telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
61+
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
62+
}
63+
64+
public async Task<bool> HandleAsync(PackageValidationMessageData message)
65+
{
66+
if (message == null)
67+
{
68+
throw new ArgumentNullException(nameof(message));
69+
}
70+
71+
using (_logger.BeginScope("Handling symbol message for {PackageId} {PackageVersion} validation set {ValidationSetId}",
72+
message.PackageId,
73+
message.PackageNormalizedVersion,
74+
message.ValidationTrackingId))
75+
{
76+
var symbolPackageEntity = _gallerySymbolService.FindPackageByIdAndVersionStrict(message.PackageId, message.PackageNormalizedVersion);
77+
78+
if (symbolPackageEntity == null)
79+
{
80+
// no package in DB yet. Might have received message a bit early, need to retry later
81+
if (message.DeliveryCount - 1 >= _configs.MissingPackageRetryCount)
82+
{
83+
_logger.LogWarning("Could not find symbols for package {PackageId} {PackageNormalizedVersion} in DB after {DeliveryCount} tries, dropping message",
84+
message.PackageId,
85+
message.PackageNormalizedVersion,
86+
message.DeliveryCount);
87+
88+
_telemetryService.TrackMissingPackageForValidationMessage(
89+
message.PackageId,
90+
message.PackageNormalizedVersion,
91+
message.ValidationTrackingId.ToString());
92+
93+
return true;
94+
}
95+
else
96+
{
97+
_logger.LogInformation("Could not find symbols for package {PackageId} {PackageNormalizedVersion} in DB, retrying",
98+
message.PackageId,
99+
message.PackageNormalizedVersion);
100+
101+
return false;
102+
}
103+
}
104+
105+
var validationSet = await _validationSetProvider.TryGetOrCreateValidationSetAsync(message, symbolPackageEntity);
106+
107+
if (validationSet == null)
108+
{
109+
_logger.LogInformation("The validation request for {PackageId} {PackageNormalizedVersion} validation set {ValidationSetId} is a duplicate. Discarding.",
110+
message.PackageId,
111+
message.PackageNormalizedVersion,
112+
message.ValidationTrackingId);
113+
return true;
114+
}
115+
116+
var processorStats = await _validationSetProcessor.ProcessValidationsAsync(validationSet);
117+
await _validationOutcomeProcessor.ProcessValidationOutcomeAsync(validationSet, symbolPackageEntity, processorStats);
118+
}
119+
120+
return true;
121+
}
122+
}
123+
}

‎src/NuGet.Services.Validation.Orchestrator/Symbols/SymbolCriteriaEvaluator.cs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using NuGetGallery;
6+
using NuGet.Services.Validation.Vcs;
7+
8+
namespace NuGet.Services.Validation
9+
{
10+
public class SymbolCriteriaEvaluator : ICriteriaEvaluator<SymbolPackage>
11+
{
12+
private readonly ICriteriaEvaluator<Package> _packageCriteriaEvaluator;
13+
14+
public SymbolCriteriaEvaluator(ICriteriaEvaluator<Package> packageCriteriaEvaluator)
15+
{
16+
_packageCriteriaEvaluator = packageCriteriaEvaluator ?? throw new ArgumentNullException(nameof(packageCriteriaEvaluator));
17+
}
18+
19+
public bool IsMatch(ICriteria criteria, SymbolPackage entity)
20+
{
21+
return _packageCriteriaEvaluator.IsMatch(criteria, entity.Package);
22+
}
23+
}
24+
}

0 commit comments

Comments
 (0)