Skip to content
This repository was archived by the owner on Jul 30, 2024. It is now read-only.

Commit 6e1a12c

Browse files
Add SymbolsIngester in Orchestrator (#501)
Add the symbols ingester as an Orchestrator validator.
1 parent 0c7b7c8 commit 6e1a12c

16 files changed

Lines changed: 934 additions & 8 deletions

NuGet.Jobs.sln

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Symbols", "src\V
149149
EndProject
150150
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Symbols.Tests", "tests\Validation.Symbols.Tests\Validation.Symbols.Tests.csproj", "{640D29AB-4D1B-4FC7-AE67-AD12EE5AC503}"
151151
EndProject
152+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Symbols.Core.Tests", "tests\Validation.Symbols.Core.Tests\Validation.Symbols.Core.Tests.csproj", "{9ED642DF-4623-4EB2-8B72-52C6489BF9D6}"
153+
EndProject
152154
Global
153155
GlobalSection(SolutionConfigurationPlatforms) = preSolution
154156
Debug|Any CPU = Debug|Any CPU
@@ -393,6 +395,10 @@ Global
393395
{640D29AB-4D1B-4FC7-AE67-AD12EE5AC503}.Debug|Any CPU.Build.0 = Debug|Any CPU
394396
{640D29AB-4D1B-4FC7-AE67-AD12EE5AC503}.Release|Any CPU.ActiveCfg = Release|Any CPU
395397
{640D29AB-4D1B-4FC7-AE67-AD12EE5AC503}.Release|Any CPU.Build.0 = Release|Any CPU
398+
{9ED642DF-4623-4EB2-8B72-52C6489BF9D6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
399+
{9ED642DF-4623-4EB2-8B72-52C6489BF9D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
400+
{9ED642DF-4623-4EB2-8B72-52C6489BF9D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
401+
{9ED642DF-4623-4EB2-8B72-52C6489BF9D6}.Release|Any CPU.Build.0 = Release|Any CPU
396402
EndGlobalSection
397403
GlobalSection(SolutionProperties) = preSolution
398404
HideSolutionNode = FALSE
@@ -457,6 +463,7 @@ Global
457463
{21C0A0EE-8696-4013-950F-D6495D0C6E40} = {6A776396-02B1-475D-A104-26940ADB04AB}
458464
{2DD07A73-8C88-4429-BB24-C2813586EF92} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
459465
{640D29AB-4D1B-4FC7-AE67-AD12EE5AC503} = {6A776396-02B1-475D-A104-26940ADB04AB}
466+
{9ED642DF-4623-4EB2-8B72-52C6489BF9D6} = {6A776396-02B1-475D-A104-26940ADB04AB}
460467
EndGlobalSection
461468
GlobalSection(ExtensibilityGlobals) = postSolution
462469
SolutionGuid = {284A7AC3-FB43-4F1F-9C9C-2AF0E1F46C2B}

src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@
5757
<Compile Include="IPackageStatusProcessor.cs" />
5858
<Compile Include="Symbols\ISymbolsMessageEnqueuer.cs" />
5959
<Compile Include="Symbols\SymbolsMessageEnqueuer.cs" />
60+
<Compile Include="Symbols\SymbolsIngesterConfiguration.cs" />
61+
<Compile Include="Symbols\SymbolsIngester.cs" />
6062
<Compile Include="Symbols\SymbolsValidator.cs" />
6163
<Compile Include="Symbols\SymbolsValidationConfiguration.cs" />
6264
<Compile Include="UsernameHelper.cs" />
Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Threading.Tasks;
6+
using Microsoft.Extensions.Logging;
7+
using NuGet.Jobs.Validation;
8+
using NuGet.Jobs.Validation.Symbols.Core;
9+
using NuGet.Services.Validation.Orchestrator;
10+
using NuGet.Services.Validation.Orchestrator.Telemetry;
11+
12+
namespace NuGet.Services.Validation.Symbols
13+
{
14+
[ValidatorName(ValidatorName.SymbolsIngester)]
15+
public class SymbolsIngester : BaseValidator, IValidator
16+
{
17+
private readonly ISymbolsValidationEntitiesService _symbolsValidationEntitiesService;
18+
private readonly ISymbolsMessageEnqueuer _symbolMessageEnqueuer;
19+
private readonly ITelemetryService _telemetryService;
20+
private readonly ILogger<SymbolsIngester> _logger;
21+
22+
public SymbolsIngester(
23+
ISymbolsValidationEntitiesService symbolsValidationEntitiesService,
24+
ISymbolsMessageEnqueuer symbolMessageEnqueuer,
25+
ITelemetryService telemetryService,
26+
ILogger<SymbolsIngester> logger)
27+
{
28+
_symbolsValidationEntitiesService = symbolsValidationEntitiesService ?? throw new ArgumentNullException(nameof(symbolsValidationEntitiesService));
29+
_symbolMessageEnqueuer = symbolMessageEnqueuer ?? throw new ArgumentNullException(nameof(symbolMessageEnqueuer));
30+
_telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
31+
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
32+
}
33+
34+
public async Task<IValidationResult> GetResultAsync(IValidationRequest request)
35+
{
36+
if (request == null)
37+
{
38+
throw new ArgumentNullException(nameof(request));
39+
}
40+
41+
var result = SymbolsValidationEntitiesService.ConvertToIValidationResult(await _symbolsValidationEntitiesService.GetSymbolsServerRequestAsync(request));
42+
_logger.LogInformation(
43+
"Symbols status {Status} for PackageId: {PackageId}, PackageNormalizedVersion {PackageNormalizedVersion}, SymbolsPackageKey {SymbolsPackageKey} ",
44+
result.Status,
45+
request.PackageId,
46+
request.PackageVersion,
47+
request.PackageKey);
48+
49+
return result;
50+
}
51+
52+
/// <summary>
53+
/// The pattern used for the StartAsync:
54+
/// 1. Check if an ingestion for the specific symbols package key was already started
55+
/// 2. Only if a ingestion was not started queue the message to be processed.
56+
/// 3. After the message is queued, update the SymbolServerRequests table.
57+
/// </summary>
58+
/// <param name="request">The request to be sent to the ingester job queue.</param>
59+
/// <returns>The operation status as <see cref="IValidationResult"/>.</returns>
60+
public async Task<IValidationResult> StartAsync(IValidationRequest request)
61+
{
62+
if (request == null)
63+
{
64+
throw new ArgumentNullException(nameof(request));
65+
}
66+
67+
var result = SymbolsValidationEntitiesService.ConvertToIValidationResult(await _symbolsValidationEntitiesService.GetSymbolsServerRequestAsync(request));
68+
69+
if (result.Status != ValidationStatus.NotStarted)
70+
{
71+
_logger.LogWarning(
72+
"Symbol ingestion for {PackageId} {PackageNormalizedVersion} {SymbolsPackageKey} has already started.",
73+
request.PackageId,
74+
request.PackageVersion,
75+
request.PackageKey);
76+
77+
return result;
78+
}
79+
80+
_telemetryService.TrackSymbolsMessageEnqueued(ValidatorName.SymbolsIngester, request.ValidationId);
81+
await _symbolMessageEnqueuer.EnqueueSymbolsValidationMessageAsync(request);
82+
83+
var newSymbolsRequest = SymbolsValidationEntitiesService.CreateFromValidationRequest(request, SymbolsPackageIngestRequestStatus.Ingesting);
84+
var savedSymbolRequest = await _symbolsValidationEntitiesService.AddSymbolsServerRequestAsync(newSymbolsRequest);
85+
86+
if(savedSymbolRequest.RequestStatusKey != SymbolsPackageIngestRequestStatus.Ingesting)
87+
{
88+
_logger.LogWarning(
89+
"The symbols ingestion request already in the database. RequestStatus:{Status} for {PackageId} {PackageNormalizedVersion} {SymbolsPackageKey}.",
90+
newSymbolsRequest.RequestStatusKey,
91+
request.PackageId,
92+
request.PackageVersion,
93+
request.PackageKey);
94+
}
95+
else
96+
{
97+
_logger.LogInformation(
98+
"The symbols ingestion request added to the database. RequestStatus:{Status} for {PackageId} {PackageNormalizedVersion} {SymbolsPackageKey}.",
99+
newSymbolsRequest.RequestStatusKey,
100+
request.PackageId,
101+
request.PackageVersion,
102+
request.PackageKey);
103+
}
104+
return SymbolsValidationEntitiesService.ConvertToIValidationResult(savedSymbolRequest);
105+
}
106+
}
107+
}
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
namespace NuGet.Services.Validation.Symbols
5+
{
6+
public class SymbolsIngesterConfiguration : SymbolsValidationConfiguration
7+
{
8+
}
9+
}

src/Validation.Common.Job/Validation.Common.Job.csproj

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@
100100
<Version>2.27.0</Version>
101101
</PackageReference>
102102
<PackageReference Include="NuGet.Services.ServiceBus">
103-
<Version>2.27.0</Version>
103+
<Version>2.28.0-master-36259</Version>
104104
</PackageReference>
105105
<PackageReference Include="NuGet.Services.Sql">
106106
<Version>2.27.0</Version>
@@ -109,7 +109,7 @@
109109
<Version>2.27.0</Version>
110110
</PackageReference>
111111
<PackageReference Include="NuGet.Services.Validation">
112-
<Version>2.27.0</Version>
112+
<Version>2.28.0-master-36259</Version>
113113
</PackageReference>
114114
<PackageReference Include="NuGet.Services.Validation.Issues">
115115
<Version>2.27.0</Version>

src/Validation.Common.Job/Validation.Common.Job.nuspec

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,10 @@
1818
<dependency id="NuGet.Packaging" version="4.8.0-preview4.5289" />
1919
<dependency id="NuGet.Services.Configuration" version="2.27.0" />
2020
<dependency id="NuGet.Services.Logging" version="2.27.0" />
21-
<dependency id="NuGet.Services.ServiceBus" version="2.27.0" />
21+
<dependency id="NuGet.Services.ServiceBus" version="2.28.0-master-36259" />
2222
<dependency id="NuGet.Services.Sql" version="2.27.0" />
2323
<dependency id="NuGet.Services.Storage" version="2.27.0" />
24-
<dependency id="NuGet.Services.Validation" version="2.27.0" />
24+
<dependency id="NuGet.Services.Validation" version="2.28.0-master-36259" />
2525
<dependency id="NuGet.Services.Validation.Issues" version="2.27.0" />
2626
<dependency id="NuGetGallery.Core" version="4.4.5-master-36524" />
2727
<dependency id="Serilog" version="2.5.0" />

src/Validation.Common.Job/Validation/ValidatorName.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,6 @@ public static class ValidatorName
1212
public const string PackageSignatureProcessor = "PackageSigningValidator";
1313
public const string PackageSignatureValidator = "PackageSigningValidator2";
1414
public const string SymbolsValidator = "SymbolsValidator";
15+
public const string SymbolsIngester = "SymbolsIngester";
1516
}
1617
}
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+

2+
//-----------------------------------------------------------------------------
3+
// Copyright (c) Microsoft Corporation. All rights reserved.
4+
//-----------------------------------------------------------------------------
5+
6+
7+
using System.Threading.Tasks;
8+
using NuGet.Services.Validation;
9+
10+
namespace NuGet.Jobs.Validation.Symbols.Core
11+
{
12+
/// <summary>
13+
/// Interface that interacts with Validation db.
14+
/// </summary>
15+
public interface ISymbolsValidationEntitiesService
16+
{
17+
/// <summary>
18+
/// Will add a new request in the SymbolsServerRequests db
19+
/// </summary>
20+
/// <param name="request">The request.</param>
21+
/// <returns>The request ingested. If the request already exists the request that is persisted is returned.</returns>
22+
Task<SymbolsServerRequest> AddSymbolsServerRequestAsync(SymbolsServerRequest request);
23+
24+
/// <summary>
25+
/// Try to update the status.
26+
/// </summary>
27+
/// <param name="request">The request.</param>
28+
/// <param name="newStatus">The new status.</param>
29+
/// <returns>True if success, false if the request is not in the database. Any exceptions will be propagated to the caller.</returns>
30+
Task<bool> TryUpdateSymbolsServerRequestAsync(SymbolsServerRequest request, SymbolsPackageIngestRequestStatus newStatus);
31+
32+
/// <summary>
33+
/// Queries and returns the <see cref="SymbolsServerRequest"/> based on the requestname and symbolsPackageKey.
34+
/// </summary>
35+
/// <param name="requestName">The request name.</param>
36+
/// <param name="symbolsPackageKey">The key of the symbols package</param>
37+
/// <returns>The <see cref="SymbolsServerRequest"/> if found. Null otherwise.</returns>
38+
Task<SymbolsServerRequest> GetSymbolsServerRequestAsync(string requestName, int symbolsPackageKey);
39+
40+
/// <summary>
41+
/// Queries and returns the <see cref="SymbolsServerRequest"/> based on the <see cref="IValidationRequest"/> information.
42+
/// </summary>
43+
/// <param name="validationRequest">The validation request.</param>
44+
/// <returns>The result transformed to <see cref="IValidationResult"/>.</returns>
45+
Task<SymbolsServerRequest> GetSymbolsServerRequestAsync(IValidationRequest validationRequest);
46+
}
47+
}
Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
//-----------------------------------------------------------------------------
2+
// Copyright (c) Microsoft Corporation. All rights reserved.
3+
//-----------------------------------------------------------------------------
4+
5+
using System;
6+
using System.Data.Entity;
7+
using System.Data.Entity.Infrastructure;
8+
using System.Linq;
9+
using System.Threading.Tasks;
10+
using NuGet.Services.Validation;
11+
12+
namespace NuGet.Jobs.Validation.Symbols.Core
13+
{
14+
public class SymbolsValidationEntitiesService : ISymbolsValidationEntitiesService
15+
{
16+
private readonly IValidationEntitiesContext _validationEntitiesContext;
17+
18+
public SymbolsValidationEntitiesService(IValidationEntitiesContext validationEntitiesContext)
19+
{
20+
_validationEntitiesContext = validationEntitiesContext ?? throw new ArgumentNullException(nameof(validationEntitiesContext));
21+
}
22+
23+
public async Task<SymbolsServerRequest> AddSymbolsServerRequestAsync(SymbolsServerRequest request)
24+
{
25+
if (request == null)
26+
{
27+
throw new ArgumentNullException(nameof(request));
28+
}
29+
var currentRequest = await GetSymbolsServerRequestAsync(request.RequestName, request.SymbolsKey);
30+
if (currentRequest != null)
31+
{
32+
return currentRequest;
33+
}
34+
35+
_validationEntitiesContext.SymbolsServerRequests.Add(request);
36+
try
37+
{
38+
await _validationEntitiesContext.SaveChangesAsync();
39+
return request;
40+
}
41+
catch (DbUpdateException e) when (e.IsUniqueConstraintViolationException())
42+
{
43+
// The request must be ingested already.
44+
return await GetSymbolsServerRequestAsync(request.RequestName, request.SymbolsKey);
45+
}
46+
}
47+
48+
public async Task<bool> TryUpdateSymbolsServerRequestAsync(SymbolsServerRequest request, SymbolsPackageIngestRequestStatus newStatus)
49+
{
50+
if (request == null)
51+
{
52+
throw new ArgumentNullException(nameof(request));
53+
}
54+
if (await GetSymbolsServerRequestAsync(request.RequestName, request.SymbolsKey) == null)
55+
{
56+
return false;
57+
}
58+
request.RequestStatusKey = newStatus;
59+
request.LastUpdated = DateTime.UtcNow;
60+
await _validationEntitiesContext.SaveChangesAsync();
61+
return true;
62+
}
63+
64+
public Task<SymbolsServerRequest> GetSymbolsServerRequestAsync(string requestName, int symbolsPackageKey)
65+
{
66+
return _validationEntitiesContext
67+
.SymbolsServerRequests
68+
.Where(s => (s.RequestName == requestName) && (s.SymbolsKey == symbolsPackageKey))
69+
.FirstOrDefaultAsync();
70+
}
71+
72+
public async Task<SymbolsServerRequest> GetSymbolsServerRequestAsync(IValidationRequest validationRequest)
73+
{
74+
string requestName = validationRequest.PackageKey.ToString();
75+
return await GetSymbolsServerRequestAsync(requestName, validationRequest.PackageKey);
76+
}
77+
78+
/// <summary>
79+
/// Creates a <see cref="SymbolsServerRequest"/> from a <see cref="IValidationRequest"/>
80+
/// </summary>
81+
/// <param name="validationRequest">The <see cref="IValidationRequest"/>.</param>
82+
/// <param name="status">The <see cref="SymbolsPackageIngestRequestStatus"/>.</param>
83+
/// <returns></returns>
84+
public static SymbolsServerRequest CreateFromValidationRequest(IValidationRequest validationRequest, SymbolsPackageIngestRequestStatus status)
85+
{
86+
return new SymbolsServerRequest()
87+
{
88+
Created = DateTime.UtcNow,
89+
LastUpdated = DateTime.UtcNow,
90+
RequestName = validationRequest.PackageKey.ToString(),
91+
RequestStatusKey = status,
92+
SymbolsKey = validationRequest.PackageKey
93+
};
94+
}
95+
96+
/// <summary>
97+
/// Converts a <see cref="SymbolsServerRequest"/> to <see cref="IValidationResult"/>
98+
/// </summary>
99+
/// <param name="symbolsServerRequest">A <see cref="SymbolsServerRequest" />.</param>
100+
/// <returns>The <see cref="IValidationResult"/>.</returns>
101+
public static IValidationResult ConvertToIValidationResult(SymbolsServerRequest symbolsServerRequest)
102+
{
103+
if (symbolsServerRequest == null)
104+
{
105+
return new ValidationResult(ValidationStatus.NotStarted);
106+
}
107+
switch (symbolsServerRequest.RequestStatusKey)
108+
{
109+
case SymbolsPackageIngestRequestStatus.FailedIngestion:
110+
return new ValidationResult(ValidationStatus.Failed);
111+
case SymbolsPackageIngestRequestStatus.Ingested:
112+
return new ValidationResult(ValidationStatus.Succeeded);
113+
case SymbolsPackageIngestRequestStatus.Ingesting:
114+
return new ValidationResult(ValidationStatus.Incomplete);
115+
default:
116+
throw new ArgumentOutOfRangeException("Unexpected SymbolsPackageIngestRequestStatus.");
117+
}
118+
}
119+
}
120+
}

0 commit comments

Comments
 (0)