Skip to content
This repository was archived by the owner on Jul 30, 2024. It is now read-only.

Commit bf51759

Browse files
loic-sharmaloic-sharma
authored
Merge pull request #511 from NuGet/dev
[ReleasePrep][2018.08.02]RI of dev into master
2 parents ee4fac7 + 97f0725 commit bf51759

25 files changed

Lines changed: 1084 additions & 37 deletions

File tree

‎NuGet.Jobs.sln‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Symbols", "src\V
149149
EndProject
150150
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Symbols.Tests", "tests\Validation.Symbols.Tests\Validation.Symbols.Tests.csproj", "{640D29AB-4D1B-4FC7-AE67-AD12EE5AC503}"
151151
EndProject
152+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Symbols.Core.Tests", "tests\Validation.Symbols.Core.Tests\Validation.Symbols.Core.Tests.csproj", "{9ED642DF-4623-4EB2-8B72-52C6489BF9D6}"
153+
EndProject
152154
Global
153155
GlobalSection(SolutionConfigurationPlatforms) = preSolution
154156
Debug|Any CPU = Debug|Any CPU
@@ -393,6 +395,10 @@ Global
393395
{640D29AB-4D1B-4FC7-AE67-AD12EE5AC503}.Debug|Any CPU.Build.0 = Debug|Any CPU
394396
{640D29AB-4D1B-4FC7-AE67-AD12EE5AC503}.Release|Any CPU.ActiveCfg = Release|Any CPU
395397
{640D29AB-4D1B-4FC7-AE67-AD12EE5AC503}.Release|Any CPU.Build.0 = Release|Any CPU
398+
{9ED642DF-4623-4EB2-8B72-52C6489BF9D6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
399+
{9ED642DF-4623-4EB2-8B72-52C6489BF9D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
400+
{9ED642DF-4623-4EB2-8B72-52C6489BF9D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
401+
{9ED642DF-4623-4EB2-8B72-52C6489BF9D6}.Release|Any CPU.Build.0 = Release|Any CPU
396402
EndGlobalSection
397403
GlobalSection(SolutionProperties) = preSolution
398404
HideSolutionNode = FALSE
@@ -457,6 +463,7 @@ Global
457463
{21C0A0EE-8696-4013-950F-D6495D0C6E40} = {6A776396-02B1-475D-A104-26940ADB04AB}
458464
{2DD07A73-8C88-4429-BB24-C2813586EF92} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
459465
{640D29AB-4D1B-4FC7-AE67-AD12EE5AC503} = {6A776396-02B1-475D-A104-26940ADB04AB}
466+
{9ED642DF-4623-4EB2-8B72-52C6489BF9D6} = {6A776396-02B1-475D-A104-26940ADB04AB}
460467
EndGlobalSection
461468
GlobalSection(ExtensibilityGlobals) = postSolution
462469
SolutionGuid = {284A7AC3-FB43-4F1F-9C9C-2AF0E1F46C2B}

‎src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,11 @@
5757
<Compile Include="IPackageStatusProcessor.cs" />
5858
<Compile Include="Symbols\ISymbolsMessageEnqueuer.cs" />
5959
<Compile Include="Symbols\SymbolsMessageEnqueuer.cs" />
60+
<Compile Include="Symbols\SymbolsIngesterConfiguration.cs" />
61+
<Compile Include="Symbols\SymbolsIngester.cs" />
6062
<Compile Include="Symbols\SymbolsValidator.cs" />
6163
<Compile Include="Symbols\SymbolsValidationConfiguration.cs" />
64+
<Compile Include="UsernameHelper.cs" />
6265
<Compile Include="ValidatingEntitites\IValidatingEntity.cs" />
6366
<Compile Include="IValidationOutcomeProcessor.cs" />
6467
<Compile Include="IValidationPackageFileService.cs" />

‎src/NuGet.Services.Validation.Orchestrator/PackageSigning/ProcessSignature/BaseSignatureProcessor.cs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
using System;
55
using System.Threading.Tasks;
66
using Microsoft.Extensions.Logging;
7-
using NuGet.Jobs.Validation;
87
using NuGet.Jobs.Validation.Storage;
98
using NuGet.Services.Validation.Orchestrator.Telemetry;
109

‎src/NuGet.Services.Validation.Orchestrator/PackageSigning/ProcessSignature/PackageSignatureValidator.cs‎

Lines changed: 55 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@
88
using Microsoft.Extensions.Options;
99
using NuGet.Jobs.Validation;
1010
using NuGet.Jobs.Validation.Storage;
11+
using NuGet.Services.Validation.Orchestrator;
1112
using NuGet.Services.Validation.Orchestrator.PackageSigning.ScanAndSign;
1213
using NuGet.Services.Validation.Orchestrator.Telemetry;
14+
using NuGetGallery;
1315

1416
namespace NuGet.Services.Validation.PackageSigning.ProcessSignature
1517
{
@@ -23,6 +25,7 @@ public class PackageSignatureValidator : BaseSignatureProcessor, IValidator
2325
private readonly IValidatorStateService _validatorStateService;
2426
private readonly IProcessSignatureEnqueuer _signatureVerificationEnqueuer;
2527
private readonly ISimpleCloudBlobProvider _blobProvider;
28+
private readonly ICorePackageService _packages;
2629
private readonly ScanAndSignConfiguration _config;
2730
private readonly ITelemetryService _telemetryService;
2831
private readonly ILogger<PackageSignatureValidator> _logger;
@@ -31,6 +34,7 @@ public PackageSignatureValidator(
3134
IValidatorStateService validatorStateService,
3235
IProcessSignatureEnqueuer signatureVerificationEnqueuer,
3336
ISimpleCloudBlobProvider blobProvider,
37+
ICorePackageService packages,
3438
IOptionsSnapshot<ScanAndSignConfiguration> configAccessor,
3539
ITelemetryService telemetryService,
3640
ILogger<PackageSignatureValidator> logger)
@@ -39,6 +43,7 @@ public PackageSignatureValidator(
3943
_validatorStateService = validatorStateService ?? throw new ArgumentNullException(nameof(validatorStateService));
4044
_signatureVerificationEnqueuer = signatureVerificationEnqueuer ?? throw new ArgumentNullException(nameof(signatureVerificationEnqueuer));
4145
_blobProvider = blobProvider ?? throw new ArgumentNullException(nameof(blobProvider));
46+
_packages = packages ?? throw new ArgumentNullException(nameof(packages));
4247
_telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
4348
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
4449

@@ -60,45 +65,57 @@ public override async Task<IValidationResult> GetResultAsync(IValidationRequest
6065
{
6166
var result = await base.GetResultAsync(request);
6267

63-
return Validate(result);
68+
return Validate(request, result);
6469
}
6570

6671
public override async Task<IValidationResult> StartAsync(IValidationRequest request)
6772
{
6873
var result = await base.StartAsync(request);
6974

70-
return Validate(result);
75+
return Validate(request, result);
7176
}
7277

73-
private IValidationResult Validate(IValidationResult result)
78+
private IValidationResult Validate(IValidationRequest request, IValidationResult result)
7479
{
7580
/// The package signature validator runs after the <see cref="PackageSignatureProcessor" />.
7681
/// All signature validation issues should be caught and handled by the processor.
7782
if (result.Status == ValidationStatus.Failed || result.NupkgUrl != null)
7883
{
79-
if (_config.RepositorySigningEnabled)
84+
if (!_config.RepositorySigningEnabled)
8085
{
81-
_logger.LogCritical(
82-
"Unexpected validation result in package signature validator. This may be caused by an invalid repository " +
83-
"signature. Throwing an exception to force this validation to dead-letter. " +
86+
_logger.LogInformation(
87+
"Ignoring invalid validation result in package signature validator as repository signing is disabled. " +
8488
"Status = {ValidationStatus}, Nupkg URL = {NupkgUrl}, validation issues = {Issues}",
8589
result.Status,
8690
result.NupkgUrl,
8791
result.Issues.Select(i => i.IssueCode));
8892

89-
throw new InvalidOperationException("Package signature validator has an unexpected validation result");
93+
return ValidationResult.Succeeded;
9094
}
91-
else
95+
96+
// TODO: Remove this.
97+
// See: https://github.com/NuGet/Engineering/issues/1592
98+
if (HasOwnerWithInvalidUsername(request))
9299
{
93-
_logger.LogInformation(
94-
"Ignoring invalid validation result in package signature validator as repository signing is disabled. " +
100+
_logger.LogWarning(
101+
"Ignoring invalid validation result in package signature validator as the package has an owner with an invalid username. " +
95102
"Status = {ValidationStatus}, Nupkg URL = {NupkgUrl}, validation issues = {Issues}",
96103
result.Status,
97104
result.NupkgUrl,
98105
result.Issues.Select(i => i.IssueCode));
99106

100107
return ValidationResult.Succeeded;
101108
}
109+
110+
_logger.LogCritical(
111+
"Unexpected validation result in package signature validator. This may be caused by an invalid repository " +
112+
"signature. Throwing an exception to force this validation to dead-letter. " +
113+
"Status = {ValidationStatus}, Nupkg URL = {NupkgUrl}, validation issues = {Issues}",
114+
result.Status,
115+
result.NupkgUrl,
116+
result.Issues.Select(i => i.IssueCode));
117+
118+
throw new InvalidOperationException("Package signature validator has an unexpected validation result");
102119
}
103120

104121
/// Suppress all validation issues. The <see cref="PackageSignatureProcessor"/> should
@@ -116,5 +133,32 @@ private IValidationResult Validate(IValidationResult result)
116133

117134
return result;
118135
}
136+
137+
private bool HasOwnerWithInvalidUsername(IValidationRequest request)
138+
{
139+
var registration = _packages.FindPackageRegistrationById(request.PackageId);
140+
141+
if (registration == null)
142+
{
143+
_logger.LogError("Attempted to validate package that has no package registration");
144+
145+
throw new InvalidOperationException($"Registration for package id {request.PackageId} does not exist");
146+
}
147+
148+
var owners = registration.Owners.Select(o => o.Username).ToList();
149+
150+
if (owners.Any(UsernameHelper.IsInvalid))
151+
{
152+
_logger.LogWarning(
153+
"Package {PackageId} {PackageVersion} has an owner with an invalid username. {Owners}",
154+
request.PackageId,
155+
request.PackageVersion,
156+
owners);
157+
158+
return true;
159+
}
160+
161+
return false;
162+
}
119163
}
120164
}

‎src/NuGet.Services.Validation.Orchestrator/PackageSigning/ScanAndSign/ScanAndSignProcessor.cs‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@ namespace NuGet.Services.Validation.Orchestrator.PackageSigning.ScanAndSign
2020
[ValidatorName(ValidatorName.ScanAndSign)]
2121
public class ScanAndSignProcessor : IProcessor
2222
{
23-
private const string UsernameRegex = @"^[A-Za-z0-9][A-Za-z0-9_\.-]+[A-Za-z0-9]$";
24-
2523
private readonly IValidationEntitiesContext _validationContext;
2624
private readonly IValidatorStateService _validatorStateService;
2725
private readonly ICorePackageService _packageService;
@@ -218,7 +216,9 @@ private async Task<bool> ShouldRepositorySignAsync(IValidationRequest request, L
218216
return false;
219217
}
220218

221-
if (owners.Any(IsInvalidUsername))
219+
// TODO: Remove this check.
220+
// See: https://github.com/NuGet/Engineering/issues/1582
221+
if (owners.Any(UsernameHelper.IsInvalid))
222222
{
223223
_logger.LogWarning(
224224
"Package {PackageId} {PackageVersion} has an owner with an invalid username. Scanning instead of signing. {Owners}",
@@ -248,10 +248,5 @@ private List<string> FindPackageOwners(IValidationRequest request)
248248
.Select(o => o.Username)
249249
.ToList();
250250
}
251-
252-
private bool IsInvalidUsername(string username)
253-
{
254-
return !Regex.IsMatch(username, UsernameRegex, RegexOptions.None, TimeSpan.FromSeconds(5));
255-
}
256251
}
257252
}

‎src/NuGet.Services.Validation.Orchestrator/Symbols/SymbolsIngester.cs‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Threading.Tasks;
6+
using Microsoft.Extensions.Logging;
7+
using NuGet.Jobs.Validation;
8+
using NuGet.Jobs.Validation.Symbols.Core;
9+
using NuGet.Services.Validation.Orchestrator;
10+
using NuGet.Services.Validation.Orchestrator.Telemetry;
11+
12+
namespace NuGet.Services.Validation.Symbols
13+
{
14+
[ValidatorName(ValidatorName.SymbolsIngester)]
15+
public class SymbolsIngester : BaseValidator, IValidator
16+
{
17+
private readonly ISymbolsValidationEntitiesService _symbolsValidationEntitiesService;
18+
private readonly ISymbolsMessageEnqueuer _symbolMessageEnqueuer;
19+
private readonly ITelemetryService _telemetryService;
20+
private readonly ILogger<SymbolsIngester> _logger;
21+
22+
public SymbolsIngester(
23+
ISymbolsValidationEntitiesService symbolsValidationEntitiesService,
24+
ISymbolsMessageEnqueuer symbolMessageEnqueuer,
25+
ITelemetryService telemetryService,
26+
ILogger<SymbolsIngester> logger)
27+
{
28+
_symbolsValidationEntitiesService = symbolsValidationEntitiesService ?? throw new ArgumentNullException(nameof(symbolsValidationEntitiesService));
29+
_symbolMessageEnqueuer = symbolMessageEnqueuer ?? throw new ArgumentNullException(nameof(symbolMessageEnqueuer));
30+
_telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
31+
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
32+
}
33+
34+
public async Task<IValidationResult> GetResultAsync(IValidationRequest request)
35+
{
36+
if (request == null)
37+
{
38+
throw new ArgumentNullException(nameof(request));
39+
}
40+
41+
var result = SymbolsValidationEntitiesService.ConvertToIValidationResult(await _symbolsValidationEntitiesService.GetSymbolsServerRequestAsync(request));
42+
_logger.LogInformation(
43+
"Symbols status {Status} for PackageId: {PackageId}, PackageNormalizedVersion {PackageNormalizedVersion}, SymbolsPackageKey {SymbolsPackageKey} ",
44+
result.Status,
45+
request.PackageId,
46+
request.PackageVersion,
47+
request.PackageKey);
48+
49+
return result;
50+
}
51+
52+
/// <summary>
53+
/// The pattern used for the StartAsync:
54+
/// 1. Check if an ingestion for the specific symbols package key was already started
55+
/// 2. Only if a ingestion was not started queue the message to be processed.
56+
/// 3. After the message is queued, update the SymbolServerRequests table.
57+
/// </summary>
58+
/// <param name="request">The request to be sent to the ingester job queue.</param>
59+
/// <returns>The operation status as <see cref="IValidationResult"/>.</returns>
60+
public async Task<IValidationResult> StartAsync(IValidationRequest request)
61+
{
62+
if (request == null)
63+
{
64+
throw new ArgumentNullException(nameof(request));
65+
}
66+
67+
var result = SymbolsValidationEntitiesService.ConvertToIValidationResult(await _symbolsValidationEntitiesService.GetSymbolsServerRequestAsync(request));
68+
69+
if (result.Status != ValidationStatus.NotStarted)
70+
{
71+
_logger.LogWarning(
72+
"Symbol ingestion for {PackageId} {PackageNormalizedVersion} {SymbolsPackageKey} has already started.",
73+
request.PackageId,
74+
request.PackageVersion,
75+
request.PackageKey);
76+
77+
return result;
78+
}
79+
80+
_telemetryService.TrackSymbolsMessageEnqueued(ValidatorName.SymbolsIngester, request.ValidationId);
81+
await _symbolMessageEnqueuer.EnqueueSymbolsValidationMessageAsync(request);
82+
83+
var newSymbolsRequest = SymbolsValidationEntitiesService.CreateFromValidationRequest(request, SymbolsPackageIngestRequestStatus.Ingesting);
84+
var savedSymbolRequest = await _symbolsValidationEntitiesService.AddSymbolsServerRequestAsync(newSymbolsRequest);
85+
86+
if(savedSymbolRequest.RequestStatusKey != SymbolsPackageIngestRequestStatus.Ingesting)
87+
{
88+
_logger.LogWarning(
89+
"The symbols ingestion request already in the database. RequestStatus:{Status} for {PackageId} {PackageNormalizedVersion} {SymbolsPackageKey}.",
90+
newSymbolsRequest.RequestStatusKey,
91+
request.PackageId,
92+
request.PackageVersion,
93+
request.PackageKey);
94+
}
95+
else
96+
{
97+
_logger.LogInformation(
98+
"The symbols ingestion request added to the database. RequestStatus:{Status} for {PackageId} {PackageNormalizedVersion} {SymbolsPackageKey}.",
99+
newSymbolsRequest.RequestStatusKey,
100+
request.PackageId,
101+
request.PackageVersion,
102+
request.PackageKey);
103+
}
104+
return SymbolsValidationEntitiesService.ConvertToIValidationResult(savedSymbolRequest);
105+
}
106+
}
107+
}

‎src/NuGet.Services.Validation.Orchestrator/Symbols/SymbolsIngesterConfiguration.cs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
namespace NuGet.Services.Validation.Symbols
5+
{
6+
public class SymbolsIngesterConfiguration : SymbolsValidationConfiguration
7+
{
8+
}
9+
}

‎src/NuGet.Services.Validation.Orchestrator/UsernameHelper.cs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Text.RegularExpressions;
6+
7+
namespace NuGet.Services.Validation.Orchestrator
8+
{
9+
// TODO: Remove this type.
10+
// See: https://github.com/NuGet/Engineering/issues/1582
11+
// See: https://github.com/NuGet/Engineering/issues/1592
12+
public static class UsernameHelper
13+
{
14+
private const string UsernameRegex = @"^[A-Za-z0-9][A-Za-z0-9_\.-]+[A-Za-z0-9]$";
15+
16+
public static bool IsInvalid(string username)
17+
{
18+
return !Regex.IsMatch(username, UsernameRegex, RegexOptions.None, TimeSpan.FromSeconds(5));
19+
}
20+
}
21+
}

‎src/Validation.Common.Job/Validation.Common.Job.csproj‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@
100100
<Version>2.27.0</Version>
101101
</PackageReference>
102102
<PackageReference Include="NuGet.Services.ServiceBus">
103-
<Version>2.27.0</Version>
103+
<Version>2.28.0-master-36259</Version>
104104
</PackageReference>
105105
<PackageReference Include="NuGet.Services.Sql">
106106
<Version>2.27.0</Version>
@@ -109,7 +109,7 @@
109109
<Version>2.27.0</Version>
110110
</PackageReference>
111111
<PackageReference Include="NuGet.Services.Validation">
112-
<Version>2.27.0</Version>
112+
<Version>2.28.0-master-36259</Version>
113113
</PackageReference>
114114
<PackageReference Include="NuGet.Services.Validation.Issues">
115115
<Version>2.27.0</Version>

‎src/Validation.Common.Job/Validation.Common.Job.nuspec‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,10 @@
1818
<dependency id="NuGet.Packaging" version="4.8.0-preview4.5289" />
1919
<dependency id="NuGet.Services.Configuration" version="2.27.0" />
2020
<dependency id="NuGet.Services.Logging" version="2.27.0" />
21-
<dependency id="NuGet.Services.ServiceBus" version="2.27.0" />
21+
<dependency id="NuGet.Services.ServiceBus" version="2.28.0-master-36259" />
2222
<dependency id="NuGet.Services.Sql" version="2.27.0" />
2323
<dependency id="NuGet.Services.Storage" version="2.27.0" />
24-
<dependency id="NuGet.Services.Validation" version="2.27.0" />
24+
<dependency id="NuGet.Services.Validation" version="2.28.0-master-36259" />
2525
<dependency id="NuGet.Services.Validation.Issues" version="2.27.0" />
2626
<dependency id="NuGetGallery.Core" version="4.4.5-master-36524" />
2727
<dependency id="Serilog" version="2.5.0" />

0 commit comments

Comments
 (0)