Skip to content
This repository was archived by the owner on Jul 30, 2024. It is now read-only.

Commit e60d1ae

Browse files
joelverhagenjoelverhagen
committed
Add job to verify hashes of existing packages based on DB value (#340)
Progress on NuGet/Engineering#1168
1 parent eecbf64 commit e60d1ae

32 files changed

Lines changed: 1058 additions & 98 deletions

‎NuGet.Jobs.sln‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.PackageSigning.C
111111
EndProject
112112
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Common.Job", "src\Validation.Common.Job\Validation.Common.Job.csproj", "{FA87D075-A934-4443-8D0B-5DB32640B6D7}"
113113
EndProject
114+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PackageHash", "src\PackageHash\PackageHash.csproj", "{40843020-6F0A-48F0-AC28-42FFE3A5C21E}"
115+
EndProject
114116
Global
115117
GlobalSection(SolutionConfigurationPlatforms) = preSolution
116118
Debug|Any CPU = Debug|Any CPU
@@ -283,6 +285,10 @@ Global
283285
{FA87D075-A934-4443-8D0B-5DB32640B6D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
284286
{FA87D075-A934-4443-8D0B-5DB32640B6D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
285287
{FA87D075-A934-4443-8D0B-5DB32640B6D7}.Release|Any CPU.Build.0 = Release|Any CPU
288+
{40843020-6F0A-48F0-AC28-42FFE3A5C21E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
289+
{40843020-6F0A-48F0-AC28-42FFE3A5C21E}.Debug|Any CPU.Build.0 = Debug|Any CPU
290+
{40843020-6F0A-48F0-AC28-42FFE3A5C21E}.Release|Any CPU.ActiveCfg = Release|Any CPU
291+
{40843020-6F0A-48F0-AC28-42FFE3A5C21E}.Release|Any CPU.Build.0 = Release|Any CPU
286292
EndGlobalSection
287293
GlobalSection(SolutionProperties) = preSolution
288294
HideSolutionNode = FALSE
@@ -329,6 +335,7 @@ Global
329335
{2C5BE067-ADFD-49E3-BA9F-13A74436E5DB} = {6A776396-02B1-475D-A104-26940ADB04AB}
330336
{B4B7564A-965B-447B-927F-6749E2C08880} = {6A776396-02B1-475D-A104-26940ADB04AB}
331337
{FA87D075-A934-4443-8D0B-5DB32640B6D7} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
338+
{40843020-6F0A-48F0-AC28-42FFE3A5C21E} = {FA5644B5-4F08-43F6-86B3-039374312A47}
332339
EndGlobalSection
333340
GlobalSection(ExtensibilityGlobals) = postSolution
334341
SolutionGuid = {284A7AC3-FB43-4F1F-9C9C-2AF0E1F46C2B}

‎src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,9 @@
9393
<Compile Include="ValidatorProvider.cs" />
9494
</ItemGroup>
9595
<ItemGroup>
96-
<None Include="App.config" />
96+
<None Include="App.config">
97+
<SubType>Designer</SubType>
98+
</None>
9799
<None Include="settings.json">
98100
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
99101
</None>
@@ -102,9 +104,6 @@
102104
<PackageReference Include="NuGet.Services.Validation.Issues">
103105
<Version>2.14.0</Version>
104106
</PackageReference>
105-
<PackageReference Include="NuGet.Versioning">
106-
<Version>4.3.0</Version>
107-
</PackageReference>
108107
</ItemGroup>
109108
<ItemGroup>
110109
<ProjectReference Include="..\NuGet.Jobs.Common\NuGet.Jobs.Common.csproj">

‎src/PackageHash/BatchProcessor.cs‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Collections.Concurrent;
6+
using System.Collections.Generic;
7+
using System.Linq;
8+
using System.Threading;
9+
using System.Threading.Tasks;
10+
using Microsoft.Extensions.Logging;
11+
using Microsoft.Extensions.Options;
12+
using NuGet.Common;
13+
14+
namespace NuGet.Services.PackageHash
15+
{
16+
public class BatchProcessor : IBatchProcessor
17+
{
18+
private readonly IPackageHashCalculator _calculator;
19+
private readonly IOptionsSnapshot<PackageHashConfiguration> _configuration;
20+
private readonly ILogger<BatchProcessor> _logger;
21+
22+
public BatchProcessor(
23+
IPackageHashCalculator calculator,
24+
IOptionsSnapshot<PackageHashConfiguration> configuration,
25+
ILogger<BatchProcessor> logger)
26+
{
27+
_calculator = calculator ?? throw new ArgumentNullException(nameof(configuration));
28+
_configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
29+
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
30+
}
31+
32+
public async Task<IReadOnlyList<InvalidPackageHash>> ProcessBatchAsync(
33+
IReadOnlyList<PackageHash> batch,
34+
string hashAlgorithmId,
35+
CancellationToken token)
36+
{
37+
// Build up a bag of work.
38+
var remainingWork = new ConcurrentBag<Work>();
39+
var failures = new ConcurrentBag<InvalidPackageHash>();
40+
foreach (var source in _configuration.Value.Sources)
41+
{
42+
foreach (var package in batch)
43+
{
44+
remainingWork.Add(new Work(source, package));
45+
}
46+
}
47+
48+
// Perform the work in parallel.
49+
_logger.LogInformation(
50+
"Starting to check hashes for {PackageCount} packages from {SourceCount} sources ({WorkCount} total" +
51+
" checks, {DegreeOfParallelism} tasks).",
52+
batch.Count,
53+
_configuration.Value.Sources.Count,
54+
remainingWork.Count,
55+
_configuration.Value.DegreeOfParallelism);
56+
57+
var tasks = Enumerable
58+
.Range(0, _configuration.Value.DegreeOfParallelism)
59+
.Select(x => ProcessWorkAsync(remainingWork, failures, hashAlgorithmId, token))
60+
.ToList();
61+
62+
await Task.WhenAll(tasks);
63+
64+
_logger.LogInformation(
65+
"Completed the batch. {FailureResultCount} failure results were found.",
66+
failures.Count);
67+
68+
return failures.ToList();
69+
}
70+
71+
private async Task ProcessWorkAsync(
72+
ConcurrentBag<Work> remainingWork,
73+
ConcurrentBag<InvalidPackageHash> failures,
74+
string hashAlgorithmId,
75+
CancellationToken token)
76+
{
77+
while (remainingWork.TryTake(out var work) && !token.IsCancellationRequested)
78+
{
79+
var actualHash = await _calculator.GetPackageHashAsync(
80+
work.Source,
81+
work.Package.Identity,
82+
hashAlgorithmId,
83+
token);
84+
85+
if (work.Package.ExpectedHash != actualHash)
86+
{
87+
failures.Add(new InvalidPackageHash(work.Source, work.Package, actualHash));
88+
}
89+
}
90+
}
91+
92+
private class Work
93+
{
94+
public Work(PackageSource source, PackageHash package)
95+
{
96+
Source = source ?? throw new ArgumentNullException(nameof(source));
97+
Package = package ?? throw new ArgumentNullException(nameof(package));
98+
}
99+
100+
public PackageSource Source { get; }
101+
public PackageHash Package { get; }
102+
}
103+
}
104+
}

‎src/PackageHash/ConsistentHash.cs‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Security.Cryptography;
6+
using System.Text;
7+
8+
namespace NuGet.Services.PackageHash
9+
{
10+
public static class ConsistentHash
11+
{
12+
public static int DetermineBucket(string key, int bucketCount)
13+
{
14+
using (var hashAlgorithm = SHA256.Create())
15+
{
16+
var hashBytes = hashAlgorithm.ComputeHash(Encoding.UTF8.GetBytes(key));
17+
var reducedHash = 0;
18+
for (var i = 0; i < hashBytes.Length; i += sizeof(int))
19+
{
20+
reducedHash ^= BitConverter.ToInt32(hashBytes, i);
21+
}
22+
23+
return reducedHash % bucketCount;
24+
}
25+
}
26+
}
27+
}

‎src/PackageHash/IBatchProcessor.cs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System.Collections.Generic;
5+
using System.Threading;
6+
using System.Threading.Tasks;
7+
8+
namespace NuGet.Services.PackageHash
9+
{
10+
public interface IBatchProcessor
11+
{
12+
Task<IReadOnlyList<InvalidPackageHash>> ProcessBatchAsync(
13+
IReadOnlyList<PackageHash> batch,
14+
string hashAlgorithm,
15+
CancellationToken token);
16+
}
17+
}

‎src/PackageHash/IPackageHashCalculator.cs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System.Threading;
5+
using System.Threading.Tasks;
6+
using NuGet.Packaging.Core;
7+
8+
namespace NuGet.Services.PackageHash
9+
{
10+
public interface IPackageHashCalculator
11+
{
12+
Task<string> GetPackageHashAsync(
13+
PackageSource source,
14+
PackageIdentity package,
15+
string hashAlgorithmId,
16+
CancellationToken token);
17+
}
18+
}

‎src/PackageHash/IPackageHashProcessor.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
using System.Threading;
2+
using System.Threading.Tasks;
3+
4+
namespace NuGet.Services.PackageHash
5+
{
6+
public interface IPackageHashProcessor
7+
{
8+
Task ExecuteAsync(int bucketNumber, int bucketCount, CancellationToken token);
9+
}
10+
}

‎src/PackageHash/IResultRecorder.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
using System.Collections.Generic;
2+
using System.Threading.Tasks;
3+
4+
namespace NuGet.Services.PackageHash
5+
{
6+
public interface IResultRecorder
7+
{
8+
Task RecordAsync(IReadOnlyList<InvalidPackageHash> results);
9+
}
10+
}

‎src/PackageHash/InvalidPackageHash.cs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
namespace NuGet.Services.PackageHash
5+
{
6+
public class InvalidPackageHash
7+
{
8+
public InvalidPackageHash(PackageSource source, PackageHash package, string invalidHash)
9+
{
10+
Source = source;
11+
Package = package;
12+
InvalidHash = invalidHash;
13+
}
14+
15+
public PackageSource Source { get; }
16+
public PackageHash Package { get; }
17+
public string InvalidHash { get; }
18+
}
19+
}

0 commit comments

Comments
 (0)