Skip to content
This repository was archived by the owner on Jul 30, 2024. It is now read-only.

Commit b01d02f

Browse files
loic-sharmaloic-sharma
authored
Improve package registration finder (#532)
Improves the code that finds the information about package registrations by: 1. Sleeping between batches of registrations 2. Creating a new scope for each batch
1 parent 352b32d commit b01d02f

6 files changed

Lines changed: 66 additions & 30 deletions

File tree

‎src/NuGet.Services.Revalidate/Initialization/IPackageFinder.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

44
using System.Collections.Generic;
5+
using System.Threading.Tasks;
56
using NuGet.Versioning;
67

78
namespace NuGet.Services.Revalidate
@@ -41,7 +42,7 @@ public interface IPackageFinder
4142
/// <param name="setName">The name of this set of packages.</param>
4243
/// <param name="packageRegistrationKeys">The set of package registration keys.</param>
4344
/// <returns>Information about each package registration, if it exists in the database.</returns>
44-
List<PackageRegistrationInformation> FindPackageRegistrationInformation(string setName, HashSet<int> packageRegistrationKeys);
45+
Task<List<PackageRegistrationInformation>> FindPackageRegistrationInformationAsync(string setName, HashSet<int> packageRegistrationKeys);
4546

4647
/// <summary>
4748
/// Find versions that are appropriate for revalidations.

‎src/NuGet.Services.Revalidate/Initialization/InitializationManager.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ private async Task ClearPackageRevalidationStateAsync()
122122

123123
private async Task InitializePackageSetAsync(string setName, HashSet<int> packageRegistrationKeys)
124124
{
125-
var packageInformations = _packageFinder.FindPackageRegistrationInformation(setName, packageRegistrationKeys);
125+
var packageInformations = await _packageFinder.FindPackageRegistrationInformationAsync(setName, packageRegistrationKeys);
126126

127127
var chunks = packageInformations
128128
.OrderByDescending(p => p.Downloads)

‎src/NuGet.Services.Revalidate/Initialization/PackageFinder.cs‎

Lines changed: 31 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
using System.Linq;
88
using System.Linq.Expressions;
99
using System.Threading;
10+
using System.Threading.Tasks;
11+
using Microsoft.Extensions.DependencyInjection;
1012
using Microsoft.Extensions.Logging;
1113
using Newtonsoft.Json;
1214
using NuGet.Versioning;
@@ -29,15 +31,18 @@ public class PackageFinder : IPackageFinder
2931
private static string MicrosoftAccountName = "Microsoft";
3032

3133
private readonly IGalleryContext _galleryContext;
34+
private readonly IServiceScopeFactory _scopeFactory;
3235
private readonly InitializationConfiguration _config;
3336
private readonly ILogger<PackageFinder> _logger;
3437

3538
public PackageFinder(
3639
IGalleryContext galleryContext,
40+
IServiceScopeFactory scopeFactory,
3741
InitializationConfiguration config,
3842
ILogger<PackageFinder> logger)
3943
{
4044
_galleryContext = galleryContext ?? throw new ArgumentNullException(nameof(galleryContext));
45+
_scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory));
4146
_config = config ?? throw new ArgumentNullException(nameof(config));
4247
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
4348
}
@@ -105,7 +110,7 @@ public HashSet<int> FindAllPackages(HashSet<int> except)
105110
return FindRegistrationKeys(RemainingSetName, r => !except.Contains(r.Key));
106111
}
107112

108-
public List<PackageRegistrationInformation> FindPackageRegistrationInformation(string setName, HashSet<int> packageRegistrationKeys)
113+
public async Task<List<PackageRegistrationInformation>> FindPackageRegistrationInformationAsync(string setName, HashSet<int> packageRegistrationKeys)
109114
{
110115
// Fetch the packages' information in batches.
111116
var batches = packageRegistrationKeys.Batch(BatchSize);
@@ -121,23 +126,38 @@ public List<PackageRegistrationInformation> FindPackageRegistrationInformation(s
121126

122127
var batch = batches[batchIndex];
123128

124-
var packages = _galleryContext.PackageRegistrations
125-
.Where(r => batch.Contains(r.Key))
126-
.Select(r => new PackageRegistrationInformation
127-
{
128-
Key = r.Key,
129-
Id = r.Id,
130-
Downloads = r.DownloadCount,
131-
Versions = r.Packages.Count(),
132-
});
129+
using (var scope = _scopeFactory.CreateScope())
130+
{
131+
var scopedContext = scope.ServiceProvider.GetRequiredService<IGalleryContext>();
132+
var packages = scopedContext
133+
.PackageRegistrations
134+
.Where(r => batch.Contains(r.Key))
135+
.Select(r => new PackageRegistrationInformation
136+
{
137+
Key = r.Key,
138+
Id = r.Id,
139+
Downloads = r.DownloadCount,
140+
Versions = r.Packages.Count(),
141+
});
133142

134-
result.AddRange(packages);
143+
result.AddRange(packages);
144+
}
135145

136146
_logger.LogInformation(
137147
"Fetched batch {Batch} of {BatchesCount} of package informations for package set {SetName}",
138148
batchIndex + 1,
139149
batches.Count,
140150
setName);
151+
152+
if (batchIndex < batches.Count - 1)
153+
{
154+
_logger.LogInformation(
155+
"Sleeping for {SleepDuration} before fetching the next batch for package set {SetName}...",
156+
_config.SleepDurationBetweenBatches,
157+
setName);
158+
159+
await Task.Delay(_config.SleepDurationBetweenBatches);
160+
}
141161
}
142162

143163
return result;

‎tests/NuGet.Services.Revalidate.Tests/Initializer/InitializationManagerFacts.cs‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,8 @@ public async Task RemovesPreviousRevalidations()
3939
_packageFinder.Setup(f => f.FindDependencyPackages(It.IsAny<HashSet<int>>())).Returns(new HashSet<int>());
4040
_packageFinder.Setup(f => f.FindAllPackages(It.IsAny<HashSet<int>>())).Returns(new HashSet<int>());
4141

42-
_packageFinder.Setup(f => f.FindPackageRegistrationInformation(It.IsAny<string>(), It.IsAny<HashSet<int>>()))
43-
.Returns(new List<PackageRegistrationInformation>());
42+
_packageFinder.Setup(f => f.FindPackageRegistrationInformationAsync(It.IsAny<string>(), It.IsAny<HashSet<int>>()))
43+
.ReturnsAsync(new List<PackageRegistrationInformation>());
4444

4545
var firstRemove = true;
4646

@@ -339,20 +339,20 @@ List<PackageRegistrationInformation> RegistrationInformation(IEnumerable<Package
339339
}
340340

341341
_packageFinder
342-
.Setup(f => f.FindPackageRegistrationInformation(PackageFinder.MicrosoftSetName, It.IsAny<HashSet<int>>()))
343-
.Returns(RegistrationInformation(microsoftPackages));
342+
.Setup(f => f.FindPackageRegistrationInformationAsync(PackageFinder.MicrosoftSetName, It.IsAny<HashSet<int>>()))
343+
.ReturnsAsync(RegistrationInformation(microsoftPackages));
344344

345345
_packageFinder
346-
.Setup(f => f.FindPackageRegistrationInformation(PackageFinder.PreinstalledSetName, It.IsAny<HashSet<int>>()))
347-
.Returns(RegistrationInformation(preinstalledPackages));
346+
.Setup(f => f.FindPackageRegistrationInformationAsync(PackageFinder.PreinstalledSetName, It.IsAny<HashSet<int>>()))
347+
.ReturnsAsync(RegistrationInformation(preinstalledPackages));
348348

349349
_packageFinder
350-
.Setup(f => f.FindPackageRegistrationInformation(PackageFinder.DependencySetName, It.IsAny<HashSet<int>>()))
351-
.Returns(RegistrationInformation(dependencyPackages));
350+
.Setup(f => f.FindPackageRegistrationInformationAsync(PackageFinder.DependencySetName, It.IsAny<HashSet<int>>()))
351+
.ReturnsAsync(RegistrationInformation(dependencyPackages));
352352

353353
_packageFinder
354-
.Setup(f => f.FindPackageRegistrationInformation(PackageFinder.RemainingSetName, It.IsAny<HashSet<int>>()))
355-
.Returns(RegistrationInformation(remainingPackages));
354+
.Setup(f => f.FindPackageRegistrationInformationAsync(PackageFinder.RemainingSetName, It.IsAny<HashSet<int>>()))
355+
.ReturnsAsync(RegistrationInformation(remainingPackages));
356356

357357
// Build the list of versions for each version of packages.
358358
var versions = microsoftPackages

‎tests/NuGet.Services.Revalidate.Tests/Initializer/PackageFinderFacts.cs‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33

44
using System;
55
using System.Collections.Generic;
6+
using System.Threading.Tasks;
7+
using Microsoft.Extensions.DependencyInjection;
68
using Microsoft.Extensions.Logging;
79
using Moq;
810
using NuGet.Versioning;
@@ -207,7 +209,7 @@ public void ExcludesIdsFromExceptParameter()
207209
public class TheFindPackageRegistrationInformationMethod : FactsBase
208210
{
209211
[Fact]
210-
public void FindsRegistrationInformation()
212+
public async Task FindsRegistrationInformation()
211213
{
212214
// Arrange
213215
// Package registration 3 isn't in the input set and should be ignored.
@@ -249,7 +251,7 @@ public void FindsRegistrationInformation()
249251
});
250252

251253
// Act and assert
252-
var actual = _target.FindPackageRegistrationInformation("Name", new HashSet<int> { 1, 2 });
254+
var actual = await _target.FindPackageRegistrationInformationAsync("Name", new HashSet<int> { 1, 2 });
253255

254256
Assert.Equal(2, actual.Count);
255257
Assert.Equal(1, actual[0].Key);
@@ -264,7 +266,7 @@ public void FindsRegistrationInformation()
264266
}
265267

266268
[Fact]
267-
public void PackageCountDoesntFilterPackagesThatWontBeRevalidated()
269+
public async Task PackageCountDoesntFilterPackagesThatWontBeRevalidated()
268270
{
269271
// Arrange
270272
// Package registration 3 isn't in the input set and should be ignored.
@@ -294,7 +296,7 @@ public void PackageCountDoesntFilterPackagesThatWontBeRevalidated()
294296
});
295297

296298
// Act & Assert
297-
var actual = _target.FindPackageRegistrationInformation("Name", new HashSet<int> { 1 });
299+
var actual = await _target.FindPackageRegistrationInformationAsync("Name", new HashSet<int> { 1 });
298300

299301
Assert.Single(actual);
300302
Assert.Equal(1, actual[0].Key);
@@ -464,17 +466,27 @@ public void SkipsPackagesThatAreNotAvailableOrDeleted()
464466
public class FactsBase
465467
{
466468
public readonly Mock<IEntitiesContext> _context;
469+
public readonly Mock<IServiceScopeFactory> _scopeFactory;
467470

468471
public readonly InitializationConfiguration _config;
469472
public readonly PackageFinder _target;
470473

471474
public FactsBase()
472475
{
473476
_context = new Mock<IEntitiesContext>();
477+
_scopeFactory = new Mock<IServiceScopeFactory>();
474478
_config = new InitializationConfiguration();
475479

480+
var scope = new Mock<IServiceScope>();
481+
var serviceProvider = new Mock<IServiceProvider>();
482+
483+
_scopeFactory.Setup(s => s.CreateScope()).Returns(scope.Object);
484+
scope.Setup(s => s.ServiceProvider).Returns(serviceProvider.Object);
485+
serviceProvider.Setup(p => p.GetService(typeof(IEntitiesContext))).Returns(_context.Object);
486+
476487
_target = new PackageFinder(
477488
_context.Object,
489+
_scopeFactory.Object,
478490
_config,
479491
Mock.Of<ILogger<PackageFinder>>());
480492
}

‎tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -311,9 +311,12 @@ public async Task EnqueuesScanAndSignEvenIfRepositorySigningIsDisabled()
311311
_request.NupkgUrl,
312312
_config.V3ServiceIndexUrl,
313313
It.Is<List<string>>(l =>
314-
l.Count() == 2 &&
315-
l.Contains("Billy") &&
316-
l.Contains("Bob"))),
314+
// Ensure that the owners are lexicographically ordered.
315+
l.Count() == 4 &&
316+
l[0] == "Annie" &&
317+
l[1] == "Bob" &&
318+
l[2] == "zack" &&
319+
l[3] == "Zorro")),
317320
Times.Once);
318321

319322
_validatorStateServiceMock

0 commit comments

Comments
 (0)