Skip to content
This repository was archived by the owner on Jul 30, 2024. It is now read-only.

Commit 6b53394

Browse files
loic-sharmaloic-sharma
authored
[Revalidation] Request revalidation (#477)
Adds the core logic of the revalidation job so that it wakes up, determines the next package to revalidate, and enqueues it to the Orchestrator. Addresses https://github.com/NuGet/Engineering/issues/1443
1 parent f1c75c6 commit 6b53394

28 files changed

Lines changed: 1298 additions & 46 deletions

‎src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,30 @@
11
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

4+
using System;
5+
46
namespace NuGet.Services.Revalidate
57
{
68
public class RevalidationConfiguration
79
{
10+
/// <summary>
11+
/// The time before the revalidation job restarts itself.
12+
/// </summary>
13+
public TimeSpan ShutdownWaitInterval { get; set; } = TimeSpan.FromDays(1);
14+
15+
/// <summary>
16+
/// How long the revalidation job should wait if a revalidation cannot be processed at this time.
17+
/// </summary>
18+
public TimeSpan RetryLaterSleep { get; set; } = TimeSpan.FromMinutes(5);
19+
820
/// <summary>
921
/// The configurations used to initialize the revalidation state.
1022
/// </summary>
1123
public InitializationConfiguration Initialization { get; set; }
24+
25+
/// <summary>
26+
/// The configurations used by the in-memory queue of revalidations to start.
27+
/// </summary>
28+
public RevalidationQueueConfiguration Queue { get; set; }
1229
}
1330
}

‎src/NuGet.Services.Revalidate/Configuration/RevalidationQueueConfiguration.cs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
6+
namespace NuGet.Services.Revalidate
7+
{
8+
public class RevalidationQueueConfiguration
9+
{
10+
/// <summary>
11+
/// The maximum times that the <see cref="RevalidationQueue"/> should look for a revalidation
12+
/// before giving up.
13+
/// </summary>
14+
public int MaximumAttempts { get; set; } = 5;
15+
16+
/// <summary>
17+
/// The time to sleep after an initialized revalidation is deemed completed.
18+
/// </summary>
19+
public TimeSpan SleepBetweenAttempts { get; set; } = TimeSpan.FromSeconds(5);
20+
}
21+
}

‎src/NuGet.Services.Revalidate/Initialization/PackageFinder.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,6 @@ private HashSet<int> FindRegistrationKeys(string setName, Expression<Func<Packag
189189
.ToList();
190190

191191
result.UnionWith(batchResults);
192-
start = batchResults.Last();
193192

194193
_logger.LogInformation("Found {Results} results for package set {SetName}", result.Count, setName);
195194

@@ -199,6 +198,8 @@ private HashSet<int> FindRegistrationKeys(string setName, Expression<Func<Packag
199198
}
200199
else
201200
{
201+
start = batchResults.Last();
202+
202203
_logger.LogInformation(
203204
"Sleeping for {SleepDuration} before searching for more package set {SetName} results",
204205
_config.SleepDurationBetweenBatches,

‎src/NuGet.Services.Revalidate/Job.cs‎

Lines changed: 39 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@
1313
using NuGet.Jobs;
1414
using NuGet.Jobs.Configuration;
1515
using NuGet.Jobs.Validation;
16+
using NuGet.Services.Logging;
17+
using NuGet.Services.ServiceBus;
18+
using NuGet.Services.Validation;
1619
using NuGetGallery;
1720

1821
namespace NuGet.Services.Revalidate
@@ -26,6 +29,8 @@ public class Job : JsonConfigurationJob
2629
private const string VerifyInitializationArgumentName = "VerifyInitialization";
2730
private const string JobConfigurationSectionName = "RevalidateJob";
2831

32+
private static readonly TimeSpan RetryLaterSleepDuration = TimeSpan.FromMinutes(5);
33+
2934
private bool _initialize;
3035
private bool _verifyInitialization;
3136

@@ -75,17 +80,23 @@ public override async Task Run()
7580
}
7681
else
7782
{
78-
// TODO: https://github.com/NuGet/Engineering/issues/1443
79-
// Send revalidation requests to the Orchestrator.
80-
throw new NotImplementedException();
83+
Logger.LogInformation("Running the revalidation service...");
84+
85+
await scope.ServiceProvider
86+
.GetRequiredService<IRevalidationService>()
87+
.RunAsync();
88+
89+
Logger.LogInformation("Revalidation service finished running");
8190
}
8291
}
8392
}
8493

8594
protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot)
8695
{
8796
services.Configure<RevalidationConfiguration>(configurationRoot.GetSection(JobConfigurationSectionName));
97+
services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value);
8898
services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.Initialization);
99+
services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.Queue);
89100

90101
services.AddScoped<IGalleryContext>(provider =>
91102
{
@@ -94,9 +105,31 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
94105
return new GalleryContext(config.ConnectionString, readOnly: false);
95106
});
96107

97-
services.AddScoped<IRevalidationStateService, RevalidationStateService>();
98-
services.AddScoped<IPackageFinder, PackageFinder>();
99-
services.AddScoped<InitializationManager>();
108+
// Core
109+
services.AddTransient<ITelemetryService, TelemetryService>();
110+
services.AddTransient<ITelemetryClient, TelemetryClientWrapper>();
111+
112+
services.AddTransient<IRevalidationStateService, RevalidationStateService>();
113+
114+
// Initialization
115+
services.AddTransient<IPackageFinder, PackageFinder>();
116+
services.AddTransient<InitializationManager>();
117+
118+
// Revalidation
119+
services.AddTransient<IHealthService, HealthService>();
120+
services.AddTransient<IRevalidationQueue, RevalidationQueue>();
121+
services.AddTransient<IRevalidationService, RevalidationService>();
122+
services.AddTransient<IRevalidationThrottler, RevalidationThrottler>();
123+
services.AddTransient<ISingletonService, SingletonService>();
124+
125+
services.AddTransient<IPackageValidationEnqueuer, PackageValidationEnqueuer>();
126+
services.AddTransient<IServiceBusMessageSerializer, ServiceBusMessageSerializer>();
127+
services.AddTransient<ITopicClient>(provider =>
128+
{
129+
var config = provider.GetRequiredService<IOptionsSnapshot<ServiceBusConfiguration>>().Value;
130+
131+
return new TopicClientWrapper(config.ConnectionString, config.TopicPath);
132+
});
100133
}
101134

102135
protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder)

‎src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,16 +44,28 @@
4444
</ItemGroup>
4545
<ItemGroup>
4646
<Compile Include="Configuration\InitializationConfiguration.cs" />
47+
<Compile Include="Configuration\RevalidationQueueConfiguration.cs" />
4748
<Compile Include="Extensions\IEnumerableExtensions.cs" />
4849
<Compile Include="Initialization\InitializationManager.cs" />
4950
<Compile Include="Configuration\RevalidationConfiguration.cs" />
5051
<Compile Include="Initialization\IPackageFinder.cs" />
5152
<Compile Include="Initialization\PackageFinder.cs" />
5253
<Compile Include="Initialization\PackageRegistrationInformation.cs" />
53-
<Compile Include="IRevalidationStateService.cs" />
54-
<Compile Include="RevalidationStateService.cs" />
55-
<Compile Include="TelemetryService.cs" />
56-
<Compile Include="ITelemetryService.cs" />
54+
<Compile Include="Services\HealthService.cs" />
55+
<Compile Include="Services\IHealthService.cs" />
56+
<Compile Include="Services\IRevalidationQueue.cs" />
57+
<Compile Include="Services\IRevalidationStateService.cs" />
58+
<Compile Include="Services\IRevalidationService.cs" />
59+
<Compile Include="Services\IRevalidationThrottler.cs" />
60+
<Compile Include="Services\ISingletonService.cs" />
61+
<Compile Include="Services\RevalidationQueue.cs" />
62+
<Compile Include="Services\RevalidationResult.cs" />
63+
<Compile Include="Services\RevalidationService.cs" />
64+
<Compile Include="Services\RevalidationStateService.cs" />
65+
<Compile Include="Services\RevalidationThrottler.cs" />
66+
<Compile Include="Services\SingletonService.cs" />
67+
<Compile Include="Services\TelemetryService.cs" />
68+
<Compile Include="Services\ITelemetryService.cs" />
5769
<Compile Include="Job.cs" />
5870
<Compile Include="Program.cs" />
5971
<Compile Include="Properties\AssemblyInfo.cs" />

‎src/NuGet.Services.Revalidate/Services/HealthService.cs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System.Threading.Tasks;
5+
6+
namespace NuGet.Services.Revalidate
7+
{
8+
public class HealthService : IHealthService
9+
{
10+
public Task<bool> IsHealthyAsync()
11+
{
12+
// TODO:
13+
// We are software gods that never make mistakes.
14+
return Task.FromResult(true);
15+
}
16+
}
17+
}

‎src/NuGet.Services.Revalidate/Services/IHealthService.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System.Threading.Tasks;
5+
6+
namespace NuGet.Services.Revalidate
7+
{
8+
public interface IHealthService
9+
{
10+
/// <summary>
11+
/// Determine whether the NuGet service is healthy.
12+
/// </summary>
13+
/// <returns>Whether the NuGet service is healthy.</returns>
14+
Task<bool> IsHealthyAsync();
15+
}
16+
}

‎src/NuGet.Services.Revalidate/Services/IRevalidationQueue.cs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System.Threading.Tasks;
5+
using NuGet.Services.Validation;
6+
7+
namespace NuGet.Services.Revalidate
8+
{
9+
public interface IRevalidationQueue
10+
{
11+
/// <summary>
12+
/// Fetch the next package to revalidate.
13+
/// </summary>
14+
/// <returns>The next package to revalidate, or null if there are no packages to revalidate at this time.</returns>
15+
Task<PackageRevalidation> NextOrNullAsync();
16+
}
17+
}

src/NuGet.Services.Revalidate/ITelemetryService.cs renamed to src/NuGet.Services.Revalidate/Services/IRevalidationService.cs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

4-
using System;
4+
using System.Threading.Tasks;
55

66
namespace NuGet.Services.Revalidate
77
{
8-
public interface ITelemetryService
8+
public interface IRevalidationService
99
{
10+
Task RunAsync();
11+
12+
Task<RevalidationResult> StartNextRevalidationAsync();
1013
}
1114
}

src/NuGet.Services.Revalidate/IRevalidationStateService.cs renamed to src/NuGet.Services.Revalidate/Services/IRevalidationStateService.cs

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

4-
using System;
54
using System.Collections.Generic;
65
using System.Threading.Tasks;
76
using NuGet.Services.Validation;
@@ -10,6 +9,12 @@ namespace NuGet.Services.Revalidate
109
{
1110
public interface IRevalidationStateService
1211
{
12+
/// <summary>
13+
/// Check whether the killswitch has been activated. If it has, all revalidation operations should be halted.
14+
/// </summary>
15+
/// <returns>Whether the killswitch has been activated.</returns>
16+
Task<bool> IsKillswitchActiveAsync();
17+
1318
/// <summary>
1419
/// Add the new revalidations to the database.
1520
/// </summary>
@@ -27,5 +32,12 @@ public interface IRevalidationStateService
2732
/// </summary>
2833
/// <returns>The count of package revalidations in the database.</returns>
2934
Task<int> PackageRevalidationCountAsync();
35+
36+
/// <summary>
37+
/// Update the package revalidation and mark is as enqueued.
38+
/// </summary>
39+
/// <param name="revalidation">The revalidation to update.</param>
40+
/// <returns>A task that completes once the revalidation has been updated.</returns>
41+
Task MarkRevalidationAsEnqueuedAsync(PackageRevalidation revalidation);
3042
}
3143
}

0 commit comments

Comments
 (0)