Skip to content
This repository was archived by the owner on Jul 30, 2024. It is now read-only.

Commit 0d7b59a

Browse files
loic-sharmaloic-sharma
authored
Merge pull request #538 from NuGet/dev
[ReleasePrep][2018.08.16]RI of dev into master
2 parents 6a34f74 + 262cd16 commit 0d7b59a

54 files changed

Lines changed: 2448 additions & 537 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

src/NuGet.Services.Revalidate/Services/IPackageRevalidationInserter.cs renamed to src/NuGet.Services.Revalidate/Initialization/IPackageRevalidationInserter.cs

File renamed without changes.

src/NuGet.Services.Revalidate/Services/PackageRevalidationInserter.cs renamed to src/NuGet.Services.Revalidate/Initialization/PackageRevalidationInserter.cs

File renamed without changes.

‎src/NuGet.Services.Revalidate/Job.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
143143
services.AddTransient<IHealthService, HealthService>();
144144
services.AddTransient<IRevalidationQueue, RevalidationQueue>();
145145
services.AddTransient<IRevalidationService, RevalidationService>();
146+
services.AddTransient<IRevalidationStarter, RevalidationStarter>();
146147
services.AddTransient<IRevalidationThrottler, RevalidationThrottler>();
147148
services.AddTransient<ISingletonService, SingletonService>();
148149

‎src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,20 +58,22 @@
5858
<Compile Include="Services\HealthService.cs" />
5959
<Compile Include="Services\IGalleryService.cs" />
6060
<Compile Include="Services\IHealthService.cs" />
61-
<Compile Include="Services\IPackageRevalidationInserter.cs" />
61+
<Compile Include="Initialization\IPackageRevalidationInserter.cs" />
6262
<Compile Include="Services\IRevalidationQueue.cs" />
6363
<Compile Include="Services\IRevalidationJobStateService.cs" />
6464
<Compile Include="Services\IPackageRevalidationStateService.cs" />
6565
<Compile Include="Services\IRevalidationService.cs" />
66+
<Compile Include="Services\IRevalidationStarter.cs" />
6667
<Compile Include="Services\IRevalidationThrottler.cs" />
6768
<Compile Include="Services\ISingletonService.cs" />
68-
<Compile Include="Services\PackageRevalidationInserter.cs" />
69+
<Compile Include="Initialization\PackageRevalidationInserter.cs" />
6970
<Compile Include="Services\RevalidationOperation.cs" />
7071
<Compile Include="Services\RevalidationQueue.cs" />
7172
<Compile Include="Services\RevalidationResult.cs" />
7273
<Compile Include="Services\RevalidationService.cs" />
7374
<Compile Include="Services\RevalidationJobStateService.cs" />
7475
<Compile Include="Services\PackageRevalidationStateService.cs" />
76+
<Compile Include="Services\RevalidationStarter.cs" />
7577
<Compile Include="Services\RevalidationThrottler.cs" />
7678
<Compile Include="Services\SingletonService.cs" />
7779
<Compile Include="Services\TelemetryService.cs" />

‎src/NuGet.Services.Revalidate/Services/IHealthService.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@ namespace NuGet.Services.Revalidate
88
public interface IHealthService
99
{
1010
/// <summary>
11-
/// Determine whether the NuGet service is healthy.
11+
/// Determine whether NuGet's ingestion pipeline is healthy.
1212
/// </summary>
13-
/// <returns>Whether the NuGet service is healthy.</returns>
13+
/// <returns>Whether the NuGet ingestion pipeline is healthy.</returns>
1414
Task<bool> IsHealthyAsync();
1515
}
1616
}

‎src/NuGet.Services.Revalidate/Services/IRevalidationService.cs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,15 @@
55

66
namespace NuGet.Services.Revalidate
77
{
8+
/// <summary>
9+
/// Starts revalidations until there are no more pending revalidations.
10+
/// </summary>
811
public interface IRevalidationService
912
{
13+
/// <summary>
14+
/// Start revalidations until there are no more pending revalidations.
15+
/// </summary>
16+
/// <returns></returns>
1017
Task RunAsync();
11-
12-
Task<RevalidationResult> StartNextRevalidationAsync();
1318
}
1419
}

‎src/NuGet.Services.Revalidate/Services/IRevalidationStarter.cs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System.Threading.Tasks;
5+
6+
namespace NuGet.Services.Revalidate
7+
{
8+
/// <summary>
9+
/// The service used to try to start revalidations.
10+
/// </summary>
11+
public interface IRevalidationStarter
12+
{
13+
/// <summary>
14+
/// Attempt to start a single revalidation. This may choose not to enqueue
15+
/// a revalidation if:
16+
///
17+
/// 1. The ingestion pipeline appears unhealthy
18+
/// 2. The ingestion pipeline is too active
19+
/// 3. The revalidation job has been killswitched
20+
/// 4. A revalidation could not be found at this time
21+
/// </summary>
22+
/// <returns>The result of the revalidation attempt.</returns>
23+
Task<RevalidationResult> StartNextRevalidationAsync();
24+
}
25+
}

‎src/NuGet.Services.Revalidate/Services/RevalidationService.cs‎

Lines changed: 8 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -4,43 +4,31 @@
44
using System;
55
using System.Diagnostics;
66
using System.Threading.Tasks;
7+
using Microsoft.Extensions.DependencyInjection;
78
using Microsoft.Extensions.Logging;
8-
using NuGet.Services.Validation;
99

1010
namespace NuGet.Services.Revalidate
1111
{
1212
public class RevalidationService : IRevalidationService
1313
{
1414
private readonly IRevalidationJobStateService _jobState;
15-
private readonly IPackageRevalidationStateService _packageState;
16-
private readonly ISingletonService _singletonService;
1715
private readonly IRevalidationThrottler _throttler;
18-
private readonly IHealthService _healthService;
19-
private readonly IRevalidationQueue _revalidationQueue;
20-
private readonly IPackageValidationEnqueuer _validationEnqueuer;
16+
private readonly IServiceScopeFactory _scopeFactory;
2117
private readonly RevalidationConfiguration _config;
2218
private readonly ITelemetryService _telemetryService;
2319
private readonly ILogger<RevalidationService> _logger;
2420

2521
public RevalidationService(
2622
IRevalidationJobStateService jobState,
27-
IPackageRevalidationStateService packageState,
28-
ISingletonService singletonService,
2923
IRevalidationThrottler throttler,
30-
IHealthService healthService,
31-
IRevalidationQueue revalidationQueue,
32-
IPackageValidationEnqueuer validationEnqueuer,
24+
IServiceScopeFactory scopeFactory,
3325
RevalidationConfiguration config,
3426
ITelemetryService telemetryService,
3527
ILogger<RevalidationService> logger)
3628
{
3729
_jobState = jobState ?? throw new ArgumentNullException(nameof(jobState));
38-
_packageState = packageState ?? throw new ArgumentNullException(nameof(packageState));
39-
_singletonService = singletonService ?? throw new ArgumentNullException(nameof(singletonService));
4030
_throttler = throttler ?? throw new ArgumentNullException(nameof(throttler));
41-
_healthService = healthService ?? throw new ArgumentNullException(nameof(healthService));
42-
_revalidationQueue = revalidationQueue ?? throw new ArgumentNullException(nameof(revalidationQueue));
43-
_validationEnqueuer = validationEnqueuer ?? throw new ArgumentNullException(nameof(validationEnqueuer));
31+
_scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory));
4432
_config = config ?? throw new ArgumentNullException(nameof(config));
4533
_telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
4634
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
@@ -91,110 +79,18 @@ public async Task RunAsync()
9179
_logger.LogInformation("Finished running after {ElapsedTime}", runTime.Elapsed);
9280
}
9381

94-
public async Task<RevalidationResult> StartNextRevalidationAsync()
82+
private async Task<RevalidationResult> StartNextRevalidationAsync()
9583
{
9684
using (var operation = _telemetryService.TrackStartNextRevalidationOperation())
85+
using (var scope = _scopeFactory.CreateScope())
9786
{
98-
var result = await StartNextRevalidationInternalAsync();
87+
var starter = scope.ServiceProvider.GetRequiredService<IRevalidationStarter>();
88+
var result = await starter.StartNextRevalidationAsync();
9989

10090
operation.Properties.Result = result;
10191

10292
return result;
10393
}
10494
}
105-
106-
public async Task<RevalidationResult> StartNextRevalidationInternalAsync()
107-
{
108-
try
109-
{
110-
// Don't start a revalidation if the job has been deactivated, if the ingestion pipeline is unhealthy,
111-
// or if we have reached our quota of desired revalidations.
112-
var checkResult = await CanStartRevalidationAsync();
113-
if (checkResult != null)
114-
{
115-
_logger.LogInformation(
116-
"Detected that a revalidation should not be started due to result {Result}",
117-
checkResult.Value);
118-
119-
return checkResult.Value;
120-
}
121-
122-
// Everything is in tip-top shape! Increase the throttling quota and start the next revalidation.
123-
await _jobState.IncreaseDesiredPackageEventRateAsync();
124-
125-
var revalidation = await _revalidationQueue.NextOrNullAsync();
126-
if (revalidation == null)
127-
{
128-
_logger.LogInformation("Could not find a package to revalidate at this time, retry later...");
129-
130-
return RevalidationResult.RetryLater;
131-
}
132-
133-
return await StartRevalidationAsync(revalidation);
134-
}
135-
catch (Exception e)
136-
{
137-
_logger.LogError(0, e, "Failed to start next validation due to exception, retry later...");
138-
139-
return RevalidationResult.RetryLater;
140-
}
141-
}
142-
143-
private async Task<RevalidationResult?> CanStartRevalidationAsync()
144-
{
145-
if (!await _singletonService.IsSingletonAsync())
146-
{
147-
_logger.LogCritical("Detected another instance of the revalidate job, cancelling revalidations!");
148-
149-
return RevalidationResult.UnrecoverableError;
150-
}
151-
152-
if (await _jobState.IsKillswitchActiveAsync())
153-
{
154-
_logger.LogWarning("Revalidation killswitch has been activated, retry later...");
155-
156-
return RevalidationResult.RetryLater;
157-
}
158-
159-
if (await _throttler.IsThrottledAsync())
160-
{
161-
_logger.LogInformation("Revalidations have reached the desired event rate, retry later...");
162-
163-
return RevalidationResult.RetryLater;
164-
}
165-
166-
if (!await _healthService.IsHealthyAsync())
167-
{
168-
_logger.LogWarning("Service appears to be unhealthy, resetting the desired package event rate. Retry later...");
169-
170-
await _jobState.ResetDesiredPackageEventRateAsync();
171-
172-
return RevalidationResult.RetryLater;
173-
}
174-
175-
if (await _jobState.IsKillswitchActiveAsync())
176-
{
177-
_logger.LogWarning("Revalidation killswitch has been activated after the throttle and health check, retry later...");
178-
179-
return RevalidationResult.RetryLater;
180-
}
181-
182-
return null;
183-
}
184-
185-
private async Task<RevalidationResult> StartRevalidationAsync(PackageRevalidation revalidation)
186-
{
187-
var message = new PackageValidationMessageData(
188-
revalidation.PackageId,
189-
revalidation.PackageNormalizedVersion,
190-
revalidation.ValidationTrackingId.Value);
191-
192-
await _validationEnqueuer.StartValidationAsync(message);
193-
await _packageState.MarkPackageRevalidationAsEnqueuedAsync(revalidation);
194-
195-
_telemetryService.TrackPackageRevalidationStarted(revalidation.PackageId, revalidation.PackageNormalizedVersion);
196-
197-
return RevalidationResult.RevalidationEnqueued;
198-
}
19995
}
20096
}

‎src/NuGet.Services.Revalidate/Services/RevalidationStarter.cs‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
using System;
2+
using System.Threading.Tasks;
3+
using Microsoft.Extensions.Logging;
4+
using NuGet.Services.Validation;
5+
6+
namespace NuGet.Services.Revalidate
7+
{
8+
public class RevalidationStarter : IRevalidationStarter
9+
{
10+
private readonly IRevalidationJobStateService _jobState;
11+
private readonly IPackageRevalidationStateService _packageState;
12+
private readonly ISingletonService _singletonService;
13+
private readonly IRevalidationThrottler _throttler;
14+
private readonly IHealthService _healthService;
15+
private readonly IRevalidationQueue _revalidationQueue;
16+
private readonly IPackageValidationEnqueuer _validationEnqueuer;
17+
private readonly ITelemetryService _telemetryService;
18+
private readonly ILogger<RevalidationStarter> _logger;
19+
20+
public RevalidationStarter(
21+
IRevalidationJobStateService jobState,
22+
IPackageRevalidationStateService packageState,
23+
ISingletonService singletonService,
24+
IRevalidationThrottler throttler,
25+
IHealthService healthService,
26+
IRevalidationQueue revalidationQueue,
27+
IPackageValidationEnqueuer validationEnqueuer,
28+
ITelemetryService telemetryService,
29+
ILogger<RevalidationStarter> logger)
30+
{
31+
_jobState = jobState ?? throw new ArgumentNullException(nameof(jobState));
32+
_packageState = packageState ?? throw new ArgumentNullException(nameof(packageState));
33+
_singletonService = singletonService ?? throw new ArgumentNullException(nameof(singletonService));
34+
_throttler = throttler ?? throw new ArgumentNullException(nameof(throttler));
35+
_healthService = healthService ?? throw new ArgumentNullException(nameof(healthService));
36+
_revalidationQueue = revalidationQueue ?? throw new ArgumentNullException(nameof(revalidationQueue));
37+
_validationEnqueuer = validationEnqueuer ?? throw new ArgumentNullException(nameof(validationEnqueuer));
38+
_telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
39+
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
40+
}
41+
42+
public async Task<RevalidationResult> StartNextRevalidationAsync()
43+
{
44+
try
45+
{
46+
// Don't start a revalidation if the job has been deactivated, if the ingestion pipeline is unhealthy,
47+
// or if we have reached our quota of desired revalidations.
48+
var checkResult = await CanStartRevalidationAsync();
49+
if (checkResult != null)
50+
{
51+
_logger.LogInformation(
52+
"Detected that a revalidation should not be started due to result {Result}",
53+
checkResult.Value);
54+
55+
return checkResult.Value;
56+
}
57+
58+
// Everything is in tip-top shape! Increase the throttling quota and start the next revalidation.
59+
await _jobState.IncreaseDesiredPackageEventRateAsync();
60+
61+
var revalidation = await _revalidationQueue.NextOrNullAsync();
62+
if (revalidation == null)
63+
{
64+
_logger.LogInformation("Could not find a package to revalidate at this time, retry later...");
65+
66+
return RevalidationResult.RetryLater;
67+
}
68+
69+
return await StartRevalidationAsync(revalidation);
70+
}
71+
catch (Exception e)
72+
{
73+
_logger.LogError(0, e, "Failed to start next validation due to exception, retry later...");
74+
75+
return RevalidationResult.RetryLater;
76+
}
77+
}
78+
79+
private async Task<RevalidationResult?> CanStartRevalidationAsync()
80+
{
81+
if (!await _singletonService.IsSingletonAsync())
82+
{
83+
_logger.LogCritical("Detected another instance of the revalidate job, cancelling revalidations!");
84+
85+
return RevalidationResult.UnrecoverableError;
86+
}
87+
88+
if (await _jobState.IsKillswitchActiveAsync())
89+
{
90+
_logger.LogWarning("Revalidation killswitch has been activated, retry later...");
91+
92+
return RevalidationResult.RetryLater;
93+
}
94+
95+
if (await _throttler.IsThrottledAsync())
96+
{
97+
_logger.LogInformation("Revalidations have reached the desired event rate, retry later...");
98+
99+
return RevalidationResult.RetryLater;
100+
}
101+
102+
if (!await _healthService.IsHealthyAsync())
103+
{
104+
_logger.LogWarning("Service appears to be unhealthy, resetting the desired package event rate. Retry later...");
105+
106+
await _jobState.ResetDesiredPackageEventRateAsync();
107+
108+
return RevalidationResult.RetryLater;
109+
}
110+
111+
if (await _jobState.IsKillswitchActiveAsync())
112+
{
113+
_logger.LogWarning("Revalidation killswitch has been activated after the throttle and health check, retry later...");
114+
115+
return RevalidationResult.RetryLater;
116+
}
117+
118+
return null;
119+
}
120+
121+
private async Task<RevalidationResult> StartRevalidationAsync(PackageRevalidation revalidation)
122+
{
123+
var message = new PackageValidationMessageData(
124+
revalidation.PackageId,
125+
revalidation.PackageNormalizedVersion,
126+
revalidation.ValidationTrackingId.Value);
127+
128+
await _validationEnqueuer.StartValidationAsync(message);
129+
await _packageState.MarkPackageRevalidationAsEnqueuedAsync(revalidation);
130+
131+
_telemetryService.TrackPackageRevalidationStarted(revalidation.PackageId, revalidation.PackageNormalizedVersion);
132+
133+
return RevalidationResult.RevalidationEnqueued;
134+
}
135+
}
136+
}

0 commit comments

Comments
 (0)