Skip to content
This repository was archived by the owner on Jul 30, 2024. It is now read-only.

Commit ff0e2cb

Browse files
loic-sharmaloic-sharma
authored
[Revalidate] Slow down based off the Gallery's event rate (#499)
This makes the revalidate job slow down if it detects package events on the Gallery. Part of: https://github.com/NuGet/Engineering/issues/1441
1 parent 6de44bb commit ff0e2cb

11 files changed

Lines changed: 190 additions & 28 deletions

File tree

‎src/NuGet.Services.Revalidate/Configuration/ApplicationInsightsConfiguration.cs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
namespace NuGet.Services.Revalidate
5+
{
6+
/// <summary>
7+
/// The configuration needed to query an Application Insights account using
8+
/// the REST endpoints.
9+
/// </summary>
10+
public class ApplicationInsightsConfiguration
11+
{
12+
/// <summary>
13+
/// The Application Insights account identifier.
14+
/// </summary>
15+
public string AppId { get; set; }
16+
17+
/// <summary>
18+
/// The API Key used to access the Application Insights account.
19+
/// </summary>
20+
public string ApiKey { get; set; }
21+
}
22+
}

‎src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,11 @@ public class RevalidationConfiguration
4040
/// </summary>
4141
public HealthConfiguration Health { get; set; }
4242

43+
/// <summary>
44+
/// The configurations to authenticate to Application Insight's REST endpoints.
45+
/// </summary>
46+
public ApplicationInsightsConfiguration AppInsights { get; set; }
47+
4348
/// <summary>
4449
/// The configurations used by the in-memory queue of revalidations to start.
4550
/// </summary>

‎src/NuGet.Services.Revalidate/Job.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
124124
services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value);
125125
services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.Initialization);
126126
services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.Health);
127+
services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.AppInsights);
127128
services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.Queue);
128129

129130
services.AddScoped<IGalleryContext>(provider =>
@@ -139,13 +140,14 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
139140

140141
services.AddTransient<IPackageRevalidationStateService, PackageRevalidationStateService>();
141142
services.AddTransient<IRevalidationJobStateService, RevalidationJobStateService>();
142-
services.AddTransient<NuGetGallery.IRevalidationStateService, NuGetGallery.RevalidationStateService>();
143+
services.AddTransient<IRevalidationStateService, RevalidationStateService>();
143144

144145
// Initialization
145146
services.AddTransient<IPackageFinder, PackageFinder>();
146147
services.AddTransient<InitializationManager>();
147148

148149
// Revalidation
150+
services.AddTransient<IGalleryService, GalleryService>();
149151
services.AddTransient<IHealthService, HealthService>();
150152
services.AddTransient<IRevalidationQueue, RevalidationQueue>();
151153
services.AddTransient<IRevalidationService, RevalidationService>();

‎src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
<ItemGroup>
3737
<Reference Include="System" />
3838
<Reference Include="System.Core" />
39+
<Reference Include="System.Web" />
3940
<Reference Include="System.Xml.Linq" />
4041
<Reference Include="System.Data.DataSetExtensions" />
4142
<Reference Include="Microsoft.CSharp" />
@@ -44,6 +45,7 @@
4445
</ItemGroup>
4546
<ItemGroup>
4647
<Compile Include="Configuration\HealthConfiguration.cs" />
48+
<Compile Include="Configuration\ApplicationInsightsConfiguration.cs" />
4749
<Compile Include="Configuration\InitializationConfiguration.cs" />
4850
<Compile Include="Configuration\RevalidationQueueConfiguration.cs" />
4951
<Compile Include="Extensions\IEnumerableExtensions.cs" />
@@ -52,7 +54,9 @@
5254
<Compile Include="Initialization\IPackageFinder.cs" />
5355
<Compile Include="Initialization\PackageFinder.cs" />
5456
<Compile Include="Initialization\PackageRegistrationInformation.cs" />
57+
<Compile Include="Services\GalleryService.cs" />
5558
<Compile Include="Services\HealthService.cs" />
59+
<Compile Include="Services\IGalleryService.cs" />
5660
<Compile Include="Services\IHealthService.cs" />
5761
<Compile Include="Services\IRevalidationQueue.cs" />
5862
<Compile Include="Services\IRevalidationJobStateService.cs" />

‎src/NuGet.Services.Revalidate/Services/GalleryService.cs‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Net.Http;
6+
using System.Threading.Tasks;
7+
using System.Web;
8+
using Microsoft.Extensions.Logging;
9+
using Newtonsoft.Json;
10+
11+
namespace NuGet.Services.Revalidate
12+
{
13+
public class GalleryService : IGalleryService
14+
{
15+
private static readonly string GalleryEventsQuery = HttpUtility.UrlPathEncode(
16+
"customMetrics | " +
17+
"where name == \"PackagePush\" or name == \"PackageUnlisted\" or name == \"PackageListed\" | " +
18+
"summarize sum(value)");
19+
20+
private readonly HttpClient _httpClient;
21+
private readonly ApplicationInsightsConfiguration _appInsightsConfig;
22+
private readonly ILogger<GalleryService> _logger;
23+
24+
public GalleryService(
25+
HttpClient httpClient,
26+
ApplicationInsightsConfiguration appInsightsConfig,
27+
ILogger<GalleryService> logger)
28+
{
29+
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
30+
_appInsightsConfig = appInsightsConfig ?? throw new ArgumentNullException(nameof(appInsightsConfig));
31+
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
32+
}
33+
34+
public async Task<int> CountEventsInPastHourAsync()
35+
{
36+
try
37+
{
38+
using (var request = new HttpRequestMessage())
39+
{
40+
request.RequestUri = new Uri($"https://api.applicationinsights.io/v1/apps/{_appInsightsConfig.AppId}/query?timespan=PT1H&query={GalleryEventsQuery}");
41+
request.Method = HttpMethod.Get;
42+
43+
request.Headers.Add("x-api-key", _appInsightsConfig.ApiKey);
44+
45+
using (var response = await _httpClient.SendAsync(request))
46+
{
47+
response.EnsureSuccessStatusCode();
48+
49+
var json = await response.Content.ReadAsStringAsync();
50+
var data = JsonConvert.DeserializeObject<QueryResult>(json);
51+
52+
if (data?.Tables?.Length != 1 ||
53+
data.Tables[0]?.Rows?.Length != 1 ||
54+
data.Tables[0].Rows[0]?.Length != 1)
55+
{
56+
throw new InvalidOperationException("Malformed response content");
57+
}
58+
59+
// Get the first row's first column's value.
60+
return data.Tables[0].Rows[0][0];
61+
}
62+
}
63+
}
64+
catch (Exception e)
65+
{
66+
_logger.LogError(0, e, "Exception thrown when getting the Gallery's package event rate.");
67+
68+
throw new InvalidOperationException("Exception thrown when getting the Gallery's package event rate.", e);
69+
}
70+
}
71+
72+
private class QueryResult
73+
{
74+
public QueryTable[] Tables { get; set; }
75+
}
76+
77+
private class QueryTable
78+
{
79+
public int[][] Rows { get; set; }
80+
}
81+
}
82+
}

‎src/NuGet.Services.Revalidate/Services/IGalleryService.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System.Threading.Tasks;
5+
6+
namespace NuGet.Services.Revalidate
7+
{
8+
public interface IGalleryService
9+
{
10+
/// <summary>
11+
/// Count the number of gallery events (package pushes, listing, and unlisting) in the past hour.
12+
/// </summary>
13+
/// <returns>The number of gallery events in the past hour.</returns>
14+
Task<int> CountEventsInPastHourAsync();
15+
}
16+
}

‎src/NuGet.Services.Revalidate/Services/RevalidationThrottler.cs‎

Lines changed: 24 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -11,30 +11,52 @@ public class RevalidationThrottler : IRevalidationThrottler
1111
{
1212
private readonly IRevalidationJobStateService _jobState;
1313
private readonly IPackageRevalidationStateService _packageState;
14+
private readonly IGalleryService _gallery;
1415
private readonly RevalidationConfiguration _config;
1516
private readonly ILogger<RevalidationThrottler> _logger;
1617

1718
public RevalidationThrottler(
1819
IRevalidationJobStateService jobState,
1920
IPackageRevalidationStateService packageState,
21+
IGalleryService gallery,
2022
RevalidationConfiguration config,
2123
ILogger<RevalidationThrottler> logger)
2224
{
2325
_jobState = jobState ?? throw new ArgumentNullException(nameof(jobState));
2426
_packageState = packageState ?? throw new ArgumentNullException(nameof(packageState));
27+
_gallery = gallery ?? throw new ArgumentNullException(nameof(gallery));
2528
_config = config ?? throw new ArgumentNullException(nameof(config));
2629
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
2730
}
2831

2932
public async Task<bool> IsThrottledAsync()
3033
{
3134
var desiredRate = await _jobState.GetDesiredPackageEventRateAsync();
32-
var recentGalleryEvents = await CountGalleryEventsInPastHourAsync();
35+
var recentGalleryEvents = await _gallery.CountEventsInPastHourAsync();
3336
var recentRevalidations = await _packageState.CountRevalidationsEnqueuedInPastHourAsync();
3437

3538
var revalidationQuota = desiredRate - recentRevalidations - recentGalleryEvents;
3639

37-
return (revalidationQuota <= 0);
40+
if (revalidationQuota <= 0)
41+
{
42+
_logger.LogInformation(
43+
"Throttling revalidations. Desired rate: {DesiredRate}, gallery events: {GalleryEvents}, recent revalidations: {RecentRevalidations}",
44+
desiredRate,
45+
recentGalleryEvents,
46+
recentRevalidations);
47+
48+
return true;
49+
}
50+
else
51+
{
52+
_logger.LogInformation(
53+
"Allowing revalidations. Desired rate: {DesiredRate}, gallery events: {GalleryEvents}, recent revalidations: {RecentRevalidations}",
54+
desiredRate,
55+
recentGalleryEvents,
56+
recentRevalidations);
57+
58+
return false;
59+
}
3860
}
3961

4062
public async Task DelayUntilNextRevalidationAsync()
@@ -55,22 +77,5 @@ public async Task DelayUntilRevalidationRetryAsync()
5577

5678
await Task.Delay(_config.RetryLaterSleep);
5779
}
58-
59-
private Task<int> CountGalleryEventsInPastHourAsync()
60-
{
61-
// TODO: Count the number of package pushes, lists, and unlists.
62-
// Run this AI query:
63-
//
64-
// customMetrics | where name == "PackagePush" or name == "PackageUnlisted" or name == "PackageListed" | summarize sum(value)
65-
//
66-
// Using this HTTP request:
67-
//
68-
// GET /v1/apps/46f13c7d-635f-42c3-8120-593edeaad426/query?timespan=P1D&query=customMetrics%20%7C%20where%20name%20%3D%3D%20%22PackagePush%22%20or%20name%20%3D%3D%20%22PackageUnlisted%22%20or%20name%20%3D%3D%20%22PackageListed%22%20%7C%20summarize%20sum(value)%20 HTTP/1.1
69-
// Host: api.applicationinsights.io
70-
// x-api-key: my-super-secret-api-key
71-
//
72-
// See: https://dev.applicationinsights.io/quickstart
73-
return Task.FromResult(0);
74-
}
7580
}
7681
}

‎src/NuGet.Services.Revalidate/Settings/dev.json‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,17 +19,17 @@
1919
"MinPackageEventRate": 120,
2020
"MaxPackageEventRate": 500,
2121

22+
"AppInsights": {
23+
"AppId": "46f13c7d-635f-42c3-8120-593edeaad426",
24+
"ApiKey": "$$Dev-ApplicationInsights-ApiKey-Gallery-RevalidationJob$$"
25+
},
26+
2227
"Queue": {
2328
"MaximumAttempts": 5,
2429
"SleepBetweenAttempts": "00:05:00"
2530
}
2631
},
2732

28-
"Storage": {
29-
"ConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$",
30-
"Container": "revalidations"
31-
},
32-
3333
"GalleryDb": {
3434
"ConnectionString": "Data Source=tcp:#{Jobs.validation.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;Integrated Security=False;User ID=$$Dev-GalleryDBReadOnly-UserName$$;Password=$$Dev-GalleryDBReadOnly-Password$$;Connect Timeout=30;Encrypt=True"
3535
},
@@ -44,6 +44,8 @@
4444
"TopicPath": "validation"
4545
},
4646

47+
"PackageDownloadTimeout": "00:10:00",
48+
4749
"KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
4850
"KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
4951
"KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",

‎src/NuGet.Services.Revalidate/Settings/int.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,11 @@
1919
"MinPackageEventRate": 120,
2020
"MaxPackageEventRate": 500,
2121

22+
"AppInsights": {
23+
"AppId": "718e0c81-9132-4bf2-b24b-aa625dafd800",
24+
"ApiKey": "$$Int-ApplicationInsights-ApiKey-Gallery-RevalidationJob$$"
25+
},
26+
2227
"Queue": {
2328
"MaximumAttempts": 5,
2429
"SleepBetweenAttempts": "00:05:00"
@@ -39,6 +44,8 @@
3944
"TopicPath": "validation"
4045
},
4146

47+
"PackageDownloadTimeout": "00:10:00",
48+
4249
"KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
4350
"KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
4451
"KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",

‎src/NuGet.Services.Revalidate/Settings/prod.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,11 @@
1919
"MinPackageEventRate": 120,
2020
"MaxPackageEventRate": 500,
2121

22+
"AppInsights": {
23+
"AppId": "338f6804-b1a9-4fe3-bba7-c93064e7ae7b",
24+
"ApiKey": "$$Prod-ApplicationInsights-ApiKey-Gallery-RevalidationJob$$"
25+
},
26+
2227
"Queue": {
2328
"MaximumAttempts": 5,
2429
"SleepBetweenAttempts": "00:05:00"
@@ -39,6 +44,8 @@
3944
"TopicPath": "validation"
4045
},
4146

47+
"PackageDownloadTimeout": "00:10:00",
48+
4249
"KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
4350
"KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
4451
"KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",

0 commit comments

Comments
 (0)