Skip to content

Commit 73b9012

Browse files
ryuyulyndaidaiidrewgillies
authored
[Stats]Enable Alternate storage for downloads.v1.json (#8380)
* Add FeatureFlag for secondary stats. * Updating Naming. * Add new Configs. Update downloadcountservice suport alternate endpoint. * Update DependencyInjection to support alternate configuration. * Fix tests for build * Add setting for alternate stats storage * Stub new config to account deleter. * read out warning message (#8382) * Add icons to project to ensure bundling (#8385) * Add FeatureFlag for secondary stats. * Updating Naming. * Add new Configs. Update downloadcountservice suport alternate endpoint. * Update DependencyInjection to support alternate configuration. * Fix tests for build * Add setting for alternate stats storage * Stub new config to account deleter. Co-authored-by: lyndaidaii <[email protected]> Co-authored-by: Drew Gillies <[email protected]>
1 parent 35ca7d7 commit 73b9012

12 files changed

Lines changed: 148 additions & 37 deletions

File tree

‎src/AccountDeleter/Configuration/GalleryConfiguration.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ public string SiteRoot
3232
public string AzureStorage_Packages_ConnectionString { get => throw new NotImplementedException(); set => throw new NotImplementedException(); }
3333
public string AzureStorage_FlatContainer_ConnectionString { get => throw new NotImplementedException(); set => throw new NotImplementedException(); }
3434
public string AzureStorage_Statistics_ConnectionString { get => throw new NotImplementedException(); set => throw new NotImplementedException(); }
35+
public string AzureStorage_Statistics_ConnectionString_Alternate { get => throw new NotImplementedException(); set => throw new NotImplementedException(); }
3536
public string AzureStorage_Uploads_ConnectionString { get => throw new NotImplementedException(); set => throw new NotImplementedException(); }
3637
public string AzureStorage_Revalidation_ConnectionString { get => throw new NotImplementedException(); set => throw new NotImplementedException(); }
3738
public bool AzureStorageReadAccessGeoRedundant { get => throw new NotImplementedException(); set => throw new NotImplementedException(); }

‎src/NuGetGallery.Services/Configuration/AppConfiguration.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ public class AppConfiguration : IAppConfiguration
6060
[DisplayName("AzureStorage.Statistics.ConnectionString")]
6161
public string AzureStorage_Statistics_ConnectionString { get; set; }
6262

63+
[DisplayName("AzureStorage.Statistics.ConnectionString.Alternate")]
64+
public string AzureStorage_Statistics_ConnectionString_Alternate { get; set; }
65+
6366
[DisplayName("AzureStorage.Uploads.ConnectionString")]
6467
public string AzureStorage_Uploads_ConnectionString { get; set; }
6568

‎src/NuGetGallery.Services/Configuration/FeatureFlagService.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ public class FeatureFlagService : IFeatureFlagService
1515
private const string GalleryPrefix = "NuGetGallery.";
1616

1717
private const string ABTestingFlightName = GalleryPrefix + "ABTesting";
18+
private const string AlternateStatisticsSourceFeatureName = GalleryPrefix + "AlternateStatisticsSource";
1819
private const string AsyncAccountDeleteFeatureName = GalleryPrefix + "AsyncAccountDelete";
1920
private const string SelfServiceAccountDeleteFeatureName = GalleryPrefix + "SelfServiceAccountDelete";
2021
private const string EmbeddedIconFlightName = GalleryPrefix + "EmbeddedIcons";
@@ -68,6 +69,11 @@ public FeatureFlagService(IFeatureFlagClient client)
6869
_client = client ?? throw new ArgumentNullException(nameof(client));
6970
}
7071

72+
public bool IsAlternateStatisticsSourceEnabled()
73+
{
74+
return _client.IsEnabled(AlternateStatisticsSourceFeatureName, defaultValue: false);
75+
}
76+
7177
public bool IsAsyncAccountDeleteEnabled()
7278
{
7379
return _client.IsEnabled(AsyncAccountDeleteFeatureName, defaultValue: false);

‎src/NuGetGallery.Services/Configuration/IAppConfiguration.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,11 @@ public interface IAppConfiguration : IMessageServiceConfiguration
7575
/// </summary>
7676
string AzureStorage_Statistics_ConnectionString { get; set; }
7777

78+
/// <summary>
79+
/// The Azure Storage connection string used for statistics. Secondary
80+
/// </summary>
81+
string AzureStorage_Statistics_ConnectionString_Alternate { get; set; }
82+
7883
/// <summary>
7984
/// The Azure Storage connection string used for package uploads, before publishing.
8085
/// </summary>

‎src/NuGetGallery.Services/Configuration/IBlobStorageConfiguration.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
namespace NuGetGallery.Services
5+
{
6+
public interface IBlobStorageConfiguration
7+
{
8+
string ConnectionString { get; }
9+
bool ReadAccessGeoRedundant { get; }
10+
}
11+
}

‎src/NuGetGallery.Services/Configuration/IFeatureFlagService.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,12 @@ namespace NuGetGallery
88
{
99
public interface IFeatureFlagService
1010
{
11+
/// <summary>
12+
/// Whether downloads.v1.json hould be pulled from primary or secondary location.
13+
/// If true, the secondary location will be used to download downloads.v1.json.
14+
/// </summary>
15+
bool IsAlternateStatisticsSourceEnabled();
16+
1117
/// <summary>
1218
/// Whether account deletes are performed asychronously or not.
1319
/// If true, account deletes will be attempted to be performed asychronously

‎src/NuGetGallery.Services/Configuration/SimpleBlobStorageConfiguration.cs‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
namespace NuGetGallery.Services
5+
{
6+
public class SimpleBlobStorageConfiguration : IBlobStorageConfiguration
7+
{
8+
public string ConnectionString
9+
{
10+
get;
11+
private set;
12+
}
13+
14+
public bool ReadAccessGeoRedundant
15+
{
16+
get;
17+
private set;
18+
}
19+
20+
public SimpleBlobStorageConfiguration(string connectionString, bool readAccessGeoRedundant)
21+
{
22+
ConnectionString = connectionString;
23+
ReadAccessGeoRedundant = readAccessGeoRedundant;
24+
}
25+
}
26+
}

‎src/NuGetGallery.Services/NuGetGallery.Services.csproj‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@
109109
<Compile Include="Configuration\FeatureFlagService.cs" />
110110
<Compile Include="Configuration\IABTestConfiguration.cs" />
111111
<Compile Include="Configuration\IAppConfiguration.cs" />
112+
<Compile Include="Configuration\IBlobStorageConfiguration.cs" />
112113
<Compile Include="Configuration\ICacheConfiguration.cs" />
113114
<Compile Include="Configuration\ICertificatesConfiguration.cs" />
114115
<Compile Include="Configuration\IFeatureFlagService.cs" />
@@ -127,6 +128,7 @@
127128
<Compile Include="Configuration\SecretReader\EmptySecretReaderFactory.cs" />
128129
<Compile Include="Configuration\SecretReader\SecretReaderFactory.cs" />
129130
<Compile Include="Configuration\ServiceBusConfiguration.cs" />
131+
<Compile Include="Configuration\SimpleBlobStorageConfiguration.cs" />
130132
<Compile Include="Configuration\SymbolsConfiguration.cs" />
131133
<Compile Include="Configuration\TyposquattingConfiguration.cs" />
132134
<Compile Include="Diagnostics\DiagnosticsService.cs" />

‎src/NuGetGallery/App_Start/DefaultDependenciesModule.cs‎

Lines changed: 49 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,9 @@ public static class BindingKeys
6767
public const string AsyncDeleteAccountName = "AsyncDeleteAccountService";
6868
public const string SyncDeleteAccountName = "SyncDeleteAccountService";
6969

70+
public const string PrimaryStatisticsKey = "PrimaryStatisticsKey";
71+
public const string AlternateStatisticsKey = "AlternateStatisticsKey";
72+
7073
public const string AccountDeleterTopic = "AccountDeleterBindingKey";
7174
public const string PackageValidationTopic = "PackageValidationBindingKey";
7275
public const string SymbolsPackageValidationTopic = "SymbolsPackageValidationBindingKey";
@@ -698,6 +701,51 @@ private static void RegisterDeleteAccountService(ContainerBuilder builder, Confi
698701
}
699702
}
700703

704+
private static void RegisterStatisticsServices(ContainerBuilder builder, IGalleryConfigurationService configuration, ITelemetryService telemetryService)
705+
{
706+
// when running on Windows Azure, we use a back-end job to calculate stats totals and store in the blobs
707+
builder.RegisterInstance(new JsonAggregateStatsService(configuration.Current.AzureStorage_Statistics_ConnectionString, configuration.Current.AzureStorageReadAccessGeoRedundant))
708+
.AsSelf()
709+
.As<IAggregateStatsService>()
710+
.SingleInstance();
711+
712+
// when running on Windows Azure, pull the statistics from the warehouse via storage
713+
builder.RegisterInstance(new CloudReportService(configuration.Current.AzureStorage_Statistics_ConnectionString, configuration.Current.AzureStorageReadAccessGeoRedundant))
714+
.AsSelf()
715+
.As<IReportService>()
716+
.SingleInstance();
717+
718+
// when running on Windows Azure, download counts come from the downloads.v1.json blob
719+
builder.Register(c => new SimpleBlobStorageConfiguration(configuration.Current.AzureStorage_Statistics_ConnectionString, configuration.Current.AzureStorageReadAccessGeoRedundant))
720+
.SingleInstance()
721+
.Keyed<IBlobStorageConfiguration>(BindingKeys.PrimaryStatisticsKey);
722+
723+
builder.Register(c => new SimpleBlobStorageConfiguration(configuration.Current.AzureStorage_Statistics_ConnectionString_Alternate, configuration.Current.AzureStorageReadAccessGeoRedundant))
724+
.SingleInstance()
725+
.Keyed<IBlobStorageConfiguration>(BindingKeys.AlternateStatisticsKey);
726+
727+
builder.Register(c =>
728+
{
729+
var primaryConfiguration = c.ResolveKeyed<IBlobStorageConfiguration>(BindingKeys.PrimaryStatisticsKey);
730+
var alternateConfiguration = c.ResolveKeyed<IBlobStorageConfiguration>(BindingKeys.AlternateStatisticsKey);
731+
var featureFlagService = c.Resolve<IFeatureFlagService>();
732+
var downloadCountService = new CloudDownloadCountService(telemetryService, featureFlagService, primaryConfiguration, alternateConfiguration);
733+
734+
var dlCountInterceptor = new DownloadCountObjectMaterializedInterceptor(downloadCountService, telemetryService);
735+
ObjectMaterializedInterception.AddInterceptor(dlCountInterceptor);
736+
737+
return downloadCountService;
738+
})
739+
.AsSelf()
740+
.As<IDownloadCountService>()
741+
.SingleInstance();
742+
743+
builder.RegisterType<JsonStatisticsService>()
744+
.AsSelf()
745+
.As<IStatisticsService>()
746+
.SingleInstance();
747+
}
748+
701749
private static void RegisterSwitchingDeleteAccountService(ContainerBuilder builder, ConfigurationService configuration)
702750
{
703751
var asyncAccountDeleteConnectionString = configuration.ServiceBus.AccountDeleter_ConnectionString;
@@ -1368,34 +1416,7 @@ private static void ConfigureForAzureStorage(ContainerBuilder builder, IGalleryC
13681416
}
13691417
}
13701418

1371-
// when running on Windows Azure, we use a back-end job to calculate stats totals and store in the blobs
1372-
builder.RegisterInstance(new JsonAggregateStatsService(configuration.Current.AzureStorage_Statistics_ConnectionString, configuration.Current.AzureStorageReadAccessGeoRedundant))
1373-
.AsSelf()
1374-
.As<IAggregateStatsService>()
1375-
.SingleInstance();
1376-
1377-
// when running on Windows Azure, pull the statistics from the warehouse via storage
1378-
builder.RegisterInstance(new CloudReportService(configuration.Current.AzureStorage_Statistics_ConnectionString, configuration.Current.AzureStorageReadAccessGeoRedundant))
1379-
.AsSelf()
1380-
.As<IReportService>()
1381-
.SingleInstance();
1382-
1383-
// when running on Windows Azure, download counts come from the downloads.v1.json blob
1384-
var downloadCountService = new CloudDownloadCountService(
1385-
telemetryService,
1386-
configuration.Current.AzureStorage_Statistics_ConnectionString,
1387-
configuration.Current.AzureStorageReadAccessGeoRedundant);
1388-
1389-
builder.RegisterInstance(downloadCountService)
1390-
.AsSelf()
1391-
.As<IDownloadCountService>()
1392-
.SingleInstance();
1393-
ObjectMaterializedInterception.AddInterceptor(new DownloadCountObjectMaterializedInterceptor(downloadCountService, telemetryService));
1394-
1395-
builder.RegisterType<JsonStatisticsService>()
1396-
.AsSelf()
1397-
.As<IStatisticsService>()
1398-
.SingleInstance();
1419+
RegisterStatisticsServices(builder, configuration, telemetryService);
13991420

14001421
builder.RegisterInstance(new TableErrorLog(configuration.Current.AzureStorage_Errors_ConnectionString, configuration.Current.AzureStorageReadAccessGeoRedundant))
14011422
.As<ErrorLog>()

‎src/NuGetGallery/Services/CloudDownloadCountService.cs‎

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
using Microsoft.WindowsAzure.Storage.RetryPolicies;
1313
using Newtonsoft.Json;
1414
using Newtonsoft.Json.Linq;
15+
using NuGetGallery.Services;
1516

1617
namespace NuGetGallery
1718
{
@@ -25,21 +26,26 @@ public class CloudDownloadCountService : IDownloadCountService
2526
private const string TelemetryOriginForRefreshMethod = "CloudDownloadCountService.Refresh";
2627

2728
private readonly ITelemetryService _telemetryService;
28-
private readonly string _connectionString;
29-
private readonly bool _readAccessGeoRedundant;
29+
private readonly IFeatureFlagService _featureFlagService;
30+
private readonly IBlobStorageConfiguration _primaryStorageConfiguration;
31+
private readonly IBlobStorageConfiguration _alternateBlobStorageConfiguration;
3032

3133
private readonly object _refreshLock = new object();
3234
private bool _isRefreshing;
3335

3436
private readonly ConcurrentDictionary<string, ConcurrentDictionary<string, int>> _downloadCounts
3537
= new ConcurrentDictionary<string, ConcurrentDictionary<string, int>>(StringComparer.OrdinalIgnoreCase);
3638

37-
public CloudDownloadCountService(ITelemetryService telemetryService, string connectionString, bool readAccessGeoRedundant)
39+
public CloudDownloadCountService(
40+
ITelemetryService telemetryService,
41+
IFeatureFlagService featureFlagService,
42+
IBlobStorageConfiguration primaryBlobStorageConfiguration,
43+
IBlobStorageConfiguration alternateBlobStorageConfiguration)
3844
{
3945
_telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
40-
41-
_connectionString = connectionString;
42-
_readAccessGeoRedundant = readAccessGeoRedundant;
46+
_featureFlagService = featureFlagService ?? throw new ArgumentNullException(nameof(featureFlagService));
47+
_primaryStorageConfiguration = primaryBlobStorageConfiguration ?? throw new ArgumentNullException(nameof(primaryBlobStorageConfiguration));
48+
_alternateBlobStorageConfiguration = alternateBlobStorageConfiguration;
4349
}
4450

4551
public bool TryGetDownloadCountForPackageRegistration(string id, out int downloadCount)
@@ -248,10 +254,19 @@ private void RefreshCore()
248254

249255
private CloudBlockBlob GetBlobReference()
250256
{
251-
var storageAccount = CloudStorageAccount.Parse(_connectionString);
257+
string connectionString = _primaryStorageConfiguration.ConnectionString;
258+
bool readAccessGeoRedundant = _primaryStorageConfiguration.ReadAccessGeoRedundant;
259+
260+
if (_alternateBlobStorageConfiguration != null && _featureFlagService.IsAlternateStatisticsSourceEnabled())
261+
{
262+
connectionString = _alternateBlobStorageConfiguration.ConnectionString;
263+
readAccessGeoRedundant = _alternateBlobStorageConfiguration.ReadAccessGeoRedundant;
264+
}
265+
266+
var storageAccount = CloudStorageAccount.Parse(connectionString);
252267
var blobClient = storageAccount.CreateCloudBlobClient();
253268

254-
if (_readAccessGeoRedundant)
269+
if (readAccessGeoRedundant)
255270
{
256271
blobClient.DefaultRequestOptions.LocationMode = LocationMode.PrimaryThenSecondary;
257272
}

0 commit comments

Comments
 (0)