Skip to content

Commit a90cbe6

Browse files
drewgilliesagr
andauthored
Job base classes MSI support (#10153)
Co-authored-by: Andrei Grigorev <[email protected]>
1 parent c542a18 commit a90cbe6

10 files changed

Lines changed: 171 additions & 59 deletions

File tree

src/NuGet.Jobs.Catalog2Registration/DependencyInjectionExtensions.cs

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,8 @@ public static ContainerBuilder AddCatalog2Registration(this ContainerBuilder con
2828
RegisterCursorStorage(containerBuilder);
2929

3030
containerBuilder
31-
.Register<ICloudBlobClient>(c =>
32-
{
33-
var options = c.Resolve<IOptionsSnapshot<Catalog2RegistrationConfiguration>>();
34-
return new CloudBlobClientWrapper(
35-
options.Value.StorageConnectionString,
36-
requestTimeout: DefaultBlobRequestOptions.ServerTimeout);
37-
});
31+
.RegisterStorageAccount<Catalog2RegistrationConfiguration>(c => c.StorageConnectionString, requestTimeout: DefaultBlobRequestOptions.ServerTimeout)
32+
.As<ICloudBlobClient>();
3833

3934
containerBuilder.Register(c => new Catalog2RegistrationCommand(
4035
c.Resolve<ICollector>(),

src/NuGet.Jobs.Common/JsonConfigurationJob.cs

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,7 @@ protected virtual void ConfigureDefaultJobServices(IServiceCollection services,
167167
services.Configure<ValidationDbConfiguration>(configurationRoot.GetSection(ValidationDbConfigurationSectionName));
168168
services.Configure<ServiceBusConfiguration>(configurationRoot.GetSection(ServiceBusConfigurationSectionName));
169169
services.Configure<ValidationStorageConfiguration>(configurationRoot.GetSection(ValidationStorageConfigurationSectionName));
170+
services.ConfigureStorageMsi(configurationRoot);
170171

171172
services.AddSingleton(new TelemetryClient(ApplicationInsightsConfiguration.TelemetryConfiguration));
172173
services.AddTransient<ITelemetryClient, TelemetryClientWrapper>();
@@ -197,13 +198,7 @@ private void AddScopedSqlConnectionFactory<TDbConfiguration>(IServiceCollection
197198
public static void ConfigureFeatureFlagAutofacServices(ContainerBuilder containerBuilder)
198199
{
199200
containerBuilder
200-
.Register(c =>
201-
{
202-
var options = c.Resolve<IOptionsSnapshot<FeatureFlagConfiguration>>();
203-
return new CloudBlobClientWrapper(
204-
options.Value.ConnectionString,
205-
requestTimeout: TimeSpan.FromMinutes(2));
206-
})
201+
.RegisterStorageAccount<FeatureFlagConfiguration>(c => c.ConnectionString, requestTimeout: TimeSpan.FromMinutes(2))
207202
.Keyed<ICloudBlobClient>(FeatureFlagBindingKey);
208203

209204
containerBuilder
Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using Autofac;
6+
using Autofac.Builder;
7+
using Microsoft.Extensions.Configuration;
8+
using Microsoft.Extensions.DependencyInjection;
9+
using Microsoft.Extensions.Options;
10+
using NuGetGallery;
11+
12+
namespace NuGet.Jobs
13+
{
14+
public static class StorageAccountHelper
15+
{
16+
private const string StorageUseManagedIdentityPropertyName = "Storage_UseManagedIdentity";
17+
private const string StorageManagedIdentityClientIdPropertyName = "Storage_ManagedIdentityClientId";
18+
19+
public static IServiceCollection ConfigureStorageMsi(
20+
this IServiceCollection serviceCollection,
21+
IConfiguration configuration,
22+
string useManageIdentityPropertyName = null,
23+
string managedIdentityClientIdPropertyName = null)
24+
{
25+
if (serviceCollection == null)
26+
{
27+
throw new ArgumentNullException(nameof(serviceCollection));
28+
}
29+
if (configuration == null)
30+
{
31+
throw new ArgumentNullException(nameof(configuration));
32+
}
33+
34+
useManageIdentityPropertyName ??= StorageUseManagedIdentityPropertyName;
35+
managedIdentityClientIdPropertyName ??= StorageManagedIdentityClientIdPropertyName;
36+
37+
string useManagedIdentityStr = configuration[useManageIdentityPropertyName];
38+
string managedIdentityClientId = configuration[managedIdentityClientIdPropertyName];
39+
bool useManagedIdentity = false;
40+
if (!string.IsNullOrWhiteSpace(useManagedIdentityStr))
41+
{
42+
useManagedIdentity = bool.Parse(useManagedIdentityStr);
43+
}
44+
return serviceCollection.Configure<StorageMsiConfiguration>(storageConfiguration =>
45+
{
46+
storageConfiguration.UseManagedIdentity = useManagedIdentity;
47+
storageConfiguration.ManagedIdentityClientId = managedIdentityClientId;
48+
});
49+
}
50+
51+
public static CloudBlobClientWrapper CreateCloudBlobClient(
52+
this IServiceProvider serviceProvider,
53+
string storageConnectionString,
54+
bool readAccessGeoRedundant = false,
55+
TimeSpan? requestTimeout = null)
56+
{
57+
if (serviceProvider == null)
58+
{
59+
throw new ArgumentNullException(nameof(serviceProvider));
60+
}
61+
if (string.IsNullOrWhiteSpace(storageConnectionString))
62+
{
63+
throw new ArgumentException($"{nameof(storageConnectionString)} cannot be null or empty.", nameof(storageConnectionString));
64+
}
65+
66+
var msiConfiguration = serviceProvider.GetRequiredService<IOptions<StorageMsiConfiguration>>().Value;
67+
return CreateCloudBlobClient(
68+
msiConfiguration,
69+
storageConnectionString,
70+
readAccessGeoRedundant,
71+
requestTimeout);
72+
}
73+
74+
public static IRegistrationBuilder<CloudBlobClientWrapper, SimpleActivatorData, SingleRegistrationStyle> RegisterStorageAccount<TConfiguration>(
75+
this ContainerBuilder builder,
76+
Func<TConfiguration, string> getConnectionString,
77+
Func<TConfiguration, bool> getReadAccessGeoRedundant = null,
78+
TimeSpan? requestTimeout = null)
79+
where TConfiguration : class, new()
80+
{
81+
if (builder == null)
82+
{
83+
throw new ArgumentNullException(nameof(builder));
84+
}
85+
if (getConnectionString == null)
86+
{
87+
throw new ArgumentNullException(nameof(getConnectionString));
88+
}
89+
90+
return builder.Register(c =>
91+
{
92+
var options = c.Resolve<IOptionsSnapshot<TConfiguration>>();
93+
string storageConnectionString = getConnectionString(options.Value);
94+
bool readAccessGeoRedundant = getReadAccessGeoRedundant?.Invoke(options.Value) ?? false;
95+
var msiConfiguration = c.Resolve<IOptions<StorageMsiConfiguration>>().Value;
96+
return CreateCloudBlobClient(
97+
msiConfiguration,
98+
storageConnectionString,
99+
readAccessGeoRedundant,
100+
requestTimeout);
101+
});
102+
}
103+
104+
private static CloudBlobClientWrapper CreateCloudBlobClient(
105+
StorageMsiConfiguration msiConfiguration,
106+
string storageConnectionString,
107+
bool readAccessGeoRedundant = false,
108+
TimeSpan? requestTimeout = null)
109+
{
110+
if (msiConfiguration.UseManagedIdentity)
111+
{
112+
if (string.IsNullOrWhiteSpace(msiConfiguration.ManagedIdentityClientId))
113+
{
114+
return CloudBlobClientWrapper.UsingDefaultAzureCredential(
115+
storageConnectionString,
116+
readAccessGeoRedundant: readAccessGeoRedundant,
117+
requestTimeout: requestTimeout);
118+
}
119+
else
120+
{
121+
return CloudBlobClientWrapper.UsingMsi(
122+
storageConnectionString,
123+
msiConfiguration.ManagedIdentityClientId,
124+
readAccessGeoRedundant,
125+
requestTimeout);
126+
}
127+
}
128+
129+
return new CloudBlobClientWrapper(
130+
storageConnectionString,
131+
readAccessGeoRedundant,
132+
requestTimeout);
133+
}
134+
}
135+
}
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
namespace NuGet.Jobs
5+
{
6+
public class StorageMsiConfiguration
7+
{
8+
public bool UseManagedIdentity { get; set; }
9+
public string ManagedIdentityClientId { get; set; }
10+
}
11+
}

src/NuGet.Jobs.GitHubIndexer/Job.cs

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,16 +42,15 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
4242
services.AddTransient<IRepositoriesCache, DiskRepositoriesCache>();
4343
services.AddTransient<IConfigFileParser, ConfigFileParser>();
4444
services.AddTransient<IRepoFetcher, RepoFetcher>();
45-
services.AddTransient<ICloudBlobClient>(provider => {
46-
var config = provider.GetRequiredService<IOptionsSnapshot<GitHubIndexerConfiguration>>();
47-
return new CloudBlobClientWrapper(config.Value.StorageConnectionString, config.Value.StorageReadAccessGeoRedundant);
48-
});
4945

5046
services.Configure<GitHubIndexerConfiguration>(configurationRoot.GetSection(GitHubIndexerConfigurationSectionName));
5147
}
5248

5349
protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder, IConfigurationRoot configurationRoot)
5450
{
51+
containerBuilder
52+
.RegisterStorageAccount<GitHubIndexerConfiguration>(c => c.StorageConnectionString, c => c.StorageReadAccessGeoRedundant)
53+
.As<ICloudBlobClient>();
5554
}
5655
}
5756
}

src/NuGet.Services.AzureSearch/DependencyInjectionExtensions.cs

Lines changed: 5 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
using Microsoft.Extensions.Logging;
2020
using Microsoft.Extensions.Options;
2121
using Microsoft.Rest;
22+
using NuGet.Jobs;
2223
using NuGet.Protocol;
2324
using NuGet.Services.AzureSearch.Auxiliary2AzureSearch;
2425
using NuGet.Services.AzureSearch.AuxiliaryFiles;
@@ -108,13 +109,7 @@ private static void RegisterIndexServices(ContainerBuilder containerBuilder, str
108109
private static void RegisterAzureSearchStorageServices(ContainerBuilder containerBuilder, string key)
109110
{
110111
containerBuilder
111-
.Register<ICloudBlobClient>(c =>
112-
{
113-
var options = c.Resolve<IOptionsSnapshot<AzureSearchConfiguration>>();
114-
return new CloudBlobClientWrapper(
115-
options.Value.StorageConnectionString,
116-
requestTimeout: DefaultBlobRequestOptions.ServerTimeout);
117-
})
112+
.RegisterStorageAccount<AzureSearchConfiguration>(c => c.StorageConnectionString, requestTimeout: DefaultBlobRequestOptions.ServerTimeout)
118113
.Keyed<ICloudBlobClient>(key);
119114

120115
containerBuilder
@@ -221,13 +216,9 @@ private static void RegisterAzureSearchStorageServices(ContainerBuilder containe
221216
private static void RegisterAuxiliaryDataStorageServices(ContainerBuilder containerBuilder, string key)
222217
{
223218
containerBuilder
224-
.Register<ICloudBlobClient>(c =>
225-
{
226-
var options = c.Resolve<IOptionsSnapshot<AuxiliaryDataStorageConfiguration>>();
227-
return new CloudBlobClientWrapper(
228-
options.Value.AuxiliaryDataStorageConnectionString,
229-
requestTimeout: DefaultBlobRequestOptions.ServerTimeout);
230-
})
219+
.RegisterStorageAccount<AuxiliaryDataStorageConfiguration>(
220+
c => c.AuxiliaryDataStorageConnectionString,
221+
requestTimeout: DefaultBlobRequestOptions.ServerTimeout)
231222
.Keyed<ICloudBlobClient>(key);
232223

233224
containerBuilder

src/NuGet.Services.Validation.Orchestrator/Job.cs

Lines changed: 5 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -182,13 +182,6 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
182182
services.AddTransient<IBrokeredMessageSerializer<PackageValidationMessageData>, PackageValidationMessageDataSerializationAdapter>();
183183
services.AddTransient<ICriteriaEvaluator<Package>, PackageCriteriaEvaluator>();
184184
services.AddTransient<IProcessSignatureEnqueuer, ProcessSignatureEnqueuer>();
185-
services.AddTransient<ICloudBlobClient>(c =>
186-
{
187-
var configurationAccessor = c.GetRequiredService<IOptionsSnapshot<ValidationConfiguration>>();
188-
return new CloudBlobClientWrapper(
189-
configurationAccessor.Value.ValidationStorageConnectionString,
190-
readAccessGeoRedundant: false);
191-
});
192185
services.AddTransient<ICloudBlobContainerInformationProvider, GalleryCloudBlobContainerInformationProvider>();
193186
services.AddTransient<ICoreFileStorageService, CloudBlobCoreFileStorageService>();
194187
services.AddTransient<IFileDownloader, FileDownloader>();
@@ -238,6 +231,10 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
238231

239232
protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder, IConfigurationRoot configurationRoot)
240233
{
234+
containerBuilder
235+
.RegisterStorageAccount<ValidationConfiguration>(c => c.ValidationStorageConnectionString)
236+
.As<ICloudBlobClient>();
237+
241238
containerBuilder
242239
.Register(c =>
243240
{
@@ -458,13 +455,7 @@ private static void ConfigureSymbolScanValidator(ContainerBuilder builder)
458455
private static void ConfigureFlatContainer(ContainerBuilder builder)
459456
{
460457
builder
461-
.Register<CloudBlobClientWrapper>(c =>
462-
{
463-
var configurationAccessor = c.Resolve<IOptionsSnapshot<FlatContainerConfiguration>>();
464-
return new CloudBlobClientWrapper(
465-
configurationAccessor.Value.ConnectionString,
466-
readAccessGeoRedundant: false);
467-
})
458+
.RegisterStorageAccount<FlatContainerConfiguration>(c => c.ConnectionString)
468459
.Keyed<ICloudBlobClient>(FlatContainerBindingKey);
469460

470461
builder

src/Validation.Common.Job/Leases/CloudBlobLeaseService.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

44
using System;
5+
using System.IO;
56
using System.Net;
67
using System.Threading;
78
using System.Threading.Tasks;

src/Validation.Common.Job/ValidationJobBase.cs

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -50,13 +50,6 @@ protected override void ConfigureDefaultJobServices(IServiceCollection services,
5050
services.AddTransient<IFileDownloader, FileDownloader>();
5151
services.AddTransient<IServiceBusMessageSerializer, ServiceBusMessageSerializer>();
5252

53-
services.AddTransient<ICloudBlobClient>(c =>
54-
{
55-
var configurationAccessor = c.GetRequiredService<IOptionsSnapshot<ValidationStorageConfiguration>>();
56-
return new CloudBlobClientWrapper(
57-
configurationAccessor.Value.ConnectionString,
58-
readAccessGeoRedundant: false);
59-
});
6053
services.AddTransient<ICoreFileStorageService, CloudBlobCoreFileStorageService>();
6154
services.AddTransient<ISharedAccessSignatureService, SharedAccessSignatureService>();
6255

@@ -97,6 +90,10 @@ protected override void ConfigureDefaultAutofacServices(ContainerBuilder contain
9790

9891
ConfigureFeatureFlagAutofacServices(containerBuilder);
9992

93+
containerBuilder
94+
.RegisterStorageAccount<ValidationStorageConfiguration>(c => c.ConnectionString)
95+
.As<ICloudBlobClient>();
96+
10097
containerBuilder
10198
.Register(c =>
10299
{

src/Validation.Symbols/Job.cs

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using Microsoft.Extensions.Configuration;
66
using Microsoft.Extensions.DependencyInjection;
77
using Microsoft.Extensions.Options;
8+
using NuGet.Jobs;
89
using NuGet.Jobs.Validation;
910
using NuGet.Jobs.Validation.Storage;
1011
using NuGet.Jobs.Validation.Symbols.Core;
@@ -32,16 +33,12 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
3233
{
3334
var configurationAccessor = c.GetRequiredService<IOptionsSnapshot<SymbolsValidatorConfiguration>>();
3435
var packageStorageService = new CloudBlobCoreFileStorageService(
35-
new CloudBlobClientWrapper(
36-
configurationAccessor.Value.PackageConnectionString,
37-
readAccessGeoRedundant: false),
36+
c.CreateCloudBlobClient(configurationAccessor.Value.PackageConnectionString),
3837
c.GetRequiredService<IDiagnosticsService>(),
3938
c.GetRequiredService<ICloudBlobContainerInformationProvider>());
4039

4140
var packageValidationStorageService = new CloudBlobCoreFileStorageService(
42-
new CloudBlobClientWrapper(
43-
configurationAccessor.Value.ValidationPackageConnectionString,
44-
readAccessGeoRedundant: false),
41+
c.CreateCloudBlobClient(configurationAccessor.Value.ValidationPackageConnectionString),
4542
c.GetRequiredService<IDiagnosticsService>(),
4643
c.GetRequiredService<ICloudBlobContainerInformationProvider>());
4744

0 commit comments

Comments
 (0)