Skip to content
This repository was archived by the owner on Jul 30, 2024. It is now read-only.

Commit 983003c

Browse files
author
Christy Henriksson
authored
Migrate Search.GenerateAuxillaryData to use JsonConfig (#518)
1 parent 91e3d1c commit 983003c

20 files changed

Lines changed: 246 additions & 62 deletions

‎src/NuGet.Jobs.Common/Configuration/StatisticsDbConfiguration.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
namespace NuGet.Jobs.Configuration
5+
{
6+
public class StatisticsDbConfiguration : IDbConfiguration
7+
{
8+
public string ConnectionString { get; set; }
9+
}
10+
}

‎src/NuGet.Jobs.Common/JsonConfigurationJob.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ public abstract class JsonConfigurationJob : JobBase
2525
{
2626
private const string InitializationConfigurationSectionName = "Initialization";
2727
private const string GalleryDbConfigurationSectionName = "GalleryDb";
28+
private const string StatisticsDbConfigurationSectionName = "StatisticsDb";
2829
private const string ValidationDbConfigurationSectionName = "ValidationDb";
2930
private const string ServiceBusConfigurationSectionName = "ServiceBus";
3031
private const string ValidationStorageConfigurationSectionName = "ValidationStorage";
@@ -109,6 +110,7 @@ private IServiceProvider GetServiceProvider(IConfigurationRoot configurationRoot
109110
protected virtual void ConfigureDefaultJobServices(IServiceCollection services, IConfigurationRoot configurationRoot)
110111
{
111112
services.Configure<GalleryDbConfiguration>(configurationRoot.GetSection(GalleryDbConfigurationSectionName));
113+
services.Configure<StatisticsDbConfiguration>(configurationRoot.GetSection(StatisticsDbConfigurationSectionName));
112114
services.Configure<ValidationDbConfiguration>(configurationRoot.GetSection(ValidationDbConfigurationSectionName));
113115
services.Configure<ServiceBusConfiguration>(configurationRoot.GetSection(ServiceBusConfigurationSectionName));
114116
services.Configure<ValidationStorageConfiguration>(configurationRoot.GetSection(ValidationStorageConfigurationSectionName));
@@ -133,6 +135,12 @@ protected virtual void RegisterDatabases(IServiceProvider serviceProvider)
133135
RegisterDatabase<GalleryDbConfiguration>(serviceProvider);
134136
}
135137

138+
var statisticsDb = serviceProvider.GetRequiredService<IOptionsSnapshot<StatisticsDbConfiguration>>();
139+
if (!string.IsNullOrEmpty(statisticsDb.Value?.ConnectionString))
140+
{
141+
RegisterDatabase<StatisticsDbConfiguration>(serviceProvider);
142+
}
143+
136144
var validationDb = serviceProvider.GetRequiredService<IOptionsSnapshot<ValidationDbConfiguration>>();
137145
if (!string.IsNullOrEmpty(validationDb.Value?.ConnectionString))
138146
{

‎src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
<Compile Include="Configuration\IDbConfiguration.cs" />
4848
<Compile Include="Configuration\JobArgumentNames.cs" />
4949
<Compile Include="Configuration\ServiceBusConfiguration.cs" />
50+
<Compile Include="Configuration\StatisticsDbConfiguration.cs" />
5051
<Compile Include="Configuration\ValidationDbConfiguration.cs" />
5152
<Compile Include="Configuration\ValidationStorageConfiguration.cs" />
5253
<Compile Include="Extensions\LoggerExtensions.cs" />

‎src/Search.GenerateAuxiliaryData/Configuration/InitializationConfiguration.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
namespace Search.GenerateAuxiliaryData
5+
{
6+
public class InitializationConfiguration
7+
{
8+
public string AzureCdnCloudStorageAccount { get; set; }
9+
10+
public string AzureCdnCloudStorageContainerName { get; set; }
11+
12+
public string PrimaryDestination { get; set; }
13+
14+
public string DestinationContainerName { get; set; }
15+
}
16+
}

‎src/Search.GenerateAuxiliaryData/Job.cs‎

Lines changed: 62 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,19 @@
44
using System;
55
using System.Collections.Generic;
66
using System.ComponentModel.Design;
7-
using System.Data.SqlClient;
87
using System.Linq;
98
using System.Threading.Tasks;
9+
using Autofac;
10+
using Microsoft.Extensions.Configuration;
11+
using Microsoft.Extensions.DependencyInjection;
1012
using Microsoft.Extensions.Logging;
1113
using Microsoft.WindowsAzure.Storage;
12-
using Microsoft.WindowsAzure.Storage.Blob;
1314
using NuGet.Jobs;
14-
using NuGet.Services.KeyVault;
15-
using NuGet.Services.Sql;
15+
using NuGet.Jobs.Configuration;
1616

1717
namespace Search.GenerateAuxiliaryData
1818
{
19-
internal class Job
20-
: JobBase
19+
public class Job : JsonConfigurationJob
2120
{
2221
private const string DefaultContainerName = "ng-search-data";
2322

@@ -40,40 +39,62 @@ internal class Job
4039
private const string StatisticsReportName = "downloads.v1.json";
4140

4241
private List<Exporter> _exportersToRun;
43-
private CloudBlobContainer _destContainer;
44-
private CloudBlobContainer _statisticsContainer;
42+
43+
private InitializationConfiguration Configuration { get; set; }
4544

4645
public override void Init(IServiceContainer serviceContainer, IDictionary<string, string> jobArgsDictionary)
4746
{
48-
var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector));
49-
50-
var packageDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase);
51-
var packageDbConnectionFactory = new AzureSqlConnectionFactory(packageDbConnectionString, secretInjector);
52-
53-
var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase);
54-
var statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector);
47+
base.Init(serviceContainer, jobArgsDictionary);
5548

56-
var statisticsStorageAccount = CloudStorageAccount.Parse(
57-
JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount));
49+
Configuration = _serviceProvider.GetRequiredService<InitializationConfiguration>();
5850

59-
var statisticsReportsContainerName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName);
51+
var destinationContainer = CloudStorageAccount.Parse(Configuration.PrimaryDestination)
52+
.CreateCloudBlobClient()
53+
.GetContainerReference(Configuration.DestinationContainerName ?? DefaultContainerName);
6054

61-
var destination = CloudStorageAccount.Parse(
62-
JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PrimaryDestination));
63-
64-
var destinationContainerName =
65-
JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.DestinationContainerName)
66-
?? DefaultContainerName;
67-
68-
_destContainer = destination.CreateCloudBlobClient().GetContainerReference(destinationContainerName);
69-
_statisticsContainer = statisticsStorageAccount.CreateCloudBlobClient().GetContainerReference(statisticsReportsContainerName);
55+
var statisticsContainer = CloudStorageAccount.Parse(Configuration.AzureCdnCloudStorageAccount)
56+
.CreateCloudBlobClient()
57+
.GetContainerReference(Configuration.AzureCdnCloudStorageContainerName);
7058

7159
_exportersToRun = new List<Exporter> {
72-
new VerifiedPackagesExporter(LoggerFactory.CreateLogger<VerifiedPackagesExporter>(), packageDbConnectionFactory, _destContainer, ScriptVerifiedPackages, OutputNameVerifiedPackages),
73-
new NestedJArrayExporter(LoggerFactory.CreateLogger<NestedJArrayExporter>(), packageDbConnectionFactory, _destContainer, ScriptCuratedFeed, OutputNameCuratedFeed, Col0CuratedFeed, Col1CuratedFeed),
74-
new NestedJArrayExporter(LoggerFactory.CreateLogger<NestedJArrayExporter>(), packageDbConnectionFactory, _destContainer, ScriptOwners, OutputNameOwners, Col0Owners, Col1Owners),
75-
new RankingsExporter(LoggerFactory.CreateLogger<RankingsExporter>(), statisticsDbConnectionFactory, _destContainer, ScriptRankingsTotal, OutputNameRankings),
76-
new BlobStorageExporter(LoggerFactory.CreateLogger<BlobStorageExporter>(), _statisticsContainer, StatisticsReportName, _destContainer, StatisticsReportName)
60+
new VerifiedPackagesExporter(
61+
LoggerFactory.CreateLogger<VerifiedPackagesExporter>(),
62+
OpenSqlConnectionAsync<GalleryDbConfiguration>,
63+
destinationContainer,
64+
ScriptVerifiedPackages,
65+
OutputNameVerifiedPackages),
66+
67+
new NestedJArrayExporter(
68+
LoggerFactory.CreateLogger<NestedJArrayExporter>(),
69+
OpenSqlConnectionAsync<GalleryDbConfiguration>,
70+
destinationContainer,
71+
ScriptCuratedFeed,
72+
OutputNameCuratedFeed,
73+
Col0CuratedFeed,
74+
Col1CuratedFeed),
75+
76+
new NestedJArrayExporter(
77+
LoggerFactory.CreateLogger<NestedJArrayExporter>(),
78+
OpenSqlConnectionAsync<GalleryDbConfiguration>,
79+
destinationContainer,
80+
ScriptOwners,
81+
OutputNameOwners,
82+
Col0Owners,
83+
Col1Owners),
84+
85+
new RankingsExporter(
86+
LoggerFactory.CreateLogger<RankingsExporter>(),
87+
OpenSqlConnectionAsync<StatisticsDbConfiguration>,
88+
destinationContainer,
89+
ScriptRankingsTotal,
90+
OutputNameRankings),
91+
92+
new BlobStorageExporter(
93+
LoggerFactory.CreateLogger<BlobStorageExporter>(),
94+
statisticsContainer,
95+
StatisticsReportName,
96+
destinationContainer,
97+
StatisticsReportName)
7798
};
7899
}
79100

@@ -100,5 +121,14 @@ public override async Task Run()
100121
throw new ExporterException($"{failedExporters.Count()} tasks failed: {string.Join(", ", failedExporters)}");
101122
}
102123
}
124+
125+
protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder)
126+
{
127+
}
128+
129+
protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot)
130+
{
131+
ConfigureInitializationSection<InitializationConfiguration>(services, configurationRoot);
132+
}
103133
}
104134
}

‎src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,10 @@
66
using System.Data;
77
using System.Data.SqlClient;
88
using System.Linq;
9+
using System.Threading.Tasks;
910
using Microsoft.Extensions.Logging;
1011
using Microsoft.WindowsAzure.Storage.Blob;
1112
using Newtonsoft.Json.Linq;
12-
using NuGet.Services.Sql;
1313

1414
namespace Search.GenerateAuxiliaryData
1515
{
@@ -20,8 +20,12 @@ class NestedJArrayExporter
2020
public string Col1 { get; }
2121
public string SqlScript { get; }
2222

23-
public NestedJArrayExporter(ILogger<NestedJArrayExporter> logger, ISqlConnectionFactory connectionFactory, CloudBlobContainer defaultDestinationContainer, string defaultSqlScript, string defaultName, string defaultCol0, string defaultCol1)
24-
: base(logger, connectionFactory, defaultDestinationContainer, defaultName)
23+
public NestedJArrayExporter(
24+
ILogger<NestedJArrayExporter> logger,
25+
Func<Task<SqlConnection>> openSqlConnectionAsync,
26+
CloudBlobContainer defaultDestinationContainer,
27+
string defaultSqlScript, string defaultName, string defaultCol0, string defaultCol1)
28+
: base(logger, openSqlConnectionAsync, defaultDestinationContainer, defaultName)
2529
{
2630
Col0 = defaultCol0;
2731
Col1 = defaultCol1;

‎src/Search.GenerateAuxiliaryData/RankingsExporter.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,10 @@
44
using System;
55
using System.Data;
66
using System.Data.SqlClient;
7+
using System.Threading.Tasks;
78
using Microsoft.Extensions.Logging;
89
using Microsoft.WindowsAzure.Storage.Blob;
910
using Newtonsoft.Json.Linq;
10-
using NuGet.Services.Sql;
1111

1212
namespace Search.GenerateAuxiliaryData
1313
{
@@ -21,11 +21,11 @@ public sealed class RankingsExporter : SqlExporter
2121

2222
public RankingsExporter(
2323
ILogger<SqlExporter> logger,
24-
ISqlConnectionFactory connectionFactory,
24+
Func<Task<SqlConnection>> openSqlConnectionAsync,
2525
CloudBlobContainer defaultDestinationContainer,
2626
string defaultRankingsScript,
2727
string defaultName)
28-
: base(logger, connectionFactory, defaultDestinationContainer, defaultName)
28+
: base(logger, openSqlConnectionAsync, defaultDestinationContainer, defaultName)
2929
{
3030
_rankingsTotalScript = defaultRankingsScript;
3131
}

‎src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.Asia.cmd‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
@echo OFF
2-
2+
3+
REM This script is the same as Search.GenerateAuxillaryData.cmd. However, this copy is required until "Jobs.ServiceNames" deployment config is consolidated.
4+
35
cd bin
46

57
:Top
6-
echo "Starting job - #{Jobs.Asia.search.generateauxiliarydata.Title}"
8+
echo "Starting job - #{Jobs.search.generateauxiliarydata.Title}"
79

8-
title #{Jobs.Asia.search.generateauxiliarydata.Title}
10+
title #{Jobs.search.generateauxiliarydata.Title}
911

10-
start /w search.generateauxiliarydata.exe -VaultName "#{Deployment.Azure.KeyVault.VaultName}" -ClientId "#{Deployment.Azure.KeyVault.ClientId}" -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" -PrimaryDestination #{Jobs.Asia.search.generateauxiliarydata.Storage.Primary} -PackageDatabase "#{Jobs.search.generateauxiliarydata.PackageDatabase}" -StatisticsDatabase "#{Jobs.search.generateauxiliarydata.StatisticsDatabase}" -AzureCdnCloudStorageAccount "#{Jobs.stats.createazurecdnwarehousereports.AzureCdn.CloudStorageAccount}" -AzureCdnCloudStorageContainerName "#{Jobs.stats.createazurecdnwarehousereports.AzureCdn.CloudStorageContainerName}" -verbose true -Sleep #{Jobs.search.generateauxiliarydata.Sleep} -InstrumentationKey "#{Jobs.search.generateauxiliarydata.ApplicationInsightsInstrumentationKey}"
12+
start /w search.generateauxiliarydata.exe -Configuration "#{Jobs.search.generateauxiliarydata.Configuration}" -verbose true -Sleep #{Jobs.search.generateauxiliarydata.Sleep} -InstrumentationKey "#{Jobs.search.generateauxiliarydata.ApplicationInsightsInstrumentationKey}"
1113

1214
echo "Finished #{Jobs.search.generateauxiliarydata.Title}"
1315

‎src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.cmd‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ cd bin
77

88
title #{Jobs.search.generateauxiliarydata.Title}
99

10-
start /w search.generateauxiliarydata.exe -VaultName "#{Deployment.Azure.KeyVault.VaultName}" -ClientId "#{Deployment.Azure.KeyVault.ClientId}" -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" -PrimaryDestination #{Jobs.search.generateauxiliarydata.Storage.Primary} -PackageDatabase "#{Jobs.search.generateauxiliarydata.PackageDatabase}" -StatisticsDatabase "#{Jobs.search.generateauxiliarydata.StatisticsDatabase}" -AzureCdnCloudStorageAccount "#{Jobs.stats.createazurecdnwarehousereports.AzureCdn.CloudStorageAccount}" -AzureCdnCloudStorageContainerName "#{Jobs.stats.createazurecdnwarehousereports.AzureCdn.CloudStorageContainerName}" -verbose true -Sleep #{Jobs.search.generateauxiliarydata.Sleep} -InstrumentationKey "#{Jobs.search.generateauxiliarydata.ApplicationInsightsInstrumentationKey}"
10+
start /w search.generateauxiliarydata.exe -Configuration "#{Jobs.search.generateauxiliarydata.Configuration}" -verbose true -Sleep #{Jobs.search.generateauxiliarydata.Sleep} -InstrumentationKey "#{Jobs.search.generateauxiliarydata.ApplicationInsightsInstrumentationKey}"
1111

1212
echo "Finished #{Jobs.search.generateauxiliarydata.Title}"
1313

‎src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
</ItemGroup>
4444
<ItemGroup>
4545
<Compile Include="BlobStorageExporter.cs" />
46+
<Compile Include="Configuration\InitializationConfiguration.cs" />
4647
<Compile Include="Exporter.cs" />
4748
<Compile Include="ExporterException.cs" />
4849
<Compile Include="Job.cs" />
@@ -57,6 +58,7 @@
5758
<None Include="App.config" />
5859
<None Include="Scripts\*" />
5960
<None Include="Search.GenerateAuxiliaryData.nuspec" />
61+
<None Include="Settings\*" />
6062
</ItemGroup>
6163
<ItemGroup>
6264
<EmbeddedResource Include="SqlScripts\CuratedFeed.sql" />
@@ -92,9 +94,6 @@
9294
<PackageReference Include="Newtonsoft.Json">
9395
<Version>9.0.1</Version>
9496
</PackageReference>
95-
<PackageReference Include="NuGet.Services.Sql">
96-
<Version>2.27.0</Version>
97-
</PackageReference>
9897
<PackageReference Include="WindowsAzure.Storage">
9998
<Version>7.1.2</Version>
10099
</PackageReference>

0 commit comments

Comments
 (0)