Skip to content
This repository was archived by the owner on Jul 30, 2024. It is now read-only.

Commit a443527

Browse files
author
Christy Henriksson
authored
Migrate ImportCdnStats to use JsonConfig (#524)
1 parent 624d0ec commit a443527

12 files changed

Lines changed: 146 additions & 64 deletions

‎src/Stats.ImportAzureCdnStatistics/Configuration/ImportAzureCdnStatisticsConfiguration.cs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
namespace Stats.ImportAzureCdnStatistics
5+
{
6+
public class ImportAzureCdnStatisticsConfiguration
7+
{
8+
public string AzureCdnCloudStorageAccount { get; set; }
9+
10+
public string AzureCdnCloudStorageContainerName { get; set; }
11+
12+
public string AzureCdnPlatform { get; set; }
13+
14+
public string AzureCdnAccountNumber { get; set; }
15+
16+
public bool AggregatesOnly { get; set; }
17+
}
18+
}

‎src/Stats.ImportAzureCdnStatistics/DataImporter.cs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,29 @@
11
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

4+
using System;
45
using System.Data;
56
using System.Data.SqlClient;
67
using System.Threading.Tasks;
7-
using NuGet.Services.Sql;
88

99
namespace Stats.ImportAzureCdnStatistics
1010
{
1111
internal class DataImporter
1212
{
13-
private readonly ISqlConnectionFactory _statisticsDbConnectionFactory;
13+
private readonly Func<Task<SqlConnection>> _openStatisticsSqlConnectionAsync;
1414
private const string _sqlSelectTop1FromTable = "SELECT TOP 1 * FROM [dbo].[{0}]";
1515

16-
public DataImporter(ISqlConnectionFactory statisticsDbConnectionFactory)
16+
public DataImporter(Func<Task<SqlConnection>> openStatisticsSqlConnectionAsync)
1717
{
18-
_statisticsDbConnectionFactory = statisticsDbConnectionFactory;
18+
_openStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync;
1919
}
2020

2121
public async Task<DataTable> GetDataTableAsync(string tableName)
2222
{
2323
var dataTable = new DataTable();
2424
var query = string.Format(_sqlSelectTop1FromTable, tableName);
2525

26-
using (var connection = await _statisticsDbConnectionFactory.CreateAsync())
26+
using (var connection = await _openStatisticsSqlConnectionAsync())
2727
{
2828
var tableAdapter = new SqlDataAdapter(query, connection)
2929
{

src/Stats.ImportAzureCdnStatistics/Job.cs renamed to src/Stats.ImportAzureCdnStatistics/ImportAzureCdnStatisticsJob.cs

Lines changed: 35 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -6,66 +6,57 @@
66
using System.ComponentModel.Design;
77
using System.Globalization;
88
using System.Threading.Tasks;
9+
using Autofac;
10+
using Microsoft.Extensions.Configuration;
11+
using Microsoft.Extensions.DependencyInjection;
12+
using Microsoft.Extensions.Options;
913
using Microsoft.WindowsAzure.Storage;
1014
using Microsoft.WindowsAzure.Storage.Blob;
1115
using Microsoft.WindowsAzure.Storage.RetryPolicies;
1216
using NuGet.Jobs;
13-
using NuGet.Services.KeyVault;
14-
using NuGet.Services.Sql;
17+
using NuGet.Jobs.Configuration;
1518
using Stats.AzureCdnLogs.Common;
1619

1720
namespace Stats.ImportAzureCdnStatistics
1821
{
19-
public class Job
20-
: JobBase
22+
public class ImportAzureCdnStatisticsJob : JsonConfigurationJob
2123
{
22-
private bool _aggregatesOnly;
23-
private string _azureCdnAccountNumber;
24-
private string _cloudStorageContainerName;
24+
private ImportAzureCdnStatisticsConfiguration _configuration;
2525
private AzureCdnPlatform _azureCdnPlatform;
26-
private ISqlConnectionFactory _statisticsDbConnectionFactory;
27-
private CloudStorageAccount _cloudStorageAccount;
2826
private CloudBlobClient _cloudBlobClient;
2927
private LogFileProvider _blobLeaseManager;
3028

3129
public override void Init(IServiceContainer serviceContainer, IDictionary<string, string> jobArgsDictionary)
3230
{
33-
var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector));
34-
var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase);
35-
_statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector);
31+
base.Init(serviceContainer, jobArgsDictionary);
3632

37-
var azureCdnPlatform = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnPlatform);
38-
var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount);
39-
_cloudStorageAccount = ValidateAzureCloudStorageAccount(cloudStorageAccountConnectionString);
33+
_configuration = _serviceProvider.GetRequiredService<IOptionsSnapshot<ImportAzureCdnStatisticsConfiguration>>().Value;
4034

41-
_azureCdnAccountNumber = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnAccountNumber);
42-
_azureCdnPlatform = ValidateAzureCdnPlatform(azureCdnPlatform);
43-
_cloudStorageContainerName = ValidateAzureContainerName(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName));
35+
_azureCdnPlatform = ValidateAzureCdnPlatform(_configuration.AzureCdnPlatform);
4436

45-
_aggregatesOnly = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.AggregatesOnly);
46-
47-
// construct a cloud blob client for the configured storage account
48-
_cloudBlobClient = _cloudStorageAccount.CreateCloudBlobClient();
37+
var cloudStorageAccount = ValidateAzureCloudStorageAccount(_configuration.AzureCdnCloudStorageAccount);
38+
_cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient();
4939
_cloudBlobClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(10), 5);
5040

51-
// Get the source blob container (containing compressed log files)
52-
// and construct a log source (fetching raw logs from the source blob container)
53-
var sourceBlobContainer = _cloudBlobClient.GetContainerReference(_cloudStorageContainerName);
54-
_blobLeaseManager = new LogFileProvider(sourceBlobContainer, LoggerFactory);
41+
_blobLeaseManager = new LogFileProvider(
42+
_cloudBlobClient.GetContainerReference(_configuration.AzureCdnCloudStorageContainerName),
43+
LoggerFactory);
5544
}
5645

5746
public override async Task Run()
5847
{
5948
// Get the target blob container (for archiving decompressed log files)
60-
var targetBlobContainer = _cloudBlobClient.GetContainerReference(_cloudStorageContainerName + "-archive");
49+
var targetBlobContainer = _cloudBlobClient.GetContainerReference(
50+
_configuration.AzureCdnCloudStorageContainerName + "-archive");
6151
await targetBlobContainer.CreateIfNotExistsAsync();
6252

6353
// Get the dead-letter table (corrupted or failed blobs will end up there)
64-
var deadLetterBlobContainer = _cloudBlobClient.GetContainerReference(_cloudStorageContainerName + "-deadletter");
54+
var deadLetterBlobContainer = _cloudBlobClient.GetContainerReference(
55+
_configuration.AzureCdnCloudStorageContainerName + "-deadletter");
6556
await deadLetterBlobContainer.CreateIfNotExistsAsync();
6657

6758
// Create a parser
68-
var warehouse = new Warehouse(LoggerFactory, _statisticsDbConnectionFactory);
59+
var warehouse = new Warehouse(LoggerFactory, OpenSqlConnectionAsync<StatisticsDbConfiguration>);
6960
var statisticsBlobContainerUtility = new StatisticsBlobContainerUtility(
7061
targetBlobContainer,
7162
deadLetterBlobContainer,
@@ -74,11 +65,13 @@ public override async Task Run()
7465
var logProcessor = new LogFileProcessor(statisticsBlobContainerUtility, LoggerFactory, warehouse);
7566

7667
// Get the next to-be-processed raw log file using the cdn raw log file name prefix
77-
var prefix = string.Format(CultureInfo.InvariantCulture, "{0}_{1}_", _azureCdnPlatform.GetRawLogFilePrefix(), _azureCdnAccountNumber);
68+
var prefix = string.Format(CultureInfo.InvariantCulture, "{0}_{1}_",
69+
_azureCdnPlatform.GetRawLogFilePrefix(),
70+
_configuration.AzureCdnAccountNumber);
7871

7972
// Get next raw log file to be processed
8073
IReadOnlyCollection<string> alreadyAggregatedLogFiles = null;
81-
if (_aggregatesOnly)
74+
if (_configuration.AggregatesOnly)
8275
{
8376
// We only want to process aggregates for the log files.
8477
// Get the list of files we already processed so we can skip them.
@@ -90,9 +83,9 @@ public override async Task Run()
9083
{
9184
var packageTranslator = new PackageTranslator("packagetranslations.json");
9285
var packageStatisticsParser = new PackageStatisticsParser(packageTranslator, LoggerFactory);
93-
await logProcessor.ProcessLogFileAsync(leasedLogFile, packageStatisticsParser, _aggregatesOnly);
86+
await logProcessor.ProcessLogFileAsync(leasedLogFile, packageStatisticsParser, _configuration.AggregatesOnly);
9487

95-
if (_aggregatesOnly)
88+
if (_configuration.AggregatesOnly)
9689
{
9790
_blobLeaseManager.TrackLastProcessedBlobUri(leasedLogFile.Uri);
9891
}
@@ -139,5 +132,14 @@ private static string ValidateAzureContainerName(string containerName)
139132
}
140133
return containerName;
141134
}
135+
136+
protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder)
137+
{
138+
}
139+
140+
protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot)
141+
{
142+
ConfigureInitializationSection<ImportAzureCdnStatisticsConfiguration>(services, configurationRoot);
143+
}
142144
}
143145
}

‎src/Stats.ImportAzureCdnStatistics/LogFileProcessor.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ public LogFileProcessor(
3131

3232
_warehouse = warehouse ?? throw new ArgumentNullException(nameof(warehouse));
3333
_statisticsBlobContainerUtility = statisticsBlobContainerUtility ?? throw new ArgumentNullException(nameof(statisticsBlobContainerUtility));
34-
_logger = loggerFactory.CreateLogger<Job>();
34+
_logger = loggerFactory.CreateLogger<ImportAzureCdnStatisticsJob>();
3535
}
3636

3737
public async Task ProcessLogFileAsync(ILeasedLogFile logFile, IPackageStatisticsParser packageStatisticsParser, bool aggregatesOnly = false)

‎src/Stats.ImportAzureCdnStatistics/Program.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ public class Program
99
{
1010
public static void Main(string[] args)
1111
{
12-
var job = new Job();
12+
var job = new ImportAzureCdnStatisticsJob();
1313
JobRunner.Run(job, args).Wait();
1414
}
1515
}

‎src/Stats.ImportAzureCdnStatistics/Scripts/Stats.ImportAzureCdnStatistics.cmd‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,16 @@
33
cd bin
44

55
:Top
6-
echo "Starting job - #{Jobs.stats.importazurecdnstatistics.Title}"
6+
echo "Starting job - #{Jobs.stats.importazurecdnstatistics.Title}"
77

8-
title #{Jobs.stats.importazurecdnstatistics.Title}
8+
title #{Jobs.stats.importazurecdnstatistics.Title}
99

10-
start /w stats.importazurecdnstatistics.exe -VaultName "#{Deployment.Azure.KeyVault.VaultName}" -ClientId "#{Deployment.Azure.KeyVault.ClientId}" -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" -AzureCdnCloudStorageAccount "#{Jobs.stats.importazurecdnstatistics.AzureCdn.CloudStorageAccount}" -AzureCdnCloudStorageContainerName "#{Jobs.stats.importazurecdnstatistics.AzureCdn.CloudStorageContainerName}" -AzureCdnPlatform "#{Jobs.stats.importazurecdnstatistics.AzureCdn.Platform}" -AzureCdnAccountNumber "#{Jobs.stats.importazurecdnstatistics.AzureCdn.AccountNumber}" -StatisticsDatabase "#{Jobs.stats.importazurecdnstatistics.StatisticsDatabase}" -InstrumentationKey "#{Jobs.stats.importazurecdnstatistics.InstrumentationKey}" -verbose true -Interval #{Jobs.stats.importazurecdnstatistics.Interval}
10+
start /w stats.importazurecdnstatistics.exe ^
11+
-Configuration "#{Jobs.stats.importazurecdnstatistics.Configuration}" ^
12+
-InstrumentationKey "#{Jobs.stats.importazurecdnstatistics.InstrumentationKey}" ^
13+
-Interval #{Jobs.stats.importazurecdnstatistics.Interval} ^
14+
-verbose true
1115

12-
echo "Finished #{Jobs.stats.importazurecdnstatistics.Title}"
16+
echo "Finished #{Jobs.stats.importazurecdnstatistics.Title}"
1317

14-
goto Top
18+
goto Top

‎src/Stats.ImportAzureCdnStatistics/Settings/dev.json‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"Initialization": {
3+
"AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$",
4+
"AzureCdnCloudStorageContainerName": "nuget-cdnlogs-raw",
5+
"AzureCdnPlatform": "HttpLargeObject",
6+
"AzureCdnAccountNumber": "$$Dev-AzureCdn-AccountNumber$$"
7+
},
8+
9+
"StatisticsDb": {
10+
"ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-dev-statistics;User ID=$$Dev-StatisticsDBWriter-UserName$$;Password=$$Dev-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
11+
},
12+
13+
"KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
14+
"KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
15+
"KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",
16+
"KeyVault_ValidateCertificate": true,
17+
"KeyVault_StoreName": "My",
18+
"KeyVault_StoreLocation": "LocalMachine"
19+
}

‎src/Stats.ImportAzureCdnStatistics/Settings/int.json‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"Initialization": {
3+
"AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$",
4+
"AzureCdnCloudStorageContainerName": "nuget-cdnlogs-raw",
5+
"AzureCdnPlatform": "HttpLargeObject",
6+
"AzureCdnAccountNumber": "$$Int-AzureCdn-AccountNumber$$"
7+
},
8+
9+
"StatisticsDb": {
10+
"ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-int-statistics;User ID=$$Int-StatisticsDBWriter-UserName$$;Password=$$Int-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
11+
},
12+
13+
"KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
14+
"KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
15+
"KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",
16+
"KeyVault_ValidateCertificate": true,
17+
"KeyVault_StoreName": "My",
18+
"KeyVault_StoreLocation": "LocalMachine"
19+
}

‎src/Stats.ImportAzureCdnStatistics/Settings/prod.json‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"Initialization": {
3+
"AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetgallery;AccountKey=$$Prod-NuGetGalleryStorage-Key$$",
4+
"AzureCdnCloudStorageContainerName": "nuget-cdnlogs-raw",
5+
"AzureCdnPlatform": "HttpLargeObject",
6+
"AzureCdnAccountNumber": "$$Prod-AzureCdn-AccountNumber$$"
7+
},
8+
9+
"StatisticsDb": {
10+
"ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-prod-statistics;User ID=$$Prod-StatisticsDBWriter-UserName$$;Password=$$Prod-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
11+
},
12+
13+
"KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
14+
"KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
15+
"KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",
16+
"KeyVault_ValidateCertificate": true,
17+
"KeyVault_StoreName": "My",
18+
"KeyVault_StoreLocation": "LocalMachine"
19+
}

‎src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
</ItemGroup>
5050
<ItemGroup>
5151
<Compile Include="ApplicationInsightsHelper.cs" />
52+
<Compile Include="Configuration\ImportAzureCdnStatisticsConfiguration.cs" />
5253
<Compile Include="IPackageStatisticsParser.cs" />
5354
<Compile Include="IStatisticsBlobContainerUtility.cs" />
5455
<Compile Include="PackageDefinition.cs" />
@@ -70,7 +71,7 @@
7071
<Compile Include="Dimensions\PlatformDimension.cs" />
7172
<Compile Include="DataImporter.cs" />
7273
<Compile Include="Dimensions\DateDimension.cs" />
73-
<Compile Include="Job.cs" />
74+
<Compile Include="ImportAzureCdnStatisticsJob.cs" />
7475
<Compile Include="Dimensions\PackageDimension.cs" />
7576
<Compile Include="Program.cs" />
7677
<Compile Include="Properties\AssemblyInfo.cs" />
@@ -91,6 +92,9 @@
9192
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
9293
</None>
9394
<None Include="Scripts\*" />
95+
<None Include="Settings\dev.json" />
96+
<None Include="Settings\int.json" />
97+
<None Include="Settings\prod.json" />
9498
<None Include="Stats.ImportAzureCdnStatistics.nuspec" />
9599
</ItemGroup>
96100
<ItemGroup>
@@ -122,12 +126,6 @@
122126
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions">
123127
<Version>1.0.0</Version>
124128
</PackageReference>
125-
<PackageReference Include="NuGet.Services.Logging">
126-
<Version>2.25.0</Version>
127-
</PackageReference>
128-
<PackageReference Include="NuGet.Services.Sql">
129-
<Version>2.27.0</Version>
130-
</PackageReference>
131129
<PackageReference Include="NuGet.Versioning">
132130
<Version>4.3.0-preview1-2524</Version>
133131
</PackageReference>

0 commit comments

Comments
 (0)