Skip to content
This repository was archived by the owner on Jul 30, 2024. It is now read-only.

Commit 626499d

Browse files
author
Christy Henriksson
authored
Migrate CreateAzureCdnWarehouseReports to use JsonConfig (#526)
1 parent a443527 commit 626499d

23 files changed

Lines changed: 387 additions & 150 deletions

‎src/Stats.CreateAzureCdnWarehouseReports/Configuration/CreateAzureCdnWarehouseReportsConfiguration.cs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
namespace Stats.CreateAzureCdnWarehouseReports
5+
{
6+
public class CreateAzureCdnWarehouseReportsConfiguration
7+
{
8+
public string AzureCdnCloudStorageAccount { get; set; }
9+
10+
public string AzureCdnCloudStorageContainerName { get; set; }
11+
12+
public string DataStorageAccount { get; set; }
13+
14+
public string DataContainerName { get; set; }
15+
16+
public int? CommandTimeOut { get; set; }
17+
18+
public int? PerPackageReportDegreeOfParallelism { get; set; }
19+
20+
public string ReportName { get; set; }
21+
}
22+
}

src/Stats.CreateAzureCdnWarehouseReports/Job.cs renamed to src/Stats.CreateAzureCdnWarehouseReports/CreateAzureCdnWarehouseReportsJob.cs

Lines changed: 131 additions & 54 deletions
Large diffs are not rendered by default.

‎src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
using Microsoft.Extensions.Logging;
1111
using Newtonsoft.Json;
1212
using Newtonsoft.Json.Linq;
13-
using NuGet.Services.Sql;
1413
using NuGet.Versioning;
1514

1615
namespace Stats.CreateAzureCdnWarehouseReports
@@ -25,22 +24,23 @@ public class DownloadCountReport
2524
public DownloadCountReport(
2625
ILogger<DownloadCountReport> logger,
2726
IEnumerable<StorageContainerTarget> targets,
28-
ISqlConnectionFactory statisticsDbConnectionFactory,
29-
ISqlConnectionFactory galleryDbConnectionFactory)
30-
: base(logger, targets, statisticsDbConnectionFactory, galleryDbConnectionFactory)
27+
Func<Task<SqlConnection>> openStatisticsSqlConnectionAsync,
28+
Func<Task<SqlConnection>> openGallerySqlConnectionAsync)
29+
: base(logger, targets, openStatisticsSqlConnectionAsync, openGallerySqlConnectionAsync)
3130
{
3231
}
3332

3433
public async Task Run()
3534
{
3635
// Gather download count data from statistics warehouse
3736
IReadOnlyCollection<DownloadCountData> downloadData;
38-
_logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...",
39-
StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog);
4037

41-
using (var connection = await StatisticsDbConnectionFactory.CreateAsync())
38+
using (var connection = await OpenStatisticsSqlConnectionAsync())
4239
using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot))
4340
{
41+
_logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...",
42+
connection.DataSource, connection.Database);
43+
4444
downloadData = (await connection.QueryWithRetryAsync<DownloadCountData>(
4545
_storedProcedureName, commandType: CommandType.StoredProcedure, transaction: transaction, commandTimeout: _defaultCommandTimeout)).ToList();
4646
}

‎src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
using Microsoft.WindowsAzure.Storage;
1212
using Newtonsoft.Json;
1313
using Newtonsoft.Json.Linq;
14-
using NuGet.Services.Sql;
1514

1615
namespace Stats.CreateAzureCdnWarehouseReports
1716
{
@@ -26,23 +25,24 @@ public DownloadsPerToolVersionReport(
2625
ILogger<DownloadsPerToolVersionReport> logger,
2726
CloudStorageAccount cloudStorageAccount,
2827
string statisticsContainerName,
29-
ISqlConnectionFactory statisticsDbConnectionFactory,
30-
ISqlConnectionFactory galleryDbConnectionFactory)
28+
Func<Task<SqlConnection>> openStatisticsSqlConnectionAsync,
29+
Func<Task<SqlConnection>> openGallerySqlConnectionAsync)
3130
: base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) },
32-
statisticsDbConnectionFactory, galleryDbConnectionFactory)
31+
openStatisticsSqlConnectionAsync, openGallerySqlConnectionAsync)
3332
{
3433
}
3534

3635
public async Task Run()
3736
{
3837
// Gather download count data from statistics warehouse
3938
IReadOnlyCollection<ToolDownloadCountData> data;
40-
_logger.LogInformation("Gathering Tools Download Counts from {DataSource}/{InitialCatalog}...",
41-
StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog);
4239

43-
using (var connection = await StatisticsDbConnectionFactory.CreateAsync())
40+
using (var connection = await OpenStatisticsSqlConnectionAsync())
4441
using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot))
4542
{
43+
_logger.LogInformation("Gathering Tools Download Counts from {DataSource}/{InitialCatalog}...",
44+
connection.DataSource, connection.Database);
45+
4646
data = (await connection.QueryWithRetryAsync<ToolDownloadCountData>(
4747
_storedProcedureName, commandType: CommandType.StoredProcedure, transaction: transaction, commandTimeout: _defaultCommandTimeout)).ToList();
4848
}

‎src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
using Microsoft.Extensions.Logging;
1010
using Microsoft.WindowsAzure.Storage;
1111
using Newtonsoft.Json;
12-
using NuGet.Services.Sql;
1312

1413
namespace Stats.CreateAzureCdnWarehouseReports
1514
{
@@ -27,23 +26,24 @@ public GalleryTotalsReport(
2726
ILogger<GalleryTotalsReport> logger,
2827
CloudStorageAccount cloudStorageAccount,
2928
string statisticsContainerName,
30-
ISqlConnectionFactory statisticsDbConnectionFactory,
31-
ISqlConnectionFactory galleryDbConnectionFactory)
29+
Func<Task<SqlConnection>> openStatisticsSqlConnectionAsync,
30+
Func<Task<SqlConnection>> openGallerySqlConnectionAsync)
3231
: base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) },
33-
statisticsDbConnectionFactory, galleryDbConnectionFactory)
32+
openStatisticsSqlConnectionAsync, openGallerySqlConnectionAsync)
3433
{
3534
}
3635

3736
public async Task Run()
3837
{
3938
// gather package numbers from gallery database
4039
GalleryTotalsData totalsData;
41-
_logger.LogInformation("Gathering Gallery Totals from {GalleryDataSource}/{GalleryInitialCatalog}...",
42-
GalleryDbConnectionFactory.DataSource, GalleryDbConnectionFactory.InitialCatalog);
4340

44-
using (var connection = await GalleryDbConnectionFactory.CreateAsync())
41+
using (var connection = await OpenGallerySqlConnectionAsync())
4542
using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot))
4643
{
44+
_logger.LogInformation("Gathering Gallery Totals from {GalleryDataSource}/{GalleryInitialCatalog}...",
45+
connection.DataSource, connection.Database);
46+
4747
totalsData = (await connection.QueryWithRetryAsync<GalleryTotalsData>(
4848
GalleryQuery, commandType: CommandType.Text, transaction: transaction)).First();
4949
}
@@ -52,12 +52,12 @@ public async Task Run()
5252
_logger.LogInformation("Unique packages: {UniquePackagesCount}", totalsData.UniquePackages);
5353

5454
// gather download count data from statistics warehouse
55-
_logger.LogInformation("Gathering Gallery Totals from {StatisticsDataSource}/{StatisticsInitialCatalog}...",
56-
StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog);
57-
58-
using (var connection = await StatisticsDbConnectionFactory.CreateAsync())
55+
using (var connection = await OpenStatisticsSqlConnectionAsync())
5956
using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot))
6057
{
58+
_logger.LogInformation("Gathering Gallery Totals from {StatisticsDataSource}/{StatisticsInitialCatalog}...",
59+
connection.DataSource, connection.Database);
60+
6161
totalsData.Downloads = (await connection.ExecuteScalarWithRetryAsync<long>(
6262
WarehouseStoredProcedureName,
6363
commandType: CommandType.StoredProcedure,

‎src/Stats.CreateAzureCdnWarehouseReports/Program.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ class Program
99
{
1010
static void Main(string[] args)
1111
{
12-
var job = new Job();
12+
var job = new CreateAzureCdnWarehouseReportsJob();
1313
JobRunner.Run(job, args).Wait();
1414
}
1515
}

‎src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@
33

44
using System;
55
using System.Collections.Generic;
6+
using System.Data.SqlClient;
67
using System.Linq;
78
using System.Threading.Tasks;
89
using Microsoft.Extensions.Logging;
910
using Microsoft.WindowsAzure.Storage.Blob;
1011
using Microsoft.WindowsAzure.Storage.RetryPolicies;
11-
using NuGet.Services.Sql;
1212

1313
namespace Stats.CreateAzureCdnWarehouseReports
1414
{
@@ -18,20 +18,20 @@ public abstract class ReportBase
1818

1919
protected readonly IReadOnlyCollection<StorageContainerTarget> Targets;
2020

21-
protected readonly ISqlConnectionFactory StatisticsDbConnectionFactory;
21+
protected readonly Func<Task<SqlConnection>> OpenStatisticsSqlConnectionAsync;
2222

23-
protected ISqlConnectionFactory GalleryDbConnectionFactory;
23+
protected Func<Task<SqlConnection>> OpenGallerySqlConnectionAsync;
2424

2525
protected ReportBase(
2626
ILogger<ReportBase> logger,
2727
IEnumerable<StorageContainerTarget> targets,
28-
ISqlConnectionFactory statisticsDbConnectionFactory,
29-
ISqlConnectionFactory galleryDbConnectionFactory)
28+
Func<Task<SqlConnection>> openStatisticsSqlConnectionAsync,
29+
Func<Task<SqlConnection>> openGallerySqlConnectionAsync)
3030
{
3131
_logger = logger;
3232
Targets = targets.ToList().AsReadOnly();
33-
StatisticsDbConnectionFactory = statisticsDbConnectionFactory;
34-
GalleryDbConnectionFactory = galleryDbConnectionFactory;
33+
OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync;
34+
OpenGallerySqlConnectionAsync = openGallerySqlConnectionAsync;
3535
}
3636

3737
protected async Task<CloudBlobContainer> GetBlobContainer(StorageContainerTarget target)

‎src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs‎

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,27 +8,26 @@
88
using System.Diagnostics;
99
using System.Threading.Tasks;
1010
using Microsoft.Extensions.Logging;
11-
using NuGet.Services.Sql;
1211

1312
namespace Stats.CreateAzureCdnWarehouseReports
1413
{
1514
internal class ReportDataCollector
1615
{
1716
private int _commandTimeoutSeconds;
1817
private readonly string _procedureName;
19-
private readonly ISqlConnectionFactory _sourceDbConnectionFactory;
18+
private readonly Func<Task<SqlConnection>> _openGallerySqlConnectionAsync;
2019

2120
private ILogger<ReportDataCollector> _logger;
2221

2322
public ReportDataCollector(
2423
ILogger<ReportDataCollector> logger,
2524
string procedureName,
26-
ISqlConnectionFactory sourceDbConnectionFactory,
25+
Func<Task<SqlConnection>> openGallerySqlConnectionAsync,
2726
int timeout)
2827
{
2928
_logger = logger;
3029
_procedureName = procedureName;
31-
_sourceDbConnectionFactory = sourceDbConnectionFactory;
30+
_openGallerySqlConnectionAsync = openGallerySqlConnectionAsync;
3231
_commandTimeoutSeconds = timeout;
3332
}
3433

@@ -48,7 +47,7 @@ public async Task<DataTable> CollectAsync(DateTime reportGenerationTime, params
4847

4948
public static async Task<IReadOnlyCollection<DirtyPackageId>> GetDirtyPackageIds(
5049
ILogger logger,
51-
ISqlConnectionFactory sourceDbConnectionFactory,
50+
Func<Task<SqlConnection>> openGallerySqlConnectionAsync,
5251
DateTime reportGenerationTime,
5352
int commandTimeout)
5453
{
@@ -57,19 +56,19 @@ public static async Task<IReadOnlyCollection<DirtyPackageId>> GetDirtyPackageIds
5756
IReadOnlyCollection<DirtyPackageId> packageIds = new List<DirtyPackageId>();
5857

5958
// Get the data
60-
await WithRetry(async () => packageIds = await GetDirtyPackageIdsFromWarehouse(sourceDbConnectionFactory, reportGenerationTime, commandTimeout), logger);
59+
await WithRetry(async () => packageIds = await GetDirtyPackageIdsFromWarehouse(openGallerySqlConnectionAsync, reportGenerationTime, commandTimeout), logger);
6160

6261
logger.LogInformation("Found {DirtyPackagesCount} dirty packages to update.", packageIds.Count);
6362

6463
return packageIds;
6564
}
6665

6766
public static async Task<IReadOnlyCollection<string>> ListInactivePackageIdReports(
68-
ISqlConnectionFactory sourceDbConnectionFactory,
67+
Func<Task<SqlConnection>> openGallerySqlConnectionAsync,
6968
DateTime reportGenerationTime,
7069
int commandTimeout)
7170
{
72-
using (var connection = await sourceDbConnectionFactory.CreateAsync())
71+
using (var connection = await openGallerySqlConnectionAsync())
7372
{
7473
var command = new SqlCommand("[dbo].[DownloadReportListInactive]", connection);
7574
command.CommandType = CommandType.StoredProcedure;
@@ -122,7 +121,7 @@ private static async Task WithRetry(Func<Task> action, ILogger logger)
122121

123122
private async Task<DataTable> ExecuteSql(DateTime reportGenerationTime, params Tuple<string, int, string>[] parameters)
124123
{
125-
using (var connection = await _sourceDbConnectionFactory.CreateAsync())
124+
using (var connection = await _openGallerySqlConnectionAsync())
126125
{
127126
var command = new SqlCommand(_procedureName, connection);
128127
command.CommandType = CommandType.StoredProcedure;
@@ -146,11 +145,11 @@ private async Task<DataTable> ExecuteSql(DateTime reportGenerationTime, params T
146145
}
147146

148147
private static async Task<IReadOnlyCollection<DirtyPackageId>> GetDirtyPackageIdsFromWarehouse(
149-
ISqlConnectionFactory sourceDbConnectionFactory,
148+
Func<Task<SqlConnection>> openGallerySqlConnectionAsync,
150149
DateTime reportGenerationTime,
151150
int commandTimeout)
152151
{
153-
using (var connection = await sourceDbConnectionFactory.CreateAsync())
152+
using (var connection = await openGallerySqlConnectionAsync())
154153
{
155154
var command = new SqlCommand("[dbo].[GetDirtyPackageIds]", connection);
156155
command.CommandType = CommandType.StoredProcedure;
@@ -172,11 +171,11 @@ private static async Task<IReadOnlyCollection<DirtyPackageId>> GetDirtyPackageId
172171
}
173172

174173
public static async Task UpdateDirtyPackageIdCursor(
175-
ISqlConnectionFactory sourceDbConnectionFactory,
174+
Func<Task<SqlConnection>> openGallerySqlConnectionAsync,
176175
DateTime runToCursor,
177176
int commandTimeout)
178177
{
179-
using (var connection = await sourceDbConnectionFactory.CreateAsync())
178+
using (var connection = await openGallerySqlConnectionAsync())
180179
{
181180
var command = new SqlCommand("[dbo].[UpdateDirtyPackageIdCursor]", connection);
182181
command.CommandType = CommandType.StoredProcedure;

‎src/Stats.CreateAzureCdnWarehouseReports/Scripts/Stats.CreateAzureCdnWarehouseReports.cmd‎

Lines changed: 9 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -3,26 +3,16 @@
33
cd bin
44

55
:Top
6-
echo "Starting job - #{Jobs.stats.createazurecdnwarehousereports.Title}"
6+
echo "Starting job - #{Jobs.stats.createazurecdnwarehousereports.Title}"
77

8-
title #{Jobs.stats.createazurecdnwarehousereports.Title}
8+
title #{Jobs.stats.createazurecdnwarehousereports.Title}
99

10-
start /w stats.createazurecdnwarehousereports.exe ^
11-
-VaultName "#{Deployment.Azure.KeyVault.VaultName}" ^
12-
-ClientId "#{Deployment.Azure.KeyVault.ClientId}" ^
13-
-CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" ^
14-
-AzureCdnCloudStorageAccount "#{Jobs.stats.createazurecdnwarehousereports.AzureCdn.CloudStorageAccount}" ^
15-
-AzureCdnCloudStorageContainerName "#{Jobs.stats.createazurecdnwarehousereports.AzureCdn.CloudStorageContainerName}" ^
16-
-StatisticsDatabase "#{Jobs.stats.createazurecdnwarehousereports.StatisticsDatabase}" ^
17-
-SourceDatabase "#{Jobs.stats.createazurecdnwarehousereports.SourceDatabase}" ^
18-
-DataStorageAccount "#{Jobs.stats.createazurecdnwarehousereports.DataStorageAccount}" ^
19-
-InstrumentationKey "#{Jobs.stats.createazurecdnwarehousereports.InstrumentationKey}" ^
20-
-DataContainerName "#{Jobs.stats.createazurecdnwarehousereports.DataContainerName}" ^
21-
-CommandTimeOut "#{Jobs.stats.createazurecdnwarehousereports.CommandTimeOut}" ^
22-
-PerPackageReportDegreeOfParallelism "#{Jobs.stats.createazurecdnwarehousereports.PerPackageReportDegreeOfParallelism}" ^
23-
-verbose true ^
24-
-Interval #{Jobs.stats.createazurecdnwarehousereports.Interval}
10+
start /w stats.createazurecdnwarehousereports.exe ^
11+
-Configuration "#{Jobs.stats.createazurecdnwarehousereports.Configuration}"
12+
-InstrumentationKey "#{Jobs.stats.createazurecdnwarehousereports.InstrumentationKey}" ^
13+
-Interval #{Jobs.stats.createazurecdnwarehousereports.Interval} ^
14+
-verbose true
2515

26-
echo "Finished #{Jobs.stats.createazurecdnwarehousereports.Title}"
16+
echo "Finished #{Jobs.stats.createazurecdnwarehousereports.Title}"
2717

28-
goto Top
18+
goto Top

‎src/Stats.CreateAzureCdnWarehouseReports/Settings/dev.json‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"Initialization": {
3+
"AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$",
4+
"AzureCdnCloudStorageContainerName": "nuget-cdnstats",
5+
"DataStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetdev0;AccountKey=$$Dev-NuGetDev0Storage-Key$$",
6+
"DataContainerName": "ng-search-data",
7+
"CommandTimeOut": "7200",
8+
"PerPackageReportDegreeOfParallelism": "64"
9+
},
10+
11+
"GalleryDb": {
12+
"ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;User ID=$$Dev-GalleryDBReadOnly-UserName$$;Password=$$Dev-GalleryDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
13+
},
14+
15+
"StatisticsDb": {
16+
"ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-dev-statistics;User ID=$$Dev-StatisticsDBWriter-UserName$$;Password=$$Dev-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
17+
},
18+
19+
"KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
20+
"KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
21+
"KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",
22+
"KeyVault_ValidateCertificate": true,
23+
"KeyVault_StoreName": "My",
24+
"KeyVault_StoreLocation": "LocalMachine"
25+
}

0 commit comments

Comments
 (0)