Migrate Stats.AggregateCdnDownloads to JsonConfig (#523)

Christy Henriksson · web-flow · commit 624d0ecc3280 · 2018-08-08T13:26:05.000-07:00
diff --git a/src/NuGet.Jobs.Common/Extensions/DapperExtensions.cs b/src/NuGet.Jobs.Common/Extensions/DapperExtensions.cs
@@ -2,7 +2,6 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System.Collections.Generic;
-using System.Linq;
 using System.Threading.Tasks;
 using Dapper;
 
diff --git a/src/Stats.AggregateCdnDownloadsInGallery/AggregateCdnDownloadsJob.cs b/src/Stats.AggregateCdnDownloadsInGallery/AggregateCdnDownloadsJob.cs
@@ -9,16 +9,18 @@
 using System.Diagnostics;
 using System.Linq;
 using System.Threading.Tasks;
+using Autofac;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 using NuGet.Jobs;
-using NuGet.Services.KeyVault;
-using NuGet.Services.Sql;
+using NuGet.Jobs.Configuration;
 using IPackageIdGroup = System.Linq.IGrouping<string, Stats.AggregateCdnDownloadsInGallery.DownloadCountData>;
 
 namespace Stats.AggregateCdnDownloadsInGallery
 {
-    public class Job
-        : JobBase
+    public class AggregateCdnDownloadsJob : JsonConfigurationJob
     {
         private const int _defaultBatchSize = 5000;
         private const int _defaultBatchSleepSeconds = 10;
@@ -55,40 +57,35 @@ GROUP BY Stats.[PackageRegistrationKey]
             DROP TABLE #AggregateCdnDownloadsInGallery";
 
         private const string _storedProcedureName = "[dbo].[SelectTotalDownloadCountsPerPackageVersion]";
-        private ISqlConnectionFactory _statisticsDbConnectionFactory;
-        private ISqlConnectionFactory _galleryDbConnectionFactory;
-        private int _batchSize;
-        private int _batchSleepSeconds;
+
+        private AggregateCdnDownloadsConfiguration _configuration;
 
         public override void Init(IServiceContainer serviceContainer, IDictionary<string, string> jobArgsDictionary)
         {
-            var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector));
-
-            var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase);
-            _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector);
-
-            var galleryDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DestinationDatabase);
-            _galleryDbConnectionFactory = new AzureSqlConnectionFactory(galleryDbConnectionString, secretInjector);
+            base.Init(serviceContainer, jobArgsDictionary);
 
-            _batchSize = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.BatchSize) ?? _defaultBatchSize;
-            _batchSleepSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.BatchSleepSeconds) ?? _defaultBatchSleepSeconds;
+            _configuration = _serviceProvider.GetRequiredService<IOptionsSnapshot<AggregateCdnDownloadsConfiguration>>().Value;
         }
 
         public override async Task Run()
         {
             // Gather download counts data from statistics warehouse
             IReadOnlyList<DownloadCountData> downloadData;
-            Logger.LogInformation("Using batch size {BatchSize} and batch sleep seconds {BatchSleepSeconds}.", _batchSize, _batchSleepSeconds);
-            Logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", _statisticsDbConnectionFactory.DataSource, _statisticsDbConnectionFactory.InitialCatalog);
+            Logger.LogInformation("Using batch size {BatchSize} and batch sleep seconds {BatchSleepSeconds}.",
+                _configuration.BatchSize,
+                _configuration.BatchSleepSeconds);
+
             var stopwatch = Stopwatch.StartNew();
 
-            using (var statisticsDatabase = await _statisticsDbConnectionFactory.CreateAsync())
-            using (var statisticsDatabaseTransaction = statisticsDatabase.BeginTransaction(IsolationLevel.Snapshot))
+            using (var connection = await OpenSqlConnectionAsync<StatisticsDbConfiguration>())
+            using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot))
             {
+                Logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", connection.DataSource, connection.Database);
+
                 downloadData = (
-                    await statisticsDatabase.QueryWithRetryAsync<DownloadCountData>(
+                    await connection.QueryWithRetryAsync<DownloadCountData>(
                         _storedProcedureName,
-                        transaction: statisticsDatabaseTransaction,
+                        transaction: transaction,
                         commandType: CommandType.StoredProcedure,
                         commandTimeout: TimeSpan.FromMinutes(15),
                         maxRetries: 3))
@@ -106,10 +103,10 @@ await statisticsDatabase.QueryWithRetryAsync<DownloadCountData>(
                 return;
             }
 
-            using (var destinationDatabase = await _galleryDbConnectionFactory.CreateAsync())
+            using (var connection = await OpenSqlConnectionAsync<GalleryDbConfiguration>())
             {
                 // Fetch package registrations so we can match package ID to package registration key.
-                var packageRegistrationLookup = await GetPackageRegistrations(destinationDatabase);
+                var packageRegistrationLookup = await GetPackageRegistrations(connection);
 
                 // Group based on package ID and store in a stack for easy incremental processing.
                 var allGroups = downloadData.GroupBy(p => p.PackageId).ToList();
@@ -126,18 +123,18 @@ await statisticsDatabase.QueryWithRetryAsync<DownloadCountData>(
                 while (remainingGroups.Any())
                 {
                     // Create a batch of one or more package registrations to update.
-                    var batch = PopGroupBatch(remainingGroups, _batchSize);
+                    var batch = PopGroupBatch(remainingGroups, _configuration.BatchSize);
 
-                    await ProcessBatch(batch, destinationDatabase, packageRegistrationLookup);
+                    await ProcessBatch(batch, connection, packageRegistrationLookup);
 
                     Logger.LogInformation(
                         "There are {GroupCount} package registration groups remaining.",
                         remainingGroups.Count);
 
                     if (remainingGroups.Any())
                     {
-                        Logger.LogInformation("Sleeping for {BatchSleepSeconds} seconds before continuing.", _batchSleepSeconds);
-                        await Task.Delay(TimeSpan.FromSeconds(_batchSleepSeconds));
+                        Logger.LogInformation("Sleeping for {BatchSleepSeconds} seconds before continuing.", _configuration.BatchSleepSeconds);
+                        await Task.Delay(TimeSpan.FromSeconds(_configuration.BatchSleepSeconds));
                     }
                 }
 
@@ -305,5 +302,14 @@ private async Task<IDictionary<string, string>> GetPackageRegistrations(SqlConne
 
             return packageRegistrationDictionary;
         }
+
+        protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder)
+        {
+        }
+
+        protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot)
+        {
+            ConfigureInitializationSection<AggregateCdnDownloadsConfiguration>(services, configurationRoot);
+        }
     }
 }
diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Configuration/AggregateCdnDownloadsConfiguration.cs b/src/Stats.AggregateCdnDownloadsInGallery/Configuration/AggregateCdnDownloadsConfiguration.cs
@@ -0,0 +1,12 @@
+﻿// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace Stats.AggregateCdnDownloadsInGallery
+{
+    public class AggregateCdnDownloadsConfiguration
+    {
+        public int BatchSize { get; set; }
+
+        public int BatchSleepSeconds { get; set; }
+    }
+}
diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Program.cs b/src/Stats.AggregateCdnDownloadsInGallery/Program.cs
@@ -9,7 +9,7 @@ public class Program
     {
         public static void Main(string[] args)
         {
-            var job = new Job();
+            var job = new AggregateCdnDownloadsJob();
             JobRunner.Run(job, args).Wait();
         }
     }
diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Scripts/Stats.AggregateCdnDownloadsInGallery.cmd b/src/Stats.AggregateCdnDownloadsInGallery/Scripts/Stats.AggregateCdnDownloadsInGallery.cmd
@@ -8,14 +8,8 @@ echo "Starting job - #{Jobs.stats.aggregatecdndownloadsingallery.Title}"
 title #{Jobs.stats.aggregatecdndownloadsingallery.Title}
 
 start /w stats.aggregatecdndownloadsingallery.exe ^
-	-VaultName "#{Deployment.Azure.KeyVault.VaultName}" ^
-	-ClientId "#{Deployment.Azure.KeyVault.ClientId}" ^
-	-CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" ^
-	-StatisticsDatabase "#{Jobs.stats.aggregatecdndownloadsingallery.StatisticsDatabase}" ^
-	-DestinationDatabase "#{Jobs.stats.aggregatecdndownloadsingallery.DestinationDatabase}" ^
+	-Configuration "#{Jobs.stats.aggregatecdndownloadsingallery.Configuration}" ^
 	-InstrumentationKey "#{Jobs.stats.aggregatecdndownloadsingallery.InstrumentationKey}" ^
-	-BatchSize "#{Jobs.stats.aggregatecdndownloadsingallery.BatchSize}" ^
-	-BatchSleepSeconds "#{Jobs.stats.aggregatecdndownloadsingallery.BatchSleepSeconds}" ^
 	-verbose true ^
 	-Interval #{Jobs.stats.aggregatecdndownloadsingallery.Interval}
 
diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Settings/dev.json b/src/Stats.AggregateCdnDownloadsInGallery/Settings/dev.json
@@ -0,0 +1,21 @@
+{
+  "Initialization": {
+    "BatchSize": 5000,
+    "BatchSleepSeconds": 10
+  },
+
+  "GalleryDb": {
+    "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;User ID=$$Dev-GalleryDBWriter-UserName$$;Password=$$Dev-GalleryDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
+  },
+
+  "StatisticsDb": {
+    "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-dev-statistics;User ID=$$Dev-StatisticsDBWriter-UserName$$;Password=$$Dev-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
+  },
+
+  "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
+  "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
+  "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",
+  "KeyVault_ValidateCertificate": true,
+  "KeyVault_StoreName": "My",
+  "KeyVault_StoreLocation": "LocalMachine"
+}
diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Settings/int.json b/src/Stats.AggregateCdnDownloadsInGallery/Settings/int.json
@@ -0,0 +1,21 @@
+{
+  "Initialization": {
+    "BatchSize": 5000,
+    "BatchSleepSeconds": 10
+  },
+
+  "GalleryDb": {
+    "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-int-0-v2gallery;User ID=$$Int-GalleryDBWriter-UserName$$;Password=$$Int-GalleryDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
+  },
+
+  "StatisticsDb": {
+    "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-int-statistics;User ID=$$Int-StatisticsDBWriter-UserName$$;Password=$$Int-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
+  },
+
+  "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
+  "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
+  "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",
+  "KeyVault_ValidateCertificate": true,
+  "KeyVault_StoreName": "My",
+  "KeyVault_StoreLocation": "LocalMachine"
+}
diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Settings/prod.json b/src/Stats.AggregateCdnDownloadsInGallery/Settings/prod.json
@@ -0,0 +1,21 @@
+{
+  "Initialization": {
+    "BatchSize": 5000,
+    "BatchSleepSeconds": 10
+  },
+
+  "GalleryDb": {
+    "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=NuGetGallery;User ID=$$Prod-GalleryDBWriter-UserName$$;Password=$$Prod-GalleryDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
+  },
+
+  "StatisticsDb": {
+    "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-prod-statistics;User ID=$$Prod-StatisticsDBWriter-UserName$$;Password=$$Prod-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
+  },
+
+  "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
+  "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
+  "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",
+  "KeyVault_ValidateCertificate": true,
+  "KeyVault_StoreName": "My",
+  "KeyVault_StoreLocation": "LocalMachine"
+}
diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj b/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj
@@ -42,15 +42,17 @@
     <Reference Include="Microsoft.CSharp" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="Configuration\AggregateCdnDownloadsConfiguration.cs" />
     <Compile Include="DownloadCountData.cs" />
-    <Compile Include="Job.cs" />
+    <Compile Include="AggregateCdnDownloadsJob.cs" />
     <Compile Include="PackageRegistrationData.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="App.config" />
     <None Include="Scripts\*" />
+    <None Include="Settings\*" />
     <None Include="Stats.AggregateCdnDownloadsInGallery.nuspec" />
   </ItemGroup>
   <ItemGroup>
@@ -75,9 +77,6 @@
     <PackageReference Include="Newtonsoft.Json">
       <Version>9.0.1</Version>
     </PackageReference>
-    <PackageReference Include="NuGet.Services.Sql">
-      <Version>2.27.0</Version>
-    </PackageReference>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <Import Project="..\..\build\sign.targets" Condition="Exists('..\..\build\sign.targets')" />
diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.nuspec b/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.nuspec
@@ -18,5 +18,7 @@
     <file src="Scripts\PreDeploy.ps1" />
     <file src="Scripts\PostDeploy.ps1" />
     <file src="Scripts\nssm.exe" />
+
+    <file src="Settings\*.json" target="bin" />
   </files>
 </package>
diff --git a/tests/Tests.Stats.AggregateCdnDownloadsInGallery/AggregateCdnDownloadsJobTests.cs b/tests/Tests.Stats.AggregateCdnDownloadsInGallery/AggregateCdnDownloadsJobTests.cs
@@ -8,7 +8,7 @@
 
 namespace Stats.AggregateCdnDownloadsInGallery
 {
-    public class JobTests
+    public class AggregateCdnDownloadsJobTests
     {
 
         public class PopGroupBatch
@@ -55,7 +55,7 @@ public void ReturnsCorrectNumberOfPackageRegistrationGroups(int batchSize, int e
                 }.GroupBy(x => x.PackageId).OrderByDescending(x => x.Key));
 
                 // Act
-                var groupBatch = Job.PopGroupBatch(data, batchSize);
+                var groupBatch = AggregateCdnDownloadsJob.PopGroupBatch(data, batchSize);
 
                 // Assert
                 Assert.Equal(expectedGroupCount, groupBatch.Count);
diff --git a/tests/Tests.Stats.AggregateCdnDownloadsInGallery/Tests.Stats.AggregateCdnDownloadsInGallery.csproj b/tests/Tests.Stats.AggregateCdnDownloadsInGallery/Tests.Stats.AggregateCdnDownloadsInGallery.csproj
@@ -45,7 +45,7 @@
     <Reference Include="System.Xml" />
   </ItemGroup>
   <ItemGroup>
-    <Compile Include="JobTests.cs" />
+    <Compile Include="AggregateCdnDownloadsJobTests.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>

Original file line number	Diff line number	Diff line change
`@@ -9,7 +9,7 @@ public class Program`
`9`	`9`	`{`
`10`	`10`	`public static void Main(string[] args)`
`11`	`11`	`{`
`12`		`- var job = new Job();`
	`12`	`+ var job = new AggregateCdnDownloadsJob();`
`13`	`13`	`JobRunner.Run(job, args).Wait();`
`14`	`14`	`}`
`15`	`15`	`}`