Skip to content
This repository was archived by the owner on Jul 30, 2024. It is now read-only.

Commit 9537d56

Browse files
author
Christy Henriksson
authored
Merge pull request #685 from NuGet/chenriks-legacy
Convert CollectCdnLogs jobs to JsonConfig
2 parents c1801e1 + ea1dca0 commit 9537d56

23 files changed

Lines changed: 411 additions & 192 deletions

‎src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,9 @@ public class AzureStatsLogDestination : ILogDestination
2424
private CloudBlobClient _cloudBlobClient;
2525
private CloudBlobContainer _cloudBlobContainer;
2626

27-
public AzureStatsLogDestination(string connectionString, string containerName)
27+
public AzureStatsLogDestination(CloudStorageAccount storageAccount, string containerName)
2828
{
29-
_azureAccount = CloudStorageAccount.Parse(connectionString);
29+
_azureAccount = storageAccount;
3030
_cloudBlobClient = _azureAccount.CreateCloudBlobClient();
3131
_cloudBlobContainer = _cloudBlobClient.GetContainerReference(containerName);
3232
_cloudBlobContainer.CreateIfNotExists();

‎src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,9 @@ public class AzureStatsLogSource : ILogSource
3333
/// </summary>
3434
/// <param name="connectionString">The connection string for the Azure account.</param>
3535
/// <param name="containerName">The container name.</param>
36-
public AzureStatsLogSource(string connectionString, string containerName, int azureServerTimeoutInSeconds)
36+
public AzureStatsLogSource(CloudStorageAccount storageAccount, string containerName, int azureServerTimeoutInSeconds)
3737
{
38-
_azureAccount = CloudStorageAccount.Parse(connectionString);
38+
_azureAccount = storageAccount;
3939
_blobClient = _azureAccount.CreateCloudBlobClient();
4040
_container = _blobClient.GetContainerReference(containerName);
4141
_blobRequestOptions = new BlobRequestOptions();

‎src/Stats.CollectAzureCdnLogs/Configuration/CollectAzureCdnLogsConfiguration.cs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
namespace Stats.CollectAzureCdnLogs
5+
{
6+
public class CollectAzureCdnLogsConfiguration
7+
{
8+
public string AzureCdnAccountNumber { get; set; }
9+
10+
public string AzureCdnCloudStorageAccount { get; set; }
11+
12+
public string AzureCdnCloudStorageContainerName { get; set; }
13+
14+
public string AzureCdnPlatform { get; set; }
15+
16+
public string FtpSourceUri { get; set; }
17+
18+
public string FtpSourceUsername { get; set; }
19+
20+
public string FtpSourcePassword { get; set; }
21+
}
22+
}

‎src/Stats.CollectAzureCdnLogs/Job.cs‎

Lines changed: 50 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,13 @@
99
using System.Text;
1010
using System.Text.RegularExpressions;
1111
using System.Threading.Tasks;
12+
using Autofac;
1213
using ICSharpCode.SharpZipLib;
1314
using ICSharpCode.SharpZipLib.GZip;
15+
using Microsoft.Extensions.Configuration;
16+
using Microsoft.Extensions.DependencyInjection;
1417
using Microsoft.Extensions.Logging;
18+
using Microsoft.Extensions.Options;
1519
using Microsoft.WindowsAzure.Storage;
1620
using NuGet.Jobs;
1721
using Stats.AzureCdnLogs.Common;
@@ -20,31 +24,48 @@
2024

2125
namespace Stats.CollectAzureCdnLogs
2226
{
23-
public class Job
24-
: JobBase
27+
public class Job : JsonConfigurationJob
2528
{
2629
private static readonly DateTime _unixTimestamp = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
2730
private Uri _ftpServerUri;
28-
private string _ftpUsername;
29-
private string _ftpPassword;
30-
private string _azureCdnAccountNumber;
3131
private AzureCdnPlatform _azureCdnPlatform;
3232
private CloudStorageAccount _cloudStorageAccount;
33-
private string _cloudStorageContainerName;
33+
34+
private CollectAzureCdnLogsConfiguration _configuration;
3435

3536
public override void Init(IServiceContainer serviceContainer, IDictionary<string, string> jobArgsDictionary)
3637
{
37-
var ftpLogFolder = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.FtpSourceUri);
38-
var azureCdnPlatform = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnPlatform);
39-
var cloudStorageAccount = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount);
40-
_cloudStorageContainerName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName);
41-
_azureCdnAccountNumber = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnAccountNumber);
42-
_ftpUsername = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.FtpSourceUsername);
43-
_ftpPassword = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.FtpSourcePassword);
44-
45-
_ftpServerUri = ValidateFtpUri(ftpLogFolder);
46-
_azureCdnPlatform = ValidateAzureCdnPlatform(azureCdnPlatform);
47-
_cloudStorageAccount = ValidateAzureCloudStorageAccount(cloudStorageAccount);
38+
base.Init(serviceContainer, jobArgsDictionary);
39+
40+
InitializeJobConfiguration(_serviceProvider);
41+
}
42+
43+
public void InitializeJobConfiguration(IServiceProvider serviceProvider)
44+
{
45+
_configuration = serviceProvider.GetRequiredService<IOptionsSnapshot<CollectAzureCdnLogsConfiguration>>().Value;
46+
47+
if (string.IsNullOrEmpty(_configuration.AzureCdnAccountNumber))
48+
{
49+
throw new ArgumentException("Configuration 'AzureCdnAccountNumber' is required", nameof(_configuration));
50+
}
51+
52+
if (string.IsNullOrEmpty(_configuration.AzureCdnCloudStorageContainerName))
53+
{
54+
throw new ArgumentException("Configuration 'AzureCdnCloudStorageContainerName' is required", nameof(_configuration));
55+
}
56+
57+
if (string.IsNullOrEmpty(_configuration.FtpSourceUsername)) {
58+
throw new ArgumentException("Configuration 'FtpSourceUsername' is required", nameof(_configuration));
59+
}
60+
61+
if (string.IsNullOrEmpty(_configuration.FtpSourcePassword))
62+
{
63+
throw new ArgumentException("Configuration 'FtpSourcePassword' is required", nameof(_configuration));
64+
}
65+
66+
_cloudStorageAccount = ValidateAzureCloudStorageAccount(_configuration.AzureCdnCloudStorageAccount);
67+
_azureCdnPlatform = ValidateAzureCdnPlatform(_configuration.AzureCdnPlatform);
68+
_ftpServerUri = ValidateFtpUri(_configuration.FtpSourceUri);
4869
}
4970

5071
private static CloudStorageAccount ValidateAzureCloudStorageAccount(string cloudStorageAccount)
@@ -110,14 +131,14 @@ private static Uri ValidateFtpUri(string serverUrl)
110131

111132
public override async Task Run()
112133
{
113-
var ftpClient = new FtpRawLogClient(LoggerFactory, _ftpUsername, _ftpPassword);
134+
var ftpClient = new FtpRawLogClient(LoggerFactory, _configuration.FtpSourceUsername, _configuration.FtpSourcePassword);
114135
var azureClient = new CloudBlobRawLogClient(LoggerFactory, _cloudStorageAccount);
115136

116137
// Collect directory listing.
117138
var rawLogFileUris = await ftpClient.GetRawLogFileUris(_ftpServerUri);
118139

119140
// Prepare cloud storage blob container.
120-
var cloudBlobContainer = await azureClient.CreateContainerIfNotExistsAsync(_cloudStorageContainerName);
141+
var cloudBlobContainer = await azureClient.CreateContainerIfNotExistsAsync(_configuration.AzureCdnCloudStorageContainerName);
121142

122143
foreach (var rawLogFileUri in rawLogFileUris)
123144
{
@@ -126,7 +147,7 @@ public override async Task Run()
126147
var rawLogFile = new RawLogFileInfo(rawLogFileUri);
127148

128149
if (_azureCdnPlatform != rawLogFile.AzureCdnPlatform
129-
|| !_azureCdnAccountNumber.Equals(rawLogFile.AzureCdnAccountNumber, StringComparison.InvariantCultureIgnoreCase))
150+
|| !_configuration.AzureCdnAccountNumber.Equals(rawLogFile.AzureCdnAccountNumber, StringComparison.InvariantCultureIgnoreCase))
130151
{
131152
// Only process the raw log files matching the target CDN platform and account number.
132153
continue;
@@ -367,5 +388,14 @@ private static string ToUnixTimeStamp(DateTime dateTime)
367388
var secondsPastEpoch = (dateTime - _unixTimestamp).TotalSeconds;
368389
return secondsPastEpoch.ToString(CultureInfo.InvariantCulture);
369390
}
391+
392+
protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder)
393+
{
394+
}
395+
396+
protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot)
397+
{
398+
ConfigureInitializationSection<CollectAzureCdnLogsConfiguration>(services, configurationRoot);
399+
}
370400
}
371401
}

‎src/Stats.CollectAzureCdnLogs/Scripts/Stats.CollectAzureCdnLogs.cmd‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,16 @@
33
cd bin
44

55
:Top
6-
echo "Starting job - #{Jobs.stats.collectazurecdnlogs.Title}"
6+
echo "Starting job - #{Jobs.stats.collectazurecdnlogs.Title}"
77

8-
title #{Jobs.stats.collectazurecdnlogs.Title}
8+
title #{Jobs.stats.collectazurecdnlogs.Title}
99

10-
start /w stats.collectazurecdnlogs.exe -VaultName "#{Deployment.Azure.KeyVault.VaultName}" -ClientId "#{Deployment.Azure.KeyVault.ClientId}" -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" -FtpSourceUri "#{Jobs.stats.collectazurecdnlogs.FtpSource.Uri}" -FtpSourceUsername "#{Jobs.stats.collectazurecdnlogs.FtpSource.Username}" -FtpSourcePassword "#{Jobs.stats.collectazurecdnlogs.FtpSource.Password}" -AzureCdnAccountNumber "#{Jobs.stats.collectazurecdnlogs.AzureCdn.AccountNumber}" -AzureCdnPlatform "#{Jobs.stats.collectazurecdnlogs.AzureCdn.Platform}" -AzureCdnCloudStorageAccount "#{Jobs.stats.collectazurecdnlogs.AzureCdn.CloudStorageAccount}" -AzureCdnCloudStorageContainerName "#{Jobs.stats.collectazurecdnlogs.AzureCdn.CloudStorageContainerName}" -InstrumentationKey "#{Jobs.stats.collectazurecdnlogs.InstrumentationKey}" -verbose true -Interval #{Jobs.stats.collectazurecdnlogs.Interval}
10+
start /w stats.collectazurecdnlogs.exe ^
11+
-Configuration "#{Jobs.stats.collectazurecdnlogs.Configuration}" ^
12+
-InstrumentationKey "#{Jobs.stats.collectazurecdnlogs.InstrumentationKey}" ^
13+
-verbose true ^
14+
-Interval #{Jobs.stats.collectazurecdnlogs.Interval}
1115

12-
echo "Finished #{Jobs.stats.collectazurecdnlogs.Title}"
16+
echo "Finished #{Jobs.stats.collectazurecdnlogs.Title}"
1317

14-
goto Top
18+
goto Top

‎src/Stats.CollectAzureCdnLogs/Settings/dev.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"Initialization": {
3+
"FtpSourceUri": "#{Jobs.stats.collectazurecdnlogs.FtpSource.Uri}",
4+
"FtpSourceUsername": "#{Jobs.stats.collectazurecdnlogs.FtpSource.Username}",
5+
"FtpSourcePassword": "#{Jobs.stats.collectazurecdnlogs.FtpSource.Password}",
6+
"AzureCdnAccountNumber": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.AccountNumber}",
7+
"AzureCdnPlatform": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.Platform}",
8+
"AzureCdnCloudStorageAccount": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.CloudStorageAccount}",
9+
"AzureCdnCloudStorageContainerName": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.CloudStorageContainerName}"
10+
},
11+
12+
"KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
13+
"KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
14+
"KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",
15+
"KeyVault_ValidateCertificate": true,
16+
"KeyVault_StoreName": "My",
17+
"KeyVault_StoreLocation": "LocalMachine"
18+
}

‎src/Stats.CollectAzureCdnLogs/Settings/int.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"Initialization": {
3+
"FtpSourceUri": "#{Jobs.stats.collectazurecdnlogs.FtpSource.Uri}",
4+
"FtpSourceUsername": "#{Jobs.stats.collectazurecdnlogs.FtpSource.Username}",
5+
"FtpSourcePassword": "#{Jobs.stats.collectazurecdnlogs.FtpSource.Password}",
6+
"AzureCdnAccountNumber": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.AccountNumber}",
7+
"AzureCdnPlatform": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.Platform}",
8+
"AzureCdnCloudStorageAccount": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.CloudStorageAccount}",
9+
"AzureCdnCloudStorageContainerName": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.CloudStorageContainerName}"
10+
},
11+
12+
"KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
13+
"KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
14+
"KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",
15+
"KeyVault_ValidateCertificate": true,
16+
"KeyVault_StoreName": "My",
17+
"KeyVault_StoreLocation": "LocalMachine"
18+
}

‎src/Stats.CollectAzureCdnLogs/Settings/prod.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"Initialization": {
3+
"FtpSourceUri": "#{Jobs.stats.collectazurecdnlogs.FtpSource.Uri}",
4+
"FtpSourceUsername": "#{Jobs.stats.collectazurecdnlogs.FtpSource.Username}",
5+
"FtpSourcePassword": "#{Jobs.stats.collectazurecdnlogs.FtpSource.Password}",
6+
"AzureCdnAccountNumber": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.AccountNumber}",
7+
"AzureCdnPlatform": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.Platform}",
8+
"AzureCdnCloudStorageAccount": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.CloudStorageAccount}",
9+
"AzureCdnCloudStorageContainerName": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.CloudStorageContainerName}"
10+
},
11+
12+
"KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
13+
"KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
14+
"KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",
15+
"KeyVault_ValidateCertificate": true,
16+
"KeyVault_StoreName": "My",
17+
"KeyVault_StoreLocation": "LocalMachine"
18+
}

‎src/Stats.CollectAzureCdnLogs/Stats.CollectAzureCdnLogs.csproj‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
</ItemGroup>
5050
<ItemGroup>
5151
<Compile Include="Blob\CloudBlobRawLogClient.cs" />
52+
<Compile Include="Configuration\CollectAzureCdnLogsConfiguration.cs" />
5253
<Compile Include="FileExtensions.cs" />
5354
<Compile Include="Ftp\FtpDownloadStream.cs" />
5455
<Compile Include="Ftp\FtpRawLogClient.cs" />
@@ -70,6 +71,9 @@
7071
<SubType>Designer</SubType>
7172
</None>
7273
<None Include="Scripts\*" />
74+
<None Include="Settings\dev.json" />
75+
<None Include="Settings\int.json" />
76+
<None Include="Settings\prod.json" />
7377
<None Include="Stats.CollectAzureCdnLogs.nuspec" />
7478
</ItemGroup>
7579
<ItemGroup>
@@ -110,6 +114,7 @@
110114
<Version>7.1.2</Version>
111115
</PackageReference>
112116
</ItemGroup>
117+
<ItemGroup />
113118
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
114119
<PropertyGroup>
115120
<SignPath>..\..\build</SignPath>

‎src/Stats.CollectAzureCdnLogs/Stats.CollectAzureCdnLogs.nuspec‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,5 +18,7 @@
1818
<file src="Scripts\PreDeploy.ps1" />
1919
<file src="Scripts\PostDeploy.ps1" />
2020
<file src="Scripts\nssm.exe" />
21+
22+
<file src="Settings\*.json" target="bin" />
2123
</files>
2224
</package>

0 commit comments

Comments
 (0)