Skip to content
This repository was archived by the owner on Jul 30, 2024. It is now read-only.

Commit 2bb06ee

Browse files
skofman1skofman1
authored
Search.GenerateAuxiliaryData should copy Downloads.v1.json to search metadata location (#398)
* Add new script to nuspec * Fix indent * Initial code * tested * Update scripts * PR feedback
1 parent b349b71 commit 2bb06ee

12 files changed

Lines changed: 158 additions & 42 deletions

File tree

‎src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ public static class JobArgumentNames
7272
public const string AzureCdnAccountNumber = "AzureCdnAccountNumber";
7373
public const string AzureCdnPlatform = "AzureCdnPlatform";
7474

75-
//Arguments shared by CollectAzureCdnLogs and ParseAzureCdnLogs
75+
//Arguments shared by CollectAzureCdnLogs, ParseAzureCdnLogs, Search.GenerateAuxiliaryData
7676
public const string AzureCdnCloudStorageAccount = "AzureCdnCloudStorageAccount";
7777
public const string AzureCdnCloudStorageContainerName = "AzureCdnCloudStorageContainerName";
7878

‎src/Search.GenerateAuxiliaryData/BlobStorageExporter.cs‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Diagnostics;
6+
using System.Threading.Tasks;
7+
using Microsoft.Extensions.Logging;
8+
using Microsoft.WindowsAzure.Storage;
9+
using Microsoft.WindowsAzure.Storage.Blob;
10+
11+
namespace Search.GenerateAuxiliaryData
12+
{
13+
/// <summary>
14+
/// Used for data that needs to be copied from a blob storage
15+
/// </summary>
16+
public class BlobStorageExporter : Exporter
17+
{
18+
private static readonly TimeSpan MaxCopyDuration = TimeSpan.FromMinutes(10);
19+
private static readonly TimeSpan CopyPollFrequency = TimeSpan.FromMilliseconds(500);
20+
21+
private CloudBlobContainer _sourceContainer;
22+
private string _sourceName;
23+
24+
public BlobStorageExporter(ILogger<Exporter> logger, CloudBlobContainer sourceContainer, string sourceName, CloudBlobContainer destinationContainer, string destinationName):
25+
base(logger, destinationContainer, destinationName)
26+
{
27+
_sourceContainer = sourceContainer ?? throw new ArgumentNullException(nameof(sourceContainer));
28+
_sourceName = sourceName;
29+
}
30+
31+
public override async Task ExportAsync()
32+
{
33+
_logger.LogInformation("Copying {ReportName} report from {ConnectionString}/{SourceName}", _name, _sourceContainer.Uri, _sourceName);
34+
35+
await _destinationContainer.CreateIfNotExistsAsync();
36+
37+
var sourceCloudBlob = _sourceContainer.GetBlockBlobReference(_sourceName);
38+
var destinationCloudBlob = _destinationContainer.GetBlockBlobReference(_name);
39+
40+
41+
await destinationCloudBlob.StartCopyAsync(sourceCloudBlob);
42+
43+
var stopwatch = Stopwatch.StartNew();
44+
while (destinationCloudBlob.CopyState.Status == CopyStatus.Pending
45+
&& stopwatch.Elapsed < MaxCopyDuration)
46+
{
47+
await destinationCloudBlob.FetchAttributesAsync();
48+
await Task.Delay(CopyPollFrequency);
49+
}
50+
51+
if (destinationCloudBlob.CopyState.Status == CopyStatus.Pending)
52+
{
53+
throw new TimeoutException($"Waiting for the blob copy operation to complete timed out after {MaxCopyDuration.TotalSeconds} seconds.");
54+
}
55+
else if (destinationCloudBlob.CopyState.Status != CopyStatus.Success)
56+
{
57+
throw new StorageException($"The blob copy operation had copy status {destinationCloudBlob.CopyState.Status} ({destinationCloudBlob.CopyState.StatusDescription}).");
58+
}
59+
60+
_logger.LogInformation("Copy of {ReportName} completed. Took: {Seconds} seconds.", _name, stopwatch.Elapsed.TotalSeconds);
61+
}
62+
}
63+
}

‎src/Search.GenerateAuxiliaryData/Exporter.cs‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Threading.Tasks;
6+
using Microsoft.Extensions.Logging;
7+
using Microsoft.WindowsAzure.Storage.Blob;
8+
9+
namespace Search.GenerateAuxiliaryData
10+
{
11+
// Public only to facilitate testing.
12+
public abstract class Exporter
13+
{
14+
protected ILogger<Exporter> _logger;
15+
protected CloudBlobContainer _destinationContainer;
16+
17+
protected string _name { get; }
18+
19+
public Exporter(ILogger<Exporter> logger, CloudBlobContainer defaultDestinationContainer, string defaultName)
20+
{
21+
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
22+
_destinationContainer = defaultDestinationContainer ?? throw new ArgumentNullException(nameof(defaultDestinationContainer));
23+
24+
_name = defaultName;
25+
}
26+
27+
public abstract Task ExportAsync();
28+
}
29+
}

‎src/Search.GenerateAuxiliaryData/ExporterException.cs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
6+
namespace Search.GenerateAuxiliaryData
7+
{
8+
public class ExporterException : Exception
9+
{
10+
public ExporterException(string message)
11+
: base(message)
12+
{
13+
}
14+
}
15+
}

‎src/Search.GenerateAuxiliaryData/Job.cs‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,11 @@ internal class Job
3434
private const string ScriptVerifiedPackages = "SqlScripts.VerifiedPackages.sql";
3535
private const string OutputNameVerifiedPackages = "verifiedPackages.json";
3636

37-
private List<SqlExporter> _sqlExportScriptsToRun;
37+
private const string StatisticsReportName = "downloads.v1.json";
38+
39+
private List<Exporter> _exportersToRun;
3840
private CloudBlobContainer _destContainer;
41+
private CloudBlobContainer _statisticsContainer;
3942

4043
public override void Init(IDictionary<string, string> jobArgsDictionary)
4144
{
@@ -45,6 +48,11 @@ public override void Init(IDictionary<string, string> jobArgsDictionary)
4548
var statisticsDatabaseConnString = new SqlConnectionStringBuilder(
4649
JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase)).ToString();
4750

51+
var statisticsStorageAccount = CloudStorageAccount.Parse(
52+
JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount));
53+
54+
var statisticsReportsContainerName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName);
55+
4856
var destination = CloudStorageAccount.Parse(
4957
JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PrimaryDestination));
5058

@@ -53,36 +61,38 @@ public override void Init(IDictionary<string, string> jobArgsDictionary)
5361
?? DefaultContainerName;
5462

5563
_destContainer = destination.CreateCloudBlobClient().GetContainerReference(destinationContainerName);
64+
_statisticsContainer = statisticsStorageAccount.CreateCloudBlobClient().GetContainerReference(statisticsReportsContainerName);
5665

57-
_sqlExportScriptsToRun = new List<SqlExporter> {
66+
_exportersToRun = new List<Exporter> {
5867
new VerifiedPackagesExporter(LoggerFactory.CreateLogger<VerifiedPackagesExporter>(), packageDatabaseConnString, _destContainer, ScriptVerifiedPackages, OutputNameVerifiedPackages),
5968
new NestedJArrayExporter(LoggerFactory.CreateLogger<NestedJArrayExporter>(), packageDatabaseConnString, _destContainer, ScriptCuratedFeed, OutputNameCuratedFeed, Col0CuratedFeed, Col1CuratedFeed),
6069
new NestedJArrayExporter(LoggerFactory.CreateLogger<NestedJArrayExporter>(), packageDatabaseConnString, _destContainer, ScriptOwners, OutputNameOwners, Col0Owners, Col1Owners),
61-
new RankingsExporter(LoggerFactory.CreateLogger<RankingsExporter>(), statisticsDatabaseConnString, _destContainer, ScriptRankingsTotal, OutputNameRankings)
70+
new RankingsExporter(LoggerFactory.CreateLogger<RankingsExporter>(), statisticsDatabaseConnString, _destContainer, ScriptRankingsTotal, OutputNameRankings),
71+
new BlobStorageExporter(LoggerFactory.CreateLogger<BlobStorageExporter>(), _statisticsContainer, StatisticsReportName, _destContainer, StatisticsReportName)
6272
};
6373
}
6474

6575
public override async Task Run()
6676
{
67-
var failedSqlExporters = new List<string>();
77+
var failedExporters = new List<string>();
6878

69-
foreach (SqlExporter exporter in _sqlExportScriptsToRun)
79+
foreach (Exporter exporter in _exportersToRun)
7080
{
7181
try
7282
{
73-
await exporter.RunSqlExportAsync();
83+
await exporter.ExportAsync();
7484
}
7585
catch (Exception e)
7686
{
7787
var exporterName = exporter.GetType().Name;
7888
Logger.LogError("SQL exporter '{ExporterName}' failed: {Exception}", exporterName, e);
79-
failedSqlExporters.Add(exporterName);
89+
failedExporters.Add(exporterName);
8090
}
8191
}
8292

83-
if (failedSqlExporters.Any())
93+
if (failedExporters.Any())
8494
{
85-
throw new SqlExporterException($"{failedSqlExporters.Count()} tasks failed: {string.Join(", ", failedSqlExporters)}");
95+
throw new ExporterException($"{failedExporters.Count()} tasks failed: {string.Join(", ", failedExporters)}");
8696
}
8797
}
8898
}

‎src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.Asia.cmd‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,10 @@ cd bin
1414
-PrimaryDestination #{Jobs.Asia.search.generateauxiliarydata.Storage.Primary} ^
1515
-PackageDatabase "#{Jobs.search.generateauxiliarydata.PackageDatabase}" ^
1616
-StatisticsDatabase "#{Jobs.search.generateauxiliarydata.StatisticsDatabase}" ^
17-
-verbose true -Sleep #{Jobs.search.generateauxiliarydata.Sleep} ^
17+
-AzureCdnCloudStorageAccount "#{Jobs.stats.createazurecdnwarehousereports.AzureCdn.CloudStorageAccount}"
18+
-AzureCdnCloudStorageContainerName "#{Jobs.stats.createazurecdnwarehousereports.AzureCdn.CloudStorageContainerName}"
19+
-verbose true ^
20+
-Sleep #{Jobs.search.generateauxiliarydata.Sleep} ^
1821
-InstrumentationKey "#{Jobs.search.generateauxiliarydata.ApplicationInsightsInstrumentationKey}"
1922

2023
echo "Finished #{Jobs.search.generateauxiliarydata.Title}"

‎src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.cmd‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,18 @@ cd bin
77

88
title #{Jobs.search.generateauxiliarydata.Title}
99

10-
start /w search.generateauxiliarydata.exe -VaultName "#{Deployment.Azure.KeyVault.VaultName}" -ClientId "#{Deployment.Azure.KeyVault.ClientId}" -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" -PrimaryDestination #{Jobs.search.generateauxiliarydata.Storage.Primary} -PackageDatabase "#{Jobs.search.generateauxiliarydata.PackageDatabase}" -StatisticsDatabase "#{Jobs.search.generateauxiliarydata.StatisticsDatabase}" -verbose true -Sleep #{Jobs.search.generateauxiliarydata.Sleep} -InstrumentationKey "#{Jobs.search.generateauxiliarydata.ApplicationInsightsInstrumentationKey}"
10+
start /w search.generateauxiliarydata.exe ^
11+
-VaultName "#{Deployment.Azure.KeyVault.VaultName}" ^
12+
-ClientId "#{Deployment.Azure.KeyVault.ClientId}" ^
13+
-CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" ^
14+
-PrimaryDestination #{Jobs.search.generateauxiliarydata.Storage.Primary} ^
15+
-PackageDatabase "#{Jobs.search.generateauxiliarydata.PackageDatabase}" ^
16+
-StatisticsDatabase "#{Jobs.search.generateauxiliarydata.StatisticsDatabase}" ^
17+
-AzureCdnCloudStorageAccount "#{Jobs.stats.createazurecdnwarehousereports.AzureCdn.CloudStorageAccount}"
18+
-AzureCdnCloudStorageContainerName "#{Jobs.stats.createazurecdnwarehousereports.AzureCdn.CloudStorageContainerName}"
19+
-verbose true ^
20+
-Sleep #{Jobs.search.generateauxiliarydata.Sleep} ^
21+
-InstrumentationKey "#{Jobs.search.generateauxiliarydata.ApplicationInsightsInstrumentationKey}"
1122

1223
echo "Finished #{Jobs.search.generateauxiliarydata.Title}"
1324

‎src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,13 +84,15 @@
8484
</Reference>
8585
</ItemGroup>
8686
<ItemGroup>
87+
<Compile Include="BlobStorageExporter.cs" />
88+
<Compile Include="Exporter.cs" />
89+
<Compile Include="ExporterException.cs" />
8790
<Compile Include="Job.cs" />
8891
<Compile Include="NestedJArrayExporter.cs" />
8992
<Compile Include="Program.cs" />
9093
<Compile Include="Properties\AssemblyInfo.cs" />
9194
<Compile Include="RankingsExporter.cs" />
9295
<Compile Include="SqlExporter.cs" />
93-
<Compile Include="SqlExporterException.cs" />
9496
<Compile Include="VerifiedPackagesExporter.cs" />
9597
</ItemGroup>
9698
<ItemGroup>

‎src/Search.GenerateAuxiliaryData/SqlExporter.cs‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
using System.Collections.Generic;
55
using System.Data;
66
using System.Data.SqlClient;
7-
using System.Diagnostics;
87
using System.IO;
98
using System.Reflection;
109
using System.Threading.Tasks;
@@ -16,22 +15,18 @@
1615
namespace Search.GenerateAuxiliaryData
1716
{
1817
// Public only to facilitate testing.
19-
public abstract class SqlExporter
18+
public abstract class SqlExporter : Exporter
2019
{
21-
private ILogger<SqlExporter> _logger;
2220
private static Assembly _executingAssembly = Assembly.GetExecutingAssembly();
2321
private static string _assemblyName = _executingAssembly.GetName().Name;
2422

2523
public string ConnectionString { get; }
26-
public CloudBlobContainer DestinationContainer { get; }
27-
public string Name { get; }
2824

2925
public SqlExporter(ILogger<SqlExporter> logger, string defaultConnectionString, CloudBlobContainer defaultDestinationContainer, string defaultName)
26+
: base(logger, defaultDestinationContainer, defaultName)
3027
{
3128
_logger = logger;
3229
ConnectionString = defaultConnectionString;
33-
DestinationContainer = defaultDestinationContainer;
34-
Name = defaultName;
3530
}
3631

3732
protected static string GetEmbeddedSqlScript(string resourceName)
@@ -40,9 +35,9 @@ protected static string GetEmbeddedSqlScript(string resourceName)
4035
return new StreamReader(stream).ReadToEnd();
4136
}
4237

43-
public async Task RunSqlExportAsync()
38+
public override async Task ExportAsync()
4439
{
45-
_logger.LogInformation("Generating {ReportName} report from {ConnectionString}.", Name, TracableConnectionString(ConnectionString));
40+
_logger.LogInformation("Generating {ReportName} report from {ConnectionString}.", _name, TracableConnectionString(ConnectionString));
4641

4742
JContainer result;
4843
using (var connection = new SqlConnection(ConnectionString))
@@ -52,7 +47,7 @@ public async Task RunSqlExportAsync()
5247
result = GetResultOfQuery(connection);
5348
}
5449

55-
await WriteToBlobAsync(_logger, DestinationContainer, result.ToString(Formatting.None), Name);
50+
await WriteToBlobAsync(_logger, _destinationContainer, result.ToString(Formatting.None), _name);
5651
}
5752

5853
protected abstract JContainer GetResultOfQuery(SqlConnection connection);
@@ -75,7 +70,7 @@ private static string TracableConnectionString(string connectionString)
7570
return connStr.ToString();
7671
}
7772

78-
private static async Task WriteToBlobAsync(ILogger<SqlExporter> logger, CloudBlobContainer container, string content, string name)
73+
private static async Task WriteToBlobAsync(ILogger<Exporter> logger, CloudBlobContainer container, string content, string name)
7974
{
8075
await container.CreateIfNotExistsAsync();
8176

‎src/Search.GenerateAuxiliaryData/SqlExporterException.cs‎

Lines changed: 0 additions & 16 deletions
This file was deleted.

0 commit comments

Comments
 (0)