Skip to content
This repository was archived by the owner on Jul 30, 2024. It is now read-only.

Commit c7cfdd0

Browse files
joelverhagenjoelverhagen
committed
Split gallery DB download count update into smaller batches (#394)
Address NuGet/Engineering#1245
1 parent 29f959b commit c7cfdd0

8 files changed

Lines changed: 351 additions & 81 deletions

File tree

‎NuGet.Jobs.sln‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.PackageSigning.R
119119
EndProject
120120
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.PackageSigning.RevalidateCertificate.Tests", "tests\Validation.PackageSigning.RevalidateCertificate.Tests\Validation.PackageSigning.RevalidateCertificate.Tests.csproj", "{64095857-E9E3-4D9C-8769-7E558CD757CB}"
121121
EndProject
122+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tests.Stats.AggregateCdnDownloadsInGallery", "tests\Tests.Stats.AggregateCdnDownloadsInGallery\Tests.Stats.AggregateCdnDownloadsInGallery.csproj", "{136411AF-B9FA-438D-B790-9FB78A5F7F54}"
123+
EndProject
122124
Global
123125
GlobalSection(SolutionConfigurationPlatforms) = preSolution
124126
Debug|Any CPU = Debug|Any CPU
@@ -307,6 +309,10 @@ Global
307309
{64095857-E9E3-4D9C-8769-7E558CD757CB}.Debug|Any CPU.Build.0 = Debug|Any CPU
308310
{64095857-E9E3-4D9C-8769-7E558CD757CB}.Release|Any CPU.ActiveCfg = Release|Any CPU
309311
{64095857-E9E3-4D9C-8769-7E558CD757CB}.Release|Any CPU.Build.0 = Release|Any CPU
312+
{136411AF-B9FA-438D-B790-9FB78A5F7F54}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
313+
{136411AF-B9FA-438D-B790-9FB78A5F7F54}.Debug|Any CPU.Build.0 = Debug|Any CPU
314+
{136411AF-B9FA-438D-B790-9FB78A5F7F54}.Release|Any CPU.ActiveCfg = Release|Any CPU
315+
{136411AF-B9FA-438D-B790-9FB78A5F7F54}.Release|Any CPU.Build.0 = Release|Any CPU
310316
EndGlobalSection
311317
GlobalSection(SolutionProperties) = preSolution
312318
HideSolutionNode = FALSE
@@ -357,6 +363,7 @@ Global
357363
{26435822-8938-48C9-96FD-0DCCF8F7CE00} = {6A776396-02B1-475D-A104-26940ADB04AB}
358364
{EA32E1E5-7E7D-44E6-B496-43E1FEDE9400} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
359365
{64095857-E9E3-4D9C-8769-7E558CD757CB} = {6A776396-02B1-475D-A104-26940ADB04AB}
366+
{136411AF-B9FA-438D-B790-9FB78A5F7F54} = {6A776396-02B1-475D-A104-26940ADB04AB}
360367
EndGlobalSection
361368
GlobalSection(ExtensibilityGlobals) = postSolution
362369
SolutionGuid = {284A7AC3-FB43-4F1F-9C9C-2AF0E1F46C2B}

‎src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,5 +117,7 @@ public static class JobArgumentNames
117117
public const string MailFrom = "MailFrom";
118118
public const string SmtpUri = "SmtpUri";
119119

120+
// Arguments specific to Stats.AggregateCdnDownloadsInGallery
121+
public static string BatchSleepSeconds = "BatchSleepSeconds";
120122
}
121123
}

‎src/Stats.AggregateCdnDownloadsInGallery/Job.cs‎

Lines changed: 177 additions & 76 deletions
Large diffs are not rendered by default.

‎src/Stats.AggregateCdnDownloadsInGallery/Scripts/Stats.AggregateCdnDownloadsInGallery.cmd‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,22 @@
33
cd bin
44

55
:Top
6-
echo "Starting job - #{Jobs.stats.aggregatecdndownloadsingallery.Title}"
6+
echo "Starting job - #{Jobs.stats.aggregatecdndownloadsingallery.Title}"
77

8-
title #{Jobs.stats.aggregatecdndownloadsingallery.Title}
8+
title #{Jobs.stats.aggregatecdndownloadsingallery.Title}
99

10-
start /w stats.aggregatecdndownloadsingallery.exe -VaultName "#{Deployment.Azure.KeyVault.VaultName}" -ClientId "#{Deployment.Azure.KeyVault.ClientId}" -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" -StatisticsDatabase "#{Jobs.stats.aggregatecdndownloadsingallery.StatisticsDatabase}" -DestinationDatabase "#{Jobs.stats.aggregatecdndownloadsingallery.DestinationDatabase}" -InstrumentationKey "#{Jobs.stats.aggregatecdndownloadsingallery.InstrumentationKey}" -verbose true -Interval #{Jobs.stats.aggregatecdndownloadsingallery.Interval}
10+
start /w stats.aggregatecdndownloadsingallery.exe ^
11+
-VaultName "#{Deployment.Azure.KeyVault.VaultName}" ^
12+
-ClientId "#{Deployment.Azure.KeyVault.ClientId}" ^
13+
-CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" ^
14+
-StatisticsDatabase "#{Jobs.stats.aggregatecdndownloadsingallery.StatisticsDatabase}" ^
15+
-DestinationDatabase "#{Jobs.stats.aggregatecdndownloadsingallery.DestinationDatabase}" ^
16+
-InstrumentationKey "#{Jobs.stats.aggregatecdndownloadsingallery.InstrumentationKey}" ^
17+
-BatchSize "#{Jobs.stats.aggregatecdndownloadsingallery.BatchSize}" ^
18+
-BatchSleepSeconds "#{Jobs.stats.aggregatecdndownloadsingallery.BatchSleepSeconds}" ^
19+
-verbose true ^
20+
-Interval #{Jobs.stats.aggregatecdndownloadsingallery.Interval}
1121

12-
echo "Finished #{Jobs.stats.aggregatecdndownloadsingallery.Title}"
22+
echo "Finished #{Jobs.stats.aggregatecdndownloadsingallery.Title}"
1323

14-
goto Top
24+
goto Top

‎tests/Tests.Stats.AggregateCdnDownloadsInGallery/App.config‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
<?xml version="1.0" encoding="utf-8" ?>
2+
<configuration>
3+
<startup>
4+
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6" />
5+
</startup>
6+
</configuration>

‎tests/Tests.Stats.AggregateCdnDownloadsInGallery/JobTests.cs‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System.Collections.Generic;
5+
using System.Linq;
6+
using Xunit;
7+
using IPackageIdGroup = System.Linq.IGrouping<string, Stats.AggregateCdnDownloadsInGallery.DownloadCountData>;
8+
9+
namespace Stats.AggregateCdnDownloadsInGallery
10+
{
11+
public class JobTests
12+
{
13+
14+
public class PopGroupBatch
15+
{
16+
[Theory]
17+
[InlineData(-1, 1)]
18+
[InlineData(0, 1)]
19+
[InlineData(1, 1)]
20+
[InlineData(2, 1)]
21+
[InlineData(3, 1)]
22+
[InlineData(4, 1)]
23+
[InlineData(5, 1)]
24+
[InlineData(6, 1)]
25+
[InlineData(7, 1)]
26+
[InlineData(8, 1)]
27+
[InlineData(9, 2)]
28+
[InlineData(10, 2)]
29+
[InlineData(11, 3)]
30+
[InlineData(12, 3)]
31+
[InlineData(13, 3)]
32+
[InlineData(14, 3)]
33+
[InlineData(15, 4)]
34+
[InlineData(16, 4)]
35+
[InlineData(17, 4)]
36+
public void ReturnsCorrectNumberOfPackageRegistrationGroups(int batchSize, int expectedGroupCount)
37+
{
38+
// Arrange
39+
var data = new Stack<IPackageIdGroup>(new[]
40+
{
41+
new DownloadCountData { PackageId = "A", PackageVersion = "4.2.0" },
42+
new DownloadCountData { PackageId = "A", PackageVersion = "4.3.0" },
43+
new DownloadCountData { PackageId = "A", PackageVersion = "4.4.0" },
44+
new DownloadCountData { PackageId = "A", PackageVersion = "4.5.0" },
45+
new DownloadCountData { PackageId = "A", PackageVersion = "4.6.0" }, // 5 versions, 1 ID (6 total records so far)
46+
47+
new DownloadCountData { PackageId = "B", PackageVersion = "4.5.0" },
48+
new DownloadCountData { PackageId = "B", PackageVersion = "4.6.0" }, // 2 versions, 1 ID (9 total records so far)
49+
50+
new DownloadCountData { PackageId = "C", PackageVersion = "4.6.0" }, // 1 version, 1 ID (11 total records)
51+
52+
new DownloadCountData { PackageId = "D", PackageVersion = "4.4.0" },
53+
new DownloadCountData { PackageId = "D", PackageVersion = "4.5.0" },
54+
new DownloadCountData { PackageId = "D", PackageVersion = "4.6.0" }, // 3 versions, 1 ID (15 total records)
55+
}.GroupBy(x => x.PackageId).OrderByDescending(x => x.Key));
56+
57+
// Act
58+
var groupBatch = Job.PopGroupBatch(data, batchSize);
59+
60+
// Assert
61+
Assert.Equal(expectedGroupCount, groupBatch.Count);
62+
}
63+
}
64+
}
65+
}

‎tests/Tests.Stats.AggregateCdnDownloadsInGallery/Properties/AssemblyInfo.cs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System.Reflection;
5+
using System.Runtime.InteropServices;
6+
7+
[assembly: AssemblyTitle("Tests.Stats.AggregateCdnDownloadsInGallery")]
8+
[assembly: ComVisible(false)]
9+
[assembly: Guid("136411af-b9fa-438d-b790-9fb78a5f7f54")]

‎tests/Tests.Stats.AggregateCdnDownloadsInGallery/Tests.Stats.AggregateCdnDownloadsInGallery.csproj‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3+
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
4+
<PropertyGroup>
5+
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
6+
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
7+
<ProjectGuid>{136411AF-B9FA-438D-B790-9FB78A5F7F54}</ProjectGuid>
8+
<OutputType>Library</OutputType>
9+
<RootNamespace>Stats.AggregateCdnDownloadsInGallery</RootNamespace>
10+
<AssemblyName>Tests.Stats.AggregateCdnDownloadsInGallery</AssemblyName>
11+
<TargetFrameworkVersion>v4.6</TargetFrameworkVersion>
12+
<FileAlignment>512</FileAlignment>
13+
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
14+
</PropertyGroup>
15+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
16+
<PlatformTarget>AnyCPU</PlatformTarget>
17+
<DebugSymbols>true</DebugSymbols>
18+
<DebugType>full</DebugType>
19+
<Optimize>false</Optimize>
20+
<OutputPath>bin\Debug\</OutputPath>
21+
<DefineConstants>DEBUG;TRACE</DefineConstants>
22+
<ErrorReport>prompt</ErrorReport>
23+
<WarningLevel>4</WarningLevel>
24+
</PropertyGroup>
25+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
26+
<PlatformTarget>AnyCPU</PlatformTarget>
27+
<DebugType>pdbonly</DebugType>
28+
<Optimize>true</Optimize>
29+
<OutputPath>bin\Release\</OutputPath>
30+
<DefineConstants>TRACE</DefineConstants>
31+
<ErrorReport>prompt</ErrorReport>
32+
<WarningLevel>4</WarningLevel>
33+
</PropertyGroup>
34+
<PropertyGroup>
35+
<StartupObject />
36+
</PropertyGroup>
37+
<ItemGroup>
38+
<Reference Include="System" />
39+
<Reference Include="System.Core" />
40+
<Reference Include="System.Xml.Linq" />
41+
<Reference Include="System.Data.DataSetExtensions" />
42+
<Reference Include="Microsoft.CSharp" />
43+
<Reference Include="System.Data" />
44+
<Reference Include="System.Net.Http" />
45+
<Reference Include="System.Xml" />
46+
</ItemGroup>
47+
<ItemGroup>
48+
<Compile Include="JobTests.cs" />
49+
<Compile Include="Properties\AssemblyInfo.cs" />
50+
</ItemGroup>
51+
<ItemGroup>
52+
<None Include="App.config" />
53+
</ItemGroup>
54+
<ItemGroup>
55+
<PackageReference Include="xunit">
56+
<Version>2.3.1</Version>
57+
</PackageReference>
58+
</ItemGroup>
59+
<ItemGroup>
60+
<ProjectReference Include="..\..\src\NuGet.Jobs.Common\NuGet.Jobs.Common.csproj">
61+
<Project>{4B4B1EFB-8F33-42E6-B79F-54E7F3293D31}</Project>
62+
<Name>NuGet.Jobs.Common</Name>
63+
</ProjectReference>
64+
<ProjectReference Include="..\..\src\Stats.AggregateCdnDownloadsInGallery\Stats.AggregateCdnDownloadsInGallery.csproj">
65+
<Project>{6e27275f-0a0b-bb64-4c28-8f3e894b8c9e}</Project>
66+
<Name>Stats.AggregateCdnDownloadsInGallery</Name>
67+
</ProjectReference>
68+
</ItemGroup>
69+
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
70+
</Project>

0 commit comments

Comments
 (0)