Skip to content
This repository was archived by the owner on Aug 3, 2024. It is now read-only.

Commit ae6a42c

Browse files
joelverhagenjoelverhagen
committed
Add batch signing for the output assemblies (#312)
Use the newer credential provider to mitigate timeout issue Progress on NuGet/Engineering#1821
1 parent f5c510e commit ae6a42c

14 files changed

Lines changed: 836 additions & 76 deletions

‎.gitignore‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,8 @@ bld/
2424

2525
# Files created and used by our build scripts
2626
AssemblyInfo.*.cs
27-
.nuget/CredentialProviderBundle.zip
28-
.nuget/CredentialProvider.VSS.exe
29-
.nuget/EULA_Microsoft Visual Studio Team Services Credential Provider.docx
30-
.nuget/ThirdPartyNotices.txt
27+
.nuget/credprovider
28+
.nuget/.marker.v*
3129
nuget.exe
3230
build/packages/
3331

‎NuGet.Server.Common.sln‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.Licenses", "
9090
EndProject
9191
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.Licenses.Tests", "tests\NuGet.Services.Licenses.Tests\NuGet.Services.Licenses.Tests.csproj", "{D0A110BD-156D-432B-9525-171430C9F51D}"
9292
EndProject
93+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.Build.Tests", "tests\NuGet.Services.Build.Tests\NuGet.Services.Build.Tests.csproj", "{AB3C7814-3AD6-41BD-8968-A3CC10A52EC5}"
94+
EndProject
9395
Global
9496
GlobalSection(SolutionConfigurationPlatforms) = preSolution
9597
Debug|Any CPU = Debug|Any CPU
@@ -248,6 +250,10 @@ Global
248250
{D0A110BD-156D-432B-9525-171430C9F51D}.Debug|Any CPU.Build.0 = Debug|Any CPU
249251
{D0A110BD-156D-432B-9525-171430C9F51D}.Release|Any CPU.ActiveCfg = Release|Any CPU
250252
{D0A110BD-156D-432B-9525-171430C9F51D}.Release|Any CPU.Build.0 = Release|Any CPU
253+
{AB3C7814-3AD6-41BD-8968-A3CC10A52EC5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
254+
{AB3C7814-3AD6-41BD-8968-A3CC10A52EC5}.Debug|Any CPU.Build.0 = Debug|Any CPU
255+
{AB3C7814-3AD6-41BD-8968-A3CC10A52EC5}.Release|Any CPU.ActiveCfg = Release|Any CPU
256+
{AB3C7814-3AD6-41BD-8968-A3CC10A52EC5}.Release|Any CPU.Build.0 = Release|Any CPU
251257
EndGlobalSection
252258
GlobalSection(SolutionProperties) = preSolution
253259
HideSolutionNode = FALSE
@@ -291,6 +297,7 @@ Global
291297
{20D89F39-6E71-4F14-B845-64F5C395954B} = {7783A106-0F4C-4055-9AB4-413FB2C7B8F0}
292298
{20EE3196-4D8B-42B6-8E5E-B8ADC2D3BF9D} = {8415FED7-1BED-4227-8B4F-BB7C24E041CD}
293299
{D0A110BD-156D-432B-9525-171430C9F51D} = {7783A106-0F4C-4055-9AB4-413FB2C7B8F0}
300+
{AB3C7814-3AD6-41BD-8968-A3CC10A52EC5} = {7783A106-0F4C-4055-9AB4-413FB2C7B8F0}
294301
EndGlobalSection
295302
GlobalSection(ExtensibilityGlobals) = postSolution
296303
SolutionGuid = {AA413DB0-5475-4B5D-A3AF-6323DA8D538B}

‎build.ps1‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ trap {
2020
exit 1
2121
}
2222

23+
# Enable TLS 1.2 since GitHub requires it.
24+
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
25+
2326
. "$PSScriptRoot\build\common.ps1"
2427

2528
Function Clean-Tests {
@@ -97,6 +100,11 @@ Invoke-BuildStep 'Building solution' { `
97100
Build-Solution $Configuration $BuildNumber -MSBuildVersion "15" $SolutionPath -SkipRestore:$SkipRestore
98101
} `
99102
-ev +BuildErrors
103+
104+
Invoke-BuildStep 'Signing the binaries' {
105+
Sign-Binaries -Configuration $Configuration -BuildNumber $BuildNumber -MSBuildVersion "15" `
106+
} `
107+
-ev +BuildErrors
100108

101109
Invoke-BuildStep 'Creating artifacts' { `
102110
$projects = `

‎build/FindDuplicateFiles.cs‎

Lines changed: 311 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,311 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.IO;
7+
using System.Linq;
8+
using System.Security.Cryptography;
9+
using Microsoft.Build.Framework;
10+
11+
namespace NuGet.Services.Build
12+
{
13+
public class FindDuplicateFiles : Microsoft.Build.Utilities.Task
14+
{
15+
[Required]
16+
public ITaskItem[] Files { get; set; }
17+
18+
[Output]
19+
public ITaskItem[] UniqueFiles { get; set; }
20+
21+
[Output]
22+
public ITaskItem[] DuplicateFiles { get; set; }
23+
24+
public override bool Execute()
25+
{
26+
var infos = GetUniqueTaskItemInfo();
27+
Log.LogMessage(
28+
MessageImportance.High,
29+
"Of the {0} items provided, {1} are unique file paths that exist. {2} items will be ignored.",
30+
Files.Length,
31+
infos.Count,
32+
Files.Length - infos.Count);
33+
34+
if (infos.Count == 0)
35+
{
36+
UniqueFiles = Array.Empty<ITaskItem>();
37+
DuplicateFiles = Array.Empty<ITaskItem>();
38+
return true;
39+
}
40+
41+
var filePathToDuplicates = FindDuplicates(infos);
42+
43+
var fileCount = filePathToDuplicates.Sum(x => x.Value.Count);
44+
var uniqueCount = filePathToDuplicates.Count;
45+
var duplicateCount = fileCount - uniqueCount;
46+
Log.LogMessage(
47+
MessageImportance.High,
48+
"Of the {0} unique file paths provided, {1} ({2:P}) are unique and {3} ({4:P}) are duplicate.",
49+
fileCount,
50+
uniqueCount,
51+
(float)uniqueCount / fileCount,
52+
duplicateCount,
53+
(float)duplicateCount / fileCount);
54+
55+
var uniqueFiles = new List<ITaskItem>();
56+
var duplicateFiles = new List<ITaskItem>();
57+
foreach (var pair in filePathToDuplicates)
58+
{
59+
foreach (var duplicate in pair.Value)
60+
{
61+
if (pair.Key == duplicate.FullPath)
62+
{
63+
uniqueFiles.Add(duplicate.Item);
64+
}
65+
else
66+
{
67+
duplicate.Item.SetMetadata("DuplicateOf", pair.Key);
68+
duplicateFiles.Add(duplicate.Item);
69+
}
70+
}
71+
}
72+
73+
UniqueFiles = uniqueFiles.ToArray();
74+
DuplicateFiles = duplicateFiles.ToArray();
75+
76+
return true;
77+
}
78+
79+
private List<TaskItemInfo> GetUniqueTaskItemInfo()
80+
{
81+
// Compare paths in a case insensitive manner.
82+
var uniqueFullPaths = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
83+
84+
var infos = new List<TaskItemInfo>();
85+
foreach (var item in Files)
86+
{
87+
var fullPath = item.GetMetadata("FullPath");
88+
if (fullPath == null || !File.Exists(fullPath))
89+
{
90+
Log.LogWarning("File '{0}' does not exist.", item.ItemSpec);
91+
continue;
92+
}
93+
94+
if (!uniqueFullPaths.Add(fullPath))
95+
{
96+
Log.LogWarning("File path '{0}' is associated with multiple items. Only the first will be used.", item.ItemSpec);
97+
continue;
98+
}
99+
100+
var info = new TaskItemInfo(item, fullPath);
101+
102+
infos.Add(info);
103+
}
104+
105+
return infos;
106+
}
107+
108+
private Dictionary<string, List<TaskItemInfo>> FindDuplicates(List<TaskItemInfo> infos)
109+
{
110+
var fileSizeToInfos = GetFileSizeToInfos(infos);
111+
112+
var filePathToDuplicates = new Dictionary<string, List<TaskItemInfo>>();
113+
114+
// Inspired by https://stackoverflow.com/a/36113168.
115+
var buffer = new byte[1024];
116+
foreach (var fileSizePair in fileSizeToInfos)
117+
{
118+
// The file size is unique, the file is unique.
119+
if (TryAddUnique("file size", filePathToDuplicates, fileSizePair))
120+
{
121+
continue;
122+
}
123+
124+
BreakTiesWithLeadingHash(filePathToDuplicates, buffer, fileSizePair);
125+
}
126+
127+
return filePathToDuplicates;
128+
}
129+
130+
private Dictionary<long, List<TaskItemInfo>> GetFileSizeToInfos(
131+
IEnumerable<TaskItemInfo> infos)
132+
{
133+
var fileSizeToInfos = new Dictionary<long, List<TaskItemInfo>>();
134+
foreach (var info in infos)
135+
{
136+
info.FileSize = new FileInfo(info.FullPath).Length;
137+
138+
List<TaskItemInfo> infosWithSameFileSize;
139+
if (!fileSizeToInfos.TryGetValue(info.FileSize, out infosWithSameFileSize))
140+
{
141+
infosWithSameFileSize = new List<TaskItemInfo> { info };
142+
fileSizeToInfos.Add(info.FileSize, infosWithSameFileSize);
143+
}
144+
else
145+
{
146+
infosWithSameFileSize.Add(info);
147+
}
148+
}
149+
150+
return fileSizeToInfos;
151+
}
152+
153+
private void BreakTiesWithLeadingHash(
154+
Dictionary<string, List<TaskItemInfo>> filePathToDuplicates,
155+
byte[] buffer,
156+
KeyValuePair<long, List<TaskItemInfo>> fileSizePair)
157+
{
158+
var leadingHashToInfos = GetLeadingHashToInfos(fileSizePair.Value, buffer);
159+
160+
foreach (var leadingHashPair in leadingHashToInfos)
161+
{
162+
// If the leading hash is unique, the file is unique.
163+
if (TryAddUnique("leading hash", filePathToDuplicates, leadingHashPair))
164+
{
165+
continue;
166+
}
167+
168+
BreakTiesWithHash(filePathToDuplicates, leadingHashPair);
169+
}
170+
}
171+
172+
private static Dictionary<string, List<TaskItemInfo>> GetLeadingHashToInfos(
173+
IEnumerable<TaskItemInfo> infos,
174+
byte[] buffer)
175+
{
176+
var leadingHashToInfos = new Dictionary<string, List<TaskItemInfo>>();
177+
foreach (var info in infos)
178+
{
179+
using (var fileStream = new FileStream(info.FullPath, FileMode.Open))
180+
{
181+
var totalRead = 0;
182+
while (totalRead < buffer.Length)
183+
{
184+
var read = fileStream.Read(buffer, totalRead, buffer.Length - totalRead);
185+
totalRead += read;
186+
if (read == 0)
187+
{
188+
break;
189+
}
190+
}
191+
192+
using (var hashAlgorithm = SHA256.Create())
193+
{
194+
var hashBytes = hashAlgorithm.ComputeHash(buffer, 0, totalRead);
195+
var hash = GetHashString(hashBytes);
196+
info.HeaderHash = hash;
197+
198+
List<TaskItemInfo> infosWithSameLeadingHash;
199+
if (!leadingHashToInfos.TryGetValue(hash, out infosWithSameLeadingHash))
200+
{
201+
infosWithSameLeadingHash = new List<TaskItemInfo> { info };
202+
leadingHashToInfos.Add(hash, infosWithSameLeadingHash);
203+
}
204+
else
205+
{
206+
infosWithSameLeadingHash.Add(info);
207+
}
208+
}
209+
}
210+
}
211+
212+
return leadingHashToInfos;
213+
}
214+
215+
private void BreakTiesWithHash(
216+
Dictionary<string, List<TaskItemInfo>> filePathToDuplicates,
217+
KeyValuePair<string, List<TaskItemInfo>> leadingHashPair)
218+
{
219+
var hashToInfos = GetHashToInfos(leadingHashPair.Value);
220+
221+
foreach (var hashPair in hashToInfos)
222+
{
223+
// If the hash is unique, the file is unique.
224+
if (TryAddUnique("hash", filePathToDuplicates, hashPair))
225+
{
226+
continue;
227+
}
228+
229+
// If multiple files has the same hash, they are duplicates.
230+
filePathToDuplicates.Add(hashPair.Value[0].FullPath, hashPair.Value);
231+
232+
Log.LogMessage(
233+
"File with {0} duplicates: {1}{2}{3}",
234+
hashPair.Value.Count - 1,
235+
hashPair.Value[0].FullPath,
236+
string.Concat(hashPair.Value.Skip(1).Select(x => Environment.NewLine + " - " + x.FullPath)),
237+
Environment.NewLine);
238+
}
239+
}
240+
241+
private bool TryAddUnique<T>(
242+
string keyName,
243+
Dictionary<string, List<TaskItemInfo>> filePathToDuplicates,
244+
KeyValuePair<T, List<TaskItemInfo>> pair)
245+
{
246+
if (pair.Value.Count == 1)
247+
{
248+
Log.LogMessage(
249+
"Unique file by {0} {1}: {2}{3}",
250+
keyName,
251+
pair.Key,
252+
pair.Value[0].FullPath,
253+
Environment.NewLine);
254+
255+
filePathToDuplicates.Add(pair.Value[0].FullPath, new List<TaskItemInfo> { pair.Value[0] });
256+
return true;
257+
}
258+
259+
return false;
260+
}
261+
262+
private static Dictionary<string, List<TaskItemInfo>> GetHashToInfos(
263+
IEnumerable<TaskItemInfo> infos)
264+
{
265+
var hashToInfos = new Dictionary<string, List<TaskItemInfo>>();
266+
foreach (var info in infos)
267+
{
268+
using (var fileStream = new FileStream(info.FullPath, FileMode.Open))
269+
using (var hashAlgorithm = SHA256.Create())
270+
{
271+
var hashBytes = hashAlgorithm.ComputeHash(fileStream);
272+
var hash = GetHashString(hashBytes);
273+
info.Hash = hash;
274+
275+
List<TaskItemInfo> infosWithSameHash;
276+
if (!hashToInfos.TryGetValue(hash, out infosWithSameHash))
277+
{
278+
infosWithSameHash = new List<TaskItemInfo> { info };
279+
hashToInfos.Add(hash, infosWithSameHash);
280+
}
281+
else
282+
{
283+
infosWithSameHash.Add(info);
284+
}
285+
}
286+
}
287+
288+
return hashToInfos;
289+
}
290+
291+
private static string GetHashString(byte[] hashBytes)
292+
{
293+
return BitConverter.ToString(hashBytes).Replace("-", string.Empty).ToLowerInvariant();
294+
}
295+
296+
private class TaskItemInfo
297+
{
298+
public TaskItemInfo(ITaskItem item, string fullPath)
299+
{
300+
Item = item;
301+
FullPath = fullPath;
302+
}
303+
304+
public ITaskItem Item { get; private set; }
305+
public string FullPath { get; private set; }
306+
public long FileSize { get; set; }
307+
public string HeaderHash { get; set; }
308+
public string Hash { get; set; }
309+
}
310+
}
311+
}

‎build/FindDuplicateFiles.targets‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
2+
<UsingTask
3+
TaskName="FindDuplicateFiles"
4+
TaskFactory="CodeTaskFactory"
5+
AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.Core.dll" >
6+
<ParameterGroup>
7+
<Files ParameterType="Microsoft.Build.Framework.ITaskItem[]" Required="true" />
8+
<UniqueFiles ParameterType="Microsoft.Build.Framework.ITaskItem[]" Output="true" />
9+
<DuplicateFiles ParameterType="Microsoft.Build.Framework.ITaskItem[]" Output="true" />
10+
</ParameterGroup>
11+
<Task>
12+
<Code Type="Class" Language="cs" Source="FindDuplicateFiles.cs" />
13+
</Task>
14+
</UsingTask>
15+
</Project>

0 commit comments

Comments
 (0)