Skip to content
This repository was archived by the owner on Jul 30, 2024. It is now read-only.

Commit f11937d

Browse files
joelverhagenjoelverhagen
committed
Each validation set gets its own copy of the package (#361)
Progress on NuGet/Engineering#1190 Address NuGet/Engineering#1030
1 parent 5ead5b6 commit f11937d

13 files changed

Lines changed: 682 additions & 125 deletions

‎src/NuGet.Services.Validation.Orchestrator/IValidationPackageFileService.cs‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Threading.Tasks;
6+
using NuGetGallery;
7+
8+
namespace NuGet.Services.Validation.Orchestrator
9+
{
10+
public interface IValidationPackageFileService : ICorePackageFileService
11+
{
12+
/// <summary>
13+
/// Copy a package from the validation container to a location specific for the validation set. This allows the
14+
/// validation set to have its own copy of the package to mutate (via <see cref="IProcessor"/>) and validate.
15+
/// </summary>
16+
/// <param name="validationSet">The validation set, containing validation set and package identifiers.</param>
17+
Task CopyValidationPackageForValidationSetAsync(PackageValidationSet validationSet);
18+
19+
/// <summary>
20+
/// Copy a package from the packages container to a location specific for the validation set. This allows the
21+
/// validation set to have its own copy of the package to mutate (via <see cref="IProcessor"/>) and validate.
22+
/// </summary>
23+
/// <param name="validationSet">The validation set, containing validation set and package identifiers.</param>
24+
Task CopyPackageFileForValidationSetAsync(PackageValidationSet validationSet);
25+
26+
/// <summary>
27+
/// Copy a package from a location specific for the validation set to the packages container.
28+
/// </summary>
29+
/// <param name="validationSet">The validation set, containing validation set and package identifiers.</param>
30+
Task CopyValidationSetPackageToPackageFileAsync(PackageValidationSet validationSet);
31+
32+
/// <summary>
33+
/// Copy a package from the validation container to the packages container.
34+
/// </summary>
35+
/// <param name="id">The package ID.</param>
36+
/// <param name="normalizedVersion">The normalized package version.</param>
37+
Task CopyValidationPackageToPackageFileAsync(string id, string normalizedVersion);
38+
39+
/// <summary>
40+
/// Delete a package from a location specific for the validation set.
41+
/// </summary>
42+
/// <param name="validationSet">The validation set, containing validation set and package identifiers.</param>
43+
Task DeletePackageForValidationSetAsync(PackageValidationSet validationSet);
44+
45+
/// <summary>
46+
/// Generates the URI for the specified validating package, which can be used to download it.
47+
/// </summary>
48+
/// <param name="validationSet">The validation set, containing validation set and package identifiers.</param>
49+
/// <param name="endOfAccess">The timestamp that limits the URI usage period.</param>
50+
/// <returns>Time limited (if implementation supports) URI for the package.</returns>
51+
Task<Uri> GetPackageForValidationSetReadUriAsync(PackageValidationSet validationSet, DateTimeOffset endOfAccess);
52+
53+
/// <summary>
54+
/// Checks whether the validation set's package file exists.
55+
/// </summary>
56+
/// <param name="validationSet">The validation set, containing validation set and package identifiers.</param>
57+
/// <returns>True if file exists, false otherwise</returns>
58+
Task<bool> DoesValidationSetPackageExistAsync(PackageValidationSet validationSet);
59+
}
60+
}

‎src/NuGet.Services.Validation.Orchestrator/Job.cs‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -287,21 +287,27 @@ private static IServiceProvider CreateProvider(IServiceCollection services)
287287
parameterKey: ValidationStorageBindingKey);
288288

289289
containerBuilder
290-
.RegisterKeyedTypeWithKeyedParameter<NuGetGallery.ICorePackageFileService, NuGetGallery.CorePackageFileService, NuGetGallery.ICoreFileStorageService>(
290+
.RegisterKeyedTypeWithKeyedParameter<IValidationPackageFileService, ValidationPackageFileService, NuGetGallery.ICoreFileStorageService>(
291291
typeKey: ValidationStorageBindingKey,
292292
parameterKey: ValidationStorageBindingKey);
293293

294294
containerBuilder
295295
.RegisterTypeWithKeyedParameter<
296296
IValidationOutcomeProcessor,
297297
ValidationOutcomeProcessor,
298-
NuGetGallery.ICorePackageFileService>(ValidationStorageBindingKey);
298+
IValidationPackageFileService>(ValidationStorageBindingKey);
299+
300+
containerBuilder
301+
.RegisterTypeWithKeyedParameter<
302+
IValidationSetProvider,
303+
ValidationSetProvider,
304+
IValidationPackageFileService>(ValidationStorageBindingKey);
299305

300306
containerBuilder
301307
.RegisterTypeWithKeyedParameter<
302308
IValidationSetProcessor,
303309
ValidationSetProcessor,
304-
NuGetGallery.ICorePackageFileService>(ValidationStorageBindingKey);
310+
IValidationPackageFileService>(ValidationStorageBindingKey);
305311

306312
containerBuilder
307313
.RegisterType<ScopedMessageHandler<PackageValidationMessageData>>()

‎src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
<Compile Include="Error.cs" />
5454
<Compile Include="IMessageService.cs" />
5555
<Compile Include="IValidationOutcomeProcessor.cs" />
56+
<Compile Include="IValidationPackageFileService.cs" />
5657
<Compile Include="IValidationSetProcessor.cs" />
5758
<Compile Include="IValidationSetProvider.cs" />
5859
<Compile Include="IValidationStorageService.cs" />
@@ -79,6 +80,7 @@
7980
<Compile Include="Configuration\ValidationConfiguration.cs" />
8081
<Compile Include="Configuration\ValidationConfigurationItem.cs" />
8182
<Compile Include="ValidationFailureBehavior.cs" />
83+
<Compile Include="ValidationPackageFileService.cs" />
8284
<Compile Include="Vcs\IPackageCriteria.cs" />
8385
<Compile Include="Vcs\IPackageCriteriaEvaluator.cs" />
8486
<Compile Include="Vcs\PackageCriteriaEvaluator.cs" />

‎src/NuGet.Services.Validation.Orchestrator/ValidationOutcomeProcessor.cs‎

Lines changed: 41 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ namespace NuGet.Services.Validation.Orchestrator
1515
public class ValidationOutcomeProcessor : IValidationOutcomeProcessor
1616
{
1717
private readonly ICorePackageService _galleryPackageService;
18-
private readonly ICorePackageFileService _packageFileService;
18+
private readonly IValidationPackageFileService _packageFileService;
1919
private readonly IPackageValidationEnqueuer _validationEnqueuer;
2020
private readonly ValidationConfiguration _validationConfiguration;
2121
private readonly IMessageService _messageService;
@@ -24,7 +24,7 @@ public class ValidationOutcomeProcessor : IValidationOutcomeProcessor
2424

2525
public ValidationOutcomeProcessor(
2626
ICorePackageService galleryPackageService,
27-
ICorePackageFileService packageFileService,
27+
IValidationPackageFileService packageFileService,
2828
IPackageValidationEnqueuer validationEnqueuer,
2929
IOptionsSnapshot<ValidationConfiguration> validationConfigurationAccessor,
3030
IMessageService messageService,
@@ -106,6 +106,8 @@ ValidationConfigurationItem GetValidationConfigurationItem(string validationName
106106
validationSet.ValidationTrackingId);
107107
}
108108

109+
await _packageFileService.DeletePackageForValidationSetAsync(validationSet);
110+
109111
TrackTotalValidationDuration(validationSet, isSuccess: false);
110112
}
111113
else if (AllValidationsSucceeded(validationSet, GetValidationConfigurationItem))
@@ -114,6 +116,7 @@ ValidationConfigurationItem GetValidationConfigurationItem(string validationName
114116
package.PackageRegistration.Id,
115117
package.NormalizedVersion,
116118
validationSet.ValidationTrackingId);
119+
117120
if (package.PackageStatusKey != PackageStatus.Available)
118121
{
119122
await MoveFileToPublicStorageAndMarkPackageAsAvailable(validationSet, package);
@@ -139,6 +142,9 @@ ValidationConfigurationItem GetValidationConfigurationItem(string validationName
139142
TrackMissingNupkgForAvailablePackage(validationSet);
140143
}
141144
}
145+
146+
await _packageFileService.DeletePackageForValidationSetAsync(validationSet);
147+
142148
_logger.LogInformation("Done processing {PackageId} {PackageVersion} {ValidationSetId}",
143149
package.PackageRegistration.Id,
144150
package.NormalizedVersion,
@@ -176,23 +182,27 @@ private async Task MoveFileToPublicStorageAndMarkPackageAsAvailable(PackageValid
176182
package.PackageRegistration.Id,
177183
package.NormalizedVersion,
178184
validationSet.ValidationTrackingId);
179-
var packageStream = await _packageFileService.DownloadValidationPackageFileAsync(package);
180185

181-
try
186+
if (await _packageFileService.DoesValidationSetPackageExistAsync(validationSet))
182187
{
183-
await _packageFileService.SavePackageFileAsync(package, packageStream);
188+
await CopyAsync(
189+
validationSet,
190+
package,
191+
x => _packageFileService.CopyValidationSetPackageToPackageFileAsync(x));
184192
}
185-
catch (InvalidOperationException)
193+
else
186194
{
187-
// The package already exists in the packages container. This can happen if the DB commit below fails
188-
// and this flow is retried. We assume that the package content has not changed. Today there is no way
189-
// for the content to change. Hard deletes (the one way a package ID and version can get different
190-
// content) delete from both the packages and validating container so this can't be a mismatch.
191195
_logger.LogInformation(
192-
"Package already exists in packages container for {PackageId} {PackageVersion}, validation set {ValidationSetId}",
196+
"The package specific to the validation set does not exist. Falling back to the validation " +
197+
"container for package {PackageId} {PackageVersion}, validation set {ValidationSetId}",
193198
package.PackageRegistration.Id,
194199
package.NormalizedVersion,
195200
validationSet.ValidationTrackingId);
201+
202+
await CopyAsync(
203+
validationSet,
204+
package,
205+
x => _packageFileService.CopyValidationPackageToPackageFileAsync(x.PackageId, x.PackageNormalizedVersion));
196206
}
197207

198208
_logger.LogInformation("Marking package {PackageId} {PackageVersion}, validation set {ValidationSetId} as {PackageStatus} in DB",
@@ -229,6 +239,26 @@ private async Task MoveFileToPublicStorageAndMarkPackageAsAvailable(PackageValid
229239
await _packageFileService.DeleteValidationPackageFileAsync(package.PackageRegistration.Id, package.Version);
230240
}
231241

242+
private async Task CopyAsync(PackageValidationSet validationSet, Package package, Func<PackageValidationSet, Task> copyAsync)
243+
{
244+
try
245+
{
246+
await copyAsync(validationSet);
247+
}
248+
catch (InvalidOperationException)
249+
{
250+
// The package already exists in the packages container. This can happen if the DB commit below fails
251+
// and this flow is retried. We assume that the package content has not changed. Today there is no way
252+
// for the content to change. Hard deletes (the one way a package ID and version can get different
253+
// content) delete from both the packages and validating container so this can't be a mismatch.
254+
_logger.LogInformation(
255+
"Package already exists in packages container for {PackageId} {PackageVersion}, validation set {ValidationSetId}",
256+
package.PackageRegistration.Id,
257+
package.NormalizedVersion,
258+
validationSet.ValidationTrackingId);
259+
}
260+
}
261+
232262
private bool AllValidationsSucceeded(
233263
PackageValidationSet packageValidationSet,
234264
Func<string, ValidationConfigurationItem> getValidationConfigurationItem)

‎src/NuGet.Services.Validation.Orchestrator/ValidationPackageFileService.cs‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Threading.Tasks;
6+
using Microsoft.Extensions.Logging;
7+
using NuGetGallery;
8+
9+
namespace NuGet.Services.Validation.Orchestrator
10+
{
11+
public class ValidationPackageFileService : CorePackageFileService, IValidationPackageFileService
12+
{
13+
private readonly ICoreFileStorageService _fileStorageService;
14+
private readonly ILogger<ValidationPackageFileService> _logger;
15+
16+
public ValidationPackageFileService(
17+
ICoreFileStorageService fileStorageService,
18+
ILogger<ValidationPackageFileService> logger) : base(fileStorageService)
19+
{
20+
_fileStorageService = fileStorageService ?? throw new ArgumentNullException(nameof(fileStorageService));
21+
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
22+
}
23+
24+
public Task CopyValidationPackageForValidationSetAsync(PackageValidationSet validationSet)
25+
{
26+
var srcFileName = BuildFileName(
27+
validationSet.PackageId,
28+
validationSet.PackageNormalizedVersion,
29+
CoreConstants.PackageFileSavePathTemplate,
30+
CoreConstants.NuGetPackageFileExtension);
31+
32+
return CopyFileAsync(
33+
CoreConstants.ValidationFolderName,
34+
srcFileName,
35+
CoreConstants.ValidationFolderName,
36+
BuildValidationSetPackageFileName(validationSet));
37+
}
38+
39+
public Task CopyPackageFileForValidationSetAsync(PackageValidationSet validationSet)
40+
{
41+
var srcFileName = BuildFileName(
42+
validationSet.PackageId,
43+
validationSet.PackageNormalizedVersion,
44+
CoreConstants.PackageFileSavePathTemplate,
45+
CoreConstants.NuGetPackageFileExtension);
46+
47+
return CopyFileAsync(
48+
CoreConstants.PackagesFolderName,
49+
srcFileName,
50+
CoreConstants.ValidationFolderName,
51+
BuildValidationSetPackageFileName(validationSet));
52+
}
53+
54+
public Task CopyValidationPackageToPackageFileAsync(string id, string normalizedVersion)
55+
{
56+
var fileName = BuildFileName(
57+
id,
58+
normalizedVersion,
59+
CoreConstants.PackageFileSavePathTemplate,
60+
CoreConstants.NuGetPackageFileExtension);
61+
62+
return CopyFileAsync(
63+
CoreConstants.ValidationFolderName,
64+
fileName,
65+
CoreConstants.PackagesFolderName,
66+
fileName);
67+
}
68+
69+
public Task CopyValidationSetPackageToPackageFileAsync(PackageValidationSet validationSet)
70+
{
71+
var srcFileName = BuildValidationSetPackageFileName(validationSet);
72+
73+
var destFileName = BuildFileName(
74+
validationSet.PackageId,
75+
validationSet.PackageNormalizedVersion,
76+
CoreConstants.PackageFileSavePathTemplate,
77+
CoreConstants.NuGetPackageFileExtension);
78+
79+
return CopyFileAsync(
80+
CoreConstants.ValidationFolderName,
81+
srcFileName,
82+
CoreConstants.PackagesFolderName,
83+
destFileName);
84+
}
85+
86+
public Task<bool> DoesValidationSetPackageExistAsync(PackageValidationSet validationSet)
87+
{
88+
var fileName = BuildValidationSetPackageFileName(validationSet);
89+
90+
return _fileStorageService.FileExistsAsync(CoreConstants.ValidationFolderName, fileName);
91+
}
92+
93+
public Task DeletePackageForValidationSetAsync(PackageValidationSet validationSet)
94+
{
95+
var fileName = BuildValidationSetPackageFileName(validationSet);
96+
97+
_logger.LogInformation(
98+
"Deleting package for validation set {ValidationTrackingId} from {FolderName}/{FileName}.",
99+
validationSet.ValidationTrackingId,
100+
CoreConstants.ValidationFolderName,
101+
fileName);
102+
103+
return _fileStorageService.DeleteFileAsync(CoreConstants.ValidationFolderName, fileName);
104+
}
105+
106+
public Task<Uri> GetPackageForValidationSetReadUriAsync(PackageValidationSet validationSet, DateTimeOffset endOfAccess)
107+
{
108+
var fileName = BuildValidationSetPackageFileName(validationSet);
109+
110+
return _fileStorageService.GetFileReadUriAsync(CoreConstants.ValidationFolderName, fileName, endOfAccess);
111+
}
112+
113+
private Task CopyFileAsync(string srcFolderName, string srcFileName, string destFolderName, string destFileName)
114+
{
115+
_logger.LogInformation(
116+
"Copying file {SrcFolderName}/{SrcFileName} to {DestFolderName}/{DestFileName}.",
117+
srcFolderName,
118+
srcFileName,
119+
destFolderName,
120+
destFileName);
121+
122+
return _fileStorageService.CopyFileAsync(
123+
srcFolderName,
124+
srcFileName,
125+
destFolderName,
126+
destFileName);
127+
}
128+
129+
private static string BuildValidationSetPackageFileName(PackageValidationSet validationSet)
130+
{
131+
return $"validation-sets/{validationSet.ValidationTrackingId}/" +
132+
$"{validationSet.PackageId.ToLowerInvariant()}." +
133+
$"{validationSet.PackageNormalizedVersion.ToLowerInvariant()}" +
134+
CoreConstants.NuGetPackageFileExtension;
135+
}
136+
}
137+
}

0 commit comments

Comments
 (0)