Skip to content
This repository was archived by the owner on Jul 30, 2024. It is now read-only.

Commit 6fabe69

Browse files
loic-sharmaloic-sharma
authored
Merge pull request #344 from NuGet/dev
[ReleasePrep][2018.02.20]RI of dev into master
2 parents dd2effb + d0addef commit 6fabe69

80 files changed

Lines changed: 4063 additions & 906 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎LICENSE.txt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
Copyright (c) .NET Foundation. All rights reserved.
1+
Copyright (c) .NET Foundation and Contributors.
2+
3+
All rights reserved.
24

35
Licensed under the Apache License, Version 2.0 (the "License"); you may not use
46
these files except in compliance with the License. You may obtain a copy of the

‎NuGet.Jobs.sln‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.PackageSigning.H
109109
EndProject
110110
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.PackageSigning.Core.Tests", "tests\Validation.PackageSigning.Core.Tests\Validation.PackageSigning.Core.Tests.csproj", "{B4B7564A-965B-447B-927F-6749E2C08880}"
111111
EndProject
112+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Common.Job", "src\Validation.Common.Job\Validation.Common.Job.csproj", "{FA87D075-A934-4443-8D0B-5DB32640B6D7}"
113+
EndProject
114+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PackageHash", "src\PackageHash\PackageHash.csproj", "{40843020-6F0A-48F0-AC28-42FFE3A5C21E}"
115+
EndProject
112116
Global
113117
GlobalSection(SolutionConfigurationPlatforms) = preSolution
114118
Debug|Any CPU = Debug|Any CPU
@@ -277,6 +281,14 @@ Global
277281
{B4B7564A-965B-447B-927F-6749E2C08880}.Debug|Any CPU.Build.0 = Debug|Any CPU
278282
{B4B7564A-965B-447B-927F-6749E2C08880}.Release|Any CPU.ActiveCfg = Release|Any CPU
279283
{B4B7564A-965B-447B-927F-6749E2C08880}.Release|Any CPU.Build.0 = Release|Any CPU
284+
{FA87D075-A934-4443-8D0B-5DB32640B6D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
285+
{FA87D075-A934-4443-8D0B-5DB32640B6D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
286+
{FA87D075-A934-4443-8D0B-5DB32640B6D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
287+
{FA87D075-A934-4443-8D0B-5DB32640B6D7}.Release|Any CPU.Build.0 = Release|Any CPU
288+
{40843020-6F0A-48F0-AC28-42FFE3A5C21E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
289+
{40843020-6F0A-48F0-AC28-42FFE3A5C21E}.Debug|Any CPU.Build.0 = Debug|Any CPU
290+
{40843020-6F0A-48F0-AC28-42FFE3A5C21E}.Release|Any CPU.ActiveCfg = Release|Any CPU
291+
{40843020-6F0A-48F0-AC28-42FFE3A5C21E}.Release|Any CPU.Build.0 = Release|Any CPU
280292
EndGlobalSection
281293
GlobalSection(SolutionProperties) = preSolution
282294
HideSolutionNode = FALSE
@@ -312,6 +324,7 @@ Global
312324
{147A757D-864B-4C74-B8CF-14DFF9793605} = {6A776396-02B1-475D-A104-26940ADB04AB}
313325
{FC0CEF12-D501-46D1-B1BF-D4134BD8D478} = {B9D03824-A9CA-43AC-86D6-8BB399B9A228}
314326
{0C887292-C5AB-4107-946C-A53B18A38D22} = {6A776396-02B1-475D-A104-26940ADB04AB}
327+
{E6D094FB-9068-4578-B176-116F97E7506B} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
315328
{A3B0B15D-22D9-4F1F-94C4-B24B28ECF632} = {6A776396-02B1-475D-A104-26940ADB04AB}
316329
{F9690B52-3C92-42A0-B41F-1A6040C2D2EE} = {6A776396-02B1-475D-A104-26940ADB04AB}
317330
{91C060DA-736F-4DA9-A57F-CB3AC0E6CB10} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
@@ -321,6 +334,8 @@ Global
321334
{5ACE7756-F8D0-4D90-9957-872DE4A1381E} = {6A776396-02B1-475D-A104-26940ADB04AB}
322335
{2C5BE067-ADFD-49E3-BA9F-13A74436E5DB} = {6A776396-02B1-475D-A104-26940ADB04AB}
323336
{B4B7564A-965B-447B-927F-6749E2C08880} = {6A776396-02B1-475D-A104-26940ADB04AB}
337+
{FA87D075-A934-4443-8D0B-5DB32640B6D7} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
338+
{40843020-6F0A-48F0-AC28-42FFE3A5C21E} = {FA5644B5-4F08-43F6-86B3-039374312A47}
324339
EndGlobalSection
325340
GlobalSection(ExtensibilityGlobals) = postSolution
326341
SolutionGuid = {284A7AC3-FB43-4F1F-9C9C-2AF0E1F46C2B}

‎src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj‎

Lines changed: 8 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -93,87 +93,27 @@
9393
<Compile Include="ValidatorProvider.cs" />
9494
</ItemGroup>
9595
<ItemGroup>
96-
<None Include="App.config" />
96+
<None Include="App.config">
97+
<SubType>Designer</SubType>
98+
</None>
9799
<None Include="settings.json">
98100
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
99101
</None>
100102
</ItemGroup>
101103
<ItemGroup>
102-
<PackageReference Include="Autofac">
103-
<Version>4.6.1</Version>
104-
</PackageReference>
105-
<PackageReference Include="Autofac.Extensions.DependencyInjection">
106-
<Version>4.2.0</Version>
107-
</PackageReference>
108-
<PackageReference Include="EntityFramework">
109-
<Version>6.1.3</Version>
110-
</PackageReference>
111-
<PackageReference Include="Microsoft.ApplicationInsights">
112-
<Version>2.2.0</Version>
113-
</PackageReference>
114-
<PackageReference Include="Microsoft.Extensions.Configuration">
115-
<Version>1.1.2</Version>
116-
</PackageReference>
117-
<PackageReference Include="Microsoft.Extensions.Configuration.Binder">
118-
<Version>1.1.2</Version>
119-
</PackageReference>
120-
<PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions">
121-
<Version>1.1.2</Version>
122-
</PackageReference>
123-
<PackageReference Include="Microsoft.Extensions.DependencyInjection">
124-
<Version>1.1.1</Version>
125-
</PackageReference>
126-
<PackageReference Include="Microsoft.Extensions.Logging">
127-
<Version>1.1.2</Version>
128-
</PackageReference>
129-
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions">
130-
<Version>1.1.2</Version>
131-
</PackageReference>
132-
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions">
133-
<Version>1.1.2</Version>
134-
</PackageReference>
135-
<PackageReference Include="NuGet.Services.Configuration">
136-
<Version>2.13.0</Version>
137-
</PackageReference>
138-
<PackageReference Include="NuGet.Services.Contracts">
139-
<Version>2.13.0</Version>
140-
</PackageReference>
141-
<PackageReference Include="NuGet.Services.KeyVault">
142-
<Version>2.13.0</Version>
143-
</PackageReference>
144-
<PackageReference Include="NuGet.Services.Logging">
145-
<Version>2.13.0</Version>
146-
</PackageReference>
147-
<PackageReference Include="NuGet.Services.ServiceBus">
148-
<Version>2.13.0</Version>
149-
</PackageReference>
150-
<PackageReference Include="NuGet.Services.Validation">
151-
<Version>2.13.0</Version>
152-
</PackageReference>
153104
<PackageReference Include="NuGet.Services.Validation.Issues">
154-
<Version>2.13.0</Version>
155-
</PackageReference>
156-
<PackageReference Include="NuGet.Versioning">
157-
<Version>4.3.0</Version>
158-
</PackageReference>
159-
<PackageReference Include="NuGetGallery.Core">
160-
<Version>4.4.4-dev-19677</Version>
161-
</PackageReference>
162-
<PackageReference Include="Serilog">
163-
<Version>2.5.0</Version>
164-
</PackageReference>
165-
<PackageReference Include="System.Net.Http">
166-
<Version>4.3.3</Version>
167-
</PackageReference>
168-
<PackageReference Include="WindowsAzure.Storage">
169-
<Version>7.1.2</Version>
105+
<Version>2.14.0</Version>
170106
</PackageReference>
171107
</ItemGroup>
172108
<ItemGroup>
173109
<ProjectReference Include="..\NuGet.Jobs.Common\NuGet.Jobs.Common.csproj">
174110
<Project>{4b4b1efb-8f33-42e6-b79f-54e7f3293d31}</Project>
175111
<Name>NuGet.Jobs.Common</Name>
176112
</ProjectReference>
113+
<ProjectReference Include="..\Validation.Common.Job\Validation.Common.Job.csproj">
114+
<Project>{fa87d075-a934-4443-8d0b-5db32640b6d7}</Project>
115+
<Name>Validation.Common.Job</Name>
116+
</ProjectReference>
177117
<ProjectReference Include="..\Validation.Common\Validation.Common.csproj">
178118
<Project>{2539ddf3-0cc5-4a03-b5f9-39b47744a7bd}</Project>
179119
<Name>Validation.Common</Name>

‎src/NuGet.Services.Validation.Orchestrator/Telemetry/ITelemetryService.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,5 +78,11 @@ public interface ITelemetryService
7878
/// <param name="validatorType">The validator type (name) the produced the issue.</param>
7979
/// <param name="clientCode">The client code.</param>
8080
void TrackClientValidationIssue(string validatorType, string clientCode);
81+
82+
/// <summary>
83+
/// A metric for the case when orchestrator sees a package marked as available, but the blob is missing
84+
/// in the public container.
85+
/// </summary>
86+
void TrackMissingNupkgForAvailablePackage(string packageId, string normalizedVersion, string validationTrackingId);
8187
}
8288
}

‎src/NuGet.Services.Validation.Orchestrator/Telemetry/TelemetryService.cs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,17 @@ public class TelemetryService : ITelemetryService
2121
private const string ValidatorDurationSeconds = Prefix + "ValidatorDurationSeconds";
2222
private const string ValidatorStarted = Prefix + "ValidatorStarted";
2323
private const string ClientValidationIssue = Prefix + "ClientValidationIssue";
24+
private const string MissingNupkgForAvailablePackage = Prefix + "MissingNupkgForAvailablePackage";
2425

2526
private const string FromStatus = "FromStatus";
2627
private const string ToStatus = "ToStatus";
2728
private const string IsSuccess = "IsSuccess";
2829
private const string ValidatorType = "ValidatorType";
2930
private const string IssueCode = "IssueCode";
3031
private const string ClientCode = "ClientCode";
32+
private const string PackageId = "PackageId";
33+
private const string NormalizedVersion = "NormalizedVersion";
34+
private const string ValidationTrackingId = "ValidationTrackingId";
3135

3236
private readonly TelemetryClient _telemetryClient;
3337

@@ -135,5 +139,16 @@ public void TrackClientValidationIssue(string validatorType, string clientCode)
135139
{ ClientCode, clientCode },
136140
});
137141
}
142+
143+
public void TrackMissingNupkgForAvailablePackage(string packageId, string normalizedVersion, string validationTrackingId)
144+
=> _telemetryClient.TrackMetric(
145+
MissingNupkgForAvailablePackage,
146+
1,
147+
new Dictionary<string, string>
148+
{
149+
{ PackageId, packageId },
150+
{ NormalizedVersion, normalizedVersion },
151+
{ ValidationTrackingId, validationTrackingId },
152+
});
138153
}
139154
}

‎src/NuGet.Services.Validation.Orchestrator/ValidationMessageHandler.cs‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -33,19 +33,27 @@ public ValidationMessageHandler(
3333

3434
public async Task<bool> HandleAsync(PackageValidationMessageData message)
3535
{
36-
var package = _galleryPackageService.FindPackageByIdAndVersionStrict(message.PackageId, message.PackageVersion);
37-
38-
if (package == null)
36+
if (message == null)
3937
{
40-
// no package in DB yet. Might have received message a bit early, need to retry later
41-
_logger.LogInformation("Did not find information in DB for package {PackageId} {PackageVersion}",
42-
message.PackageId,
43-
message.PackageVersion);
44-
return false;
38+
throw new ArgumentNullException(nameof(message));
4539
}
4640

47-
using (_logger.BeginScope("Handling message for {PackageId} {PackageVersion} validation set {ValidationSetId}", message.PackageId, message.PackageVersion, message.ValidationTrackingId))
41+
using (_logger.BeginScope("Handling message for {PackageId} {PackageVersion} validation set {ValidationSetId}",
42+
message.PackageId,
43+
message.PackageVersion,
44+
message.ValidationTrackingId))
4845
{
46+
var package = _galleryPackageService.FindPackageByIdAndVersionStrict(message.PackageId, message.PackageVersion);
47+
48+
if (package == null)
49+
{
50+
// no package in DB yet. Might have received message a bit early, need to retry later
51+
_logger.LogInformation("Did not find information in DB for package {PackageId} {PackageVersion}",
52+
message.PackageId,
53+
message.PackageVersion);
54+
return false;
55+
}
56+
4957
var validationSet = await _validationSetProvider.TryGetOrCreateValidationSetAsync(message.ValidationTrackingId, package);
5058

5159
if (validationSet == null)

‎src/NuGet.Services.Validation.Orchestrator/ValidationOutcomeProcessor.cs‎

Lines changed: 33 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,13 @@ ValidationConfigurationItem GetValidationConfigurationItem(string validationName
6060

6161
if (AnyValidationFailed(validationSet, GetValidationConfigurationItem))
6262
{
63+
var failedValidations = GetFailedValidations(validationSet, GetValidationConfigurationItem).ToList();
64+
6365
_logger.LogWarning("Some validations failed for package {PackageId} {PackageVersion}, validation set {ValidationSetId}: {FailedValidations}",
6466
package.PackageRegistration.Id,
6567
package.NormalizedVersion,
6668
validationSet.ValidationTrackingId,
67-
GetFailedValidations(validationSet, GetValidationConfigurationItem));
69+
failedValidations);
6870

6971
// The only way we can move to the failed validation state is if the package is currently in the
7072
// validating state. This has a beneficial side effect of only sending a failed validation email to the
@@ -122,6 +124,20 @@ ValidationConfigurationItem GetValidationConfigurationItem(string validationName
122124
package.PackageRegistration.Id,
123125
package.NormalizedVersion,
124126
validationSet.ValidationTrackingId);
127+
128+
if (!await _packageFileService.DoesPackageFileExistAsync(package))
129+
{
130+
var validationPackageAvailable = await _packageFileService.DoesValidationPackageFileExistAsync(package);
131+
132+
_logger.LogWarning("Package {PackageId} {PackageVersion} is marked as available, but does not exist " +
133+
"in public container. Does package exist in validation container: {ExistsInValidation}",
134+
package.PackageRegistration.Id,
135+
package.NormalizedVersion,
136+
validationPackageAvailable);
137+
138+
// report missing package, don't try to fix up anything. This shouldn't happen and needs an investigation.
139+
TrackMissingNupkgForAvailablePackage(validationSet);
140+
}
125141
}
126142
_logger.LogInformation("Done processing {PackageId} {PackageVersion} {ValidationSetId}",
127143
package.PackageRegistration.Id,
@@ -139,6 +155,14 @@ ValidationConfigurationItem GetValidationConfigurationItem(string validationName
139155
}
140156
}
141157

158+
private void TrackMissingNupkgForAvailablePackage(PackageValidationSet validationSet)
159+
{
160+
_telemetryService.TrackMissingNupkgForAvailablePackage(
161+
validationSet.PackageId,
162+
validationSet.PackageNormalizedVersion,
163+
validationSet.ValidationTrackingId.ToString());
164+
}
165+
142166
private void TrackTotalValidationDuration(PackageValidationSet validationSet, bool isSuccess)
143167
{
144168
_telemetryService.TrackTotalValidationDuration(
@@ -155,17 +179,15 @@ private async Task MoveFileToPublicStorageAndMarkPackageAsAvailable(PackageValid
155179
var packageStream = await _packageFileService.DownloadValidationPackageFileAsync(package);
156180
await _packageFileService.SavePackageFileAsync(package, packageStream);
157181

182+
_logger.LogInformation("Marking package {PackageId} {PackageVersion}, validation set {ValidationSetId} as {PackageStatus} in DB",
183+
package.PackageRegistration.Id,
184+
package.NormalizedVersion,
185+
validationSet.ValidationTrackingId,
186+
PackageStatus.Available);
187+
158188
try
159189
{
160-
_logger.LogInformation("Marking package {PackageId} {PackageVersion}, validation set {ValidationSetId} as {PackageStatus} in DB",
161-
package.PackageRegistration.Id,
162-
package.NormalizedVersion,
163-
validationSet.ValidationTrackingId,
164-
PackageStatus.Available);
165-
166190
await UpdatePackageStatusAsync(package, PackageStatus.Available);
167-
168-
_messageService.SendPackagePublishedMessage(package);
169191
}
170192
catch (Exception e)
171193
{
@@ -182,6 +204,8 @@ private async Task MoveFileToPublicStorageAndMarkPackageAsAvailable(PackageValid
182204
throw;
183205
}
184206

207+
_messageService.SendPackagePublishedMessage(package);
208+
185209
_logger.LogInformation("Deleting from the source for package {PackageId} {PackageVersion}, validation set {ValidationSetId}",
186210
package.PackageRegistration.Id,
187211
package.NormalizedVersion,

‎src/NuGet.Services.Validation.Orchestrator/ValidationStorageService.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -137,12 +137,13 @@ private void TrackValidationStatus(PackageValidation packageValidation)
137137
packageValidation.Type,
138138
isSuccess);
139139

140+
var issues = (packageValidation.PackageValidationIssues ?? Enumerable.Empty<PackageValidationIssue>()).ToList();
140141
_telemetryService.TrackValidationIssueCount(
141-
packageValidation.PackageValidationIssues.Count,
142+
issues.Count,
142143
packageValidation.Type,
143144
isSuccess);
144145

145-
foreach (var issue in packageValidation?.PackageValidationIssues ?? Enumerable.Empty<PackageValidationIssue>())
146+
foreach (var issue in issues)
146147
{
147148
_telemetryService.TrackValidationIssue(packageValidation.Type, issue.IssueCode);
148149

0 commit comments

Comments
 (0)