Skip to content
This repository was archived by the owner on Jul 30, 2024. It is now read-only.

Commit a1ad85a

Browse files
joelverhagenjoelverhagen
committed
Merge branch 'dev' into master (#665)
2 parents b926704 + 7ed9eae commit a1ad85a

18 files changed

Lines changed: 361 additions & 52 deletions

File tree

‎build.ps1‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ param (
99
[string]$SemanticVersion = '1.0.0-zlocal',
1010
[string]$Branch = 'zlocal',
1111
[string]$CommitSHA,
12-
[string]$BuildBranch = '001c5deac1f1e2e194c1d0d5eec14183bb78e4cf'
12+
[string]$BuildBranch = 'b5f9d1c89da96c462935e2195ceb00e69287b93e'
1313
)
1414

1515
$msBuildVersion = 15;
@@ -117,7 +117,7 @@ Invoke-BuildStep 'Restoring solution packages' { `
117117

118118
Invoke-BuildStep 'Building solution' {
119119
param($Configuration, $BuildNumber, $SolutionPath, $SkipRestore)
120-
Build-Solution $Configuration $BuildNumber -MSBuildVersion "$msBuildVersion" $SolutionPath -MSBuildProperties "/m" -SkipRestore:$SkipRestore `
120+
Build-Solution $Configuration $BuildNumber -MSBuildVersion "$msBuildVersion" $SolutionPath -SkipRestore:$SkipRestore `
121121
} `
122122
-args $Configuration, $BuildNumber, (Join-Path $PSScriptRoot "NuGet.Jobs.sln"), $SkipRestore `
123123
-ev +BuildErrors

‎src/Monitoring.RebootSearchInstance/SearchInstanceRebooter.cs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using System;
55
using System.Diagnostics;
66
using System.Linq;
7+
using System.Net;
78
using System.Threading;
89
using System.Threading.Tasks;
910
using Microsoft.Extensions.Logging;
@@ -220,6 +221,20 @@ private async Task<InstanceHealth> DetermineInstanceHealthAsync(
220221
{
221222
commitDateTime = await _searchServiceClient.GetCommitDateTimeAsync(instance, token);
222223
}
224+
catch (HttpResponseException ex) when (ex.StatusCode == HttpStatusCode.ServiceUnavailable
225+
|| ex.StatusCode == HttpStatusCode.InternalServerError)
226+
{
227+
_logger.LogInformation(
228+
(EventId)0,
229+
ex,
230+
"The HTTP response when hitting {DiagUrl} was {StatusCode} {ReasonPhrase}. Considering this " +
231+
"instance as an unhealthy state.",
232+
instance.DiagUrl,
233+
(int)ex.StatusCode,
234+
ex.ReasonPhrase);
235+
236+
return InstanceHealth.Unhealthy;
237+
}
223238
catch (Exception ex)
224239
{
225240
_logger.LogInformation(

‎src/NuGet.Services.Validation.Orchestrator/IValidationPackageFileService.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,14 @@ Task CopyValidationSetPackageToPackageFileAsync(
124124
/// successive read and write operations.</exception>
125125
Task<PackageStreamMetadata> UpdatePackageBlobMetadataAsync(PackageValidationSet validationSet);
126126

127+
/// <summary>
128+
/// Updates package blob properties.
129+
/// </summary>
130+
/// <param name="validationSet">A validationSet that will identify the package that will have its blob metadata updated.</param>
131+
/// <exception cref="Microsoft.WindowsAzure.Storage.StorageException">Thrown if the blob has changed between
132+
/// successive read and write operations.</exception>
133+
Task UpdatePackageBlobPropertiesAsync(PackageValidationSet validationSet);
134+
127135
/// <summary>
128136
/// Reads the ETag for the package in the public container.
129137
/// </summary>

‎src/NuGet.Services.Validation.Orchestrator/PackageStatusProcessor.cs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -94,13 +94,16 @@ protected virtual async Task MakePackageAvailableAsync(IValidatingEntity<T> vali
9494
// 1) Operate on blob storage.
9595
var copied = await UpdatePublicPackageAsync(validationSet);
9696

97-
// 2) Update the package's blob metadata in the packages blob storage container.
97+
// 2) Update the package's blob metadata in the public blob storage container.
9898
var metadata = await _packageFileService.UpdatePackageBlobMetadataAsync(validationSet);
9999

100-
// 3) Operate on the database.
100+
// 3) Update the package's blob properties in the public blob storage container.
101+
await _packageFileService.UpdatePackageBlobPropertiesAsync(validationSet);
102+
103+
// 4) Operate on the database.
101104
var fromStatus = await MarkPackageAsAvailableAsync(validationSet, validatingEntity, metadata, copied);
102105

103-
// 4) Emit telemetry and clean up.
106+
// 5) Emit telemetry and clean up.
104107
if (fromStatus != PackageStatus.Available)
105108
{
106109
_telemetryService.TrackPackageStatusChange(fromStatus, PackageStatus.Available);
@@ -113,7 +116,7 @@ protected virtual async Task MakePackageAvailableAsync(IValidatingEntity<T> vali
113116
await _packageFileService.DeleteValidationPackageFileAsync(validationSet);
114117
}
115118

116-
// 4) Verify the package still exists (we've had bugs here before).
119+
// 5) Verify the package still exists (we've had bugs here before).
117120
if (validatingEntity.Status == PackageStatus.Available
118121
&& !await _packageFileService.DoesPackageFileExistAsync(validationSet))
119122
{

‎src/NuGet.Services.Validation.Orchestrator/ValidationPackageFileService.cs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,30 @@ public Task CopyPackageUrlForValidationSetAsync(PackageValidationSet validationS
223223
AccessConditionWrapper.GenerateEmptyCondition());
224224
}
225225

226+
public async Task UpdatePackageBlobPropertiesAsync(PackageValidationSet validationSet)
227+
{
228+
var fileName = BuildFileName(
229+
validationSet,
230+
_fileMetadataService.FileSavePathTemplate,
231+
_fileMetadataService.FileExtension);
232+
233+
// This will throw if the ETag changes between read and write operations.
234+
await _fileStorageService.SetPropertiesAsync(
235+
_fileMetadataService.FileFolderName,
236+
fileName,
237+
async (lazyStream, blobProperties) =>
238+
{
239+
// Update the cache control only if the cache control is not the same as the default value.
240+
if (!string.Equals(blobProperties.CacheControl, CoreConstants.DefaultCacheControl, StringComparison.OrdinalIgnoreCase))
241+
{
242+
blobProperties.CacheControl = CoreConstants.DefaultCacheControl;
243+
return await Task.FromResult(true);
244+
}
245+
246+
return await Task.FromResult(false);
247+
});
248+
}
249+
226250
public async Task<PackageStreamMetadata> UpdatePackageBlobMetadataAsync(PackageValidationSet validationSet)
227251
{
228252
var fileName = BuildFileName(

‎src/NuGet.Services.Validation.Orchestrator/ValidationSetProvider.cs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,11 @@ private PackageValidationSet InitializeValidationSet(PackageValidationMessageDat
152152
ValidatingType = message.ValidatingType
153153
};
154154

155-
foreach (var validation in _validationConfiguration.Validations)
155+
var validationsToStart = _validationConfiguration
156+
.Validations
157+
.Where(v => v.ShouldStart);
158+
159+
foreach (var validation in validationsToStart)
156160
{
157161
var packageValidation = new PackageValidation
158162
{

‎src/PackageLagMonitor/HttpResponseException.cs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Net;
6+
7+
namespace NuGet.Jobs.Montoring.PackageLag
8+
{
9+
public class HttpResponseException : Exception
10+
{
11+
public HttpResponseException(HttpStatusCode statusCode, string reasonPhrase, string message)
12+
: base(message)
13+
{
14+
StatusCode = statusCode;
15+
ReasonPhrase = reasonPhrase;
16+
}
17+
18+
public HttpStatusCode StatusCode { get; }
19+
public string ReasonPhrase { get; }
20+
}
21+
}

‎src/PackageLagMonitor/Monitoring.PackageLag.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
</PropertyGroup>
4949
<ItemGroup>
5050
<Compile Include="AzureManagementAPIWrapperConfiguration.cs" />
51+
<Compile Include="HttpResponseException.cs" />
5152
<Compile Include="Instance.cs" />
5253
<Compile Include="ISearchServiceClient.cs" />
5354
<Compile Include="Job.cs" />

‎src/PackageLagMonitor/SearchServiceClient.cs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,15 @@ public async Task<DateTimeOffset> GetCommitDateTimeAsync(Instance instance, Canc
4646
HttpCompletionOption.ResponseContentRead,
4747
token))
4848
{
49+
if (!diagResponse.IsSuccessStatusCode)
50+
{
51+
throw new HttpResponseException(
52+
diagResponse.StatusCode,
53+
diagResponse.ReasonPhrase,
54+
$"The HTTP response when hitting {instance.DiagUrl} was {(int)diagResponse.StatusCode} " +
55+
$"{diagResponse.ReasonPhrase}, which is not successful.");
56+
}
57+
4958
var diagContent = diagResponse.Content;
5059
var searchDiagResultRaw = await diagContent.ReadAsStringAsync();
5160
var searchDiagResultObject = JsonConvert.DeserializeObject<SearchDiagnosticResponse>(searchDiagResultRaw);

‎src/Stats.ImportAzureCdnStatistics/UserAgentParser.cs‎

Lines changed: 52 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,21 +4,47 @@
44
using System;
55
using System.IO;
66
using System.Reflection;
7+
using System.Text.RegularExpressions;
78
using UAParser;
89

910
namespace Stats.ImportAzureCdnStatistics
1011
{
1112
public class UserAgentParser
1213
{
1314
private static readonly Parser _defaultParser;
14-
private static readonly Parser _customParser;
15+
private static readonly Parser _knownClientsParser;
16+
private static readonly Parser _knownClientsInChinaParser;
1517

1618
static UserAgentParser()
1719
{
1820
_defaultParser = Parser.GetDefault();
1921

2022
var yaml = ReadKnownClientsYaml();
21-
_customParser = Parser.FromYaml(yaml);
23+
_knownClientsParser = Parser.FromYaml(yaml);
24+
25+
var patchedYaml = AddSupportForChinaCdn(yaml);
26+
_knownClientsInChinaParser = Parser.FromYaml(patchedYaml);
27+
}
28+
29+
private static string AddSupportForChinaCdn(string yaml)
30+
{
31+
// Seems like user agent headers from requests hitting the China CDN endpoints
32+
// are using '+' characters instead of whitespace characters
33+
34+
var patchedYaml = Regex.Replace(
35+
yaml,
36+
@"(?:[:]\s'\()+([\w-.\s]+)(?:\))+", // Look for any matches of : '(user agent)
37+
ReplaceWhitespaceWithPlusSign, // Replace whitespace ' ' character with '+' character in the user agent matches
38+
RegexOptions.Compiled);
39+
40+
return patchedYaml;
41+
}
42+
43+
private static string ReplaceWhitespaceWithPlusSign(Match match)
44+
{
45+
// The + sign needs to be escaped by a \
46+
// as it is output to another regex in YAML.
47+
return ": '(" + match.Groups[1].Value.Replace(" ", @"\+") + ")";
2248
}
2349

2450
private static string ReadKnownClientsYaml()
@@ -35,7 +61,14 @@ private static string ReadKnownClientsYaml()
3561
public UserAgent ParseUserAgent(string userAgent)
3662
{
3763
// try custom parser first
38-
var parsedResult = _customParser.ParseUserAgent(userAgent);
64+
var parsedResult = _knownClientsParser.ParseUserAgent(userAgent);
65+
66+
if (string.Equals(parsedResult.Family, "other", StringComparison.InvariantCultureIgnoreCase))
67+
{
68+
// fallback to China parser
69+
parsedResult = _knownClientsInChinaParser.ParseUserAgent(userAgent);
70+
}
71+
3972
if (string.Equals(parsedResult.Family, "other", StringComparison.InvariantCultureIgnoreCase))
4073
{
4174
// fallback to default parser
@@ -47,7 +80,14 @@ public UserAgent ParseUserAgent(string userAgent)
4780
public OS ParseOS(string userAgent)
4881
{
4982
// try custom parser first
50-
var parsedResult = _customParser.ParseOS(userAgent);
83+
var parsedResult = _knownClientsParser.ParseOS(userAgent);
84+
85+
if (string.Equals(parsedResult.Family, "other", StringComparison.InvariantCultureIgnoreCase))
86+
{
87+
// fallback to China parser
88+
parsedResult = _knownClientsInChinaParser.ParseOS(userAgent);
89+
}
90+
5191
if (string.Equals(parsedResult.Family, "other", StringComparison.InvariantCultureIgnoreCase))
5292
{
5393
// fallback to default parser
@@ -60,7 +100,14 @@ public OS ParseOS(string userAgent)
60100
public Device ParseDevice(string userAgent)
61101
{
62102
// try custom parser first
63-
var parsedResult = _customParser.ParseDevice(userAgent);
103+
var parsedResult = _knownClientsParser.ParseDevice(userAgent);
104+
105+
if (string.Equals(parsedResult.Family, "other", StringComparison.InvariantCultureIgnoreCase))
106+
{
107+
// fallback to China parser
108+
parsedResult = _knownClientsInChinaParser.ParseDevice(userAgent);
109+
}
110+
64111
if (string.Equals(parsedResult.Family, "other", StringComparison.InvariantCultureIgnoreCase))
65112
{
66113
// fallback to default parser

0 commit comments

Comments
 (0)