Skip to content

Commit dc7abf2

Browse files
agragr
authored
Merge pull request #10144 from NuGet/dev
[ReleasePrep][2024.08.20] RI dev to main
2 parents 907f3ab + 146473e commit dc7abf2

23 files changed

Lines changed: 237 additions & 199 deletions

File tree

‎build/common.ps1‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,6 @@ Function Invoke-BuildStep {
161161
if ($env:TF_BUILD) {
162162
Write-Output "##[group]$BuildStep"
163163
}
164-
165164
Trace-Log "[BEGIN] $BuildStep"
166165
$sw = [Diagnostics.Stopwatch]::StartNew()
167166
$completed = $false
@@ -173,6 +172,9 @@ Function Invoke-BuildStep {
173172
finally {
174173
$sw.Stop()
175174
Reset-Colors
175+
if ($env:TF_BUILD) {
176+
Write-Output "##[endgroup]"
177+
}
176178
if ($completed) {
177179
Trace-Log "[DONE +$(Format-ElapsedTime $sw.Elapsed)] $BuildStep"
178180
}
@@ -184,14 +186,10 @@ Function Invoke-BuildStep {
184186
Error-Log "[FAILED +$(Format-ElapsedTime $sw.Elapsed)] $BuildStep"
185187
}
186188
}
187-
188-
if ($env:TF_BUILD) {
189-
Write-Output "##[endgroup]"
190-
}
191189
}
192190
}
193191
else {
194-
Warning-Log "[SKIP] $BuildStep"
192+
Trace-Log "[SKIP] $BuildStep"
195193
}
196194
}
197195

‎src/Catalog/Downloads/DownloadsV1JsonClient.cs‎

Lines changed: 20 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
1-
// Copyright (c) .NET Foundation. All rights reserved.
1+
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

44
using System;
55
using System.Diagnostics;
66
using System.IO;
7-
using System.Net.Http;
87
using System.Threading.Tasks;
8+
using Azure;
9+
using Azure.Storage.Blobs;
10+
using Azure.Storage.Blobs.Models;
911
using Microsoft.Extensions.Logging;
1012
using Newtonsoft.Json;
1113
using NuGet.Services.Metadata.Catalog.Helpers;
@@ -14,46 +16,43 @@ namespace NuGet.Services.Metadata.Catalog
1416
{
1517
public class DownloadsV1JsonClient : IDownloadsV1JsonClient
1618
{
17-
private readonly HttpClient _httpClient;
19+
private readonly BlobClient _blobClient;
1820
private readonly ILogger<DownloadsV1JsonClient> _logger;
1921

20-
public DownloadsV1JsonClient(HttpClient httpClient, ILogger<DownloadsV1JsonClient> logger)
22+
public DownloadsV1JsonClient(BlobClient blobClient, ILogger<DownloadsV1JsonClient> logger)
2123
{
22-
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
24+
_blobClient = blobClient ?? throw new ArgumentNullException(nameof(blobClient));
2325
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
2426
}
2527

26-
public async Task<DownloadData> ReadAsync(string url)
28+
public async Task<DownloadData> ReadAsync()
2729
{
2830
var downloadData = new DownloadData();
29-
await ReadAsync(url, downloadData.SetDownloadCount);
31+
await ReadAsync(downloadData.SetDownloadCount);
3032
return downloadData;
3133
}
3234

33-
public async Task ReadAsync(string url, AddDownloadCount addCount)
35+
public async Task ReadAsync(AddDownloadCount addCount)
3436
{
3537
var stopwatch = Stopwatch.StartNew();
3638
var packageCount = 0;
37-
39+
3840
await Retry.IncrementalAsync(
3941
async () =>
4042
{
41-
_logger.LogInformation("Attempting to download {Url}", url);
42-
using (var response = await _httpClient.GetAsync(url))
43+
_logger.LogInformation("Attempting to download {Url}", _blobClient.Uri.GetLeftPart(UriPartial.Path));
44+
using (BlobDownloadStreamingResult result = await _blobClient.DownloadStreamingAsync())
45+
using (var textReader = new StreamReader(result.Content))
46+
using (var jsonReader = new JsonTextReader(textReader))
4347
{
44-
response.EnsureSuccessStatusCode();
45-
using (var textReader = new StreamReader(await response.Content.ReadAsStreamAsync()))
46-
using (var jsonReader = new JsonTextReader(textReader))
48+
DownloadsV1Reader.Load(jsonReader, (id, version, count) =>
4749
{
48-
DownloadsV1Reader.Load(jsonReader, (id, version, count) =>
49-
{
50-
packageCount++;
51-
addCount(id, version, count);
52-
});
53-
}
50+
packageCount++;
51+
addCount(id, version, count);
52+
});
5453
}
5554
},
56-
ex => ex is HttpRequestException,
55+
ex => ex is RequestFailedException,
5756
maxRetries: 5,
5857
initialWaitInterval: TimeSpan.Zero,
5958
waitIncrement: TimeSpan.FromSeconds(20));

‎src/Catalog/Downloads/IDownloadsV1JsonClient.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) .NET Foundation. All rights reserved.
1+
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

44
using System.Threading.Tasks;
@@ -9,7 +9,7 @@ namespace NuGet.Services.Metadata.Catalog
99

1010
public interface IDownloadsV1JsonClient
1111
{
12-
Task<DownloadData> ReadAsync(string url);
13-
Task ReadAsync(string url, AddDownloadCount addCount);
12+
Task<DownloadData> ReadAsync();
13+
Task ReadAsync(AddDownloadCount addCount);
1414
}
15-
}
15+
}

‎src/Catalog/NuGet.Services.Metadata.Catalog.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@
7373

7474
<ItemGroup>
7575
<ProjectReference Include="..\NuGet.Protocol.Catalog\NuGet.Protocol.Catalog.csproj" />
76+
<ProjectReference Include="..\NuGet.Services.Configuration\NuGet.Services.Configuration.csproj" />
7677
<ProjectReference Include="..\NuGet.Services.Logging\NuGet.Services.Logging.csproj" />
7778
<ProjectReference Include="..\NuGet.Services.Sql\NuGet.Services.Sql.csproj" />
7879
<ProjectReference Include="..\NuGetGallery.Core\NuGetGallery.Core.csproj" />

‎src/Catalog/ServiceCollectionExtensions.cs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using Azure.Storage.Blobs;
6+
using Microsoft.Extensions.Configuration;
7+
using Microsoft.Extensions.Logging;
8+
using NuGet.Services.Configuration;
9+
using NuGet.Services.Metadata.Catalog;
10+
11+
namespace Microsoft.Extensions.DependencyInjection
12+
{
13+
public static class ServiceCollectionExtensions
14+
{
15+
public static IServiceCollection AddDownloadsV1JsonClient(this IServiceCollection services, Func<IServiceProvider, string> urlFactory)
16+
{
17+
services.AddSingleton<IDownloadsV1JsonClient>(provider =>
18+
{
19+
var url = urlFactory(provider);
20+
21+
var configuration = provider.GetRequiredService<IConfiguration>();
22+
var blobClient = new BlobClient(new Uri(url), configuration.GetTokenCredential());
23+
24+
var logger = provider.GetRequiredService<ILogger<DownloadsV1JsonClient>>();
25+
26+
return new DownloadsV1JsonClient(blobClient, logger);
27+
});
28+
return services;
29+
}
30+
}
31+
}

‎src/NuGet.Jobs.Auxiliary2AzureSearch/Job.cs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
1-
// Copyright (c) .NET Foundation. All rights reserved.
1+
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

44
using Microsoft.Extensions.Configuration;
55
using Microsoft.Extensions.DependencyInjection;
6+
using Microsoft.Extensions.Options;
67
using NuGet.Services.AzureSearch;
78
using NuGet.Services.AzureSearch.Auxiliary2AzureSearch;
89

@@ -19,6 +20,11 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
1920
services.Configure<Auxiliary2AzureSearchConfiguration>(configurationRoot.GetSection(ConfigurationSectionName));
2021
services.Configure<AzureSearchJobConfiguration>(configurationRoot.GetSection(ConfigurationSectionName));
2122
services.Configure<AzureSearchConfiguration>(configurationRoot.GetSection(ConfigurationSectionName));
23+
services.AddDownloadsV1JsonClient(provider =>
24+
{
25+
var jsonConfigurationAccessor = provider.GetRequiredService<IOptionsSnapshot<Auxiliary2AzureSearchConfiguration>>();
26+
return jsonConfigurationAccessor.Value.DownloadsV1JsonUrl;
27+
});
2228
}
2329
}
2430
}

‎src/NuGet.Jobs.Db2AzureSearch/Job.cs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
1-
// Copyright (c) .NET Foundation. All rights reserved.
1+
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

44
using Microsoft.Extensions.Configuration;
55
using Microsoft.Extensions.DependencyInjection;
6+
using Microsoft.Extensions.Options;
67
using NuGet.Services.AzureSearch;
78
using NuGet.Services.AzureSearch.AuxiliaryFiles;
89
using NuGet.Services.AzureSearch.Db2AzureSearch;
@@ -26,6 +27,11 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
2627
configurationRoot.GetSection(DevelopmentConfigurationSectionName));
2728
services.Configure<Db2AzureSearchDevelopmentConfiguration>(
2829
configurationRoot.GetSection(DevelopmentConfigurationSectionName));
30+
services.AddDownloadsV1JsonClient(provider =>
31+
{
32+
var jsonConfigurationAccessor = provider.GetRequiredService<IOptionsSnapshot<Db2AzureSearchConfiguration>>();
33+
return jsonConfigurationAccessor.Value.DownloadsV1JsonUrl;
34+
});
2935
}
3036
}
3137
}

‎src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/UpdateDownloadsCommand.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) .NET Foundation. All rights reserved.
1+
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

44
using System;
@@ -117,7 +117,7 @@ private async Task<bool> PushIndexChangesAsync()
117117

118118
// The "new" data in this case is from the statistics pipeline.
119119
_logger.LogInformation("Fetching new download count data by URL.");
120-
var newData = await _downloadsV1JsonClient.ReadAsync(_options.Value.DownloadsV1JsonUrl);
120+
var newData = await _downloadsV1JsonClient.ReadAsync();
121121

122122
_logger.LogInformation("Removing invalid IDs and versions from the old downloads data.");
123123
CleanDownloadData(oldResult.Data);
@@ -436,4 +436,4 @@ private void CleanDownloadData(DownloadData data)
436436
nonNormalizedVersionCount);
437437
}
438438
}
439-
}
439+
}

‎src/NuGet.Services.AzureSearch/Db2AzureSearch/NewPackageRegistrationFromDbProducer.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) .NET Foundation. All rights reserved.
1+
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

44
using System;
@@ -94,7 +94,7 @@ public async Task<InitialAuxiliaryData> ProduceWorkAsync(
9494
// Fetch the download data from the auxiliary file, since this is what is used for displaying download
9595
// counts in the search service. We don't use the gallery DB values as they are different from the
9696
// auxiliary file.
97-
var downloads = await _downloadsV1JsonClient.ReadAsync(_options.Value.DownloadsV1JsonUrl);
97+
var downloads = await _downloadsV1JsonClient.ReadAsync();
9898
var popularityTransfers = await GetPopularityTransfersAsync();
9999

100100
// Apply changes from popularity transfers.
@@ -442,4 +442,4 @@ public PackageRegistrationInfo(int key, string id, string[] owners, bool isVerif
442442
public bool IsVerified { get; }
443443
}
444444
}
445-
}
445+
}

‎src/NuGet.Services.AzureSearch/DependencyInjectionExtensions.cs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) .NET Foundation. All rights reserved.
1+
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

44
using System;
@@ -26,7 +26,6 @@
2626
using NuGet.Services.AzureSearch.Db2AzureSearch;
2727
using NuGet.Services.AzureSearch.SearchService;
2828
using NuGet.Services.AzureSearch.Wrappers;
29-
using NuGet.Services.Metadata.Catalog;
3029
using NuGet.Services.Metadata.Catalog.Persistence;
3130
using NuGet.Services.V3;
3231
using NuGetGallery;
@@ -316,7 +315,6 @@ public static IServiceCollection AddAzureSearch(
316315
}
317316
});
318317

319-
services.AddTransient<IDownloadsV1JsonClient, DownloadsV1JsonClient>();
320318
services.AddSingleton<IAuxiliaryDataCache, AuxiliaryDataCache>();
321319
services.AddScoped(p => p.GetRequiredService<IAuxiliaryDataCache>().Get());
322320
services.AddSingleton<IAuxiliaryFileReloader, AuxiliaryFileReloader>();

0 commit comments

Comments
 (0)