Skip to content

Commit 13375e3

Browse files
LanaparezaninLana Parezanin
Lanaparezanin
and
Lana Parezanin
authored
Migrating Stats.PostProcessReports to use MSI (#10356)
* Fixing gallery build error * Testfix * Bugfix * Added TopLevel * Has right .net sdk version now * Changed toplevel error * Changed toplevel error pt 2 * Changed toplevel error pt 3 * Bugfix * Bugfix2 * Added a comment * Clarified comment * AzureStatsLogDestination is now using new SDK * Temp changes * Changes * AzureStatsLogSource.cs has been updated * Migrated more files and tests * Final fixes? * Fixed failing tests * Fixed blobleasemanager problem * Fixing * Test fixes * Buildfix * Fixed cancellation problem * Latest changes * Polished BlobLeaseService.cs * Polished AzureBlobLockResult.cs * Polished AzureBlobLeaseManager.cs * Polished Job.cs * Fixed CDNLogsSanitizer scripts * Polished tests * First change * Changes new * Fixed tests * Initial changes * FIxed path * Nit * Nit * Nit * Removed unneccessary changes * Added extra authentication * Fixed some URI issues * Formatting * Testfix? * Fixed test --------- Co-authored-by: Lana Parezanin <[email protected]>
1 parent 3ebf56c commit 13375e3

3 files changed

Lines changed: 40 additions & 12 deletions

File tree

‎src/Stats.PostProcessReports/DetailedReportPostProcessor.cs‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ private async Task ProcessBlobs(List<StorageListItem> jsonBlobs, CancellationTok
121121
foreach (var sourceBlob in jsonBlobs)
122122
{
123123
var blobName = GetBlobName(sourceBlob);
124-
var workBlobUri = _workStorage.ResolveUri(_configuration.WorkPath + blobName);
124+
var workBlobUri = _workStorage.ResolveUri(_configuration.WorkPath + '/' + blobName);
125125
var sourceBlobStats = new BlobStatistics();
126126
var individualReports = await ProcessSourceBlobAsync(sourceBlob, sourceBlobStats, totals);
127127
using (_logger.BeginScope("Processing {BlobName}", blobName))
@@ -155,7 +155,7 @@ private async Task ProcessBlobs(List<StorageListItem> jsonBlobs, CancellationTok
155155
}
156156
}
157157
}
158-
var jobSucceededUrl = _workStorage.ResolveUri(_configuration.WorkPath + JobSucceededFilename);
158+
var jobSucceededUrl = _workStorage.ResolveUri(_configuration.WorkPath + '/' + JobSucceededFilename);
159159
var jobSucceededContent = new StringStorageContent("", TextContentType);
160160
await _workStorage.Save(jobSucceededUrl, jobSucceededContent, overwrite: true, cancellationToken: cancellationToken);
161161
_telemetryService.ReportTotals(totals.SourceFilesProcessed, totals.TotalLinesProcessed, totals.TotalFilesCreated, totals.TotalLinesFailed);
@@ -203,13 +203,13 @@ private async Task CopySourceBlobsAsync(List<StorageListItem> jsonBlobs, Cancell
203203
foreach (var sourceBlob in jsonBlobs)
204204
{
205205
var blobName = GetBlobName(sourceBlob);
206-
var targetUrl = _workStorage.ResolveUri(_configuration.WorkPath + blobName);
206+
var targetUrl = _workStorage.ResolveUri(_configuration.WorkPath + '/' + blobName);
207207
_logger.LogInformation("{SourceBlobUri} ({BlobName})", sourceBlob.Uri.AbsoluteUri, blobName);
208208
_logger.LogInformation("{WorkBlobUrl}", targetUrl);
209209
await _sourceStorage.CopyAsync(sourceBlob.Uri, _workStorage, targetUrl, destinationProperties: null, cancellationToken);
210210
}
211211
var copySucceededContent = new StringStorageContent("", TextContentType);
212-
var copySucceededUrl = _workStorage.ResolveUri(_configuration.WorkPath + CopySucceededFilename);
212+
var copySucceededUrl = _workStorage.ResolveUri(_configuration.WorkPath + '/' + CopySucceededFilename);
213213
await _workStorage.Save(copySucceededUrl, copySucceededContent, overwrite: true, cancellationToken: cancellationToken);
214214
}
215215

@@ -246,7 +246,7 @@ private async Task<ConcurrentBag<LineProcessingContext>> ProcessSourceBlobAsync(
246246
var sw = Stopwatch.StartNew();
247247
var numLines = 0;
248248
var individualReports = new ConcurrentBag<LineProcessingContext>();
249-
var workStorageUrl = _workStorage.ResolveUri(_configuration.WorkPath + GetBlobName(sourceBlob));
249+
var workStorageUrl = _workStorage.ResolveUri(_configuration.WorkPath + '/' + GetBlobName(sourceBlob));
250250
var storageContent = await _workStorage.Load(workStorageUrl, CancellationToken.None);
251251
using (var sourceStream = storageContent.GetContentStream())
252252
using (var streamReader = new StreamReader(sourceStream))
@@ -318,7 +318,6 @@ private static string GetBlobName(StorageListItem blob)
318318
{
319319
throw new ArgumentException($"Blob URI path does not contain '/': {blob.Uri.AbsolutePath}", nameof(blob));
320320
}
321-
322321
return path.Substring(lastSlash + 1);
323322
}
324323

@@ -352,7 +351,7 @@ private async Task WriteReports(
352351
continue;
353352
}
354353
var outFilename = $"recentpopularitydetail_{data.PackageId.ToLowerInvariant()}.json";
355-
var destinationUri = _destinationStorage.ResolveUri(_configuration.DestinationPath + outFilename);
354+
var destinationUri = _destinationStorage.ResolveUri(_configuration.DestinationPath + '/' + outFilename);
356355
var storageContent = new StringStorageContent(details.Data, JsonContentType);
357356

358357
await _destinationStorage.Save(destinationUri, storageContent, overwrite: true, cancellationToken: cancellationToken);

‎src/Stats.PostProcessReports/Job.cs‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

4+
using System;
45
using System.Threading.Tasks;
56
using Autofac;
67
using Autofac.Core;
8+
using Azure.Identity;
79
using Azure.Storage.Blobs;
810
using Microsoft.Extensions.Configuration;
911
using Microsoft.Extensions.DependencyInjection;
@@ -25,7 +27,7 @@ public override async Task Run()
2527
protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot)
2628
{
2729
services.Configure<PostProcessReportsConfiguration>(configurationRoot.GetSection("Configuration"));
28-
30+
services.ConfigureStorageMsi(configurationRoot);
2931
services.AddTransient<ITelemetryService, TelemetryService>();
3032
}
3133

@@ -38,8 +40,36 @@ protected override void ConfigureAutofacServices(ContainerBuilder containerBuild
3840
containerBuilder
3941
.Register(c =>
4042
{
41-
var cfg = c.Resolve<IOptionsSnapshot<PostProcessReportsConfiguration>>().Value;
42-
return new BlobServiceClientFactory(AzureStorageFactory.PrepareConnectionString(cfg.StorageAccount));
43+
PostProcessReportsConfiguration cfg = c.Resolve<IOptionsSnapshot<PostProcessReportsConfiguration>>().Value;
44+
StorageMsiConfiguration storageMsiConfiguration = _serviceProvider.GetRequiredService<IOptionsSnapshot<StorageMsiConfiguration>>().Value;
45+
try
46+
{
47+
if (storageMsiConfiguration.UseManagedIdentity)
48+
{
49+
string connectionString = cfg.StorageAccount.Replace("BlobEndPoint=", "");
50+
Uri blobEndpointUri = new Uri(connectionString);
51+
52+
if (string.IsNullOrWhiteSpace(storageMsiConfiguration.ManagedIdentityClientId))
53+
{
54+
// 1. Using MSI with DefaultAzureCredential (local debugging)
55+
return new BlobServiceClientFactory(blobEndpointUri, new DefaultAzureCredential());
56+
}
57+
else
58+
{
59+
// 2. Using MSI with ClientId
60+
return new BlobServiceClientFactory(blobEndpointUri, new ManagedIdentityCredential(storageMsiConfiguration.ManagedIdentityClientId));
61+
}
62+
}
63+
else
64+
{
65+
// 3. Using SAS token
66+
return new BlobServiceClientFactory(AzureStorageFactory.PrepareConnectionString(cfg.StorageAccount));
67+
}
68+
}
69+
catch (Exception ex)
70+
{
71+
throw new ArgumentException("Job parameter for Azure CDN Blob Service Client is invalid.", ex);
72+
}
4373
})
4474
.AsSelf();
4575

@@ -107,6 +137,5 @@ protected override void ConfigureAutofacServices(ContainerBuilder containerBuild
107137
(_, ctx) => ctx.ResolveKeyed<IStorage>(destinationKey)))
108138
.As<IDetailedReportPostProcessor>();
109139
}
110-
111140
}
112141
}

‎tests/Stats.PostProcessReports.Tests/DetailedReportPostProcessorFacts.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,7 @@ public async Task SplitsIncomingFiles()
217217
"file1.json"
218218
};
219219

220-
var fileUrl = Blob(_workStorageMock, "file1.json").Uri;
220+
var fileUrl = Blob(_workStorageMock, _configuration.WorkPath + '/' + "file1.json").Uri;
221221
const string line1 = "{\"PackageId\": \"Foo\", \"Otherstuff\": 123}";
222222
const string line2 = "{\"PackageId\": \"Bar\", \"Otherstuff\": 321}";
223223
const string content = line1 + "\n" + line2 + "\n";

0 commit comments

Comments
 (0)