Migrate Validation orchestrator jobs to new SDK and Managed Identities (#10161)

Author: advay26

src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj

@@ -29,6 +29,7 @@
     <ProjectReference Include="..\NuGet.Services.FeatureFlags\NuGet.Services.FeatureFlags.csproj" />
     <ProjectReference Include="..\NuGet.Services.Logging\NuGet.Services.Logging.csproj" />
     <ProjectReference Include="..\NuGet.Services.Sql\NuGet.Services.Sql.csproj" />
+    <ProjectReference Include="..\NuGet.Services.Storage\NuGet.Services.Storage.csproj" />
     <ProjectReference Include="..\NuGetGallery.Core\NuGetGallery.Core.csproj" />
   </ItemGroup>
 

src/NuGet.Jobs.Common/StorageAccountExtensions.cs

@@ -6,10 +6,12 @@
 using Autofac.Builder;
 using Azure.Data.Tables;
 using Azure.Identity;
+using Azure.Storage.Blobs;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
 using NuGet.Services.Configuration;
+using NuGet.Services.Storage;
 using NuGetGallery;
 
 namespace NuGet.Jobs
@@ -180,6 +182,48 @@ private static CloudBlobClientWrapper CreateCloudBlobClient(
                 requestTimeout);
         }
 
+        public static BlobServiceClient CreateBlobServiceClient(
+            StorageMsiConfiguration storageMsiConfiguration,
+            string storageConnectionString,
+            TimeSpan? requestTimeout = null)
+        {
+            BlobClientOptions blobClientOptions = new BlobClientOptions();
+            if (requestTimeout.HasValue)
+            {
+                blobClientOptions.Retry.NetworkTimeout = requestTimeout.Value;
+            }
+
+            if (storageMsiConfiguration.UseManagedIdentity)
+            {
+                Uri blobEndpointUri = AzureStorage.GetPrimaryServiceUri(storageConnectionString);
+
+                if (string.IsNullOrWhiteSpace(storageMsiConfiguration.ManagedIdentityClientId))
+                {
+                    // 1. Using MSI with DefaultAzureCredential (local debugging)
+                    return new BlobServiceClient(
+                        blobEndpointUri,
+                        new DefaultAzureCredential(),
+                        blobClientOptions);
+                }
+                else
+                {
+                    // 2. Using MSI with ClientId
+                    return new BlobServiceClient(
+                        blobEndpointUri,
+                        new ManagedIdentityCredential(storageMsiConfiguration.ManagedIdentityClientId),
+                        blobClientOptions);
+                }
+            }
+            else
+            {
+                // 3. Using SAS token
+                // workaround for https://github.com/Azure/azure-sdk-for-net/issues/44373
+                var connectionString = storageConnectionString.Replace("SharedAccessSignature=?", "SharedAccessSignature=");
+
+                return new BlobServiceClient(connectionString, blobClientOptions);
+            }
+        }
+
         private static TableServiceClient CreateTableServiceClientClient(
             StorageMsiConfiguration msiConfiguration,
             string tableStorageConnectionString)

src/NuGet.Services.V3/NuGet.Services.V3.csproj

@@ -7,6 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="WindowsAzure.Storage" />
     <ProjectReference Include="..\Catalog\NuGet.Services.Metadata.Catalog.csproj" />
     <ProjectReference Include="..\Validation.Common.Job\Validation.Common.Job.csproj" />
   </ItemGroup>

src/NuGet.Services.Validation.Orchestrator/Job.cs

@@ -386,11 +386,9 @@ private static void ConfigureLeaseService(ContainerBuilder builder)
                 .Register(c =>
                 {
                     LeaseConfiguration config = c.Resolve<IOptionsSnapshot<LeaseConfiguration>>().Value;
+                    StorageMsiConfiguration storageMsiConfiguration = c.Resolve<IOptionsSnapshot<StorageMsiConfiguration>>().Value;
 
-                    // workaround for https://github.com/Azure/azure-sdk-for-net/issues/44373
-                    var connectionString = config.ConnectionString.Replace("SharedAccessSignature=?", "SharedAccessSignature=");
-
-                    BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString);
+                    BlobServiceClient blobServiceClient = StorageAccountHelper.CreateBlobServiceClient(storageMsiConfiguration, config.ConnectionString);
                     return new CloudBlobLeaseService(blobServiceClient, config.ContainerName, config.StoragePath);
                 })
                 .As<ILeaseService>();

src/Validation.Common.Job/Leases/CloudBlobLeaseService.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -10,7 +10,6 @@
 using Azure.Storage.Blobs;
 using Azure.Storage.Blobs.Specialized;
 using Azure.Storage.Blobs.Models;
-using System.IO;
 
 namespace NuGet.Jobs.Validation.Leases
 {

src/Validation.Common.Job/Validation.Common.Job.csproj

@@ -28,7 +28,6 @@
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" />
     <PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" />
     <PackageReference Include="NuGet.Packaging" />
-    <PackageReference Include="WindowsAzure.Storage" />
     <PackageReference Include="System.Formats.Asn1" />
   </ItemGroup>
 

tests/Validation.Common.Job.Tests/Leases/BlobStorageFixture.cs

@@ -1,9 +1,8 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
-using Microsoft.WindowsAzure.Storage;
-using Microsoft.WindowsAzure.Storage.Blob;
+using Azure.Storage.Blobs;
 
 namespace Validation.Common.Job.Tests.Leases
 {
@@ -30,23 +29,21 @@ public BlobStorageFixture()
             TestRunId = Guid.NewGuid().ToString();
             ConnectionString = GetEnvironmentVariable(ConnectionStringName, required: true);
 
-            GetContainerReference().CreateIfNotExists();
+            GetBlobContainerClient().CreateIfNotExists();
         }
 
         public string TestRunId { get; }
         public string ConnectionString { get; }
 
         public void Dispose()
         {
-            GetContainerReference().DeleteIfExists();
+            GetBlobContainerClient().DeleteIfExists();
         }
 
-        private CloudBlobContainer GetContainerReference()
+        private BlobContainerClient GetBlobContainerClient()
         {
-            var account = CloudStorageAccount.Parse(ConnectionString);
-            var blobClient = account.CreateCloudBlobClient();
-            var container = blobClient.GetContainerReference(TestRunId);
-            return container;
+            var blobServiceClient = new BlobServiceClient(ConnectionString);
+            return blobServiceClient.GetBlobContainerClient(TestRunId);
         }
 
         private static string GetEnvironmentVariable(string name, bool required)