Skip to content

Commit 72dfdf8

Browse files
drewgilliesdrewgillies
authored
Migrate ArchivePackages to using MSI for storage access (#10281)
1 parent ec52555 commit 72dfdf8

1 file changed

Lines changed: 68 additions & 84 deletions

File tree

‎src/ArchivePackages/ArchivePackages.Job.cs‎

Lines changed: 68 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -1,140 +1,130 @@
1-
// Copyright (c) .NET Foundation. All rights reserved.
1+
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

44
using System;
55
using System.Collections.Generic;
66
using System.ComponentModel.Design;
77
using System.Data.SqlClient;
8+
using System.IO;
89
using System.Linq;
10+
using System.Text;
911
using System.Threading.Tasks;
1012
using Autofac;
1113
using Dapper;
1214
using Microsoft.Extensions.Configuration;
1315
using Microsoft.Extensions.DependencyInjection;
1416
using Microsoft.Extensions.Options;
15-
using Microsoft.WindowsAzure.Storage;
16-
using Microsoft.WindowsAzure.Storage.Blob;
1717
using Newtonsoft.Json.Linq;
18+
using NuGetGallery;
1819
using NuGet.Jobs;
1920
using NuGet.Jobs.Configuration;
21+
using Autofac.Core;
2022

2123
namespace ArchivePackages
2224
{
2325
public class Job : JsonConfigurationJob
2426
{
25-
private readonly JobEventSource JobEventSourceLog = JobEventSource.Log;
2627
private const string ContentTypeJson = "application/json";
2728
private const string DateTimeFormatSpecifier = "O";
2829
private const string CursorDateTimeKey = "cursorDateTime";
2930
private const string DefaultPackagesContainerName = "packages";
3031
private const string DefaultPackagesArchiveContainerName = "ng-backups";
3132
private const string DefaultCursorBlobName = "cursor.json";
3233

33-
private InitializationConfiguration Configuration { get; set; }
34+
private readonly JobEventSource _jobEventSourceLog = JobEventSource.Log;
35+
private InitializationConfiguration _configuration;
3436

35-
/// <summary>
36-
/// Gets or sets an Azure Storage Uri referring to a container to use as the source for package blobs
37-
/// </summary>
38-
public CloudStorageAccount Source { get; set; }
39-
40-
public string SourceContainerName { get; set; }
41-
42-
/// <summary>
43-
/// Gets or sets an Azure Storage Uri referring to a container to use as the destination
44-
/// </summary>
45-
public CloudStorageAccount PrimaryDestination { get; set; }
37+
private string _sourceContainerName;
38+
private string _destinationContainerName;
4639

47-
/// <summary>
48-
/// Gets or sets an Azure Storage Uri referring to a container to use as the secondary destination
49-
/// DestinationContainerName should be same as the primary destination
50-
/// </summary>
51-
public CloudStorageAccount SecondaryDestination { get; set; }
40+
private string _sourceAccount;
41+
private string _primaryDestinationAccount;
42+
private string _secondaryDestinationAccount;
5243

53-
/// <summary>
54-
/// Destination Container name for both Primary and Secondary destinations. Also, for the cursor blob
55-
/// </summary>
56-
public string DestinationContainerName { get; set; }
44+
private ICloudBlobContainer _sourceContainer;
45+
private ICloudBlobContainer _primaryDestinationContainer;
46+
private ICloudBlobContainer _secondaryDestinationContainer;
5747

58-
/// <summary>
59-
/// Blob containing the cursor data. Cursor data comprises of cursorDateTime
60-
/// </summary>
61-
public string CursorBlobName { get; set; }
48+
private string _cursorBlobName;
6249

6350
/// <summary>
6451
/// Gallery database registration, for diagnostics.
6552
/// </summary>
66-
private SqlConnectionStringBuilder GalleryDatabase { get; set; }
53+
private SqlConnectionStringBuilder _galleryDatabase;
6754

68-
protected CloudBlobContainer SourceContainer { get; private set; }
69-
70-
protected CloudBlobContainer PrimaryDestinationContainer { get; private set; }
71-
72-
protected CloudBlobContainer SecondaryDestinationContainer { get; private set; }
7355

7456
public Job() : base(JobEventSource.Log) { }
7557

7658
public override void Init(IServiceContainer serviceContainer, IDictionary<string, string> jobArgsDictionary)
7759
{
7860
base.Init(serviceContainer, jobArgsDictionary);
7961

80-
Configuration = _serviceProvider.GetRequiredService<IOptionsSnapshot<InitializationConfiguration>>().Value;
62+
_configuration = _serviceProvider.GetRequiredService<IOptionsSnapshot<InitializationConfiguration>>().Value;
63+
64+
_galleryDatabase = GetDatabaseRegistration<GalleryDbConfiguration>();
8165

82-
GalleryDatabase = GetDatabaseRegistration<GalleryDbConfiguration>();
66+
_sourceContainerName = _configuration.SourceContainerName ?? DefaultPackagesContainerName;
67+
_sourceAccount = _configuration.Source;
8368

84-
Source = CloudStorageAccount.Parse(Configuration.Source);
69+
var sourceBlobClient = _serviceProvider.CreateCloudBlobClient(
70+
$"BlobEndPoint=https://{_sourceAccount}.blob.core.windows.net");
71+
_sourceContainer = sourceBlobClient.GetContainerReference(_sourceContainerName);
8572

86-
PrimaryDestination = CloudStorageAccount.Parse(Configuration.PrimaryDestination);
73+
_destinationContainerName = _configuration.DestinationContainerName ?? DefaultPackagesArchiveContainerName;
74+
_primaryDestinationAccount = _configuration.PrimaryDestination;
75+
var primaryDestinationBlobClient = _serviceProvider.CreateCloudBlobClient(
76+
$"BlobEndPoint=https://{_primaryDestinationAccount}.blob.core.windows.net");
77+
_primaryDestinationContainer = primaryDestinationBlobClient.GetContainerReference(_destinationContainerName);
8778

88-
if (!string.IsNullOrEmpty(Configuration.SecondaryDestination))
79+
if (!string.IsNullOrEmpty(_configuration.SecondaryDestination))
8980
{
90-
SecondaryDestination = CloudStorageAccount.Parse(Configuration.SecondaryDestination);
81+
_secondaryDestinationAccount = _configuration.SecondaryDestination;
82+
var secondaryDestinationBlobClient = _serviceProvider.CreateCloudBlobClient(
83+
$"BlobEndPoint=https://{_primaryDestinationAccount}.blob.core.windows.net");
84+
_secondaryDestinationContainer = secondaryDestinationBlobClient.GetContainerReference(_destinationContainerName);
9185
}
9286

93-
SourceContainerName = Configuration.SourceContainerName ?? DefaultPackagesContainerName;
94-
DestinationContainerName = Configuration.DestinationContainerName ?? DefaultPackagesArchiveContainerName;
95-
96-
SourceContainer = Source.CreateCloudBlobClient().GetContainerReference(SourceContainerName);
97-
PrimaryDestinationContainer = PrimaryDestination.CreateCloudBlobClient().GetContainerReference(DestinationContainerName);
98-
SecondaryDestinationContainer = SecondaryDestination?.CreateCloudBlobClient().GetContainerReference(DestinationContainerName);
99-
100-
CursorBlobName = Configuration.CursorBlob ?? DefaultCursorBlobName;
87+
_cursorBlobName = _configuration.CursorBlob ?? DefaultCursorBlobName;
10188
}
10289

10390
public override async Task Run()
10491
{
105-
JobEventSourceLog.PreparingToArchive(Source.Credentials.AccountName, SourceContainer.Name, PrimaryDestination.Credentials.AccountName, PrimaryDestinationContainer.Name, GalleryDatabase.DataSource, GalleryDatabase.InitialCatalog);
106-
await Archive(PrimaryDestinationContainer);
92+
_jobEventSourceLog.PreparingToArchive(_sourceAccount, _sourceContainerName, _primaryDestinationAccount, _destinationContainerName, _galleryDatabase.DataSource, _galleryDatabase.InitialCatalog);
93+
await Archive(_primaryDestinationContainer);
10794

10895
// todo: consider reusing package query for primary and secondary archives
109-
if (SecondaryDestinationContainer != null)
96+
if (_secondaryDestinationContainer != null)
11097
{
111-
JobEventSourceLog.PreparingToArchive2(SecondaryDestination.Credentials.AccountName, SecondaryDestinationContainer.Name);
112-
await Archive(SecondaryDestinationContainer);
98+
_jobEventSourceLog.PreparingToArchive2(_secondaryDestinationAccount, _destinationContainerName);
99+
await Archive(_secondaryDestinationContainer);
113100
}
114101
}
115102

116-
private static async Task<JObject> GetJObject(CloudBlobContainer container, string blobName)
103+
private static async Task<JObject> GetJObject(ICloudBlobContainer container, string blobName)
117104
{
118-
var blob = container.GetBlockBlobReference(blobName);
119-
var json = await blob.DownloadTextAsync();
105+
var blob = container.GetBlobReference(blobName);
106+
var json = await blob.DownloadTextIfExistsAsync();
120107
return JObject.Parse(json);
121108
}
122109

123-
private static async Task SetJObject(CloudBlobContainer container, string blobName, JObject jObject)
110+
private static async Task SetJObject(ICloudBlobContainer container, string blobName, JObject jObject)
124111
{
125-
var blob = container.GetBlockBlobReference(blobName);
112+
var blob = container.GetBlobReference(blobName);
126113
blob.Properties.ContentType = ContentTypeJson;
127-
await blob.UploadTextAsync(jObject.ToString());
114+
using (Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(jObject.ToString())))
115+
{
116+
await blob.UploadFromStreamAsync(stream, overwrite: true);
117+
}
128118
}
129119

130-
private async Task Archive(CloudBlobContainer destinationContainer)
120+
private async Task Archive(ICloudBlobContainer destinationContainer)
131121
{
132-
var cursorJObject = await GetJObject(destinationContainer, CursorBlobName);
122+
var cursorJObject = await GetJObject(destinationContainer, _cursorBlobName);
133123
var cursorDateTime = cursorJObject[CursorDateTimeKey].Value<DateTime>();
134124

135-
JobEventSourceLog.CursorData(cursorDateTime.ToString(DateTimeFormatSpecifier));
125+
_jobEventSourceLog.CursorData(cursorDateTime.ToString(DateTimeFormatSpecifier));
136126

137-
JobEventSourceLog.GatheringPackagesToArchiveFromDb(GalleryDatabase.DataSource, GalleryDatabase.InitialCatalog);
127+
_jobEventSourceLog.GatheringPackagesToArchiveFromDb(_galleryDatabase.DataSource, _galleryDatabase.InitialCatalog);
138128
List<PackageRef> packages;
139129
using (var connection = await OpenSqlConnectionAsync<GalleryDbConfiguration>())
140130
{
@@ -145,24 +135,21 @@ FROM Packages p
145135
WHERE Published > @cursorDateTime OR LastEdited > @cursorDateTime", new { cursorDateTime = cursorDateTime }))
146136
.ToList();
147137
}
148-
JobEventSourceLog.GatheredPackagesToArchiveFromDb(packages.Count, GalleryDatabase.DataSource, GalleryDatabase.InitialCatalog);
138+
_jobEventSourceLog.GatheredPackagesToArchiveFromDb(packages.Count, _galleryDatabase.DataSource, _galleryDatabase.InitialCatalog);
149139

150140
var archiveSet = packages
151141
.AsParallel()
152142
.Select(r => Tuple.Create(StorageHelpers.GetPackageBlobName(r.Id, r.Version), StorageHelpers.GetPackageBackupBlobName(r.Id, r.Version, r.Hash)))
153143
.ToList();
154144

155-
//if (!WhatIf)
156-
{
157-
await destinationContainer.CreateIfNotExistsAsync();
158-
}
145+
await destinationContainer.CreateIfNotExistAsync(enablePublicAccess: false);
159146

160147
if (archiveSet.Count > 0)
161148
{
162-
JobEventSourceLog.StartingArchive(archiveSet.Count);
149+
_jobEventSourceLog.StartingArchive(archiveSet.Count);
163150
foreach (var archiveItem in archiveSet)
164151
{
165-
await ArchivePackage(archiveItem.Item1, archiveItem.Item2, SourceContainer, destinationContainer);
152+
await ArchivePackage(archiveItem.Item1, archiveItem.Item2, _sourceContainer, destinationContainer);
166153
}
167154

168155
var maxLastEdited = packages.Max(p => p.LastEdited);
@@ -172,35 +159,32 @@ FROM Packages p
172159
var newCursorDateTime = maxLastEdited > maxPublished ? new DateTime(maxLastEdited.Value.Ticks, DateTimeKind.Utc) : new DateTime(maxPublished.Value.Ticks, DateTimeKind.Utc);
173160
var newCursorDateTimeString = newCursorDateTime.ToString(DateTimeFormatSpecifier);
174161

175-
JobEventSourceLog.NewCursorData(newCursorDateTimeString);
162+
_jobEventSourceLog.NewCursorData(newCursorDateTimeString);
176163
cursorJObject[CursorDateTimeKey] = newCursorDateTimeString;
177-
await SetJObject(destinationContainer, CursorBlobName, cursorJObject);
164+
await SetJObject(destinationContainer, _cursorBlobName, cursorJObject);
178165
}
179166
}
180167

181-
private async Task ArchivePackage(string sourceBlobName, string destinationBlobName, CloudBlobContainer sourceContainer, CloudBlobContainer destinationContainer)
168+
private async Task ArchivePackage(string sourceBlobName, string destinationBlobName, ICloudBlobContainer sourceContainer, ICloudBlobContainer destinationContainer)
182169
{
183170
// Identify the source and destination blobs
184-
var sourceBlob = sourceContainer.GetBlockBlobReference(sourceBlobName);
185-
var destBlob = destinationContainer.GetBlockBlobReference(destinationBlobName);
171+
var sourceBlob = sourceContainer.GetBlobReference(sourceBlobName);
172+
var destBlob = destinationContainer.GetBlobReference(destinationBlobName);
186173

187174
if (await destBlob.ExistsAsync())
188175
{
189-
JobEventSourceLog.ArchiveExists(destBlob.Name);
176+
_jobEventSourceLog.ArchiveExists(destBlob.Name);
190177
}
191178
else if (!await sourceBlob.ExistsAsync())
192179
{
193-
JobEventSourceLog.SourceBlobMissing(sourceBlob.Name);
180+
_jobEventSourceLog.SourceBlobMissing(sourceBlob.Name);
194181
}
195182
else
196183
{
197184
// Start the copy
198-
JobEventSourceLog.StartingCopy(sourceBlob.Name, destBlob.Name);
199-
//if (!WhatIf)
200-
{
201-
await destBlob.StartCopyAsync(sourceBlob);
202-
}
203-
JobEventSourceLog.StartedCopy(sourceBlob.Name, destBlob.Name);
185+
_jobEventSourceLog.StartingCopy(sourceBlob.Name, destBlob.Name);
186+
await destBlob.StartCopyAsync(sourceBlob, sourceAccessCondition: AccessConditionWrapper.GenerateEmptyCondition(), destAccessCondition: AccessConditionWrapper.GenerateEmptyCondition());
187+
_jobEventSourceLog.StartedCopy(sourceBlob.Name, destBlob.Name);
204188
}
205189
}
206190

0 commit comments

Comments
 (0)