Skip to content
This repository was archived by the owner on Aug 3, 2024. It is now read-only.

Commit 6addfb0

Browse files
agragr
authored
Blob copy routines (#383)
* Storage copy routines. * Ability to request metadata from Azure storage when listing blobs. * Method for setting blob metadata. * Save blob logs are under verbose flag now.
1 parent 6e7b563 commit 6addfb0

11 files changed

Lines changed: 255 additions & 28 deletions

File tree

‎src/NuGet.Services.Storage/AggregateStorage.cs‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,17 @@ public AggregateStorage(Uri baseAddress, Storage primaryStorage, Storage[] secon
3333

3434
BaseAddress = _primaryStorage.BaseAddress;
3535
}
36-
36+
37+
protected override Task OnCopyAsync(
38+
Uri sourceUri,
39+
IStorage destinationStorage,
40+
Uri destinationUri,
41+
IReadOnlyDictionary<string, string> destinationProperties,
42+
CancellationToken cancellationToken)
43+
{
44+
throw new NotImplementedException();
45+
}
46+
3747
protected override Task OnSave(Uri resourceUri, StorageContent content, bool overwrite, CancellationToken cancellationToken)
3848
{
3949
var tasks = new List<Task>();
@@ -91,9 +101,14 @@ public override Task<bool> ExistsAsync(string fileName, CancellationToken cancel
91101
return _primaryStorage.ExistsAsync(fileName, cancellationToken);
92102
}
93103

94-
public override Task<IEnumerable<StorageListItem>> List(CancellationToken cancellationToken)
104+
public override Task<IEnumerable<StorageListItem>> List(bool getMetadata, CancellationToken cancellationToken)
105+
{
106+
return _primaryStorage.List(getMetadata, cancellationToken);
107+
}
108+
109+
public override Task SetMetadataAsync(Uri resourceUri, IDictionary<string, string> metadata)
95110
{
96-
return _primaryStorage.List(cancellationToken);
111+
throw new NotImplementedException();
97112
}
98113
}
99114
}

‎src/NuGet.Services.Storage/AzureStorage.cs‎

Lines changed: 121 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,26 +11,46 @@
1111
using Microsoft.Extensions.Logging;
1212
using Microsoft.WindowsAzure.Storage;
1313
using Microsoft.WindowsAzure.Storage.Blob;
14+
using Microsoft.WindowsAzure.Storage.DataMovement;
1415

1516
namespace NuGet.Services.Storage
1617
{
1718
public class AzureStorage : Storage
1819
{
1920
private readonly ILogger<AzureStorage> _logger;
2021
private readonly CloudBlobDirectory _directory;
21-
22-
public AzureStorage(CloudStorageAccount account, string containerName, string path, Uri baseAddress, ILogger<AzureStorage> logger)
23-
: this(account.CreateCloudBlobClient().GetContainerReference(containerName).GetDirectoryReference(path), baseAddress, logger)
22+
private readonly bool _useServerSideCopy;
23+
24+
public AzureStorage(
25+
CloudStorageAccount account,
26+
string containerName,
27+
string path,
28+
Uri baseAddress,
29+
bool useServerSideCopy,
30+
bool initializeContainer,
31+
ILogger<AzureStorage> logger)
32+
: this(
33+
account.CreateCloudBlobClient().GetContainerReference(containerName).GetDirectoryReference(path),
34+
baseAddress,
35+
useServerSideCopy,
36+
initializeContainer,
37+
logger)
2438
{
2539
}
2640

27-
private AzureStorage(CloudBlobDirectory directory, Uri baseAddress, ILogger<AzureStorage> logger)
41+
private AzureStorage(
42+
CloudBlobDirectory directory,
43+
Uri baseAddress,
44+
bool useServerSideCopy,
45+
bool initializeContainer,
46+
ILogger<AzureStorage> logger)
2847
: base(baseAddress ?? GetDirectoryUri(directory), logger)
2948
{
3049
_logger = logger;
3150
_directory = directory;
51+
_useServerSideCopy = useServerSideCopy;
3252

33-
if (_directory.Container.CreateIfNotExists())
53+
if (initializeContainer && _directory.Container.CreateIfNotExists())
3454
{
3555
_directory.Container.SetPermissions(new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Blob });
3656

@@ -97,18 +117,88 @@ public override async Task<bool> ExistsAsync(string fileName, CancellationToken
97117
return false;
98118
}
99119

100-
public override async Task<IEnumerable<StorageListItem>> List(CancellationToken cancellationToken)
120+
public override async Task<IEnumerable<StorageListItem>> List(bool getMetadata, CancellationToken cancellationToken)
101121
{
102-
var files = await _directory.ListBlobsAsync(cancellationToken);
122+
var files = await _directory.ListBlobsAsync(getMetadata, cancellationToken);
103123

104124
return files.Select(GetStorageListItem).AsEnumerable();
105125
}
106126

127+
public override async Task SetMetadataAsync(Uri resourceUri, IDictionary<string, string> metadata)
128+
{
129+
var blob = GetBlockBlobReference(GetName(resourceUri));
130+
131+
foreach (var kvp in metadata)
132+
{
133+
blob.Metadata[kvp.Key] = kvp.Value;
134+
}
135+
136+
await blob.SetMetadataAsync();
137+
}
138+
107139
private StorageListItem GetStorageListItem(IListBlobItem listBlobItem)
108140
{
109-
var lastModified = (listBlobItem as CloudBlockBlob)?.Properties.LastModified?.UtcDateTime;
141+
var cloudBlockBlob = (listBlobItem as CloudBlockBlob);
142+
var lastModified = cloudBlockBlob?.Properties.LastModified?.UtcDateTime;
143+
144+
return new StorageListItem(listBlobItem.Uri, lastModified, cloudBlockBlob?.Metadata);
145+
}
146+
147+
protected override async Task OnCopyAsync(
148+
Uri sourceUri,
149+
IStorage destinationStorage,
150+
Uri destinationUri,
151+
IReadOnlyDictionary<string, string> destinationProperties,
152+
CancellationToken cancellationToken)
153+
{
154+
var azureDestinationStorage = destinationStorage as AzureStorage;
155+
156+
if (azureDestinationStorage == null)
157+
{
158+
throw new NotImplementedException("Copying is only supported from Azure storage to Azure storage.");
159+
}
160+
161+
string sourceName = GetName(sourceUri);
162+
string destinationName = azureDestinationStorage.GetName(destinationUri);
163+
164+
CloudBlockBlob sourceBlob = GetBlockBlobReference(sourceName);
165+
CloudBlockBlob destinationBlob = azureDestinationStorage.GetBlockBlobReference(destinationName);
166+
167+
var context = new SingleTransferContext();
168+
169+
if (destinationProperties?.Count > 0)
170+
{
171+
context.SetAttributesCallbackAsync = new SetAttributesCallbackAsync((destination) =>
172+
{
173+
var blob = (CloudBlockBlob)destination;
174+
175+
// The copy statement copied all properties from the source blob to the destination blob; however,
176+
// there may be required properties on destination blob, all of which may have not already existed
177+
// on the source blob at the time of copy.
178+
foreach (var property in destinationProperties)
179+
{
180+
switch (property.Key)
181+
{
182+
case StorageConstants.CacheControl:
183+
blob.Properties.CacheControl = property.Value;
184+
break;
185+
186+
case StorageConstants.ContentType:
187+
blob.Properties.ContentType = property.Value;
188+
break;
189+
190+
default:
191+
throw new NotImplementedException($"Storage property '{property.Value}' is not supported.");
192+
}
193+
}
194+
195+
return Task.CompletedTask;
196+
});
197+
}
198+
199+
context.ShouldOverwriteCallbackAsync = new ShouldOverwriteCallbackAsync((source, destination) => Task.FromResult(true));
110200

111-
return new StorageListItem(listBlobItem.Uri, lastModified);
201+
await TransferManager.CopyAsync(sourceBlob, destinationBlob, _useServerSideCopy, options: null, context: context);
112202
}
113203

114204
// save
@@ -143,7 +233,10 @@ await blob.UploadFromStreamAsync(
143233
operationContext: null,
144234
cancellationToken: cancellationToken);
145235

146-
_logger.LogInformation("Saved compressed blob {BlobUri} to container {ContainerName}", blob.Uri.ToString(), _directory.Container.Name);
236+
if (Verbose)
237+
{
238+
_logger.LogInformation("Saved compressed blob {BlobUri} to container {ContainerName}", blob.Uri.ToString(), _directory.Container.Name);
239+
}
147240
}
148241
}
149242
else
@@ -157,7 +250,10 @@ await blob.UploadFromStreamAsync(
157250
operationContext: null,
158251
cancellationToken: cancellationToken);
159252

160-
_logger.LogInformation("Saved uncompressed blob {BlobUri} to container {ContainerName}", blob.Uri.ToString(), _directory.Container.Name);
253+
if (Verbose)
254+
{
255+
_logger.LogInformation("Saved uncompressed blob {BlobUri} to container {ContainerName}", blob.Uri.ToString(), _directory.Container.Name);
256+
}
161257
}
162258
}
163259
}
@@ -217,5 +313,19 @@ protected override async Task OnDelete(Uri resourceUri, CancellationToken cancel
217313

218314
await blob.DeleteAsync(cancellationToken);
219315
}
316+
317+
private CloudBlockBlob GetBlockBlobReference(string blobName)
318+
{
319+
var blob = _directory.GetBlockBlobReference(blobName);
320+
321+
ApplyBlobRequestOptions(blob);
322+
323+
return blob;
324+
}
325+
326+
private void ApplyBlobRequestOptions(CloudBlockBlob blob)
327+
{
328+
blob.ServiceClient.DefaultRequestOptions = _directory.ServiceClient.DefaultRequestOptions;
329+
}
220330
}
221331
}

‎src/NuGet.Services.Storage/AzureStorageFactory.cs‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,25 @@ public class AzureStorageFactory : StorageFactory
1414
string _path;
1515
private Uri _differentBaseAddress = null;
1616
private readonly ILogger<AzureStorage> _azureStorageLogger;
17+
private readonly bool _useServerSideCopy;
18+
private readonly bool _initializeContainer;
1719

18-
public AzureStorageFactory(CloudStorageAccount account, string containerName, ILogger<AzureStorage> azureStorageLogger, string path = null, Uri baseAddress = null)
20+
public AzureStorageFactory(
21+
CloudStorageAccount account,
22+
string containerName,
23+
ILogger<AzureStorage> azureStorageLogger,
24+
string path = null,
25+
Uri baseAddress = null,
26+
bool useServerSideCopy = true,
27+
bool initializeContainer = true
28+
)
1929
{
2030
_account = account;
2131
_containerName = containerName;
2232
_path = null;
2333
_azureStorageLogger = azureStorageLogger;
34+
_useServerSideCopy = useServerSideCopy;
35+
_initializeContainer = initializeContainer;
2436

2537
if (path != null)
2638
{
@@ -71,7 +83,7 @@ public override Storage Create(string name = null)
7183
newBase = new Uri(_differentBaseAddress, name + "/");
7284
}
7385

74-
return new AzureStorage(_account, _containerName, path, newBase, _azureStorageLogger) { Verbose = Verbose, CompressContent = CompressContent };
86+
return new AzureStorage(_account, _containerName, path, newBase, _useServerSideCopy, _initializeContainer, _azureStorageLogger) { Verbose = Verbose, CompressContent = CompressContent };
7587
}
7688
}
7789
}

‎src/NuGet.Services.Storage/CloudBlobStorageExtensions.cs‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,19 +11,21 @@ namespace NuGet.Services.Storage
1111
internal static class CloudBlobStorageExtensions
1212
{
1313
public static async Task<IEnumerable<IListBlobItem>> ListBlobsAsync(
14-
this CloudBlobDirectory directory, CancellationToken cancellationToken)
14+
this CloudBlobDirectory directory,
15+
bool getMetadata,
16+
CancellationToken cancellationToken)
1517
{
1618
var items = new List<IListBlobItem>();
1719
BlobContinuationToken continuationToken = null;
1820
do
1921
{
2022
var segment = await directory.ListBlobsSegmentedAsync(
21-
useFlatBlobListing: true,
22-
blobListingDetails: BlobListingDetails.None,
23-
maxResults: null,
23+
useFlatBlobListing: true,
24+
blobListingDetails: getMetadata ? BlobListingDetails.Metadata : BlobListingDetails.None,
25+
maxResults: null,
2426
currentToken: continuationToken,
25-
options: null,
26-
operationContext: null,
27+
options: null,
28+
operationContext: null,
2729
cancellationToken: cancellationToken);
2830

2931
continuationToken = segment.ContinuationToken;

‎src/NuGet.Services.Storage/FileStorage.cs‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public override Task<bool> ExistsAsync(string fileName, CancellationToken cancel
4141
return Task.FromResult(Exists(fileName));
4242
}
4343

44-
public override Task<IEnumerable<StorageListItem>> List(CancellationToken cancellationToken)
44+
public override Task<IEnumerable<StorageListItem>> List(bool getMetadata, CancellationToken cancellationToken)
4545
{
4646
DirectoryInfo directoryInfo = new DirectoryInfo(Path);
4747
var files = directoryInfo.GetFiles("*", SearchOption.AllDirectories)
@@ -57,8 +57,17 @@ public string Path
5757
set;
5858
}
5959

60-
// save
60+
protected override Task OnCopyAsync(
61+
Uri sourceUri,
62+
IStorage destinationStorage,
63+
Uri destinationUri,
64+
IReadOnlyDictionary<string, string> destinationProperties,
65+
CancellationToken cancellationToken)
66+
{
67+
throw new NotImplementedException();
68+
}
6169

70+
// save
6271
protected override async Task OnSave(Uri resourceUri, StorageContent content, bool overwrite, CancellationToken cancellationToken)
6372
{
6473
SaveCount++;
@@ -163,5 +172,10 @@ protected override async Task OnDelete(Uri resourceUri, CancellationToken cancel
163172
await Task.Run(() => { fileInfo.Delete(); },cancellationToken);
164173
}
165174
}
175+
176+
public override Task SetMetadataAsync(Uri resourceUri, IDictionary<string, string> metadata)
177+
{
178+
throw new NotImplementedException();
179+
}
166180
}
167181
}

‎src/NuGet.Services.Storage/IStorage.cs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,13 @@ public interface IStorage
1818
Task<string> LoadString(Uri resourceUri, CancellationToken cancellationToken);
1919
Uri BaseAddress { get; }
2020
Uri ResolveUri(string relativeUri);
21-
Task<IEnumerable<StorageListItem>> List(CancellationToken cancellationToken);
21+
Task<IEnumerable<StorageListItem>> List(bool getMetadata, CancellationToken cancellationToken);
22+
Task CopyAsync(
23+
Uri sourceUri,
24+
IStorage destinationStorage,
25+
Uri destinationUri,
26+
IReadOnlyDictionary<string, string> destinationProperties,
27+
CancellationToken cancellation);
28+
Task SetMetadataAsync(Uri resourceUri, IDictionary<string, string> metadata);
2229
}
2330
}

‎src/NuGet.Services.Storage/NuGet.Services.Storage.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
</PropertyGroup>
77

88
<ItemGroup>
9+
<PackageReference Include="Microsoft.Azure.Storage.DataMovement" Version="0.9.0" />
910
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions">
1011
<Version>2.2.0</Version>
1112
</PackageReference>

0 commit comments

Comments
 (0)