Fix Db2Catalog and Catalog2Dnx with new SDK changes (#10110)

* Sanitize URLs for logging
* Fix container URI construction when directory is provided
* Remove flaky assertion

Author: joelverhagen

src/Catalog/Persistence/AzureStorage.cs

@@ -129,9 +129,9 @@ public AzureStorage(
 
             if (initializeContainer)
             {
-                BlobContainerInfo blobContainerInfo = _blobContainerClientWrapper.ContainerClient.CreateIfNotExists(PublicAccessType.Blob);
+                Response<BlobContainerInfo> blobContainerInfoResponse = _blobContainerClientWrapper.ContainerClient.CreateIfNotExists(PublicAccessType.Blob);
 
-                if (blobContainerInfo != null && Verbose)
+                if (blobContainerInfoResponse?.Value != null && Verbose)
                 {
                     Trace.WriteLine($"Created '{_blobContainerClientWrapper.ContainerClient.Name}' public container");
                 }
@@ -192,10 +192,10 @@ public override async Task<IEnumerable<StorageListItem>> ListAsync(CancellationT
             var blobs = new List<StorageListItem>();
 
             BlobContainerClient blobContainerClient = _blobContainerClientWrapper.ContainerClient;
-            await foreach (var blobItem in blobContainerClient.GetBlobsAsync(prefix: _directory.DirectoryPrefix, cancellationToken: cancellationToken))
+            await foreach (var blobItem in blobContainerClient.GetBlobsByHierarchyAsync(prefix: _directory.DirectoryPrefix, cancellationToken: cancellationToken))
             {
-                var lastModified = blobItem.Properties.LastModified?.UtcDateTime;
-                blobs.Add(new StorageListItem(new Uri(blobContainerClient.Uri, blobItem.Name), lastModified));
+                var lastModified = blobItem.Blob.Properties.LastModified?.UtcDateTime;
+                blobs.Add(new StorageListItem(blobContainerClient.GetBlobClient(blobItem.Blob.Name).Uri, lastModified));
             }
 
             return blobs;
@@ -314,7 +314,7 @@ protected override async Task OnSaveAsync(Uri resourceUri, StorageContent conten
 
                     await blockBlobClient.UploadAsync(destinationStream, headers, conditions: blobRequestConditions, cancellationToken: cancellationToken);
 
-                    Trace.WriteLine($"Saved compressed blob {blockBlobClient.Uri} to container {_blobContainerClientWrapper.ContainerClient.Name}");
+                    Trace.WriteLine($"Saved compressed blob {RemoveQueryString(blockBlobClient.Uri)} to container {_blobContainerClientWrapper.ContainerClient.Name}");
                 }
             }
             else
@@ -323,7 +323,7 @@ protected override async Task OnSaveAsync(Uri resourceUri, StorageContent conten
                 {
                     await blockBlobClient.UploadAsync(stream, headers, cancellationToken: cancellationToken);
 
-                    Trace.WriteLine($"Saved uncompressed blob {blockBlobClient.Uri} to container {_blobContainerClientWrapper.ContainerClient.Name}");
+                    Trace.WriteLine($"Saved uncompressed blob {RemoveQueryString(blockBlobClient.Uri)} to container {_blobContainerClientWrapper.ContainerClient.Name}");
                 }
             }
 
@@ -351,14 +351,14 @@ private async Task<bool> TryTakeBlobSnapshotAsync(BlockBlobClient blobBlockClien
                 {
                     var response = await blobBlockClient.CreateSnapshotAsync();
                     stopwatch.Stop();
-                    Trace.WriteLine($"SnapshotCreated:milliseconds={stopwatch.ElapsedMilliseconds}:{blobBlockClient.Uri.ToString()}:{response?.Value.Snapshot}");
+                    Trace.WriteLine($"SnapshotCreated:milliseconds={stopwatch.ElapsedMilliseconds}:{RemoveQueryString(blobBlockClient.Uri)}:{response?.Value.Snapshot}");
                 }
                 return true;
             }
             catch (RequestFailedException e)
             {
                 stopwatch.Stop();
-                Trace.WriteLine($"EXCEPTION:milliseconds={stopwatch.ElapsedMilliseconds}:CreateSnapshot: Failed to take the snapshot for blob {blobBlockClient.Uri.ToString()}. Exception{e.ToString()}");
+                Trace.WriteLine($"EXCEPTION:milliseconds={stopwatch.ElapsedMilliseconds}:CreateSnapshot: Failed to take the snapshot for blob {RemoveQueryString(blobBlockClient.Uri)}. Exception{e.ToString()}");
                 return false;
             }
         }
@@ -429,7 +429,7 @@ protected override async Task OnDeleteAsync(Uri resourceUri, DeleteRequestOption
 
         public override Uri GetUri(string name)
         {
-            var baseUri = _directory.Uri.AbsoluteUri;
+            var baseUri = RemoveQueryString(_directory.Uri);
 
             if (baseUri.EndsWith("/"))
             {
@@ -465,22 +465,22 @@ public async Task<bool> AreSynchronized(ICloudBlockBlob sourceBlockBlob, ICloudB
                     var destinationBlobHasSha512Hash = destinationBlobMetadata.TryGetValue(Sha512HashAlgorithmId, out var destinationBlobSha512Hash);
                     if (!sourceBlobHasSha512Hash)
                     {
-                        Trace.TraceWarning($"The source blob ({sourceBlockBlob.Uri}) doesn't have the SHA512 hash.");
+                        Trace.TraceWarning($"The source blob ({RemoveQueryString(sourceBlockBlob.Uri)}) doesn't have the SHA512 hash.");
                     }
                     if (!destinationBlobHasSha512Hash)
                     {
-                        Trace.TraceWarning($"The destination blob ({destinationBlockBlob.Uri}) doesn't have the SHA512 hash.");
+                        Trace.TraceWarning($"The destination blob ({RemoveQueryString(destinationBlockBlob.Uri)}) doesn't have the SHA512 hash.");
                     }
                     if (sourceBlobHasSha512Hash && destinationBlobHasSha512Hash)
                     {
                         if (sourceBlobSha512Hash == destinationBlobSha512Hash)
                         {
-                            Trace.WriteLine($"The source blob ({sourceBlockBlob.Uri}) and destination blob ({destinationBlockBlob.Uri}) have the same SHA512 hash and are synchronized.");
+                            Trace.WriteLine($"The source blob ({RemoveQueryString(sourceBlockBlob.Uri)}) and destination blob ({RemoveQueryString(destinationBlockBlob.Uri)}) have the same SHA512 hash and are synchronized.");
                             return true;
                         }
 
                         // The SHA512 hash between the source and destination blob should be always same.
-                        Trace.TraceWarning($"The source blob ({sourceBlockBlob.Uri}) and destination blob ({destinationBlockBlob.Uri}) have the different SHA512 hash and are not synchronized. " +
+                        Trace.TraceWarning($"The source blob ({RemoveQueryString(sourceBlockBlob.Uri)}) and destination blob ({RemoveQueryString(destinationBlockBlob.Uri)}) have the different SHA512 hash and are not synchronized. " +
                             $"The source blob hash is {sourceBlobSha512Hash} while the destination blob hash is {destinationBlobSha512Hash}");
                     }
 
@@ -524,7 +524,7 @@ public async Task<bool> HasPropertiesAsync(Uri blobUri, string contentType, stri
         private BlockBlobClient GetBlockBlobReference(string blobName)
         {
             IBlobContainerClientWrapper containerClient = _directory.ContainerClientWrapper;
-            BlockBlobClient blobClient = containerClient.GetBlockBlobClient(blobName);
+            BlockBlobClient blobClient = containerClient.GetBlockBlobClient(_directory.DirectoryPrefix + "/" + blobName);
 
             // ApplyBlobRequestOptions(blobClient) is not needed as the options should be set at the client level
             // when creating the BlobServiceClient or BlobContainerClient.

src/Catalog/Persistence/AzureStorageBlobs.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -19,12 +19,12 @@ public class CloudBlobDirectoryWrapper : ICloudBlobDirectory
         private readonly BlobClientOptions _defaultClientOptions;
 
         public BlobServiceClient ServiceClient => _containerClient.GetParentBlobServiceClient();
-        public Uri Uri => new Uri(_containerClient.Uri, _directoryPrefix);
+        public Uri Uri { get; }
         public string DirectoryPrefix => _directoryPrefix;
         public BlobClientOptions ContainerOptions => _defaultClientOptions;
         public IBlobContainerClientWrapper ContainerClientWrapper => _blobContainerClientWrapper;
 
-        public CloudBlobDirectoryWrapper(BlobServiceClient serviceClient, string containerName, string directoryPrefix, BlobClientOptions  blobClientOptions = null)
+        public CloudBlobDirectoryWrapper(BlobServiceClient serviceClient, string containerName, string directoryPrefix, BlobClientOptions blobClientOptions = null)
         {
             _directoryPrefix = directoryPrefix ?? throw new ArgumentNullException(nameof(directoryPrefix));
 
@@ -48,7 +48,9 @@ public CloudBlobDirectoryWrapper(BlobServiceClient serviceClient, string contain
             {
                 _containerClient = serviceClient.GetBlobContainerClient(containerName);
             }
-            
+
+            Uri = new Uri(Storage.RemoveQueryString(_containerClient.Uri).TrimEnd('/') + "/" + _directoryPrefix);
+
             _blobContainerClientWrapper = new BlobContainerClientWrapper(_containerClient);
         }
 

src/Catalog/Persistence/CloudBlobDirectoryWrapper.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -233,6 +233,11 @@ protected void TraceMethod(string method, Uri resourceUri)
             }
         }
 
+        public static string RemoveQueryString(Uri storageUri)
+        {
+            return storageUri.GetLeftPart(UriPartial.Path);
+        }
+
         private string TraceException(string method, Uri resourceUri, Exception exception)
         {
             string message = $"{method} EXCEPTION: {GetUri(GetName(resourceUri))} {exception.ToString()}";
@@ -246,4 +251,4 @@ private void TraceExecutionTime(string method, Uri resourceUri, long executionTi
             Trace.WriteLine(message);
         }
     }
-}
+}