HandlePackageEdits (#181)

* Update RepositoryUrl in DB and ReadMe in blob storage on package edit (#174)

* Update HasReadMe depending on ReadMeState

* Edits/deletes ReadMe from blob storage

* remove unneeded commented code

* make blob readme id folder lowercase

* code clean up

* moved ReadMe blob storage update to its own function

* Scott's updates

* changed HasReadMe update location + other minor fixes

* PR quick changes

* changed back to updating ReadMe, then DB

* added rollback if DB changes fail

* moved readme to a different function

* moved code to rollback function

* minor text fixes

* include modification of both .md and .html file

* indentation

* cleaned up template

* changed GetReadMeBlobPath to private

* tuple to private class

* roll back trace error and while not completed loop

* Added ReadMeState for upload and minor text fixes (#180)

* Update HasReadMe depending on ReadMeState

* Edits/deletes ReadMe from blob storage

* remove unneeded commented code

* make blob readme id folder lowercase

* code clean up

* moved ReadMe blob storage update to its own function

* Scott's updates

* changed HasReadMe update location + other minor fixes

* PR quick changes

* changed back to updating ReadMe, then DB

* added rollback if DB changes fail

* moved readme to a different function

* moved code to rollback function

* minor text fixes

* include modification of both .md and .html file

* indentation

* cleaned up template

* changed GetReadMeBlobPath to private

* tuple to private class

* roll back trace error and while not completed loop

* added uploaded ReadMeState because of snapshots and rollback scenarios

* delete upload state and check table instead

Author: laurenshen

src/HandlePackageEdits/HandlePackageEdits.Job.cs

@@ -26,15 +26,21 @@ internal class Job : JobBase
         public static readonly long DefaultMaxAllowedManifestBytes = 10 /* Mb */ * 1024 /* Kb */ * 1024; /* b */
 
         public static readonly string GetEditsBaseSql = @"
-            SELECT pr.Id, p.NormalizedVersion AS Version, p.Hash, e.*
+            SELECT pr.Id, p.NormalizedVersion AS Version, p.Hash, p.HasReadMe, e.*
             FROM PackageEdits e
             INNER JOIN Packages p ON p.[Key] = e.PackageKey
             INNER JOIN PackageRegistrations pr ON pr.[Key] = p.PackageRegistrationKey";
 
         public const string DefaultSourceContainerName = "packages";
         public const string DefaultBackupContainerName = "package-backups";
+        public const string DefaultReadMeContainerName = "readmes";
         public const int DefaultMaxRetryCount = 10;
 
+        private const string ReadMeChanged = "changed";
+        private const string ReadMeDeleted = "deleted";
+        private const string MarkdownExtension = "md";
+        private const string HtmlExtension = "html";
+
         /// <summary>
         /// Gets or sets an Azure Storage Uri referring to a container to use as the source for package blobs
         /// </summary>
@@ -52,18 +58,31 @@ FROM PackageEdits e
         public CloudStorageAccount Backups { get; set; }
         public string BackupsContainerName { get; set; }
 
+        /// <summary>
+        /// Gets or sets an Azure Storage Uri referring to a container to use as the storage for ReadMes
+        /// </summary>
+        public string ReadMeContainerName { get; set; }
+
         /// <summary>
         /// Gets or sets the maximum number of tries that are allowed before considering an edit failed.
         /// </summary>
         public int? MaxTryCount { get; set; }
 
         public long? MaxAllowedManifestBytes { get; set; }
 
-        protected CloudBlobContainer SourceContainer { get; set; }
-        protected CloudBlobContainer BackupsContainer { get; set; }
+        private CloudBlobContainer SourceContainer { get; set; }
+        private CloudBlobContainer BackupsContainer { get; set; }
+        private CloudBlobContainer ReadMeContainer { get; set; }
         protected long MaxManifestSize { get; set; }
         private ILogger _logger;
 
+        private class ReadMeBlobs
+        {
+            public CloudBlockBlob activeBlob = null;
+            public CloudBlockBlob pendingBlob = null;
+            public CloudBlockBlob activeSnapshot = null;
+        }
+
         public override bool Init(IDictionary<string, string> jobArgsDictionary)
         {
             try
@@ -86,12 +105,13 @@ public override bool Init(IDictionary<string, string> jobArgsDictionary)
                                            JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.SourceStorage));
                 Backups = CloudStorageAccount.Parse(
                                            JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.BackupStorage));
-
                 SourceContainerName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.SourceContainerName) ?? DefaultSourceContainerName;
                 BackupsContainerName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.BackupContainerName) ?? DefaultBackupContainerName;
+                ReadMeContainerName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.ReadMeContainerName) ?? DefaultReadMeContainerName;
 
                 SourceContainer = Source.CreateCloudBlobClient().GetContainerReference(SourceContainerName);
                 BackupsContainer = Backups.CreateCloudBlobClient().GetContainerReference(BackupsContainerName);
+                ReadMeContainer = Source.CreateCloudBlobClient().GetContainerReference(ReadMeContainerName);
 
                 MaxTryCount = DefaultMaxRetryCount;
             }
@@ -154,13 +174,14 @@ public override async Task<bool> Run()
                     await UpdatePackageEditDbWithError(exception, edit.Key);
                 }
             }
-
             return true;
         }
 
         private async Task ApplyEdit(PackageEdit edit)
         {
             string originalPath = null;
+            string originalReadMeMDPath = null;
+            string originalReadMeHTMLPath = null;
 
             try
             {
@@ -172,6 +193,8 @@ private async Task ApplyEdit(PackageEdit edit)
                 }
 
                 originalPath = Path.Combine(directory, "original.nupkg");
+                Trace.TraceInformation($"Downloading nupkg to {originalPath}");
+
                 var sourceBlob = SourceContainer.GetBlockBlobReference(
                     StorageHelpers.GetPackageBlobName(edit.Id, edit.Version));
                 Trace.TraceInformation($"Name is {sourceBlob.Name}, storage uri is {sourceBlob.StorageUri}");
@@ -223,8 +246,12 @@ private async Task ApplyEdit(PackageEdit edit)
                     hash = Convert.ToBase64String(
                         hashAlgorithm.ComputeHash(originalStream));
                 }
+                
+                // ReadMe update
+                var readMeMDBlob = await UpdateReadMeAsync(edit, directory, originalReadMeMDPath, MarkdownExtension);
+
+                var readMeHTMLBlob = await UpdateReadMeAsync(edit, directory, originalReadMeHTMLPath, HtmlExtension);
 
-                // Update the database
                 try
                 {
                     Trace.TraceInformation($"Updating package record for {edit.Id} {edit.Version}");
@@ -234,20 +261,52 @@ private async Task ApplyEdit(PackageEdit edit)
                 catch (Exception exception)
                 {
                     // Error occurred while updaing database, roll back the blob to the snapshot
-                    // Can't do "await" in a catch block, but this should be pretty quick since it just starts the copy
                     Trace.TraceError($"Failed to update database! {exception}");
                     Trace.TraceWarning(
                         $"Rolling back updated blob for {edit.Id} {edit.Version}. Copying snapshot {sourceSnapshot.Uri.AbsoluteUri} to {sourceBlob.Uri.AbsoluteUri}");
-                    sourceBlob.StartCopy(sourceSnapshot);
+                    await sourceBlob.StartCopyAsync(sourceSnapshot);
+                    while (sourceBlob.CopyState.Status != CopyStatus.Success)
+                    {
+                        await Task.Delay(1000);
+                    }
                     Trace.TraceWarning(
                         $"Rolled back updated blob for {edit.Id} {edit.Version}. Copying snapshot {sourceSnapshot.Uri.AbsoluteUri} to {sourceBlob.Uri.AbsoluteUri}");
 
+                    await RollBackReadMeAsync(edit, directory, originalReadMeMDPath, readMeMDBlob.activeSnapshot, readMeMDBlob.activeBlob);
+                    await RollBackReadMeAsync(edit, directory, originalReadMeHTMLPath, readMeHTMLBlob.activeSnapshot, readMeHTMLBlob.activeBlob);
+
                     throw;
                 }
 
+                if (edit.ReadMeState == ReadMeChanged)
+                {
+                    // Delete pending ReadMes
+                    Trace.TraceInformation($"Deleting pending ReadMe for {edit.Id} {edit.Version} from {readMeMDBlob.pendingBlob.Uri.AbsoluteUri}");
+                    await readMeMDBlob.pendingBlob.DeleteIfExistsAsync();
+                    Trace.TraceInformation($"Deleted pending ReadMe for {edit.Id} {edit.Version} from {readMeMDBlob.pendingBlob.Uri.AbsoluteUri}");
+
+                    Trace.TraceInformation($"Deleting pending ReadMe for {edit.Id} {edit.Version} from {readMeHTMLBlob.pendingBlob.Uri.AbsoluteUri}");
+                    await readMeHTMLBlob.pendingBlob.DeleteIfExistsAsync();
+                    Trace.TraceInformation($"Deleted pending ReadMe for {edit.Id} {edit.Version} from {readMeHTMLBlob.pendingBlob.Uri.AbsoluteUri}");
+                }
+
                 Trace.TraceInformation("Deleting snapshot blob {2} for {0} {1}.", edit.Id, edit.Version, sourceSnapshot.Uri.AbsoluteUri);
                 await sourceSnapshot.DeleteAsync();
                 Trace.TraceInformation("Deleted snapshot blob {2} for {0} {1}.", edit.Id, edit.Version, sourceSnapshot.Uri.AbsoluteUri);
+
+                if (readMeMDBlob.activeSnapshot != null)
+                {
+                    Trace.TraceInformation("Deleting snapshot ReadMe {2} for {0} {1}.", edit.Id, edit.Version, readMeMDBlob.activeSnapshot.Uri.AbsoluteUri);
+                    await readMeMDBlob.activeSnapshot.DeleteAsync();
+                    Trace.TraceInformation("Deleted snapshot ReadMe{2} for {0} {1}.", edit.Id, edit.Version, readMeMDBlob.activeSnapshot.Uri.AbsoluteUri);
+                }
+
+                if (readMeHTMLBlob.activeSnapshot != null)
+                {
+                    Trace.TraceInformation("Deleting snapshot ReadMe {2} for {0} {1}.", edit.Id, edit.Version, readMeHTMLBlob.activeSnapshot.Uri.AbsoluteUri);
+                    await readMeHTMLBlob.activeSnapshot.DeleteAsync();
+                    Trace.TraceInformation("Deleted snapshot ReadMe {2} for {0} {1}.", edit.Id, edit.Version, readMeHTMLBlob.activeSnapshot.Uri.AbsoluteUri);
+                }
             }
             finally
             {
@@ -277,6 +336,8 @@ private async Task UpdateDatabaseWithEdit(PackageEdit edit, string hash, long si
                     edit.IconUrl,
                     edit.LicenseUrl,
                     edit.ProjectUrl,
+                    edit.RepositoryUrl,
+                    edit.ReadMeState,
                     edit.ReleaseNotes,
                     edit.RequiresLicenseAcceptance,
                     edit.Summary,
@@ -290,6 +351,20 @@ private async Task UpdateDatabaseWithEdit(PackageEdit edit, string hash, long si
                     HashAlgorithm = HashAlgorithmName
                 });
 
+                // Update parameters with new HasReadMe value
+                var HasReadMeSQL = string.Empty;
+                if (edit.ReadMeState == ReadMeChanged)
+                {
+                    parameters.Add("HasReadMe", 1);
+                    HasReadMeSQL = ", HasReadMe = @HasReadMe";
+
+                }
+                else if (edit.ReadMeState == ReadMeDeleted)
+                {
+                    parameters.Add("HasReadMe", 0);
+                    HasReadMeSQL = ", HasReadMe = @HasReadMe";
+                }
+
                 // Prep SQL for merging in authors
                 var loadAuthorsSql = new StringBuilder();
                 var authors = edit.Authors.Split(',');
@@ -299,7 +374,7 @@ private async Task UpdateDatabaseWithEdit(PackageEdit edit, string hash, long si
                     parameters.Add("Author" + i, authors[i]);
                 }
 
-                await connection.QueryAsync<int>(@"
+                await connection.QueryAsync<int>($@"
                             BEGIN TRANSACTION
                                 -- Form a comma-separated list of authors
                                 DECLARE @existingAuthors nvarchar(MAX)
@@ -327,7 +402,8 @@ INSERT INTO [PackageHistories]
                                             HashAlgorithm,
                                             PackageFileSize,
                                             LastUpdated,
-                                            Published
+                                            Published,
+                                            RepositoryUrl
                                 FROM        [Packages]
                                 WHERE       [Key] = @PackageKey
 
@@ -349,7 +425,9 @@ UPDATE  [Packages]
                                         Hash = @Hash,
                                         HashAlgorithm = @HashAlgorithm,
                                         PackageFileSize = @PackageFileSize,
-                                        FlattenedAuthors = @Authors
+                                        FlattenedAuthors = @Authors,
+                                        RepositoryUrl = @RepositoryUrl
+                                        {HasReadMeSQL}
                                 WHERE   [Key] = @PackageKey
 
                                 -- Update Authors
@@ -389,7 +467,123 @@ UPDATE  PackageEdits
             {
                 Trace.TraceError($"Error updating the package edit database with error! {ex}");
             }
+        }
+
+        private async Task<ReadMeBlobs> UpdateReadMeAsync(PackageEdit edit, 
+            string directory, string originalReadMePath, string readMeExtension)
+        {
+            ReadMeBlobs currentBlob = new ReadMeBlobs();
 
+            // ReadMeState of null means unmodified
+            if (edit.ReadMeState == null)
+            {
+                return currentBlob;
+            }
+
+            originalReadMePath = Path.Combine(directory, "readme" + readMeExtension);
+            Trace.TraceInformation($"Attempting to save ReadMe at {originalReadMePath}");
+
+            currentBlob.activeBlob = ReadMeContainer.GetBlockBlobReference(
+                        StorageHelpers.GetActiveReadMeBlobNamePath(edit.Id, edit.Version, readMeExtension));
+            Trace.TraceInformation($"Found active ReadMe {currentBlob.activeBlob.Name} at storage URI {currentBlob.activeBlob.StorageUri}");
+
+            // Update ReadMe in blob storage
+            if (edit.ReadMeState == ReadMeChanged)
+            {
+                // Do the blob update
+                try
+                {
+                    currentBlob.pendingBlob = ReadMeContainer.GetBlockBlobReference(
+                        StorageHelpers.GetPendingReadMeBlobNamePath(edit.Id, edit.Version, readMeExtension));
+                    Trace.TraceInformation($"Found pending ReadMe {currentBlob.pendingBlob.Name} at storage URI {currentBlob.pendingBlob.StorageUri}");
+
+                    if (edit.HasReadMe)
+                    {
+                        // Snapshot the original blob if it exists (so if it's an edit, not an upload)
+                        Trace.TraceInformation($"Snapshotting original blob for {edit.Id} {edit.Version} ({currentBlob.activeBlob.Uri.AbsoluteUri}).");
+                        currentBlob.activeSnapshot = await currentBlob.activeBlob.CreateSnapshotAsync();
+                        Trace.TraceInformation($"Snapshotted original blob for {edit.Id} {edit.Version} ({currentBlob.activeBlob.Uri.AbsoluteUri}).");
+                    }
+
+                    // Download pending ReadMe
+                    Trace.TraceInformation($"Downloading new ReadMe for {edit.Id} {edit.Version}");
+                    await currentBlob.pendingBlob.DownloadToFileAsync(originalReadMePath, FileMode.Create);
+                    Trace.TraceInformation($"Downloaded new ReadMe for {edit.Id} {edit.Version}");
+
+                    // Upload pending ReadMe to active
+                    Trace.TraceInformation($"Uploading new ReadMe for {edit.Id} {edit.Version} to {currentBlob.activeBlob.Uri.AbsoluteUri}");
+                    await currentBlob.activeBlob.UploadFromFileAsync(originalReadMePath);
+                    Trace.TraceInformation($"Uploaded new ReadMe for {edit.Id} {edit.Version} to {currentBlob.activeBlob.Uri.AbsoluteUri}");
+                }
+                finally
+                {
+                    if (!string.IsNullOrEmpty(originalReadMePath) && File.Exists(originalReadMePath))
+                    {
+                        File.Delete(originalReadMePath);
+                    }
+                }
+            }
+            // Delete ReadMe in blob storage
+            else if (edit.ReadMeState == ReadMeDeleted)
+            {
+                // Download active ReadMe
+                Trace.TraceInformation($"Downloading old ReadMe for {edit.Id} {edit.Version}");
+                await currentBlob.activeBlob.DownloadToFileAsync(originalReadMePath, FileMode.Create);
+                Trace.TraceInformation($"Downloaded old ReadMe for {edit.Id} {edit.Version}");
+
+                // Delete active ReadMe
+                Trace.TraceInformation($"Deleting ReadMe of {edit.Id} {edit.Version} from {currentBlob.activeBlob.Uri.AbsoluteUri}");
+                await currentBlob.activeBlob.DeleteIfExistsAsync();
+                Trace.TraceInformation($"Deleted ReadMe of {edit.Id} {edit.Version} from {currentBlob.activeBlob.Uri.AbsoluteUri}");
+            }
+            return currentBlob;
+        }
+
+        private async Task RollBackReadMeAsync(PackageEdit edit, string directory, string originalReadMePath, 
+            CloudBlockBlob activeReadMeSnapshot, CloudBlockBlob activeReadMeBlob)
+        {
+            if (edit.ReadMeState != null)
+            {
+                if (edit.ReadMeState == ReadMeChanged)
+                {
+                    if (edit.HasReadMe)
+                    {
+                        Trace.TraceWarning(
+                        $"Rolling back ReadMe blob for {edit.Id} {edit.Version}. Copying snapshot {activeReadMeSnapshot.Uri.AbsoluteUri} to {activeReadMeBlob.Uri.AbsoluteUri}");
+                        activeReadMeBlob.StartCopy(activeReadMeSnapshot);
+                        while (activeReadMeBlob.CopyState.Status != CopyStatus.Success)
+                        {
+                            await Task.Delay(1000);
+                        }
+                        Trace.TraceWarning(
+                            $"Rolled back ReadMe blob for {edit.Id} {edit.Version}. Copying snapshot {activeReadMeSnapshot.Uri.AbsoluteUri} to {activeReadMeBlob.Uri.AbsoluteUri}");
+                    }
+                    else
+                    {
+                        // Delete ReadMes from active
+                        Trace.TraceInformation($"Deleting ReadMe of {edit.Id} {edit.Version} from {activeReadMeBlob.Uri.AbsoluteUri}");
+                        await activeReadMeBlob.DeleteIfExistsAsync();
+                        Trace.TraceInformation($"Deleted ReadMe of {edit.Id} {edit.Version} from {activeReadMeBlob.Uri.AbsoluteUri}");
+                    }
+                }
+                else if (edit.ReadMeState == ReadMeDeleted)
+                {
+                    try
+                    {
+                        // Upload original ReadMe back to active
+                        Trace.TraceInformation($"Uploading old ReadMe for {edit.Id} {edit.Version} to {activeReadMeBlob.Uri.AbsoluteUri}");
+                        await activeReadMeBlob.UploadFromFileAsync(originalReadMePath);
+                        Trace.TraceInformation($"Uploaded old ReadMe for {edit.Id} {edit.Version} to {activeReadMeBlob.Uri.AbsoluteUri}");
+                    }
+                    finally
+                    {
+                        if (!string.IsNullOrEmpty(originalReadMePath) && File.Exists(originalReadMePath))
+                        {
+                            File.Delete(originalReadMePath);
+                        }
+                    }
+                }
+            }
         }
     }
 }