Markdig version update (#8292)

* update markdig to 0.22.0

* migrate to markdig with feature flag

* add unit test

* add disable html extension, auto link, emjo, table, tasklist

Author: lyndaidaii

src/NuGetGallery.Services/Configuration/FeatureFlagService.cs

@@ -40,6 +40,7 @@ public class FeatureFlagService : IFeatureFlagService
         private const string PackageRenamesFeatureName = GalleryPrefix + "PackageRenames";
         private const string EmbeddedReadmeFlightName = GalleryPrefix + "EmbeddedReadmes";
         private const string LicenseMdRenderingFlightName = GalleryPrefix + "LicenseMdRendering";
+        private const string MarkdigMdRenderingFlightName = GalleryPrefix + "MarkdigMdRendering";
         private const string DeletePackageApiFlightName = GalleryPrefix + "DeletePackageApi";
 
         private const string ODataV1GetAllNonHijackedFeatureName = GalleryPrefix + "ODataV1GetAllNonHijacked";
@@ -286,6 +287,11 @@ public bool IsODataV2SearchCountNonHijackedEnabled()
             return _client.IsEnabled(ODataV2SearchCountNonHijackedFeatureName, defaultValue: true);
         }
 
+        public bool IsMarkdigMdRenderingEnabled()
+        {
+            return _client.IsEnabled(MarkdigMdRenderingFlightName, defaultValue: false);
+        }
+        
         public bool IsDeletePackageApiEnabled(User user)
         {
             return _client.IsEnabled(DeletePackageApiFlightName, user, defaultValue: false);

src/NuGetGallery.Services/Configuration/IFeatureFlagService.cs

@@ -227,6 +227,10 @@ public interface IFeatureFlagService
         bool IsODataV2SearchCountNonHijackedEnabled();
 
         /// <summary>
+        /// Whether rendering Markdown content to HTML using Markdig is enabled
+        /// </summary>
+        bool IsMarkdigMdRenderingEnabled();
+        
         /// Whether or not the user can delete a package through the API.
         /// </summary>
         bool IsDeletePackageApiEnabled(User user);

src/NuGetGallery/Services/MarkdownService.cs

@@ -3,25 +3,74 @@
 
 using System;
 using System.IO;
+using System.Linq;
+using System.Management;
 using System.Text.RegularExpressions;
+using System.Timers;
 using System.Web;
 using CommonMark;
 using CommonMark.Syntax;
+using Markdig;
+using Markdig.Parsers;
+using Markdig.Renderers;
+using Markdig.Syntax;
+using Markdig.Syntax.Inlines;
 
 namespace NuGetGallery
 {
     public class MarkdownService : IMarkdownService
     {
         private static readonly TimeSpan RegexTimeout = TimeSpan.FromMinutes(1);
         private static readonly Regex EncodedBlockQuotePattern = new Regex("^ {0,3}>", RegexOptions.Multiline, RegexTimeout);
-        private static readonly Regex CommonMarkLinkPattern = new Regex("<a href=([\"\']).*?\\1", RegexOptions.None, RegexTimeout);
+        private static readonly Regex LinkPattern = new Regex("<a href=([\"\']).*?\\1", RegexOptions.None, RegexTimeout);
+
+        private readonly IFeatureFlagService _features;
+
+        public MarkdownService(IFeatureFlagService features)
+        {
+            _features = features ?? throw new ArgumentNullException(nameof(features));
+        }
 
         public RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString)
         {
-            return GetHtmlFromMarkdown(markdownString, 1);
+            if (markdownString == null)
+            {
+                throw new ArgumentNullException(nameof(markdownString));
+            }
+
+            if (_features.IsMarkdigMdRenderingEnabled()) 
+            { 
+                return GetHtmlFromMarkdownMarkdig(markdownString, 1);
+            }
+            else
+            {
+                return GetHtmlFromMarkdownCommonMark(markdownString, 1);
+            }
         }
 
         public RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString, int incrementHeadersBy)
+        {
+            if (markdownString == null)
+            {
+                throw new ArgumentNullException(nameof(markdownString));
+            }
+
+            if (incrementHeadersBy < 0)
+            {
+                throw new ArgumentOutOfRangeException(nameof(incrementHeadersBy), $"{nameof(incrementHeadersBy)} must be greater than or equal to 0");
+            }
+
+            if (_features.IsMarkdigMdRenderingEnabled())
+            {
+                return GetHtmlFromMarkdownMarkdig(markdownString, incrementHeadersBy);
+            }
+            else
+            {
+                return GetHtmlFromMarkdownCommonMark(markdownString, incrementHeadersBy);
+            }
+        }
+
+        private RenderedMarkdownResult GetHtmlFromMarkdownCommonMark(string markdownString, int incrementHeadersBy)
         {
             var output = new RenderedMarkdownResult()
             {
@@ -106,7 +155,84 @@ public RenderedMarkdownResult GetHtmlFromMarkdown(string markdownString, int inc
             {
                 CommonMarkConverter.ProcessStage3(document, htmlWriter, settings);
 
-                output.Content = CommonMarkLinkPattern.Replace(htmlWriter.ToString(), "$0" + " rel=\"nofollow\"").Trim();
+                output.Content = LinkPattern.Replace(htmlWriter.ToString(), "$0" + " rel=\"nofollow\"").Trim();
+                return output;
+            }
+        }
+
+        private RenderedMarkdownResult GetHtmlFromMarkdownMarkdig(string markdownString, int incrementHeadersBy)
+        {
+            var output = new RenderedMarkdownResult()
+            {
+                ImagesRewritten = false,
+                Content = ""
+            };
+
+            var readmeWithoutBom = markdownString.TrimStart('\ufeff');
+
+            var pipeline = new MarkdownPipelineBuilder()
+                .UseGridTables()
+                .UsePipeTables()
+                .UseListExtras()
+                .UseTaskLists()
+                .UseSoftlineBreakAsHardlineBreak()
+                .UseEmojiAndSmiley()
+                .UseAutoLinks()
+                .UseReferralLinks("nofollow")
+                .DisableHtml() //block inline html
+                .Build();
+
+            using (var htmlWriter = new StringWriter())
+            {
+                var renderer = new HtmlRenderer(htmlWriter);
+                pipeline.Setup(renderer);
+
+                var document = Markdown.Parse(readmeWithoutBom, pipeline);
+
+                foreach (var node in document.Descendants())
+                {
+                    if (node is Markdig.Syntax.Block)
+                    {
+                        // Demote heading tags so they don't overpower expander headings.
+                        if (node is HeadingBlock heading)
+                        {
+                            heading.Level = Math.Min(heading.Level + incrementHeadersBy, 6);
+                        }
+                    }
+                    else if (node is Markdig.Syntax.Inlines.Inline)
+                    {
+                        if (node is LinkInline linkInline)
+                        {
+                            if (linkInline.IsImage)
+                            {
+                                if (!PackageHelper.TryPrepareUrlForRendering(linkInline.Url, out string readyUriString, rewriteAllHttp: true))
+                                {
+                                    linkInline.Url = string.Empty;
+                                }
+                                else
+                                {
+                                    output.ImagesRewritten = output.ImagesRewritten || (linkInline.Url != readyUriString);
+                                    linkInline.Url = readyUriString;
+                                }
+                            }
+                            else
+                            {
+                                // Allow only http or https links in markdown. Transform link to https for known domains.
+                                if (!PackageHelper.TryPrepareUrlForRendering(linkInline.Url, out string readyUriString))
+                                {
+                                    linkInline.Url = string.Empty;
+                                }
+                                else
+                                {
+                                    linkInline.Url = readyUriString;
+                                }
+                            }
+                        }
+                    }
+                }
+
+                renderer.Render(document);
+                output.Content = htmlWriter.ToString().Trim();
                 return output;
             }
         }