Add BlockSearchEngineIndexing to allow site-wide search engine blocking (#6385)

Progress on #6381

Author: joelverhagen

src/NuGetGallery/Configuration/AppConfiguration.cs

@@ -342,5 +342,7 @@ public string ExternalBrandingMessage
         public bool RejectSignedPackagesWithNoRegisteredCertificate { get; set; }
 
         public bool RejectPackagesWithTooManyPackageEntries { get; set; }
+
+        public bool BlockSearchEngineIndexing { get; set; }
     }
 }

src/NuGetGallery/Configuration/IAppConfiguration.cs

@@ -344,5 +344,10 @@ public interface IAppConfiguration : ICoreMessageServiceConfiguration
         /// Whether or not to synchronously reject packages on push/upload that have too many package entries.
         /// </summary>
         bool RejectPackagesWithTooManyPackageEntries { get; set; }
+
+        /// <summary>
+        /// Whether or not to block search engines from indexing the web pages using the "noindex" meta tag.
+        /// </summary>
+        bool BlockSearchEngineIndexing { get; set; }
     }
 }

src/NuGetGallery/Views/Packages/DisplayPackage.cshtml

@@ -5,6 +5,7 @@
 @{
     ViewBag.Title = Model.Id + " " + Model.Version;
     ViewBag.Tab = "Packages";
+    ViewBag.BlockSearchEngineIndexing = !Model.Listed || !Model.Available;
 
     var absolutePackageUrl = Url.Absolute(Url.Package(Model.Id));
 
@@ -74,12 +75,6 @@
     <meta property="og:determiner" content="a" />
     <meta property="og:image" content="@(PackageHelper.ShouldRenderUrl(Model.IconUrl) ? Model.IconUrl : Url.Absolute("~/Content/gallery/img/default-package-icon-256x256.png"))" />
 }
-@section Meta {
-    @if (!Model.Listed || !Model.Available)
-    {
-        <meta name="robots" content="noindex">
-    }
-}
 
 @helper VersionListDivider(int rowCount, bool versionsExpanded)
 {

src/NuGetGallery/Views/Shared/Gallery/Layout.cshtml

@@ -4,6 +4,10 @@
     <meta charset="utf-8" />
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
+    @if (Config.Current.BlockSearchEngineIndexing || ViewBag.BlockSearchEngineIndexing == true)
+    {
+        <meta name="robots" content="noindex">
+    }
     @RenderSection("SocialMeta", required: false)
     @RenderSection("Meta", required: false)
 

src/NuGetGallery/Web.config

@@ -147,6 +147,7 @@
     <add key="Gallery.EnforceDefaultSecurityPolicies" value="false"/>
     <add key="Gallery.RejectSignedPackagesWithNoRegisteredCertificate" value="false"/>
     <add key="Gallery.RejectPackagesWithTooManyPackageEntries" value="false"/>
+    <add key="Gallery.BlockSearchEngineIndexing" value="false"/>
     <!-- This is also the default so you can remove this setting if you really want. -->
     <!-- Set this to false if you want to disable search indexing in the background. -->
     <add key="Gallery.AutoUpdateSearchIndex" value="true"/>