[Revalidation] Pause if the ingestion pipeline is unhealthy (#504)

This integrates with @scottbommarito's status work to pause the job if the ingestion pipeline is unhealthy.

Part of https://github.com/NuGet/Engineering/issues/1128

Author: loic-sharma

src/NuGet.Services.Revalidate/Configuration/HealthConfiguration.cs

@@ -0,0 +1,24 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace NuGet.Services.Revalidate
+{
+    public class HealthConfiguration
+    {
+        /// <summary>
+        /// The name of the Azure Blob Storage container that stores status information.
+        /// </summary>
+        public string ContainerName { get; set; }
+
+        /// <summary>
+        /// The name of the Azure Blob Storage blob that stores status information.
+        /// </summary>
+        public string StatusBlobName { get; set; }
+
+        /// <summary>
+        /// The path to the component that the revalidation job will monitor. The revalidation job will
+        /// pause if this component isn't healthy.
+        /// </summary>
+        public string ComponentPath { get; set; }
+    }
+}

src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs

@@ -35,6 +35,11 @@ public class RevalidationConfiguration
         /// </summary>
         public InitializationConfiguration Initialization { get; set; }
 
+        /// <summary>
+        /// The configurations used to determine the health of the ingestion pipeline.
+        /// </summary>
+        public HealthConfiguration Health { get; set; }
+
         /// <summary>
         /// The configurations used by the in-memory queue of revalidations to start.
         /// </summary>

src/NuGet.Services.Revalidate/Job.cs

@@ -96,6 +96,7 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
             services.Configure<RevalidationConfiguration>(configurationRoot.GetSection(JobConfigurationSectionName));
             services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value);
             services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.Initialization);
+            services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.Health);
             services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.Queue);
 
             services.AddScoped<IGalleryContext>(provider =>

src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj

@@ -43,6 +43,7 @@
     <Reference Include="System.Xml" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="Configuration\HealthConfiguration.cs" />
     <Compile Include="Configuration\InitializationConfiguration.cs" />
     <Compile Include="Configuration\RevalidationQueueConfiguration.cs" />
     <Compile Include="Extensions\IEnumerableExtensions.cs" />
@@ -103,6 +104,9 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
+    <PackageReference Include="NuGet.Services.Status">
+      <Version>2.27.0</Version>
+    </PackageReference>
     <PackageReference Include="NuGet.Services.Storage">
       <Version>2.27.0</Version>
     </PackageReference>

src/NuGet.Services.Revalidate/Services/HealthService.cs

@@ -1,17 +1,52 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
+using System.IO;
 using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
+using NuGet.Services.Status;
+using NuGetGallery;
 
 namespace NuGet.Services.Revalidate
 {
     public class HealthService : IHealthService
     {
-        public Task<bool> IsHealthyAsync()
+        private readonly ICoreFileStorageService _storage;
+        private readonly HealthConfiguration _config;
+        private readonly ILogger<HealthService> _logger;
+
+        public HealthService(
+            ICoreFileStorageService storage,
+            HealthConfiguration config,
+            ILogger<HealthService> logger)
+        {
+            _storage = storage ?? throw new ArgumentNullException(nameof(storage));
+            _config = config ?? throw new ArgumentNullException(nameof(config));
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        }
+
+        public async Task<bool> IsHealthyAsync()
         {
-            // TODO:
-            // We are software gods that never make mistakes.
-            return Task.FromResult(true);
+            using (var stream = await _storage.GetFileAsync(_config.ContainerName, _config.StatusBlobName))
+            using (var reader = new StreamReader(stream))
+            {
+                var json = await reader.ReadToEndAsync();
+                var status = JsonConvert.DeserializeObject<ServiceStatus>(json);
+                var component = status.ServiceRootComponent.GetByPath(_config.ComponentPath);
+
+                if (component == null)
+                {
+                    _logger.LogError(
+                        "Assuming that the service is unhealthy as the component path {ComponentPath} could not be found",
+                        _config.ComponentPath);
+
+                    return false;
+                }
+
+                return component.Status == ComponentStatus.Up;
+            }
         }
     }
-}
+}

src/NuGet.Services.Revalidate/Settings/dev.json

@@ -10,6 +10,12 @@
       "SleepDurationBetweenBatches": "00:00:01"
     },
 
+    "Health": {
+      "ContainerName": "status",
+      "StatusBlobName": "status.json",
+      "ComponentPath":  "NuGet/Package Publishing"
+    },
+
     "MinPackageEventRate": 120,
     "MaxPackageEventRate": 500,
 

src/NuGet.Services.Revalidate/Settings/int.json

@@ -10,6 +10,12 @@
       "SleepDurationBetweenBatches": "00:00:30"
     },
 
+    "Health": {
+      "ContainerName": "status",
+      "StatusBlobName": "status.json",
+      "ComponentPath": "NuGet/Package Publishing"
+    },
+
     "MinPackageEventRate": 120,
     "MaxPackageEventRate": 500,
 

src/NuGet.Services.Revalidate/Settings/prod.json

@@ -10,6 +10,12 @@
       "SleepDurationBetweenBatches": "00:00:30"
     },
 
+    "Health": {
+      "ContainerName": "status",
+      "StatusBlobName": "status.json",
+      "ComponentPath": "NuGet/Package Publishing"
+    },
+
     "MinPackageEventRate": 120,
     "MaxPackageEventRate": 500,
 

src/Validation.Common.Job/Validation.Common.Job.csproj

@@ -115,7 +115,7 @@
       <Version>2.27.0</Version>
     </PackageReference>
     <PackageReference Include="NuGetGallery.Core">
-      <Version>4.4.5-dev-36354</Version>
+      <Version>4.4.5-master-36524</Version>
     </PackageReference>
     <PackageReference Include="Serilog">
       <Version>2.5.0</Version>

src/Validation.Common.Job/Validation.Common.Job.nuspec

@@ -23,7 +23,7 @@
       <dependency id="NuGet.Services.Storage" version="2.27.0" />
       <dependency id="NuGet.Services.Validation" version="2.27.0" />
       <dependency id="NuGet.Services.Validation.Issues" version="2.27.0" />
-      <dependency id="NuGetGallery.Core" version="4.4.5-dev-36354" />
+      <dependency id="NuGetGallery.Core" version="4.4.5-master-36524" />
       <dependency id="Serilog" version="2.5.0" />
       <dependency id="System.Net.Http" version="4.3.3" />
     </dependencies>