[Revalidation] Pause if the ingestion pipeline is unhealthy

src/NuGet.Services.Revalidate/Configuration/HealthConfiguration.cs

@@ -0,0 +1,24 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace NuGet.Services.Revalidate
+{
+    public class HealthConfiguration
+    {
+        /// <summary>
+        /// The name of the Azure Blob Storage container that stores status information.
+        /// </summary>
+        public string ContainerName { get; set; }
+
+        /// <summary>
+        /// The name of the Azure Blob Storage blob that stores status information.
+        /// </summary>
+        public string StatusBlobName { get; set; }
+
+        /// <summary>
+        /// The path to the component that the revalidation job will monitor. The revalidation job will
+        /// pause if this component isn't healthy.
+        /// </summary>
+        public string ComponentPath { get; set; }
+    }
+}

src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs

@@ -35,6 +35,11 @@ public class RevalidationConfiguration
         /// </summary>
         public InitializationConfiguration Initialization { get; set; }
 
+        /// <summary>
+        /// The configurations used to determine the health of the ingestion pipeline.
+        /// </summary>
+        public HealthConfiguration Health { get; set; }
+
         /// <summary>
         /// The configurations used by the in-memory queue of revalidations to start.
         /// </summary>

src/NuGet.Services.Revalidate/Job.cs

@@ -96,6 +96,7 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
             services.Configure<RevalidationConfiguration>(configurationRoot.GetSection(JobConfigurationSectionName));
             services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value);
             services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.Initialization);
+            services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.Health);
             services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.Queue);
 
             services.AddScoped<IGalleryContext>(provider =>

src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj

@@ -43,6 +43,7 @@
     <Reference Include="System.Xml" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="Configuration\HealthConfiguration.cs" />
     <Compile Include="Configuration\InitializationConfiguration.cs" />
     <Compile Include="Configuration\RevalidationQueueConfiguration.cs" />
     <Compile Include="Extensions\IEnumerableExtensions.cs" />
@@ -103,6 +104,9 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
+    <PackageReference Include="NuGet.Services.Status">
+      <Version>2.27.0</Version>
+    </PackageReference>
     <PackageReference Include="NuGet.Services.Storage">
       <Version>2.27.0</Version>
     </PackageReference>

src/NuGet.Services.Revalidate/Services/HealthService.cs

@@ -1,17 +1,52 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
+using System.IO;
 using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
+using NuGet.Services.Status;
+using NuGetGallery;
 
 namespace NuGet.Services.Revalidate
 {
     public class HealthService : IHealthService
     {
-        public Task<bool> IsHealthyAsync()
+        private readonly ICoreFileStorageService _storage;
+        private readonly HealthConfiguration _config;
+        private readonly ILogger<HealthService> _logger;
+
+        public HealthService(
+            ICoreFileStorageService storage,
+            HealthConfiguration config,
+            ILogger<HealthService> logger)
+        {
+            _storage = storage ?? throw new ArgumentNullException(nameof(storage));
+            _config = config ?? throw new ArgumentNullException(nameof(config));
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        }
+
+        public async Task<bool> IsHealthyAsync()
         {
-            // TODO:
-            // We are software gods that never make mistakes.
-            return Task.FromResult(true);
+            using (var stream = await _storage.GetFileAsync(_config.ContainerName, _config.StatusBlobName))
+            using (var reader = new StreamReader(stream))
+            {
+                var json = await reader.ReadToEndAsync();
+                var status = JsonConvert.DeserializeObject<ServiceStatus>(json);
+                var component = status.ServiceRootComponent.GetByPath(_config.ComponentPath);
+
+                if (component == null)
+                {
+                    _logger.LogError(
+                        "Assuming that the service is unhealthy as the component path {ComponentPath} could not be found",
+                        _config.ComponentPath);
+
+                    return false;
+                }
+
+                return component.Status == ComponentStatus.Up;
+            }
         }
     }
-}
+}

src/NuGet.Services.Revalidate/Settings/dev.json

@@ -10,6 +10,12 @@
       "SleepDurationBetweenBatches": "00:00:01"
     },
 
+    "Health": {
+      "ContainerName": "status",
+      "StatusBlobName": "status.json",
+      "ComponentPath":  "NuGet/Package Publishing"
+    },
+
     "MinPackageEventRate": 120,
     "MaxPackageEventRate": 500,
 

src/NuGet.Services.Revalidate/Settings/int.json

@@ -10,6 +10,12 @@
       "SleepDurationBetweenBatches": "00:00:30"
     },
 
+    "Health": {
+      "ContainerName": "status",
+      "StatusBlobName": "status.json",
+      "ComponentPath": "NuGet/Package Publishing"
+    },
+
     "MinPackageEventRate": 120,
     "MaxPackageEventRate": 500,
 

src/NuGet.Services.Revalidate/Settings/prod.json

@@ -10,6 +10,12 @@
       "SleepDurationBetweenBatches": "00:00:30"
     },
 
+    "Health": {
+      "ContainerName": "status",
+      "StatusBlobName": "status.json",
+      "ComponentPath": "NuGet/Package Publishing"
+    },
+
     "MinPackageEventRate": 120,
     "MaxPackageEventRate": 500,
 

src/Validation.Common.Job/Validation.Common.Job.csproj

@@ -115,7 +115,7 @@
       <Version>2.27.0</Version>
     </PackageReference>
     <PackageReference Include="NuGetGallery.Core">
-      <Version>4.4.5-dev-36354</Version>
+      <Version>4.4.5-loshar-health-36421</Version>
     </PackageReference>
     <PackageReference Include="Serilog">
       <Version>2.5.0</Version>

src/Validation.Common.Job/Validation.Common.Job.nuspec

@@ -23,7 +23,7 @@
       <dependency id="NuGet.Services.Storage" version="2.27.0" />
       <dependency id="NuGet.Services.Validation" version="2.27.0" />
       <dependency id="NuGet.Services.Validation.Issues" version="2.27.0" />
-      <dependency id="NuGetGallery.Core" version="4.4.5-dev-36354" />
+      <dependency id="NuGetGallery.Core" version="4.4.5-loshar-health-36421" />
       <dependency id="Serilog" version="2.5.0" />
       <dependency id="System.Net.Http" version="4.3.3" />
     </dependencies>

tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj

@@ -66,11 +66,13 @@
     <Compile Include="Initializer\InitializationManagerFacts.cs" />
     <Compile Include="Initializer\PackageFinderFacts.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Services\HealthServiceFacts.cs" />
     <Compile Include="Services\RevalidationJobStateServiceFacts.cs" />
     <Compile Include="Services\PackageRevalidationStateServiceFacts.cs" />
     <Compile Include="Services\RevalidationQueueFacts.cs" />
     <Compile Include="Services\RevalidationServiceFacts.cs" />
     <Compile Include="Services\RevalidationThrottlerFacts.cs" />
+    <Compile Include="TestData\TestResources.cs" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\..\src\NuGet.Services.Revalidate\NuGet.Services.Revalidate.csproj">
@@ -82,5 +84,11 @@
       <Name>Tests.ContextHelpers</Name>
     </ProjectReference>
   </ItemGroup>
+  <ItemGroup>
+    <EmbeddedResource Include="TestData\PackagePublishingDownStatus.json" />
+    <EmbeddedResource Include="TestData\PackagePublishingDegradedStatus.json" />
+    <EmbeddedResource Include="TestData\PackagePublishingMissingStatus.json" />
+    <EmbeddedResource Include="TestData\PackagePublishingUpStatus.json" />
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
 </Project>

tests/NuGet.Services.Revalidate.Tests/Services/HealthServiceFacts.cs

@@ -0,0 +1,72 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Moq;
+using NuGet.Services.Revalidate.Tests.TestData;
+using NuGetGallery;
+using Xunit;
+
+namespace NuGet.Services.Revalidate.Tests.Services
+{
+    public class HealthServiceFacts
+    {
+        private readonly Mock<ICoreFileStorageService> _storage;
+        private readonly HealthConfiguration _config;
+        private readonly HealthService _target;
+
+        public HealthServiceFacts()
+        {
+            _storage = new Mock<ICoreFileStorageService>();
+            _config = new HealthConfiguration
+            {
+                ContainerName = "status",
+                StatusBlobName = "status.json",
+                ComponentPath = "NuGet/Package Publishing"
+            };
+
+            _target = new HealthService(_storage.Object, _config, Mock.Of<ILogger<HealthService>>());
+        }
+
+        [Theory]
+        [InlineData(TestResources.PackagePublishingDegradedStatus, false)]
+        [InlineData(TestResources.PackagePublishingDownStatus, false)]
+        [InlineData(TestResources.PackagePublishingUpStatus, true)]
+        public async Task ReturnsHealthyIfStatusBlobIndicatesHealthyComponent(string resourceName, bool expectsHealthy)
+        {
+            _storage
+                .Setup(s => s.GetFileAsync(_config.ContainerName, _config.StatusBlobName))
+                .ReturnsAsync(TestResources.GetResourceStream(resourceName));
+
+            Assert.Equal(expectsHealthy, await _target.IsHealthyAsync());
+        }
+
+        [Fact]
+        public async Task AssumesUnhealthyIfComponentCannotBeFoundInStatusBlob()
+        {
+            _storage
+                .Setup(s => s.GetFileAsync(_config.ContainerName, _config.StatusBlobName))
+                .ReturnsAsync(TestResources.GetResourceStream(TestResources.PackagePublishingMissingStatus));
+
+            Assert.False(await _target.IsHealthyAsync());
+        }
+
+        [Fact]
+        public async Task ThrowsIfStorageServiceThrows()
+        {
+            // This may happen if the status blob can't be found.
+            var expectedException = new Exception("Look ma, I'm an exception!");
+
+            _storage
+                .Setup(s => s.GetFileAsync(_config.ContainerName, _config.StatusBlobName))
+                .ThrowsAsync(expectedException);
+
+            // Act & Assert
+            var actualException = await Assert.ThrowsAsync<Exception>(() => _target.IsHealthyAsync());
+
+            Assert.Same(expectedException, actualException);
+        }
+    }
+}

tests/NuGet.Services.Revalidate.Tests/TestData/PackagePublishingDegradedStatus.json

@@ -0,0 +1,17 @@
+{
+  "LastUpdated": "2018-01-01T12:00:00.0000000Z",
+  "ServiceRootComponent": {
+    "Name": "NuGet",
+    "Status": "Up",
+    "SubComponents": [
+      {
+        "Status": "Degraded",
+        "Name": "Package Publishing",
+        "Description": "Uploading new packages to NuGet.org",
+        "Path": "NuGet/Package Publishing"
+      }
+    ]
+
+  },
+  "Events": []
+}

tests/NuGet.Services.Revalidate.Tests/TestData/PackagePublishingDownStatus.json

@@ -0,0 +1,16 @@
+{
+  "LastUpdated": "2018-01-01T12:00:00.0000000Z",
+  "ServiceRootComponent": {
+    "Name": "NuGet",
+    "Status": "Up",
+    "SubComponents": [
+      {
+        "Status": "Down",
+        "Name": "Package Publishing",
+        "Description": "Uploading new packages to NuGet.org",
+        "Path": "NuGet/Package Publishing"
+      }
+    ]
+  },
+  "Events": []
+}

tests/NuGet.Services.Revalidate.Tests/TestData/PackagePublishingMissingStatus.json

@@ -0,0 +1,16 @@
+{
+  "LastUpdated": "2018-01-01T12:00:00.0000000Z",
+  "ServiceRootComponent": {
+    "Name": "NuGet",
+    "Status": "Up",
+    "SubComponents": [
+      {
+        "Status": "Up",
+        "Name": "Not Package Publishing",
+        "Description": "Uploading new packages to NuGet.org",
+        "Path": "NuGet/Not Package Publishing"
+      }
+    ]
+  },
+  "Events": []
+}

tests/NuGet.Services.Revalidate.Tests/TestData/PackagePublishingUpStatus.json

@@ -0,0 +1,16 @@
+{
+  "LastUpdated": "2018-01-01T12:00:00.0000000Z",
+  "ServiceRootComponent": {
+    "Name": "NuGet",
+    "Status": "Up",
+    "SubComponents": [
+      {
+        "Status": "Up",
+        "Name": "Package Publishing",
+        "Description": "Uploading new packages to NuGet.org",
+        "Path": "NuGet/Package Publishing"
+      }
+    ]
+  },
+  "Events": []
+}

tests/NuGet.Services.Revalidate.Tests/TestData/TestResources.cs

@@ -0,0 +1,42 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.IO;
+
+namespace NuGet.Services.Revalidate.Tests.TestData
+{
+    internal class TestResources
+    {
+        private const string ResourceNamespace = "NuGet.Services.Revalidate.Tests.TestData";
+
+        public const string PackagePublishingDegradedStatus = ResourceNamespace + ".PackagePublishingDegradedStatus.json";
+        public const string PackagePublishingDownStatus = ResourceNamespace + ".PackagePublishingDownStatus.json";
+        public const string PackagePublishingUpStatus = ResourceNamespace + ".PackagePublishingUpStatus.json";
+
+        public const string PackagePublishingMissingStatus = ResourceNamespace + ".PackagePublishingMissingStatus.json";
+
+        /// <summary>
+        /// Buffer the resource stream into memory so the caller doesn't have to dispose.
+        /// </summary>
+        public static MemoryStream GetResourceStream(string resourceName)
+        {
+            var resourceStream = typeof(TestResources)
+                .Assembly
+                .GetManifestResourceStream(resourceName);
+
+            if (resourceStream == null)
+            {
+                return null;
+            }
+
+            var bufferedStream = new MemoryStream();
+            using (resourceStream)
+            {
+                resourceStream.CopyTo(bufferedStream);
+            }
+
+            bufferedStream.Position = 0;
+            return bufferedStream;
+        }
+    }
+}