SIDM-8803 ignore am password grant failures after start up

build.gradle

@@ -9,7 +9,7 @@ plugins {
 }
 
 group = 'uk.gov.hmcts.reform'
-version = '2.3.3'
+version = '2.3.4'
 sourceCompatibility = 1.8
 generateLombokConfig.enabled = false
 

src/main/java/uk/gov/hmcts/reform/idam/health/am/AmHealthProbeConfiguration.java

@@ -40,7 +40,7 @@ public ScheduledHealthProbeIndicator amReadyScheduledHealthProbe(AMReadyHealthPr
     public ScheduledHealthProbeIndicator amPasswordGrantScheduledHealthProbe(AmPasswordGrantHealthProbe amPasswordGrantHealthProbe) {
         return new ScheduledHealthProbeIndicator(
                 amPasswordGrantHealthProbe,
-                HealthProbeFailureHandling.MARK_AS_DOWN,
+                HealthProbeFailureHandling.IGNORE_ONCE_READY,
                 taskScheduler,
                 amHealthProbeProperties.getPasswordGrant().getFreshnessInterval(),
                 amHealthProbeProperties.getPasswordGrant().getCheckInterval());

src/main/java/uk/gov/hmcts/reform/idam/health/probe/HealthProbeFailureHandling.java

@@ -1,5 +1,5 @@
 package uk.gov.hmcts.reform.idam.health.probe;
 
 public enum HealthProbeFailureHandling {
-    MARK_AS_DOWN, IGNORE
+    MARK_AS_DOWN, IGNORE, IGNORE_ONCE_READY
 }

src/main/java/uk/gov/hmcts/reform/idam/health/probe/ScheduledHealthProbeIndicator.java

@@ -70,7 +70,7 @@ protected void refresh() {
 
             this.status = newStatus;
             this.statusDateTime = LocalDateTime.now(clock);
-        } else if (failureHandling == HealthProbeFailureHandling.IGNORE) {
+        } else {
             log.warn("{}: DOWN state ignored", this.healthProbe.getName());
         }
     }

src/test/java/uk/gov/hmcts/reform/idam/health/probe/ScheduledHealthProbeIndicatorTest.java

@@ -36,16 +36,20 @@ public class ScheduledHealthProbeIndicatorTest {
 
     private ScheduledHealthProbeIndicator strictScheduledHealthProbe;
     private ScheduledHealthProbeIndicator ignoringScheduledHealthProbe;
+    private ScheduledHealthProbeIndicator ignoringOnceReadyScheduledHealthProbe;
 
     @Before
     public void setup() {
         strictScheduledHealthProbe = new ScheduledHealthProbeIndicator(
                 healthProbe, HealthProbeFailureHandling.MARK_AS_DOWN, taskScheduler, 40000L, 30000L);
         ignoringScheduledHealthProbe = new ScheduledHealthProbeIndicator(
                 healthProbe, HealthProbeFailureHandling.IGNORE, taskScheduler, 40000L, 30000L);
+        ignoringOnceReadyScheduledHealthProbe = new ScheduledHealthProbeIndicator(
+                healthProbe, HealthProbeFailureHandling.IGNORE_ONCE_READY, taskScheduler, 40000L, 30000L);
+
 
         when(healthProbe.getName()).thenReturn("testprobe");
-        verify(taskScheduler, times(2)).scheduleWithFixedDelay(any(Runnable.class), anyLong());
+        verify(taskScheduler, times(3)).scheduleWithFixedDelay(any(Runnable.class), anyLong());
     }
 
     @Test
@@ -112,6 +116,18 @@ public void testIsOkay_ignoreDown() {
         verify(healthProbe, times(2)).probe();
     }
 
+    @Test
+    public void testIsOkay_ignoreDownForIgnoreOnceReady() {
+        when(healthProbe.probe()).thenReturn(true);
+        ignoringOnceReadyScheduledHealthProbe.changeClock(Clock.fixed(Instant.ofEpochSecond(EPOCH_1AM), ZoneId.systemDefault()));
+        ignoringOnceReadyScheduledHealthProbe.refresh();
+        when(healthProbe.probe()).thenReturn(false);
+        ignoringOnceReadyScheduledHealthProbe.changeClock(Clock.fixed(Instant.ofEpochSecond(EPOCH_1AM + 31), ZoneId.systemDefault()));
+        ignoringOnceReadyScheduledHealthProbe.refresh();
+        assertThat(ignoringOnceReadyScheduledHealthProbe.isOkay(), is(true));
+        verify(healthProbe, times(2)).probe();
+    }
+
     @Test
     public void testIsOkay_alwaysSuccessWhenIgnoreProbe() {
         when(healthProbe.probe()).thenReturn(false);
@@ -123,6 +139,17 @@ public void testIsOkay_alwaysSuccessWhenIgnoreProbe() {
         verify(healthProbe, times(3)).probe();
     }
 
+    @Test
+    public void testIsOkay_returnDownAtStartupForIgnoreOnceReady() {
+        when(healthProbe.probe()).thenReturn(false);
+        assertThat(ignoringOnceReadyScheduledHealthProbe.isOkay(), is(false));
+        when(healthProbe.probe()).thenReturn(true);
+        assertThat(ignoringOnceReadyScheduledHealthProbe.isOkay(), is(true));
+        ignoringOnceReadyScheduledHealthProbe.refresh();
+        assertThat(ignoringOnceReadyScheduledHealthProbe.isOkay(), is(true));
+        verify(healthProbe, times(3)).probe();
+    }
+
     @Test
     public void testIsOkay_successWhenNotFreshAndIgnore() {
         when(healthProbe.probe()).thenReturn(true);