diff --git a/operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java b/operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java index 57164921262..d259106dee6 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java +++ b/operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java @@ -746,7 +746,7 @@ private void addServerToMaps(Map serverHealthMap, */ class MakeRightDomainOperationImpl implements MakeRightDomainOperation { - private final DomainPresenceInfo liveInfo; + private DomainPresenceInfo liveInfo; private boolean explicitRecheck; private boolean deleting; private boolean willInterrupt; @@ -851,6 +851,22 @@ public void setInspectionRun() { inspectionRun = true; } + @Override + public void setLiveInfo(DomainPresenceInfo info) { + this.liveInfo = info; + } + + @Override + public void clear() { + this.liveInfo = null; + this.eventData = null; + this.explicitRecheck = false; + this.deleting = false; + this.willInterrupt = false; + this.inspectionRun = false; + } + + @Override public boolean wasInspectionRun() { return inspectionRun; diff --git a/operator/src/main/java/oracle/kubernetes/operator/JobWatcher.java b/operator/src/main/java/oracle/kubernetes/operator/JobWatcher.java index 250792f2552..31ef8b72902 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/JobWatcher.java +++ b/operator/src/main/java/oracle/kubernetes/operator/JobWatcher.java @@ -12,6 +12,7 @@ import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import javax.annotation.Nonnull; @@ -25,13 +26,16 @@ import io.kubernetes.client.util.Watchable; import oracle.kubernetes.operator.TuningParameters.WatchTuning; import oracle.kubernetes.operator.builders.WatchBuilder; +import oracle.kubernetes.operator.calls.CallResponse; import oracle.kubernetes.operator.helpers.CallBuilder; import oracle.kubernetes.operator.helpers.KubernetesUtils; import oracle.kubernetes.operator.helpers.ResponseStep; import oracle.kubernetes.operator.logging.LoggingFacade; import oracle.kubernetes.operator.logging.LoggingFactory; import oracle.kubernetes.operator.logging.MessageKeys; +import oracle.kubernetes.operator.steps.DefaultResponseStep; import oracle.kubernetes.operator.watcher.WatchListener; +import oracle.kubernetes.operator.work.NextAction; import oracle.kubernetes.operator.work.Packet; import oracle.kubernetes.operator.work.Step; import oracle.kubernetes.utils.SystemClock; @@ -243,6 +247,11 @@ boolean isReady(V1Job job) { return isComplete(job) || isFailed(job); } + @Override + boolean onReadNotFoundForCachedResource(V1Job cachedJob, boolean isNotFoundOnRead) { + return false; + } + // Ignore modified callbacks from different jobs (identified by having different creation times) or those // where the job is not yet ready. @Override @@ -302,6 +311,21 @@ Throwable createTerminationException(V1Job job) { void logWaiting(String name) { LOGGER.fine(MessageKeys.WAITING_FOR_JOB_READY, name); } + + @Override + protected DefaultResponseStep resumeIfReady(Callback callback) { + return new DefaultResponseStep<>(null) { + @Override + public NextAction onSuccess(Packet packet, CallResponse callResponse) { + if (isReady(callResponse.getResult()) || callback.didResumeFiber()) { + callback.proceedFromWait(callResponse.getResult()); + return doNext(packet); + } + return doDelay(createReadAndIfReadyCheckStep(callback), packet, + getWatchBackstopRecheckDelaySeconds(), TimeUnit.SECONDS); + } + }; + } } static class DeadlineExceededException extends Exception { diff --git a/operator/src/main/java/oracle/kubernetes/operator/MakeRightDomainOperation.java b/operator/src/main/java/oracle/kubernetes/operator/MakeRightDomainOperation.java index df2414966fe..1ddde62fb9a 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/MakeRightDomainOperation.java +++ b/operator/src/main/java/oracle/kubernetes/operator/MakeRightDomainOperation.java @@ -36,6 +36,10 @@ public interface MakeRightDomainOperation { void setInspectionRun(); + void setLiveInfo(DomainPresenceInfo info); + + void clear(); + boolean wasInspectionRun(); private static boolean wasInspectionRun(Packet packet) { diff --git a/operator/src/main/java/oracle/kubernetes/operator/PodWatcher.java b/operator/src/main/java/oracle/kubernetes/operator/PodWatcher.java index 9f34fd907b9..de5dc1881e4 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/PodWatcher.java +++ b/operator/src/main/java/oracle/kubernetes/operator/PodWatcher.java @@ -11,6 +11,7 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import javax.annotation.Nonnull; @@ -28,7 +29,9 @@ import io.kubernetes.client.util.Watchable; import oracle.kubernetes.operator.TuningParameters.WatchTuning; import oracle.kubernetes.operator.builders.WatchBuilder; +import oracle.kubernetes.operator.calls.CallResponse; import oracle.kubernetes.operator.helpers.CallBuilder; +import oracle.kubernetes.operator.helpers.DomainPresenceInfo; import oracle.kubernetes.operator.helpers.KubernetesUtils; import oracle.kubernetes.operator.helpers.LegalNames; import oracle.kubernetes.operator.helpers.PodHelper; @@ -36,9 +39,16 @@ import oracle.kubernetes.operator.logging.LoggingFacade; import oracle.kubernetes.operator.logging.LoggingFactory; import oracle.kubernetes.operator.logging.MessageKeys; +import oracle.kubernetes.operator.steps.DefaultResponseStep; import oracle.kubernetes.operator.watcher.WatchListener; +import oracle.kubernetes.operator.work.NextAction; +import oracle.kubernetes.operator.work.Packet; import oracle.kubernetes.operator.work.Step; +import static oracle.kubernetes.operator.ProcessingConstants.SERVER_NAME; +import static oracle.kubernetes.operator.logging.MessageKeys.EXECUTE_MAKE_RIGHT_DOMAIN; +import static oracle.kubernetes.operator.logging.MessageKeys.LOG_WAITING_COUNT; + /** * Watches for changes to pods. */ @@ -305,6 +315,8 @@ public Step waitForDelete(V1Pod pod, Step next) { private abstract static class WaitForPodStatusStep extends WaitForReadyStep { + public static final int RECHECK_DEBUG_COUNT = 10; + private WaitForPodStatusStep(V1Pod pod, Step next) { super(pod, next); } @@ -322,6 +334,67 @@ V1ObjectMeta getMetadata(V1Pod pod) { Step createReadAsyncStep(String name, String namespace, String domainUid, ResponseStep responseStep) { return new CallBuilder().readPodAsync(name, namespace, domainUid, responseStep); } + + protected DefaultResponseStep resumeIfReady(Callback callback) { + return new DefaultResponseStep<>(getNext()) { + @Override + public NextAction onSuccess(Packet packet, CallResponse callResponse) { + + DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class); + String serverName = (String)packet.get(SERVER_NAME); + String resource = initialResource == null ? resourceName : getMetadata(initialResource).getName(); + if ((info != null) && (callResponse != null)) { + Optional.ofNullable(callResponse.getResult()).ifPresent(result -> + info.setServerPodFromEvent(getPodLabel(result), result)); + if (onReadNotFoundForCachedResource(getServerPod(info, serverName), isNotFoundOnRead(callResponse))) { + LOGGER.fine(EXECUTE_MAKE_RIGHT_DOMAIN, serverName, callback.getRecheckCount()); + removeCallback(resource, callback); + return doNext(NEXT_STEP_FACTORY.createMakeDomainRightStep(callback, info, getNext()), packet); + } + } + + if (isReady(callResponse.getResult()) || callback.didResumeFiber()) { + callback.proceedFromWait(callResponse.getResult()); + return null; + } + + if (shouldWait()) { + if ((callback.getRecheckCount() % RECHECK_DEBUG_COUNT) == 0) { + LOGGER.fine(LOG_WAITING_COUNT, serverName, callback.getRecheckCount()); + } + // Watch backstop recheck count is less than or equal to the configured recheck count, delay. + return doDelay(createReadAndIfReadyCheckStep(callback), packet, + getWatchBackstopRecheckDelaySeconds(), TimeUnit.SECONDS); + } else { + LOGGER.fine(EXECUTE_MAKE_RIGHT_DOMAIN, serverName, callback.getRecheckCount()); + removeCallback(resource, callback); + // Watch backstop recheck count is more than configured recheck count, proceed to make-right step. + return doNext(NEXT_STEP_FACTORY.createMakeDomainRightStep(callback, info, getNext()), packet); + } + } + + private String getPodLabel(V1Pod pod) { + return Optional.ofNullable(pod) + .map(V1Pod::getMetadata) + .map(V1ObjectMeta::getLabels) + .map(m -> m.get(LabelConstants.SERVERNAME_LABEL)) + .orElse(null); + } + + private V1Pod getServerPod(DomainPresenceInfo info, String serverName) { + return Optional.ofNullable(serverName).map(info::getServerPod).orElse(null); + } + + private boolean isNotFoundOnRead(CallResponse callResponse) { + return callResponse.getResult() == null; + } + + private boolean shouldWait() { + return callback.incrementAndGetRecheckCount() <= getWatchBackstopRecheckCount(); + } + }; + } + } private class WaitForPodReadyStep extends WaitForPodStatusStep { @@ -360,6 +433,13 @@ protected void removeCallback(String podName, Consumer callback) { protected void logWaiting(String name) { LOGGER.fine(MessageKeys.WAITING_FOR_POD_READY, name); } + + @Override + protected boolean onReadNotFoundForCachedResource(V1Pod cachedPod, boolean isNotFoundOnRead) { + // Return true if cached pod is not null but pod not found in explicit read, false otherwise. + return (cachedPod != null) && isNotFoundOnRead; + } + } private class WaitForPodDeleteStep extends WaitForPodStatusStep { @@ -367,6 +447,11 @@ private WaitForPodDeleteStep(V1Pod pod, Step next) { super(pod, next); } + @Override + protected boolean onReadNotFoundForCachedResource(V1Pod cachedPod, boolean isNotFoundOnRead) { + return false; + } + // A pod is considered deleted when reading its value from Kubernetes returns null. @Override protected boolean isReady(V1Pod result) { diff --git a/operator/src/main/java/oracle/kubernetes/operator/TuningParameters.java b/operator/src/main/java/oracle/kubernetes/operator/TuningParameters.java index 041f59a1f35..445a97a44bd 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/TuningParameters.java +++ b/operator/src/main/java/oracle/kubernetes/operator/TuningParameters.java @@ -194,6 +194,7 @@ class WatchTuning { public final int watchLifetime; public final int watchMinimumDelay; public final int watchBackstopRecheckDelay; + public final int watchBackstopRecheckCount; /** * Create watch tuning. @@ -201,10 +202,12 @@ class WatchTuning { * @param watchMinimumDelay Minimum delay before accepting new events to prevent hot loops * @param watchBackstopRecheckDelay Recheck delay for get while waiting for a status to backstop missed watch events */ - public WatchTuning(int watchLifetime, int watchMinimumDelay, int watchBackstopRecheckDelay) { + public WatchTuning(int watchLifetime, int watchMinimumDelay, int watchBackstopRecheckDelay, + int watchBackstopRecheckCount) { this.watchLifetime = watchLifetime; this.watchMinimumDelay = watchMinimumDelay; this.watchBackstopRecheckDelay = watchBackstopRecheckDelay; + this.watchBackstopRecheckCount = watchBackstopRecheckCount; } @Override diff --git a/operator/src/main/java/oracle/kubernetes/operator/TuningParametersImpl.java b/operator/src/main/java/oracle/kubernetes/operator/TuningParametersImpl.java index cdfe665d9d3..749c32f69f7 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/TuningParametersImpl.java +++ b/operator/src/main/java/oracle/kubernetes/operator/TuningParametersImpl.java @@ -75,7 +75,8 @@ private void update() { new WatchTuning( (int) readTuningParameter("watchLifetime", 300), (int) readTuningParameter("watchMinimumDelay", 5), - (int) readTuningParameter("watchBackstopRecheckDelaySeconds", 5)); + (int) readTuningParameter("watchBackstopRecheckDelaySeconds", 5), + (int) readTuningParameter("watchBackstopRecheckCount", 60)); PodTuning pod = new PodTuning( diff --git a/operator/src/main/java/oracle/kubernetes/operator/WaitForReadyStep.java b/operator/src/main/java/oracle/kubernetes/operator/WaitForReadyStep.java index ce82cd3529b..392028e5b52 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/WaitForReadyStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/WaitForReadyStep.java @@ -4,21 +4,25 @@ package oracle.kubernetes.operator; import java.util.Optional; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import io.kubernetes.client.openapi.models.V1ObjectMeta; -import io.kubernetes.client.openapi.models.V1Pod; import oracle.kubernetes.operator.calls.CallResponse; +import oracle.kubernetes.operator.helpers.CallBuilder; import oracle.kubernetes.operator.helpers.DomainPresenceInfo; import oracle.kubernetes.operator.helpers.ResponseStep; +import oracle.kubernetes.operator.logging.LoggingFacade; +import oracle.kubernetes.operator.logging.LoggingFactory; import oracle.kubernetes.operator.steps.DefaultResponseStep; import oracle.kubernetes.operator.work.AsyncFiber; import oracle.kubernetes.operator.work.NextAction; import oracle.kubernetes.operator.work.Packet; import oracle.kubernetes.operator.work.Step; +import oracle.kubernetes.weblogic.domain.model.Domain; +import static oracle.kubernetes.operator.ProcessingConstants.MAKE_RIGHT_DOMAIN_OPERATION; import static oracle.kubernetes.operator.helpers.KubernetesUtils.getDomainUidLabel; /** @@ -27,7 +31,18 @@ * @param the type of resource handled by this step */ abstract class WaitForReadyStep extends Step { + private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); private static final int DEFAULT_RECHECK_SECONDS = 5; + private static final int DEFAULT_RECHECK_COUNT = 60; + + static NextStepFactory NEXT_STEP_FACTORY = + (callback, info, next) -> createMakeDomainRightStep(callback, info, next); + + protected static Step createMakeDomainRightStep(WaitForReadyStep.Callback callback, + DomainPresenceInfo info, Step next) { + return new CallBuilder().readDomainAsync(info.getDomainUid(), + info.getNamespace(), new MakeRightDomainStep(callback, null)); + } static int getWatchBackstopRecheckDelaySeconds() { return Optional.ofNullable(TuningParameters.getInstance()) @@ -36,8 +51,15 @@ static int getWatchBackstopRecheckDelaySeconds() { .orElse(DEFAULT_RECHECK_SECONDS); } - private final T initialResource; - private final String resourceName; + static int getWatchBackstopRecheckCount() { + return Optional.ofNullable(TuningParameters.getInstance()) + .map(TuningParameters::getWatchTuning) + .map(t -> t.watchBackstopRecheckCount) + .orElse(DEFAULT_RECHECK_COUNT); + } + + final T initialResource; + final String resourceName; /** * Creates a step which will only proceed once the specified resource is ready. @@ -61,6 +83,15 @@ static int getWatchBackstopRecheckDelaySeconds() { */ abstract boolean isReady(T resource); + /** + * Returns true if the cached resource is not found during periodic listing. + * @param cachedResource cached resource to check + * @param isNotFoundOnRead Boolean indicating if resource is not found in call response. + * + * @return true if cached resource not found on read + */ + abstract boolean onReadNotFoundForCachedResource(T cachedResource, boolean isNotFoundOnRead); + /** * Returns true if the callback for this resource should be processed. This is typically used to exclude * resources which have changed but are not yet ready, or else different instances with the same name. @@ -173,14 +204,16 @@ private void checkUpdatedResource(Packet packet, AsyncFiber fiber, Callback call null); } - private Step createReadAndIfReadyCheckStep(Callback callback) { + Step createReadAndIfReadyCheckStep(Callback callback) { if (initialResource != null) { return createReadAsyncStep(getName(), getNamespace(), getDomainUid(), resumeIfReady(callback)); } else { - return new ReadAndIfReadyCheckStep(getName(), callback, getNext()); + return new ReadAndIfReadyCheckStep(getName(), resumeIfReady(callback), getNext()); } } + protected abstract ResponseStep resumeIfReady(Callback callback); + private String getNamespace() { return getMetadata(initialResource).getNamespace(); } @@ -193,56 +226,55 @@ public String getName() { return initialResource != null ? getMetadata(initialResource).getName() : resourceName; } - private DefaultResponseStep resumeIfReady(Callback callback) { - return new DefaultResponseStep<>(null) { - @Override - public NextAction onSuccess(Packet packet, CallResponse callResponse) { - if ((callResponse != null) && (callResponse.getResult() instanceof V1Pod)) { - V1Pod pod = (V1Pod) callResponse.getResult(); - Optional.ofNullable(packet.getSpi(DomainPresenceInfo.class)) - .ifPresent(i -> i.setServerPodFromEvent(getPodLabel(pod, LabelConstants.SERVERNAME_LABEL), pod)); - } - if (isReady(callResponse.getResult())) { - callback.proceedFromWait(callResponse.getResult()); - return doNext(packet); - } - return doDelay(createReadAndIfReadyCheckStep(callback), packet, - getWatchBackstopRecheckDelaySeconds(), TimeUnit.SECONDS); - } - - private String getPodLabel(V1Pod pod, String labelName) { - return Optional.ofNullable(pod) - .map(V1Pod::getMetadata) - .map(V1ObjectMeta::getLabels) - .map(m -> m.get(labelName)) - .orElse(null); - } - }; - } private class ReadAndIfReadyCheckStep extends Step { - private final Callback callback; private final String resourceName; + private final ResponseStep responseStep; - ReadAndIfReadyCheckStep(String resourceName, Callback callback, Step next) { + ReadAndIfReadyCheckStep(String resourceName, ResponseStep responseStep, Step next) { super(next); - this.callback = callback; this.resourceName = resourceName; + this.responseStep = responseStep; } @Override public NextAction apply(Packet packet) { DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class); return doNext(createReadAsyncStep(resourceName, info.getNamespace(), - info.getDomainUid(), resumeIfReady(callback)), packet); + info.getDomainUid(), responseStep), packet); + } + + } + + static class MakeRightDomainStep extends DefaultResponseStep { + public static final String WAIT_TIMEOUT_EXCEEDED = "Wait timeout exceeded"; + private final WaitForReadyStep.Callback callback; + + MakeRightDomainStep(WaitForReadyStep.Callback callback, Step next) { + super(next); + this.callback = callback; + } + + @Override + public NextAction onSuccess(Packet packet, CallResponse callResponse) { + MakeRightDomainOperation makeRightDomainOperation = + (MakeRightDomainOperation)packet.get(MAKE_RIGHT_DOMAIN_OPERATION); + if (makeRightDomainOperation != null) { + makeRightDomainOperation.clear(); + makeRightDomainOperation.setLiveInfo(new DomainPresenceInfo((Domain) callResponse.getResult())); + makeRightDomainOperation.withExplicitRecheck().interrupt().execute(); + } + callback.fiber.terminate(new Exception(WAIT_TIMEOUT_EXCEEDED), packet); + return super.onSuccess(packet, callResponse); } } - private class Callback implements Consumer { + class Callback implements Consumer { private final AsyncFiber fiber; private final Packet packet; private final AtomicBoolean didResume = new AtomicBoolean(false); + private final AtomicInteger recheckCount = new AtomicInteger(0); Callback(AsyncFiber fiber, Packet packet) { this.fiber = fiber; @@ -258,7 +290,7 @@ public void accept(T resource) { } // The resource has now either completed or failed, so we can continue processing. - private void proceedFromWait(T resource) { + void proceedFromWait(T resource) { removeCallback(getName(), this); if (mayResumeFiber()) { handleResourceReady(fiber, packet, resource); @@ -271,6 +303,18 @@ private void proceedFromWait(T resource) { private boolean mayResumeFiber() { return didResume.compareAndSet(false, true); } + + boolean didResumeFiber() { + return didResume.get(); + } + + int incrementAndGetRecheckCount() { + return recheckCount.incrementAndGet(); + } + + int getRecheckCount() { + return recheckCount.get(); + } } private void handleResourceReady(AsyncFiber fiber, Packet packet, T resource) { @@ -279,4 +323,11 @@ private void handleResourceReady(AsyncFiber fiber, Packet packet, T resource) { fiber.terminate(createTerminationException(resource), packet); } } + + // an interface to provide a hook for unit testing. + interface NextStepFactory { + Step createMakeDomainRightStep(WaitForReadyStep.Callback callback, + DomainPresenceInfo info, Step next); + } + } diff --git a/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java b/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java index b73ee66d0c7..4049ed381ef 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java +++ b/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java @@ -140,6 +140,8 @@ public class MessageKeys { public static final String HTTP_REQUEST_GOT_THROWABLE = "WLSKO-0189"; public static final String DOMAIN_ROLL_STARTING = "WLSKO-0190"; public static final String DOMAIN_ROLL_COMPLETED = "WLSKO-0191"; + public static final String EXECUTE_MAKE_RIGHT_DOMAIN = "WLSKO-0192"; + public static final String LOG_WAITING_COUNT = "WLSKO-0193"; // domain status messages public static final String DUPLICATE_SERVER_NAME_FOUND = "WLSDO-0001"; diff --git a/operator/src/main/resources/Operator.properties b/operator/src/main/resources/Operator.properties index 87d432e3c77..6d1d85acc8c 100644 --- a/operator/src/main/resources/Operator.properties +++ b/operator/src/main/resources/Operator.properties @@ -139,6 +139,8 @@ WLSKO-0187=Stop managing namespace {0} WLSKO-0189=HTTP request method {0} to {1} failed with exception {2}. WLSKO-0190=Rolling restart WebLogic server pods in domain {0} because {1} WLSKO-0191=Rolling restart of domain {0} completed +WLSKO-0192=Executing make right domain operation, recheck count for server {0} is {1}. +WLSKO-0193=Waiting for server {0} to start, recheck count is {1}. # Domain status messages diff --git a/operator/src/test/java/oracle/kubernetes/operator/PodWatcherTest.java b/operator/src/test/java/oracle/kubernetes/operator/PodWatcherTest.java index 42e5a915251..fb5aeb53d7c 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/PodWatcherTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/PodWatcherTest.java @@ -34,7 +34,9 @@ import static oracle.kubernetes.operator.LabelConstants.CREATEDBYOPERATOR_LABEL; import static oracle.kubernetes.operator.LabelConstants.DOMAINUID_LABEL; import static oracle.kubernetes.operator.helpers.LegalNames.DEFAULT_INTROSPECTOR_JOB_NAME_SUFFIX; +import static oracle.kubernetes.operator.logging.MessageKeys.EXECUTE_MAKE_RIGHT_DOMAIN; import static oracle.kubernetes.operator.logging.MessageKeys.INTROSPECTOR_POD_FAILED; +import static oracle.kubernetes.utils.LogMatcher.containsFine; import static oracle.kubernetes.utils.LogMatcher.containsInfo; import static org.hamcrest.Matchers.both; import static org.hamcrest.Matchers.hasEntry; @@ -69,7 +71,8 @@ public void setUp() throws Exception { private String[] getMessageKeys() { return new String[] { - getPodFailedMessageKey() + getPodFailedMessageKey(), + getMakeRightDomainStepKey() }; } @@ -77,6 +80,10 @@ private String getPodFailedMessageKey() { return INTROSPECTOR_POD_FAILED; } + private String getMakeRightDomainStepKey() { + return EXECUTE_MAKE_RIGHT_DOMAIN; + } + @Override public void receivedResponse(Watch.Response response) { recordCallBack(response); @@ -245,6 +252,16 @@ public void whenPodCreatedAndReadyLater_runNextStep() { assertThat(terminalStep.wasRun(), is(true)); } + @Test + public void whenPodCreatedAndNotReadyAfterTimeout_executeMakeRightDomain() { + executeWaitForReady(); + + testSupport.setTime(10, TimeUnit.SECONDS); + + assertThat(terminalStep.wasRun(), is(true)); + assertThat(logRecords, containsFine(getMakeRightDomainStepKey())); + } + @Test public void whenPodNotReadyLater_dontRunNextStep() { sendPodModifiedWatchAfterWaitForReady(this::dontChangePod); @@ -318,6 +335,19 @@ private void sendPodModifiedWatchAfterResourceCreatedAndWaitForReady(Function... modifiers) { diff --git a/operator/src/test/java/oracle/kubernetes/operator/WatcherTestBase.java b/operator/src/test/java/oracle/kubernetes/operator/WatcherTestBase.java index 16f11f7660a..a19ea254be4 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/WatcherTestBase.java +++ b/operator/src/test/java/oracle/kubernetes/operator/WatcherTestBase.java @@ -9,11 +9,15 @@ import java.util.concurrent.atomic.AtomicBoolean; import com.meterware.simplestub.Memento; +import com.meterware.simplestub.StaticStubSupport; import io.kubernetes.client.openapi.models.V1ObjectMeta; import io.kubernetes.client.util.Watch; import oracle.kubernetes.operator.TuningParameters.WatchTuning; import oracle.kubernetes.operator.builders.StubWatchFactory; import oracle.kubernetes.operator.builders.WatchEvent; +import oracle.kubernetes.operator.helpers.DomainPresenceInfo; +import oracle.kubernetes.operator.helpers.TuningParametersStub; +import oracle.kubernetes.operator.work.Step; import oracle.kubernetes.utils.TestUtils; import oracle.kubernetes.utils.TestUtils.ConsoleHandlerMemento; import org.junit.jupiter.api.AfterEach; @@ -40,7 +44,7 @@ public abstract class WatcherTestBase extends ThreadFactoryTestBase implements A private final List mementos = new ArrayList<>(); private final List> callBacks = new ArrayList<>(); private final AtomicBoolean stopping = new AtomicBoolean(false); - final WatchTuning tuning = new WatchTuning(30, 0, 5); + final WatchTuning tuning = new WatchTuning(30, 0, 5, 24); private BigInteger resourceVersion = INITIAL_RESOURCE_VERSION; private V1ObjectMeta createMetaData() { @@ -80,8 +84,10 @@ protected ConsoleHandlerMemento configureOperatorLogger() { return TestUtils.silenceOperatorLogger().ignoringLoggedExceptions(hasNextException); } - final void addMemento(Memento memento) { + final void addMemento(Memento memento) throws NoSuchFieldException { mementos.add(memento); + mementos.add(TuningParametersStub.install()); + mementos.add(TestStepFactory.install()); } @AfterEach @@ -253,4 +259,19 @@ private Watcher createAndRunWatcher(String nameSpace, AtomicBoolean stopping, } protected abstract Watcher createWatcher(String ns, AtomicBoolean stopping, BigInteger rv); + + static class TestStepFactory implements WaitForReadyStep.NextStepFactory { + + private static TestStepFactory factory = new TestStepFactory(); + + private static Memento install() throws NoSuchFieldException { + return StaticStubSupport.install(WaitForReadyStep.class, "NEXT_STEP_FACTORY", factory); + } + + @Override + public Step createMakeDomainRightStep(WaitForReadyStep.Callback callback, + DomainPresenceInfo info, Step next) { + return next; + } + } } diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/TuningParametersStub.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/TuningParametersStub.java index 3cddec8c6dc..e40bb720b4c 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/TuningParametersStub.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/TuningParametersStub.java @@ -79,7 +79,7 @@ public CallBuilderTuning getCallBuilderTuning() { @Override public WatchTuning getWatchTuning() { - return null; + return new TuningParameters.WatchTuning(30, 0, 5, 1); } @Override diff --git a/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStepTest.java b/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStepTest.java index c2ba266c0e5..73e4b1ef8df 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStepTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStepTest.java @@ -95,7 +95,7 @@ public class ManagedServerUpIteratorStepTest extends ThreadFactoryTestBase imple private final AtomicBoolean stopping = new AtomicBoolean(false); private static final BigInteger INITIAL_RESOURCE_VERSION = new BigInteger("234"); private final PodWatcher watcher = createWatcher(NS, stopping, INITIAL_RESOURCE_VERSION); - final TuningParameters.WatchTuning tuning = new TuningParameters.WatchTuning(30, 0, 5); + final TuningParameters.WatchTuning tuning = new TuningParameters.WatchTuning(30, 0, 5, 24); @Nonnull private static String getManagedServerName(int n) {