From ad87657776859d1bed3d1ddf9d4fbf6c13859760 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Wed, 12 May 2021 15:57:41 -0400 Subject: [PATCH 01/10] add scopes to compatibility check logic, and add three roll events and log messages --- .../kubernetes/operator/EventConstants.java | 6 + .../operator/ProcessingConstants.java | 1 + .../helpers/CollectiveCompatibility.java | 23 +- .../operator/helpers/CompatibilityCheck.java | 64 +++ .../operator/helpers/CompatibleMaps.java | 106 +++- .../operator/helpers/CompatibleSets.java | 16 +- .../kubernetes/operator/helpers/Equality.java | 23 +- .../operator/helpers/EventHelper.java | 76 ++- .../operator/helpers/PodCompatibility.java | 107 +++- .../operator/helpers/PodHelper.java | 4 +- .../operator/helpers/PodStepContext.java | 65 ++- .../operator/helpers/ResponseStep.java | 2 +- .../operator/helpers/RollingHelper.java | 46 +- .../operator/logging/MessageKeys.java | 2 + .../steps/ManagedServerUpAfterStep.java | 4 +- .../oracle/kubernetes/operator/work/Step.java | 4 + .../src/main/resources/Operator.properties | 2 + .../kubernetes/operator/EventTestUtils.java | 2 +- .../operator/helpers/AdminPodHelperTest.java | 46 ++ .../operator/helpers/EventHelperTest.java | 60 +++ .../helpers/PodCompatibilityTest.java | 464 ++++++++++++++++++ .../operator/helpers/PodHelperTestBase.java | 228 +++++++++ .../operator/helpers/RollingHelperTest.java | 152 +++++- 23 files changed, 1454 insertions(+), 49 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/EventConstants.java b/operator/src/main/java/oracle/kubernetes/operator/EventConstants.java index d19acad09dd..08900a676c8 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/EventConstants.java +++ b/operator/src/main/java/oracle/kubernetes/operator/EventConstants.java @@ -14,7 +14,10 @@ public interface EventConstants { String DOMAIN_PROCESSING_FAILED_EVENT = "DomainProcessingFailed"; String DOMAIN_PROCESSING_RETRYING_EVENT = "DomainProcessingRetrying"; String DOMAIN_PROCESSING_ABORTED_EVENT = "DomainProcessingAborted"; + String DOMAIN_ROLL_COMPLETED_EVENT = "DomainRollCompleted"; + String DOMAIN_ROLL_STARTING_EVENT = "DomainRollStarting"; String DOMAIN_VALIDATION_ERROR_EVENT = "DomainValidationError"; + String POD_CYCLE_STARTING_EVENT = "PodCycleStarting"; String EVENT_NORMAL = "Normal"; String EVENT_WARNING = "Warning"; String WEBLOGIC_OPERATOR_COMPONENT = "weblogic.operator"; @@ -34,6 +37,7 @@ public interface EventConstants { = "Aborting the processing of domain resource %s permanently due to: %s"; String DOMAIN_VALIDATION_ERROR_PATTERN = "Validation error in domain resource %s: %s"; + String POD_CYCLE_STARTING_PATTERN = "Replacing pod %s because: %s"; String NAMESPACE_WATCHING_STARTED_EVENT = "NamespaceWatchingStarted"; String NAMESPACE_WATCHING_STARTED_PATTERN = "Started watching namespace %s"; String NAMESPACE_WATCHING_STOPPED_EVENT = "NamespaceWatchingStopped"; @@ -47,4 +51,6 @@ public interface EventConstants { String STOP_MANAGING_NAMESPACE_PATTERN = "Stop managing namespace %s"; String START_MANAGING_NAMESPACE_FAILED_EVENT = "StartManagingNamespaceFailed"; String START_MANAGING_NAMESPACE_FAILED_PATTERN = "Start managing namespace %s failed due to an authorization error"; + String DOMAIN_ROLL_STARTING_PATTERN = "Rolling restart WebLogic server pods in domain %s because: %s"; + String DOMAIN_ROLL_COMPLETED_PATTERN = "Rolling restart of domain %s completed"; } diff --git a/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java b/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java index c16d8f49cdb..4b605d55486 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java +++ b/operator/src/main/java/oracle/kubernetes/operator/ProcessingConstants.java @@ -35,6 +35,7 @@ public interface ProcessingConstants { String MII_DYNAMIC_UPDATE_WDTROLLBACKFILE = "miiDynamicUpdateRollbackFile"; String MII_DYNAMIC_UPDATE_SUCCESS = "0"; String MII_DYNAMIC_UPDATE_RESTART_REQUIRED = "103"; + String DOMAIN_ROLL_START_EVENT_GENERATED = "domainRollStartEventGenerated"; String DOMAIN_VALIDATION_ERRORS = "domainValidationErrors"; String INTROSPECTOR_JOB_FAILURE_LOGGED = "introspectorJobFailureLogged"; diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CollectiveCompatibility.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CollectiveCompatibility.java index a002e4fac63..46dbabb3e8f 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CollectiveCompatibility.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CollectiveCompatibility.java @@ -41,9 +41,30 @@ public String getIncompatibility() { reasons.add(getIndent() + check.getIncompatibility()); } } - return reasons.isEmpty() ? null : getHeader() + String.join("\n", reasons); + return reasons.isEmpty() ? null : getHeader() + String.join(",\n", reasons); } + @Override + public String getScopedIncompatibility(CompatibilityScope scope) { + final List reasons = new ArrayList<>(); + for (CompatibilityCheck check : checks) { + if (!check.isCompatible() && check.getScopedIncompatibility(scope) != null) { + reasons.add(getIndent() + check.getScopedIncompatibility(scope)); + } + } + return reasons.isEmpty() + ? null + : scope == CompatibilityScope.DOMAIN + ? String.join(",\n", reasons) + : getHeader() + String.join(",\n", reasons); + } + + @Override + public CompatibilityScope getScope() { + return CompatibilityScope.MINIMUM; + } + + void addSets(String description, List expected, List actual) { add(CheckFactory.create(description, expected, actual)); } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibilityCheck.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibilityCheck.java index 633da53dc33..1a2fe2fe49e 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibilityCheck.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibilityCheck.java @@ -4,10 +4,74 @@ package oracle.kubernetes.operator.helpers; interface CompatibilityCheck { + enum CompatibilityScope { + DOMAIN { + @Override + public boolean contains(CompatibilityScope scope) { + switch (scope) { + case DOMAIN: + case MINIMUM: + return true; + case POD: + case UNKNOWN: + default: + return false; + } + } + }, + POD { + @Override + public boolean contains(CompatibilityScope scope) { + switch (scope) { + case DOMAIN: + case POD: + case UNKNOWN: + case MINIMUM: + return true; + default: + return false; + + } + } + }, + UNKNOWN { + @Override + public boolean contains(CompatibilityScope scope) { + switch (scope) { + case DOMAIN: + case POD: + default: + return false; + case UNKNOWN: + case MINIMUM: + return true; + } + } + }, + MINIMUM { + @Override + public boolean contains(CompatibilityScope scope) { + switch (scope) { + case DOMAIN: + case POD: + case UNKNOWN: + default: + return false; + } + } + }; + + public abstract boolean contains(CompatibilityScope scope); + } + boolean isCompatible(); String getIncompatibility(); + String getScopedIncompatibility(CompatibilityScope scope); + + CompatibilityScope getScope(); + default CompatibilityCheck ignoring(String... keys) { return this; } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleMaps.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleMaps.java index fd676e78ad2..5ecace6eb3c 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleMaps.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleMaps.java @@ -5,14 +5,31 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import io.kubernetes.client.openapi.models.V1EnvVar; + import static oracle.kubernetes.operator.helpers.PodCompatibility.getMissingElements; +import static oracle.kubernetes.weblogic.domain.model.ServerEnvVars.DOMAIN_HOME; +import static oracle.kubernetes.weblogic.domain.model.ServerEnvVars.LOG_HOME; +import static oracle.kubernetes.weblogic.domain.model.ServerEnvVars.SERVER_OUT_IN_POD_LOG; class CompatibleMaps implements CompatibilityCheck { + private static final List DOMAIN_FIELDS = Collections.singletonList("env"); + private static final List DOMAIN_ENV_KEYS = Arrays.asList(LOG_HOME, SERVER_OUT_IN_POD_LOG, DOMAIN_HOME); + private static final HashMap ELEMENT_NAMES_MAP = new HashMap<>(); + + static { + ELEMENT_NAMES_MAP.put(LOG_HOME, "logHome"); + ELEMENT_NAMES_MAP.put(SERVER_OUT_IN_POD_LOG, "isIncludeServerOutInPodLog"); + ELEMENT_NAMES_MAP.put(DOMAIN_HOME, "domainHome"); + } + private final String description; private final Map expected; private final Map actual; @@ -52,19 +69,102 @@ public String getIncompatibility() { Set missingKeys = getMissingElements(expected.keySet(), actual.keySet()); if (!missingKeys.isEmpty()) { - sb.append(String.format("actual %s has no entry for '%s'%n", description, missingKeys)); + sb.append(String.format("%s changed and contains '%s' as well%n", description, missingKeys)); } for (K key : expected.keySet()) { if (isKeyToCheck(key) && actual.containsKey(key) && valuesDiffer(key)) { sb.append( String.format( - "actual %s has entry '%s' with value '%s' rather than '%s'%n", + "%s '%s' changed from '%s' to '%s'%n", + description, key, getValue(actual.get(key)), getValue(expected.get(key)))); + } + } + + return sb.length() == 0 ? null : sb.toString(); + } + + private String getDomainIncompatibility() { + StringBuilder sb = new StringBuilder(); + + Set missingKeys = getMissingElements(expected.keySet(), actual.keySet()); + if (!missingKeys.isEmpty()) { + sb.append(String.format("%s changed and contains '%s' as well%n", description, missingKeys)); + } + + for (K key : expected.keySet()) { + if (isKeyToCheck(key) && actual.containsKey(key) && valuesDiffer(key) && isDomainKey(key)) { + sb.append( + String.format( + "'%s' changed from '%s' to '%s'%n", + ELEMENT_NAMES_MAP.get(key), getValue(actual.get(key)), getValue(expected.get(key)))); + } + } + + return sb.length() == 0 ? null : sb.toString(); + } + + private Object getValue(Object obj) { + if (obj instanceof V1EnvVar) { + return ((V1EnvVar) obj).getValue(); + } + return obj; + } + + private String getUnknownIncompatibility() { + StringBuilder sb = new StringBuilder(); + + + Set missingKeys = getMissingElements(expected.keySet(), actual.keySet()); + if (!missingKeys.isEmpty()) { + sb.append(String.format("%s changed and contains '%s' as well%n", description, missingKeys)); + } + + for (K key : expected.keySet()) { + if (isKeyToCheck(key) && actual.containsKey(key) && valuesDiffer(key) && !isDomainKey(key)) { + sb.append( + String.format( + "%s '%s' changed from '%s' to '%s'%n", description, key, actual.get(key), expected.get(key))); } } - return sb.toString(); + return sb.length() == 0 ? null : sb.toString(); + } + + private boolean isDomainKey(K key) { + return DOMAIN_ENV_KEYS.contains(key); + } + + private boolean containsDomainKeys(Set missingKeys) { + for (K key : missingKeys) { + if (isDomainKey(key)) { + return true; + } + } + return false; + } + + @Override + public String getScopedIncompatibility(CompatibilityScope scope) { + switch (scope) { + case DOMAIN: + return getDomainIncompatibility(); + case UNKNOWN: + return getUnknownIncompatibility(); + case POD: + return getIncompatibility(); + default: + return null; + } + } + + @Override + public CompatibilityScope getScope() { + if (DOMAIN_FIELDS.contains(description)) { + return CompatibilityScope.MINIMUM; + } + return CompatibilityScope.UNKNOWN; } @Override diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleSets.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleSets.java index 341d22bb1a5..7b1a9d34f22 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleSets.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleSets.java @@ -30,6 +30,20 @@ public boolean isCompatible() { @Override public String getIncompatibility() { return String.format( - "actual %s does not have %s", description, getMissingElements(expected, actual)); + "%s contains additional elements %s", description, getMissingElements(expected, actual)); } + + @Override + public String getScopedIncompatibility(CompatibilityScope scope) { + if (scope.contains(getScope())) { + return getIncompatibility(); + } + return null; + } + + @Override + public CompatibilityScope getScope() { + return CompatibilityScope.UNKNOWN; + } + } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/Equality.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/Equality.java index 7015aec5c4c..4f9342ea356 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/Equality.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/Equality.java @@ -3,9 +3,13 @@ package oracle.kubernetes.operator.helpers; +import java.util.Arrays; +import java.util.List; import java.util.Objects; class Equality implements CompatibilityCheck { + private static final List DOMAIN_FIELDS = Arrays.asList("image", "imagePullPolicy"); + private final String description; private final Object expected; private final Object actual; @@ -23,6 +27,23 @@ public boolean isCompatible() { @Override public String getIncompatibility() { - return description + " expected: " + expected + " but was: " + actual; + return description + " changed from " + "'" + actual + "'" + " to " + "'" + expected + "'"; + } + + @Override + public String getScopedIncompatibility(CompatibilityScope scope) { + if (scope.contains(getScope())) { + return getIncompatibility(); + } + return null; } + + @Override + public CompatibilityScope getScope() { + if (DOMAIN_FIELDS.contains(description)) { + return CompatibilityScope.DOMAIN; + } + return CompatibilityScope.POD; + } + } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/EventHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/EventHelper.java index 9cba245dadf..7153c6bd4f3 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/EventHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/EventHelper.java @@ -44,12 +44,15 @@ import static oracle.kubernetes.operator.EventConstants.DOMAIN_PROCESSING_RETRYING_PATTERN; import static oracle.kubernetes.operator.EventConstants.DOMAIN_PROCESSING_STARTING_EVENT; import static oracle.kubernetes.operator.EventConstants.DOMAIN_PROCESSING_STARTING_PATTERN; +import static oracle.kubernetes.operator.EventConstants.DOMAIN_ROLL_STARTING_EVENT; import static oracle.kubernetes.operator.EventConstants.DOMAIN_VALIDATION_ERROR_EVENT; import static oracle.kubernetes.operator.EventConstants.DOMAIN_VALIDATION_ERROR_PATTERN; import static oracle.kubernetes.operator.EventConstants.EVENT_NORMAL; import static oracle.kubernetes.operator.EventConstants.EVENT_WARNING; import static oracle.kubernetes.operator.EventConstants.NAMESPACE_WATCHING_STARTED_PATTERN; import static oracle.kubernetes.operator.EventConstants.NAMESPACE_WATCHING_STOPPED_EVENT; +import static oracle.kubernetes.operator.EventConstants.POD_CYCLE_STARTING_EVENT; +import static oracle.kubernetes.operator.EventConstants.POD_CYCLE_STARTING_PATTERN; import static oracle.kubernetes.operator.EventConstants.WEBLOGIC_OPERATOR_COMPONENT; import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_PROCESSING_ABORTED; import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_PROCESSING_COMPLETED; @@ -223,10 +226,6 @@ private void clearNamespaceStartingFlag() { domainNamespaces.clearNamespaceStartingFlag(eventData.getNamespace()); } } - - private boolean isForbiddenForNamespaceWatchingStoppedEvent(CallResponse callResponse) { - return isForbidden(callResponse) && NAMESPACE_WATCHING_STOPPED == eventData.eventItem; - } } private class ReplaceEventResponseStep extends ResponseStep { @@ -281,12 +280,12 @@ private Step createEventRefreshStep(CoreV1Event event) { } private boolean isForbiddenForNSWatchStoppedEvent( - ResponseStep responseStep, CallResponse callResponse) { + ResponseStep responseStep, CallResponse callResponse) { return responseStep.isForbidden(callResponse) && NAMESPACE_WATCHING_STOPPED == eventData.eventItem; } private boolean hasLoggedForbiddenNSWatchStoppedEvent( - ResponseStep responseStep, CallResponse callResponse) { + ResponseStep responseStep, CallResponse callResponse) { if (isForbiddenForNSWatchStoppedEvent(responseStep, callResponse)) { LOGGER.info(MessageKeys.CREATING_EVENT_FORBIDDEN, eventData.eventItem.getReason(), eventData.getNamespace()); return true; @@ -453,6 +452,35 @@ public String getMessage(EventData eventData) { return getMessageFromEventData(eventData); } + }, + DOMAIN_ROLL_STARTING { + @Override + public String getReason() { + return DOMAIN_ROLL_STARTING_EVENT; + } + + @Override + public String getPattern() { + return EventConstants.DOMAIN_ROLL_STARTING_PATTERN; + } + + @Override + public String getMessage(EventData eventData) { + return getMessageFromEventData(eventData); + } + + }, + DOMAIN_ROLL_COMPLETED { + @Override + public String getReason() { + return EventConstants.DOMAIN_ROLL_COMPLETED_EVENT; + } + + @Override + public String getPattern() { + return EventConstants.DOMAIN_ROLL_COMPLETED_PATTERN; + } + }, DOMAIN_VALIDATION_ERROR { @Override @@ -475,6 +503,27 @@ public String getMessage(EventData eventData) { return getMessageFromEventData(eventData); } }, + POD_CYCLE_STARTING { + @Override + protected String getType() { + return EVENT_NORMAL; + } + + @Override + public String getReason() { + return POD_CYCLE_STARTING_EVENT; + } + + @Override + public String getPattern() { + return POD_CYCLE_STARTING_PATTERN; + } + + @Override + public String getMessage(EventData eventData) { + return getMessageFromEventDataWithPod(eventData); + } + }, NAMESPACE_WATCHING_STARTED { @Override public String getReason() { @@ -606,6 +655,11 @@ private static String getMessageFromEventData(EventData eventData) { eventData.getResourceNameFromInfo(), Optional.ofNullable(eventData.message).orElse("")); } + private static String getMessageFromEventDataWithPod(EventData eventData) { + return String.format(eventData.eventItem.getPattern(), + eventData.getPodName(), Optional.ofNullable(eventData.message).orElse("")); + } + private static void addCreatedByOperatorLabel(V1ObjectMeta metadata) { metadata.putLabelsItem(LabelConstants.CREATEDBYOPERATOR_LABEL, "true"); } @@ -680,6 +734,7 @@ public static class EventData { private String message; private String namespace; private String resourceName; + private String podName; private DomainPresenceInfo info; public EventData(EventItem eventItem) { @@ -706,6 +761,11 @@ public EventData resourceName(String resourceName) { return this; } + public EventData podName(String podName) { + this.podName = podName; + return this; + } + public EventData domainPresenceInfo(DomainPresenceInfo info) { this.info = info; return this; @@ -720,6 +780,10 @@ public String getNamespace() { .map(DomainPresenceInfo::getNamespace).orElse("")); } + public String getPodName() { + return podName; + } + public String getResourceName() { return Optional.ofNullable(resourceName).orElse(this.getResourceNameFromInfo()); } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodCompatibility.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodCompatibility.java index 73aad33bc2c..a9bdfd0cafe 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodCompatibility.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodCompatibility.java @@ -29,7 +29,6 @@ /** A class which defines the compatibility rules for existing vs. specified pods. */ class PodCompatibility extends CollectiveCompatibility { PodCompatibility(V1Pod expected, V1Pod actual) { - add("sha256Hash", AnnotationHelper.getHash(expected), AnnotationHelper.getHash(actual)); add(new PodMetadataCompatibility(expected.getMetadata(), actual.getMetadata())); add(new PodSpecCompatibility(Objects.requireNonNull(expected.getSpec()), Objects.requireNonNull(actual.getSpec()))); } @@ -70,23 +69,56 @@ && isLabelSame(CLUSTERRESTARTVERSION_LABEL) private boolean isLabelSame(String labelName) { return Objects.equals( - Objects.requireNonNull(expected.getLabels()).get(labelName), - Objects.requireNonNull(actual.getLabels()).get(labelName) + getExpected(labelName), + getActual(labelName) ); } + private String getExpected(String labelName) { + return Objects.requireNonNull(expected.getLabels()).get(labelName); + } + + private String getActual(String labelName) { + return Objects.requireNonNull(actual.getLabels()).get(labelName); + } + @Override public String getIncompatibility() { if (!isLabelSame(DOMAINRESTARTVERSION_LABEL)) { - return "domain restart label changed."; + return "domain restart version changed from '" + getActual(DOMAINRESTARTVERSION_LABEL) + + "' to '" + getExpected(DOMAINRESTARTVERSION_LABEL) + "'"; } else if (!isLabelSame(CLUSTERRESTARTVERSION_LABEL)) { - return "cluster restart label changed."; + return "cluster restart version changed from '" + getActual(CLUSTERRESTARTVERSION_LABEL) + + "' to '" + getExpected(CLUSTERRESTARTVERSION_LABEL) + "'"; } else if (!isLabelSame(SERVERRESTARTVERSION_LABEL)) { - return "server restart label changed."; + return "server restart version changed from '" + getActual(SERVERRESTARTVERSION_LABEL) + + "' to '" + getExpected(SERVERRESTARTVERSION_LABEL) + "'"; } else { return null; } } + + @Override + public String getScopedIncompatibility(CompatibilityScope scope) { + if (scope.contains(getScope())) { + return getIncompatibility(); + } + return null; + } + + @Override + public CompatibilityScope getScope() { + if (!isLabelSame(DOMAINRESTARTVERSION_LABEL)) { + return CompatibilityScope.DOMAIN; + } else if (!isLabelSame(CLUSTERRESTARTVERSION_LABEL)) { + return CompatibilityScope.DOMAIN; + } else if (!isLabelSame(SERVERRESTARTVERSION_LABEL)) { + return CompatibilityScope.POD; + } else { + return CompatibilityScope.UNKNOWN; + } + } + } static class PodSpecCompatibility extends CollectiveCompatibility { @@ -111,13 +143,13 @@ private void addContainerChecks( if (actual.containsKey(name)) { containerChecks.add(createCompatibilityCheck(expected.get(name), actual.get(name))); } else { - containerChecks.add(new Mismatch("Expected container '%s' not found", name)); + containerChecks.add(new Mismatch("additional container '%s' added", name)); } } for (String name : actual.keySet()) { if (!expected.containsKey(name)) { - containerChecks.add(new Mismatch("Found unexpected container '%s'", name)); + containerChecks.add(new Mismatch("container '%s' removed", name)); } } @@ -191,6 +223,19 @@ public boolean isCompatible() { public String getIncompatibility() { return errorMessage; } + + @Override + public String getScopedIncompatibility(CompatibilityScope scope) { + if (scope.contains(getScope())) { + return getIncompatibility(); + } + return null; + } + + @Override + public CompatibilityScope getScope() { + return CompatibilityScope.UNKNOWN; + } } static class Probes implements CompatibilityCheck { @@ -215,15 +260,28 @@ public boolean isCompatible() { @Override public String getIncompatibility() { return String.format( - "Expected %s probe with initial delay %d, timeout %d and period %d \n" - + " but found initial delay %d, timeout %d and period %d.", + "%s probe changed from " + + "'initial delay %d, timeout %d and period %d' to 'initial delay %d, timeout %d and period %d'\n", description, - expected.getInitialDelaySeconds(), - expected.getTimeoutSeconds(), - expected.getPeriodSeconds(), actual.getInitialDelaySeconds(), actual.getTimeoutSeconds(), - actual.getPeriodSeconds()); + actual.getPeriodSeconds(), + expected.getInitialDelaySeconds(), + expected.getTimeoutSeconds(), + expected.getPeriodSeconds()); + } + + @Override + public String getScopedIncompatibility(CompatibilityScope scope) { + if (scope.contains(getScope())) { + return getIncompatibility(); + } + return null; + } + + @Override + public CompatibilityScope getScope() { + return CompatibilityScope.DOMAIN; } } @@ -256,16 +314,29 @@ public String getIncompatibility() { if (!KubernetesUtils.mapEquals(getLimits(expected), getLimits(actual))) { sb.append( String.format( - "Expected resource limits: %s but found %s", - getLimits(expected), getLimits(actual))); + "resource limits changed from '%s' to '%s'", + getLimits(actual), getLimits(expected))); } if (!KubernetesUtils.mapEquals(getRequests(expected), getRequests(actual))) { sb.append( String.format( - "Expected resource requests: %s but found %s", + "resource requests changed from '%s' to '%s'", getRequests(expected), getRequests(actual))); } - return sb.toString(); + return sb.length() == 0 ? null : sb.toString(); + } + + @Override + public String getScopedIncompatibility(CompatibilityScope scope) { + if (scope.contains(getScope())) { + return getIncompatibility(); + } + return null; + } + + @Override + public CompatibilityScope getScope() { + return CompatibilityScope.UNKNOWN; } } } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java index 265d1c1064b..9a530822f84 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java @@ -323,7 +323,7 @@ Step replaceCurrentPod(V1Pod pod, Step next) { if (MakeRightDomainOperation.isInspectionRequired(packet)) { return createProgressingStep(MakeRightDomainOperation.createStepsToRerunWithIntrospection(packet)); } else { - return createProgressingStep(createCyclePodStep(pod, next)); + return createProgressingStep(createDomainRollStartEventIfNeeded(pod, createCyclePodStep(pod, next))); } } @@ -463,7 +463,7 @@ String getServerName() { @Override // let the pod rolling step update the pod Step replaceCurrentPod(V1Pod pod, Step next) { - return deferProcessing(createCyclePodStep(pod, next)); + return createDomainRollStartEventIfNeeded(pod, deferProcessing(createCyclePodStep(pod, next))); } private Step deferProcessing(Step deferredStep) { diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java index c7bcbf27b31..2056ecd673f 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java @@ -50,6 +50,8 @@ import oracle.kubernetes.operator.WebLogicConstants; import oracle.kubernetes.operator.calls.CallResponse; import oracle.kubernetes.operator.calls.UnrecoverableErrorBuilder; +import oracle.kubernetes.operator.helpers.CompatibilityCheck.CompatibilityScope; +import oracle.kubernetes.operator.helpers.EventHelper.EventData; import oracle.kubernetes.operator.logging.LoggingFacade; import oracle.kubernetes.operator.logging.LoggingFactory; import oracle.kubernetes.operator.logging.MessageKeys; @@ -72,9 +74,14 @@ import static oracle.kubernetes.operator.LabelConstants.MII_UPDATED_RESTART_REQUIRED_LABEL; import static oracle.kubernetes.operator.LabelConstants.MODEL_IN_IMAGE_DOMAINZIP_HASH; import static oracle.kubernetes.operator.LabelConstants.OPERATOR_VERSION; +import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_ROLL_START_EVENT_GENERATED; import static oracle.kubernetes.operator.ProcessingConstants.MII_DYNAMIC_UPDATE; import static oracle.kubernetes.operator.ProcessingConstants.MII_DYNAMIC_UPDATE_SUCCESS; import static oracle.kubernetes.operator.helpers.AnnotationHelper.SHA256_ANNOTATION; +import static oracle.kubernetes.operator.helpers.CompatibilityCheck.CompatibilityScope.DOMAIN; +import static oracle.kubernetes.operator.helpers.CompatibilityCheck.CompatibilityScope.UNKNOWN; +import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_ROLL_STARTING; +import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.POD_CYCLE_STARTING; public abstract class PodStepContext extends BasePodStepContext { @@ -450,6 +457,40 @@ private void logPodReplaced() { LOGGER.info(getPodReplacedMessageKey(), getDomainUid(), getServerName()); } + protected Step createDomainRollStartEventIfNeeded(V1Pod pod, Step next) { + if ("true".equals(packet.getValue(DOMAIN_ROLL_START_EVENT_GENERATED))) { + return next; + } + + String domainIncompatibility = getReasonToRecycle(pod, DOMAIN); + if (hasChangesInExpectedScope(domainIncompatibility)) { + return createDomainRollStartEvent(next, domainIncompatibility); + } + + if (hasChangesInExpectedScope(getReasonToRecycle(pod, UNKNOWN))) { + return createDomainRollStartEvent(next, "domain resource changed"); + } + + if (!canUseNewDomainZip(pod)) { + return createDomainRollStartEvent(next, "WebLogic domain configuration changed"); + } + + return next; + } + + private Step createDomainRollStartEvent(Step next, String domainIncompatibility) { + LOGGER.info(MessageKeys.DOMAIN_ROLL_STARTING, getDomainUid(), domainIncompatibility); + packet.put(DOMAIN_ROLL_START_EVENT_GENERATED, "true"); + return Step.chain( + EventHelper.createEventStep( + new EventData(DOMAIN_ROLL_STARTING, domainIncompatibility.trim())), + next); + } + + private boolean hasChangesInExpectedScope(String domainIncompatibility) { + return domainIncompatibility != null && domainIncompatibility.length() != 0; + } + abstract String getPodCreatedMessageKey(); abstract String getPodExistsMessageKey(); @@ -546,7 +587,7 @@ private V1Pod withLegacyDomainHash(V1Pod pod, String oldDomainHash) { return pod; } - private boolean canUseNewDomainZip(V1Pod currentPod) { + protected boolean canUseNewDomainZip(V1Pod currentPod) { String dynamicUpdateResult = packet.getValue(MII_DYNAMIC_UPDATE); if (miiDomainZipHash == null || isDomainZipUnchanged(currentPod)) { @@ -567,9 +608,9 @@ private boolean isDomainZipUnchanged(V1Pod currentPod) { return formatHashLabel(miiDomainZipHash).equals(getLabel(currentPod, MODEL_IN_IMAGE_DOMAINZIP_HASH)); } - private String getReasonToRecycle(V1Pod currentPod) { + protected String getReasonToRecycle(V1Pod currentPod, CompatibilityScope scope) { PodCompatibility compatibility = new PodCompatibility(getPodModel(), currentPod); - return compatibility.getIncompatibility(); + return compatibility.getScopedIncompatibility(scope); } private ResponseStep createResponse(Step next) { @@ -966,7 +1007,7 @@ private Step getConflictStep() { } } - private class CyclePodStep extends BaseStep { + public class CyclePodStep extends BaseStep { private final V1Pod pod; CyclePodStep(V1Pod pod, Step next) { @@ -976,8 +1017,18 @@ private class CyclePodStep extends BaseStep { @Override public NextAction apply(Packet packet) { + String reason = getReasonToRecycle(pod, CompatibilityScope.POD); + LOGGER.info( + MessageKeys.CYCLING_POD, + Objects.requireNonNull(pod.getMetadata()).getName(), + reason); markBeingDeleted(); - return doNext(deletePod(pod, getNext()), packet); + return doNext(createCyclePodEventStep(reason, deletePod(pod, getNext())), packet); + } + + private Step createCyclePodEventStep(String reason, Step next) { + return Step.chain(EventHelper.createEventStep(new EventData(POD_CYCLE_STARTING, reason).podName(getPodName())), + next); } } @@ -999,10 +1050,6 @@ public NextAction apply(Packet packet) { if (currentPod == null) { return doNext(createNewPod(getNext()), packet); } else if (!canUseCurrentPod(currentPod)) { - LOGGER.info( - MessageKeys.CYCLING_POD, - Objects.requireNonNull(currentPod.getMetadata()).getName(), - getReasonToRecycle(currentPod)); return doNext(replaceCurrentPod(currentPod, getNext()), packet); } else if (mustPatchPod(currentPod)) { return doNext(patchCurrentPod(currentPod, getNext()), packet); diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/ResponseStep.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/ResponseStep.java index a6ac21ab84a..1ca0b4d9cd8 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/ResponseStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/ResponseStep.java @@ -117,7 +117,7 @@ protected final NextAction doContinueListOrNext(CallResponse callResponse, Pa */ protected final NextAction doContinueListOrNext(CallResponse callResponse, Packet packet, Step next) { String cont = accessContinue(callResponse.getResult()); - if (callResponse != null && cont != null) { + if (cont != null) { packet.put(CONTINUE, cont); // Since the continue value is present, invoking the original request will return // the next window of data. diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/RollingHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/RollingHelper.java index 819ac543285..53186ff0c44 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/RollingHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/RollingHelper.java @@ -28,6 +28,8 @@ import oracle.kubernetes.utils.OperatorUtils; import oracle.kubernetes.weblogic.domain.model.Domain; +import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_ROLL_START_EVENT_GENERATED; + /** * After the {@link PodHelper} identifies servers that are presently running, but that are using an * out-of-date specification, it defers the processing of these servers to the RollingHelper. This @@ -134,7 +136,6 @@ public NextAction apply(Packet packet) { } if (!clusteredRestarts.isEmpty()) { - LOGGER.fine("Restarting server " + packet.get(ProcessingConstants.SERVER_NAME)); for (Map.Entry> entry : clusteredRestarts.entrySet()) { work.add( new StepAndPacket( @@ -143,11 +144,48 @@ public NextAction apply(Packet packet) { } if (!work.isEmpty()) { - return doForkJoin(getNext(), packet, work); + return doForkJoin(createAfterRollStep(getNext()), packet, work); } - return doNext(packet); + return doNext(createAfterRollStep(getNext()), packet); + } + + private Step createAfterRollStep(Step next) { + return new AfterRollStep(next); + } + } + + private static class AfterRollStep extends Step { + public AfterRollStep(Step next) { + super(next); + } + + @Override + public NextAction apply(Packet packet) { + return doNext(createDomainRollCompletedEventStepIfNeeded(getNext(), packet), packet); + } + + } + + /** + * Create DOMAIN_ROLL_COMPLETED event if a roll started earlier. + * + * @param next next step + * @param packet packet to use + * @return step chain + */ + public static Step createDomainRollCompletedEventStepIfNeeded(Step next, Packet packet) { + if ("true".equals(packet.remove(DOMAIN_ROLL_START_EVENT_GENERATED))) { + LOGGER.info(MessageKeys.DOMAIN_ROLL_COMPLETED, getDomainUid(packet)); + return Step.chain( + EventHelper.createEventStep(new EventHelper.EventData(EventHelper.EventItem.DOMAIN_ROLL_COMPLETED)), + next); } + return next; + } + + private static String getDomainUid(Packet packet) { + return packet.getSpi(DomainPresenceInfo.class).getDomainUid(); } private static class ServersThatCanRestartNowStep extends Step { @@ -209,7 +247,7 @@ public NextAction apply(Packet packet) { } } - LOGGER.info(MessageKeys.ROLLING_SERVERS, dom.getDomainUid(), servers, readyServers); + LOGGER.info(MessageKeys.ROLLING_SERVERS, dom.getDomainUid(), servers.toString(), readyServers); int countToRestartNow = countReady - dom.getMinAvailable(clusterName); Collection restarts = new ArrayList<>(); diff --git a/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java b/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java index 95b26cdd78e..f479ad8efd1 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java +++ b/operator/src/main/java/oracle/kubernetes/operator/logging/MessageKeys.java @@ -140,6 +140,8 @@ public class MessageKeys { public static final String END_MANAGING_NAMESPACE = "WLSKO-0187"; public static final String MII_DOMAIN_DYNAMICALLY_UPDATED = "WLSKO-0188"; public static final String HTTP_REQUEST_GOT_THROWABLE = "WLSKO-0189"; + public static final String DOMAIN_ROLL_STARTING = "WLSKO-0190"; + public static final String DOMAIN_ROLL_COMPLETED = "WLSKO-0191"; // domain status messages public static final String DUPLICATE_SERVER_NAME_FOUND = "WLSDO-0001"; diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpAfterStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpAfterStep.java index 3e8bb609a72..c324f96c5e5 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpAfterStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpAfterStep.java @@ -19,6 +19,8 @@ import oracle.kubernetes.operator.work.Packet; import oracle.kubernetes.operator.work.Step; +import static oracle.kubernetes.operator.helpers.RollingHelper.createDomainRollCompletedEventStepIfNeeded; + public class ManagedServerUpAfterStep extends Step { private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator"); @@ -29,7 +31,7 @@ public ManagedServerUpAfterStep(Step next) { @Override public NextAction apply(Packet packet) { if (getServersToRoll(packet).isEmpty()) { - return doNext(packet); + return doNext(createDomainRollCompletedEventStepIfNeeded(getNext(), packet), packet); } else if (MakeRightDomainOperation.isInspectionRequired(packet)) { return doNext(MakeRightDomainOperation.createStepsToRerunWithIntrospection(packet), packet); } else { diff --git a/operator/src/main/java/oracle/kubernetes/operator/work/Step.java b/operator/src/main/java/oracle/kubernetes/operator/work/Step.java index 9a463ce761b..a313d255a7a 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/work/Step.java +++ b/operator/src/main/java/oracle/kubernetes/operator/work/Step.java @@ -341,5 +341,9 @@ public StepAndPacket(Step step, Packet packet) { this.step = step; this.packet = packet; } + + public Step getStep() { + return step; + } } } diff --git a/operator/src/main/resources/Operator.properties b/operator/src/main/resources/Operator.properties index 07a83798927..dd4c72adb1d 100644 --- a/operator/src/main/resources/Operator.properties +++ b/operator/src/main/resources/Operator.properties @@ -139,6 +139,8 @@ WLSKO-0186=Start managing namespace {0} WLSKO-0187=Stop managing namespace {0} WLSKO-0188=Model in Image domain with DomainUID ''{0}'' and server name ''{1}'' online updated successfully. No restart is necessary WLSKO-0189=HTTP request method {0} to {1} failed with exception {2}. +WLSKO-0190=Rolling restart WebLogic server pods in domain {0} because {1} +WLSKO-0191=Rolling restart of domain {0} completed # Domain status messages diff --git a/operator/src/test/java/oracle/kubernetes/operator/EventTestUtils.java b/operator/src/test/java/oracle/kubernetes/operator/EventTestUtils.java index 46dda8c4440..62a4fca7f9a 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/EventTestUtils.java +++ b/operator/src/test/java/oracle/kubernetes/operator/EventTestUtils.java @@ -19,7 +19,7 @@ import static oracle.kubernetes.operator.EventConstants.WEBLOGIC_OPERATOR_COMPONENT; public class EventTestUtils { - private static List getEventsWithReason(@NotNull List events, String reason) { + public static List getEventsWithReason(@NotNull List events, String reason) { return events.stream().filter(event -> reasonMatches(event, reason)).collect(Collectors.toList()); } diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/AdminPodHelperTest.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/AdminPodHelperTest.java index 37d83840d55..5a7ed0cf750 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/AdminPodHelperTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/AdminPodHelperTest.java @@ -21,12 +21,14 @@ import oracle.kubernetes.operator.work.FiberTestSupport; import oracle.kubernetes.operator.work.Packet; import oracle.kubernetes.operator.work.Step; +import oracle.kubernetes.utils.TestUtils; import oracle.kubernetes.weblogic.domain.ServerConfigurator; import org.junit.jupiter.api.Test; import static oracle.kubernetes.operator.WebLogicConstants.ADMIN_STATE; import static oracle.kubernetes.operator.WebLogicConstants.RUNNING_STATE; import static oracle.kubernetes.operator.helpers.DomainStatusMatcher.hasStatus; +import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.POD_CYCLE_STARTING; import static oracle.kubernetes.operator.helpers.Matchers.hasContainer; import static oracle.kubernetes.operator.helpers.Matchers.hasEnvVar; import static oracle.kubernetes.operator.helpers.Matchers.hasInitContainer; @@ -51,6 +53,7 @@ import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.stringContainsInOrder; import static org.hamcrest.junit.MatcherAssert.assertThat; @SuppressWarnings("SameParameterValue") @@ -62,6 +65,8 @@ public class AdminPodHelperTest extends PodHelperTestBase { private static final String RAW_MOUNT_PATH_1 = "$(DOMAIN_HOME)/servers/$(SERVER_NAME)"; private static final String END_MOUNT_PATH_1 = "/u01/oracle/user_projects/domains/servers/ADMIN_SERVER"; + private TestUtils.ConsoleHandlerMemento consoleHandlerMemento = TestUtils.silenceOperatorLogger(); + public AdminPodHelperTest() { super(ADMIN_SERVER, ADMIN_PORT); } @@ -750,6 +755,47 @@ public void whenDomainAndAdminHasRestartVersion_createAdminPodWithRestartVersion assertThat(podLabels, hasKey(not(LabelConstants.CLUSTERRESTARTVERSION_LABEL))); } + + @Test + public void whenDomainHomeChanged_podCycleEventCreatedWithCorrectMessage() + throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { + initializeExistingPod(); + getConfiguredDomainSpec().setDomainHome("adfgg"); + + testSupport.runSteps(getStepFactory(), terminalStep); + logRecords.clear(); + + assertThat( + "Expected Event " + POD_CYCLE_STARTING + " expected with message not found", + getExpectedEventMessage(POD_CYCLE_STARTING), + stringContainsInOrder("Replacing ", getPodName(), "DOMAIN_HOME", "changed from", "adfgg")); + } + + @Test + public void whenDomainHomeChanged_podCycleEventCreatedWithCorrectNS() + throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { + initializeExistingPod(); + getConfiguredDomainSpec().setDomainHome("adfgg"); + + testSupport.runSteps(getStepFactory(), terminalStep); + logRecords.clear(); + + assertContainsEventWithNamespace(POD_CYCLE_STARTING, NS); + } + + @Test + public void whenDomainHomeChanged_generateExpectedLogMessage() + throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { + consoleHandlerMemento.collectLogMessages(logRecords, getCyclePodKey()); + initializeExistingPod(); + getConfiguredDomainSpec().setDomainHome("adfgg"); + + testSupport.runSteps(getStepFactory(), terminalStep); + + assertThat(logRecords, containsInfo(getReplacedMessageKey())); + assertThat(logRecords, containsInfo(getCyclePodKey())); + } + private V1Pod createTestPodModel() { return new V1Pod().metadata(createPodMetadata()).spec(createPodSpec()); } diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/EventHelperTest.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/EventHelperTest.java index 492138144b9..16e6f906a26 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/EventHelperTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/EventHelperTest.java @@ -53,8 +53,11 @@ import static oracle.kubernetes.operator.EventConstants.DOMAIN_PROCESSING_RETRYING_PATTERN; import static oracle.kubernetes.operator.EventConstants.DOMAIN_PROCESSING_STARTING_EVENT; import static oracle.kubernetes.operator.EventConstants.DOMAIN_PROCESSING_STARTING_PATTERN; +import static oracle.kubernetes.operator.EventConstants.DOMAIN_ROLL_COMPLETED_EVENT; +import static oracle.kubernetes.operator.EventConstants.DOMAIN_ROLL_STARTING_EVENT; import static oracle.kubernetes.operator.EventConstants.NAMESPACE_WATCHING_STARTED_EVENT; import static oracle.kubernetes.operator.EventConstants.NAMESPACE_WATCHING_STOPPED_EVENT; +import static oracle.kubernetes.operator.EventConstants.POD_CYCLE_STARTING_EVENT; import static oracle.kubernetes.operator.EventConstants.START_MANAGING_NAMESPACE_FAILED_EVENT; import static oracle.kubernetes.operator.EventConstants.STOP_MANAGING_NAMESPACE_EVENT; import static oracle.kubernetes.operator.EventTestUtils.containsEvent; @@ -79,8 +82,11 @@ import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_PROCESSING_FAILED; import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_PROCESSING_RETRYING; import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_PROCESSING_STARTING; +import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_ROLL_COMPLETED; +import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_ROLL_STARTING; import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.NAMESPACE_WATCHING_STARTED; import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.NAMESPACE_WATCHING_STOPPED; +import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.POD_CYCLE_STARTING; import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.START_MANAGING_NAMESPACE; import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.STOP_MANAGING_NAMESPACE; import static oracle.kubernetes.operator.helpers.EventHelper.createEventStep; @@ -836,6 +842,60 @@ public void whenNSWatchStoppedEventCreatedTwice_fail503OnReplace_eventCreatedOnc NAMESPACE_WATCHING_STOPPED_EVENT, 2), is(true)); } + @Test + public void whenDomainRollStartingEventCreateCalled_domainRollStartingEventCreatedWithExpectedCount() { + testSupport.runSteps(createEventStep(new EventData(DOMAIN_ROLL_STARTING))); + + assertThat("Found DOMAIN_ROLL_STARTING event with expected count", + containsOneEventWithCount(getEvents(testSupport), DOMAIN_ROLL_STARTING_EVENT, 1), is(true)); + } + + @Test + public void whenDomainRollStartingEventCreateCalled_domainRollStartingEventCreatedWithExpectedMessage() { + testSupport.runSteps(createEventStep(new EventData(DOMAIN_ROLL_STARTING).message("abcde"))); + + assertThat("Found DOMAIN_ROLL_STARTING event with expected message", + containsEventWithMessage(getEvents(testSupport), + DOMAIN_ROLL_STARTING_EVENT, + String.format(EventConstants.DOMAIN_ROLL_STARTING_PATTERN, UID, "abcde")), is(true)); + } + + @Test + public void whenDomainRollCompletedEventCreateCalled_domainRollCompletedEventCreatedWithExpectedCount() { + testSupport.runSteps(createEventStep(new EventData(DOMAIN_ROLL_COMPLETED))); + + assertThat("Found DOMAIN_ROLL_COMPLETED event with expected count", + containsOneEventWithCount(getEvents(testSupport), DOMAIN_ROLL_COMPLETED_EVENT, 1), is(true)); + } + + @Test + public void whenDomainRollCompletedEventCreateCalled_domainRollCompletedEventCreatedWithExpectedMessage() { + testSupport.runSteps(createEventStep(new EventData(DOMAIN_ROLL_COMPLETED))); + + assertThat("Found DOMAIN_ROLL_COMPLETED event with expected message", + containsEventWithMessage(getEvents(testSupport), + DOMAIN_ROLL_COMPLETED_EVENT, + String.format(EventConstants.DOMAIN_ROLL_COMPLETED_PATTERN, UID)), is(true)); + } + + @Test + public void whenPodCycleStartingEventCreateCalled_podCycleStartingEventCreatedWithExpectedCount() { + testSupport.runSteps(createEventStep(new EventData(POD_CYCLE_STARTING))); + + assertThat("Found POD_CYCLE_STARTING event with expected count", + containsOneEventWithCount(getEvents(testSupport), POD_CYCLE_STARTING_EVENT, 1), is(true)); + } + + @Test + public void whenPodCycleStartingEventCreateCalled_podCycleStartingEventCreatedWithExpectedMessage() { + testSupport.runSteps(createEventStep(new EventData(POD_CYCLE_STARTING).podName("12345").message("abcde"))); + + assertThat("Found POD_CYCLE_STARTING event with expected message", + containsEventWithMessage(getEvents(testSupport), + POD_CYCLE_STARTING_EVENT, + String.format(EventConstants.POD_CYCLE_STARTING_PATTERN, "12345", "abcde")), is(true)); + } + private void dispatchAddedEventWatches() { List events = getEvents(testSupport); for (CoreV1Event event : events) { diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/PodCompatibilityTest.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/PodCompatibilityTest.java index 924af88b62d..4d635e89bb1 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/PodCompatibilityTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/PodCompatibilityTest.java @@ -12,12 +12,20 @@ import io.kubernetes.client.openapi.models.V1Container; import io.kubernetes.client.openapi.models.V1ContainerPort; import io.kubernetes.client.openapi.models.V1EnvVar; +import io.kubernetes.client.openapi.models.V1ObjectMeta; import io.kubernetes.client.openapi.models.V1Probe; import io.kubernetes.client.openapi.models.V1ResourceRequirements; +import io.kubernetes.client.openapi.models.V1VolumeMount; +import oracle.kubernetes.operator.LabelConstants; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.junit.jupiter.api.Test; +import static oracle.kubernetes.operator.helpers.CompatibilityCheck.CompatibilityScope.DOMAIN; +import static oracle.kubernetes.operator.helpers.CompatibilityCheck.CompatibilityScope.POD; +import static oracle.kubernetes.operator.helpers.CompatibilityCheck.CompatibilityScope.UNKNOWN; import static oracle.kubernetes.operator.helpers.PodHelper.AdminPodStepContext.INTERNAL_OPERATOR_CERT_ENV; +import static oracle.kubernetes.weblogic.domain.model.ServerEnvVars.DOMAIN_HOME; +import static oracle.kubernetes.weblogic.domain.model.ServerEnvVars.LOG_HOME; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.blankOrNullString; import static org.hamcrest.Matchers.both; @@ -25,6 +33,7 @@ import static org.hamcrest.Matchers.emptyOrNullString; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.stringContainsInOrder; public class PodCompatibilityTest { @Test @@ -219,6 +228,461 @@ public void whenActualKeysHaveDifferentValues_reportMissingElements() { assertThat(check.getIncompatibility(), not(containsString("gamma"))); } + @Test + public void whenImagesDontMatch_createDomainAndPodScopeErrorMessage() { + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container().image("abcde"), new V1Container().image("cdefg")); + + assertThat( + compatibility.getScopedIncompatibility(DOMAIN), + both(containsString("abcde")).and(containsString("cdefg"))); + assertThat( + compatibility.getScopedIncompatibility(POD), + both(containsString("abcde")).and(containsString("cdefg"))); + } + + @Test + public void whenImagesDontMatch_dontCreateUnknownScopeErrorMessage() { + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container().image("abcde"), new V1Container().image("cdefg")); + + assertThat( + compatibility.getScopedIncompatibility(UNKNOWN), blankOrNullString()); + } + + + @Test + public void whenImagePullPoliciesDontMatch_createDomainAndPodScopeErrorMessage() { + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container().imagePullPolicy("abcde"), new V1Container().imagePullPolicy("cdefg")); + + assertThat( + compatibility.getScopedIncompatibility(DOMAIN), + both(containsString("abcde")).and(containsString("cdefg"))); + assertThat( + compatibility.getScopedIncompatibility(POD), + both(containsString("abcde")).and(containsString("cdefg"))); + } + + @Test + public void whenImagePullPoliciesDontMatch_dontCreateUnknownScopeErrorMessage() { + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container().imagePullPolicy("abcde"), new V1Container().imagePullPolicy("cdefg")); + + assertThat( + compatibility.getScopedIncompatibility(UNKNOWN), blankOrNullString()); + } + + + @Test + public void whenVolumeMountsDontMatch_createUnknownAndPodScopeErrorMessage() { + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container().name("c1").addVolumeMountsItem(new V1VolumeMount().name("V1").mountPath("abcde")), + new V1Container().name("c1").addVolumeMountsItem(new V1VolumeMount().name("V1").mountPath("cdefg"))); + + assertThat( + compatibility.getScopedIncompatibility(UNKNOWN), + stringContainsInOrder("c1", "volumeMount", "changed", "cdefg", "to", "abcde")); + assertThat( + compatibility.getScopedIncompatibility(POD), + stringContainsInOrder("c1", "volumeMount", "changed", "cdefg", "to", "abcde")); + } + + @Test + public void whenVolumeMountsDontMatch_dontCreateDomainScopeErrorMessage() { + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container().name("c1").addVolumeMountsItem(new V1VolumeMount().name("V1").mountPath("abcde")), + new V1Container().name("c1").addVolumeMountsItem(new V1VolumeMount().name("V1").mountPath("cdefg"))); + + assertThat( + compatibility.getScopedIncompatibility(DOMAIN), blankOrNullString()); + } + + @Test + public void whenRestartVersionsDontMatch_createDomainAndPodScopeErrorMessage() { + PodCompatibility.PodMetadataCompatibility compatibility = + new PodCompatibility.PodMetadataCompatibility( + new V1ObjectMeta().putLabelsItem(LabelConstants.DOMAINRESTARTVERSION_LABEL, "12345"), + new V1ObjectMeta().putLabelsItem(LabelConstants.DOMAINRESTARTVERSION_LABEL, "abcde")); + + assertThat( + compatibility.getScopedIncompatibility(DOMAIN), + stringContainsInOrder("domain restart version changed from", "abcde", "to", "12345")); + assertThat( + compatibility.getScopedIncompatibility(POD), + stringContainsInOrder("domain restart version changed from", "abcde", "to", "12345")); + } + + @Test + public void whenRestartVersionsDontMatch_dontCreateUnknownScopeErrorMessage() { + PodCompatibility.PodMetadataCompatibility compatibility = + new PodCompatibility.PodMetadataCompatibility( + new V1ObjectMeta().putLabelsItem(LabelConstants.DOMAINRESTARTVERSION_LABEL, "12345"), + new V1ObjectMeta().putLabelsItem(LabelConstants.DOMAINRESTARTVERSION_LABEL, "abcde")); + + assertThat( + compatibility.getScopedIncompatibility(UNKNOWN), blankOrNullString()); + } + + @Test + public void whenLivenessProbesDontMatch_createDomainAndPodScopeErrorMessage() { + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container() + .livenessProbe( + new V1Probe().initialDelaySeconds(1).timeoutSeconds(5).periodSeconds(3)), + new V1Container() + .livenessProbe( + new V1Probe().initialDelaySeconds(1).timeoutSeconds(2).periodSeconds(3))); + + assertThat( + compatibility.getScopedIncompatibility(DOMAIN), + both(containsString("timeout")).and(containsString("2"))); + assertThat( + compatibility.getScopedIncompatibility(POD), + both(containsString("timeout")).and(containsString("2"))); + } + + @Test + public void whenLivenessProbesDontMatch_dontCreateUnknownScopeErrorMessage() { + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container() + .livenessProbe( + new V1Probe().initialDelaySeconds(1).timeoutSeconds(5).periodSeconds(3)), + new V1Container() + .livenessProbe( + new V1Probe().initialDelaySeconds(1).timeoutSeconds(2).periodSeconds(3))); + + assertThat( + compatibility.getScopedIncompatibility(UNKNOWN), blankOrNullString()); + } + + @Test + public void whenResourcesDontMatch_dontCreateDomainScopeErrorMessage() { + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container() + .resources(new V1ResourceRequirements().putLimitsItem("time", new Quantity("20"))), + new V1Container()); + + assertThat( + compatibility.getScopedIncompatibility(DOMAIN), blankOrNullString()); + } + + @Test + public void whenResourcesDontMatch_createUnknownAndPodScopeScopeErrorMessage() { + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container() + .resources(new V1ResourceRequirements().putLimitsItem("time", new Quantity("20"))), + new V1Container()); + + assertThat( + compatibility.getScopedIncompatibility(UNKNOWN), + both(containsString("time")).and(containsString("limits"))); + assertThat( + compatibility.getScopedIncompatibility(POD), + both(containsString("time")).and(containsString("limits"))); + } + + @Test + public void whenPortsDontMatch_dontCreateDomainScopeErrorMessage() { + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container().addPortsItem(new V1ContainerPort().containerPort(1100)), + new V1Container().addPortsItem(new V1ContainerPort().containerPort(1234))); + + assertThat( + compatibility.getScopedIncompatibility(DOMAIN), blankOrNullString()); + } + + @Test + public void whenPortsDontMatch_createUnknownAndPodScopeErrorMessage() { + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container().addPortsItem(new V1ContainerPort().containerPort(1100)), + new V1Container().addPortsItem(new V1ContainerPort().containerPort(1234))); + + assertThat( + compatibility.getScopedIncompatibility(UNKNOWN), + both(containsString("1100")).and(not(containsString("1234")))); + assertThat( + compatibility.getScopedIncompatibility(POD), + both(containsString("1100")).and(not(containsString("1234")))); + } + + @Test + public void whenEnvVarsDontMatch_dontCreateDomainScopeErrorMessage() { + String name = "aa"; + String value = "bb"; + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container().addEnvItem(envVar(name, value)), + new V1Container().addEnvItem(envVar("aa", "cc"))); + + assertThat( + compatibility.getScopedIncompatibility(DOMAIN), blankOrNullString()); + } + + @Test + public void whenEnvVarsDontMatch_createUnknownAndPodScopeErrorMessage() { + String name = "aa"; + String value = "bb"; + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container().addEnvItem(envVar(name, value)), + new V1Container().addEnvItem(envVar("aa", "cc"))); + + assertThat( + compatibility.getScopedIncompatibility(UNKNOWN), both(containsString("aa")).and(containsString("bb"))); + assertThat( + compatibility.getScopedIncompatibility(POD), both(containsString("aa")).and(containsString("bb"))); + } + + @Test + public void whenDomainHomeEnvVarsDontMatch_createDomainAndPodScopeErrorMessage() { + String name = DOMAIN_HOME; + String value = "/myhome"; + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container().addEnvItem(envVar(name, value)), + new V1Container().addEnvItem(envVar(name, "cc"))); + + assertThat( + compatibility.getScopedIncompatibility(DOMAIN), both(containsString("domainHome")).and(containsString(value))); + assertThat( + compatibility.getScopedIncompatibility(POD), both(containsString(name)).and(containsString(value))); + } + + @Test + public void whenDomainHomeEnvVarsDontMatch_dontCreateUnknownScopeErrorMessage() { + String name = DOMAIN_HOME; + String value = "/myhome"; + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container().addEnvItem(envVar(name, value)), + new V1Container().addEnvItem(envVar(name, "cc"))); + + assertThat( + compatibility.getScopedIncompatibility(UNKNOWN), blankOrNullString()); + } + + @Test + public void whenLogHomeEnvVarsDontMatch_createDomainAndPodScopeErrorMessage() { + String name = LOG_HOME; + String value = "/myhome"; + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container().addEnvItem(envVar(name, value)), + new V1Container().addEnvItem(envVar(name, "cc"))); + + assertThat( + compatibility.getScopedIncompatibility(DOMAIN), both(containsString("logHome")).and(containsString(value))); + assertThat( + compatibility.getScopedIncompatibility(POD), both(containsString(name)).and(containsString(value))); + } + + @Test + public void whenLogHomeEnvVarsDontMatch_dontCreateUnknownScopeErrorMessage() { + String name = LOG_HOME; + String value = "/myhome"; + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container().addEnvItem(envVar(name, value)), + new V1Container().addEnvItem(envVar(name, "cc"))); + + assertThat( + compatibility.getScopedIncompatibility(UNKNOWN), blankOrNullString()); + } + + @Test + public void whenOnlyCertificateForEnvVarsDontMatch_dontCreateAnyScopeErrorMessage() { + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container().name("test").addEnvItem(envVar(INTERNAL_OPERATOR_CERT_ENV, "bb")), + new V1Container()); + + assertThat(compatibility.getScopedIncompatibility(DOMAIN), blankOrNullString()); + assertThat(compatibility.getScopedIncompatibility(POD), blankOrNullString()); + assertThat(compatibility.getScopedIncompatibility(UNKNOWN), blankOrNullString()); + } + + @Test + public void whenExpectedNotSubsetOfActual_dontReportDomainScopeMissingElements() { + CompatibilityCheck check = + new CompatibleSets<>( + "letters", + ImmutableSet.of("alpha", "beta", "delta"), + ImmutableSet.of("beta", "gamma", "alpha")); + + assertThat(check.getScopedIncompatibility(DOMAIN), blankOrNullString()); + } + + @Test + public void whenExpectedNotSubsetOfActual_reportUnknownAndPodScopeMissingElements() { + CompatibilityCheck check = + new CompatibleSets<>( + "letters", + ImmutableSet.of("alpha", "beta", "delta"), + ImmutableSet.of("beta", "gamma", "alpha")); + + assertThat(check.getScopedIncompatibility(UNKNOWN), containsString("delta")); + assertThat(check.getScopedIncompatibility(UNKNOWN), not(containsString("alpha"))); + assertThat(check.getScopedIncompatibility(UNKNOWN), not(containsString("gamma"))); + assertThat(check.getScopedIncompatibility(POD), containsString("delta")); + assertThat(check.getScopedIncompatibility(POD), not(containsString("alpha"))); + assertThat(check.getScopedIncompatibility(POD), not(containsString("gamma"))); + } + + @Test + public void whenCanBeMapsAndExpectedAndActualDifferentValues_dontReportDomainScopeChangedElements() { + CompatibilityCheck check = + CheckFactory.create( + "letters", + Arrays.asList(object("alpha", 1), object("beta", 2), object("gamma", 3)), + Arrays.asList(object("beta", 222), object("gamma", 3), object("alpha", 1))); + + assertThat(check.getScopedIncompatibility(DOMAIN), blankOrNullString()); + } + + @Test + public void whenCanBeMapsAndExpectedAndActualDifferentValues_reportUnknownScopeChangedElements() { + CompatibilityCheck check = + CheckFactory.create( + "letters", + Arrays.asList(object("alpha", 1), object("beta", 2), object("gamma", 3)), + Arrays.asList(object("beta", 222), object("gamma", 3), object("alpha", 1))); + + assertThat(check.getScopedIncompatibility(UNKNOWN), both(containsString("beta")).and(containsString("222"))); + assertThat(check.getScopedIncompatibility(UNKNOWN), not(containsString("alpha"))); + } + + @Test + public void ignoreAllScopeCertForComparisons() { + CompatibilityCheck check = + CheckFactory.create( + "envVars", + Arrays.asList(object("alpha", 1), object(INTERNAL_OPERATOR_CERT_ENV, 3)), + Arrays.asList(object(INTERNAL_OPERATOR_CERT_ENV, 700), object("alpha", 1))) + .ignoring(INTERNAL_OPERATOR_CERT_ENV); + + assertThat(check.getScopedIncompatibility(DOMAIN), emptyOrNullString()); + assertThat(check.getScopedIncompatibility(POD), emptyOrNullString()); + assertThat(check.getScopedIncompatibility(UNKNOWN), emptyOrNullString()); + } + + @Test + public void whenExpectedNotSubmapOfActual_reportDomainScopeMissingElements() { + CompatibilityCheck check = + new CompatibleMaps<>( + "letters", + ImmutableMap.of("alpha", 1, "beta", 2, "delta", 4), + ImmutableMap.of("beta", 2, "gamma", 3, "alpha", 1)); + + assertThat(check.getScopedIncompatibility(DOMAIN), containsString("delta")); + assertThat(check.getScopedIncompatibility(DOMAIN), not(containsString("alpha"))); + assertThat(check.getScopedIncompatibility(DOMAIN), not(containsString("gamma"))); + } + + @Test + public void whenExpectedNotSubmapOfActual_reportUnknownAndPodScopeMissingElements() { + CompatibilityCheck check = + new CompatibleMaps<>( + "letters", + ImmutableMap.of("alpha", 1, "beta", 2, "delta", 4), + ImmutableMap.of("beta", 2, "gamma", 3, "alpha", 1)); + + assertThat(check.getScopedIncompatibility(UNKNOWN), containsString("delta")); + assertThat(check.getScopedIncompatibility(UNKNOWN), not(containsString("alpha"))); + assertThat(check.getScopedIncompatibility(UNKNOWN), not(containsString("gamma"))); + assertThat(check.getScopedIncompatibility(POD), containsString("delta")); + assertThat(check.getScopedIncompatibility(POD), not(containsString("alpha"))); + assertThat(check.getScopedIncompatibility(POD), not(containsString("gamma"))); + } + + @Test + public void whenActualKeysHaveDifferentValues_dontReportDomainScopeAdditionalElements() { + CompatibilityCheck check = + new CompatibleMaps<>( + "letters", + ImmutableMap.of("alpha", 1, "beta", 2), + ImmutableMap.of("beta", 5, "gamma", 3, "alpha", 1)); + + assertThat(check.getScopedIncompatibility(DOMAIN), emptyOrNullString()); + } + + @Test + public void whenActualKeysHaveDifferentValues_reportUnknownAndPodScopeMissingElements() { + CompatibilityCheck check = + new CompatibleMaps<>( + "letters", + ImmutableMap.of("alpha", 1, "beta", 2), + ImmutableMap.of("beta", 5, "gamma", 3, "alpha", 1)); + + assertThat(check.getScopedIncompatibility(UNKNOWN), both(containsString("beta")).and(containsString("5"))); + assertThat(check.getScopedIncompatibility(UNKNOWN), not(containsString("alpha"))); + assertThat(check.getScopedIncompatibility(UNKNOWN), not(containsString("gamma"))); + assertThat(check.getScopedIncompatibility(POD), both(containsString("beta")).and(containsString("5"))); + assertThat(check.getScopedIncompatibility(POD), not(containsString("alpha"))); + assertThat(check.getScopedIncompatibility(POD), not(containsString("gamma"))); + } + + @Test + public void whenImagesDontMatch_andResourcesDontMatch_createDomainAScopeErrorMessageForImageOnly() { + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container().image("abcde") + .resources(new V1ResourceRequirements().putLimitsItem("time", new Quantity("20"))), + new V1Container().image("cdefg")); + + assertThat( + compatibility.getScopedIncompatibility(DOMAIN), + both(containsString("abcde")).and(containsString("cdefg"))); + assertThat( + compatibility.getScopedIncompatibility(POD), + stringContainsInOrder("resource limits changed from", "{}", "to", "time=Quantity")); + } + + @Test + public void whenImagesDontMatch_andResourcesDontMatch_createUnknownScopeErrorMessageForResourcesOnly() { + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container().image("abcde") + .resources(new V1ResourceRequirements().putLimitsItem("time", new Quantity("20"))), + new V1Container().image("cdefg")); + + assertThat( + compatibility.getScopedIncompatibility(UNKNOWN), + not(both(containsString("abcde")).and(containsString("cdefg")))); + assertThat( + compatibility.getScopedIncompatibility(POD), + both(containsString("time")).and(containsString("limits"))); + } + + @Test + public void whenImagesDontMatch_andResourcesDontMatch_createPodScopeErrorMessageForBoth() { + PodCompatibility.ContainerCompatibility compatibility = + new PodCompatibility.ContainerCompatibility( + new V1Container().image("abcde") + .resources(new V1ResourceRequirements().putLimitsItem("time", new Quantity("20"))), + new V1Container().image("cdefg")); + + assertThat( + compatibility.getScopedIncompatibility(POD), + both(containsString("abcde")).and(containsString("cdefg"))); + assertThat( + compatibility.getScopedIncompatibility(POD), + both(containsString("time")).and(containsString("limits"))); + } + + static class ObjectWithName { private final String name; private final int value; diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java index 274bad26310..2cc9f12c117 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java @@ -11,6 +11,9 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.LogRecord; @@ -18,6 +21,7 @@ import com.meterware.simplestub.Memento; import com.meterware.simplestub.StaticStubSupport; import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.models.CoreV1Event; import io.kubernetes.client.openapi.models.V1Affinity; import io.kubernetes.client.openapi.models.V1ConfigMapKeySelector; import io.kubernetes.client.openapi.models.V1Container; @@ -48,6 +52,7 @@ import io.kubernetes.client.openapi.models.V1Volume; import io.kubernetes.client.openapi.models.V1VolumeMount; import io.kubernetes.client.openapi.models.V1WeightedPodAffinityTerm; +import oracle.kubernetes.operator.DomainProcessorImpl; import oracle.kubernetes.operator.DomainSourceType; import oracle.kubernetes.operator.KubernetesConstants; import oracle.kubernetes.operator.LabelConstants; @@ -56,6 +61,7 @@ import oracle.kubernetes.operator.PodAwaiterStepFactory; import oracle.kubernetes.operator.ProcessingConstants; import oracle.kubernetes.operator.calls.unprocessable.UnrecoverableErrorBuilderImpl; +import oracle.kubernetes.operator.logging.MessageKeys; import oracle.kubernetes.operator.utils.InMemoryCertificates; import oracle.kubernetes.operator.utils.WlsDomainConfigSupport; import oracle.kubernetes.operator.wlsconfig.NetworkAccessPoint; @@ -74,6 +80,7 @@ import oracle.kubernetes.weblogic.domain.model.DomainSpec; import oracle.kubernetes.weblogic.domain.model.DomainValidationBaseTest; import oracle.kubernetes.weblogic.domain.model.ServerEnvVars; +import org.hamcrest.junit.MatcherAssert; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -81,6 +88,9 @@ import static com.meterware.simplestub.Stub.createStrictStub; import static com.meterware.simplestub.Stub.createStub; +import static oracle.kubernetes.operator.EventConstants.DOMAIN_ROLL_STARTING_EVENT; +import static oracle.kubernetes.operator.EventTestUtils.containsEventWithNamespace; +import static oracle.kubernetes.operator.EventTestUtils.getEventsWithReason; import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.DOMAINZIP_HASH; import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.INTROSPECTOR_CONFIG_MAP_NAME_SUFFIX; import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.NUM_CONFIG_MAPS; @@ -93,6 +103,7 @@ import static oracle.kubernetes.operator.KubernetesConstants.SCRIPT_CONFIG_MAP_NAME; import static oracle.kubernetes.operator.LabelConstants.MII_UPDATED_RESTART_REQUIRED_LABEL; import static oracle.kubernetes.operator.LabelConstants.OPERATOR_VERSION; +import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_ROLL_START_EVENT_GENERATED; import static oracle.kubernetes.operator.ProcessingConstants.MAKE_RIGHT_DOMAIN_OPERATION; import static oracle.kubernetes.operator.ProcessingConstants.MII_DYNAMIC_UPDATE; import static oracle.kubernetes.operator.ProcessingConstants.MII_DYNAMIC_UPDATE_RESTART_REQUIRED; @@ -100,6 +111,7 @@ import static oracle.kubernetes.operator.ProcessingConstants.SERVER_SCAN; import static oracle.kubernetes.operator.helpers.AnnotationHelper.SHA256_ANNOTATION; import static oracle.kubernetes.operator.helpers.DomainStatusMatcher.hasStatus; +import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_ROLL_STARTING; import static oracle.kubernetes.operator.helpers.KubernetesTestSupport.DOMAIN; import static oracle.kubernetes.operator.helpers.KubernetesTestSupport.POD; import static oracle.kubernetes.operator.helpers.Matchers.ProbeMatcher.hasExpectedTuning; @@ -122,6 +134,7 @@ import static oracle.kubernetes.operator.helpers.TuningParametersStub.READINESS_INITIAL_DELAY; import static oracle.kubernetes.operator.helpers.TuningParametersStub.READINESS_PERIOD; import static oracle.kubernetes.operator.helpers.TuningParametersStub.READINESS_TIMEOUT; +import static oracle.kubernetes.operator.logging.MessageKeys.CYCLING_POD; import static oracle.kubernetes.utils.LogMatcher.containsFine; import static oracle.kubernetes.utils.LogMatcher.containsInfo; import static org.hamcrest.Matchers.allOf; @@ -137,6 +150,7 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.stringContainsInOrder; import static org.hamcrest.junit.MatcherAssert.assertThat; @SuppressWarnings({"SameParameterValue", "ConstantConditions", "OctalInteger", "unchecked"}) @@ -179,6 +193,9 @@ public abstract class PodHelperTestBase extends DomainValidationBaseTest { protected final V1SecurityContext containerSecurityContext = createSecurityContext(222L); protected final V1Affinity affinity = createAffinity(); private Memento hashMemento; + private final Map> domainEventObjects = new ConcurrentHashMap<>(); + + private TestUtils.ConsoleHandlerMemento consoleHandlerMemento = TestUtils.silenceOperatorLogger(); PodHelperTestBase(String serverName, int listenPort) { this.serverName = serverName; @@ -251,6 +268,7 @@ public void setUp() throws Exception { .collectLogMessages(logRecords, getMessageKeys()) .withLogLevel(Level.FINE) .ignoringLoggedExceptions(ApiException.class)); + mementos.add(StaticStubSupport.install(DomainProcessorImpl.class, "domainEventK8SObjects", domainEventObjects)); WlsDomainConfigSupport configSupport = new WlsDomainConfigSupport(DOMAIN_NAME); configSupport.addWlsServer(ADMIN_SERVER, ADMIN_PORT); @@ -946,6 +964,10 @@ void initializeExistingPod(V1Pod pod) { domainPresenceInfo.setServerPod(getServerName(), pod); } + void initializeExistingPodWithMii() { + initializeExistingPod(createPodModel()); + } + void initializeExistingPodWithIntrospectVersion(String introspectVersion) { initializeExistingPodWithIntrospectVersion(createPodModel(), introspectVersion); } @@ -1720,6 +1742,212 @@ public void whenPodCreated_createPodWithOwnerReference() { assertThat(getCreatedPod().getMetadata().getOwnerReferences(), contains(expectedReference)); } + + @Test + public void whenDomainHomeChanged_domainRollStartEventCreatedWithCorrectMessage() + throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { + initializeExistingPod(); + getConfiguredDomainSpec().setDomainHome("adfgg"); + + testSupport.runSteps(getStepFactory(), terminalStep); + logRecords.clear(); + + assertThat( + "Expected Event " + DOMAIN_ROLL_STARTING + " expected with message not found", + getExpectedEventMessage(DOMAIN_ROLL_STARTING), + stringContainsInOrder("Rolling restart", UID, "domainHome", " changed from", "to", "adfgg")); + } + + @Test + public void whenDomainHomeChanged_domainRollStartEventCreatedWithCorrectNS() + throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { + initializeExistingPod(); + getConfiguredDomainSpec().setDomainHome("adfgg"); + + testSupport.runSteps(getStepFactory(), terminalStep); + logRecords.clear(); + + assertContainsEventWithNamespace(DOMAIN_ROLL_STARTING, NS); + } + + @Test + public void whenDomainHomeChanged_butEventAlreadyGenerated_dontCreateDomainRollStartEvent() + throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { + initializeExistingPod(); + testSupport.addToPacket(DOMAIN_ROLL_START_EVENT_GENERATED, "true"); + getConfiguredDomainSpec().setDomainHome("adfgg"); + + testSupport.runSteps(getStepFactory(), terminalStep); + logRecords.clear(); + + assertThat( + "Found unexpected Event " + DOMAIN_ROLL_STARTING, + getEventsWithReason(getEvents(), DOMAIN_ROLL_STARTING_EVENT), empty()); + } + + @Test + public void whenImageChanged_domainRollStartEventCreatedWithCorrectMessage() + throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { + initializeExistingPod(); + getConfiguredDomainSpec().setImage("adfgg"); + + testSupport.runSteps(getStepFactory(), terminalStep); + logRecords.clear(); + + assertThat( + "Expected Event " + DOMAIN_ROLL_STARTING + " expected with message not found", + getExpectedEventMessage(DOMAIN_ROLL_STARTING), + stringContainsInOrder("Rolling restart", UID, "image changed","adfgg")); + } + + @Test + public void whenImageChanged_expectedLogMessageFound() + throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { + consoleHandlerMemento.collectLogMessages(logRecords, getDomainRollStartingKey()); + initializeExistingPod(); + getConfiguredDomainSpec().setImage("adfgg"); + + testSupport.runSteps(getStepFactory(), terminalStep); + + assertThat(logRecords, containsInfo(getDomainRollStartingKey())); + logRecords.clear(); + } + + @Test + public void whenInitContainerLivenessProbeChanged_domainRollStartEventCreatedWithCorrectMessage() { + initializeExistingPod(); + getConfigurator() + .withContainer(new V1Container().livenessProbe(new V1Probe().periodSeconds(123))); + + testSupport.runSteps(getStepFactory(), terminalStep); + logRecords.clear(); + + assertThat( + "Expected Event " + DOMAIN_ROLL_STARTING + " expected with message not found", + getExpectedEventMessage(DOMAIN_ROLL_STARTING), + stringContainsInOrder("Rolling restart", UID, "domain resource changed")); + } + + @Test + public void whenDefaultReadinessProbeChanged_domainRollStartEventCreatedWithCorrectMessage() { + initializeExistingPod(); + getConfigurator() + .withDefaultReadinessProbeSettings(12, 23, 45); + + testSupport.runSteps(getStepFactory(), terminalStep); + logRecords.clear(); + + assertThat( + "Expected Event " + DOMAIN_ROLL_STARTING + " expected with message not found", + getExpectedEventMessage(DOMAIN_ROLL_STARTING), + stringContainsInOrder("Rolling restart", UID, + "readiness probe", "changed from", "1", "2", "3", "to", "12", "23", "45")); + } + + @Test + public void whenDomainZipHashChanged_domainRollStartEventCreatedWithCorrectMessage() { + initializeExistingPod(); + disableAutoIntrospectOnNewMiiPods(); + testSupport.addToPacket(DOMAINZIP_HASH, "1234"); + + testSupport.runSteps(getStepFactory(), terminalStep); + logRecords.clear(); + + assertThat( + "Expected Event " + DOMAIN_ROLL_STARTING + " expected with message not found", + getExpectedEventMessage(DOMAIN_ROLL_STARTING), + stringContainsInOrder("Rolling restart", UID, "WebLogic domain configuration changed")); + } + + @Test + public void whenDomainZipHashChanged_butIsMIIDynamicUpdate_dontCreateDomainRollStartEvent() { + initializeExistingPod(); + disableAutoIntrospectOnNewMiiPods(); + testSupport.addToPacket(DOMAINZIP_HASH, "1234"); + testSupport.addToPacket(MII_DYNAMIC_UPDATE, MII_DYNAMIC_UPDATE_SUCCESS); + + getConfigurator().withMIIOnlineUpdate(); + + testSupport.runSteps(getStepFactory(), terminalStep); + logRecords.clear(); + + assertThat( + "Found unexpected event " + DOMAIN_ROLL_STARTING, + getEventsWithReason(getEvents(), DOMAIN_ROLL_STARTING_EVENT), empty()); + } + + @Test + public void whenImageDomainHomeAndRestartVersionChanged_expectedLogMessageFound() + throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { + consoleHandlerMemento.collectLogMessages(logRecords, getDomainRollStartingKey()); + initializeExistingPod(); + getConfiguredDomainSpec().setImage("adfgg"); + getConfiguredDomainSpec().setDomainHome("12345"); + getConfigurator().withRestartVersion("domainRestartV1"); + + testSupport.runSteps(getStepFactory(), terminalStep); + + assertThat(logRecords, containsInfo(getDomainRollStartingKey())); + logRecords.clear(); + } + + @Test + public void whenImageDomainHomeAndRestartVersionChanged_domainRollStartEventCreatedWithCorrectMessage() + throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { + consoleHandlerMemento.collectLogMessages(logRecords, getDomainRollStartingKey()); + initializeExistingPod(); + getConfiguredDomainSpec().setImage("adfgg"); + getConfiguredDomainSpec().setDomainHome("12345"); + getConfigurator().withRestartVersion("domainRestartV1"); + + testSupport.runSteps(getStepFactory(), terminalStep); + + logRecords.clear(); + + /* + message: Rolling restart the pods in domain uid1 because domain restart version changed. + image changed from image:latest to adfgg + imagePullPolicy changed from image:latest to IfNotPresent + 'domainHome' changed from '/u01/oracle/user_projects/domains' to '12345' + */ + assertThat( + "Expected Event " + DOMAIN_ROLL_STARTING + " expected with message not found", + getExpectedEventMessage(DOMAIN_ROLL_STARTING), + stringContainsInOrder("Rolling restart", UID, + "domain restart version changed", + "image changed", "adfgg", + "domainHome", "changed", "12345")); + } + + + protected static String getCyclePodKey() { + return CYCLING_POD; + } + + protected static String getDomainRollStartingKey() { + return MessageKeys.DOMAIN_ROLL_STARTING; + } + + protected void assertContainsEventWithNamespace(EventHelper.EventItem event, String ns) { + MatcherAssert.assertThat( + "Expected Event " + event.getReason() + " was not created", + containsEventWithNamespace(getEvents(), event.getReason(), ns), + is(true)); + } + + protected String getExpectedEventMessage(EventHelper.EventItem event) { + List events = getEventsWithReason(getEvents(), event.getReason()); + return Optional.ofNullable(events) + .filter(list -> list.size() != 0) + .map(n -> n.get(0)) + .map(CoreV1Event::getMessage) + .orElse("Event not found"); + } + + private List getEvents() { + return testSupport.getResources(KubernetesTestSupport.EVENT); + } + interface PodMutator { void mutate(V1Pod pod); } diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/RollingHelperTest.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/RollingHelperTest.java index 6334ee86eb0..2944bff6f18 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/RollingHelperTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/RollingHelperTest.java @@ -8,20 +8,27 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; import java.util.logging.LogRecord; import com.meterware.simplestub.Memento; +import com.meterware.simplestub.StaticStubSupport; +import io.kubernetes.client.openapi.models.CoreV1Event; import io.kubernetes.client.openapi.models.V1ObjectMeta; import io.kubernetes.client.openapi.models.V1Pod; import io.kubernetes.client.openapi.models.V1PodCondition; import io.kubernetes.client.openapi.models.V1PodStatus; import io.kubernetes.client.openapi.models.V1SecretReference; +import oracle.kubernetes.operator.DomainProcessorImpl; import oracle.kubernetes.operator.DomainStatusUpdater; +import oracle.kubernetes.operator.LabelConstants; import oracle.kubernetes.operator.PodAwaiterStepFactory; import oracle.kubernetes.operator.ProcessingConstants; import oracle.kubernetes.operator.helpers.PodHelper.ManagedPodStepContext; import oracle.kubernetes.operator.helpers.PodHelperTestBase.PassthroughPodAwaiterStepFactory; +import oracle.kubernetes.operator.logging.MessageKeys; import oracle.kubernetes.operator.utils.WlsDomainConfigSupport; import oracle.kubernetes.operator.wlsconfig.WlsDomainConfig; import oracle.kubernetes.operator.work.Packet; @@ -31,16 +38,26 @@ import oracle.kubernetes.utils.TestUtils; import oracle.kubernetes.weblogic.domain.model.Domain; import oracle.kubernetes.weblogic.domain.model.DomainSpec; +import org.hamcrest.junit.MatcherAssert; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import static oracle.kubernetes.operator.EventTestUtils.containsEventWithMessage; +import static oracle.kubernetes.operator.EventTestUtils.getEventsWithReason; +import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_ROLL_START_EVENT_GENERATED; import static oracle.kubernetes.operator.ProcessingConstants.SERVERS_TO_ROLL; import static oracle.kubernetes.operator.ProcessingConstants.SERVER_SCAN; +import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_ROLL_COMPLETED; +import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_ROLL_STARTING; +import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.POD_CYCLE_STARTING; +import static oracle.kubernetes.operator.logging.MessageKeys.CYCLING_POD; import static oracle.kubernetes.operator.logging.MessageKeys.MANAGED_POD_REPLACED; import static oracle.kubernetes.utils.LogMatcher.containsInOrder; import static oracle.kubernetes.utils.LogMatcher.containsInfo; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.stringContainsInOrder; import static org.hamcrest.collection.IsEmptyCollection.empty; public class RollingHelperTest { @@ -68,6 +85,8 @@ public class RollingHelperTest { protected final KubernetesTestSupport testSupport = new KubernetesTestSupport(); protected final List mementos = new ArrayList<>(); protected final List logRecords = new ArrayList<>(); + private final Map> domainEventObjects = new ConcurrentHashMap<>(); + private final Map nsEventObjects = new ConcurrentHashMap<>(); private WlsDomainConfig domainTopology; @@ -95,6 +114,8 @@ public void setUp() throws Exception { new PassthroughPodAwaiterStepFactory()); testSupport.addToPacket(ProcessingConstants.CLUSTER_NAME, CLUSTER_NAME); + mementos.add(StaticStubSupport.install(DomainProcessorImpl.class, "domainEventK8SObjects", domainEventObjects)); + mementos.add(StaticStubSupport.install(DomainProcessorImpl.class, "namespaceEventK8SObjects", nsEventObjects)); } @AfterEach @@ -134,7 +155,7 @@ private DomainPresenceInfo createDomainPresenceInfo(Domain domain) { private String[] getMessageKeys() { return new String[] { - MANAGED_POD_REPLACED + MANAGED_POD_REPLACED, CYCLING_POD, MessageKeys.DOMAIN_ROLL_COMPLETED }; } @@ -150,6 +171,16 @@ private Step.StepAndPacket createRollingStepAndPacket(String serverName) { LegalNames.toPodName(UID, serverName)), null)), packet); } + private Step.StepAndPacket createRollingStepAndPacket(V1Pod pod, String serverName) { + Packet packet = testSupport.getPacket().copy(); + packet.put(SERVER_SCAN, domainTopology.getServerConfig(serverName)); + return new Step.StepAndPacket(DomainStatusUpdater.createProgressingStep( + DomainStatusUpdater.MANAGED_SERVERS_STARTING_PROGRESS_REASON, + false, + new ManagedPodStepContext(terminalStep, packet).createCyclePodStep( + pod, null)), packet); + } + private void initializeExistingPods() { SERVER_NAMES.forEach(this::initializeExistingPod); } @@ -176,6 +207,7 @@ public void verifyThatManagedServerPodsAreReplacedInOrder() { containsInfo(MANAGED_POD_REPLACED, SERVER2_NAME), containsInfo(MANAGED_POD_REPLACED, SERVER10_NAME) )); + logRecords.clear(); } @Test @@ -188,4 +220,122 @@ public void verifyThatWhenRollingIsEmpty_NoManagedServerPodsAreReplaced() { assertThat(logRecords, empty()); } + @Test + public void afterRoll_domainRollCompletedEventCreated() { + initializeExistingPods(); + testSupport.addToPacket(SERVERS_TO_ROLL, rolling); + testSupport.addToPacket(DOMAIN_ROLL_START_EVENT_GENERATED, "true"); + SERVER_NAMES.forEach(s -> + rolling.put(s, createRollingStepAndPacket(modifyRestartVersion(createPodModel(s), "V10"), s))); + + testSupport.runSteps(RollingHelper.rollServers(rolling, terminalStep)); + logRecords.clear(); + + assertContainsEventWithMessage(DOMAIN_ROLL_COMPLETED, UID); + } + + @Test + public void afterRoll_expectedLogMessageFound() { + initializeExistingPods(); + testSupport.addToPacket(SERVERS_TO_ROLL, rolling); + testSupport.addToPacket(DOMAIN_ROLL_START_EVENT_GENERATED, "true"); + SERVER_NAMES.forEach(s -> + rolling.put(s, createRollingStepAndPacket(modifyRestartVersion(createPodModel(s), "V11"), s))); + + testSupport.runSteps(RollingHelper.rollServers(rolling, terminalStep)); + + // printLogRecords(); + SERVER_NAMES.forEach(s -> assertThat(logRecords, + containsInfo(CYCLING_POD, getPodName(s), "domain restart version changed from 'V11' to 'null'"))); + SERVER_NAMES.forEach(s -> assertThat(logRecords, + containsInfo(MANAGED_POD_REPLACED, s))); + assertThat(logRecords, + containsInfo(MessageKeys.DOMAIN_ROLL_COMPLETED, UID)); + } + + @Test + public void whenDomainHomeChanged_podCycleEventCreatedWithCorrectMessage() { + initializeExistingPods(); + testSupport.addToPacket(SERVERS_TO_ROLL, rolling); + testSupport.addToPacket(DOMAIN_ROLL_START_EVENT_GENERATED, "true"); + SERVER_NAMES.forEach(s -> + rolling.put(s, createRollingStepAndPacket(modifyRestartVersion(createPodModel(s), "V3"), s))); + + testSupport.runSteps(RollingHelper.rollServers(rolling, terminalStep)); + logRecords.clear(); + + SERVER_NAMES.forEach(s -> assertThat( + "Expected Event " + DOMAIN_ROLL_STARTING + " expected with message not found", + getExpectedEventMessage(POD_CYCLE_STARTING, getPodName(s), NS), + stringContainsInOrder("Replacing ", getPodName(s), "domain restart version changed"))); + } + + @Test + public void whenDomainHomeChanged_generateExpectedLogMessage() { + initializeExistingPods(); + testSupport.addToPacket(SERVERS_TO_ROLL, rolling); + testSupport.addToPacket(DOMAIN_ROLL_START_EVENT_GENERATED, "true"); + SERVER_NAMES.forEach(s -> + rolling.put(s, createRollingStepAndPacket(modifyRestartVersion(createPodModel(s), "V5"), s))); + + testSupport.runSteps(RollingHelper.rollServers(rolling, terminalStep)); + + // printLogRecords(); + SERVER_NAMES.forEach(s -> assertThat(logRecords, + containsInfo(CYCLING_POD, getPodName(s), "domain restart version changed from 'V5' to 'null'"))); + SERVER_NAMES.forEach(s -> assertThat(logRecords, + containsInfo(MANAGED_POD_REPLACED, s))); + assertThat(logRecords, + containsInfo(MessageKeys.DOMAIN_ROLL_COMPLETED, UID)); + } + + private void printLogRecords() { + String str = ""; + for (LogRecord r : logRecords) { + str += r.getLevel() + " " + r.getMessage(); + for (Object o : r.getParameters()) { + str += o.toString(); + } + } + System.out.println(str); + } + + private String getPodName(String s) { + return getPodNameFromMetadata(domainPresenceInfo.getServerPod(s)); + } + + private String getPodNameFromMetadata(V1Pod serverPod) { + return Optional.ofNullable(serverPod).map(V1Pod::getMetadata).map(V1ObjectMeta::getName).orElse(""); + } + + protected String getExpectedEventMessage(EventHelper.EventItem event, String name, String ns) { + List events = getEventsWithReason(getEvents(), event.getReason()); + System.out.println(events); + for (CoreV1Event e : events) { + if (e.getMessage().contains(name) && e.getMetadata().getNamespace().equals(ns)) { + return e.getMessage(); + } + } + return "Event not found"; + } + + private V1Pod modifyRestartVersion(V1Pod pod, String restartVersion) { + pod.setStatus(new V1PodStatus().phase("Running").addConditionsItem( + new V1PodCondition().type("Ready").status("True"))); + pod.getMetadata().getLabels().remove(LabelConstants.DOMAINRESTARTVERSION_LABEL); + pod.getMetadata().getLabels().put(LabelConstants.DOMAINRESTARTVERSION_LABEL, restartVersion); + return pod; + } + + private void assertContainsEventWithMessage(EventHelper.EventItem event, Object...params) { + String message = String.format(event.getPattern(), params); + MatcherAssert.assertThat( + "Expected Event " + event.getReason() + " with message" + message + " was not created", + containsEventWithMessage(getEvents(), event.getReason(), message), is(true)); + } + + private List getEvents() { + return testSupport.getResources(KubernetesTestSupport.EVENT); + } + } From fac122f373d3ee4e03003cd06f97fdaef2ae7412 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Wed, 12 May 2021 17:20:52 -0400 Subject: [PATCH 02/10] refactoring --- .../operator/helpers/CompatibleMaps.java | 68 ++++++++++--------- .../operator/helpers/EventHelper.java | 5 -- .../operator/helpers/PodCompatibility.java | 24 +++---- .../operator/helpers/PodStepContext.java | 11 +-- .../operator/helpers/RollingHelper.java | 2 +- .../helpers/PodCompatibilityTest.java | 2 +- 6 files changed, 52 insertions(+), 60 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleMaps.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleMaps.java index 5ecace6eb3c..217b119f7ce 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleMaps.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleMaps.java @@ -67,17 +67,11 @@ private boolean valuesDiffer(K key) { public String getIncompatibility() { StringBuilder sb = new StringBuilder(); - Set missingKeys = getMissingElements(expected.keySet(), actual.keySet()); - if (!missingKeys.isEmpty()) { - sb.append(String.format("%s changed and contains '%s' as well%n", description, missingKeys)); - } + handleMissingElements(sb); for (K key : expected.keySet()) { - if (isKeyToCheck(key) && actual.containsKey(key) && valuesDiffer(key)) { - sb.append( - String.format( - "%s '%s' changed from '%s' to '%s'%n", - description, key, getValue(actual.get(key)), getValue(expected.get(key)))); + if (isKeyChanged(key)) { + sb.append(getGeneralScopedFormat(key)); } } @@ -87,49 +81,57 @@ public String getIncompatibility() { private String getDomainIncompatibility() { StringBuilder sb = new StringBuilder(); - Set missingKeys = getMissingElements(expected.keySet(), actual.keySet()); - if (!missingKeys.isEmpty()) { - sb.append(String.format("%s changed and contains '%s' as well%n", description, missingKeys)); - } + handleMissingElements(sb); for (K key : expected.keySet()) { - if (isKeyToCheck(key) && actual.containsKey(key) && valuesDiffer(key) && isDomainKey(key)) { - sb.append( - String.format( - "'%s' changed from '%s' to '%s'%n", - ELEMENT_NAMES_MAP.get(key), getValue(actual.get(key)), getValue(expected.get(key)))); + if (isKeyChanged(key) && isDomainKey(key)) { + sb.append(getDomainScopedFormat(ELEMENT_NAMES_MAP.get(key), key)); } } return sb.length() == 0 ? null : sb.toString(); } - private Object getValue(Object obj) { - if (obj instanceof V1EnvVar) { - return ((V1EnvVar) obj).getValue(); + private String getUnknownIncompatibility() { + StringBuilder sb = new StringBuilder(); + + handleMissingElements(sb); + + for (K key : expected.keySet()) { + if (isKeyChanged(key) && !isDomainKey(key)) { + sb.append(getGeneralScopedFormat(key)); + } } - return obj; + + return sb.length() == 0 ? null : sb.toString(); } - private String getUnknownIncompatibility() { - StringBuilder sb = new StringBuilder(); + private boolean isKeyChanged(K key) { + return isKeyToCheck(key) && actual.containsKey(key) && valuesDiffer(key); + } + private String getDomainScopedFormat(String name, K key) { + return String.format( + "'%s' changed from '%s' to '%s'%n", + name, getValue(actual.get(key)), getValue(expected.get(key))); + } + + private String getGeneralScopedFormat(K key) { + return String.format("%s %s", description, getDomainScopedFormat((String)key, key)); + } + private void handleMissingElements(StringBuilder sb) { Set missingKeys = getMissingElements(expected.keySet(), actual.keySet()); if (!missingKeys.isEmpty()) { sb.append(String.format("%s changed and contains '%s' as well%n", description, missingKeys)); } + } - for (K key : expected.keySet()) { - if (isKeyToCheck(key) && actual.containsKey(key) && valuesDiffer(key) && !isDomainKey(key)) { - sb.append( - String.format( - "%s '%s' changed from '%s' to '%s'%n", - description, key, actual.get(key), expected.get(key))); - } + private Object getValue(Object obj) { + if (obj instanceof V1EnvVar) { + return ((V1EnvVar) obj).getValue(); } - - return sb.length() == 0 ? null : sb.toString(); + return obj; } private boolean isDomainKey(K key) { diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/EventHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/EventHelper.java index 7153c6bd4f3..57019ff7002 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/EventHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/EventHelper.java @@ -504,11 +504,6 @@ public String getMessage(EventData eventData) { } }, POD_CYCLE_STARTING { - @Override - protected String getType() { - return EVENT_NORMAL; - } - @Override public String getReason() { return POD_CYCLE_STARTING_EVENT; diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodCompatibility.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodCompatibility.java index a9bdfd0cafe..7e9899e841d 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodCompatibility.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodCompatibility.java @@ -68,31 +68,25 @@ && isLabelSame(CLUSTERRESTARTVERSION_LABEL) } private boolean isLabelSame(String labelName) { - return Objects.equals( - getExpected(labelName), - getActual(labelName) + return Objects.equals(getLabel(expected, labelName), getLabel(actual, labelName) ); } - private String getExpected(String labelName) { - return Objects.requireNonNull(expected.getLabels()).get(labelName); - } - - private String getActual(String labelName) { - return Objects.requireNonNull(actual.getLabels()).get(labelName); + private String getLabel(V1ObjectMeta metadata, String labelName) { + return Objects.requireNonNull(metadata.getLabels()).get(labelName); } @Override public String getIncompatibility() { if (!isLabelSame(DOMAINRESTARTVERSION_LABEL)) { - return "domain restart version changed from '" + getActual(DOMAINRESTARTVERSION_LABEL) - + "' to '" + getExpected(DOMAINRESTARTVERSION_LABEL) + "'"; + return "domain restart version changed from '" + getLabel(actual, DOMAINRESTARTVERSION_LABEL) + + "' to '" + getLabel(expected, DOMAINRESTARTVERSION_LABEL) + "'"; } else if (!isLabelSame(CLUSTERRESTARTVERSION_LABEL)) { - return "cluster restart version changed from '" + getActual(CLUSTERRESTARTVERSION_LABEL) - + "' to '" + getExpected(CLUSTERRESTARTVERSION_LABEL) + "'"; + return "cluster restart version changed from '" + getLabel(actual, CLUSTERRESTARTVERSION_LABEL) + + "' to '" + getLabel(expected, CLUSTERRESTARTVERSION_LABEL) + "'"; } else if (!isLabelSame(SERVERRESTARTVERSION_LABEL)) { - return "server restart version changed from '" + getActual(SERVERRESTARTVERSION_LABEL) - + "' to '" + getExpected(SERVERRESTARTVERSION_LABEL) + "'"; + return "server restart version changed from '" + getLabel(actual, SERVERRESTARTVERSION_LABEL) + + "' to '" + getLabel(expected, SERVERRESTARTVERSION_LABEL) + "'"; } else { return null; } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java index 2056ecd673f..0c95f9ff3d2 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java @@ -1017,16 +1017,17 @@ public class CyclePodStep extends BaseStep { @Override public NextAction apply(Packet packet) { + + markBeingDeleted(); + return doNext(createCyclePodEventStep(deletePod(pod, getNext())), packet); + } + + private Step createCyclePodEventStep(Step next) { String reason = getReasonToRecycle(pod, CompatibilityScope.POD); LOGGER.info( MessageKeys.CYCLING_POD, Objects.requireNonNull(pod.getMetadata()).getName(), reason); - markBeingDeleted(); - return doNext(createCyclePodEventStep(reason, deletePod(pod, getNext())), packet); - } - - private Step createCyclePodEventStep(String reason, Step next) { return Step.chain(EventHelper.createEventStep(new EventData(POD_CYCLE_STARTING, reason).podName(getPodName())), next); } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/RollingHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/RollingHelper.java index 53186ff0c44..15206ddf140 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/RollingHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/RollingHelper.java @@ -247,7 +247,7 @@ public NextAction apply(Packet packet) { } } - LOGGER.info(MessageKeys.ROLLING_SERVERS, dom.getDomainUid(), servers.toString(), readyServers); + LOGGER.info(MessageKeys.ROLLING_SERVERS, dom.getDomainUid(), servers, readyServers); int countToRestartNow = countReady - dom.getMinAvailable(clusterName); Collection restarts = new ArrayList<>(); diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/PodCompatibilityTest.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/PodCompatibilityTest.java index 4d635e89bb1..4c97dace0c6 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/PodCompatibilityTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/PodCompatibilityTest.java @@ -95,7 +95,7 @@ public void whenEnvVarsDontMatch_createErrorMessage() { PodCompatibility.ContainerCompatibility compatibility = new PodCompatibility.ContainerCompatibility( new V1Container().addEnvItem(envVar(name, value)), - new V1Container().addEnvItem(envVar("aa", "cc"))); + new V1Container().addEnvItem(envVar(name, "cc"))); assertThat( compatibility.getIncompatibility(), both(containsString("aa")).and(containsString("bb"))); From d144b55c66b1113c946530cd089cfd02cee9f8b6 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Thu, 13 May 2021 14:22:45 -0400 Subject: [PATCH 03/10] minor adjustment based on manual testing --- .../operator/helpers/CompatibleMaps.java | 27 +++++--- .../operator/helpers/PodStepContext.java | 25 ++++--- .../helpers/PodCompatibilityTest.java | 28 +++++++- .../operator/helpers/PodHelperTestBase.java | 69 ++++++++++++++++++- .../operator/helpers/RollingHelperTest.java | 34 ++++++--- 5 files changed, 153 insertions(+), 30 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleMaps.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleMaps.java index 217b119f7ce..a905cb9ab1f 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleMaps.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleMaps.java @@ -7,6 +7,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -67,7 +68,7 @@ private boolean valuesDiffer(K key) { public String getIncompatibility() { StringBuilder sb = new StringBuilder(); - handleMissingElements(sb); + handleMissingElements(sb, getMissingElements(expected.keySet(), actual.keySet())); for (K key : expected.keySet()) { if (isKeyChanged(key)) { @@ -81,7 +82,7 @@ public String getIncompatibility() { private String getDomainIncompatibility() { StringBuilder sb = new StringBuilder(); - handleMissingElements(sb); + handleMissingElements(sb, getDomainScopedKeys(getMissingElements(expected.keySet(), actual.keySet()))); for (K key : expected.keySet()) { if (isKeyChanged(key) && isDomainKey(key)) { @@ -95,7 +96,7 @@ private String getDomainIncompatibility() { private String getUnknownIncompatibility() { StringBuilder sb = new StringBuilder(); - handleMissingElements(sb); + handleMissingElements(sb, getUnknownScopedKeys(getMissingElements(expected.keySet(), actual.keySet()))); for (K key : expected.keySet()) { if (isKeyChanged(key) && !isDomainKey(key)) { @@ -120,8 +121,7 @@ private String getGeneralScopedFormat(K key) { return String.format("%s %s", description, getDomainScopedFormat((String)key, key)); } - private void handleMissingElements(StringBuilder sb) { - Set missingKeys = getMissingElements(expected.keySet(), actual.keySet()); + private void handleMissingElements(StringBuilder sb, Set missingKeys) { if (!missingKeys.isEmpty()) { sb.append(String.format("%s changed and contains '%s' as well%n", description, missingKeys)); } @@ -138,13 +138,24 @@ private boolean isDomainKey(K key) { return DOMAIN_ENV_KEYS.contains(key); } - private boolean containsDomainKeys(Set missingKeys) { + private Set getDomainScopedKeys(Set missingKeys) { + Set newSet = new HashSet<>(); for (K key : missingKeys) { if (isDomainKey(key)) { - return true; + newSet.add(key); } } - return false; + return newSet; + } + + private Set getUnknownScopedKeys(Set missingKeys) { + Set newSet = new HashSet<>(); + for (K key : missingKeys) { + if (!isDomainKey(key)) { + newSet.add(key); + } + } + return newSet; } @Override diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java index 0c95f9ff3d2..b2a0b1b16c0 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java @@ -462,20 +462,29 @@ protected Step createDomainRollStartEventIfNeeded(V1Pod pod, Step next) { return next; } - String domainIncompatibility = getReasonToRecycle(pod, DOMAIN); - if (hasChangesInExpectedScope(domainIncompatibility)) { + String domainIncompatibility = getDomainIncompatibility(pod); + if (haveReasonsToRoll(domainIncompatibility)) { return createDomainRollStartEvent(next, domainIncompatibility); } - if (hasChangesInExpectedScope(getReasonToRecycle(pod, UNKNOWN))) { - return createDomainRollStartEvent(next, "domain resource changed"); + return next; + } + + private String getDomainIncompatibility(V1Pod pod) { + String domainIncompatibility = getReasonToRecycle(pod, DOMAIN); + if (!haveReasonsToRoll(domainIncompatibility) + && haveReasonsToRoll(getReasonToRecycle(pod, UNKNOWN))) { + domainIncompatibility = "domain resource changed"; } if (!canUseNewDomainZip(pod)) { - return createDomainRollStartEvent(next, "WebLogic domain configuration changed"); + if (haveReasonsToRoll(domainIncompatibility)) { + domainIncompatibility += ",\nWebLogic domain configuration changed"; + } else { + domainIncompatibility = "WebLogic domain configuration changed"; + } } - - return next; + return domainIncompatibility; } private Step createDomainRollStartEvent(Step next, String domainIncompatibility) { @@ -487,7 +496,7 @@ private Step createDomainRollStartEvent(Step next, String domainIncompatibility) next); } - private boolean hasChangesInExpectedScope(String domainIncompatibility) { + private boolean haveReasonsToRoll(String domainIncompatibility) { return domainIncompatibility != null && domainIncompatibility.length() != 0; } diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/PodCompatibilityTest.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/PodCompatibilityTest.java index 4c97dace0c6..8b78ccdf550 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/PodCompatibilityTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/PodCompatibilityTest.java @@ -580,17 +580,43 @@ public void ignoreAllScopeCertForComparisons() { @Test public void whenExpectedNotSubmapOfActual_reportDomainScopeMissingElements() { + CompatibilityCheck check = + new CompatibleMaps<>( + "env", + ImmutableMap.of("alpha", 1, "beta", 2, "DOMAIN_HOME", 4), + ImmutableMap.of("beta", 2, "gamma", 3, "alpha", 1)); + + assertThat(check.getScopedIncompatibility(DOMAIN), containsString("DOMAIN_HOME")); + assertThat(check.getScopedIncompatibility(DOMAIN), not(containsString("alpha"))); + assertThat(check.getScopedIncompatibility(DOMAIN), not(containsString("gamma"))); + } + + @Test + public void whenExpectedNonDomainScopedNotSubmapOfActual_dontReportDomainScopeMissingElements() { CompatibilityCheck check = new CompatibleMaps<>( "letters", ImmutableMap.of("alpha", 1, "beta", 2, "delta", 4), ImmutableMap.of("beta", 2, "gamma", 3, "alpha", 1)); - assertThat(check.getScopedIncompatibility(DOMAIN), containsString("delta")); + assertThat(check.getScopedIncompatibility(DOMAIN), not(containsString("delta"))); assertThat(check.getScopedIncompatibility(DOMAIN), not(containsString("alpha"))); assertThat(check.getScopedIncompatibility(DOMAIN), not(containsString("gamma"))); } + @Test + public void whenExpectedNonDomainScopedNotSubmapOfActual_dontReportUnknownScopeMissingElements() { + CompatibilityCheck check = + new CompatibleMaps<>( + "letters", + ImmutableMap.of("alpha", 1, "beta", 2, "delta", 4), + ImmutableMap.of("beta", 2, "gamma", 3, "alpha", 1)); + + assertThat(check.getScopedIncompatibility(UNKNOWN), containsString("delta")); + assertThat(check.getScopedIncompatibility(UNKNOWN), not(containsString("alpha"))); + assertThat(check.getScopedIncompatibility(UNKNOWN), not(containsString("gamma"))); + } + @Test public void whenExpectedNotSubmapOfActual_reportUnknownAndPodScopeMissingElements() { CompatibilityCheck check = diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java index 2cc9f12c117..01ef1d681ea 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java @@ -1844,6 +1844,22 @@ public void whenDefaultReadinessProbeChanged_domainRollStartEventCreatedWithCorr "readiness probe", "changed from", "1", "2", "3", "to", "12", "23", "45")); } + @Test + public void whenDefaultLivenessProbeChanged_domainRollStartEventCreatedWithCorrectMessage() { + initializeExistingPod(); + getConfigurator() + .withDefaultLivenessProbeSettings(12, 23, 45); + + testSupport.runSteps(getStepFactory(), terminalStep); + logRecords.clear(); + + assertThat( + "Expected Event " + DOMAIN_ROLL_STARTING + " expected with message not found", + getExpectedEventMessage(DOMAIN_ROLL_STARTING), + stringContainsInOrder("Rolling restart", UID, + "liveness probe", "changed from", "4", "5", "6", "to", "12", "23", "45")); + } + @Test public void whenDomainZipHashChanged_domainRollStartEventCreatedWithCorrectMessage() { initializeExistingPod(); @@ -1905,9 +1921,9 @@ public void whenImageDomainHomeAndRestartVersionChanged_domainRollStartEventCrea logRecords.clear(); /* - message: Rolling restart the pods in domain uid1 because domain restart version changed. - image changed from image:latest to adfgg - imagePullPolicy changed from image:latest to IfNotPresent + message: Rolling restart the pods in domain uid1 because domain restart version changed, + image changed from image:latest to adfgg, + imagePullPolicy changed from image:latest to IfNotPresent, 'domainHome' changed from '/u01/oracle/user_projects/domains' to '12345' */ assertThat( @@ -1919,6 +1935,52 @@ public void whenImageDomainHomeAndRestartVersionChanged_domainRollStartEventCrea "domainHome", "changed", "12345")); } + @Test + public void whenImageDomainHomeAndWebLogicZipHashChanged_domainRollStartEventCreatedWithCorrectMessage() + throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { + consoleHandlerMemento.collectLogMessages(logRecords, getDomainRollStartingKey()); + initializeExistingPod(); + getConfiguredDomainSpec().setImage("adfgg"); + getConfiguredDomainSpec().setDomainHome("12345"); + testSupport.addToPacket(DOMAINZIP_HASH, "1234"); + testSupport.addToPacket(MII_DYNAMIC_UPDATE, MII_DYNAMIC_UPDATE_SUCCESS); + + testSupport.runSteps(getStepFactory(), terminalStep); + + logRecords.clear(); + + /* + message: Rolling restart the pods in domain uid1 because domain restart version changed, + imagePullPolicy changed from image:latest to IfNotPresent, + 'domainHome' changed from '/u01/oracle/user_projects/domains' to '12345', + WebLogic domain configuration changed + */ + assertThat( + "Expected Event " + DOMAIN_ROLL_STARTING + " expected with message not found", + getExpectedEventMessage(DOMAIN_ROLL_STARTING), + stringContainsInOrder("Rolling restart", UID, + "image changed", "adfgg", + "domainHome", "changed", "12345", + "WebLogic domain configuration changed")); + } + + @Test + public void whenInitContainerLivenessProbeAndWebLogicZipHashChanged_domainRollStartEventCreatedWithCorrectMessage() { + initializeExistingPod(); + getConfigurator() + .withContainer(new V1Container().livenessProbe(new V1Probe().periodSeconds(123))); + testSupport.addToPacket(DOMAINZIP_HASH, "1234"); + testSupport.addToPacket(MII_DYNAMIC_UPDATE, MII_DYNAMIC_UPDATE_SUCCESS); + + testSupport.runSteps(getStepFactory(), terminalStep); + logRecords.clear(); + + assertThat( + "Expected Event " + DOMAIN_ROLL_STARTING + " expected with message not found", + getExpectedEventMessage(DOMAIN_ROLL_STARTING), + stringContainsInOrder("Rolling restart", UID, "domain resource changed", + "WebLogic domain configuration changed")); + } protected static String getCyclePodKey() { return CYCLING_POD; @@ -1937,6 +1999,7 @@ protected void assertContainsEventWithNamespace(EventHelper.EventItem event, Str protected String getExpectedEventMessage(EventHelper.EventItem event) { List events = getEventsWithReason(getEvents(), event.getReason()); + //System.out.println(events); return Optional.ofNullable(events) .filter(list -> list.size() != 0) .map(n -> n.get(0)) diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/RollingHelperTest.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/RollingHelperTest.java index d9516f4b2f8..96774cde4cf 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/RollingHelperTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/RollingHelperTest.java @@ -16,6 +16,7 @@ import com.meterware.simplestub.Memento; import com.meterware.simplestub.StaticStubSupport; import io.kubernetes.client.openapi.models.CoreV1Event; +import io.kubernetes.client.openapi.models.V1EnvVar; import io.kubernetes.client.openapi.models.V1ObjectMeta; import io.kubernetes.client.openapi.models.V1Pod; import io.kubernetes.client.openapi.models.V1PodCondition; @@ -254,7 +255,7 @@ public void afterRoll_expectedLogMessageFound() { } @Test - public void whenDomainHomeChanged_podCycleEventCreatedWithCorrectMessage() { + public void whenRolling_podCycleEventCreatedWithCorrectMessage() { initializeExistingPods(); testSupport.addToPacket(SERVERS_TO_ROLL, rolling); testSupport.addToPacket(DOMAIN_ROLL_START_EVENT_GENERATED, "true"); @@ -271,22 +272,22 @@ public void whenDomainHomeChanged_podCycleEventCreatedWithCorrectMessage() { } @Test - public void whenDomainHomeChanged_generateExpectedLogMessage() { + public void whenDomainHomeAndRestartVersionChanged_podCycleEventCreatedWithCorrectMessage() { initializeExistingPods(); testSupport.addToPacket(SERVERS_TO_ROLL, rolling); testSupport.addToPacket(DOMAIN_ROLL_START_EVENT_GENERATED, "true"); SERVER_NAMES.forEach(s -> - rolling.put(s, createRollingStepAndPacket(modifyRestartVersion(createPodModel(s), "V5"), s))); + rolling.put(s, createRollingStepAndPacket( + modifyDomainHome(modifyRestartVersion(createPodModel(s), "V5"), "xxxx"), s))); testSupport.runSteps(RollingHelper.rollServers(rolling, terminalStep)); + logRecords.clear(); - // printLogRecords(); - SERVER_NAMES.forEach(s -> assertThat(logRecords, - containsInfo(CYCLING_POD, getPodName(s), "domain restart version changed from 'V5' to 'null'"))); - SERVER_NAMES.forEach(s -> assertThat(logRecords, - containsInfo(MANAGED_POD_REPLACED, s))); - assertThat(logRecords, - containsInfo(MessageKeys.DOMAIN_ROLL_COMPLETED, UID)); + SERVER_NAMES.forEach(s -> assertThat( + "Expected Event " + DOMAIN_ROLL_STARTING + " expected with message not found", + getExpectedEventMessage(POD_CYCLE_STARTING, getPodName(s), NS), + stringContainsInOrder("Replacing ", getPodName(s), + "domain restart version changed", "V5", "DOMAIN_HOME", "changed", "xxxx"))); } private void printLogRecords() { @@ -326,6 +327,19 @@ private V1Pod modifyRestartVersion(V1Pod pod, String restartVersion) { return pod; } + private V1Pod modifyDomainHome(V1Pod pod, String domainHome) { + pod.setStatus(new V1PodStatus().phase("Running").addConditionsItem( + new V1PodCondition().type("Ready").status("True"))); + List envList = pod.getSpec().getContainers().get(0).getEnv(); + for (V1EnvVar env : envList) { + if (env.getName().equals("DOMAIN_HOME")) { + env.setValue(domainHome); + return pod; + } + } + return pod; + } + private void assertContainsEventWithMessage(EventHelper.EventItem event, Object...params) { String message = String.format(event.getPattern(), params); MatcherAssert.assertThat( From cea3930b999378adb377409eaff25fac45351487 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Thu, 13 May 2021 16:20:48 -0400 Subject: [PATCH 04/10] minor changes to event message format --- .../helpers/CollectiveCompatibility.java | 6 ++-- .../operator/helpers/CompatibleMaps.java | 30 +++++++++---------- .../operator/helpers/CompatibleSets.java | 2 +- .../kubernetes/operator/helpers/Equality.java | 2 +- .../operator/helpers/PodCompatibility.java | 2 +- .../operator/helpers/PodHelperTestBase.java | 11 ++++--- 6 files changed, 26 insertions(+), 27 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CollectiveCompatibility.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CollectiveCompatibility.java index 46dbabb3e8f..94af3a53e08 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CollectiveCompatibility.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CollectiveCompatibility.java @@ -41,7 +41,7 @@ public String getIncompatibility() { reasons.add(getIndent() + check.getIncompatibility()); } } - return reasons.isEmpty() ? null : getHeader() + String.join(",\n", reasons); + return reasons.isEmpty() ? null : getHeader() + String.join(",%n", reasons); } @Override @@ -55,8 +55,8 @@ public String getScopedIncompatibility(CompatibilityScope scope) { return reasons.isEmpty() ? null : scope == CompatibilityScope.DOMAIN - ? String.join(",\n", reasons) - : getHeader() + String.join(",\n", reasons); + ? String.join(",%n", reasons) + : getHeader() + String.join(",%n", reasons); } @Override diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleMaps.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleMaps.java index a905cb9ab1f..436ff3806b0 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleMaps.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleMaps.java @@ -66,45 +66,45 @@ private boolean valuesDiffer(K key) { @Override public String getIncompatibility() { - StringBuilder sb = new StringBuilder(); + final List reasons = new ArrayList<>(); - handleMissingElements(sb, getMissingElements(expected.keySet(), actual.keySet())); + handleMissingElements(reasons, getMissingElements(expected.keySet(), actual.keySet())); for (K key : expected.keySet()) { if (isKeyChanged(key)) { - sb.append(getGeneralScopedFormat(key)); + reasons.add(getGeneralScopedFormat(key)); } } - return sb.length() == 0 ? null : sb.toString(); + return reasons.isEmpty() ? null : String.join(",%n", reasons); } private String getDomainIncompatibility() { - StringBuilder sb = new StringBuilder(); + final List reasons = new ArrayList<>(); - handleMissingElements(sb, getDomainScopedKeys(getMissingElements(expected.keySet(), actual.keySet()))); + handleMissingElements(reasons, getDomainScopedKeys(getMissingElements(expected.keySet(), actual.keySet()))); for (K key : expected.keySet()) { if (isKeyChanged(key) && isDomainKey(key)) { - sb.append(getDomainScopedFormat(ELEMENT_NAMES_MAP.get(key), key)); + reasons.add(getDomainScopedFormat(ELEMENT_NAMES_MAP.get(key), key)); } } - return sb.length() == 0 ? null : sb.toString(); + return reasons.isEmpty() ? null : String.join(",%n", reasons); } private String getUnknownIncompatibility() { - StringBuilder sb = new StringBuilder(); + final List reasons = new ArrayList<>(); - handleMissingElements(sb, getUnknownScopedKeys(getMissingElements(expected.keySet(), actual.keySet()))); + handleMissingElements(reasons, getUnknownScopedKeys(getMissingElements(expected.keySet(), actual.keySet()))); for (K key : expected.keySet()) { if (isKeyChanged(key) && !isDomainKey(key)) { - sb.append(getGeneralScopedFormat(key)); + reasons.add(getGeneralScopedFormat(key)); } } - return sb.length() == 0 ? null : sb.toString(); + return reasons.isEmpty() ? null : String.join(",%n", reasons); } private boolean isKeyChanged(K key) { @@ -113,7 +113,7 @@ private boolean isKeyChanged(K key) { private String getDomainScopedFormat(String name, K key) { return String.format( - "'%s' changed from '%s' to '%s'%n", + "'%s' changed from '%s' to '%s'", name, getValue(actual.get(key)), getValue(expected.get(key))); } @@ -121,9 +121,9 @@ private String getGeneralScopedFormat(K key) { return String.format("%s %s", description, getDomainScopedFormat((String)key, key)); } - private void handleMissingElements(StringBuilder sb, Set missingKeys) { + private void handleMissingElements(List reasons, Set missingKeys) { if (!missingKeys.isEmpty()) { - sb.append(String.format("%s changed and contains '%s' as well%n", description, missingKeys)); + reasons.add(String.format("%s changed and contains '%s' as well", description, missingKeys)); } } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleSets.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleSets.java index 7b1a9d34f22..29bfd49f4da 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleSets.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleSets.java @@ -30,7 +30,7 @@ public boolean isCompatible() { @Override public String getIncompatibility() { return String.format( - "%s contains additional elements %s", description, getMissingElements(expected, actual)); + "'%s' contains additional elements '%s'", description, getMissingElements(expected, actual)); } @Override diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/Equality.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/Equality.java index 4f9342ea356..4e92075b7ff 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/Equality.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/Equality.java @@ -27,7 +27,7 @@ public boolean isCompatible() { @Override public String getIncompatibility() { - return description + " changed from " + "'" + actual + "'" + " to " + "'" + expected + "'"; + return "'" + description + "'" + " changed from '" + actual + "' to '" + expected + "'"; } @Override diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodCompatibility.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodCompatibility.java index 7e9899e841d..e99814da3f6 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodCompatibility.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodCompatibility.java @@ -255,7 +255,7 @@ public boolean isCompatible() { public String getIncompatibility() { return String.format( "%s probe changed from " - + "'initial delay %d, timeout %d and period %d' to 'initial delay %d, timeout %d and period %d'\n", + + "'initial delay %d, timeout %d and period %d' to 'initial delay %d, timeout %d and period %d'", description, actual.getInitialDelaySeconds(), actual.getTimeoutSeconds(), diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java index 01ef1d681ea..3896dac13ce 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/PodHelperTestBase.java @@ -1797,7 +1797,7 @@ public void whenImageChanged_domainRollStartEventCreatedWithCorrectMessage() assertThat( "Expected Event " + DOMAIN_ROLL_STARTING + " expected with message not found", getExpectedEventMessage(DOMAIN_ROLL_STARTING), - stringContainsInOrder("Rolling restart", UID, "image changed","adfgg")); + stringContainsInOrder("Rolling restart", UID, "image", "changed","adfgg")); } @Test @@ -1922,8 +1922,7 @@ public void whenImageDomainHomeAndRestartVersionChanged_domainRollStartEventCrea /* message: Rolling restart the pods in domain uid1 because domain restart version changed, - image changed from image:latest to adfgg, - imagePullPolicy changed from image:latest to IfNotPresent, + 'image' changed from image:latest to adfgg, 'domainHome' changed from '/u01/oracle/user_projects/domains' to '12345' */ assertThat( @@ -1931,7 +1930,7 @@ public void whenImageDomainHomeAndRestartVersionChanged_domainRollStartEventCrea getExpectedEventMessage(DOMAIN_ROLL_STARTING), stringContainsInOrder("Rolling restart", UID, "domain restart version changed", - "image changed", "adfgg", + "image", "changed", "adfgg", "domainHome", "changed", "12345")); } @@ -1951,7 +1950,7 @@ public void whenImageDomainHomeAndWebLogicZipHashChanged_domainRollStartEventCre /* message: Rolling restart the pods in domain uid1 because domain restart version changed, - imagePullPolicy changed from image:latest to IfNotPresent, + 'image' changed from image:latest to 'adcgg', 'domainHome' changed from '/u01/oracle/user_projects/domains' to '12345', WebLogic domain configuration changed */ @@ -1959,7 +1958,7 @@ public void whenImageDomainHomeAndWebLogicZipHashChanged_domainRollStartEventCre "Expected Event " + DOMAIN_ROLL_STARTING + " expected with message not found", getExpectedEventMessage(DOMAIN_ROLL_STARTING), stringContainsInOrder("Rolling restart", UID, - "image changed", "adfgg", + "image", "changed", "adfgg", "domainHome", "changed", "12345", "WebLogic domain configuration changed")); } From 9cdff47e2f6f6dfaa2d11a61a5590ffffcba6fb5 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Thu, 13 May 2021 17:14:54 -0400 Subject: [PATCH 05/10] more format/style changes --- .../operator/helpers/CollectiveCompatibility.java | 6 +++--- .../oracle/kubernetes/operator/helpers/CompatibleMaps.java | 6 +++--- .../kubernetes/operator/helpers/PodCompatibility.java | 3 +-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CollectiveCompatibility.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CollectiveCompatibility.java index 94af3a53e08..46dbabb3e8f 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CollectiveCompatibility.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CollectiveCompatibility.java @@ -41,7 +41,7 @@ public String getIncompatibility() { reasons.add(getIndent() + check.getIncompatibility()); } } - return reasons.isEmpty() ? null : getHeader() + String.join(",%n", reasons); + return reasons.isEmpty() ? null : getHeader() + String.join(",\n", reasons); } @Override @@ -55,8 +55,8 @@ public String getScopedIncompatibility(CompatibilityScope scope) { return reasons.isEmpty() ? null : scope == CompatibilityScope.DOMAIN - ? String.join(",%n", reasons) - : getHeader() + String.join(",%n", reasons); + ? String.join(",\n", reasons) + : getHeader() + String.join(",\n", reasons); } @Override diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleMaps.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleMaps.java index 436ff3806b0..969889f8930 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleMaps.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CompatibleMaps.java @@ -76,7 +76,7 @@ public String getIncompatibility() { } } - return reasons.isEmpty() ? null : String.join(",%n", reasons); + return reasons.isEmpty() ? null : String.join(",\n", reasons); } private String getDomainIncompatibility() { @@ -90,7 +90,7 @@ private String getDomainIncompatibility() { } } - return reasons.isEmpty() ? null : String.join(",%n", reasons); + return reasons.isEmpty() ? null : String.join(",\n", reasons); } private String getUnknownIncompatibility() { @@ -104,7 +104,7 @@ private String getUnknownIncompatibility() { } } - return reasons.isEmpty() ? null : String.join(",%n", reasons); + return reasons.isEmpty() ? null : String.join(",\n", reasons); } private boolean isKeyChanged(K key) { diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodCompatibility.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodCompatibility.java index e99814da3f6..8f9d112d413 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodCompatibility.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodCompatibility.java @@ -254,8 +254,7 @@ public boolean isCompatible() { @Override public String getIncompatibility() { return String.format( - "%s probe changed from " - + "'initial delay %d, timeout %d and period %d' to 'initial delay %d, timeout %d and period %d'", + "%s probe {initial delay, timeout, period} changed from " + "'{%d, %d, %d}' to '{%d, %d, %d}'", description, actual.getInitialDelaySeconds(), actual.getTimeoutSeconds(), From 5582024d60f5430e23cd681680b68a5d7be09ef6 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Fri, 14 May 2021 15:50:22 -0400 Subject: [PATCH 06/10] fix an issue in DomainProcessingCompleted logic --- .../operator/helpers/EventHelper.java | 44 ++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/EventHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/EventHelper.java index 57019ff7002..99461f1d4e3 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/EventHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/EventHelper.java @@ -193,7 +193,7 @@ public NextAction onSuccess(Packet packet, CallResponse callRespons } Optional.ofNullable(packet.getSpi(DomainPresenceInfo.class)) - .ifPresent(dpi -> dpi.setLastEventItem(eventData.eventItem)); + .ifPresent(dpi -> setLastEventItemIfNeeded(dpi, eventData.eventItem)); return doNext(packet); } @@ -228,6 +228,12 @@ private void clearNamespaceStartingFlag() { } } + private void setLastEventItemIfNeeded(DomainPresenceInfo dpi, EventItem eventItem) { + if (eventItem.shouldSetLastEventItem()) { + dpi.setLastEventItem(eventItem); + } + } + private class ReplaceEventResponseStep extends ResponseStep { Step replaceEventStep; CoreV1Event existingEvent; @@ -241,7 +247,7 @@ private class ReplaceEventResponseStep extends ResponseStep { @Override public NextAction onSuccess(Packet packet, CallResponse callResponse) { Optional.ofNullable(packet.getSpi(DomainPresenceInfo.class)) - .ifPresent(dpi -> dpi.setLastEventItem(eventData.eventItem)); + .ifPresent(dpi -> setLastEventItemIfNeeded(dpi, eventData.eventItem)); return doNext(packet); } @@ -386,6 +392,11 @@ public String getReason() { public String getPattern() { return DOMAIN_PROCESSING_STARTING_PATTERN; } + + @Override + public boolean shouldSetLastEventItem() { + return true; + } }, DOMAIN_PROCESSING_COMPLETED { @Override @@ -397,6 +408,11 @@ public String getReason() { public String getPattern() { return DOMAIN_PROCESSING_COMPLETED_PATTERN; } + + @Override + public boolean shouldSetLastEventItem() { + return true; + } }, DOMAIN_PROCESSING_FAILED { @Override @@ -419,6 +435,11 @@ public String getMessage(EventData eventData) { return getMessageFromEventData(eventData); } + @Override + public boolean shouldSetLastEventItem() { + return true; + } + }, DOMAIN_PROCESSING_RETRYING { @Override @@ -430,6 +451,11 @@ public String getReason() { public String getPattern() { return DOMAIN_PROCESSING_RETRYING_PATTERN; } + + @Override + public boolean shouldSetLastEventItem() { + return true; + } }, DOMAIN_PROCESSING_ABORTED { @Override @@ -452,6 +478,11 @@ public String getMessage(EventData eventData) { return getMessageFromEventData(eventData); } + @Override + public boolean shouldSetLastEventItem() { + return true; + } + }, DOMAIN_ROLL_STARTING { @Override @@ -502,6 +533,11 @@ public String getPattern() { public String getMessage(EventData eventData) { return getMessageFromEventData(eventData); } + + @Override + public boolean shouldSetLastEventItem() { + return true; + } }, POD_CYCLE_STARTING { @Override @@ -719,6 +755,10 @@ String getType() { return EVENT_NORMAL; } + boolean shouldSetLastEventItem() { + return false; + } + public abstract String getPattern(); public abstract String getReason(); From e0c12e9f5c7fbdda3583ef9738676ef640ca5d3e Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Mon, 17 May 2021 22:20:54 -0400 Subject: [PATCH 07/10] doc --- .../managing-domains/domain-events.md | 102 +++++++++++++++++- 1 file changed, 100 insertions(+), 2 deletions(-) diff --git a/documentation/staging/content/userguide/managing-domains/domain-events.md b/documentation/staging/content/userguide/managing-domains/domain-events.md index 6429b90e213..b4bd5228d7f 100644 --- a/documentation/staging/content/userguide/managing-domains/domain-events.md +++ b/documentation/staging/content/userguide/managing-domains/domain-events.md @@ -27,9 +27,12 @@ The operator generates these event types in a domain namespace, which indicate t * `DomainDeleted`: An existing domain has been deleted. * `DomainProcessingStarting`: The operator has started to process a new domain or to update an existing domain. This event may be a result of a `DomainCreate`, `DomainChanged`, or `DomainDeleted` event, or a result of a retry after a failed attempt. * `DomainProcessingFailed`: The operator has encountered a problem while it was processing the domain resource. The failure either could be a configuration error or a Kubernetes API error. - * `DomainProcessingRetrying`: The operator is going to retry the processing of a domain after it encountered an failure. + * `DomainProcessingRetrying`: The operator is going to retry the processing of a domain after it encountered a failure. * `DomainProcessingCompleted`: The operator successfully completed the processing of a domain resource. * `DomainProcessingAborted`: The operator stopped processing a domain when the operator encountered a fatal error or a failure that persisted after the specified maximum number of retries. + * `DomainRollStarting`: The operator has started a rolling restart of a domain after it detects a change or changes in the domain resource or WebLogic domain configuration, which require a restart of all running pods in the domain. Please refer to [Fields in domain resource that cause servers to be restarted]({{< relref "/userguide/managing-domains/domain-lifecycle/startup#fields-that-cause-servers-to-be-restarted" >}}). + * `DomainRollCompleted`: The operator has successfully completed a rolling restart of a domain. + * `PodCycleStarting`: The operator has started to replace a server pod after it detects that the current pod is not conformed to the current domain resource or WebLogic domain configuration. * `DomainValidationError`: A validation error or warning is found in a domain resource. Please refer to the event message for details. * `NamespaceWatchingStarted`: The operator has started watching for domains in a namespace. * `NamespaceWatchingStopped`: The operator has stopped watching for domains in a namespace. Note that the creation of this event in a domain namespace is the operator's best effort only; the event will not be generated if the required Kubernetes privilege is removed when a namespace is no longer managed by the operator. @@ -85,7 +88,7 @@ $ kubectl get events -n [namespace] --selector=weblogic.createdByOperator=true To get all the events that are generated by the operator for a particular domain resource, for example `sample-domain1`, run: ```shell -$ kubectl get events -n [namespace] --selector=weblogic.domainUID=sample-domain1,weblogic.createdByOperator=true --sort-by=lastTimestamp +$ kubectl get events -n [namespace] --selector=weblogic.domainUID=sample-domain1&&weblogic.createdByOperator=true --sort-by=lastTimestamp ``` #### Examples of generated events @@ -285,3 +288,98 @@ Source: Type: Normal Events: ``` + +Example of the sequence of operator generated events in a domain rolling restart after the domain resource's `image` and `logHomeEnabled` changed. + +``` +LAST SEEN TYPE REASON OBJECT MESSAGE +2m58s Normal DomainChanged domain/domain1 Domain resource domain1 was changed +2m58s Normal DomainProcessingStarting domain/domain1 Creating or updating Kubernetes presence for WebLogic Domain with UID domain1 +2m58s Normal DomainRollStarting domain/domain1 Rolling restart WebLogic server pods in domain domain1 because: 'image' changed from 'oracle/weblogic' to 'oracle/weblogic:14.1.1.0', + 'logHome' changed from 'null' to '/shared/logs/domain1' +2m58s Normal PodCycleStarting domain/domain1 Replacing pod domain1-adminserver because: In container 'weblogic-server': + 'image' changed from 'oracle/weblogic' to 'oracle/weblogic:14.1.1.0', + env 'LOG_HOME' changed from 'null' to '/shared/logs/domain1' +2m7s Normal PodCycleStarting domain/domain1 Replacing pod domain1-managed-server1 because: In container 'weblogic-server': + 'image' changed from 'oracle/weblogic' to 'oracle/weblogic:14.1.1.0', + env 'LOG_HOME' changed from 'null' to '/shared/logs/domain1' +71s Normal PodCycleStarting domain/domain1 Replacing pod domain1-managed-server2 because: In container 'weblogic-server': + 'image' changed from 'oracle/weblogic' to 'oracle/weblogic:14.1.1.0', + env 'LOG_HOME' changed from 'null' to '/shared/logs/domain1' +19s Normal DomainRollCompleted domain/domain1 Rolling restart of domain domain1 completed +19s Normal DomainProcessingCompleted domain/domain1 Successfully completed processing domain resource domain1 + +``` + +Example of a `DomainRollStarting` event: + +``` +Name: domain1.DomainRollStarting.7d33e9b787e9c318 +Namespace: weblogic-domain +Labels: weblogic.createdByOperator=true + weblogic.domainUID=domain1 +Annotations: +API Version: v1 +Count: 1 +Event Time: +First Timestamp: 2021-05-18T02:00:24Z +Involved Object: + API Version: weblogic.oracle/v8 + Kind: Domain + Name: domain1 + Namespace: weblogic-domain + UID: 5df7dcda-d606-4509-9a06-32f25e16e166 +Kind: Event +Last Timestamp: 2021-05-18T02:00:24Z +Message: Rolling restart WebLogic server pods in domain domain1 because: 'image' changed from 'oracle/weblogic' to 'oracle/weblogic:14.1.1.0', + 'logHome' changed from 'null' to '/shared/logs/domain1' +Metadata: + Creation Timestamp: 2021-05-18T02:00:24Z + Resource Version: 12842363 + Self Link: /api/v1/namespaces/weblogic-domain/events/domain1.DomainRollStarting.7d33e9b787e9c318 + UID: 6ec92655-9d06-43b1-8b26-c01ebccadecf +Reason: DomainRollStarting +Reporting Component: weblogic.operator +Reporting Instance: weblogic-operator-fc4ccc8b5-rh4v6 +Source: +Type: Normal +Events: + +``` + +Example of a `PodCycleStarting` event: + +``` +Name: domain1.PodCycleStarting.7d34bc3232231f49 +Namespace: weblogic-domain +Labels: weblogic.createdByOperator=true + weblogic.domainUID=domain1 +Annotations: +API Version: v1 +Count: 1 +Event Time: +First Timestamp: 2021-05-18T02:01:18Z +Involved Object: + API Version: weblogic.oracle/v8 + Kind: Domain + Name: domain1 + Namespace: weblogic-domain + UID: 5df7dcda-d606-4509-9a06-32f25e16e166 +Kind: Event +Last Timestamp: 2021-05-18T02:01:18Z +Message: Replacing pod domain1-managed-server1 because: In container 'weblogic-server': + 'image' changed from 'oracle/weblogic' to 'oracle/weblogic:14.1.1.0', + env 'LOG_HOME' changed from 'null' to '/shared/logs/domain1' +Metadata: + Creation Timestamp: 2021-05-18T02:01:18Z + Resource Version: 12842530 + Self Link: /api/v1/namespaces/weblogic-domain/events/domain1.PodCycleStarting.7d34bc3232231f49 + UID: 4c6a203e-9b93-4b46-b9e3-1a448b52c7ca +Reason: PodCycleStarting +Reporting Component: weblogic.operator +Reporting Instance: weblogic-operator-fc4ccc8b5-rh4v6 +Source: +Type: Normal +Events: + +``` \ No newline at end of file From 63f17c91dff51793cf37c7297810c40e27663ccd Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Tue, 18 May 2021 09:27:11 -0400 Subject: [PATCH 08/10] more doc --- .../managing-domains/domain-events.md | 60 ++++++++++--------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/documentation/staging/content/userguide/managing-domains/domain-events.md b/documentation/staging/content/userguide/managing-domains/domain-events.md index b4bd5228d7f..eaaa7dce1be 100644 --- a/documentation/staging/content/userguide/managing-domains/domain-events.md +++ b/documentation/staging/content/userguide/managing-domains/domain-events.md @@ -30,7 +30,7 @@ The operator generates these event types in a domain namespace, which indicate t * `DomainProcessingRetrying`: The operator is going to retry the processing of a domain after it encountered a failure. * `DomainProcessingCompleted`: The operator successfully completed the processing of a domain resource. * `DomainProcessingAborted`: The operator stopped processing a domain when the operator encountered a fatal error or a failure that persisted after the specified maximum number of retries. - * `DomainRollStarting`: The operator has started a rolling restart of a domain after it detects a change or changes in the domain resource or WebLogic domain configuration, which require a restart of all running pods in the domain. Please refer to [Fields in domain resource that cause servers to be restarted]({{< relref "/userguide/managing-domains/domain-lifecycle/startup#fields-that-cause-servers-to-be-restarted" >}}). + * `DomainRollStarting`: The operator has started a rolling restart of a domain after it detects a change or changes in the domain resource or WebLogic domain configuration, which require a restart of all running pods in the domain. The operator makes its best efforts in providing the reason of the rolling restart. If the rolling restart is caused by one or more of the following fields in the domain resource, the event message contains the fields that have changed as well as their old and new values: image, imagePullPolicy, livenessProbe, readinessProbe, domain restartVersion, domainHome, includeServerOutInPodLog, and logHome. When the rolling restart is caused by changes in the other domain resource fields (Please refer to [Fields in domain resource that cause servers to be restarted]({{< relref "/userguide/managing-domains/domain-lifecycle/startup#fields-that-cause-servers-to-be-restarted" >}})), the event message simply indicates that the domain resource has changed. If the cause of the rolling restart is in the WebLogic domain configuration, the event message indicates that as well without the details. * `DomainRollCompleted`: The operator has successfully completed a rolling restart of a domain. * `PodCycleStarting`: The operator has started to replace a server pod after it detects that the current pod is not conformed to the current domain resource or WebLogic domain configuration. * `DomainValidationError`: A validation error or warning is found in a domain resource. Please refer to the event message for details. @@ -291,33 +291,35 @@ Events: Example of the sequence of operator generated events in a domain rolling restart after the domain resource's `image` and `logHomeEnabled` changed. +The output of command `kubectl get events -n sample-domain1-ns --selector=weblogic.domainUID=sample-domain1,weblogic.createdByOperator=true --sort-by=lastTimestamp'. + ``` LAST SEEN TYPE REASON OBJECT MESSAGE -2m58s Normal DomainChanged domain/domain1 Domain resource domain1 was changed -2m58s Normal DomainProcessingStarting domain/domain1 Creating or updating Kubernetes presence for WebLogic Domain with UID domain1 -2m58s Normal DomainRollStarting domain/domain1 Rolling restart WebLogic server pods in domain domain1 because: 'image' changed from 'oracle/weblogic' to 'oracle/weblogic:14.1.1.0', - 'logHome' changed from 'null' to '/shared/logs/domain1' -2m58s Normal PodCycleStarting domain/domain1 Replacing pod domain1-adminserver because: In container 'weblogic-server': +2m58s Normal DomainChanged domain/sample-domain1 Domain resource sample-domain1 was changed +2m58s Normal DomainProcessingStarting domain/sample-domain1 Creating or updating Kubernetes presence for WebLogic Domain with UID sample-domain1 +2m58s Normal DomainRollStarting domain/sample-domain1 Rolling restart WebLogic server pods in domain sample-domain1 because: 'image' changed from 'oracle/weblogic' to 'oracle/weblogic:14.1.1.0', + 'logHome' changed from 'null' to '/shared/logs/sample-domain1' +2m58s Normal PodCycleStarting domain/sample-domain1 Replacing pod sample-domain1-adminserver because: In container 'weblogic-server': 'image' changed from 'oracle/weblogic' to 'oracle/weblogic:14.1.1.0', - env 'LOG_HOME' changed from 'null' to '/shared/logs/domain1' -2m7s Normal PodCycleStarting domain/domain1 Replacing pod domain1-managed-server1 because: In container 'weblogic-server': + env 'LOG_HOME' changed from 'null' to '/shared/logs/sample-domain1' +2m7s Normal PodCycleStarting domain/sample-domain1 Replacing pod sample-domain1-managed-server1 because: In container 'weblogic-server': 'image' changed from 'oracle/weblogic' to 'oracle/weblogic:14.1.1.0', - env 'LOG_HOME' changed from 'null' to '/shared/logs/domain1' -71s Normal PodCycleStarting domain/domain1 Replacing pod domain1-managed-server2 because: In container 'weblogic-server': + env 'LOG_HOME' changed from 'null' to '/shared/logs/sample-domain1' +71s Normal PodCycleStarting domain/sample-domain1 Replacing pod sample-domain1-managed-server2 because: In container 'weblogic-server': 'image' changed from 'oracle/weblogic' to 'oracle/weblogic:14.1.1.0', - env 'LOG_HOME' changed from 'null' to '/shared/logs/domain1' -19s Normal DomainRollCompleted domain/domain1 Rolling restart of domain domain1 completed -19s Normal DomainProcessingCompleted domain/domain1 Successfully completed processing domain resource domain1 + env 'LOG_HOME' changed from 'null' to '/shared/logs/sample-domain1' +19s Normal DomainRollCompleted domain/sample-domain1 Rolling restart of domain sample-domain1 completed +19s Normal DomainProcessingCompleted domain/sample-domain1 Successfully completed processing domain resource sample-domain1 ``` Example of a `DomainRollStarting` event: ``` -Name: domain1.DomainRollStarting.7d33e9b787e9c318 -Namespace: weblogic-domain +Name: sample-domain1.DomainRollStarting.7d33e9b787e9c318 +Namespace: sample-domain1-ns Labels: weblogic.createdByOperator=true - weblogic.domainUID=domain1 + weblogic.domainUID=sample-domain1 Annotations: API Version: v1 Count: 1 @@ -326,17 +328,17 @@ First Timestamp: 2021-05-18T02:00:24Z Involved Object: API Version: weblogic.oracle/v8 Kind: Domain - Name: domain1 - Namespace: weblogic-domain + Name: sample-domain1 + Namespace: sample-domain1-ns UID: 5df7dcda-d606-4509-9a06-32f25e16e166 Kind: Event Last Timestamp: 2021-05-18T02:00:24Z -Message: Rolling restart WebLogic server pods in domain domain1 because: 'image' changed from 'oracle/weblogic' to 'oracle/weblogic:14.1.1.0', - 'logHome' changed from 'null' to '/shared/logs/domain1' +Message: Rolling restart WebLogic server pods in domain sample-domain1 because: 'image' changed from 'oracle/weblogic' to 'oracle/weblogic:14.1.1.0', + 'logHome' changed from 'null' to '/shared/logs/sample-domain1' Metadata: Creation Timestamp: 2021-05-18T02:00:24Z Resource Version: 12842363 - Self Link: /api/v1/namespaces/weblogic-domain/events/domain1.DomainRollStarting.7d33e9b787e9c318 + Self Link: /api/v1/namespaces/sample-domain1-ns/events/sample-domain1.DomainRollStarting.7d33e9b787e9c318 UID: 6ec92655-9d06-43b1-8b26-c01ebccadecf Reason: DomainRollStarting Reporting Component: weblogic.operator @@ -350,10 +352,10 @@ Events: Example of a `PodCycleStarting` event: ``` -Name: domain1.PodCycleStarting.7d34bc3232231f49 -Namespace: weblogic-domain +Name: sample-domain1.PodCycleStarting.7d34bc3232231f49 +Namespace: sample-domain1-ns Labels: weblogic.createdByOperator=true - weblogic.domainUID=domain1 + weblogic.domainUID=sample-domain1 Annotations: API Version: v1 Count: 1 @@ -362,18 +364,18 @@ First Timestamp: 2021-05-18T02:01:18Z Involved Object: API Version: weblogic.oracle/v8 Kind: Domain - Name: domain1 - Namespace: weblogic-domain + Name: sample-domain1 + Namespace: sample-domain1-ns UID: 5df7dcda-d606-4509-9a06-32f25e16e166 Kind: Event Last Timestamp: 2021-05-18T02:01:18Z -Message: Replacing pod domain1-managed-server1 because: In container 'weblogic-server': +Message: Replacing pod sample-domain1-managed-server1 because: In container 'weblogic-server': 'image' changed from 'oracle/weblogic' to 'oracle/weblogic:14.1.1.0', - env 'LOG_HOME' changed from 'null' to '/shared/logs/domain1' + env 'LOG_HOME' changed from 'null' to '/shared/logs/sample-domain1' Metadata: Creation Timestamp: 2021-05-18T02:01:18Z Resource Version: 12842530 - Self Link: /api/v1/namespaces/weblogic-domain/events/domain1.PodCycleStarting.7d34bc3232231f49 + Self Link: /api/v1/namespaces/sample-domain1-ns/events/sample-domain1.PodCycleStarting.7d34bc3232231f49 UID: 4c6a203e-9b93-4b46-b9e3-1a448b52c7ca Reason: PodCycleStarting Reporting Component: weblogic.operator From c8938b7008c80da2031e3d7e19126678a7912324 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Tue, 18 May 2021 13:10:15 -0400 Subject: [PATCH 09/10] address review comments to the doc section, plus minor refactoring --- .../managing-domains/domain-events.md | 20 +++++++++++++------ .../kubernetes/operator/EventConstants.java | 2 ++ .../operator/helpers/PodStepContext.java | 8 +++++--- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/documentation/staging/content/userguide/managing-domains/domain-events.md b/documentation/staging/content/userguide/managing-domains/domain-events.md index eaaa7dce1be..18126180049 100644 --- a/documentation/staging/content/userguide/managing-domains/domain-events.md +++ b/documentation/staging/content/userguide/managing-domains/domain-events.md @@ -30,9 +30,19 @@ The operator generates these event types in a domain namespace, which indicate t * `DomainProcessingRetrying`: The operator is going to retry the processing of a domain after it encountered a failure. * `DomainProcessingCompleted`: The operator successfully completed the processing of a domain resource. * `DomainProcessingAborted`: The operator stopped processing a domain when the operator encountered a fatal error or a failure that persisted after the specified maximum number of retries. - * `DomainRollStarting`: The operator has started a rolling restart of a domain after it detects a change or changes in the domain resource or WebLogic domain configuration, which require a restart of all running pods in the domain. The operator makes its best efforts in providing the reason of the rolling restart. If the rolling restart is caused by one or more of the following fields in the domain resource, the event message contains the fields that have changed as well as their old and new values: image, imagePullPolicy, livenessProbe, readinessProbe, domain restartVersion, domainHome, includeServerOutInPodLog, and logHome. When the rolling restart is caused by changes in the other domain resource fields (Please refer to [Fields in domain resource that cause servers to be restarted]({{< relref "/userguide/managing-domains/domain-lifecycle/startup#fields-that-cause-servers-to-be-restarted" >}})), the event message simply indicates that the domain resource has changed. If the cause of the rolling restart is in the WebLogic domain configuration, the event message indicates that as well without the details. + * `DomainRollStarting`: The operator has detected domain resource or Model in Image model + updates that require it to perform a rolling restart of the domain. + If the domain roll is due to a change to domain resource fields + `image`, `imagePullPolicy`, `livenessProbe`, `readinessProbe`, `restartVersion`, + `domainHome`, `includeServerOutInPodLog`, or `logHome`, then + the event message reports the field name plus its old and new values. + If the domain roll is due to other domain resource changes that cause servers to be restarted + (see [full list of fields that cause servers to be restarted]({{< relref "/userguide/managing-domains/domain-lifecycle/startup#fields-that-cause-servers-to-be-restarted" >}})), + then the event message simply reports that the domain resource has changed. + If the domain roll is due to a Model in Image model update, + then the event message reports there has been a change in the WebLogic domain configuration without the details. * `DomainRollCompleted`: The operator has successfully completed a rolling restart of a domain. - * `PodCycleStarting`: The operator has started to replace a server pod after it detects that the current pod is not conformed to the current domain resource or WebLogic domain configuration. + * `PodCycleStarting`: The operator has started to replace a server pod after it detects that the current pod does not conform to the current domain resource or WebLogic domain configuration. * `DomainValidationError`: A validation error or warning is found in a domain resource. Please refer to the event message for details. * `NamespaceWatchingStarted`: The operator has started watching for domains in a namespace. * `NamespaceWatchingStopped`: The operator has stopped watching for domains in a namespace. Note that the creation of this event in a domain namespace is the operator's best effort only; the event will not be generated if the required Kubernetes privilege is removed when a namespace is no longer managed by the operator. @@ -88,7 +98,7 @@ $ kubectl get events -n [namespace] --selector=weblogic.createdByOperator=true To get all the events that are generated by the operator for a particular domain resource, for example `sample-domain1`, run: ```shell -$ kubectl get events -n [namespace] --selector=weblogic.domainUID=sample-domain1&&weblogic.createdByOperator=true --sort-by=lastTimestamp +$ kubectl get events -n [namespace] --selector=weblogic.domainUID=sample-domain1,weblogic.createdByOperator=true --sort-by=lastTimestamp ``` #### Examples of generated events @@ -289,9 +299,7 @@ Type: Normal Events: ``` -Example of the sequence of operator generated events in a domain rolling restart after the domain resource's `image` and `logHomeEnabled` changed. - -The output of command `kubectl get events -n sample-domain1-ns --selector=weblogic.domainUID=sample-domain1,weblogic.createdByOperator=true --sort-by=lastTimestamp'. +Example of the sequence of operator generated events in a domain rolling restart after the domain resource's `image` and `logHomeEnabled` changed, which is the output of the command `kubectl get events -n sample-domain1-ns --selector=weblogic.domainUID=sample-domain1,weblogic.createdByOperator=true --sort-by=lastTimestamp'. ``` LAST SEEN TYPE REASON OBJECT MESSAGE diff --git a/operator/src/main/java/oracle/kubernetes/operator/EventConstants.java b/operator/src/main/java/oracle/kubernetes/operator/EventConstants.java index 08900a676c8..5ff2ba69310 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/EventConstants.java +++ b/operator/src/main/java/oracle/kubernetes/operator/EventConstants.java @@ -53,4 +53,6 @@ public interface EventConstants { String START_MANAGING_NAMESPACE_FAILED_PATTERN = "Start managing namespace %s failed due to an authorization error"; String DOMAIN_ROLL_STARTING_PATTERN = "Rolling restart WebLogic server pods in domain %s because: %s"; String DOMAIN_ROLL_COMPLETED_PATTERN = "Rolling restart of domain %s completed"; + String ROLL_REASON_DOMAIN_RESOURCE_CHANGED = "domain resource changed"; + String ROLL_REASON_WEBLOGIC_CONFIGURATION_CHANGED = "WebLogic domain configuration changed due to a Model in Image model update"; } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java index b2a0b1b16c0..fc5b2e8fa44 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/PodStepContext.java @@ -69,6 +69,8 @@ import oracle.kubernetes.weblogic.domain.model.Shutdown; import org.apache.commons.lang3.builder.EqualsBuilder; +import static oracle.kubernetes.operator.EventConstants.ROLL_REASON_DOMAIN_RESOURCE_CHANGED; +import static oracle.kubernetes.operator.EventConstants.ROLL_REASON_WEBLOGIC_CONFIGURATION_CHANGED; import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.NUM_CONFIG_MAPS; import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_STATE_LABEL; import static oracle.kubernetes.operator.LabelConstants.MII_UPDATED_RESTART_REQUIRED_LABEL; @@ -474,14 +476,14 @@ private String getDomainIncompatibility(V1Pod pod) { String domainIncompatibility = getReasonToRecycle(pod, DOMAIN); if (!haveReasonsToRoll(domainIncompatibility) && haveReasonsToRoll(getReasonToRecycle(pod, UNKNOWN))) { - domainIncompatibility = "domain resource changed"; + domainIncompatibility = ROLL_REASON_DOMAIN_RESOURCE_CHANGED; } if (!canUseNewDomainZip(pod)) { if (haveReasonsToRoll(domainIncompatibility)) { - domainIncompatibility += ",\nWebLogic domain configuration changed"; + domainIncompatibility += ",\n" + ROLL_REASON_WEBLOGIC_CONFIGURATION_CHANGED; } else { - domainIncompatibility = "WebLogic domain configuration changed"; + domainIncompatibility = ROLL_REASON_WEBLOGIC_CONFIGURATION_CHANGED; } } return domainIncompatibility; From f2dde5ddea22ce73f5566784d64b698b2cf0bd36 Mon Sep 17 00:00:00 2001 From: Dongbo Xiao Date: Tue, 18 May 2021 13:11:28 -0400 Subject: [PATCH 10/10] minor style fix --- .../main/java/oracle/kubernetes/operator/EventConstants.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/operator/src/main/java/oracle/kubernetes/operator/EventConstants.java b/operator/src/main/java/oracle/kubernetes/operator/EventConstants.java index 5ff2ba69310..9be424e8b90 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/EventConstants.java +++ b/operator/src/main/java/oracle/kubernetes/operator/EventConstants.java @@ -54,5 +54,6 @@ public interface EventConstants { String DOMAIN_ROLL_STARTING_PATTERN = "Rolling restart WebLogic server pods in domain %s because: %s"; String DOMAIN_ROLL_COMPLETED_PATTERN = "Rolling restart of domain %s completed"; String ROLL_REASON_DOMAIN_RESOURCE_CHANGED = "domain resource changed"; - String ROLL_REASON_WEBLOGIC_CONFIGURATION_CHANGED = "WebLogic domain configuration changed due to a Model in Image model update"; + String ROLL_REASON_WEBLOGIC_CONFIGURATION_CHANGED + = "WebLogic domain configuration changed due to a Model in Image model update"; }