diff --git a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java index 6b80fcea5d8..c215d10fb81 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java +++ b/operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServersUpStep.java @@ -30,9 +30,11 @@ import oracle.kubernetes.operator.work.NextAction; import oracle.kubernetes.operator.work.Packet; import oracle.kubernetes.operator.work.Step; +import oracle.kubernetes.utils.OperatorUtils; import oracle.kubernetes.weblogic.domain.model.Domain; import oracle.kubernetes.weblogic.domain.model.ServerSpec; +import static java.util.Comparator.comparing; import static oracle.kubernetes.operator.DomainStatusUpdater.MANAGED_SERVERS_STARTING_PROGRESS_REASON; import static oracle.kubernetes.operator.DomainStatusUpdater.createProgressingStep; @@ -118,23 +120,30 @@ public NextAction apply(Packet packet) { private void addServersToFactory(@Nonnull ServersUpStepFactory factory, @Nonnull WlsDomainConfig wlsDomainConfig) { Set clusteredServers = new HashSet<>(); + List pendingServers = new ArrayList<>(); wlsDomainConfig.getClusterConfigs().values() - .forEach(wlsClusterConfig -> addClusteredServersToFactory(factory, clusteredServers, wlsClusterConfig)); + .forEach(wlsClusterConfig -> addClusteredServersToFactory( + factory, clusteredServers, wlsClusterConfig, pendingServers)); wlsDomainConfig.getServerConfigs().values().stream() .filter(wlsServerConfig -> !clusteredServers.contains(wlsServerConfig.getName())) - .forEach(wlsServerConfig -> factory.addServerIfNeeded(wlsServerConfig, null)); + .forEach(wlsServerConfig -> factory.addServerIfAlways(wlsServerConfig, null, pendingServers)); + + for (ServerConfig serverConfig : pendingServers) { + factory.addServerIfNeeded(serverConfig.wlsServerConfig, serverConfig.wlsClusterConfig); + } } - private void addClusteredServersToFactory(@Nonnull ServersUpStepFactory factory, Set clusteredServers, - @Nonnull WlsClusterConfig wlsClusterConfig) { + private void addClusteredServersToFactory( + @Nonnull ServersUpStepFactory factory, Set clusteredServers, + @Nonnull WlsClusterConfig wlsClusterConfig, List pendingServers) { factory.logIfInvalidReplicaCount(wlsClusterConfig); // We depend on 'getServerConfigs()' returning an ascending 'numero-lexi' // sorted list so that a cluster's "lowest named" servers have precedence // when the cluster's replica count is lower than the WL cluster size. wlsClusterConfig.getServerConfigs() .forEach(wlsServerConfig -> { - factory.addServerIfNeeded(wlsServerConfig, wlsClusterConfig); + factory.addServerIfAlways(wlsServerConfig, wlsClusterConfig, pendingServers); clusteredServers.add(wlsServerConfig.getName()); }); } @@ -148,7 +157,7 @@ Step createServerStep( static class ServersUpStepFactory { final WlsDomainConfig domainTopology; final Domain domain; - Collection startupInfos; + List startupInfos; List shutdownInfos = new ArrayList<>(); final Collection servers = new ArrayList<>(); final Collection preCreateServers = new ArrayList<>(); @@ -176,20 +185,15 @@ boolean shouldPrecreateServerService(ServerSpec server) { private void addServerIfNeeded(@Nonnull WlsServerConfig serverConfig, WlsClusterConfig clusterConfig) { String serverName = serverConfig.getName(); - if (servers.contains(serverName) || serverName.equals(domainTopology.getAdminServerName())) { + if (adminServerOrDone(serverName)) { return; } - String clusterName = clusterConfig == null ? null : clusterConfig.getClusterName(); + String clusterName = getClusterName(clusterConfig); ServerSpec server = domain.getServer(serverName, clusterName); if (server.shouldStart(getReplicaCount(clusterName))) { - servers.add(serverName); - if (shouldPrecreateServerService(server)) { - preCreateServers.add(serverName); - } - addStartupInfo(new ServerStartupInfo(serverConfig, clusterName, server)); - addToCluster(clusterName); + addServerToStart(serverConfig, clusterName, server); } else if (shouldPrecreateServerService(server)) { preCreateServers.add(serverName); addShutdownInfo(new ServerShutdownInfo(serverConfig, clusterName, server, true)); @@ -198,6 +202,15 @@ private void addServerIfNeeded(@Nonnull WlsServerConfig serverConfig, WlsCluster } } + private void addServerToStart(@Nonnull WlsServerConfig serverConfig, String clusterName, ServerSpec server) { + servers.add(serverConfig.getName()); + if (shouldPrecreateServerService(server)) { + preCreateServers.add(serverConfig.getName()); + } + addStartupInfo(new ServerStartupInfo(serverConfig, clusterName, server)); + addToCluster(clusterName); + } + boolean exceedsMaxConfiguredClusterSize(WlsClusterConfig clusterConfig) { if (clusterConfig != null) { String clusterName = clusterConfig.getClusterName(); @@ -218,6 +231,11 @@ private Step createNextStep(Step next) { } Collection getStartupInfos() { + if (startupInfos != null) { + Collections.sort( + startupInfos, + comparing((ServerStartupInfo sinfo) -> OperatorUtils.getSortingString(sinfo.getServerName()))); + } return startupInfos; } @@ -291,5 +309,41 @@ private void logIfInvalidReplicaCount(WlsClusterConfig clusterConfig) { logIfReplicasExceedsClusterServersMax(clusterConfig); logIfReplicasLessThanClusterServersMin(clusterConfig); } + + private void addServerIfAlways( + WlsServerConfig wlsServerConfig, + WlsClusterConfig wlsClusterConfig, + List pendingServers) { + String serverName = wlsServerConfig.getName(); + if (adminServerOrDone(serverName)) { + return; + } + String clusterName = getClusterName(wlsClusterConfig); + ServerSpec server = domain.getServer(serverName, clusterName); + if (server.alwaysStart()) { + addServerToStart(wlsServerConfig, clusterName, server); + } else { + pendingServers.add(new ServerConfig(wlsClusterConfig, wlsServerConfig)); + } + } + + private boolean adminServerOrDone(String serverName) { + return servers.contains(serverName) || serverName.equals(domainTopology.getAdminServerName()); + } + + private static String getClusterName(WlsClusterConfig clusterConfig) { + return clusterConfig == null ? null : clusterConfig.getClusterName(); + } + + } + + private static class ServerConfig { + protected WlsServerConfig wlsServerConfig; + protected WlsClusterConfig wlsClusterConfig; + + ServerConfig(WlsClusterConfig cluster, WlsServerConfig server) { + this.wlsClusterConfig = cluster; + this.wlsServerConfig = server; + } } } diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/ManagedServerSpecCommonImpl.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/ManagedServerSpecCommonImpl.java index dda92ad366c..24d66e6d44a 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/ManagedServerSpecCommonImpl.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/ManagedServerSpecCommonImpl.java @@ -25,4 +25,12 @@ public boolean shouldStart(int currentReplicas) { } return super.shouldStart(currentReplicas); } + + @Override + public boolean alwaysStart() { + if (isStartAdminServerOnly()) { + return false; + } + return super.alwaysStart(); + } } diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/ServerSpec.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/ServerSpec.java index 620d4226a2a..b357bad34bf 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/ServerSpec.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/ServerSpec.java @@ -160,4 +160,6 @@ public interface ServerSpec { String getClusterRestartVersion(); String getServerRestartVersion(); + + boolean alwaysStart(); } diff --git a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/ServerSpecCommonImpl.java b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/ServerSpecCommonImpl.java index 592bf6c9523..66389fc07e5 100644 --- a/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/ServerSpecCommonImpl.java +++ b/operator/src/main/java/oracle/kubernetes/weblogic/domain/model/ServerSpecCommonImpl.java @@ -142,6 +142,10 @@ private ServerStartPolicy getEffectiveServerStartPolicy() { .orElse(ServerStartPolicy.getDefaultPolicy()); } + public boolean alwaysStart() { + return ServerStartPolicy.ALWAYS.equals(getEffectiveServerStartPolicy()); + } + @Nonnull @Override public ProbeTuning getLivenessProbe() { diff --git a/operator/src/test/java/oracle/kubernetes/operator/DomainProcessorTest.java b/operator/src/test/java/oracle/kubernetes/operator/DomainProcessorTest.java index 4519a0752dc..3034cae2a6b 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/DomainProcessorTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/DomainProcessorTest.java @@ -79,6 +79,8 @@ import static oracle.kubernetes.operator.helpers.KubernetesTestSupport.SERVICE; import static oracle.kubernetes.operator.logging.MessageKeys.NOT_STARTING_DOMAINUID_THREAD; import static oracle.kubernetes.utils.LogMatcher.containsFine; +import static oracle.kubernetes.weblogic.domain.model.ConfigurationConstants.START_ALWAYS; +import static oracle.kubernetes.weblogic.domain.model.ConfigurationConstants.START_NEVER; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.empty; @@ -281,6 +283,224 @@ public void whenDomainShutDown_ignoreNonOperatorServices() { assertThat(getRunningPods(), empty()); } + @Test + public void whenClusterReplicas2_server3WithAlwaysPolicy_establishMatchingPresence() { + domainConfigurator.configureCluster(CLUSTER).withReplicas(2); + domainConfigurator.configureServer(MS_PREFIX + 3).withServerStartPolicy(START_ALWAYS); + + DomainPresenceInfo info = new DomainPresenceInfo(newDomain); + processor.createMakeRightOperation(info).execute(); + + List runningPods = getRunningPods(); + //one introspector pod, one admin server pod and two managed server pods + assertThat(runningPods.size(), equalTo(4)); + + assertServerPodAndServicePresent(info, ADMIN_NAME); + for (Integer i : Arrays.asList(1,3)) { + assertServerPodAndServicePresent(info, MS_PREFIX + i); + } + assertServerPodNotPresent(info, MS_PREFIX + 2); + + assertThat(info.getClusterService(CLUSTER), notNullValue()); + } + + @Test + public void whenClusterScaleUpToReplicas3_fromReplicas2_server3WithAlwaysPolicy_establishMatchingPresence() + throws JsonProcessingException { + establishPreviousIntrospection(null, Arrays.asList(1, 3)); + + domainConfigurator.configureCluster(CLUSTER).withReplicas(3); + domainConfigurator.configureServer(MS_PREFIX + 3).withServerStartPolicy(START_ALWAYS); + + DomainPresenceInfo info = new DomainPresenceInfo(newDomain); + processor.createMakeRightOperation(info).execute(); + + List runningPods = getRunningPods(); + //one introspector pod, one admin server pod and three managed server pods + assertThat(runningPods.size(), equalTo(5)); + + assertServerPodAndServicePresent(info, ADMIN_NAME); + for (Integer i : Arrays.asList(1,2,3)) { + assertServerPodAndServicePresent(info, MS_PREFIX + i); + } + assertThat(info.getClusterService(CLUSTER), notNullValue()); + + } + + @Test + public void whenClusterScaleDownToReplicas1_fromReplicas2_server3WithAlwaysPolicy_establishMatchingPresence() + throws JsonProcessingException { + establishPreviousIntrospection(null, Arrays.asList(1,3)); + + domainConfigurator.configureCluster(CLUSTER).withReplicas(1); + domainConfigurator.configureServer(MS_PREFIX + 3).withServerStartPolicy(START_ALWAYS); + + DomainPresenceInfo info = new DomainPresenceInfo(newDomain); + processor.createMakeRightOperation(info).execute(); + + logRecords.clear(); + + List runningPods = getRunningPods(); + //one introspector pod, one admin server pod and one managed server pods + assertThat(runningPods.size(), equalTo(3)); + + assertServerPodAndServicePresent(info, ADMIN_NAME); + assertServerPodAndServicePresent(info, MS_PREFIX + 3); + for (Integer i : Arrays.asList(1,2)) { + assertServerPodNotPresent(info, MS_PREFIX + i); + } + + assertThat(info.getClusterService(CLUSTER), notNullValue()); + } + + @Test + public void whenClusterReplicas3_server3And4WithAlwaysPolicy_establishMatchingPresence() { + domainConfigurator.configureCluster(CLUSTER).withReplicas(3); + + for (Integer i : Arrays.asList(3,4)) { + domainConfigurator.configureServer(MS_PREFIX + i).withServerStartPolicy(START_ALWAYS); + } + + DomainPresenceInfo info = new DomainPresenceInfo(newDomain); + processor.createMakeRightOperation(info).execute(); + + List runningPods = getRunningPods(); + //one introspector pod, one admin server pod and three managed server pods + assertThat(runningPods.size(), equalTo(5)); + + assertServerPodAndServicePresent(info, ADMIN_NAME); + for (Integer i : Arrays.asList(1,3,4)) { + assertServerPodAndServicePresent(info, MS_PREFIX + i); + } + assertServerPodNotPresent(info, MS_PREFIX + 2); + + assertThat(info.getClusterService(CLUSTER), notNullValue()); + } + + @Test + public void whenClusterScaleUpToReplicas4_fromReplicas2_server3And4WithAlwaysPolicy_establishMatchingPresence() + throws JsonProcessingException { + establishPreviousIntrospection(null, Arrays.asList(1, 3, 4)); + + for (Integer i : Arrays.asList(3,4)) { + domainConfigurator.configureServer(MS_PREFIX + i).withServerStartPolicy(START_ALWAYS); + } + + domainConfigurator.configureCluster(CLUSTER).withReplicas(4); + + DomainPresenceInfo info = new DomainPresenceInfo(newDomain); + + processor.createMakeRightOperation(info).execute(); + + List runningPods = getRunningPods(); + //one introspector pod, one admin server pod and four managed server pods + assertThat(runningPods.size(), equalTo(6)); + + assertServerPodAndServicePresent(info, ADMIN_NAME); + for (Integer i : Arrays.asList(1,2,3,4)) { + assertServerPodAndServicePresent(info, MS_PREFIX + i); + } + + assertThat(info.getClusterService(CLUSTER), notNullValue()); + } + + @Test + public void whenClusterReplicas2_server1And2And3WithAlwaysPolicy_establishMatchingPresence() { + domainConfigurator.configureCluster(CLUSTER).withReplicas(2); + + for (Integer i : Arrays.asList(1,2,3)) { + domainConfigurator.configureServer(MS_PREFIX + i).withServerStartPolicy(START_ALWAYS); + } + + DomainPresenceInfo info = new DomainPresenceInfo(newDomain); + processor.createMakeRightOperation(info).execute(); + + List runningPods = getRunningPods(); + //one introspector pod, one admin server pod and three managed server pods + assertThat(runningPods.size(), equalTo(5)); + + assertServerPodAndServicePresent(info, ADMIN_NAME); + for (Integer i : Arrays.asList(1,2,3)) { + assertServerPodAndServicePresent(info, MS_PREFIX + i); + } + + assertThat(info.getClusterService(CLUSTER), notNullValue()); + } + + @Test + public void whenClusterScaleDownToReplicas1_fromReplicas2_server1And2And3WithAlwaysPolicy_establishMatchingPresence() + throws JsonProcessingException { + establishPreviousIntrospection(null, Arrays.asList(1, 2, 3)); + + // now scale down the cluster + domainConfigurator.configureCluster(CLUSTER).withReplicas(1); + + for (Integer i : Arrays.asList(1,2,3)) { + domainConfigurator.configureServer(MS_PREFIX + i).withServerStartPolicy(START_ALWAYS); + } + + DomainPresenceInfo info = new DomainPresenceInfo(newDomain); + processor.createMakeRightOperation(info).execute(); + logRecords.clear(); + + List runningPods = getRunningPods(); + //one introspector pod, one admin server pod and three managed server pods + assertThat(runningPods.size(), equalTo(5)); + + assertServerPodAndServicePresent(info, ADMIN_NAME); + for (Integer i : Arrays.asList(1,2,3)) { + assertServerPodAndServicePresent(info, MS_PREFIX + i); + } + assertThat(info.getClusterService(CLUSTER), notNullValue()); + } + + @Test + public void whenClusterReplicas2_server2NeverPolicy_establishMatchingPresence() { + domainConfigurator.configureCluster(CLUSTER).withReplicas(2); + domainConfigurator.configureServer(MS_PREFIX + 2).withServerStartPolicy(START_NEVER); + DomainPresenceInfo info = new DomainPresenceInfo(newDomain); + processor.createMakeRightOperation(info).execute(); + + List runningPods = getRunningPods(); + //one introspector pod, one admin server pod and two managed server pods + assertThat(runningPods.size(), equalTo(4)); + + assertServerPodAndServicePresent(info, ADMIN_NAME); + for (Integer i : Arrays.asList(1,3)) { + assertServerPodAndServicePresent(info, MS_PREFIX + i); + } + assertServerPodNotPresent(info, MS_PREFIX + 2); + assertThat(info.getClusterService(CLUSTER), notNullValue()); + } + + @Test + public void whenClusterReplicas2_allServersExcept5NeverPolicy_establishMatchingPresence() { + domainConfigurator.configureCluster(CLUSTER).withReplicas(3); + int[] servers = IntStream.rangeClosed(1, MAX_SERVERS).toArray(); + for (int i : servers) { + if (i != 5) { + domainConfigurator.configureServer(MS_PREFIX + i).withServerStartPolicy(START_NEVER); + } + } + DomainPresenceInfo info = new DomainPresenceInfo(newDomain); + processor.createMakeRightOperation(info).execute(); + + List runningPods = getRunningPods(); + //one introspector pod, one admin server pod and one managed server pods + assertThat(runningPods.size(), equalTo(3)); + + assertServerPodAndServicePresent(info, ADMIN_NAME); + for (int i : servers) { + if (i != 5) { + assertServerPodAndServiceNotPresent(info, MS_PREFIX + i); + } else { + assertServerPodAndServicePresent(info, MS_PREFIX + i); + } + } + + assertThat(info.getClusterService(CLUSTER), notNullValue()); + } + private V1Service createNonOperatorService() { return new V1Service() .metadata( @@ -386,14 +606,20 @@ public void whenIntrospectionJobNotComplete_waitForIt() throws Exception { } private void establishPreviousIntrospection(Consumer domainSetup) throws JsonProcessingException { + establishPreviousIntrospection(domainSetup, Arrays.asList(1,2)); + } + + private void establishPreviousIntrospection(Consumer domainSetup, List msNumbers) + throws JsonProcessingException { if (domainSetup != null) { domainSetup.accept(domain); domainSetup.accept(newDomain); } domainConfigurator.configureCluster(CLUSTER).withReplicas(MIN_REPLICAS); defineServerResources(ADMIN_NAME); - defineServerResources(getManagedServerName(1)); - defineServerResources(getManagedServerName(2)); + for (Integer i : msNumbers) { + defineServerResources(getManagedServerName(i)); + } DomainProcessorImpl.registerDomainPresenceInfo(new DomainPresenceInfo(domain)); testSupport.defineResources(createIntrospectorConfigMap(OLD_INTROSPECTION_STATE)); testSupport.doOnCreate(KubernetesTestSupport.JOB, j -> recordJob((V1Job) j)); @@ -778,6 +1004,14 @@ private void assertServerPodAndServicePresent(DomainPresenceInfo info, String se assertThat(serverName + " pod", info.getServerPod(serverName), notNullValue()); } + private void assertServerPodNotPresent(DomainPresenceInfo info, String serverName) { + assertThat(serverName + " pod", isServerInactive(info, serverName), is(Boolean.TRUE)); + } + + private boolean isServerInactive(DomainPresenceInfo info, String serverName) { + return info.isServerPodBeingDeleted(serverName) || info.getServerPod(serverName) == null; + } + @Test public void whenDomainIsNotValid_dontBringUpServers() { defineDuplicateServerNames(); @@ -793,7 +1027,7 @@ public void whenDomainIsNotValid_dontBringUpServers() { private void assertServerPodAndServiceNotPresent(DomainPresenceInfo info, String serverName) { assertThat(serverName + " server service", info.getServerService(serverName), nullValue()); - assertThat(serverName + " pod", info.getServerPod(serverName), nullValue()); + assertThat(serverName + " pod", isServerInactive(info, serverName), is(Boolean.TRUE)); } @Test