Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

OWLS-90180 Optimize detection of when to run introspection #2430

Merged
merged 11 commits into from
Jul 8, 2021
Merged

OWLS-90180 Optimize detection of when to run introspection #2430

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
8d9fd6a
OWLS-90180 - Optimize introspector job run detection.
ankedia Jun 28, 2021
21e87c2
Use domainPresenceRecheckIntervalSeconds as the timeout interval for …
ankedia Jun 28, 2021
3438caa
Minor code cleanup.
ankedia Jun 28, 2021
6a29a8b
Fix comments.
ankedia Jun 28, 2021
462b5d6
Minor cleanup.
ankedia Jun 28, 2021
db5cc7c
PR review comment - set intro job START_TIME to job creation time in …
ankedia Jun 28, 2021
aec66ce
Additional validations.
ankedia Jun 30, 2021
f30a87b
Minor change.
ankedia Jun 30, 2021
8fe6448
Minor changes.
ankedia Jun 30, 2021
f07b126
Minor changes.
ankedia Jul 6, 2021
fedcece
Review comments and minor cleanup.
ankedia Jul 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,15 +303,14 @@ private static Step domainIntrospectionSteps(DomainPresenceInfo info) {
return Step.chain(
ConfigMapHelper.readIntrospectionVersionStep(info.getNamespace(), info.getDomainUid()),
new IntrospectionRequestStep(info),
JobHelper.deleteDomainIntrospectorJobStep(null),
JobHelper.createDomainIntrospectorJobStep(null));
JobHelper.replaceOrCreateDomainIntrospectorJobStep(null));
}

private static class IntrospectionRequestStep extends Step {
public static class IntrospectionRequestStep extends Step {

private final String requestedIntrospectVersion;

IntrospectionRequestStep(DomainPresenceInfo info) {
public IntrospectionRequestStep(DomainPresenceInfo info) {
this.requestedIntrospectVersion = info.getDomain().getIntrospectVersion();
}

Expand Down
1 change: 1 addition & 0 deletions operator/src/main/java/oracle/kubernetes/operator/LabelConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ public interface LabelConstants {
String MODEL_IN_IMAGE_DOMAINZIP_HASH = "weblogic.modelInImageDomainZipHash";
String INTROSPECTION_STATE_LABEL = "weblogic.introspectVersion";
String MII_UPDATED_RESTART_REQUIRED_LABEL = "weblogic.configChangesPendingRestart";
String INTROSPECTION_DOMAIN_SPEC_GENERATION = "weblogic.domainSpecGeneration";

static String forDomainUidSelector(String uid) {
return String.format("%s=%s", DOMAINUID_LABEL, uid);
Expand Down
25 changes: 22 additions & 3 deletions operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import java.io.BufferedReader;
import java.io.IOException;
import java.io.StringReader;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
Expand Down Expand Up @@ -49,6 +50,7 @@
import org.yaml.snakeyaml.Yaml;

import static java.lang.System.lineSeparator;
import static java.time.temporal.ChronoUnit.MILLIS;
import static oracle.kubernetes.operator.DomainStatusUpdater.BAD_TOPOLOGY;
import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.DOMAINZIP_HASH;
import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.DOMAIN_INPUTS_HASH;
Expand All @@ -57,6 +59,7 @@
import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.SECRETS_MD_5;
import static oracle.kubernetes.operator.IntrospectorConfigMapConstants.SIT_CONFIG_FILE_PREFIX;
import static oracle.kubernetes.operator.KubernetesConstants.SCRIPT_CONFIG_MAP_NAME;
import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_DOMAIN_SPEC_GENERATION;
import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_STATE_LABEL;
import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_VALIDATION_ERRORS;
import static oracle.kubernetes.operator.helpers.KubernetesUtils.getDomainUidLabel;
Expand Down Expand Up @@ -319,8 +322,11 @@ class ReadResponseStep extends DefaultResponseStep<V1ConfigMap> {

@Override
public NextAction onSuccess(Packet packet, CallResponse<V1ConfigMap> callResponse) {
DomainPresenceInfo.fromPacket(packet).map(DomainPresenceInfo::getDomain).map(Domain::getIntrospectVersion)
Domain domain = DomainPresenceInfo.fromPacket(packet).map(DomainPresenceInfo::getDomain).orElse(null);
Optional.ofNullable(domain).map(Domain::getIntrospectVersion)
.ifPresent(value -> addLabel(INTROSPECTION_STATE_LABEL, value));
Optional.ofNullable(domain).map(Domain::getMetadata).map(V1ObjectMeta::getGeneration)
.ifPresent(value -> addLabel(INTROSPECTION_DOMAIN_SPEC_GENERATION, value.toString()));
V1ConfigMap existingMap = withoutTransientData(callResponse.getResult());
if (existingMap == null) {
return doNext(createConfigMap(getNext()), packet);
Expand Down Expand Up @@ -504,9 +510,8 @@ public NextAction apply(Packet packet) {
}

private long timeSinceJobStart(Packet packet) {
return System.currentTimeMillis() - ((Long) packet.get(JobHelper.START_TIME));
return ((OffsetDateTime)packet.get(JobHelper.START_TIME)).until(OffsetDateTime.now(), MILLIS);
}

}

static class IntrospectionLoader {
Expand Down Expand Up @@ -870,12 +875,26 @@ public NextAction onSuccess(Packet packet, CallResponse<V1ConfigMap> callRespons

if (domainTopology != null) {
recordTopology(packet, packet.getSpi(DomainPresenceInfo.class), domainTopology);
recordIntrospectVersionAndGeneration(result, packet);
return doNext(DomainValidationSteps.createValidateDomainTopologyStep(getNext()), packet);
} else {
return doNext(packet);
}
}

private void recordIntrospectVersionAndGeneration(V1ConfigMap result, Packet packet) {
Map<String, String> labels = Optional.ofNullable(result)
.map(V1ConfigMap::getMetadata)
.map(V1ObjectMeta::getLabels).orElse(null);

Optional.ofNullable(labels).map(l -> l.get(INTROSPECTION_STATE_LABEL))
.ifPresentOrElse(
version -> packet.put(INTROSPECTION_STATE_LABEL, version),
() -> packet.remove(INTROSPECTION_STATE_LABEL));
Optional.ofNullable(labels).map(l -> l.get(INTROSPECTION_DOMAIN_SPEC_GENERATION))
.ifPresent(generation -> packet.put(INTROSPECTION_DOMAIN_SPEC_GENERATION, generation));
}

private String getTopologyYaml(Map<String, String> data) {
return data.get(IntrospectorConfigMapConstants.TOPOLOGY_YAML);
}
Expand Down
177 changes: 136 additions & 41 deletions operator/src/main/java/oracle/kubernetes/operator/helpers/JobHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

package oracle.kubernetes.operator.helpers;

import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
Expand All @@ -21,6 +22,7 @@
import io.kubernetes.client.openapi.models.V1PodSpec;
import io.kubernetes.client.openapi.models.V1Volume;
import io.kubernetes.client.openapi.models.V1VolumeMount;
import oracle.kubernetes.operator.DomainProcessorImpl;
import oracle.kubernetes.operator.DomainStatusUpdater;
import oracle.kubernetes.operator.IntrospectorConfigMapConstants;
import oracle.kubernetes.operator.JobWatcher;
Expand Down Expand Up @@ -49,9 +51,12 @@
import oracle.kubernetes.weblogic.domain.model.ManagedServer;
import oracle.kubernetes.weblogic.domain.model.ServerEnvVars;

import static java.time.temporal.ChronoUnit.SECONDS;
import static oracle.kubernetes.operator.DomainSourceType.FromModel;
import static oracle.kubernetes.operator.DomainStatusUpdater.INSPECTING_DOMAIN_PROGRESS_REASON;
import static oracle.kubernetes.operator.DomainStatusUpdater.createProgressingStartedEventStep;
import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_DOMAIN_SPEC_GENERATION;
import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_STATE_LABEL;
import static oracle.kubernetes.operator.logging.MessageKeys.INTROSPECTOR_JOB_FAILED;
import static oracle.kubernetes.operator.logging.MessageKeys.INTROSPECTOR_JOB_FAILED_DETAIL;

Expand Down Expand Up @@ -90,18 +95,40 @@ private static boolean runIntrospector(Packet packet, DomainPresenceInfo info) {
LOGGER.fine("runIntrospector topology: " + topology);
LOGGER.fine("runningServersCount: " + runningServersCount(info));
LOGGER.fine("creatingServers: " + creatingServers(info));
LOGGER.fine("isModelInImageUpdate: " + isModelInImageUpdate(packet, info));
return topology == null
|| isBringingUpNewDomain(info)
|| introspectionRequested(packet)
|| isModelInImageUpdate(packet, info);
|| isBringingUpNewDomain(packet, info)
|| isIntrospectionRequestedAndRemove(packet)
|| isModelInImageUpdate(packet, info)
|| isIntrospectVersionChanged(packet, info);
}

private static boolean isBringingUpNewDomain(DomainPresenceInfo info) {
return runningServersCount(info) == 0 && creatingServers(info);
private static boolean isBringingUpNewDomain(Packet packet, DomainPresenceInfo info) {
return runningServersCount(info) == 0 && creatingServers(info) && isGenerationChanged(packet, info);
}

private static boolean introspectionRequested(Packet packet) {
return packet.containsKey(ProcessingConstants.DOMAIN_INTROSPECT_REQUESTED);
private static boolean isIntrospectionRequestedAndRemove(Packet packet) {
return packet.remove(ProcessingConstants.DOMAIN_INTROSPECT_REQUESTED) != null;
}

private static boolean isIntrospectVersionChanged(Packet packet, DomainPresenceInfo info) {
return Optional.ofNullable(packet.get(INTROSPECTION_STATE_LABEL))
.map(introspectVersionLabel -> !introspectVersionLabel.equals(getIntrospectVersion(info))).orElse(false);
}

private static boolean isGenerationChanged(Packet packet, DomainPresenceInfo info) {
return Optional.ofNullable(packet.get(INTROSPECTION_DOMAIN_SPEC_GENERATION))
.map(gen -> !gen.equals(getGeneration(info))).orElse(true);
}

private static String getIntrospectVersion(DomainPresenceInfo info) {
return Optional.ofNullable(info.getDomain()).map(Domain::getSpec).map(s -> s.getIntrospectVersion())
.orElse("");
}

private static String getGeneration(DomainPresenceInfo info) {
return Optional.ofNullable(info.getDomain()).map(Domain::getMetadata).map(m -> m.getGeneration().toString())
.orElse("");
}

private static boolean isModelInImageUpdate(Packet packet, DomainPresenceInfo info) {
Expand Down Expand Up @@ -178,13 +205,13 @@ static boolean creatingServers(DomainPresenceInfo info) {
}

/**
* Factory for {@link Step} that deletes WebLogic domain introspector job.
* Factory for {@link Step} that replaces or creates WebLogic domain introspector job.
*
* @param next Next processing step
* @return Step for deleting the domain introsepctor jod
* @return Step for replacing or creating the domain introsepctor jod
*/
public static Step deleteDomainIntrospectorJobStep(Step next) {
return new DeleteIntrospectorJobStep(next);
public static Step replaceOrCreateDomainIntrospectorJobStep(Step next) {
return new ReplaceOrCreateIntrospectorJobStep(next);
}

private static Step createWatchDomainIntrospectorJobReadyStep(Step next) {
Expand Down Expand Up @@ -374,7 +401,7 @@ public NextAction apply(Packet packet) {
if (runIntrospector(packet, info)) {
JobStepContext context = new DomainIntrospectorJobStepContext(packet);

packet.putIfAbsent(START_TIME, System.currentTimeMillis());
packet.putIfAbsent(START_TIME, OffsetDateTime.now());

return doNext(
Step.chain(
Expand All @@ -392,50 +419,60 @@ public NextAction apply(Packet packet) {
}
}

private static class DeleteIntrospectorJobStep extends Step {
private static class ReplaceOrCreateIntrospectorJobStep extends Step {

static final int JOB_DELETE_TIMEOUT_SECONDS = 1;

DeleteIntrospectorJobStep(Step next) {
ReplaceOrCreateIntrospectorJobStep(Step next) {
super(next);
}

@Override
public NextAction apply(Packet packet) {
return doNext(deleteJob(packet, getNext()), packet);
return doNext(replaceOrCreateJob(packet, getNext()), packet);
}

String getJobDeletedMessageKey() {
return MessageKeys.JOB_DELETED;

private Step replaceOrCreateJob(Packet packet, Step next) {
DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
return new CallBuilder().readJobAsync(JobHelper.createJobName(info.getDomain().getDomainUid()),
info.getNamespace(), info.getDomain().getDomainUid(),
new ReplaceOrCreateStep(next));
}

void logJobDeleted(String domainUid, String namespace, String jobName, Packet packet) {
V1Job domainIntrospectorJob =
(V1Job) packet.remove(ProcessingConstants.DOMAIN_INTROSPECTOR_JOB);
private class ReplaceOrCreateStep extends DefaultResponseStep {

packet.remove(ProcessingConstants.INTROSPECTOR_JOB_FAILURE_LOGGED);
if (domainIntrospectorJob != null
&& !JobWatcher.isComplete(domainIntrospectorJob)) {
logIntrospectorFailure(packet, domainIntrospectorJob);
ReplaceOrCreateStep(Step next) {
super(next);
}
packet.remove(ProcessingConstants.JOB_POD_NAME);

LOGGER.fine(getJobDeletedMessageKey(), domainUid, namespace, jobName);
}
@Override
public NextAction onSuccess(Packet packet, CallResponse callResponse) {
DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
String namespace = info.getNamespace();
V1Job job = (V1Job) callResponse.getResult();
if ((job != null) && (packet.get(ProcessingConstants.DOMAIN_INTROSPECTOR_JOB) == null)) {
packet.put(ProcessingConstants.DOMAIN_INTROSPECTOR_JOB, job);
}

private Step deleteJob(Packet packet, Step next) {
DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
java.lang.String domainUid = info.getDomain().getDomainUid();
java.lang.String namespace = info.getNamespace();
String jobName = JobHelper.createJobName(domainUid);
logJobDeleted(domainUid, namespace, jobName, packet);
return new CallBuilder().withTimeoutSeconds(JOB_DELETE_TIMEOUT_SECONDS)
.deleteJobAsync(
jobName,
namespace,
domainUid,
new V1DeleteOptions().propagationPolicy("Foreground"),
new DefaultResponseStep<>(next));
if (job != null) {
packet.putIfAbsent(START_TIME, Optional.ofNullable(job.getMetadata())
.map(m -> m.getCreationTimestamp()).orElse(OffsetDateTime.now()));
return doNext(Step.chain(
createProgressingStartedEventStep(info, INSPECTING_DOMAIN_PROGRESS_REASON, true, null),
readDomainIntrospectorPodLogStep(null),
deleteDomainIntrospectorJobStep(null),
ConfigMapHelper.createIntrospectorConfigMapStep(null),
ConfigMapHelper.readExistingIntrospectorConfigMap(namespace, info.getDomainUid()),
new DomainProcessorImpl.IntrospectionRequestStep(info),
createDomainIntrospectorJobStep(getNext())), packet);
} else {
packet.putIfAbsent(START_TIME, OffsetDateTime.now());
return doNext(Step.chain(
ConfigMapHelper.readExistingIntrospectorConfigMap(namespace, info.getDomainUid()),
createDomainIntrospectorJobStep(getNext())), packet);
}
}
}
}

Expand Down Expand Up @@ -507,17 +544,32 @@ public NextAction onSuccess(Packet packet, CallResponse<String> callResponse) {
jobConditionsReason.add(DomainStatusUpdater.ERR_INTROSPECTOR);
}
//Introspector job is incomplete, update domain status and terminate processing
Step nextStep = null;
int retryIntervalSeconds = TuningParameters.getInstance().getMainTuning().domainPresenceRecheckIntervalSeconds;

if (OffsetDateTime.now().isAfter(
getJobCreationTime(domainIntrospectorJob).plus(retryIntervalSeconds, SECONDS))) {
//Introspector job is incomplete and current time is greater than the lazy deletion time for the job,
//update the domain status and execute the next step
nextStep = getNext();
}

return doNext(
DomainStatusUpdater.createFailureRelatedSteps(
onSeparateLines(jobConditionsReason),
onSeparateLines(severeStatuses),
null),
nextStep),
packet);
}

return doNext(packet);
}

private OffsetDateTime getJobCreationTime(V1Job domainIntrospectorJob) {
return Optional.ofNullable(domainIntrospectorJob.getMetadata())
.map(m -> m.getCreationTimestamp()).orElse(OffsetDateTime.now());
}

private boolean isNotComplete(V1Job domainIntrospectorJob) {
return !JobWatcher.isComplete(domainIntrospectorJob);
}
Expand Down Expand Up @@ -611,6 +663,49 @@ private static void logIntrospectorFailure(Packet packet, V1Job domainIntrospect
}
}

static class DeleteDomainIntrospectorJobStep extends Step {

DeleteDomainIntrospectorJobStep(Step next) {
super(next);
}

@Override
public NextAction apply(Packet packet) {
DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
String jobName = JobHelper.createJobName(info.getDomainUid());
logJobDeleted(info.getDomainUid(), info.getNamespace(), jobName, packet);
return doNext(new CallBuilder().withTimeoutSeconds(ReplaceOrCreateIntrospectorJobStep.JOB_DELETE_TIMEOUT_SECONDS)
.deleteJobAsync(
jobName,
info.getNamespace(),
info.getDomainUid(),
new V1DeleteOptions().propagationPolicy("Foreground"),
new DefaultResponseStep<>(getNext())), packet);
}
}

public static Step deleteDomainIntrospectorJobStep(Step next) {
return new DeleteDomainIntrospectorJobStep(next);
}

static void logJobDeleted(String domainUid, String namespace, String jobName, Packet packet) {
V1Job domainIntrospectorJob =
(V1Job) packet.remove(ProcessingConstants.DOMAIN_INTROSPECTOR_JOB);

packet.remove(ProcessingConstants.INTROSPECTOR_JOB_FAILURE_LOGGED);
if (domainIntrospectorJob != null
&& !JobWatcher.isComplete(domainIntrospectorJob)) {
logIntrospectorFailure(packet, domainIntrospectorJob);
}
packet.remove(ProcessingConstants.JOB_POD_NAME);

LOGGER.fine(getJobDeletedMessageKey(), domainUid, namespace, jobName);
}

static String getJobDeletedMessageKey() {
return MessageKeys.JOB_DELETED;
}

private static class ReadDomainIntrospectorPodStep extends Step {

ReadDomainIntrospectorPodStep(Step next) {
Expand Down
2 changes: 1 addition & 1 deletion operator/src/test/java/oracle/kubernetes/operator/DomainUpPlanTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ public void useSequenceBeforeAdminServerStep() {
plan,
hasChainWithStepsInOrder(
"DomainPresenceStep",
"DomainIntrospectorJobStep",
"ReplaceOrCreateIntrospectorJobStep",
"BeforeAdminServiceStep",
"AdminPodStep",
"ForServerStep",
Expand Down
Loading