Skip to content

Commit

Permalink
chore: logging the changes in the probe
Browse files Browse the repository at this point in the history
  • Loading branch information
@novoj
novoj committed May 6, 2024
1 parent 5754620 commit 3e17056
Showing 1 changed file with 82 additions and 53 deletions.
135 changes: 82 additions & 53 deletions ...rc/main/java/io/evitadb/externalApi/observability/metric/ObservabilityProbesDetector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import io.evitadb.externalApi.observability.ObservabilityManager;
import io.evitadb.externalApi.observability.ObservabilityProvider;
import io.evitadb.utils.CollectionUtils;
import lombok.extern.slf4j.Slf4j;
import org.jboss.threads.EnhancedQueueExecutor;

import javax.annotation.Nonnull;
Expand All @@ -48,6 +49,7 @@
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;

/**
* This class is responsible for detecting health problems in the system. It monitors:
Expand All @@ -63,6 +65,7 @@
* unhealthy.
* 5. The readiness of the external APIs. If at least one external API is not ready, the system is considered unhealthy.
*/
@Slf4j
public class ObservabilityProbesDetector implements ProbesProvider {
private static final Set<HealthProblem> NO_HEALTH_PROBLEMS = EnumSet.noneOf(HealthProblem.class);
private static final Set<String> OLD_GENERATION_GC_NAMES = Set.of("G1 Old Generation", "PS MarkSweep", "ConcurrentMarkSweep");
Expand All @@ -72,8 +75,6 @@ public class ObservabilityProbesDetector implements ProbesProvider {
.stream()
.filter(gc -> OLD_GENERATION_GC_NAMES.contains(gc.getName()))
.toList();

private ObservabilityManager observabilityManager;
private final AtomicLong lastSeenRejectedTaskCount = new AtomicLong(0L);
private final AtomicLong lastSeenSubmittedTaskCount = new AtomicLong(0L);
private final AtomicLong lastSeenJavaErrorCount = new AtomicLong(0L);
Expand All @@ -82,6 +83,33 @@ public class ObservabilityProbesDetector implements ProbesProvider {
private final AtomicLong lastSeenJavaGarbageCollections = new AtomicLong(0L);
private final AtomicBoolean seenReady = new AtomicBoolean();
private final AtomicReference<Readiness> lastReadinessSeen = new AtomicReference<>();
private ObservabilityManager observabilityManager;

/**
* Records the result of the health problem check.
*
* @param healthProblem the result of the health problem check
* @param healthProblems the set of health problems
* @param theObservabilityManager the observability manager for recording the health problem
*/
private static void recordResult(
@Nonnull HealthProblemCheckResult healthProblem,
@Nonnull Set<HealthProblem> healthProblems,
@Nullable ObservabilityManager theObservabilityManager
) {
if (healthProblem.present()) {
if (healthProblem.healthProblem() != null) {
healthProblems.add(healthProblem.healthProblem());
}
if (theObservabilityManager != null) {
theObservabilityManager.recordHealthProblem(healthProblem.healthProblemName());
}
} else {
if (theObservabilityManager != null) {
theObservabilityManager.clearHealthProblem(healthProblem.healthProblemName());
}
}
}

@Nonnull
@Override
Expand All @@ -101,36 +129,59 @@ public Set<HealthProblem> getHealthProblems(@Nonnull EvitaContract evitaContract

recordResult(checkApiReadiness(), healthProblems, theObservabilityManager);

return healthProblems.isEmpty() ? NO_HEALTH_PROBLEMS : healthProblems;
if (healthProblems.isEmpty()) {
return NO_HEALTH_PROBLEMS;
} else {
log.error(
"Detected health problems: {}",
healthProblems.stream().map(HealthProblem::name).collect(Collectors.joining(", "))
);
return healthProblems;
}
}

/**
* Records the result of the health problem check.
* @param healthProblem the result of the health problem check
* @param healthProblems the set of health problems
* @param theObservabilityManager the observability manager for recording the health problem
*/
private static void recordResult(
@Nonnull HealthProblemCheckResult healthProblem,
@Nonnull Set<HealthProblem> healthProblems,
@Nullable ObservabilityManager theObservabilityManager
) {
if (healthProblem.present()) {
if (healthProblem.healthProblem() != null) {
healthProblems.add(healthProblem.healthProblem());
}
if (theObservabilityManager != null) {
theObservabilityManager.recordHealthProblem(healthProblem.healthProblemName());
}
} else {
if (theObservabilityManager != null) {
theObservabilityManager.clearHealthProblem(healthProblem.healthProblemName());
@Nonnull
@Override
public Readiness getReadiness(@Nonnull EvitaContract evitaContract, @Nonnull ExternalApiServer externalApiServer, @Nonnull String... apiCodes) {
final Optional<ObservabilityManager> theObservabilityManager = getObservabilityManager(externalApiServer);
// check the end-points availability
//noinspection rawtypes
final Collection<ExternalApiProviderRegistrar> availableExternalApis = ExternalApiServer.gatherExternalApiProviders();
final Map<String, Boolean> readiness = CollectionUtils.createHashMap(availableExternalApis.size());
for (String apiCode : apiCodes) {
final ExternalApiProvider<?> apiProvider = externalApiServer.getExternalApiProviderByCode(apiCode);
readiness.put(apiProvider.getCode(), apiProvider.isReady());
theObservabilityManager.ifPresent(it -> it.recordReadiness(apiProvider.getCode(), apiProvider.isReady()));
}
final boolean ready = readiness.values().stream().allMatch(Boolean::booleanValue);
if (ready) {
this.seenReady.set(true);
}
final Readiness currentReadiness = new Readiness(
ready ? ReadinessState.READY : (this.seenReady.get() ? ReadinessState.STALLING : ReadinessState.STARTING),
readiness.entrySet().stream()
.map(entry -> new ApiState(entry.getKey(), entry.getValue()))
.toArray(ApiState[]::new)
);

final Readiness previousReadiness = this.lastReadinessSeen.get();
if (previousReadiness == null || currentReadiness.state() != previousReadiness.state()) {
if (previousReadiness == null || previousReadiness.state() != ReadinessState.STALLING) {
log.info("System readiness changed to {}", currentReadiness.state());
} else if (currentReadiness.state() == ReadinessState.SHUTDOWN) {
log.warn("System readiness changed to {}", currentReadiness.state());
} else {
log.error("System readiness changed to {}", currentReadiness.state());
}
}

this.lastReadinessSeen.set(currentReadiness);
return currentReadiness;
}

/**
* Checks the readiness of the external APIs.
*
* @return the result of the check
*/
@Nonnull
Expand All @@ -145,6 +196,7 @@ private HealthProblemCheckResult checkApiReadiness() {
/**
* Checks the memory shortage. If the current memory usage is above 90% of the total available memory and the number
* of errors has increased since the last check, the system is considered unhealthy.
*
* @param theObservabilityManager the observability manager
* @return the result of the check
*/
Expand All @@ -160,7 +212,7 @@ private HealthProblemCheckResult checkMemoryShortage(@Nonnull ObservabilityManag
usedMemory > 0.9f &&
(
javaOOMErrorCount > this.lastSeenJavaOOMErrorCount.get() ||
oldGenerationCollectionCount > this.lastSeenJavaGarbageCollections.get()
oldGenerationCollectionCount > this.lastSeenJavaGarbageCollections.get()
)
);
this.lastSeenJavaOOMErrorCount.set(javaOOMErrorCount);
Expand All @@ -170,6 +222,7 @@ private HealthProblemCheckResult checkMemoryShortage(@Nonnull ObservabilityManag

/**
* Checks the number of Java internal errors.
*
* @param theObservabilityManager the observability manager
* @return the result of the check
*/
Expand All @@ -187,6 +240,7 @@ private HealthProblemCheckResult checkJavaErrors(@Nonnull ObservabilityManager t

/**
* Checks the rejection / submission rate of the executor input queues.
*
* @param evita the Evita instance
* @return the result of the check
*/
Expand Down Expand Up @@ -224,35 +278,9 @@ private HealthProblemCheckResult checkEvitaErrors(@Nonnull ObservabilityManager
return result;
}

@Nonnull
@Override
public Readiness getReadiness(@Nonnull EvitaContract evitaContract, @Nonnull ExternalApiServer externalApiServer, @Nonnull String... apiCodes) {
final Optional<ObservabilityManager> theObservabilityManager = getObservabilityManager(externalApiServer);
// check the end-points availability
//noinspection rawtypes
final Collection<ExternalApiProviderRegistrar> availableExternalApis = ExternalApiServer.gatherExternalApiProviders();
final Map<String, Boolean> readiness = CollectionUtils.createHashMap(availableExternalApis.size());
for (String apiCode : apiCodes) {
final ExternalApiProvider<?> apiProvider = externalApiServer.getExternalApiProviderByCode(apiCode);
readiness.put(apiProvider.getCode(), apiProvider.isReady());
theObservabilityManager.ifPresent(it -> it.recordReadiness(apiProvider.getCode(), apiProvider.isReady()));
}
final boolean ready = readiness.values().stream().allMatch(Boolean::booleanValue);
if (ready) {
this.seenReady.set(true);
}
final Readiness currentReadiness = new Readiness(
ready ? ReadinessState.READY : (this.seenReady.get() ? ReadinessState.STALLING : ReadinessState.STARTING),
readiness.entrySet().stream()
.map(entry -> new ApiState(entry.getKey(), entry.getValue()))
.toArray(ApiState[]::new)
);
this.lastReadinessSeen.set(currentReadiness);
return currentReadiness;
}

/**
* Returns the observability manager from the external API server.
*
* @param externalApiServer the external API server
* @return the observability manager or NULL if it is not available
*/
Expand All @@ -271,8 +299,9 @@ private Optional<ObservabilityManager> getObservabilityManager(@Nonnull External

/**
* This record represents the result of a health problem check.
*
* @param healthProblem type of health problem
* @param present true if the health problem is present, false otherwise
* @param present true if the health problem is present, false otherwise
*/
private record HealthProblemCheckResult(
@Nullable HealthProblem healthProblem,
Expand Down

0 comments on commit 3e17056

Please sign in to comment.