diff --git a/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java b/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java index 4b0a79783885f..97f3e765fecbc 100644 --- a/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java +++ b/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java @@ -96,7 +96,7 @@ public class FsHealthService extends AbstractLifecycleComponent implements NodeH ); public static final Setting REFRESH_INTERVAL_SETTING = Setting.timeSetting( "monitor.fs.health.refresh_interval", - TimeValue.timeValueSeconds(60), + TimeValue.timeValueSeconds(1), TimeValue.timeValueMillis(1), Setting.Property.NodeScope ); @@ -228,6 +228,7 @@ private void monitorFSHealth() { } Files.delete(tempDataPath); final long elapsedTime = currentTimeMillisSupplier.getAsLong() - executionStartTime; + logger.info("health check took {}", elapsedTime); if (elapsedTime > slowPathLoggingThreshold.millis()) { logger.warn( "health check of [{}] took [{}ms] which is above the warn threshold of [{}]", diff --git a/server/src/main/java/org/opensearch/throttling/tracker/AverageIOUsageTracker.java b/server/src/main/java/org/opensearch/throttling/tracker/AverageIOUsageTracker.java index 1235ac3e37cf0..e7c0f19be60c1 100644 --- a/server/src/main/java/org/opensearch/throttling/tracker/AverageIOUsageTracker.java +++ b/server/src/main/java/org/opensearch/throttling/tracker/AverageIOUsageTracker.java @@ -145,6 +145,7 @@ private void recordUsage(IoUsageFetcher.DiskStats usage) { protected void doStart() { scheduledFuture = threadPool.scheduleWithFixedDelay(() -> { IoUsageFetcher.DiskStats usage = getUsage(); + if(usage == null) return; recordUsage(usage); }, pollingInterval, ThreadPool.Names.GENERIC); } diff --git a/server/src/main/java/org/opensearch/throttling/tracker/IoUsageFetcher.java b/server/src/main/java/org/opensearch/throttling/tracker/IoUsageFetcher.java index 478179b852194..3b7a9713bbe44 100644 --- a/server/src/main/java/org/opensearch/throttling/tracker/IoUsageFetcher.java +++ b/server/src/main/java/org/opensearch/throttling/tracker/IoUsageFetcher.java @@ -81,9 +81,12 @@ public DiskStats getDiskUtilizationStats() { double writeLatency = 0.0; double readOps = 0.0; double writeOps = 0.0; + if(this.fsService.stats().getIoStats() == null) { + return null; + } for (FsInfo.DeviceStats devicesStat : this.fsService.stats().getIoStats().getDevicesStats()) { if (previousIOTimeMap != null && previousIOTimeMap.containsKey(devicesStat.getDeviceName())){ - logger.info(this.fsService.stats().getTimestamp()); + //logger.info(this.fsService.stats().getTimestamp()); long ioSpentTime = devicesStat.getCurrentIOTime() - previousIOTimeMap.get(devicesStat.getDeviceName()).ioTime; ioUsePercent = (ioSpentTime * 100) / (1000); readOps += devicesStat.currentReadOperations() - previousIOTimeMap.get(devicesStat.getDeviceName()).readOps; @@ -104,11 +107,11 @@ public DiskStats getDiskUtilizationStats() { devicesStat.getCurrentReadKilobytes(), devicesStat.getCurrentWriteKilobytes()); currentIOTimeMap.put(devicesStat.getDeviceName(), ps); } - logger.info("Read in MB : {} , Write in MB : {}", readkb/1000, writekb/1000); + // logger.info("Read in MB : {} , Write in MB : {}", readkb/1000, writekb/1000); // readLatency += (readOps / readTime) * 100; // writeLatency += (writeOps / writeTime) * 100; - logger.info("read ops : {} , writeops : {} , readtime: {} , writetime: {}", readOps, writeOps, readTime, writeTime); - logger.info("Read latency : {} write latency : {}" , readLatency, writeLatency); +// logger.info("read ops : {} , writeops : {} , readtime: {} , writetime: {}", readOps, writeOps, readTime, writeTime); +// logger.info("Read latency : {} write latency : {}" , readLatency, writeLatency); logger.info("IO use percent : {}", ioUsePercent); previousIOTimeMap = currentIOTimeMap; diff --git a/server/src/main/java/org/opensearch/throttling/tracker/PerformanceTrackerSettings.java b/server/src/main/java/org/opensearch/throttling/tracker/PerformanceTrackerSettings.java index 66f4e3df4b66e..cce796fd86b96 100644 --- a/server/src/main/java/org/opensearch/throttling/tracker/PerformanceTrackerSettings.java +++ b/server/src/main/java/org/opensearch/throttling/tracker/PerformanceTrackerSettings.java @@ -23,7 +23,7 @@ private static class Defaults { private static final long WINDOW_DURATION = 30; private static final long REFRESH_INTERVAL = 1000; - private static final long IO_POLLING_INTERVAL = 60000; + private static final long IO_POLLING_INTERVAL = 1000; private static final long IO_WINDOW_DURATION = 60; }