Add Shard Indexing Pressure Memory Manager (#478) (#945)

It introduces a Memory Manager for Shard Indexing Pressure. It is responsible for increasing and decreasing the allocated shard limit based on incoming requests, and validate the current values against the thresholds. Signed-off-by: Saurabh Singh <sisurab@amazon.com>
opensearch-project · Sep 15, 2021 · 347bcc5 · 347bcc5
1 parent 19cc0eb
commit 347bcc5
Show file tree

Hide file tree

Showing 4 changed files with 1,047 additions and 8 deletions.
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -33,6 +33,12 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.opensearch.action.main.TransportMainAction;
+import org.opensearch.index.IndexModule;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.IndexingPressure;
+import org.opensearch.index.ShardIndexingPressureMemoryManager;
+import org.opensearch.index.ShardIndexingPressureSettings;
+import org.opensearch.index.ShardIndexingPressureStore;
 import org.opensearch.watcher.ResourceWatcherService;
 import org.opensearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction;
 import org.opensearch.action.admin.indices.close.TransportCloseIndexAction;
@@ -100,11 +106,6 @@
 import org.opensearch.gateway.GatewayService;
 import org.opensearch.gateway.PersistedClusterStateService;
 import org.opensearch.http.HttpTransportSettings;
-import org.opensearch.index.IndexModule;
-import org.opensearch.index.IndexSettings;
-import org.opensearch.index.IndexingPressure;
-import org.opensearch.index.ShardIndexingPressureSettings;
-import org.opensearch.index.ShardIndexingPressureStore;
 import org.opensearch.indices.IndexingMemoryController;
 import org.opensearch.indices.IndicesQueryCache;
 import org.opensearch.indices.IndicesRequestCache;
@@ -587,7 +588,14 @@ public void apply(Settings value, Settings current, Settings previous) {
             ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENFORCED,
             ShardIndexingPressureSettings.REQUEST_SIZE_WINDOW,
             ShardIndexingPressureSettings.SHARD_MIN_LIMIT,
-            ShardIndexingPressureStore.MAX_COLD_STORE_SIZE)));
+            ShardIndexingPressureStore.MAX_COLD_STORE_SIZE,
+            ShardIndexingPressureMemoryManager.LOWER_OPERATING_FACTOR,
+            ShardIndexingPressureMemoryManager.OPTIMAL_OPERATING_FACTOR,
+            ShardIndexingPressureMemoryManager.UPPER_OPERATING_FACTOR,
+            ShardIndexingPressureMemoryManager.NODE_SOFT_LIMIT,
+            ShardIndexingPressureMemoryManager.THROUGHPUT_DEGRADATION_LIMITS,
+            ShardIndexingPressureMemoryManager.SUCCESSFUL_REQUEST_ELAPSED_TIMEOUT,
+            ShardIndexingPressureMemoryManager.MAX_OUTSTANDING_REQUESTS)));
 
     public static List<SettingUpgrader<?>> BUILT_IN_SETTING_UPGRADERS = Collections.unmodifiableList(Arrays.asList(
             SniffConnectionStrategy.SEARCH_REMOTE_CLUSTER_SEEDS_UPGRADER,