[ML] Add effective max model memory limit to ML info (#55529)

The ML info endpoint returns the max_model_memory_limit setting
if one is configured.  However, it is still possible to create
a job that cannot run anywhere in the current cluster because
no node in the cluster has enough memory to accommodate it.

This change adds an extra piece of information,
limits.effective_max_model_memory_limit, to the ML info
response that returns the biggest model memory limit that could
be run in the current cluster assuming no other jobs were
running.

The idea is that the ML UI will be able to warn users who try to
create jobs with higher model memory limits that their jobs will
not be able to start unless they add a bigger ML node to their
cluster.

Relates elastic/kibana#63942

Author: David Roberts

docs/reference/ml/anomaly-detection/apis/get-ml-info.asciidoc

@@ -113,9 +113,12 @@ This is a possible response:
     "version": "7.0.0",
     "build_hash": "99a07c016d5a73"
   },
-  "limits" : { }
+  "limits" : {
+    "effective_max_model_memory_limit": "28961mb"
+  }
 }
 ----
 // TESTRESPONSE[s/"upgrade_mode": false/"upgrade_mode": $body.upgrade_mode/]
 // TESTRESPONSE[s/"version": "7.0.0",/"version": "$body.native_code.version",/]
 // TESTRESPONSE[s/"build_hash": "99a07c016d5a73"/"build_hash": "$body.native_code.build_hash"/]
+// TESTRESPONSE[s/"effective_max_model_memory_limit": "28961mb"/"effective_max_model_memory_limit": "$body.limits.effective_max_model_memory_limit"/]

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java

@@ -10,6 +10,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -21,9 +23,11 @@
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.action.MlInfoAction;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits;
 import org.elasticsearch.xpack.core.ml.job.config.CategorizationAnalyzerConfig;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
+import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.ml.process.MlControllerHolder;
 
 import java.io.IOException;
@@ -106,11 +110,50 @@ private Map<String, Object> datafeedsDefaults() {
         return anomalyDetectorsDefaults;
     }
 
+    static ByteSizeValue calculateEffectiveMaxModelMemoryLimit(int maxMachineMemoryPercent, DiscoveryNodes nodes) {
+
+        long maxMlMemory = -1;
+
+        for (DiscoveryNode node : nodes) {
+
+            Map<String, String> nodeAttributes = node.getAttributes();
+            String machineMemoryStr = nodeAttributes.get(MachineLearning.MACHINE_MEMORY_NODE_ATTR);
+            if (machineMemoryStr == null) {
+                continue;
+            }
+            long machineMemory;
+            try {
+                machineMemory = Long.parseLong(machineMemoryStr);
+            } catch (NumberFormatException e) {
+                continue;
+            }
+            maxMlMemory = Math.max(maxMlMemory, machineMemory * maxMachineMemoryPercent / 100);
+        }
+
+        if (maxMlMemory <= 0) {
+            // This implies there are currently no ML nodes in the cluster, so we
+            // have no idea what the effective limit would be if one were added
+            return null;
+        }
+
+        maxMlMemory -= Math.max(Job.PROCESS_MEMORY_OVERHEAD.getBytes(), DataFrameAnalyticsConfig.PROCESS_MEMORY_OVERHEAD.getBytes());
+        maxMlMemory -= MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes();
+        return new ByteSizeValue(Math.max(0L, maxMlMemory) / 1024 / 1024, ByteSizeUnit.MB);
+    }
+
     private Map<String, Object> limits() {
         Map<String, Object> limits = new HashMap<>();
+        ByteSizeValue effectiveMaxModelMemoryLimit = calculateEffectiveMaxModelMemoryLimit(
+            clusterService.getClusterSettings().get(MachineLearning.MAX_MACHINE_MEMORY_PERCENT), clusterService.state().getNodes());
         ByteSizeValue maxModelMemoryLimit = clusterService.getClusterSettings().get(MachineLearningField.MAX_MODEL_MEMORY_LIMIT);
         if (maxModelMemoryLimit != null && maxModelMemoryLimit.getBytes() > 0) {
-            limits.put("max_model_memory_limit", maxModelMemoryLimit);
+            limits.put("max_model_memory_limit", maxModelMemoryLimit.getStringRep());
+            if (effectiveMaxModelMemoryLimit == null || effectiveMaxModelMemoryLimit.compareTo(maxModelMemoryLimit) > 0) {
+                effectiveMaxModelMemoryLimit = maxModelMemoryLimit;
+            }
+        }
+        if (effectiveMaxModelMemoryLimit != null) {
+            limits.put("effective_max_model_memory_limit", effectiveMaxModelMemoryLimit.getStringRep());
         }
         return limits;
     }

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportMlInfoActionTests.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.ml.action;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.job.config.Job;
+import org.elasticsearch.xpack.ml.MachineLearning;
+
+import java.net.InetAddress;
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class TransportMlInfoActionTests extends ESTestCase {
+
+    public void testCalculateEffectiveMaxModelMemoryLimit() {
+
+        int mlMemoryPercent = randomIntBetween(5, 90);
+        long highestMlMachineMemory = -1;
+
+        DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
+        for (int i = randomIntBetween(1, 10); i > 0; --i) {
+            String nodeName = "_node_name" + i;
+            String nodeId = "_node_id" + i;
+            TransportAddress ta = new TransportAddress(InetAddress.getLoopbackAddress(), 9300 + i);
+            if (randomBoolean()) {
+                // Not an ML node
+                builder.add(new DiscoveryNode(nodeName, nodeId, ta, Collections.emptyMap(), Collections.emptySet(), Version.CURRENT));
+            } else {
+                // ML node
+                long machineMemory = randomLongBetween(2000000000L, 100000000000L);
+                highestMlMachineMemory = Math.max(machineMemory, highestMlMachineMemory);
+                builder.add(new DiscoveryNode(nodeName, nodeId, ta,
+                    Collections.singletonMap(MachineLearning.MACHINE_MEMORY_NODE_ATTR, String.valueOf(machineMemory)),
+                    Collections.emptySet(), Version.CURRENT));
+            }
+        }
+        DiscoveryNodes nodes = builder.build();
+
+        ByteSizeValue effectiveMaxModelMemoryLimit =
+            TransportMlInfoAction.calculateEffectiveMaxModelMemoryLimit(mlMemoryPercent, nodes);
+
+        if (highestMlMachineMemory < 0) {
+            assertThat(effectiveMaxModelMemoryLimit, nullValue());
+        } else {
+            assertThat(effectiveMaxModelMemoryLimit, notNullValue());
+            assertThat(effectiveMaxModelMemoryLimit.getBytes()
+                    + Math.max(Job.PROCESS_MEMORY_OVERHEAD.getBytes(), DataFrameAnalyticsConfig.PROCESS_MEMORY_OVERHEAD.getBytes())
+                    + MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(),
+                lessThanOrEqualTo(highestMlMachineMemory * mlMemoryPercent / 100));
+        }
+    }
+}

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/ml_info.yml

@@ -15,7 +15,9 @@ teardown:
   - match: { defaults.anomaly_detectors.categorization_examples_limit: 4 }
   - match: { defaults.anomaly_detectors.model_snapshot_retention_days: 1 }
   - match: { defaults.datafeeds.scroll_size: 1000 }
-  - match: { limits: {} }
+  - is_false: limits.max_model_memory_limit
+  # We cannot assert an exact value for the next one as it will vary depending on the test machine
+  - match: { limits.effective_max_model_memory_limit: "/\\d+[kmg]?b/" }
   - match: { upgrade_mode: false }
 
   - do:
@@ -32,6 +34,8 @@ teardown:
   - match: { defaults.anomaly_detectors.model_snapshot_retention_days: 1 }
   - match: { defaults.datafeeds.scroll_size: 1000 }
   - match: { limits.max_model_memory_limit: "512mb" }
+  # We cannot assert an exact value for the next one as it will vary depending on the test machine
+  - match: { limits.effective_max_model_memory_limit: "/\\d+[kmg]?b/" }
   - match: { upgrade_mode: false }
 
   - do:
@@ -48,4 +52,42 @@ teardown:
   - match: { defaults.anomaly_detectors.model_snapshot_retention_days: 1 }
   - match: { defaults.datafeeds.scroll_size: 1000 }
   - match: { limits.max_model_memory_limit: "6gb" }
+  # We cannot assert an exact value for the next one as it will vary depending on the test machine
+  - match: { limits.effective_max_model_memory_limit: "/\\d+[kmg]?b/" }
+  - match: { upgrade_mode: false }
+
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            xpack.ml.max_model_memory_limit: "6gb"
+
+  - do:
+      ml.info: {}
+  - match: { defaults.anomaly_detectors.categorization_analyzer.tokenizer: "ml_classic" }
+  - match: { defaults.anomaly_detectors.model_memory_limit: "1gb" }
+  - match: { defaults.anomaly_detectors.categorization_examples_limit: 4 }
+  - match: { defaults.anomaly_detectors.model_snapshot_retention_days: 1 }
+  - match: { defaults.datafeeds.scroll_size: 1000 }
+  - match: { limits.max_model_memory_limit: "6gb" }
+  # We cannot assert an exact value for the next one as it will vary depending on the test machine
+  - match: { limits.effective_max_model_memory_limit: "/\\d+[kmg]?b/" }
+  - match: { upgrade_mode: false }
+
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            xpack.ml.max_model_memory_limit: "1mb"
+
+  - do:
+      ml.info: {}
+  - match: { defaults.anomaly_detectors.categorization_analyzer.tokenizer: "ml_classic" }
+  - match: { defaults.anomaly_detectors.model_memory_limit: "1mb" }
+  - match: { defaults.anomaly_detectors.categorization_examples_limit: 4 }
+  - match: { defaults.anomaly_detectors.model_snapshot_retention_days: 1 }
+  - match: { defaults.datafeeds.scroll_size: 1000 }
+  - match: { limits.max_model_memory_limit: "1mb" }
+  # This time we can assert an exact value for the next one because the hard limit is so low
+  - match: { limits.effective_max_model_memory_limit: "1mb" }
   - match: { upgrade_mode: false }