Merge branch 'main' of https://github.com/elastic/elasticsearch into …

…111712_fix

docs/changelog/111437.yaml

@@ -0,0 +1,5 @@
+pr: 111437
+summary: "[ES|QL] Create `Range` in `PushFiltersToSource` for qualified pushable filters on the same field"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/111779.yaml

@@ -0,0 +1,7 @@
+pr: 111779
+summary: "ESQL: Fix serialization during `can_match`"
+area: ES|QL
+type: bug
+issues:
+ - 111701
+ - 111726

muted-tests.yml

@@ -127,6 +127,12 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/111721
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
   issue: https://github.com/elastic/elasticsearch/issues/111765
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testSnapshotRestore {cluster=OLD}
+  issue: https://github.com/elastic/elasticsearch/issues/111777
+- class: org.elasticsearch.xpack.restart.CoreFullClusterRestartIT
+  method: testSnapshotRestore {cluster=OLD}
+  issue: https://github.com/elastic/elasticsearch/issues/111775
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java

@@ -370,7 +370,7 @@ public static DataStreamLifecycle fromXContent(XContentParser parser) throws IOE
      * Adds a retention param to signal that this serialisation should include the effective retention metadata
      */
     public static ToXContent.Params maybeAddEffectiveRetentionParams(ToXContent.Params params) {
-        boolean shouldAddEffectiveRetention = Objects.equals(params.param(RestRequest.PATH_RESTRICTED), "serverless");
+        boolean shouldAddEffectiveRetention = params.paramAsBoolean(RestRequest.SERVERLESS_REQUEST, false);
         return new DelegatingMapParams(
             Map.of(INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME, Boolean.toString(shouldAddEffectiveRetention)),
             params

server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java

@@ -125,6 +125,14 @@ public boolean contains(String nodeId, SingleNodeShutdownMetadata.Type type) {
         return get(nodeId, type) != null;
     }
 
+    /**
+     * Checks if the provided node is scheduled for being permanently removed from the cluster.
+     */
+    public boolean isNodeMarkedForRemoval(String nodeId) {
+        var singleNodeShutdownMetadata = get(nodeId);
+        return singleNodeShutdownMetadata != null && singleNodeShutdownMetadata.getType().isRemovalType();
+    }
+
     /**
      * Add or update the shutdown metadata for a single node.
      * @param nodeShutdownMetadata The single node shutdown metadata to add or update.

server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java

@@ -83,7 +83,14 @@ public final void handleRequest(RestRequest request, RestChannel channel, NodeCl
         // check if the query has any parameters that are not in the supported set (if declared)
         Set<String> supported = allSupportedParameters();
         if (supported != null) {
-            var allSupported = Sets.union(RestResponse.RESPONSE_PARAMS, ALWAYS_SUPPORTED, supported);
+            var allSupported = Sets.union(
+                RestResponse.RESPONSE_PARAMS,
+                ALWAYS_SUPPORTED,
+                // these internal parameters cannot be set by end-users, but are used by Elasticsearch internally.
+                // they must be accepted by all handlers
+                RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS,
+                supported
+            );
             if (allSupported.containsAll(request.params().keySet()) == false) {
                 Set<String> unsupported = Sets.difference(request.params().keySet(), allSupported);
                 throw new IllegalArgumentException(unrecognized(request, unsupported, allSupported, "parameter"));

server/src/main/java/org/elasticsearch/rest/RestController.java

@@ -480,6 +480,14 @@ private void dispatchRequest(
             } else {
                 threadContext.putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, Boolean.TRUE.toString());
             }
+
+            if (apiProtections.isEnabled()) {
+                // API protections are only enabled in serverless; therefore we can use this as an indicator to mark the
+                // request as a serverless mode request here, so downstream handlers can use the marker
+                request.markAsServerlessRequest();
+                logger.trace("Marked request for uri [{}] as serverless request", request.uri());
+            }
+
             final var finalChannel = responseChannel;
             this.interceptor.intercept(request, responseChannel, handler.getConcreteRestHandler(), new ActionListener<>() {
                 @Override

server/src/main/java/org/elasticsearch/rest/RestRequest.java

@@ -48,7 +48,31 @@
 
 public class RestRequest implements ToXContent.Params, Traceable {
 
-    public static final String PATH_RESTRICTED = "pathRestricted";
+    /**
+     * Internal marker request parameter to indicate that a request was made in serverless mode. Use this parameter, together with
+     * {@link #OPERATOR_REQUEST} if you need to toggle behavior for serverless, for example to enforce partial API restrictions
+     * (prevent request fields, omit response fields) for an API.
+     * Requests not made in serverless mode, will *not* have this parameter set.
+     * Given a request instance, you can use {@link #isServerlessRequest()} to determine if the parameter is set or not.
+     * This is also available from {@code ToXContent.Params}. For example:
+     * {@code params.paramAsBoolean(RestRequest.SERVERLESS_REQUEST, false)}
+     */
+    public static final String SERVERLESS_REQUEST = "serverlessRequest";
+    /**
+     * Internal marker request parameter to indicate that a request was made by an operator user.
+     * Requests made by regular users (users without operator privileges), will *not* have this parameter set.
+     * Given a request instance, you can use {@link #isOperatorRequest()} to determine if the parameter is set or not.
+     * This is also available from {@code ToXContent.Params}. For example:
+     * {@code params.paramAsBoolean(RestRequest.OPERATOR_REQUEST, false)}
+     */
+    public static final String OPERATOR_REQUEST = "operatorRequest";
+
+    /**
+     * Internal request parameters used as markers to indicate various operations modes such as serverless mode, or operator mode.
+     * These can never be set directly by end-users. Instead, they are set internally by Elasticsearch and must be supported by all
+     * request handlers.
+     */
+    public static final Set<String> INTERNAL_MARKER_REQUEST_PARAMETERS = Set.of(SERVERLESS_REQUEST, OPERATOR_REQUEST);
     // tchar pattern as defined by RFC7230 section 3.2.6
     private static final Pattern TCHAR_PATTERN = Pattern.compile("[a-zA-Z0-9!#$%&'*+\\-.\\^_`|~]+");
 
@@ -616,13 +640,41 @@ public boolean hasExplicitRestApiVersion() {
         return restApiVersion.isPresent();
     }
 
-    public void markPathRestricted(String restriction) {
-        if (params.containsKey(PATH_RESTRICTED)) {
-            throw new IllegalArgumentException("The parameter [" + PATH_RESTRICTED + "] is already defined.");
+    /**
+     * See {@link #SERVERLESS_REQUEST}
+     */
+    public void markAsServerlessRequest() {
+        setParamTrueOnceAndConsume(SERVERLESS_REQUEST);
+    }
+
+    /**
+     * See {@link #SERVERLESS_REQUEST}
+     */
+    public boolean isServerlessRequest() {
+        return paramAsBoolean(SERVERLESS_REQUEST, false);
+    }
+
+    /**
+     * See {@link #OPERATOR_REQUEST}
+     */
+    public void markAsOperatorRequest() {
+        setParamTrueOnceAndConsume(OPERATOR_REQUEST);
+    }
+
+    /**
+     * See {@link #OPERATOR_REQUEST}
+     */
+    public boolean isOperatorRequest() {
+        return paramAsBoolean(OPERATOR_REQUEST, false);
+    }
+
+    private void setParamTrueOnceAndConsume(String param) {
+        if (params.containsKey(param)) {
+            throw new IllegalArgumentException("The parameter [" + param + "] is already defined.");
         }
-        params.put(PATH_RESTRICTED, restriction);
+        params.put(param, "true");
         // this parameter is intended be consumed via ToXContent.Params.param(..), not this.params(..) so don't require it is consumed here
-        consumedParams.add(PATH_RESTRICTED);
+        consumedParams.add(param);
     }
 
     @Override

server/src/main/java/org/elasticsearch/rest/RestUtils.java

@@ -23,7 +23,7 @@
 import java.util.regex.Pattern;
 
 import static org.elasticsearch.action.support.master.AcknowledgedRequest.DEFAULT_ACK_TIMEOUT;
-import static org.elasticsearch.rest.RestRequest.PATH_RESTRICTED;
+import static org.elasticsearch.rest.RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS;
 
 public class RestUtils {
 
@@ -85,8 +85,10 @@ private static String decodeQueryStringParam(final String s) {
     }
 
     private static void addParam(Map<String, String> params, String name, String value) {
-        if (PATH_RESTRICTED.equalsIgnoreCase(name)) {
-            throw new IllegalArgumentException("parameter [" + PATH_RESTRICTED + "] is reserved and may not set");
+        for (var reservedParameter : INTERNAL_MARKER_REQUEST_PARAMETERS) {
+            if (reservedParameter.equalsIgnoreCase(name)) {
+                throw new IllegalArgumentException("parameter [" + name + "] is reserved and may not be set");
+            }
         }
         params.put(name, value);
     }

server/src/main/java/org/elasticsearch/rest/ServerlessScope.java

@@ -22,9 +22,4 @@
 @Target(ElementType.TYPE)
 public @interface ServerlessScope {
     Scope value();
-
-    /**
-     * A value used when restricting a response of a serverless endpoints.
-     */
-    String SERVERLESS_RESTRICTION = "serverless";
 }

server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java

@@ -22,7 +22,7 @@
 import java.io.IOException;
 import java.util.Map;
 
-import static org.elasticsearch.rest.RestRequest.PATH_RESTRICTED;
+import static org.elasticsearch.rest.RestRequest.SERVERLESS_REQUEST;
 import static org.hamcrest.Matchers.equalTo;
 
 public class GetDataStreamLifecycleActionTests extends ESTestCase {
@@ -75,7 +75,7 @@ private Map<String, Object> getXContentMap(
         TimeValue globalMaxRetention
     ) throws IOException {
         try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
-            ToXContent.Params params = new ToXContent.MapParams(Map.of(PATH_RESTRICTED, "serverless"));
+            ToXContent.Params params = new ToXContent.MapParams(Map.of(SERVERLESS_REQUEST, "true"));
             RolloverConfiguration rolloverConfiguration = null;
             DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(globalDefaultRetention, globalMaxRetention);
             dataStreamLifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention);

server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java

@@ -39,7 +39,7 @@
 import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.RetentionSource.DATA_STREAM_CONFIGURATION;
 import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.RetentionSource.DEFAULT_GLOBAL_RETENTION;
 import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.RetentionSource.MAX_GLOBAL_RETENTION;
-import static org.elasticsearch.rest.RestRequest.PATH_RESTRICTED;
+import static org.elasticsearch.rest.RestRequest.SERVERLESS_REQUEST;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
@@ -354,13 +354,7 @@ public void testEffectiveRetentionParams() {
         }
         {
             ToXContent.Params params = DataStreamLifecycle.maybeAddEffectiveRetentionParams(
-                new ToXContent.MapParams(Map.of(PATH_RESTRICTED, "not-serverless"))
-            );
-            assertThat(params.paramAsBoolean(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME, false), equalTo(false));
-        }
-        {
-            ToXContent.Params params = DataStreamLifecycle.maybeAddEffectiveRetentionParams(
-                new ToXContent.MapParams(Map.of(PATH_RESTRICTED, "serverless"))
+                new ToXContent.MapParams(Map.of(SERVERLESS_REQUEST, "true"))
             );
             assertThat(params.paramAsBoolean(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME, false), equalTo(true));
         }

server/src/test/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadataTests.java

@@ -34,6 +34,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -121,6 +122,39 @@ public void testSigtermIsRemoveInOlderVersions() throws IOException {
         assertThat(new SingleNodeShutdownMetadata(out.bytes().streamInput()).getType(), equalTo(SingleNodeShutdownMetadata.Type.SIGTERM));
     }
 
+    public void testIsNodeMarkedForRemoval() {
+        SingleNodeShutdownMetadata.Type type;
+        SingleNodeShutdownMetadata.Builder builder = SingleNodeShutdownMetadata.builder()
+            .setNodeId("thenode")
+            .setReason("myReason")
+            .setStartedAtMillis(0L);
+        switch (type = randomFrom(SingleNodeShutdownMetadata.Type.values())) {
+            case SIGTERM -> {
+                var metadata = new NodesShutdownMetadata(
+                    Map.of("thenode", builder.setType(type).setGracePeriod(TimeValue.ONE_MINUTE).build())
+                );
+                assertThat(metadata.isNodeMarkedForRemoval("thenode"), is(true));
+                assertThat(metadata.isNodeMarkedForRemoval("anotherNode"), is(false));
+            }
+            case REMOVE -> {
+                var metadata = new NodesShutdownMetadata(Map.of("thenode", builder.setType(type).build()));
+                assertThat(metadata.isNodeMarkedForRemoval("thenode"), is(true));
+                assertThat(metadata.isNodeMarkedForRemoval("anotherNode"), is(false));
+            }
+            case REPLACE -> {
+                var metadata = new NodesShutdownMetadata(
+                    Map.of("thenode", builder.setType(type).setTargetNodeName("newnodecoming").build())
+                );
+                assertThat(metadata.isNodeMarkedForRemoval("thenode"), is(true));
+                assertThat(metadata.isNodeMarkedForRemoval("anotherNode"), is(false));
+            }
+            case RESTART -> {
+                var metadata = new NodesShutdownMetadata(Map.of("thenode", builder.setType(type).build()));
+                assertThat(metadata.isNodeMarkedForRemoval("thenode"), is(false));
+            }
+        }
+    }
+
     @Override
     protected Writeable.Reader<Diff<Metadata.Custom>> diffReader() {
         return NodesShutdownMetadata.NodeShutdownMetadataDiff::new;

server/src/test/java/org/elasticsearch/rest/RestRequestTests.java

@@ -31,7 +31,8 @@
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonMap;
-import static org.elasticsearch.rest.RestRequest.PATH_RESTRICTED;
+import static org.elasticsearch.rest.RestRequest.OPERATOR_REQUEST;
+import static org.elasticsearch.rest.RestRequest.SERVERLESS_REQUEST;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
@@ -249,16 +250,30 @@ public void testRequiredContent() {
         assertEquals("unknown content type", e.getMessage());
     }
 
-    public void testMarkPathRestricted() {
+    public void testIsServerlessRequest() {
         RestRequest request1 = contentRestRequest("content", new HashMap<>());
-        request1.markPathRestricted("foo");
-        assertEquals(request1.param(PATH_RESTRICTED), "foo");
-        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> request1.markPathRestricted("foo"));
-        assertThat(exception.getMessage(), is("The parameter [" + PATH_RESTRICTED + "] is already defined."));
-
-        RestRequest request2 = contentRestRequest("content", Map.of(PATH_RESTRICTED, "foo"));
-        exception = expectThrows(IllegalArgumentException.class, () -> request2.markPathRestricted("bar"));
-        assertThat(exception.getMessage(), is("The parameter [" + PATH_RESTRICTED + "] is already defined."));
+        request1.markAsServerlessRequest();
+        assertEquals(request1.param(SERVERLESS_REQUEST), "true");
+        assertTrue(request1.isServerlessRequest());
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, request1::markAsServerlessRequest);
+        assertThat(exception.getMessage(), is("The parameter [" + SERVERLESS_REQUEST + "] is already defined."));
+
+        RestRequest request2 = contentRestRequest("content", Map.of(SERVERLESS_REQUEST, "true"));
+        exception = expectThrows(IllegalArgumentException.class, request2::markAsServerlessRequest);
+        assertThat(exception.getMessage(), is("The parameter [" + SERVERLESS_REQUEST + "] is already defined."));
+    }
+
+    public void testIsOperatorRequest() {
+        RestRequest request1 = contentRestRequest("content", new HashMap<>());
+        request1.markAsOperatorRequest();
+        assertEquals(request1.param(OPERATOR_REQUEST), "true");
+        assertTrue(request1.isOperatorRequest());
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, request1::markAsOperatorRequest);
+        assertThat(exception.getMessage(), is("The parameter [" + OPERATOR_REQUEST + "] is already defined."));
+
+        RestRequest request2 = contentRestRequest("content", Map.of(OPERATOR_REQUEST, "true"));
+        exception = expectThrows(IllegalArgumentException.class, request2::markAsOperatorRequest);
+        assertThat(exception.getMessage(), is("The parameter [" + OPERATOR_REQUEST + "] is already defined."));
     }
 
     public static RestRequest contentRestRequest(String content, Map<String, String> params) {

server/src/test/java/org/elasticsearch/rest/RestUtilsTests.java

@@ -18,7 +18,7 @@
 import java.util.Map;
 import java.util.regex.Pattern;
 
-import static org.elasticsearch.rest.RestRequest.PATH_RESTRICTED;
+import static org.elasticsearch.rest.RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
@@ -160,13 +160,15 @@ public void testCrazyURL() {
     }
 
     public void testReservedParameters() {
-        Map<String, String> params = new HashMap<>();
-        String uri = "something?" + PATH_RESTRICTED + "=value";
-        IllegalArgumentException exception = expectThrows(
-            IllegalArgumentException.class,
-            () -> RestUtils.decodeQueryString(uri, uri.indexOf('?') + 1, params)
-        );
-        assertEquals(exception.getMessage(), "parameter [" + PATH_RESTRICTED + "] is reserved and may not set");
+        for (var reservedParam : INTERNAL_MARKER_REQUEST_PARAMETERS) {
+            Map<String, String> params = new HashMap<>();
+            String uri = "something?" + reservedParam + "=value";
+            IllegalArgumentException exception = expectThrows(
+                IllegalArgumentException.class,
+                () -> RestUtils.decodeQueryString(uri, uri.indexOf('?') + 1, params)
+            );
+            assertEquals(exception.getMessage(), "parameter [" + reservedParam + "] is reserved and may not be set");
+        }
     }
 
     private void assertCorsSettingRegexIsNull(String settingsValue) {