Merge remote-tracking branch 'nik9000/shape_multifield_warning' into …

…shape_multifield_warning

docs/changelog/120852.yaml

@@ -0,0 +1,5 @@
+pr: 120852
+summary: Correct line and column numbers of missing named parameters
+area: ES|QL
+type: bug
+issues: []

docs/changelog/121324.yaml

@@ -0,0 +1,6 @@
+pr: 121324
+summary: Support duplicate suggestions in completion field
+area: Suggesters
+type: bug
+issues:
+ - 82432

libs/core/src/main/java/org/elasticsearch/core/CheckedSupplier.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.core;
+
+/**
+ * A {@link java.util.function.Supplier}-like interface which allows throwing checked exceptions.
+ */
+@FunctionalInterface
+public interface CheckedSupplier<T, E extends Exception> {
+    T get() throws E;
+}

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/DummyImplementations.java

@@ -52,10 +52,9 @@
  * <p>
  * A bit like Mockito but way more painful.
  */
-class DummyImplementations {
-
-    static class DummyLocaleServiceProvider extends LocaleServiceProvider {
+public class DummyImplementations {
 
+    public static class DummyLocaleServiceProvider extends LocaleServiceProvider {
         @Override
         public Locale[] getAvailableLocales() {
             throw unexpected();

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java

@@ -96,6 +96,9 @@ static CheckAction alwaysDenied(CheckedRunnable<Exception> action) {
 
     private static final Map<String, CheckAction> checkActions = Stream.concat(
         Stream.<Entry<String, CheckAction>>of(
+            entry("static_reflection", deniedToPlugins(RestEntitlementsCheckAction::staticMethodNeverEntitledViaReflection)),
+            entry("nonstatic_reflection", deniedToPlugins(RestEntitlementsCheckAction::nonstaticMethodNeverEntitledViaReflection)),
+            entry("constructor_reflection", deniedToPlugins(RestEntitlementsCheckAction::constructorNeverEntitledViaReflection)),
             entry("runtime_exit", deniedToPlugins(RestEntitlementsCheckAction::runtimeExit)),
             entry("runtime_halt", deniedToPlugins(RestEntitlementsCheckAction::runtimeHalt)),
             entry("system_exit", deniedToPlugins(RestEntitlementsCheckAction::systemExit)),
@@ -338,6 +341,11 @@ private static void systemExit() {
         System.exit(123);
     }
 
+    private static void staticMethodNeverEntitledViaReflection() throws Exception {
+        Method systemExit = System.class.getMethod("exit", int.class);
+        systemExit.invoke(null, 123);
+    }
+
     private static void createClassLoader() throws IOException {
         try (var classLoader = new URLClassLoader("test", new URL[0], RestEntitlementsCheckAction.class.getClassLoader())) {
             logger.info("Created URLClassLoader [{}]", classLoader.getName());
@@ -348,6 +356,11 @@ private static void processBuilder_start() throws IOException {
         new ProcessBuilder("").start();
     }
 
+    private static void nonstaticMethodNeverEntitledViaReflection() throws Exception {
+        Method processBuilderStart = ProcessBuilder.class.getMethod("start");
+        processBuilderStart.invoke(new ProcessBuilder(""));
+    }
+
     private static void processBuilder_startPipeline() throws IOException {
         ProcessBuilder.startPipeline(List.of());
     }
@@ -386,6 +399,10 @@ private static void setHttpsConnectionProperties() {
         new DummyLocaleServiceProvider();
     }
 
+    private static void constructorNeverEntitledViaReflection() throws Exception {
+        DummyLocaleServiceProvider.class.getConstructor().newInstance();
+    }
+
     private static void breakIteratorProvider$() {
         new DummyBreakIteratorProvider();
     }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java

@@ -14,6 +14,8 @@
 import com.sun.tools.attach.AttachNotSupportedException;
 import com.sun.tools.attach.VirtualMachine;
 
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.core.CheckedSupplier;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.entitlement.initialization.EntitlementInitialization;
 import org.elasticsearch.entitlement.runtime.api.NotEntitledException;
@@ -22,8 +24,10 @@
 import org.elasticsearch.logging.Logger;
 
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.attribute.FileAttribute;
 import java.util.Map;
 import java.util.function.Function;
 
@@ -144,30 +148,31 @@ private static String findAgentJar() {
      * @throws IllegalStateException if the entitlements system can't prevent an unauthorized action of our choosing
      */
     private static void selfTest() {
-        ensureCannotStartProcess();
-        ensureCanCreateTempFile();
+        ensureCannotStartProcess(ProcessBuilder::start);
+        ensureCanCreateTempFile(EntitlementBootstrap::createTempFile);
+
+        // Try again with reflection
+        ensureCannotStartProcess(EntitlementBootstrap::reflectiveStartProcess);
+        ensureCanCreateTempFile(EntitlementBootstrap::reflectiveCreateTempFile);
     }
 
-    private static void ensureCannotStartProcess() {
+    private static void ensureCannotStartProcess(CheckedConsumer<ProcessBuilder, ?> startProcess) {
         try {
             // The command doesn't matter; it doesn't even need to exist
-            new ProcessBuilder("").start();
+            startProcess.accept(new ProcessBuilder(""));
         } catch (NotEntitledException e) {
             logger.debug("Success: Entitlement protection correctly prevented process creation");
             return;
-        } catch (IOException e) {
+        } catch (Exception e) {
             throw new IllegalStateException("Failed entitlement protection self-test", e);
         }
         throw new IllegalStateException("Entitlement protection self-test was incorrectly permitted");
     }
 
-    /**
-     * Originally {@code Security.selfTest}.
-     */
     @SuppressForbidden(reason = "accesses jvm default tempdir as a self-test")
-    private static void ensureCanCreateTempFile() {
+    private static void ensureCanCreateTempFile(CheckedSupplier<Path, ?> createTempFile) {
         try {
-            Path p = Files.createTempFile(null, null);
+            Path p = createTempFile.get();
             p.toFile().deleteOnExit();
 
             // Make an effort to clean up the file immediately; also, deleteOnExit leaves the file if the JVM exits abnormally.
@@ -184,5 +189,24 @@ private static void ensureCanCreateTempFile() {
         logger.debug("Success: Entitlement protection correctly permitted temp file creation");
     }
 
+    @SuppressForbidden(reason = "accesses jvm default tempdir as a self-test")
+    private static Path createTempFile() throws Exception {
+        return Files.createTempFile(null, null);
+    }
+
+    private static void reflectiveStartProcess(ProcessBuilder pb) throws Exception {
+        try {
+            var start = ProcessBuilder.class.getMethod("start");
+            start.invoke(pb);
+        } catch (InvocationTargetException e) {
+            throw (Exception) e.getCause();
+        }
+    }
+
+    private static Path reflectiveCreateTempFile() throws Exception {
+        return (Path) Files.class.getMethod("createTempFile", String.class, String.class, FileAttribute[].class)
+            .invoke(null, null, null, new FileAttribute<?>[0]);
+    }
+
     private static final Logger logger = LogManager.getLogger(EntitlementBootstrap.class);
 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/suggest/30_context.yml

@@ -395,3 +395,75 @@ setup:
                 field: suggest_multi_contexts
                 contexts:
                   location: []
+
+---
+"Duplicate suggestions in different contexts":
+  - requires:
+      cluster_features: [ "search.completion_field.duplicate.support" ]
+      reason: "Support for duplicate suggestions in different contexts"
+
+  - do:
+      index:
+        refresh: true
+        index: test
+        id:    "1"
+        body:
+          suggest_context:
+            -
+              input: "foox"
+              weight: 2
+              contexts:
+                color: ["red", "yellow"]
+            -
+              input: "foox"
+              weight: 3
+              contexts:
+                color: ["blue", "green", "yellow"]
+  - do:
+      search:
+        body:
+          suggest:
+            result:
+              text: "foo"
+              completion:
+                field: suggest_context
+                contexts:
+                  color: "red"
+
+  - length: { suggest.result: 1  }
+  - length: { suggest.result.0.options: 1  }
+  - match:  { suggest.result.0.options.0.text: "foox" }
+  - match:  { suggest.result.0.options.0._score: 2 }
+
+  - do:
+      search:
+        body:
+          suggest:
+            result:
+              text: "foo"
+              completion:
+                field: suggest_context
+                contexts:
+                  color: "yellow"
+
+  - length: { suggest.result: 1  }
+  - length: { suggest.result.0.options: 1  }
+  - match:  { suggest.result.0.options.0.text: "foox" }
+  # the highest weight wins
+  - match:  { suggest.result.0.options.0._score: 3 }
+
+  - do:
+      search:
+        body:
+          suggest:
+            result:
+              text: "foo"
+              completion:
+                field: suggest_context
+                contexts:
+                  color: "blue"
+
+  - length: { suggest.result: 1 }
+  - length: { suggest.result.0.options: 1 }
+  - match: { suggest.result.0.options.0.text: "foox" }
+  - match: { suggest.result.0.options.0._score: 3 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/suggest/50_completion_with_multi_fields.yml

@@ -268,3 +268,80 @@
 
   - length: { suggest.result: 1  }
   - length: { suggest.result.0.options: 1  }
+
+---
+"Duplicate suggestions in different contexts in sub-fields":
+  - requires:
+      cluster_features: [ "search.completion_field.duplicate.support" ]
+      reason: "Support for duplicate suggestions in different contexts"
+
+  - do:
+      indices.create:
+        index: completion_with_context
+        body:
+          mappings:
+            "properties":
+              "suggest_1":
+                "type": "completion"
+                "contexts":
+                  -
+                    "name": "color"
+                    "type": "category"
+                "fields":
+                  "suggest_2":
+                    "type": "completion"
+                    "contexts":
+                      -
+                        "name": "color"
+                        "type": "category"
+
+
+  - do:
+      index:
+        refresh: true
+        index: completion_with_context
+        id:    "1"
+        body:
+          suggest_1:
+            -
+              input: "foox"
+              weight: 2
+              contexts:
+                color: ["red"]
+            -
+              input: "foox"
+              weight: 3
+              contexts:
+                color: ["blue", "green"]
+  - do:
+      search:
+        body:
+          suggest:
+            result:
+              text: "foo"
+              completion:
+                field: suggest_1.suggest_2
+                contexts:
+                  color: "red"
+
+  - length: { suggest.result: 1  }
+  - length: { suggest.result.0.options: 1  }
+  - match:  { suggest.result.0.options.0.text: "foox" }
+  - match:  { suggest.result.0.options.0._score: 2 }
+
+
+  - do:
+      search:
+        body:
+          suggest:
+            result:
+              text: "foo"
+              completion:
+                field: suggest_1.suggest_2
+                contexts:
+                  color: "blue"
+
+  - length: { suggest.result: 1 }
+  - length: { suggest.result.0.options: 1 }
+  - match: { suggest.result.0.options.0.text: "foox" }
+  - match: { suggest.result.0.options.0._score: 3 }

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -614,6 +614,7 @@ public void apply(Settings value, Settings current, Settings previous) {
         DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING,
         IndicesClusterStateService.SHARD_LOCK_RETRY_INTERVAL_SETTING,
         IndicesClusterStateService.SHARD_LOCK_RETRY_TIMEOUT_SETTING,
+        IndicesClusterStateService.CONCURRENT_SHARD_CLOSE_LIMIT,
         IngestSettings.GROK_WATCHDOG_INTERVAL,
         IngestSettings.GROK_WATCHDOG_MAX_EXECUTION_TIME,
         TDigestExecutionHint.SETTING,