Merge branch 'main' into smarter_field_caps_subscribableListener

.editorconfig

@@ -209,7 +209,7 @@ indent_size = 4
 max_line_length = 140
 ij_java_class_count_to_use_import_on_demand = 999
 ij_java_names_count_to_use_import_on_demand = 999
-ij_java_imports_layout = *,|,com.**,|,org.**,|,java.**,javax.**,|,$*
+ij_java_imports_layout = *,|,com.**,|,org.**,|,java.**,|,javax.**,|,$*
 
 [*.json]
 indent_size = 2

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -142,13 +142,18 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     description = 'Enables preview features on native library module'
     dependsOn tasks.named("enableExternalConfiguration")
 
-    doLast {
-      ['main', 'test'].each { sourceSet ->
-        modifyXml(".idea/modules/libs/native/elasticsearch.libs.native.${sourceSet}.iml") { xml ->
-          xml.component.find { it.'@name' == 'NewModuleRootManager' }?.'@LANGUAGE_LEVEL' = 'JDK_21_PREVIEW'
+    ext {
+      enablePreview = { moduleFile, languageLevel ->
+        modifyXml(moduleFile) { xml ->
+          xml.component.find { it.'@name' == 'NewModuleRootManager' }?.'@LANGUAGE_LEVEL' = languageLevel
         }
       }
     }
+
+    doLast {
+      enablePreview('.idea/modules/libs/native/elasticsearch.libs.native.main.iml', 'JDK_21_PREVIEW')
+      enablePreview('.idea/modules/libs/native/elasticsearch.libs.native.test.iml', 'JDK_21_PREVIEW')
+    }
   }
 
   tasks.register('buildDependencyArtifacts') {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -22,7 +22,7 @@ public enum DockerBase {
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:55b297da5151d2a2997e8ab9729fe1304e4869389d7090ab7031cc29530f69f8",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:32f06b169bb4b0f257fbb10e8c8379f06d3ee1355c89b3327cb623781a29590e",
         "-wolfi",
         "apk"
     ),

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -98,7 +98,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private static final int ES_DESTROY_TIMEOUT = 20;
     private static final TimeUnit ES_DESTROY_TIMEOUT_UNIT = TimeUnit.SECONDS;
 
-    private static final int NODE_UP_TIMEOUT = 2;
+    private static final int NODE_UP_TIMEOUT = 3;
     private static final TimeUnit NODE_UP_TIMEOUT_UNIT = TimeUnit.MINUTES;
     private static final int ADDITIONAL_CONFIG_TIMEOUT = 15;
     private static final TimeUnit ADDITIONAL_CONFIG_TIMEOUT_UNIT = TimeUnit.SECONDS;

docs/changelog/117235.yaml

@@ -0,0 +1,5 @@
+pr: 117235
+summary: "Deprecate `ChunkingOptions` parameter"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/117595.yaml

@@ -0,0 +1,5 @@
+pr: 117595
+summary: Fix for Deberta tokenizer when input sequence exceeds 512 tokens
+area: Machine Learning
+type: bug
+issues: []

docs/reference/inference/service-elser.asciidoc

@@ -102,10 +102,39 @@ If `adaptive_allocations` is enabled, do not set this value, because it's automa
 Sets the number of threads used by each model allocation during inference. This generally increases the speed per inference request. The inference process is a compute-bound process; `threads_per_allocations` must not exceed the number of available allocated processors per node.
 Must be a power of 2. Max allowed value is 32.
 
+[discrete]
+[[inference-example-elser-adaptive-allocation]]
+==== ELSER service example with adaptive allocations
+
+When adaptive allocations are enabled, the number of allocations of the model is set automatically based on the current load.
+
+NOTE: For more information on how to optimize your ELSER endpoints, refer to {ml-docs}/ml-nlp-elser.html#elser-recommendations[the ELSER recommendations] section in the model documentation.
+To learn more about model autoscaling, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] page.
+
+The following example shows how to create an {infer} endpoint called `my-elser-model` to perform a `sparse_embedding` task type and configure adaptive allocations.
+
+The request below will automatically download the ELSER model if it isn't already downloaded and then deploy the model.
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/sparse_embedding/my-elser-model
+{
+  "service": "elser",
+  "service_settings": {
+    "adaptive_allocations": {
+      "enabled": true,
+      "min_number_of_allocations": 3,
+      "max_number_of_allocations": 10
+    },
+    "num_threads": 1
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
 
 [discrete]
 [[inference-example-elser]]
-==== ELSER service example
+==== ELSER service example without adaptive allocations
 
 The following example shows how to create an {infer} endpoint called `my-elser-model` to perform a `sparse_embedding` task type.
 Refer to the {ml-docs}/ml-nlp-elser.html[ELSER model documentation] for more info.
@@ -151,32 +180,4 @@ You might see a 502 bad gateway error in the response when using the {kib} Conso
 This error usually just reflects a timeout, while the model downloads in the background.
 You can check the download progress in the {ml-app} UI.
 If using the Python client, you can set the `timeout` parameter to a higher value.
-====
-
-[discrete]
-[[inference-example-elser-adaptive-allocation]]
-==== Setting adaptive allocations for the ELSER service
-
-NOTE: For more information on how to optimize your ELSER endpoints, refer to {ml-docs}/ml-nlp-elser.html#elser-recommendations[the ELSER recommendations] section in the model documentation.
-To learn more about model autoscaling, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] page.
-
-The following example shows how to create an {infer} endpoint called `my-elser-model` to perform a `sparse_embedding` task type and configure adaptive allocations.
-
-The request below will automatically download the ELSER model if it isn't already downloaded and then deploy the model.
-
-[source,console]
-------------------------------------------------------------
-PUT _inference/sparse_embedding/my-elser-model
-{
-  "service": "elser",
-  "service_settings": {
-    "adaptive_allocations": {
-      "enabled": true,
-      "min_number_of_allocations": 3,
-      "max_number_of_allocations": 10
-    },
-    "num_threads": 1
-  }
-}
-------------------------------------------------------------
-// TEST[skip:TBD]
+====

libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java

@@ -91,15 +91,18 @@ static MethodKey parseCheckerMethodSignature(String checkerMethodName, Type[] ch
                 String.format(
                     Locale.ROOT,
                     "Checker method %s has incorrect name format. "
-                        + "It should be either check$$methodName (instance) or check$package_ClassName$methodName (static)",
+                        + "It should be either check$$methodName (instance), check$package_ClassName$methodName (static) or "
+                        + "check$package_ClassName$ (ctor)",
                     checkerMethodName
                 )
             );
         }
 
-        // No "className" (check$$methodName) -> method is static, and we'll get the class from the actual typed argument
+        // No "className" (check$$methodName) -> method is instance, and we'll get the class from the actual typed argument
         final boolean targetMethodIsStatic = classNameStartIndex + 1 != classNameEndIndex;
-        final String targetMethodName = checkerMethodName.substring(classNameEndIndex + 1);
+        // No "methodName" (check$package_ClassName$) -> method is ctor
+        final boolean targetMethodIsCtor = classNameEndIndex + 1 == checkerMethodName.length();
+        final String targetMethodName = targetMethodIsCtor ? "<init>" : checkerMethodName.substring(classNameEndIndex + 1);
 
         final String targetClassName;
         final List<String> targetParameterTypes;

libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java

@@ -154,11 +154,12 @@ public MethodVisitor visitMethod(int access, String name, String descriptor, Str
             var mv = super.visitMethod(access, name, descriptor, signature, exceptions);
             if (isAnnotationPresent == false) {
                 boolean isStatic = (access & ACC_STATIC) != 0;
+                boolean isCtor = "<init>".equals(name);
                 var key = new MethodKey(className, name, Stream.of(Type.getArgumentTypes(descriptor)).map(Type::getInternalName).toList());
                 var instrumentationMethod = instrumentationMethods.get(key);
                 if (instrumentationMethod != null) {
                     // LOGGER.debug("Will instrument method {}", key);
-                    return new EntitlementMethodVisitor(Opcodes.ASM9, mv, isStatic, descriptor, instrumentationMethod);
+                    return new EntitlementMethodVisitor(Opcodes.ASM9, mv, isStatic, isCtor, descriptor, instrumentationMethod);
                 } else {
                     // LOGGER.trace("Will not instrument method {}", key);
                 }
@@ -187,6 +188,7 @@ private void addClassAnnotationIfNeeded() {
 
     class EntitlementMethodVisitor extends MethodVisitor {
         private final boolean instrumentedMethodIsStatic;
+        private final boolean instrumentedMethodIsCtor;
         private final String instrumentedMethodDescriptor;
         private final CheckerMethod instrumentationMethod;
         private boolean hasCallerSensitiveAnnotation = false;
@@ -195,11 +197,13 @@ class EntitlementMethodVisitor extends MethodVisitor {
             int api,
             MethodVisitor methodVisitor,
             boolean instrumentedMethodIsStatic,
+            boolean instrumentedMethodIsCtor,
             String instrumentedMethodDescriptor,
             CheckerMethod instrumentationMethod
         ) {
             super(api, methodVisitor);
             this.instrumentedMethodIsStatic = instrumentedMethodIsStatic;
+            this.instrumentedMethodIsCtor = instrumentedMethodIsCtor;
             this.instrumentedMethodDescriptor = instrumentedMethodDescriptor;
             this.instrumentationMethod = instrumentationMethod;
         }
@@ -260,14 +264,15 @@ private void pushCallerClass() {
 
         private void forwardIncomingArguments() {
             int localVarIndex = 0;
-            if (instrumentedMethodIsStatic == false) {
+            if (instrumentedMethodIsCtor) {
+                localVarIndex++;
+            } else if (instrumentedMethodIsStatic == false) {
                 mv.visitVarInsn(Opcodes.ALOAD, localVarIndex++);
             }
             for (Type type : Type.getArgumentTypes(instrumentedMethodDescriptor)) {
                 mv.visitVarInsn(type.getOpcode(Opcodes.ILOAD), localVarIndex);
                 localVarIndex += type.getSize();
             }
-
         }
 
         private void invokeInstrumentationMethod() {

libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java

@@ -45,6 +45,12 @@ interface TestCheckerOverloads {
         void check$org_example_TestTargetClass$staticMethodWithOverload(Class<?> clazz, int x, String y);
     }
 
+    interface TestCheckerCtors {
+        void check$org_example_TestTargetClass$(Class<?> clazz);
+
+        void check$org_example_TestTargetClass$(Class<?> clazz, int x, String y);
+    }
+
     public void testInstrumentationTargetLookup() throws IOException, ClassNotFoundException {
         Map<MethodKey, CheckerMethod> methodsMap = instrumentationService.lookupMethodsToInstrument(TestChecker.class.getName());
 
@@ -142,6 +148,38 @@ public void testInstrumentationTargetLookupWithOverloads() throws IOException, C
         );
     }
 
+    public void testInstrumentationTargetLookupWithCtors() throws IOException, ClassNotFoundException {
+        Map<MethodKey, CheckerMethod> methodsMap = instrumentationService.lookupMethodsToInstrument(TestCheckerCtors.class.getName());
+
+        assertThat(methodsMap, aMapWithSize(2));
+        assertThat(
+            methodsMap,
+            hasEntry(
+                equalTo(new MethodKey("org/example/TestTargetClass", "<init>", List.of("I", "java/lang/String"))),
+                equalTo(
+                    new CheckerMethod(
+                        "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestCheckerCtors",
+                        "check$org_example_TestTargetClass$",
+                        List.of("Ljava/lang/Class;", "I", "Ljava/lang/String;")
+                    )
+                )
+            )
+        );
+        assertThat(
+            methodsMap,
+            hasEntry(
+                equalTo(new MethodKey("org/example/TestTargetClass", "<init>", List.of())),
+                equalTo(
+                    new CheckerMethod(
+                        "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestCheckerCtors",
+                        "check$org_example_TestTargetClass$",
+                        List.of("Ljava/lang/Class;")
+                    )
+                )
+            )
+        );
+    }
+
     public void testParseCheckerMethodSignatureStaticMethod() {
         var methodKey = InstrumentationServiceImpl.parseCheckerMethodSignature(
             "check$org_example_TestClass$staticMethod",
@@ -169,6 +207,24 @@ public void testParseCheckerMethodSignatureStaticMethodInnerClass() {
         assertThat(methodKey, equalTo(new MethodKey("org/example/TestClass$InnerClass", "staticMethod", List.of())));
     }
 
+    public void testParseCheckerMethodSignatureCtor() {
+        var methodKey = InstrumentationServiceImpl.parseCheckerMethodSignature(
+            "check$org_example_TestClass$",
+            new Type[] { Type.getType(Class.class) }
+        );
+
+        assertThat(methodKey, equalTo(new MethodKey("org/example/TestClass", "<init>", List.of())));
+    }
+
+    public void testParseCheckerMethodSignatureCtorWithArgs() {
+        var methodKey = InstrumentationServiceImpl.parseCheckerMethodSignature(
+            "check$org_example_TestClass$",
+            new Type[] { Type.getType(Class.class), Type.getType("I"), Type.getType(String.class) }
+        );
+
+        assertThat(methodKey, equalTo(new MethodKey("org/example/TestClass", "<init>", List.of("I", "java/lang/String"))));
+    }
+
     public void testParseCheckerMethodSignatureIncorrectName() {
         var exception = assertThrows(
             IllegalArgumentException.class,