Merge branch 'main' of https://github.com/elastic/elasticsearch into …

…smarter_field_caps_subscribableListener

build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt

@@ -167,5 +167,3 @@ org.elasticsearch.cluster.SnapshotDeletionsInProgress$Entry#<init>(java.lang.Str
 @defaultMessage Use a Thread constructor with a name, anonymous threads are more difficult to debug
 java.lang.Thread#<init>(java.lang.Runnable)
 java.lang.Thread#<init>(java.lang.ThreadGroup, java.lang.Runnable)
-
-org.elasticsearch.common.bytes.BytesReference#copyBytes(org.elasticsearch.common.bytes.BytesReference) @ This method is a subject for removal. Copying bytes is prone to performance regressions and unnecessary allocations.

docs/changelog/116739.yaml

@@ -0,0 +1,5 @@
+pr: 116739
+summary: Change default Docker image to be based on UBI minimal instead of Ubuntu
+area: Infra/Core
+type: enhancement
+issues: []

docs/changelog/117303.yaml

@@ -0,0 +1,5 @@
+pr: 117303
+summary: Remove HTTP content copies
+area: Network
+type: enhancement
+issues: []

docs/changelog/117551.yaml

@@ -0,0 +1,5 @@
+pr: 117551
+summary: Fix stats by constant expresson with alias
+area: ES|QL
+type: bug
+issues: []

docs/reference/connector/docs/connectors-content-syncs.asciidoc

@@ -52,7 +52,7 @@ However, a fast, accessible third-party data source that stores huge amounts of
 
 [NOTE]
 ====
-Incremental syncs for the SharePoint Online connector use specific logic.
+Incremental syncs for <<es-connectors-sharepoint-online,SharePoint Online>> and <<es-connectors-google-drive,Google Drive>> connectors use specific logic.
 All other connectors use the same shared connector framework logic for incremental syncs.
 ====
 

docs/reference/data-streams/tsds.asciidoc

@@ -339,4 +339,5 @@ include::tsds-index-settings.asciidoc[]
 include::downsampling.asciidoc[]
 include::downsampling-ilm.asciidoc[]
 include::downsampling-manual.asciidoc[]
+include::downsampling-dsl.asciidoc[]
 include::tsds-reindex.asciidoc[]

docs/reference/intro.asciidoc

@@ -85,7 +85,7 @@ You can deploy {es} in various ways.
 **Hosted options**
 
 * {cloud}/ec-getting-started-trial.html[*Elastic Cloud Hosted*]: {es} is available as part of the hosted Elastic Stack offering, deployed in the cloud with your provider of choice. Sign up for a https://cloud.elastic.co/registration[14-day free trial].
-* {serverless-docs}/general/sign-up-trial[*Elastic Cloud Serverless* (technical preview)]: Create serverless projects for autoscaled and fully managed {es} deployments. Sign up for a https://cloud.elastic.co/serverless-registration[14-day free trial].
+* {serverless-docs}/general/sign-up-trial[*Elastic Cloud Serverless*]: Create serverless projects for autoscaled and fully managed {es} deployments. Sign up for a https://cloud.elastic.co/serverless-registration[14-day free trial].
 
 **Advanced options**
 

modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java

@@ -33,7 +33,7 @@
 public class InternalAdjacencyMatrix extends InternalMultiBucketAggregation<InternalAdjacencyMatrix, InternalAdjacencyMatrix.InternalBucket>
     implements
         AdjacencyMatrix {
-    public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucket implements AdjacencyMatrix.Bucket {
+    public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucketWritable implements AdjacencyMatrix.Bucket {
 
         private final String key;
         private final long docCount;

modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java

@@ -34,7 +34,7 @@ public class InternalTimeSeries extends InternalMultiBucketAggregation<InternalT
     /**
      * A bucket associated with a specific time series (identified by its key)
      */
-    public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucket {
+    public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucketWritable {
         protected long bucketOrd;
         protected final BytesRef key;
         // TODO: make computing docCount optional

modules/lang-mustache/src/internalClusterTest/java/org/elasticsearch/script/mustache/SearchTemplateIT.java

@@ -13,12 +13,10 @@
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptAction;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
-import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.TransportDeleteStoredScriptAction;
 import org.elasticsearch.action.admin.cluster.storedscripts.TransportPutStoredScriptAction;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptType;
@@ -39,6 +37,7 @@
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 
+import static org.elasticsearch.action.admin.cluster.storedscripts.StoredScriptIntegTestUtils.newPutStoredScriptTestRequest;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
@@ -467,12 +466,6 @@ public static void assertHitCount(SearchTemplateRequestBuilder requestBuilder, l
     }
 
     private void putJsonStoredScript(String id, String jsonContent) {
-        assertAcked(
-            safeExecute(
-                TransportPutStoredScriptAction.TYPE,
-                new PutStoredScriptRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).id(id)
-                    .content(new BytesArray(jsonContent), XContentType.JSON)
-            )
-        );
+        assertAcked(safeExecute(TransportPutStoredScriptAction.TYPE, newPutStoredScriptTestRequest(id, jsonContent)));
     }
 }

modules/repository-s3/build.gradle

@@ -46,6 +46,7 @@ dependencies {
   yamlRestTestImplementation project(":test:framework")
   yamlRestTestImplementation project(':test:fixtures:s3-fixture')
   yamlRestTestImplementation project(':test:fixtures:ec2-imds-fixture')
+  yamlRestTestImplementation project(':test:fixtures:aws-sts-fixture')
   yamlRestTestImplementation project(':test:fixtures:minio-fixture')
   internalClusterTestImplementation project(':test:fixtures:minio-fixture')
 

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RestReloadCredentialsIT.java

@@ -35,7 +35,14 @@ public class RepositoryS3RestReloadCredentialsIT extends ESRestTestCase {
     private static final String BUCKET = "RepositoryS3RestReloadCredentialsIT-bucket-" + HASHED_SEED;
     private static final String BASE_PATH = "RepositoryS3RestReloadCredentialsIT-base-path-" + HASHED_SEED;
 
-    public static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, "ignored");
+    private static volatile String repositoryAccessKey;
+
+    public static final S3HttpFixture s3Fixture = new S3HttpFixture(
+        true,
+        BUCKET,
+        BASE_PATH,
+        S3HttpFixture.mutableAccessKey(() -> repositoryAccessKey)
+    );
 
     private static final MutableSettingsProvider keystoreSettings = new MutableSettingsProvider();
 
@@ -68,7 +75,7 @@ public void testReloadCredentialsFromKeystore() throws IOException {
 
         // Set up initial credentials
         final var accessKey1 = randomIdentifier();
-        s3Fixture.setAccessKey(accessKey1);
+        repositoryAccessKey = accessKey1;
         keystoreSettings.put("s3.client.default.access_key", accessKey1);
         keystoreSettings.put("s3.client.default.secret_key", randomIdentifier());
         cluster.updateStoredSecureSettings();
@@ -79,14 +86,14 @@ public void testReloadCredentialsFromKeystore() throws IOException {
 
         // Rotate credentials in blob store
         final var accessKey2 = randomValueOtherThan(accessKey1, ESTestCase::randomIdentifier);
-        s3Fixture.setAccessKey(accessKey2);
+        repositoryAccessKey = accessKey2;
 
         // Ensure that initial credentials now invalid
         final var accessDeniedException2 = expectThrows(ResponseException.class, () -> client().performRequest(verifyRequest));
         assertThat(accessDeniedException2.getResponse().getStatusLine().getStatusCode(), equalTo(500));
         assertThat(
             accessDeniedException2.getMessage(),
-            allOf(containsString("Bad access key"), containsString("Status Code: 403"), containsString("Error Code: AccessDenied"))
+            allOf(containsString("Access denied"), containsString("Status Code: 403"), containsString("Error Code: AccessDenied"))
         );
 
         // Set up refreshed credentials

modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java

@@ -10,8 +10,8 @@
 package org.elasticsearch.repositories.s3;
 
 import fixture.aws.imds.Ec2ImdsHttpFixture;
+import fixture.s3.DynamicS3Credentials;
 import fixture.s3.S3HttpFixture;
-import fixture.s3.S3HttpFixtureWithSessionToken;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
@@ -34,27 +34,30 @@ public class RepositoryS3ClientYamlTestSuiteIT extends AbstractRepositoryS3Clien
 
     private static final String HASHED_SEED = Integer.toString(Murmur3HashFunction.hash(System.getProperty("tests.seed")));
     private static final String TEMPORARY_SESSION_TOKEN = "session_token-" + HASHED_SEED;
-    private static final String IMDS_ACCESS_KEY = "imds-access-key-" + HASHED_SEED;
-    private static final String IMDS_SESSION_TOKEN = "imds-session-token-" + HASHED_SEED;
 
     private static final S3HttpFixture s3Fixture = new S3HttpFixture();
 
-    private static final S3HttpFixtureWithSessionToken s3HttpFixtureWithSessionToken = new S3HttpFixtureWithSessionToken(
+    private static final S3HttpFixture s3HttpFixtureWithSessionToken = new S3HttpFixture(
+        true,
         "session_token_bucket",
         "session_token_base_path_integration_tests",
-        System.getProperty("s3TemporaryAccessKey"),
-        TEMPORARY_SESSION_TOKEN
+        S3HttpFixture.fixedAccessKeyAndToken(System.getProperty("s3TemporaryAccessKey"), TEMPORARY_SESSION_TOKEN)
     );
 
-    private static final S3HttpFixtureWithSessionToken s3HttpFixtureWithImdsSessionToken = new S3HttpFixtureWithSessionToken(
+    private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials();
+
+    private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
+        dynamicS3Credentials::addValidCredentials,
+        Set.of()
+    );
+
+    private static final S3HttpFixture s3HttpFixtureWithImdsSessionToken = new S3HttpFixture(
+        true,
         "ec2_bucket",
         "ec2_base_path",
-        IMDS_ACCESS_KEY,
-        IMDS_SESSION_TOKEN
+        dynamicS3Credentials::isAuthorized
     );
 
-    private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(IMDS_ACCESS_KEY, IMDS_SESSION_TOKEN, Set.of());
-
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-s3")
         .keystore("s3.client.integration_test_permanent.access_key", System.getProperty("s3PermanentAccessKey"))

modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsClientYamlTestSuiteIT.java

@@ -10,12 +10,12 @@
 package org.elasticsearch.repositories.s3;
 
 import fixture.aws.imds.Ec2ImdsHttpFixture;
-import fixture.s3.S3HttpFixtureWithSessionToken;
+import fixture.s3.DynamicS3Credentials;
+import fixture.s3.S3HttpFixture;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
-import org.elasticsearch.cluster.routing.Murmur3HashFunction;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.junit.ClassRule;
@@ -26,23 +26,20 @@
 
 public class RepositoryS3EcsClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT {
 
-    private static final String HASHED_SEED = Integer.toString(Murmur3HashFunction.hash(System.getProperty("tests.seed")));
-    private static final String ECS_ACCESS_KEY = "ecs-access-key-" + HASHED_SEED;
-    private static final String ECS_SESSION_TOKEN = "ecs-session-token-" + HASHED_SEED;
-
-    private static final S3HttpFixtureWithSessionToken s3Fixture = new S3HttpFixtureWithSessionToken(
-        "ecs_bucket",
-        "ecs_base_path",
-        ECS_ACCESS_KEY,
-        ECS_SESSION_TOKEN
-    );
+    private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials();
 
     private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
-        ECS_ACCESS_KEY,
-        ECS_SESSION_TOKEN,
+        dynamicS3Credentials::addValidCredentials,
         Set.of("/ecs_credentials_endpoint")
     );
 
+    private static final S3HttpFixture s3Fixture = new S3HttpFixture(
+        true,
+        "ecs_bucket",
+        "ecs_base_path",
+        dynamicS3Credentials::isAuthorized
+    );
+
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-s3")
         .setting("s3.client.integration_test_ecs.endpoint", s3Fixture::getAddress)

modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3StsClientYamlTestSuiteIT.java

@@ -9,8 +9,9 @@
 
 package org.elasticsearch.repositories.s3;
 
+import fixture.aws.sts.AwsStsHttpFixture;
+import fixture.s3.DynamicS3Credentials;
 import fixture.s3.S3HttpFixture;
-import fixture.s3.S3HttpFixtureWithSTS;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
@@ -24,13 +25,27 @@
 
 public class RepositoryS3StsClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT {
 
-    public static final S3HttpFixture s3Fixture = new S3HttpFixture();
-    private static final S3HttpFixtureWithSTS s3Sts = new S3HttpFixtureWithSTS();
+    private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials();
+
+    private static final S3HttpFixture s3HttpFixture = new S3HttpFixture(
+        true,
+        "sts_bucket",
+        "sts_base_path",
+        dynamicS3Credentials::isAuthorized
+    );
+
+    private static final AwsStsHttpFixture stsHttpFixture = new AwsStsHttpFixture(dynamicS3Credentials::addValidCredentials, """
+        Atza|IQEBLjAsAhRFiXuWpUXuRvQ9PZL3GMFcYevydwIUFAHZwXZXXXXXXXXJnrulxKDHwy87oGKPznh0D6bEQZTSCzyoCtL_8S07pLpr0zMbn6w1lfVZKNTBdDans\
+        FBmtGnIsIapjI6xKR02Yc_2bQ8LZbUXSGm6Ry6_BG7PrtLZtj_dfCTj92xNGed-CrKqjG7nPBjNIL016GGvuS5gSvPRUxWES3VYfm1wl7WTI7jn-Pcb6M-buCgHhFO\
+        zTQxod27L9CqnOLio7N3gZAGpsp6n1-AJBOCJckcyXe2c6uD0srOJeZlKUm2eTDVMf8IehDVI0r1QOnTV6KzzAI3OY87Vd_cVMQ""");
 
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-s3")
-        .setting("s3.client.integration_test_sts.endpoint", s3Sts::getAddress)
-        .systemProperty("com.amazonaws.sdk.stsMetadataServiceEndpointOverride", () -> s3Sts.getAddress() + "/assume-role-with-web-identity")
+        .setting("s3.client.integration_test_sts.endpoint", s3HttpFixture::getAddress)
+        .systemProperty(
+            "com.amazonaws.sdk.stsMetadataServiceEndpointOverride",
+            () -> stsHttpFixture.getAddress() + "/assume-role-with-web-identity"
+        )
         .configFile("repository-s3/aws-web-identity-token-file", Resource.fromClasspath("aws-web-identity-token-file"))
         .environment("AWS_WEB_IDENTITY_TOKEN_FILE", System.getProperty("awsWebIdentityTokenExternalLocation"))
         // // The AWS STS SDK requires the role and session names to be set. We can verify that they are sent to S3S in the
@@ -40,7 +55,7 @@ public class RepositoryS3StsClientYamlTestSuiteIT extends AbstractRepositoryS3Cl
         .build();
 
     @ClassRule
-    public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(s3Sts).around(cluster);
+    public static TestRule ruleChain = RuleChain.outerRule(s3HttpFixture).around(stsHttpFixture).around(cluster);
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() throws Exception {

modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4TrashingAllocatorIT.java

@@ -89,7 +89,7 @@ public List<Route> routes() {
 
                 @Override
                 protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-                    var content = request.releasableContent();
+                    var content = request.content();
                     var iter = content.iterator();
                     return (chan) -> {
                         request.getHttpRequest().release();

muted-tests.yml

@@ -226,9 +226,6 @@ tests:
 - class: org.elasticsearch.xpack.inference.InferenceRestIT
   method: test {p0=inference/30_semantic_text_inference/Calculates embeddings using the default ELSER 2 endpoint}
   issue: https://github.com/elastic/elasticsearch/issues/117349
-- class: org.elasticsearch.xpack.security.operator.OperatorPrivilegesIT
-  method: testEveryActionIsEitherOperatorOnlyOrNonOperator
-  issue: https://github.com/elastic/elasticsearch/issues/102992
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_reset/Test reset running transform}
   issue: https://github.com/elastic/elasticsearch/issues/117473
@@ -240,6 +237,12 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/117524
 - class: org.elasticsearch.repositories.s3.RepositoryS3EcsClientYamlTestSuiteIT
   issue: https://github.com/elastic/elasticsearch/issues/117525
+- class: org.elasticsearch.xpack.esql.qa.mixed.FieldExtractorIT
+  method: testConstantKeywordField
+  issue: https://github.com/elastic/elasticsearch/issues/117531
+- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
+  method: test {p0=synonyms/90_synonyms_reloading_for_synset/Reload analyzers for specific synonym set}
+  issue: https://github.com/elastic/elasticsearch/issues/116777
 
 # Examples:
 #

qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java

@@ -10,6 +10,7 @@
 
 package org.elasticsearch.system.indices;
 
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.internal.node.NodeClient;
@@ -177,12 +178,12 @@ public List<Route> routes() {
 
         @Override
         protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+            var content = request.requiredContent();
             IndexRequest indexRequest = new IndexRequest(".net-new-system-index-primary");
-            indexRequest.source(request.requiredContent(), request.getXContentType());
+            indexRequest.source(content, request.getXContentType());
             indexRequest.id(request.param("id"));
             indexRequest.setRefreshPolicy(request.param("refresh"));
-
-            return channel -> client.index(indexRequest, new RestToXContentListener<>(channel));
+            return channel -> client.index(indexRequest, ActionListener.withRef(new RestToXContentListener<>(channel), content));
         }
 
         @Override

server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.monitor.jvm.HotThreads;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.MockLog;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.hamcrest.Matcher;
 
@@ -31,6 +32,7 @@
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.test.MockLog.assertThatLogger;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
 import static org.hamcrest.CoreMatchers.equalTo;
@@ -211,4 +213,25 @@ public void testLogLocalHotThreads() {
             )
         );
     }
+
+    @TestLogging(reason = "testing logging at various levels", value = "org.elasticsearch.action.admin.HotThreadsIT:TRACE")
+    public void testLogLocalCurrentThreadsInPlainText() {
+        final var level = randomFrom(Level.TRACE, Level.DEBUG, Level.INFO, Level.WARN, Level.ERROR);
+        assertThatLogger(
+            () -> HotThreads.logLocalCurrentThreads(logger, level, getTestName()),
+            HotThreadsIT.class,
+            new MockLog.SeenEventExpectation(
+                "Should log hot threads header in plain text",
+                HotThreadsIT.class.getCanonicalName(),
+                level,
+                "testLogLocalCurrentThreadsInPlainText: Hot threads at"
+            ),
+            new MockLog.SeenEventExpectation(
+                "Should log hot threads cpu usage in plain text",
+                HotThreadsIT.class.getCanonicalName(),
+                level,
+                "cpu usage by thread"
+            )
+        );
+    }
 }