Merge branch 'main' into do_not_throw_on_null_flat_object

Signed-off-by: Michael Froh <froh@amazon.com>

CHANGELOG.md

@@ -46,6 +46,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.azure:azure-core-http-netty` from 1.15.1 to 1.15.3 ([#15300](https://github.com/opensearch-project/OpenSearch/pull/15300))
 - Bump `com.gradle.develocity` from 3.17.6 to 3.18 ([#15297](https://github.com/opensearch-project/OpenSearch/pull/15297))
 - Bump `commons-cli:commons-cli` from 1.8.0 to 1.9.0 ([#15298](https://github.com/opensearch-project/OpenSearch/pull/15298))
+- Bump `opentelemetry` from 1.40.0 to 1.41.0 ([#15361](https://github.com/opensearch-project/OpenSearch/pull/15361))
+- Bump `opentelemetry-semconv` from 1.26.0-alpha to 1.27.0-alpha ([#15361](https://github.com/opensearch-project/OpenSearch/pull/15361))
 
 ### Changed
 - Add lower limit for primary and replica batch allocators timeout ([#14979](https://github.com/opensearch-project/OpenSearch/pull/14979))
@@ -66,6 +68,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix range aggregation optimization ignoring top level queries ([#15194](https://github.com/opensearch-project/OpenSearch/pull/15194))
 - Fix incorrect parameter names in MinHash token filter configuration handling ([#15233](https://github.com/opensearch-project/OpenSearch/pull/15233))
 - Fix indexing error when flat_object field is explicitly null ([#15375](https://github.com/opensearch-project/OpenSearch/pull/15375))
+- Fix split response processor not included in allowlist ([#15393](https://github.com/opensearch-project/OpenSearch/pull/15393))
+- Fix unchecked cast in dynamic action map getter ([#15394](https://github.com/opensearch-project/OpenSearch/pull/15394))
 
 ### Security
 

buildSrc/version.properties

@@ -74,5 +74,5 @@ jzlib             = 1.1.3
 resteasy          = 6.2.4.Final
 
 # opentelemetry dependencies
-opentelemetry         = 1.40.0
-opentelemetrysemconv  = 1.26.0-alpha
+opentelemetry         = 1.41.0
+opentelemetrysemconv  = 1.27.0-alpha

modules/search-pipeline-common/src/main/java/org/opensearch/search/pipeline/common/SearchPipelineCommonModulePlugin.java

@@ -97,6 +97,8 @@ public Map<String, Processor.Factory<SearchResponseProcessor>> getResponseProces
                 new TruncateHitsResponseProcessor.Factory(),
                 CollapseResponseProcessor.TYPE,
                 new CollapseResponseProcessor.Factory(),
+                SplitResponseProcessor.TYPE,
+                new SplitResponseProcessor.Factory(),
                 SortResponseProcessor.TYPE,
                 new SortResponseProcessor.Factory()
             )

modules/search-pipeline-common/src/test/java/org/opensearch/search/pipeline/common/SearchPipelineCommonModulePluginTests.java

@@ -47,6 +47,7 @@ public void testResponseProcessorAllowlist() throws IOException {
             List.of("rename_field", "truncate_hits", "collapse"),
             SearchPipelineCommonModulePlugin::getResponseProcessors
         );
+        runAllowlistTest(key, List.of("split", "sort"), SearchPipelineCommonModulePlugin::getResponseProcessors);
 
         final IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
@@ -82,7 +83,7 @@ public void testAllowlistNotSpecified() throws IOException {
         try (SearchPipelineCommonModulePlugin plugin = new SearchPipelineCommonModulePlugin()) {
             assertEquals(Set.of("oversample", "filter_query", "script"), plugin.getRequestProcessors(createParameters(settings)).keySet());
             assertEquals(
-                Set.of("rename_field", "truncate_hits", "collapse", "sort"),
+                Set.of("rename_field", "truncate_hits", "collapse", "split", "sort"),
                 plugin.getResponseProcessors(createParameters(settings)).keySet()
             );
             assertEquals(Set.of(), plugin.getSearchPhaseResultsProcessors(createParameters(settings)).keySet());

plugins/telemetry-otel/build.gradle

@@ -86,7 +86,9 @@ thirdPartyAudit {
     'io.opentelemetry.sdk.autoconfigure.spi.traces.ConfigurableSpanExporterProvider',
     'kotlin.io.path.PathsKt',
     'io.opentelemetry.sdk.autoconfigure.spi.traces.ConfigurableSpanExporterProvider',
-    'io.opentelemetry.sdk.autoconfigure.spi.internal.AutoConfigureListener'
+    'io.opentelemetry.sdk.autoconfigure.spi.internal.AutoConfigureListener',
+    'io.opentelemetry.sdk.autoconfigure.spi.internal.ComponentProvider',
+    'io.opentelemetry.sdk.autoconfigure.spi.internal.StructuredConfigProperties'
   )
 }
 

plugins/telemetry-otel/licenses/opentelemetry-api-1.41.0.jar.sha1

@@ -0,0 +1 @@
+ec5ad3b420c9fba4b340e85a3199fd0f2accd023

plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.41.0-alpha.jar.sha1

@@ -0,0 +1 @@
+fd387313cc37a6e93062e9a80a2526634d22cb19

plugins/telemetry-otel/licenses/opentelemetry-context-1.41.0.jar.sha1

@@ -0,0 +1 @@
+3d7cf15ef425053e24e825160ca7b4ac08d721aa

plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.41.0.jar.sha1

@@ -0,0 +1 @@
+cf92f4c1b60c2359c12f6f323f6a2a623c333910

plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.41.0.jar.sha1

@@ -0,0 +1 @@
+8dee21440b811004ecc1c36c1cd44f9d3494546c

plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-1.41.0.jar.sha1

@@ -0,0 +1 @@
+d86e60b6d49e389ebe5797d42a7288a20d30c162

plugins/telemetry-otel/licenses/opentelemetry-exporter-otlp-common-1.41.0.jar.sha1

@@ -0,0 +1 @@
+aeba3075b8dfd97779edadc0a3711d999bb0e396

plugins/telemetry-otel/licenses/opentelemetry-exporter-sender-okhttp-1.41.0.jar.sha1

@@ -0,0 +1 @@
+368d7905d6a0a313c63e3a91f895a3a08500519e

plugins/telemetry-otel/licenses/opentelemetry-sdk-1.41.0.jar.sha1

@@ -0,0 +1 @@
+c740e8f7d0d914d6acd310ac53901bb8753c6e8d

plugins/telemetry-otel/licenses/opentelemetry-sdk-common-1.41.0.jar.sha1

@@ -0,0 +1 @@
+b820861f85ba83db0ad896c47f723208d7473d5a

plugins/telemetry-otel/licenses/opentelemetry-sdk-logs-1.41.0.jar.sha1

@@ -0,0 +1 @@
+f88ee292f5605c87dfe85c8d90131bce9f0b3b8e

plugins/telemetry-otel/licenses/opentelemetry-sdk-metrics-1.41.0.jar.sha1

@@ -0,0 +1 @@
+9d1200befb28e3e9f61073ac3de23cc55e509dc7

plugins/telemetry-otel/licenses/opentelemetry-sdk-trace-1.41.0.jar.sha1

@@ -0,0 +1 @@
+d9bbc2e2e800317d72fbf3141ae8391e95fa6229

plugins/telemetry-otel/licenses/opentelemetry-semconv-1.27.0-alpha.jar.sha1

@@ -0,0 +1 @@
+906d916bee46f60260c09314284b5948c54a0662

server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java

@@ -50,15 +50,22 @@
 public class RemoteStatePublicationIT extends RemoteStoreBaseIntegTestCase {
 
     private static String INDEX_NAME = "test-index";
+    private boolean isRemoteStateEnabled = true;
+    private String isRemotePublicationEnabled = "true";
 
     @Before
     public void setup() {
         asyncUploadMockFsRepo = false;
+        isRemoteStateEnabled = true;
+        isRemotePublicationEnabled = "true";
     }
 
     @Override
     protected Settings featureFlagSettings() {
-        return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL, "true").build();
+        return Settings.builder()
+            .put(super.featureFlagSettings())
+            .put(FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL, isRemotePublicationEnabled)
+            .build();
     }
 
     @Override
@@ -76,7 +83,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
         );
         return Settings.builder()
             .put(super.nodeSettings(nodeOrdinal))
-            .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true)
+            .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), isRemoteStateEnabled)
             .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, routingTableRepoName)
             .put(routingTableRepoTypeAttributeKey, ReloadableFsRepository.TYPE)
             .put(routingTableRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath)
@@ -136,6 +143,18 @@ public void testPublication() throws Exception {
         }
     }
 
+    public void testRemotePublicationDisableIfRemoteStateDisabled() {
+        // only disable remote state
+        isRemoteStateEnabled = false;
+        // create cluster with multi node with in-consistent settings
+        prepareCluster(3, 2, INDEX_NAME, 1, 2);
+        // assert cluster is stable, ensuring publication falls back to legacy transport with inconsistent settings
+        ensureStableCluster(5);
+        ensureGreen(INDEX_NAME);
+
+        assertNull(internalCluster().getCurrentClusterManagerNodeInstance(RemoteClusterStateService.class));
+    }
+
     private Map<String, Integer> getMetadataFiles(BlobStoreRepository repository, String subDirectory) throws IOException {
         BlobPath metadataPath = repository.basePath()
             .add(
@@ -151,5 +170,4 @@ private Map<String, Integer> getMetadataFiles(BlobStoreRepository repository, St
             return fileName.split(DELIMITER)[0];
         }).collect(Collectors.toMap(Function.identity(), key -> 1, Integer::sum));
     }
-
 }

server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java

@@ -265,13 +265,9 @@ public void testValidCompositeIndex() {
                     assertEquals(expectedTimeUnits, dateDim.getIntervals());
                     assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField());
                     assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField());
-                    List<MetricStat> expectedMetrics = Arrays.asList(
-                        MetricStat.AVG,
-                        MetricStat.VALUE_COUNT,
-                        MetricStat.SUM,
-                        MetricStat.MAX,
-                        MetricStat.MIN
-                    );
+
+                    // Assert default metrics
+                    List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.VALUE_COUNT, MetricStat.SUM, MetricStat.AVG);
                     assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
                     assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
                     assertEquals(
@@ -349,13 +345,9 @@ public void testUpdateIndexWhenMappingIsSame() {
                     assertEquals(expectedTimeUnits, dateDim.getIntervals());
                     assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField());
                     assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField());
-                    List<MetricStat> expectedMetrics = Arrays.asList(
-                        MetricStat.AVG,
-                        MetricStat.VALUE_COUNT,
-                        MetricStat.SUM,
-                        MetricStat.MAX,
-                        MetricStat.MIN
-                    );
+
+                    // Assert default metrics
+                    List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.VALUE_COUNT, MetricStat.SUM, MetricStat.AVG);
                     assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
                     assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
                     assertEquals(

server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java

@@ -34,6 +34,12 @@
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.Weight;
 import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.opensearch.action.admin.cluster.node.stats.NodeStats;
 import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
@@ -56,7 +62,10 @@
 import org.opensearch.env.NodeEnvironment;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.cache.request.RequestCacheStats;
+import org.opensearch.index.query.QueryBuilder;
 import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.index.query.QueryShardContext;
+import org.opensearch.index.query.TermQueryBuilder;
 import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
 import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.opensearch.search.aggregations.bucket.histogram.Histogram;
@@ -65,6 +74,7 @@
 import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase;
 import org.opensearch.test.hamcrest.OpenSearchAssertions;
 
+import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.ZoneId;
@@ -768,6 +778,59 @@ public void testDeleteAndCreateSameIndexShardOnSameNode() throws Exception {
         assertTrue(stats.getMemorySizeInBytes() == 0);
     }
 
+    public void testTimedOutQuery() throws Exception {
+        // A timed out query should be cached and then invalidated
+        Client client = client();
+        String index = "index";
+        assertAcked(
+            client.admin()
+                .indices()
+                .prepareCreate(index)
+                .setMapping("k", "type=keyword")
+                .setSettings(
+                    Settings.builder()
+                        .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        // Disable index refreshing to avoid cache being invalidated mid-test
+                        .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(-1))
+                )
+                .get()
+        );
+        indexRandom(true, client.prepareIndex(index).setSource("k", "hello"));
+        ensureSearchable(index);
+        // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
+        forceMerge(client, index);
+
+        QueryBuilder timeoutQueryBuilder = new TermQueryBuilder("k", "hello") {
+            @Override
+            protected Query doToQuery(QueryShardContext context) {
+                return new TermQuery(new Term("k", "hello")) {
+                    @Override
+                    public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
+                        // Create the weight before sleeping. Otherwise, TermStates.build() (in the call to super.createWeight()) will
+                        // sometimes throw an exception on timeout, rather than timing out gracefully.
+                        Weight result = super.createWeight(searcher, scoreMode, boost);
+                        try {
+                            Thread.sleep(500);
+                        } catch (InterruptedException ignored) {}
+                        return result;
+                    }
+                };
+            }
+        };
+
+        SearchResponse resp = client.prepareSearch(index)
+            .setRequestCache(true)
+            .setQuery(timeoutQueryBuilder)
+            .setTimeout(TimeValue.ZERO)
+            .get();
+        assertTrue(resp.isTimedOut());
+        RequestCacheStats requestCacheStats = getRequestCacheStats(client, index);
+        // The cache should be empty as the timed-out query was invalidated
+        assertEquals(0, requestCacheStats.getMemorySizeInBytes());
+    }
+
     private Path[] shardDirectory(String server, Index index, int shard) {
         NodeEnvironment env = internalCluster().getInstance(NodeEnvironment.class, server);
         final Path[] paths = env.availableShardPaths(new ShardId(index, shard));

server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStorePinnedTimestampsIT.java

@@ -9,8 +9,10 @@
 package org.opensearch.remotestore;
 
 import org.opensearch.common.collect.Tuple;
+import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.action.ActionListener;
+import org.opensearch.indices.RemoteStoreSettings;
 import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
@@ -20,6 +22,14 @@
 public class RemoteStorePinnedTimestampsIT extends RemoteStoreBaseIntegTestCase {
     static final String INDEX_NAME = "remote-store-test-idx-1";
 
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal))
+            .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true)
+            .build();
+    }
+
     ActionListener<Void> noOpActionListener = new ActionListener<>() {
         @Override
         public void onResponse(Void unused) {}