Merge remote-tracking branch 'ossupstream/main' into 0.18.0_script

karenyrx · karenyrx · commit 0b995e8ebf9b · 2025-09-29T18:30:48.000Z
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -40,6 +40,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Replace centos:8 with almalinux:8 since centos docker images are deprecated ([#19154](https://github.com/opensearch-project/OpenSearch/pull/19154))
 - Add CompletionStage variants to IndicesAdminClient as an alternative to ActionListener ([#19161](https://github.com/opensearch-project/OpenSearch/pull/19161))
 - Remove cap on Java version used by forbidden APIs ([#19163](https://github.com/opensearch-project/OpenSearch/pull/19163))
+- Omit maxScoreCollector for field collapsing when sort by score descending ([#19181](https://github.com/opensearch-project/OpenSearch/pull/19181))
 - Disable pruning for `doc_values` for the wildcard field mapper ([#18568](https://github.com/opensearch-project/OpenSearch/pull/18568))
 - Make all methods in Engine.Result public ([#19276](https://github.com/opensearch-project/OpenSearch/pull/19275))
 - Create and attach interclusterTest and yamlRestTest code coverage reports to gradle check task([#19165](https://github.com/opensearch-project/OpenSearch/pull/19165))
diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
@@ -76,9 +76,9 @@ Fork [opensearch-project/OpenSearch](https://github.com/opensearch-project/OpenS
 
 #### JDK
 
-OpenSearch recommends building with the [Temurin/Adoptium](https://adoptium.net/temurin/releases/) distribution. JDK 11 is the minimum supported, and JDK-24 is the newest supported. You must have a supported JDK installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-21`. 
+OpenSearch recommends building with the [Temurin/Adoptium](https://adoptium.net/temurin/releases/) distribution. JDK 11 is the minimum supported, and JDK-24 is the newest supported. You must have a supported JDK installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-21`.
 
-Download Java 11 from [here](https://adoptium.net/releases.html?variant=openjdk11). 
+Download Java 11 from [here](https://adoptium.net/releases.html?variant=openjdk11).
 
 
 In addition, certain backward compatibility tests check out and compile the previous major version of OpenSearch, and therefore require installing [JDK 11](https://adoptium.net/temurin/releases/?version=11) and [JDK 17](https://adoptium.net/temurin/releases/?version=17) and setting the `JAVA11_HOME` and `JAVA17_HOME` environment variables. More to that, since 8.10 release, Gradle has deprecated the usage of the any JDKs below JDK-16. For smooth development experience, the recommendation is to install at least [JDK 17](https://adoptium.net/temurin/releases/?version=17) or [JDK 21](https://adoptium.net/temurin/releases/?version=21). If you still want to build with JDK-11 only, please add `-Dorg.gradle.warning.mode=none` when invoking any Gradle build task from command line, for example:
@@ -178,6 +178,23 @@ Run OpenSearch using `gradlew run`.
 ./gradlew run -PinstalledPlugins="['plugin1', 'plugin2']"
 ```
 
+External plugins may also be fetched and installed from maven snapshots:
+
+```bash
+./gradlew run -PinstalledPlugins="['opensearch-job-scheduler', 'opensearch-sql-plugin']"
+```
+
+You can specify a plugin version to pull to test a specific version in the org.opensearch.plugin groupId:
+```bash
+./gradlew run -PinstalledPlugins="['opensearch-job-scheduler:3.3.x.x']"
+```
+
+or install with fully qualified maven coordinates:
+```bash
+./gradlew run -PinstalledPlugins="['com.example:my-cool-plugin:3.3.x.x']"
+```
+
+
 That will build OpenSearch and start it, writing its log above Gradle's status message. We log a lot of stuff on startup, specifically these lines tell you that OpenSearch is ready.
 
 ```
diff --git a/gradle/run.gradle b/gradle/run.gradle
@@ -28,6 +28,7 @@
  * under the License.
  */
 import org.opensearch.gradle.testclusters.RunTask
+import org.opensearch.gradle.VersionProperties
 
 apply plugin: 'opensearch.testclusters'
 
@@ -41,8 +42,45 @@ testClusters {
     if (numNodes > 1) numberOfNodes = numNodes
     if (findProperty("installedPlugins")) {
       installedPlugins = Eval.me(installedPlugins)
+
+      def resolveMavenPlugin = { coords ->
+        // Add default groupId if not fully qualified (less than 2 colons)
+        String[] parts = coords.split(':')
+        if (parts.length == 2 && parts[0].contains('.')) {
+          throw new IllegalArgumentException("version is required if groupdId is specified '${coords}' Use format: groupId:artifactId:version")
+        }
+        String fullCoords = parts.length < 3 ? 'org.opensearch.plugin:' + coords : coords
+        def config = project.configurations.detachedConfiguration(
+          project.dependencies.create(fullCoords)
+        )
+        config.resolutionStrategy.cacheChangingModulesFor 0, 'seconds'
+        plugin(project.layout.file(project.provider { config.singleFile }))
+      }
+
       for (String p : installedPlugins) {
-        plugin('plugins:'.concat(p))
+        // check if its a local plugin first
+        if (project.findProject(':plugins:' + p) != null) {
+          plugin('plugins:' + p)
+        } else {
+          // attempt to fetch it from maven
+          project.repositories.mavenLocal()
+          project.repositories {
+            maven {
+              name = 'OpenSearch Snapshots'
+              url = 'https://central.sonatype.com/repository/maven-snapshots/'
+            }
+          }
+          if (p.contains(':')) {
+            // Maven coordinates with version specified
+            String coords = p.contains('@') ? p : (p + '@zip')
+            resolveMavenPlugin(coords)
+          } else {
+            // Not found locally, try Maven with current OS version + 0
+            String version = VersionProperties.getOpenSearch().replace('-SNAPSHOT', '.0-SNAPSHOT')
+            String coords = p + ':' + version + '@zip'
+            resolveMavenPlugin(coords)
+          }
+        }
         if (p.equals("arrow-flight-rpc")) {
           // Add system properties for Netty configuration
           systemProperty 'io.netty.allocator.numDirectArenas', '1'
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/400_max_score.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/400_max_score.yml
@@ -91,3 +91,71 @@ teardown:
   - match: { hits.total: 2 }
   - length: { hits.hits: 2 }
   - match: { hits.max_score: null }
+
+
+# related issue: https://github.com/opensearch-project/OpenSearch/issues/19179
+---
+"Test max score with sorting on score primarily for field collapsing":
+  - skip:
+      version: " - 3.2.99"
+      reason: Fixed in 3.3.0
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test_1
+        body:
+          query: { term: { foo: bar} }
+          collapse: { field: foo }
+          sort: [{ _score: desc }, { _doc: desc }]
+  - match: { hits.total: 2 }
+  - length: { hits.hits: 1 }
+  - match: { hits.max_score: 1.0 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test_1
+        body:
+          query: { term: { foo: bar} }
+          collapse: { field: foo }
+          sort: [{ _score: asc }, { _doc: desc }]
+  - match: { hits.total: 2 }
+  - length: { hits.hits: 1 }
+  - match: { hits.max_score: null }
+
+---
+"Test max score with sorting on score firstly primarily for field collapsing with concurrent segment search enabled":
+  - skip:
+      version: " - 3.2.99"
+      reason: Fixed in 3.3.0
+
+  - do:
+      indices.put_settings:
+        index: test_1
+        body:
+          index.search.concurrent_segment_search.mode: 'all'
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test_1
+        body:
+          query: { term: { foo: bar} }
+          collapse: { field: foo }
+          sort: [{ _score: desc }, { _doc: desc }]
+  - match: { hits.total: 2 }
+  - length: { hits.hits: 1 }
+  - match: { hits.max_score: 1.0 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test_1
+        body:
+          query: { term: { foo: bar} }
+          collapse: { field: foo }
+          sort: [{ _score: asc }, { _doc: desc }]
+  - match: { hits.total: 2 }
+  - length: { hits.hits: 1 }
+  - match: { hits.max_score: null }
diff --git a/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java b/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java
@@ -86,6 +86,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Objects;
+import java.util.function.Function;
 import java.util.function.Supplier;
 
 import static org.opensearch.search.profile.query.CollectorResult.REASON_SEARCH_COUNT;
@@ -228,10 +229,11 @@ static class CollapsingTopDocsCollectorContext extends TopDocsCollectorContext {
         private final DocValueFormat[] sortFmt;
         private final CollapsingTopDocsCollector<?> topDocsCollector;
         private final Collector collector;
-        private final Supplier<Float> maxScoreSupplier;
+        private final Function<CollapseTopFieldDocs, Float> maxScoreSupplier;
         private final CollapseContext collapseContext;
         private final boolean trackMaxScore;
         private final Sort sort;
+        private final boolean sortByScore;
         private final FieldDoc searchAfter;
 
         /**
@@ -274,14 +276,24 @@ private CollapsingTopDocsCollectorContext(
             this.searchAfter = searchAfter;
             this.topDocsCollector = collapseContext.createTopDocs(sort, numHits, searchAfter);
             this.trackMaxScore = trackMaxScore;
+            this.sortByScore = sortAndFormats == null || SortField.FIELD_SCORE.equals(sortAndFormats.sort.getSort()[0]);
 
-            MaxScoreCollector maxScoreCollector;
-            if (trackMaxScore) {
+            final MaxScoreCollector maxScoreCollector;
+            if (sortByScore) {
+                maxScoreCollector = null;
+                maxScoreSupplier = (topDocs) -> {
+                    if (topDocs.scoreDocs.length == 0) {
+                        return Float.NaN;
+                    } else {
+                        return topDocs.scoreDocs[0].score;
+                    }
+                };
+            } else if (trackMaxScore) {
                 maxScoreCollector = new MaxScoreCollector();
-                maxScoreSupplier = maxScoreCollector::getMaxScore;
+                maxScoreSupplier = (topDocs) -> maxScoreCollector.getMaxScore();
             } else {
                 maxScoreCollector = null;
-                maxScoreSupplier = () -> Float.NaN;
+                maxScoreSupplier = (topDocs) -> Float.NaN;
             }
 
             this.collector = MultiCollector.wrap(topDocsCollector, maxScoreCollector);
@@ -296,7 +308,7 @@ Collector create(Collector in) throws IOException {
         @Override
         void postProcess(QuerySearchResult result) throws IOException {
             final CollapseTopFieldDocs topDocs = topDocsCollector.getTopDocs();
-            result.topDocs(new TopDocsAndMaxScore(topDocs, maxScoreSupplier.get()), sortFmt);
+            result.topDocs(new TopDocsAndMaxScore(topDocs, maxScoreSupplier.apply(topDocs)), sortFmt);
         }
 
         @Override
@@ -305,8 +317,8 @@ CollectorManager<?, ReduceableSearchResult> createManager(CollectorManager<?, Re
                 @Override
                 public Collector newCollector() throws IOException {
                     MaxScoreCollector maxScoreCollector = null;
-
-                    if (trackMaxScore) {
+                    // if sort by score in descending order, MaxScoreCollector is not needed
+                    if (!sortByScore && trackMaxScore) {
                         maxScoreCollector = new MaxScoreCollector();
                     }
 
@@ -353,7 +365,15 @@ protected ReduceableSearchResult reduceWith(final Collection<CollapseTopFieldDoc
                     numHits,
                     topFieldDocs.toArray(new CollapseTopFieldDocs[0])
                 );
-                result.topDocs(new TopDocsAndMaxScore(topDocs, maxScore), sortFmt);
+                TopDocsAndMaxScore topDocsAndMaxScore;
+                // if sort by score in descending order, we can get the max score from the first matched document directly
+                // if no matched document, max score is Float.NaN
+                if (sortByScore && topDocs.scoreDocs.length > 0) {
+                    topDocsAndMaxScore = new TopDocsAndMaxScore(topDocs, topDocs.scoreDocs[0].score);
+                } else {
+                    topDocsAndMaxScore = new TopDocsAndMaxScore(topDocs, maxScore);
+                }
+                result.topDocs(topDocsAndMaxScore, sortFmt);
             };
         }
     }
@@ -835,13 +855,12 @@ public static TopDocsCollectorContext createTopDocsCollectorContext(SearchContex
                 hasFilterCollector
             );
         } else if (searchContext.collapse() != null) {
-            boolean trackScores = searchContext.sort() == null ? true : searchContext.trackScores();
             int numDocs = Math.min(searchContext.from() + searchContext.size(), totalNumDocs);
             return new CollapsingTopDocsCollectorContext(
                 searchContext.collapse(),
                 searchContext.sort(),
                 numDocs,
-                trackScores,
+                searchContext.trackScores(),
                 searchContext.searchAfter()
             );
         } else {
diff --git a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java
@@ -1088,6 +1088,67 @@ public void testMaxScore() throws Exception {
         dir.close();
     }
 
+    public void testMaxScoreWithSortOnScoreAndCollapsingResults() throws Exception {
+        Directory dir = newDirectory();
+        IndexWriterConfig iwc = newIndexWriterConfig();
+        RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+
+        // Always end up with uneven buckets so collapsing is predictable
+        final int numDocs = 2 * scaledRandomIntBetween(600, 900) - 1;
+        for (int i = 0; i < numDocs; i++) {
+            Document doc = new Document();
+            doc.add(new StringField("foo", "bar", Store.NO));
+            doc.add(new NumericDocValuesField("user", i & 1));
+            w.addDocument(doc);
+        }
+        w.close();
+
+        IndexReader reader = DirectoryReader.open(dir);
+        QueryShardContext queryShardContext = mock(QueryShardContext.class);
+        when(queryShardContext.fieldMapper("user")).thenReturn(
+            new NumberFieldType("user", NumberType.INTEGER, true, false, true, false, false, null, Collections.emptyMap())
+        );
+
+        TestSearchContext context = new TestSearchContext(queryShardContext, indexShard, newContextSearcher(reader, executor));
+        context.collapse(new CollapseBuilder("user").build(context.getQueryShardContext()));
+        context.trackScores(false);
+        context.parsedQuery(new ParsedQuery(new TermQuery(new Term("foo", "bar"))));
+        context.setTask(new SearchShardTask(123L, "", "", "", null, Collections.emptyMap()));
+        context.setSize(2);
+        context.trackTotalHitsUpTo(5);
+
+        QueryPhase.executeInternal(context.withCleanQueryResult(), queryPhaseSearcher);
+        assertFalse(Float.isNaN(context.queryResult().getMaxScore()));
+        assertEquals(2, context.queryResult().topDocs().topDocs.scoreDocs.length);
+        assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs));
+        assertThat(context.queryResult().topDocs().topDocs, instanceOf(CollapseTopFieldDocs.class));
+
+        CollapseTopFieldDocs topDocs = (CollapseTopFieldDocs) context.queryResult().topDocs().topDocs;
+        assertThat(topDocs.collapseValues.length, equalTo(2));
+        assertThat(topDocs.collapseValues[0], equalTo(0L)); // user == 0
+        assertThat(topDocs.collapseValues[1], equalTo(1L)); // user == 1
+
+        assertThat(context.queryResult().topDocs().topDocs.scoreDocs[0].score, equalTo(context.queryResult().getMaxScore()));
+
+        Sort sort = new Sort(new SortField(null, SortField.Type.SCORE), new SortField(null, SortField.Type.DOC));
+        SortAndFormats sortAndFormats = new SortAndFormats(sort, new DocValueFormat[] { DocValueFormat.RAW, DocValueFormat.RAW });
+        context.sort(sortAndFormats);
+        QueryPhase.executeInternal(context.withCleanQueryResult(), queryPhaseSearcher);
+        assertFalse(Float.isNaN(context.queryResult().getMaxScore()));
+        assertEquals(2, context.queryResult().topDocs().topDocs.scoreDocs.length);
+        assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs));
+        assertThat(context.queryResult().topDocs().topDocs, instanceOf(CollapseTopFieldDocs.class));
+
+        topDocs = (CollapseTopFieldDocs) context.queryResult().topDocs().topDocs;
+        assertThat(topDocs.collapseValues.length, equalTo(2));
+        assertThat(topDocs.collapseValues[0], equalTo(0L)); // user == 0
+        assertThat(topDocs.collapseValues[1], equalTo(1L)); // user == 1
+        assertThat(context.queryResult().topDocs().topDocs.scoreDocs[0].score, equalTo(context.queryResult().getMaxScore()));
+
+        reader.close();
+        dir.close();
+    }
+
     public void testCollapseQuerySearchResults() throws Exception {
         Directory dir = newDirectory();
         final Sort sort = new Sort(new SortField("user", SortField.Type.INT));
