Update Apache Lucene to 9.7.0-snapshot-4d1ed9e (opensearch-project#7605)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
Arpit-Bandejiya · May 19, 2023 · 3732ddb · 3732ddb
1 parent 30a1a43
commit 3732ddb
Show file tree

Hide file tree

Showing 51 changed files with 53 additions and 44 deletions.
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.6.0
+lucene            = 9.7.0-snapshot-4d1ed9e
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 19.0.2+7

diff --git a/libs/core/licenses/lucene-core-9.6.0.jar.sha1 b/libs/core/licenses/lucene-core-9.6.0.jar.sha1
diff --git a/libs/core/licenses/lucene-core-9.7.0-snapshot-4d1ed9e.jar.sha1 b/libs/core/licenses/lucene-core-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+8e87d93868271da754a7c22517b3dbbcf84fc7f9
diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java
@@ -88,7 +88,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_7_0 = new Version(2070099, org.apache.lucene.util.Version.LUCENE_9_5_0);
     public static final Version V_2_7_1 = new Version(2070199, org.apache.lucene.util.Version.LUCENE_9_5_0);
     public static final Version V_2_8_0 = new Version(2080099, org.apache.lucene.util.Version.LUCENE_9_6_0);
-    public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_6_0);
+    public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_7_0);
     public static final Version CURRENT = V_3_0_0;
 
     public static Version fromId(int id) {

diff --git a/modules/lang-expression/licenses/lucene-expressions-9.6.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.6.0.jar.sha1
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.7.0-snapshot-4d1ed9e.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+441c6341e75f06b834c992084f21556681963ea6
diff --git a/modules/parent-join/src/test/java/org/opensearch/join/query/HasChildQueryBuilderTests.java b/modules/parent-join/src/test/java/org/opensearch/join/query/HasChildQueryBuilderTests.java
@@ -33,6 +33,7 @@
 package org.opensearch.join.query;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
@@ -303,10 +304,9 @@ static void assertLateParsingQuery(Query query, String type, String id) throws I
         assertThat(booleanQuery.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.MUST));
         assertThat(booleanQuery.clauses().get(0).getQuery(), instanceOf(TermInSetQuery.class));
         TermInSetQuery termsQuery = (TermInSetQuery) booleanQuery.clauses().get(0).getQuery();
-        Query rewrittenTermsQuery = termsQuery.rewrite(null);
         // The query is of type MultiTermQueryConstantScoreBlendedWrapper and is sealed inside Apache Lucene,
         // no access to inner queries without using the reflection, falling back to stringified query comparison
-        assertThat(rewrittenTermsQuery.toString(), equalTo("_id:([ff 69 64])"));
+        assertThat(termsQuery.toString(), equalTo("_id:([ff 69 64])"));
         // check the type filter
         assertThat(booleanQuery.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.FILTER));
         assertEquals(new TermQuery(new Term("join_field", type)), booleanQuery.clauses().get(1).getQuery());

diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.6.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.6.0.jar.sha1
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0-snapshot-4d1ed9e.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+5da04f8baf0d0961eee28fc1ca0dde93f58bd3e5
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.6.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.6.0.jar.sha1
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0-snapshot-4d1ed9e.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+c5307fbb67bd0901e07d2987d4c3e33dc7d9dd62
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.6.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.6.0.jar.sha1
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0-snapshot-4d1ed9e.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+668dd1d3506840f6a258efda8a4f5e0ae0f069eb
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.6.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.6.0.jar.sha1
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0-snapshot-4d1ed9e.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+df49e950d6463df63546b0989b46a717424b48c0
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.6.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.6.0.jar.sha1
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0-snapshot-4d1ed9e.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+06abcf12423dc8f115f2bb4c269b6ba2202245c2
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.6.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.6.0.jar.sha1
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0-snapshot-4d1ed9e.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+0c2f7b681c54ff2f174a894c0bcc26030d1644c2
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.6.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.6.0.jar.sha1
diff --git a/...ns/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0-snapshot-4d1ed9e.jar.sha1 b/...ns/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+ac785becbaf914830deeb950626b850cd89efadf
diff --git a/server/licenses/lucene-analysis-common-9.6.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-analysis-common-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-analysis-common-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+d478df2acb5b224038a2380c15eec26a77bbce8b
diff --git a/server/licenses/lucene-backward-codecs-9.6.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-backward-codecs-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-backward-codecs-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+28c618f5e814cf05af94032f340d7cb79ea7dec2
diff --git a/server/licenses/lucene-core-9.6.0.jar.sha1 b/server/licenses/lucene-core-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-core-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-core-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+8e87d93868271da754a7c22517b3dbbcf84fc7f9
diff --git a/server/licenses/lucene-grouping-9.6.0.jar.sha1 b/server/licenses/lucene-grouping-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-grouping-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-grouping-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+b1b0674d8afa4ad78c1eccfb0b73064b5826631d
diff --git a/server/licenses/lucene-highlighter-9.6.0.jar.sha1 b/server/licenses/lucene-highlighter-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-highlighter-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-highlighter-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+d376f4bad9ba6b9a2cc3fb4201cccfd7465cb54c
diff --git a/server/licenses/lucene-join-9.6.0.jar.sha1 b/server/licenses/lucene-join-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-join-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-join-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+757e3bdda4a34fb4a5ded85d34df5ea00dc4e310
diff --git a/server/licenses/lucene-memory-9.6.0.jar.sha1 b/server/licenses/lucene-memory-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-memory-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-memory-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+daaf3d5424e6e474e860a7db92af4f490c627457
diff --git a/server/licenses/lucene-misc-9.6.0.jar.sha1 b/server/licenses/lucene-misc-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-misc-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-misc-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+c3c52d69784be7787e239631cc7b64f3e32ded4f
diff --git a/server/licenses/lucene-queries-9.6.0.jar.sha1 b/server/licenses/lucene-queries-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-queries-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-queries-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+8f10d4ff130928351a8c03f9c2d10188d0267d74
diff --git a/server/licenses/lucene-queryparser-9.6.0.jar.sha1 b/server/licenses/lucene-queryparser-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-queryparser-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-queryparser-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+22bc9f5ca57b5b3c35b10075716ca6b2ab1e2f4c
diff --git a/server/licenses/lucene-sandbox-9.6.0.jar.sha1 b/server/licenses/lucene-sandbox-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-sandbox-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-sandbox-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+e88b9b8adb0e206a20482f47340cb65794d23330
diff --git a/server/licenses/lucene-spatial-extras-9.6.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-spatial-extras-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-spatial-extras-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+85b32ddf62395219fe5bd7d5d9f70e81c9a41d34
diff --git a/server/licenses/lucene-spatial3d-9.6.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-spatial3d-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-spatial3d-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+387188af30ac63d8e970b2f3043003b01ed001ca
diff --git a/server/licenses/lucene-suggest-9.6.0.jar.sha1 b/server/licenses/lucene-suggest-9.6.0.jar.sha1
diff --git a/server/licenses/lucene-suggest-9.7.0-snapshot-4d1ed9e.jar.sha1 b/server/licenses/lucene-suggest-9.7.0-snapshot-4d1ed9e.jar.sha1
@@ -0,0 +1 @@
+c9aa971ca1beaa1fe407417587343c0a356b9ad2
diff --git a/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java b/server/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java
@@ -38,6 +38,7 @@
 import org.apache.lucene.queries.spans.SpanTermQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MultiPhraseQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
@@ -66,45 +67,45 @@ public CustomFieldQuery(Query query, IndexReader reader, boolean phraseHighlight
     }
 
     @Override
-    protected void flatten(Query sourceQuery, IndexReader reader, Collection<Query> flatQueries, float boost) throws IOException {
+    protected void flatten(Query sourceQuery, IndexSearcher searcher, Collection<Query> flatQueries, float boost) throws IOException {
         if (sourceQuery instanceof BoostQuery) {
             BoostQuery bq = (BoostQuery) sourceQuery;
             sourceQuery = bq.getQuery();
             boost *= bq.getBoost();
-            flatten(sourceQuery, reader, flatQueries, boost);
+            flatten(sourceQuery, searcher, flatQueries, boost);
         } else if (sourceQuery instanceof SpanTermQuery) {
-            super.flatten(new TermQuery(((SpanTermQuery) sourceQuery).getTerm()), reader, flatQueries, boost);
+            super.flatten(new TermQuery(((SpanTermQuery) sourceQuery).getTerm()), searcher, flatQueries, boost);
         } else if (sourceQuery instanceof ConstantScoreQuery) {
-            flatten(((ConstantScoreQuery) sourceQuery).getQuery(), reader, flatQueries, boost);
+            flatten(((ConstantScoreQuery) sourceQuery).getQuery(), searcher, flatQueries, boost);
         } else if (sourceQuery instanceof FunctionScoreQuery) {
-            flatten(((FunctionScoreQuery) sourceQuery).getSubQuery(), reader, flatQueries, boost);
+            flatten(((FunctionScoreQuery) sourceQuery).getSubQuery(), searcher, flatQueries, boost);
         } else if (sourceQuery instanceof MultiPhrasePrefixQuery) {
-            flatten(sourceQuery.rewrite(reader), reader, flatQueries, boost);
+            flatten(sourceQuery.rewrite(searcher), searcher, flatQueries, boost);
         } else if (sourceQuery instanceof MultiPhraseQuery) {
             MultiPhraseQuery q = ((MultiPhraseQuery) sourceQuery);
-            convertMultiPhraseQuery(0, new int[q.getTermArrays().length], q, q.getTermArrays(), q.getPositions(), reader, flatQueries);
+            convertMultiPhraseQuery(0, new int[q.getTermArrays().length], q, q.getTermArrays(), q.getPositions(), searcher, flatQueries);
         } else if (sourceQuery instanceof BlendedTermQuery) {
             final BlendedTermQuery blendedTermQuery = (BlendedTermQuery) sourceQuery;
-            flatten(blendedTermQuery.rewrite(reader), reader, flatQueries, boost);
+            flatten(blendedTermQuery.rewrite(searcher), searcher, flatQueries, boost);
         } else if (sourceQuery instanceof org.apache.lucene.queries.function.FunctionScoreQuery) {
             org.apache.lucene.queries.function.FunctionScoreQuery funcScoreQuery =
                 (org.apache.lucene.queries.function.FunctionScoreQuery) sourceQuery;
             // flatten query with query boost
-            flatten(funcScoreQuery.getWrappedQuery(), reader, flatQueries, boost);
+            flatten(funcScoreQuery.getWrappedQuery(), searcher, flatQueries, boost);
         } else if (sourceQuery instanceof SynonymQuery) {
             // SynonymQuery should be handled by the parent class directly.
             // This statement should be removed when https://issues.apache.org/jira/browse/LUCENE-7484 is merged.
             SynonymQuery synQuery = (SynonymQuery) sourceQuery;
             for (Term term : synQuery.getTerms()) {
-                flatten(new TermQuery(term), reader, flatQueries, boost);
+                flatten(new TermQuery(term), searcher, flatQueries, boost);
             }
         } else if (sourceQuery instanceof OpenSearchToParentBlockJoinQuery) {
             Query childQuery = ((OpenSearchToParentBlockJoinQuery) sourceQuery).getChildQuery();
             if (childQuery != null) {
-                flatten(childQuery, reader, flatQueries, boost);
+                flatten(childQuery, searcher, flatQueries, boost);
             }
         } else {
-            super.flatten(sourceQuery, reader, flatQueries, boost);
+            super.flatten(sourceQuery, searcher, flatQueries, boost);
         }
     }
 
@@ -114,7 +115,7 @@ private void convertMultiPhraseQuery(
         MultiPhraseQuery orig,
         Term[][] terms,
         int[] pos,
-        IndexReader reader,
+        IndexSearcher searcher,
         Collection<Query> flatQueries
     ) throws IOException {
         if (currentPos == 0) {
@@ -126,7 +127,7 @@ private void convertMultiPhraseQuery(
             if (numTerms > 16) {
                 for (Term[] currentPosTerm : terms) {
                     for (Term term : currentPosTerm) {
-                        super.flatten(new TermQuery(term), reader, flatQueries, 1F);
+                        super.flatten(new TermQuery(term), searcher, flatQueries, 1F);
                     }
                 }
                 return;
@@ -143,12 +144,12 @@ private void convertMultiPhraseQuery(
                 queryBuilder.add(terms[i][termsIdx[i]], pos[i]);
             }
             Query query = queryBuilder.build();
-            this.flatten(query, reader, flatQueries, 1F);
+            this.flatten(query, searcher, flatQueries, 1F);
         } else {
             Term[] t = terms[currentPos];
             for (int i = 0; i < t.length; i++) {
                 termsIdx[currentPos] = i;
-                convertMultiPhraseQuery(currentPos + 1, termsIdx, orig, terms, pos, reader, flatQueries);
+                convertMultiPhraseQuery(currentPos + 1, termsIdx, orig, terms, pos, searcher, flatQueries);
             }
         }
     }

diff --git a/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java b/server/src/test/java/org/opensearch/lucene/queries/SpanMatchNoDocsQueryTests.java
@@ -56,11 +56,19 @@
 
 public class SpanMatchNoDocsQueryTests extends OpenSearchTestCase {
     public void testSimple() throws Exception {
+        Directory dir = newDirectory();
+        IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig());
+        IndexReader ir = DirectoryReader.open(iw);
+
         SpanMatchNoDocsQuery query = new SpanMatchNoDocsQuery("field", "a good reason");
         assertEquals(query.toString(), "SpanMatchNoDocsQuery(\"a good reason\")");
-        Query rewrite = query.rewrite(null);
+        Query rewrite = query.rewrite(ir);
         assertTrue(rewrite instanceof SpanMatchNoDocsQuery);
         assertEquals(rewrite.toString(), "SpanMatchNoDocsQuery(\"a good reason\")");
+
+        iw.close();
+        ir.close();
+        dir.close();
     }
 
     public void testQuery() throws Exception {