Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Upgrade] to lucene-9.4.0-snapshot-ddf0d0a #4183

Merged
merged 3 commits into from
Aug 15, 2022
Merged

[Upgrade] to lucene-9.4.0-snapshot-ddf0d0a #4183

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
86c95e4
[Upgrade] to lucene-9.4.0-snapshot-ddf0d0a
nknize Aug 10, 2022
02ce7aa
add fieldname to BytesRefFieldComparatorSource
nknize Aug 12, 2022
38c237c
set to not skip docs
nknize Aug 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion buildSrc/version.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
opensearch = 3.0.0
lucene = 9.3.0
lucene = 9.4.0-snapshot-ddf0d0a

bundled_jdk_vendor = adoptium
bundled_jdk = 17.0.4+8
Expand Down
1 change: 0 additions & 1 deletion modules/lang-expression/licenses/lucene-expressions-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions modules/lang-expression/licenses/lucene-expressions-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
9f23e695b0c864fa9722e4f67d950266ca64d37b
1 change: 0 additions & 1 deletion plugins/analysis-icu/licenses/lucene-analysis-icu-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions plugins/analysis-icu/licenses/lucene-analysis-icu-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
2f6cb0fd7387c6e0db3b86eef7d8677cea3e88a0
1 change: 0 additions & 1 deletion plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
6aff23715a2fba88d844ac83c61decce8ed480bd
1 change: 0 additions & 1 deletion plugins/analysis-nori/licenses/lucene-analysis-nori-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions plugins/analysis-nori/licenses/lucene-analysis-nori-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
f82d3eba195134f663865e9de3f511e16fbc7351
1 change: 0 additions & 1 deletion plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
2af6e1996e696b1721a2ec7382bac9aa5096eecb
1 change: 0 additions & 1 deletion plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ec01d7f91f711abd75b539bb66a437db7cf1ca67
1 change: 0 additions & 1 deletion plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7041b3fa92b8687a84c4ce666b5718bbbc315db1
1 change: 0 additions & 1 deletion plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions ...ns/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0a5ec9a237c2539e3cbabfadff707252e33b3599
1 change: 0 additions & 1 deletion server/licenses/lucene-analysis-common-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-analysis-common-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3920c527fd5eee69e09f614391cef4e05c581c7f
1 change: 0 additions & 1 deletion server/licenses/lucene-backward-codecs-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-backward-codecs-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
d1dfcd42ea257355d5cbc64ac2f47142a550ae52
1 change: 0 additions & 1 deletion server/licenses/lucene-core-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-core-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ae4757f88e97036b30eb1eac1d21da6dabc85a5e
1 change: 0 additions & 1 deletion server/licenses/lucene-grouping-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-grouping-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
75485e3222b096027463378fe3bb2c8d1f529d25
1 change: 0 additions & 1 deletion server/licenses/lucene-highlighter-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-highlighter-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
d2a3d1f326f6d3bd6033b5620dc84f3c20a0412d
1 change: 0 additions & 1 deletion server/licenses/lucene-join-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-join-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
2c6f6058c765a955e0544c6050aeee3a5e376e47
1 change: 0 additions & 1 deletion server/licenses/lucene-memory-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-memory-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
2a155679022106c7db356da32563580d8de043d7
1 change: 0 additions & 1 deletion server/licenses/lucene-misc-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-misc-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7c7ac2027a12bf02657ec3a421c8039736b98344
1 change: 0 additions & 1 deletion server/licenses/lucene-queries-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-queries-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
25978bb82b9f78537f0511f0be64253be19de6fd
1 change: 0 additions & 1 deletion server/licenses/lucene-queryparser-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-queryparser-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
e3b6ce41d5bd73fdcc80b5b2a40283c03525aa96
1 change: 0 additions & 1 deletion server/licenses/lucene-sandbox-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-sandbox-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ae8649d2d01a416acdbe7c29f14b47a5594acb85
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial-extras-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-spatial-extras-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
58049352bb5fc8683c389eb2eb879fb4f16ff9b3
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial3d-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-spatial3d-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
2d3a8f802e1bb439d945de81ba6b16d01b24d58a
1 change: 0 additions & 1 deletion server/licenses/lucene-suggest-9.3.0.jar.sha1
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-suggest-9.4.0-snapshot-ddf0d0a.jar.sha1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
11cdb21cf08feb19e074b4a101e1550dfd544379
2 changes: 1 addition & 1 deletion server/src/main/java/org/opensearch/Version.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
public static final Version V_2_1_1 = new Version(2010199, org.apache.lucene.util.Version.LUCENE_9_2_0);
public static final Version V_2_2_0 = new Version(2020099, org.apache.lucene.util.Version.LUCENE_9_3_0);
public static final Version V_2_3_0 = new Version(2030099, org.apache.lucene.util.Version.LUCENE_9_3_0);
public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_3_0);
public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_4_0);
public static final Version CURRENT = V_3_0_0;

public static Version readVersion(StreamInput in) throws IOException {
Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/org/opensearch/common/lucene/Lucene.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@
* @opensearch.internal
*/
public class Lucene {
public static final String LATEST_CODEC = "Lucene92";
public static final String LATEST_CODEC = "Lucene94";

public static final String SOFT_DELETES_FIELD = "__soft_deletes";

Expand Down
8 changes: 4 additions & 4 deletions server/src/main/java/org/opensearch/index/codec/CodecService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@

import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
import org.apache.lucene.codecs.lucene92.Lucene92Codec.Mode;
import org.apache.lucene.codecs.lucene94.Lucene94Codec;
import org.apache.lucene.codecs.lucene94.Lucene94Codec.Mode;
import org.opensearch.common.Nullable;
import org.opensearch.common.collect.MapBuilder;
import org.opensearch.index.mapper.MapperService;
Expand All @@ -62,8 +62,8 @@ public class CodecService {
public CodecService(@Nullable MapperService mapperService, Logger logger) {
final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
if (mapperService == null) {
codecs.put(DEFAULT_CODEC, new Lucene92Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene92Codec(Mode.BEST_COMPRESSION));
codecs.put(DEFAULT_CODEC, new Lucene94Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene94Codec(Mode.BEST_COMPRESSION));
} else {
codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
Expand Down
4 changes: 2 additions & 2 deletions server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
import org.apache.lucene.codecs.lucene94.Lucene94Codec;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.opensearch.common.lucene.Lucene;
import org.opensearch.index.mapper.CompletionFieldMapper;
Expand All @@ -53,7 +53,7 @@
*
* @opensearch.internal
*/
public class PerFieldMappingPostingFormatCodec extends Lucene92Codec {
public class PerFieldMappingPostingFormatCodec extends Lucene94Codec {
private final Logger logger;
private final MapperService mapperService;
private final DocValuesFormat dvFormat = new Lucene90DocValuesFormat();
Expand Down
11 changes: 4 additions & 7 deletions ...in/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.Scorable;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.comparators.TermOrdValComparator;
import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.BytesRef;
import org.opensearch.common.util.BigArrays;
Expand Down Expand Up @@ -99,7 +100,7 @@ public FieldComparator<?> newComparator(String fieldname, int numHits, boolean e
final boolean sortMissingLast = sortMissingLast(missingValue) ^ reversed;
final BytesRef missingBytes = (BytesRef) missingObject(missingValue, reversed);
if (indexFieldData instanceof IndexOrdinalsFieldData) {
return new FieldComparator.TermOrdValComparator(numHits, null, sortMissingLast) {
FieldComparator<?> cmp = new TermOrdValComparator(numHits, indexFieldData.getFieldName(), sortMissingLast, reversed) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@nknize This may have caused a regression, see https://build.ci.opensearch.org/job/gradle-check/1941/

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(Also in https://build.ci.opensearch.org/job/gradle-check/1752/ associated with this PR)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Disregard the above. Found the problem elsewhere.


@Override
protected SortedDocValues getSortedDocValues(LeafReaderContext context, String field) throws IOException {
Expand All @@ -121,13 +122,9 @@ protected SortedDocValues getSortedDocValues(LeafReaderContext context, String f
return new ReplaceMissing(selectedValues, missingBytes);
}
}

@Override
public void setScorer(Scorable scorer) {
BytesRefFieldComparatorSource.this.setScorer(scorer);
}
Comment on lines -125 to -128
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not sure what was the use of this and when we are removing it will this impact anything.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It doesn't impact anything since the setScorer method is a noop


};
cmp.disableSkipping();
return cmp;
}

return new FieldComparator.TermValComparator(numHits, null, sortMissingLast) {
Expand Down
12 changes: 6 additions & 6 deletions server/src/test/java/org/opensearch/index/codec/CodecTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@

import org.apache.logging.log4j.LogManager;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
import org.apache.lucene.codecs.lucene94.Lucene94Codec;
import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
Expand Down Expand Up @@ -65,21 +65,21 @@ public class CodecTests extends OpenSearchTestCase {
public void testResolveDefaultCodecs() throws Exception {
CodecService codecService = createCodecService();
assertThat(codecService.codec("default"), instanceOf(PerFieldMappingPostingFormatCodec.class));
assertThat(codecService.codec("default"), instanceOf(Lucene92Codec.class));
assertThat(codecService.codec("default"), instanceOf(Lucene94Codec.class));
}

public void testDefault() throws Exception {
Codec codec = createCodecService().codec("default");
assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_SPEED, codec);
assertStoredFieldsCompressionEquals(Lucene94Codec.Mode.BEST_SPEED, codec);
}

public void testBestCompression() throws Exception {
Codec codec = createCodecService().codec("best_compression");
assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_COMPRESSION, codec);
assertStoredFieldsCompressionEquals(Lucene94Codec.Mode.BEST_COMPRESSION, codec);
}

// write some docs with it, inspect .si to see this was the used compression
private void assertStoredFieldsCompressionEquals(Lucene92Codec.Mode expected, Codec actual) throws Exception {
private void assertStoredFieldsCompressionEquals(Lucene94Codec.Mode expected, Codec actual) throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(null);
iwc.setCodec(actual);
Expand All @@ -91,7 +91,7 @@ private void assertStoredFieldsCompressionEquals(Lucene92Codec.Mode expected, Co
SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader();
String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY);
assertNotNull(v);
assertEquals(expected, Lucene92Codec.Mode.valueOf(v));
assertEquals(expected, Lucene94Codec.Mode.valueOf(v));
ir.close();
dir.close();
}
Expand Down
4 changes: 2 additions & 2 deletions server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
package org.opensearch.index.engine;

import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
import org.apache.lucene.codecs.lucene94.Lucene94Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
Expand Down Expand Up @@ -70,7 +70,7 @@ public void testExceptionsAreNotCached() {
public void testCompletionStatsCache() throws IOException, InterruptedException {
final IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
final PostingsFormat postingsFormat = new Completion90PostingsFormat();
indexWriterConfig.setCodec(new Lucene92Codec() {
indexWriterConfig.setCodec(new Lucene94Codec() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
return postingsFormat; // all fields are suggest fields
Expand Down