Update to Apache Lucene 9.11.0-snapshot-4be6531 (#13850)

Signed-off-by: Navneet Verma <navneev@amazon.com>
opensearch-project · May 28, 2024 · 45360db · 45360db
1 parent 994bfae
commit 45360db
Show file tree

Hide file tree

Showing 50 changed files with 40 additions and 27 deletions.
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 opensearch = 3.0.0
-lucene = 9.11.0-snapshot-fb97840
+lucene = 9.11.0-snapshot-4be6531
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 21.0.3+9

diff --git a/libs/core/licenses/lucene-core-9.11.0-snapshot-4be6531.jar.sha1 b/libs/core/licenses/lucene-core-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+3c2361bd633374ae3814b175cc25ccf773f67026
diff --git a/libs/core/licenses/lucene-core-9.11.0-snapshot-fb97840.jar.sha1 b/libs/core/licenses/lucene-core-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.11.0-snapshot-4be6531.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+8752daf173a642ae02e081cc0398f2ce59278200
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.11.0-snapshot-fb97840.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0-snapshot-4be6531.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+12630ff9c56e2a372ba57f519c579ff9e728208a
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0-snapshot-fb97840.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0-snapshot-4be6531.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+752bfc61c7829be6c27d9c1764250196e2c6b06b
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0-snapshot-fb97840.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0-snapshot-4be6531.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+5ca56d42b24498a226cf91f48b94e010b6af5fe2
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0-snapshot-fb97840.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0-snapshot-4be6531.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+8eb59a89aa8984457798ccffb8e97e5351bebc1f
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0-snapshot-fb97840.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0-snapshot-4be6531.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+851c1bd99eaef368e84335853dd448e4f56cdbc8
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0-snapshot-fb97840.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0-snapshot-4be6531.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+63647085d41ae231733580c20a498ce7c9134ce5
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0-snapshot-fb97840.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/...s/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0-snapshot-4be6531.jar.sha1 b/...s/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+a3ba7dd03b1df9efed08eb544689d51d2be22aa5
diff --git a/...s/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0-snapshot-fb97840.jar.sha1 b/...s/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/server/licenses/lucene-analysis-common-9.11.0-snapshot-4be6531.jar.sha1 b/server/licenses/lucene-analysis-common-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+9cc4e600289bf1171b47de74536bd34c476f85a8
diff --git a/server/licenses/lucene-analysis-common-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-analysis-common-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/server/licenses/lucene-backward-codecs-9.11.0-snapshot-4be6531.jar.sha1 b/server/licenses/lucene-backward-codecs-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+8babfe85be7e36c893741e08072c11e71db09715
diff --git a/server/licenses/lucene-backward-codecs-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-backward-codecs-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/server/licenses/lucene-core-9.11.0-snapshot-4be6531.jar.sha1 b/server/licenses/lucene-core-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+3c2361bd633374ae3814b175cc25ccf773f67026
diff --git a/server/licenses/lucene-core-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-core-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/server/licenses/lucene-grouping-9.11.0-snapshot-4be6531.jar.sha1 b/server/licenses/lucene-grouping-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+d9f29b49cd1e0a061ff7fa4a53e8605bd49bd3d0
diff --git a/server/licenses/lucene-grouping-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-grouping-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/server/licenses/lucene-highlighter-9.11.0-snapshot-4be6531.jar.sha1 b/server/licenses/lucene-highlighter-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+33bc26d46d62bb1cf3bf725db637226a43db7625
diff --git a/server/licenses/lucene-highlighter-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-highlighter-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/server/licenses/lucene-join-9.11.0-snapshot-4be6531.jar.sha1 b/server/licenses/lucene-join-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+82966698abdb8f0367a162f642560566a6085dc8
diff --git a/server/licenses/lucene-join-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-join-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/server/licenses/lucene-memory-9.11.0-snapshot-4be6531.jar.sha1 b/server/licenses/lucene-memory-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+746f392e7ec27a7cd6ca2add7dd8441d2a6085da
diff --git a/server/licenses/lucene-memory-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-memory-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/server/licenses/lucene-misc-9.11.0-snapshot-4be6531.jar.sha1 b/server/licenses/lucene-misc-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+0eb06ecc39c0ec0db380a6e5aad1b16907e0bfd9
diff --git a/server/licenses/lucene-misc-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-misc-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/server/licenses/lucene-queries-9.11.0-snapshot-4be6531.jar.sha1 b/server/licenses/lucene-queries-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+0e56eb18cceffcd5ce2e47b679e873420254df74
diff --git a/server/licenses/lucene-queries-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-queries-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/server/licenses/lucene-queryparser-9.11.0-snapshot-4be6531.jar.sha1 b/server/licenses/lucene-queryparser-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+dee3997a72eeae905e92930f53e724b6bef279da
diff --git a/server/licenses/lucene-queryparser-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-queryparser-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/server/licenses/lucene-sandbox-9.11.0-snapshot-4be6531.jar.sha1 b/server/licenses/lucene-sandbox-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+946bc45b87b3d770ab6828b0d0a5f8684f2c3624
diff --git a/server/licenses/lucene-sandbox-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-sandbox-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/server/licenses/lucene-spatial-extras-9.11.0-snapshot-4be6531.jar.sha1 b/server/licenses/lucene-spatial-extras-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+d73667f61fb5e7fde4cec52fcfbbfd9847068aec
diff --git a/server/licenses/lucene-spatial-extras-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-spatial-extras-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/server/licenses/lucene-spatial3d-9.11.0-snapshot-4be6531.jar.sha1 b/server/licenses/lucene-spatial3d-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+a8e8ab80bfb6abd70932e50fe31e13ecf2e00987
diff --git a/server/licenses/lucene-spatial3d-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-spatial3d-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/server/licenses/lucene-suggest-9.11.0-snapshot-4be6531.jar.sha1 b/server/licenses/lucene-suggest-9.11.0-snapshot-4be6531.jar.sha1
@@ -0,0 +1 @@
+45d6f0facd45d4e49585f0dabfa62ed5a1883033
diff --git a/server/licenses/lucene-suggest-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-suggest-9.11.0-snapshot-fb97840.jar.sha1
diff --git a/server/src/main/java/org/apache/lucene/search/uhighlight/CustomFieldHighlighter.java b/server/src/main/java/org/apache/lucene/search/uhighlight/CustomFieldHighlighter.java
@@ -51,6 +51,8 @@
 class CustomFieldHighlighter extends FieldHighlighter {
  private static final Passage[] EMPTY_PASSAGE = new Passage[0];
 
+ private static final Comparator<Passage> DEFAULT_PASSAGE_SORT_COMPARATOR = Comparator.comparingInt(Passage::getStartOffset);
+
  private final Locale breakIteratorLocale;
  private final int noMatchSize;
  private String fieldValue;
@@ -66,7 +68,16 @@ class CustomFieldHighlighter extends FieldHighlighter {
  PassageFormatter passageFormatter,
  int noMatchSize
  ) {
- super(field, fieldOffsetStrategy, breakIterator, passageScorer, maxPassages, maxNoHighlightPassages, passageFormatter);
+ super(
+ field,
+ fieldOffsetStrategy,
+ breakIterator,
+ passageScorer,
+ maxPassages,
+ maxNoHighlightPassages,
+ passageFormatter,
+ DEFAULT_PASSAGE_SORT_COMPARATOR
+ );
  this.breakIteratorLocale = breakIteratorLocale;
  this.noMatchSize = noMatchSize;
  }

diff --git a/...r/src/main/java/org/opensearch/search/fetch/subphase/highlight/FastVectorHighlighter.java b/...r/src/main/java/org/opensearch/search/fetch/subphase/highlight/FastVectorHighlighter.java
@@ -304,7 +304,7 @@ private static BoundaryScanner getBoundaryScanner(Field field) {
  return DEFAULT_WORD_BOUNDARY_SCANNER;
  case CHARS:
  if (fieldOptions.boundaryMaxScan() != SimpleBoundaryScanner.DEFAULT_MAX_SCAN
- || fieldOptions.boundaryChars() != SimpleBoundaryScanner.DEFAULT_BOUNDARY_CHARS) {
+ || fieldOptions.boundaryChars() != HighlightBuilder.DEFAULT_BOUNDARY_CHARS) {
  return new SimpleBoundaryScanner(fieldOptions.boundaryMaxScan(), fieldOptions.boundaryChars());
  }
  return DEFAULT_SIMPLE_BOUNDARY_SCANNER;

diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilder.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilder.java
@@ -111,6 +111,8 @@ public class HighlightBuilder extends AbstractHighlighterBuilder<HighlightBuilde
  /** the default closing tags when {@code tag_schema = "styled"} */
  public static final String[] DEFAULT_STYLED_POST_TAGS = { "</em>" };
 
+ static final Character[] DEFAULT_BOUNDARY_CHARS = HighlightBuilder.convertCharArray(SimpleBoundaryScanner.DEFAULT_BOUNDARY_CHARS);
+
  /**
  * a {@link FieldOptions} with default settings
  */
@@ -124,7 +126,7 @@ public class HighlightBuilder extends AbstractHighlighterBuilder<HighlightBuilde
  .numberOfFragments(DEFAULT_NUMBER_OF_FRAGMENTS)
  .encoder(DEFAULT_ENCODER)
  .boundaryMaxScan(SimpleBoundaryScanner.DEFAULT_MAX_SCAN)
- .boundaryChars(SimpleBoundaryScanner.DEFAULT_BOUNDARY_CHARS)
+ .boundaryChars(DEFAULT_BOUNDARY_CHARS)
  .boundaryScannerLocale(Locale.ROOT)
  .noMatchSize(DEFAULT_NO_MATCH_SIZE)
  .phraseLimit(DEFAULT_PHRASE_LIMIT)