Merge branch 'main' into dependabot/gradle/plugins/repository-hdfs/co…

…m.google.protobuf-protobuf-java-3.21.11

.ci/bwcVersions

@@ -11,4 +11,5 @@ BWC_VERSION:
   - "2.3.1"
   - "2.4.0"
   - "2.4.1"
+  - "2.4.2"
   - "2.5.0"

CHANGELOG.md

@@ -52,6 +52,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Update Apache Lucene to 9.5.0-snapshot-a4ef70f ([#4979](https://github.com/opensearch-project/OpenSearch/pull/4979))
 - Update to Gradle 7.6 and JDK-19 ([#4973](https://github.com/opensearch-project/OpenSearch/pull/4973))
 - Bumps `protobuf-java` from 3.21.9 to 3.21.11 in /plugins/repository-hdfs ([#5519](https://github.com/opensearch-project/OpenSearch/pull/5519))
+- Update Apache Lucene to 9.5.0-snapshot-d5cef1c ([#5570](https://github.com/opensearch-project/OpenSearch/pull/5570))
 
 ### Changed
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
@@ -102,6 +103,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump gradle-extra-configurations-plugin from 7.0.0 to 8.0.0 ([#4808](https://github.com/opensearch-project/OpenSearch/pull/4808))
 ### Changed
 ### Deprecated
+- Refactor fuzziness interface on query builders ([#5433](https://github.com/opensearch-project/OpenSearch/pull/5433))
+
 ### Removed
 ### Fixed
 - Fix 1.x compatibility bug with stored Tasks ([#5412](https://github.com/opensearch-project/OpenSearch/pull/5412))

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.5.0-snapshot-a4ef70f
+lucene            = 9.5.0-snapshot-d5cef1c
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 19.0.1+10

modules/lang-expression/licenses/lucene-expressions-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+afe877ebf5ec4be7d17636b695015c449a523a3b

plugins/analysis-icu/licenses/lucene-analysis-icu-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+670d8f48ea9cba542e263d3ec6c3e2a33accc561

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+b1f42bad26470c8ef88096e0c8564a74223c52c9

plugins/analysis-nori/licenses/lucene-analysis-nori-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+f3cf74fa91da5133667f8916f93071fed231f2ee

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+e633c2d0cd677e4f1cef5aadc6bdc65e8e898d98

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+c12a2943e6f4977f15d489ac3e9802c5dfb3c4cc

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+5d257928a34e586a7de9fc7d4a013868f7a1db74

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+3d88f80ad07421b9470cb44a6f5b67dd47047b13

server/licenses/lucene-analysis-common-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+f2440fe126dad78e95f901c0f7a6eeb66da09938

server/licenses/lucene-backward-codecs-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+51677b84f823e352ab366f6a6bf87de8816650c4

server/licenses/lucene-core-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+b11b5c54ab26152c0db003c7a514f4e6c6825fdd

server/licenses/lucene-grouping-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+58bd60d4c3ec753eef0d904601ab6d726633a8db

server/licenses/lucene-highlighter-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+97f11221b89e37c575ed9538d88b1529872abd80

server/licenses/lucene-join-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+84980688c8eb9fbdb597a4291713ee630653392d

server/licenses/lucene-memory-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+23d3d1eefec90950b34ddef988be21b8fdfeb415

server/licenses/lucene-misc-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+e5f164991c11efeebf0697c76ffbe986af5341d5

server/licenses/lucene-queries-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+8c382224253727794557200e97717b927ad8fa74

server/licenses/lucene-queryparser-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+18cb84b504b8a57075efca72f4701aa8a720a057

server/licenses/lucene-sandbox-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+76cc7f77a30864e853a6662a5b7f4937023bd5e6

server/licenses/lucene-spatial-extras-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+70df807e8504f2fb1fe28ceaf33373e3de51aec8

server/licenses/lucene-spatial3d-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+75aaac030d36ddf7cdb09632fe1293c6ecd756ce

server/licenses/lucene-suggest-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+384e589a3d90773ff47ffbaa2797afe95609f183

server/src/internalClusterTest/java/org/opensearch/search/query/MultiMatchQueryIT.java

@@ -37,6 +37,7 @@
 import org.opensearch.action.index.IndexRequestBuilder;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.Fuzziness;
 import org.opensearch.common.util.set.Sets;
 import org.opensearch.common.xcontent.XContentBuilder;
 import org.opensearch.common.xcontent.XContentFactory;
@@ -1024,7 +1025,7 @@ public void testFuzzyFieldLevelBoosting() throws InterruptedException, Execution
 
         SearchResponse searchResponse = client().prepareSearch(idx)
             .setExplain(true)
-            .setQuery(multiMatchQuery("foo").field("title", 100).field("body").fuzziness(0))
+            .setQuery(multiMatchQuery("foo").field("title", 100).field("body").fuzziness(Fuzziness.ZERO))
             .get();
         SearchHit[] hits = searchResponse.getHits().getHits();
         assertNotEquals("both documents should be on different shards", hits[0].getShard().getShardId(), hits[1].getShard().getShardId());

server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java

@@ -49,6 +49,7 @@
 import org.opensearch.common.regex.Regex;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.time.DateFormatter;
+import org.opensearch.common.unit.Fuzziness;
 import org.opensearch.common.xcontent.XContentBuilder;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.common.xcontent.XContentType;
@@ -762,21 +763,21 @@ public void testMatchQueryFuzzy() throws Exception {
             client().prepareIndex("test").setId("2").setSource("text", "Unity")
         );
 
-        SearchResponse searchResponse = client().prepareSearch().setQuery(matchQuery("text", "uniy").fuzziness("0")).get();
+        SearchResponse searchResponse = client().prepareSearch().setQuery(matchQuery("text", "uniy").fuzziness(Fuzziness.ZERO)).get();
         assertHitCount(searchResponse, 0L);
 
-        searchResponse = client().prepareSearch().setQuery(matchQuery("text", "uniy").fuzziness("1")).get();
+        searchResponse = client().prepareSearch().setQuery(matchQuery("text", "uniy").fuzziness(Fuzziness.ONE)).get();
         assertHitCount(searchResponse, 2L);
         assertSearchHits(searchResponse, "1", "2");
 
-        searchResponse = client().prepareSearch().setQuery(matchQuery("text", "uniy").fuzziness("AUTO")).get();
+        searchResponse = client().prepareSearch().setQuery(matchQuery("text", "uniy").fuzziness(Fuzziness.AUTO)).get();
         assertHitCount(searchResponse, 2L);
         assertSearchHits(searchResponse, "1", "2");
 
-        searchResponse = client().prepareSearch().setQuery(matchQuery("text", "uniy").fuzziness("AUTO:5,7")).get();
+        searchResponse = client().prepareSearch().setQuery(matchQuery("text", "uniy").fuzziness(Fuzziness.customAuto(5, 7))).get();
         assertHitCount(searchResponse, 0L);
 
-        searchResponse = client().prepareSearch().setQuery(matchQuery("text", "unify").fuzziness("AUTO:5,7")).get();
+        searchResponse = client().prepareSearch().setQuery(matchQuery("text", "unify").fuzziness(Fuzziness.customAuto(5, 7))).get();
         assertHitCount(searchResponse, 1L);
         assertSearchHits(searchResponse, "2");
     }

server/src/main/java/org/opensearch/Version.java

@@ -83,16 +83,11 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_2_1 = new Version(2020199, org.apache.lucene.util.Version.LUCENE_9_3_0);
     public static final Version V_2_3_0 = new Version(2030099, org.apache.lucene.util.Version.LUCENE_9_3_0);
     public static final Version V_2_4_0 = new Version(2040099, org.apache.lucene.util.Version.LUCENE_9_4_1);
+    public static final Version V_2_4_1 = new Version(2040199, org.apache.lucene.util.Version.LUCENE_9_4_2);
 
     // UNRELEASED
-    public static final Version V_2_4_1 = new Version(
-        2040199,
-        org.apache.lucene.util.Version.fromBits(9, 4, 2) /** needs updated 9.5.0 snapshots */
-    );
-    public static final Version V_2_5_0 = new Version(
-        2050099,
-        org.apache.lucene.util.Version.fromBits(9, 4, 2) /** needs updated 9.5.0 snapshots */
-    );
+    public static final Version V_2_4_2 = new Version(2040299, org.apache.lucene.util.Version.LUCENE_9_4_2);
+    public static final Version V_2_5_0 = new Version(2050099, org.apache.lucene.util.Version.LUCENE_9_4_2);
     public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_5_0);
     public static final Version CURRENT = V_3_0_0;
 

server/src/main/java/org/opensearch/common/lucene/Lucene.java

@@ -109,7 +109,7 @@
  * @opensearch.internal
  */
 public class Lucene {
-    public static final String LATEST_CODEC = "Lucene94";
+    public static final String LATEST_CODEC = "Lucene95";
 
     public static final String SOFT_DELETES_FIELD = "__soft_deletes";
 

server/src/main/java/org/opensearch/common/unit/Fuzziness.java

@@ -139,6 +139,16 @@ public static Fuzziness build(Object fuzziness) {
         return new Fuzziness(string);
     }
 
+    /***
+     * Creates a {@link Fuzziness} instance from lowDistance and highDistance.
+     * where the edit distance is 0 for strings shorter than lowDistance,
+     * 1 for strings where its length between lowDistance and highDistance (inclusive),
+     * and 2 for strings longer than highDistance.
+     */
+    public static Fuzziness customAuto(int lowDistance, int highDistance) {
+        return new Fuzziness("AUTO", lowDistance, highDistance);
+    }
+
     private static Fuzziness parseCustomAuto(final String string) {
         assert string.toUpperCase(Locale.ROOT).startsWith(AUTO.asString() + ":");
         String[] fuzzinessLimit = string.substring(AUTO.asString().length() + 1).split(",");

server/src/main/java/org/opensearch/index/codec/CodecService.java

@@ -34,8 +34,8 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec.Mode;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec.Mode;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.collect.MapBuilder;
 import org.opensearch.index.mapper.MapperService;
@@ -62,8 +62,8 @@ public class CodecService {
     public CodecService(@Nullable MapperService mapperService, Logger logger) {
         final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
         if (mapperService == null) {
-            codecs.put(DEFAULT_CODEC, new Lucene94Codec());
-            codecs.put(BEST_COMPRESSION_CODEC, new Lucene94Codec(Mode.BEST_COMPRESSION));
+            codecs.put(DEFAULT_CODEC, new Lucene95Codec());
+            codecs.put(BEST_COMPRESSION_CODEC, new Lucene95Codec(Mode.BEST_COMPRESSION));
         } else {
             codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
             codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));

server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java

@@ -36,7 +36,7 @@
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
 import org.opensearch.common.lucene.Lucene;
 import org.opensearch.index.mapper.CompletionFieldMapper;
@@ -53,7 +53,7 @@
  *
  * @opensearch.internal
  */
-public class PerFieldMappingPostingFormatCodec extends Lucene94Codec {
+public class PerFieldMappingPostingFormatCodec extends Lucene95Codec {
     private final Logger logger;
     private final MapperService mapperService;
     private final DocValuesFormat dvFormat = new Lucene90DocValuesFormat();

server/src/main/java/org/opensearch/index/engine/RecoverySourcePruneMergePolicy.java

@@ -238,8 +238,8 @@ public void close() throws IOException {
             }
 
             @Override
-            public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
-                in.visitDocument(docID, visitor);
+            public void document(int docID, StoredFieldVisitor visitor) throws IOException {
+                in.document(docID, visitor);
             }
 
             @Override
@@ -268,11 +268,11 @@ private static class RecoverySourcePruningStoredFieldsReader extends FilterStore
             }
 
             @Override
-            public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
+            public void document(int docID, StoredFieldVisitor visitor) throws IOException {
                 if (recoverySourceToKeep != null && recoverySourceToKeep.get(docID)) {
-                    super.visitDocument(docID, visitor);
+                    super.document(docID, visitor);
                 } else {
-                    super.visitDocument(docID, new FilterStoredFieldVisitor(visitor) {
+                    super.document(docID, new FilterStoredFieldVisitor(visitor) {
                         @Override
                         public Status needsField(FieldInfo fieldInfo) throws IOException {
                             if (recoverySourceField.equals(fieldInfo.name)) {
@@ -293,7 +293,6 @@ public StoredFieldsReader getMergeInstance() {
             public StoredFieldsReader clone() {
                 return new RecoverySourcePruningStoredFieldsReader(in.clone(), recoverySourceToKeep, recoverySourceField);
             }
-
         }
 
         /**

server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java

@@ -45,6 +45,8 @@
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.StoredFields;
+import org.apache.lucene.index.TermVectors;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.VectorEncoding;
 import org.apache.lucene.index.VectorSimilarityFunction;
@@ -201,6 +203,11 @@ public Fields getTermVectors(int docID) {
         throw new UnsupportedOperationException();
     }
 
+    @Override
+    public TermVectors termVectors() throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
     @Override
     public int numDocs() {
         return 1;
@@ -232,6 +239,11 @@ public void document(int docID, StoredFieldVisitor visitor) throws IOException {
         }
     }
 
+    @Override
+    public StoredFields storedFields() throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
     @Override
     protected void doClose() {
 
@@ -251,4 +263,9 @@ public VectorValues getVectorValues(String field) throws IOException {
     public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
         throw new UnsupportedOperationException();
     }
+
+    @Override
+    public TopDocs searchNearestVectors(String field, BytesRef target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
+        throw new UnsupportedOperationException();
+    }
 }