iter

lucene/CHANGES.txt

@@ -76,7 +76,8 @@ New Features
 Improvements
 ---------------------
 
-* LUCENE-10078: Merge-on-refresh is now enabled by default. (Adrien Grand)
+* LUCENE-10078: Merge on full flush is now enabled by default with a timeout of
+  500ms. (Adrien Grand)
 
 * LUCENE-10585: Facet module code cleanup (copy/paste scrubbing, simplification and some very minor
   optimization tweaks). (Greg Miller)

lucene/core/src/java/org/apache/lucene/index/FilterMergePolicy.java

@@ -124,4 +124,9 @@ public int numDeletesToMerge(
   public MergePolicy unwrap() {
     return in;
   }
+
+  @Override
+  protected long maxFullFlushMergeSize() {
+    return in.maxFullFlushMergeSize();
+  }
 }

lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java

@@ -460,7 +460,7 @@ public IndexWriterConfig setCommitOnClose(boolean commitOnClose) {
    * <p>Note: Which segments would get merged depends on the implementation of {@link
    * MergePolicy#findFullFlushMerges(MergeTrigger, SegmentInfos, MergePolicy.MergeContext)}
    *
-   * <p>Note: Set to 0 to disable merge-on-refresh.
+   * <p>Note: Set to 0 to disable merging on full flush.
    */
   public IndexWriterConfig setMaxFullFlushMergeWaitMillis(long maxFullFlushMergeWaitMillis) {
     this.maxFullFlushMergeWaitMillis = maxFullFlushMergeWaitMillis;

lucene/core/src/java/org/apache/lucene/index/LogByteSizeMergePolicy.java

@@ -97,10 +97,10 @@ public double getMaxMergeMBForForcedMerge() {
 
   /**
    * Sets the minimum size for the lowest level segments. Any segments below this size are
-   * considered to be on the same level (even if they vary drastically in size) and will be merged
-   * whenever there are mergeFactor of them. This effectively truncates the "long tail" of small
-   * segments that would otherwise be created into a single level. If you set this too large, it
-   * could greatly increase the merging cost during indexing (if you flush many small segments).
+   * candidates for full-flush merges and merged more aggressively. This effectively reduces chances
+   * to get a "long tail" of small segments that would otherwise be created into a single level. If
+   * you set this too large, it could greatly increase the merging cost during indexing (if you
+   * flush many small segments).
    */
   public void setMinMergeMB(double mb) {
     minMergeSize = (long) (mb * 1024 * 1024);

lucene/core/src/java/org/apache/lucene/index/LogDocMergePolicy.java

@@ -44,10 +44,10 @@ protected long size(SegmentCommitInfo info, MergeContext mergeContext) throws IO
 
   /**
    * Sets the minimum size for the lowest level segments. Any segments below this size are
-   * considered to be on the same level (even if they vary drastically in size) and will be merged
-   * whenever there are mergeFactor of them. This effectively truncates the "long tail" of small
-   * segments that would otherwise be created into a single level. If you set this too large, it
-   * could greatly increase the merging cost during indexing (if you flush many small segments).
+   * candidates for full-flush merges and merged more aggressively. This effectively reduces chances
+   * to get a "long tail" of small segments that would otherwise be created into a single level. If
+   * you set this too large, it could greatly increase the merging cost during indexing (if you
+   * flush many small segments).
    */
   public void setMinMergeDocs(int minMergeDocs) {
     minMergeSize = minMergeDocs;

lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java

@@ -68,8 +68,8 @@ public abstract class LogMergePolicy extends MergePolicy {
   protected int mergeFactor = DEFAULT_MERGE_FACTOR;
 
   /**
-   * Any segments whose size is smaller than this value will be rounded up to this value. This
-   * ensures that tiny segments are aggressively merged.
+   * Any segments whose size is smaller than this value will be candidates for full-flush merges and
+   * merged more aggressively.
    */
   protected long minMergeSize;
 
@@ -184,6 +184,11 @@ protected boolean isMerged(
         && (numToMerge != 1 || !segmentIsOriginal || isMerged(infos, mergeInfo, mergeContext));
   }
 
+  @Override
+  protected long maxFullFlushMergeSize() {
+    return minMergeSize;
+  }
+
   /**
    * Returns the merges necessary to merge the index, taking the max merge size or max merge docs
    * into consideration. This method attempts to respect the {@code maxNumSegments} parameter,

lucene/core/src/java/org/apache/lucene/index/MergePolicy.java

@@ -697,7 +697,7 @@ protected long size(SegmentCommitInfo info, MergeContext mergeContext) throws IO
    * implementation of {@link #findFullFlushMerges}.
    */
   protected long maxFullFlushMergeSize() {
-    return Long.MAX_VALUE;
+    return 0L;
   }
 
   /** Asserts that the delCount for this SegmentCommitInfo is valid */

lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java

@@ -201,6 +201,11 @@ public double getFloorSegmentMB() {
     return floorSegmentBytes / (1024 * 1024.);
   }
 
+  @Override
+  protected long maxFullFlushMergeSize() {
+    return floorSegmentBytes;
+  }
+
   /**
    * When forceMergeDeletes is called, we only merge away a segment if its delete percentage is over
    * this threshold. Default is 10%.

lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java

@@ -815,7 +815,8 @@ public void testNPEAfterInvalidReindex2() throws Exception {
   /** test reopening backwards from a non-NRT reader (with document deletes) */
   public void testNRTMdeletes() throws Exception {
     Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    IndexWriterConfig iwc =
+        new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE);
     SnapshotDeletionPolicy snapshotter =
         new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
     iwc.setIndexDeletionPolicy(snapshotter);
@@ -865,7 +866,8 @@ public void testNRTMdeletes() throws Exception {
   /** test reopening backwards from an NRT reader (with document deletes) */
   public void testNRTMdeletes2() throws Exception {
     Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()))
+        .setMergePolicy(NoMergePolicy.INSTANCE);
     SnapshotDeletionPolicy snapshotter =
         new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
     iwc.setIndexDeletionPolicy(snapshotter);

lucene/core/src/test/org/apache/lucene/index/TestLogMergePolicy.java

@@ -57,10 +57,7 @@ public void testFullFlushMerges() throws IOException {
 
     LogMergePolicy mp = mergePolicy();
 
-    for (int i = 0; i < 2 * mp.getMergeFactor() + 3; ++i) {
-      if (i < mp.getMergeFactor()) {
-        assertNull(mp.findFullFlushMerges(MergeTrigger.FULL_FLUSH, segmentInfos, mergeContext));
-      }
+    for (int i = 0; i < mp.getMergeFactor(); ++i) {
       segmentInfos.add(
           makeSegmentCommitInfo(
               "_" + segNameGenerator.getAndIncrement(),
@@ -76,11 +73,6 @@ public void testFullFlushMerges() throws IOException {
       segmentInfos =
           applyMerge(segmentInfos, merge, "_" + segNameGenerator.getAndIncrement(), stats);
     }
-    assertEquals(5, segmentInfos.size());
-    assertEquals(mp.getMergeFactor(), segmentInfos.info(0).info.maxDoc());
-    assertEquals(mp.getMergeFactor(), segmentInfos.info(1).info.maxDoc());
-    assertEquals(1, segmentInfos.info(2).info.maxDoc());
-    assertEquals(1, segmentInfos.info(3).info.maxDoc());
-    assertEquals(1, segmentInfos.info(4).info.maxDoc());
+    assertEquals(1, segmentInfos.size());
   }
 }

lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java

@@ -922,13 +922,13 @@ public void testSimulateUpdates() throws IOException {
 
   public void testFullFlushMerges() throws IOException {
     AtomicLong segNameGenerator = new AtomicLong();
+    IOStats stats = new IOStats();
     MergeContext mergeContext = new MockMergeContext(SegmentCommitInfo::getDelCount);
     SegmentInfos segmentInfos = new SegmentInfos(Version.LATEST.major);
 
-    TieredMergePolicy mp = mergePolicy();
+    TieredMergePolicy mp = new TieredMergePolicy();
 
-    for (int i = 0; i < mp.getSegmentsPerTier(); ++i) {
-      assertNull(mp.findFullFlushMerges(MergeTrigger.FULL_FLUSH, segmentInfos, mergeContext));
+    for (int i = 0; i < 11; ++i) {
       segmentInfos.add(
           makeSegmentCommitInfo(
               "_" + segNameGenerator.getAndIncrement(),
@@ -937,6 +937,13 @@ public void testFullFlushMerges() throws IOException {
               Double.MIN_VALUE,
               IndexWriter.SOURCE_FLUSH));
     }
-    assertNotNull(mp.findFullFlushMerges(MergeTrigger.FULL_FLUSH, segmentInfos, mergeContext));
+    MergeSpecification spec =
+        mp.findFullFlushMerges(MergeTrigger.FULL_FLUSH, segmentInfos, mergeContext);
+    assertNotNull(spec);
+    for (OneMerge merge : spec.merges) {
+      segmentInfos =
+          applyMerge(segmentInfos, merge, "_" + segNameGenerator.getAndIncrement(), stats);
+    }
+    assertEquals(2, segmentInfos.size());
   }
 }

lucene/core/src/test/org/apache/lucene/index/TestTragicIndexWriterDeadlock.java

@@ -105,7 +105,10 @@ public void run() {
   // LUCENE-7570
   public void testDeadlockStalledMerges() throws Exception {
     Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig();
+    IndexWriterConfig iwc =
+        new IndexWriterConfig()
+            // nocommit: deadlock if you don't disable merge-on-full-flush
+            .setMaxFullFlushMergeWaitMillis(0);
 
     // so we merge every 2 segments:
     LogMergePolicy mp = new LogDocMergePolicy();