IGNITE-13786 Add defragmentation-specific B+Tree optimizations - Fixes …

…#8560. Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
apache · Dec 10, 2020 · dc4b71b · dc4b71b
1 parent 3c2535e
commit dc4b71b
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 35 deletions.
diff --git a/...al/processors/cache/persistence/defragmentation/CachePartitionDefragmentationManager.java b/...al/processors/cache/persistence/defragmentation/CachePartitionDefragmentationManager.java
@@ -93,6 +93,7 @@
 import static org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.DEFRAGMENTATION_MAPPING_REGION_NAME;
 import static org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.DEFRAGMENTATION_PART_REGION_NAME;
 import static org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.batchRenameDefragmentedCacheGroupPartitions;
+import static org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.defragmentedIndexFile;
 import static org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.defragmentedIndexTmpFile;
 import static org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.defragmentedPartFile;
 import static org.apache.ignite.internal.processors.cache.persistence.defragmentation.DefragmentationFileUtils.defragmentedPartMappingFile;
@@ -386,9 +387,9 @@ public void executeDefragmentation() throws IgniteCheckedException {
                             partCtx.partPageMemory
                         );
 
-                            partCtx.createNewCacheDataStore(offheap);
+                        partCtx.createNewCacheDataStore(offheap);
 
-                            copyPartitionData(partCtx, treeIter);
+                        copyPartitionData(partCtx, treeIter);
 
                         DefragmentationPageReadWriteManager pageMgr = (DefragmentationPageReadWriteManager)partCtx.partPageMemory.pageManager();
 
@@ -450,7 +451,21 @@ public void executeDefragmentation() throws IgniteCheckedException {
                             .futureFor(CheckpointState.FINISHED);
                     }
 
+                    PageStore oldIdxPageStore = filePageStoreMgr.getStore(grpId, INDEX_PARTITION);
+
                     idxDfrgFut = idxDfrgFut.chain(fut -> {
+                        if (log.isDebugEnabled()) {
+                            log.debug(S.toString(
+                                "Index partition defragmented",
+                                "grpId", grpId, false,
+                                "oldPages", oldIdxPageStore.pages(), false,
+                                "newPages", idxAllocationTracker.get() + 1, false,
+                                "pageSize", pageSize, false,
+                                "partFile", defragmentedIndexFile(workDir).getName(), false,
+                                "workDir", workDir, false
+                            ));
+                        }
+
                         oldPageMem.invalidate(grpId, PageIdAllocator.INDEX_PARTITION);
 
                         PageMemoryEx partPageMem = (PageMemoryEx)partDataRegion.pageMemory();
@@ -476,8 +491,6 @@ public void executeDefragmentation() throws IgniteCheckedException {
                         return null;
                     });
 
-                    PageStore oldIdxPageStore = filePageStoreMgr.getStore(grpId, INDEX_PARTITION);
-
                     status.onIndexDefragmented(
                         oldGrpCtx,
                         oldIdxPageStore.size(),
@@ -614,6 +627,9 @@ private void copyPartitionData(
         CacheDataTree tree = partCtx.oldCacheDataStore.tree();
 
         CacheDataTree newTree = partCtx.newCacheDataStore.tree();
+
+        newTree.enableSequentialWriteMode();
+
         PendingEntriesTree newPendingTree = partCtx.newCacheDataStore.pendingTree();
         AbstractFreeList<CacheDataRow> freeList = partCtx.newCacheDataStore.getCacheStoreFreeList();
 

diff --git a/...src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java b/...src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
@@ -63,7 +63,6 @@
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.LongListReuseBag;
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseBag;
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
-import org.apache.ignite.internal.processors.cache.persistence.tree.util.InsertLast;
 import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
 import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandlerWrapper;
 import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageLockListener;
@@ -154,6 +153,9 @@ public abstract class BPlusTree<L, T extends L> extends DataStructure implements
     /** Failure processor. */
     private final FailureProcessor failureProcessor;
 
+    /** Flag for enabling single-threaded append-only tree creation. */
+    private boolean sequentialWriteOptsEnabled;
+
     /** */
     private final GridTreePrinter<Long> treePrinter = new GridTreePrinter<Long>() {
         /** */
@@ -884,6 +886,11 @@ public final String getName() {
         return name;
     }
 
+    /** Flag for enabling single-threaded append-only tree creation. */
+    public void enableSequentialWriteMode() {
+        sequentialWriteOptsEnabled = true;
+    }
+
     /**
      * Initialize new tree.
      *
@@ -1398,6 +1405,8 @@ public final <R> R findOne(L row, TreeRowClosure<L, T> c, Object x) throws Ignit
      * @throws IgniteCheckedException If failed.
      */
     private void doFind(Get g) throws IgniteCheckedException {
+        assert !sequentialWriteOptsEnabled;
+
         for (;;) { // Go down with retries.
             g.init();
 
@@ -2054,6 +2063,8 @@ private Result invokeDown(final Invoke x, final long pageId, final long backId,
      * @throws IgniteCheckedException If failed.
      */
     private T doRemove(L row, boolean needOld) throws IgniteCheckedException {
+        assert !sequentialWriteOptsEnabled;
+
         checkDestroyed();
 
         Remove r = new Remove(row, needOld);
@@ -2711,7 +2722,8 @@ private boolean splitPage(
         long pageId, long page, long pageAddr, BPlusIO io, long fwdId, long fwdBuf, int idx
     ) throws IgniteCheckedException {
         int cnt = io.getCount(pageAddr);
-        int mid = cnt >>> 1;
+
+        int mid = sequentialWriteOptsEnabled ? (int)(cnt * 0.85) : cnt >>> 1;
 
         boolean res = false;
 
@@ -2767,7 +2779,7 @@ private Result askNeighbor(long pageId, Get g, boolean back) throws IgniteChecke
      * @return Result code.
      * @throws IgniteCheckedException If failed.
      */
-    private Result putDown(final Put p, final long pageId, final long fwdId, final int lvl)
+    private Result putDown(final Put p, final long pageId, final long fwdId, int lvl)
         throws IgniteCheckedException {
         assert lvl >= 0 : lvl;
 
@@ -5302,8 +5314,11 @@ private int findInsertionPoint(int lvl, BPlusIO<L> io, long buf, int low, int cn
         throws IgniteCheckedException {
         assert row != null;
 
-        if (row instanceof InsertLast)
+        if (sequentialWriteOptsEnabled) {
+            assert io.getForward(buf) == 0L;
+
             return -cnt - 1;
+        }
 
         int high = cnt - 1;
 

diff --git a/...in/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/InsertLast.java b/...in/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/InsertLast.java
diff --git a/...org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDefragmentationTest.java b/...org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDefragmentationTest.java
@@ -80,7 +80,7 @@ public class IgnitePdsDefragmentationTest extends GridCommonAbstractTest {
     public static final int PARTS = 5;
 
     /** */
-    public static final int ADDED_KEYS_COUNT = 150;
+    public static final int ADDED_KEYS_COUNT = 1500;
 
     /** */
     protected static final String GRP_NAME = "group";

diff --git a/...g/apache/ignite/internal/processors/query/h2/defragmentation/IndexingDefragmentation.java b/...g/apache/ignite/internal/processors/query/h2/defragmentation/IndexingDefragmentation.java
@@ -40,7 +40,6 @@
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusMetaIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIoResolver;
-import org.apache.ignite.internal.processors.cache.persistence.tree.util.InsertLast;
 import org.apache.ignite.internal.processors.cache.tree.mvcc.data.MvccDataRow;
 import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
 import org.apache.ignite.internal.processors.query.h2.database.H2Tree;
@@ -158,6 +157,8 @@ public void defragment(
                 for (int i = 0; i < segments; i++) {
                     H2Tree tree = oldH2Idx.treeForRead(i);
 
+                    newIdx.treeForRead(i).enableSequentialWriteMode();
+
                     treeIterator.iterate(tree, oldCachePageMem, (theTree, io, pageAddr, idx) -> {
                         cancellationChecker.run();
 
@@ -396,7 +397,7 @@ public BPlusLeafIoDelegate(IO io) {
     /**
      * H2CacheRow with stored index values
      */
-    private static class H2CacheRowWithIndex extends H2CacheRow implements InsertLast {
+    private static class H2CacheRowWithIndex extends H2CacheRow {
         /** List of index values. */
         private final List<Value> values;
 
@@ -406,6 +407,7 @@ public H2CacheRowWithIndex(GridH2RowDescriptor desc, CacheDataRow row, List<Valu
             this.values = values;
         }
 
+        /** */
         public static H2CacheRowWithIndex create(
             GridH2RowDescriptor desc,
             long newLink,