Add a pagestorage v3 controller. (#4320)

ref #3594

dbms/src/Storages/Page/V3/BlobStore.cpp

@@ -325,7 +325,7 @@ std::pair<BlobFileId, BlobFileOffset> BlobStore::getPosFromStats(size_t size)
     // Can't insert into this spacemap
     if (offset == INVALID_BLOBFILE_OFFSET)
     {
-        stat->smap->logStats();
+        stat->smap->logDebugString();
         throw Exception(fmt::format("Get postion from BlobStat failed, it may caused by `sm_max_caps` is no correct. [size={}] [old_max_caps={}] [max_caps={}] [blob_id={}]",
                                     size,
                                     old_max_cap,
@@ -1207,7 +1207,7 @@ bool BlobStore::BlobStats::BlobStat::removePosFromStat(BlobFileOffset offset, si
 {
     if (!smap->markFree(offset, buf_size))
     {
-        smap->logStats();
+        smap->logDebugString();
         throw Exception(fmt::format("Remove postion from BlobStat failed, [offset={} , buf_size={}, blob_id={}] is invalid.",
                                     offset,
                                     buf_size,
@@ -1224,7 +1224,7 @@ void BlobStore::BlobStats::BlobStat::restoreSpaceMap(BlobFileOffset offset, size
 {
     if (!smap->markUsed(offset, buf_size))
     {
-        smap->logStats();
+        smap->logDebugString();
         throw Exception(fmt::format("Restore postion from BlobStat failed, [offset={}] [buf_size={}] [blob_id={}] is used or subspan is used",
                                     offset,
                                     buf_size,

dbms/src/Storages/Page/V3/BlobStore.h

@@ -269,6 +269,8 @@ class BlobStore : private Allocator<false>
     BlobFilePtr getBlobFile(BlobFileId blob_id);
 
     friend class PageDirectoryFactory;
+    friend class PageStorageControl;
+
 #ifndef DBMS_PUBLIC_GTEST
 private:
 #endif

dbms/src/Storages/Page/V3/PageDirectory.h

@@ -252,6 +252,7 @@ class VersionedPageEntries
             being_ref_count,
             entries.size());
     }
+    friend class PageStorageControl;
 
 private:
     mutable std::mutex m;
@@ -365,6 +366,7 @@ class PageDirectory
     PageDirectory & operator=(PageDirectory && rhs) = delete;
 
     friend class PageDirectoryFactory;
+    friend class PageStorageControl;
 
 private:
     // Only `std::map` is allow for `MVCCMap`. Cause `std::map::insert` ensure that

dbms/src/Storages/Page/V3/PageStorageImpl.h

@@ -82,6 +82,7 @@ class PageStorageImpl : public DB::PageStorage
 #endif
 
     friend class PageDirectoryFactory;
+    friend class PageStorageControl;
 #ifndef DBMS_PUBLIC_GTEST
 private:
 #endif

dbms/src/Storages/Page/V3/spacemap/SpaceMap.cpp

@@ -59,11 +59,12 @@ bool SpaceMap::checkSpace(UInt64 offset, size_t size) const
     return (offset < start) || (offset > end) || (offset + size - 1 > end);
 }
 
-void SpaceMap::logStats()
+void SpaceMap::logDebugString()
 {
-    smapStats();
+    LOG_DEBUG(log, toDebugString());
 }
 
+
 bool SpaceMap::markFree(UInt64 offset, size_t length)
 {
     if (checkSpace(offset, length))

dbms/src/Storages/Page/V3/spacemap/SpaceMap.h

@@ -118,7 +118,12 @@ class SpaceMap
     /**
      * Log the status of space map
      */
-    void logStats();
+    void logDebugString();
+
+    /**
+     * return the status of space map
+     */
+    virtual String toDebugString() = 0;
 
     SpaceMapType getType() const
     {
@@ -143,9 +148,6 @@ class SpaceMap
 
     virtual ~SpaceMap() = default;
 
-    /* Print space maps status  */
-    virtual void smapStats() = 0;
-
     // Return true if space [offset, offset+size) are all free
     virtual bool isMarkUnused(UInt64 offset, size_t size) = 0;
 

dbms/src/Storages/Page/V3/spacemap/SpaceMapRBTree.cpp

@@ -464,36 +464,28 @@ void RBTreeSpaceMap::freeSmap()
     }
 }
 
-void RBTreeSpaceMap::smapStats()
+String RBTreeSpaceMap::toDebugString()
 {
     struct rb_node * node = nullptr;
     struct SmapRbEntry * entry;
     UInt64 count = 0;
-    UInt64 max_size = 0;
-    UInt64 min_size = ULONG_MAX;
+    FmtBuffer fmt_buffer;
 
     if (rb_tree->root.rb_node == nullptr)
     {
-        LOG_ERROR(log, "Tree have not been inited.");
-        return;
+        fmt_buffer.append("Tree have not been inited.");
+        return fmt_buffer.toString();
     }
 
-    LOG_DEBUG(log, "RB-Tree entries status: ");
+    fmt_buffer.append("    RB-Tree entries status: \n");
     for (node = rb_tree_first(&rb_tree->root); node != nullptr; node = rb_tree_next(node))
     {
         entry = node_to_entry(node);
-        LOG_FMT_DEBUG(log, "  Space: {} start: {} size: {}", count, entry->start, entry->count);
+        fmt_buffer.fmtAppend("      Space: {} start: {} size: {} \n", count, entry->start, entry->count);
         count++;
-        if (entry->count > max_size)
-        {
-            max_size = entry->count;
-        }
-
-        if (entry->count < min_size)
-        {
-            min_size = entry->count;
-        }
     }
+
+    return fmt_buffer.toString();
 }
 
 bool RBTreeSpaceMap::isMarkUnused(UInt64 offset, size_t length)

dbms/src/Storages/Page/V3/spacemap/SpaceMapRBTree.h

@@ -56,7 +56,7 @@ class RBTreeSpaceMap
 
     void freeSmap();
 
-    void smapStats() override;
+    String toDebugString() override;
 
     bool isMarkUnused(UInt64 offset, size_t length) override;
 

dbms/src/Storages/Page/V3/spacemap/SpaceMapSTDMap.h

@@ -57,16 +57,21 @@ class STDMapSpaceMap
         free_map.insert({start, end});
     }
 
-    void smapStats() override
+    String toDebugString() override
     {
         UInt64 count = 0;
 
-        LOG_FMT_DEBUG(log, "STD-Map entries status: ");
+        FmtBuffer fmt_buffer;
+        fmt_buffer.append("    STD-Map entries status: \n");
+
+        // Need use `count`,so can't use `joinStr` here.
         for (auto it = free_map.begin(); it != free_map.end(); it++)
         {
-            LOG_FMT_DEBUG(log, "  Space: {} start: {} size : {}", count, it->first, it->second);
+            fmt_buffer.fmtAppend("      Space: {} start: {} size : {}\n", count, it->first, it->second);
             count++;
         }
+
+        return fmt_buffer.toString();
     }
 
     std::pair<UInt64, UInt64> getSizes() const override

dbms/src/Storages/Page/V3/tests/CMakeLists.txt

@@ -26,4 +26,10 @@ add_executable(gtests_page_storage_v3 ${ps_v3_gtest_sources} ${TiFlash_SOURCE_DI
 target_link_libraries(gtests_page_storage_v3 page_storage_v3 gtest_main)
 target_compile_options(gtests_page_storage_v3 PRIVATE -Wno-unknown-pragmas)
 target_compile_definitions(gtests_page_storage_v3 PRIVATE DBMS_PUBLIC_GTEST)
-add_check(gtests_page_storage_v3)
+add_check(gtests_page_storage_v3)
+
+
+add_executable(page_storage_ctl EXCLUDE_FROM_ALL page_storage_ctl.cpp)
+target_compile_definitions(page_storage_ctl PUBLIC DBMS_PUBLIC_GTEST)
+target_link_libraries(page_storage_ctl dbms page_storage_v3)
+target_compile_options(page_storage_ctl PRIVATE -Wno-format -lc++) # turn off printf format check

dbms/src/Storages/Page/V3/tests/gtest_free_map.cpp

@@ -54,7 +54,7 @@ TEST_P(SpaceMapTest, InitAndDestory)
 {
     SpaceMapPtr smap = SpaceMap::createSpaceMap(test_type, 0, 100);
 
-    smap->logStats();
+    smap->logDebugString();
 }
 
 
@@ -256,11 +256,11 @@ TEST_P(SpaceMapTest, TestMargins2)
     // Right margin in marked used space
     // Left margin contain freed space
     ASSERT_FALSE(smap->markFree(49, 10));
-    smap->logStats();
+    smap->logDebugString();
     // Left margin align with marked used space left margin
     // But right margin contain freed space
     ASSERT_FALSE(smap->markFree(51, 20));
-    smap->logStats();
+    smap->logDebugString();
     // Right margin align with marked used space right margin
     // But left margin contain freed space
     ASSERT_FALSE(smap->markUsed(40, 19));