PageStorage: Report the file usage of BlobStore

dbms/src/Interpreters/AsynchronousMetrics.cpp

@@ -21,8 +21,12 @@
 #include <IO/UncompressedCache.h>
 #include <Interpreters/AsynchronousMetrics.h>
 #include <Storages/DeltaMerge/DeltaMergeStore.h>
+#include <Storages/DeltaMerge/StoragePool.h>
 #include <Storages/MarkCache.h>
+#include <Storages/Page/FileUsage.h>
 #include <Storages/StorageDeltaMerge.h>
+#include <Storages/Transaction/KVStore.h>
+#include <Storages/Transaction/TMTContext.h>
 #include <common/config_common.h>
 
 #include <chrono>
@@ -125,6 +129,26 @@ static void calculateMaxAndSum(Max & max, Sum & sum, T x)
         max = x;
 }
 
+FileUsageStatistics AsynchronousMetrics::getPageStorageFileUsage()
+{
+    // Get from RegionPersister
+    auto & tmt = context.getTMTContext();
+    auto & kvstore = tmt.getKVStore();
+    FileUsageStatistics usage = kvstore->getFileUsageStatistics();
+
+    // Get the blob file status from all PS V3 instances
+    if (auto global_storage_pool = context.getGlobalStoragePool(); global_storage_pool != nullptr)
+    {
+        const auto log_usage = global_storage_pool->log_storage->getFileUsageStatistics();
+        const auto meta_usage = global_storage_pool->meta_storage->getFileUsageStatistics();
+        const auto data_usage = global_storage_pool->data_storage->getFileUsageStatistics();
+
+        usage.total_file_num += log_usage.total_file_num + meta_usage.total_file_num + data_usage.total_file_num;
+        usage.total_disk_size += log_usage.total_disk_size + meta_usage.total_disk_size + data_usage.total_disk_size;
+        usage.total_valid_size += log_usage.total_valid_size + meta_usage.total_valid_size + data_usage.total_valid_size;
+    }
+    return usage;
+}
 
 void AsynchronousMetrics::update()
 {
@@ -147,6 +171,7 @@ void AsynchronousMetrics::update()
     set("Uptime", context.getUptimeSeconds());
 
     {
+        // Get the snapshot status from all delta tree tables
         auto databases = context.getDatabases();
 
         double max_dt_stable_oldest_snapshot_lifetime = 0.0;
@@ -177,6 +202,13 @@ void AsynchronousMetrics::update()
         set("MaxDTBackgroundTasksLength", max_dt_background_tasks_length);
     }
 
+    {
+        const FileUsageStatistics usage = getPageStorageFileUsage();
+        set("BlobFileNums", usage.total_file_num);
+        set("BlobDiskBytes", usage.total_disk_size);
+        set("BlobValidBytes", usage.total_valid_size);
+    }
+
 #if USE_TCMALLOC
     {
         /// tcmalloc related metrics. Remove if you switch to different allocator.

dbms/src/Interpreters/AsynchronousMetrics.h

@@ -14,6 +14,8 @@
 
 #pragma once
 
+#include <Storages/Page/FileUsage.h>
+
 #include <condition_variable>
 #include <mutex>
 #include <string>
@@ -47,6 +49,9 @@ class AsynchronousMetrics
     /// Returns copy of all values.
     Container getValues() const;
 
+private:
+    FileUsageStatistics getPageStorageFileUsage();
+
 private:
     Context & context;
 

dbms/src/Server/Server.cpp

@@ -464,7 +464,7 @@ struct RaftStoreProxyRunner : boost::noncopyable
     }
 
     RunRaftStoreProxyParms parms;
-    pthread_t thread;
+    pthread_t thread{};
     Poco::Logger * log;
 };
 
@@ -477,6 +477,11 @@ void initStores(Context & global_context, Poco::Logger * log, bool lazily_init_s
         int err_cnt = 0;
         for (auto & [table_id, storage] : storages)
         {
+            // This will skip the init of storages that do not contain any data. TiFlash now sync the schema and
+            // create all tables regardless the table have define TiFlash replica or not, so there may be lots
+            // of empty tables in TiFlash.
+            // Note that we still need to init stores that contains data (defined by the stable dir of this storage
+            // is exist), or the data used size reported to PD is not correct.
             try
             {
                 init_cnt += storage->initStoreIfDataDirExist() ? 1 : 0;
@@ -498,6 +503,7 @@ void initStores(Context & global_context, Poco::Logger * log, bool lazily_init_s
     if (lazily_init_store)
     {
         LOG_FMT_INFO(log, "Lazily init store.");
+        // apply the inited in another thread to shorten the start time of TiFlash
         std::thread(do_init_stores).detach();
     }
     else
@@ -1149,7 +1155,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
 
     /// Try to increase limit on number of open files.
     {
-        rlimit rlim;
+        rlimit rlim{};
         if (getrlimit(RLIMIT_NOFILE, &rlim))
             throw Poco::Exception("Cannot getrlimit");
 
@@ -1437,6 +1443,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
         }
 
         /// This object will periodically calculate some metrics.
+        /// should init after `createTMTContext` cause we collect some data from the TiFlash context object.
         AsynchronousMetrics async_metrics(*global_context);
         attachSystemTablesAsync(*global_context->getDatabase("system"), async_metrics);
 

dbms/src/Storages/DeltaMerge/StoragePool.h

@@ -28,6 +28,7 @@ struct Settings;
 class Context;
 class StoragePathPool;
 class StableDiskDelegator;
+class AsynchronousMetrics;
 
 namespace DM
 {
@@ -50,6 +51,7 @@ class GlobalStoragePool : private boost::noncopyable
     void restore();
 
     friend class StoragePool;
+    friend class ::DB::AsynchronousMetrics;
 
     // GC immediately
     // Only used on dbgFuncMisc

dbms/src/Storages/Page/FileUsage.h

@@ -0,0 +1,29 @@
+// Copyright 2022 PingCAP, Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include <stdint.h>
+
+#include <cstddef>
+
+namespace DB
+{
+struct FileUsageStatistics
+{
+    size_t total_disk_size = 0;
+    size_t total_valid_size = 0;
+    size_t total_file_num = 0;
+};
+
+} // namespace DB

dbms/src/Storages/Page/PageStorage.cpp

@@ -66,6 +66,8 @@ class PageReaderImpl : private boost::noncopyable
     // Get some statistics of all living snapshots and the oldest living snapshot.
     virtual SnapshotsStatistics getSnapshotsStat() const = 0;
 
+    virtual FileUsageStatistics getFileUsageStatistics() const = 0;
+
     virtual void traverse(const std::function<void(const DB::Page & page)> & acceptor, bool only_v2, bool only_v3) const = 0;
 };
 
@@ -137,6 +139,11 @@ class PageReaderImplNormal : public PageReaderImpl
         storage->traverse(acceptor, nullptr);
     }
 
+    FileUsageStatistics getFileUsageStatistics() const override
+    {
+        return storage->getFileUsageStatistics();
+    }
+
 private:
     NamespaceId ns_id;
     PageStoragePtr storage;
@@ -294,6 +301,11 @@ class PageReaderImplMixed : public PageReaderImpl
         return statistics_total;
     }
 
+    FileUsageStatistics getFileUsageStatistics() const override
+    {
+        return storage_v3->getFileUsageStatistics();
+    }
+
     void traverse(const std::function<void(const DB::Page & page)> & acceptor, bool only_v2, bool only_v3) const override
     {
         // Used by RegionPersister::restore
@@ -424,6 +436,12 @@ SnapshotsStatistics PageReader::getSnapshotsStat() const
     return impl->getSnapshotsStat();
 }
 
+
+FileUsageStatistics PageReader::getFileUsageStatistics() const
+{
+    return impl->getFileUsageStatistics();
+}
+
 void PageReader::traverse(const std::function<void(const DB::Page & page)> & acceptor, bool only_v2, bool only_v3) const
 {
     impl->traverse(acceptor, only_v2, only_v3);

dbms/src/Storages/Page/PageStorage.h

@@ -19,6 +19,7 @@
 #include <Interpreters/SettingsCommon.h>
 #include <Storages/FormatVersion.h>
 #include <Storages/Page/Config.h>
+#include <Storages/Page/FileUsage.h>
 #include <Storages/Page/Page.h>
 #include <Storages/Page/PageDefines.h>
 #include <Storages/Page/PageUtil.h>
@@ -251,6 +252,12 @@ class PageStorage : private boost::noncopyable
     // Get some statistics of all living snapshots and the oldest living snapshot.
     virtual SnapshotsStatistics getSnapshotsStat() const = 0;
 
+    virtual FileUsageStatistics getFileUsageStatistics() const
+    {
+        // return all zeros by default
+        return FileUsageStatistics{};
+    }
+
     virtual size_t getNumberOfPages() = 0;
 
     virtual std::set<PageId> getAliveExternalPageIds(NamespaceId ns_id) = 0;
@@ -380,6 +387,8 @@ class PageReader : private boost::noncopyable
     // Get some statistics of all living snapshots and the oldest living snapshot.
     SnapshotsStatistics getSnapshotsStat() const;
 
+    FileUsageStatistics getFileUsageStatistics() const;
+
     void traverse(const std::function<void(const DB::Page & page)> & acceptor, bool only_v2 = false, bool only_v3 = false) const;
 
 private:

dbms/src/Storages/Page/Snapshot.h

@@ -61,7 +61,7 @@ class PageStorageSnapshotMixed : public PageStorageSnapshot
 };
 using PageStorageSnapshotMixedPtr = std::shared_ptr<PageStorageSnapshotMixed>;
 
-static inline PageStorageSnapshotMixedPtr
+inline PageStorageSnapshotMixedPtr
 toConcreteMixedSnapshot(const PageStorageSnapshotPtr & ptr)
 {
     return std::static_pointer_cast<PageStorageSnapshotMixed>(ptr);

dbms/src/Storages/Page/V3/BlobStore.cpp

@@ -20,6 +20,7 @@
 #include <Common/StringUtils/StringUtils.h>
 #include <Common/TiFlashMetrics.h>
 #include <Poco/File.h>
+#include <Storages/Page/FileUsage.h>
 #include <Storages/Page/PageDefines.h>
 #include <Storages/Page/V3/BlobStore.h>
 #include <Storages/Page/V3/PageDirectory.h>
@@ -66,7 +67,7 @@ using ChecksumClass = Digest::CRC64;
   * BlobStore methods *
   *********************/
 
-BlobStore::BlobStore(String storage_name, const FileProviderPtr & file_provider_, PSDiskDelegatorPtr delegator_, BlobStore::Config config_)
+BlobStore::BlobStore(String storage_name, const FileProviderPtr & file_provider_, PSDiskDelegatorPtr delegator_, const BlobStore::Config & config_)
     : delegator(std::move(delegator_))
     , file_provider(file_provider_)
     , config(config_)
@@ -115,6 +116,38 @@ void BlobStore::registerPaths()
     }
 }
 
+FileUsageStatistics BlobStore::getFileUsageStatistics() const
+{
+    FileUsageStatistics usage;
+
+    // Get a copy of stats map to avoid the big lock on stats map
+    const auto stats_list = blob_stats.getStats();
+
+    for (const auto & [path, stats] : stats_list)
+    {
+        (void)path;
+        for (const auto & stat : stats)
+        {
+            // We can access to these type without any locking.
+            if (stat->isReadOnly() || stat->isBigBlob())
+            {
+                usage.total_disk_size += stat->sm_total_size;
+                usage.total_valid_size += stat->sm_valid_size;
+            }
+            else
+            {
+                // Else the stat may being updated, acquire a lock to avoid data race.
+                auto lock = stat->lock();
+                usage.total_disk_size += stat->sm_total_size;
+                usage.total_valid_size += stat->sm_valid_size;
+            }
+        }
+        usage.total_file_num += stats.size();
+    }
+
+    return usage;
+}
+
 PageEntriesEdit BlobStore::handleLargeWrite(DB::WriteBatch & wb, const WriteLimiterPtr & write_limiter)
 {
     auto ns_id = wb.getNamespaceId();
@@ -872,6 +905,7 @@ struct BlobStoreGCInfo
 
 std::vector<BlobFileId> BlobStore::getGCStats()
 {
+    // Get a copy of stats map to avoid the big lock on stats map
     const auto stats_list = blob_stats.getStats();
     std::vector<BlobFileId> blob_need_gc;
     BlobStoreGCInfo blobstore_gc_info;
@@ -1211,7 +1245,7 @@ BlobStatPtr BlobStore::BlobStats::createStat(BlobFileId blob_file_id, const std:
     // New blob file id won't bigger than roll_id
     if (blob_file_id > roll_id)
     {
-        throw Exception(fmt::format("BlobStats won't create [blob_id={}], which is bigger than [RollMaxId={}]",
+        throw Exception(fmt::format("BlobStats won't create [blob_id={}], which is bigger than [roll_id={}]",
                                     blob_file_id,
                                     roll_id),
                         ErrorCodes::LOGICAL_ERROR);
@@ -1274,8 +1308,7 @@ BlobStatPtr BlobStore::BlobStats::createBigPageStatNotChecking(BlobFileId blob_f
     BlobStatPtr stat = std::make_shared<BlobStat>(
         blob_file_id,
         SpaceMap::SpaceMapType::SMAP64_BIG,
-        config.file_limit_size,
-        BlobStatType::BIG_BLOB);
+        config.file_limit_size);
 
     PageFileIdAndLevel id_lvl{blob_file_id, 0};
     stats_map[delegator->choosePath(id_lvl)].emplace_back(stat);
@@ -1453,7 +1486,7 @@ bool BlobStore::BlobStats::BlobStat::removePosFromStat(BlobFileOffset offset, si
     if (!smap->markFree(offset, buf_size))
     {
         smap->logDebugString();
-        throw Exception(fmt::format("Remove postion from BlobStat failed, [offset={} , buf_size={}, blob_id={}] is invalid.",
+        throw Exception(fmt::format("Remove postion from BlobStat failed, invalid position [offset={}] [buf_size={}] [blob_id={}]",
                                     offset,
                                     buf_size,
                                     id),
@@ -1470,7 +1503,7 @@ void BlobStore::BlobStats::BlobStat::restoreSpaceMap(BlobFileOffset offset, size
     if (!smap->markUsed(offset, buf_size))
     {
         smap->logDebugString();
-        throw Exception(fmt::format("Restore postion from BlobStat failed, [offset={}] [buf_size={}] [blob_id={}] is used or subspan is used",
+        throw Exception(fmt::format("Restore postion from BlobStat failed, the space/subspace is already being used [offset={}] [buf_size={}] [blob_id={}]",
                                     offset,
                                     buf_size,
                                     id),