From f988991379ea5f4897c6a27caa27d91ca1e1e63f Mon Sep 17 00:00:00 2001 From: Doodle <13706157+critical27@users.noreply.github.com> Date: Mon, 5 Jul 2021 12:44:27 +0800 Subject: [PATCH 1/4] fix disk manager thread conflict --- src/kvstore/DiskManager.cpp | 2 ++ src/kvstore/DiskManager.h | 3 +++ src/kvstore/raftex/RaftPart.cpp | 8 +++++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/kvstore/DiskManager.cpp b/src/kvstore/DiskManager.cpp index daed01cb5..0264bc405 100644 --- a/src/kvstore/DiskManager.cpp +++ b/src/kvstore/DiskManager.cpp @@ -70,6 +70,7 @@ StatusOr DiskManager::path(GraphSpaceID spaceId, PartitionID partId void DiskManager::addPartToPath(GraphSpaceID spaceId, PartitionID partId, const std::string& path) { + std::lock_guard lg(lock_); try { auto canonical = boost::filesystem::canonical(path); auto dataPath = canonical.parent_path().parent_path(); @@ -85,6 +86,7 @@ void DiskManager::addPartToPath(GraphSpaceID spaceId, void DiskManager::removePartFromPath(GraphSpaceID spaceId, PartitionID partId, const std::string& path) { + std::lock_guard lg(lock_); try { auto canonical = boost::filesystem::canonical(path); auto dataPath = canonical.parent_path().parent_path(); diff --git a/src/kvstore/DiskManager.h b/src/kvstore/DiskManager.h index 9ea1b7e39..2632d74c7 100644 --- a/src/kvstore/DiskManager.h +++ b/src/kvstore/DiskManager.h @@ -65,6 +65,9 @@ class DiskManager { // the index in dataPaths_ for a given space + part std::unordered_map> partIndex_; + + // lock used to protect partPath_ and partIndex_ + std::mutex lock_; }; } // namespace kvstore diff --git a/src/kvstore/raftex/RaftPart.cpp b/src/kvstore/raftex/RaftPart.cpp index c1856afad..4be5ceeb7 100644 --- a/src/kvstore/raftex/RaftPart.cpp +++ b/src/kvstore/raftex/RaftPart.cpp @@ -768,7 +768,7 @@ void RaftPart::appendLogsInternal(AppendLogsIterator iter, TermID termId) { } while (false); if (!checkAppendLogResult(res)) { - LOG(ERROR) << idStr_ << "Failed append logs"; + LOG_EVERY_N(WARNING, 100) << idStr_ << "Failed to write wal"; return; } // Step 2: Replicate to followers @@ -805,7 +805,7 @@ void RaftPart::replicateLogs(folly::EventBase* eb, } while (false); if (!checkAppendLogResult(res)) { - LOG(ERROR) << idStr_ << "Replicate logs failed"; + LOG(WARNING) << idStr_ << "replicateLogs failed because of not leader or term changed"; return; } @@ -911,7 +911,9 @@ void RaftPart::processAppendLogResponses( } while (false); if (!checkAppendLogResult(res)) { - LOG(ERROR) << idStr_ << "processAppendLogResponses failed!"; + LOG(WARNING) + << idStr_ + << "processAppendLogResponses failed because of not leader or term changed"; return; } From 223f8a8a494904e3b5131a55d7be3bc2a976838b Mon Sep 17 00:00:00 2001 From: Doodle <13706157+critical27@users.noreply.github.com> Date: Mon, 5 Jul 2021 14:53:33 +0800 Subject: [PATCH 2/4] add default value of minimum_reserved_bytes --- conf/nebula-storaged.conf.default | 3 +++ conf/nebula-storaged.conf.production | 3 +++ 2 files changed, 6 insertions(+) diff --git a/conf/nebula-storaged.conf.default b/conf/nebula-storaged.conf.default index f45e3003e..e068ca92a 100644 --- a/conf/nebula-storaged.conf.default +++ b/conf/nebula-storaged.conf.default @@ -54,6 +54,9 @@ # One path per Rocksdb instance. --data_path=data/storage +# Minimum reserved bytes of each data path +--minimum_reserved_bytes=268435456 + # The default reserved bytes for one batch operation --rocksdb_batch_size=4096 # The default block cache size used in BlockBasedTable. diff --git a/conf/nebula-storaged.conf.production b/conf/nebula-storaged.conf.production index f44888e5f..712302ab8 100644 --- a/conf/nebula-storaged.conf.production +++ b/conf/nebula-storaged.conf.production @@ -54,6 +54,9 @@ # One path per Rocksdb instance. --data_path=data/storage +# Minimum reserved bytes of each data path +--minimum_reserved_bytes=268435456 + # The default reserved bytes for one batch operation --rocksdb_batch_size=4096 # The default block cache size used in BlockBasedTable. (MB) From caf703bdb1c5e24efac95d445c7ce51f24fdfee5 Mon Sep 17 00:00:00 2001 From: Doodle <13706157+critical27@users.noreply.github.com> Date: Tue, 6 Jul 2021 11:08:26 +0800 Subject: [PATCH 3/4] fix gcc 7.5 build warning --- src/kvstore/DiskManager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kvstore/DiskManager.cpp b/src/kvstore/DiskManager.cpp index 0264bc405..8ecb4af54 100644 --- a/src/kvstore/DiskManager.cpp +++ b/src/kvstore/DiskManager.cpp @@ -48,8 +48,8 @@ StatusOr> DiskManager::path(GraphSpaceID spaceId) { return Status::Error("Space not found"); } std::vector paths; - for (const auto& [path, _] : spaceIt->second) { - paths.emplace_back(path); + for (const auto& partEntry : spaceIt->second) { + paths.emplace_back(partEntry.first); } return paths; } From 3ca07109205b45cb6a03f6d095fa16b2dc91a0ea Mon Sep 17 00:00:00 2001 From: Doodle <13706157+critical27@users.noreply.github.com> Date: Tue, 6 Jul 2021 14:07:32 +0800 Subject: [PATCH 4/4] fix wal_path --- src/kvstore/NebulaStore.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kvstore/NebulaStore.cpp b/src/kvstore/NebulaStore.cpp index 392909c09..24cb8149f 100644 --- a/src/kvstore/NebulaStore.cpp +++ b/src/kvstore/NebulaStore.cpp @@ -338,7 +338,7 @@ std::shared_ptr NebulaStore::newPart(GraphSpaceID spaceId, KVEngine* engine, bool asLearner, const std::vector& defaultPeers) { - auto walPath = folly::stringPrintf("%s/wal/%d", engine->getDataRoot(), partId); + auto walPath = folly::stringPrintf("%s/wal/%d", engine->getWalRoot(), partId); auto part = std::make_shared(spaceId, partId, raftAddr_,