From e3c77dd765432d8bc88d6f520220e725c354ba58 Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 13:00:57 +0300 Subject: [PATCH 01/34] explicit type Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 61 +++++++++++++----------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index e6593725e3..898e9a328a 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -484,7 +484,7 @@ namespace kagome::blockchain { const primitives::BlockHash &BlockTreeImpl::getGenesisBlockHash() const { return block_tree_data_ .sharedAccess( - [&](const auto &p) + [&](const BlockTreeData &p) -> std::reference_wrapper { if (p.genesis_block_hash_.has_value()) { return p.genesis_block_hash_.value(); @@ -506,7 +506,7 @@ namespace kagome::blockchain { outcome::result BlockTreeImpl::addBlockHeader( const primitives::BlockHeader &header) { return block_tree_data_.exclusiveAccess( - [&](auto &p) -> outcome::result { + [&](BlockTreeData &p) -> outcome::result { auto parent = p.tree_->getRoot().findByHash(header.parent_hash); if (!parent) { return BlockTreeError::NO_PARENT; @@ -542,7 +542,7 @@ namespace kagome::blockchain { outcome::result BlockTreeImpl::addBlock( const primitives::Block &block) { return block_tree_data_.exclusiveAccess( - [&](auto &p) -> outcome::result { + [&](BlockTreeData &p) -> outcome::result { // Check if we know parent of this block; if not, we cannot insert it auto parent = p.tree_->getRoot().findByHash(block.header.parent_hash); if (!parent) { @@ -611,7 +611,7 @@ namespace kagome::blockchain { outcome::result BlockTreeImpl::removeLeaf( const primitives::BlockHash &block_hash) { return block_tree_data_.exclusiveAccess( - [&](auto &p) -> outcome::result { + [&](BlockTreeData &p) -> outcome::result { // Check if block is leaf if (p.tree_->getMetadata().leaves.count(block_hash) == 0) { return BlockTreeError::BLOCK_IS_NOT_LEAF; @@ -655,7 +655,7 @@ namespace kagome::blockchain { outcome::result BlockTreeImpl::markAsParachainDataBlock( const primitives::BlockHash &block_hash) { return block_tree_data_.exclusiveAccess( - [&](auto &p) -> outcome::result { + [&](BlockTreeData &p) -> outcome::result { SL_TRACE(log_, "Trying to adjust weight for block {}", block_hash); auto node = p.tree_->getRoot().findByHash(block_hash); @@ -825,7 +825,7 @@ namespace kagome::blockchain { const primitives::BlockHash &block_hash, const primitives::BlockHeader &block_header) { return block_tree_data_.exclusiveAccess( - [&](auto &p) -> outcome::result { + [&](BlockTreeData &p) -> outcome::result { return addExistingBlockNoLock(p, block_hash, block_header); }); } @@ -834,7 +834,7 @@ namespace kagome::blockchain { const primitives::BlockHash &block_hash, const primitives::BlockBody &body) { return block_tree_data_.exclusiveAccess( - [&](auto &p) -> outcome::result { + [&](BlockTreeData &p) -> outcome::result { return p.storage_->putBlockBody(block_hash, body); }); } @@ -842,7 +842,7 @@ namespace kagome::blockchain { outcome::result BlockTreeImpl::finalize( const primitives::BlockHash &block_hash, const primitives::Justification &justification) { - return block_tree_data_.exclusiveAccess([&](auto &p) + return block_tree_data_.exclusiveAccess([&](BlockTreeData &p) -> outcome::result { const auto node = p.tree_->getRoot().findByHash(block_hash); if (!node) { @@ -967,7 +967,7 @@ namespace kagome::blockchain { outcome::result> BlockTreeImpl::getBlockHash(primitives::BlockNumber block_number) const { return block_tree_data_.sharedAccess( - [&](const auto &p) + [&](const BlockTreeData &p) -> outcome::result> { OUTCOME_TRY(hash_opt, p.storage_->getBlockHash(block_number)); return hash_opt; @@ -976,8 +976,9 @@ namespace kagome::blockchain { outcome::result BlockTreeImpl::hasBlockHeader( const primitives::BlockHash &block_hash) const { - return block_tree_data_.sharedAccess( - [&](const auto &p) { return p.storage_->hasBlockHeader(block_hash); }); + return block_tree_data_.sharedAccess([&](const BlockTreeData &p) { + return p.storage_->hasBlockHeader(block_hash); + }); } outcome::result BlockTreeImpl::getBlockHeaderNoLock( @@ -992,7 +993,8 @@ namespace kagome::blockchain { outcome::result BlockTreeImpl::getBlockHeader( const primitives::BlockHash &block_hash) const { return block_tree_data_.sharedAccess( - [&](const auto &p) -> outcome::result { + [&](const BlockTreeData &p) + -> outcome::result { return getBlockHeaderNoLock(p, block_hash); }); } @@ -1000,7 +1002,7 @@ namespace kagome::blockchain { outcome::result BlockTreeImpl::getBlockBody( const primitives::BlockHash &block_hash) const { return block_tree_data_.sharedAccess( - [&](const auto &p) -> outcome::result { + [&](const BlockTreeData &p) -> outcome::result { OUTCOME_TRY(body, p.storage_->getBlockBody(block_hash)); if (body.has_value()) { return body.value(); @@ -1013,7 +1015,8 @@ namespace kagome::blockchain { BlockTreeImpl::getBlockJustification( const primitives::BlockHash &block_hash) const { return block_tree_data_.sharedAccess( - [&](const auto &p) -> outcome::result { + [&](const BlockTreeData &p) + -> outcome::result { OUTCOME_TRY(justification, p.storage_->getJustification(block_hash)); if (justification.has_value()) { return justification.value(); @@ -1024,7 +1027,7 @@ namespace kagome::blockchain { BlockTree::BlockHashVecRes BlockTreeImpl::getBestChainFromBlock( const primitives::BlockHash &block, uint64_t maximum) const { - return block_tree_data_.sharedAccess([&](const auto &p) + return block_tree_data_.sharedAccess([&](const BlockTreeData &p) -> BlockTree::BlockHashVecRes { auto block_number_res = p.header_repo_->getNumberByHash(block); if (block_number_res.has_error()) { @@ -1118,7 +1121,7 @@ namespace kagome::blockchain { BlockTree::BlockHashVecRes BlockTreeImpl::getDescendingChainToBlock( const primitives::BlockHash &to_block, uint64_t maximum) const { - return block_tree_data_.sharedAccess([&](const auto &p) { + return block_tree_data_.sharedAccess([&](const BlockTreeData &p) { return getDescendingChainToBlockNoLock(p, to_block, maximum); }); } @@ -1127,7 +1130,7 @@ namespace kagome::blockchain { const primitives::BlockHash &ancestor, const primitives::BlockHash &descendant) const { return block_tree_data_.sharedAccess( - [&](const auto &p) -> BlockTreeImpl::BlockHashVecRes { + [&](const BlockTreeData &p) -> BlockTreeImpl::BlockHashVecRes { OUTCOME_TRY(from, p.header_repo_->getNumberByHash(ancestor)); OUTCOME_TRY(to, p.header_repo_->getNumberByHash(descendant)); if (to < from) { @@ -1250,7 +1253,7 @@ namespace kagome::blockchain { bool BlockTreeImpl::hasDirectChain( const primitives::BlockHash &ancestor, const primitives::BlockHash &descendant) const { - return block_tree_data_.sharedAccess([&](const auto &p) { + return block_tree_data_.sharedAccess([&](const BlockTreeData &p) { return hasDirectChainNoLock(p, ancestor, descendant); }); } @@ -1272,13 +1275,13 @@ namespace kagome::blockchain { primitives::BlockInfo BlockTreeImpl::bestBlock() const { return block_tree_data_.sharedAccess( - [&](const auto &p) { return bestBlockNoLock(p); }); + [&](const BlockTreeData &p) { return bestBlockNoLock(p); }); } outcome::result BlockTreeImpl::getBestContaining( const primitives::BlockHash &target_hash) const { return block_tree_data_.sharedAccess( - [&](const auto &p) -> outcome::result { + [&](const BlockTreeData &p) -> outcome::result { auto &root = p.tree_->getRoot(); auto target = root.findByHash(target_hash); @@ -1345,12 +1348,12 @@ namespace kagome::blockchain { std::vector BlockTreeImpl::getLeaves() const { return block_tree_data_.sharedAccess( - [&](const auto &p) { return getLeavesNoLock(p); }); + [&](const BlockTreeData &p) { return getLeavesNoLock(p); }); } BlockTreeImpl::BlockHashVecRes BlockTreeImpl::getChildren( const primitives::BlockHash &block) const { - return block_tree_data_.sharedAccess([&](const auto &p) + return block_tree_data_.sharedAccess([&](const BlockTreeData &p) -> BlockTreeImpl::BlockHashVecRes { if (auto node = p.tree_->getRoot().findByHash(block); node != nullptr) { std::vector result; @@ -1381,7 +1384,7 @@ namespace kagome::blockchain { primitives::BlockInfo BlockTreeImpl::getLastFinalized() const { return block_tree_data_.sharedAccess( - [&](const auto &p) { return getLastFinalizedNoLock(p); }); + [&](const BlockTreeData &p) { return getLastFinalizedNoLock(p); }); } outcome::result BlockTreeImpl::pruneNoLock( @@ -1467,7 +1470,7 @@ namespace kagome::blockchain { retired_hashes{std::move(retired_hashes)}]() mutable { if (auto self = wself.lock()) { auto eo = self->block_tree_data_.sharedAccess( - [&](auto const &p) { return p.extrinsic_observer_; }); + [&](const BlockTreeData &p) { return p.extrinsic_observer_; }); for (auto &&extrinsic : extrinsics) { auto result = eo->onTxMessage(extrinsic); @@ -1585,10 +1588,12 @@ namespace kagome::blockchain { auto node = std::make_shared( block_info.hash, block_info.number, nullptr, true, false); auto meta = std::make_shared(node); - const auto leaves_size = block_tree_data_.exclusiveAccess([&](auto &p) { - p.tree_ = std::make_unique(std::move(node), std::move(meta)); - return p.tree_->getMetadata().leaves.size(); - }); + const auto leaves_size = + block_tree_data_.exclusiveAccess([&](BlockTreeData &p) { + p.tree_ = + std::make_unique(std::move(node), std::move(meta)); + return p.tree_->getMetadata().leaves.size(); + }); metric_known_chain_leaves_->set(leaves_size); metric_best_block_height_->set(block_info.number); From 802c67e85b764ba8422ad60560cb267f47fbdb99 Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 13:33:04 +0300 Subject: [PATCH 02/34] root constructor Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 8 ++------ core/blockchain/impl/cached_tree.cpp | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 898e9a328a..cd4f01725d 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -270,10 +270,7 @@ namespace kagome::blockchain { // Prepare and create block tree basing last finalized block auto tree = std::make_shared(last_finalized_block_info.hash, - last_finalized_block_info.number, - nullptr, - true, - false); + last_finalized_block_info.number); SL_DEBUG(log, "Last finalized block {}", tree->getBlockInfo()); auto meta = std::make_shared(tree); @@ -1585,8 +1582,7 @@ namespace kagome::blockchain { } void BlockTreeImpl::warp(const primitives::BlockInfo &block_info) { - auto node = std::make_shared( - block_info.hash, block_info.number, nullptr, true, false); + auto node = std::make_shared(block_info.hash, block_info.number); auto meta = std::make_shared(node); const auto leaves_size = block_tree_data_.exclusiveAccess([&](BlockTreeData &p) { diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index 686e9bee5a..9e8e0d5e15 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -42,7 +42,7 @@ namespace kagome::blockchain { finalized{finalized}, babe_primary{babe_primary}, contains_approved_para_block{false}, - reverted{parent ? parent->reverted : false} {} + reverted{parent->reverted} {} outcome::result TreeNode::applyToChain( const primitives::BlockInfo &chain_end, From 4977a4fe4aa9dfd944dfd932f678343e5c3161fa Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 13:35:00 +0300 Subject: [PATCH 03/34] warp mutex Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 25 +++++++++++------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index cd4f01725d..1e6c100f2c 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -1582,20 +1582,17 @@ namespace kagome::blockchain { } void BlockTreeImpl::warp(const primitives::BlockInfo &block_info) { - auto node = std::make_shared(block_info.hash, block_info.number); - auto meta = std::make_shared(node); - const auto leaves_size = - block_tree_data_.exclusiveAccess([&](BlockTreeData &p) { - p.tree_ = - std::make_unique(std::move(node), std::move(meta)); - return p.tree_->getMetadata().leaves.size(); - }); - - metric_known_chain_leaves_->set(leaves_size); - metric_best_block_height_->set(block_info.number); - telemetry_->notifyBlockFinalized(block_info); - telemetry_->pushBlockStats(); - metric_finalized_block_height_->set(block_info.number); + block_tree_data_.exclusiveAccess([&](BlockTreeData &p) { + auto node = + std::make_shared(block_info.hash, block_info.number); + auto meta = std::make_shared(node); + p.tree_ = std::make_unique(std::move(node), std::move(meta)); + metric_known_chain_leaves_->set(1); + metric_best_block_height_->set(block_info.number); + telemetry_->notifyBlockFinalized(block_info); + telemetry_->pushBlockStats(); + metric_finalized_block_height_->set(block_info.number); + }); } void BlockTreeImpl::notifyBestAndFinalized() { From b57405c4f2a1dee4be7ddfb9d57846f81fd01d59 Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 14:07:48 +0300 Subject: [PATCH 04/34] remove unfinalized Signed-off-by: turuslan --- core/blockchain/block_tree.hpp | 5 +++ core/blockchain/impl/block_tree_impl.cpp | 33 +++++++++++++++++ core/blockchain/impl/block_tree_impl.hpp | 2 ++ .../consensus/timeline/impl/timeline_impl.cpp | 35 +------------------ test/mock/core/blockchain/block_tree_mock.hpp | 2 ++ 5 files changed, 43 insertions(+), 34 deletions(-) diff --git a/core/blockchain/block_tree.hpp b/core/blockchain/block_tree.hpp index 82e161783e..521985368a 100644 --- a/core/blockchain/block_tree.hpp +++ b/core/blockchain/block_tree.hpp @@ -256,6 +256,11 @@ namespace kagome::blockchain { * Notify best and finalized block to subscriptions. */ virtual void notifyBestAndFinalized() = 0; + + /** + * Used when switching from fast-sync to full-sync. + */ + virtual void removeUnfinalized() = 0; }; } // namespace kagome::blockchain diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 1e6c100f2c..90f81bb982 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -1605,4 +1605,37 @@ namespace kagome::blockchain { chain_events_engine_->notify( primitives::events::ChainEventType::kFinalizedHeads, finalized_header); } + + void BlockTreeImpl::removeUnfinalized() { + auto r = block_tree_data_.exclusiveAccess( + [&](BlockTreeData &p) -> outcome::result { + auto finalized = p.tree_->getRoot().getBlockInfo(); + auto nodes = std::move(p.tree_->getRoot().children); + primitives::BlockNumber max = 0; + for (size_t i = 0; i < nodes.size(); ++i) { + auto children = std::move(nodes[i]->children); + for (auto &node : children) { + max = std::max(max, node->depth); + nodes.emplace_back(std::move(node)); + } + } + auto node = + std::make_shared(finalized.hash, finalized.number); + auto meta = std::make_shared(node); + p.tree_ = + std::make_unique(std::move(node), std::move(meta)); + OUTCOME_TRY(p.storage_->setBlockTreeLeaves({finalized.hash})); + for (auto i = max; i > finalized.number; --i) { + OUTCOME_TRY(p.storage_->deassignNumberToHash(i)); + } + std::reverse(nodes.begin(), nodes.end()); + for (auto &node : nodes) { + OUTCOME_TRY(p.storage_->removeBlock(node->block_hash)); + } + return outcome::success(); + }); + if (not r) { + SL_ERROR(log_, "removeUnfinalized error: {}", r.error()); + } + } } // namespace kagome::blockchain diff --git a/core/blockchain/impl/block_tree_impl.hpp b/core/blockchain/impl/block_tree_impl.hpp index 6b16f72d42..f94fa334d2 100644 --- a/core/blockchain/impl/block_tree_impl.hpp +++ b/core/blockchain/impl/block_tree_impl.hpp @@ -144,6 +144,8 @@ namespace kagome::blockchain { void notifyBestAndFinalized() override; + void removeUnfinalized() override; + private: struct BlockTreeData { std::shared_ptr header_repo_; diff --git a/core/consensus/timeline/impl/timeline_impl.cpp b/core/consensus/timeline/impl/timeline_impl.cpp index 80d5f644d6..ef7402df72 100644 --- a/core/consensus/timeline/impl/timeline_impl.cpp +++ b/core/consensus/timeline/impl/timeline_impl.cpp @@ -611,40 +611,7 @@ namespace kagome::consensus { "Rolling back non-finalized blocks. Last known finalized is {}", block_at_state); - // Next do-while-loop serves for removal non finalized blocks - bool affected; - do { - affected = false; - - auto block_tree_leaves = block_tree_->getLeaves(); - - for (const auto &hash : block_tree_leaves) { - if (hash == block_at_state.hash) { - continue; - } - - auto header_res = block_tree_->getBlockHeader(hash); - if (header_res.has_error()) { - SL_CRITICAL(log_, - "Can't get header of one of removing leave_block: {}", - header_res.error()); - continue; - } - - const auto &header = header_res.value(); - - // Block before last finalized must not be a leave. - // Don't touch it just in case - if (header.number < block_at_state.number) { - continue; - } - - std::ignore = consistency_keeper_->start( - primitives::BlockInfo(header.number, hash)); - - affected = true; - } - } while (affected); + block_tree_->removeUnfinalized(); SL_TRACE(log_, "Trying to sync state on block {} from {}", diff --git a/test/mock/core/blockchain/block_tree_mock.hpp b/test/mock/core/blockchain/block_tree_mock.hpp index 2cd1fc4874..e9fdaf9bbc 100644 --- a/test/mock/core/blockchain/block_tree_mock.hpp +++ b/test/mock/core/blockchain/block_tree_mock.hpp @@ -131,5 +131,7 @@ namespace kagome::blockchain { MOCK_METHOD(void, warp, (const primitives::BlockInfo &), (override)); MOCK_METHOD(void, notifyBestAndFinalized, (), (override)); + + MOCK_METHOD(void, removeUnfinalized, (), (override)); }; } // namespace kagome::blockchain From 80fbae4b3ef048edc4b06e34240a7e28eacf25a4 Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 14:09:34 +0300 Subject: [PATCH 05/34] remove unused Signed-off-by: turuslan --- core/blockchain/impl/cached_tree.cpp | 54 ----------------- core/blockchain/impl/cached_tree.hpp | 21 ------- test/core/blockchain/block_tree_test.cpp | 76 ------------------------ 3 files changed, 151 deletions(-) diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index 9e8e0d5e15..fd6145bba9 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -44,60 +44,6 @@ namespace kagome::blockchain { contains_approved_para_block{false}, reverted{parent->reverted} {} - outcome::result TreeNode::applyToChain( - const primitives::BlockInfo &chain_end, - const std::function(const TreeNode &node)> &op) - const { - using ChildIdx = size_t; - std::map fork_choice; - - const auto *current_node = findByHash(chain_end.hash).get(); - if (current_node == nullptr) { - return Error::NO_CHAIN_BETWEEN_BLOCKS; - } - - // mostly to catch typos in tests, but who knows - BOOST_ASSERT(current_node->depth == chain_end.number); - // now we must memorize where to go on forks in order to traverse - // from this to chain_end - while (current_node->depth > this->depth) { - auto curren_parent = current_node->parent.lock(); - BOOST_ASSERT(curren_parent != nullptr); - // if there's a fork at the parent, memorize which branch to take - if (curren_parent->children.size() > 1) { - const auto child_idx = std::find_if(curren_parent->children.begin(), - curren_parent->children.end(), - [current_node](auto &sptr) { - return sptr.get() == current_node; - }) - - curren_parent->children.begin(); - fork_choice[curren_parent->block_hash] = child_idx; - } - current_node = curren_parent.get(); - } - if (current_node->block_hash != this->block_hash) { - return Error::NO_CHAIN_BETWEEN_BLOCKS; - } - - current_node = this; - do { - OUTCOME_TRY(exit_token, op(*current_node)); - if (exit_token == ExitToken::EXIT) { - return outcome::success(); - } - if (current_node->children.size() == 1) { - current_node = current_node->children.front().get(); - } else if (current_node->children.size() > 1) { - auto child_idx = fork_choice[current_node->block_hash]; - current_node = current_node->children[child_idx].get(); - } else { - break; - } - } while (current_node->depth <= chain_end.number); - - return outcome::success(); - } - std::shared_ptr TreeNode::findByHash( const primitives::BlockHash &hash) const { // standard BFS diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index 9eabe80c81..a5a228f3c4 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -55,27 +55,6 @@ namespace kagome::blockchain { std::as_const(*this).findByHash(hash)); } - /** - * Exit token for applyToChain method. - * Simply a value denoting whether applyToChain should stop. - */ - enum class ExitToken { EXIT, CONTINUE }; - - /** - * Applies \arg op for each node starting from this and ending with \arg - * chain_end. If op returns success(EXIT), stops. - * @note if \arg op returns an error, execution will stop immediately and - * the rest of nodes will not be processed - * @note inefficient in case chain_end is not in chain with the current - * node. Will traverse the whole subtree trying to find it. - * @return error if chain doesn't exist or op() returns an error on a - * call, success otherwise - */ - outcome::result applyToChain( - const primitives::BlockInfo &chain_end, - const std::function(const TreeNode &node)> - &op) const; - primitives::BlockInfo getBlockInfo() const { return {depth, block_hash}; } diff --git a/test/core/blockchain/block_tree_test.cpp b/test/core/blockchain/block_tree_test.cpp index 2dda81161d..2337947140 100644 --- a/test/core/blockchain/block_tree_test.cpp +++ b/test/core/blockchain/block_tree_test.cpp @@ -631,82 +631,6 @@ struct NodeProcessor { MOCK_METHOD(void, foo, (const TreeNode &), (const)); }; -/** - * Call applyToChain targeting the rightmost leaf in the tree - * (so that the whole tree is traversed on its lookup) - */ -TEST_F(BlockTreeTest, TreeNode_applyToChain_lastLeaf) { - auto tree = makeFullTree(3, 2); - - NodeProcessor p; - EXPECT_CALL(p, foo(*tree)); - EXPECT_CALL(p, foo(*tree->children[1])); - EXPECT_CALL(p, foo(*tree->children[1]->children[1])); - - ASSERT_OUTCOME_SUCCESS_TRY(tree->applyToChain( - {2, tree->children[1]->children[1]->block_hash}, [&p](auto &node) { - p.foo(node); - return TreeNode::ExitToken::CONTINUE; - })); -} - -/** - * Call applyToChain targeting the tree root - */ -TEST_F(BlockTreeTest, TreeNode_applyToChain_root) { - auto tree = makeFullTree(3, 2); - - NodeProcessor p; - EXPECT_CALL(p, foo(*tree)); - - ASSERT_OUTCOME_SUCCESS_TRY( - tree->applyToChain({0, tree->block_hash}, [&p](auto &node) { - p.foo(node); - return TreeNode::ExitToken::CONTINUE; - })); -} - -/** - * Call apply to chain targeting a node not present in the tree - */ -TEST_F(BlockTreeTest, TreeNode_applyToChain_invalidNode) { - auto tree = makeFullTree(3, 2); - - // p.foo() should not be called - StrictMock p; - - ASSERT_OUTCOME_SOME_ERROR( - tree->applyToChain({42, "213232"_hash256}, [&p](auto &node) { - p.foo(node); - return outcome::success(TreeNode::ExitToken::CONTINUE); - })); -} - -/** - * Call apply to chain with a functor that return ExitToken::EXIT on the second - * processed node - */ -TEST_F(BlockTreeTest, TreeNode_applyToChain_exitTokenWorks) { - auto tree = makeFullTree(3, 2); - - NodeProcessor p; - EXPECT_CALL(p, foo(*tree)); - EXPECT_CALL(p, foo(*tree->children[1])); - // shouldn't be called because of exit token - // EXPECT_CALL(p, foo(*tree->children[1]->children[1])); - - size_t counter = 0; - ASSERT_OUTCOME_SUCCESS_TRY( - tree->applyToChain({2, tree->children[1]->children[1]->block_hash}, - [&p, &counter](auto &node) { - p.foo(node); - if (counter++ == 1) { - return TreeNode::ExitToken::EXIT; - } - return TreeNode::ExitToken::CONTINUE; - })); -} - /** * @given block tree with at least three blocks inside * @when asking for chain from the given block to top From 726aaeb45a132774fe771d11eddbe3e94109b7a8 Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 14:18:36 +0300 Subject: [PATCH 06/34] finalized has_justification Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 27 ++++++------------------ core/blockchain/impl/cached_tree.cpp | 3 --- core/blockchain/impl/cached_tree.hpp | 3 --- 3 files changed, 6 insertions(+), 27 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 90f81bb982..e9e92d6441 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -512,7 +512,7 @@ namespace kagome::blockchain { // update local meta with the new block auto new_node = std::make_shared( - block_hash, header.number, parent, false, isPrimary(header)); + block_hash, header.number, parent, isPrimary(header)); p.tree_->updateMeta(new_node); @@ -550,11 +550,8 @@ namespace kagome::blockchain { OUTCOME_TRY(block_hash, p.storage_->putBlock(block)); // Update local meta with the block - auto new_node = std::make_shared(block_hash, - block.header.number, - parent, - false, - isPrimary(block.header)); + auto new_node = std::make_shared( + block_hash, block.header.number, parent, isPrimary(block.header)); p.tree_->updateMeta(new_node); @@ -793,11 +790,8 @@ namespace kagome::blockchain { } // Update local meta with the block - auto new_node = std::make_shared(block_hash, - block_header.number, - parent, - false, - isPrimary(block_header)); + auto new_node = std::make_shared( + block_hash, block_header.number, parent, isPrimary(block_header)); p.tree_->updateMeta(new_node); @@ -863,10 +857,6 @@ namespace kagome::blockchain { OUTCOME_TRY(p.storage_->putJustification(justification, block_hash)); justification_stored = true; - // update our local meta - node->finalized = true; - node->has_justification = true; - OUTCOME_TRY(pruneNoLock(p, node)); OUTCOME_TRY(pruneTrie(p, node->getBlockInfo().number)); @@ -910,10 +900,6 @@ namespace kagome::blockchain { return outcome::success(); } else if (hasDirectChainNoLock( p, block_hash, last_finalized_block_info.hash)) { - if (node->has_justification) { - // block already has justification (in memory), fine - return outcome::success(); - } OUTCOME_TRY(justification_opt, p.storage_->getJustification(block_hash)); if (justification_opt.has_value()) { @@ -1390,8 +1376,7 @@ namespace kagome::blockchain { auto following_node = lastFinalizedNode; - for (auto current_node = following_node->parent.lock(); - current_node && !current_node->finalized; + for (auto current_node = following_node->parent.lock(); current_node; current_node = current_node->parent.lock()) { // DFS-on-deque to_remove.emplace_back(); // Waterbreak diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index fd6145bba9..8b6c26b40d 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -26,7 +26,6 @@ namespace kagome::blockchain { : block_hash{hash}, depth{depth}, parent{}, - finalized{true}, babe_primary{false}, contains_approved_para_block{false}, reverted{false} {} @@ -34,12 +33,10 @@ namespace kagome::blockchain { TreeNode::TreeNode(const primitives::BlockHash &hash, primitives::BlockNumber depth, const std::shared_ptr &parent, - bool finalized, bool babe_primary) : block_hash{hash}, depth{depth}, parent{parent}, - finalized{finalized}, babe_primary{babe_primary}, contains_approved_para_block{false}, reverted{parent->reverted} {} diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index a5a228f3c4..f10d839143 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -29,14 +29,11 @@ namespace kagome::blockchain { TreeNode(const primitives::BlockHash &hash, primitives::BlockNumber depth, const std::shared_ptr &parent, - bool finalized, bool babe_primary); primitives::BlockHash block_hash; primitives::BlockNumber depth; std::weak_ptr parent; - bool finalized; - bool has_justification = false; bool babe_primary; bool contains_approved_para_block; bool reverted; From f351cbb1dc93aeaa46ae4917bf186ee19ffd1fbf Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 14:21:23 +0300 Subject: [PATCH 07/34] unused Signed-off-by: turuslan --- core/blockchain/impl/cached_tree.cpp | 9 --------- core/blockchain/impl/cached_tree.hpp | 4 ---- 2 files changed, 13 deletions(-) diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index 8b6c26b40d..c8979fcd99 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -11,15 +11,6 @@ #include #include -OUTCOME_CPP_DEFINE_CATEGORY(kagome::blockchain, TreeNode::Error, e) { - using E = kagome::blockchain::TreeNode::Error; - switch (e) { - case E::NO_CHAIN_BETWEEN_BLOCKS: - return "no chain exists between given blocks"; - } - return "unknown error"; -} - namespace kagome::blockchain { TreeNode::TreeNode(const primitives::BlockHash &hash, primitives::BlockNumber depth) diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index f10d839143..3d3fb0e9bf 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -22,8 +22,6 @@ namespace kagome::blockchain { */ class TreeNode : public std::enable_shared_from_this { public: - enum class Error { NO_CHAIN_BETWEEN_BLOCKS = 1 }; - TreeNode(const primitives::BlockHash &hash, primitives::BlockNumber depth); TreeNode(const primitives::BlockHash &hash, @@ -150,5 +148,3 @@ namespace kagome::blockchain { std::shared_ptr metadata_; }; } // namespace kagome::blockchain - -OUTCOME_HPP_DECLARE_ERROR(kagome::blockchain, TreeNode::Error); From b24810bd193ab48e426ff701ac015dbf98d76ef6 Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 14:26:59 +0300 Subject: [PATCH 08/34] block weight Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 2 +- core/blockchain/impl/cached_tree.cpp | 32 +++++++---------------- core/blockchain/impl/cached_tree.hpp | 33 ++++-------------------- 3 files changed, 15 insertions(+), 52 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index e9e92d6441..d729f48dae 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -1307,7 +1307,7 @@ namespace kagome::blockchain { continue; } - if (metadata.getWeight(best) < metadata.getWeight(tree_node) + if (best->weight() < tree_node->weight() and hasDirectChainNoLock( p, target_hash, tree_node->block_hash)) { best = tree_node; diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index c8979fcd99..2f6cc3eee5 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -17,7 +17,7 @@ namespace kagome::blockchain { : block_hash{hash}, depth{depth}, parent{}, - babe_primary{false}, + babe_primary_weight{0}, contains_approved_para_block{false}, reverted{false} {} @@ -28,10 +28,15 @@ namespace kagome::blockchain { : block_hash{hash}, depth{depth}, parent{parent}, - babe_primary{babe_primary}, + babe_primary_weight{parent->babe_primary_weight + + (babe_primary ? 1 : 0)}, contains_approved_para_block{false}, reverted{parent->reverted} {} + BlockWeight TreeNode::weight() const { + return {babe_primary_weight, depth}; + } + std::shared_ptr TreeNode::findByHash( const primitives::BlockHash &hash) const { // standard BFS @@ -88,25 +93,6 @@ namespace kagome::blockchain { handle(subtree_root_node); } - TreeMeta::Weight TreeMeta::getWeight(std::shared_ptr node) const { - auto finalized = last_finalized.lock(); - BOOST_ASSERT(finalized); - Weight weight{WeightInfo(0ull), node->depth}; - while (node != finalized) { - BOOST_ASSERT(node->depth > finalized->depth); - if (node->babe_primary) { - ++weight.first.data.babe_primary; - } - if (node->contains_approved_para_block) { - ++weight.first.data.parachain_payload; - } - auto parent = node->parent.lock(); - BOOST_ASSERT(parent); - node = std::move(parent); - } - return weight; - } - bool TreeMeta::chooseBest(std::shared_ptr node) { if (node->reverted) { return false; @@ -114,7 +100,7 @@ namespace kagome::blockchain { auto best = best_block.lock(); BOOST_ASSERT(best); BOOST_ASSERT(not best->reverted); - if (getWeight(node) > getWeight(best)) { + if (node->weight() > best->weight()) { best_block = node; return true; } @@ -152,7 +138,7 @@ namespace kagome::blockchain { continue; } - if (getWeight(best) < getWeight(tree_node)) { + if (best->weight() < tree_node->weight()) { best = tree_node; } } diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index 3d3fb0e9bf..cb0b6d3f9e 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -15,6 +15,8 @@ #include "primitives/justification.hpp" namespace kagome::blockchain { + using BlockWeight = std::pair; + /** * In-memory light representation of the tree, used for efficiency and usage * convenience - we would only ask the database for some info, when directly @@ -32,12 +34,14 @@ namespace kagome::blockchain { primitives::BlockHash block_hash; primitives::BlockNumber depth; std::weak_ptr parent; - bool babe_primary; + uint32_t babe_primary_weight; bool contains_approved_para_block; bool reverted; std::vector> children{}; + BlockWeight weight() const; + /** * Get a node of the tree, containing block with the specified hash, if it * can be found @@ -63,35 +67,8 @@ namespace kagome::blockchain { * the operations faster */ struct TreeMeta { - union WeightInfo { - BOOST_STATIC_ASSERT( -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - false -#else //__BYTE_ORDER__ - true -#endif //__BYTE_ORDER__ - ); - - struct { - uint64_t parachain_payload : 48; - uint64_t babe_primary : 16; - } data; - uint64_t value; - - WeightInfo(uint64_t v) : value(v) {} - bool operator==(const WeightInfo &r) const { - return value == r.value; - } - bool operator<(const WeightInfo &r) const { - return value < r.value; - } - }; - using Weight = std::pair; - explicit TreeMeta(const std::shared_ptr &subtree_root_node); - Weight getWeight(std::shared_ptr node) const; - /** * Compare node weight with best and replace if heavier. * @return true if heavier and replaced. From 19cd0ea78712b2cf1c10914da0c20a8e6edfb3ad Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 14:39:56 +0300 Subject: [PATCH 09/34] fix test Signed-off-by: turuslan --- test/core/blockchain/block_tree_test.cpp | 35 ++++++------------------ 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/test/core/blockchain/block_tree_test.cpp b/test/core/blockchain/block_tree_test.cpp index 2337947140..34763e8aae 100644 --- a/test/core/blockchain/block_tree_test.cpp +++ b/test/core/blockchain/block_tree_test.cpp @@ -343,6 +343,9 @@ struct BlockTreeTest : public testing::Test { auto it = std::find_if(num_to_hash_.begin(), num_to_hash_.end(), [&](const auto &it) { return it.second == hash; }); + if (it == num_to_hash_.end()) { + return; + } num_to_hash_.erase(it); } void delNumToHash(BlockNumber number) { @@ -608,8 +611,7 @@ std::shared_ptr makeFullTree(size_t depth, size_t branching_factor) { auto &make_subtree) { primitives::BlockHash hash{}; std::copy_n(name.begin(), name.size(), hash.begin()); - auto node = - std::make_shared(hash, current_depth, parent, false, false); + auto node = std::make_shared(hash, current_depth, parent, false); if (current_depth + 1 == max_depth) { return node; } @@ -927,34 +929,15 @@ TEST_F(BlockTreeTest, GetBestBlock) { // --------------------------------------------------------------------------- - auto D1_hash = addHeaderToRepository(C1_hash, 47, SlotType::Primary); - ASSERT_OUTCOME_SUCCESS_TRY(block_tree_->markAsParachainDataBlock(D1_hash)); - - // 42 43 44 45 46 47 48 49 50 - // - // C1 - D1** - // / - // LF - T - A - B - C2 - D2 - E2* - // \ - // C3 - D3 - E3 - F3 - - { - ASSERT_OUTCOME_SUCCESS(best_info, block_tree_->getBestContaining(T_hash)); - ASSERT_EQ(best_info.hash, D1_hash); - } - - // --------------------------------------------------------------------------- - auto G3_hash = addHeaderToRepository(F3_hash, 50, SlotType::Primary); - ASSERT_OUTCOME_SUCCESS_TRY(block_tree_->markAsParachainDataBlock(G3_hash)); // 42 43 44 45 46 47 48 49 50 // - // C1 - D1** + // C1 // / // LF - T - A - B - C2 - D2 - E2* // \ - // C3 - D3 - E3 - F3 - G3*** + // C3 - D3 - E3 - F3 - G3** { ASSERT_OUTCOME_SUCCESS(best_info, block_tree_->getBestContaining(T_hash)); @@ -967,14 +950,14 @@ TEST_F(BlockTreeTest, GetBestBlock) { // 42 43 44 45 46 47 48 49 50 // - // C1 - D1** + // C1 // / // LF - T - A - B - C2 - D2 - E2* // \ - // C3 - D3 - E3 - F3 - G3*** + // C3 - D3 - E3 - F3 - G3** { ASSERT_OUTCOME_SUCCESS(best_info, block_tree_->getBestContaining(T_hash)); - ASSERT_EQ(best_info.hash, D1_hash); + ASSERT_EQ(best_info.hash, E2_hash); } } From 07df71557c9c8c4000080048850b7e022b0d105c Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 14:59:14 +0300 Subject: [PATCH 10/34] constructor Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 17 +++++++---------- core/blockchain/impl/cached_tree.cpp | 15 ++++++--------- core/blockchain/impl/cached_tree.hpp | 6 ++---- test/core/blockchain/block_tree_test.cpp | 3 ++- 4 files changed, 17 insertions(+), 24 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index d729f48dae..8e258d23d7 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -269,8 +269,7 @@ namespace kagome::blockchain { } // Prepare and create block tree basing last finalized block - auto tree = std::make_shared(last_finalized_block_info.hash, - last_finalized_block_info.number); + auto tree = std::make_shared(last_finalized_block_info); SL_DEBUG(log, "Last finalized block {}", tree->getBlockInfo()); auto meta = std::make_shared(tree); @@ -512,7 +511,7 @@ namespace kagome::blockchain { // update local meta with the new block auto new_node = std::make_shared( - block_hash, header.number, parent, isPrimary(header)); + header.blockInfo(), parent, isPrimary(header)); p.tree_->updateMeta(new_node); @@ -551,7 +550,7 @@ namespace kagome::blockchain { // Update local meta with the block auto new_node = std::make_shared( - block_hash, block.header.number, parent, isPrimary(block.header)); + block.header.blockInfo(), parent, isPrimary(block.header)); p.tree_->updateMeta(new_node); @@ -629,7 +628,7 @@ namespace kagome::blockchain { primitives::BlockInfo block{node->depth - 1, hash_opt.value()}; - auto tree = std::make_shared(block.hash, block.number); + auto tree = std::make_shared(block); auto meta = std::make_shared(tree); p.tree_ = std::make_unique(std::move(tree), std::move(meta)); @@ -791,7 +790,7 @@ namespace kagome::blockchain { // Update local meta with the block auto new_node = std::make_shared( - block_hash, block_header.number, parent, isPrimary(block_header)); + block_header.blockInfo(), parent, isPrimary(block_header)); p.tree_->updateMeta(new_node); @@ -1568,8 +1567,7 @@ namespace kagome::blockchain { void BlockTreeImpl::warp(const primitives::BlockInfo &block_info) { block_tree_data_.exclusiveAccess([&](BlockTreeData &p) { - auto node = - std::make_shared(block_info.hash, block_info.number); + auto node = std::make_shared(block_info); auto meta = std::make_shared(node); p.tree_ = std::make_unique(std::move(node), std::move(meta)); metric_known_chain_leaves_->set(1); @@ -1604,8 +1602,7 @@ namespace kagome::blockchain { nodes.emplace_back(std::move(node)); } } - auto node = - std::make_shared(finalized.hash, finalized.number); + auto node = std::make_shared(finalized); auto meta = std::make_shared(node); p.tree_ = std::make_unique(std::move(node), std::move(meta)); diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index 2f6cc3eee5..5fe9492750 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -12,21 +12,18 @@ #include namespace kagome::blockchain { - TreeNode::TreeNode(const primitives::BlockHash &hash, - primitives::BlockNumber depth) - : block_hash{hash}, - depth{depth}, - parent{}, + TreeNode::TreeNode(const primitives::BlockInfo &info) + : block_hash{info.hash}, + depth{info.number}, babe_primary_weight{0}, contains_approved_para_block{false}, reverted{false} {} - TreeNode::TreeNode(const primitives::BlockHash &hash, - primitives::BlockNumber depth, + TreeNode::TreeNode(const primitives::BlockInfo &info, const std::shared_ptr &parent, bool babe_primary) - : block_hash{hash}, - depth{depth}, + : block_hash{info.hash}, + depth{info.number}, parent{parent}, babe_primary_weight{parent->babe_primary_weight + (babe_primary ? 1 : 0)}, diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index cb0b6d3f9e..fb79f8b5e8 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -24,10 +24,8 @@ namespace kagome::blockchain { */ class TreeNode : public std::enable_shared_from_this { public: - TreeNode(const primitives::BlockHash &hash, primitives::BlockNumber depth); - - TreeNode(const primitives::BlockHash &hash, - primitives::BlockNumber depth, + TreeNode(const primitives::BlockInfo &info); + TreeNode(const primitives::BlockInfo &info, const std::shared_ptr &parent, bool babe_primary); diff --git a/test/core/blockchain/block_tree_test.cpp b/test/core/blockchain/block_tree_test.cpp index 34763e8aae..e8832c6ef9 100644 --- a/test/core/blockchain/block_tree_test.cpp +++ b/test/core/blockchain/block_tree_test.cpp @@ -611,7 +611,8 @@ std::shared_ptr makeFullTree(size_t depth, size_t branching_factor) { auto &make_subtree) { primitives::BlockHash hash{}; std::copy_n(name.begin(), name.size(), hash.begin()); - auto node = std::make_shared(hash, current_depth, parent, false); + auto node = std::make_shared( + primitives::BlockInfo{hash, current_depth}, parent, false); if (current_depth + 1 == max_depth) { return node; } From 8431645fcb5e4c2d7daa966578f56f75f573a8f3 Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 18:05:20 +0300 Subject: [PATCH 11/34] meta finalized Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 7 ++----- core/blockchain/impl/cached_tree.cpp | 4 ++++ core/blockchain/impl/cached_tree.hpp | 3 +++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 8e258d23d7..5852a688c0 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -454,8 +454,7 @@ namespace kagome::blockchain { metric_finalized_block_height_ = metrics_registry_->registerGaugeMetric( blockHeightMetricName, {{"status", "finalized"}}); - metric_finalized_block_height_->set( - p.tree_->getMetadata().last_finalized.lock()->depth); + metric_finalized_block_height_->set(p.tree_->finalized()->depth); metrics_registry_->registerGaugeFamily( knownChainLeavesMetricName, @@ -1359,9 +1358,7 @@ namespace kagome::blockchain { primitives::BlockInfo BlockTreeImpl::getLastFinalizedNoLock( const BlockTreeData &p) const { - const auto &last = p.tree_->getMetadata().last_finalized.lock(); - BOOST_ASSERT(last != nullptr); - return last->getBlockInfo(); + return p.tree_->finalized()->getBlockInfo(); } primitives::BlockInfo BlockTreeImpl::getLastFinalized() const { diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index 5fe9492750..a0a014e3cd 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -224,4 +224,8 @@ namespace kagome::blockchain { } } } + + const std::shared_ptr &CachedTree::finalized() const { + return root_; + } } // namespace kagome::blockchain diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index fb79f8b5e8..53868b032b 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -93,6 +93,9 @@ namespace kagome::blockchain { BOOST_ASSERT(root_ != nullptr); BOOST_ASSERT(metadata_ != nullptr); } + + const std::shared_ptr &finalized() const; + /** * Remove nodes in block tree from current tree_ to {\arg new_trie_root}. * Needed to avoid cascade shared_ptr destructor calls which break From 3b89aeecc942645e09f06e3dea73b6e288e98491 Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 18:06:21 +0300 Subject: [PATCH 12/34] meta best Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 18 ++++++------------ core/blockchain/impl/cached_tree.cpp | 5 +++++ core/blockchain/impl/cached_tree.hpp | 2 ++ 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 5852a688c0..c618c94c25 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -449,8 +449,7 @@ namespace kagome::blockchain { metric_best_block_height_ = metrics_registry_->registerGaugeMetric( blockHeightMetricName, {{"status", "best"}}); - metric_best_block_height_->set( - p.tree_->getMetadata().best_block.lock()->depth); + metric_best_block_height_->set(p.tree_->best()->depth); metric_finalized_block_height_ = metrics_registry_->registerGaugeMetric( blockHeightMetricName, {{"status", "finalized"}}); @@ -521,8 +520,7 @@ namespace kagome::blockchain { p.tree_->getMetadata().leaves.end()})); metric_known_chain_leaves_->set(p.tree_->getMetadata().leaves.size()); - metric_best_block_height_->set( - p.tree_->getMetadata().best_block.lock()->depth); + metric_best_block_height_->set(p.tree_->best()->depth); notifyChainEventsEngine(primitives::events::ChainEventType::kNewHeads, header); @@ -579,8 +577,7 @@ namespace kagome::blockchain { } metric_known_chain_leaves_->set(p.tree_->getMetadata().leaves.size()); - metric_best_block_height_->set( - p.tree_->getMetadata().best_block.lock()->depth); + metric_best_block_height_->set(p.tree_->best()->depth); SL_VERBOSE(log_, "Block {} has been added into block tree", @@ -800,8 +797,7 @@ namespace kagome::blockchain { p.tree_->getMetadata().leaves.end()})); metric_known_chain_leaves_->set(p.tree_->getMetadata().leaves.size()); - metric_best_block_height_->set( - p.tree_->getMetadata().best_block.lock()->depth); + metric_best_block_height_->set(p.tree_->best()->depth); SL_VERBOSE(log_, "Block {} has been restored in block tree from storage", @@ -1022,7 +1018,7 @@ namespace kagome::blockchain { return std::vector{block}; } - auto best_block = p.tree_->getMetadata().best_block.lock(); + auto best_block = p.tree_->best(); BOOST_ASSERT(best_block != nullptr); auto current_depth = best_block->depth; @@ -1249,9 +1245,7 @@ namespace kagome::blockchain { primitives::BlockInfo BlockTreeImpl::bestBlockNoLock( const BlockTreeData &p) const { - auto best_block = p.tree_->getMetadata().best_block.lock(); - BOOST_ASSERT(best_block != nullptr); - return best_block->getBlockInfo(); + return p.tree_->best()->getBlockInfo(); } primitives::BlockInfo BlockTreeImpl::bestBlock() const { diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index a0a014e3cd..d1b3a66959 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -228,4 +228,9 @@ namespace kagome::blockchain { const std::shared_ptr &CachedTree::finalized() const { return root_; } + + const std::shared_ptr &CachedTree::best() const { + (decltype(best_) &)best_ = metadata_->best_block.lock(); // TODO: refactor + return best_; + } } // namespace kagome::blockchain diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index 53868b032b..245546f52c 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -95,6 +95,7 @@ namespace kagome::blockchain { } const std::shared_ptr &finalized() const; + const std::shared_ptr &best() const; /** * Remove nodes in block tree from current tree_ to {\arg new_trie_root}. @@ -123,6 +124,7 @@ namespace kagome::blockchain { private: std::shared_ptr root_; + std::shared_ptr best_; std::shared_ptr metadata_; }; } // namespace kagome::blockchain From e43f846f92490793e342c827bb539b7ef369819e Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 18:13:16 +0300 Subject: [PATCH 13/34] meta leaf count Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 10 +++++----- core/blockchain/impl/cached_tree.cpp | 5 +++++ core/blockchain/impl/cached_tree.hpp | 1 + 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index c618c94c25..d34de224d7 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -461,7 +461,7 @@ namespace kagome::blockchain { metric_known_chain_leaves_ = metrics_registry_->registerGaugeMetric(knownChainLeavesMetricName); - metric_known_chain_leaves_->set(p.tree_->getMetadata().leaves.size()); + metric_known_chain_leaves_->set(p.tree_->leafCount()); telemetry_->setGenesisBlockHash(getGenesisBlockHash()); }); @@ -519,7 +519,7 @@ namespace kagome::blockchain { {p.tree_->getMetadata().leaves.begin(), p.tree_->getMetadata().leaves.end()})); - metric_known_chain_leaves_->set(p.tree_->getMetadata().leaves.size()); + metric_known_chain_leaves_->set(p.tree_->leafCount()); metric_best_block_height_->set(p.tree_->best()->depth); notifyChainEventsEngine(primitives::events::ChainEventType::kNewHeads, @@ -576,7 +576,7 @@ namespace kagome::blockchain { } } - metric_known_chain_leaves_->set(p.tree_->getMetadata().leaves.size()); + metric_known_chain_leaves_->set(p.tree_->leafCount()); metric_best_block_height_->set(p.tree_->best()->depth); SL_VERBOSE(log_, @@ -796,7 +796,7 @@ namespace kagome::blockchain { p.storage_->setBlockTreeLeaves({p.tree_->getMetadata().leaves.begin(), p.tree_->getMetadata().leaves.end()})); - metric_known_chain_leaves_->set(p.tree_->getMetadata().leaves.size()); + metric_known_chain_leaves_->set(p.tree_->leafCount()); metric_best_block_height_->set(p.tree_->best()->depth); SL_VERBOSE(log_, @@ -1313,7 +1313,7 @@ namespace kagome::blockchain { std::vector BlockTreeImpl::getLeavesNoLock( const BlockTreeData &p) const { std::vector result; - result.reserve(p.tree_->getMetadata().leaves.size()); + result.reserve(p.tree_->leafCount()); std::transform(p.tree_->getMetadata().leaves.begin(), p.tree_->getMetadata().leaves.end(), std::back_inserter(result), diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index d1b3a66959..bcba79572f 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -233,4 +233,9 @@ namespace kagome::blockchain { (decltype(best_) &)best_ = metadata_->best_block.lock(); // TODO: refactor return best_; } + + size_t CachedTree::leafCount() const { + auto &leaves_ = metadata_->leaves; // TODO: refactor + return leaves_.size(); + } } // namespace kagome::blockchain diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index 245546f52c..aa0c21c9dc 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -96,6 +96,7 @@ namespace kagome::blockchain { const std::shared_ptr &finalized() const; const std::shared_ptr &best() const; + size_t leafCount() const; /** * Remove nodes in block tree from current tree_ to {\arg new_trie_root}. From 8796ad032723d90b5c25fcfafb9615870e25fcfc Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 18:15:55 +0300 Subject: [PATCH 14/34] meta leaf hashes Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 28 +++++------------------- core/blockchain/impl/cached_tree.cpp | 5 +++++ core/blockchain/impl/cached_tree.hpp | 1 + 3 files changed, 12 insertions(+), 22 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index d34de224d7..466ec2828a 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -515,9 +515,7 @@ namespace kagome::blockchain { OUTCOME_TRY(reorganizeNoLock(p)); - OUTCOME_TRY(p.storage_->setBlockTreeLeaves( - {p.tree_->getMetadata().leaves.begin(), - p.tree_->getMetadata().leaves.end()})); + OUTCOME_TRY(p.storage_->setBlockTreeLeaves(p.tree_->leafHashes())); metric_known_chain_leaves_->set(p.tree_->leafCount()); metric_best_block_height_->set(p.tree_->best()->depth); @@ -553,9 +551,7 @@ namespace kagome::blockchain { OUTCOME_TRY(reorganizeNoLock(p)); - OUTCOME_TRY(p.storage_->setBlockTreeLeaves( - {p.tree_->getMetadata().leaves.begin(), - p.tree_->getMetadata().leaves.end()})); + OUTCOME_TRY(p.storage_->setBlockTreeLeaves(p.tree_->leafHashes())); notifyChainEventsEngine(primitives::events::ChainEventType::kNewHeads, block.header); @@ -633,9 +629,7 @@ namespace kagome::blockchain { // Remove from storage OUTCOME_TRY(p.storage_->removeBlock(node->block_hash)); - OUTCOME_TRY(p.storage_->setBlockTreeLeaves( - {p.tree_->getMetadata().leaves.begin(), - p.tree_->getMetadata().leaves.end()})); + OUTCOME_TRY(p.storage_->setBlockTreeLeaves(p.tree_->leafHashes())); return outcome::success(); }); @@ -792,9 +786,7 @@ namespace kagome::blockchain { OUTCOME_TRY(reorganizeNoLock(p)); - OUTCOME_TRY( - p.storage_->setBlockTreeLeaves({p.tree_->getMetadata().leaves.begin(), - p.tree_->getMetadata().leaves.end()})); + OUTCOME_TRY(p.storage_->setBlockTreeLeaves(p.tree_->leafHashes())); metric_known_chain_leaves_->set(p.tree_->leafCount()); metric_best_block_height_->set(p.tree_->best()->depth); @@ -858,9 +850,7 @@ namespace kagome::blockchain { OUTCOME_TRY(reorganizeNoLock(p)); - OUTCOME_TRY(p.storage_->setBlockTreeLeaves( - {p.tree_->getMetadata().leaves.begin(), - p.tree_->getMetadata().leaves.end()})); + OUTCOME_TRY(p.storage_->setBlockTreeLeaves(p.tree_->leafHashes())); notifyChainEventsEngine( primitives::events::ChainEventType::kFinalizedHeads, header); @@ -1312,13 +1302,7 @@ namespace kagome::blockchain { std::vector BlockTreeImpl::getLeavesNoLock( const BlockTreeData &p) const { - std::vector result; - result.reserve(p.tree_->leafCount()); - std::transform(p.tree_->getMetadata().leaves.begin(), - p.tree_->getMetadata().leaves.end(), - std::back_inserter(result), - [](const auto &hash) { return hash; }); - return result; + return p.tree_->leafHashes(); } std::vector BlockTreeImpl::getLeaves() const { diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index bcba79572f..b20e6adb72 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -238,4 +238,9 @@ namespace kagome::blockchain { auto &leaves_ = metadata_->leaves; // TODO: refactor return leaves_.size(); } + + std::vector CachedTree::leafHashes() const { + auto &leaves_ = metadata_->leaves; // TODO: refactor + return {leaves_.begin(), leaves_.end()}; + } } // namespace kagome::blockchain diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index aa0c21c9dc..9784d0ef15 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -97,6 +97,7 @@ namespace kagome::blockchain { const std::shared_ptr &finalized() const; const std::shared_ptr &best() const; size_t leafCount() const; + std::vector leafHashes() const; /** * Remove nodes in block tree from current tree_ to {\arg new_trie_root}. From a38ecd53e6ad895b094bb5ec57451283aaa351a7 Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 18:19:02 +0300 Subject: [PATCH 15/34] reuse Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 466ec2828a..5a9a4db8fc 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -453,7 +453,7 @@ namespace kagome::blockchain { metric_finalized_block_height_ = metrics_registry_->registerGaugeMetric( blockHeightMetricName, {{"status", "finalized"}}); - metric_finalized_block_height_->set(p.tree_->finalized()->depth); + metric_finalized_block_height_->set(getLastFinalizedNoLock(p).number); metrics_registry_->registerGaugeFamily( knownChainLeavesMetricName, From 6364ec3bfa987f02739983893f6d16aded17e060 Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 18:21:15 +0300 Subject: [PATCH 16/34] reuse Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 5a9a4db8fc..0bf41c4eeb 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -449,7 +449,7 @@ namespace kagome::blockchain { metric_best_block_height_ = metrics_registry_->registerGaugeMetric( blockHeightMetricName, {{"status", "best"}}); - metric_best_block_height_->set(p.tree_->best()->depth); + metric_best_block_height_->set(bestBlockNoLock(p).number); metric_finalized_block_height_ = metrics_registry_->registerGaugeMetric( blockHeightMetricName, {{"status", "finalized"}}); @@ -518,7 +518,7 @@ namespace kagome::blockchain { OUTCOME_TRY(p.storage_->setBlockTreeLeaves(p.tree_->leafHashes())); metric_known_chain_leaves_->set(p.tree_->leafCount()); - metric_best_block_height_->set(p.tree_->best()->depth); + metric_best_block_height_->set(bestBlockNoLock(p).number); notifyChainEventsEngine(primitives::events::ChainEventType::kNewHeads, header); @@ -573,7 +573,7 @@ namespace kagome::blockchain { } metric_known_chain_leaves_->set(p.tree_->leafCount()); - metric_best_block_height_->set(p.tree_->best()->depth); + metric_best_block_height_->set(bestBlockNoLock(p).number); SL_VERBOSE(log_, "Block {} has been added into block tree", @@ -789,7 +789,7 @@ namespace kagome::blockchain { OUTCOME_TRY(p.storage_->setBlockTreeLeaves(p.tree_->leafHashes())); metric_known_chain_leaves_->set(p.tree_->leafCount()); - metric_best_block_height_->set(p.tree_->best()->depth); + metric_best_block_height_->set(bestBlockNoLock(p).number); SL_VERBOSE(log_, "Block {} has been restored in block tree from storage", From 4b130abfd9b7b888c6643cb4c1f9ec2b0badb9a6 Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 18:43:41 +0300 Subject: [PATCH 17/34] meta is leaf Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 3 ++- core/blockchain/impl/cached_tree.cpp | 5 +++++ core/blockchain/impl/cached_tree.hpp | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 0bf41c4eeb..4795044422 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -598,7 +598,7 @@ namespace kagome::blockchain { return block_tree_data_.exclusiveAccess( [&](BlockTreeData &p) -> outcome::result { // Check if block is leaf - if (p.tree_->getMetadata().leaves.count(block_hash) == 0) { + if (p.tree_->isLeaf(block_hash)) { return BlockTreeError::BLOCK_IS_NOT_LEAF; } @@ -1269,6 +1269,7 @@ namespace kagome::blockchain { auto metadata = p.tree_->getMetadata(); + // TODO(turuslan): comparator std::set> candidates; for (auto &leaf : metadata.leaves) { if (auto node = target->findByHash(leaf)) { diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index b20e6adb72..3a00fd0942 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -243,4 +243,9 @@ namespace kagome::blockchain { auto &leaves_ = metadata_->leaves; // TODO: refactor return {leaves_.begin(), leaves_.end()}; } + + bool CachedTree::isLeaf(const primitives::BlockHash &hash) const { + auto &leaves_ = metadata_->leaves; // TODO: refactor + return leaves_.find(hash) != leaves_.end(); + } } // namespace kagome::blockchain diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index 9784d0ef15..264d330676 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -98,6 +98,7 @@ namespace kagome::blockchain { const std::shared_ptr &best() const; size_t leafCount() const; std::vector leafHashes() const; + bool isLeaf(const primitives::BlockHash &hash) const; /** * Remove nodes in block tree from current tree_ to {\arg new_trie_root}. From 5995ba99a548724fbaceaa100bb75e342a39c093 Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 18:53:52 +0300 Subject: [PATCH 18/34] cmp Signed-off-by: turuslan --- core/blockchain/impl/cached_tree.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index 3a00fd0942..e33e0d8d68 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -104,16 +104,17 @@ namespace kagome::blockchain { return false; } + struct Cmp { + bool operator()(const std::shared_ptr &lhs, + const std::shared_ptr &rhs) const { + BOOST_ASSERT(lhs and rhs); + return std::tie(lhs->depth, lhs->block_hash) + > std::tie(rhs->depth, rhs->block_hash); + } + }; + void TreeMeta::forceRefreshBest() { auto root = last_finalized.lock(); - struct Cmp { - bool operator()(const std::shared_ptr &lhs, - const std::shared_ptr &rhs) const { - BOOST_ASSERT(lhs and rhs); - return lhs->depth < rhs->depth - or (lhs->depth == rhs->depth and lhs->block_hash < rhs->block_hash); - } - }; std::set, Cmp> candidates; for (auto &leaf : leaves) { @@ -124,7 +125,7 @@ namespace kagome::blockchain { auto best = std::move(root); while (not candidates.empty()) { - auto node = candidates.extract((++candidates.rbegin()).base()); + auto node = candidates.extract(candidates.begin()); BOOST_ASSERT(not node.empty()); auto &tree_node = node.value(); From 8f600648b309c4e6da2f0571dd0fb0e024743020 Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 19:05:55 +0300 Subject: [PATCH 19/34] can descend Signed-off-by: turuslan --- core/blockchain/impl/cached_tree.cpp | 20 ++++++++++++++++++++ core/blockchain/impl/cached_tree.hpp | 3 +++ 2 files changed, 23 insertions(+) diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index e33e0d8d68..2e2f6800d8 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -66,6 +66,26 @@ namespace kagome::blockchain { return !(*this == other); } + template + bool descend(std::shared_ptr from, + const std::shared_ptr &to, + const F &f) { + while (from != to) { + if (from->depth <= to->depth) { + return false; + } + f(from); + from = from->parent.lock(); + BOOST_ASSERT(from); + } + return true; + } + + bool canDescend(std::shared_ptr from, + const std::shared_ptr &to) { + return descend(from, to, [](const std::shared_ptr) {}); + } + TreeMeta::TreeMeta(const std::shared_ptr &subtree_root_node) : best_block{subtree_root_node}, last_finalized{subtree_root_node} { std::function)> handle = diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index 264d330676..00073a29e4 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -60,6 +60,9 @@ namespace kagome::blockchain { bool operator!=(const TreeNode &other) const; }; + bool canDescend(std::shared_ptr from, + const std::shared_ptr &to); + /** * Useful information about the tree & blocks it contains to make some of * the operations faster From 0289defad53c4e97f476b235006ff5bcf7d98c18 Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 19:08:24 +0300 Subject: [PATCH 20/34] best with Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 35 +++--------------------- core/blockchain/impl/cached_tree.cpp | 29 ++++++++++++++++++++ core/blockchain/impl/cached_tree.hpp | 2 ++ 3 files changed, 35 insertions(+), 31 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 4795044422..e76b3bbe3e 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -1247,6 +1247,9 @@ namespace kagome::blockchain { const primitives::BlockHash &target_hash) const { return block_tree_data_.sharedAccess( [&](const BlockTreeData &p) -> outcome::result { + if (getLastFinalizedNoLock(p).hash == target_hash) { + return bestBlockNoLock(p); + } auto &root = p.tree_->getRoot(); auto target = root.findByHash(target_hash); @@ -1267,37 +1270,7 @@ namespace kagome::blockchain { return bestBlockNoLock(p); } - auto metadata = p.tree_->getMetadata(); - - // TODO(turuslan): comparator - std::set> candidates; - for (auto &leaf : metadata.leaves) { - if (auto node = target->findByHash(leaf)) { - candidates.emplace(std::move(node)); - } - } - - auto best = target; - while (not candidates.empty()) { - auto node = candidates.extract((++candidates.rbegin()).base()); - BOOST_ASSERT(not node.empty()); - - auto &tree_node = node.value(); - if (tree_node->reverted) { - if (auto parent = tree_node->parent.lock()) { - candidates.emplace(std::move(parent)); - } - continue; - } - - if (best->weight() < tree_node->weight() - and hasDirectChainNoLock( - p, target_hash, tree_node->block_hash)) { - best = tree_node; - } - } - - return best->getBlockInfo(); + return p.tree_->bestWith(target); }); } diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index 2e2f6800d8..ac67b4113a 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -269,4 +269,33 @@ namespace kagome::blockchain { auto &leaves_ = metadata_->leaves; // TODO: refactor return leaves_.find(hash) != leaves_.end(); } + + primitives::BlockInfo CachedTree::bestWith( + const std::shared_ptr &required) const { + auto &leaves_ = metadata_->leaves; // TODO: refactor + std::set, Cmp> candidates; + for (auto &leaf : leaves_) { + if (auto node = required->findByHash(leaf)) { + candidates.emplace(std::move(node)); + } + } + auto best = required; + while (not candidates.empty()) { + auto _node = candidates.extract(candidates.begin()); + auto &node = _node.value(); + if (node->depth <= required->depth) { + continue; + } + if (node->reverted) { + auto parent = node->parent.lock(); + BOOST_ASSERT(parent); + candidates.emplace(std::move(parent)); + continue; + } + if (node->weight() > best->weight() and canDescend(node, required)) { + best = node; + } + } + return best->getBlockInfo(); + } } // namespace kagome::blockchain diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index 00073a29e4..a7f709d689 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -102,6 +102,8 @@ namespace kagome::blockchain { size_t leafCount() const; std::vector leafHashes() const; bool isLeaf(const primitives::BlockHash &hash) const; + primitives::BlockInfo bestWith( + const std::shared_ptr &required) const; /** * Remove nodes in block tree from current tree_ to {\arg new_trie_root}. From b1b99a8534d73790b2b5e7954ecfb1ab3698ec4e Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 19:28:33 +0300 Subject: [PATCH 21/34] remove leaf Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index e76b3bbe3e..eacae69184 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -598,7 +598,8 @@ namespace kagome::blockchain { return block_tree_data_.exclusiveAccess( [&](BlockTreeData &p) -> outcome::result { // Check if block is leaf - if (p.tree_->isLeaf(block_hash)) { + if (block_hash == getLastFinalizedNoLock(p).hash + or p.tree_->isLeaf(block_hash)) { return BlockTreeError::BLOCK_IS_NOT_LEAF; } @@ -606,25 +607,9 @@ namespace kagome::blockchain { BOOST_ASSERT_MSG(node != nullptr, "As checked before, block exists as one of leaves"); - if (not node->parent.expired()) { - // Remove from block tree, ... - p.tree_->removeFromMeta(node); + p.tree_->removeFromMeta(node); - OUTCOME_TRY(reorganizeNoLock(p)); - - } else { - // ... or repair tree by parent of root - OUTCOME_TRY(hash_opt, p.storage_->getBlockHash(node->depth - 1)); - BOOST_ASSERT_MSG(hash_opt.has_value(), - "Non genesis block must have parent"); - - primitives::BlockInfo block{node->depth - 1, hash_opt.value()}; - - auto tree = std::make_shared(block); - auto meta = std::make_shared(tree); - p.tree_ = - std::make_unique(std::move(tree), std::move(meta)); - } + OUTCOME_TRY(reorganizeNoLock(p)); // Remove from storage OUTCOME_TRY(p.storage_->removeBlock(node->block_hash)); From 091ff94b45c2abbac1d489033fe6353faa129324 Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 19:35:35 +0300 Subject: [PATCH 22/34] remove meta Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 39 ++++------ core/blockchain/impl/cached_tree.cpp | 93 +++++++----------------- core/blockchain/impl/cached_tree.hpp | 40 +++------- 3 files changed, 50 insertions(+), 122 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index eacae69184..3f265f2483 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -269,22 +269,20 @@ namespace kagome::blockchain { } // Prepare and create block tree basing last finalized block - auto tree = std::make_shared(last_finalized_block_info); - SL_DEBUG(log, "Last finalized block {}", tree->getBlockInfo()); - auto meta = std::make_shared(tree); - - std::shared_ptr block_tree( - new BlockTreeImpl(std::move(header_repo), - std::move(storage), - std::make_unique(tree, meta), - std::move(extrinsic_observer), - std::move(hasher), - std::move(chain_events_engine), - std::move(extrinsic_events_engine), - std::move(extrinsic_event_key_repo), - std::move(justification_storage_policy), - state_pruner, - std::move(io_context))); + SL_DEBUG(log, "Last finalized block {}", last_finalized_block_info); + + std::shared_ptr block_tree(new BlockTreeImpl( + std::move(header_repo), + std::move(storage), + std::make_unique(last_finalized_block_info), + std::move(extrinsic_observer), + std::move(hasher), + std::move(chain_events_engine), + std::move(extrinsic_events_engine), + std::move(extrinsic_event_key_repo), + std::move(justification_storage_policy), + state_pruner, + std::move(io_context))); // Add non-finalized block to the block tree for (auto &e : collected) { @@ -1501,9 +1499,7 @@ namespace kagome::blockchain { void BlockTreeImpl::warp(const primitives::BlockInfo &block_info) { block_tree_data_.exclusiveAccess([&](BlockTreeData &p) { - auto node = std::make_shared(block_info); - auto meta = std::make_shared(node); - p.tree_ = std::make_unique(std::move(node), std::move(meta)); + p.tree_ = std::make_unique(block_info); metric_known_chain_leaves_->set(1); metric_best_block_height_->set(block_info.number); telemetry_->notifyBlockFinalized(block_info); @@ -1536,10 +1532,7 @@ namespace kagome::blockchain { nodes.emplace_back(std::move(node)); } } - auto node = std::make_shared(finalized); - auto meta = std::make_shared(node); - p.tree_ = - std::make_unique(std::move(node), std::move(meta)); + p.tree_ = std::make_unique(finalized); OUTCOME_TRY(p.storage_->setBlockTreeLeaves({finalized.hash})); for (auto i = max; i > finalized.number; --i) { OUTCOME_TRY(p.storage_->deassignNumberToHash(i)); diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index ac67b4113a..56bf7191ab 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -86,39 +86,13 @@ namespace kagome::blockchain { return descend(from, to, [](const std::shared_ptr) {}); } - TreeMeta::TreeMeta(const std::shared_ptr &subtree_root_node) - : best_block{subtree_root_node}, last_finalized{subtree_root_node} { - std::function)> handle = - [&](std::shared_ptr node) { - // avoid deep recursion - while (node->children.size() == 1) { - node = node->children.front(); - } - - // is a leaf - if (node->children.empty()) { - leaves.emplace(node->block_hash); - chooseBest(node); - } else { - // follow descendants recursively - for (const auto &child : node->children) { - handle(child); - } - } - }; - - handle(subtree_root_node); - } - - bool TreeMeta::chooseBest(std::shared_ptr node) { + bool CachedTree::chooseBest(std::shared_ptr node) { if (node->reverted) { return false; } - auto best = best_block.lock(); - BOOST_ASSERT(best); - BOOST_ASSERT(not best->reverted); - if (node->weight() > best->weight()) { - best_block = node; + BOOST_ASSERT(not best_->reverted); + if (node->weight() > best_->weight()) { + best_ = node; return true; } return false; @@ -133,17 +107,15 @@ namespace kagome::blockchain { } }; - void TreeMeta::forceRefreshBest() { - auto root = last_finalized.lock(); - + void CachedTree::forceRefreshBest() { std::set, Cmp> candidates; - for (auto &leaf : leaves) { - if (auto node = root->findByHash(leaf)) { + for (auto &leaf : leaves_) { + if (auto node = root_->findByHash(leaf)) { candidates.emplace(std::move(node)); } } - auto best = std::move(root); + best_ = root_; while (not candidates.empty()) { auto node = candidates.extract(candidates.begin()); BOOST_ASSERT(not node.empty()); @@ -156,12 +128,10 @@ namespace kagome::blockchain { continue; } - if (best->weight() < tree_node->weight()) { - best = tree_node; + if (best_->weight() < tree_node->weight()) { + best_ = tree_node; } } - - best_block = best; } void CachedTree::updateTreeRoot(std::shared_ptr new_trie_root) { @@ -179,7 +149,6 @@ namespace kagome::blockchain { prev_node = prev_node->parent.lock(); } - metadata_ = std::make_shared(root_); root_->parent.reset(); } @@ -193,22 +162,13 @@ namespace kagome::blockchain { return *root_; } - const TreeMeta &CachedTree::getMetadata() const { - BOOST_ASSERT(metadata_ != nullptr); - return *metadata_; - } - void CachedTree::updateMeta(const std::shared_ptr &new_node) { auto parent = new_node->parent.lock(); parent->children.push_back(new_node); - metadata_->leaves.insert(new_node->block_hash); - metadata_->leaves.erase(parent->block_hash); - metadata_->chooseBest(new_node); - } - - void CachedTree::forceRefreshBest() { - metadata_->forceRefreshBest(); + leaves_.insert(new_node->block_hash); + leaves_.erase(parent->block_hash); + chooseBest(new_node); } void CachedTree::removeFromMeta(const std::shared_ptr &node) { @@ -223,56 +183,53 @@ namespace kagome::blockchain { parent->children.erase(it); } - metadata_->leaves.erase(node->block_hash); + leaves_.erase(node->block_hash); if (parent->children.empty()) { - metadata_->leaves.insert(parent->block_hash); + leaves_.insert(parent->block_hash); } - auto best = metadata_->best_block.lock(); - BOOST_ASSERT(best); - if (node == best) { - metadata_->best_block = parent; - for (auto it = metadata_->leaves.begin(); - it != metadata_->leaves.end();) { + if (node == best_) { + best_ = parent; + for (auto it = leaves_.begin(); it != leaves_.end();) { const auto &hash = *it++; const auto leaf_node = root_->findByHash(hash); if (leaf_node == nullptr) { // Already removed with removed subtree - metadata_->leaves.erase(hash); - } else if (metadata_->chooseBest(leaf_node)) { + leaves_.erase(hash); + } else if (chooseBest(leaf_node)) { break; } } } } + CachedTree::CachedTree(const primitives::BlockInfo &root) + : root_{std::make_shared(root)}, + best_{root_}, + leaves_{root.hash} {} + const std::shared_ptr &CachedTree::finalized() const { return root_; } const std::shared_ptr &CachedTree::best() const { - (decltype(best_) &)best_ = metadata_->best_block.lock(); // TODO: refactor return best_; } size_t CachedTree::leafCount() const { - auto &leaves_ = metadata_->leaves; // TODO: refactor return leaves_.size(); } std::vector CachedTree::leafHashes() const { - auto &leaves_ = metadata_->leaves; // TODO: refactor return {leaves_.begin(), leaves_.end()}; } bool CachedTree::isLeaf(const primitives::BlockHash &hash) const { - auto &leaves_ = metadata_->leaves; // TODO: refactor return leaves_.find(hash) != leaves_.end(); } primitives::BlockInfo CachedTree::bestWith( const std::shared_ptr &required) const { - auto &leaves_ = metadata_->leaves; // TODO: refactor std::set, Cmp> candidates; for (auto &leaf : leaves_) { if (auto node = required->findByHash(leaf)) { diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index a7f709d689..52e21ac8dc 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -63,39 +63,12 @@ namespace kagome::blockchain { bool canDescend(std::shared_ptr from, const std::shared_ptr &to); - /** - * Useful information about the tree & blocks it contains to make some of - * the operations faster - */ - struct TreeMeta { - explicit TreeMeta(const std::shared_ptr &subtree_root_node); - - /** - * Compare node weight with best and replace if heavier. - * @return true if heavier and replaced. - */ - bool chooseBest(std::shared_ptr node); - - /// Force find and update actual best block - void forceRefreshBest(); - - std::unordered_set leaves; - std::weak_ptr best_block; - - std::weak_ptr last_finalized; - }; - /** * Non-finalized part of block tree */ class CachedTree { public: - explicit CachedTree(std::shared_ptr root, - std::shared_ptr metadata) - : root_{std::move(root)}, metadata_{std::move(metadata)} { - BOOST_ASSERT(root_ != nullptr); - BOOST_ASSERT(metadata_ != nullptr); - } + explicit CachedTree(const primitives::BlockInfo &root); const std::shared_ptr &finalized() const; const std::shared_ptr &best() const; @@ -115,6 +88,7 @@ namespace kagome::blockchain { void updateMeta(const std::shared_ptr &new_node); + /// Force find and update actual best block void forceRefreshBest(); /** @@ -128,11 +102,15 @@ namespace kagome::blockchain { const TreeNode &getRoot() const; TreeNode &getRoot(); - const TreeMeta &getMetadata() const; - private: + /** + * Compare node weight with best and replace if heavier. + * @return true if heavier and replaced. + */ + bool chooseBest(std::shared_ptr node); + std::shared_ptr root_; std::shared_ptr best_; - std::shared_ptr metadata_; + std::unordered_set leaves_; }; } // namespace kagome::blockchain From 90a4e7eb07a56b86830125843396e297d27c5a57 Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 19:46:10 +0300 Subject: [PATCH 23/34] weak ptr Signed-off-by: turuslan --- core/utils/wptr.hpp | 47 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 core/utils/wptr.hpp diff --git a/core/utils/wptr.hpp b/core/utils/wptr.hpp new file mode 100644 index 0000000000..5efb54e03d --- /dev/null +++ b/core/utils/wptr.hpp @@ -0,0 +1,47 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace kagome { + /** + * `bool operator==(std::weak_ptr, std::weak_ptr)`. + */ + template + bool wptrEq(const std::weak_ptr &l, const std::weak_ptr &r) { + return not l.owner_before(r) and not r.owner_before(l); + } + + /** + * `wptrEmpty(std::weak_ptr{}) == true`. + * `wptrEmpty(expired) == false`. + */ + template + bool wptrEmpty(const std::weak_ptr &w) { + return wptrEq(w, {}); + } + + /** + * Assert that `std::weak_ptr` is not expired. + */ + template + std::shared_ptr wptrLock(const std::weak_ptr &w) { + auto s = w.lock(); + [[unlikely]] if (not s and not wptrEmpty(w)) { throw std::bad_weak_ptr{}; } + return s; + } + + /** + * Assert that `std::weak_ptr` is neither empty nor expired. + */ + template + std::shared_ptr wptrMustLock(const std::weak_ptr &w) { + auto s = w.lock(); + [[unlikely]] if (not s) { throw std::bad_weak_ptr{}; } + return s; + } +} // namespace kagome From 50eb803ff3dd6591caa2ebf533d37d3b3c5569a5 Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 20:02:16 +0300 Subject: [PATCH 24/34] can descend Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 3f265f2483..c5632b00c3 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -1145,18 +1145,7 @@ namespace kagome::blockchain { // if both nodes are in our light tree, we can use this representation // only if (ancestor_node_ptr && descendant_node_ptr) { - auto current_node = descendant_node_ptr; - while (current_node != ancestor_node_ptr) { - if (current_node->depth <= ancestor_node_ptr->depth) { - return false; - } - if (auto parent = current_node->parent; !parent.expired()) { - current_node = parent.lock(); - } else { - return false; - } - } - return true; + return canDescend(descendant_node_ptr, ancestor_node_ptr); } // else, we need to use a database From 55948f7601f56df4718c90816227f637d80231ae Mon Sep 17 00:00:00 2001 From: turuslan Date: Mon, 9 Oct 2023 20:04:32 +0300 Subject: [PATCH 25/34] weak parent Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 6 ++-- core/blockchain/impl/cached_tree.cpp | 39 +++++++++++------------- core/blockchain/impl/cached_tree.hpp | 3 +- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index c5632b00c3..643a6f0c6b 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -1037,7 +1037,7 @@ namespace kagome::blockchain { // Try to retrieve from cached tree if (auto node = p.tree_->getRoot().findByHash(hash)) { while (maximum > chain.size()) { - auto parent = node->parent.lock(); + auto parent = node->parent(); if (not parent) { hash = node->block_hash; break; @@ -1296,8 +1296,8 @@ namespace kagome::blockchain { auto following_node = lastFinalizedNode; - for (auto current_node = following_node->parent.lock(); current_node; - current_node = current_node->parent.lock()) { + for (auto current_node = following_node->parent(); current_node; + current_node = current_node->parent()) { // DFS-on-deque to_remove.emplace_back(); // Waterbreak std::copy_if(current_node->children.begin(), diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index 56bf7191ab..c3e84da9be 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -7,10 +7,10 @@ #include "blockchain/impl/cached_tree.hpp" #include - -#include #include +#include "utils/wptr.hpp" + namespace kagome::blockchain { TreeNode::TreeNode(const primitives::BlockInfo &info) : block_hash{info.hash}, @@ -24,12 +24,16 @@ namespace kagome::blockchain { bool babe_primary) : block_hash{info.hash}, depth{info.number}, - parent{parent}, + weak_parent{parent}, babe_primary_weight{parent->babe_primary_weight + (babe_primary ? 1 : 0)}, contains_approved_para_block{false}, reverted{parent->reverted} {} + std::shared_ptr TreeNode::parent() const { + return wptrLock(weak_parent); + } + BlockWeight TreeNode::weight() const { return {babe_primary_weight, depth}; } @@ -53,13 +57,7 @@ namespace kagome::blockchain { } bool TreeNode::operator==(const TreeNode &other) const { - const auto &other_parent = other.parent; - auto parents_equal = (parent.expired() && other_parent.expired()) - || (!parent.expired() && !other_parent.expired() - && parent.lock() == other_parent.lock()); - - return parents_equal && block_hash == other.block_hash - && depth == other.depth; + return block_hash == other.block_hash && depth == other.depth; } bool TreeNode::operator!=(const TreeNode &other) const { @@ -75,8 +73,7 @@ namespace kagome::blockchain { return false; } f(from); - from = from->parent.lock(); - BOOST_ASSERT(from); + from = wptrMustLock(from->weak_parent); } return true; } @@ -122,7 +119,7 @@ namespace kagome::blockchain { auto &tree_node = node.value(); if (tree_node->reverted) { - if (auto parent = tree_node->parent.lock()) { + if (auto parent = tree_node->parent()) { candidates.emplace(std::move(parent)); } continue; @@ -136,7 +133,7 @@ namespace kagome::blockchain { void CachedTree::updateTreeRoot(std::shared_ptr new_trie_root) { auto prev_root = root_; - auto prev_node = new_trie_root->parent.lock(); + auto prev_node = new_trie_root->parent(); // now node won't be deleted while cleaning children root_ = std::move(new_trie_root); @@ -146,10 +143,10 @@ namespace kagome::blockchain { // recursive calls of shared pointer destructors break the stack while (prev_node && prev_node != prev_root) { prev_node->children.clear(); - prev_node = prev_node->parent.lock(); + prev_node = prev_node->parent(); } - root_->parent.reset(); + root_->weak_parent.reset(); } const TreeNode &CachedTree::getRoot() const { @@ -163,7 +160,7 @@ namespace kagome::blockchain { } void CachedTree::updateMeta(const std::shared_ptr &new_node) { - auto parent = new_node->parent.lock(); + auto parent = wptrMustLock(new_node->weak_parent); parent->children.push_back(new_node); leaves_.insert(new_node->block_hash); @@ -172,7 +169,7 @@ namespace kagome::blockchain { } void CachedTree::removeFromMeta(const std::shared_ptr &node) { - auto parent = node->parent.lock(); + auto parent = node->parent(); if (parent == nullptr) { // Already removed with removed subtree return; @@ -244,9 +241,9 @@ namespace kagome::blockchain { continue; } if (node->reverted) { - auto parent = node->parent.lock(); - BOOST_ASSERT(parent); - candidates.emplace(std::move(parent)); + if (auto parent = node->parent()) { + candidates.emplace(std::move(parent)); + } continue; } if (node->weight() > best->weight() and canDescend(node, required)) { diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index 52e21ac8dc..5f22cf69b0 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -31,13 +31,14 @@ namespace kagome::blockchain { primitives::BlockHash block_hash; primitives::BlockNumber depth; - std::weak_ptr parent; + std::weak_ptr weak_parent; uint32_t babe_primary_weight; bool contains_approved_para_block; bool reverted; std::vector> children{}; + std::shared_ptr parent() const; BlockWeight weight() const; /** From db8c3537100c172968f1a7ca4d6a185690f49cc2 Mon Sep 17 00:00:00 2001 From: turuslan Date: Tue, 10 Oct 2023 06:36:20 +0300 Subject: [PATCH 26/34] info Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 56 ++++++++++++------------ core/blockchain/impl/cached_tree.cpp | 29 ++++++------ core/blockchain/impl/cached_tree.hpp | 7 +-- 3 files changed, 42 insertions(+), 50 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 643a6f0c6b..5260151ec9 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -610,7 +610,7 @@ namespace kagome::blockchain { OUTCOME_TRY(reorganizeNoLock(p)); // Remove from storage - OUTCOME_TRY(p.storage_->removeBlock(node->block_hash)); + OUTCOME_TRY(p.storage_->removeBlock(node->info.hash)); OUTCOME_TRY(p.storage_->setBlockTreeLeaves(p.tree_->leafHashes())); @@ -657,7 +657,7 @@ namespace kagome::blockchain { reverting_tree_node->reverted = true; - if (reverting_tree_node->getBlockInfo() == best) { + if (reverting_tree_node->info == best) { need_to_refresh_best = true; } @@ -808,16 +808,16 @@ namespace kagome::blockchain { if (!node) { return BlockTreeError::NON_FINALIZED_BLOCK_NOT_FOUND; } - const auto block = node->getBlockInfo(); + const auto block = node->info; auto last_finalized_block_info = getLastFinalizedNoLock(p); auto justification_stored = false; - if (node->depth > last_finalized_block_info.number) { + if (node->info.number > last_finalized_block_info.number) { SL_DEBUG(log_, "Finalizing block {}", block); - OUTCOME_TRY(header_opt, p.storage_->getBlockHeader(node->block_hash)); + OUTCOME_TRY(header_opt, p.storage_->getBlockHeader(node->info.hash)); if (not header_opt.has_value()) { return BlockTreeError::HEADER_NOT_FOUND; } @@ -827,7 +827,7 @@ namespace kagome::blockchain { justification_stored = true; OUTCOME_TRY(pruneNoLock(p, node)); - OUTCOME_TRY(pruneTrie(p, node->getBlockInfo().number)); + OUTCOME_TRY(pruneTrie(p, node->info.number)); p.tree_->updateTreeRoot(node); @@ -838,7 +838,7 @@ namespace kagome::blockchain { notifyChainEventsEngine( primitives::events::ChainEventType::kFinalizedHeads, header); - OUTCOME_TRY(body, p.storage_->getBlockBody(node->block_hash)); + OUTCOME_TRY(body, p.storage_->getBlockBody(node->info.hash)); if (body.has_value()) { for (auto &ext : body.value()) { auto extrinsic_hash = p.hasher_->blake2b_256(ext.data); @@ -860,9 +860,9 @@ namespace kagome::blockchain { log_->info("Finalized block {}", block); telemetry_->notifyBlockFinalized(block); telemetry_->pushBlockStats(); - metric_finalized_block_height_->set(node->depth); + metric_finalized_block_height_->set(node->info.number); - } else if (node->block_hash == last_finalized_block_info.hash) { + } else if (node->info.hash == last_finalized_block_info.hash) { // block is current last finalized, fine return outcome::success(); } else if (hasDirectChainNoLock( @@ -882,7 +882,7 @@ namespace kagome::blockchain { } SL_DEBUG(log_, "Store justification for finalized block {}", block); - if (last_finalized_block_info.number < node->depth) { + if (last_finalized_block_info.number < node->info.number) { // we store justification for last finalized block only as long as it is // last finalized (if it doesn't meet other justification storage rules, // e.g. its number a multiple of 512) @@ -993,7 +993,7 @@ namespace kagome::blockchain { auto best_block = p.tree_->best(); BOOST_ASSERT(best_block != nullptr); - auto current_depth = best_block->depth; + auto current_depth = best_block->info.number; if (start_block_number >= current_depth) { return std::vector{block}; @@ -1039,10 +1039,10 @@ namespace kagome::blockchain { while (maximum > chain.size()) { auto parent = node->parent(); if (not parent) { - hash = node->block_hash; + hash = node->info.hash; break; } - chain.emplace_back(node->block_hash); + chain.emplace_back(node->info.hash); node = parent; } } @@ -1117,7 +1117,7 @@ namespace kagome::blockchain { primitives::BlockNumber ancestor_depth = 0u; primitives::BlockNumber descendant_depth = 0u; if (ancestor_node_ptr) { - ancestor_depth = ancestor_node_ptr->depth; + ancestor_depth = ancestor_node_ptr->info.number; } else { auto number_res = p.header_repo_->getNumberByHash(ancestor); if (!number_res) { @@ -1126,7 +1126,7 @@ namespace kagome::blockchain { ancestor_depth = number_res.value(); } if (descendant_node_ptr) { - descendant_depth = descendant_node_ptr->depth; + descendant_depth = descendant_node_ptr->info.number; } else { auto number_res = p.header_repo_->getNumberByHash(descendant); if (!number_res) { @@ -1207,7 +1207,7 @@ namespace kagome::blockchain { primitives::BlockInfo BlockTreeImpl::bestBlockNoLock( const BlockTreeData &p) const { - return p.tree_->best()->getBlockInfo(); + return p.tree_->best()->info; } primitives::BlockInfo BlockTreeImpl::bestBlock() const { @@ -1264,7 +1264,7 @@ namespace kagome::blockchain { std::vector result; result.reserve(node->children.size()); for (const auto &child : node->children) { - result.push_back(child->block_hash); + result.push_back(child->info.hash); } return result; } @@ -1282,7 +1282,7 @@ namespace kagome::blockchain { primitives::BlockInfo BlockTreeImpl::getLastFinalizedNoLock( const BlockTreeData &p) const { - return p.tree_->finalized()->getBlockInfo(); + return p.tree_->finalized()->info; } primitives::BlockInfo BlockTreeImpl::getLastFinalized() const { @@ -1327,8 +1327,8 @@ namespace kagome::blockchain { retired_hashes.reserve(to_remove.size()); for (const auto &node : to_remove) { OUTCOME_TRY(block_header_opt, - p.storage_->getBlockHeader(node->block_hash)); - OUTCOME_TRY(block_body_opt, p.storage_->getBlockBody(node->block_hash)); + p.storage_->getBlockHeader(node->info.hash)); + OUTCOME_TRY(block_body_opt, p.storage_->getBlockBody(node->info.hash)); if (block_body_opt.has_value()) { extrinsics.reserve(extrinsics.size() + block_body_opt.value().size()); for (auto &ext : block_body_opt.value()) { @@ -1336,7 +1336,7 @@ namespace kagome::blockchain { if (auto key = p.extrinsic_event_key_repo_->get(extrinsic_hash)) { main_thread_.execute([wself{weak_from_this()}, key{key.value()}, - block_hash{node->block_hash}]() { + block_hash{node->info.hash}]() { if (auto self = wself.lock()) { self->extrinsic_events_engine_->notify( key, @@ -1351,14 +1351,14 @@ namespace kagome::blockchain { OUTCOME_TRY(p.state_pruner_->pruneDiscarded(block_header_opt.value())); } - retired_hashes.emplace_back(node->block_hash); + retired_hashes.emplace_back(node->info.hash); p.tree_->removeFromMeta(node); - OUTCOME_TRY(p.storage_->removeBlock(node->block_hash)); + OUTCOME_TRY(p.storage_->removeBlock(node->info.hash)); } auto last_finalized_block_info = getLastFinalizedNoLock(p); for (primitives::BlockNumber n = last_finalized_block_info.number; - n < lastFinalizedNode->getBlockInfo().number; + n < lastFinalizedNode->info.number; ++n) { if (auto result = p.storage_->getBlockHash(n); result.has_value() && result.value()) { @@ -1403,7 +1403,7 @@ namespace kagome::blockchain { BOOST_ASSERT(!last_pruned.has_value() || last_pruned.value().number - <= block_tree_data.tree_->getRoot().depth); + <= block_tree_data.tree_->getRoot().info.number); auto next_pruned_number = last_pruned ? last_pruned->number + 1 : 0; OUTCOME_TRY(hash_opt, getBlockHash(next_pruned_number)); @@ -1511,13 +1511,13 @@ namespace kagome::blockchain { void BlockTreeImpl::removeUnfinalized() { auto r = block_tree_data_.exclusiveAccess( [&](BlockTreeData &p) -> outcome::result { - auto finalized = p.tree_->getRoot().getBlockInfo(); + auto finalized = p.tree_->getRoot().info; auto nodes = std::move(p.tree_->getRoot().children); primitives::BlockNumber max = 0; for (size_t i = 0; i < nodes.size(); ++i) { auto children = std::move(nodes[i]->children); for (auto &node : children) { - max = std::max(max, node->depth); + max = std::max(max, node->info.number); nodes.emplace_back(std::move(node)); } } @@ -1528,7 +1528,7 @@ namespace kagome::blockchain { } std::reverse(nodes.begin(), nodes.end()); for (auto &node : nodes) { - OUTCOME_TRY(p.storage_->removeBlock(node->block_hash)); + OUTCOME_TRY(p.storage_->removeBlock(node->info.hash)); } return outcome::success(); }); diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index c3e84da9be..0763517e22 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -13,8 +13,7 @@ namespace kagome::blockchain { TreeNode::TreeNode(const primitives::BlockInfo &info) - : block_hash{info.hash}, - depth{info.number}, + : info{info}, babe_primary_weight{0}, contains_approved_para_block{false}, reverted{false} {} @@ -22,8 +21,7 @@ namespace kagome::blockchain { TreeNode::TreeNode(const primitives::BlockInfo &info, const std::shared_ptr &parent, bool babe_primary) - : block_hash{info.hash}, - depth{info.number}, + : info{info}, weak_parent{parent}, babe_primary_weight{parent->babe_primary_weight + (babe_primary ? 1 : 0)}, @@ -35,7 +33,7 @@ namespace kagome::blockchain { } BlockWeight TreeNode::weight() const { - return {babe_primary_weight, depth}; + return {babe_primary_weight, info.number}; } std::shared_ptr TreeNode::findByHash( @@ -45,7 +43,7 @@ namespace kagome::blockchain { nodes_to_scan.push(shared_from_this()); while (!nodes_to_scan.empty()) { const auto &node = nodes_to_scan.front(); - if (node->block_hash == hash) { + if (node->info.hash == hash) { return node; } for (const auto &child : node->children) { @@ -57,7 +55,7 @@ namespace kagome::blockchain { } bool TreeNode::operator==(const TreeNode &other) const { - return block_hash == other.block_hash && depth == other.depth; + return info == other.info; } bool TreeNode::operator!=(const TreeNode &other) const { @@ -69,7 +67,7 @@ namespace kagome::blockchain { const std::shared_ptr &to, const F &f) { while (from != to) { - if (from->depth <= to->depth) { + if (from->info.number <= to->info.number) { return false; } f(from); @@ -99,8 +97,7 @@ namespace kagome::blockchain { bool operator()(const std::shared_ptr &lhs, const std::shared_ptr &rhs) const { BOOST_ASSERT(lhs and rhs); - return std::tie(lhs->depth, lhs->block_hash) - > std::tie(rhs->depth, rhs->block_hash); + return lhs->info > rhs->info; } }; @@ -163,8 +160,8 @@ namespace kagome::blockchain { auto parent = wptrMustLock(new_node->weak_parent); parent->children.push_back(new_node); - leaves_.insert(new_node->block_hash); - leaves_.erase(parent->block_hash); + leaves_.insert(new_node->info.hash); + leaves_.erase(parent->info.hash); chooseBest(new_node); } @@ -180,9 +177,9 @@ namespace kagome::blockchain { parent->children.erase(it); } - leaves_.erase(node->block_hash); + leaves_.erase(node->info.hash); if (parent->children.empty()) { - leaves_.insert(parent->block_hash); + leaves_.insert(parent->info.hash); } if (node == best_) { @@ -237,7 +234,7 @@ namespace kagome::blockchain { while (not candidates.empty()) { auto _node = candidates.extract(candidates.begin()); auto &node = _node.value(); - if (node->depth <= required->depth) { + if (node->info.number <= required->info.number) { continue; } if (node->reverted) { @@ -250,6 +247,6 @@ namespace kagome::blockchain { best = node; } } - return best->getBlockInfo(); + return best->info; } } // namespace kagome::blockchain diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index 5f22cf69b0..beaabebf47 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -29,8 +29,7 @@ namespace kagome::blockchain { const std::shared_ptr &parent, bool babe_primary); - primitives::BlockHash block_hash; - primitives::BlockNumber depth; + primitives::BlockInfo info; std::weak_ptr weak_parent; uint32_t babe_primary_weight; bool contains_approved_para_block; @@ -53,10 +52,6 @@ namespace kagome::blockchain { std::as_const(*this).findByHash(hash)); } - primitives::BlockInfo getBlockInfo() const { - return {depth, block_hash}; - } - bool operator==(const TreeNode &other) const; bool operator!=(const TreeNode &other) const; }; From d30fc38b7ecb948a55e55234b2edf824a34044c8 Mon Sep 17 00:00:00 2001 From: turuslan Date: Tue, 10 Oct 2023 06:37:53 +0300 Subject: [PATCH 27/34] unused Signed-off-by: turuslan --- core/blockchain/impl/cached_tree.cpp | 8 -------- core/blockchain/impl/cached_tree.hpp | 3 --- 2 files changed, 11 deletions(-) diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index 0763517e22..cfbd74c2cf 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -54,14 +54,6 @@ namespace kagome::blockchain { return nullptr; } - bool TreeNode::operator==(const TreeNode &other) const { - return info == other.info; - } - - bool TreeNode::operator!=(const TreeNode &other) const { - return !(*this == other); - } - template bool descend(std::shared_ptr from, const std::shared_ptr &to, diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index beaabebf47..bd146c6c76 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -51,9 +51,6 @@ namespace kagome::blockchain { return std::const_pointer_cast( std::as_const(*this).findByHash(hash)); } - - bool operator==(const TreeNode &other) const; - bool operator!=(const TreeNode &other) const; }; bool canDescend(std::shared_ptr from, From 359ce474f55ddd5b6e4ce31d72c46573ce47968b Mon Sep 17 00:00:00 2001 From: turuslan Date: Tue, 10 Oct 2023 06:49:49 +0300 Subject: [PATCH 28/34] find Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 31 ++++++++++++------------ core/blockchain/impl/cached_tree.cpp | 11 ++++++--- core/blockchain/impl/cached_tree.hpp | 1 + 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 5260151ec9..2b831f22c6 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -499,7 +499,7 @@ namespace kagome::blockchain { const primitives::BlockHeader &header) { return block_tree_data_.exclusiveAccess( [&](BlockTreeData &p) -> outcome::result { - auto parent = p.tree_->getRoot().findByHash(header.parent_hash); + auto parent = p.tree_->find(header.parent_hash); if (!parent) { return BlockTreeError::NO_PARENT; } @@ -533,7 +533,7 @@ namespace kagome::blockchain { return block_tree_data_.exclusiveAccess( [&](BlockTreeData &p) -> outcome::result { // Check if we know parent of this block; if not, we cannot insert it - auto parent = p.tree_->getRoot().findByHash(block.header.parent_hash); + auto parent = p.tree_->find(block.header.parent_hash); if (!parent) { return BlockTreeError::NO_PARENT; } @@ -601,7 +601,7 @@ namespace kagome::blockchain { return BlockTreeError::BLOCK_IS_NOT_LEAF; } - auto node = p.tree_->getRoot().findByHash(block_hash); + auto node = p.tree_->find(block_hash); BOOST_ASSERT_MSG(node != nullptr, "As checked before, block exists as one of leaves"); @@ -624,7 +624,7 @@ namespace kagome::blockchain { [&](BlockTreeData &p) -> outcome::result { SL_TRACE(log_, "Trying to adjust weight for block {}", block_hash); - auto node = p.tree_->getRoot().findByHash(block_hash); + auto node = p.tree_->find(block_hash); if (node == nullptr) { SL_WARN(log_, "Block {} doesn't exists in block tree", block_hash); return BlockTreeError::BLOCK_NOT_EXISTS; @@ -642,7 +642,7 @@ namespace kagome::blockchain { bool need_to_refresh_best = false; auto best = bestBlockNoLock(p); for (const auto &block_hash : block_hashes) { - auto tree_node = p.tree_->getRoot().findByHash(block_hash); + auto tree_node = p.tree_->find(block_hash); if (tree_node == nullptr) { SL_WARN( log_, "Block {} doesn't exists in block tree", block_hash); @@ -686,7 +686,7 @@ namespace kagome::blockchain { "Trying to add block {} into block tree", primitives::BlockInfo(block_header.number, block_hash)); - auto node = p.tree_->getRoot().findByHash(block_hash); + auto node = p.tree_->find(block_hash); // Check if tree doesn't have this block; if not, we skip that if (node != nullptr) { SL_TRACE(log_, @@ -695,7 +695,7 @@ namespace kagome::blockchain { return BlockTreeError::BLOCK_EXISTS; } - auto parent = p.tree_->getRoot().findByHash(block_header.parent_hash); + auto parent = p.tree_->find(block_header.parent_hash); // Check if we know parent of this block; if not, we cannot insert it if (parent == nullptr) { @@ -729,7 +729,7 @@ namespace kagome::blockchain { to_add.emplace(hash, std::move(header)); - if (p.tree_->getRoot().findByHash(header.parent_hash) != nullptr) { + if (p.tree_->find(header.parent_hash) != nullptr) { SL_TRACE(log_, "Block {} parent of {} has found in block tree", primitives::BlockInfo(*header.parentInfo()), @@ -752,7 +752,7 @@ namespace kagome::blockchain { to_add.pop(); } - parent = p.tree_->getRoot().findByHash(block_header.parent_hash); + parent = p.tree_->find(block_header.parent_hash); BOOST_ASSERT_MSG(parent != nullptr, "Parent must be restored at this moment"); @@ -804,7 +804,7 @@ namespace kagome::blockchain { const primitives::Justification &justification) { return block_tree_data_.exclusiveAccess([&](BlockTreeData &p) -> outcome::result { - const auto node = p.tree_->getRoot().findByHash(block_hash); + const auto node = p.tree_->find(block_hash); if (!node) { return BlockTreeError::NON_FINALIZED_BLOCK_NOT_FOUND; } @@ -1035,7 +1035,7 @@ namespace kagome::blockchain { auto hash = to_block; // Try to retrieve from cached tree - if (auto node = p.tree_->getRoot().findByHash(hash)) { + if (auto node = p.tree_->find(hash)) { while (maximum > chain.size()) { auto parent = node->parent(); if (not parent) { @@ -1104,8 +1104,8 @@ namespace kagome::blockchain { const BlockTreeData &p, const primitives::BlockHash &ancestor, const primitives::BlockHash &descendant) const { - auto ancestor_node_ptr = p.tree_->getRoot().findByHash(ancestor); - auto descendant_node_ptr = p.tree_->getRoot().findByHash(descendant); + auto ancestor_node_ptr = p.tree_->find(ancestor); + auto descendant_node_ptr = p.tree_->find(descendant); /* * check that ancestor is above descendant @@ -1222,9 +1222,8 @@ namespace kagome::blockchain { if (getLastFinalizedNoLock(p).hash == target_hash) { return bestBlockNoLock(p); } - auto &root = p.tree_->getRoot(); - auto target = root.findByHash(target_hash); + auto target = p.tree_->find(target_hash); // If target has not found in block tree (in memory), // it means block finalized or discarded @@ -1260,7 +1259,7 @@ namespace kagome::blockchain { const primitives::BlockHash &block) const { return block_tree_data_.sharedAccess([&](const BlockTreeData &p) -> BlockTreeImpl::BlockHashVecRes { - if (auto node = p.tree_->getRoot().findByHash(block); node != nullptr) { + if (auto node = p.tree_->find(block); node != nullptr) { std::vector result; result.reserve(node->children.size()); for (const auto &child : node->children) { diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index cfbd74c2cf..eef73abe0a 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -96,7 +96,7 @@ namespace kagome::blockchain { void CachedTree::forceRefreshBest() { std::set, Cmp> candidates; for (auto &leaf : leaves_) { - if (auto node = root_->findByHash(leaf)) { + if (auto node = find(leaf)) { candidates.emplace(std::move(node)); } } @@ -178,7 +178,7 @@ namespace kagome::blockchain { best_ = parent; for (auto it = leaves_.begin(); it != leaves_.end();) { const auto &hash = *it++; - const auto leaf_node = root_->findByHash(hash); + const auto leaf_node = find(hash); if (leaf_node == nullptr) { // Already removed with removed subtree leaves_.erase(hash); @@ -218,7 +218,7 @@ namespace kagome::blockchain { const std::shared_ptr &required) const { std::set, Cmp> candidates; for (auto &leaf : leaves_) { - if (auto node = required->findByHash(leaf)) { + if (auto node = find(leaf)) { candidates.emplace(std::move(node)); } } @@ -241,4 +241,9 @@ namespace kagome::blockchain { } return best->info; } + + std::shared_ptr CachedTree::find( + const primitives::BlockHash &hash) const { + return root_->findByHash(hash); + } } // namespace kagome::blockchain diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index bd146c6c76..c2351bc748 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -70,6 +70,7 @@ namespace kagome::blockchain { bool isLeaf(const primitives::BlockHash &hash) const; primitives::BlockInfo bestWith( const std::shared_ptr &required) const; + std::shared_ptr find(const primitives::BlockHash &hash) const; /** * Remove nodes in block tree from current tree_ to {\arg new_trie_root}. From d7f20ed70104d3ba9294f0af8c2ce19a3292042e Mon Sep 17 00:00:00 2001 From: turuslan Date: Tue, 10 Oct 2023 06:51:21 +0300 Subject: [PATCH 29/34] no shared Signed-off-by: turuslan --- core/blockchain/impl/cached_tree.cpp | 32 +++++++++++----------------- core/blockchain/impl/cached_tree.hpp | 14 +----------- 2 files changed, 14 insertions(+), 32 deletions(-) diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index eef73abe0a..2d8ce0565a 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -36,24 +36,6 @@ namespace kagome::blockchain { return {babe_primary_weight, info.number}; } - std::shared_ptr TreeNode::findByHash( - const primitives::BlockHash &hash) const { - // standard BFS - std::queue> nodes_to_scan; - nodes_to_scan.push(shared_from_this()); - while (!nodes_to_scan.empty()) { - const auto &node = nodes_to_scan.front(); - if (node->info.hash == hash) { - return node; - } - for (const auto &child : node->children) { - nodes_to_scan.push(child); - } - nodes_to_scan.pop(); - } - return nullptr; - } - template bool descend(std::shared_ptr from, const std::shared_ptr &to, @@ -244,6 +226,18 @@ namespace kagome::blockchain { std::shared_ptr CachedTree::find( const primitives::BlockHash &hash) const { - return root_->findByHash(hash); + std::queue> queue; + queue.push(root_); + while (not queue.empty()) { + auto &node = queue.front(); + if (node->info.hash == hash) { + return node; + } + for (auto &child : node->children) { + queue.push(child); + } + queue.pop(); + } + return nullptr; } } // namespace kagome::blockchain diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index c2351bc748..fe380e3a7a 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -22,7 +22,7 @@ namespace kagome::blockchain { * convenience - we would only ask the database for some info, when directly * requested */ - class TreeNode : public std::enable_shared_from_this { + class TreeNode { public: TreeNode(const primitives::BlockInfo &info); TreeNode(const primitives::BlockInfo &info, @@ -39,18 +39,6 @@ namespace kagome::blockchain { std::shared_ptr parent() const; BlockWeight weight() const; - - /** - * Get a node of the tree, containing block with the specified hash, if it - * can be found - */ - std::shared_ptr findByHash( - const primitives::BlockHash &hash) const; - - std::shared_ptr findByHash(const primitives::BlockHash &hash) { - return std::const_pointer_cast( - std::as_const(*this).findByHash(hash)); - } }; bool canDescend(std::shared_ptr from, From 7917b1500ed825cb8bc6a49a25cde3e6ef906d0b Mon Sep 17 00:00:00 2001 From: turuslan Date: Tue, 10 Oct 2023 06:55:02 +0300 Subject: [PATCH 30/34] unused Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 6 +++--- core/blockchain/impl/cached_tree.cpp | 10 ---------- core/blockchain/impl/cached_tree.hpp | 3 --- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 2b831f22c6..559e2e56ef 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -1402,7 +1402,7 @@ namespace kagome::blockchain { BOOST_ASSERT(!last_pruned.has_value() || last_pruned.value().number - <= block_tree_data.tree_->getRoot().info.number); + <= block_tree_data.tree_->finalized()->info.number); auto next_pruned_number = last_pruned ? last_pruned->number + 1 : 0; OUTCOME_TRY(hash_opt, getBlockHash(next_pruned_number)); @@ -1510,8 +1510,8 @@ namespace kagome::blockchain { void BlockTreeImpl::removeUnfinalized() { auto r = block_tree_data_.exclusiveAccess( [&](BlockTreeData &p) -> outcome::result { - auto finalized = p.tree_->getRoot().info; - auto nodes = std::move(p.tree_->getRoot().children); + auto finalized = p.tree_->finalized()->info; + auto nodes = std::move(p.tree_->finalized()->children); primitives::BlockNumber max = 0; for (size_t i = 0; i < nodes.size(); ++i) { auto children = std::move(nodes[i]->children); diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index 2d8ce0565a..0e764e5884 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -120,16 +120,6 @@ namespace kagome::blockchain { root_->weak_parent.reset(); } - const TreeNode &CachedTree::getRoot() const { - BOOST_ASSERT(root_ != nullptr); - return *root_; - } - - TreeNode &CachedTree::getRoot() { - BOOST_ASSERT(root_ != nullptr); - return *root_; - } - void CachedTree::updateMeta(const std::shared_ptr &new_node) { auto parent = wptrMustLock(new_node->weak_parent); parent->children.push_back(new_node); diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index fe380e3a7a..1774061d75 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -81,9 +81,6 @@ namespace kagome::blockchain { */ void removeFromMeta(const std::shared_ptr &node); - const TreeNode &getRoot() const; - TreeNode &getRoot(); - private: /** * Compare node weight with best and replace if heavier. From f720844de935a11ec86620dd3414e40f400ea5b6 Mon Sep 17 00:00:00 2001 From: turuslan Date: Tue, 10 Oct 2023 09:26:08 +0300 Subject: [PATCH 31/34] finalize Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 559e2e56ef..3c4e7d6259 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -804,20 +804,20 @@ namespace kagome::blockchain { const primitives::Justification &justification) { return block_tree_data_.exclusiveAccess([&](BlockTreeData &p) -> outcome::result { + auto last_finalized_block_info = getLastFinalizedNoLock(p); + if (block_hash == last_finalized_block_info.hash) { + return outcome::success(); + } const auto node = p.tree_->find(block_hash); if (!node) { return BlockTreeError::NON_FINALIZED_BLOCK_NOT_FOUND; } - const auto block = node->info; - - auto last_finalized_block_info = getLastFinalizedNoLock(p); - auto justification_stored = false; if (node->info.number > last_finalized_block_info.number) { - SL_DEBUG(log_, "Finalizing block {}", block); + SL_DEBUG(log_, "Finalizing block {}", node->info); - OUTCOME_TRY(header_opt, p.storage_->getBlockHeader(node->info.hash)); + OUTCOME_TRY(header_opt, p.storage_->getBlockHeader(block_hash)); if (not header_opt.has_value()) { return BlockTreeError::HEADER_NOT_FOUND; } @@ -838,7 +838,7 @@ namespace kagome::blockchain { notifyChainEventsEngine( primitives::events::ChainEventType::kFinalizedHeads, header); - OUTCOME_TRY(body, p.storage_->getBlockBody(node->info.hash)); + OUTCOME_TRY(body, p.storage_->getBlockBody(block_hash)); if (body.has_value()) { for (auto &ext : body.value()) { auto extrinsic_hash = p.hasher_->blake2b_256(ext.data); @@ -857,14 +857,10 @@ namespace kagome::blockchain { } } - log_->info("Finalized block {}", block); - telemetry_->notifyBlockFinalized(block); + log_->info("Finalized block {}", node->info); + telemetry_->notifyBlockFinalized(node->info); telemetry_->pushBlockStats(); metric_finalized_block_height_->set(node->info.number); - - } else if (node->info.hash == last_finalized_block_info.hash) { - // block is current last finalized, fine - return outcome::success(); } else if (hasDirectChainNoLock( p, block_hash, last_finalized_block_info.hash)) { OUTCOME_TRY(justification_opt, @@ -880,7 +876,7 @@ namespace kagome::blockchain { if (not justification_stored) { OUTCOME_TRY(p.storage_->putJustification(justification, block_hash)); } - SL_DEBUG(log_, "Store justification for finalized block {}", block); + SL_DEBUG(log_, "Store justification for finalized block {}", node->info); if (last_finalized_block_info.number < node->info.number) { // we store justification for last finalized block only as long as it is From 6e9988594cf68e30d0005ad17616976e8be78e64 Mon Sep 17 00:00:00 2001 From: turuslan Date: Tue, 10 Oct 2023 09:53:08 +0300 Subject: [PATCH 32/34] finalize justification Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 56 +++++++++++------------- 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 3c4e7d6259..77a657639e 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -809,12 +809,7 @@ namespace kagome::blockchain { return outcome::success(); } const auto node = p.tree_->find(block_hash); - if (!node) { - return BlockTreeError::NON_FINALIZED_BLOCK_NOT_FOUND; - } - auto justification_stored = false; - - if (node->info.number > last_finalized_block_info.number) { + if (node) { SL_DEBUG(log_, "Finalizing block {}", node->info); OUTCOME_TRY(header_opt, p.storage_->getBlockHeader(block_hash)); @@ -824,7 +819,6 @@ namespace kagome::blockchain { auto &header = header_opt.value(); OUTCOME_TRY(p.storage_->putJustification(justification, block_hash)); - justification_stored = true; OUTCOME_TRY(pruneNoLock(p, node)); OUTCOME_TRY(pruneTrie(p, node->info.number)); @@ -861,31 +855,13 @@ namespace kagome::blockchain { telemetry_->notifyBlockFinalized(node->info); telemetry_->pushBlockStats(); metric_finalized_block_height_->set(node->info.number); - } else if (hasDirectChainNoLock( - p, block_hash, last_finalized_block_info.hash)) { - OUTCOME_TRY(justification_opt, - p.storage_->getJustification(block_hash)); - if (justification_opt.has_value()) { - // block already has justification (in DB), fine - return outcome::success(); - } - } - KAGOME_PROFILE_START(justification_store) - - if (not justification_stored) { - OUTCOME_TRY(p.storage_->putJustification(justification, block_hash)); - } - SL_DEBUG(log_, "Store justification for finalized block {}", node->info); - - if (last_finalized_block_info.number < node->info.number) { // we store justification for last finalized block only as long as it is // last finalized (if it doesn't meet other justification storage rules, // e.g. its number a multiple of 512) - OUTCOME_TRY(last_finalized_header_opt, - p.storage_->getBlockHeader(last_finalized_block_info.hash)); - // SAFETY: header for the last finalized block must be present - auto &last_finalized_header = last_finalized_header_opt.value(); + OUTCOME_TRY( + last_finalized_header, + p.header_repo_->getBlockHeader(last_finalized_block_info.hash)); OUTCOME_TRY( shouldStoreLastFinalized, p.justification_storage_policy_->shouldStoreFor( @@ -902,10 +878,28 @@ namespace kagome::blockchain { last_finalized_block_info.hash)); } } + } else { + OUTCOME_TRY(header, p.header_repo_->getBlockHeader(block_hash)); + if (header.number >= last_finalized_block_info.number) { + return BlockTreeError::NON_FINALIZED_BLOCK_NOT_FOUND; + } + OUTCOME_TRY(canon_hash, p.header_repo_->getHashByNumber(header.number)); + if (block_hash != canon_hash) { + return BlockTreeError::BLOCK_ON_DEAD_END; + } + if (not p.justification_storage_policy_ + ->shouldStoreFor(header, last_finalized_block_info.number) + .value()) { + return outcome::success(); + } + OUTCOME_TRY(justification_opt, + p.storage_->getJustification(block_hash)); + if (justification_opt.has_value()) { + // block already has justification (in DB), fine + return outcome::success(); + } + OUTCOME_TRY(p.storage_->putJustification(justification, block_hash)); } - - KAGOME_PROFILE_END(justification_store) - return outcome::success(); }); } From 525821b52d68579406f0436d2802ad465a2c9b3f Mon Sep 17 00:00:00 2001 From: turuslan Date: Tue, 10 Oct 2023 15:25:10 +0300 Subject: [PATCH 33/34] remove unfinalized, reorg and prune Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 56 +++++++++++++----------- core/blockchain/impl/block_tree_impl.hpp | 5 ++- core/blockchain/impl/cached_tree.cpp | 40 +++++++++++++++++ core/blockchain/impl/cached_tree.hpp | 26 +++++++++++ 4 files changed, 100 insertions(+), 27 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 77a657639e..495fe1f107 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -1279,6 +1279,30 @@ namespace kagome::blockchain { [&](const BlockTreeData &p) { return getLastFinalizedNoLock(p); }); } + outcome::result BlockTreeImpl::reorgAndPrune( + BlockTreeData &p, const ReorgAndPrune &changes) { + OUTCOME_TRY(p.storage_->setBlockTreeLeaves(p.tree_->leafHashes())); + metric_known_chain_leaves_->set(p.tree_->leafCount()); + if (changes.reorg) { + for (auto &block : changes.reorg->revert) { + OUTCOME_TRY(p.storage_->deassignNumberToHash(block.number)); + } + for (auto &block : changes.reorg->apply) { + OUTCOME_TRY(p.storage_->assignNumberToHash(block)); + } + if (not changes.reorg->apply.empty()) { + metric_best_block_height_->set(changes.reorg->apply.back().number); + } else { + metric_best_block_height_->set(changes.reorg->common.number); + } + } + for (auto &block : changes.prune) { + OUTCOME_TRY(p.storage_->removeBlock(block.hash)); + } + // TODO(turuslan): #1679, move code from pruneNoLock + return outcome::success(); + } + outcome::result BlockTreeImpl::pruneNoLock( BlockTreeData &p, const std::shared_ptr &lastFinalizedNode) { std::deque> to_remove; @@ -1498,31 +1522,11 @@ namespace kagome::blockchain { } void BlockTreeImpl::removeUnfinalized() { - auto r = block_tree_data_.exclusiveAccess( - [&](BlockTreeData &p) -> outcome::result { - auto finalized = p.tree_->finalized()->info; - auto nodes = std::move(p.tree_->finalized()->children); - primitives::BlockNumber max = 0; - for (size_t i = 0; i < nodes.size(); ++i) { - auto children = std::move(nodes[i]->children); - for (auto &node : children) { - max = std::max(max, node->info.number); - nodes.emplace_back(std::move(node)); - } - } - p.tree_ = std::make_unique(finalized); - OUTCOME_TRY(p.storage_->setBlockTreeLeaves({finalized.hash})); - for (auto i = max; i > finalized.number; --i) { - OUTCOME_TRY(p.storage_->deassignNumberToHash(i)); - } - std::reverse(nodes.begin(), nodes.end()); - for (auto &node : nodes) { - OUTCOME_TRY(p.storage_->removeBlock(node->info.hash)); - } - return outcome::success(); - }); - if (not r) { - SL_ERROR(log_, "removeUnfinalized error: {}", r.error()); - } + block_tree_data_.exclusiveAccess([&](BlockTreeData &p) { + auto changes = p.tree_->removeUnfinalized(); + if (auto r = reorgAndPrune(p, changes); r.has_error()) { + SL_WARN(log_, "removeUnfinalized error: {}", r.error()); + } + }); } } // namespace kagome::blockchain diff --git a/core/blockchain/impl/block_tree_impl.hpp b/core/blockchain/impl/block_tree_impl.hpp index f94fa334d2..78af4c0599 100644 --- a/core/blockchain/impl/block_tree_impl.hpp +++ b/core/blockchain/impl/block_tree_impl.hpp @@ -37,7 +37,7 @@ namespace kagome::storage::trie_pruner { } namespace kagome::blockchain { - + struct ReorgAndPrune; class TreeNode; class CachedTree; @@ -181,6 +181,9 @@ namespace kagome::blockchain { std::shared_ptr state_pruner, std::shared_ptr<::boost::asio::io_context> io_context); + outcome::result reorgAndPrune(BlockTreeData &p, + const ReorgAndPrune &changes); + outcome::result pruneNoLock( BlockTreeData &p, const std::shared_ptr &lastFinalizedNode); diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index 0e764e5884..2b3aea0cdd 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -12,6 +12,10 @@ #include "utils/wptr.hpp" namespace kagome::blockchain { + bool Reorg::empty() const { + return revert.empty() and apply.empty(); + } + TreeNode::TreeNode(const primitives::BlockInfo &info) : info{info}, babe_primary_weight{0}, @@ -36,6 +40,22 @@ namespace kagome::blockchain { return {babe_primary_weight, info.number}; } + Reorg reorg(std::shared_ptr from, std::shared_ptr to) { + Reorg reorg; + while (from != to) { + if (from->info.number > to->info.number) { + reorg.revert.emplace_back(from->info); + from = wptrMustLock(from->weak_parent); + } else { + reorg.apply.emplace_back(to->info); + to = wptrMustLock(to->weak_parent); + } + } + reorg.common = to->info; + std::reverse(reorg.apply.begin(), reorg.apply.end()); + return reorg; + } + template bool descend(std::shared_ptr from, const std::shared_ptr &to, @@ -230,4 +250,24 @@ namespace kagome::blockchain { } return nullptr; } + + ReorgAndPrune CachedTree::removeUnfinalized() { + ReorgAndPrune changes; + if (best_ != root_) { + changes.reorg = reorg(best_, root_); + } + std::deque> queue{root_}; + while (not queue.empty()) { + auto parent = std::move(queue.front()); + queue.pop_front(); + for (auto &child : parent->children) { + changes.prune.emplace_back(child->info); + queue.emplace_back(child); + } + parent->children.clear(); + } + std::reverse(changes.prune.begin(), changes.prune.end()); + *this = CachedTree{root_->info}; + return changes; + } } // namespace kagome::blockchain diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index 1774061d75..01fb8e5694 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -17,6 +17,26 @@ namespace kagome::blockchain { using BlockWeight = std::pair; + /** + * Used to update hashes of best chain by number. + */ + struct Reorg { + primitives::BlockInfo common; + std::vector revert; + std::vector apply; + + bool empty() const; + }; + + /** + * Used to enqueue blocks for removal. + * Children are removed before parent. + */ + struct ReorgAndPrune { + std::optional reorg; + std::vector prune; + }; + /** * In-memory light representation of the tree, used for efficiency and usage * convenience - we would only ask the database for some info, when directly @@ -41,6 +61,8 @@ namespace kagome::blockchain { BlockWeight weight() const; }; + Reorg reorg(std::shared_ptr from, std::shared_ptr to); + bool canDescend(std::shared_ptr from, const std::shared_ptr &to); @@ -59,6 +81,10 @@ namespace kagome::blockchain { primitives::BlockInfo bestWith( const std::shared_ptr &required) const; std::shared_ptr find(const primitives::BlockHash &hash) const; + /** + * Used when switching from fast-sync to full-sync. + */ + ReorgAndPrune removeUnfinalized(); /** * Remove nodes in block tree from current tree_ to {\arg new_trie_root}. From f20e71d6ad1711b74c801cdd2f4673b95ee42603 Mon Sep 17 00:00:00 2001 From: turuslan Date: Tue, 10 Oct 2023 15:29:34 +0300 Subject: [PATCH 34/34] return info Signed-off-by: turuslan --- core/blockchain/impl/block_tree_impl.cpp | 10 ++++------ core/blockchain/impl/cached_tree.cpp | 8 ++++---- core/blockchain/impl/cached_tree.hpp | 4 ++-- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 495fe1f107..2bca032175 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -981,9 +981,7 @@ namespace kagome::blockchain { return std::vector{block}; } - auto best_block = p.tree_->best(); - BOOST_ASSERT(best_block != nullptr); - auto current_depth = best_block->info.number; + auto current_depth = bestBlockNoLock(p).number; if (start_block_number >= current_depth) { return std::vector{block}; @@ -1197,7 +1195,7 @@ namespace kagome::blockchain { primitives::BlockInfo BlockTreeImpl::bestBlockNoLock( const BlockTreeData &p) const { - return p.tree_->best()->info; + return p.tree_->best(); } primitives::BlockInfo BlockTreeImpl::bestBlock() const { @@ -1271,7 +1269,7 @@ namespace kagome::blockchain { primitives::BlockInfo BlockTreeImpl::getLastFinalizedNoLock( const BlockTreeData &p) const { - return p.tree_->finalized()->info; + return p.tree_->finalized(); } primitives::BlockInfo BlockTreeImpl::getLastFinalized() const { @@ -1416,7 +1414,7 @@ namespace kagome::blockchain { BOOST_ASSERT(!last_pruned.has_value() || last_pruned.value().number - <= block_tree_data.tree_->finalized()->info.number); + <= getLastFinalizedNoLock(block_tree_data).number); auto next_pruned_number = last_pruned ? last_pruned->number + 1 : 0; OUTCOME_TRY(hash_opt, getBlockHash(next_pruned_number)); diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index 2b3aea0cdd..22b6b63c17 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -186,12 +186,12 @@ namespace kagome::blockchain { best_{root_}, leaves_{root.hash} {} - const std::shared_ptr &CachedTree::finalized() const { - return root_; + primitives::BlockInfo CachedTree::finalized() const { + return root_->info; } - const std::shared_ptr &CachedTree::best() const { - return best_; + primitives::BlockInfo CachedTree::best() const { + return best_->info; } size_t CachedTree::leafCount() const { diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index 01fb8e5694..279e0873bb 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -73,8 +73,8 @@ namespace kagome::blockchain { public: explicit CachedTree(const primitives::BlockInfo &root); - const std::shared_ptr &finalized() const; - const std::shared_ptr &best() const; + primitives::BlockInfo finalized() const; + primitives::BlockInfo best() const; size_t leafCount() const; std::vector leafHashes() const; bool isLeaf(const primitives::BlockHash &hash) const;