From 37695eee0e64ff4b594587e4d5bd811acc4c6151 Mon Sep 17 00:00:00 2001 From: turuslan Date: Fri, 16 Jun 2023 13:21:25 +0300 Subject: [PATCH 01/12] babe indexer Signed-off-by: turuslan --- .../impl/kagome_application_impl.cpp | 3 + .../impl/kagome_application_impl.hpp | 2 +- core/blockchain/block_tree.hpp | 7 + core/blockchain/impl/block_tree_impl.cpp | 8 + core/blockchain/impl/block_tree_impl.hpp | 2 + core/blockchain/impl/digest_tracker_impl.cpp | 36 +- core/blockchain/impl/digest_tracker_impl.hpp | 11 +- core/blockchain/indexer.hpp | 246 +++++ core/consensus/CMakeLists.txt | 1 - .../consensus/babe/babe_config_repository.hpp | 8 +- core/consensus/babe/babe_digest_observer.hpp | 41 - .../babe/has_authority_set_change.hpp | 48 + core/consensus/babe/impl/babe_config_node.cpp | 45 - core/consensus/babe/impl/babe_config_node.hpp | 67 -- .../babe/impl/babe_config_repository_impl.cpp | 907 ++++-------------- .../babe/impl/babe_config_repository_impl.hpp | 93 +- .../consensus/babe/impl/babe_digests_util.cpp | 38 - .../consensus/babe/impl/babe_digests_util.hpp | 6 - core/consensus/babe/impl/babe_impl.cpp | 33 +- .../babe/impl/block_appender_base.cpp | 9 +- core/consensus/grandpa/structs.hpp | 25 +- core/injector/application_injector.cpp | 440 ++++----- .../impl/state_protocol_observer_impl.cpp | 5 +- core/network/impl/synchronizer_impl.cpp | 6 +- .../types/block_announce_handshake.hpp | 14 +- core/network/types/roles.hpp | 41 +- core/network/warp/sync.cpp | 5 + core/network/warp/sync.hpp | 8 + core/primitives/apply_result.hpp | 15 +- core/primitives/block_header.hpp | 7 + core/primitives/check_inherents_result.hpp | 9 +- core/primitives/common.hpp | 21 +- core/primitives/scheduled_change.hpp | 19 +- core/scale/tie.hpp | 31 +- core/storage/face/map_cursor.hpp | 22 + core/storage/in_memory/cursor.hpp | 74 ++ core/storage/in_memory/in_memory_storage.cpp | 3 +- core/storage/in_memory/in_memory_storage.hpp | 2 + core/storage/predefined_keys.hpp | 7 +- core/telemetry/service.cpp | 3 + .../babe/babe_config_repository_test.cpp | 19 +- test/core/consensus/babe/babe_test.cpp | 2 +- .../consensus/babe/block_executor_test.cpp | 2 +- .../network/state_protocol_observer_test.cpp | 4 +- test/mock/core/blockchain/block_tree_mock.hpp | 5 + .../babe/babe_config_repository_mock.hpp | 10 +- 46 files changed, 990 insertions(+), 1420 deletions(-) create mode 100644 core/blockchain/indexer.hpp delete mode 100644 core/consensus/babe/babe_digest_observer.hpp create mode 100644 core/consensus/babe/has_authority_set_change.hpp delete mode 100644 core/consensus/babe/impl/babe_config_node.cpp delete mode 100644 core/consensus/babe/impl/babe_config_node.hpp create mode 100644 core/storage/in_memory/cursor.hpp diff --git a/core/application/impl/kagome_application_impl.cpp b/core/application/impl/kagome_application_impl.cpp index 0c7defdc8a..f3b6ca35d8 100644 --- a/core/application/impl/kagome_application_impl.cpp +++ b/core/application/impl/kagome_application_impl.cpp @@ -15,6 +15,9 @@ #include "telemetry/service.hpp" namespace kagome::application { + KagomeApplicationImpl::~KagomeApplicationImpl() { + kagome::telemetry::setTelemetryService(nullptr); + } KagomeApplicationImpl::KagomeApplicationImpl( std::shared_ptr app_config) diff --git a/core/application/impl/kagome_application_impl.hpp b/core/application/impl/kagome_application_impl.hpp index c8cda51dca..c9682cdc4f 100644 --- a/core/application/impl/kagome_application_impl.hpp +++ b/core/application/impl/kagome_application_impl.hpp @@ -22,7 +22,7 @@ namespace kagome::application { using uptr = std::unique_ptr; public: - ~KagomeApplicationImpl() override = default; + ~KagomeApplicationImpl() override; explicit KagomeApplicationImpl(std::shared_ptr config); diff --git a/core/blockchain/block_tree.hpp b/core/blockchain/block_tree.hpp index 91ce2c2485..592583e1c4 100644 --- a/core/blockchain/block_tree.hpp +++ b/core/blockchain/block_tree.hpp @@ -192,6 +192,13 @@ namespace kagome::blockchain { const primitives::BlockHash &ancestor, const primitives::BlockHash &descendant) const = 0; + bool hasDirectChain(const primitives::BlockInfo &ancestor, + const primitives::BlockInfo &descendant) const { + return hasDirectChain(ancestor.hash, descendant.hash); + } + + virtual bool isFinalized(const primitives::BlockInfo &block) const = 0; + /** * Get a best leaf of the tree * @return best leaf diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index 6f94619aaf..0f2739eea2 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -1130,6 +1130,14 @@ namespace kagome::blockchain { }); } + bool BlockTreeImpl::isFinalized(const primitives::BlockInfo &block) const { + return block_tree_data_.sharedAccess([&](const BlockTreeData &p) { + return block.number <= getLastFinalizedNoLock(p).number + and p.header_repo_->getHashByNumber(block.number) + == outcome::success(block.hash); + }); + } + primitives::BlockInfo BlockTreeImpl::bestLeafNoLock( const BlockTreeData &p) const { auto leaf = p.tree_->getMetadata().best_leaf.lock(); diff --git a/core/blockchain/impl/block_tree_impl.hpp b/core/blockchain/impl/block_tree_impl.hpp index 4bdcecd984..f754426882 100644 --- a/core/blockchain/impl/block_tree_impl.hpp +++ b/core/blockchain/impl/block_tree_impl.hpp @@ -119,6 +119,8 @@ namespace kagome::blockchain { bool hasDirectChain(const primitives::BlockHash &ancestor, const primitives::BlockHash &descendant) const override; + bool isFinalized(const primitives::BlockInfo &block) const override; + primitives::BlockInfo bestLeaf() const override; outcome::result getBestContaining( diff --git a/core/blockchain/impl/digest_tracker_impl.cpp b/core/blockchain/impl/digest_tracker_impl.cpp index 073d78a005..e98deb4059 100644 --- a/core/blockchain/impl/digest_tracker_impl.cpp +++ b/core/blockchain/impl/digest_tracker_impl.cpp @@ -6,19 +6,15 @@ #include "digest_tracker_impl.hpp" #include "common/visitor.hpp" -#include "consensus/babe/babe_digest_observer.hpp" #include "consensus/grandpa/grandpa_digest_observer.hpp" namespace kagome::blockchain { DigestTrackerImpl::DigestTrackerImpl( - std::shared_ptr babe_update_observer, std::shared_ptr grandpa_digest_observer) - : babe_digest_observer_(std::move(babe_update_observer)), - grandpa_digest_observer_(std::move(grandpa_digest_observer)), + : grandpa_digest_observer_(std::move(grandpa_digest_observer)), logger_(log::createLogger("DigestTracker", "digest_tracker")) { - BOOST_ASSERT(babe_digest_observer_ != nullptr); BOOST_ASSERT(grandpa_digest_observer_ != nullptr); } @@ -44,11 +40,7 @@ namespace kagome::blockchain { return outcome::success(); // It does not processed by tracker }, [&](const primitives::PreRuntime &item) { - SL_TRACE(logger_, - "PreRuntime-digest on block {}, engine '{}'", - context.block_info, - item.consensus_engine_id.toString()); - return onPreRuntime(context, item); + return outcome::success(); }, [&](const primitives::RuntimeEnvironmentUpdated &item) { SL_TRACE(logger_, @@ -70,9 +62,6 @@ namespace kagome::blockchain { } void DigestTrackerImpl::cancel(const primitives::BlockInfo &block) { - // Cancel tracked babe digest - babe_digest_observer_->cancel(block); - // Cancel tracked grandpa digest grandpa_digest_observer_->cancel(block); } @@ -83,7 +72,7 @@ namespace kagome::blockchain { if (message.consensus_engine_id == primitives::kBabeEngineId) { OUTCOME_TRY(digest, scale::decode(message.data)); - return babe_digest_observer_->onDigest(context, digest); + return outcome::success(); } else if (message.consensus_engine_id == primitives::kGrandpaEngineId) { OUTCOME_TRY(digest, @@ -109,23 +98,4 @@ namespace kagome::blockchain { return outcome::success(); } } - - outcome::result DigestTrackerImpl::onPreRuntime( - const primitives::BlockContext &context, - const primitives::PreRuntime &message) { - if (message.consensus_engine_id == primitives::kBabeEngineId) { - OUTCOME_TRY( - digest, - scale::decode(message.data)); - - return babe_digest_observer_->onDigest(context, digest); - } else { - SL_WARN(logger_, - "Unknown consensus engine id in block {}: {}", - context.block_info, - message.consensus_engine_id.toString()); - return outcome::success(); - } - } - } // namespace kagome::blockchain diff --git a/core/blockchain/impl/digest_tracker_impl.hpp b/core/blockchain/impl/digest_tracker_impl.hpp index 1e551efa97..3cbe612d91 100644 --- a/core/blockchain/impl/digest_tracker_impl.hpp +++ b/core/blockchain/impl/digest_tracker_impl.hpp @@ -13,17 +13,12 @@ namespace kagome::consensus::grandpa { class GrandpaDigestObserver; } -namespace kagome::consensus::babe { - class BabeDigestObserver; -} namespace kagome::blockchain { class DigestTrackerImpl final : public DigestTracker { public: - DigestTrackerImpl(std::shared_ptr - babe_update_observer, - std::shared_ptr + DigestTrackerImpl(std::shared_ptr grandpa_digest_observer); outcome::result onDigest(const primitives::BlockContext &context, @@ -32,14 +27,10 @@ namespace kagome::blockchain { void cancel(const primitives::BlockInfo &block) override; private: - outcome::result onPreRuntime(const primitives::BlockContext &context, - const primitives::PreRuntime &message); - outcome::result onConsensus( const primitives::BlockContext &context, const primitives::Consensus &consensus_message); - std::shared_ptr babe_digest_observer_; std::shared_ptr grandpa_digest_observer_; diff --git a/core/blockchain/indexer.hpp b/core/blockchain/indexer.hpp new file mode 100644 index 0000000000..b3fcaa90e0 --- /dev/null +++ b/core/blockchain/indexer.hpp @@ -0,0 +1,246 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef KAGOME_BLOCKCHAIN_INDEXER_HPP +#define KAGOME_BLOCKCHAIN_INDEXER_HPP + +#include + +#include "blockchain/block_tree.hpp" +#include "storage/buffer_map_types.hpp" + +namespace kagome::blockchain { + struct Descent { + Descent(std::shared_ptr block_tree, + primitives::BlockInfo start) + : block_tree_{std::move(block_tree)}, path_{start} {} + + bool can(const primitives::BlockInfo &to) { + if (to == path_.front()) { + return true; + } + if (to.number >= path_.front().number) { + return false; + } + auto i = indexFor(to.number); + if (i >= path_.size()) { + if (not update_path_) { + return block_tree_->hasDirectChain(to, path_.back()); + } + auto chain_res = block_tree_->getDescendingChainToBlock( + path_.back().hash, path_.back().number - to.number + 1); + if (not chain_res) { + return false; + } + auto &chain = chain_res.value(); + if (chain.size() <= 1) { + return false; + } + for (size_t j = 1; j < chain.size(); ++j) { + path_.emplace_back(path_.back().number - 1, chain[j]); + } + if (i >= path_.size()) { + return false; + } + } + return to == path_.at(i); + } + + size_t indexFor(primitives::BlockNumber n) { + BOOST_ASSERT(n <= path_.front().number); + return path_.front().number - n; + } + + std::shared_ptr block_tree_; + std::vector path_; + bool update_path_ = true; + }; + + template + struct Indexed { + SCALE_TIE_ONLY(value, prev); + + std::optional value; + std::optional prev; + bool inherit = false; + }; + + template + struct Indexer { + static common::Blob<4 + 32> toKey(const primitives::BlockInfo &info) { + common::Blob<4 + 32> key; + boost::endian::store_big_u32(key.data(), info.number); + std::copy_n(info.hash.data(), 32, key.data() + 4); + return key; + } + + static primitives::BlockInfo fromKey(common::BufferView key) { + BOOST_ASSERT(key.size() == 4 + 32); + primitives::BlockInfo info; + info.number = boost::endian::load_big_u32(key.data()); + std::copy_n(key.data() + 4, 32, info.hash.data()); + return info; + } + + Indexer(std::shared_ptr db, + std::shared_ptr block_tree, + const primitives::BlockInfo &genesis) + : db_{std::move(db)}, block_tree_{std::move(block_tree)} { + map_.emplace(genesis, get(genesis).value_or(Indexed{})); + } + + Descent descend(const primitives::BlockInfo &from) const { + return {block_tree_, from}; + } + + std::optional> get(const primitives::BlockInfo &block) const { + if (auto it = map_.find(block); it != map_.end()) { + return it->second; + } + if (auto r = db_->tryGet(toKey(block)).value()) { + return scale::decode>(*r).value(); + } + return std::nullopt; + } + + void put(const primitives::BlockInfo &block, + const Indexed &indexed, + bool db) { + map_[block] = indexed; + if (db) { + db_->put(toKey(block), scale::encode(indexed).value()).value(); + } + } + + void remove(const primitives::BlockInfo &block) { + map_.erase(block); + db_->remove(toKey(block)).value(); + } + + void filterUnfinalized() { + for (auto it = map_.begin(); it != map_.end();) { + if (not block_tree_->isFinalized(it->first)) { + it = map_.erase(it); + } else { + ++it; + } + } + } + + primitives::BlockInfo removeUnfinalized() { + for (auto it = map_.begin(); it != map_.end();) { + if (not block_tree_->isFinalized(it->first)) { + it = map_.erase(it); + } else { + ++it; + } + } + primitives::BlockInfo max{0, block_tree_->getGenesisBlockHash()}; + auto batch = db_->batch(); + auto db_cur = db_->cursor(); + for (db_cur->seekLast().value(); db_cur->isValid();) { + auto info = fromKey(*db_cur->key()); + db_cur->prev().value(); + if (not block_tree_->isFinalized(info)) { + batch->remove(toKey(info)).value(); + } else if (max < info) { + max = info; + break; + } + } + batch->commit().value(); + return max; + } + + void writeFinalized(primitives::BlockNumber first, + primitives::BlockNumber last) { + auto begin = map_.lower_bound({first, {}}), + end = map_.lower_bound({last + 1, {}}); + auto batch = db_->batch(); + for (auto it = begin; it != end; ++it) { + if (block_tree_->isFinalized(it->first)) { + batch->put(toKey(it->first), scale::encode(it->second).value()) + .value(); + } + } + batch->commit().value(); + } + + using Search = std::pair>; + struct SearchRaw { + Search kv; + primitives::BlockInfo last; + }; + + std::optional searchRaw(Descent &descent, + const primitives::BlockInfo &block) { + auto map_it = map_.lower_bound(block); + while (true) { + if (map_it != map_.end() and descent.can(map_it->first)) { + if (not map_it->second.inherit) { + return SearchRaw{*map_it, map_it->first}; + } + BOOST_ASSERT(map_it->second.prev); + auto r = get(*map_it->second.prev); + BOOST_ASSERT(r); + return SearchRaw{ + {*map_it->second.prev, std::move(*r)}, + map_it->first, + }; + } + if (map_it == map_.begin()) { + break; + } + --map_it; + } + auto db_cur = db_->cursor(); + for (db_cur->seekReverse(toKey(block)).value(); db_cur->isValid(); + db_cur->prev().value()) { + auto info = fromKey(*db_cur->key()); + if (not descent.can(info)) { + continue; + } + return SearchRaw{ + {info, scale::decode>(*db_cur->value()).value()}, + info, + }; + } + return std::nullopt; + } + + template + std::optional search(Descent &descent, + const primitives::BlockInfo &block, + const Cb &cb) { + descent.update_path_ = true; + auto raw = searchRaw(descent, block); + if (not raw) { + return std::nullopt; + } + BOOST_ASSERT(not raw->kv.second.inherit); + if (not raw->kv.second.value or raw->last != block) { + auto prev = raw->kv.second.value ? raw->kv.first : raw->kv.second.prev; + auto i_first = + descent.indexFor(raw->last.number + (raw->kv.second.value ? 1 : 0)); + BOOST_ASSERT(i_first < descent.path_.size()); + auto i_last = descent.indexFor(block.number); + BOOST_ASSERT(i_last < descent.path_.size()); + cb(prev, i_first, i_last); + descent.update_path_ = false; + raw = searchRaw(descent, block); + if (not raw or not raw->kv.second.value or raw->last != block) { + return std::nullopt; + } + } + return raw->kv; + } + + std::shared_ptr db_; + std::shared_ptr block_tree_; + std::map> map_; + }; +} // namespace kagome::blockchain + +#endif // KAGOME_BLOCKCHAIN_INDEXER_HPP diff --git a/core/consensus/CMakeLists.txt b/core/consensus/CMakeLists.txt index 01ce7d7cbf..fe62864471 100644 --- a/core/consensus/CMakeLists.txt +++ b/core/consensus/CMakeLists.txt @@ -5,7 +5,6 @@ add_library(consensus babe/impl/babe_digests_util.cpp - babe/impl/babe_config_node.cpp babe/impl/block_executor_impl.cpp babe/impl/babe_impl.cpp babe/impl/threshold_util.cpp diff --git a/core/consensus/babe/babe_config_repository.hpp b/core/consensus/babe/babe_config_repository.hpp index adad3b1080..9b9c5157d9 100644 --- a/core/consensus/babe/babe_config_repository.hpp +++ b/core/consensus/babe/babe_config_repository.hpp @@ -25,12 +25,12 @@ namespace kagome::consensus::babe { /// Returns the actual babe configuration /// @return the actual babe configuration - virtual std::optional< - std::reference_wrapper> - config(const primitives::BlockContext &context, + virtual outcome::result< + std::shared_ptr> + config(const primitives::BlockInfo &parent_info, EpochNumber epoch_number) const = 0; - virtual void readFromState(const primitives::BlockInfo &block) = 0; + virtual void warp(const primitives::BlockInfo &block) = 0; }; } // namespace kagome::consensus::babe diff --git a/core/consensus/babe/babe_digest_observer.hpp b/core/consensus/babe/babe_digest_observer.hpp deleted file mode 100644 index 5f5c3504df..0000000000 --- a/core/consensus/babe/babe_digest_observer.hpp +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright Soramitsu Co., Ltd. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef KAGOME_CONSENSUS_BABEDIGESTOBSERVER -#define KAGOME_CONSENSUS_BABEDIGESTOBSERVER - -#include "outcome/outcome.hpp" -#include "primitives/block_data.hpp" -#include "primitives/common.hpp" -#include "primitives/digest.hpp" - -namespace kagome::consensus::babe { - - class BabeDigestObserver { - public: - virtual ~BabeDigestObserver() = default; - - /// Observes PreRuntime of block - /// @param context - data of accorded block - /// @param digest - BabeBlockHeader as decoded content of PreRuntime digest - /// @return failure or nothing - virtual outcome::result onDigest( - const primitives::BlockContext &context, - const BabeBlockHeader &digest) = 0; - - /// Observes ConsensusLog of block - /// @param context - data of accorded block - /// @param digest - BabeDigest as particular variant of ConsensusLog digest - /// @return failure or nothing - virtual outcome::result onDigest( - const primitives::BlockContext &context, - const primitives::BabeDigest &digest) = 0; - - virtual void cancel(const primitives::BlockInfo &block) = 0; - }; - -} // namespace kagome::consensus::babe - -#endif // KAGOME_CONSENSUS_BABEDIGESTOBSERVER diff --git a/core/consensus/babe/has_authority_set_change.hpp b/core/consensus/babe/has_authority_set_change.hpp new file mode 100644 index 0000000000..235d5b62bd --- /dev/null +++ b/core/consensus/babe/has_authority_set_change.hpp @@ -0,0 +1,48 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef KAGOME_CONSENSUS_BABE_HAS_AUTHORITY_SET_CHANGE_HPP +#define KAGOME_CONSENSUS_BABE_HAS_AUTHORITY_SET_CHANGE_HPP + +#include "primitives/block_header.hpp" + +namespace kagome::consensus::babe { + struct HasAuthoritySetChange { + HasAuthoritySetChange(const primitives::BlockHeader &block) { + for (auto &digest : block.digest) { + auto consensus = boost::get(&digest); + if (not consensus) { + continue; + } + auto decoded_res = consensus->decode(); + if (not decoded_res) { + continue; + } + auto &decoded = decoded_res.value(); + auto babe = boost::get(&decoded.digest); + if (not babe) { + continue; + } + if (auto item = boost::get(babe)) { + epoch = std::move(*item); + continue; + } + if (auto item = boost::get(babe)) { + config = boost::get(*item); + continue; + } + } + } + + operator bool() const { + return epoch.has_value(); + } + + std::optional epoch; + std::optional config; + }; +} // namespace kagome::consensus::babe + +#endif // KAGOME_CONSENSUS_BABE_HAS_AUTHORITY_SET_CHANGE_HPP diff --git a/core/consensus/babe/impl/babe_config_node.cpp b/core/consensus/babe/impl/babe_config_node.cpp deleted file mode 100644 index f1b32cdc5f..0000000000 --- a/core/consensus/babe/impl/babe_config_node.cpp +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Copyright Soramitsu Co., Ltd. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "babe_config_node.hpp" - -namespace kagome::consensus::babe { - - BabeConfigNode::BabeConfigNode( - const std::shared_ptr &ancestor, - primitives::BlockInfo block) - : block(block), parent(ancestor) { - BOOST_ASSERT(ancestor != nullptr); - } - - std::shared_ptr BabeConfigNode::createAsRoot( - primitives::BlockInfo block, - std::shared_ptr config) { - auto fake_parent = std::make_shared(); - auto node = std::make_shared(fake_parent, block); - node->epoch = std::numeric_limitsepoch)>::max(); - node->config = std::move(config); - return node; - } - - std::shared_ptr BabeConfigNode::makeDescendant( - const primitives::BlockInfo &target_block, - std::optional target_epoch_number) const { - auto node = - std::make_shared(shared_from_this(), target_block); - node->epoch = target_epoch_number.value_or(epoch); - node->epoch_changed = node->epoch != epoch; - if (not node->epoch_changed) { - node->config = config; - node->next_config = next_config; - } else { - node->config = next_config.value_or(config); - node->next_config.reset(); - } - - return node; - } - -} // namespace kagome::consensus::babe diff --git a/core/consensus/babe/impl/babe_config_node.hpp b/core/consensus/babe/impl/babe_config_node.hpp deleted file mode 100644 index f1ed7f534a..0000000000 --- a/core/consensus/babe/impl/babe_config_node.hpp +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Copyright Soramitsu Co., Ltd. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef KAGOME_CONSENSUS_BABECONFIGNODE -#define KAGOME_CONSENSUS_BABECONFIGNODE - -#include - -#include "primitives/block.hpp" -#include "scale/scale.hpp" - -namespace kagome::consensus::babe { - - using IsBlockFinalized = Tagged; - - class BabeConfigNode final - : public std::enable_shared_from_this { - public: - BabeConfigNode() = default; - - BabeConfigNode(const std::shared_ptr &ancestor, - primitives::BlockInfo block); - - /// Creates node as root - /// @param block - target block - /// @param config - config associated with provided block - /// @result node - static std::shared_ptr createAsRoot( - primitives::BlockInfo block, - std::shared_ptr config); - - /// Creates descendant schedule node for block - /// @param block - target block - /// @param epoch_number - optional number to inform if provided block of - /// other epoch - /// @result descendant node - std::shared_ptr makeDescendant( - const primitives::BlockInfo &block, - std::optional epoch_number = std::nullopt) const; - - friend inline ::scale::ScaleEncoderStream &operator<<( - ::scale::ScaleEncoderStream &s, const BabeConfigNode &node) { - return s << node.block << node.epoch << node.config << node.next_config; - } - - friend inline ::scale::ScaleDecoderStream &operator>>( - ::scale::ScaleDecoderStream &s, BabeConfigNode &node) { - return s >> const_cast(node.block) >> node.epoch - >> node.config >> node.next_config; - } - - const primitives::BlockInfo block{}; - std::weak_ptr parent; - std::vector> descendants{}; - - EpochNumber epoch{}; - bool epoch_changed = false; - std::shared_ptr config; - std::optional> - next_config; - }; - -}; // namespace kagome::consensus::babe - -#endif // KAGOME_CONSENSUS_BABECONFIGNODE diff --git a/core/consensus/babe/impl/babe_config_repository_impl.cpp b/core/consensus/babe/impl/babe_config_repository_impl.cpp index 077dc955a9..713ef5ae2e 100644 --- a/core/consensus/babe/impl/babe_config_repository_impl.cpp +++ b/core/consensus/babe/impl/babe_config_repository_impl.cpp @@ -16,10 +16,27 @@ #include "primitives/block_header.hpp" #include "runtime/runtime_api/babe_api.hpp" #include "scale/scale.hpp" +#include "storage/map_prefix/prefix.hpp" #include "storage/predefined_keys.hpp" #include "storage/trie/trie_storage.hpp" +OUTCOME_CPP_DEFINE_CATEGORY(kagome::consensus::babe, + BabeConfigRepositoryImpl::Error, + e) { + using E = decltype(e); + switch (e) { + case E::ERROR: + return "error"; + } +} + namespace kagome::consensus::babe { + constexpr size_t kDontIndexFinalizedBlocks = 10000; + + inline primitives::NextConfigDataV1 getConfig( + const primitives::BabeConfiguration &state) { + return {state.leadership_rate, state.allowed_slots}; + } BabeConfigRepositoryImpl::BabeConfigRepositoryImpl( application::AppStateManager &app_state_manager, @@ -37,6 +54,13 @@ namespace kagome::consensus::babe { config_warp_sync_{app_config.syncMethod() == application::AppConfiguration::SyncMethod::Warp}, block_tree_(std::move(block_tree)), + indexer_{ + std::make_shared( + storage::kBabeConfigRepositoryImplIndexerPrefix, + persistent_storage_), + block_tree_, + {0, block_tree_->getGenesisBlockHash()}, + }, header_repo_(std::move(header_repo)), babe_api_(std::move(babe_api)), hasher_(std::move(hasher)), @@ -58,17 +82,32 @@ namespace kagome::consensus::babe { } bool BabeConfigRepositoryImpl::prepare() { - auto load_res = load(); - auto best_info = block_tree_->bestLeaf(); - auto best_block = block_tree_->getBlockHeader(best_info.hash).value(); - if ((not load_res or not config({best_info}, 0)) - and trie_storage_->getEphemeralBatchAt(best_block.state_root)) { - readFromState(best_info); - load_res = outcome::success(); + auto last_indexed = indexer_.removeUnfinalized(); + auto finalized = block_tree_->getLastFinalized(); + auto finalized_header = block_tree_->getBlockHeader(finalized.hash).value(); + if (finalized.number - last_indexed.number > kDontIndexFinalizedBlocks + and trie_storage_->getEphemeralBatchAt(finalized_header.state_root)) { + warp(finalized); + } + + auto genesis_res = config({block_tree_->getGenesisBlockHash(), 0}, false); + if (not genesis_res) { + SL_ERROR(logger_, "get config at genesis error: {}", genesis_res.error()); + return false; } - if (load_res.has_error()) { - SL_VERBOSE(logger_, "Can not load state: {}", load_res.error()); - return config_warp_sync_; + auto &genesis = genesis_res.value(); + slot_duration_ = genesis->slot_duration; + epoch_length_ = genesis->epoch_length; + + auto best = block_tree_->bestLeaf(); + auto best_header = block_tree_->getBlockHeader(best.hash).value(); + if (auto res = config(best, true); not res and not config_warp_sync_) { + SL_ERROR(logger_, "get config at best {} error: {}", best, res.error()); + if (not trie_storage_->getEphemeralBatchAt(best_header.state_root)) { + SL_ERROR(logger_, + "warp sync was not completed, restart with \"--sync Warp\""); + } + return false; } chain_sub_->subscribe(chain_sub_->generateSubscriptionSetId(), @@ -83,16 +122,10 @@ namespace kagome::consensus::babe { if (auto self = wp.lock()) { const auto &header = boost::get(event).get(); - auto hash = - self->hasher_->blake2b_256(scale::encode(header).value()); - - auto save_res = self->save(); - if (save_res.has_error()) { - SL_WARN(self->logger_, - "Can not save state at finalization: {}", - save_res.error()); - } - self->prune({header.number, hash}); + self->indexer_.filterUnfinalized(); + self->indexer_.writeFinalized(self->last_saved_state_block_, + header.number); + self->last_saved_state_block_ = header.number; } } }); @@ -100,381 +133,17 @@ namespace kagome::consensus::babe { return true; } - outcome::result BabeConfigRepositoryImpl::load() { - const auto finalized_block = block_tree_->getLastFinalized(); - - // First, look up slot number of block number 1 sync epochs - if (finalized_block.number > 0) { - OUTCOME_TRY(first_block_hash, block_tree_->getBlockHash(1)); - - OUTCOME_TRY(first_block_header, - block_tree_->getBlockHeader(first_block_hash)); - - auto babe_digest_res = getBabeDigests(first_block_header); - BOOST_ASSERT_MSG(babe_digest_res.has_value(), - "Any non genesis block must contain babe digest"); - auto first_slot_number = babe_digest_res.value().second.slot_number; - - syncEpoch([&] { return std::tuple(first_slot_number, true); }); - } - - // 1. Load last state - OUTCOME_TRY(encoded_last_state_opt, - persistent_storage_->tryGet( - storage::kBabeConfigRepoStateLookupKey("last"))); - - if (encoded_last_state_opt.has_value()) { - auto last_state_res = scale::decode>( - encoded_last_state_opt.value()); - - if (last_state_res.has_value()) { - auto &last_state = last_state_res.value(); - if (last_state->block.number <= finalized_block.number) { - root_ = std::move(last_state); - SL_DEBUG(logger_, - "State was initialized by last saved on block {}", - root_->block); - } else { - SL_WARN( - logger_, - "Last state not match with last finalized; Try to use savepoint"); - } - } else { - SL_WARN( - logger_, "Can not decode last state: {}", last_state_res.error()); - std::ignore = persistent_storage_->remove( - storage::kBabeConfigRepoStateLookupKey("last")); - } - } - - // 2. Load from last control point, if state is still not found - if (root_ == nullptr) { - for (auto block_number = - (finalized_block.number / kSavepointBlockInterval) - * kSavepointBlockInterval; - block_number > 0; - block_number -= kSavepointBlockInterval) { - OUTCOME_TRY(encoded_saved_state_opt, - persistent_storage_->tryGet( - storage::kBabeConfigRepoStateLookupKey(block_number))); - - if (not encoded_saved_state_opt.has_value()) { - continue; - } - - auto saved_state_res = scale::decode>( - encoded_saved_state_opt.value()); - - if (saved_state_res.has_error()) { - SL_WARN(logger_, - "Can not decode state saved on block {}: {}", - block_number, - saved_state_res.error()); - std::ignore = persistent_storage_->remove( - storage::kBabeConfigRepoStateLookupKey(block_number)); - continue; - } - - root_ = std::move(saved_state_res.value()); - SL_VERBOSE(logger_, - "State was initialized by savepoint on block {}", - root_->block); - break; - } - } - - // 3. Load state from genesis, if state is still not found - if (root_ == nullptr) { - auto genesis_hash = block_tree_->getGenesisBlockHash(); - auto babe_config_res = babe_api_->configuration(genesis_hash); - if (babe_config_res.has_error()) { - SL_WARN(logger_, - "Can't get babe config over babe API on genesis block: {}", - babe_config_res.error()); - return babe_config_res.as_failure(); - } - auto &babe_config = babe_config_res.value(); - - root_ = BabeConfigNode::createAsRoot( - {0, genesis_hash}, - std::make_shared( - std::move(babe_config))); - SL_VERBOSE(logger_, "State was initialized by genesis block"); - } - - BOOST_ASSERT_MSG(root_ != nullptr, "The root must be initialized by now"); - - // Init slot duration and epoch length - auto slot_duration = std::chrono::duration_cast( - root_->config->slot_duration); - BOOST_ASSERT_MSG(slot_duration.count() > 0, - "Slot duration must be greater zero"); - const_cast(slot_duration_) = slot_duration; - auto epoch_length = root_->config->epoch_length; - BOOST_ASSERT_MSG(epoch_length, "Epoch length must be greater zero"); - const_cast(epoch_length_) = epoch_length; - - // 4. Apply digests before last finalized - bool need_to_save = false; - for (auto block_number = root_->block.number + 1; - block_number <= finalized_block.number; - ++block_number) { - auto block_hash_res = block_tree_->getBlockHash(block_number); - if (block_hash_res.has_error()) { - SL_WARN(logger_, - "Can't get hash of an already finalized block #{}: {}", - block_number, - block_hash_res.error()); - return block_hash_res.as_failure(); - } - const auto &block_hash = block_hash_res.value(); - - auto block_header_res = block_tree_->getBlockHeader(block_hash); - if (block_header_res.has_error()) { - SL_WARN(logger_, - "Can't get header of an already finalized block #{}: {}", - block_number, - block_header_res.error()); - return block_header_res.as_failure(); - } - const auto &block_header = block_header_res.value(); - - primitives::BlockContext context{ - .block_info = {block_number, block_hash}}; - - for (auto &item : block_header.digest) { - auto res = visit_in_place( - item, - [&](const primitives::PreRuntime &msg) -> outcome::result { - if (msg.consensus_engine_id == primitives::kBabeEngineId) { - OUTCOME_TRY(digest_item, - scale::decode(msg.data)); - - return onDigest(context, digest_item); - } - return outcome::success(); - }, - [&](const primitives::Consensus &msg) -> outcome::result { - if (msg.consensus_engine_id == primitives::kBabeEngineId) { - OUTCOME_TRY(digest_item, - scale::decode(msg.data)); - - return onDigest(context, digest_item); - } - return outcome::success(); - }, - [](const auto &) { return outcome::success(); }); - if (res.has_error()) { - SL_WARN(logger_, - "Can't apply babe digest of finalized block {}: {}", - context.block_info, - res.error()); - return res.as_failure(); - } - } - - prune(context.block_info); - - if (context.block_info.number % (kSavepointBlockInterval / 10) == 0) { - // Make savepoint - auto save_res = save(); - if (save_res.has_error()) { - SL_WARN(logger_, "Can't re-make savepoint: {}", save_res.error()); - } else { - need_to_save = false; - } - } else { - need_to_save = true; - } - } - - // Save state on finalized part of blockchain - if (need_to_save) { - if (auto save_res = save(); save_res.has_error()) { - SL_WARN(logger_, "Can't re-save state: {}", save_res.error()); - } - } - - // 4. Collect and apply digests of non-finalized blocks - auto leaves = block_tree_->getLeaves(); - std::map>> - digests; - // 4.1 Collect digests - for (auto &leave_hash : leaves) { - for (auto hash = leave_hash;;) { - auto block_header_res = block_tree_->getBlockHeader(hash); - if (block_header_res.has_error()) { - SL_WARN(logger_, - "Can't get header of non-finalized block {}: {}", - hash, - block_header_res.error()); - return block_header_res.as_failure(); - } - const auto &block_header = block_header_res.value(); - - // This block is finalized - if (block_header.number <= finalized_block.number) { - break; - } - - primitives::BlockContext context{ - .block_info = {block_header.number, hash}}; - - // This block was meet earlier - if (digests.find(context) != digests.end()) { - break; - } - - auto &digest_of_block = digests[context]; - - // Search and collect babe digests - for (auto &item : block_header.digest) { - auto res = visit_in_place( - item, - [&](const primitives::PreRuntime &msg) -> outcome::result { - if (msg.consensus_engine_id == primitives::kBabeEngineId) { - auto res = - scale::decode(msg.data); - if (res.has_error()) { - return res.as_failure(); - } - const auto &digest_item = res.value(); - - digest_of_block.emplace_back(digest_item); - } - return outcome::success(); - }, - [&](const primitives::Consensus &msg) -> outcome::result { - if (msg.consensus_engine_id == primitives::kBabeEngineId) { - auto res = scale::decode(msg.data); - if (res.has_error()) { - return res.as_failure(); - } - const auto &digest_item = res.value(); - - digest_of_block.emplace_back(digest_item); - } - return outcome::success(); - }, - [](const auto &) { return outcome::success(); }); - if (res.has_error()) { - SL_WARN(logger_, - "Can't collect babe digest of non-finalized block {}: {}", - context.block_info, - res.error()); - return res.as_failure(); - } - } - - hash = block_header.parent_hash; - } - } - // 4.2 Apply digests - for (const auto &[context_tmp, digests_of_block] : digests) { - const auto &context = context_tmp; - for (const auto &digest : digests_of_block) { - auto res = visit_in_place(digest, [&](const auto &digest_item) { - return onDigest(context, digest_item); - }); - if (res.has_error()) { - SL_WARN(logger_, - "Can't apply babe digest of non-finalized block {}: {}", - context.block_info, - res.error()); - return res.as_failure(); - } - } - } - - prune(finalized_block); - - return outcome::success(); - } - - outcome::result BabeConfigRepositoryImpl::save() { - const auto finalized_block = block_tree_->getLastFinalized(); - - BOOST_ASSERT(last_saved_state_block_ <= finalized_block.number); - - auto saving_state_node = getNode({.block_info = finalized_block}); - BOOST_ASSERT_MSG(saving_state_node != nullptr, - "Finalized block must have associated node"); - const auto saving_state_block = saving_state_node->block; - - // Does not need to save - if (last_saved_state_block_ >= saving_state_block.number) { - return outcome::success(); - } - - const auto last_savepoint = - (last_saved_state_block_ / kSavepointBlockInterval) - * kSavepointBlockInterval; - - const auto new_savepoint = - (saving_state_block.number / kSavepointBlockInterval) - * kSavepointBlockInterval; - - // It's time to make savepoint - if (new_savepoint > last_savepoint) { - auto hash_res = header_repo_->getHashByNumber(new_savepoint); - if (hash_res.has_value()) { - primitives::BlockInfo savepoint_block(new_savepoint, hash_res.value()); - - auto ancestor_node = getNode({.block_info = savepoint_block}); - if (ancestor_node != nullptr) { - auto node = ancestor_node->block == savepoint_block - ? ancestor_node - : ancestor_node->makeDescendant(savepoint_block); - auto res = persistent_storage_->put( - storage::kBabeConfigRepoStateLookupKey(new_savepoint), - storage::Buffer(scale::encode(node).value())); - if (res.has_error()) { - SL_WARN(logger_, - "Can't make savepoint on block {}: {}", - savepoint_block, - hash_res.error()); - return res.as_failure(); - } - SL_DEBUG(logger_, "Savepoint has made on block {}", savepoint_block); - } - } else { - SL_WARN(logger_, - "Can't take hash of savepoint block {}: {}", - new_savepoint, - hash_res.error()); - } - } - - auto res = persistent_storage_->put( - storage::kBabeConfigRepoStateLookupKey("last"), - storage::Buffer(scale::encode(saving_state_node).value())); - if (res.has_error()) { - SL_WARN(logger_, - "Can't save last state on block {}: {}", - saving_state_block, - res.error()); - return res.as_failure(); - } - SL_DEBUG(logger_, "Last state has saved on block {}", saving_state_block); - - last_saved_state_block_ = saving_state_block.number; - - return outcome::success(); - } - - std::optional> - BabeConfigRepositoryImpl::config(const primitives::BlockContext &context, + outcome::result> + BabeConfigRepositoryImpl::config(const primitives::BlockInfo &parent_info, EpochNumber epoch_number) const { - auto node = getNode(context); - if (node) { - if (epoch_number > node->epoch) { - return *node->next_config.value_or(node->config); - } - return *node->config; - } - return std::nullopt; + auto epoch_changed = true; + if (parent_info.number != 0) { + OUTCOME_TRY(parent_header, block_tree_->getBlockHeader(parent_info.hash)); + OUTCOME_TRY(parent_digest, getBabeDigests(parent_header)); + auto parent_epoch = slotToEpoch(parent_digest.second.slot_number); + epoch_changed = epoch_number != parent_epoch; + } + return config(parent_info, epoch_changed); } BabeDuration BabeConfigRepositoryImpl::slotDuration() const { @@ -488,286 +157,6 @@ namespace kagome::consensus::babe { return epoch_length_; } - outcome::result BabeConfigRepositoryImpl::onDigest( - const primitives::BlockContext &context, - const consensus::babe::BabeBlockHeader &digest) { - EpochNumber epoch_number = slotToEpoch(digest.slot_number); - - auto node = getNode(context); - BOOST_ASSERT(node != nullptr); - - SL_LOG(logger_, - node->epoch != epoch_number ? log::Level::DEBUG : log::Level::TRACE, - "BabeBlockHeader babe-digest on block {}: " - "slot {}, epoch {}, authority #{}, {}", - context.block_info, - digest.slot_number, - epoch_number, - digest.authority_index, - to_string(digest.slotType())); - - if (node->block == context.block_info) { - return BabeError::BAD_ORDER_OF_DIGEST_ITEM; - } - - // Create descendant if and only if epoch is changed - if (node->epoch != epoch_number) { - auto new_node = node->makeDescendant(context.block_info, epoch_number); - - node->descendants.emplace_back(std::move(new_node)); - } - - return outcome::success(); - } - - outcome::result BabeConfigRepositoryImpl::onDigest( - const primitives::BlockContext &context, - const primitives::BabeDigest &digest) { - return visit_in_place( - digest, - [&](const primitives::NextEpochData &msg) -> outcome::result { - SL_DEBUG(logger_, - "NextEpochData babe-digest on block {}: " - "{} authorities, randomness {}", - context.block_info, - msg.authorities.size(), - msg.randomness); - return onNextEpochData(context, msg); - }, - [&](const primitives::OnDisabled &msg) { - SL_TRACE( - logger_, - "OnDisabled babe-digest on block {}: " - "disable authority #{}; ignored (it is checked only by runtime)", - context.block_info, - msg.authority_index); - // Implemented sending of OnDisabled events before actually preventing - // disabled validators from authoring, so it's possible that there are - // blocks on the chain that came from disabled validators (before they - // were booted from the set at the end of epoch). Currently, the - // runtime prevents disabled validators from authoring (it will just - // panic), so we don't do any client-side handling in substrate - // https://matrix.to/#/!oZltgdfyakVMtEAWCI:web3.foundation/$hArAlUKaxvquGdaRG9W8ihcsNrO6wD4Q2CQjDIb3MMY?via=web3.foundation&via=matrix.org&via=matrix.parity.io - return outcome::success(); - }, - [&](const primitives::NextConfigData &msg) { - return visit_in_place( - msg, - [&](const primitives::NextConfigDataV1 &msg) { - SL_DEBUG(logger_, - "NextConfigData babe-digest on block {}: " - "ratio={}/{}, second_slot={}", - context.block_info, - msg.ratio.first, - msg.ratio.second, - to_string(msg.second_slot)); - return onNextConfigData(context, msg); - }, - [&](const auto &) { - SL_WARN(logger_, - "Unsupported NextConfigData babe-digest on block {}: " - "variant #{}", - context.block_info, - digest.which()); - return BabeError::UNKNOWN_DIGEST_TYPE; - }); - }, - [&](auto &) { - SL_WARN(logger_, - "Unsupported babe-digest on block {}: variant #{}", - context.block_info, - digest.which()); - return BabeError::UNKNOWN_DIGEST_TYPE; - }); - } - - outcome::result BabeConfigRepositoryImpl::onNextEpochData( - const primitives::BlockContext &context, - const primitives::NextEpochData &msg) { - auto node = getNode(context); - - if (node->block != context.block_info) { - return BabeError::BAD_ORDER_OF_DIGEST_ITEM; - } - - auto config = node->next_config.value_or(node->config); - - if (config->authorities != msg.authorities - or config->randomness != msg.randomness) { - auto new_config = - std::make_shared(*config); - new_config->authorities = msg.authorities; - new_config->randomness = msg.randomness; - node->next_config = std::move(new_config); - } - - return outcome::success(); - } - - outcome::result BabeConfigRepositoryImpl::onNextConfigData( - const primitives::BlockContext &context, - const primitives::NextConfigDataV1 &msg) { - auto node = getNode(context); - - if (node->block != context.block_info) { - return BabeError::BAD_ORDER_OF_DIGEST_ITEM; - } - - auto config = node->next_config.value_or(node->config); - - if (config->leadership_rate != msg.ratio - or config->allowed_slots != msg.second_slot) { - auto new_config = - std::make_shared(*config); - new_config->leadership_rate = msg.ratio; - new_config->allowed_slots = msg.second_slot; - node->next_config = std::move(new_config); - } - - return outcome::success(); - } - - std::shared_ptr BabeConfigRepositoryImpl::getNode( - const primitives::BlockContext &context) const { - BOOST_ASSERT(root_ != nullptr); - - // Lazy getter of direct chain best block ('cause it may be not used) - auto get_block = - [&, block = std::optional()]() mutable { - if (not block.has_value()) { - if (context.header.has_value()) { - const auto &header = context.header.value().get(); - block.emplace(header.number - 1, header.parent_hash); - } else { - block.emplace(context.block_info); - } - } - return block.value(); - }; - - // Target block is not descendant of the current root - if (root_->block.number > context.block_info.number - || (root_->block != context.block_info - && not directChainExists(root_->block, get_block()))) { - return nullptr; - } - - std::shared_ptr ancestor = root_; - while (ancestor->block != context.block_info) { - bool goto_next_generation = false; - for (const auto &node : ancestor->descendants) { - if (node->block == context.block_info) { - return node; - } - if (directChainExists(node->block, get_block())) { - ancestor = node; - goto_next_generation = true; - break; - } - } - if (not goto_next_generation) { - break; - } - } - return ancestor; - } - - bool BabeConfigRepositoryImpl::directChainExists( - const primitives::BlockInfo &ancestor, - const primitives::BlockInfo &descendant) const { - SL_TRACE(logger_, - "Looking if direct chain exists between {} and {}", - ancestor, - descendant); - // Check if it's one-block chain - if (ancestor == descendant) { - return true; - } - // Any block is descendant of genesis - if (ancestor.number == 0) { - return true; - } - // No direct chain if order is wrong - if (ancestor.number > descendant.number) { - return false; - } - auto result = block_tree_->hasDirectChain(ancestor.hash, descendant.hash); - return result; - } - - void BabeConfigRepositoryImpl::prune(const primitives::BlockInfo &block) { - if (block == root_->block) { - return; - } - - if (block.number < root_->block.number) { - return; - } - - auto node = getNode({.block_info = block}); - - if (not node) { - return; - } - - if (node->block != block) { - // Reorganize ancestry - auto new_node = node->makeDescendant(block); - auto descendants = std::move(node->descendants); - for (auto &descendant : descendants) { - if (directChainExists(block, descendant->block)) { - new_node->descendants.emplace_back(std::move(descendant)); - } - } - node = std::move(new_node); - } - - root_ = std::move(node); - - SL_TRACE(logger_, "Prune upto block {}", block); - } - - void BabeConfigRepositoryImpl::cancel(const primitives::BlockInfo &block) { - auto ancestor = getNode({.block_info = block}); - - if (ancestor == nullptr) { - SL_TRACE(logger_, "Can't remove node of block {}: no ancestor", block); - return; - } - - if (ancestor == root_) { - // Can't remove root - SL_TRACE(logger_, "Can't remove node of block {}: it is root", block); - return; - } - - if (ancestor->block == block) { - ancestor = - std::const_pointer_cast(ancestor->parent.lock()); - BOOST_ASSERT_MSG(ancestor != nullptr, "Non root node must have a parent"); - } - - auto it = std::find_if(ancestor->descendants.begin(), - ancestor->descendants.end(), - [&](std::shared_ptr node) { - return node->block == block; - }); - - if (it != ancestor->descendants.end()) { - if (not(*it)->descendants.empty()) { - // Has descendants - is not a leaf - SL_TRACE(logger_, - "Can't remove node of block {}: " - "not found such descendant of ancestor", - block); - return; - } - - ancestor->descendants.erase(it); - SL_DEBUG(logger_, "Node of block {} has removed", block); - } - } - BabeSlotNumber BabeConfigRepositoryImpl::syncEpoch( std::function()> &&f) { if (not is_first_block_finalized_) { @@ -839,47 +228,147 @@ namespace kagome::consensus::babe { return 0; } - void BabeConfigRepositoryImpl::readFromState( - const primitives::BlockInfo &block) { - if (auto r = readFromStateOutcome(block); not r) { - logger_->error("readFromState {}, error {}", block, r.error()); + void BabeConfigRepositoryImpl::warp(const primitives::BlockInfo &block) { + indexer_.put(block, {}, true); + } + + outcome::result> + BabeConfigRepositoryImpl::config(const primitives::BlockInfo &block, + bool next) const { + auto descent = indexer_.descend(block); + outcome::result cb_res = outcome::success(); + auto cb = [&](std::optional prev, + size_t i_first, + size_t i_last) { + cb_res = [&]() -> outcome::result { + BOOST_ASSERT(i_first >= i_last); + auto info = descent.path_.at(i_first); + std::shared_ptr prev_state; + if (not prev) { + OUTCOME_TRY(_state, babe_api_->configuration(info.hash)); + auto state = std::make_shared( + std::move(_state)); + BabeIndexedValue value{getConfig(*state), state, state}; + if (info.number == 0) { + indexer_.put(info, {value, std::nullopt}, true); + } else { + std::vector refs; + while (true) { + OUTCOME_TRY(header, block_tree_->getBlockHeader(info.hash)); + if (HasAuthoritySetChange digests{header}) { + value.next_state = applyDigests(value.config, digests); + indexer_.put(info, {value, std::nullopt}, true); + if (not refs.empty()) { + indexer_.remove(refs.front()); + } + break; + } + refs.emplace_back(info); + info = *header.parentInfo(); + } + std::reverse(refs.begin(), refs.end()); + for (auto &block : refs) { + indexer_.put(block, {std::nullopt, info, true}, false); + } + } + if (i_first == i_last) { + return outcome::success(); + } + prev = info; + prev_state = *value.next_state; + --i_first; + } + while (true) { + info = descent.path_.at(i_first); + OUTCOME_TRY(header, block_tree_->getBlockHeader(info.hash)); + if (HasAuthoritySetChange digests{header}) { + if (not prev_state) { + BOOST_OUTCOME_TRY(prev_state, loadPrev(prev)); + } + auto state = applyDigests(getConfig(*prev_state), digests); + BabeIndexedValue value{getConfig(*state), std::nullopt, state}; + indexer_.put(info, {value, prev}, block_tree_->isFinalized(info)); + prev = info; + prev_state = state; + } else { + indexer_.put(info, {std::nullopt, prev, true}, false); + } + if (i_first == i_last) { + break; + } + --i_first; + } + return outcome::success(); + }(); + }; + auto r = indexer_.search(descent, block, cb); + OUTCOME_TRY(cb_res); + if (not r) { + return Error::ERROR; + } + if (not next and r->second.value->state) { + return *r->second.value->state; + } + if (next) { + OUTCOME_TRY(load(r->first, r->second)); + return *r->second.value->next_state; } + if (not r->second.prev) { + return Error::ERROR; + } + return loadPrev(*r->second.prev); + } + + std::shared_ptr + BabeConfigRepositoryImpl::applyDigests( + const primitives::NextConfigDataV1 &config, + const HasAuthoritySetChange &digests) const { + BOOST_ASSERT(digests); + auto state = std::make_shared(); + state->slot_duration = slot_duration_; + state->epoch_length = epoch_length_; + if (digests.config) { + state->leadership_rate = digests.config->ratio; + state->allowed_slots = digests.config->second_slot; + } else { + state->leadership_rate = config.ratio; + state->allowed_slots = config.second_slot; + } + state->authorities = digests.epoch->authorities; + state->randomness = digests.epoch->randomness; + return state; } - outcome::result BabeConfigRepositoryImpl::readFromStateOutcome( - const primitives::BlockInfo &block) { - OUTCOME_TRY(hash1, block_tree_->getBlockHash(1)); - OUTCOME_TRY(header1, block_tree_->getBlockHeader(hash1)); - OUTCOME_TRY(header, block_tree_->getBlockHeader(block.hash)); - auto parent = header; - std::optional next_epoch; - while (parent.number != 0) { - if (auto _digest = getNextEpochDigest(parent)) { - next_epoch = std::move(_digest.value()); - break; + outcome::result BabeConfigRepositoryImpl::load( + const primitives::BlockInfo &block, + blockchain::Indexed &item) const { + if (not item.value->next_state) { + if (block.number == 0) { + BOOST_ASSERT(item.value->state); + item.value->next_state = item.value->state; + } else { + OUTCOME_TRY(header, block_tree_->getBlockHeader(block.hash)); + item.value->next_state = applyDigests(item.value->config, {header}); + indexer_.put(block, item, false); } - OUTCOME_TRY(header, block_tree_->getBlockHeader(parent.parent_hash)); - parent = std::move(header); } - OUTCOME_TRY(config, babe_api_->configuration(block.hash)); - root_ = BabeConfigNode::createAsRoot( - block, - std::make_shared(std::move(config))); - if (block.number != 0) { - OUTCOME_TRY(digests, getBabeDigests(header)); - root_->epoch = slotToEpoch(digests.second.slot_number); + return outcome::success(); + } + + outcome::result> + BabeConfigRepositoryImpl::loadPrev( + const std::optional &prev) const { + if (not prev) { + return Error::ERROR; + } + auto r = indexer_.get(*prev); + if (not r) { + return Error::ERROR; } - if (next_epoch) { - auto config = - std::make_shared(*root_->config); - config->authorities = std::move(next_epoch->authorities); - config->randomness = next_epoch->randomness; - root_->next_config = config; + if (not r->value) { + return Error::ERROR; } - OUTCOME_TRY( - persistent_storage_->put(storage::kBabeConfigRepoStateLookupKey("last"), - scale::encode(root_).value())); - SL_INFO(logger_, "Read state at {}", block); - return outcome::success(); + OUTCOME_TRY(load(*prev, *r)); + return *r->value->next_state; } } // namespace kagome::consensus::babe diff --git a/core/consensus/babe/impl/babe_config_repository_impl.hpp b/core/consensus/babe/impl/babe_config_repository_impl.hpp index b2a9ee13f3..89c0e2a9f0 100644 --- a/core/consensus/babe/impl/babe_config_repository_impl.hpp +++ b/core/consensus/babe/impl/babe_config_repository_impl.hpp @@ -7,13 +7,14 @@ #define KAGOME_CONSENSUS_BABE_BABECONFIGREPOSITORYIMPL #include "consensus/babe/babe_config_repository.hpp" -#include "consensus/babe/babe_digest_observer.hpp" #include "consensus/babe/babe_util.hpp" -#include "consensus/babe/impl/babe_config_node.hpp" +#include "blockchain/indexer.hpp" +#include "consensus/babe/has_authority_set_change.hpp" #include "log/logger.hpp" #include "primitives/block_data.hpp" #include "primitives/event_types.hpp" +#include "primitives/scheduled_change.hpp" #include "storage/spaced_storage.hpp" namespace kagome::application { @@ -35,15 +36,24 @@ namespace kagome::storage::trie { } // namespace kagome::storage::trie namespace kagome::consensus::babe { + struct BabeIndexedValue { + SCALE_TIE_ONLY(config, state); + + primitives::NextConfigDataV1 config; + std::optional> state; + std::optional> + next_state; + }; class BabeConfigRepositoryImpl final : public BabeConfigRepository, - public BabeDigestObserver, public BabeUtil, public std::enable_shared_from_this { - static const primitives::BlockNumber kSavepointBlockInterval = 100000; - public: + enum class Error { + ERROR = 1, + }; + BabeConfigRepositoryImpl( application::AppStateManager &app_state_manager, std::shared_ptr persistent_storage, @@ -58,29 +68,16 @@ namespace kagome::consensus::babe { bool prepare(); - // BabeDigestObserver - - outcome::result onDigest(const primitives::BlockContext &context, - const BabeBlockHeader &digest) override; - - outcome::result onDigest( - const primitives::BlockContext &context, - const primitives::BabeDigest &digest) override; - - void cancel(const primitives::BlockInfo &block) override; - // BabeConfigRepository BabeDuration slotDuration() const override; EpochLength epochLength() const override; - std::optional> - config(const primitives::BlockContext &context, + outcome::result> + config(const primitives::BlockInfo &parent_info, EpochNumber epoch_number) const override; - void readFromState(const primitives::BlockInfo &block) override; - // BabeUtil BabeSlotNumber syncEpoch( @@ -96,55 +93,38 @@ namespace kagome::consensus::babe { EpochNumber slotToEpoch(BabeSlotNumber slot) const override; BabeSlotNumber slotInEpoch(BabeSlotNumber slot) const override; - private: - outcome::result load(); - outcome::result save(); - - void prune(const primitives::BlockInfo &block); - - outcome::result onNextEpochData( - const primitives::BlockContext &context, - const primitives::NextEpochData &msg); - - outcome::result onNextConfigData( - const primitives::BlockContext &context, - const primitives::NextConfigDataV1 &msg); - - /** - * @brief Find node according to the block - * @param block for which to find the schedule node - * @return oldest node according to the block - */ - std::shared_ptr getNode( - const primitives::BlockContext &context) const; - - /** - * @brief Check if one block is direct ancestor of second one - * @param ancestor - hash of block, which is at the top of the chain - * @param descendant - hash of block, which is the bottom of the chain - * @return true if \param ancestor is direct ancestor of \param descendant - */ - bool directChainExists(const primitives::BlockInfo &ancestor, - const primitives::BlockInfo &descendant) const; + void warp(const primitives::BlockInfo &block) override; + private: BabeSlotNumber getFirstBlockSlotNumber(); - outcome::result readFromStateOutcome( - const primitives::BlockInfo &block); + outcome::result> + config(const primitives::BlockInfo &block, bool next) const; + + std::shared_ptr applyDigests( + const primitives::NextConfigDataV1 &config, + const HasAuthoritySetChange &digests) const; + + outcome::result load( + const primitives::BlockInfo &block, + blockchain::Indexed &item) const; + + outcome::result> + loadPrev(const std::optional &prev) const; std::shared_ptr persistent_storage_; bool config_warp_sync_; std::shared_ptr block_tree_; + mutable blockchain::Indexer indexer_; std::shared_ptr header_repo_; std::shared_ptr babe_api_; std::shared_ptr hasher_; std::shared_ptr trie_storage_; std::shared_ptr chain_sub_; - const BabeDuration slot_duration_{}; - const EpochLength epoch_length_{}; + BabeDuration slot_duration_{}; + EpochLength epoch_length_{}; - std::shared_ptr root_; primitives::BlockNumber last_saved_state_block_ = 0; const BabeClock &clock_; @@ -156,4 +136,7 @@ namespace kagome::consensus::babe { } // namespace kagome::consensus::babe +OUTCOME_HPP_DECLARE_ERROR(kagome::consensus::babe, + BabeConfigRepositoryImpl::Error) + #endif // KAGOME_CONSENSUS_BABE_BABECONFIGREPOSITORYIMPL diff --git a/core/consensus/babe/impl/babe_digests_util.cpp b/core/consensus/babe/impl/babe_digests_util.cpp index 4bfb4d88d0..212737c730 100644 --- a/core/consensus/babe/impl/babe_digests_util.cpp +++ b/core/consensus/babe/impl/babe_digests_util.cpp @@ -15,10 +15,6 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::consensus::babe, DigestError, e) { "header and seal digests"; case E::NO_TRAILING_SEAL_DIGEST: return "the block must contain a seal digest as the last digest"; - case E::MULTIPLE_EPOCH_CHANGE_DIGESTS: - return "the block contains multiple epoch change digests"; - case E::NEXT_EPOCH_DIGEST_DOES_NOT_EXIST: - return "next epoch digest does not exist"; } return "unknown error"; } @@ -56,38 +52,4 @@ namespace kagome::consensus::babe { return DigestError::REQUIRED_DIGESTS_NOT_FOUND; } - - outcome::result getNextEpochDigest( - const primitives::BlockHeader &header) { - // https://github.com/paritytech/substrate/blob/d8df977d024ebeb5330bacac64cf7193a7c242ed/core/consensus/babe/src/lib.rs#L497 - outcome::result epoch_digest = - DigestError::NEXT_EPOCH_DIGEST_DOES_NOT_EXIST; - - for (const auto &log : header.digest) { - visit_in_place( - log, - [&epoch_digest](const primitives::Consensus &consensus) { - if (consensus.consensus_engine_id == primitives::kBabeEngineId) { - auto consensus_log_res = - scale::decode(consensus.data); - if (not consensus_log_res) { - return; - } - - visit_in_place( - consensus_log_res.value(), - [&epoch_digest](const primitives::NextEpochData &next_epoch) { - if (not epoch_digest) { - epoch_digest = static_cast(next_epoch); - } else { - epoch_digest = DigestError::MULTIPLE_EPOCH_CHANGE_DIGESTS; - } - }, - [](const auto &) {}); - } - }, - [](const auto &) {}); - } - return epoch_digest; - } } // namespace kagome::consensus::babe diff --git a/core/consensus/babe/impl/babe_digests_util.hpp b/core/consensus/babe/impl/babe_digests_util.hpp index a225cf5da1..42bb858629 100644 --- a/core/consensus/babe/impl/babe_digests_util.hpp +++ b/core/consensus/babe/impl/babe_digests_util.hpp @@ -21,8 +21,6 @@ namespace kagome::consensus::babe { enum class DigestError { REQUIRED_DIGESTS_NOT_FOUND = 1, NO_TRAILING_SEAL_DIGEST, - MULTIPLE_EPOCH_CHANGE_DIGESTS, - NEXT_EPOCH_DIGEST_DOES_NOT_EXIST }; template @@ -39,10 +37,6 @@ namespace kagome::consensus::babe { outcome::result> getBabeDigests( const primitives::BlockHeader &header); - - outcome::result getNextEpochDigest( - const primitives::BlockHeader &header); - } // namespace kagome::consensus::babe OUTCOME_HPP_DECLARE_ERROR(kagome::consensus::babe, DigestError) diff --git a/core/consensus/babe/impl/babe_impl.cpp b/core/consensus/babe/impl/babe_impl.cpp index 033d0d1d67..9617a20e04 100644 --- a/core/consensus/babe/impl/babe_impl.cpp +++ b/core/consensus/babe/impl/babe_impl.cpp @@ -226,9 +226,9 @@ namespace kagome::consensus::babe { current_epoch_.epoch_number, current_epoch_.start_slot); - auto babe_config = babe_config_repo_->config({.block_info = best_block_}, - current_epoch_.epoch_number); - if (not babe_config.has_value()) { + auto babe_config = + babe_config_repo_->config(best_block_, current_epoch_.epoch_number); + if (not babe_config and app_config_.syncMethod() != SyncMethod::Warp) { SL_CRITICAL( log_, "Can't obtain digest of epoch {} from block tree for block {}", @@ -236,11 +236,14 @@ namespace kagome::consensus::babe { best_block_); return false; } - const auto &authorities = babe_config->get().authorities; - if (authorities.size() == 1 && session_keys_->getBabeKeyPair(authorities)) { - SL_INFO(log_, "Starting single validating node."); - onSynchronized(); - return true; + if (babe_config) { + const auto &authorities = babe_config.value()->authorities; + if (authorities.size() == 1 + && session_keys_->getBabeKeyPair(authorities)) { + SL_INFO(log_, "Starting single validating node."); + onSynchronized(); + return true; + } } switch (app_config_.syncMethod()) { @@ -354,10 +357,9 @@ namespace kagome::consensus::babe { SL_DEBUG(log_, "Starting an epoch {}. Secondary slots allowed={}", epoch.epoch_number, - babe_config_repo_ - ->config({.block_info = best_block_}, epoch.epoch_number) - ->get() - .isSecondarySlotsAllowed()); + babe_config_repo_->config(best_block_, epoch.epoch_number) + .value() + ->isSecondarySlotsAllowed()); current_epoch_ = epoch; current_slot_ = current_epoch_.start_slot; @@ -694,7 +696,6 @@ namespace kagome::consensus::babe { } self->adjustEpochDescriptor(); - self->babe_config_repo_->readFromState(block_at_state); self->justification_observer_->reload(); self->block_tree_->notifyBestAndFinalized(); @@ -803,10 +804,10 @@ namespace kagome::consensus::babe { BOOST_ASSERT(babe_digests_res.has_value()); } - auto babe_config_opt = babe_config_repo_->config( - {.block_info = best_block_}, current_epoch_.epoch_number); + auto babe_config_opt = + babe_config_repo_->config(best_block_, current_epoch_.epoch_number); if (babe_config_opt) { - auto &babe_config = babe_config_opt.value().get(); + auto &babe_config = *babe_config_opt.value(); auto keypair = session_keys_->getBabeKeyPair(babe_config.authorities); if (not keypair) { SL_ERROR(log_, diff --git a/core/consensus/babe/impl/block_appender_base.cpp b/core/consensus/babe/impl/block_appender_base.cpp index 4fb1475511..28ada59221 100644 --- a/core/consensus/babe/impl/block_appender_base.cpp +++ b/core/consensus/babe/impl/block_appender_base.cpp @@ -219,11 +219,10 @@ namespace kagome::consensus::babe { return digest_tracking_res.as_failure(); } - auto babe_config_opt = babe_config_repo_->config(context, epoch_number); - if (!babe_config_opt.has_value()) { - return BlockAdditionError::ORPHAN_BLOCK; - } - auto &babe_config = babe_config_opt.value().get(); + OUTCOME_TRY( + babe_config_opt, + babe_config_repo_->config(*block.header.parentInfo(), epoch_number)); + auto &babe_config = *babe_config_opt; SL_TRACE(logger_, "Actual epoch digest to apply block {} (slot {}, epoch {}). " diff --git a/core/consensus/grandpa/structs.hpp b/core/consensus/grandpa/structs.hpp index 82f2c7420d..6397740bb4 100644 --- a/core/consensus/grandpa/structs.hpp +++ b/core/consensus/grandpa/structs.hpp @@ -8,7 +8,6 @@ #include "common/visitor.hpp" #include "consensus/grandpa/common.hpp" -#include "log/logger.hpp" #include "primitives/block_header.hpp" #include "primitives/common.hpp" @@ -136,34 +135,14 @@ namespace kagome::consensus::grandpa { // justification that contains a list of signed precommits justifying the // validity of the block struct GrandpaJustification { + SCALE_TIE(4); + RoundNumber round_number; primitives::BlockInfo block_info; std::vector items{}; std::vector votes_ancestries{}; }; - template > - Stream &operator<<(Stream &s, const GrandpaJustification &v) { - return s << v.round_number << v.block_info << v.items << v.votes_ancestries; - } - - template > - Stream &operator>>(Stream &s, GrandpaJustification &v) { - s >> v.round_number >> v.block_info >> v.items; - // TODO(turuslan): remove after merging - // https://github.com/soramitsu/kagome/pull/1491 - if (not s.hasMore(1)) { - log::createLogger("GrandpaJustification") - ->error( - "decode error, missing `votes_ancestries`. Remove database files " - "and re-sync your node."); - } - s >> v.votes_ancestries; - return s; - } - /// A commit message which is an aggregate of precommits. struct Commit { primitives::BlockInfo vote; diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index 4a43ba75cf..8fb14b9511 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -533,240 +533,224 @@ namespace { host_api::OffchainExtensionConfig offchain_ext_config{ config->isOffchainIndexingEnabled()}; - return di:: - make_injector( - // bind configs - useConfig(rpc_thread_pool_config), - useConfig(ws_config), - useConfig(pool_moderator_config), - useConfig(tp_pool_limits), - useConfig(ping_config), - useConfig(offchain_ext_config), - - // inherit host injector - libp2p::injector::makeHostInjector( - libp2p::injector::useWssPem(config->nodeWssPem()), - libp2p::injector::useSecurityAdaptors< - libp2p::security::Noise>()[di::override]), - - // inherit kademlia injector - libp2p::injector::makeKademliaInjector(), - bind_by_lambda( - [random_walk{config->getRandomWalkInterval()}]( - const auto &injector) { - auto &chain_spec = - injector.template create(); - return get_kademlia_config(chain_spec, random_walk); - })[boost::di::override], - - di::bind.template to(), - di::bind.to(config), - bind_by_lambda( - [](auto &&injector) { - return std::const_pointer_cast< - primitives::CodeSubstituteBlockIds>( - injector.template create() - .codeSubstitutes()); - }), - - // compose peer keypair - bind_by_lambda([](const auto &injector) { - auto &app_config = - injector - .template create(); - auto &chain = - injector.template create(); - auto &crypto_provider = - injector.template create(); - auto &csprng = injector.template create(); - auto &crypto_store = - injector.template create(); - return injector::get_peer_keypair( - app_config, chain, crypto_provider, csprng, crypto_store); + return di::make_injector( + // bind configs + useConfig(rpc_thread_pool_config), + useConfig(ws_config), + useConfig(pool_moderator_config), + useConfig(tp_pool_limits), + useConfig(ping_config), + useConfig(offchain_ext_config), + + // inherit host injector + libp2p::injector::makeHostInjector( + libp2p::injector::useWssPem(config->nodeWssPem()), + libp2p::injector::useSecurityAdaptors< + libp2p::security::Noise>()[di::override]), + + // inherit kademlia injector + libp2p::injector::makeKademliaInjector(), + bind_by_lambda( + [random_walk{config->getRandomWalkInterval()}]( + const auto &injector) { + auto &chain_spec = + injector.template create(); + return get_kademlia_config(chain_spec, random_walk); })[boost::di::override], - di::bind() // NOLINT - .template to(), - di::bind() // NOLINT - .template to(), - - // starting metrics interfaces - di::bind.template to(), - di::bind.template to(), - di::bind.to([](const auto - &injector) { - return metrics::Exposer::Configuration{ - injector - .template create() - .openmetricsHttpEndpoint()}; - }), - // hardfix for Mac clang - di::bind.to( - [](const auto &injector) { - return metrics::Session::Configuration{}; - }), - // ending metrics interfaces - di::bind.template to(), - di::bind.to(config->roles()), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - bind_by_lambda([](const auto &injector) { - const application::AppConfiguration &config = - injector - .template create(); - auto chain_spec = - injector.template create>(); - BOOST_ASSERT( // since rocksdb is the only possible option now - config.storageBackend() - == application::AppConfiguration::StorageBackend::RocksDB); - return get_rocks_db(config, chain_spec); - }), - bind_by_lambda([](const auto &injector) { - auto root = - injector::calculate_genesis_state( - injector - .template create(), - injector - .template create(), - injector - .template create()) - .value(); - const auto &hasher = - injector.template create>(); - const auto &storage = - injector.template create>(); - return blockchain::BlockStorageImpl::create(root, storage, hasher) - .value(); - }), - di::bind.template to(), - bind_by_lambda( - [](const auto &injector) { return get_block_tree(injector); }), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - bind_by_lambda([](const auto &injector) { - const application::AppConfiguration &config = - injector - .template create(); - auto chain_spec = - injector.template create>(); + di::bind.template to(), + di::bind.to(config), + bind_by_lambda([](auto &&injector) { + return std::const_pointer_cast( + injector.template create() + .codeSubstitutes()); + }), - return get_key_file_storage(config, chain_spec); - }), - di::bind.template to(), - di::bind.template to(), - makeRuntimeInjector(config->runtimeExecMethod()), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - bind_by_lambda( - [](const auto &injector) { return get_thread_pool(injector); }), - bind_by_lambda( - [](const auto &injector) { - auto storage = - injector.template create>(); - return get_trie_storage_backend(storage); - }), - bind_by_lambda([](const auto - &injector) { - return storage::trie::TrieStorageImpl::createEmpty( - injector.template create< - sptr>(), - injector.template create>(), - injector.template create< - sptr>()) + // compose peer keypair + bind_by_lambda([](const auto &injector) { + auto &app_config = + injector.template create(); + auto &chain = + injector.template create(); + auto &crypto_provider = + injector.template create(); + auto &csprng = injector.template create(); + auto &crypto_store = + injector.template create(); + return injector::get_peer_keypair( + app_config, chain, crypto_provider, csprng, crypto_store); + })[boost::di::override], + + di::bind() // NOLINT + .template to(), + di::bind() // NOLINT + .template to(), + + // starting metrics interfaces + di::bind.template to(), + di::bind.template to(), + di::bind.to([](const auto &injector) { + return metrics::Exposer::Configuration{ + injector.template create() + .openmetricsHttpEndpoint()}; + }), + // hardfix for Mac clang + di::bind.to([](const auto &injector) { + return metrics::Session::Configuration{}; + }), + // ending metrics interfaces + di::bind.template to(), + di::bind.to(config->roles()), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + bind_by_lambda([](const auto &injector) { + const application::AppConfiguration &config = + injector.template create(); + auto chain_spec = + injector.template create>(); + BOOST_ASSERT( // since rocksdb is the only possible option now + config.storageBackend() + == application::AppConfiguration::StorageBackend::RocksDB); + return get_rocks_db(config, chain_spec); + }), + bind_by_lambda([](const auto &injector) { + auto root = + injector::calculate_genesis_state( + injector.template create(), + injector.template create(), + injector.template create()) .value(); + const auto &hasher = injector.template create>(); + const auto &storage = + injector.template create>(); + return blockchain::BlockStorageImpl::create(root, storage, hasher) + .value(); + }), + di::bind.template to(), + bind_by_lambda( + [](const auto &injector) { return get_block_tree(injector); }), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + bind_by_lambda([](const auto &injector) { + const application::AppConfiguration &config = + injector.template create(); + auto chain_spec = + injector.template create>(); + + return get_key_file_storage(config, chain_spec); + }), + di::bind.template to(), + di::bind.template to(), + makeRuntimeInjector(config->runtimeExecMethod()), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + bind_by_lambda( + [](const auto &injector) { return get_thread_pool(injector); }), + bind_by_lambda( + [](const auto &injector) { + auto storage = + injector.template create>(); + return get_trie_storage_backend(storage); }), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - bind_by_lambda([](const auto &injector) { - const application::AppConfiguration &config = - injector - .template create(); - return get_chain_spec(config); + bind_by_lambda([](const auto &injector) { + return storage::trie::TrieStorageImpl::createEmpty( + injector.template create< + sptr>(), + injector.template create>(), + injector.template create< + sptr>()) + .value(); + }), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + bind_by_lambda([](const auto &injector) { + const application::AppConfiguration &config = + injector.template create(); + return get_chain_spec(config); + }), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + bind_by_lambda( + [](const auto &injector) { + return get_genesis_block_header(injector); }), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - bind_by_lambda( - [](const auto &injector) { - return get_genesis_block_header(injector); - }), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - di::bind.template to(), - - // user-defined overrides... - std::forward(args)...); + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + di::bind.template to(), + + // user-defined overrides... + std::forward(args)...); } template diff --git a/core/network/impl/state_protocol_observer_impl.cpp b/core/network/impl/state_protocol_observer_impl.cpp index bfcc13b177..48f7fc9008 100644 --- a/core/network/impl/state_protocol_observer_impl.cpp +++ b/core/network/impl/state_protocol_observer_impl.cpp @@ -64,7 +64,7 @@ namespace kagome::network { entry.entries.emplace_back( StateEntry{cursor->key().value(), {*value_opt}}); size += entry.entries.back().key.size() - + entry.entries.back().value.size(); + + entry.entries.back().value.size(); } } res = cursor->next(); @@ -89,9 +89,6 @@ namespace kagome::network { : cursor->seekUpperBound(request.start[0])); unsigned size = 0; KeyValueStateEntry entry; - // main state storage hash is marked with zeros in response - entry.state_root = - storage::trie::RootHash::fromSpan(std::vector(32, 0)).value(); StateResponse response; response.entries.emplace_back(std::move(entry)); diff --git a/core/network/impl/synchronizer_impl.cpp b/core/network/impl/synchronizer_impl.cpp index 9c2e8104e5..9c3f38c45e 100644 --- a/core/network/impl/synchronizer_impl.cpp +++ b/core/network/impl/synchronizer_impl.cpp @@ -9,7 +9,7 @@ #include "application/app_configuration.hpp" #include "blockchain/block_tree_error.hpp" -#include "consensus/babe/impl/babe_digests_util.hpp" +#include "consensus/babe/has_authority_set_change.hpp" #include "consensus/grandpa/environment.hpp" #include "consensus/grandpa/has_authority_set_change.hpp" #include "network/helpers/peer_id_formatter.hpp" @@ -1020,7 +1020,7 @@ namespace kagome::network { while (block.number != 0) { if (auto _header = block_tree_->getBlockHeader(block.hash)) { auto &header = _header.value(); - if (consensus::babe::getNextEpochDigest(header)) { + if (consensus::babe::HasAuthoritySetChange(header)) { break; } block = {header.number - 1, header.parent_hash}; @@ -1058,7 +1058,7 @@ namespace kagome::network { if (block.number < self->block_tree_->getLastFinalized().number) { self->block_storage_->assignNumberToHash(block).value(); } - if (consensus::babe::getNextEpochDigest(*header)) { + if (consensus::babe::HasAuthoritySetChange(*header)) { cb(outcome::success()); return; } diff --git a/core/network/types/block_announce_handshake.hpp b/core/network/types/block_announce_handshake.hpp index 68dd6efb87..0be0f68eaf 100644 --- a/core/network/types/block_announce_handshake.hpp +++ b/core/network/types/block_announce_handshake.hpp @@ -26,23 +26,13 @@ namespace kagome::network { * posibility of the correct communication with it. */ struct BlockAnnounceHandshake { + SCALE_TIE_ONLY(roles, best_block.number, best_block.hash, genesis_hash); + Roles roles; //!< Supported roles. primitives::BlockInfo best_block; //!< Best block. BlockHash genesis_hash; //!< Genesis block hash. - - friend inline scale::ScaleEncoderStream &operator<<( - scale::ScaleEncoderStream &s, const BlockAnnounceHandshake &v) { - return s << v.roles << v.best_block.number << v.best_block.hash - << v.genesis_hash; - } - - friend inline scale::ScaleDecoderStream &operator>>( - scale::ScaleDecoderStream &s, BlockAnnounceHandshake &v) { - return s >> v.roles >> v.best_block.number >> v.best_block.hash - >> v.genesis_hash; - } }; } // namespace kagome::network diff --git a/core/network/types/roles.hpp b/core/network/types/roles.hpp index 2080159d9d..068b2208a4 100644 --- a/core/network/types/roles.hpp +++ b/core/network/types/roles.hpp @@ -6,9 +6,13 @@ #ifndef KAGOME_CORE_NETWORK_TYPES_ROLES_HPP #define KAGOME_CORE_NETWORK_TYPES_ROLES_HPP +#include "scale/tie.hpp" + namespace kagome::network { union Roles { + SCALE_TIE_ONLY(value); + struct { /** * Full node, does not participate in consensus. @@ -45,43 +49,6 @@ namespace kagome::network { } return to_string(r.value); } - - /** - * @brief compares two Roles instances - * @param lhs first instance - * @param rhs second instance - * @return true if equal false otherwise - */ - inline bool operator==(const Roles &lhs, const Roles &rhs) { - return lhs.value == rhs.value; - } - - /** - * @brief outputs object of type Roles to stream - * @tparam Stream output stream type - * @param s stream reference - * @param v value to output - * @return reference to stream - */ - template > - Stream &operator<<(Stream &s, const Roles &v) { - return s << v.value; - } - - /** - * @brief decodes object of type Roles from stream - * @tparam Stream input stream type - * @param s stream reference - * @param v value to decode - * @return reference to stream - */ - template > - Stream &operator>>(Stream &s, Roles &v) { - return s >> v.value; - } - } // namespace kagome::network #endif // KAGOME_CORE_NETWORK_TYPES_ROLES_HPP diff --git a/core/network/warp/sync.cpp b/core/network/warp/sync.cpp index be9377010c..683d90d55a 100644 --- a/core/network/warp/sync.cpp +++ b/core/network/warp/sync.cpp @@ -7,6 +7,7 @@ #include "blockchain/block_storage.hpp" #include "blockchain/block_tree.hpp" +#include "consensus/babe/babe_config_repository.hpp" #include "consensus/grandpa/authority_manager.hpp" #include "consensus/grandpa/has_authority_set_change.hpp" #include "consensus/grandpa/justification_observer.hpp" @@ -24,12 +25,15 @@ namespace kagome::network { std::shared_ptr block_storage, std::shared_ptr warp_sync_cache, std::shared_ptr authority_manager, + std::shared_ptr + babe_config_repository, std::shared_ptr block_tree) : hasher_{std::move(hasher)}, grandpa_{std::move(grandpa)}, block_storage_{std::move(block_storage)}, warp_sync_cache_{std::move(warp_sync_cache)}, authority_manager_{std::move(authority_manager)}, + babe_config_repository_{std::move(babe_config_repository)}, block_tree_{std::move(block_tree)}, db_{db.getSpace(storage::Space::kDefault)} { app_state_manager.atLaunch([this] { @@ -109,6 +113,7 @@ namespace kagome::network { block_storage_->setBlockTreeLeaves({op.block_info.hash}).value(); warp_sync_cache_->warp(op.block_info); authority_manager_->warp(op.block_info, op.header, op.authorities); + babe_config_repository_->warp(op.block_info); block_tree_->warp(op.block_info); db_->remove(storage::kWarpSyncOp).value(); } diff --git a/core/network/warp/sync.hpp b/core/network/warp/sync.hpp index d16cdae7ec..aeec8ca5bd 100644 --- a/core/network/warp/sync.hpp +++ b/core/network/warp/sync.hpp @@ -18,6 +18,10 @@ namespace kagome::blockchain { class BlockStorage; } // namespace kagome::blockchain +namespace kagome::consensus::babe { + class BabeConfigRepository; +} // namespace kagome::consensus::babe + namespace kagome::consensus::grandpa { struct JustificationObserver; class AuthorityManager; @@ -59,6 +63,8 @@ namespace kagome::network { std::shared_ptr block_storage, std::shared_ptr warp_sync_cache, std::shared_ptr authority_manager, + std::shared_ptr + babe_config_repository, std::shared_ptr block_tree); /** @@ -84,6 +90,8 @@ namespace kagome::network { std::shared_ptr block_storage_; std::shared_ptr warp_sync_cache_; std::shared_ptr authority_manager_; + std::shared_ptr + babe_config_repository_; std::shared_ptr block_tree_; std::shared_ptr db_; bool done_ = false; diff --git a/core/primitives/apply_result.hpp b/core/primitives/apply_result.hpp index 8eb3284cb4..0de8c1660d 100644 --- a/core/primitives/apply_result.hpp +++ b/core/primitives/apply_result.hpp @@ -40,6 +40,8 @@ namespace kagome::primitives { SCALE_EMPTY_CODER(BadOrigin); /// A custom error in a module. struct Module { + SCALE_TIE_ONLY(index, error); + /// Module index, matching the metadata module index. uint8_t index; /// Module specific error value. @@ -49,19 +51,6 @@ namespace kagome::primitives { message; // not currently used in rust impl, thus not scale encoded }; - template > - Stream &operator<<(Stream &s, const Module &v) { - return s << v.index << v.error; - } - - template > - Stream &operator>>(Stream &s, Module &v) { - s >> v.index >> v.error; - return s; - } - /// At least one consumer is remaining so the account cannot be destroyed. struct ConsumerRemaining {}; SCALE_EMPTY_CODER(ConsumerRemaining); diff --git a/core/primitives/block_header.hpp b/core/primitives/block_header.hpp index 8647dffadf..1abcdd9dbd 100644 --- a/core/primitives/block_header.hpp +++ b/core/primitives/block_header.hpp @@ -41,6 +41,13 @@ namespace kagome::primitives { bool operator!=(const BlockHeader &rhs) const { return !operator==(rhs); } + + std::optional parentInfo() const { + if (number != 0) { + return primitives::BlockInfo{number - 1, parent_hash}; + } + return std::nullopt; + } }; struct BlockHeaderReflection { diff --git a/core/primitives/check_inherents_result.hpp b/core/primitives/check_inherents_result.hpp index c1dbeac8e5..0c2f07368c 100644 --- a/core/primitives/check_inherents_result.hpp +++ b/core/primitives/check_inherents_result.hpp @@ -13,6 +13,8 @@ namespace kagome::primitives { * @brief result of check_inherents method of BlockBuilder runtime api */ struct CheckInherentsResult { + SCALE_TIE(3); + /// Did the check succeed? bool is_okay = false; /// Did we encounter a fatal error? @@ -20,13 +22,6 @@ namespace kagome::primitives { /// We use the `InherentData` to store our errors. primitives::InherentData errors; }; - - template > - Stream &operator>>(Stream &s, CheckInherentsResult &v) { - return s >> v.is_okay >> v.is_fatal_error >> v.errors; - } - } // namespace kagome::primitives #endif // KAGOME_CORE_PRIMITIVES_CHECK_INHERENTS_RESULT_HPP diff --git a/core/primitives/common.hpp b/core/primitives/common.hpp index 4e4ebd928e..c897e2edf8 100644 --- a/core/primitives/common.hpp +++ b/core/primitives/common.hpp @@ -13,6 +13,7 @@ #include "common/blob.hpp" #include "macro/endianness_utils.hpp" +#include "scale/tie.hpp" namespace kagome::primitives { using BlockNumber = uint32_t; @@ -25,6 +26,8 @@ namespace kagome::primitives { template struct BlockInfoT : public boost::equality_comparable>, public boost::less_than_comparable> { + SCALE_TIE_ONLY(hash, number); + BlockInfoT() = default; BlockInfoT(const BlockNumber &n, const BlockHash &h) @@ -36,28 +39,10 @@ namespace kagome::primitives { BlockNumber number{}; BlockHash hash{}; - bool operator==(const BlockInfoT &o) const { - return number == o.number && hash == o.hash; - } - bool operator<(const BlockInfoT &o) const { return number < o.number or (number == o.number and hash < o.hash); } }; - - template > - Stream &operator<<(Stream &s, const BlockInfoT &msg) { - return s << msg.hash << msg.number; - } - - template > - Stream &operator>>(Stream &s, BlockInfoT &msg) { - return s >> msg.hash >> msg.number; - } } // namespace detail using BlockInfo = detail::BlockInfoT; diff --git a/core/primitives/scheduled_change.hpp b/core/primitives/scheduled_change.hpp index 2bd0e03fb9..17bc020e93 100644 --- a/core/primitives/scheduled_change.hpp +++ b/core/primitives/scheduled_change.hpp @@ -19,18 +19,15 @@ namespace kagome::primitives { DelayInChain() = default; explicit DelayInChain(uint32_t delay) : subchain_length(delay) {} - virtual ~DelayInChain() = default; }; struct AuthorityListChange { - SCALE_TIE(2); AuthorityList authorities{}; uint32_t subchain_length = 0; AuthorityListChange() = default; AuthorityListChange(AuthorityList authorities, uint32_t delay) : authorities(std::move(authorities)), subchain_length(delay) {} - virtual ~AuthorityListChange() = default; }; struct NextEpochData final : public consensus::babe::EpochDigest { @@ -45,10 +42,14 @@ namespace kagome::primitives { using NextConfigData = boost::variant, NextConfigDataV1>; struct ScheduledChange final : public AuthorityListChange { + SCALE_TIE_ONLY(authorities, subchain_length); + using AuthorityListChange::AuthorityListChange; }; struct ForcedChange final : public AuthorityListChange { + SCALE_TIE_ONLY(delay_start, authorities, subchain_length); + ForcedChange() = default; ForcedChange(AuthorityList authorities, @@ -57,18 +58,6 @@ namespace kagome::primitives { : AuthorityListChange(authorities, delay), delay_start{delay_start} {} BlockNumber delay_start; - - friend scale::ScaleDecoderStream &operator>>(scale::ScaleDecoderStream &s, - ForcedChange &change) { - return s >> change.delay_start >> change.authorities - >> change.subchain_length; - } - - friend scale::ScaleEncoderStream &operator<<(scale::ScaleEncoderStream &s, - const ForcedChange &change) { - return s << change.delay_start << change.authorities - << change.subchain_length; - } }; struct OnDisabled { diff --git a/core/scale/tie.hpp b/core/scale/tie.hpp index fd88901f74..475a2ed4cf 100644 --- a/core/scale/tie.hpp +++ b/core/scale/tie.hpp @@ -8,8 +8,7 @@ #include -#define SCALE_TIE(N) \ - static constexpr size_t scale_tie = N; \ +#define SCALE_TIE_EQ \ template \ bool operator==(const _ScaleTieType &r) const { \ using ThisT = std::decay_t; \ @@ -25,10 +24,28 @@ return !operator==(r); \ } +#define SCALE_TIE_ONLY(...) \ + auto as_tie() { return std::tie(__VA_ARGS__); } \ + SCALE_TIE_EQ + +#define SCALE_TIE(N) \ + static constexpr size_t scale_tie = N; \ + SCALE_TIE_EQ + namespace scale { class ScaleEncoderStream; class ScaleDecoderStream; + template >>() + .as_tie())> + auto as_tie(T &&v, F &&f) { + return f(const_cast> &>(v) + .as_tie()); + } + // generated by housekeeping/scale_tie.py template + constexpr auto as_tie_sfinae = [](auto &&) {}; + + template (), + as_tie_sfinae))> ScaleEncoderStream &operator<<(ScaleEncoderStream &s, const T &v) { as_tie(v, [&](auto v) { std::apply([&](const auto &...v) { (..., (s << v)); }, v); @@ -79,7 +100,9 @@ namespace scale { return s; } - template + template (), + as_tie_sfinae))> ScaleDecoderStream &operator>>(ScaleDecoderStream &s, T &v) { as_tie(v, [&](auto v) { std::apply([&](auto &...v) { (..., (s >> v)); }, v); diff --git a/core/storage/face/map_cursor.hpp b/core/storage/face/map_cursor.hpp index 2bff481090..d01102d63d 100644 --- a/core/storage/face/map_cursor.hpp +++ b/core/storage/face/map_cursor.hpp @@ -35,6 +35,28 @@ namespace kagome::storage::face { */ virtual outcome::result seek(const View &key) = 0; + /** + * Lower bound in reverse order. + * rocks_db.put(2) + * rocks_db.seek(1) -> 2 + * rocks_db.seek(2) -> 2 + * rocks_db.seek(3) -> none + * seekReverse(rocks_db, 1) -> none + * seekReverse(rocks_db, 2) -> 2 + * seekReverse(rocks_db, 3) -> 2 + */ + outcome::result seekReverse(const View &prefix) { + OUTCOME_TRY(ok, seek(prefix)); + if (not ok) { + return seekLast(); + } + if (View{*key()} > prefix) { + OUTCOME_TRY(prev()); + return isValid(); + } + return true; + } + /** * @brief Same as std::rbegin(...);, e.g. points to the last valid element * @return error if any, true if trie is not empty, false otherwise diff --git a/core/storage/in_memory/cursor.hpp b/core/storage/in_memory/cursor.hpp new file mode 100644 index 0000000000..85de6e225e --- /dev/null +++ b/core/storage/in_memory/cursor.hpp @@ -0,0 +1,74 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef KAGOME_STORAGE_IN_MEMORY_CURSOR_HPP +#define KAGOME_STORAGE_IN_MEMORY_CURSOR_HPP + +#include "common/buffer.hpp" +#include "storage/in_memory/in_memory_storage.hpp" + +namespace kagome::storage { + class InMemoryCursor : public BufferStorageCursor { + public: + explicit InMemoryCursor(InMemoryStorage &db) : db{db} {} + + outcome::result seekFirst() override { + return seek(db.storage.begin()); + } + + outcome::result seek(const BufferView &key) override { + return seek(db.storage.lower_bound(key.toHex())); + } + + outcome::result seekLast() override { + return seek(db.storage.empty() ? db.storage.end() + : std::prev(db.storage.end())); + } + + bool isValid() const override { + return kv.has_value(); + } + + outcome::result next() override { + seek(db.storage.upper_bound(kv->first.toHex())); + return outcome::success(); + } + + outcome::result prev() override { + auto it = db.storage.lower_bound(kv->first.toHex()); + seek(it == db.storage.begin() ? db.storage.end() : std::prev(it)); + return outcome::success(); + } + + std::optional key() const override { + if (kv) { + return kv->first; + } + return std::nullopt; + } + + std::optional value() const override { + if (kv) { + return BufferView{kv->second}; + } + return std::nullopt; + } + + private: + bool seek(decltype(InMemoryStorage::storage)::iterator it) { + if (it == db.storage.end()) { + kv.reset(); + } else { + kv.emplace(Buffer::fromHex(it->first).value(), it->second); + } + return isValid(); + } + + InMemoryStorage &db; + std::optional> kv; + }; +} // namespace kagome::storage + +#endif // KAGOME_STORAGE_IN_MEMORY_CURSOR_HPP diff --git a/core/storage/in_memory/in_memory_storage.cpp b/core/storage/in_memory/in_memory_storage.cpp index b9b67cf7d3..cddf8d2be9 100644 --- a/core/storage/in_memory/in_memory_storage.cpp +++ b/core/storage/in_memory/in_memory_storage.cpp @@ -6,6 +6,7 @@ #include "storage/in_memory/in_memory_storage.hpp" #include "storage/database_error.hpp" +#include "storage/in_memory/cursor.hpp" #include "storage/in_memory/in_memory_batch.hpp" using kagome::common::Buffer; @@ -65,7 +66,7 @@ namespace kagome::storage { } std::unique_ptr InMemoryStorage::cursor() { - return nullptr; + return std::make_unique(*this); } size_t InMemoryStorage::size() const { diff --git a/core/storage/in_memory/in_memory_storage.hpp b/core/storage/in_memory/in_memory_storage.hpp index 6d3d4f929d..ef0fc57bea 100644 --- a/core/storage/in_memory/in_memory_storage.hpp +++ b/core/storage/in_memory/in_memory_storage.hpp @@ -48,6 +48,8 @@ namespace kagome::storage { private: std::map storage; size_t size_ = 0; + + friend class InMemoryCursor; }; } // namespace kagome::storage diff --git a/core/storage/predefined_keys.hpp b/core/storage/predefined_keys.hpp index 025653f14b..fd330ac601 100644 --- a/core/storage/predefined_keys.hpp +++ b/core/storage/predefined_keys.hpp @@ -40,11 +40,8 @@ namespace kagome::storage { inline const common::Buffer kWarpSyncOp = ":kagome:WarpSync:op"_buf; - template - inline common::Buffer kBabeConfigRepoStateLookupKey(Tag tag) { - return common::Buffer::fromString( - fmt::format(":kagome:babe_config_repo_state:{}", tag)); - } + inline const common::Buffer kBabeConfigRepositoryImplIndexerPrefix = + ":kagome:BabeConfigRepositoryImpl:Indexer:"_buf; template inline common::Buffer kAuthorityManagerStateLookupKey(Tag tag) { diff --git a/core/telemetry/service.cpp b/core/telemetry/service.cpp index 93413e6a0c..e058172a94 100644 --- a/core/telemetry/service.cpp +++ b/core/telemetry/service.cpp @@ -42,6 +42,9 @@ namespace kagome::telemetry { void setActualImplementation(Telemetry service) { service_ = std::move(service); + if (not service_) { + return; + } if (was_synchronized_) { service_->notifyWasSynchronized(); } diff --git a/test/core/consensus/babe/babe_config_repository_test.cpp b/test/core/consensus/babe/babe_config_repository_test.cpp index 5af1973ca0..30c9592cf2 100644 --- a/test/core/consensus/babe/babe_config_repository_test.cpp +++ b/test/core/consensus/babe/babe_config_repository_test.cpp @@ -15,10 +15,10 @@ #include "mock/core/clock/clock_mock.hpp" #include "mock/core/crypto/hasher_mock.hpp" #include "mock/core/runtime/babe_api_mock.hpp" -#include "mock/core/storage/persistent_map_mock.hpp" #include "mock/core/storage/spaced_storage_mock.hpp" #include "mock/core/storage/trie/trie_storage_mock.hpp" #include "primitives/babe_configuration.hpp" +#include "storage/in_memory/in_memory_storage.hpp" #include "testutil/literals.hpp" #include "testutil/prepare_loggers.hpp" @@ -38,7 +38,7 @@ using primitives::BlockId; using primitives::BlockInfo; using primitives::events::ChainSubscriptionEngine; using runtime::BabeApiMock; -using storage::BufferStorageMock; +using storage::InMemoryStorage; using storage::SpacedStorageMock; using storage::trie::TrieStorageMock; @@ -62,9 +62,7 @@ class BabeConfigRepositoryTest : public testing::Test { app_state_manager = std::make_shared(); EXPECT_CALL(*app_state_manager, atPrepare(_)).WillOnce(Return()); - persistent_storage = std::make_shared(); - EXPECT_CALL(*persistent_storage, tryGetMock(_)) - .WillRepeatedly(Return(std::nullopt)); + persistent_storage = std::make_shared(); spaced_storage = std::make_shared(); EXPECT_CALL(*spaced_storage, getSpace(_)) @@ -72,10 +70,12 @@ class BabeConfigRepositoryTest : public testing::Test { app_config = std::make_shared(); block_tree = std::make_shared(); + primitives::BlockInfo genesis{0, "genesis"_hash256}; EXPECT_CALL(*block_tree, getLastFinalized()) .WillOnce(Return(BlockInfo{0, "genesis"_hash256})); + EXPECT_CALL(*block_tree, isFinalized(genesis)).WillRepeatedly(Return(true)); EXPECT_CALL(*block_tree, getGenesisBlockHash()) - .WillOnce(testing::ReturnRefOfCopy("genesis"_hash256)); + .WillRepeatedly(testing::ReturnRefOfCopy(genesis.hash)); header_repo = std::make_shared(); @@ -106,7 +106,7 @@ class BabeConfigRepositoryTest : public testing::Test { std::shared_ptr app_state_manager; std::shared_ptr spaced_storage; std::shared_ptr app_config; - std::shared_ptr persistent_storage; + std::shared_ptr persistent_storage; std::shared_ptr block_tree; std::shared_ptr header_repo; std::shared_ptr babe_api; @@ -125,7 +125,10 @@ class BabeConfigRepositoryTest : public testing::Test { */ TEST_F(BabeConfigRepositoryTest, getCurrentSlot) { EXPECT_CALL(*block_tree, getBlockHeader(_)) - .WillOnce(Return(outcome::success())); + .WillRepeatedly(Return(outcome::success())); + EXPECT_CALL(*trie_storage, getEphemeralBatchAt(_)).WillOnce([] { + return nullptr; + }); babe_config_repo_->prepare(); auto time = std::chrono::system_clock::now(); EXPECT_CALL(*clock, now()).Times(1).WillOnce(Return(time)); diff --git a/test/core/consensus/babe/babe_test.cpp b/test/core/consensus/babe/babe_test.cpp index 6bd6a97b9f..fa8818fb8a 100644 --- a/test/core/consensus/babe/babe_test.cpp +++ b/test/core/consensus/babe/babe_test.cpp @@ -126,7 +126,7 @@ class BabeTest : public testing::Test { babe_config_repo_ = std::make_shared(); ON_CALL(*babe_config_repo_, config(_, _)) - .WillByDefault(Return(*babe_config_)); + .WillByDefault(Return(babe_config_)); ON_CALL(*babe_config_repo_, epochLength()) .WillByDefault(Return(babe_config_->epoch_length)); diff --git a/test/core/consensus/babe/block_executor_test.cpp b/test/core/consensus/babe/block_executor_test.cpp index 9eb15df17c..332061fca8 100644 --- a/test/core/consensus/babe/block_executor_test.cpp +++ b/test/core/consensus/babe/block_executor_test.cpp @@ -121,7 +121,7 @@ class BlockExecutorTest : public testing::Test { babe_config_repo_ = std::make_shared(); ON_CALL(*babe_config_repo_, config(_, _)) - .WillByDefault(Return(std::make_optional(std::cref(*babe_config_)))); + .WillByDefault(Return(babe_config_)); block_validator_ = std::make_shared(); grandpa_environment_ = std::make_shared(); diff --git a/test/core/network/state_protocol_observer_test.cpp b/test/core/network/state_protocol_observer_test.cpp index c231ec4812..28fabcb99c 100644 --- a/test/core/network/state_protocol_observer_test.cpp +++ b/test/core/network/state_protocol_observer_test.cpp @@ -124,7 +124,7 @@ TEST_F(StateProtocolObserverTest, Simple) { StateRequest request{ .hash = "1"_hash256, .start = {}, - .no_proof = false + .no_proof = false, }; EXPECT_OUTCOME_TRUE(response, @@ -132,7 +132,7 @@ TEST_F(StateProtocolObserverTest, Simple) { StateResponse ref = { .entries = {{ - .state_root = RootHash::fromSpan(std::vector(32, 0)).value(), + .state_root = {}, .entries = {{.key = "abc"_buf, .value = "123"_buf}, {.key = "cde"_buf, .value = "345"_buf}}, .complete = true, diff --git a/test/mock/core/blockchain/block_tree_mock.hpp b/test/mock/core/blockchain/block_tree_mock.hpp index 27724117e3..55d10f180e 100644 --- a/test/mock/core/blockchain/block_tree_mock.hpp +++ b/test/mock/core/blockchain/block_tree_mock.hpp @@ -99,6 +99,11 @@ namespace kagome::blockchain { (const primitives::BlockHash &, const primitives::BlockHash &), (const, override)); + MOCK_METHOD(bool, + isFinalized, + (const primitives::BlockInfo &), + (const, override)); + MOCK_METHOD(outcome::result, getBestContaining, (const primitives::BlockHash &, diff --git a/test/mock/core/consensus/babe/babe_config_repository_mock.hpp b/test/mock/core/consensus/babe/babe_config_repository_mock.hpp index 0fc2dfd6cb..3ac565a768 100644 --- a/test/mock/core/consensus/babe/babe_config_repository_mock.hpp +++ b/test/mock/core/consensus/babe/babe_config_repository_mock.hpp @@ -19,16 +19,12 @@ namespace kagome::consensus::babe { MOCK_METHOD(EpochLength, epochLength, (), (const, override)); MOCK_METHOD( - std::optional< - std::reference_wrapper>, + outcome::result>, config, - (const primitives::BlockContext &, EpochNumber), + (const primitives::BlockInfo &, EpochNumber), (const, override)); - MOCK_METHOD(void, - readFromState, - (const primitives::BlockInfo &), - (override)); + MOCK_METHOD(void, warp, (const primitives::BlockInfo &), (override)); }; } // namespace kagome::consensus::babe From 36d8119bafd1bf7f56d0436aa012e93c9707ef7b Mon Sep 17 00:00:00 2001 From: turuslan Date: Fri, 16 Jun 2023 15:45:29 +0300 Subject: [PATCH 02/12] minideb Signed-off-by: turuslan --- core/consensus/babe/impl/babe_config_repository_impl.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/core/consensus/babe/impl/babe_config_repository_impl.cpp b/core/consensus/babe/impl/babe_config_repository_impl.cpp index 713ef5ae2e..50a1e7ba20 100644 --- a/core/consensus/babe/impl/babe_config_repository_impl.cpp +++ b/core/consensus/babe/impl/babe_config_repository_impl.cpp @@ -28,6 +28,7 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::consensus::babe, case E::ERROR: return "error"; } + return fmt::format("BabeConfigRepositoryImpl::Error({})", e); } namespace kagome::consensus::babe { From c7eb7304b7cf39f19e278463b1144e1359a65a3d Mon Sep 17 00:00:00 2001 From: turuslan Date: Thu, 22 Jun 2023 17:12:45 +0300 Subject: [PATCH 03/12] load db into map, no inherit=true below last finalized indexed, fix finalized already indexed Signed-off-by: turuslan --- core/blockchain/indexer.hpp | 106 ++++++++---------- .../babe/impl/babe_config_repository_impl.cpp | 12 +- .../babe/impl/babe_config_repository_impl.hpp | 2 - core/network/warp/sync.cpp | 2 +- 4 files changed, 51 insertions(+), 71 deletions(-) diff --git a/core/blockchain/indexer.hpp b/core/blockchain/indexer.hpp index b3fcaa90e0..772492516b 100644 --- a/core/blockchain/indexer.hpp +++ b/core/blockchain/indexer.hpp @@ -85,10 +85,24 @@ namespace kagome::blockchain { } Indexer(std::shared_ptr db, - std::shared_ptr block_tree, - const primitives::BlockInfo &genesis) + std::shared_ptr block_tree) : db_{std::move(db)}, block_tree_{std::move(block_tree)} { - map_.emplace(genesis, get(genesis).value_or(Indexed{})); + primitives::BlockInfo genesis{0, block_tree_->getGenesisBlockHash()}; + last_finalized_indexed_ = genesis; + auto batch = db_->batch(); + auto db_cur = db_->cursor(); + for (db_cur->seekFirst().value(); db_cur->isValid(); + db_cur->next().value()) { + auto info = fromKey(*db_cur->key()); + if (not block_tree_->isFinalized(info)) { + batch->remove(toKey(info)).value(); + continue; + } + last_finalized_indexed_ = info; + map_.emplace(info, scale::decode>(*db_cur->value()).value()); + } + map_.emplace(genesis, Indexed{}); + batch->commit().value(); } Descent descend(const primitives::BlockInfo &from) const { @@ -108,6 +122,9 @@ namespace kagome::blockchain { void put(const primitives::BlockInfo &block, const Indexed &indexed, bool db) { + if (indexed.inherit and block.number <= last_finalized_indexed_.number) { + return; + } map_[block] = indexed; if (db) { db_->put(toKey(block), scale::encode(indexed).value()).value(); @@ -119,50 +136,30 @@ namespace kagome::blockchain { db_->remove(toKey(block)).value(); } - void filterUnfinalized() { - for (auto it = map_.begin(); it != map_.end();) { - if (not block_tree_->isFinalized(it->first)) { - it = map_.erase(it); - } else { - ++it; - } - } - } - - primitives::BlockInfo removeUnfinalized() { - for (auto it = map_.begin(); it != map_.end();) { - if (not block_tree_->isFinalized(it->first)) { - it = map_.erase(it); - } else { - ++it; - } - } - primitives::BlockInfo max{0, block_tree_->getGenesisBlockHash()}; + void finalize() { auto batch = db_->batch(); - auto db_cur = db_->cursor(); - for (db_cur->seekLast().value(); db_cur->isValid();) { - auto info = fromKey(*db_cur->key()); - db_cur->prev().value(); - if (not block_tree_->isFinalized(info)) { - batch->remove(toKey(info)).value(); - } else if (max < info) { - max = info; - break; + auto finalized = block_tree_->getLastFinalized(); + auto first = last_finalized_indexed_.number + 1; + for (auto map_it = map_.lower_bound({first, {}}); map_it != map_.end();) { + auto &[info, indexed] = *map_it; + if (block_tree_->isFinalized(info)) { + if (not indexed.inherit) { + batch->put(toKey(info), scale::encode(indexed).value()).value(); + last_finalized_indexed_ = info; + } + } else if (not block_tree_->hasDirectChain(finalized, info)) { + map_it = map_.erase(map_it); + continue; } + ++map_it; } - batch->commit().value(); - return max; - } - - void writeFinalized(primitives::BlockNumber first, - primitives::BlockNumber last) { - auto begin = map_.lower_bound({first, {}}), - end = map_.lower_bound({last + 1, {}}); - auto batch = db_->batch(); - for (auto it = begin; it != end; ++it) { - if (block_tree_->isFinalized(it->first)) { - batch->put(toKey(it->first), scale::encode(it->second).value()) - .value(); + for (auto map_it = map_.lower_bound({first, {}}); + map_it != map_.end() + and map_it->first.number < last_finalized_indexed_.number;) { + if (map_it->second.inherit) { + map_it = map_.erase(map_it); + } else { + ++map_it; } } batch->commit().value(); @@ -191,23 +188,10 @@ namespace kagome::blockchain { }; } if (map_it == map_.begin()) { - break; + return std::nullopt; } --map_it; } - auto db_cur = db_->cursor(); - for (db_cur->seekReverse(toKey(block)).value(); db_cur->isValid(); - db_cur->prev().value()) { - auto info = fromKey(*db_cur->key()); - if (not descent.can(info)) { - continue; - } - return SearchRaw{ - {info, scale::decode>(*db_cur->value()).value()}, - info, - }; - } - return std::nullopt; } template @@ -220,7 +204,10 @@ namespace kagome::blockchain { return std::nullopt; } BOOST_ASSERT(not raw->kv.second.inherit); - if (not raw->kv.second.value or raw->last != block) { + if (not raw->kv.second.value + or (raw->last != block + and (block.number > last_finalized_indexed_.number + or not block_tree_->isFinalized(block)))) { auto prev = raw->kv.second.value ? raw->kv.first : raw->kv.second.prev; auto i_first = descent.indexFor(raw->last.number + (raw->kv.second.value ? 1 : 0)); @@ -239,6 +226,7 @@ namespace kagome::blockchain { std::shared_ptr db_; std::shared_ptr block_tree_; + primitives::BlockInfo last_finalized_indexed_; std::map> map_; }; } // namespace kagome::blockchain diff --git a/core/consensus/babe/impl/babe_config_repository_impl.cpp b/core/consensus/babe/impl/babe_config_repository_impl.cpp index 50a1e7ba20..053aaadfe9 100644 --- a/core/consensus/babe/impl/babe_config_repository_impl.cpp +++ b/core/consensus/babe/impl/babe_config_repository_impl.cpp @@ -60,7 +60,6 @@ namespace kagome::consensus::babe { storage::kBabeConfigRepositoryImplIndexerPrefix, persistent_storage_), block_tree_, - {0, block_tree_->getGenesisBlockHash()}, }, header_repo_(std::move(header_repo)), babe_api_(std::move(babe_api)), @@ -83,10 +82,10 @@ namespace kagome::consensus::babe { } bool BabeConfigRepositoryImpl::prepare() { - auto last_indexed = indexer_.removeUnfinalized(); auto finalized = block_tree_->getLastFinalized(); auto finalized_header = block_tree_->getBlockHeader(finalized.hash).value(); - if (finalized.number - last_indexed.number > kDontIndexFinalizedBlocks + if (finalized.number - indexer_.last_finalized_indexed_.number + > kDontIndexFinalizedBlocks and trie_storage_->getEphemeralBatchAt(finalized_header.state_root)) { warp(finalized); } @@ -121,12 +120,7 @@ namespace kagome::consensus::babe { const primitives::events::ChainEventParams &event) { if (type == primitives::events::ChainEventType::kFinalizedHeads) { if (auto self = wp.lock()) { - const auto &header = - boost::get(event).get(); - self->indexer_.filterUnfinalized(); - self->indexer_.writeFinalized(self->last_saved_state_block_, - header.number); - self->last_saved_state_block_ = header.number; + self->indexer_.finalize(); } } }); diff --git a/core/consensus/babe/impl/babe_config_repository_impl.hpp b/core/consensus/babe/impl/babe_config_repository_impl.hpp index 89c0e2a9f0..e10b24e5d4 100644 --- a/core/consensus/babe/impl/babe_config_repository_impl.hpp +++ b/core/consensus/babe/impl/babe_config_repository_impl.hpp @@ -125,8 +125,6 @@ namespace kagome::consensus::babe { BabeDuration slot_duration_{}; EpochLength epoch_length_{}; - primitives::BlockNumber last_saved_state_block_ = 0; - const BabeClock &clock_; std::optional first_block_slot_number_; bool is_first_block_finalized_ = false; diff --git a/core/network/warp/sync.cpp b/core/network/warp/sync.cpp index 683d90d55a..c7cc3ba02d 100644 --- a/core/network/warp/sync.cpp +++ b/core/network/warp/sync.cpp @@ -113,8 +113,8 @@ namespace kagome::network { block_storage_->setBlockTreeLeaves({op.block_info.hash}).value(); warp_sync_cache_->warp(op.block_info); authority_manager_->warp(op.block_info, op.header, op.authorities); - babe_config_repository_->warp(op.block_info); block_tree_->warp(op.block_info); + babe_config_repository_->warp(op.block_info); db_->remove(storage::kWarpSyncOp).value(); } } // namespace kagome::network From fa869a07784e91e31bf0ed3d8304869a7e04d6f2 Mon Sep 17 00:00:00 2001 From: turuslan Date: Thu, 22 Jun 2023 17:24:20 +0300 Subject: [PATCH 04/12] mutex Signed-off-by: turuslan --- core/consensus/babe/impl/babe_config_repository_impl.cpp | 4 ++++ core/consensus/babe/impl/babe_config_repository_impl.hpp | 3 +++ 2 files changed, 7 insertions(+) diff --git a/core/consensus/babe/impl/babe_config_repository_impl.cpp b/core/consensus/babe/impl/babe_config_repository_impl.cpp index 053aaadfe9..03f783522d 100644 --- a/core/consensus/babe/impl/babe_config_repository_impl.cpp +++ b/core/consensus/babe/impl/babe_config_repository_impl.cpp @@ -82,6 +82,7 @@ namespace kagome::consensus::babe { } bool BabeConfigRepositoryImpl::prepare() { + std::unique_lock lock{indexer_mutex_}; auto finalized = block_tree_->getLastFinalized(); auto finalized_header = block_tree_->getBlockHeader(finalized.hash).value(); if (finalized.number - indexer_.last_finalized_indexed_.number @@ -120,6 +121,7 @@ namespace kagome::consensus::babe { const primitives::events::ChainEventParams &event) { if (type == primitives::events::ChainEventType::kFinalizedHeads) { if (auto self = wp.lock()) { + std::unique_lock lock{self->indexer_mutex_}; self->indexer_.finalize(); } } @@ -138,6 +140,7 @@ namespace kagome::consensus::babe { auto parent_epoch = slotToEpoch(parent_digest.second.slot_number); epoch_changed = epoch_number != parent_epoch; } + std::unique_lock lock{indexer_mutex_}; return config(parent_info, epoch_changed); } @@ -224,6 +227,7 @@ namespace kagome::consensus::babe { } void BabeConfigRepositoryImpl::warp(const primitives::BlockInfo &block) { + std::unique_lock lock{indexer_mutex_}; indexer_.put(block, {}, true); } diff --git a/core/consensus/babe/impl/babe_config_repository_impl.hpp b/core/consensus/babe/impl/babe_config_repository_impl.hpp index e10b24e5d4..5bfc30b422 100644 --- a/core/consensus/babe/impl/babe_config_repository_impl.hpp +++ b/core/consensus/babe/impl/babe_config_repository_impl.hpp @@ -9,6 +9,8 @@ #include "consensus/babe/babe_config_repository.hpp" #include "consensus/babe/babe_util.hpp" +#include + #include "blockchain/indexer.hpp" #include "consensus/babe/has_authority_set_change.hpp" #include "log/logger.hpp" @@ -115,6 +117,7 @@ namespace kagome::consensus::babe { std::shared_ptr persistent_storage_; bool config_warp_sync_; std::shared_ptr block_tree_; + mutable std::mutex indexer_mutex_; mutable blockchain::Indexer indexer_; std::shared_ptr header_repo_; std::shared_ptr babe_api_; From 4791e77cd16edb0d444e48dd5f03f97a9419634e Mon Sep 17 00:00:00 2001 From: turuslan Date: Thu, 22 Jun 2023 18:01:19 +0300 Subject: [PATCH 05/12] pr comments Signed-off-by: turuslan --- core/blockchain/indexer.hpp | 52 +++++++------------ .../babe/impl/babe_config_repository_impl.cpp | 18 ++++--- .../babe/impl/babe_config_repository_impl.hpp | 3 +- .../babe/impl/block_appender_base.cpp | 4 +- core/utils/block_info_key.hpp | 39 ++++++++++++++ 5 files changed, 73 insertions(+), 43 deletions(-) create mode 100644 core/utils/block_info_key.hpp diff --git a/core/blockchain/indexer.hpp b/core/blockchain/indexer.hpp index 772492516b..4810af0975 100644 --- a/core/blockchain/indexer.hpp +++ b/core/blockchain/indexer.hpp @@ -6,10 +6,9 @@ #ifndef KAGOME_BLOCKCHAIN_INDEXER_HPP #define KAGOME_BLOCKCHAIN_INDEXER_HPP -#include - #include "blockchain/block_tree.hpp" #include "storage/buffer_map_types.hpp" +#include "utils/block_info_key.hpp" namespace kagome::blockchain { struct Descent { @@ -17,7 +16,7 @@ namespace kagome::blockchain { primitives::BlockInfo start) : block_tree_{std::move(block_tree)}, path_{start} {} - bool can(const primitives::BlockInfo &to) { + bool descends(const primitives::BlockInfo &to) const { if (to == path_.front()) { return true; } @@ -48,13 +47,13 @@ namespace kagome::blockchain { return to == path_.at(i); } - size_t indexFor(primitives::BlockNumber n) { + size_t indexFor(primitives::BlockNumber n) const { BOOST_ASSERT(n <= path_.front().number); return path_.front().number - n; } std::shared_ptr block_tree_; - std::vector path_; + mutable std::vector path_; bool update_path_ = true; }; @@ -69,21 +68,6 @@ namespace kagome::blockchain { template struct Indexer { - static common::Blob<4 + 32> toKey(const primitives::BlockInfo &info) { - common::Blob<4 + 32> key; - boost::endian::store_big_u32(key.data(), info.number); - std::copy_n(info.hash.data(), 32, key.data() + 4); - return key; - } - - static primitives::BlockInfo fromKey(common::BufferView key) { - BOOST_ASSERT(key.size() == 4 + 32); - primitives::BlockInfo info; - info.number = boost::endian::load_big_u32(key.data()); - std::copy_n(key.data() + 4, 32, info.hash.data()); - return info; - } - Indexer(std::shared_ptr db, std::shared_ptr block_tree) : db_{std::move(db)}, block_tree_{std::move(block_tree)} { @@ -93,9 +77,9 @@ namespace kagome::blockchain { auto db_cur = db_->cursor(); for (db_cur->seekFirst().value(); db_cur->isValid(); db_cur->next().value()) { - auto info = fromKey(*db_cur->key()); + auto info = BlockInfoKey::decode(*db_cur->key()).value(); if (not block_tree_->isFinalized(info)) { - batch->remove(toKey(info)).value(); + batch->remove(BlockInfoKey::encode(info)).value(); continue; } last_finalized_indexed_ = info; @@ -113,7 +97,7 @@ namespace kagome::blockchain { if (auto it = map_.find(block); it != map_.end()) { return it->second; } - if (auto r = db_->tryGet(toKey(block)).value()) { + if (auto r = db_->tryGet(BlockInfoKey::encode(block)).value()) { return scale::decode>(*r).value(); } return std::nullopt; @@ -127,13 +111,14 @@ namespace kagome::blockchain { } map_[block] = indexed; if (db) { - db_->put(toKey(block), scale::encode(indexed).value()).value(); + db_->put(BlockInfoKey::encode(block), scale::encode(indexed).value()) + .value(); } } void remove(const primitives::BlockInfo &block) { map_.erase(block); - db_->remove(toKey(block)).value(); + db_->remove(BlockInfoKey::encode(block)).value(); } void finalize() { @@ -144,7 +129,10 @@ namespace kagome::blockchain { auto &[info, indexed] = *map_it; if (block_tree_->isFinalized(info)) { if (not indexed.inherit) { - batch->put(toKey(info), scale::encode(indexed).value()).value(); + batch + ->put(BlockInfoKey::encode(info), + scale::encode(indexed).value()) + .value(); last_finalized_indexed_ = info; } } else if (not block_tree_->hasDirectChain(finalized, info)) { @@ -165,9 +153,9 @@ namespace kagome::blockchain { batch->commit().value(); } - using Search = std::pair>; + using KeyValue = std::pair>; struct SearchRaw { - Search kv; + KeyValue kv; primitives::BlockInfo last; }; @@ -175,7 +163,7 @@ namespace kagome::blockchain { const primitives::BlockInfo &block) { auto map_it = map_.lower_bound(block); while (true) { - if (map_it != map_.end() and descent.can(map_it->first)) { + if (map_it != map_.end() and descent.descends(map_it->first)) { if (not map_it->second.inherit) { return SearchRaw{*map_it, map_it->first}; } @@ -195,9 +183,9 @@ namespace kagome::blockchain { } template - std::optional search(Descent &descent, - const primitives::BlockInfo &block, - const Cb &cb) { + std::optional search(Descent &descent, + const primitives::BlockInfo &block, + const Cb &cb) { descent.update_path_ = true; auto raw = searchRaw(descent, block); if (not raw) { diff --git a/core/consensus/babe/impl/babe_config_repository_impl.cpp b/core/consensus/babe/impl/babe_config_repository_impl.cpp index 03f783522d..71607e2be0 100644 --- a/core/consensus/babe/impl/babe_config_repository_impl.cpp +++ b/core/consensus/babe/impl/babe_config_repository_impl.cpp @@ -25,8 +25,10 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::consensus::babe, e) { using E = decltype(e); switch (e) { - case E::ERROR: - return "error"; + case E::NOT_FOUND: + return "babe config not found"; + case E::PREVIOUS_NOT_FOUND: + return "previous babe config not found"; } return fmt::format("BabeConfigRepositoryImpl::Error({})", e); } @@ -34,7 +36,7 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::consensus::babe, namespace kagome::consensus::babe { constexpr size_t kDontIndexFinalizedBlocks = 10000; - inline primitives::NextConfigDataV1 getConfig( + inline static primitives::NextConfigDataV1 getConfig( const primitives::BabeConfiguration &state) { return {state.leadership_rate, state.allowed_slots}; } @@ -303,7 +305,7 @@ namespace kagome::consensus::babe { auto r = indexer_.search(descent, block, cb); OUTCOME_TRY(cb_res); if (not r) { - return Error::ERROR; + return Error::NOT_FOUND; } if (not next and r->second.value->state) { return *r->second.value->state; @@ -313,7 +315,7 @@ namespace kagome::consensus::babe { return *r->second.value->next_state; } if (not r->second.prev) { - return Error::ERROR; + return Error::PREVIOUS_NOT_FOUND; } return loadPrev(*r->second.prev); } @@ -358,14 +360,14 @@ namespace kagome::consensus::babe { BabeConfigRepositoryImpl::loadPrev( const std::optional &prev) const { if (not prev) { - return Error::ERROR; + return Error::PREVIOUS_NOT_FOUND; } auto r = indexer_.get(*prev); if (not r) { - return Error::ERROR; + return Error::PREVIOUS_NOT_FOUND; } if (not r->value) { - return Error::ERROR; + return Error::PREVIOUS_NOT_FOUND; } OUTCOME_TRY(load(*prev, *r)); return *r->value->next_state; diff --git a/core/consensus/babe/impl/babe_config_repository_impl.hpp b/core/consensus/babe/impl/babe_config_repository_impl.hpp index 5bfc30b422..7391e13d88 100644 --- a/core/consensus/babe/impl/babe_config_repository_impl.hpp +++ b/core/consensus/babe/impl/babe_config_repository_impl.hpp @@ -53,7 +53,8 @@ namespace kagome::consensus::babe { public std::enable_shared_from_this { public: enum class Error { - ERROR = 1, + NOT_FOUND = 1, + PREVIOUS_NOT_FOUND, }; BabeConfigRepositoryImpl( diff --git a/core/consensus/babe/impl/block_appender_base.cpp b/core/consensus/babe/impl/block_appender_base.cpp index 28ada59221..969100a90e 100644 --- a/core/consensus/babe/impl/block_appender_base.cpp +++ b/core/consensus/babe/impl/block_appender_base.cpp @@ -220,9 +220,9 @@ namespace kagome::consensus::babe { } OUTCOME_TRY( - babe_config_opt, + babe_config_ptr, babe_config_repo_->config(*block.header.parentInfo(), epoch_number)); - auto &babe_config = *babe_config_opt; + auto &babe_config = *babe_config_ptr; SL_TRACE(logger_, "Actual epoch digest to apply block {} (slot {}, epoch {}). " diff --git a/core/utils/block_info_key.hpp b/core/utils/block_info_key.hpp new file mode 100644 index 0000000000..6376bcf3cc --- /dev/null +++ b/core/utils/block_info_key.hpp @@ -0,0 +1,39 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef KAGOME_UTILS_BLOCK_INFO_KEY_HPP +#define KAGOME_UTILS_BLOCK_INFO_KEY_HPP + +#include +#include + +#include "primitives/common.hpp" + +namespace kagome { + struct BlockInfoKey { + static constexpr size_t kNumberSize = sizeof(primitives::BlockNumber); + static constexpr size_t kHashSize = primitives::BlockHash::size(); + using Key = common::Blob; + + static Key encode(const primitives::BlockInfo &info) { + Key key; + boost::endian::store_big_u32(key.data(), info.number); + std::copy_n(info.hash.data(), kHashSize, key.data() + kNumberSize); + return key; + } + + static std::optional decode(common::BufferView key) { + if (key.size() != Key::size()) { + return std::nullopt; + } + primitives::BlockInfo info; + info.number = boost::endian::load_big_u32(key.data()); + std::copy_n(key.data() + kNumberSize, kHashSize, info.hash.data()); + return info; + } + }; +} // namespace kagome + +#endif // KAGOME_UTILS_BLOCK_INFO_KEY_HPP From e77eb9210dbf3eeca76ce91df10a952dc915c849 Mon Sep 17 00:00:00 2001 From: turuslan Date: Thu, 22 Jun 2023 18:21:43 +0300 Subject: [PATCH 06/12] comment Signed-off-by: turuslan --- core/blockchain/indexer.hpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/core/blockchain/indexer.hpp b/core/blockchain/indexer.hpp index 4810af0975..dbd4ddb1b4 100644 --- a/core/blockchain/indexer.hpp +++ b/core/blockchain/indexer.hpp @@ -66,6 +66,11 @@ namespace kagome::blockchain { bool inherit = false; }; + /** + * Used to store and query inheritable values for blocks. + * Used to store changes from digests. + * Ensures that no block between found and requested change that value. + */ template struct Indexer { Indexer(std::shared_ptr db, @@ -182,6 +187,14 @@ namespace kagome::blockchain { } } + /** + * Search first inherited value for `block` descending by `descent`. + * Unindexed blocks are indexed with `cb`. + * `cb` args: + * - `Option` of previous block with value. + * - `size_t` and `size_t` [first..last] indices into `descent.path_` + * blocks. `descent.path_` is reversed, so indices decrease. + */ template std::optional search(Descent &descent, const primitives::BlockInfo &block, From b99385e1aa5ae82cda8ff9d020715bed23decd1b Mon Sep 17 00:00:00 2001 From: turuslan Date: Thu, 22 Jun 2023 19:10:27 +0300 Subject: [PATCH 07/12] fix merge Signed-off-by: turuslan --- core/consensus/babe/impl/babe_impl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/consensus/babe/impl/babe_impl.cpp b/core/consensus/babe/impl/babe_impl.cpp index a12efaa5e6..19803e3341 100644 --- a/core/consensus/babe/impl/babe_impl.cpp +++ b/core/consensus/babe/impl/babe_impl.cpp @@ -331,7 +331,7 @@ namespace kagome::consensus::babe { auto babe_config = babe_config_repo_->config(best_block_, current_epoch_.epoch_number); - if (not babe_config and app_config_.syncMethod() != SyncMethod::Warp) { + if (not babe_config and sync_method_ != SyncMethod::Warp) { SL_CRITICAL( log_, "Can't obtain digest of epoch {} from block tree for block {}", From 7143dd5398942781179f966e4aeced56a3d47d97 Mon Sep 17 00:00:00 2001 From: turuslan Date: Thu, 22 Jun 2023 20:47:17 +0300 Subject: [PATCH 08/12] pr comments Signed-off-by: turuslan --- ...ange.hpp => has_babe_consensus_digest.hpp} | 22 ++++++++++++++----- .../babe/impl/babe_config_repository_impl.cpp | 15 ++++++++----- .../babe/impl/babe_config_repository_impl.hpp | 4 ++-- core/network/impl/synchronizer_impl.cpp | 6 ++--- 4 files changed, 32 insertions(+), 15 deletions(-) rename core/consensus/babe/{has_authority_set_change.hpp => has_babe_consensus_digest.hpp} (63%) diff --git a/core/consensus/babe/has_authority_set_change.hpp b/core/consensus/babe/has_babe_consensus_digest.hpp similarity index 63% rename from core/consensus/babe/has_authority_set_change.hpp rename to core/consensus/babe/has_babe_consensus_digest.hpp index 235d5b62bd..8e798fbdbc 100644 --- a/core/consensus/babe/has_authority_set_change.hpp +++ b/core/consensus/babe/has_babe_consensus_digest.hpp @@ -3,14 +3,20 @@ * SPDX-License-Identifier: Apache-2.0 */ -#ifndef KAGOME_CONSENSUS_BABE_HAS_AUTHORITY_SET_CHANGE_HPP -#define KAGOME_CONSENSUS_BABE_HAS_AUTHORITY_SET_CHANGE_HPP +#ifndef KAGOME_CONSENSUS_BABE_HAS_BABE_CONSENSUS_DIGEST_HPP +#define KAGOME_CONSENSUS_BABE_HAS_BABE_CONSENSUS_DIGEST_HPP #include "primitives/block_header.hpp" +#include "log/logger.hpp" + namespace kagome::consensus::babe { - struct HasAuthoritySetChange { - HasAuthoritySetChange(const primitives::BlockHeader &block) { + struct HasBabeConsensusDigest { + static auto logger() { + return log::createLogger("HasBabeConsensusDigest", "babe"); + } + + HasBabeConsensusDigest(const primitives::BlockHeader &block) { for (auto &digest : block.digest) { auto consensus = boost::get(&digest); if (not consensus) { @@ -18,6 +24,12 @@ namespace kagome::consensus::babe { } auto decoded_res = consensus->decode(); if (not decoded_res) { + SL_WARN(logger(), + "error decoding digest block={} engine={} digest={}: {}", + block.number, + consensus->consensus_engine_id.toHex(), + consensus->data.toHex(), + decoded_res.error()); continue; } auto &decoded = decoded_res.value(); @@ -45,4 +57,4 @@ namespace kagome::consensus::babe { }; } // namespace kagome::consensus::babe -#endif // KAGOME_CONSENSUS_BABE_HAS_AUTHORITY_SET_CHANGE_HPP +#endif // KAGOME_CONSENSUS_BABE_HAS_BABE_CONSENSUS_DIGEST_HPP diff --git a/core/consensus/babe/impl/babe_config_repository_impl.cpp b/core/consensus/babe/impl/babe_config_repository_impl.cpp index 71607e2be0..7577736e30 100644 --- a/core/consensus/babe/impl/babe_config_repository_impl.cpp +++ b/core/consensus/babe/impl/babe_config_repository_impl.cpp @@ -34,7 +34,12 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::consensus::babe, } namespace kagome::consensus::babe { - constexpr size_t kDontIndexFinalizedBlocks = 10000; + /** + * If there is more than `kMaxUnindexedBlocksNum` unindexed finalized blocks + * and last finalized block has state, then babe won't index all of them, but + * recover with runtime call and latest block with digest. + */ + constexpr size_t kMaxUnindexedBlocksNum = 10000; inline static primitives::NextConfigDataV1 getConfig( const primitives::BabeConfiguration &state) { @@ -88,7 +93,7 @@ namespace kagome::consensus::babe { auto finalized = block_tree_->getLastFinalized(); auto finalized_header = block_tree_->getBlockHeader(finalized.hash).value(); if (finalized.number - indexer_.last_finalized_indexed_.number - > kDontIndexFinalizedBlocks + > kMaxUnindexedBlocksNum and trie_storage_->getEphemeralBatchAt(finalized_header.state_root)) { warp(finalized); } @@ -256,7 +261,7 @@ namespace kagome::consensus::babe { std::vector refs; while (true) { OUTCOME_TRY(header, block_tree_->getBlockHeader(info.hash)); - if (HasAuthoritySetChange digests{header}) { + if (HasBabeConsensusDigest digests{header}) { value.next_state = applyDigests(value.config, digests); indexer_.put(info, {value, std::nullopt}, true); if (not refs.empty()) { @@ -282,7 +287,7 @@ namespace kagome::consensus::babe { while (true) { info = descent.path_.at(i_first); OUTCOME_TRY(header, block_tree_->getBlockHeader(info.hash)); - if (HasAuthoritySetChange digests{header}) { + if (HasBabeConsensusDigest digests{header}) { if (not prev_state) { BOOST_OUTCOME_TRY(prev_state, loadPrev(prev)); } @@ -323,7 +328,7 @@ namespace kagome::consensus::babe { std::shared_ptr BabeConfigRepositoryImpl::applyDigests( const primitives::NextConfigDataV1 &config, - const HasAuthoritySetChange &digests) const { + const HasBabeConsensusDigest &digests) const { BOOST_ASSERT(digests); auto state = std::make_shared(); state->slot_duration = slot_duration_; diff --git a/core/consensus/babe/impl/babe_config_repository_impl.hpp b/core/consensus/babe/impl/babe_config_repository_impl.hpp index 7391e13d88..6641b4f302 100644 --- a/core/consensus/babe/impl/babe_config_repository_impl.hpp +++ b/core/consensus/babe/impl/babe_config_repository_impl.hpp @@ -12,7 +12,7 @@ #include #include "blockchain/indexer.hpp" -#include "consensus/babe/has_authority_set_change.hpp" +#include "consensus/babe/has_babe_consensus_digest.hpp" #include "log/logger.hpp" #include "primitives/block_data.hpp" #include "primitives/event_types.hpp" @@ -106,7 +106,7 @@ namespace kagome::consensus::babe { std::shared_ptr applyDigests( const primitives::NextConfigDataV1 &config, - const HasAuthoritySetChange &digests) const; + const HasBabeConsensusDigest &digests) const; outcome::result load( const primitives::BlockInfo &block, diff --git a/core/network/impl/synchronizer_impl.cpp b/core/network/impl/synchronizer_impl.cpp index c523cffa2b..866fa5fe2e 100644 --- a/core/network/impl/synchronizer_impl.cpp +++ b/core/network/impl/synchronizer_impl.cpp @@ -9,7 +9,7 @@ #include "application/app_configuration.hpp" #include "blockchain/block_tree_error.hpp" -#include "consensus/babe/has_authority_set_change.hpp" +#include "consensus/babe/has_babe_consensus_digest.hpp" #include "consensus/grandpa/environment.hpp" #include "consensus/grandpa/has_authority_set_change.hpp" #include "network/helpers/peer_id_formatter.hpp" @@ -1022,7 +1022,7 @@ namespace kagome::network { while (block.number != 0) { if (auto _header = block_tree_->getBlockHeader(block.hash)) { auto &header = _header.value(); - if (consensus::babe::HasAuthoritySetChange(header)) { + if (consensus::babe::HasBabeConsensusDigest(header)) { break; } block = {header.number - 1, header.parent_hash}; @@ -1060,7 +1060,7 @@ namespace kagome::network { if (block.number < self->block_tree_->getLastFinalized().number) { self->block_storage_->assignNumberToHash(block).value(); } - if (consensus::babe::HasAuthoritySetChange(*header)) { + if (consensus::babe::HasBabeConsensusDigest(*header)) { cb(outcome::success()); return; } From b5502f3f9010331e9a6565c74129be87462a877a Mon Sep 17 00:00:00 2001 From: turuslan Date: Thu, 22 Jun 2023 21:31:36 +0300 Subject: [PATCH 09/12] minideb Signed-off-by: turuslan --- core/consensus/babe/impl/babe_impl.cpp | 2 ++ core/utils/block_info_key.hpp | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/core/consensus/babe/impl/babe_impl.cpp b/core/consensus/babe/impl/babe_impl.cpp index 19803e3341..9a96fd3cff 100644 --- a/core/consensus/babe/impl/babe_impl.cpp +++ b/core/consensus/babe/impl/babe_impl.cpp @@ -269,6 +269,7 @@ namespace kagome::consensus::babe { SL_INFO(log_, "Warp sync would be faster than Fast sync that was selected"); } + break; case SyncMethod::Warp: if (full_sync_duration < warp_sync_duration and full_sync_available) { @@ -278,6 +279,7 @@ namespace kagome::consensus::babe { SL_INFO(log_, "Fast sync would be faster than Warp sync that was selected"); } + break; } current_epoch_ = initial_epoch_res.value(); diff --git a/core/utils/block_info_key.hpp b/core/utils/block_info_key.hpp index 6376bcf3cc..4ebc5fc046 100644 --- a/core/utils/block_info_key.hpp +++ b/core/utils/block_info_key.hpp @@ -7,6 +7,7 @@ #define KAGOME_UTILS_BLOCK_INFO_KEY_HPP #include +#include #include #include "primitives/common.hpp" @@ -25,7 +26,7 @@ namespace kagome { } static std::optional decode(common::BufferView key) { - if (key.size() != Key::size()) { + if (libp2p::spanSize(key) != Key::size()) { return std::nullopt; } primitives::BlockInfo info; From 48dd9716808adbce31ca658652c6ef3695c4381e Mon Sep 17 00:00:00 2001 From: turuslan Date: Fri, 23 Jun 2023 16:55:15 +0300 Subject: [PATCH 10/12] comments Signed-off-by: turuslan --- core/blockchain/indexer.hpp | 20 +++++++++++++++++++ .../babe/impl/babe_config_repository_impl.cpp | 6 +++--- .../babe/impl/babe_config_repository_impl.hpp | 12 ++++++++++- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/core/blockchain/indexer.hpp b/core/blockchain/indexer.hpp index dbd4ddb1b4..499a4ae39b 100644 --- a/core/blockchain/indexer.hpp +++ b/core/blockchain/indexer.hpp @@ -11,11 +11,18 @@ #include "utils/block_info_key.hpp" namespace kagome::blockchain { + /** + * Cached ancestry check. + */ struct Descent { Descent(std::shared_ptr block_tree, primitives::BlockInfo start) : block_tree_{std::move(block_tree)}, path_{start} {} + /** + * Checks if `to` is ancestor of `start`. + * Caches intermediate blocks if `update_path_` is true. + */ bool descends(const primitives::BlockInfo &to) const { if (to == path_.front()) { return true; @@ -47,6 +54,9 @@ namespace kagome::blockchain { return to == path_.at(i); } + /** + * Get index in `path_` for block. + */ size_t indexFor(primitives::BlockNumber n) const { BOOST_ASSERT(n <= path_.front().number); return path_.front().number - n; @@ -61,8 +71,18 @@ namespace kagome::blockchain { struct Indexed { SCALE_TIE_ONLY(value, prev); + /** + * Empty `value` means that blocks from `prev` to current have been indexed, + * and current block doesn't have own `value`. + */ std::optional value; + /** + * Previous block with value. + */ std::optional prev; + /** + * Does this block inherit value from `prev` or has own `value`. + */ bool inherit = false; }; diff --git a/core/consensus/babe/impl/babe_config_repository_impl.cpp b/core/consensus/babe/impl/babe_config_repository_impl.cpp index 7577736e30..4c3f3e46ec 100644 --- a/core/consensus/babe/impl/babe_config_repository_impl.cpp +++ b/core/consensus/babe/impl/babe_config_repository_impl.cpp @@ -240,7 +240,7 @@ namespace kagome::consensus::babe { outcome::result> BabeConfigRepositoryImpl::config(const primitives::BlockInfo &block, - bool next) const { + bool next_epoch) const { auto descent = indexer_.descend(block); outcome::result cb_res = outcome::success(); auto cb = [&](std::optional prev, @@ -312,10 +312,10 @@ namespace kagome::consensus::babe { if (not r) { return Error::NOT_FOUND; } - if (not next and r->second.value->state) { + if (not next_epoch and r->second.value->state) { return *r->second.value->state; } - if (next) { + if (next_epoch) { OUTCOME_TRY(load(r->first, r->second)); return *r->second.value->next_state; } diff --git a/core/consensus/babe/impl/babe_config_repository_impl.hpp b/core/consensus/babe/impl/babe_config_repository_impl.hpp index 6641b4f302..72ea518548 100644 --- a/core/consensus/babe/impl/babe_config_repository_impl.hpp +++ b/core/consensus/babe/impl/babe_config_repository_impl.hpp @@ -41,8 +41,18 @@ namespace kagome::consensus::babe { struct BabeIndexedValue { SCALE_TIE_ONLY(config, state); + /** + * `NextConfigData` is rare digest, so always store recent config. + */ primitives::NextConfigDataV1 config; + /** + * Current epoch read from runtime. + * Used at genesis and after warp sync. + */ std::optional> state; + /** + * Next epoch lazily computed from `config` and digests. + */ std::optional> next_state; }; @@ -102,7 +112,7 @@ namespace kagome::consensus::babe { BabeSlotNumber getFirstBlockSlotNumber(); outcome::result> - config(const primitives::BlockInfo &block, bool next) const; + config(const primitives::BlockInfo &block, bool next_epoch) const; std::shared_ptr applyDigests( const primitives::NextConfigDataV1 &config, From 5cb35fa5e1685553b34bb8752fcb8507cea4cc97 Mon Sep 17 00:00:00 2001 From: turuslan Date: Tue, 27 Jun 2023 16:46:00 +0300 Subject: [PATCH 11/12] outcome init Signed-off-by: turuslan --- core/blockchain/indexer.hpp | 22 ++++++++++++------- .../babe/impl/babe_config_repository_impl.cpp | 4 ++++ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/core/blockchain/indexer.hpp b/core/blockchain/indexer.hpp index 499a4ae39b..94560b9031 100644 --- a/core/blockchain/indexer.hpp +++ b/core/blockchain/indexer.hpp @@ -98,20 +98,26 @@ namespace kagome::blockchain { : db_{std::move(db)}, block_tree_{std::move(block_tree)} { primitives::BlockInfo genesis{0, block_tree_->getGenesisBlockHash()}; last_finalized_indexed_ = genesis; + map_.emplace(genesis, Indexed{}); + } + + outcome::result init() { auto batch = db_->batch(); auto db_cur = db_->cursor(); - for (db_cur->seekFirst().value(); db_cur->isValid(); - db_cur->next().value()) { + OUTCOME_TRY(db_cur->seekFirst()); + while (db_cur->isValid()) { auto info = BlockInfoKey::decode(*db_cur->key()).value(); if (not block_tree_->isFinalized(info)) { - batch->remove(BlockInfoKey::encode(info)).value(); - continue; + OUTCOME_TRY(batch->remove(BlockInfoKey::encode(info))); + } else { + last_finalized_indexed_ = info; + BOOST_OUTCOME_TRY(map_[info], + scale::decode>(*db_cur->value())); } - last_finalized_indexed_ = info; - map_.emplace(info, scale::decode>(*db_cur->value()).value()); + OUTCOME_TRY(db_cur->next()); } - map_.emplace(genesis, Indexed{}); - batch->commit().value(); + OUTCOME_TRY(batch->commit()); + return outcome::success(); } Descent descend(const primitives::BlockInfo &from) const { diff --git a/core/consensus/babe/impl/babe_config_repository_impl.cpp b/core/consensus/babe/impl/babe_config_repository_impl.cpp index 4c3f3e46ec..b5a494044a 100644 --- a/core/consensus/babe/impl/babe_config_repository_impl.cpp +++ b/core/consensus/babe/impl/babe_config_repository_impl.cpp @@ -85,6 +85,10 @@ namespace kagome::consensus::babe { BOOST_ASSERT(babe_api_ != nullptr); BOOST_ASSERT(hasher_ != nullptr); + if (auto r = indexer_.init(); not r) { + logger_->error("Indexer::init error: {}", r.error()); + } + app_state_manager.atPrepare([this] { return prepare(); }); } From 3763389cd85ab6e9a7b62e3a16e3607296996c61 Mon Sep 17 00:00:00 2001 From: turuslan Date: Wed, 19 Jul 2023 14:49:45 +0300 Subject: [PATCH 12/12] pr comment Signed-off-by: turuslan --- core/blockchain/indexer.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/core/blockchain/indexer.hpp b/core/blockchain/indexer.hpp index 94560b9031..06a9e53a7c 100644 --- a/core/blockchain/indexer.hpp +++ b/core/blockchain/indexer.hpp @@ -44,6 +44,7 @@ namespace kagome::blockchain { if (chain.size() <= 1) { return false; } + path_.reserve(path_.size() + chain.size() - 1); for (size_t j = 1; j < chain.size(); ++j) { path_.emplace_back(path_.back().number - 1, chain[j]); }