diff --git a/core/blockchain/impl/digest_tracker_impl.cpp b/core/blockchain/impl/digest_tracker_impl.cpp index e98deb4059..6ea8abcfc1 100644 --- a/core/blockchain/impl/digest_tracker_impl.cpp +++ b/core/blockchain/impl/digest_tracker_impl.cpp @@ -80,10 +80,10 @@ namespace kagome::blockchain { return grandpa_digest_observer_->onDigest(context, digest); + } else if (message.consensus_engine_id == primitives::kBeefyEngineId) { + return outcome::success(); } else if (message.consensus_engine_id - == primitives::kUnsupportedEngineId_BEEF - or message.consensus_engine_id - == primitives::kUnsupportedEngineId_POL1) { + == primitives::kUnsupportedEngineId_POL1) { SL_TRACE(logger_, "Unsupported consensus engine id in block {}: {}", context.block_info, diff --git a/core/consensus/beefy/digest.hpp b/core/consensus/beefy/digest.hpp new file mode 100644 index 0000000000..0a9a73aabd --- /dev/null +++ b/core/consensus/beefy/digest.hpp @@ -0,0 +1,57 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "consensus/beefy/types.hpp" +#include "primitives/block_header.hpp" + +namespace kagome { + inline std::optional beefyValidatorsDigest( + const primitives::BlockHeader &block) { + for (auto &digest : block.digest) { + auto consensus = boost::get(&digest); + if (not consensus) { + continue; + } + if (consensus->consensus_engine_id != primitives::kBeefyEngineId) { + continue; + } + auto decoded_res = + scale::decode(consensus->data); + if (not decoded_res) { + continue; + } + auto &decoded = decoded_res.value(); + if (auto item = boost::get(&decoded)) { + return std::move(*item); + } + } + return std::nullopt; + } + + inline std::optional beefyMmrDigest( + const primitives::BlockHeader &block) { + for (auto &digest : block.digest) { + auto consensus = boost::get(&digest); + if (not consensus) { + continue; + } + if (consensus->consensus_engine_id != primitives::kBeefyEngineId) { + continue; + } + auto decoded_res = + scale::decode(consensus->data); + if (not decoded_res) { + continue; + } + auto &decoded = decoded_res.value(); + if (auto item = boost::get(&decoded)) { + return *item; + } + } + return std::nullopt; + } +} // namespace kagome diff --git a/core/consensus/beefy/sig.hpp b/core/consensus/beefy/sig.hpp index dc68b9e301..b5db42d6d3 100644 --- a/core/consensus/beefy/sig.hpp +++ b/core/consensus/beefy/sig.hpp @@ -27,9 +27,8 @@ namespace kagome::consensus::beefy { } inline bool verify(const crypto::EcdsaProvider &ecdsa, - const BeefyJustification &justification_v1, + const SignedCommitment &justification, const ValidatorSet &validators) { - auto &justification = boost::get(justification_v1); if (justification.commitment.validator_set_id != validators.id) { return false; } @@ -50,4 +49,11 @@ namespace kagome::consensus::beefy { } return valid >= threshold(total); } + + inline bool verify(const crypto::EcdsaProvider &ecdsa, + const BeefyJustification &justification_v1, + const ValidatorSet &validators) { + return verify( + ecdsa, boost::get(justification_v1), validators); + } } // namespace kagome::consensus::beefy diff --git a/core/consensus/beefy/types.hpp b/core/consensus/beefy/types.hpp index 387851e705..0cb9c9a02e 100644 --- a/core/consensus/beefy/types.hpp +++ b/core/consensus/beefy/types.hpp @@ -25,6 +25,15 @@ namespace kagome::consensus::beefy { std::vector validators; primitives::AuthoritySetId id; + + std::optional find( + const crypto::EcdsaPublicKey &key) const { + auto it = std::find(validators.begin(), validators.end(), key); + if (it == validators.end()) { + return std::nullopt; + } + return it - validators.begin(); + } }; using ConsensusDigest = @@ -34,6 +43,7 @@ namespace kagome::consensus::beefy { MmrRootHash>; using PayloadId = common::Blob<2>; + constexpr PayloadId kMmr{{'m', 'h'}}; struct Commitment { SCALE_TIE(3); @@ -55,8 +65,8 @@ namespace kagome::consensus::beefy { Commitment commitment; std::vector> signatures; }; - scale::ScaleEncoderStream &operator<<(scale::ScaleEncoderStream &s, - const SignedCommitment &v) { + inline scale::ScaleEncoderStream &operator<<(scale::ScaleEncoderStream &s, + const SignedCommitment &v) { s << v.commitment; size_t count = 0; common::Buffer bits; @@ -80,8 +90,8 @@ namespace kagome::consensus::beefy { } return s; } - scale::ScaleDecoderStream &operator>>(scale::ScaleDecoderStream &s, - SignedCommitment &v) { + inline scale::ScaleDecoderStream &operator>>(scale::ScaleDecoderStream &s, + SignedCommitment &v) { s >> v.commitment; common::Buffer bits; s >> bits; @@ -111,4 +121,6 @@ namespace kagome::consensus::beefy { } using BeefyJustification = boost::variant, SignedCommitment>; + + using BeefyGossipMessage = boost::variant; } // namespace kagome::consensus::beefy diff --git a/core/crypto/crypto_store/key_type.cpp b/core/crypto/crypto_store/key_type.cpp index d62598effe..2df6cbf1be 100644 --- a/core/crypto/crypto_store/key_type.cpp +++ b/core/crypto/crypto_store/key_type.cpp @@ -21,6 +21,7 @@ namespace kagome::crypto { KEY_TYPE_ASGN, KEY_TYPE_AUDI, KEY_TYPE_ACCO, + KEY_TYPE_BEEF, }; return supported_types.count(k) > 0; diff --git a/core/crypto/crypto_store/key_type.hpp b/core/crypto/crypto_store/key_type.hpp index e4bdc4fd7d..b418a182f3 100644 --- a/core/crypto/crypto_store/key_type.hpp +++ b/core/crypto/crypto_store/key_type.hpp @@ -34,6 +34,7 @@ namespace kagome::crypto { KEY_TYPE_AUDI = 0x69647561u, // Account discovery [sr25519, ed25519, secp256k1] KEY_TYPE_ASGN = 0x6e677361u, // ASGN KEY_TYPE_PARA = 0x61726170u, // PARA + KEY_TYPE_BEEF = 0x66656562u, // Beefy, secp256k1 // clang-format on }; diff --git a/core/crypto/crypto_store/session_keys.cpp b/core/crypto/crypto_store/session_keys.cpp index ee68ba62b9..b5892210a2 100644 --- a/core/crypto/crypto_store/session_keys.cpp +++ b/core/crypto/crypto_store/session_keys.cpp @@ -70,6 +70,7 @@ namespace kagome::crypto { store_->generateSr25519Keypair(KEY_TYPE_AUDI, *dev).value(); store_->generateSr25519Keypair(KEY_TYPE_ASGN, *dev).value(); store_->generateSr25519Keypair(KEY_TYPE_PARA, *dev).value(); + store_->generateEcdsaKeypair(KEY_TYPE_BEEF, *dev).value(); } } @@ -123,4 +124,12 @@ namespace kagome::crypto { return nullptr; } + SessionKeys::KeypairWithIndexOpt + SessionKeysImpl::getBeefKeyPair( + const std::vector &authorities) { + return find( + beef_key_pair_, KEY_TYPE_BEEF, authorities, std::equal_to{}); + } } // namespace kagome::crypto diff --git a/core/crypto/crypto_store/session_keys.hpp b/core/crypto/crypto_store/session_keys.hpp index 67d141cffe..4e029a29d8 100644 --- a/core/crypto/crypto_store/session_keys.hpp +++ b/core/crypto/crypto_store/session_keys.hpp @@ -8,6 +8,7 @@ #include "common/blob.hpp" #include "crypto/crypto_store/key_type.hpp" +#include "crypto/ecdsa_types.hpp" #include "network/types/roles.hpp" #include "primitives/authority.hpp" #include "primitives/authority_discovery_id.hpp" @@ -65,6 +66,12 @@ namespace kagome::crypto { */ virtual std::shared_ptr getAudiKeyPair( const std::vector &authorities) = 0; + + /** + * @return current BEEF session key pair + */ + virtual KeypairWithIndexOpt getBeefKeyPair( + const std::vector &authorities) = 0; }; class SessionKeysImpl : public SessionKeys { @@ -72,6 +79,7 @@ namespace kagome::crypto { KeypairWithIndexOpt gran_key_pair_; KeypairWithIndexOpt para_key_pair_; KeypairWithIndexOpt audi_key_pair_; + KeypairWithIndexOpt beef_key_pair_; network::Roles roles_; std::shared_ptr store_; @@ -107,6 +115,9 @@ namespace kagome::crypto { std::shared_ptr getAudiKeyPair( const std::vector &authorities) override; + + KeypairWithIndexOpt getBeefKeyPair( + const std::vector &authorities) override; }; } // namespace kagome::crypto diff --git a/core/injector/CMakeLists.txt b/core/injector/CMakeLists.txt index f94caac06c..3755f44882 100644 --- a/core/injector/CMakeLists.txt +++ b/core/injector/CMakeLists.txt @@ -7,6 +7,7 @@ add_library(application_injector application_injector.cpp ) target_link_libraries(application_injector + beefy_api Boost::Boost.DI runtime_wavm account_nonce_api diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index f02c51c339..b864c3cc58 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -99,6 +99,7 @@ #include "metrics/impl/metrics_watcher.hpp" #include "metrics/impl/prometheus/handler_impl.hpp" #include "metrics/metrics.hpp" +#include "network/beefy/beefy.hpp" #include "network/impl/block_announce_transmitter_impl.hpp" #include "network/impl/extrinsic_observer_impl.hpp" #include "network/impl/grandpa_transmitter_impl.hpp" @@ -143,6 +144,7 @@ #include "runtime/runtime_api/impl/account_nonce_api.hpp" #include "runtime/runtime_api/impl/authority_discovery_api.hpp" #include "runtime/runtime_api/impl/babe_api.hpp" +#include "runtime/runtime_api/impl/beefy.hpp" #include "runtime/runtime_api/impl/block_builder.hpp" #include "runtime/runtime_api/impl/core.hpp" #include "runtime/runtime_api/impl/grandpa_api.hpp" @@ -502,6 +504,7 @@ namespace { di::bind.template to(), di::bind.template to(), di::bind.template to(), + di::bind.template to(), di::bind.template to(), di::bind.template to(), di::bind.template to(), @@ -796,6 +799,7 @@ namespace { di::bind.template to(), di::bind.template to(), di::bind.template to(), + di::bind.template to(), di::bind.template to(), di::bind.template to(), di::bind.template to(), diff --git a/core/metrics/histogram_timer.hpp b/core/metrics/histogram_timer.hpp index 5b652027e0..ea30c09948 100644 --- a/core/metrics/histogram_timer.hpp +++ b/core/metrics/histogram_timer.hpp @@ -21,6 +21,20 @@ namespace kagome::metrics { return buckets; } + struct GaugeHelper { + GaugeHelper(const std::string &name, const std::string &help) { + registry_->registerGaugeFamily(name, help); + metric_ = registry_->registerGaugeMetric(name); + } + + auto *operator->() { + return metric_; + } + + metrics::RegistryPtr registry_ = metrics::createRegistry(); + metrics::Gauge *metric_; + }; + struct HistogramHelper { HistogramHelper(const std::string &name, const std::string &help, diff --git a/core/network/CMakeLists.txt b/core/network/CMakeLists.txt index c3c84e06ba..741cd5de9c 100644 --- a/core/network/CMakeLists.txt +++ b/core/network/CMakeLists.txt @@ -7,6 +7,8 @@ add_subdirectory(protobuf) add_library(network + beefy/beefy.cpp + beefy/protocol.cpp impl/protocols/light.cpp impl/state_protocol_observer_impl.cpp impl/state_sync_request_flow.cpp @@ -38,6 +40,7 @@ target_link_libraries(network core_api light_api_proto p2p::p2p_basic_scheduler + runtime_transaction_error executor scale_libp2p_types blockchain diff --git a/core/network/adapters/protobuf_block_response.hpp b/core/network/adapters/protobuf_block_response.hpp index 03afb16b17..f07e54d8d2 100644 --- a/core/network/adapters/protobuf_block_response.hpp +++ b/core/network/adapters/protobuf_block_response.hpp @@ -58,7 +58,7 @@ namespace kagome::network { vec.emplace_back(primitives::kGrandpaEngineId, gran->data); } if (auto &beef = src_block.beefy_justification) { - vec.emplace_back(primitives::kUnsupportedEngineId_BEEF, beef->data); + vec.emplace_back(primitives::kBeefyEngineId, beef->data); } dst_block->set_justifications( common::Buffer{scale::encode(vec).value()}.toString()); @@ -124,7 +124,7 @@ namespace kagome::network { if (engine == primitives::kGrandpaEngineId) { justification = primitives::Justification{std::move(raw)}; } - if (engine == primitives::kUnsupportedEngineId_BEEF) { + if (engine == primitives::kBeefyEngineId) { beefy_justification = primitives::Justification{std::move(raw)}; } } diff --git a/core/network/beefy/beefy.cpp b/core/network/beefy/beefy.cpp new file mode 100644 index 0000000000..c3a1760efb --- /dev/null +++ b/core/network/beefy/beefy.cpp @@ -0,0 +1,446 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "network/beefy/beefy.hpp" + +#include "application/app_state_manager.hpp" +#include "application/chain_spec.hpp" +#include "blockchain/block_tree.hpp" +#include "blockchain/block_tree_error.hpp" +#include "consensus/beefy/digest.hpp" +#include "consensus/beefy/sig.hpp" +#include "crypto/crypto_store/session_keys.hpp" +#include "metrics/histogram_timer.hpp" +#include "network/beefy/protocol.hpp" +#include "runtime/common/runtime_transaction_error.hpp" +#include "runtime/runtime_api/beefy.hpp" +#include "storage/spaced_storage.hpp" +#include "utils/block_number_key.hpp" +#include "utils/thread_pool.hpp" + +// TODO(turuslan): #1651, report equivocation + +namespace kagome::network { + metrics::GaugeHelper metric_validator_set_id{ + "kagome_beefy_validator_set_id", + "Current BEEFY active validator set id.", + }; + metrics::GaugeHelper metric_finalized{ + "kagome_beefy_best_block", + "Best block finalized by BEEFY", + }; + + Beefy::Beefy(application::AppStateManager &app_state_manager, + const application::ChainSpec &chain_spec, + std::shared_ptr block_tree, + std::shared_ptr beefy_api, + std::shared_ptr ecdsa, + std::shared_ptr db, + std::shared_ptr thread_pool, + std::shared_ptr main_thread, + std::shared_ptr session_keys, + LazySPtr beefy_protocol, + std::shared_ptr + chain_sub_engine) + : block_tree_{std::move(block_tree)}, + beefy_api_{std::move(beefy_api)}, + ecdsa_{std::move(ecdsa)}, + db_{db->getSpace(storage::Space::kBeefyJustification)}, + strand_inner_{thread_pool->io_context()}, + strand_{*strand_inner_}, + main_thread_{std::move(main_thread)}, + session_keys_{std::move(session_keys)}, + beefy_protocol_{std::move(beefy_protocol)}, + min_delta_{chain_spec.isWococo() ? 4u : 8u}, + log_{log::createLogger("Beefy")} { + app_state_manager.atLaunch([=]() mutable { + start(std::move(chain_sub_engine)); + return true; + }); + } + + outcome::result> + Beefy::getJustification(primitives::BlockNumber block) const { + OUTCOME_TRY(raw, db_->tryGet(BlockNumberKey::encode(block))); + if (raw) { + OUTCOME_TRY(r, scale::decode(*raw)); + return outcome::success(std::make_optional(std::move(r))); + } + return outcome::success(std::nullopt); + } + + void Beefy::onJustification(const primitives::BlockHash &block_hash, + primitives::Justification raw) { + strand_.post([weak = weak_from_this(), block_hash, raw = std::move(raw)] { + if (auto self = weak.lock()) { + std::ignore = self->onJustificationOutcome(block_hash, std::move(raw)); + } + }); + } + + outcome::result Beefy::onJustificationOutcome( + const primitives::BlockHash &block_hash, primitives::Justification raw) { + if (not beefy_genesis_) { + return outcome::success(); + } + OUTCOME_TRY(justification_v1, + scale::decode(raw.data)); + auto &justification = + boost::get(justification_v1); + OUTCOME_TRY(header, block_tree_->getBlockHeader(block_hash)); + if (justification.commitment.block_number != header.number) { + return outcome::success(); + } + return onJustification(std::move(justification)); + } + + void Beefy::onMessage(consensus::beefy::BeefyGossipMessage message) { + if (not strand_.running_in_this_thread()) { + return strand_.post( + [weak = weak_from_this(), message = std::move(message)] { + if (auto self = weak.lock()) { + self->onMessage(std::move(message)); + } + }); + } + if (not beefy_genesis_) { + return; + } + if (auto justification_v1 = + boost::get(&message)) { + auto &justification = + boost::get(*justification_v1); + if (justification.commitment.block_number + > block_tree_->bestBlock().number) { + return; + } + std::ignore = onJustification(std::move(justification)); + } else { + onVote(boost::get(message), false); + } + } + + void Beefy::onVote(consensus::beefy::VoteMessage vote, bool broadcast) { + auto block_number = vote.commitment.block_number; + if (block_number < *beefy_genesis_) { + SL_VERBOSE(log_, "vote for block {} before genesis", block_number); + return; + } + if (block_number <= beefy_finalized_) { + return; + } + if (block_number >= next_digest_) { + SL_VERBOSE(log_, "ignoring vote for unindexed block {}", block_number); + return; + } + auto next_session = sessions_.upper_bound(block_number); + if (next_session == sessions_.begin()) { + return; + } + auto &session = std::prev(next_session)->second; + if (vote.commitment.validator_set_id != session.validators.id) { + SL_VERBOSE(log_, "wrong validator set id for block {}", block_number); + return; + } + auto index = session.validators.find(vote.id); + if (not index) { + SL_VERBOSE(log_, "unknown validator for block {}", block_number); + return; + } + if (not verify(*ecdsa_, vote)) { + SL_VERBOSE(log_, "wrong vote for block {}", block_number); + return; + } + auto total = session.validators.validators.size(); + auto round = session.rounds.find(block_number); + if (round == session.rounds.end()) { + round = + session.rounds + .emplace(block_number, + consensus::beefy::SignedCommitment{vote.commitment, {}}) + .first; + round->second.signatures.resize(total); + } + if (round->second.signatures[*index]) { + return; + } + round->second.signatures[*index] = vote.signature; + size_t count = 0; + for (auto &sig : round->second.signatures) { + if (sig) { + ++count; + } + } + if (count >= consensus::beefy::threshold(total)) { + std::ignore = apply(session.rounds.extract(round).mapped(), true); + } else if (broadcast) { + main_thread_->post( + [protocol{beefy_protocol_.get()}, + message{std::make_shared( + std::move(vote))}] { protocol->broadcast(std::move(message)); }); + } + } + + void Beefy::start(std::shared_ptr + chain_sub_engine) { + auto cursor = db_->cursor(); + std::ignore = cursor->seekLast(); + if (cursor->isValid()) { + beefy_finalized_ = BlockNumberKey::decode(*cursor->key()).value(); + metric_finalized->set(beefy_finalized_); + } + SL_INFO(log_, "last finalized {}", beefy_finalized_); + chain_sub_ = std::make_shared( + chain_sub_engine); + chain_sub_->subscribe(chain_sub_->generateSubscriptionSetId(), + primitives::events::ChainEventType::kFinalizedHeads); + auto on_finalize = [weak = weak_from_this()]( + subscription::SubscriptionSetId, + auto &&, + primitives::events::ChainEventType, + const primitives::events::ChainEventParams &) { + if (auto self = weak.lock()) { + self->strand_.post([weak] { + if (auto self = weak.lock()) { + std::ignore = self->update(); + } + }); + } + }; + chain_sub_->setCallback(std::move(on_finalize)); + strand_.post([weak = weak_from_this()] { + if (auto self = weak.lock()) { + std::ignore = self->update(); + } + }); + } + + bool Beefy::hasJustification(primitives::BlockNumber block) const { + auto r = db_->contains(BlockNumberKey::encode(block)); + return r and r.value(); + } + + outcome::result Beefy::findValidators( + primitives::BlockNumber max, primitives::BlockNumber min) const { + OUTCOME_TRY(opt, block_tree_->getBlockHash(max)); + if (not opt) { + return blockchain::BlockTreeError::HEADER_NOT_FOUND; + } + primitives::BlockInfo info{max, *opt}; + while (true) { + if (info.number <= *beefy_genesis_) { + // bug: beefy pallet doesn't produce digest with first validators + OUTCOME_TRY(validators, beefy_api_->validatorSet(info.hash)); + if (not validators) { + return runtime::RuntimeTransactionError::EXPORT_FUNCTION_NOT_FOUND; + } + return std::make_pair(info.number, std::move(*validators)); + } + OUTCOME_TRY(header, block_tree_->getBlockHeader(info.hash)); + if (auto validators = beefyValidatorsDigest(header)) { + return std::make_pair(info.number, std::move(*validators)); + } + if (info.number <= min) { + return std::nullopt; + } + info = *header.parentInfo(); + } + } + + outcome::result Beefy::onJustification( + consensus::beefy::SignedCommitment justification) { + auto block_number = justification.commitment.block_number; + if (block_number < *beefy_genesis_) { + return outcome::success(); + } + pending_justifications_.emplace(block_number, std::move(justification)); + return update(); + } + + outcome::result Beefy::apply( + consensus::beefy::SignedCommitment justification, bool broadcast) { + auto block_number = justification.commitment.block_number; + if (block_number == beefy_finalized_) { + return outcome::success(); + } + if (hasJustification(block_number)) { + return outcome::success(); + } + FindValidatorsResult found; + if (block_number <= beefy_finalized_) { + BOOST_OUTCOME_TRY(found, findValidators(block_number, block_number)); + if (not found) { + return outcome::success(); + } + } else if (block_number >= next_digest_) { + BOOST_OUTCOME_TRY(found, findValidators(block_number, next_digest_)); + } + auto session = sessions_.end(); + if (not found) { + if (block_number <= beefy_finalized_) { + return outcome::success(); + } + auto next_session = sessions_.upper_bound(block_number); + if (next_session == sessions_.begin()) { + SL_TRACE(log_, "no session for block {}", block_number); + return outcome::success(); + } + session = std::prev(next_session); + } + auto &first = found ? found->first : session->first; + auto &validators = found ? found->second : session->second.validators; + if (justification.commitment.validator_set_id != validators.id) { + SL_VERBOSE(log_, "wrong validator set id for block {}", block_number); + return outcome::success(); + } + if (not verify(*ecdsa_, justification, validators)) { + SL_VERBOSE(log_, "wrong justification for block {}", block_number); + return outcome::success(); + } + consensus::beefy::BeefyJustification justification_v1{ + std::move(justification)}; + OUTCOME_TRY(db_->put(BlockNumberKey::encode(block_number), + scale::encode(justification_v1).value())); + if (beefy_finalized_ > *beefy_genesis_ + and sessions_.count(beefy_finalized_) == 0) { + OUTCOME_TRY(last_hash, block_tree_->getBlockHash(beefy_finalized_)); + if (last_hash) { + if (auto r = block_tree_->getBlockHeader(*last_hash)) { + if (not beefyValidatorsDigest(r.value())) { + OUTCOME_TRY(db_->remove(BlockNumberKey::encode(beefy_finalized_))); + } + } + } + } + if (block_number <= beefy_finalized_) { + return outcome::success(); + } + sessions_.erase(sessions_.begin(), session); + if (session != sessions_.end()) { + session->second.rounds.erase( + session->second.rounds.begin(), + session->second.rounds.upper_bound(block_number)); + } + if (found) { + sessions_.emplace(first, Session{std::move(validators), {}}); + metricValidatorSetId(); + } + SL_INFO(log_, "finalized {}", block_number); + beefy_finalized_ = block_number; + metric_finalized->set(beefy_finalized_); + next_digest_ = std::max(next_digest_, block_number + 1); + if (broadcast) { + main_thread_->post( + [protocol{beefy_protocol_.get()}, + message{std::make_shared( + std::move(justification_v1))}] { + protocol->broadcast(std::move(message)); + }); + } + return outcome::success(); + } + + outcome::result Beefy::update() { + auto grandpa_finalized = block_tree_->getLastFinalized(); + if (not beefy_genesis_) { + BOOST_OUTCOME_TRY(beefy_genesis_, + beefy_api_->genesis(grandpa_finalized.hash)); + if (not beefy_genesis_) { + SL_TRACE(log_, "no beefy pallet yet"); + return outcome::success(); + } + next_digest_ = std::max(beefy_finalized_, *beefy_genesis_); + } + if (grandpa_finalized.number < *beefy_genesis_) { + return outcome::success(); + } + for (auto pending_it = pending_justifications_.begin(); + pending_it != pending_justifications_.end() + and pending_it->first <= grandpa_finalized.number;) { + std::ignore = + apply(pending_justifications_.extract(pending_it++).mapped(), false); + } + while (next_digest_ <= grandpa_finalized.number) { + OUTCOME_TRY( + found, + findValidators(next_digest_, + sessions_.empty() ? *beefy_genesis_ : next_digest_)); + if (found) { + sessions_.emplace(found->first, Session{std::move(found->second), {}}); + metricValidatorSetId(); + } + ++next_digest_; + } + std::ignore = vote(); + return outcome::success(); + } + + outcome::result Beefy::vote() { + auto next_session = sessions_.upper_bound(beefy_finalized_ + 1); + if (next_session == sessions_.begin()) { + SL_VERBOSE(log_, "can't vote: no sessions"); + return outcome::success(); + } + auto session = std::prev(next_session); + auto grandpa_finalized = block_tree_->getLastFinalized().number; + auto target = session->first; + if (target <= beefy_finalized_) { + auto diff = grandpa_finalized - beefy_finalized_ + 1; + target = beefy_finalized_ + + std::max( + min_delta_, math::nextHighPowerOf2(diff / 2)); + if (next_session != sessions_.end() and target >= next_session->first) { + target = next_session->first; + session = next_session; + } + } + if (target > grandpa_finalized) { + return outcome::success(); + } + if (target <= last_voted_) { + return outcome::success(); + } + auto key = + session_keys_->getBeefKeyPair(session->second.validators.validators); + if (not key) { + SL_TRACE(log_, + "can't vote: not validator of set {}", + session->second.validators.id); + return outcome::success(); + } + OUTCOME_TRY(block_hash, block_tree_->getBlockHash(target)); + if (not block_hash) { + SL_VERBOSE(log_, "can't vote: no block {}", target); + return outcome::success(); + } + OUTCOME_TRY(header, block_tree_->getBlockHeader(*block_hash)); + auto mmr = beefyMmrDigest(header); + if (not mmr) { + SL_VERBOSE(log_, "can't vote: no mmr digest in block {}", target); + return outcome::success(); + } + consensus::beefy::VoteMessage vote; + vote.commitment = { + {{consensus::beefy::kMmr, common::Buffer{*mmr}}}, + target, + session->second.validators.id, + }; + vote.id = key->first->public_key; + BOOST_OUTCOME_TRY( + vote.signature, + ecdsa_->signPrehashed(consensus::beefy::prehash(vote.commitment), + key->first->secret_key)); + onVote(std::move(vote), true); + last_voted_ = target; + return outcome::success(); + } + + void Beefy::metricValidatorSetId() { + if (not sessions_.empty()) { + metric_validator_set_id->set( + std::prev(sessions_.end())->second.validators.id); + } + } +} // namespace kagome::network diff --git a/core/network/beefy/beefy.hpp b/core/network/beefy/beefy.hpp new file mode 100644 index 0000000000..5859d461c2 --- /dev/null +++ b/core/network/beefy/beefy.hpp @@ -0,0 +1,119 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +#include "consensus/beefy/types.hpp" +#include "injector/lazy.hpp" +#include "log/logger.hpp" +#include "network/beefy/i_beefy.hpp" +#include "primitives/event_types.hpp" +#include "primitives/justification.hpp" +#include "storage/buffer_map_types.hpp" + +namespace kagome { + class ThreadPool; +} // namespace kagome + +namespace kagome::application { + class AppStateManager; + class ChainSpec; +} // namespace kagome::application + +namespace kagome::blockchain { + class BlockTree; +} // namespace kagome::blockchain + +namespace kagome::crypto { + class EcdsaProvider; + class SessionKeys; +} // namespace kagome::crypto + +namespace kagome::runtime { + class BeefyApi; +} // namespace kagome::runtime + +namespace kagome::storage { + class SpacedStorage; +} // namespace kagome::storage + +namespace kagome::network { + class BeefyProtocol; + + class Beefy : public IBeefy, public std::enable_shared_from_this { + public: + Beefy(application::AppStateManager &app_state_manager, + const application::ChainSpec &chain_spec, + std::shared_ptr block_tree, + std::shared_ptr beefy_api, + std::shared_ptr ecdsa, + std::shared_ptr db, + std::shared_ptr thread_pool, + std::shared_ptr main_thread, + std::shared_ptr session_keys, + LazySPtr beefy_protocol, + std::shared_ptr + chain_sub_engine); + + outcome::result> + getJustification(primitives::BlockNumber block) const override; + + void onJustification(const primitives::BlockHash &block_hash, + primitives::Justification raw) override; + + void onMessage(consensus::beefy::BeefyGossipMessage message); + + private: + struct Session { + consensus::beefy::ValidatorSet validators; + std::map + rounds; + }; + using Sessions = std::map; + + void start(std::shared_ptr + chain_sub_engine); + bool hasJustification(primitives::BlockNumber block) const; + using FindValidatorsResult = std::optional< + std::pair>; + outcome::result findValidators( + primitives::BlockNumber max, primitives::BlockNumber min) const; + outcome::result onJustificationOutcome( + const primitives::BlockHash &block_hash, primitives::Justification raw); + outcome::result onJustification( + consensus::beefy::SignedCommitment justification); + void onVote(consensus::beefy::VoteMessage vote, bool broadcast); + outcome::result apply( + consensus::beefy::SignedCommitment justification, bool broadcast); + outcome::result update(); + outcome::result vote(); + void metricValidatorSetId(); + + std::shared_ptr block_tree_; + std::shared_ptr beefy_api_; + std::shared_ptr ecdsa_; + std::shared_ptr db_; + std::shared_ptr strand_inner_; + boost::asio::io_context::strand strand_; + std::shared_ptr main_thread_; + std::shared_ptr session_keys_; + LazySPtr beefy_protocol_; + primitives::BlockNumber min_delta_; + std::shared_ptr chain_sub_; + + std::optional beefy_genesis_; + primitives::BlockNumber beefy_finalized_ = 0; + primitives::BlockNumber next_digest_ = 0; + primitives::BlockNumber last_voted_ = 0; + Sessions sessions_; + std::map + pending_justifications_; + + log::Logger log_; + }; +} // namespace kagome::network diff --git a/core/network/beefy/i_beefy.hpp b/core/network/beefy/i_beefy.hpp new file mode 100644 index 0000000000..848dc5726c --- /dev/null +++ b/core/network/beefy/i_beefy.hpp @@ -0,0 +1,22 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "consensus/beefy/types.hpp" +#include "primitives/justification.hpp" + +namespace kagome::network { + class IBeefy { + public: + virtual ~IBeefy() = default; + + virtual outcome::result> + getJustification(primitives::BlockNumber block) const = 0; + + virtual void onJustification(const primitives::BlockHash &block_hash, + primitives::Justification raw) = 0; + }; +} // namespace kagome::network diff --git a/core/network/beefy/protocol.cpp b/core/network/beefy/protocol.cpp new file mode 100644 index 0000000000..9a36be7907 --- /dev/null +++ b/core/network/beefy/protocol.cpp @@ -0,0 +1,107 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "network/beefy/protocol.hpp" + +#include "blockchain/genesis_block_hash.hpp" +#include "network/beefy/beefy.hpp" +#include "network/common.hpp" +#include "network/impl/protocols/protocol_error.hpp" +#include "network/impl/stream_engine.hpp" +#include "network/notifications/connect_and_handshake.hpp" +#include "network/notifications/handshake_and_read_messages.hpp" + +namespace kagome::network { + BeefyJustificationProtocol::BeefyJustificationProtocol(libp2p::Host &host, + const blockchain::GenesisBlockHash &genesis, + std::shared_ptr beefy) + : RequestResponseProtocolType{ + kName, + host, + make_protocols(kBeefyJustificationProtocol, genesis), + log::createLogger(kName), + }, + beefy_{std::move(beefy)} {} + + std::optional> + BeefyJustificationProtocol::onRxRequest(RequestType block, + std::shared_ptr) { + OUTCOME_TRY(opt, beefy_->getJustification(block)); + if (opt) { + return outcome::success(std::move(*opt)); + } + return outcome::failure(ProtocolError::NO_RESPONSE); + } + + BeefyProtocol::BeefyProtocol(libp2p::Host &host, + const blockchain::GenesisBlockHash &genesis, + Roles roles, + std::shared_ptr beefy, + std::shared_ptrstream_engine + ) + : base_{ + kName, + host, + make_protocols(kBeefyProtocol, genesis), + log::createLogger(kName), + }, + roles_{roles}, + beefy_{std::move(beefy)}, + stream_engine_{std::move(stream_engine)} + {} + + bool BeefyProtocol::start() { + return base_.start(weak_from_this()); + } + + const std::string &BeefyProtocol::protocolName() const { + return base_.protocolName(); + } + + void BeefyProtocol::onIncomingStream(std::shared_ptr stream) { + auto on_handshake = [](std::shared_ptr self, + std::shared_ptr stream, + Roles) { + self->stream_engine_->addIncoming(stream, self); + return true; + }; + auto on_message = [](std::shared_ptr self, + consensus::beefy::BeefyGossipMessage message) { + self->beefy_->onMessage(std::move(message)); + return true; + }; + notifications::handshakeAndReadMessages< + consensus::beefy::BeefyGossipMessage>(weak_from_this(), + std::move(stream), + roles_, + std::move(on_handshake), + std::move(on_message)); + } + + void BeefyProtocol::newOutgoingStream( + const PeerInfo &peer, + std::function>)> &&cb) { + auto on_handshake = + [cb = std::move(cb)]( + std::shared_ptr self, + outcome::result> + r) mutable { + if (not r) { + cb(r.error()); + return; + } + auto &stream = std::get<0>(r.value()); + self->stream_engine_->addOutgoing(stream, self); + cb(std::move(stream)); + }; + notifications::connectAndHandshake( + weak_from_this(), base_, peer, roles_, std::move(on_handshake)); + } + + void BeefyProtocol::broadcast( + std::shared_ptr message) { + stream_engine_->broadcast(shared_from_this(), message); + } +} // namespace kagome::network diff --git a/core/network/beefy/protocol.hpp b/core/network/beefy/protocol.hpp new file mode 100644 index 0000000000..52373f473c --- /dev/null +++ b/core/network/beefy/protocol.hpp @@ -0,0 +1,70 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "consensus/beefy/types.hpp" +#include "network/helpers/scale_message_read_writer.hpp" +#include "network/impl/protocols/request_response_protocol.hpp" +#include "network/types/roles.hpp" + +namespace kagome::blockchain { + class GenesisBlockHash; +} // namespace kagome::blockchain + +namespace kagome::network { + class Beefy; + struct StreamEngine; + + class BeefyJustificationProtocol + : public RequestResponseProtocol { + static constexpr auto kName = "BeefyJustificationProtocol"; + + public: + BeefyJustificationProtocol(libp2p::Host &host, + const blockchain::GenesisBlockHash &genesis, + std::shared_ptr beefy); + + std::optional> onRxRequest( + RequestType block, std::shared_ptr) override; + + void onTxRequest(const RequestType &) override {} + + private: + std::shared_ptr beefy_; + }; + + class BeefyProtocol final + : public ProtocolBase, + public std::enable_shared_from_this { + static constexpr auto kName = "BeefyProtocol"; + + public: + BeefyProtocol(libp2p::Host &host, + const blockchain::GenesisBlockHash &genesis, + Roles roles, + std::shared_ptr beefy, + std::shared_ptr stream_engine); + + bool start() override; + const std::string &protocolName() const override; + void onIncomingStream(std::shared_ptr stream) override; + void newOutgoingStream( + const PeerInfo &peer, + std::function>)> &&cb) + override; + + void broadcast( + std::shared_ptr message); + + private: + ProtocolBaseImpl base_; + Roles roles_; + std::shared_ptr beefy_; + std::shared_ptr stream_engine_; + }; +} // namespace kagome::network diff --git a/core/network/common.hpp b/core/network/common.hpp index 27d2db5b0b..27323fa583 100644 --- a/core/network/common.hpp +++ b/core/network/common.hpp @@ -9,7 +9,7 @@ #include #include "application/chain_spec.hpp" -#include "primitives/common.hpp" +#include "blockchain/genesis_block_hash.hpp" namespace kagome::network { /// Current protocol version. @@ -36,33 +36,38 @@ namespace kagome::network { const libp2p::peer::ProtocolName kFetchStatementProtocol{ "/{}/req_statement/1"}; const libp2p::peer::ProtocolName kSendDisputeProtocol{"/{}/send_dispute/1"}; + const libp2p::peer::ProtocolName kBeefyProtocol{"/{}/beefy/2"}; + const libp2p::peer::ProtocolName kBeefyJustificationProtocol{ + "/{}/beefy/justifications/1"}; + + struct ProtocolPrefix { + std::string_view prefix; + }; + constexpr ProtocolPrefix kProtocolPrefixParitytech{"paritytech"}; + constexpr ProtocolPrefix kProtocolPrefixPolkadot{"polkadot"}; template libp2p::StreamProtocols make_protocols(std::string_view format, const Args &...args) { - libp2p::StreamProtocols protocols; - auto instantiate = [&](const auto &arg) { - if constexpr (std::is_same_v, - std::decay_t>) { + struct Visitor { + std::string_view format; + libp2p::StreamProtocols protocols; + + void visit(const blockchain::GenesisBlockHash &arg) { auto x = hex_lower(arg); protocols.emplace_back(fmt::vformat(format, fmt::make_format_args(x))); - } else if constexpr (std::is_same_v< - std::decay_t, - std::decay_t>) { - auto x = hex_lower(arg.hash); - protocols.emplace_back(fmt::vformat(format, fmt::make_format_args(x))); - } else if constexpr (std::is_same_v< - std::decay_t, - std::decay_t>) { + } + void visit(const application::ChainSpec &arg) { protocols.emplace_back( fmt::vformat(format, fmt::make_format_args(arg.protocolId()))); - } else { + } + void visit(const ProtocolPrefix &arg) { protocols.emplace_back( - fmt::vformat(format, fmt::make_format_args(arg))); + fmt::vformat(format, fmt::make_format_args(arg.prefix))); } - }; - (instantiate(args), ...); - return protocols; + } visitor{format, {}}; + (visitor.visit(args), ...); + return visitor.protocols; } } // namespace kagome::network diff --git a/core/network/impl/peer_manager_impl.cpp b/core/network/impl/peer_manager_impl.cpp index b200ec4b54..ec26d39594 100644 --- a/core/network/impl/peer_manager_impl.cpp +++ b/core/network/impl/peer_manager_impl.cpp @@ -11,6 +11,7 @@ #include +#include "network/beefy/protocol.hpp" #include "outcome/outcome.hpp" #include "scale/libp2p_types.hpp" #include "storage/predefined_keys.hpp" @@ -794,6 +795,11 @@ namespace kagome::network { self->tryOpenGrandpaProtocol(peer_info, peer_state.value().get()); self->tryOpenValidationProtocol(peer_info, peer_state.value().get()); + openOutgoing(self->stream_engine_, + self->router_->getBeefyProtocol(), + peer_info, + [](outcome::result< + std::shared_ptr>) {}); } }); diff --git a/core/network/impl/protocols/grandpa_protocol.cpp b/core/network/impl/protocols/grandpa_protocol.cpp index c3d48bb068..91aa80b6de 100644 --- a/core/network/impl/protocols/grandpa_protocol.cpp +++ b/core/network/impl/protocols/grandpa_protocol.cpp @@ -35,7 +35,8 @@ namespace kagome::network { std::shared_ptr scheduler) : base_(kGrandpaProtocolName, host, - make_protocols(kGrandpaProtocol, genesis_hash, "paritytech"), + make_protocols( + kGrandpaProtocol, genesis_hash, kProtocolPrefixParitytech), log::createLogger(kGrandpaProtocolName, "grandpa_protocol")), hasher_{std::move(hasher)}, io_context_(std::move(io_context)), diff --git a/core/network/impl/protocols/light.cpp b/core/network/impl/protocols/light.cpp index fa09249fcd..db242bba8c 100644 --- a/core/network/impl/protocols/light.cpp +++ b/core/network/impl/protocols/light.cpp @@ -16,7 +16,7 @@ namespace kagome::network { LightProtocol::LightProtocol( libp2p::Host &host, const application::ChainSpec &chain_spec, - const primitives::GenesisBlockHeader &genesis, + const blockchain::GenesisBlockHash &genesis, std::shared_ptr repository, std::shared_ptr storage, std::shared_ptr module_repo, diff --git a/core/network/impl/protocols/light.hpp b/core/network/impl/protocols/light.hpp index e9772630e9..58c0381571 100644 --- a/core/network/impl/protocols/light.hpp +++ b/core/network/impl/protocols/light.hpp @@ -14,9 +14,9 @@ namespace kagome::application { class ChainSpec; } // namespace kagome::application -namespace kagome::primitives { - struct GenesisBlockHeader; -} // namespace kagome::primitives +namespace kagome::blockchain { + class GenesisBlockHash; +} // namespace kagome::blockchain namespace kagome::runtime { class ModuleRepository; @@ -45,7 +45,7 @@ namespace kagome::network { public: LightProtocol(libp2p::Host &host, const application::ChainSpec &chain_spec, - const primitives::GenesisBlockHeader &genesis, + const blockchain::GenesisBlockHash &genesis, std::shared_ptr repository, std::shared_ptr storage, std::shared_ptr module_repo, diff --git a/core/network/impl/protocols/parachain_protocol.hpp b/core/network/impl/protocols/parachain_protocol.hpp index 2c9c7c484d..cdea9fcada 100644 --- a/core/network/impl/protocols/parachain_protocol.hpp +++ b/core/network/impl/protocols/parachain_protocol.hpp @@ -60,7 +60,7 @@ namespace kagome::network { log::Logger logger) : base_(kParachainProtocolName, host, - make_protocols(protocol, genesis_hash, "polkadot"), + make_protocols(protocol, genesis_hash, kProtocolPrefixPolkadot), std::move(logger)), observer_(std::move(observer)), roles_{roles}, diff --git a/core/network/impl/protocols/protocol_fetch_available_data.hpp b/core/network/impl/protocols/protocol_fetch_available_data.hpp index 0c3516eaee..48c8cb02c8 100644 --- a/core/network/impl/protocols/protocol_fetch_available_data.hpp +++ b/core/network/impl/protocols/protocol_fetch_available_data.hpp @@ -35,7 +35,7 @@ namespace kagome::network { host, make_protocols(kFetchAvailableDataProtocol, genesis_hash, - "polkadot"), + kProtocolPrefixPolkadot), log::createLogger( kName, "req_available_data_protocol")}, av_store_{std::move(av_store)} {} @@ -73,7 +73,7 @@ namespace kagome::network { host, make_protocols(kFetchStatementProtocol, genesis_hash, - "polkadot"), + kProtocolPrefixPolkadot), log::createLogger( kName, "req_statement_protocol")}, backing_store_{std::move(backing_store)} {} diff --git a/core/network/impl/protocols/protocol_fetch_chunk.hpp b/core/network/impl/protocols/protocol_fetch_chunk.hpp index cd1b5d12cf..e58f5c378e 100644 --- a/core/network/impl/protocols/protocol_fetch_chunk.hpp +++ b/core/network/impl/protocols/protocol_fetch_chunk.hpp @@ -46,7 +46,7 @@ namespace kagome::network { host, make_protocols(kFetchChunkProtocol, genesis_hash, - "polkadot"), + kProtocolPrefixPolkadot), log::createLogger(kFetchChunkProtocolName, "req_chunk_protocol")}, pp_{std::move(pp)} { diff --git a/core/network/impl/protocols/protocol_req_collation.cpp b/core/network/impl/protocols/protocol_req_collation.cpp index 8086c15258..e5b0067458 100644 --- a/core/network/impl/protocols/protocol_req_collation.cpp +++ b/core/network/impl/protocols/protocol_req_collation.cpp @@ -29,7 +29,7 @@ namespace kagome::network { host, make_protocols(kReqCollationProtocol, genesis_hash, - "polkadot"), + kProtocolPrefixPolkadot), log::createLogger( kReqCollationProtocolName, "req_collation_protocol")}, diff --git a/core/network/impl/protocols/protocol_req_pov.cpp b/core/network/impl/protocols/protocol_req_pov.cpp index 8c78b84b89..9a53fb1976 100644 --- a/core/network/impl/protocols/protocol_req_pov.cpp +++ b/core/network/impl/protocols/protocol_req_pov.cpp @@ -28,7 +28,7 @@ namespace kagome::network { host, make_protocols(kReqPovProtocol, genesis_hash, - "polkadot"), + kProtocolPrefixPolkadot), log::createLogger(kReqPovProtocolName, "req_pov_protocol")}, observer_{std::move(observer)} {} diff --git a/core/network/impl/protocols/send_dispute_protocol.hpp b/core/network/impl/protocols/send_dispute_protocol.hpp index 79019b62a5..f22a835959 100644 --- a/core/network/impl/protocols/send_dispute_protocol.hpp +++ b/core/network/impl/protocols/send_dispute_protocol.hpp @@ -49,7 +49,7 @@ namespace kagome::network { host, make_protocols(kSendDisputeProtocol, genesis_hash, - "polkadot"), + kProtocolPrefixPolkadot), log::createLogger(kSendDisputeProtocolName, "dispute_protocol")}, dispute_request_observer_{std::move(dispute_request_observer)} { diff --git a/core/network/impl/router_libp2p.cpp b/core/network/impl/router_libp2p.cpp index 83cfcf979f..e6ab654fff 100644 --- a/core/network/impl/router_libp2p.cpp +++ b/core/network/impl/router_libp2p.cpp @@ -17,6 +17,8 @@ namespace kagome::network { LazySPtr sync_protocol, LazySPtr state_protocol, LazySPtr warp_protocol, + LazySPtr beefy_protocol, + LazySPtr beefy_justifications_protocol, LazySPtr light_protocol, LazySPtr propagate_transactions_protocol, LazySPtr validation_protocol, @@ -37,6 +39,9 @@ namespace kagome::network { sync_protocol_(std::move(sync_protocol)), state_protocol_(std::move(state_protocol)), warp_protocol_{std::move(warp_protocol)}, + beefy_protocol_{std::move(beefy_protocol)}, + beefy_justifications_protocol_{ + std::move(beefy_justifications_protocol)}, light_protocol_{std::move(light_protocol)}, propagate_transactions_protocol_( std::move(propagate_transactions_protocol)), @@ -76,6 +81,8 @@ namespace kagome::network { app_state_manager_->takeControl(*sync_protocol_.get()); app_state_manager_->takeControl(*state_protocol_.get()); app_state_manager_->takeControl(*warp_protocol_.get()); + app_state_manager_->takeControl(*beefy_protocol_.get()); + app_state_manager_->takeControl(*beefy_justifications_protocol_.get()); app_state_manager_->takeControl(*light_protocol_.get()); app_state_manager_->takeControl(*propagate_transactions_protocol_.get()); @@ -220,6 +227,10 @@ namespace kagome::network { return send_dispute_protocol_.get(); } + std::shared_ptr RouterLibp2p::getBeefyProtocol() const { + return beefy_protocol_.get(); + } + std::shared_ptr RouterLibp2p::getPingProtocol() const { return ping_protocol_.get(); diff --git a/core/network/impl/router_libp2p.hpp b/core/network/impl/router_libp2p.hpp index 972214ab4c..1c3cb5b8c4 100644 --- a/core/network/impl/router_libp2p.hpp +++ b/core/network/impl/router_libp2p.hpp @@ -15,6 +15,7 @@ #include "libp2p/host/host.hpp" #include "libp2p/multi/multiaddress.hpp" #include "libp2p/protocol/ping.hpp" +#include "network/beefy/protocol.hpp" #include "network/impl/protocols/light.hpp" #include "network/sync_protocol_observer.hpp" #include "network/types/bootstrap_nodes.hpp" @@ -44,6 +45,8 @@ namespace kagome::network { LazySPtr sync_protocol, LazySPtr state_protocol, LazySPtr warp_protocol, + LazySPtr beefy_protocol, + LazySPtr beefy_justifications_protocol, LazySPtr light_protocol, LazySPtr propagate_transactions_protocol, LazySPtr validation_protocol, @@ -93,6 +96,8 @@ namespace kagome::network { std::shared_ptr getPingProtocol() const override; + std::shared_ptr getBeefyProtocol() const override; + private: /** * Appends /p2p/ part to ip4 and ip6 addresses which then passed to @@ -115,6 +120,8 @@ namespace kagome::network { LazySPtr sync_protocol_; LazySPtr state_protocol_; LazySPtr warp_protocol_; + LazySPtr beefy_protocol_; + LazySPtr beefy_justifications_protocol_; LazySPtr light_protocol_; LazySPtr propagate_transactions_protocol_; diff --git a/core/network/impl/sync_protocol_observer_impl.cpp b/core/network/impl/sync_protocol_observer_impl.cpp index bc0896deb2..5a644ffbfb 100644 --- a/core/network/impl/sync_protocol_observer_impl.cpp +++ b/core/network/impl/sync_protocol_observer_impl.cpp @@ -8,6 +8,7 @@ #include #include "application/app_configuration.hpp" +#include "network/beefy/i_beefy.hpp" #include "log/formatters/variant.hpp" #include "network/common.hpp" #include "primitives/common.hpp" @@ -28,9 +29,11 @@ namespace kagome::network { SyncProtocolObserverImpl::SyncProtocolObserverImpl( std::shared_ptr block_tree, std::shared_ptr blocks_headers, + std::shared_ptr beefy, std::shared_ptr peer_manager) : block_tree_{std::move(block_tree)}, blocks_headers_{std::move(blocks_headers)}, + beefy_{std::move(beefy)}, peer_manager_{std::move(peer_manager)}, log_(log::createLogger("SyncProtocolObserver", "network")) { BOOST_ASSERT(block_tree_); @@ -187,7 +190,21 @@ namespace kagome::network { new_block.justification = std::move(justification_res.value()); } if (request.multiple_justifications) { - // TODO(turuslan): #1651, beefy_justification + std::optional number; + if (new_block.header) { + number = new_block.header->number; + } else if (auto r = blocks_headers_->getNumberByHash(hash)) { + number = r.value(); + } + if (number) { + if (auto r = beefy_->getJustification(*number)) { + if (auto &opt = r.value()) { + new_block.beefy_justification = primitives::Justification{ + common::Buffer{scale::encode(*opt).value()}, + }; + } + } + } } } } diff --git a/core/network/impl/sync_protocol_observer_impl.hpp b/core/network/impl/sync_protocol_observer_impl.hpp index d617ea812c..fa01b73d75 100644 --- a/core/network/impl/sync_protocol_observer_impl.hpp +++ b/core/network/impl/sync_protocol_observer_impl.hpp @@ -19,6 +19,7 @@ #include "primitives/common.hpp" namespace kagome::network { + class IBeefy; class SyncProtocolObserverImpl : public SyncProtocolObserver, @@ -29,6 +30,7 @@ namespace kagome::network { SyncProtocolObserverImpl( std::shared_ptr block_tree, std::shared_ptr blocks_headers, + std::shared_ptr beefy, std::shared_ptr peer_manager); ~SyncProtocolObserverImpl() override = default; @@ -49,6 +51,7 @@ namespace kagome::network { std::shared_ptr block_tree_; std::shared_ptr blocks_headers_; + std::shared_ptr beefy_; mutable std::unordered_set requested_ids_; std::shared_ptr peer_manager_; diff --git a/core/network/impl/synchronizer_impl.cpp b/core/network/impl/synchronizer_impl.cpp index ce8d4a1c23..2dc68cb16e 100644 --- a/core/network/impl/synchronizer_impl.cpp +++ b/core/network/impl/synchronizer_impl.cpp @@ -12,6 +12,7 @@ #include "consensus/babe/has_babe_consensus_digest.hpp" #include "consensus/grandpa/environment.hpp" #include "consensus/grandpa/has_authority_set_change.hpp" +#include "network/beefy/i_beefy.hpp" #include "network/types/block_attributes.hpp" #include "primitives/common.hpp" #include "storage/predefined_keys.hpp" @@ -86,6 +87,7 @@ namespace kagome::network { std::shared_ptr scheduler, std::shared_ptr hasher, primitives::events::ChainSubscriptionEnginePtr chain_sub_engine, + std::shared_ptr beefy, std::shared_ptr grandpa_environment) : app_state_manager_(std::move(app_state_manager)), block_tree_(std::move(block_tree)), @@ -97,6 +99,7 @@ namespace kagome::network { router_(std::move(router)), scheduler_(std::move(scheduler)), hasher_(std::move(hasher)), + beefy_{std::move(beefy)}, grandpa_environment_{std::move(grandpa_environment)}, chain_sub_engine_(std::move(chain_sub_engine)) { BOOST_ASSERT(app_state_manager_); @@ -846,12 +849,13 @@ namespace kagome::network { scheduleRecentRequestRemoval(peer_id, request_fingerprint); + using Result = outcome::result; auto response_handler = [wp = weak_from_this(), peer_id, target_block, limit, handler = std::move(handler)]( - auto &&response_res) mutable { + Result response_res) mutable { auto self = wp.lock(); if (not self) { return; @@ -919,6 +923,10 @@ namespace kagome::network { *block.justification); } } + if (block.beefy_justification) { + self->beefy_->onJustification(block.hash, + std::move(*block.beefy_justification)); + } } if (justification_received) { @@ -1240,6 +1248,11 @@ namespace kagome::network { }); } } + + if (block_data.beefy_justification) { + beefy_->onJustification(block_data.hash, + std::move(*block_data.beefy_justification)); + } } } } diff --git a/core/network/impl/synchronizer_impl.hpp b/core/network/impl/synchronizer_impl.hpp index 0aa58c1af7..1c8dab4b22 100644 --- a/core/network/impl/synchronizer_impl.hpp +++ b/core/network/impl/synchronizer_impl.hpp @@ -47,6 +47,7 @@ namespace kagome::storage::trie { } // namespace kagome::storage::trie namespace kagome::network { + class IBeefy; class SynchronizerImpl : public Synchronizer, @@ -98,6 +99,7 @@ namespace kagome::network { std::shared_ptr scheduler, std::shared_ptr hasher, primitives::events::ChainSubscriptionEnginePtr chain_sub_engine, + std::shared_ptr beefy, std::shared_ptr grandpa_environment); /** @see AppStateManager::takeControl */ @@ -217,6 +219,7 @@ namespace kagome::network { std::shared_ptr router_; std::shared_ptr scheduler_; std::shared_ptr hasher_; + std::shared_ptr beefy_; std::shared_ptr grandpa_environment_; primitives::events::ChainSubscriptionEnginePtr chain_sub_engine_; diff --git a/core/network/router.hpp b/core/network/router.hpp index 071080de37..a03d3ef35a 100644 --- a/core/network/router.hpp +++ b/core/network/router.hpp @@ -22,6 +22,8 @@ #include "network/protocols/sync_protocol.hpp" namespace kagome::network { + class BeefyProtocol; + /** * Router, which reads and delivers different network messages to the * observers, responsible for their processing @@ -51,6 +53,7 @@ namespace kagome::network { virtual std::shared_ptr getGrandpaProtocol() const = 0; virtual std::shared_ptr getSendDisputeProtocol() const = 0; + virtual std::shared_ptr getBeefyProtocol() const = 0; virtual std::shared_ptr getPingProtocol() const = 0; }; } // namespace kagome::network diff --git a/core/network/warp/protocol.hpp b/core/network/warp/protocol.hpp index 7a4682efeb..50976c56c4 100644 --- a/core/network/warp/protocol.hpp +++ b/core/network/warp/protocol.hpp @@ -20,7 +20,7 @@ namespace kagome::network { public: WarpProtocol(libp2p::Host &host, const application::ChainSpec &chain_spec, - const primitives::GenesisBlockHeader &genesis, + const blockchain::GenesisBlockHash &genesis, std::shared_ptr cache) : RequestResponseProtocolType{ kName, diff --git a/core/parachain/validator/impl/parachain_processor.cpp b/core/parachain/validator/impl/parachain_processor.cpp index 2f86710721..e980a4a2a5 100644 --- a/core/parachain/validator/impl/parachain_processor.cpp +++ b/core/parachain/validator/impl/parachain_processor.cpp @@ -136,7 +136,7 @@ namespace kagome::parachain { [log{logger_}, wptr_self{weak_from_this()}]( const primitives::BlockHash &relay_parent, const network::SignedBitfield &bitfield) { - log->info("Distribute bitfield on {}", relay_parent); + SL_VERBOSE(log, "Distribute bitfield on {}", relay_parent); if (auto self = wptr_self.lock()) { auto msg = std::make_shared< network::WireMessage>( @@ -346,12 +346,12 @@ namespace kagome::parachain { } } - logger_->info( - "Inited new backing task.(assignment={}, our index={}, relay " - "parent={})", - assignment, - validator->validatorIndex(), - relay_parent); + SL_VERBOSE(logger_, + "Inited new backing task.(assignment={}, our index={}, relay " + "parent={})", + assignment, + validator->validatorIndex(), + relay_parent); return RelayParentState{ .assignment = assignment, @@ -377,7 +377,7 @@ namespace kagome::parachain { auto rps_result = initNewBackingTask(relay_parent); if (rps_result.has_value()) { storeStateByRelayParent(relay_parent, std::move(rps_result.value())); - } else { + } else if (rps_result.error() != Error::KEY_NOT_PRESENT) { logger_->error( "Relay parent state was not created. (relay parent={}, error={})", relay_parent, diff --git a/core/primitives/digest.hpp b/core/primitives/digest.hpp index 6c3c145e96..9f1e744c08 100644 --- a/core/primitives/digest.hpp +++ b/core/primitives/digest.hpp @@ -31,7 +31,7 @@ namespace kagome::primitives { inline const auto kUnsupportedEngineId_POL1 = ConsensusEngineId::fromString("POL1").value(); - inline const auto kUnsupportedEngineId_BEEF = + inline const auto kBeefyEngineId = ConsensusEngineId::fromString("BEEF").value(); struct Other : public common::Buffer {}; @@ -89,7 +89,7 @@ namespace kagome::primitives { } else if (engine_id == primitives::kUnsupportedEngineId_POL1) { OUTCOME_TRY(payload, scale::decode(data)); msg.digest = std::move(payload); - } else if (engine_id == primitives::kUnsupportedEngineId_BEEF) { + } else if (engine_id == primitives::kBeefyEngineId) { OUTCOME_TRY(payload, scale::decode(data)); msg.digest = std::move(payload); } else { diff --git a/core/primitives/math.hpp b/core/primitives/math.hpp index 8f0cbd8004..0e959189db 100644 --- a/core/primitives/math.hpp +++ b/core/primitives/math.hpp @@ -35,6 +35,17 @@ namespace kagome::math { return res; } + inline bool isPowerOf2(size_t x) { + return ((x > 0ull) && ((x & (x - 1ull)) == 0)); + } + + inline size_t nextHighPowerOf2(size_t k) { + if (isPowerOf2(k)) { + return k; + } + const auto p = k == 0ull ? 0ull : 64ull - __builtin_clzll(k); + return (1ull << p); + } } // namespace kagome::math #endif // KAGOME_MATH_HPP diff --git a/core/runtime/common/memory_allocator.cpp b/core/runtime/common/memory_allocator.cpp index 54430b7b7b..b000167179 100644 --- a/core/runtime/common/memory_allocator.cpp +++ b/core/runtime/common/memory_allocator.cpp @@ -39,7 +39,7 @@ namespace kagome::runtime { } const size_t chunk_size = - nextHighPowerOf2(roundUpAlign(size) + AllocationHeaderSz); + math::nextHighPowerOf2(roundUpAlign(size) + AllocationHeaderSz); const auto ptr = offset_; const auto new_offset = ptr + chunk_size; // align @@ -78,7 +78,7 @@ namespace kagome::runtime { .allocation_sz = 0, }; header.deserialize(ptr - AllocationHeaderSz, memory_); - BOOST_ASSERT(isPowerOf2(header.chunk_sz)); + BOOST_ASSERT(math::isPowerOf2(header.chunk_sz)); available_[header.chunk_sz].push_back(ptr - AllocationHeaderSz); BOOST_ASSERT(!available_.empty()); @@ -131,7 +131,7 @@ namespace kagome::runtime { .allocation_sz = 0, }; header.deserialize(ptr - AllocationHeaderSz, memory_); - BOOST_ASSERT(isPowerOf2(header.chunk_sz)); + BOOST_ASSERT(math::isPowerOf2(header.chunk_sz)); return header.allocation_sz; } diff --git a/core/runtime/common/memory_allocator.hpp b/core/runtime/common/memory_allocator.hpp index c8c48e803e..9015e77c52 100644 --- a/core/runtime/common/memory_allocator.hpp +++ b/core/runtime/common/memory_allocator.hpp @@ -39,18 +39,6 @@ namespace kagome::runtime { return math::roundUp(t); } - inline bool isPowerOf2(size_t x) { - return ((x > 0ull) && ((x & (x - 1ull)) == 0)); - } - - inline size_t nextHighPowerOf2(size_t k) { - if (isPowerOf2(k)) { - return k; - } - const auto p = k == 0ull ? 0ull : 64ull - __builtin_clzll(k); - return (1ull << p); - } - /** * Implementation of allocator for the runtime memory * Combination of monotonic and free-list allocator diff --git a/core/runtime/runtime_api/beefy.hpp b/core/runtime/runtime_api/beefy.hpp index 15ce76b403..55ee7d0314 100644 --- a/core/runtime/runtime_api/beefy.hpp +++ b/core/runtime/runtime_api/beefy.hpp @@ -13,7 +13,13 @@ namespace kagome::runtime { virtual ~BeefyApi() = default; /** - * Get validator set if beefy is supported. + * Get genesis if beefy is supported. + */ + virtual outcome::result> genesis( + const primitives::BlockHash &block) = 0; + + /** + * Get validator set. */ virtual outcome::result> validatorSet(const primitives::BlockHash &block) = 0; diff --git a/core/runtime/runtime_api/impl/beefy.cpp b/core/runtime/runtime_api/impl/beefy.cpp index 5f46eee36a..d42312d949 100644 --- a/core/runtime/runtime_api/impl/beefy.cpp +++ b/core/runtime/runtime_api/impl/beefy.cpp @@ -14,10 +14,10 @@ namespace kagome::runtime { BOOST_ASSERT(executor_); } - outcome::result> - BeefyApiImpl::validatorSet(const primitives::BlockHash &block) { - auto r = executor_->callAt( - block, "BeefyApi_validator_set"); + outcome::result> BeefyApiImpl::genesis( + const primitives::BlockHash &block) { + auto r = executor_->callAt>( + block, "BeefyApi_beefy_genesis"); if (r) { return std::move(r.value()); } @@ -26,4 +26,10 @@ namespace kagome::runtime { } return r.error(); } + + outcome::result> + BeefyApiImpl::validatorSet(const primitives::BlockHash &block) { + return executor_->callAt>( + block, "BeefyApi_validator_set"); + } } // namespace kagome::runtime diff --git a/core/runtime/runtime_api/impl/beefy.hpp b/core/runtime/runtime_api/impl/beefy.hpp index ff632db2b5..f48feaf755 100644 --- a/core/runtime/runtime_api/impl/beefy.hpp +++ b/core/runtime/runtime_api/impl/beefy.hpp @@ -14,6 +14,8 @@ namespace kagome::runtime { public: BeefyApiImpl(std::shared_ptr executor); + outcome::result> genesis( + const primitives::BlockHash &block) override; outcome::result> validatorSet( const primitives::BlockHash &block) override; diff --git a/core/storage/rocksdb/rocksdb_spaces.cpp b/core/storage/rocksdb/rocksdb_spaces.cpp index 8757ff85e1..70162eb60a 100644 --- a/core/storage/rocksdb/rocksdb_spaces.cpp +++ b/core/storage/rocksdb/rocksdb_spaces.cpp @@ -20,6 +20,7 @@ namespace kagome::storage { "justification", "trie_node", "dispute_data", + "beefy_justification", }; static_assert(kNames.size() == Space::kTotal - 1); diff --git a/core/storage/spaces.hpp b/core/storage/spaces.hpp index 76bc021423..caaa32e3f1 100644 --- a/core/storage/spaces.hpp +++ b/core/storage/spaces.hpp @@ -21,6 +21,7 @@ namespace kagome::storage { kJustification, kTrieNode, kDisputeData, + kBeefyJustification, kTotal }; diff --git a/core/utils/block_number_key.hpp b/core/utils/block_number_key.hpp new file mode 100644 index 0000000000..eb86e7ef36 --- /dev/null +++ b/core/utils/block_number_key.hpp @@ -0,0 +1,32 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include + +#include "primitives/common.hpp" + +namespace kagome { + struct BlockNumberKey { + using Key = common::Blob; + + static Key encode(primitives::BlockNumber number) { + Key key; + boost::endian::store_big_u32(key.data(), number); + return key; + } + + static std::optional decode( + common::BufferView key) { + if (libp2p::spanSize(key) != Key::size()) { + return std::nullopt; + } + return boost::endian::load_big_u32(key.data()); + } + }; +} // namespace kagome diff --git a/test/core/network/sync_protocol_observer_test.cpp b/test/core/network/sync_protocol_observer_test.cpp index 7a47be3993..702e325f58 100644 --- a/test/core/network/sync_protocol_observer_test.cpp +++ b/test/core/network/sync_protocol_observer_test.cpp @@ -13,6 +13,7 @@ #include "application/app_configuration.hpp" #include "mock/core/blockchain/block_header_repository_mock.hpp" #include "mock/core/blockchain/block_tree_mock.hpp" +#include "mock/core/network/beefy.hpp" #include "mock/core/network/peer_manager_mock.hpp" #include "mock/libp2p/host/host_mock.hpp" #include "primitives/block.hpp" @@ -45,7 +46,7 @@ class SynchronizerTest : public testing::Test { void SetUp() override { peer_manager_mock_ = std::make_shared(); sync_protocol_observer_ = std::make_shared( - tree_, headers_, peer_manager_mock_); + tree_, headers_, beefy_, peer_manager_mock_); } std::shared_ptr host_ = std::make_shared(); @@ -57,6 +58,7 @@ class SynchronizerTest : public testing::Test { std::shared_ptr sync_protocol_observer_; std::shared_ptr peer_manager_mock_; + std::shared_ptr beefy_ = std::make_shared(); const Hash256 block2_hash_ = "2"_hash256; const Block block3_{{3, block2_hash_, {}, {}, {}}, @@ -100,6 +102,10 @@ TEST_F(SynchronizerTest, ProcessRequest) { .WillOnce(Return(::outcome::failure(boost::system::error_code{}))); EXPECT_CALL(*peer_manager_mock_, reserveStatusStreams(peer_info_.id)); + EXPECT_CALL(*beefy_, getJustification(_)).WillRepeatedly([] { + return ::outcome::success(std::nullopt); + }); + // WHEN EXPECT_OUTCOME_TRUE(response, sync_protocol_observer_->onBlocksRequest(received_request, diff --git a/test/core/network/synchronizer_test.cpp b/test/core/network/synchronizer_test.cpp index b7b43cb130..b314424241 100644 --- a/test/core/network/synchronizer_test.cpp +++ b/test/core/network/synchronizer_test.cpp @@ -94,6 +94,7 @@ class SynchronizerTest scheduler, hasher, chain_sub_engine, + nullptr, grandpa_environment); } diff --git a/test/mock/core/crypto/session_keys_mock.hpp b/test/mock/core/crypto/session_keys_mock.hpp index f866feee98..82e4e09c3b 100644 --- a/test/mock/core/crypto/session_keys_mock.hpp +++ b/test/mock/core/crypto/session_keys_mock.hpp @@ -37,6 +37,11 @@ namespace kagome::crypto { getAudiKeyPair, (const std::vector &), (override)); + + MOCK_METHOD(KeypairWithIndexOpt, + getBeefKeyPair, + (const std::vector &), + (override)); }; } // namespace kagome::crypto diff --git a/test/mock/core/network/beefy.hpp b/test/mock/core/network/beefy.hpp new file mode 100644 index 0000000000..7f8dac1db6 --- /dev/null +++ b/test/mock/core/network/beefy.hpp @@ -0,0 +1,25 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "network/beefy/i_beefy.hpp" + +#include + +namespace kagome::network { + struct BeefyMock : IBeefy { + MOCK_METHOD( + outcome::result>, + getJustification, + (primitives::BlockNumber), + (const, override)); + + MOCK_METHOD(void, + onJustification, + (const primitives::BlockHash &, primitives::Justification), + (override)); + }; +} // namespace kagome::network diff --git a/test/mock/core/network/router_mock.hpp b/test/mock/core/network/router_mock.hpp index 900f6295d0..34f585abad 100644 --- a/test/mock/core/network/router_mock.hpp +++ b/test/mock/core/network/router_mock.hpp @@ -83,6 +83,11 @@ namespace kagome::network { (), (const, override)); + MOCK_METHOD(std::shared_ptr, + getBeefyProtocol, + (), + (const, override)); + MOCK_METHOD(std::shared_ptr, getPingProtocol, (), diff --git a/zombienet/0013-beefy-and-mmr/0013-beefy-and-mmr.toml b/zombienet/0013-beefy-and-mmr/0013-beefy-and-mmr.toml new file mode 100644 index 0000000000..ef0fb2c87b --- /dev/null +++ b/zombienet/0013-beefy-and-mmr/0013-beefy-and-mmr.toml @@ -0,0 +1,17 @@ +[settings] +timeout = 1000 + +[relaychain] +chain = "rococo-local" +default_command = "polkadot" + +[[relaychain.node_groups]] +name = "validator" +count = 3 +command = "kagome" +prometheus_prefix = "kagome" +args = ["--enable-offchain-indexing=true", "--wasm-execution", "Compiled", "--unsafe-cached-wavm-runtime"] + +[[relaychain.nodes]] +name = "validator-unstable" +args = ["--log=beefy=debug", "--enable-offchain-indexing=true"] diff --git a/zombienet/0013-beefy-and-mmr/0013-beefy-and-mmr.zndsl b/zombienet/0013-beefy-and-mmr/0013-beefy-and-mmr.zndsl new file mode 100644 index 0000000000..4e1007ab2b --- /dev/null +++ b/zombienet/0013-beefy-and-mmr/0013-beefy-and-mmr.zndsl @@ -0,0 +1,32 @@ +Description: Test BEEFY voting and finality, test MMR proofs. Assumes Rococo sessions of 1 minute. +Network: ./0013-beefy-and-mmr.toml +Creds: config + +# Check authority status. +validator: reports node_roles is 4 +validator-unstable: reports node_roles is 4 + +# BEEFY sanity checks. +validator: reports kagome_beefy_validator_set_id is 0 +validator-unstable: reports substrate_beefy_validator_set_id is 0 + +# Verify voting happens and 1st mandatory block is finalized within 1st session. +validator: reports kagome_beefy_best_block is at least 1 within 60 seconds +validator-unstable: reports substrate_beefy_best_block is at least 1 within 60 seconds + +# Pause validator-unstable and test chain is making progress without it. +validator-unstable: pause + +# Verify validator sets get changed on new sessions. +validator: reports kagome_beefy_validator_set_id is at least 1 within 70 seconds +# Check next session too. +validator: reports kagome_beefy_validator_set_id is at least 2 within 130 seconds + +# Verify voting happens and blocks are being finalized for new sessions too: +# since we verified we're at least in the 3rd session, verify BEEFY finalized mandatory #21. +validator: reports kagome_beefy_best_block is at least 21 within 130 seconds + +# Resume validator-unstable and verify it imports all BEEFY justification and catches up. +validator-unstable: resume +validator-unstable: reports substrate_beefy_validator_set_id is at least 2 within 30 seconds +validator-unstable: reports substrate_beefy_best_block is at least 21 within 30 seconds